From f03af539c9d624d0263ebba0114592ef3d906ff0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Jul 2024 16:54:15 +0000
Subject: [PATCH 001/128] Bump setuptools from 59.6.0 to 70.0.0

Bumps [setuptools](https://github.com/pypa/setuptools) from 59.6.0 to 70.0.0.
- [Release notes](https://github.com/pypa/setuptools/releases)
- [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst)
- [Commits](https://github.com/pypa/setuptools/compare/v59.6.0...v70.0.0)

---
updated-dependencies:
- dependency-name: setuptools
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 6d4b5c79dd..ef8bade305 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -21,6 +21,6 @@ semver==2.13.0; python_version == '3.6'
 semver==3.0.2; python_version >= '3.7'
 setuptools==59.6.0; python_version == '3.6'
 setuptools==68.0.0; python_version == '3.7'
-setuptools==70.1.1; python_version >= '3.8'
+setuptools==70.0.0; python_version >= '3.8'
 wcwidth==0.2.13
 #+pygelf%pygelf==0.4.0

From 0b3df639c5c4cea9da1d66134f8c006112288817 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Tue, 16 Jul 2024 10:20:59 +0200
Subject: [PATCH 002/128] Revert "Bump setuptools from 70.1.1 to 70.0.0"

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index ef8bade305..6d4b5c79dd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -21,6 +21,6 @@ semver==2.13.0; python_version == '3.6'
 semver==3.0.2; python_version >= '3.7'
 setuptools==59.6.0; python_version == '3.6'
 setuptools==68.0.0; python_version == '3.7'
-setuptools==70.0.0; python_version >= '3.8'
+setuptools==70.1.1; python_version >= '3.8'
 wcwidth==0.2.13
 #+pygelf%pygelf==0.4.0

From d85f5471e2ecbc8b988779ceb4a73a77bcc37304 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Tue, 16 Jul 2024 11:59:32 +0300
Subject: [PATCH 003/128] Update disclaimer on the support of EOL Python
 versions

---
 docs/started.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/started.rst b/docs/started.rst
index ccfc1f57ba..3ab07c0295 100644
--- a/docs/started.rst
+++ b/docs/started.rst
@@ -20,8 +20,8 @@ Requirements
 
 
 .. warning::
-   Although ReFrame supports Python 3.6, you should note that Python 3.6 has reached its end-of-life and you are advised to use a newer Python version.
-   ReFrame installations on Python 3.6 may use out-of-date dependencies due to incompatibilities of their newer versions with Python 3.6.
+   Although ReFrame supports Python 3.6 and 3.7, you should note that these Python versions have reached end-of-life and you are strongly advised to use a newer version.
+   ReFrame installations on these Python versions may use out-of-date dependencies due to incompatibilities.
 
 
 Getting the Framework

From 16ea44250248e5ae900c25089f9a24d38b0cfd20 Mon Sep 17 00:00:00 2001
From: crivella <davide.grassano@epfl.ch>
Date: Fri, 12 Jul 2024 16:08:17 +0200
Subject: [PATCH 004/128] Added benchmark tests for MetalWalls

---
 hpctestlib/sciapps/metalwalls/benchmarks.py | 267 ++++++++++++++++++++
 1 file changed, 267 insertions(+)
 create mode 100644 hpctestlib/sciapps/metalwalls/benchmarks.py

diff --git a/hpctestlib/sciapps/metalwalls/benchmarks.py b/hpctestlib/sciapps/metalwalls/benchmarks.py
new file mode 100644
index 0000000000..450a06d877
--- /dev/null
+++ b/hpctestlib/sciapps/metalwalls/benchmarks.py
@@ -0,0 +1,267 @@
+"""ReFrame benchmark for MetalWalls"""
+import re
+from typing import TypeVar
+
+import reframe as rfm
+import reframe.utility.sanity as sn
+from reframe.core.builtins import (performance_function, run_after, run_before,
+                                   sanity_function)
+from reframe.core.parameters import TestParam as parameter
+from reframe.core.variables import TestVar as variable
+
+R = TypeVar('R')
+
+address_tpl = (
+    'https://gitlab.com/ampere2/metalwalls/-/raw/{version}/{bench}/{file}'
+    '?ref_type=tags\&inline=false'
+)
+
+extract_fields = [
+    #######
+    ('Ions->Atoms Coulomb potential', 'long range', 'I->A Cpot'),
+    ('Ions->Atoms Coulomb potential', 'k==0', 'I->A Cpot'),
+    ('Ions->Atoms Coulomb potential', 'short range', 'I->A Cpot'),
+
+    #######
+    ('Atoms->Atoms Coulomb potential', 'long range', 'A->A Cpot'),
+    ('Atoms->Atoms Coulomb potential', 'k==0', 'A->A Cpot'),
+    ('Atoms->Atoms Coulomb potential', 'short range', 'A->A Cpot'),
+    ('Atoms->Atoms Coulomb potential', 'self', 'A->A Cpot'),
+
+    #######
+    ('Atoms->Atoms Coulomb grad Q', 'long range', 'A->A gradQ'),
+    ('Atoms->Atoms Coulomb grad Q', 'k==0', 'A->A gradQ'),
+    ('Atoms->Atoms Coulomb grad Q', 'short range', 'A->A gradQ'),
+    ('Atoms->Atoms Coulomb grad Q', 'self', 'A->A gradQ'),
+
+    #######
+    ('Ions Coulomb forces', 'long range', 'Ions Cfrc'),
+    ('Ions Coulomb forces', 'k==0', 'Ions Cfrc'),
+    ('Ions Coulomb forces', 'short range', 'Ions Cfrc'),
+    ('Ions Coulomb forces', 'intramolecular', 'Ions Cfrc'),
+
+    #######
+    ('Ions Coulomb potential', 'long range', 'Ions Cpot'),
+    ('Ions Coulomb potential', 'k==0', 'Ions Cpot'),
+    ('Ions Coulomb potential', 'short range', 'Ions Cpot'),
+    ('Ions Coulomb potential', 'intramolecular', 'Ions Cpot'),
+    ('Ions Coulomb potential', 'self', 'Ions Cpot'),
+
+    #######
+    (
+        'Ions Coulomb electric field (due to charges)',
+        'long range',
+        'Ions Cfield(charges)'
+    ),
+    (
+        'Ions Coulomb electric field (due to charges)',
+        'k==0',
+        'Ions Cfield(charges)'
+    ),
+    (
+        'Ions Coulomb electric field (due to charges)',
+        'short range',
+        'Ions Cfield(charges)'
+    ),
+    (
+        'Ions Coulomb electric field (due to charges)',
+        'intramolecular',
+        'Ions Cfield(charges)'
+    ),
+
+    #######
+    (
+        'Ions Coulomb electric field (due to dipoles)',
+        'long range',
+        'Ions Cfield(dipoles)'
+    ),
+    (
+        'Ions Coulomb electric field (due to dipoles)',
+        'k==0',
+        'Ions Cfield(dipoles)'
+    ),
+    (
+        'Ions Coulomb electric field (due to dipoles)',
+        'short range',
+        'Ions Cfield(dipoles)'
+    ),
+    (
+        'Ions Coulomb electric field (due to dipoles)',
+        'intramolecular',
+        'Ions Cfield(dipoles)'
+    ),
+    (
+        'Ions Coulomb electric field (due to dipoles)',
+        'self',
+        'Ions Cfield(dipoles)'
+    ),
+
+    #######
+    ('Ions Coulomb electric field gradient', 'long range', 'Ions Cfield grad'),
+    ('Ions Coulomb electric field gradient', 'short range', 'Ions Cfield grad'),
+    ('Ions Coulomb electric field gradient', 'self', 'Ions Cfield grad'),
+
+    #######
+    ('Ions Coulomb gradient mu', 'long range', 'Ions C mu_grad'),
+    ('Ions Coulomb gradient mu', 'k==0', 'Ions C mu_grad'),
+    ('Ions Coulomb gradient mu', 'short range', 'Ions C mu_grad'),
+    ('Ions Coulomb gradient mu', 'self', 'Ions C mu_grad'),
+
+    #######
+    ('Rattle', 'positions', 'Rattle'),
+    ('Rattle', 'velocities', 'Rattle'),
+
+    #######
+    ('van der Waals', 'vdW  forces', 'van der Waals'),
+    ('van der Waals', 'vdW  potential', 'van der Waals'),
+
+    #######
+    ('Intramolecular', 'Intramolecular  forces', 'Intramolecular'),
+    ('Intramolecular', 'Intramolecular potential', 'Intramolecular'),
+
+    #######
+    ('Additional degrees', 'Electrode charge computation', 'Deg of freedom'),
+    ('Additional degrees', 'Inversion of the matrix', 'Deg of freedom'),
+    ('Additional degrees', 'One matrix-vector product', 'Deg of freedom'),
+    ('Additional degrees', 'Melt dipoles computation', 'Deg of freedom'),
+    ('Additional degrees', 'Inversion of the matrix', 'Deg of freedom'),
+    ('Additional degrees', 'One matrix-vector product', 'Deg of freedom'),
+    ('Additional degrees', 'AIM DOFs computation', 'Deg of freedom'),
+
+    #######
+    ('Diagnostics', 'diagnostics computations', 'Diagnostics'),
+    ('Diagnostics', 'IO', 'Diagnostics'),
+]
+
+@rfm.simple_test
+class MetalWallsCheck(rfm.RunOnlyRegressionTest):
+    """MetalWalls benchmark test.
+    
+    `MetalWalls <https://gitlab.com/ampere2/metalwalls>`__ is a molecular 
+    dynamics code dedicated to the modelling of electrochemical systems.
+    Its main originality is the inclusion of a series of methods allowing to
+    apply a constant potential within the electrode materials.
+
+    The benchmarks consist of a set of different inputs files that vary in the
+    number of atoms and the type of simulation performed.
+    They can be found in the following repository, which is also versioned:
+    https://gitlab.com/ampere2/metalwalls/.
+    
+    """
+
+    #: The name of the output files to keep.
+    #:
+    #: :type: :class:`List[str]`
+    #: :default: ``'run.out'``
+    keep_files = ['run.out', ]
+
+    #: The version of the benchmark suite to use.
+    #:
+    #: :type: :class:`str`
+    #: :default: ``'21.06.1'``
+    benchmark_version = variable(str, value='21.06.1', loggable=True)
+
+    #: Collect and report detailed performance metrics.
+    #:
+    #: :type: :class:`bool`
+    #: :default: ``False``
+    debug_metrics = variable(bool, value=False, loggable=True)
+
+    #: Parameter pack encoding the benchmark information.
+    #:
+    #: The first element of the tuple refers to the benchmark name,
+    #:
+    #: :type: `Tuple[str,]`
+    #: :values:
+    benchmark_info = parameter([
+        ('hackathonGPU/benchmark',),
+        ('hackathonGPU/benchmark2',),
+        ('hackathonGPU/benchmark3',),
+        ('hackathonGPU/benchmark4',),
+        ('hackathonGPU/benchmark5',),
+        ('hackathonGPU/benchmark6',),
+    ], fmt=lambda x: x[0], loggable=True)
+
+    @run_after('init')
+    def prepare_test(self):
+        """Hook to the set the downloading of the pseudo-potentials"""
+        self.__bench, = self.benchmark_info
+        self.descr = f'MetalWalls {self.__bench} benchmark'
+        files_addresses = [
+            address_tpl.format(
+                version=self.benchmark_version,
+                bench=self.__bench,
+                file=_
+            ) for _ in ['data.inpt', 'runtime.inpt']
+        ]
+        self.prerun_cmds += [
+            f"curl -LJO '{_}'" for _ in files_addresses
+        ]
+
+    @performance_function('s')
+    def total_elapsed_time(self):
+        """Extract the total elapsed time from the output file"""
+        return sn.extractsingle(
+            r'Total elapsed time:\s+(?P<time>\S+)', 'run.out', 'time', float
+            )
+
+    @performance_function('s')
+    def extract_time(
+        self, name: str = None, parent: str = None, kind: str = None
+        ) -> float:
+        """Extract the time from a specific report section of the output file
+        
+        Args:
+            name (str): The name of the report to extract
+            parent (str): The parent section of the report
+            kind (str): The kind of time to extract (avg or cumul)
+        """
+        kind = kind.lower()
+        if kind == 'avg':
+            tag = 1
+        elif kind == 'cumul':
+            tag = 2
+        else:
+            raise ValueError(f'Unknown kind: {kind}')
+        
+        # Example Fromat:
+        # Ions->Atoms Coulomb potential
+        # -----------------------------
+        #   long range                       9.55040E-03  9.64590E-01         0.71
+        #   k==0                             2.48271E-02  2.50754E+00         1.84
+        #   short range                      8.72464E-02  8.81189E+00         6.46
+        #
+        # NEXT SECTION
+        res = -1
+        rgx = re.compile(rf'^ +{name}\s+(\S+) +(\S+)')
+        flag = False
+        with open('run.out', 'r') as file:
+            for line in file:
+                if flag:
+                    if rgx.match(line):
+                        res = float(rgx.match(line).group(tag))
+                        break
+                    elif not line.strip():
+                        break
+                else:
+                    if parent in line:
+                        flag = True
+
+        return res
+
+    @run_before('performance')
+    def set_perf_variables(self):
+        """Build a dictionary of performance variables"""
+        self.perf_variables['total_elapsed_time'] = self.total_elapsed_time()
+
+        if self.debug_metrics:
+            for parent, name, short in extract_fields:
+                name2 = name.replace(' ', '_')
+                for kind in ['avg', 'cumul']:
+                    app = self.extract_time(name, parent, kind)
+                    self.perf_variables[f'{short}__{name2}__{kind}'] = app
+
+    @sanity_function
+    def assert_job_finished(self):
+        """Check if the job finished successfully"""
+        return sn.assert_found(r'Total elapsed time', 'run.out')

From 34676944d79b59bc4f9499dd5f569dfd11b14e17 Mon Sep 17 00:00:00 2001
From: crivella <davide.grassano@epfl.ch>
Date: Fri, 12 Jul 2024 16:10:24 +0200
Subject: [PATCH 005/128] Added DOC section

---
 docs/hpctestlib.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/hpctestlib.rst b/docs/hpctestlib.rst
index 599aedab5e..a1db5c6ade 100644
--- a/docs/hpctestlib.rst
+++ b/docs/hpctestlib.rst
@@ -106,6 +106,10 @@ Scientific Applications
    :members:
    :show-inheritance:
 
+.. automodule:: hpctestlib.sciapps.metalwalls.benchmarks
+   :members:
+   :show-inheritance:
+
 System
 =======================
 

From 531ce9dd4500f60547aeaa8564378af2cf407dc3 Mon Sep 17 00:00:00 2001
From: crivella <davide.grassano@epfl.ch>
Date: Fri, 12 Jul 2024 16:12:49 +0200
Subject: [PATCH 006/128] Removed unused

---
 hpctestlib/sciapps/metalwalls/benchmarks.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/hpctestlib/sciapps/metalwalls/benchmarks.py b/hpctestlib/sciapps/metalwalls/benchmarks.py
index 450a06d877..aad7a8875a 100644
--- a/hpctestlib/sciapps/metalwalls/benchmarks.py
+++ b/hpctestlib/sciapps/metalwalls/benchmarks.py
@@ -1,6 +1,5 @@
 """ReFrame benchmark for MetalWalls"""
 import re
-from typing import TypeVar
 
 import reframe as rfm
 import reframe.utility.sanity as sn
@@ -9,8 +8,6 @@
 from reframe.core.parameters import TestParam as parameter
 from reframe.core.variables import TestVar as variable
 
-R = TypeVar('R')
-
 address_tpl = (
     'https://gitlab.com/ampere2/metalwalls/-/raw/{version}/{bench}/{file}'
     '?ref_type=tags\&inline=false'

From de48ee76d074d68989eab33f4bc7e439d8e6862b Mon Sep 17 00:00:00 2001
From: crivella <davide.grassano@epfl.ch>
Date: Fri, 12 Jul 2024 16:23:25 +0200
Subject: [PATCH 007/128] PEP

---
 hpctestlib/sciapps/metalwalls/benchmarks.py | 29 ++++++++++++---------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/hpctestlib/sciapps/metalwalls/benchmarks.py b/hpctestlib/sciapps/metalwalls/benchmarks.py
index aad7a8875a..83f924331f 100644
--- a/hpctestlib/sciapps/metalwalls/benchmarks.py
+++ b/hpctestlib/sciapps/metalwalls/benchmarks.py
@@ -10,7 +10,7 @@
 
 address_tpl = (
     'https://gitlab.com/ampere2/metalwalls/-/raw/{version}/{bench}/{file}'
-    '?ref_type=tags\&inline=false'
+    '?ref_type=tags&inline=false'
 )
 
 extract_fields = [
@@ -95,7 +95,11 @@
 
     #######
     ('Ions Coulomb electric field gradient', 'long range', 'Ions Cfield grad'),
-    ('Ions Coulomb electric field gradient', 'short range', 'Ions Cfield grad'),
+    (
+        'Ions Coulomb electric field gradient',
+        'short range',
+        'Ions Cfield grad'
+    ),
     ('Ions Coulomb electric field gradient', 'self', 'Ions Cfield grad'),
 
     #######
@@ -130,11 +134,12 @@
     ('Diagnostics', 'IO', 'Diagnostics'),
 ]
 
+
 @rfm.simple_test
 class MetalWallsCheck(rfm.RunOnlyRegressionTest):
     """MetalWalls benchmark test.
-    
-    `MetalWalls <https://gitlab.com/ampere2/metalwalls>`__ is a molecular 
+
+    `MetalWalls <https://gitlab.com/ampere2/metalwalls>`__ is a molecular
     dynamics code dedicated to the modelling of electrochemical systems.
     Its main originality is the inclusion of a series of methods allowing to
     apply a constant potential within the electrode materials.
@@ -143,7 +148,7 @@ class MetalWallsCheck(rfm.RunOnlyRegressionTest):
     number of atoms and the type of simulation performed.
     They can be found in the following repository, which is also versioned:
     https://gitlab.com/ampere2/metalwalls/.
-    
+
     """
 
     #: The name of the output files to keep.
@@ -200,14 +205,14 @@ def total_elapsed_time(self):
         """Extract the total elapsed time from the output file"""
         return sn.extractsingle(
             r'Total elapsed time:\s+(?P<time>\S+)', 'run.out', 'time', float
-            )
+        )
 
     @performance_function('s')
     def extract_time(
         self, name: str = None, parent: str = None, kind: str = None
-        ) -> float:
+    ) -> float:
         """Extract the time from a specific report section of the output file
-        
+
         Args:
             name (str): The name of the report to extract
             parent (str): The parent section of the report
@@ -220,13 +225,13 @@ def extract_time(
             tag = 2
         else:
             raise ValueError(f'Unknown kind: {kind}')
-        
+
         # Example Fromat:
         # Ions->Atoms Coulomb potential
         # -----------------------------
-        #   long range                       9.55040E-03  9.64590E-01         0.71
-        #   k==0                             2.48271E-02  2.50754E+00         1.84
-        #   short range                      8.72464E-02  8.81189E+00         6.46
+        #   long range                       9.55040E-03  9.64590E-01      0.71
+        #   k==0                             2.48271E-02  2.50754E+00      1.84
+        #   short range                      8.72464E-02  8.81189E+00      6.46
         #
         # NEXT SECTION
         res = -1

From 0cf86d0a8d5c3da42e5f15de0de87056bdef6ce9 Mon Sep 17 00:00:00 2001
From: crivella <davide.grassano@epfl.ch>
Date: Mon, 15 Jul 2024 11:04:45 +0200
Subject: [PATCH 008/128] Added executable, tag and description

---
 hpctestlib/sciapps/metalwalls/benchmarks.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/hpctestlib/sciapps/metalwalls/benchmarks.py b/hpctestlib/sciapps/metalwalls/benchmarks.py
index 83f924331f..a3b83c9433 100644
--- a/hpctestlib/sciapps/metalwalls/benchmarks.py
+++ b/hpctestlib/sciapps/metalwalls/benchmarks.py
@@ -163,6 +163,10 @@ class MetalWallsCheck(rfm.RunOnlyRegressionTest):
     #: :default: ``'21.06.1'``
     benchmark_version = variable(str, value='21.06.1', loggable=True)
 
+    executable = 'mw'
+    tags = {'sciapp', 'chemistry'}
+    descr = 'MetalWalls `mw` benchmark'
+
     #: Collect and report detailed performance metrics.
     #:
     #: :type: :class:`bool`

From 15e796699c00e75410e6e39b3f55074055f80e09 Mon Sep 17 00:00:00 2001
From: crivella <davide.grassano@epfl.ch>
Date: Mon, 15 Jul 2024 16:09:19 +0200
Subject: [PATCH 009/128] Remove warning on performance function not running

---
 hpctestlib/sciapps/metalwalls/benchmarks.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/hpctestlib/sciapps/metalwalls/benchmarks.py b/hpctestlib/sciapps/metalwalls/benchmarks.py
index a3b83c9433..da75bf3559 100644
--- a/hpctestlib/sciapps/metalwalls/benchmarks.py
+++ b/hpctestlib/sciapps/metalwalls/benchmarks.py
@@ -222,6 +222,9 @@ def extract_time(
             parent (str): The parent section of the report
             kind (str): The kind of time to extract (avg or cumul)
         """
+        if kind is None:
+            return 0
+        
         kind = kind.lower()
         if kind == 'avg':
             tag = 1

From d192c3f53119f8950ce9a24955e2988b383d2265 Mon Sep 17 00:00:00 2001
From: crivella <davide.grassano@epfl.ch>
Date: Mon, 15 Jul 2024 16:09:48 +0200
Subject: [PATCH 010/128] Add sanity check on expected final energy and
 temperature

---
 hpctestlib/sciapps/metalwalls/benchmarks.py | 45 ++++++++++++++++-----
 1 file changed, 36 insertions(+), 9 deletions(-)

diff --git a/hpctestlib/sciapps/metalwalls/benchmarks.py b/hpctestlib/sciapps/metalwalls/benchmarks.py
index da75bf3559..d6f0b1ff92 100644
--- a/hpctestlib/sciapps/metalwalls/benchmarks.py
+++ b/hpctestlib/sciapps/metalwalls/benchmarks.py
@@ -176,22 +176,25 @@ class MetalWallsCheck(rfm.RunOnlyRegressionTest):
     #: Parameter pack encoding the benchmark information.
     #:
     #: The first element of the tuple refers to the benchmark name,
+    #: the second is the final kinetic energy the third is the related 
+    #: tolerance, the fourth is the absolute temperature and the fifth is
+    #: the related tolerance
     #:
-    #: :type: `Tuple[str,]`
+    #: :type: `Tuple[str, float, float, float, float]`
     #: :values:
     benchmark_info = parameter([
-        ('hackathonGPU/benchmark',),
-        ('hackathonGPU/benchmark2',),
-        ('hackathonGPU/benchmark3',),
-        ('hackathonGPU/benchmark4',),
-        ('hackathonGPU/benchmark5',),
-        ('hackathonGPU/benchmark6',),
+        ('hackathonGPU/benchmark', 14.00, 0.05, 301.74, 0.5),
+        ('hackathonGPU/benchmark2', 14.00, 0.05, 301.74, 0.5),
+        ('hackathonGPU/benchmark3', 16.08, 0.05, 293.42, 0.5),
+        ('hackathonGPU/benchmark4', 16.08, 0.05, 293.42, 0.5),
+        ('hackathonGPU/benchmark5', 25.72, 0.05, 297.47, 0.5),
+        ('hackathonGPU/benchmark6', 25.72, 0.05, 297.47, 0.5),
     ], fmt=lambda x: x[0], loggable=True)
 
     @run_after('init')
     def prepare_test(self):
         """Hook to the set the downloading of the pseudo-potentials"""
-        self.__bench, = self.benchmark_info
+        self.__bench, _, _, _, _ = self.benchmark_info
         self.descr = f'MetalWalls {self.__bench} benchmark'
         files_addresses = [
             address_tpl.format(
@@ -210,6 +213,21 @@ def total_elapsed_time(self):
         return sn.extractsingle(
             r'Total elapsed time:\s+(?P<time>\S+)', 'run.out', 'time', float
         )
+    
+    @sn.deferrable
+    def extract_kinetic_energy(self):
+        """Extract the final kinetic energy from the output file"""
+        rgx = r'\|step\| +kinetic energy: +(?P<flag>\S+)'
+        app = sn.extractall(rgx, 'run.out', 'flag', float)
+        return app[-1]
+    
+    @sn.deferrable
+    def extract_temperature(self):
+        """Extract the final temperature from the output file"""
+        rgx = r'\|step\| +temperature: +(?P<flag>\S+)'
+        app = sn.extractall(rgx, 'run.out', 'flag', float)
+        return app[-1]
+
 
     @performance_function('s')
     def extract_time(
@@ -273,4 +291,13 @@ def set_perf_variables(self):
     @sanity_function
     def assert_job_finished(self):
         """Check if the job finished successfully"""
-        return sn.assert_found(r'Total elapsed time', 'run.out')
+        energy = self.extract_kinetic_energy()
+        temp = self.extract_temperature()
+        _, energy_ref, energy_tol, temp_ref, temp_tol = self.benchmark_info
+        en_rtol = energy_tol / energy_ref
+        t_rtol = temp_tol / temp_ref
+        return sn.all([
+            sn.assert_found(r'Total elapsed time', 'run.out'),
+            sn.assert_reference(energy, energy_ref, -en_rtol, en_rtol),
+            sn.assert_reference(temp, temp_ref, -t_rtol, t_rtol)
+        ])

From 20ccf7af2187853e9b75af31cfb477060da5165a Mon Sep 17 00:00:00 2001
From: crivella <davide.grassano@epfl.ch>
Date: Mon, 15 Jul 2024 16:16:00 +0200
Subject: [PATCH 011/128] PEP

---
 hpctestlib/sciapps/metalwalls/benchmarks.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/hpctestlib/sciapps/metalwalls/benchmarks.py b/hpctestlib/sciapps/metalwalls/benchmarks.py
index d6f0b1ff92..8d250bb7c1 100644
--- a/hpctestlib/sciapps/metalwalls/benchmarks.py
+++ b/hpctestlib/sciapps/metalwalls/benchmarks.py
@@ -176,7 +176,7 @@ class MetalWallsCheck(rfm.RunOnlyRegressionTest):
     #: Parameter pack encoding the benchmark information.
     #:
     #: The first element of the tuple refers to the benchmark name,
-    #: the second is the final kinetic energy the third is the related 
+    #: the second is the final kinetic energy the third is the related
     #: tolerance, the fourth is the absolute temperature and the fifth is
     #: the related tolerance
     #:
@@ -213,14 +213,14 @@ def total_elapsed_time(self):
         return sn.extractsingle(
             r'Total elapsed time:\s+(?P<time>\S+)', 'run.out', 'time', float
         )
-    
+
     @sn.deferrable
     def extract_kinetic_energy(self):
         """Extract the final kinetic energy from the output file"""
         rgx = r'\|step\| +kinetic energy: +(?P<flag>\S+)'
         app = sn.extractall(rgx, 'run.out', 'flag', float)
         return app[-1]
-    
+
     @sn.deferrable
     def extract_temperature(self):
         """Extract the final temperature from the output file"""
@@ -228,7 +228,6 @@ def extract_temperature(self):
         app = sn.extractall(rgx, 'run.out', 'flag', float)
         return app[-1]
 
-
     @performance_function('s')
     def extract_time(
         self, name: str = None, parent: str = None, kind: str = None
@@ -242,7 +241,7 @@ def extract_time(
         """
         if kind is None:
             return 0
-        
+
         kind = kind.lower()
         if kind == 'avg':
             tag = 1

From 22cb34014779f5ce3ff542d323148a45d34388c7 Mon Sep 17 00:00:00 2001
From: Theofilos Manitaras <manitaras@cscs.ch>
Date: Tue, 23 Jul 2024 15:19:00 +0200
Subject: [PATCH 012/128] Fix distribute documentation bug

Signed-off-by: Theofilos Manitaras <manitaras@cscs.ch>
---
 docs/manpage.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index 44054f6555..6bcea6b6b5 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -874,7 +874,7 @@ The way the tests are generated and how they interact with the test filtering op
    - ``avail``: Tests will run on all the nodes of their respective valid partitions that are available for running jobs.
      Note that if a node is currently allocated to another job it is still considered as "available."
    - ``NODESTATE``: Tests will run on all the nodes of their respective valid partitions that are exclusively in state ``NODESTATE``.
-     aIf ``NODESTATE`` is not specified, ``idle`` is assumed.
+     If ``NODESTATE`` is not specified, ``idle`` is assumed.
    - ``NODESTATE*``: Tests will run on all the nodes of their respective valid partitions that are at least in state ``NODESTATE``.
 
    The state of the nodes will be determined once, before beginning the

From 91396e7cbf17097d8dc7b9e47448ff40b34db034 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 1 Aug 2024 15:26:17 +0000
Subject: [PATCH 013/128] Bump pytest from 8.2.2 to 8.3.2

Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.2.2 to 8.3.2.
- [Release notes](https://github.com/pytest-dev/pytest/releases)
- [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest/compare/8.2.2...8.3.2)

---
updated-dependencies:
- dependency-name: pytest
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 6d4b5c79dd..f975d4c81f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,7 @@ jsonschema==3.2.0
 lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64'
 lxml==5.2.2; python_version >= '3.8' or platform_machine != 'aarch64'
 pytest==7.0.1; python_version < '3.8'
-pytest==8.2.2; python_version >= '3.8'
+pytest==8.3.2; python_version >= '3.8'
 pytest-forked==1.4.0; python_version == '3.6'
 pytest-forked==1.6.0; python_version >= '3.7'
 pytest-parallel==0.1.1

From b1a1c09bf99a851a26f6e6ab152a33ca8f2d599e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 5 Aug 2024 14:42:09 +0000
Subject: [PATCH 014/128] Bump setuptools from 59.6.0 to 72.1.0

Bumps [setuptools](https://github.com/pypa/setuptools) from 59.6.0 to 72.1.0.
- [Release notes](https://github.com/pypa/setuptools/releases)
- [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst)
- [Commits](https://github.com/pypa/setuptools/compare/v59.6.0...v72.1.0)

---
updated-dependencies:
- dependency-name: setuptools
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index f975d4c81f..0e9fc5409b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -21,6 +21,6 @@ semver==2.13.0; python_version == '3.6'
 semver==3.0.2; python_version >= '3.7'
 setuptools==59.6.0; python_version == '3.6'
 setuptools==68.0.0; python_version == '3.7'
-setuptools==70.1.1; python_version >= '3.8'
+setuptools==72.1.0; python_version >= '3.8'
 wcwidth==0.2.13
 #+pygelf%pygelf==0.4.0

From 5e90f6918917452d252d28c3382dd16a8a3f8e91 Mon Sep 17 00:00:00 2001
From: Eirini Koutsaniti <ekoutsaniti@gmail.com>
Date: Tue, 14 May 2024 11:07:19 +0200
Subject: [PATCH 015/128] Index job reports

---
 reframe/frontend/cli.py       |  17 +++-
 reframe/frontend/runreport.py | 176 ++++++++++++++++++++++++++++++----
 reframe/schemas/config.json   |   2 +
 3 files changed, 176 insertions(+), 19 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 2fe96a3eb6..56ee2f1e00 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -564,6 +564,10 @@ def main():
         metavar='PARAM',
         help='Print the value of configuration parameter PARAM and exit'
     )
+    misc_options.add_argument(
+        '--index-db', action='store_true',
+        help='Index old job reports in the database',
+    )
     misc_options.add_argument(
         '--system', action='store', help='Load configuration for SYSTEM',
         envvar='RFM_SYSTEM'
@@ -906,14 +910,23 @@ def restrict_logging():
                 with open(topofile, 'w') as fp:
                     json.dump(s_cpuinfo, fp, indent=2)
                     fp.write('\n')
-            except OSError as e:
-                getlogger().error(
+            except OSError:
+                logging.getlogger().error(
                     f'could not write topology file: {topofile!r}'
                 )
                 sys.exit(1)
 
         sys.exit(0)
 
+    if options.index_db:
+        try:
+            runreport.reindex_db()
+        except Exception as e:
+            printer.error(f'could not index old job reports: {e}')
+            sys.exit(1)
+
+        sys.exit(0)
+
     autodetect.detect_topology()
     printer.debug(format_env(options.env_vars))
 
diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py
index 47b59df0c7..d0bdbe0d48 100644
--- a/reframe/frontend/runreport.py
+++ b/reframe/frontend/runreport.py
@@ -5,14 +5,17 @@
 
 import decimal
 import functools
+import glob
 import json
 import jsonschema
 import lxml.etree as etree
 import os
 import re
+import sqlite3
 
 import reframe as rfm
 import reframe.core.exceptions as errors
+import reframe.core.runtime as runtime
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
 from reframe.core.logging import getlogger
@@ -190,26 +193,41 @@ def write_report(report, filename, compress=False, link_to_last=False):
             jsonext.dump(report, fp, indent=2)
             fp.write('\n')
 
-        if not link_to_last:
-            return
+    # Try to save the report in the database
+    try:
+        site_config = runtime.runtime().site_config
+        database_file = osext.expandvars(
+            site_config.get('general/0/report_database_file')
+        )
+        con, cur = initialize_db(database_file)
+        add_report_in_db(report, cur, filename)
+        con.commit()
+        con.close()
+    except Exception as e:
+        getlogger().error(
+            f"Could not save report {filename} in the database: {e}"
+        )
 
-        # Add a symlink to the latest report
-        basedir = os.path.dirname(filename)
-        with osext.change_dir(basedir):
-            link_name = 'latest.json'
-            create_symlink = functools.partial(
-                os.symlink, os.path.basename(filename), link_name
-            )
-            if not os.path.exists(link_name):
+    if not link_to_last:
+        return
+
+    # Add a symlink to the latest report
+    basedir = os.path.dirname(filename)
+    with osext.change_dir(basedir):
+        link_name = 'latest.json'
+        create_symlink = functools.partial(
+            os.symlink, os.path.basename(filename), link_name
+        )
+        if not os.path.exists(link_name):
+            create_symlink()
+        else:
+            if os.path.islink(link_name):
+                os.remove(link_name)
                 create_symlink()
             else:
-                if os.path.islink(link_name):
-                    os.remove(link_name)
-                    create_symlink()
-                else:
-                    getlogger().warning('could not create a symlink '
-                                        'to the latest report file: '
-                                        'path exists and is not a symlink')
+                getlogger().warning('could not create a symlink '
+                                    'to the latest report file: '
+                                    'path exists and is not a symlink')
 
 
 def junit_xml_report(json_report):
@@ -270,3 +288,127 @@ def junit_dump(xml, fp):
         etree.tostring(xml, encoding='utf8', pretty_print=True,
                        method='xml', xml_declaration=True).decode()
     )
+
+
+def get_reports_files(directory):
+    return [f for f in glob.glob(f"{directory}/*")
+            if os.path.isfile(f) and not f.endswith('/latest.json')]
+
+
+def initialize_db(database_file):
+    con = sqlite3.connect(database_file)
+    cur = con.cursor()
+    cur.execute("""
+    CREATE TABLE IF NOT EXISTS testcases(
+        name TEXT,
+        hash TEXT,
+        system TEXT,
+        partition TEXT,
+        environment TEXT,
+        time_start TEXT,
+        time_end TEXT,
+        run_index INTEGER,
+        test_index INTEGER,
+        json_file TEXT
+    )
+    """)
+    con.commit()
+    return (con, cur)
+
+
+def reindex_db():
+    site_config = runtime.runtime().site_config
+    database_file = osext.expandvars(
+        site_config.get('general/0/report_database_file')
+    )
+    # Check if the database file exists and remove it
+    if os.path.exists(database_file):
+        os.remove(database_file)
+
+    con, cur = initialize_db(database_file)
+
+    # Populate the database with old reports
+    reports_directory = os.path.dirname(
+        osext.expandvars(site_config.get('general/0/report_file'))
+    )
+    for f_path in get_reports_files(reports_directory):
+        with open(f_path, 'r') as file:
+            report = json.load(file)
+            #TODO: Validate report with schema
+
+        add_report_in_db(report, cur, f_path)
+        con.commit()
+
+    con.close()
+
+
+def add_report_in_db(report, cursor, report_file_path):
+    time_start = report['session_info']['time_start']
+    time_end = report['session_info']['time_end']
+    for run_index, run in enumerate(report['runs']):
+        for test_index, test in enumerate(run['testcases']):
+            name = test['name']
+            hash = test['hash']
+            system = test['system'].split(':')[0]
+            partition = test['system'].split(':')[1]
+            environment = test['environment']
+            run_index = run_index
+            test_index = test_index
+            json_file = report_file_path
+
+            cursor.execute(
+                (
+                    "INSERT INTO testcases VALUES(?, ?, ?, ?, ?, ?, ?, ?, "
+                    "?, ?)"
+                ),
+                (
+                    name,
+                    hash,
+                    system,
+                    partition,
+                    environment,
+                    time_start,
+                    time_end,
+                    run_index,
+                    test_index,
+                    json_file
+                )
+            )
+
+
+def search_db(name=None, hash=None, system=None, partition=None,
+              environment=None, time_start=None, time_end=None,
+              run_index=None, test_index=None):
+    site_config = runtime.runtime().site_config
+    database_file = osext.expandvars(
+        site_config.get('general/0/report_database_file')
+    )
+    con = sqlite3.connect(database_file)
+    cur = con.cursor()
+
+    query = "SELECT test_index, run_index, json_file FROM testcases"
+    first_condition = False
+    for condition in ['name', 'hash', 'system', 'partition', 'environment',
+                      'time_start', 'time_end', 'run_index', 'test_index']:
+        value = eval(condition)
+        if value:
+            if not first_condition:
+                query += " WHERE "
+                first_condition = True
+            else:
+                query += " AND "
+            query += f"{condition}='{eval(condition)}'"
+
+    res = cur.execute(query)
+    results = res.fetchall()
+    con.close()
+
+    # Retrieve files
+    testcases = []
+    for test_index, run_index, json_file in results:
+        with open(json_file, 'r') as file:
+            report = json.load(file)
+
+        testcases.append(report['runs'][run_index]['testcases'][test_index])
+
+    return testcases
diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index 61d9c58e18..2dcbec053c 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -518,6 +518,7 @@
                     "purge_environment": {"type": "boolean"},
                     "remote_detect": {"type": "boolean"},
                     "remote_workdir": {"type": "string"},
+                    "report_database_file": {"type": "string"},
                     "report_file": {"type": "string"},
                     "report_junit": {"type": ["string", "null"]},
                     "resolve_module_conflicts": {"type": "boolean"},
@@ -569,6 +570,7 @@
         "general/purge_environment": false,
         "general/remote_detect": false,
         "general/remote_workdir": ".",
+        "general/report_database_file": "${HOME}/.reframe/reports_index.db",
         "general/report_file": "${HOME}/.reframe/reports/run-report-{sessionid}.json",
         "general/report_junit": null,
         "general/resolve_module_conflicts": true,

From 36a3efdd1d68d8e9418cb917d21c316c37413d3d Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 22 May 2024 10:56:27 +0200
Subject: [PATCH 016/128] Refine report indexing

---
 reframe/core/logging.py                |   2 +-
 reframe/frontend/cli.py                |  50 +++---
 reframe/frontend/executors/__init__.py |  13 ++
 reframe/frontend/runreport.py          | 202 +++++++++----------------
 reframe/frontend/statistics.py         |  79 +++-------
 reframe/schemas/runreport.json         |  61 +-------
 6 files changed, 141 insertions(+), 266 deletions(-)

diff --git a/reframe/core/logging.py b/reframe/core/logging.py
index d34006fed1..e52de9e221 100644
--- a/reframe/core/logging.py
+++ b/reframe/core/logging.py
@@ -862,7 +862,7 @@ def log_performance(self, level, task, msg=None, multiline=False):
 
         self.extra['check_partition'] = task.testcase.partition.name
         self.extra['check_environ'] = task.testcase.environ.name
-        self.extra['check_result'] = 'pass' if task.succeeded else 'fail'
+        self.extra['check_result'] = task.result
         fail_reason = what(*task.exc_info) if not task.succeeded else None
         self.extra['check_fail_reason'] = fail_reason
         if msg is None:
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 56ee2f1e00..f9ed7ccb6c 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -584,6 +584,10 @@ def main():
         '-q', '--quiet', action='count', default=0,
         help='Decrease verbosity level of output',
     )
+    misc_options.add_argument(
+        '--fetch-cases', action='store',
+        help='Experimental: query report'
+    )
 
     # Options not associated with command-line arguments
     argparser.add_argument(
@@ -727,7 +731,7 @@ def restrict_logging():
 
         '''
 
-        if (options.show_config or
+        if (options.show_config or options.fetch_cases or
             options.detect_host_topology or options.describe):
             logging.getlogger().setLevel(logging.ERROR)
             return True
@@ -918,18 +922,14 @@ def restrict_logging():
 
         sys.exit(0)
 
-    if options.index_db:
-        try:
-            runreport.reindex_db()
-        except Exception as e:
-            printer.error(f'could not index old job reports: {e}')
-            sys.exit(1)
-
-        sys.exit(0)
-
     autodetect.detect_topology()
     printer.debug(format_env(options.env_vars))
 
+    if options.fetch_cases:
+        testcases = runreport.fetch_cases_raw(options.fetch_cases)
+        print(jsonext.dumps(testcases, indent=2))
+        sys.exit(0)
+
     # Setup the check loader
     if options.restore_session is not None:
         # We need to load the failed checks only from a list of reports
@@ -1478,19 +1478,35 @@ def module_unuse(*paths):
                 for c in restored_cases:
                     json_report['restored_cases'].append(report.case(*c))
 
+            # Save the report file
             report_file = runreport.next_report_filename(report_file)
-            default_loc = os.path.dirname(
-                osext.expandvars(rt.get_default('general/report_file'))
-            )
             try:
-                runreport.write_report(json_report, report_file,
-                                       rt.get_option(
-                                           'general/0/compress_report'),
-                                       os.path.dirname(report_file) == default_loc)
+                runreport.save_report(
+                    json_report, report_file,
+                    rt.get_option('general/0/compress_report')
+                )
             except OSError as e:
                 printer.warning(
                     f'failed to generate report in {report_file!r}: {e}'
                 )
+            else:
+                default_loc = os.path.dirname(
+                    osext.expandvars(rt.get_default('general/report_file'))
+                )
+                if default_loc == os.path.dirname(report_file):
+                    try:
+                        runreport.link_latest_report(report_file, 'latest.json')
+                    except Exception as e:
+                        printer.warning(
+                            f'failed to create symlink to latest report: {e}'
+                        )
+
+            # Index the generated report
+            try:
+                runreport.index_report(json_report, report_file)
+            except Exception as e:
+                printer.warning(f'failed to index the report: {e}')
+                raise
 
             # Generate the junit xml report for this session
             junit_report_file = rt.get_option('general/0/report_junit')
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index dd95746987..6c741d6e61 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -282,6 +282,19 @@ def aborted(self):
     def skipped(self):
         return self._skipped
 
+    @property
+    def result(self):
+        if self.succeeded:
+            return 'pass'
+        elif self.failed:
+            return 'fail'
+        elif self.aborted:
+            return 'abort'
+        elif self.skipped:
+            return 'skip'
+        else:
+            return '<unknown>'
+
     def _notify_listeners(self, callback_name):
         for l in self._listeners:
             callback = getattr(l, callback_name)
diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py
index d0bdbe0d48..1368ea33e9 100644
--- a/reframe/frontend/runreport.py
+++ b/reframe/frontend/runreport.py
@@ -24,7 +24,7 @@
 # The schema data version
 # Major version bumps are expected to break the validation of previous schemas
 
-DATA_VERSION = '3.1'
+DATA_VERSION = '4.0'
 _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json')
 
 
@@ -169,7 +169,7 @@ def _load_report(filename):
         raise errors.ReframeError(
             f'invalid report {filename!r} '
             f'(required data version: {DATA_VERSION}), found: {found_ver})'
-        ) from e
+        )
 
     return _RunReport(report)
 
@@ -185,7 +185,7 @@ def load_report(*filenames):
     return rpt
 
 
-def write_report(report, filename, compress=False, link_to_last=False):
+def save_report(report, filename, compress=False):
     with open(filename, 'w') as fp:
         if compress:
             jsonext.dump(report, fp)
@@ -193,31 +193,10 @@ def write_report(report, filename, compress=False, link_to_last=False):
             jsonext.dump(report, fp, indent=2)
             fp.write('\n')
 
-    # Try to save the report in the database
-    try:
-        site_config = runtime.runtime().site_config
-        database_file = osext.expandvars(
-            site_config.get('general/0/report_database_file')
-        )
-        con, cur = initialize_db(database_file)
-        add_report_in_db(report, cur, filename)
-        con.commit()
-        con.close()
-    except Exception as e:
-        getlogger().error(
-            f"Could not save report {filename} in the database: {e}"
-        )
-
-    if not link_to_last:
-        return
-
-    # Add a symlink to the latest report
-    basedir = os.path.dirname(filename)
-    with osext.change_dir(basedir):
-        link_name = 'latest.json'
-        create_symlink = functools.partial(
-            os.symlink, os.path.basename(filename), link_name
-        )
+def link_latest_report(filename, link_name):
+    prefix, target_name = os.path.split(filename)
+    with osext.change_dir(prefix):
+        create_symlink = functools.partial(os.symlink, target_name, link_name)
         if not os.path.exists(link_name):
             create_symlink()
         else:
@@ -225,9 +204,7 @@ def write_report(report, filename, compress=False, link_to_last=False):
                 os.remove(link_name)
                 create_symlink()
             else:
-                getlogger().warning('could not create a symlink '
-                                    'to the latest report file: '
-                                    'path exists and is not a symlink')
+                raise errors.ReframeError('path exists and is not a symlink')
 
 
 def junit_xml_report(json_report):
@@ -251,11 +228,8 @@ def junit_xml_report(json_report):
                 'timestamp': json_report['session_info']['time_start'][:-5],
             }
         )
-        testsuite_properties = etree.SubElement(xml_testsuite, 'properties')
         for tc in rfm_run['testcases']:
-            casename = (
-                f"{tc['unique_name']}[{tc['system']}, {tc['environment']}]"
-            )
+            casename = f"{tc['name']} @{tc['system']}+{tc['environ']}"
             testcase = etree.SubElement(
                 xml_testsuite, 'testcase',
                 attrib={
@@ -268,10 +242,10 @@ def junit_xml_report(json_report):
                     'time': str(decimal.Decimal(tc['time_total'] or 0)),
                 }
             )
-            if tc['result'] == 'failure':
+            if tc['result'] == 'fail':
                 testcase_msg = etree.SubElement(
-                    testcase, 'failure', attrib={'type': 'failure',
-                                                 'message': tc['fail_phase']}
+                    testcase, 'fail', attrib={'type': 'fail',
+                                              'message': tc['fail_phase']}
                 )
                 testcase_msg.text = f"{tc['fail_phase']}: {tc['fail_reason']}"
 
@@ -295,113 +269,77 @@ def get_reports_files(directory):
             if os.path.isfile(f) and not f.endswith('/latest.json')]
 
 
-def initialize_db(database_file):
-    con = sqlite3.connect(database_file)
-    cur = con.cursor()
-    cur.execute("""
-    CREATE TABLE IF NOT EXISTS testcases(
+def index_report(report, filename):
+    prefix = os.path.dirname(filename)
+    index_file = os.path.join(prefix, 'index.db')
+    if not os.path.exists(index_file):
+        getlogger().info('Re-building the report index...')
+        _build_index(index_file)
+
+    with sqlite3.connect(index_file) as conn:
+        _index_report(conn, report, filename)
+
+
+def _build_index(filename):
+    with sqlite3.connect(filename) as conn:
+        conn.execute(
+'''CREATE TABLE IF NOT EXISTS testcases(
         name TEXT,
-        hash TEXT,
         system TEXT,
         partition TEXT,
-        environment TEXT,
+        environ TEXT,
         time_start TEXT,
         time_end TEXT,
         run_index INTEGER,
         test_index INTEGER,
-        json_file TEXT
-    )
-    """)
-    con.commit()
-    return (con, cur)
-
-
-def reindex_db():
-    site_config = runtime.runtime().site_config
-    database_file = osext.expandvars(
-        site_config.get('general/0/report_database_file')
-    )
-    # Check if the database file exists and remove it
-    if os.path.exists(database_file):
-        os.remove(database_file)
-
-    con, cur = initialize_db(database_file)
-
-    # Populate the database with old reports
-    reports_directory = os.path.dirname(
-        osext.expandvars(site_config.get('general/0/report_file'))
-    )
-    for f_path in get_reports_files(reports_directory):
-        with open(f_path, 'r') as file:
-            report = json.load(file)
-            #TODO: Validate report with schema
-
-        add_report_in_db(report, cur, f_path)
-        con.commit()
-
-    con.close()
-
-
-def add_report_in_db(report, cursor, report_file_path):
+        report_file TEXT
+)'''
+        )
+        prefix = os.path.dirname(filename)
+        for report_file in glob.iglob(f'{prefix}/*'):
+            if not os.path.islink(report_file):
+                try:
+                    report = load_report(report_file)
+                # except errors.ReframeError as e:
+                except Exception as e:
+                    getlogger().info(f'ignoring report file {report_file!r}: {e}')
+                else:
+                    _index_report(conn, report, report_file)
+
+
+def _index_report(conn, report, report_file_path):
     time_start = report['session_info']['time_start']
     time_end = report['session_info']['time_end']
-    for run_index, run in enumerate(report['runs']):
-        for test_index, test in enumerate(run['testcases']):
-            name = test['name']
-            hash = test['hash']
-            system = test['system'].split(':')[0]
-            partition = test['system'].split(':')[1]
-            environment = test['environment']
-            run_index = run_index
-            test_index = test_index
-            json_file = report_file_path
-
-            cursor.execute(
-                (
-                    "INSERT INTO testcases VALUES(?, ?, ?, ?, ?, ?, ?, ?, "
-                    "?, ?)"
-                ),
-                (
-                    name,
-                    hash,
-                    system,
-                    partition,
-                    environment,
-                    time_start,
-                    time_end,
-                    run_index,
-                    test_index,
-                    json_file
-                )
+    for run_idx, run in enumerate(report['runs']):
+        for test_idx, testcase in enumerate(run['testcases']):
+            sys, part = testcase['system'], testcase['partition']
+            conn.execute(
+'''INSERT INTO testcases VALUES(:name, :system, :partition,
+                                :environ, :time_start, :time_end,
+                                :run_index, :test_index, :report_file)''',
+                {
+                    'name': testcase['name'],
+                    'system': sys,
+                    'partition': part,
+                    'environ': testcase['environ'],
+                    'run_index': run_idx,
+                    'time_start': time_start,
+                    'time_end': time_end,
+                    'test_index': test_idx,
+                    'report_file': report_file_path
+                }
             )
 
 
-def search_db(name=None, hash=None, system=None, partition=None,
-              environment=None, time_start=None, time_end=None,
-              run_index=None, test_index=None):
+def fetch_cases_raw(condition):
     site_config = runtime.runtime().site_config
-    database_file = osext.expandvars(
-        site_config.get('general/0/report_database_file')
-    )
-    con = sqlite3.connect(database_file)
-    cur = con.cursor()
-
-    query = "SELECT test_index, run_index, json_file FROM testcases"
-    first_condition = False
-    for condition in ['name', 'hash', 'system', 'partition', 'environment',
-                      'time_start', 'time_end', 'run_index', 'test_index']:
-        value = eval(condition)
-        if value:
-            if not first_condition:
-                query += " WHERE "
-                first_condition = True
-            else:
-                query += " AND "
-            query += f"{condition}='{eval(condition)}'"
-
-    res = cur.execute(query)
-    results = res.fetchall()
-    con.close()
+    prefix = os.path.dirname(osext.expandvars(
+        site_config.get('general/0/report_file')
+    ))
+    with sqlite3.connect(os.path.join(prefix, 'index.db')) as conn:
+        query = f'SELECT test_index, run_index, report_file FROM testcases where {condition}'
+        getlogger().debug(query)
+        results = conn.execute(query).fetchall()
 
     # Retrieve files
     testcases = []
diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py
index 84b3d129cd..c13c4409e7 100644
--- a/reframe/frontend/statistics.py
+++ b/reframe/frontend/statistics.py
@@ -110,6 +110,7 @@ def json(self, force=False):
                 check = t.check
                 partition = check.current_partition
                 entry = {
+                    'build_jobid': None,
                     'build_stderr': None,
                     'build_stdout': None,
                     'dependencies_actual': [
@@ -120,35 +121,20 @@ def json(self, force=False):
                     'dependencies_conceptual': [
                         d[0] for d in t.check.user_deps()
                     ],
-                    'description': check.descr,
-                    'display_name': check.display_name,
-                    'environment': None,
                     'fail_phase': None,
                     'fail_reason': None,
                     'filename': inspect.getfile(type(check)),
                     'fixture': check.is_fixture(),
-                    'hash': check.hashcode,
-                    'jobid': None,
                     'job_stderr': None,
                     'job_stdout': None,
-                    'maintainers': check.maintainers,
-                    'name': check.name,
-                    'nodelist': [],
-                    'outputdir': None,
-                    'perfvars': None,
-                    'prefix': check.prefix,
                     'result': None,
-                    'stagedir': check.stagedir,
                     'scheduler': None,
-                    'system': check.current_system.name,
-                    'tags': list(check.tags),
                     'time_compile': t.duration('compile_complete'),
                     'time_performance': t.duration('performance'),
                     'time_run': t.duration('run_complete'),
                     'time_sanity': t.duration('sanity'),
                     'time_setup': t.duration('setup'),
-                    'time_total': t.duration('total'),
-                    'unique_name': check.unique_name
+                    'time_total': t.duration('total')
                 }
 
                 # We take partition and environment from the test case and not
@@ -156,25 +142,24 @@ def json(self, force=False):
                 # these are not set inside the check.
                 partition = t.testcase.partition
                 environ = t.testcase.environ
-                entry['system'] = partition.fullname
+                entry['partition'] = partition.name
+                entry['environ'] = environ.name
                 entry['scheduler'] = partition.scheduler.registered_name
-                entry['environment'] = environ.name
                 if check.job:
-                    entry['jobid'] = str(check.job.jobid)
                     entry['job_stderr'] = check.stderr.evaluate()
                     entry['job_stdout'] = check.stdout.evaluate()
-                    entry['nodelist'] = check.job.nodelist or []
 
                 if check.build_job:
                     entry['build_stderr'] = check.build_stderr.evaluate()
                     entry['build_stdout'] = check.build_stdout.evaluate()
 
+                entry['result'] = t.result
                 if t.failed:
                     num_failures += 1
-                    entry['result'] = 'failure'
                 elif t.aborted:
-                    entry['result'] = 'aborted'
                     num_aborted += 1
+                elif t.skipped:
+                    num_skipped += 1
 
                 if t.failed or t.aborted:
                     entry['fail_phase'] = t.failed_stage
@@ -186,43 +171,17 @@ def json(self, force=False):
                             'traceback': t.exc_info[2]
                         }
                         entry['fail_severe'] = errors.is_severe(*t.exc_info)
-                elif t.skipped:
-                    entry['result'] = 'skipped'
-                    num_skipped += 1
-                else:
-                    entry['result'] = 'success'
+                elif t.succeeded:
                     entry['outputdir'] = check.outputdir
 
-                if check.perfvalues:
-                    # Record performance variables
-                    entry['perfvars'] = []
-                    for key, ref in check.perfvalues.items():
-                        var = key.split(':')[-1]
-                        val, ref, lower, upper, unit = ref
-                        entry['perfvars'].append({
-                            'name': var,
-                            'reference': ref,
-                            'thres_lower': lower,
-                            'thres_upper': upper,
-                            'unit': unit,
-                            'value': val
-                        })
-
                 # Add any loggable variables and parameters
-                entry['check_vars'] = {}
-                test_cls = type(check)
-                for name, var in test_cls.var_space.items():
-                    if var.is_loggable():
-                        try:
-                            entry['check_vars'][name] = _getattr(check, name)
-                        except AttributeError:
-                            entry['check_vars'][name] = '<undefined>'
-
-                entry['check_params'] = {}
                 test_cls = type(check)
-                for name, param in test_cls.param_space.items():
-                    if param.is_loggable():
-                        entry['check_params'][name] = _getattr(check, name)
+                for name, alt_name in test_cls.loggable_attrs():
+                    key = alt_name if alt_name else name
+                    try:
+                        entry[key] = _getattr(check, name)
+                    except AttributeError:
+                        entry[key] = '<undefined>'
 
                 testcases.append(entry)
 
@@ -264,12 +223,12 @@ def _print_failure_info(rec, runid, total_runs):
             printer.info(line_width * '-')
             printer.info(f"FAILURE INFO for {rec['display_name']} "
                          f"(run: {runid}/{total_runs})")
-            printer.info(f"  * Description: {rec['description']}")
+            printer.info(f"  * Description: {rec['descr']}")
             printer.info(f"  * System partition: {rec['system']}")
-            printer.info(f"  * Environment: {rec['environment']}")
+            printer.info(f"  * Environment: {rec['environ']}")
             printer.info(f"  * Stage directory: {rec['stagedir']}")
             printer.info(
-                f"  * Node list: {util.nodelist_abbrev(rec['nodelist'])}"
+                f"  * Node list: {util.nodelist_abbrev(rec['job_nodelist'])}"
             )
             job_type = 'local' if rec['scheduler'] == 'local' else 'batch job'
             printer.info(f"  * Job type: {job_type} (id={rec['jobid']})")
@@ -280,8 +239,8 @@ def _print_failure_info(rec, runid, total_runs):
             printer.info(f"  * Maintainers: {rec['maintainers']}")
             printer.info(f"  * Failing phase: {rec['fail_phase']}")
             if rerun_info and not rec['fixture']:
-                printer.info(f"  * Rerun with '-n /{rec['hash']}"
-                             f" -p {rec['environment']} --system "
+                printer.info(f"  * Rerun with '-n /{rec['hashcode']}"
+                             f" -p {rec['environ']} --system "
                              f"{rec['system']} -r'")
 
             msg = rec['fail_reason']
diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json
index 7c1efca680..e989559476 100644
--- a/reframe/schemas/runreport.json
+++ b/reframe/schemas/runreport.json
@@ -6,10 +6,9 @@
         "testcase_type": {
             "type": "object",
             "properties": {
+                "build_jobid": {"type": ["string", "null"]},
                 "build_stderr": {"type": ["string", "null"]},
                 "build_stdout": {"type": ["string", "null"]},
-                "check_params": {"type": "object"},
-                "check_vars": {"type": "object"},
                 "dependencies_actual": {
                     "type": "array",
                     "items": {
@@ -23,9 +22,6 @@
                     "type": "array",
                     "items": {"type": "string"}
                 },
-                "description": {"type": "string"},
-                "display_name": {"type": "string"},
-                "environment": {"type": ["string", "null"]},
                 "fail_info": {
                     "type": ["object", "null"],
                     "properties": {
@@ -43,65 +39,18 @@
                 "fail_severe": {"type": "boolean"},
                 "filename": {"type": "string"},
                 "fixture": {"type": "boolean"},
-                "jobid": {"type": ["string", "null"]},
                 "job_stderr": {"type": ["string", "null"]},
                 "job_stdout": {"type": ["string", "null"]},
-                "maintainers": {
-                    "type": "array",
-                    "items": {"type": "string"}
-                },
-                "name": {"type": "string"},
-                "nodelist": {
-                    "type": "array",
-                    "items": {"type": "string"}
-                },
-                "outputdir": {"type": ["string", "null"]},
-                "perfvars": {
-                    "type": ["array", "null"],
-                    "items": {
-                        "type": "object",
-                        "properties": {
-                            "name": {"type": "string"},
-                            "reference": {
-                                "type": ["number", "null"]
-                            },
-                            "thres_lower": {
-                                "type": ["number", "null"]
-                            },
-                            "thres_upper": {
-                                "type": ["number", "null"]
-                            },
-                            "unit": {"type": ["string", "null"]},
-                            "value": {"type": "number"}
-                        },
-                        "required": [
-                            "name", "reference",
-                            "thres_lower", "thres_upper",
-                            "unit", "value"
-                        ]
-                    }
-                },
-                "prefix": {"type": "string"},
-                "result": {
-                    "type": "string",
-                    "enum": ["success", "failure", "aborted", "skipped"]
-                },
-                "scheduler": {"type": ["string", "null"]},
-                "stagedir": {"type": ["string", "null"]},
-                "system": {"type": "string"},
-                "tags": {
-                    "type": "array",
-                    "items": {"type": "string"}
-                },
+                "result": {"type": "string"},
                 "time_compile": {"type": ["number", "null"]},
                 "time_performance": {"type": ["number", "null"]},
                 "time_run": {"type": ["number", "null"]},
                 "time_sanity": {"type": ["number", "null"]},
                 "time_setup": {"type": ["number", "null"]},
-                "time_total": {"type": ["number", "null"]},
-                "unique_name": {"type": "string"}
+                "time_total": {"type": ["number", "null"]}
             },
-            "required": ["environment", "stagedir", "system", "unique_name"]
+            "required": ["name", "system", "partition", "environ",
+                         "perfvalues", "result", "fail_reason", "file"]
         }
     },
     "type": "object",

From 4438503bfe230a30966afce1537e62709974b3b2 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 27 May 2024 18:04:12 +0200
Subject: [PATCH 017/128] Performance comparison of past results

---
 reframe/core/logging.py             |   5 +-
 reframe/core/pipeline.py            |   3 +-
 reframe/core/schedulers/__init__.py |  11 +-
 reframe/frontend/cli.py             |  62 +++--
 reframe/frontend/runreport.py       | 383 +++++++++++++++++++++++-----
 reframe/frontend/statistics.py      |  56 ++--
 reframe/schemas/runreport.json      |  29 ++-
 requirements.txt                    |   1 +
 unittests/test_cli.py               |  11 +-
 unittests/test_policies.py          |   4 +-
 unittests/test_schedulers.py        |   4 +-
 11 files changed, 435 insertions(+), 134 deletions(-)

diff --git a/reframe/core/logging.py b/reframe/core/logging.py
index e52de9e221..223840f281 100644
--- a/reframe/core/logging.py
+++ b/reframe/core/logging.py
@@ -860,8 +860,9 @@ def log_performance(self, level, task, msg=None, multiline=False):
         if self.check is None or not self.check.is_performance_check():
             return
 
-        self.extra['check_partition'] = task.testcase.partition.name
-        self.extra['check_environ'] = task.testcase.environ.name
+        _, part, env = task.testcase
+        self.extra['check_partition'] = part.name
+        self.extra['check_environ'] = env.name
         self.extra['check_result'] = task.result
         fail_reason = what(*task.exc_info) if not task.succeeded else None
         self.extra['check_fail_reason'] = fail_reason
diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
index 462b103b4b..b532e7b36f 100644
--- a/reframe/core/pipeline.py
+++ b/reframe/core/pipeline.py
@@ -1517,8 +1517,7 @@ def _job_exitcode(self):
     @loggable_as('job_nodelist')
     @property
     def _job_nodelist(self):
-        if self.job:
-            return self.job.nodelist
+        return self.job.nodelist if self.job else []
 
     def info(self):
         '''Provide live information for this test.
diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py
index 0266e43235..eaeb9125c9 100644
--- a/reframe/core/schedulers/__init__.py
+++ b/reframe/core/schedulers/__init__.py
@@ -180,8 +180,6 @@ def filter_nodes_by_state(nodelist, state):
             }
 
     return nodelist
-    nodes[part.fullname] = [n.name for n in nodelist]
-
 
 
 class Job(jsonext.JSONSerializable, metaclass=JobMeta):
@@ -377,7 +375,7 @@ def __init__(self,
         self._jobid = None
         self._exitcode = None
         self._state = None
-        self._nodelist = None
+        self._nodelist = []
         self._submit_time = None
         self._completion_time = None
 
@@ -515,7 +513,7 @@ def nodelist(self):
         This attribute is supported by the ``local``, ``pbs``, ``slurm``,
         ``squeue``, ``ssh``, and ``torque`` scheduler backends.
 
-        This attribute is :class:`None` if no nodes are assigned to the job
+        This attribute is an empty list if no nodes are assigned to the job
         yet.
 
         The ``squeue`` scheduler backend, i.e., Slurm *without* accounting,
@@ -531,7 +529,10 @@ def nodelist(self):
 
         .. versionadded:: 2.17
 
-        :type: :class:`List[str]` or :class:`None`
+        .. versionchanged:: 4.7
+           Default value is the empty list.
+
+        :type: :class:`List[str]`
         '''
         return self._nodelist
 
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index f9ed7ccb6c..de5529b1ad 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -13,6 +13,7 @@
 import sys
 import time
 import traceback
+from tabulate import tabulate
 
 import reframe.core.config as config
 import reframe.core.exceptions as errors
@@ -28,7 +29,6 @@
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
 import reframe.utility.typecheck as typ
-
 from reframe.frontend.testgenerators import (distribute_tests,
                                              getallnodes, repeat_tests,
                                              parameterize_tests)
@@ -556,7 +556,12 @@ def main():
         envvar='RFM_COLORIZE', configvar='general/colorize'
     )
     misc_options.add_argument(
-        '--performance-report', action='store_true',
+        '--performance-compare', metavar='CMPSPEC', action='store',
+        help='Compare past performance results'
+    )
+    misc_options.add_argument(
+        '--performance-report', action='store', nargs='?',
+        const='19700101T0000Z:now/last:+job_nodelist/+result',
         help='Print a report for performance tests'
     )
     misc_options.add_argument(
@@ -584,10 +589,6 @@ def main():
         '-q', '--quiet', action='count', default=0,
         help='Decrease verbosity level of output',
     )
-    misc_options.add_argument(
-        '--fetch-cases', action='store',
-        help='Experimental: query report'
-    )
 
     # Options not associated with command-line arguments
     argparser.add_argument(
@@ -731,7 +732,7 @@ def restrict_logging():
 
         '''
 
-        if (options.show_config or options.fetch_cases or
+        if (options.show_config or
             options.detect_host_topology or options.describe):
             logging.getlogger().setLevel(logging.ERROR)
             return True
@@ -925,11 +926,6 @@ def restrict_logging():
     autodetect.detect_topology()
     printer.debug(format_env(options.env_vars))
 
-    if options.fetch_cases:
-        testcases = runreport.fetch_cases_raw(options.fetch_cases)
-        print(jsonext.dumps(testcases, indent=2))
-        sys.exit(0)
-
     # Setup the check loader
     if options.restore_session is not None:
         # We need to load the failed checks only from a list of reports
@@ -941,7 +937,12 @@ def restrict_logging():
                 new=False
             )]
 
-        report = runreport.load_report(*filenames)
+        try:
+            report = runreport.load_report(*filenames)
+        except errors.ReframeError as err:
+            printer.error(f'failed to load restore session: {err}')
+            sys.exit(1)
+
         check_search_path = list(report.slice('filename', unique=True))
         check_search_recursive = False
 
@@ -1137,12 +1138,11 @@ def _case_failed(t):
                 if not rec:
                     return False
 
-                return (rec['result'] == 'failure' or
-                        rec['result'] == 'aborted')
+                return rec['result'] == 'fail' or rec['result'] == 'abort'
 
             testcases = list(filter(_case_failed, testcases))
             printer.verbose(
-                f'Filtering successful test case(s): '
+                f'Filtering out successful test case(s): '
                 f'{len(testcases)} remaining'
             )
 
@@ -1290,6 +1290,13 @@ def _sort_testcases(testcases):
             )
             sys.exit(0)
 
+        if options.performance_compare:
+            data, header = runreport.performance_compare_data(
+                options.performance_compare
+            )
+            print(tabulate(data, headers=header, tablefmt='mixed_grid'))
+            sys.exit(0)
+
         if not options.run and not options.dry_run:
             printer.error("No action option specified. Available options:\n"
                           "  - `-l'/`-L' for listing\n"
@@ -1453,7 +1460,20 @@ def module_unuse(*paths):
                     )
 
             if options.performance_report and not options.dry_run:
-                printer.info(runner.stats.performance_report())
+                try:
+                    data, header = runreport.performance_report_data(
+                        runner.stats.json(),
+                        options.performance_report
+                    )
+                except errors.ReframeError as err:
+                    printer.warning(
+                        f'failed to generate performance report: {err}'
+                    )
+                else:
+                    print(
+                        tabulate(data, headers=header, tablefmt='mixed_grid')
+                    )
+                # printer.info(runner.stats.performance_report())
 
             # Generate the report for this session
             report_file = os.path.normpath(
@@ -1467,7 +1487,9 @@ def module_unuse(*paths):
             run_stats = runner.stats.json()
             session_info.update({
                 'num_cases': run_stats[0]['num_cases'],
-                'num_failures': run_stats[-1]['num_failures']
+                'num_failures': run_stats[-1]['num_failures'],
+                'num_aborted': run_stats[-1]['num_aborted'],
+                'num_skipped': run_stats[-1]['num_skipped']
             })
             json_report = {
                 'session_info': session_info,
@@ -1503,9 +1525,9 @@ def module_unuse(*paths):
 
             # Index the generated report
             try:
-                runreport.index_report(json_report, report_file)
+                runreport.store_results(json_report, report_file)
             except Exception as e:
-                printer.warning(f'failed to index the report: {e}')
+                printer.warning(f'failed to store results in the database: {e}')
                 raise
 
             # Generate the junit xml report for this session
diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py
index 1368ea33e9..c1de31cf7f 100644
--- a/reframe/frontend/runreport.py
+++ b/reframe/frontend/runreport.py
@@ -3,23 +3,31 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
+import abc
 import decimal
 import functools
 import glob
 import json
 import jsonschema
 import lxml.etree as etree
+import math
 import os
 import re
 import sqlite3
+import statistics
+import types
+from collections.abc import Hashable
+from datetime import datetime, timedelta
 
 import reframe as rfm
-import reframe.core.exceptions as errors
 import reframe.core.runtime as runtime
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
+from reframe.core.exceptions import ReframeError
 from reframe.core.logging import getlogger
 from reframe.core.warnings import suppress_deprecations
+from reframe.utility import nodelist_abbrev
+
 
 # The schema data version
 # Major version bumps are expected to break the validation of previous schemas
@@ -27,6 +35,13 @@
 DATA_VERSION = '4.0'
 _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json')
 
+def _tc_info(tc_entry, name='unique_name'):
+    name = tc_entry[name]
+    system = tc_entry['system']
+    partition = tc_entry['partition']
+    environ = tc_entry['environ']
+    return f'{name}@{system}:{partition}+{environ}'
+
 
 class _RunReport:
     '''A wrapper to the run report providing some additional functionality'''
@@ -40,13 +55,11 @@ def __init__(self, report):
         self._cases_index = {}
         for run in self._report['runs']:
             for tc in run['testcases']:
-                c, p, e = tc['unique_name'], tc['system'], tc['environment']
-                self._cases_index[c, p, e] = tc
+                self._cases_index[_tc_info(tc)] = tc
 
         # Index also the restored cases
         for tc in self._report['restored_cases']:
-            c, p, e = tc['unique_name'], tc['system'], tc['environment']
-            self._cases_index[c, p, e] = tc
+            self._cases_index[_tc_info(tc)] = tc
 
     def __getitem__(self, key):
         return self._report[key]
@@ -81,12 +94,12 @@ def slice(self, prop, when=None, unique=False):
                 yield val
 
     def case(self, check, part, env):
-        c, p, e = check.unique_name, part.fullname, env.name
-        ret = self._cases_index.get((c, p, e))
+        key = f'{check.unique_name}@{part.fullname}+{env.name}'
+        ret = self._cases_index.get(key)
         if ret is None:
             # Look up the case in the fallback reports
             for rpt in self._fallbacks:
-                ret = rpt._cases_index.get((c, p, e))
+                ret = rpt._cases_index.get(key)
                 if ret is not None:
                     break
 
@@ -110,7 +123,7 @@ def restore_dangling(self, graph):
     def _do_restore(self, testcase):
         tc = self.case(*testcase)
         if tc is None:
-            raise errors.ReframeError(
+            raise ReframeError(
                 f'could not restore testcase {testcase!r}: '
                 f'not found in the report files'
             )
@@ -120,7 +133,7 @@ def _do_restore(self, testcase):
             with open(dump_file) as fp:
                 testcase._check = jsonext.load(fp)
         except (OSError, json.JSONDecodeError) as e:
-            raise errors.ReframeError(
+            raise ReframeError(
                 f'could not restore testcase {testcase!r}') from e
 
 
@@ -148,10 +161,10 @@ def _load_report(filename):
         with open(filename) as fp:
             report = json.load(fp)
     except OSError as e:
-        raise errors.ReframeError(
+        raise ReframeError(
             f'failed to load report file {filename!r}') from e
     except json.JSONDecodeError as e:
-        raise errors.ReframeError(
+        raise ReframeError(
             f'report file {filename!r} is not a valid JSON file') from e
 
     # Validate the report
@@ -166,10 +179,12 @@ def _load_report(filename):
         except KeyError:
             found_ver = 'n/a'
 
-        raise errors.ReframeError(
-            f'invalid report {filename!r} '
-            f'(required data version: {DATA_VERSION}), found: {found_ver})'
-        )
+        getlogger().verbose(f'JSON validation error: {e}')
+        raise ReframeError(
+            f'failed to validate report {filename!r}: {e.args[0]} '
+            f'(check report data version: required {DATA_VERSION}, '
+            f'found: {found_ver})'
+        ) from None
 
     return _RunReport(report)
 
@@ -204,7 +219,7 @@ def link_latest_report(filename, link_name):
                 os.remove(link_name)
                 create_symlink()
             else:
-                raise errors.ReframeError('path exists and is not a symlink')
+                raise ReframeError('path exists and is not a symlink')
 
 
 def junit_xml_report(json_report):
@@ -223,13 +238,13 @@ def junit_xml_report(json_report):
                 'package': 'reframe',
                 'tests': str(rfm_run['num_cases']),
                 'time': str(json_report['session_info']['time_elapsed']),
-
                 # XSD schema does not like the timezone format, so we remove it
                 'timestamp': json_report['session_info']['time_start'][:-5],
             }
         )
+        etree.SubElement(xml_testsuite, 'properties')
         for tc in rfm_run['testcases']:
-            casename = f"{tc['name']} @{tc['system']}+{tc['environ']}"
+            casename = f'{_tc_info(tc, name="name")}'
             testcase = etree.SubElement(
                 xml_testsuite, 'testcase',
                 attrib={
@@ -244,8 +259,8 @@ def junit_xml_report(json_report):
             )
             if tc['result'] == 'fail':
                 testcase_msg = etree.SubElement(
-                    testcase, 'fail', attrib={'type': 'fail',
-                                              'message': tc['fail_phase']}
+                    testcase, 'failure', attrib={'type': 'failure',
+                                                 'message': tc['fail_phase']}
                 )
                 testcase_msg.text = f"{tc['fail_phase']}: {tc['fail_reason']}"
 
@@ -269,84 +284,320 @@ def get_reports_files(directory):
             if os.path.isfile(f) and not f.endswith('/latest.json')]
 
 
-def index_report(report, filename):
-    prefix = os.path.dirname(filename)
-    index_file = os.path.join(prefix, 'index.db')
-    if not os.path.exists(index_file):
-        getlogger().info('Re-building the report index...')
-        _build_index(index_file)
+def _db_file():
+    site_config = runtime.runtime().site_config
+    prefix = os.path.dirname(osext.expandvars(
+        site_config.get('general/0/report_file')
+    ))
+    filename = os.path.join(prefix, 'results.db')
+    if not os.path.exists(filename):
+        # Create subdirs if needed
+        if prefix:
+            os.makedirs(prefix, exist_ok=True)
+
+        getlogger().debug(f'Creating the results database in {filename}...')
+        _db_create(filename)
+
+    return filename
+
 
-    with sqlite3.connect(index_file) as conn:
-        _index_report(conn, report, filename)
+def store_results(report, report_file):
+    with sqlite3.connect(_db_file()) as conn:
+        _db_store_report(conn, report, report_file)
 
 
-def _build_index(filename):
+def _db_create(filename):
     with sqlite3.connect(filename) as conn:
         conn.execute(
+'''CREATE TABLE IF NOT EXISTS sessions(
+        id INTEGER PRIMARY KEY,
+        json_blob TEXT,
+        report_file TEXT
+)'''
+        )
+        conn.execute(
 '''CREATE TABLE IF NOT EXISTS testcases(
         name TEXT,
         system TEXT,
         partition TEXT,
         environ TEXT,
-        time_start TEXT,
-        time_end TEXT,
+        job_completion_time_unix REAL,
+        session_id INTEGER,
         run_index INTEGER,
         test_index INTEGER,
-        report_file TEXT
+        FOREIGN KEY(session_id) REFERENCES sessions(session_id)
 )'''
         )
-        prefix = os.path.dirname(filename)
-        for report_file in glob.iglob(f'{prefix}/*'):
-            if not os.path.islink(report_file):
-                try:
-                    report = load_report(report_file)
-                # except errors.ReframeError as e:
-                except Exception as e:
-                    getlogger().info(f'ignoring report file {report_file!r}: {e}')
-                else:
-                    _index_report(conn, report, report_file)
-
-
-def _index_report(conn, report, report_file_path):
-    time_start = report['session_info']['time_start']
-    time_end = report['session_info']['time_end']
+
+
+def _db_store_report(conn, report, report_file_path):
+    session_start = report['session_info']['time_start']
     for run_idx, run in enumerate(report['runs']):
         for test_idx, testcase in enumerate(run['testcases']):
             sys, part = testcase['system'], testcase['partition']
+            cursor = conn.execute(
+'''INSERT INTO sessions VALUES(:session_id, :json_blob, :report_file)''',
+                         {'session_id': None,
+                          'json_blob': jsonext.dumps(report),
+                          'report_file': report_file_path})
             conn.execute(
-'''INSERT INTO testcases VALUES(:name, :system, :partition,
-                                :environ, :time_start, :time_end,
-                                :run_index, :test_index, :report_file)''',
+'''INSERT INTO testcases VALUES(:name, :system, :partition, :environ,
+                                :job_completion_time_unix,
+                                :session_id, :run_index, :test_index)''',
                 {
                     'name': testcase['name'],
                     'system': sys,
                     'partition': part,
                     'environ': testcase['environ'],
+                    'job_completion_time_unix': testcase[
+                        'job_completion_time_unix'
+                    ],
+                    'session_id': cursor.lastrowid,
                     'run_index': run_idx,
-                    'time_start': time_start,
-                    'time_end': time_end,
-                    'test_index': test_idx,
-                    'report_file': report_file_path
+                    'test_index': test_idx
                 }
             )
 
 
-def fetch_cases_raw(condition):
-    site_config = runtime.runtime().site_config
-    prefix = os.path.dirname(osext.expandvars(
-        site_config.get('general/0/report_file')
-    ))
-    with sqlite3.connect(os.path.join(prefix, 'index.db')) as conn:
-        query = f'SELECT test_index, run_index, report_file FROM testcases where {condition}'
+def _fetch_cases_raw(condition):
+    with sqlite3.connect(_db_file()) as conn:
+        query = (f'SELECT session_id, run_index, test_index, json_blob FROM '
+                 f'testcases JOIN sessions ON session_id==id '
+                 f'WHERE {condition}')
         getlogger().debug(query)
         results = conn.execute(query).fetchall()
 
     # Retrieve files
     testcases = []
-    for test_index, run_index, json_file in results:
-        with open(json_file, 'r') as file:
-            report = json.load(file)
-
+    sessions = {}
+    for session_id, run_index, test_index, json_blob in results:
+        report = json.loads(sessions.setdefault(session_id, json_blob))
         testcases.append(report['runs'][run_index]['testcases'][test_index])
 
     return testcases
+
+
+def _fetch_cases_time_period(ts_start, ts_end):
+    return _fetch_cases_raw(
+        f'(job_completion_time_unix >= {ts_start} AND '
+        f'job_completion_time_unix < {ts_end}) '
+        'ORDER BY job_completion_time_unix'
+    )
+
+
+def _group_key(groups, testcase):
+    key = []
+    for grp in groups:
+        val = testcase[grp]
+        if grp == 'job_nodelist':
+            # Fold nodelist before adding as a key element
+            key.append(nodelist_abbrev(val))
+        elif not isinstance(val, Hashable):
+            key.append(str(val))
+        else:
+            key.append(val)
+
+    return tuple(key)
+
+
+def parse_timestamp(s):
+    now = datetime.now()
+    def _do_parse(s):
+        if s == 'now':
+            return now
+
+        formats = [r'%Y%m%d', r'%Y%m%dT%H%M',
+                   r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z']
+        for fmt in formats:
+            try:
+                return datetime.strptime(s, fmt)
+            except ValueError:
+                continue
+
+        raise ValueError(f'invalid timestamp: {s}')
+
+
+    try:
+        ts = _do_parse(s)
+    except ValueError as err:
+        # Try the relative timestamps
+        match = re.match(r'(?P<ts>.*)(?P<amount>[\+|-]\d+)(?P<unit>[hdms])', s)
+        if not match:
+            raise err
+
+        ts = _do_parse(match.group('ts'))
+        amount = int(match.group('amount'))
+        unit = match.group('unit')
+        if unit == 'd':
+            ts += timedelta(days=amount)
+        elif unit == 'm':
+            ts += timedelta(minutes=amount)
+        elif unit == 'h':
+            ts += timedelta(hours=amount)
+        elif unit == 's':
+            ts += timedelta(seconds=amount)
+
+    return ts.timestamp()
+
+
+def _group_testcases(testcases, group_by, extra_cols):
+    grouped = {}
+    for tc in testcases:
+        for pvar, reftuple in tc['perfvalues'].items():
+            pvar = pvar.split(':')[-1]
+            pval, pref, plower, pupper, punit = reftuple
+            plower = pref * (1 + plower) if plower is not None else -math.inf
+            pupper = pref * (1 + pupper) if pupper is not None else math.inf
+            record = {
+                'pvar': pvar,
+                'pval': pval,
+                'plower': plower,
+                'pupper': pupper,
+                'punit': punit,
+                **{k: tc[k] for k in group_by + extra_cols if k in tc}
+            }
+            key = _group_key(group_by, record)
+            grouped.setdefault(key, [])
+            grouped[key].append(record)
+
+    return grouped
+
+def _aggregate_perf(grouped_testcases, aggr_fn, cols):
+    other_aggr = _JoinUniqueValues('|')
+    aggr_data = {}
+    for key, seq in grouped_testcases.items():
+        aggr_data.setdefault(key, {})
+        aggr_data[key]['pval'] = aggr_fn(tc['pval'] for tc in seq)
+        for c in cols:
+            aggr_data[key][c] = other_aggr(
+                nodelist_abbrev(tc[c]) if c == 'job_nodelist' else tc[c]
+                for tc in seq
+            )
+
+    return aggr_data
+
+def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
+                          extra_group_by=None, extra_cols=None):
+    extra_group_by = extra_group_by or []
+    extra_cols = extra_cols or []
+    group_by = ['name', 'pvar', 'punit'] + extra_group_by
+
+    grouped_base = _group_testcases(base_testcases, group_by, extra_cols)
+    grouped_target = _group_testcases(target_testcases, group_by, extra_cols)
+    pbase = _aggregate_perf(grouped_base, base_fn, extra_cols)
+    ptarget = _aggregate_perf(grouped_target, target_fn, [])
+
+    # Build the final table data
+    data = []
+    for key, aggr_data in pbase.items():
+        pval = aggr_data['pval']
+        try:
+            target_pval = ptarget[key]['pval']
+        except KeyError:
+            pdiff = 'n/a'
+        else:
+            pdiff = (pval - target_pval) / target_pval
+            pdiff = '{:+7.2%}'.format(pdiff)
+
+        name, pvar, punit, *extras = key
+        line = [name, pvar, pval, punit, pdiff, *extras]
+        # Add the extra columns
+        line += [aggr_data[c] for c in extra_cols]
+        data.append(line)
+
+    return (data, ['name', 'pvar', 'pval', 'punit', 'pdiff'] + extra_group_by + extra_cols)
+
+class _Aggregator:
+    @classmethod
+    def create(cls, name):
+        if name == 'first':
+            return _First()
+        elif name == 'last':
+            return _Last()
+        elif name == 'mean':
+            return _Mean()
+        elif name == 'median':
+            return _Median()
+        elif name == 'min':
+            return _Min()
+        elif name == 'max':
+            return _Max()
+        else:
+            raise ValueError(f'unknown aggregation function: {name!r}')
+
+
+    @abc.abstractmethod
+    def __call__(self, iterable):
+        pass
+
+class _First(_Aggregator):
+    def __call__(self, iterable):
+        for i, elem in enumerate(iterable):
+            if i == 0:
+                return elem
+
+class _Last(_Aggregator):
+    def __call__(self, iterable):
+        if not isinstance(iterable, types.GeneratorType):
+            return iterable[-1]
+
+        for elem in iterable:
+            pass
+
+        return elem
+
+
+class _Mean(_Aggregator):
+    def __call__(self, iterable):
+        return statistics.mean(iterable)
+
+
+class _Median(_Aggregator):
+    def __call__(self, iterable):
+        return statistics.median(iterable)
+
+
+class _Min(_Aggregator):
+    def __call__(self, iterable):
+        return min(iterable)
+
+
+class _Max(_Aggregator):
+    def __call__(self, iterable):
+        return max(iterable)
+
+class _JoinUniqueValues(_Aggregator):
+    def __init__(self, delim):
+        self.__delim = delim
+
+    def __call__(self, iterable):
+        unique_vals = {str(elem) for elem in iterable}
+        return self.__delim.join(unique_vals)
+
+def performance_report_data(run_stats, report_spec):
+    period, aggr, cols = report_spec.split('/')
+    ts_start, ts_end = [parse_timestamp(ts) for ts in period.split(':')]
+    op, extra_groups = aggr.split(':')
+    aggr_fn = _Aggregator.create(op)
+    extra_groups = extra_groups.split('+')[1:]
+    extra_cols = cols.split('+')[1:]
+    testcases = run_stats[0]['testcases']
+    target_testcases = _fetch_cases_time_period(ts_start, ts_end)
+    return compare_testcase_data(testcases, target_testcases, _First(),
+                                 aggr_fn, extra_groups, extra_cols)
+
+
+def performance_compare_data(spec):
+    period_base, period_target, aggr, cols = spec.split('/')
+    base_testcases = _fetch_cases_time_period(
+        *(parse_timestamp(ts) for ts in period_base.split(':'))
+    )
+    target_testcases = _fetch_cases_time_period(
+        *(parse_timestamp(ts) for ts in period_target.split(':'))
+    )
+    op, extra_groups = aggr.split(':')
+    aggr_fn = _Aggregator.create(op)
+    extra_groups = extra_groups.split('+')[1:]
+    extra_cols = cols.split('+')[1:]
+    return compare_testcase_data(base_testcases, target_testcases, aggr_fn,
+                                 aggr_fn, extra_groups, extra_cols)
diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py
index c13c4409e7..4ad0a4e680 100644
--- a/reframe/frontend/statistics.py
+++ b/reframe/frontend/statistics.py
@@ -7,11 +7,13 @@
 import itertools
 import os
 import shutil
+import time
 import traceback
 
 import reframe.core.runtime as rt
 import reframe.core.exceptions as errors
 import reframe.utility as util
+from reframe.core.logging import _format_time_rfc3339
 from reframe.core.warnings import suppress_deprecations
 
 
@@ -107,8 +109,10 @@ def json(self, force=False):
             num_aborted = 0
             num_skipped = 0
             for t in run:
-                check = t.check
-                partition = check.current_partition
+                # We take partition and environment from the test case and not
+                # from the check, since if the test fails before `setup()`,
+                # these are not set inside the check.
+                check, partition, environ = t.testcase
                 entry = {
                     'build_jobid': None,
                     'build_stderr': None,
@@ -121,14 +125,18 @@ def json(self, force=False):
                     'dependencies_conceptual': [
                         d[0] for d in t.check.user_deps()
                     ],
+                    'environ': environ.name,
                     'fail_phase': None,
                     'fail_reason': None,
                     'filename': inspect.getfile(type(check)),
                     'fixture': check.is_fixture(),
+                    'job_completion_time': None,
+                    'job_completion_time_unix': None,
                     'job_stderr': None,
                     'job_stdout': None,
-                    'result': None,
-                    'scheduler': None,
+                    'partition': partition.name,
+                    'result': t.result,
+                    'scheduler': partition.scheduler.registered_name,
                     'time_compile': t.duration('compile_complete'),
                     'time_performance': t.duration('performance'),
                     'time_run': t.duration('run_complete'),
@@ -136,15 +144,6 @@ def json(self, force=False):
                     'time_setup': t.duration('setup'),
                     'time_total': t.duration('total')
                 }
-
-                # We take partition and environment from the test case and not
-                # from the check, since if the test fails before `setup()`,
-                # these are not set inside the check.
-                partition = t.testcase.partition
-                environ = t.testcase.environ
-                entry['partition'] = partition.name
-                entry['environ'] = environ.name
-                entry['scheduler'] = partition.scheduler.registered_name
                 if check.job:
                     entry['job_stderr'] = check.stderr.evaluate()
                     entry['job_stdout'] = check.stdout.evaluate()
@@ -153,7 +152,6 @@ def json(self, force=False):
                     entry['build_stderr'] = check.build_stderr.evaluate()
                     entry['build_stdout'] = check.build_stdout.evaluate()
 
-                entry['result'] = t.result
                 if t.failed:
                     num_failures += 1
                 elif t.aborted:
@@ -177,12 +175,22 @@ def json(self, force=False):
                 # Add any loggable variables and parameters
                 test_cls = type(check)
                 for name, alt_name in test_cls.loggable_attrs():
+                    if alt_name == 'partition' or alt_name == 'environ':
+                        # We set those from the testcase
+                        continue
+
                     key = alt_name if alt_name else name
                     try:
                         entry[key] = _getattr(check, name)
                     except AttributeError:
                         entry[key] = '<undefined>'
 
+                if entry['job_completion_time_unix']:
+                    entry['job_completion_time'] = _format_time_rfc3339(
+                        time.localtime(entry['job_completion_time_unix']),
+                        '%FT%T%:z'
+                    )
+
                 testcases.append(entry)
 
             self._run_data.append({
@@ -221,15 +229,14 @@ def _head_n(filename, prefix, num_lines=10):
 
         def _print_failure_info(rec, runid, total_runs):
             printer.info(line_width * '-')
-            printer.info(f"FAILURE INFO for {rec['display_name']} "
+            printer.info(f"FAILURE INFO for {rec['name']} "
                          f"(run: {runid}/{total_runs})")
             printer.info(f"  * Description: {rec['descr']}")
             printer.info(f"  * System partition: {rec['system']}")
             printer.info(f"  * Environment: {rec['environ']}")
             printer.info(f"  * Stage directory: {rec['stagedir']}")
-            printer.info(
-                f"  * Node list: {util.nodelist_abbrev(rec['job_nodelist'])}"
-            )
+            printer.info(f"  * Node list: "
+                         f"{util.nodelist_abbrev(rec['job_nodelist'])}")
             job_type = 'local' if rec['scheduler'] == 'local' else 'batch job'
             printer.info(f"  * Job type: {job_type} (id={rec['jobid']})")
             printer.info(f"  * Dependencies (conceptual): "
@@ -269,7 +276,7 @@ def _print_failure_info(rec, runid, total_runs):
                 continue
 
             for r in run_info['testcases']:
-                if r['result'] in {'success', 'aborted', 'skipped'}:
+                if r['result'] in {'pass', 'abort', 'skip'}:
                     continue
 
                 _print_failure_info(r, run_no, len(run_report))
@@ -353,11 +360,12 @@ def performance_report(self):
 
         for testcases in perf_records.values():
             for tc in testcases:
-                name = tc['display_name']
-                hash = tc['hash']
-                env  = tc['environment']
-                part = tc['system']
-                lines.append(f'[{name} /{hash} @{part}:{env}]')
+                name = tc['name']
+                hash = tc['hashcode']
+                env  = tc['environ']
+                system = tc['system']
+                part = tc['partition']
+                lines.append(f'[{name} /{hash} @{system}:{part}+{env}]')
                 for v in interesting_vars:
                     val = tc['check_vars'][v]
                     if val is not None:
diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json
index e989559476..bf60343235 100644
--- a/reframe/schemas/runreport.json
+++ b/reframe/schemas/runreport.json
@@ -22,6 +22,7 @@
                     "type": "array",
                     "items": {"type": "string"}
                 },
+                "environ": {"type": ["string", "null"]},
                 "fail_info": {
                     "type": ["object", "null"],
                     "properties": {
@@ -39,18 +40,29 @@
                 "fail_severe": {"type": "boolean"},
                 "filename": {"type": "string"},
                 "fixture": {"type": "boolean"},
+                "job_completion_time": {"type": ["string", "null"]},
+                "job_completion_time_unix": {"type": ["number", "null"]},
                 "job_stderr": {"type": ["string", "null"]},
                 "job_stdout": {"type": ["string", "null"]},
+                "name": {"type": "string"},
+                "outputdir": {"type": ["string", "null"]},
+                "perfvalues": {"type": "object"},
+                "partition": {"type": ["string", "null"]},
                 "result": {"type": "string"},
+                "scheduler": {"type": "string"},
+                "system": {"type": "string"},
                 "time_compile": {"type": ["number", "null"]},
                 "time_performance": {"type": ["number", "null"]},
                 "time_run": {"type": ["number", "null"]},
                 "time_sanity": {"type": ["number", "null"]},
                 "time_setup": {"type": ["number", "null"]},
-                "time_total": {"type": ["number", "null"]}
+                "time_total": {"type": ["number", "null"]},
+                "unique_name": {"type": "string"}
             },
-            "required": ["name", "system", "partition", "environ",
-                         "perfvalues", "result", "fail_reason", "file"]
+            "required": ["environ", "fail_phase", "fail_reason", "filename",
+                         "job_completion_time_unix", "name", "perfvalues",
+                         "partition", "result", "system", "time_total",
+                         "unique_name"]
         }
     },
     "type": "object",
@@ -72,6 +84,7 @@
                 "num_cases": {"type": "number"},
                 "num_failures": {"type": "number"},
                 "num_aborted": {"type": "number"},
+                "num_skipped": {"type": "number"},
                 "prefix_output": {"type": "string"},
                 "prefix_stage": {"type": "string"},
                 "time_elapsed": {"type": "number"},
@@ -81,7 +94,8 @@
                 "version": {"type": "string"},
                 "workdir": {"type": "string"}
             },
-            "required": ["data_version"]
+            "required": ["data_version", "hostname",
+                         "time_elapsed", "time_end", "time_start"]
         },
         "restored_cases": {
             "type": "array",
@@ -92,16 +106,17 @@
             "items": {
                 "type": "object",
                 "properties": {
+                    "num_aborted": {"type": "number"},
                     "num_cases": {"type": "number"},
                     "num_failures": {"type": "number"},
-                    "num_aborted": {"type": "number"},
-                    "runid": {"type": "number"},
+                    "num_skipped": {"type": "number"},
+                    "runid": {"typeid": "number"},
                     "testcases": {
                         "type": "array",
                         "items": {"$ref": "#/defs/testcase_type"}
                     }
                 },
-                "required": ["testcases"]
+                "required": ["num_cases", "num_failures", "testcases"]
             }
         }
     },
diff --git a/requirements.txt b/requirements.txt
index 0e9fc5409b..e01146fe10 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -22,5 +22,6 @@ semver==3.0.2; python_version >= '3.7'
 setuptools==59.6.0; python_version == '3.6'
 setuptools==68.0.0; python_version == '3.7'
 setuptools==72.1.0; python_version >= '3.8'
+tabulate==0.9.0
 wcwidth==0.2.13
 #+pygelf%pygelf==0.4.0
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 7fc8edd9fd..04fcaad50c 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -421,17 +421,20 @@ def test_perflogdir_from_env(run_reframe, tmp_path, monkeypatch):
 
 
 def test_performance_report(run_reframe, run_action):
-    returncode, stdout, _ = run_reframe(
+    returncode, stdout, stderr = run_reframe(
         checkpath=['unittests/resources/checks/frontend_checks.py'],
         more_options=['-n', '^PerformanceFailureCheck',
                       '--performance-report'],
         action=run_action
     )
+
     if run_action == 'run':
-        assert r'PERFORMANCE REPORT' in stdout
-        assert r'perf: 10 Gflop/s' in stdout
+        assert returncode == 1
     else:
-        assert r'PERFORMANCE REPORT' not in stdout
+        assert returncode == 0
+
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
 
 
 def test_skip_system_check_option(run_reframe, run_action):
diff --git a/unittests/test_policies.py b/unittests/test_policies.py
index 04e0edb78c..5d8efda6d8 100644
--- a/unittests/test_policies.py
+++ b/unittests/test_policies.py
@@ -291,7 +291,7 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path):
         jsonext.dump(report, fp)
 
     with pytest.raises(ReframeError,
-                       match=r'invalid report'):
+                       match=r'failed to validate report'):
         runreport.load_report(tmp_path / 'invalid-version.json')
 
 
@@ -391,7 +391,7 @@ def assert_reported_skipped(num_skipped):
 
         num_reported = 0
         for tc in report[0]['testcases']:
-            if tc['result'] == 'skipped':
+            if tc['result'] == 'skip':
                 num_reported += 1
 
         assert num_reported == num_skipped
diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py
index 609efc8f68..cd1c3413d0 100644
--- a/unittests/test_schedulers.py
+++ b/unittests/test_schedulers.py
@@ -476,9 +476,9 @@ def test_prepare_nodes_option_minimal(make_exec_ctx, make_job, slurm_only):
 def test_submit(make_job, exec_ctx):
     minimal_job = make_job(sched_access=exec_ctx.access)
     prepare_job(minimal_job)
-    assert minimal_job.nodelist is None
+    assert minimal_job.nodelist == []
     submit_job(minimal_job)
-    assert minimal_job.jobid is not None
+    assert minimal_job.jobid != []
     minimal_job.wait()
 
     # Additional scheduler-specific checks

From 71c777dbf8dc49d2ccd39420a2ea9d49e7e5ac66 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Tue, 18 Jun 2024 16:37:18 +0200
Subject: [PATCH 018/128] Refactor reporting code

---
 reframe/frontend/cli.py                       | 117 ++---
 reframe/frontend/executors/__init__.py        |  50 +-
 reframe/frontend/printer.py                   | 161 ++++++
 reframe/frontend/reporting/__init__.py        | 457 ++++++++++++++++++
 .../{runreport.py => reporting/analytics.py}  | 429 ++++------------
 reframe/frontend/statistics.py                | 396 ---------------
 unittests/test_cli.py                         |   8 +-
 unittests/test_policies.py                    | 122 ++---
 8 files changed, 877 insertions(+), 863 deletions(-)
 create mode 100644 reframe/frontend/reporting/__init__.py
 rename reframe/frontend/{runreport.py => reporting/analytics.py} (52%)
 delete mode 100644 reframe/frontend/statistics.py

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index de5529b1ad..fcd1972717 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -24,7 +24,8 @@
 import reframe.frontend.ci as ci
 import reframe.frontend.dependencies as dependencies
 import reframe.frontend.filters as filters
-import reframe.frontend.runreport as runreport
+import reframe.frontend.reporting as reporting
+import reframe.frontend.reporting.analytics as analytics
 import reframe.utility as util
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
@@ -932,18 +933,18 @@ def restrict_logging():
         if options.restore_session:
             filenames = options.restore_session.split(',')
         else:
-            filenames = [runreport.next_report_filename(
-                osext.expandvars(site_config.get('general/0/report_file')),
-                new=False
-            )]
+            filenames = [
+                osext.expandvars(site_config.get('general/0/report_file'))
+            ]
 
         try:
-            report = runreport.load_report(*filenames)
+            restored_session = reporting.restore_session(*filenames)
         except errors.ReframeError as err:
             printer.error(f'failed to load restore session: {err}')
             sys.exit(1)
 
-        check_search_path = list(report.slice('filename', unique=True))
+        check_search_path = list(restored_session.slice('filename',
+                                                        unique=True))
         check_search_recursive = False
 
         # If `-c` or `-R` are passed explicitly outside the configuration
@@ -954,9 +955,7 @@ def restrict_logging():
                 'search path set explicitly in the command-line or '
                 'the environment'
             )
-            check_search_path = site_config.get(
-                'general/0/check_search_path'
-            )
+            check_search_path = site_config.get('general/0/check_search_path')
 
         if site_config.is_sticky_option('general/check_search_recursive'):
             printer.warning(
@@ -998,20 +997,20 @@ def print_infoline(param, value):
         param = param + ':'
         printer.info(f"  {param.ljust(18)} {value}")
 
-    session_info = {
+    report = reporting.RunReport()
+    report.update_session_info({
         'cmdline': ' '.join(shlex.quote(arg) for arg in sys.argv),
         'config_files': rt.site_config.sources,
-        'data_version': runreport.DATA_VERSION,
-        'hostname': socket.gethostname(),
         'log_files': logging.log_files(),
         'prefix_output': rt.output_prefix,
         'prefix_stage': rt.stage_prefix,
         'user': osext.osuser(),
         'version': osext.reframe_version(),
         'workdir': os.getcwd(),
-    }
+    })
 
     # Print command line
+    session_info = report['session_info']
     printer.info(f"[ReFrame Setup]")
     print_infoline('version', session_info['version'])
     print_infoline('command', repr(session_info['cmdline']))
@@ -1134,7 +1133,7 @@ def print_infoline(param, value):
                 sys.exit(1)
 
             def _case_failed(t):
-                rec = report.case(*t)
+                rec = restored_session.case(*t)
                 if not rec:
                     return False
 
@@ -1233,7 +1232,7 @@ def _sort_testcases(testcases):
             printer.debug('Pruned test DAG')
             printer.debug(dependencies.format_deps(testgraph))
             if options.restore_session is not None:
-                testgraph, restored_cases = report.restore_dangling(testgraph)
+                testgraph, restored_cases = restored_session.restore_dangling(testgraph)
 
         testcases = dependencies.toposort(
             testgraph,
@@ -1291,10 +1290,10 @@ def _sort_testcases(testcases):
             sys.exit(0)
 
         if options.performance_compare:
-            data, header = runreport.performance_compare_data(
+            data, header = reporting.performance_compare(
                 options.performance_compare
             )
-            print(tabulate(data, headers=header, tablefmt='mixed_grid'))
+            printer(tabulate(data, headers=header, tablefmt='mixed_grid'))
             sys.exit(0)
 
         if not options.run and not options.dry_run:
@@ -1436,34 +1435,39 @@ def module_unuse(*paths):
             )
             runner.runall(testcases, restored_cases)
         finally:
+            # Build final JSON report
             time_end = time.time()
-            session_info['time_end'] = time.strftime(
-                '%FT%T%z', time.localtime(time_end)
-            )
-            session_info['time_elapsed'] = time_end - time_start
+            report.update_session_info({
+                'time_end': time.strftime(r'%FT%T%z',
+                                          time.localtime(time_end)),
+                'time_elapsed': time_end - time_start
+            })
+            report.update_run_stats(runner.stats)
+            if options.restore_session is not None:
+                report.update_restored_cases(restored_cases, restored_session)
 
             # Print a retry report if we did any retries
             if options.max_retries and runner.stats.failed(run=0):
-                printer.info(runner.stats.retry_report())
+                printer.retry_report(report)
 
             # Print a failure report if we had failures in the last run
             success = True
             if runner.stats.failed():
                 success = False
-                runner.stats.print_failure_report(
-                    printer, not options.distribute,
-                    options.duration or options.reruns
+                printer.failure_report(
+                    report,
+                    rerun_info=not options.distribute,
+                    global_stats=options.duration or options.reruns
                 )
                 if options.failure_stats:
-                    runner.stats.print_failure_stats(
-                        printer, options.duration or options.reruns
+                    printer.failure_stats(
+                        report, global_stats=options.duration or options.reruns
                     )
 
             if options.performance_report and not options.dry_run:
                 try:
-                    data, header = runreport.performance_report_data(
-                        runner.stats.json(),
-                        options.performance_report
+                    data, header = reporting.performance_compare(
+                        options.performance_report, report
                     )
                 except errors.ReframeError as err:
                     printer.warning(
@@ -1483,49 +1487,28 @@ def module_unuse(*paths):
             if basedir:
                 os.makedirs(basedir, exist_ok=True)
 
-            # Build final JSON report
-            run_stats = runner.stats.json()
-            session_info.update({
-                'num_cases': run_stats[0]['num_cases'],
-                'num_failures': run_stats[-1]['num_failures'],
-                'num_aborted': run_stats[-1]['num_aborted'],
-                'num_skipped': run_stats[-1]['num_skipped']
-            })
-            json_report = {
-                'session_info': session_info,
-                'runs': run_stats,
-                'restored_cases': []
-            }
-            if options.restore_session is not None:
-                for c in restored_cases:
-                    json_report['restored_cases'].append(report.case(*c))
-
             # Save the report file
-            report_file = runreport.next_report_filename(report_file)
             try:
-                runreport.save_report(
-                    json_report, report_file,
-                    rt.get_option('general/0/compress_report')
+                default_loc = os.path.dirname(
+                    osext.expandvars(rt.get_default('general/report_file'))
                 )
+                report.save(
+                    report_file,
+                    compress=rt.get_option('general/0/compress_report'),
+                    link_to_last=(default_loc == os.path.dirname(report_file))
+            )
             except OSError as e:
                 printer.warning(
                     f'failed to generate report in {report_file!r}: {e}'
                 )
-            else:
-                default_loc = os.path.dirname(
-                    osext.expandvars(rt.get_default('general/report_file'))
+            except errors.ReframeError as e:
+                printer.warning(
+                    f'failed to create symlink to latest report: {e}'
                 )
-                if default_loc == os.path.dirname(report_file):
-                    try:
-                        runreport.link_latest_report(report_file, 'latest.json')
-                    except Exception as e:
-                        printer.warning(
-                            f'failed to create symlink to latest report: {e}'
-                        )
-
-            # Index the generated report
+
+            # Store the generated report for analytics
             try:
-                runreport.store_results(json_report, report_file)
+                analytics.store_report(report, report.filename)
             except Exception as e:
                 printer.warning(f'failed to store results in the database: {e}')
                 raise
@@ -1535,10 +1518,8 @@ def module_unuse(*paths):
             if junit_report_file:
                 # Expand variables in filename
                 junit_report_file = osext.expandvars(junit_report_file)
-                junit_xml = runreport.junit_xml_report(json_report)
                 try:
-                    with open(junit_report_file, 'w') as fp:
-                        runreport.junit_dump(junit_xml, fp)
+                    report.save_junit(junit_report_file)
                 except OSError as e:
                     printer.warning(
                         f'failed to generate report in {junit_report_file!r}: '
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index 6c741d6e61..90b81c74c2 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -6,6 +6,7 @@
 import abc
 import contextlib
 import copy
+import itertools
 import os
 import signal
 import sys
@@ -24,15 +25,62 @@
                                      ForceExitError,
                                      RunSessionTimeout,
                                      SkipTestError,
+                                     StatisticsError,
                                      TaskExit)
 from reframe.core.schedulers.local import LocalJobScheduler
 from reframe.frontend.printer import PrettyPrinter
-from reframe.frontend.statistics import TestStats
 
 ABORT_REASONS = (AssertionError, FailureLimitError,
                  KeyboardInterrupt, ForceExitError, RunSessionTimeout)
 
 
+class TestStats:
+    '''Stores test case statistics.'''
+
+    def __init__(self):
+        # Tasks per run stored as follows: [[run0_tasks], [run1_tasks], ...]
+        self._alltasks = [[]]
+
+    def add_task(self, task):
+        current_run = runtime.runtime().current_run
+        if current_run == len(self._alltasks):
+            self._alltasks.append([])
+
+        self._alltasks[current_run].append(task)
+
+    def runs(self):
+        for runid, tasks in enumerate(self._alltasks):
+            yield runid, tasks
+
+    def tasks(self, run=-1):
+        if run is None:
+            yield from itertools.chain(*self._alltasks)
+        else:
+            try:
+                yield from self._alltasks[run]
+            except IndexError:
+                raise StatisticsError(f'no such run: {run}') from None
+
+    def failed(self, run=-1):
+        return [t for t in self.tasks(run) if t.failed]
+
+    def skipped(self, run=-1):
+        return [t for t in self.tasks(run) if t.skipped]
+
+    def aborted(self, run=-1):
+        return [t for t in self.tasks(run) if t.aborted]
+
+    def completed(self, run=-1):
+        return [t for t in self.tasks(run) if t.completed]
+
+    def num_cases(self, run=-1):
+        return sum(1 for _ in self.tasks(run))
+
+    @property
+    def num_runs(self):
+        return len(self._alltasks)
+
+
 class TestCase:
     '''A combination of a regression check, a system partition
     and a programming environment.
diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index 789889e3d3..b6530f05bb 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -3,9 +3,16 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
+import os
+import shutil
 import time
+import traceback
 import reframe.core.logging as logging
+import reframe.core.runtime as rt
 import reframe.utility.color as color
+from reframe.core.exceptions import SanityError
+from reframe.frontend.reporting import format_testcase
+from reframe.utility import nodelist_abbrev
 
 
 class PrettyPrinter:
@@ -14,6 +21,8 @@ class PrettyPrinter:
     It takes care of formatting the progress output and adds some more
     cosmetics to specific levels of messages, such as warnings and errors.
 
+    It also takes care of formatting and printing the various reports.
+
     The actual printing is delegated to an internal logger, which is
     responsible for printing.
     '''
@@ -84,3 +93,155 @@ def __setattr__(self, attr, value):
             self.__dict__['colorize'] = value
         else:
             super().__setattr__(attr, value)
+
+    def failure_report(self, report, rerun_info=True, global_stats=False):
+        '''Print a failure report'''
+
+        def _head_n(filename, prefix, num_lines=10):
+            # filename and prefix are `None` before setup
+            if filename is None or prefix is None:
+                return []
+
+            try:
+                with open(os.path.join(prefix, filename)) as fp:
+                    lines = [
+                        f'--- {filename} (first {num_lines} lines) ---'
+                    ]
+                    for i, line in enumerate(fp):
+                        if i < num_lines:
+                            # Remove trailing '\n'
+                            lines.append(line.rstrip())
+
+                lines += [f'--- {filename} ---']
+            except OSError as e:
+                lines = [f'--- {filename} ({e}) ---']
+
+            return lines
+
+        def _print_failure_info(rec, runid, total_runs):
+            self.info(line_width * '-')
+            self.info(f"FAILURE INFO for {rec['name']} "
+                         f"(run: {runid}/{total_runs})")
+            self.info(f"  * Description: {rec['descr']}")
+            self.info(f"  * System partition: {rec['system']}")
+            self.info(f"  * Environment: {rec['environ']}")
+            self.info(f"  * Stage directory: {rec['stagedir']}")
+            self.info(f"  * Node list: "
+                         f"{nodelist_abbrev(rec['job_nodelist'])}")
+            job_type = 'local' if rec['scheduler'] == 'local' else 'batch job'
+            self.info(f"  * Job type: {job_type} (id={rec['jobid']})")
+            self.info(f"  * Dependencies (conceptual): "
+                         f"{rec['dependencies_conceptual']}")
+            self.info(f"  * Dependencies (actual): "
+                         f"{rec['dependencies_actual']}")
+            self.info(f"  * Maintainers: {rec['maintainers']}")
+            self.info(f"  * Failing phase: {rec['fail_phase']}")
+            if rerun_info and not rec['fixture']:
+                self.info(f"  * Rerun with '-n /{rec['hashcode']}"
+                             f" -p {rec['environ']} --system "
+                             f"{rec['system']} -r'")
+
+            msg = rec['fail_reason']
+            if isinstance(rec['fail_info']['exc_value'], SanityError):
+                lines = [msg]
+                lines += _head_n(rec['job_stdout'], prefix=rec['stagedir'])
+                lines += _head_n(rec['job_stderr'], prefix=rec['stagedir'])
+                msg = '\n'.join(lines)
+
+            self.info(f"  * Reason: {msg}")
+
+            tb = ''.join(traceback.format_exception(
+                *rec['fail_info'].values()))
+            if rec['fail_severe']:
+                self.info(tb)
+            else:
+                self.verbose(tb)
+
+        line_width = shutil.get_terminal_size()[0]
+        self.info(line_width * '=')
+        self.info('SUMMARY OF FAILURES')
+
+        for run_no, run_info in enumerate(report['runs'], start=1):
+            if not global_stats and run_no != len(report['runs']):
+                continue
+
+            for r in run_info['testcases']:
+                if r['result'] in {'pass', 'abort', 'skip'}:
+                    continue
+
+                _print_failure_info(r, run_no, len(report['runs']))
+
+        self.info(line_width * '-')
+
+    def failure_stats(self, report, global_stats=False):
+        current_run = rt.runtime().current_run
+        failures = {}
+        for runid, run_data in enumerate(report['runs']):
+            if not global_stats and runid != current_run:
+                continue
+
+            for tc in run_data['testcases']:
+                info = f'{tc["display_name"]}'
+                info += f' @{tc["system"]}:{tc["partition"]}+{tc["environ"]}'
+
+                failed_stage = tc['fail_phase']
+                failures.setdefault(failed_stage, [])
+                failures[failed_stage].append(info)
+
+        line_width = shutil.get_terminal_size()[0]
+        stats_start = line_width * '='
+        stats_title = 'FAILURE STATISTICS'
+        stats_end = line_width * '-'
+        stats_body = []
+        row_format = "{:<13} {:<5} {}"
+        stats_hline = row_format.format(13*'-', 5*'-', 60*'-')
+        stats_header = row_format.format('Phase', '#', 'Failing test cases')
+        if global_stats:
+            num_tests = report['session_info']['num_cases']
+        else:
+            num_tests = report['runs'][current_run]['num_cases']
+
+        num_failures = 0
+        for fl in failures.values():
+            num_failures += len(fl)
+
+        stats_body = ['']
+        stats_body.append(f'Total number of test cases: {num_tests}')
+        stats_body.append(f'Total number of failures: {num_failures}')
+        stats_body.append('')
+        stats_body.append(stats_header)
+        stats_body.append(stats_hline)
+        for p, l in failures.items():
+            stats_body.append(row_format.format(p, len(l), l[0]))
+            for f in l[1:]:
+                stats_body.append(row_format.format('', '', str(f)))
+
+        if stats_body:
+            for line in (stats_start, stats_title, *stats_body, stats_end):
+                self.info(line)
+
+    def retry_report(self, report):
+        '''Print a report for test retries'''
+
+        if not rt.runtime().current_run:
+            # Do nothing if no retries
+            return
+
+        line_width = shutil.get_terminal_size()[0]
+        lines = [line_width * '=']
+        lines.append('SUMMARY OF RETRIES')
+        lines.append(line_width * '-')
+        messages = {}
+        for i, run in enumerate(report['runs'][1:], start=1):
+            for tc in run['testcases']:
+                # Overwrite entry from previous run if available
+                tc_info = format_testcase(tc)
+                messages[tc_info] = (
+                    f"  * Test {tc_info} was retried {i} time(s) and"
+                    f" {'failed' if tc['result'] == 'fail' else 'passed'}."
+                )
+
+        for msg in sorted(messages):
+            lines.append(msg)
+
+        self.info('\n'.join(lines))
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
new file mode 100644
index 0000000000..97f1203949
--- /dev/null
+++ b/reframe/frontend/reporting/__init__.py
@@ -0,0 +1,457 @@
+# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# ReFrame Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+import decimal
+import functools
+import inspect
+import json
+import jsonschema
+import lxml.etree as etree
+import os
+import re
+import socket
+import time
+from datetime import datetime
+
+import reframe as rfm
+import reframe.utility.jsonext as jsonext
+import reframe.utility.osext as osext
+from reframe.core.exceptions import ReframeError, what, is_severe
+from reframe.core.logging import getlogger, _format_time_rfc3339
+from reframe.core.warnings import suppress_deprecations
+from .analytics import (parse_cmp_spec,
+                        fetch_testcases_time_period,
+                        compare_testcase_data)
+
+
+# The schema data version
+# Major version bumps are expected to break the validation of previous schemas
+
+DATA_VERSION = '4.0'
+_SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json')
+
+
+def format_testcase(json, name='unique_name'):
+    '''Format test case from its json representation'''
+    name = json[name]
+    system = json['system']
+    partition = json['partition']
+    environ = json['environ']
+    return f'{name}@{system}:{partition}+{environ}'
+
+
+class _RestoredSessionInfo:
+    '''A restored session with some additional functionality.'''
+
+    def __init__(self, report):
+        self._report = report
+        self._fallbacks = []    # fallback reports
+
+        # Index all runs by test case; if a test case has run multiple times,
+        # only the last time will be indexed
+        self._cases_index = {}
+        for run in self._report['runs']:
+            for tc in run['testcases']:
+                self._cases_index[format_testcase(tc)] = tc
+
+        # Index also the restored cases
+        for tc in self._report['restored_cases']:
+            self._cases_index[format_testcase(tc)] = tc
+
+    def __getitem__(self, key):
+        return self._report[key]
+
+    def __getattr__(self, name):
+        with suppress_deprecations():
+            return getattr(self._report, name)
+
+    def add_fallback(self, report):
+        self._fallbacks.append(report)
+
+    def slice(self, prop, when=None, unique=False):
+        '''Slice the report on property ``prop``.'''
+
+        if unique:
+            returned = set()
+
+        for tc in self._report['runs'][-1]['testcases']:
+            val = tc[prop]
+            if unique and val in returned:
+                continue
+
+            if when is None:
+                if unique:
+                    returned.add(val)
+
+                yield val
+            elif tc[when[0]] == when[1]:
+                if unique:
+                    returned.add(val)
+
+                yield val
+
+    def case(self, check, part, env):
+        key = f'{check.unique_name}@{part.fullname}+{env.name}'
+        ret = self._cases_index.get(key)
+        if ret is None:
+            # Look up the case in the fallback reports
+            for rpt in self._fallbacks:
+                ret = rpt._cases_index.get(key)
+                if ret is not None:
+                    break
+
+        return ret
+
+    def restore_dangling(self, graph):
+        '''Restore dangling dependencies in graph from the report data.
+
+        Returns the updated graph.
+        '''
+
+        restored = []
+        for tc, deps in graph.items():
+            for d in deps:
+                if d not in graph:
+                    restored.append(d)
+                    self._do_restore(d)
+
+        return graph, restored
+
+    def _do_restore(self, testcase):
+        tc = self.case(*testcase)
+        if tc is None:
+            raise ReframeError(
+                f'could not restore testcase {testcase!r}: '
+                f'not found in the report files'
+            )
+
+        dump_file = os.path.join(tc['stagedir'], '.rfm_testcase.json')
+        try:
+            with open(dump_file) as fp:
+                testcase._check = jsonext.load(fp)
+        except (OSError, json.JSONDecodeError) as e:
+            raise ReframeError(
+                f'could not restore testcase {testcase!r}') from e
+
+
+def _expand_report_filename(filepatt, *, newfile):
+    if '{sessionid}' not in os.fspath(filepatt):
+        return filepatt
+
+    search_patt = os.path.basename(filepatt).replace('{sessionid}', r'(\d+)')
+    new_id = -1
+    basedir = os.path.dirname(filepatt) or '.'
+    for filename in os.listdir(basedir):
+        match = re.match(search_patt, filename)
+        if match:
+            found_id = int(match.group(1))
+            new_id = max(found_id, new_id)
+
+    if newfile:
+        new_id += 1
+
+    return filepatt.format(sessionid=new_id)
+
+
+def _restore_session(filename):
+    filename = _expand_report_filename(filename, newfile=False)
+    try:
+        with open(filename) as fp:
+            report = json.load(fp)
+    except OSError as e:
+        raise ReframeError(
+            f'failed to load report file {filename!r}') from e
+    except json.JSONDecodeError as e:
+        raise ReframeError(
+            f'report file {filename!r} is not a valid JSON file') from e
+
+    # Validate the report
+    with open(_SCHEMA) as fp:
+        schema = json.load(fp)
+
+    try:
+        jsonschema.validate(report, schema)
+    except jsonschema.ValidationError as e:
+        try:
+            found_ver = report['session_info']['data_version']
+        except KeyError:
+            found_ver = 'n/a'
+
+        getlogger().verbose(f'JSON validation error: {e}')
+        raise ReframeError(
+            f'failed to validate report {filename!r}: {e.args[0]} '
+            f'(check report data version: required {DATA_VERSION}, '
+            f'found: {found_ver})'
+        ) from None
+
+    return _RestoredSessionInfo(report)
+
+
+def restore_session(*filenames):
+    primary = filenames[0]
+    restored = _restore_session(primary)
+
+    # Add fallback reports
+    for f in filenames[1:]:
+        restored.add_fallback(_restore_session(f))
+
+    return restored
+
+
+class RunReport:
+    '''Internal representation of a run report
+
+    This class provides direct access to the underlying report and provides
+    convenience functions for constructing a new report.
+    '''
+    def __init__(self):
+        # Initialize the report with the required fields
+        now = datetime.now().strftime(r'%FT%T%z')
+        self.__report = {
+            'session_info': {
+                'data_version': DATA_VERSION,
+                'hostname': socket.gethostname(),
+                'time_start': now,
+                'time_end': now,
+                'time_elapsed': 0.0
+            },
+            'runs': [],
+            'restored_cases': []
+        }
+        self.__filename = None
+
+    @property
+    def filename(self):
+        return self.__filename
+
+    def __getattr__(self, name):
+        return getattr(self.__report, name)
+
+    def __getitem__(self, key):
+        return self.__report[key]
+
+    def __rfm_json_encode__(self):
+        return self.__report
+
+    def update_session_info(self, session_info):
+        self.__report['session_info'].update(session_info)
+
+    def update_restored_cases(self, restored_cases, restored_session):
+        self.__report['restored_cases'] = [restored_session.case(*c)
+                                           for c in restored_cases]
+
+    def update_run_stats(self, stats):
+        for runid, tasks in stats.runs():
+            testcases = []
+            num_failures = 0
+            num_aborted = 0
+            num_skipped = 0
+            for t in tasks:
+                # We take partition and environment from the test case and not
+                # from the check, since if the test fails before `setup()`,
+                # these are not set inside the check.
+                check, partition, environ = t.testcase
+                entry = {
+                    'build_jobid': None,
+                    'build_stderr': None,
+                    'build_stdout': None,
+                    'dependencies_actual': [
+                        (d.check.unique_name,
+                         d.partition.fullname, d.environ.name)
+                        for d in t.testcase.deps
+                    ],
+                    'dependencies_conceptual': [
+                        d[0] for d in t.check.user_deps()
+                    ],
+                    'environ': environ.name,
+                    'fail_phase': None,
+                    'fail_reason': None,
+                    'filename': inspect.getfile(type(check)),
+                    'fixture': check.is_fixture(),
+                    'job_completion_time': None,
+                    'job_completion_time_unix': None,
+                    'job_stderr': None,
+                    'job_stdout': None,
+                    'partition': partition.name,
+                    'result': t.result,
+                    'scheduler': partition.scheduler.registered_name,
+                    'time_compile': t.duration('compile_complete'),
+                    'time_performance': t.duration('performance'),
+                    'time_run': t.duration('run_complete'),
+                    'time_sanity': t.duration('sanity'),
+                    'time_setup': t.duration('setup'),
+                    'time_total': t.duration('total')
+                }
+                if check.job:
+                    entry['job_stderr'] = check.stderr.evaluate()
+                    entry['job_stdout'] = check.stdout.evaluate()
+
+                if check.build_job:
+                    entry['build_stderr'] = check.build_stderr.evaluate()
+                    entry['build_stdout'] = check.build_stdout.evaluate()
+
+                if t.failed:
+                    num_failures += 1
+                elif t.aborted:
+                    num_aborted += 1
+                elif t.skipped:
+                    num_skipped += 1
+
+                if t.failed or t.aborted:
+                    entry['fail_phase'] = t.failed_stage
+                    if t.exc_info is not None:
+                        entry['fail_reason'] = what(*t.exc_info)
+                        entry['fail_info'] = {
+                            'exc_type':  t.exc_info[0],
+                            'exc_value': t.exc_info[1],
+                            'traceback': t.exc_info[2]
+                        }
+                        entry['fail_severe'] = is_severe(*t.exc_info)
+                elif t.succeeded:
+                    entry['outputdir'] = check.outputdir
+
+                # Add any loggable variables and parameters
+                test_cls = type(check)
+                for name, alt_name in test_cls.loggable_attrs():
+                    if alt_name == 'partition' or alt_name == 'environ':
+                        # We set those from the testcase
+                        continue
+
+                    key = alt_name if alt_name else name
+                    try:
+                        with suppress_deprecations():
+                            entry[key] = getattr(check, name)
+                    except AttributeError:
+                        entry[key] = '<undefined>'
+
+                if entry['job_completion_time_unix']:
+                    entry['job_completion_time'] = _format_time_rfc3339(
+                        time.localtime(entry['job_completion_time_unix']),
+                        '%FT%T%:z'
+                    )
+
+                testcases.append(entry)
+
+            self.__report['runs'].append({
+                'num_cases': len(tasks),
+                'num_failures': num_failures,
+                'num_aborted': num_aborted,
+                'num_skipped': num_skipped,
+                'runid': runid,
+                'testcases': testcases
+            })
+
+        # Update session info from stats
+        self.__report['session_info'].update({
+            'num_cases': self.__report['runs'][0]['num_cases'],
+            'num_failures': self.__report['runs'][-1]['num_failures'],
+            'num_aborted': self.__report['runs'][-1]['num_aborted'],
+            'num_skipped': self.__report['runs'][-1]['num_skipped']
+        })
+
+    def save(self, filename, compress=False, link_to_last=True):
+        filename = _expand_report_filename(filename, newfile=True)
+        with open(filename, 'w') as fp:
+            if compress:
+                jsonext.dump(self.__report, fp)
+            else:
+                jsonext.dump(self.__report, fp, indent=2)
+                fp.write('\n')
+
+        self.__filename = filename
+        if not link_to_last:
+            return
+
+        link_name = 'latest.json'
+        prefix, target_name = os.path.split(filename)
+        with osext.change_dir(prefix):
+            create_symlink = functools.partial(os.symlink,
+                                               target_name, link_name)
+            if not os.path.exists(link_name):
+                create_symlink()
+            else:
+                if os.path.islink(link_name):
+                    os.remove(link_name)
+                    create_symlink()
+                else:
+                    raise ReframeError('path exists and is not a symlink')
+
+    def generate_xml_report(self):
+        '''Generate a JUnit report from a standard ReFrame JSON report.'''
+
+        report = self.__report
+        xml_testsuites = etree.Element('testsuites')
+        for run_id, rfm_run in enumerate(report['runs']):
+            xml_testsuite = etree.SubElement(
+                xml_testsuites, 'testsuite',
+                attrib={
+                    'errors': '0',
+                    'failures': str(rfm_run['num_failures']),
+                    'hostname': report['session_info']['hostname'],
+                    'id': str(run_id),
+                    'name': f'ReFrame run {run_id}',
+                    'package': 'reframe',
+                    'tests': str(rfm_run['num_cases']),
+                    'time': str(report['session_info']['time_elapsed']),
+                    # XSD schema does not like the timezone format,
+                    # so we remove it
+                    'timestamp': report['session_info']['time_start'][:-5],
+                }
+            )
+            etree.SubElement(xml_testsuite, 'properties')
+            for tc in rfm_run['testcases']:
+                casename = f'{format_testcase(tc, name="name")}'
+                testcase = etree.SubElement(
+                    xml_testsuite, 'testcase',
+                    attrib={
+                        'classname': tc['filename'],
+                        'name': casename,
+
+                        # XSD schema does not like the exponential format and
+                        # since we do not want to impose a fixed width, we pass
+                        # it to `Decimal` to format it automatically.
+                        'time': str(decimal.Decimal(tc['time_total'] or 0)),
+                    }
+                )
+                if tc['result'] == 'fail':
+                    testcase_msg = etree.SubElement(
+                        testcase, 'failure', attrib={'type': 'failure',
+                                                    'message': tc['fail_phase']}
+                    )
+                    testcase_msg.text = f"{tc['fail_phase']}: {tc['fail_reason']}"
+
+            testsuite_stdout = etree.SubElement(xml_testsuite, 'system-out')
+            testsuite_stdout.text = ''
+            testsuite_stderr = etree.SubElement(xml_testsuite, 'system-err')
+            testsuite_stderr.text = ''
+
+        return xml_testsuites
+
+    def save_junit(self, filename):
+        with open(filename, 'w') as fp:
+            xml = self.generate_xml_report()
+            fp.write(
+                etree.tostring(xml, encoding='utf8', pretty_print=True,
+                               method='xml', xml_declaration=True).decode()
+            )
+
+def performance_compare(cmp, report=None):
+    match = parse_cmp_spec(cmp)
+    if match.period_base is None:
+        if report is None:
+            raise ValueError('report cannot be `None` '
+                             'for current run comparisons')
+        try:
+            tcs_base = report['runs'][0]['testcases']
+        except IndexError:
+            tcs_base = []
+
+    else:
+        tcs_base = fetch_testcases_time_period(*match.period_base)
+
+    tcs_target = fetch_testcases_time_period(*match.period_target)
+    return compare_testcase_data(tcs_base, tcs_target, match.aggregator,
+                                 match.aggregator, match.extra_groups,
+                                 match.extra_cols)
diff --git a/reframe/frontend/runreport.py b/reframe/frontend/reporting/analytics.py
similarity index 52%
rename from reframe/frontend/runreport.py
rename to reframe/frontend/reporting/analytics.py
index c1de31cf7f..bdba66c21e 100644
--- a/reframe/frontend/runreport.py
+++ b/reframe/frontend/reporting/analytics.py
@@ -4,286 +4,23 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import abc
-import decimal
-import functools
-import glob
-import json
-import jsonschema
-import lxml.etree as etree
-import math
 import os
+import math
 import re
 import sqlite3
 import statistics
 import types
+from collections import namedtuple
 from collections.abc import Hashable
 from datetime import datetime, timedelta
 
-import reframe as rfm
 import reframe.core.runtime as runtime
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
-from reframe.core.exceptions import ReframeError
 from reframe.core.logging import getlogger
-from reframe.core.warnings import suppress_deprecations
 from reframe.utility import nodelist_abbrev
 
 
-# The schema data version
-# Major version bumps are expected to break the validation of previous schemas
-
-DATA_VERSION = '4.0'
-_SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json')
-
-def _tc_info(tc_entry, name='unique_name'):
-    name = tc_entry[name]
-    system = tc_entry['system']
-    partition = tc_entry['partition']
-    environ = tc_entry['environ']
-    return f'{name}@{system}:{partition}+{environ}'
-
-
-class _RunReport:
-    '''A wrapper to the run report providing some additional functionality'''
-
-    def __init__(self, report):
-        self._report = report
-        self._fallbacks = []    # fallback reports
-
-        # Index all runs by test case; if a test case has run multiple times,
-        # only the last time will be indexed
-        self._cases_index = {}
-        for run in self._report['runs']:
-            for tc in run['testcases']:
-                self._cases_index[_tc_info(tc)] = tc
-
-        # Index also the restored cases
-        for tc in self._report['restored_cases']:
-            self._cases_index[_tc_info(tc)] = tc
-
-    def __getitem__(self, key):
-        return self._report[key]
-
-    def __getattr__(self, name):
-        with suppress_deprecations():
-            return getattr(self._report, name)
-
-    def add_fallback(self, report):
-        self._fallbacks.append(report)
-
-    def slice(self, prop, when=None, unique=False):
-        '''Slice the report on property ``prop``.'''
-
-        if unique:
-            returned = set()
-
-        for tc in self._report['runs'][-1]['testcases']:
-            val = tc[prop]
-            if unique and val in returned:
-                continue
-
-            if when is None:
-                if unique:
-                    returned.add(val)
-
-                yield val
-            elif tc[when[0]] == when[1]:
-                if unique:
-                    returned.add(val)
-
-                yield val
-
-    def case(self, check, part, env):
-        key = f'{check.unique_name}@{part.fullname}+{env.name}'
-        ret = self._cases_index.get(key)
-        if ret is None:
-            # Look up the case in the fallback reports
-            for rpt in self._fallbacks:
-                ret = rpt._cases_index.get(key)
-                if ret is not None:
-                    break
-
-        return ret
-
-    def restore_dangling(self, graph):
-        '''Restore dangling dependencies in graph from the report data.
-
-        Returns the updated graph.
-        '''
-
-        restored = []
-        for tc, deps in graph.items():
-            for d in deps:
-                if d not in graph:
-                    restored.append(d)
-                    self._do_restore(d)
-
-        return graph, restored
-
-    def _do_restore(self, testcase):
-        tc = self.case(*testcase)
-        if tc is None:
-            raise ReframeError(
-                f'could not restore testcase {testcase!r}: '
-                f'not found in the report files'
-            )
-
-        dump_file = os.path.join(tc['stagedir'], '.rfm_testcase.json')
-        try:
-            with open(dump_file) as fp:
-                testcase._check = jsonext.load(fp)
-        except (OSError, json.JSONDecodeError) as e:
-            raise ReframeError(
-                f'could not restore testcase {testcase!r}') from e
-
-
-def next_report_filename(filepatt, new=True):
-    if '{sessionid}' not in filepatt:
-        return filepatt
-
-    search_patt = os.path.basename(filepatt).replace('{sessionid}', r'(\d+)')
-    new_id = -1
-    basedir = os.path.dirname(filepatt) or '.'
-    for filename in os.listdir(basedir):
-        match = re.match(search_patt, filename)
-        if match:
-            found_id = int(match.group(1))
-            new_id = max(found_id, new_id)
-
-    if new:
-        new_id += 1
-
-    return filepatt.format(sessionid=new_id)
-
-
-def _load_report(filename):
-    try:
-        with open(filename) as fp:
-            report = json.load(fp)
-    except OSError as e:
-        raise ReframeError(
-            f'failed to load report file {filename!r}') from e
-    except json.JSONDecodeError as e:
-        raise ReframeError(
-            f'report file {filename!r} is not a valid JSON file') from e
-
-    # Validate the report
-    with open(_SCHEMA) as fp:
-        schema = json.load(fp)
-
-    try:
-        jsonschema.validate(report, schema)
-    except jsonschema.ValidationError as e:
-        try:
-            found_ver = report['session_info']['data_version']
-        except KeyError:
-            found_ver = 'n/a'
-
-        getlogger().verbose(f'JSON validation error: {e}')
-        raise ReframeError(
-            f'failed to validate report {filename!r}: {e.args[0]} '
-            f'(check report data version: required {DATA_VERSION}, '
-            f'found: {found_ver})'
-        ) from None
-
-    return _RunReport(report)
-
-
-def load_report(*filenames):
-    primary = filenames[0]
-    rpt = _load_report(primary)
-
-    # Add fallback reports
-    for f in filenames[1:]:
-        rpt.add_fallback(_load_report(f))
-
-    return rpt
-
-
-def save_report(report, filename, compress=False):
-    with open(filename, 'w') as fp:
-        if compress:
-            jsonext.dump(report, fp)
-        else:
-            jsonext.dump(report, fp, indent=2)
-            fp.write('\n')
-
-def link_latest_report(filename, link_name):
-    prefix, target_name = os.path.split(filename)
-    with osext.change_dir(prefix):
-        create_symlink = functools.partial(os.symlink, target_name, link_name)
-        if not os.path.exists(link_name):
-            create_symlink()
-        else:
-            if os.path.islink(link_name):
-                os.remove(link_name)
-                create_symlink()
-            else:
-                raise ReframeError('path exists and is not a symlink')
-
-
-def junit_xml_report(json_report):
-    '''Generate a JUnit report from a standard ReFrame JSON report.'''
-
-    xml_testsuites = etree.Element('testsuites')
-    for run_id, rfm_run in enumerate(json_report['runs']):
-        xml_testsuite = etree.SubElement(
-            xml_testsuites, 'testsuite',
-            attrib={
-                'errors': '0',
-                'failures': str(rfm_run['num_failures']),
-                'hostname': json_report['session_info']['hostname'],
-                'id': str(run_id),
-                'name': f'ReFrame run {run_id}',
-                'package': 'reframe',
-                'tests': str(rfm_run['num_cases']),
-                'time': str(json_report['session_info']['time_elapsed']),
-                # XSD schema does not like the timezone format, so we remove it
-                'timestamp': json_report['session_info']['time_start'][:-5],
-            }
-        )
-        etree.SubElement(xml_testsuite, 'properties')
-        for tc in rfm_run['testcases']:
-            casename = f'{_tc_info(tc, name="name")}'
-            testcase = etree.SubElement(
-                xml_testsuite, 'testcase',
-                attrib={
-                    'classname': tc['filename'],
-                    'name': casename,
-
-                    # XSD schema does not like the exponential format and since
-                    # we do not want to impose a fixed width, we pass it to
-                    # `Decimal` to format it automatically.
-                    'time': str(decimal.Decimal(tc['time_total'] or 0)),
-                }
-            )
-            if tc['result'] == 'fail':
-                testcase_msg = etree.SubElement(
-                    testcase, 'failure', attrib={'type': 'failure',
-                                                 'message': tc['fail_phase']}
-                )
-                testcase_msg.text = f"{tc['fail_phase']}: {tc['fail_reason']}"
-
-        testsuite_stdout = etree.SubElement(xml_testsuite, 'system-out')
-        testsuite_stdout.text = ''
-        testsuite_stderr = etree.SubElement(xml_testsuite, 'system-err')
-        testsuite_stderr.text = ''
-
-    return xml_testsuites
-
-
-def junit_dump(xml, fp):
-    fp.write(
-        etree.tostring(xml, encoding='utf8', pretty_print=True,
-                       method='xml', xml_declaration=True).decode()
-    )
-
-
-def get_reports_files(directory):
-    return [f for f in glob.glob(f"{directory}/*")
-            if os.path.isfile(f) and not f.endswith('/latest.json')]
-
-
 def _db_file():
     site_config = runtime.runtime().site_config
     prefix = os.path.dirname(osext.expandvars(
@@ -301,11 +38,6 @@ def _db_file():
     return filename
 
 
-def store_results(report, report_file):
-    with sqlite3.connect(_db_file()) as conn:
-        _db_store_report(conn, report, report_file)
-
-
 def _db_create(filename):
     with sqlite3.connect(filename) as conn:
         conn.execute(
@@ -331,7 +63,6 @@ def _db_create(filename):
 
 
 def _db_store_report(conn, report, report_file_path):
-    session_start = report['session_info']['time_start']
     for run_idx, run in enumerate(report['runs']):
         for test_idx, testcase in enumerate(run['testcases']):
             sys, part = testcase['system'], testcase['partition']
@@ -359,7 +90,12 @@ def _db_store_report(conn, report, report_file_path):
             )
 
 
-def _fetch_cases_raw(condition):
+def store_report(report, report_file):
+    with sqlite3.connect(_db_file()) as conn:
+        _db_store_report(conn, report, report_file)
+
+
+def _fetch_testcases_raw(condition):
     with sqlite3.connect(_db_file()) as conn:
         query = (f'SELECT session_id, run_index, test_index, json_blob FROM '
                  f'testcases JOIN sessions ON session_id==id '
@@ -371,14 +107,14 @@ def _fetch_cases_raw(condition):
     testcases = []
     sessions = {}
     for session_id, run_index, test_index, json_blob in results:
-        report = json.loads(sessions.setdefault(session_id, json_blob))
+        report = jsonext.loads(sessions.setdefault(session_id, json_blob))
         testcases.append(report['runs'][run_index]['testcases'][test_index])
 
     return testcases
 
 
-def _fetch_cases_time_period(ts_start, ts_end):
-    return _fetch_cases_raw(
+def fetch_testcases_time_period(ts_start, ts_end):
+    return _fetch_testcases_raw(
         f'(job_completion_time_unix >= {ts_start} AND '
         f'job_completion_time_unix < {ts_end}) '
         'ORDER BY job_completion_time_unix'
@@ -400,46 +136,6 @@ def _group_key(groups, testcase):
     return tuple(key)
 
 
-def parse_timestamp(s):
-    now = datetime.now()
-    def _do_parse(s):
-        if s == 'now':
-            return now
-
-        formats = [r'%Y%m%d', r'%Y%m%dT%H%M',
-                   r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z']
-        for fmt in formats:
-            try:
-                return datetime.strptime(s, fmt)
-            except ValueError:
-                continue
-
-        raise ValueError(f'invalid timestamp: {s}')
-
-
-    try:
-        ts = _do_parse(s)
-    except ValueError as err:
-        # Try the relative timestamps
-        match = re.match(r'(?P<ts>.*)(?P<amount>[\+|-]\d+)(?P<unit>[hdms])', s)
-        if not match:
-            raise err
-
-        ts = _do_parse(match.group('ts'))
-        amount = int(match.group('amount'))
-        unit = match.group('unit')
-        if unit == 'd':
-            ts += timedelta(days=amount)
-        elif unit == 'm':
-            ts += timedelta(minutes=amount)
-        elif unit == 'h':
-            ts += timedelta(hours=amount)
-        elif unit == 's':
-            ts += timedelta(seconds=amount)
-
-    return ts.timestamp()
-
-
 def _group_testcases(testcases, group_by, extra_cols):
     grouped = {}
     for tc in testcases:
@@ -525,7 +221,6 @@ def create(cls, name):
         else:
             raise ValueError(f'unknown aggregation function: {name!r}')
 
-
     @abc.abstractmethod
     def __call__(self, iterable):
         pass
@@ -574,26 +269,114 @@ def __call__(self, iterable):
         unique_vals = {str(elem) for elem in iterable}
         return self.__delim.join(unique_vals)
 
+
+def _parse_timestamp(s):
+    now = datetime.now()
+    def _do_parse(s):
+        if s == 'now':
+            return now
+
+        formats = [r'%Y%m%d', r'%Y%m%dT%H%M',
+                   r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z']
+        for fmt in formats:
+            try:
+                return datetime.strptime(s, fmt)
+            except ValueError:
+                continue
+
+        raise ValueError(f'invalid timestamp: {s}')
+
+    try:
+        ts = _do_parse(s)
+    except ValueError as err:
+        # Try the relative timestamps
+        match = re.match(r'(?P<ts>.*)(?P<amount>[\+|-]\d+)(?P<unit>[hdms])', s)
+        if not match:
+            raise err
+
+        ts = _do_parse(match.group('ts'))
+        amount = int(match.group('amount'))
+        unit = match.group('unit')
+        if unit == 'd':
+            ts += timedelta(days=amount)
+        elif unit == 'm':
+            ts += timedelta(minutes=amount)
+        elif unit == 'h':
+            ts += timedelta(hours=amount)
+        elif unit == 's':
+            ts += timedelta(seconds=amount)
+
+    return ts.timestamp()
+
+def _parse_time_period(s):
+    try:
+        ts_start, ts_end = s.split(':')
+    except ValueError:
+        raise ValueError(f'invalid time period spec: {s}') from None
+
+    return _parse_timestamp(ts_start), _parse_timestamp(ts_end)
+
+def _parse_extra_cols(s):
+    try:
+        extra_cols = s.split('+')[1:]
+    except (ValueError, IndexError):
+        raise ValueError(f'invalid extra groups spec: {s}') from None
+
+    return extra_cols
+
+
+def _parse_aggregation(s):
+    try:
+        op, extra_groups = s.split(':')
+    except ValueError:
+        raise ValueError(f'invalid aggregate function spec: {s}') from None
+
+    return _Aggregator.create(op), _parse_extra_cols(extra_groups)
+
+
+_Match = namedtuple('_Match', ['period_base', 'period_target',
+                               'aggregator', 'extra_groups', 'extra_cols'])
+
+def parse_cmp_spec(spec):
+    parts = spec.split('/')
+    if len(parts) == 3:
+        period_base, period_target, aggr, cols = None, *parts
+    elif len(parts) == 4:
+        period_base, period_target, aggr, cols = parts
+    else:
+        raise ValueError(f'invalid cmp spec: {spec}')
+
+    if period_base is not None:
+        period_base = _parse_time_period(period_base)
+
+    period_target = _parse_time_period(period_target)
+    aggr_fn, extra_groups = _parse_aggregation(aggr)
+    extra_cols = _parse_extra_cols(cols)
+    return _Match(period_base, period_target,
+                  aggr_fn, extra_groups, extra_cols)
+
+
 def performance_report_data(run_stats, report_spec):
     period, aggr, cols = report_spec.split('/')
-    ts_start, ts_end = [parse_timestamp(ts) for ts in period.split(':')]
+    ts_start, ts_end = [_parse_timestamp(ts) for ts in period.split(':')]
     op, extra_groups = aggr.split(':')
     aggr_fn = _Aggregator.create(op)
     extra_groups = extra_groups.split('+')[1:]
     extra_cols = cols.split('+')[1:]
     testcases = run_stats[0]['testcases']
-    target_testcases = _fetch_cases_time_period(ts_start, ts_end)
-    return compare_testcase_data(testcases, target_testcases, _First(),
+    target_testcases = fetch_testcases_time_period(ts_start, ts_end)
+    return compare_testcase_data(testcases, target_testcases,
+                                 _Aggregator.create('first'),
                                  aggr_fn, extra_groups, extra_cols)
 
 
 def performance_compare_data(spec):
     period_base, period_target, aggr, cols = spec.split('/')
-    base_testcases = _fetch_cases_time_period(
-        *(parse_timestamp(ts) for ts in period_base.split(':'))
+    base_testcases = fetch_testcases_time_period(
+        *(_parse_timestamp(ts) for ts in period_base.split(':'))
     )
-    target_testcases = _fetch_cases_time_period(
-        *(parse_timestamp(ts) for ts in period_target.split(':'))
+    target_testcases = fetch_testcases_time_period(
+        *(_parse_timestamp(ts) for ts in period_target.split(':'))
     )
     op, extra_groups = aggr.split(':')
     aggr_fn = _Aggregator.create(op)
diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py
deleted file mode 100644
index 4ad0a4e680..0000000000
--- a/reframe/frontend/statistics.py
+++ /dev/null
@@ -1,396 +0,0 @@
-# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
-# ReFrame Project Developers. See the top-level LICENSE file for details.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-import inspect
-import itertools
-import os
-import shutil
-import time
-import traceback
-
-import reframe.core.runtime as rt
-import reframe.core.exceptions as errors
-import reframe.utility as util
-from reframe.core.logging import _format_time_rfc3339
-from reframe.core.warnings import suppress_deprecations
-
-
-def _getattr(obj, attr):
-    with suppress_deprecations():
-        return getattr(obj, attr)
-
-
-class TestStats:
-    '''Stores test case statistics.'''
-
-    def __init__(self):
-        # Tasks per run stored as follows: [[run0_tasks], [run1_tasks], ...]
-        self._alltasks = [[]]
-
-        # Data collected for all the runs of this session in JSON format
-        self._run_data = []
-
-    def add_task(self, task):
-        current_run = rt.runtime().current_run
-        if current_run == len(self._alltasks):
-            self._alltasks.append([])
-
-        self._alltasks[current_run].append(task)
-
-    def tasks(self, run=-1):
-        if run is None:
-            yield from itertools.chain(*self._alltasks)
-        else:
-            try:
-                yield from self._alltasks[run]
-            except IndexError:
-                raise errors.StatisticsError(f'no such run: {run}') from None
-
-    def failed(self, run=-1):
-        return [t for t in self.tasks(run) if t.failed]
-
-    def skipped(self, run=-1):
-        return [t for t in self.tasks(run) if t.skipped]
-
-    def aborted(self, run=-1):
-        return [t for t in self.tasks(run) if t.aborted]
-
-    def completed(self, run=-1):
-        return [t for t in self.tasks(run) if t.completed]
-
-    def num_cases(self, run=-1):
-        return sum(1 for _ in self.tasks(run))
-
-    @property
-    def num_runs(self):
-        return len(self._alltasks)
-
-    def retry_report(self):
-        # Return an empty report if no retries were done.
-        if not rt.runtime().current_run:
-            return ''
-
-        line_width = shutil.get_terminal_size()[0]
-        report = [line_width * '=']
-        report.append('SUMMARY OF RETRIES')
-        report.append(line_width * '-')
-        messages = {}
-        for run in range(1, len(self._alltasks)):
-            for t in self.tasks(run):
-                partition_name = ''
-                environ_name = ''
-                if t.check.current_partition:
-                    partition_name = t.check.current_partition.fullname
-
-                if t.check.current_environ:
-                    environ_name = t.check.current_environ.name
-
-                # Overwrite entry from previous run if available
-                key = f"{t.check.unique_name}:{partition_name}:{environ_name}"
-                messages[key] = (
-                    f"  * Test {t.check.info()} was retried {run} time(s) and "
-                    f"{'failed' if t.failed else 'passed'}."
-                )
-
-        for key in sorted(messages.keys()):
-            report.append(messages[key])
-
-        return '\n'.join(report)
-
-    def json(self, force=False):
-        if not force and self._run_data:
-            return self._run_data
-
-        for runid, run in enumerate(self._alltasks):
-            testcases = []
-            num_failures = 0
-            num_aborted = 0
-            num_skipped = 0
-            for t in run:
-                # We take partition and environment from the test case and not
-                # from the check, since if the test fails before `setup()`,
-                # these are not set inside the check.
-                check, partition, environ = t.testcase
-                entry = {
-                    'build_jobid': None,
-                    'build_stderr': None,
-                    'build_stdout': None,
-                    'dependencies_actual': [
-                        (d.check.unique_name,
-                         d.partition.fullname, d.environ.name)
-                        for d in t.testcase.deps
-                    ],
-                    'dependencies_conceptual': [
-                        d[0] for d in t.check.user_deps()
-                    ],
-                    'environ': environ.name,
-                    'fail_phase': None,
-                    'fail_reason': None,
-                    'filename': inspect.getfile(type(check)),
-                    'fixture': check.is_fixture(),
-                    'job_completion_time': None,
-                    'job_completion_time_unix': None,
-                    'job_stderr': None,
-                    'job_stdout': None,
-                    'partition': partition.name,
-                    'result': t.result,
-                    'scheduler': partition.scheduler.registered_name,
-                    'time_compile': t.duration('compile_complete'),
-                    'time_performance': t.duration('performance'),
-                    'time_run': t.duration('run_complete'),
-                    'time_sanity': t.duration('sanity'),
-                    'time_setup': t.duration('setup'),
-                    'time_total': t.duration('total')
-                }
-                if check.job:
-                    entry['job_stderr'] = check.stderr.evaluate()
-                    entry['job_stdout'] = check.stdout.evaluate()
-
-                if check.build_job:
-                    entry['build_stderr'] = check.build_stderr.evaluate()
-                    entry['build_stdout'] = check.build_stdout.evaluate()
-
-                if t.failed:
-                    num_failures += 1
-                elif t.aborted:
-                    num_aborted += 1
-                elif t.skipped:
-                    num_skipped += 1
-
-                if t.failed or t.aborted:
-                    entry['fail_phase'] = t.failed_stage
-                    if t.exc_info is not None:
-                        entry['fail_reason'] = errors.what(*t.exc_info)
-                        entry['fail_info'] = {
-                            'exc_type':  t.exc_info[0],
-                            'exc_value': t.exc_info[1],
-                            'traceback': t.exc_info[2]
-                        }
-                        entry['fail_severe'] = errors.is_severe(*t.exc_info)
-                elif t.succeeded:
-                    entry['outputdir'] = check.outputdir
-
-                # Add any loggable variables and parameters
-                test_cls = type(check)
-                for name, alt_name in test_cls.loggable_attrs():
-                    if alt_name == 'partition' or alt_name == 'environ':
-                        # We set those from the testcase
-                        continue
-
-                    key = alt_name if alt_name else name
-                    try:
-                        entry[key] = _getattr(check, name)
-                    except AttributeError:
-                        entry[key] = '<undefined>'
-
-                if entry['job_completion_time_unix']:
-                    entry['job_completion_time'] = _format_time_rfc3339(
-                        time.localtime(entry['job_completion_time_unix']),
-                        '%FT%T%:z'
-                    )
-
-                testcases.append(entry)
-
-            self._run_data.append({
-                'num_cases': len(run),
-                'num_failures': num_failures,
-                'num_aborted': num_aborted,
-                'num_skipped': num_skipped,
-                'runid': runid,
-                'testcases': testcases
-            })
-
-        return self._run_data
-
-    def print_failure_report(self, printer, rerun_info=True,
-                             global_stats=False):
-        def _head_n(filename, prefix, num_lines=10):
-            # filename and prefix are `None` before setup
-            if filename is None or prefix is None:
-                return []
-
-            try:
-                with open(os.path.join(prefix, filename)) as fp:
-                    lines = [
-                        f'--- {filename} (first {num_lines} lines) ---'
-                    ]
-                    for i, line in enumerate(fp):
-                        if i < num_lines:
-                            # Remove trailing '\n'
-                            lines.append(line.rstrip())
-
-                lines += [f'--- {filename} ---']
-            except OSError as e:
-                lines = [f'--- {filename} ({e}) ---']
-
-            return lines
-
-        def _print_failure_info(rec, runid, total_runs):
-            printer.info(line_width * '-')
-            printer.info(f"FAILURE INFO for {rec['name']} "
-                         f"(run: {runid}/{total_runs})")
-            printer.info(f"  * Description: {rec['descr']}")
-            printer.info(f"  * System partition: {rec['system']}")
-            printer.info(f"  * Environment: {rec['environ']}")
-            printer.info(f"  * Stage directory: {rec['stagedir']}")
-            printer.info(f"  * Node list: "
-                         f"{util.nodelist_abbrev(rec['job_nodelist'])}")
-            job_type = 'local' if rec['scheduler'] == 'local' else 'batch job'
-            printer.info(f"  * Job type: {job_type} (id={rec['jobid']})")
-            printer.info(f"  * Dependencies (conceptual): "
-                         f"{rec['dependencies_conceptual']}")
-            printer.info(f"  * Dependencies (actual): "
-                         f"{rec['dependencies_actual']}")
-            printer.info(f"  * Maintainers: {rec['maintainers']}")
-            printer.info(f"  * Failing phase: {rec['fail_phase']}")
-            if rerun_info and not rec['fixture']:
-                printer.info(f"  * Rerun with '-n /{rec['hashcode']}"
-                             f" -p {rec['environ']} --system "
-                             f"{rec['system']} -r'")
-
-            msg = rec['fail_reason']
-            if isinstance(rec['fail_info']['exc_value'], errors.SanityError):
-                lines = [msg]
-                lines += _head_n(rec['job_stdout'], prefix=rec['stagedir'])
-                lines += _head_n(rec['job_stderr'], prefix=rec['stagedir'])
-                msg = '\n'.join(lines)
-
-            printer.info(f"  * Reason: {msg}")
-
-            tb = ''.join(traceback.format_exception(
-                *rec['fail_info'].values()))
-            if rec['fail_severe']:
-                printer.info(tb)
-            else:
-                printer.verbose(tb)
-
-        line_width = shutil.get_terminal_size()[0]
-        printer.info(line_width * '=')
-        printer.info('SUMMARY OF FAILURES')
-
-        run_report = self.json()
-        for run_no, run_info in enumerate(run_report, start=1):
-            if not global_stats and run_no != len(run_report):
-                continue
-
-            for r in run_info['testcases']:
-                if r['result'] in {'pass', 'abort', 'skip'}:
-                    continue
-
-                _print_failure_info(r, run_no, len(run_report))
-
-        printer.info(line_width * '-')
-
-    def print_failure_stats(self, printer, global_stats=False):
-        if global_stats:
-            runid = None
-        else:
-            runid = rt.runtime().current_run
-
-        failures = {}
-        for tf in (t for t in self.tasks(runid) if t.failed):
-            check, partition, environ = tf.testcase
-            info = f'[{check.display_name}]'
-            if partition:
-                info += f' @{partition.fullname}'
-
-            if environ:
-                info += f'+{environ.name}'
-
-            if tf.failed_stage not in failures:
-                failures[tf.failed_stage] = []
-
-            failures[tf.failed_stage].append(info)
-
-        line_width = shutil.get_terminal_size()[0]
-        stats_start = line_width * '='
-        stats_title = 'FAILURE STATISTICS'
-        stats_end = line_width * '-'
-        stats_body = []
-        row_format = "{:<13} {:<5} {}"
-        stats_hline = row_format.format(13*'-', 5*'-', 60*'-')
-        stats_header = row_format.format('Phase', '#', 'Failing test cases')
-        num_tests = self.num_cases(runid)
-        num_failures = 0
-        for fl in failures.values():
-            num_failures += len(fl)
-
-        stats_body = ['']
-        stats_body.append(f'Total number of test cases: {num_tests}')
-        stats_body.append(f'Total number of failures: {num_failures}')
-        stats_body.append('')
-        stats_body.append(stats_header)
-        stats_body.append(stats_hline)
-        for p, l in failures.items():
-            stats_body.append(row_format.format(p, len(l), l[0]))
-            for f in l[1:]:
-                stats_body.append(row_format.format('', '', str(f)))
-
-        if stats_body:
-            for line in (stats_start, stats_title, *stats_body, stats_end):
-                printer.info(line)
-
-    def performance_report(self):
-        width = shutil.get_terminal_size()[0]
-        lines = ['', width*'=', 'PERFORMANCE REPORT', width*'-']
-
-        # Collect all the records from performance tests
-        perf_records = {}
-        for run in self.json():
-            for tc in run['testcases']:
-                if tc['perfvars']:
-                    key = tc['unique_name']
-                    perf_records.setdefault(key, [])
-                    perf_records[key].append(tc)
-
-        if not perf_records:
-            return ''
-
-        interesting_vars = {
-            'num_cpus_per_task',
-            'num_gpus_per_node',
-            'num_tasks',
-            'num_tasks_per_core',
-            'num_tasks_per_node',
-            'num_tasks_per_socket',
-            'use_multithreading'
-        }
-
-        for testcases in perf_records.values():
-            for tc in testcases:
-                name = tc['name']
-                hash = tc['hashcode']
-                env  = tc['environ']
-                system = tc['system']
-                part = tc['partition']
-                lines.append(f'[{name} /{hash} @{system}:{part}+{env}]')
-                for v in interesting_vars:
-                    val = tc['check_vars'][v]
-                    if val is not None:
-                        lines.append(f'  {v}: {val}')
-
-                lines.append('  performance:')
-                for v in tc['perfvars']:
-                    name = v['name']
-                    val  = v['value']
-                    ref  = v['reference']
-                    unit = v['unit']
-                    lthr = v['thres_lower']
-                    uthr = v['thres_upper']
-                    if lthr is not None:
-                        lthr *= 100
-                    else:
-                        lthr = '-inf'
-
-                    if uthr is not None:
-                        uthr *= 100
-                    else:
-                        uthr = 'inf'
-
-                    lines.append(f'    - {name}: {val} {unit} '
-                                 f'(r: {ref} {unit} l: {lthr}% u: +{uthr}%)')
-
-        lines.append(width*'-')
-        return '\n'.join(lines)
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 04fcaad50c..c1a44fd494 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -17,7 +17,7 @@
 import reframe.core.environments as env
 import reframe.core.logging as logging
 import reframe.core.runtime as rt
-import reframe.frontend.runreport as runreport
+import reframe.frontend.reporting as reporting
 import reframe.utility.osext as osext
 import unittests.utility as test_util
 from reframe import INSTALL_PREFIX
@@ -168,7 +168,7 @@ def test_check_restore_session_failed(run_reframe, tmp_path):
         checkpath=[],
         more_options=['--restore-session', '--failed']
     )
-    report = runreport.load_report(f'{tmp_path}/.reframe/reports/latest.json')
+    report = reporting.restore_session(f'{tmp_path}/.reframe/reports/latest.json')
     assert set(report.slice('name', when=('fail_phase', 'sanity'))) == {'T2'}
     assert set(report.slice('name',
                             when=('fail_phase', 'startup'))) == {'T7', 'T9'}
@@ -188,7 +188,7 @@ def test_check_restore_session_succeeded_test(run_reframe, tmp_path):
         checkpath=[],
         more_options=['--restore-session', '-n', 'T1']
     )
-    report = runreport.load_report(f'{tmp_path}/.reframe/reports/latest.json')
+    report = reporting.restore_session(f'{tmp_path}/.reframe/reports/latest.json')
     assert report['runs'][-1]['num_cases'] == 1
     assert report['runs'][-1]['testcases'][0]['name'] == 'T1'
 
@@ -926,7 +926,7 @@ def test_failure_stats(run_reframe, run_action):
     else:
         assert returncode != 0
         assert r'FAILURE STATISTICS' in stdout
-        assert r'sanity        1     [SanityFailureCheck' in stdout
+        assert r'sanity        1     SanityFailureCheck' in stdout
 
 
 def test_maxfail_option(run_reframe):
diff --git a/unittests/test_policies.py b/unittests/test_policies.py
index 5d8efda6d8..265f6aa52f 100644
--- a/unittests/test_policies.py
+++ b/unittests/test_policies.py
@@ -19,7 +19,7 @@
 import reframe.frontend.dependencies as dependencies
 import reframe.frontend.executors as executors
 import reframe.frontend.executors.policies as policies
-import reframe.frontend.runreport as runreport
+import reframe.frontend.reporting as reporting
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
 import reframe.utility.sanity as sn
@@ -33,6 +33,7 @@
                                      RunSessionTimeout,
                                      TaskDependencyError)
 from reframe.frontend.loader import RegressionCheckLoader
+from reframe.frontend.reporting import RunReport
 from unittests.resources.checks.hellocheck import HelloTest
 from unittests.resources.checks.frontend_checks import (
     BadSetupCheck,
@@ -214,31 +215,33 @@ def _validate_junit_report(report):
     schema.assert_(report)
 
 
-def _generate_runreport(run_stats, time_start, time_end):
-    return {
-        'session_info': {
-            'cmdline': ' '.join(sys.argv),
-            'config_files': rt.runtime().site_config.sources,
-            'data_version': runreport.DATA_VERSION,
-            'hostname': socket.gethostname(),
-            'num_cases': run_stats[0]['num_cases'],
-            'num_failures': run_stats[-1]['num_failures'],
-            'prefix_output': rt.runtime().output_prefix,
-            'prefix_stage': rt.runtime().stage_prefix,
-            'time_elapsed': time_end - time_start,
-            'time_end': time.strftime(
-                '%FT%T%z', time.localtime(time_end),
-            ),
-            'time_start': time.strftime(
-                '%FT%T%z', time.localtime(time_start),
-            ),
-            'user': osext.osuser(),
-            'version': osext.reframe_version(),
-            'workdir': os.getcwd()
-        },
-        'restored_cases': [],
-        'runs': run_stats
-    }
+def _generate_runreport(run_stats, time_start=None, time_end=None):
+    report = RunReport()
+    report.update_session_info({
+        'cmdline': ' '.join(sys.argv),
+        'config_files': rt.runtime().site_config.sources,
+        'data_version': reporting.DATA_VERSION,
+        'hostname': socket.gethostname(),
+        'prefix_output': rt.runtime().output_prefix,
+        'prefix_stage': rt.runtime().stage_prefix,
+        'user': osext.osuser(),
+        'version': osext.reframe_version(),
+        'workdir': os.getcwd()
+    })
+    if time_start and time_end:
+        time_elapsed = time_end - time_start
+        time_start = time.strftime(r'%FT%T%z', time.localtime(time_start))
+        time_end = time.strftime(r'%FT%T%z', time.localtime(time_end))
+        report.update_session_info({
+            'time_elapsed': time_elapsed,
+            'time_end': time_end,
+            'time_start': time_start
+        })
+
+    if run_stats:
+        report.update_run_stats(run_stats)
+
+    return report
 
 
 def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path):
@@ -254,28 +257,23 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path):
     assert 1 == num_failures_stage(runner, 'performance')
     assert 1 == num_failures_stage(runner, 'cleanup')
 
-    # Create a run report and validate it
-    report = _generate_runreport(runner.stats.json(), *tm.timestamps())
-
     # We dump the report first, in order to get any object conversions right
-    report_file = tmp_path / 'report.json'
-    with open(report_file, 'w') as fp:
-        jsonext.dump(report, fp)
+    report = _generate_runreport(runner.stats, *tm.timestamps())
+    report.save(tmp_path / 'report.json')
 
     # We explicitly set `time_total` to `None` in the last test case, in order
-    # to test the proper handling of `None`.`
+    # to test the proper handling of `None`.
     report['runs'][0]['testcases'][-1]['time_total'] = None
 
     # Validate the junit report
-    xml_report = runreport.junit_xml_report(report)
-    _validate_junit_report(xml_report)
+    _validate_junit_report(report.generate_xml_report())
 
-    # Read and validate the report using the runreport module
-    runreport.load_report(report_file)
+    # Read and validate the report using the `reporting` module
+    reporting.restore_session(tmp_path / 'report.json')
 
     # Try to load a non-existent report
     with pytest.raises(ReframeError, match='failed to load report file'):
-        runreport.load_report(tmp_path / 'does_not_exist.json')
+        reporting.restore_session(tmp_path / 'does_not_exist.json')
 
     # Generate an invalid JSON
     with open(tmp_path / 'invalid.json', 'w') as fp:
@@ -283,16 +281,14 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path):
         fp.write('invalid')
 
     with pytest.raises(ReframeError, match=r'is not a valid JSON file'):
-        runreport.load_report(tmp_path / 'invalid.json')
+        reporting.restore_session(tmp_path / 'invalid.json')
 
     # Generate a report that does not comply to the schema
     del report['session_info']['data_version']
-    with open(tmp_path / 'invalid-version.json', 'w') as fp:
-        jsonext.dump(report, fp)
-
+    report.save(tmp_path / 'invalid-version.json')
     with pytest.raises(ReframeError,
                        match=r'failed to validate report'):
-        runreport.load_report(tmp_path / 'invalid-version.json')
+        reporting.restore_session(tmp_path / 'invalid-version.json')
 
 
 def test_runall_skip_system_check(make_runner, make_cases, common_exec_ctx):
@@ -386,11 +382,11 @@ def test_runall_skip_tests(make_runner, make_cases,
     runner.runall(cases)
 
     def assert_reported_skipped(num_skipped):
-        report = runner.stats.json()
-        assert report[0]['num_skipped'] == num_skipped
+        run_report = _generate_runreport(runner.stats)['runs']
+        assert run_report[0]['num_skipped'] == num_skipped
 
         num_reported = 0
-        for tc in report[0]['testcases']:
+        for tc in run_report[0]['testcases']:
             if tc['result'] == 'skip':
                 num_reported += 1
 
@@ -456,9 +452,6 @@ def test_retries_bad_check(make_runner, make_cases, common_exec_ctx):
     assert runner.max_retries == rt.runtime().current_run
     assert 2 == len(runner.stats.failed())
 
-    # Ensure that the report does not raise any exception
-    runner.stats.retry_report()
-
 
 def test_retries_good_check(make_runner, make_cases, common_exec_ctx):
     runner = make_runner(max_retries=2)
@@ -911,11 +904,9 @@ def report_file(make_runner, dep_cases, common_exec_ctx, tmp_path):
     with timer() as tm:
         runner.runall(dep_cases)
 
-    report = _generate_runreport(runner.stats.json(), *tm.timestamps())
     filename = tmp_path / 'report.json'
-    with open(filename, 'w') as fp:
-        jsonext.dump(report, fp)
-
+    report = _generate_runreport(runner.stats, *tm.timestamps())
+    report.save(filename)
     return filename
 
 
@@ -928,7 +919,7 @@ def test_restore_session(report_file, make_runner,
     )
 
     # Restore the required test cases
-    report = runreport.load_report(report_file)
+    report = reporting.restore_session(report_file)
     testgraph, restored_cases = report.restore_dangling(testgraph)
 
     assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'}
@@ -938,29 +929,18 @@ def test_restore_session(report_file, make_runner,
     with timer() as tm:
         runner.runall(selected, restored_cases)
 
-    new_report = _generate_runreport(runner.stats.json(), *tm.timestamps())
+    new_report = _generate_runreport(runner.stats, *tm.timestamps())
     assert new_report['runs'][0]['num_cases'] == 1
     assert new_report['runs'][0]['testcases'][0]['name'] == 'T1'
 
     # Generate an empty report and load it as primary with the original report
     # as a fallback, in order to test if the dependencies are still resolved
     # correctly
-    empty_report = tmp_path / 'empty.json'
-
-    with open(empty_report, 'w') as fp:
-        empty_run = [
-            {
-                'num_cases': 0,
-                'num_failures': 0,
-                'num_aborted': 0,
-                'num_skipped': 0,
-                'runid': 0,
-                'testcases': []
-            }
-        ]
-        jsonext.dump(_generate_runreport(empty_run, *tm.timestamps()), fp)
-
-    report2 = runreport.load_report(empty_report, report_file)
+    empty_report = _generate_runreport(None, *tm.timestamps())
+    empty_report_file = tmp_path / 'empty.json'
+    empty_report.save(empty_report_file)
+
+    report2 = reporting.restore_session(empty_report_file, report_file)
     restored_cases = report2.restore_dangling(testgraph)[1]
     assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'}
 

From e73e7d69c87453db7223719819897eb804080b77 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 28 Jun 2024 17:04:28 +0200
Subject: [PATCH 019/128] Support for selecting sessions explicitly

---
 reframe/frontend/cli.py                 | 36 +++++-----
 reframe/frontend/reporting/__init__.py  | 27 +++++---
 reframe/frontend/reporting/analytics.py | 89 +++++++++++++++----------
 reframe/schemas/runreport.json          |  2 +
 unittests/test_policies.py              |  9 +--
 5 files changed, 92 insertions(+), 71 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index fcd1972717..f1f0fad523 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -233,6 +233,9 @@ def main():
     testgen_options = argparser.add_argument_group(
         'Options for generating tests dynamically'
     )
+    reporting_options = argparser.add_argument_group(
+        'Options related to results reporting'
+    )
     misc_options = argparser.add_argument_group('Miscellaneous options')
 
     # Output directory options
@@ -535,6 +538,17 @@ def main():
         help='Repeat selected tests N times'
     )
 
+    # Reporting options
+    reporting_options.add_argument(
+        '--performance-compare', metavar='CMPSPEC', action='store',
+        help='Compare past performance results'
+    )
+    reporting_options.add_argument(
+        '--performance-report', action='store', nargs='?',
+        const='19700101T0000Z:now/last:+job_nodelist/+result',
+        help='Print a report for performance tests'
+    )
+
     # Miscellaneous options
     misc_options.add_argument(
         '-C', '--config-file', action='append', metavar='FILE',
@@ -556,15 +570,6 @@ def main():
         help='Disable coloring of output',
         envvar='RFM_COLORIZE', configvar='general/colorize'
     )
-    misc_options.add_argument(
-        '--performance-compare', metavar='CMPSPEC', action='store',
-        help='Compare past performance results'
-    )
-    misc_options.add_argument(
-        '--performance-report', action='store', nargs='?',
-        const='19700101T0000Z:now/last:+job_nodelist/+result',
-        help='Print a report for performance tests'
-    )
     misc_options.add_argument(
         '--show-config', action='store', nargs='?', const='all',
         metavar='PARAM',
@@ -1430,18 +1435,11 @@ def module_unuse(*paths):
                         options.maxfail, options.reruns, options.duration)
         try:
             time_start = time.time()
-            session_info['time_start'] = time.strftime(
-                '%FT%T%z', time.localtime(time_start),
-            )
             runner.runall(testcases, restored_cases)
         finally:
             # Build final JSON report
             time_end = time.time()
-            report.update_session_info({
-                'time_end': time.strftime(r'%FT%T%z',
-                                          time.localtime(time_end)),
-                'time_elapsed': time_end - time_start
-            })
+            report.update_timestamps(time_start, time_end)
             report.update_run_stats(runner.stats)
             if options.restore_session is not None:
                 report.update_restored_cases(restored_cases, restored_session)
@@ -1508,10 +1506,12 @@ def module_unuse(*paths):
 
             # Store the generated report for analytics
             try:
-                analytics.store_report(report, report.filename)
+                sess_uuid = analytics.store_report(report, report.filename)
             except Exception as e:
                 printer.warning(f'failed to store results in the database: {e}')
                 raise
+            else:
+                printer.info(f'Current session stored with UUID: {sess_uuid}')
 
             # Generate the junit xml report for this session
             junit_report_file = rt.get_option('general/0/report_junit')
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 97f1203949..d3dd344328 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -13,7 +13,6 @@
 import re
 import socket
 import time
-from datetime import datetime
 
 import reframe as rfm
 import reframe.utility.jsonext as jsonext
@@ -208,19 +207,17 @@ class RunReport:
     '''
     def __init__(self):
         # Initialize the report with the required fields
-        now = datetime.now().strftime(r'%FT%T%z')
+        self.__filename = None
         self.__report = {
             'session_info': {
                 'data_version': DATA_VERSION,
-                'hostname': socket.gethostname(),
-                'time_start': now,
-                'time_end': now,
-                'time_elapsed': 0.0
+                'hostname': socket.gethostname()
             },
             'runs': [],
             'restored_cases': []
         }
-        self.__filename = None
+        now = time.time()
+        self.update_timestamps(now, now)
 
     @property
     def filename(self):
@@ -236,12 +233,26 @@ def __rfm_json_encode__(self):
         return self.__report
 
     def update_session_info(self, session_info):
-        self.__report['session_info'].update(session_info)
+        # Remove timestamps
+        for key, val in session_info.items():
+            if not key.startswith('time_'):
+                self.__report['session_info'][key] = val
 
     def update_restored_cases(self, restored_cases, restored_session):
         self.__report['restored_cases'] = [restored_session.case(*c)
                                            for c in restored_cases]
 
+    def update_timestamps(self, ts_start, ts_end):
+        fmt = r'%FT%T%z'
+        self.__report['session_info'].update({
+            'time_start': time.strftime(fmt, time.localtime(ts_start)),
+            'time_start_unix': ts_start,
+            'time_end': time.strftime(fmt, time.localtime(ts_end)),
+            'time_end_unix': ts_end,
+            'time_elapsed': ts_end - ts_start
+        })
+        print(self.__report['session_info'])
+
     def update_run_stats(self, stats):
         for runid, tasks in stats.runs():
             testcases = []
diff --git a/reframe/frontend/reporting/analytics.py b/reframe/frontend/reporting/analytics.py
index bdba66c21e..8cf3d77057 100644
--- a/reframe/frontend/reporting/analytics.py
+++ b/reframe/frontend/reporting/analytics.py
@@ -13,6 +13,7 @@
 from collections import namedtuple
 from collections.abc import Hashable
 from datetime import datetime, timedelta
+from numbers import Number
 
 import reframe.core.runtime as runtime
 import reframe.utility.jsonext as jsonext
@@ -43,6 +44,9 @@ def _db_create(filename):
         conn.execute(
 '''CREATE TABLE IF NOT EXISTS sessions(
         id INTEGER PRIMARY KEY,
+        session_uuid TEXT,
+        session_start_unix REAL,
+        session_end_unix REAL,
         json_blob TEXT,
         report_file TEXT
 )'''
@@ -63,12 +67,22 @@ def _db_create(filename):
 
 
 def _db_store_report(conn, report, report_file_path):
+    session_start_unix = report['session_info']['time_start_unix']
+    session_end_unix = report['session_info']['time_end_unix']
+    session_uuid = datetime.fromtimestamp(session_start_unix).strftime(
+        r'%Y%m%dT%H%M%S%z'
+    )
     for run_idx, run in enumerate(report['runs']):
         for test_idx, testcase in enumerate(run['testcases']):
             sys, part = testcase['system'], testcase['partition']
             cursor = conn.execute(
-'''INSERT INTO sessions VALUES(:session_id, :json_blob, :report_file)''',
+'''INSERT INTO sessions VALUES(:session_id, :session_uuid,
+                               :session_start_unix, :session_end_unix,
+                               :json_blob, :report_file)''',
                          {'session_id': None,
+                          'session_uuid': session_uuid,
+                          'session_start_unix': session_start_unix,
+                          'session_end_unix': session_end_unix,
                           'json_blob': jsonext.dumps(report),
                           'report_file': report_file_path})
             conn.execute(
@@ -89,10 +103,12 @@ def _db_store_report(conn, report, report_file_path):
                 }
             )
 
+    return session_uuid
+
 
 def store_report(report, report_file):
     with sqlite3.connect(_db_file()) as conn:
-        _db_store_report(conn, report, report_file)
+        return _db_store_report(conn, report, report_file)
 
 
 def _fetch_testcases_raw(condition):
@@ -113,10 +129,22 @@ def _fetch_testcases_raw(condition):
     return testcases
 
 
+def _fetch_session_time_period(condition):
+    with sqlite3.connect(_db_file()) as conn:
+        query = (f'SELECT session_start_unix, session_end_unix FROM sessions '
+                 f'WHERE {condition} LIMIT 1')
+        getlogger().debug(query)
+        results = conn.execute(query).fetchall()
+        if results:
+            return results[0]
+
+        return None, None
+
+
 def fetch_testcases_time_period(ts_start, ts_end):
     return _fetch_testcases_raw(
         f'(job_completion_time_unix >= {ts_start} AND '
-        f'job_completion_time_unix < {ts_end}) '
+        f'job_completion_time_unix <= {ts_end}) '
         'ORDER BY job_completion_time_unix'
     )
 
@@ -203,6 +231,7 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
 
     return (data, ['name', 'pvar', 'pval', 'punit', 'pdiff'] + extra_group_by + extra_cols)
 
+
 class _Aggregator:
     @classmethod
     def create(cls, name):
@@ -271,6 +300,9 @@ def __call__(self, iterable):
 
 
 def _parse_timestamp(s):
+    if isinstance(s, Number):
+        return s
+
     now = datetime.now()
     def _do_parse(s):
         if s == 'now':
@@ -309,10 +341,23 @@ def _do_parse(s):
     return ts.timestamp()
 
 def _parse_time_period(s):
-    try:
-        ts_start, ts_end = s.split(':')
-    except ValueError:
-        raise ValueError(f'invalid time period spec: {s}') from None
+    if s.startswith('^'):
+        # Retrieve the period of a full session
+        try:
+            session_uuid = s[1:]
+        except IndexError:
+            raise ValueError(f'invalid session uuid: {s}') from None
+        else:
+            ts_start, ts_end = _fetch_session_time_period(
+                f'session_uuid == "{session_uuid}"'
+            )
+            if not ts_start or not ts_end:
+                raise ValueError(f'no such session: {session_uuid}')
+    else:
+        try:
+            ts_start, ts_end = s.split(':')
+        except ValueError:
+            raise ValueError(f'invalid time period spec: {s}') from None
 
     return _parse_timestamp(ts_start), _parse_timestamp(ts_end)
 
@@ -354,33 +399,3 @@ def parse_cmp_spec(spec):
     extra_cols = _parse_extra_cols(cols)
     return _Match(period_base, period_target,
                   aggr_fn, extra_groups, extra_cols)
-
-
-def performance_report_data(run_stats, report_spec):
-    period, aggr, cols = report_spec.split('/')
-    ts_start, ts_end = [_parse_timestamp(ts) for ts in period.split(':')]
-    op, extra_groups = aggr.split(':')
-    aggr_fn = _Aggregator.create(op)
-    extra_groups = extra_groups.split('+')[1:]
-    extra_cols = cols.split('+')[1:]
-    testcases = run_stats[0]['testcases']
-    target_testcases = fetch_testcases_time_period(ts_start, ts_end)
-    return compare_testcase_data(testcases, target_testcases,
-                                 _Aggregator.create('first'),
-                                 aggr_fn, extra_groups, extra_cols)
-
-
-def performance_compare_data(spec):
-    period_base, period_target, aggr, cols = spec.split('/')
-    base_testcases = fetch_testcases_time_period(
-        *(_parse_timestamp(ts) for ts in period_base.split(':'))
-    )
-    target_testcases = fetch_testcases_time_period(
-        *(_parse_timestamp(ts) for ts in period_target.split(':'))
-    )
-    op, extra_groups = aggr.split(':')
-    aggr_fn = _Aggregator.create(op)
-    extra_groups = extra_groups.split('+')[1:]
-    extra_cols = cols.split('+')[1:]
-    return compare_testcase_data(base_testcases, target_testcases, aggr_fn,
-                                 aggr_fn, extra_groups, extra_cols)
diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json
index bf60343235..85a902e6ec 100644
--- a/reframe/schemas/runreport.json
+++ b/reframe/schemas/runreport.json
@@ -89,7 +89,9 @@
                 "prefix_stage": {"type": "string"},
                 "time_elapsed": {"type": "number"},
                 "time_end": {"type": "string"},
+                "time_end_unix": {"type": "number"},
                 "time_start": {"type": "string"},
+                "time_start_unix": {"type": "number"},
                 "user": {"type": "string"},
                 "version": {"type": "string"},
                 "workdir": {"type": "string"}
diff --git a/unittests/test_policies.py b/unittests/test_policies.py
index 265f6aa52f..ced716424b 100644
--- a/unittests/test_policies.py
+++ b/unittests/test_policies.py
@@ -229,14 +229,7 @@ def _generate_runreport(run_stats, time_start=None, time_end=None):
         'workdir': os.getcwd()
     })
     if time_start and time_end:
-        time_elapsed = time_end - time_start
-        time_start = time.strftime(r'%FT%T%z', time.localtime(time_start))
-        time_end = time.strftime(r'%FT%T%z', time.localtime(time_end))
-        report.update_session_info({
-            'time_elapsed': time_elapsed,
-            'time_end': time_end,
-            'time_start': time_start
-        })
+        report.update_timestamps(time_start, time_end)
 
     if run_stats:
         report.update_run_stats(run_stats)

From e6afa06626106055ec546f1587e5338f624410a7 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Sat, 29 Jun 2024 00:56:43 +0200
Subject: [PATCH 020/128] Fix unit tests

---
 reframe/frontend/reporting/__init__.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index d3dd344328..366587f2b3 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -251,7 +251,6 @@ def update_timestamps(self, ts_start, ts_end):
             'time_end_unix': ts_end,
             'time_elapsed': ts_end - ts_start
         })
-        print(self.__report['session_info'])
 
     def update_run_stats(self, stats):
         for runid, tasks in stats.runs():
@@ -427,11 +426,13 @@ def generate_xml_report(self):
                     }
                 )
                 if tc['result'] == 'fail':
+                    fail_phase = tc['fail_phase']
+                    fail_reason = tc['fail_reason']
                     testcase_msg = etree.SubElement(
                         testcase, 'failure', attrib={'type': 'failure',
-                                                    'message': tc['fail_phase']}
+                                                     'message': fail_phase}
                     )
-                    testcase_msg.text = f"{tc['fail_phase']}: {tc['fail_reason']}"
+                    testcase_msg.text = f"{tc['fail_phase']}: {fail_reason}"
 
             testsuite_stdout = etree.SubElement(xml_testsuite, 'system-out')
             testsuite_stdout.text = ''
@@ -448,6 +449,7 @@ def save_junit(self, filename):
                                method='xml', xml_declaration=True).decode()
             )
 
+
 def performance_compare(cmp, report=None):
     match = parse_cmp_spec(cmp)
     if match.period_base is None:

From 7a307885bcb4f709e36a91826f12bdb210b7a063 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Sat, 29 Jun 2024 01:47:31 +0200
Subject: [PATCH 021/128] Fix coding style issues

---
 reframe/core/logging.py                 |  4 +-
 reframe/core/pipeline.py                | 13 ++--
 reframe/core/schedulers/__init__.py     |  4 +-
 reframe/frontend/cli.py                 | 40 ++++++------
 reframe/frontend/executors/__init__.py  | 10 +--
 reframe/frontend/printer.py             | 12 ++--
 reframe/frontend/reporting/analytics.py | 82 ++++++++++++++-----------
 unittests/test_cli.py                   | 10 ++-
 unittests/test_policies.py              |  3 +-
 unittests/test_schedulers.py            | 82 ++++++++++++-------------
 10 files changed, 136 insertions(+), 124 deletions(-)

diff --git a/reframe/core/logging.py b/reframe/core/logging.py
index 223840f281..d91b067fbe 100644
--- a/reframe/core/logging.py
+++ b/reframe/core/logging.py
@@ -433,7 +433,7 @@ def _create_file_handler(site_config, config_prefix):
 
 def _create_filelog_handler(site_config, config_prefix):
     basedir = os.path.abspath(os.path.join(
-        site_config.get(f'systems/0/prefix'),
+        site_config.get('systems/0/prefix'),
         osext.expandvars(site_config.get(f'{config_prefix}/basedir'))
     ))
     prefix  = osext.expandvars(site_config.get(f'{config_prefix}/prefix'))
@@ -581,7 +581,7 @@ def _create_httpjson_handler(site_config, config_prefix):
 
 def _record_to_json(record, extras, ignore_keys):
     def _can_send(key):
-        return not key.startswith('_') and not key in ignore_keys
+        return not key.startswith('_') and key not in ignore_keys
 
     def _sanitize(s):
         return re.sub(r'\W', '_', s)
diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
index b532e7b36f..f928a406f5 100644
--- a/reframe/core/pipeline.py
+++ b/reframe/core/pipeline.py
@@ -46,7 +46,8 @@
 
 
 class _NoRuntime(ContainerPlatform):
-    '''Proxy container runtime for storing container platform info early enough.
+    '''Proxy container runtime for storing container platform info early
+    enough.
 
     This will be replaced by the framework with a concrete implementation
     based on the current partition info.
@@ -847,8 +848,8 @@ def pipeline_hooks(cls):
     #: .. deprecated:: 4.0.0
     #:    Please use :attr:`env_vars` instead.
     variables = deprecate(variable(alias=env_vars),
-                          f"the use of 'variables' is deprecated; "
-                          f"please use 'env_vars' instead")
+                          "the use of 'variables' is deprecated; "
+                          "please use 'env_vars' instead")
 
     #: Time limit for this test.
     #:
@@ -1704,7 +1705,7 @@ def _setup_build_job(self, **job_opts):
         )
 
     def _setup_run_job(self, **job_opts):
-        self._job = self._create_job(f'run', self.local, **job_opts)
+        self._job = self._create_job('run', self.local, **job_opts)
 
     def _setup_container_platform(self):
         try:
@@ -2216,7 +2217,7 @@ def check_performance(self):
 
         if perf_patterns is not None and self.perf_variables:
             raise ReframeSyntaxError(
-                f"you cannot mix 'perf_patterns' and 'perf_variables' syntax"
+                "you cannot mix 'perf_patterns' and 'perf_variables' syntax"
             )
 
         # Convert `perf_patterns` to `perf_variables`
@@ -2364,7 +2365,7 @@ def cleanup(self, remove_files=False):
         aliased = os.path.samefile(self._stagedir, self._outputdir)
         if aliased:
             self.logger.debug(
-                f'outputdir and stagedir are the same; copying skipped'
+                'outputdir and stagedir are the same; copying skipped'
             )
         else:
             self._copy_to_outputdir()
diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py
index eaeb9125c9..a8565a99bc 100644
--- a/reframe/core/schedulers/__init__.py
+++ b/reframe/core/schedulers/__init__.py
@@ -555,7 +555,7 @@ def prepare(self, commands, environs=None, prepare_cmds=None,
                 strict_flex=False, **gen_opts):
         environs = environs or []
         if self.num_tasks is not None and self.num_tasks <= 0:
-            getlogger().debug(f'[F] Flexible node allocation requested')
+            getlogger().debug('[F] Flexible node allocation requested')
             num_tasks_per_node = self.num_tasks_per_node or 1
             min_num_tasks = (-self.num_tasks if self.num_tasks else
                              num_tasks_per_node)
@@ -636,7 +636,7 @@ def finished(self):
         return done
 
     def __eq__(self, other):
-        return type(self) == type(other) and self.jobid == other.jobid
+        return type(self) is type(other) and self.jobid == other.jobid
 
     def __hash__(self):
         return hash(self.jobid)
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index f1f0fad523..268caf8dc9 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -9,7 +9,6 @@
 import os
 import random
 import shlex
-import socket
 import sys
 import time
 import traceback
@@ -90,7 +89,8 @@ def dep_lines(u, *, prefix, depth=0, lines=None, printed=None,
             else:
                 fmt_fixt_vars = ''
 
-            name_info = f'{u.check.display_name}{fmt_fixt_vars} /{u.check.hashcode}'
+            name_info = (f'{u.check.display_name}{fmt_fixt_vars} '
+                         f'/{u.check.hashcode}')
             tc_info = ''
             details = ''
             if concretized:
@@ -98,7 +98,8 @@ def dep_lines(u, *, prefix, depth=0, lines=None, printed=None,
 
             location = inspect.getfile(type(u.check))
             if detailed:
-                details = f' [variant: {u.check.variant_num}, file: {location!r}]'
+                details = (f' [variant: {u.check.variant_num}, '
+                           f'file: {location!r}]')
 
             lines.append(f'{prefix}^{name_info}{tc_info}{details}')
 
@@ -1016,12 +1017,12 @@ def print_infoline(param, value):
 
     # Print command line
     session_info = report['session_info']
-    printer.info(f"[ReFrame Setup]")
+    printer.info('[ReFrame Setup]')
     print_infoline('version', session_info['version'])
     print_infoline('command', repr(session_info['cmdline']))
     print_infoline(
-        f"launched by",
-        f"{session_info['user'] or '<unknown>'}@{session_info['hostname']}"
+        'launched by',
+        f'{session_info["user"] or "<unknown>"}@{session_info["hostname"]}'
     )
     print_infoline('working directory', repr(session_info['workdir']))
     print_infoline(
@@ -1237,7 +1238,9 @@ def _sort_testcases(testcases):
             printer.debug('Pruned test DAG')
             printer.debug(dependencies.format_deps(testgraph))
             if options.restore_session is not None:
-                testgraph, restored_cases = restored_session.restore_dangling(testgraph)
+                testgraph, restored_cases = restored_session.restore_dangling(
+                    testgraph
+                )
 
         testcases = dependencies.toposort(
             testgraph,
@@ -1320,7 +1323,7 @@ def _sort_testcases(testcases):
 
         # Load the environment for the current system
         try:
-            printer.debug(f'Loading environment for current system')
+            printer.debug('Loading environment for current system')
             runtime.loadenv(rt.system.preload_environ)
         except errors.EnvironError as e:
             printer.error("failed to load current system's environment; "
@@ -1332,14 +1335,14 @@ def module_use(*paths):
             try:
                 rt.modules_system.searchpath_add(*paths)
             except errors.EnvironError as e:
-                printer.warning(f'could not add module paths correctly')
+                printer.warning('could not add module paths correctly')
                 printer.debug(str(e))
 
         def module_unuse(*paths):
             try:
                 rt.modules_system.searchpath_remove(*paths)
             except errors.EnvironError as e:
-                printer.warning(f'could not remove module paths correctly')
+                printer.warning('could not remove module paths correctly')
                 printer.debug(str(e))
 
         printer.debug('(Un)using module paths from command line')
@@ -1417,18 +1420,19 @@ def module_unuse(*paths):
         exec_policy.sched_options = parsed_job_options
         if options.maxfail < 0:
             raise errors.CommandLineError(
-                f'--maxfail should be a non-negative integer: '
+                '--maxfail should be a non-negative integer: '
                 f'{options.maxfail}'
             )
 
         if options.reruns and options.duration:
             raise errors.CommandLineError(
-                f"'--reruns' option cannot be combined with '--duration'"
+                "'--reruns' option cannot be combined with '--duration'"
             )
 
         if options.reruns < 0:
             raise errors.CommandLineError(
-                f"'--reruns' should be a non-negative integer: {options.reruns}"
+                "'--reruns' should be a non-negative integer: "
+                f"{options.reruns}"
             )
 
         runner = Runner(exec_policy, printer, options.max_retries,
@@ -1494,7 +1498,7 @@ def module_unuse(*paths):
                     report_file,
                     compress=rt.get_option('general/0/compress_report'),
                     link_to_last=(default_loc == os.path.dirname(report_file))
-            )
+                )
             except OSError as e:
                 printer.warning(
                     f'failed to generate report in {report_file!r}: {e}'
@@ -1508,8 +1512,9 @@ def module_unuse(*paths):
             try:
                 sess_uuid = analytics.store_report(report, report.filename)
             except Exception as e:
-                printer.warning(f'failed to store results in the database: {e}')
-                raise
+                printer.warning(
+                    f'failed to store results in the database: {e}'
+                )
             else:
                 printer.info(f'Current session stored with UUID: {sess_uuid}')
 
@@ -1551,9 +1556,8 @@ def module_unuse(*paths):
     finally:
         try:
             logging.getprofiler().exit_region()     # region: 'test processing'
-            log_files = logging.log_files()
             if site_config.get('general/0/save_log_files'):
-                log_files = logging.save_log_files(rt.output_prefix)
+                logging.save_log_files(rt.output_prefix)
         except OSError as e:
             printer.error(f'could not save log file: {e}')
             sys.exit(1)
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index 90b81c74c2..8bf3f58117 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -163,12 +163,10 @@ def generate_testcases(checks, prepare=False):
     '''Generate concrete test cases from checks.
 
     If `prepare` is true then each of the cases will also be prepared for
-    being sent to the test pipeline. Note that setting this to true may slow down
-    the test case generation.
-
+    being sent to the test pipeline. Note that setting this to true may slow
+    down the test case generation.
     '''
 
-    rt = runtime.runtime()
     cases = []
     for c in checks:
         valid_comb = runtime.valid_sysenv_comb(c.valid_systems,
@@ -410,7 +408,6 @@ def temp_dry_run(check):
             with temp_dry_run(self.check):
                 return fn(*args, **kwargs)
 
-
     @logging.time_function
     def setup(self, *args, **kwargs):
         self.testcase.prepare()
@@ -647,9 +644,6 @@ def runall(self, testcases, restored_cases=None):
             runid = None if self._global_stats else -1
             num_aborted = len(self._stats.aborted(runid))
             num_failures = len(self._stats.failed(runid))
-            num_completed = len(self._stats.completed(runid))
-            num_skipped = len(self._stats.skipped(runid))
-            num_tasks = self._stats.num_cases(runid)
             if num_failures > 0:
                 status = 'FAILED'
             elif num_aborted > 0:
diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index b6530f05bb..d94444fade 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -121,25 +121,25 @@ def _head_n(filename, prefix, num_lines=10):
         def _print_failure_info(rec, runid, total_runs):
             self.info(line_width * '-')
             self.info(f"FAILURE INFO for {rec['name']} "
-                         f"(run: {runid}/{total_runs})")
+                      f"(run: {runid}/{total_runs})")
             self.info(f"  * Description: {rec['descr']}")
             self.info(f"  * System partition: {rec['system']}")
             self.info(f"  * Environment: {rec['environ']}")
             self.info(f"  * Stage directory: {rec['stagedir']}")
             self.info(f"  * Node list: "
-                         f"{nodelist_abbrev(rec['job_nodelist'])}")
+                      f"{nodelist_abbrev(rec['job_nodelist'])}")
             job_type = 'local' if rec['scheduler'] == 'local' else 'batch job'
             self.info(f"  * Job type: {job_type} (id={rec['jobid']})")
             self.info(f"  * Dependencies (conceptual): "
-                         f"{rec['dependencies_conceptual']}")
+                      f"{rec['dependencies_conceptual']}")
             self.info(f"  * Dependencies (actual): "
-                         f"{rec['dependencies_actual']}")
+                      f"{rec['dependencies_actual']}")
             self.info(f"  * Maintainers: {rec['maintainers']}")
             self.info(f"  * Failing phase: {rec['fail_phase']}")
             if rerun_info and not rec['fixture']:
                 self.info(f"  * Rerun with '-n /{rec['hashcode']}"
-                             f" -p {rec['environ']} --system "
-                             f"{rec['system']} -r'")
+                          f" -p {rec['environ']} --system "
+                          f"{rec['system']} -r'")
 
             msg = rec['fail_reason']
             if isinstance(rec['fail_info']['exc_value'], SanityError):
diff --git a/reframe/frontend/reporting/analytics.py b/reframe/frontend/reporting/analytics.py
index 8cf3d77057..d8d79da25d 100644
--- a/reframe/frontend/reporting/analytics.py
+++ b/reframe/frontend/reporting/analytics.py
@@ -41,29 +41,24 @@ def _db_file():
 
 def _db_create(filename):
     with sqlite3.connect(filename) as conn:
-        conn.execute(
-'''CREATE TABLE IF NOT EXISTS sessions(
-        id INTEGER PRIMARY KEY,
-        session_uuid TEXT,
-        session_start_unix REAL,
-        session_end_unix REAL,
-        json_blob TEXT,
-        report_file TEXT
-)'''
-        )
-        conn.execute(
-'''CREATE TABLE IF NOT EXISTS testcases(
-        name TEXT,
-        system TEXT,
-        partition TEXT,
-        environ TEXT,
-        job_completion_time_unix REAL,
-        session_id INTEGER,
-        run_index INTEGER,
-        test_index INTEGER,
-        FOREIGN KEY(session_id) REFERENCES sessions(session_id)
-)'''
-        )
+        conn.execute('CREATE TABLE IF NOT EXISTS sessions('
+                     'id INTEGER PRIMARY KEY, '
+                     'session_uuid TEXT, '
+                     'session_start_unix REAL, '
+                     'session_end_unix REAL, '
+                     'json_blob TEXT, '
+                     'report_file TEXT)')
+        conn.execute('CREATE TABLE IF NOT EXISTS testcases('
+                     'name TEXT,'
+                     'system TEXT,'
+                     'partition TEXT,'
+                     'environ TEXT,'
+                     'job_completion_time_unix REAL,'
+                     'session_id INTEGER,'
+                     'run_index INTEGER,'
+                     'test_index INTEGER,'
+                     'FOREIGN KEY(session_id) '
+                     'REFERENCES sessions(session_id))')
 
 
 def _db_store_report(conn, report, report_file_path):
@@ -76,19 +71,24 @@ def _db_store_report(conn, report, report_file_path):
         for test_idx, testcase in enumerate(run['testcases']):
             sys, part = testcase['system'], testcase['partition']
             cursor = conn.execute(
-'''INSERT INTO sessions VALUES(:session_id, :session_uuid,
-                               :session_start_unix, :session_end_unix,
-                               :json_blob, :report_file)''',
-                         {'session_id': None,
-                          'session_uuid': session_uuid,
-                          'session_start_unix': session_start_unix,
-                          'session_end_unix': session_end_unix,
-                          'json_blob': jsonext.dumps(report),
-                          'report_file': report_file_path})
+                'INSERT INTO sessions VALUES('
+                ':session_id, :session_uuid, '
+                ':session_start_unix, :session_end_unix, '
+                ':json_blob, :report_file)',
+                {
+                    'session_id': None,
+                    'session_uuid': session_uuid,
+                    'session_start_unix': session_start_unix,
+                    'session_end_unix': session_end_unix,
+                    'json_blob': jsonext.dumps(report),
+                    'report_file': report_file_path
+                }
+            )
             conn.execute(
-'''INSERT INTO testcases VALUES(:name, :system, :partition, :environ,
-                                :job_completion_time_unix,
-                                :session_id, :run_index, :test_index)''',
+                'INSERT INTO testcases VALUES('
+                ':name, :system, :partition, :environ, '
+                ':job_completion_time_unix, '
+                ':session_id, :run_index, :test_index)',
                 {
                     'name': testcase['name'],
                     'system': sys,
@@ -186,6 +186,7 @@ def _group_testcases(testcases, group_by, extra_cols):
 
     return grouped
 
+
 def _aggregate_perf(grouped_testcases, aggr_fn, cols):
     other_aggr = _JoinUniqueValues('|')
     aggr_data = {}
@@ -200,6 +201,7 @@ def _aggregate_perf(grouped_testcases, aggr_fn, cols):
 
     return aggr_data
 
+
 def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
                           extra_group_by=None, extra_cols=None):
     extra_group_by = extra_group_by or []
@@ -229,7 +231,8 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
         line += [aggr_data[c] for c in extra_cols]
         data.append(line)
 
-    return (data, ['name', 'pvar', 'pval', 'punit', 'pdiff'] + extra_group_by + extra_cols)
+    return (data, (['name', 'pvar', 'pval', 'punit', 'pdiff'] +
+                   extra_group_by + extra_cols))
 
 
 class _Aggregator:
@@ -254,12 +257,14 @@ def create(cls, name):
     def __call__(self, iterable):
         pass
 
+
 class _First(_Aggregator):
     def __call__(self, iterable):
         for i, elem in enumerate(iterable):
             if i == 0:
                 return elem
 
+
 class _Last(_Aggregator):
     def __call__(self, iterable):
         if not isinstance(iterable, types.GeneratorType):
@@ -290,6 +295,7 @@ class _Max(_Aggregator):
     def __call__(self, iterable):
         return max(iterable)
 
+
 class _JoinUniqueValues(_Aggregator):
     def __init__(self, delim):
         self.__delim = delim
@@ -304,6 +310,7 @@ def _parse_timestamp(s):
         return s
 
     now = datetime.now()
+
     def _do_parse(s):
         if s == 'now':
             return now
@@ -340,6 +347,7 @@ def _do_parse(s):
 
     return ts.timestamp()
 
+
 def _parse_time_period(s):
     if s.startswith('^'):
         # Retrieve the period of a full session
@@ -361,6 +369,7 @@ def _parse_time_period(s):
 
     return _parse_timestamp(ts_start), _parse_timestamp(ts_end)
 
+
 def _parse_extra_cols(s):
     try:
         extra_cols = s.split('+')[1:]
@@ -382,6 +391,7 @@ def _parse_aggregation(s):
 _Match = namedtuple('_Match', ['period_base', 'period_target',
                                'aggregator', 'extra_groups', 'extra_cols'])
 
+
 def parse_cmp_spec(spec):
     parts = spec.split('/')
     if len(parts) == 3:
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index c1a44fd494..0f9f10880d 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -168,7 +168,9 @@ def test_check_restore_session_failed(run_reframe, tmp_path):
         checkpath=[],
         more_options=['--restore-session', '--failed']
     )
-    report = reporting.restore_session(f'{tmp_path}/.reframe/reports/latest.json')
+    report = reporting.restore_session(
+        f'{tmp_path}/.reframe/reports/latest.json'
+    )
     assert set(report.slice('name', when=('fail_phase', 'sanity'))) == {'T2'}
     assert set(report.slice('name',
                             when=('fail_phase', 'startup'))) == {'T7', 'T9'}
@@ -188,7 +190,9 @@ def test_check_restore_session_succeeded_test(run_reframe, tmp_path):
         checkpath=[],
         more_options=['--restore-session', '-n', 'T1']
     )
-    report = reporting.restore_session(f'{tmp_path}/.reframe/reports/latest.json')
+    report = reporting.restore_session(
+        f'{tmp_path}/.reframe/reports/latest.json'
+    )
     assert report['runs'][-1]['num_cases'] == 1
     assert report['runs'][-1]['testcases'][0]['name'] == 'T1'
 
@@ -201,7 +205,7 @@ def test_check_restore_session_check_search_path(run_reframe, tmp_path):
         checkpath=['unittests/resources/checks_unlisted/deps_complex.py']
     )
     returncode, stdout, _ = run_reframe(
-        checkpath=[f'foo/'],
+        checkpath=['foo/'],
         more_options=['--restore-session', '-n', 'T1', '-R'],
         action='list'
     )
diff --git a/unittests/test_policies.py b/unittests/test_policies.py
index ced716424b..66ed64389b 100644
--- a/unittests/test_policies.py
+++ b/unittests/test_policies.py
@@ -1322,7 +1322,8 @@ def test_perf_logging_lazy(make_runner, make_exec_ctx, lazy_perf_test,
     testcases = executors.generate_testcases([lazy_perf_test])
     _assert_no_logging_error(runner.runall, testcases)
 
-    logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_LazyPerfTest.log'
+    logfile = (tmp_path / 'perflogs' / 'generic' / 'default' /
+               '_LazyPerfTest.log')
     assert os.path.exists(logfile)
 
 
diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py
index cd1c3413d0..b2da80bcfb 100644
--- a/unittests/test_schedulers.py
+++ b/unittests/test_schedulers.py
@@ -158,23 +158,23 @@ def assert_job_script_sanity(job):
 
 def _expected_lsf_directives(job):
     return set([
-        f'#BSUB -J testjob',
+        '#BSUB -J testjob',
         f'#BSUB -o {job.stdout}',
         f'#BSUB -e {job.stderr}',
         f'#BSUB -nnodes {job.num_tasks // job.num_tasks_per_node}',
         f'#BSUB -W {int(job.time_limit // 60)}',
         f'#BSUB -R "affinity[core({job.num_cpus_per_task})]"',
-        f'#BSUB -x',
-        f'#BSUB --account=spam',
-        f'#BSUB --gres=gpu:4',
-        f'#DW jobdw capacity=100GB',
-        f'#DW stage_in source=/foo',
+        '#BSUB -x',
+        '#BSUB --account=spam',
+        '#BSUB --gres=gpu:4',
+        '#DW jobdw capacity=100GB',
+        '#DW stage_in source=/foo',
     ])
 
 
 def _expected_lsf_directives_minimal(job):
     return set([
-        f'#BSUB -J testjob',
+        '#BSUB -J testjob',
         f'#BSUB -o {job.stdout}',
         f'#BSUB -e {job.stderr}',
         f'#BSUB -n {job.num_tasks}'
@@ -183,7 +183,7 @@ def _expected_lsf_directives_minimal(job):
 
 def _expected_lsf_directives_no_tasks(job):
     return set([
-        f'#BSUB -J testjob',
+        '#BSUB -J testjob',
         f'#BSUB -o {job.stdout}',
         f'#BSUB -e {job.stderr}'
     ])
@@ -202,24 +202,22 @@ def _expected_flux_directives_no_tasks(job):
 
 
 def _expected_sge_directives(job):
-    num_nodes = job.num_tasks // job.num_tasks_per_node
-    num_cpus_per_node = job.num_cpus_per_task * job.num_tasks_per_node
     return set([
-        f'#$ -N "testjob"',
-        f'#$ -l h_rt=0:5:0',
+        '#$ -N "testjob"',
+        '#$ -l h_rt=0:5:0',
         f'#$ -o {job.stdout}',
         f'#$ -e {job.stderr}',
         f'#$ -wd {job.workdir}',
-        f'#$ --gres=gpu:4',
-        f'#$ --account=spam',
-        f'#DW jobdw capacity=100GB',
-        f'#DW stage_in source=/foo'
+        '#$ --gres=gpu:4',
+        '#$ --account=spam',
+        '#DW jobdw capacity=100GB',
+        '#DW stage_in source=/foo'
     ])
 
 
 def _expected_sge_directives_minimal(job):
     return set([
-        f'#$ -N "testjob"',
+        '#$ -N "testjob"',
         f'#$ -o {job.stdout}',
         f'#$ -e {job.stderr}',
         f'#$ -wd {job.workdir}'
@@ -276,24 +274,24 @@ def _expected_pbs_directives(job):
     num_nodes = job.num_tasks // job.num_tasks_per_node
     num_cpus_per_node = job.num_cpus_per_task * job.num_tasks_per_node
     return set([
-        f'#PBS -N testjob',
-        f'#PBS -l walltime=0:5:0',
+        '#PBS -N testjob',
+        '#PBS -l walltime=0:5:0',
         f'#PBS -o {job.stdout}',
         f'#PBS -e {job.stderr}',
         f'#PBS -l select={num_nodes}:mpiprocs={job.num_tasks_per_node}:ncpus={num_cpus_per_node}:mem=100GB:cpu_type=haswell',    # noqa: E501
-        f'#PBS --account=spam',
-        f'#PBS --gres=gpu:4',
-        f'#DW jobdw capacity=100GB',
-        f'#DW stage_in source=/foo'
+        '#PBS --account=spam',
+        '#PBS --gres=gpu:4',
+        '#DW jobdw capacity=100GB',
+        '#DW stage_in source=/foo'
     ])
 
 
 def _expected_pbs_directives_minimal(job):
     return set([
-        f'#PBS -N testjob',
+        '#PBS -N testjob',
         f'#PBS -o {job.stdout}',
         f'#PBS -e {job.stderr}',
-        f'#PBS -l select=1:mpiprocs=1:ncpus=1'
+        '#PBS -l select=1:mpiprocs=1:ncpus=1'
     ])
 
 
@@ -304,25 +302,25 @@ def _expected_torque_directives(job):
     num_nodes = job.num_tasks // job.num_tasks_per_node
     num_cpus_per_node = job.num_cpus_per_task * job.num_tasks_per_node
     return set([
-        f'#PBS -N testjob',
-        f'#PBS -l walltime=0:5:0',
+        '#PBS -N testjob',
+        '#PBS -l walltime=0:5:0',
         f'#PBS -o {job.stdout}',
         f'#PBS -e {job.stderr}',
         f'#PBS -l nodes={num_nodes}:ppn={num_cpus_per_node}:haswell',
-        f'#PBS -l mem=100GB',
-        f'#PBS --account=spam',
-        f'#PBS --gres=gpu:4',
-        f'#DW jobdw capacity=100GB',
-        f'#DW stage_in source=/foo'
+        '#PBS -l mem=100GB',
+        '#PBS --account=spam',
+        '#PBS --gres=gpu:4',
+        '#DW jobdw capacity=100GB',
+        '#DW stage_in source=/foo'
     ])
 
 
 def _expected_torque_directives_minimal(job):
     return set([
-        f'#PBS -N testjob',
+        '#PBS -N testjob',
         f'#PBS -o {job.stdout}',
         f'#PBS -e {job.stderr}',
-        f'#PBS -l nodes=1:ppn=1'
+        '#PBS -l nodes=1:ppn=1'
     ])
 
 
@@ -333,23 +331,23 @@ def _expected_oar_directives(job):
     num_nodes = job.num_tasks // job.num_tasks_per_node
     num_tasks_per_node = job.num_tasks_per_node
     return set([
-        f'#OAR -n "testjob"',
+        '#OAR -n "testjob"',
         f'#OAR -O {job.stdout}',
         f'#OAR -E {job.stderr}',
         f'#OAR -l /host={num_nodes}/core={num_tasks_per_node},walltime=0:5:0',
-        f'#OAR --account=spam',
-        f'#OAR --gres=gpu:4',
-        f'#DW jobdw capacity=100GB',
-        f'#DW stage_in source=/foo'
+        '#OAR --account=spam',
+        '#OAR --gres=gpu:4',
+        '#DW jobdw capacity=100GB',
+        '#DW stage_in source=/foo'
     ])
 
 
 def _expected_oar_directives_minimal(job):
     return set([
-        f'#OAR -n "testjob"',
+        '#OAR -n "testjob"',
         f'#OAR -O {job.stdout}',
         f'#OAR -E {job.stderr}',
-        f'#OAR -l /host=1/core=1'
+        '#OAR -l /host=1/core=1'
     ])
 
 
@@ -652,7 +650,7 @@ def test_sched_access_in_submit(make_job):
     job.scheduler._sched_access_in_submit = True
 
     if job.scheduler.registered_name in ('flux', 'local', 'ssh'):
-        pytest.skip(f'not relevant for this scheduler backend')
+        pytest.skip('not relevant for this scheduler backend')
 
     prepare_job(job)
     with open(job.script_filename) as fp:

From fe36ed5a44cc894a024b8422156a7eaab0934740 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 1 Jul 2024 10:20:12 +0200
Subject: [PATCH 022/128] Fix `tabulate` requirement for Python 3.6

---
 requirements.txt | 3 ++-
 setup.cfg        | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index e01146fe10..e6df345eb9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -22,6 +22,7 @@ semver==3.0.2; python_version >= '3.7'
 setuptools==59.6.0; python_version == '3.6'
 setuptools==68.0.0; python_version == '3.7'
 setuptools==72.1.0; python_version >= '3.8'
-tabulate==0.9.0
+tabulate==0.8.10; python_version == '3.6'
+tabulate==0.9.0; python_version >= '3.7'
 wcwidth==0.2.13
 #+pygelf%pygelf==0.4.0
diff --git a/setup.cfg b/setup.cfg
index cbbf1b17e4..5ae3959849 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -37,6 +37,8 @@ install_requires =
     requests <= 2.27.1; python_version == '3.6'
     semver
     semver <= 2.13.0; python_version == '3.6'
+    tabulate
+    tabulate <= 0.8.10; python_version == '3.6'
 
 [options.packages.find]
 include = reframe,reframe.*,hpctestlib.*

From f56d1ed3103c9f67ac91735bdff6434199ee7901 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 1 Jul 2024 11:48:08 +0200
Subject: [PATCH 023/128] Add an API around results storage and merge
 `analytics` with `reporting`

---
 reframe/frontend/cli.py                 |   3 +-
 reframe/frontend/reporting/__init__.py  | 292 ++++++++++++++++-
 reframe/frontend/reporting/analytics.py | 411 ------------------------
 3 files changed, 287 insertions(+), 419 deletions(-)
 delete mode 100644 reframe/frontend/reporting/analytics.py

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 268caf8dc9..a7be337061 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -24,7 +24,6 @@
 import reframe.frontend.dependencies as dependencies
 import reframe.frontend.filters as filters
 import reframe.frontend.reporting as reporting
-import reframe.frontend.reporting.analytics as analytics
 import reframe.utility as util
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
@@ -1510,7 +1509,7 @@ def module_unuse(*paths):
 
             # Store the generated report for analytics
             try:
-                sess_uuid = analytics.store_report(report, report.filename)
+                sess_uuid = report.store()
             except Exception as e:
                 printer.warning(
                     f'failed to store results in the database: {e}'
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 366587f2b3..6a96d73afb 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -3,16 +3,24 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
+import abc
 import decimal
 import functools
 import inspect
 import json
 import jsonschema
 import lxml.etree as etree
+import math
 import os
 import re
 import socket
+import statistics
 import time
+import types
+from collections import namedtuple
+from collections.abc import Hashable
+from datetime import datetime, timedelta
+from numbers import Number
 
 import reframe as rfm
 import reframe.utility.jsonext as jsonext
@@ -20,10 +28,8 @@
 from reframe.core.exceptions import ReframeError, what, is_severe
 from reframe.core.logging import getlogger, _format_time_rfc3339
 from reframe.core.warnings import suppress_deprecations
-from .analytics import (parse_cmp_spec,
-                        fetch_testcases_time_period,
-                        compare_testcase_data)
-
+from reframe.utility import nodelist_abbrev
+from .storage import StorageBackend
 
 # The schema data version
 # Major version bumps are expected to break the validation of previous schemas
@@ -41,6 +47,9 @@ def format_testcase(json, name='unique_name'):
     return f'{name}@{system}:{partition}+{environ}'
 
 
+_ReportStorage = StorageBackend.create('sqlite')
+
+
 class _RestoredSessionInfo:
     '''A restored session with some additional functionality.'''
 
@@ -388,6 +397,11 @@ def save(self, filename, compress=False, link_to_last=True):
                 else:
                     raise ReframeError('path exists and is not a symlink')
 
+    def store(self):
+        '''Store the report in the results storage.'''
+
+        return _ReportStorage.store(self, self.filename)
+
     def generate_xml_report(self):
         '''Generate a JUnit report from a standard ReFrame JSON report.'''
 
@@ -450,6 +464,268 @@ def save_junit(self, filename):
             )
 
 
+def _group_key(groups, testcase):
+    key = []
+    for grp in groups:
+        val = testcase[grp]
+        if grp == 'job_nodelist':
+            # Fold nodelist before adding as a key element
+            key.append(nodelist_abbrev(val))
+        elif not isinstance(val, Hashable):
+            key.append(str(val))
+        else:
+            key.append(val)
+
+    return tuple(key)
+
+
+def _group_testcases(testcases, group_by, extra_cols):
+    grouped = {}
+    for tc in testcases:
+        for pvar, reftuple in tc['perfvalues'].items():
+            pvar = pvar.split(':')[-1]
+            pval, pref, plower, pupper, punit = reftuple
+            plower = pref * (1 + plower) if plower is not None else -math.inf
+            pupper = pref * (1 + pupper) if pupper is not None else math.inf
+            record = {
+                'pvar': pvar,
+                'pval': pval,
+                'plower': plower,
+                'pupper': pupper,
+                'punit': punit,
+                **{k: tc[k] for k in group_by + extra_cols if k in tc}
+            }
+            key = _group_key(group_by, record)
+            grouped.setdefault(key, [])
+            grouped[key].append(record)
+
+    return grouped
+
+
+def _aggregate_perf(grouped_testcases, aggr_fn, cols):
+    other_aggr = _JoinUniqueValues('|')
+    aggr_data = {}
+    for key, seq in grouped_testcases.items():
+        aggr_data.setdefault(key, {})
+        aggr_data[key]['pval'] = aggr_fn(tc['pval'] for tc in seq)
+        for c in cols:
+            aggr_data[key][c] = other_aggr(
+                nodelist_abbrev(tc[c]) if c == 'job_nodelist' else tc[c]
+                for tc in seq
+            )
+
+    return aggr_data
+
+
+def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
+                          extra_group_by=None, extra_cols=None):
+    extra_group_by = extra_group_by or []
+    extra_cols = extra_cols or []
+    group_by = ['name', 'pvar', 'punit'] + extra_group_by
+
+    grouped_base = _group_testcases(base_testcases, group_by, extra_cols)
+    grouped_target = _group_testcases(target_testcases, group_by, extra_cols)
+    pbase = _aggregate_perf(grouped_base, base_fn, extra_cols)
+    ptarget = _aggregate_perf(grouped_target, target_fn, [])
+
+    # Build the final table data
+    data = []
+    for key, aggr_data in pbase.items():
+        pval = aggr_data['pval']
+        try:
+            target_pval = ptarget[key]['pval']
+        except KeyError:
+            pdiff = 'n/a'
+        else:
+            pdiff = (pval - target_pval) / target_pval
+            pdiff = '{:+7.2%}'.format(pdiff)
+
+        name, pvar, punit, *extras = key
+        line = [name, pvar, pval, punit, pdiff, *extras]
+        # Add the extra columns
+        line += [aggr_data[c] for c in extra_cols]
+        data.append(line)
+
+    return (data, (['name', 'pvar', 'pval', 'punit', 'pdiff'] +
+                   extra_group_by + extra_cols))
+
+
+class _Aggregator:
+    @classmethod
+    def create(cls, name):
+        if name == 'first':
+            return _First()
+        elif name == 'last':
+            return _Last()
+        elif name == 'mean':
+            return _Mean()
+        elif name == 'median':
+            return _Median()
+        elif name == 'min':
+            return _Min()
+        elif name == 'max':
+            return _Max()
+        else:
+            raise ValueError(f'unknown aggregation function: {name!r}')
+
+    @abc.abstractmethod
+    def __call__(self, iterable):
+        pass
+
+
+class _First(_Aggregator):
+    def __call__(self, iterable):
+        for i, elem in enumerate(iterable):
+            if i == 0:
+                return elem
+
+
+class _Last(_Aggregator):
+    def __call__(self, iterable):
+        if not isinstance(iterable, types.GeneratorType):
+            return iterable[-1]
+
+        for elem in iterable:
+            pass
+
+        return elem
+
+
+class _Mean(_Aggregator):
+    def __call__(self, iterable):
+        return statistics.mean(iterable)
+
+
+class _Median(_Aggregator):
+    def __call__(self, iterable):
+        return statistics.median(iterable)
+
+
+class _Min(_Aggregator):
+    def __call__(self, iterable):
+        return min(iterable)
+
+
+class _Max(_Aggregator):
+    def __call__(self, iterable):
+        return max(iterable)
+
+
+class _JoinUniqueValues(_Aggregator):
+    def __init__(self, delim):
+        self.__delim = delim
+
+    def __call__(self, iterable):
+        unique_vals = {str(elem) for elem in iterable}
+        return self.__delim.join(unique_vals)
+
+
+def _parse_timestamp(s):
+    if isinstance(s, Number):
+        return s
+
+    now = datetime.now()
+
+    def _do_parse(s):
+        if s == 'now':
+            return now
+
+        formats = [r'%Y%m%d', r'%Y%m%dT%H%M',
+                   r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z']
+        for fmt in formats:
+            try:
+                return datetime.strptime(s, fmt)
+            except ValueError:
+                continue
+
+        raise ValueError(f'invalid timestamp: {s}')
+
+    try:
+        ts = _do_parse(s)
+    except ValueError as err:
+        # Try the relative timestamps
+        match = re.match(r'(?P<ts>.*)(?P<amount>[\+|-]\d+)(?P<unit>[hdms])', s)
+        if not match:
+            raise err
+
+        ts = _do_parse(match.group('ts'))
+        amount = int(match.group('amount'))
+        unit = match.group('unit')
+        if unit == 'd':
+            ts += timedelta(days=amount)
+        elif unit == 'm':
+            ts += timedelta(minutes=amount)
+        elif unit == 'h':
+            ts += timedelta(hours=amount)
+        elif unit == 's':
+            ts += timedelta(seconds=amount)
+
+    return ts.timestamp()
+
+
+def _parse_time_period(s):
+    if s.startswith('^'):
+        # Retrieve the period of a full session
+        try:
+            session_uuid = s[1:]
+        except IndexError:
+            raise ValueError(f'invalid session uuid: {s}') from None
+        else:
+            ts_start, ts_end = _ReportStorage.fetch_session_time_period(
+                f'session_uuid == "{session_uuid}"'
+            )
+            if not ts_start or not ts_end:
+                raise ValueError(f'no such session: {session_uuid}')
+    else:
+        try:
+            ts_start, ts_end = s.split(':')
+        except ValueError:
+            raise ValueError(f'invalid time period spec: {s}') from None
+
+    return _parse_timestamp(ts_start), _parse_timestamp(ts_end)
+
+
+def _parse_extra_cols(s):
+    try:
+        extra_cols = s.split('+')[1:]
+    except (ValueError, IndexError):
+        raise ValueError(f'invalid extra groups spec: {s}') from None
+
+    return extra_cols
+
+
+def _parse_aggregation(s):
+    try:
+        op, extra_groups = s.split(':')
+    except ValueError:
+        raise ValueError(f'invalid aggregate function spec: {s}') from None
+
+    return _Aggregator.create(op), _parse_extra_cols(extra_groups)
+
+
+_Match = namedtuple('_Match', ['period_base', 'period_target',
+                               'aggregator', 'extra_groups', 'extra_cols'])
+
+
+def parse_cmp_spec(spec):
+    parts = spec.split('/')
+    if len(parts) == 3:
+        period_base, period_target, aggr, cols = None, *parts
+    elif len(parts) == 4:
+        period_base, period_target, aggr, cols = parts
+    else:
+        raise ValueError(f'invalid cmp spec: {spec}')
+
+    if period_base is not None:
+        period_base = _parse_time_period(period_base)
+
+    period_target = _parse_time_period(period_target)
+    aggr_fn, extra_groups = _parse_aggregation(aggr)
+    extra_cols = _parse_extra_cols(cols)
+    return _Match(period_base, period_target,
+                  aggr_fn, extra_groups, extra_cols)
+
+
 def performance_compare(cmp, report=None):
     match = parse_cmp_spec(cmp)
     if match.period_base is None:
@@ -462,9 +738,13 @@ def performance_compare(cmp, report=None):
             tcs_base = []
 
     else:
-        tcs_base = fetch_testcases_time_period(*match.period_base)
+        tcs_base = _ReportStorage.fetch_testcases_time_period(
+            *match.period_base
+        )
 
-    tcs_target = fetch_testcases_time_period(*match.period_target)
+    tcs_target = _ReportStorage.fetch_testcases_time_period(
+        *match.period_target
+    )
     return compare_testcase_data(tcs_base, tcs_target, match.aggregator,
                                  match.aggregator, match.extra_groups,
                                  match.extra_cols)
diff --git a/reframe/frontend/reporting/analytics.py b/reframe/frontend/reporting/analytics.py
deleted file mode 100644
index d8d79da25d..0000000000
--- a/reframe/frontend/reporting/analytics.py
+++ /dev/null
@@ -1,411 +0,0 @@
-# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
-# ReFrame Project Developers. See the top-level LICENSE file for details.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-import abc
-import os
-import math
-import re
-import sqlite3
-import statistics
-import types
-from collections import namedtuple
-from collections.abc import Hashable
-from datetime import datetime, timedelta
-from numbers import Number
-
-import reframe.core.runtime as runtime
-import reframe.utility.jsonext as jsonext
-import reframe.utility.osext as osext
-from reframe.core.logging import getlogger
-from reframe.utility import nodelist_abbrev
-
-
-def _db_file():
-    site_config = runtime.runtime().site_config
-    prefix = os.path.dirname(osext.expandvars(
-        site_config.get('general/0/report_file')
-    ))
-    filename = os.path.join(prefix, 'results.db')
-    if not os.path.exists(filename):
-        # Create subdirs if needed
-        if prefix:
-            os.makedirs(prefix, exist_ok=True)
-
-        getlogger().debug(f'Creating the results database in {filename}...')
-        _db_create(filename)
-
-    return filename
-
-
-def _db_create(filename):
-    with sqlite3.connect(filename) as conn:
-        conn.execute('CREATE TABLE IF NOT EXISTS sessions('
-                     'id INTEGER PRIMARY KEY, '
-                     'session_uuid TEXT, '
-                     'session_start_unix REAL, '
-                     'session_end_unix REAL, '
-                     'json_blob TEXT, '
-                     'report_file TEXT)')
-        conn.execute('CREATE TABLE IF NOT EXISTS testcases('
-                     'name TEXT,'
-                     'system TEXT,'
-                     'partition TEXT,'
-                     'environ TEXT,'
-                     'job_completion_time_unix REAL,'
-                     'session_id INTEGER,'
-                     'run_index INTEGER,'
-                     'test_index INTEGER,'
-                     'FOREIGN KEY(session_id) '
-                     'REFERENCES sessions(session_id))')
-
-
-def _db_store_report(conn, report, report_file_path):
-    session_start_unix = report['session_info']['time_start_unix']
-    session_end_unix = report['session_info']['time_end_unix']
-    session_uuid = datetime.fromtimestamp(session_start_unix).strftime(
-        r'%Y%m%dT%H%M%S%z'
-    )
-    for run_idx, run in enumerate(report['runs']):
-        for test_idx, testcase in enumerate(run['testcases']):
-            sys, part = testcase['system'], testcase['partition']
-            cursor = conn.execute(
-                'INSERT INTO sessions VALUES('
-                ':session_id, :session_uuid, '
-                ':session_start_unix, :session_end_unix, '
-                ':json_blob, :report_file)',
-                {
-                    'session_id': None,
-                    'session_uuid': session_uuid,
-                    'session_start_unix': session_start_unix,
-                    'session_end_unix': session_end_unix,
-                    'json_blob': jsonext.dumps(report),
-                    'report_file': report_file_path
-                }
-            )
-            conn.execute(
-                'INSERT INTO testcases VALUES('
-                ':name, :system, :partition, :environ, '
-                ':job_completion_time_unix, '
-                ':session_id, :run_index, :test_index)',
-                {
-                    'name': testcase['name'],
-                    'system': sys,
-                    'partition': part,
-                    'environ': testcase['environ'],
-                    'job_completion_time_unix': testcase[
-                        'job_completion_time_unix'
-                    ],
-                    'session_id': cursor.lastrowid,
-                    'run_index': run_idx,
-                    'test_index': test_idx
-                }
-            )
-
-    return session_uuid
-
-
-def store_report(report, report_file):
-    with sqlite3.connect(_db_file()) as conn:
-        return _db_store_report(conn, report, report_file)
-
-
-def _fetch_testcases_raw(condition):
-    with sqlite3.connect(_db_file()) as conn:
-        query = (f'SELECT session_id, run_index, test_index, json_blob FROM '
-                 f'testcases JOIN sessions ON session_id==id '
-                 f'WHERE {condition}')
-        getlogger().debug(query)
-        results = conn.execute(query).fetchall()
-
-    # Retrieve files
-    testcases = []
-    sessions = {}
-    for session_id, run_index, test_index, json_blob in results:
-        report = jsonext.loads(sessions.setdefault(session_id, json_blob))
-        testcases.append(report['runs'][run_index]['testcases'][test_index])
-
-    return testcases
-
-
-def _fetch_session_time_period(condition):
-    with sqlite3.connect(_db_file()) as conn:
-        query = (f'SELECT session_start_unix, session_end_unix FROM sessions '
-                 f'WHERE {condition} LIMIT 1')
-        getlogger().debug(query)
-        results = conn.execute(query).fetchall()
-        if results:
-            return results[0]
-
-        return None, None
-
-
-def fetch_testcases_time_period(ts_start, ts_end):
-    return _fetch_testcases_raw(
-        f'(job_completion_time_unix >= {ts_start} AND '
-        f'job_completion_time_unix <= {ts_end}) '
-        'ORDER BY job_completion_time_unix'
-    )
-
-
-def _group_key(groups, testcase):
-    key = []
-    for grp in groups:
-        val = testcase[grp]
-        if grp == 'job_nodelist':
-            # Fold nodelist before adding as a key element
-            key.append(nodelist_abbrev(val))
-        elif not isinstance(val, Hashable):
-            key.append(str(val))
-        else:
-            key.append(val)
-
-    return tuple(key)
-
-
-def _group_testcases(testcases, group_by, extra_cols):
-    grouped = {}
-    for tc in testcases:
-        for pvar, reftuple in tc['perfvalues'].items():
-            pvar = pvar.split(':')[-1]
-            pval, pref, plower, pupper, punit = reftuple
-            plower = pref * (1 + plower) if plower is not None else -math.inf
-            pupper = pref * (1 + pupper) if pupper is not None else math.inf
-            record = {
-                'pvar': pvar,
-                'pval': pval,
-                'plower': plower,
-                'pupper': pupper,
-                'punit': punit,
-                **{k: tc[k] for k in group_by + extra_cols if k in tc}
-            }
-            key = _group_key(group_by, record)
-            grouped.setdefault(key, [])
-            grouped[key].append(record)
-
-    return grouped
-
-
-def _aggregate_perf(grouped_testcases, aggr_fn, cols):
-    other_aggr = _JoinUniqueValues('|')
-    aggr_data = {}
-    for key, seq in grouped_testcases.items():
-        aggr_data.setdefault(key, {})
-        aggr_data[key]['pval'] = aggr_fn(tc['pval'] for tc in seq)
-        for c in cols:
-            aggr_data[key][c] = other_aggr(
-                nodelist_abbrev(tc[c]) if c == 'job_nodelist' else tc[c]
-                for tc in seq
-            )
-
-    return aggr_data
-
-
-def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
-                          extra_group_by=None, extra_cols=None):
-    extra_group_by = extra_group_by or []
-    extra_cols = extra_cols or []
-    group_by = ['name', 'pvar', 'punit'] + extra_group_by
-
-    grouped_base = _group_testcases(base_testcases, group_by, extra_cols)
-    grouped_target = _group_testcases(target_testcases, group_by, extra_cols)
-    pbase = _aggregate_perf(grouped_base, base_fn, extra_cols)
-    ptarget = _aggregate_perf(grouped_target, target_fn, [])
-
-    # Build the final table data
-    data = []
-    for key, aggr_data in pbase.items():
-        pval = aggr_data['pval']
-        try:
-            target_pval = ptarget[key]['pval']
-        except KeyError:
-            pdiff = 'n/a'
-        else:
-            pdiff = (pval - target_pval) / target_pval
-            pdiff = '{:+7.2%}'.format(pdiff)
-
-        name, pvar, punit, *extras = key
-        line = [name, pvar, pval, punit, pdiff, *extras]
-        # Add the extra columns
-        line += [aggr_data[c] for c in extra_cols]
-        data.append(line)
-
-    return (data, (['name', 'pvar', 'pval', 'punit', 'pdiff'] +
-                   extra_group_by + extra_cols))
-
-
-class _Aggregator:
-    @classmethod
-    def create(cls, name):
-        if name == 'first':
-            return _First()
-        elif name == 'last':
-            return _Last()
-        elif name == 'mean':
-            return _Mean()
-        elif name == 'median':
-            return _Median()
-        elif name == 'min':
-            return _Min()
-        elif name == 'max':
-            return _Max()
-        else:
-            raise ValueError(f'unknown aggregation function: {name!r}')
-
-    @abc.abstractmethod
-    def __call__(self, iterable):
-        pass
-
-
-class _First(_Aggregator):
-    def __call__(self, iterable):
-        for i, elem in enumerate(iterable):
-            if i == 0:
-                return elem
-
-
-class _Last(_Aggregator):
-    def __call__(self, iterable):
-        if not isinstance(iterable, types.GeneratorType):
-            return iterable[-1]
-
-        for elem in iterable:
-            pass
-
-        return elem
-
-
-class _Mean(_Aggregator):
-    def __call__(self, iterable):
-        return statistics.mean(iterable)
-
-
-class _Median(_Aggregator):
-    def __call__(self, iterable):
-        return statistics.median(iterable)
-
-
-class _Min(_Aggregator):
-    def __call__(self, iterable):
-        return min(iterable)
-
-
-class _Max(_Aggregator):
-    def __call__(self, iterable):
-        return max(iterable)
-
-
-class _JoinUniqueValues(_Aggregator):
-    def __init__(self, delim):
-        self.__delim = delim
-
-    def __call__(self, iterable):
-        unique_vals = {str(elem) for elem in iterable}
-        return self.__delim.join(unique_vals)
-
-
-def _parse_timestamp(s):
-    if isinstance(s, Number):
-        return s
-
-    now = datetime.now()
-
-    def _do_parse(s):
-        if s == 'now':
-            return now
-
-        formats = [r'%Y%m%d', r'%Y%m%dT%H%M',
-                   r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z']
-        for fmt in formats:
-            try:
-                return datetime.strptime(s, fmt)
-            except ValueError:
-                continue
-
-        raise ValueError(f'invalid timestamp: {s}')
-
-    try:
-        ts = _do_parse(s)
-    except ValueError as err:
-        # Try the relative timestamps
-        match = re.match(r'(?P<ts>.*)(?P<amount>[\+|-]\d+)(?P<unit>[hdms])', s)
-        if not match:
-            raise err
-
-        ts = _do_parse(match.group('ts'))
-        amount = int(match.group('amount'))
-        unit = match.group('unit')
-        if unit == 'd':
-            ts += timedelta(days=amount)
-        elif unit == 'm':
-            ts += timedelta(minutes=amount)
-        elif unit == 'h':
-            ts += timedelta(hours=amount)
-        elif unit == 's':
-            ts += timedelta(seconds=amount)
-
-    return ts.timestamp()
-
-
-def _parse_time_period(s):
-    if s.startswith('^'):
-        # Retrieve the period of a full session
-        try:
-            session_uuid = s[1:]
-        except IndexError:
-            raise ValueError(f'invalid session uuid: {s}') from None
-        else:
-            ts_start, ts_end = _fetch_session_time_period(
-                f'session_uuid == "{session_uuid}"'
-            )
-            if not ts_start or not ts_end:
-                raise ValueError(f'no such session: {session_uuid}')
-    else:
-        try:
-            ts_start, ts_end = s.split(':')
-        except ValueError:
-            raise ValueError(f'invalid time period spec: {s}') from None
-
-    return _parse_timestamp(ts_start), _parse_timestamp(ts_end)
-
-
-def _parse_extra_cols(s):
-    try:
-        extra_cols = s.split('+')[1:]
-    except (ValueError, IndexError):
-        raise ValueError(f'invalid extra groups spec: {s}') from None
-
-    return extra_cols
-
-
-def _parse_aggregation(s):
-    try:
-        op, extra_groups = s.split(':')
-    except ValueError:
-        raise ValueError(f'invalid aggregate function spec: {s}') from None
-
-    return _Aggregator.create(op), _parse_extra_cols(extra_groups)
-
-
-_Match = namedtuple('_Match', ['period_base', 'period_target',
-                               'aggregator', 'extra_groups', 'extra_cols'])
-
-
-def parse_cmp_spec(spec):
-    parts = spec.split('/')
-    if len(parts) == 3:
-        period_base, period_target, aggr, cols = None, *parts
-    elif len(parts) == 4:
-        period_base, period_target, aggr, cols = parts
-    else:
-        raise ValueError(f'invalid cmp spec: {spec}')
-
-    if period_base is not None:
-        period_base = _parse_time_period(period_base)
-
-    period_target = _parse_time_period(period_target)
-    aggr_fn, extra_groups = _parse_aggregation(aggr)
-    extra_cols = _parse_extra_cols(cols)
-    return _Match(period_base, period_target,
-                  aggr_fn, extra_groups, extra_cols)

From ed4f728c3d0a0f9d7af647529cbf3bbce8643ac1 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 1 Jul 2024 16:22:21 +0200
Subject: [PATCH 024/128] Add missing file

---
 reframe/frontend/reporting/storage.py | 174 ++++++++++++++++++++++++++
 1 file changed, 174 insertions(+)
 create mode 100644 reframe/frontend/reporting/storage.py

diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
new file mode 100644
index 0000000000..66cea655c8
--- /dev/null
+++ b/reframe/frontend/reporting/storage.py
@@ -0,0 +1,174 @@
+# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# ReFrame Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+import abc
+import os
+import sqlite3
+from datetime import datetime
+
+import reframe.utility.jsonext as jsonext
+import reframe.utility.osext as osext
+from reframe.core.logging import getlogger
+from reframe.core.runtime import runtime
+
+
+class StorageBackend:
+    '''Abstract class that represents the results backend storage'''
+
+    @classmethod
+    def create(cls, backend, *args, **kwargs):
+        '''Factory method for creating storage backends'''
+        if backend == 'sqlite':
+            return _SqliteStorage(*args, **kwargs)
+        else:
+            raise NotImplementedError
+
+    @classmethod
+    def default(cls):
+        '''Return default storage backend'''
+        return cls.create('sqlite')
+
+    @abc.abstractmethod
+    def store(self, report, report_file):
+        '''Store the given report'''
+
+    @abc.abstractmethod
+    def fetch_session_time_period(self, session_uuid):
+        '''Fetch the time period from specific session'''
+
+    @abc.abstractmethod
+    def fetch_testcases_time_period(self, ts_start, ts_end):
+        '''Fetch all test cases from specified period'''
+
+
+class _SqliteStorage(StorageBackend):
+    def __init__(self):
+        site_config = runtime().site_config
+        prefix = os.path.dirname(osext.expandvars(
+            site_config.get('general/0/report_file')
+        ))
+        self.__db_file = os.path.join(prefix, 'results.db')
+
+    def _db_file(self):
+        prefix = os.path.dirname(self.__db_file)
+        if not os.path.exists(self.__db_file):
+            # Create subdirs if needed
+            if prefix:
+                os.makedirs(prefix, exist_ok=True)
+
+            self._db_create()
+
+        return self.__db_file
+
+    def _db_create(self):
+        clsname = type(self).__name__
+        getlogger().debug(
+            f'{clsname}: creating results database in {self.__db_file}...'
+        )
+        with sqlite3.connect(self.__db_file) as conn:
+            conn.execute('CREATE TABLE IF NOT EXISTS sessions('
+                        'id INTEGER PRIMARY KEY, '
+                        'session_uuid TEXT, '
+                        'session_start_unix REAL, '
+                        'session_end_unix REAL, '
+                        'json_blob TEXT, '
+                        'report_file TEXT)')
+            conn.execute('CREATE TABLE IF NOT EXISTS testcases('
+                        'name TEXT,'
+                        'system TEXT,'
+                        'partition TEXT,'
+                        'environ TEXT,'
+                        'job_completion_time_unix REAL,'
+                        'session_id INTEGER,'
+                        'run_index INTEGER,'
+                        'test_index INTEGER,'
+                        'FOREIGN KEY(session_id) '
+                        'REFERENCES sessions(session_id))')
+
+    def _db_store_report(self, conn, report, report_file_path):
+        session_start_unix = report['session_info']['time_start_unix']
+        session_end_unix = report['session_info']['time_end_unix']
+        session_uuid = datetime.fromtimestamp(session_start_unix).strftime(
+            r'%Y%m%dT%H%M%S%z'
+        )
+        for run_idx, run in enumerate(report['runs']):
+            for test_idx, testcase in enumerate(run['testcases']):
+                sys, part = testcase['system'], testcase['partition']
+                cursor = conn.execute(
+                    'INSERT INTO sessions VALUES('
+                    ':session_id, :session_uuid, '
+                    ':session_start_unix, :session_end_unix, '
+                    ':json_blob, :report_file)',
+                    {
+                        'session_id': None,
+                        'session_uuid': session_uuid,
+                        'session_start_unix': session_start_unix,
+                        'session_end_unix': session_end_unix,
+                        'json_blob': jsonext.dumps(report),
+                        'report_file': report_file_path
+                    }
+                )
+                conn.execute(
+                    'INSERT INTO testcases VALUES('
+                    ':name, :system, :partition, :environ, '
+                    ':job_completion_time_unix, '
+                    ':session_id, :run_index, :test_index)',
+                    {
+                        'name': testcase['name'],
+                        'system': sys,
+                        'partition': part,
+                        'environ': testcase['environ'],
+                        'job_completion_time_unix': testcase[
+                            'job_completion_time_unix'
+                        ],
+                        'session_id': cursor.lastrowid,
+                        'run_index': run_idx,
+                        'test_index': test_idx
+                    }
+                )
+
+        return session_uuid
+
+    def store(self,report, report_file):
+        with sqlite3.connect(self._db_file()) as conn:
+            return self._db_store_report(conn, report, report_file)
+
+    def _fetch_testcases_raw(self, condition):
+        with sqlite3.connect(self._db_file()) as conn:
+            query = (f'SELECT session_id, run_index, test_index, json_blob FROM '
+                    f'testcases JOIN sessions ON session_id==id '
+                    f'WHERE {condition}')
+            getlogger().debug(query)
+            results = conn.execute(query).fetchall()
+
+        # Retrieve files
+        testcases = []
+        sessions = {}
+        for session_id, run_index, test_index, json_blob in results:
+            report = jsonext.loads(sessions.setdefault(session_id, json_blob))
+            testcases.append(report['runs'][run_index]['testcases'][test_index])
+
+        return testcases
+
+    def fetch_session_time_period(self, session_uuid):
+        with sqlite3.connect(self._db_file()) as conn:
+            query = ('SELECT session_start_unix, session_end_unix '
+                     f'FROM sessions WHERE session_uuid == "{session_uuid}" '
+                     'LIMIT 1')
+            getlogger().debug(query)
+            results = conn.execute(query).fetchall()
+            if results:
+                return results[0]
+
+            return None, None
+
+    def fetch_testcases_time_period(self, ts_start, ts_end):
+        return self._fetch_testcases_raw(
+            f'(job_completion_time_unix >= {ts_start} AND '
+            f'job_completion_time_unix <= {ts_end}) '
+            'ORDER BY job_completion_time_unix'
+        )
+
+

From f68a092209b5d88b4e672385e78d95fc50f47d39 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 1 Jul 2024 12:04:54 +0200
Subject: [PATCH 025/128] Enhance printer to print tabulated data

---
 reframe/frontend/cli.py                | 11 ++++-------
 reframe/frontend/printer.py            |  7 +++++++
 reframe/frontend/reporting/__init__.py | 12 ++++++------
 3 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index a7be337061..fe8d7d449a 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -12,7 +12,6 @@
 import sys
 import time
 import traceback
-from tabulate import tabulate
 
 import reframe.core.config as config
 import reframe.core.exceptions as errors
@@ -1297,10 +1296,9 @@ def _sort_testcases(testcases):
             sys.exit(0)
 
         if options.performance_compare:
-            data, header = reporting.performance_compare(
-                options.performance_compare
+            printer.table(
+                *reporting.performance_compare(options.performance_compare)
             )
-            printer(tabulate(data, headers=header, tablefmt='mixed_grid'))
             sys.exit(0)
 
         if not options.run and not options.dry_run:
@@ -1475,9 +1473,8 @@ def module_unuse(*paths):
                         f'failed to generate performance report: {err}'
                     )
                 else:
-                    print(
-                        tabulate(data, headers=header, tablefmt='mixed_grid')
-                    )
+                    printer.table(data, header)
+
                 # printer.info(runner.stats.performance_report())
 
             # Generate the report for this session
diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index d94444fade..6dac37713c 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -7,6 +7,8 @@
 import shutil
 import time
 import traceback
+from tabulate import tabulate
+
 import reframe.core.logging as logging
 import reframe.core.runtime as rt
 import reframe.utility.color as color
@@ -245,3 +247,8 @@ def retry_report(self, report):
             lines.append(msg)
 
         self.info('\n'.join(lines))
+
+    def table(self, data, header):
+        '''Print a table'''
+
+        self.info(tabulate(data, headers=header, tablefmt='mixed_grid'))
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 6a96d73afb..85b50e0cc8 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -47,7 +47,7 @@ def format_testcase(json, name='unique_name'):
     return f'{name}@{system}:{partition}+{environ}'
 
 
-_ReportStorage = StorageBackend.create('sqlite')
+_ReportStorage = functools.partial(StorageBackend.create, backend='sqlite')
 
 
 class _RestoredSessionInfo:
@@ -400,7 +400,7 @@ def save(self, filename, compress=False, link_to_last=True):
     def store(self):
         '''Store the report in the results storage.'''
 
-        return _ReportStorage.store(self, self.filename)
+        return _ReportStorage().store(self, self.filename)
 
     def generate_xml_report(self):
         '''Generate a JUnit report from a standard ReFrame JSON report.'''
@@ -671,8 +671,8 @@ def _parse_time_period(s):
         except IndexError:
             raise ValueError(f'invalid session uuid: {s}') from None
         else:
-            ts_start, ts_end = _ReportStorage.fetch_session_time_period(
-                f'session_uuid == "{session_uuid}"'
+            ts_start, ts_end = _ReportStorage().fetch_session_time_period(
+                session_uuid
             )
             if not ts_start or not ts_end:
                 raise ValueError(f'no such session: {session_uuid}')
@@ -738,11 +738,11 @@ def performance_compare(cmp, report=None):
             tcs_base = []
 
     else:
-        tcs_base = _ReportStorage.fetch_testcases_time_period(
+        tcs_base = _ReportStorage().fetch_testcases_time_period(
             *match.period_base
         )
 
-    tcs_target = _ReportStorage.fetch_testcases_time_period(
+    tcs_target = _ReportStorage().fetch_testcases_time_period(
         *match.period_target
     )
     return compare_testcase_data(tcs_base, tcs_target, match.aggregator,

From 77aac04d4b7818900f91f6353b7048b224240c00 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 1 Jul 2024 15:49:43 +0200
Subject: [PATCH 026/128] Move comparison spec code to a separate utility
 package

---
 reframe/frontend/reporting/__init__.py | 184 +-----------------------
 reframe/frontend/reporting/utility.py  | 192 +++++++++++++++++++++++++
 2 files changed, 194 insertions(+), 182 deletions(-)
 create mode 100644 reframe/frontend/reporting/utility.py

diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 85b50e0cc8..7092861350 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -14,13 +14,8 @@
 import os
 import re
 import socket
-import statistics
 import time
-import types
-from collections import namedtuple
 from collections.abc import Hashable
-from datetime import datetime, timedelta
-from numbers import Number
 
 import reframe as rfm
 import reframe.utility.jsonext as jsonext
@@ -30,6 +25,7 @@
 from reframe.core.warnings import suppress_deprecations
 from reframe.utility import nodelist_abbrev
 from .storage import StorageBackend
+from .utility import Aggregator, parse_cmp_spec
 
 # The schema data version
 # Major version bumps are expected to break the validation of previous schemas
@@ -503,7 +499,7 @@ def _group_testcases(testcases, group_by, extra_cols):
 
 
 def _aggregate_perf(grouped_testcases, aggr_fn, cols):
-    other_aggr = _JoinUniqueValues('|')
+    other_aggr = Aggregator.create('join_uniq', '|')
     aggr_data = {}
     for key, seq in grouped_testcases.items():
         aggr_data.setdefault(key, {})
@@ -550,182 +546,6 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
                    extra_group_by + extra_cols))
 
 
-class _Aggregator:
-    @classmethod
-    def create(cls, name):
-        if name == 'first':
-            return _First()
-        elif name == 'last':
-            return _Last()
-        elif name == 'mean':
-            return _Mean()
-        elif name == 'median':
-            return _Median()
-        elif name == 'min':
-            return _Min()
-        elif name == 'max':
-            return _Max()
-        else:
-            raise ValueError(f'unknown aggregation function: {name!r}')
-
-    @abc.abstractmethod
-    def __call__(self, iterable):
-        pass
-
-
-class _First(_Aggregator):
-    def __call__(self, iterable):
-        for i, elem in enumerate(iterable):
-            if i == 0:
-                return elem
-
-
-class _Last(_Aggregator):
-    def __call__(self, iterable):
-        if not isinstance(iterable, types.GeneratorType):
-            return iterable[-1]
-
-        for elem in iterable:
-            pass
-
-        return elem
-
-
-class _Mean(_Aggregator):
-    def __call__(self, iterable):
-        return statistics.mean(iterable)
-
-
-class _Median(_Aggregator):
-    def __call__(self, iterable):
-        return statistics.median(iterable)
-
-
-class _Min(_Aggregator):
-    def __call__(self, iterable):
-        return min(iterable)
-
-
-class _Max(_Aggregator):
-    def __call__(self, iterable):
-        return max(iterable)
-
-
-class _JoinUniqueValues(_Aggregator):
-    def __init__(self, delim):
-        self.__delim = delim
-
-    def __call__(self, iterable):
-        unique_vals = {str(elem) for elem in iterable}
-        return self.__delim.join(unique_vals)
-
-
-def _parse_timestamp(s):
-    if isinstance(s, Number):
-        return s
-
-    now = datetime.now()
-
-    def _do_parse(s):
-        if s == 'now':
-            return now
-
-        formats = [r'%Y%m%d', r'%Y%m%dT%H%M',
-                   r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z']
-        for fmt in formats:
-            try:
-                return datetime.strptime(s, fmt)
-            except ValueError:
-                continue
-
-        raise ValueError(f'invalid timestamp: {s}')
-
-    try:
-        ts = _do_parse(s)
-    except ValueError as err:
-        # Try the relative timestamps
-        match = re.match(r'(?P<ts>.*)(?P<amount>[\+|-]\d+)(?P<unit>[hdms])', s)
-        if not match:
-            raise err
-
-        ts = _do_parse(match.group('ts'))
-        amount = int(match.group('amount'))
-        unit = match.group('unit')
-        if unit == 'd':
-            ts += timedelta(days=amount)
-        elif unit == 'm':
-            ts += timedelta(minutes=amount)
-        elif unit == 'h':
-            ts += timedelta(hours=amount)
-        elif unit == 's':
-            ts += timedelta(seconds=amount)
-
-    return ts.timestamp()
-
-
-def _parse_time_period(s):
-    if s.startswith('^'):
-        # Retrieve the period of a full session
-        try:
-            session_uuid = s[1:]
-        except IndexError:
-            raise ValueError(f'invalid session uuid: {s}') from None
-        else:
-            ts_start, ts_end = _ReportStorage().fetch_session_time_period(
-                session_uuid
-            )
-            if not ts_start or not ts_end:
-                raise ValueError(f'no such session: {session_uuid}')
-    else:
-        try:
-            ts_start, ts_end = s.split(':')
-        except ValueError:
-            raise ValueError(f'invalid time period spec: {s}') from None
-
-    return _parse_timestamp(ts_start), _parse_timestamp(ts_end)
-
-
-def _parse_extra_cols(s):
-    try:
-        extra_cols = s.split('+')[1:]
-    except (ValueError, IndexError):
-        raise ValueError(f'invalid extra groups spec: {s}') from None
-
-    return extra_cols
-
-
-def _parse_aggregation(s):
-    try:
-        op, extra_groups = s.split(':')
-    except ValueError:
-        raise ValueError(f'invalid aggregate function spec: {s}') from None
-
-    return _Aggregator.create(op), _parse_extra_cols(extra_groups)
-
-
-_Match = namedtuple('_Match', ['period_base', 'period_target',
-                               'aggregator', 'extra_groups', 'extra_cols'])
-
-
-def parse_cmp_spec(spec):
-    parts = spec.split('/')
-    if len(parts) == 3:
-        period_base, period_target, aggr, cols = None, *parts
-    elif len(parts) == 4:
-        period_base, period_target, aggr, cols = parts
-    else:
-        raise ValueError(f'invalid cmp spec: {spec}')
-
-    if period_base is not None:
-        period_base = _parse_time_period(period_base)
-
-    period_target = _parse_time_period(period_target)
-    aggr_fn, extra_groups = _parse_aggregation(aggr)
-    extra_cols = _parse_extra_cols(cols)
-    return _Match(period_base, period_target,
-                  aggr_fn, extra_groups, extra_cols)
-
-
 def performance_compare(cmp, report=None):
     match = parse_cmp_spec(cmp)
     if match.period_base is None:
diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py
new file mode 100644
index 0000000000..9cbc1e6666
--- /dev/null
+++ b/reframe/frontend/reporting/utility.py
@@ -0,0 +1,192 @@
+# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# ReFrame Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+import abc
+import re
+import statistics
+import types
+from collections import namedtuple
+from datetime import datetime, timedelta
+from numbers import Number
+from .storage import StorageBackend
+
+class Aggregator:
+    @classmethod
+    def create(cls, name, *args, **kwargs):
+        if name == 'first':
+            return AggrFirst(*args, **kwargs)
+        elif name == 'last':
+            return AggrLast(*args, **kwargs)
+        elif name == 'mean':
+            return AggrMean(*args, **kwargs)
+        elif name == 'median':
+            return AggrMedian(*args, **kwargs)
+        elif name == 'min':
+            return AggrMin(*args, **kwargs)
+        elif name == 'max':
+            return AggrMax(*args, **kwargs)
+        elif name == 'join_uniq':
+            return AggrJoinUniqueValues(*args, **kwargs)
+        else:
+            raise ValueError(f'unknown aggregation function: {name!r}')
+
+    @abc.abstractmethod
+    def __call__(self, iterable):
+        pass
+
+
+class AggrFirst(Aggregator):
+    def __call__(self, iterable):
+        for i, elem in enumerate(iterable):
+            if i == 0:
+                return elem
+
+
+class AggrLast(Aggregator):
+    def __call__(self, iterable):
+        if not isinstance(iterable, types.GeneratorType):
+            return iterable[-1]
+
+        for elem in iterable:
+            pass
+
+        return elem
+
+
+class AggrMean(Aggregator):
+    def __call__(self, iterable):
+        return statistics.mean(iterable)
+
+
+class AggrMedian(Aggregator):
+    def __call__(self, iterable):
+        return statistics.median(iterable)
+
+
+class AggrMin(Aggregator):
+    def __call__(self, iterable):
+        return min(iterable)
+
+
+class AggrMax(Aggregator):
+    def __call__(self, iterable):
+        return max(iterable)
+
+
+class AggrJoinUniqueValues(Aggregator):
+    def __init__(self, delim):
+        self.__delim = delim
+
+    def __call__(self, iterable):
+        unique_vals = {str(elem) for elem in iterable}
+        return self.__delim.join(unique_vals)
+
+
+def _parse_timestamp(s):
+    if isinstance(s, Number):
+        return s
+
+    now = datetime.now()
+
+    def _do_parse(s):
+        if s == 'now':
+            return now
+
+        formats = [r'%Y%m%d', r'%Y%m%dT%H%M',
+                   r'%Y%m%dT%H%M%S', r'%Y%m%dT%H%M%S%z']
+        for fmt in formats:
+            try:
+                return datetime.strptime(s, fmt)
+            except ValueError:
+                continue
+
+        raise ValueError(f'invalid timestamp: {s}')
+
+    try:
+        ts = _do_parse(s)
+    except ValueError as err:
+        # Try the relative timestamps
+        match = re.match(r'(?P<ts>.*)(?P<amount>[\+|-]\d+)(?P<unit>[hdms])', s)
+        if not match:
+            raise err
+
+        ts = _do_parse(match.group('ts'))
+        amount = int(match.group('amount'))
+        unit = match.group('unit')
+        if unit == 'd':
+            ts += timedelta(days=amount)
+        elif unit == 'm':
+            ts += timedelta(minutes=amount)
+        elif unit == 'h':
+            ts += timedelta(hours=amount)
+        elif unit == 's':
+            ts += timedelta(seconds=amount)
+
+    return ts.timestamp()
+
+
+def _parse_time_period(s):
+    if s.startswith('^'):
+        # Retrieve the period of a full session
+        try:
+            session_uuid = s[1:]
+        except IndexError:
+            raise ValueError(f'invalid session uuid: {s}') from None
+        else:
+            backend = StorageBackend('sqlite')
+            ts_start, ts_end = backend.fetch_session_time_period(
+                session_uuid
+            )
+            if not ts_start or not ts_end:
+                raise ValueError(f'no such session: {session_uuid}')
+    else:
+        try:
+            ts_start, ts_end = s.split(':')
+        except ValueError:
+            raise ValueError(f'invalid time period spec: {s}') from None
+
+    return _parse_timestamp(ts_start), _parse_timestamp(ts_end)
+
+
+def _parse_extra_cols(s):
+    try:
+        extra_cols = s.split('+')[1:]
+    except (ValueError, IndexError):
+        raise ValueError(f'invalid extra groups spec: {s}') from None
+
+    return extra_cols
+
+
+def _parse_aggregation(s):
+    try:
+        op, extra_groups = s.split(':')
+    except ValueError:
+        raise ValueError(f'invalid aggregate function spec: {s}') from None
+
+    return Aggregator.create(op), _parse_extra_cols(extra_groups)
+
+
+_Match = namedtuple('_Match', ['period_base', 'period_target',
+                               'aggregator', 'extra_groups', 'extra_cols'])
+
+
+def parse_cmp_spec(spec):
+    parts = spec.split('/')
+    if len(parts) == 3:
+        period_base, period_target, aggr, cols = None, *parts
+    elif len(parts) == 4:
+        period_base, period_target, aggr, cols = parts
+    else:
+        raise ValueError(f'invalid cmp spec: {spec}')
+
+    if period_base is not None:
+        period_base = _parse_time_period(period_base)
+
+    period_target = _parse_time_period(period_target)
+    aggr_fn, extra_groups = _parse_aggregation(aggr)
+    extra_cols = _parse_extra_cols(cols)
+    return _Match(period_base, period_target,
+                  aggr_fn, extra_groups, extra_cols)
+

From 0b52501c2ad91fbc50a032e62d53e563c68b5c0e Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 1 Jul 2024 15:56:03 +0200
Subject: [PATCH 027/128] Improve access to default storage backend

---
 reframe/frontend/reporting/__init__.py | 9 +++------
 reframe/frontend/reporting/utility.py  | 2 +-
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 7092861350..28b580fb45 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -43,9 +43,6 @@ def format_testcase(json, name='unique_name'):
     return f'{name}@{system}:{partition}+{environ}'
 
 
-_ReportStorage = functools.partial(StorageBackend.create, backend='sqlite')
-
-
 class _RestoredSessionInfo:
     '''A restored session with some additional functionality.'''
 
@@ -396,7 +393,7 @@ def save(self, filename, compress=False, link_to_last=True):
     def store(self):
         '''Store the report in the results storage.'''
 
-        return _ReportStorage().store(self, self.filename)
+        return StorageBackend.default().store(self, self.filename)
 
     def generate_xml_report(self):
         '''Generate a JUnit report from a standard ReFrame JSON report.'''
@@ -558,11 +555,11 @@ def performance_compare(cmp, report=None):
             tcs_base = []
 
     else:
-        tcs_base = _ReportStorage().fetch_testcases_time_period(
+        tcs_base = StorageBackend.default().fetch_testcases_time_period(
             *match.period_base
         )
 
-    tcs_target = _ReportStorage().fetch_testcases_time_period(
+    tcs_target = StorageBackend.default().fetch_testcases_time_period(
         *match.period_target
     )
     return compare_testcase_data(tcs_base, tcs_target, match.aggregator,
diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py
index 9cbc1e6666..2d9a839968 100644
--- a/reframe/frontend/reporting/utility.py
+++ b/reframe/frontend/reporting/utility.py
@@ -135,7 +135,7 @@ def _parse_time_period(s):
         except IndexError:
             raise ValueError(f'invalid session uuid: {s}') from None
         else:
-            backend = StorageBackend('sqlite')
+            backend = StorageBackend.default()
             ts_start, ts_end = backend.fetch_session_time_period(
                 session_uuid
             )

From 5a5b3812538dea7cfca69a2b753a89c9cc817a79 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 1 Jul 2024 16:11:10 +0200
Subject: [PATCH 028/128] Add configuration option for specifying the perf.
 report spec

---
 reframe/frontend/cli.py     | 8 ++++++--
 reframe/schemas/config.json | 2 ++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index fe8d7d449a..b6f38674f8 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -544,7 +544,11 @@ def main():
     )
     reporting_options.add_argument(
         '--performance-report', action='store', nargs='?',
-        const='19700101T0000Z:now/last:+job_nodelist/+result',
+        # a non-empty (unused) token to ensure that option will be set
+        # even if no argument is passed
+        const='<token>',
+        configvar='general/0/perf_report_spec',
+        envvar='RFM_PERF_REPORT_SPEC',
         help='Print a report for performance tests'
     )
 
@@ -1466,7 +1470,7 @@ def module_unuse(*paths):
             if options.performance_report and not options.dry_run:
                 try:
                     data, header = reporting.performance_compare(
-                        options.performance_report, report
+                        rt.get_option('general/0/perf_report_spec'), report
                     )
                 except errors.ReframeError as err:
                     printer.warning(
diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index 2dcbec053c..bcea54e185 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -514,6 +514,7 @@
                     "non_default_craype": {"type": "boolean"},
                     "dump_pipeline_progress": {"type": "boolean"},
                     "perf_info_level": {"$ref": "#/defs/loglevel"},
+                    "perf_report_spec": {"type": "string"},
                     "pipeline_timeout": {"type": ["number", "null"]},
                     "purge_environment": {"type": "boolean"},
                     "remote_detect": {"type": "boolean"},
@@ -567,6 +568,7 @@
         "general/module_mappings": [],
         "general/non_default_craype": false,
         "general/perf_info_level": "info",
+        "general/perf_report_spec": "19700101T0000Z:now/last:+job_nodelist/+result",
         "general/purge_environment": false,
         "general/remote_detect": false,
         "general/remote_workdir": ".",

From 864917ccf73807949b9825b7eec16bd8907044f3 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 1 Jul 2024 17:16:56 +0200
Subject: [PATCH 029/128] Remove unused import

---
 reframe/frontend/reporting/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 28b580fb45..bde0857e63 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-import abc
 import decimal
 import functools
 import inspect

From 3137f15044205a70749111cdd626f8a6822a0ed0 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 1 Jul 2024 17:30:11 +0200
Subject: [PATCH 030/128] Python 3.6 compatible default value for
 `perf_report_spec`

---
 reframe/schemas/config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index bcea54e185..3a9d6d80f5 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -568,7 +568,7 @@
         "general/module_mappings": [],
         "general/non_default_craype": false,
         "general/perf_info_level": "info",
-        "general/perf_report_spec": "19700101T0000Z:now/last:+job_nodelist/+result",
+        "general/perf_report_spec": "19700101T0000+0000:now/last:+job_nodelist/+result",
         "general/purge_environment": false,
         "general/remote_detect": false,
         "general/remote_workdir": ".",

From 15dd76cf0d8a6c4a1e5a88fe30d63e873f79c7af Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Tue, 2 Jul 2024 10:18:55 +0200
Subject: [PATCH 031/128] Fix coding style issues

---
 reframe/frontend/reporting/storage.py | 46 +++++++++++++--------------
 reframe/frontend/reporting/utility.py |  2 +-
 setup.cfg                             |  3 +-
 3 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 66cea655c8..0422539713 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -69,23 +69,23 @@ def _db_create(self):
         )
         with sqlite3.connect(self.__db_file) as conn:
             conn.execute('CREATE TABLE IF NOT EXISTS sessions('
-                        'id INTEGER PRIMARY KEY, '
-                        'session_uuid TEXT, '
-                        'session_start_unix REAL, '
-                        'session_end_unix REAL, '
-                        'json_blob TEXT, '
-                        'report_file TEXT)')
+                         'id INTEGER PRIMARY KEY, '
+                         'session_uuid TEXT, '
+                         'session_start_unix REAL, '
+                         'session_end_unix REAL, '
+                         'json_blob TEXT, '
+                         'report_file TEXT)')
             conn.execute('CREATE TABLE IF NOT EXISTS testcases('
-                        'name TEXT,'
-                        'system TEXT,'
-                        'partition TEXT,'
-                        'environ TEXT,'
-                        'job_completion_time_unix REAL,'
-                        'session_id INTEGER,'
-                        'run_index INTEGER,'
-                        'test_index INTEGER,'
-                        'FOREIGN KEY(session_id) '
-                        'REFERENCES sessions(session_id))')
+                         'name TEXT,'
+                         'system TEXT,'
+                         'partition TEXT,'
+                         'environ TEXT,'
+                         'job_completion_time_unix REAL,'
+                         'session_id INTEGER,'
+                         'run_index INTEGER,'
+                         'test_index INTEGER,'
+                         'FOREIGN KEY(session_id) '
+                         'REFERENCES sessions(session_id))')
 
     def _db_store_report(self, conn, report, report_file_path):
         session_start_unix = report['session_info']['time_start_unix']
@@ -131,15 +131,15 @@ def _db_store_report(self, conn, report, report_file_path):
 
         return session_uuid
 
-    def store(self,report, report_file):
+    def store(self, report, report_file):
         with sqlite3.connect(self._db_file()) as conn:
             return self._db_store_report(conn, report, report_file)
 
     def _fetch_testcases_raw(self, condition):
         with sqlite3.connect(self._db_file()) as conn:
-            query = (f'SELECT session_id, run_index, test_index, json_blob FROM '
-                    f'testcases JOIN sessions ON session_id==id '
-                    f'WHERE {condition}')
+            query = ('SELECT session_id, run_index, test_index, json_blob '
+                     'FROM testcases JOIN sessions ON session_id==id '
+                     f'WHERE {condition}')
             getlogger().debug(query)
             results = conn.execute(query).fetchall()
 
@@ -148,7 +148,9 @@ def _fetch_testcases_raw(self, condition):
         sessions = {}
         for session_id, run_index, test_index, json_blob in results:
             report = jsonext.loads(sessions.setdefault(session_id, json_blob))
-            testcases.append(report['runs'][run_index]['testcases'][test_index])
+            testcases.append(
+                report['runs'][run_index]['testcases'][test_index]
+            )
 
         return testcases
 
@@ -170,5 +172,3 @@ def fetch_testcases_time_period(self, ts_start, ts_end):
             f'job_completion_time_unix <= {ts_end}) '
             'ORDER BY job_completion_time_unix'
         )
-
-
diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py
index 2d9a839968..05cc0cf337 100644
--- a/reframe/frontend/reporting/utility.py
+++ b/reframe/frontend/reporting/utility.py
@@ -12,6 +12,7 @@
 from numbers import Number
 from .storage import StorageBackend
 
+
 class Aggregator:
     @classmethod
     def create(cls, name, *args, **kwargs):
@@ -189,4 +190,3 @@ def parse_cmp_spec(spec):
     extra_cols = _parse_extra_cols(cols)
     return _Match(period_base, period_target,
                   aggr_fn, extra_groups, extra_cols)
-
diff --git a/setup.cfg b/setup.cfg
index 5ae3959849..974e38855d 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -47,4 +47,5 @@ include = reframe,reframe.*,hpctestlib.*
 reframe = schemas/*
 
 [flake8]
-ignore = E129,E221,E226,E241,E402,E272,E741,E742,E743,W504
+extend-ignore = E129,E221,E226,E241,E402,E272,E741,E742,E743,F821,W504
+exclude = .git,__pycache__,docs/conf.py,external

From 2ca850c855dd624321f9cd1a23b6aa2e810efe46 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Tue, 2 Jul 2024 11:54:06 +0200
Subject: [PATCH 032/128] Improve formatting of performance report

---
 reframe/frontend/cli.py     |  4 +---
 reframe/frontend/printer.py | 13 ++++++++++---
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index b6f38674f8..9d55d34541 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -1477,9 +1477,7 @@ def module_unuse(*paths):
                         f'failed to generate performance report: {err}'
                     )
                 else:
-                    printer.table(data, header)
-
-                # printer.info(runner.stats.performance_report())
+                    printer.performance_report(data, header)
 
             # Generate the report for this session
             report_file = os.path.normpath(
diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index 6dac37713c..2409637a20 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -159,9 +159,8 @@ def _print_failure_info(rec, runid, total_runs):
             else:
                 self.verbose(tb)
 
-        line_width = shutil.get_terminal_size()[0]
-        self.info(line_width * '=')
-        self.info('SUMMARY OF FAILURES')
+        line_width = min(80, shutil.get_terminal_size()[0])
+        self.info(' SUMMARY OF FAILURES '.center(line_width, '='))
 
         for run_no, run_info in enumerate(report['runs'], start=1):
             if not global_stats and run_no != len(report['runs']):
@@ -248,6 +247,14 @@ def retry_report(self, report):
 
         self.info('\n'.join(lines))
 
+    def performance_report(self, data, header):
+        width = min(80, shutil.get_terminal_size()[0])
+        self.info('')
+        self.info(' PERFORMANCE REPORT '.center(width, '='))
+        self.info('')
+        self.table(data, header)
+        self.info('')
+
     def table(self, data, header):
         '''Print a table'''
 

From 9d6d1bba6bb2dbfec441a21a9195fac75295765c Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 8 Jul 2024 17:15:19 +0300
Subject: [PATCH 033/128] Improve error handling and fix handling of
 `perf_report_spec`

---
 reframe/frontend/argparse.py           |  2 +-
 reframe/frontend/cli.py                | 28 ++++++++++++++------------
 reframe/frontend/reporting/__init__.py |  7 +++++--
 reframe/schemas/config.json            |  2 +-
 4 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/reframe/frontend/argparse.py b/reframe/frontend/argparse.py
index e6c37b7484..82c86af1a3 100644
--- a/reframe/frontend/argparse.py
+++ b/reframe/frontend/argparse.py
@@ -127,7 +127,7 @@ def update_config(self, site_config):
 
     def __repr__(self):
         return (f'{type(self).__name__}({self.__namespace!r}, '
-                '{self.__option_map})')
+                f'{self.__option_map})')
 
 
 class _ArgumentHolder:
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 9d55d34541..8c0058bc09 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -293,10 +293,9 @@ def main():
         envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files'
     )
     output_options.add_argument(
-        '--timestamp', action='store', nargs='?', const='%y%m%dT%H%M%S%z',
-        metavar='TIMEFMT',
+        '--timestamp', action='store', nargs='?', metavar='TIMEFMT',
         help=('Append a timestamp to the output and stage directory prefixes '
-              '(default: "%%FT%%T")'),
+              r'(default: "%y%m%dT%H%M%S%z")'),
         envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs'
     )
 
@@ -544,12 +543,10 @@ def main():
     )
     reporting_options.add_argument(
         '--performance-report', action='store', nargs='?',
-        # a non-empty (unused) token to ensure that option will be set
-        # even if no argument is passed
-        const='<token>',
-        configvar='general/0/perf_report_spec',
+        configvar='general/perf_report_spec',
         envvar='RFM_PERF_REPORT_SPEC',
-        help='Print a report for performance tests'
+        help=('Print a report for performance tests '
+              '(default: "19700101T0000+0000:now/last:+job_nodelist/+result")')
     )
 
     # Miscellaneous options
@@ -1300,10 +1297,15 @@ def _sort_testcases(testcases):
             sys.exit(0)
 
         if options.performance_compare:
-            printer.table(
-                *reporting.performance_compare(options.performance_compare)
-            )
-            sys.exit(0)
+            try:
+                printer.table(
+                    *reporting.performance_compare(options.performance_compare)
+                )
+            except (errors.ReframeError, ValueError) as err:
+                printer.error(f'failed to generate performance report: {err}')
+                sys.exit(1)
+            else:
+                sys.exit(0)
 
         if not options.run and not options.dry_run:
             printer.error("No action option specified. Available options:\n"
@@ -1472,7 +1474,7 @@ def module_unuse(*paths):
                     data, header = reporting.performance_compare(
                         rt.get_option('general/0/perf_report_spec'), report
                     )
-                except errors.ReframeError as err:
+                except (errors.ReframeError, ValueError) as err:
                     printer.warning(
                         f'failed to generate performance report: {err}'
                     )
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index bde0857e63..e808dd4bf6 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -529,8 +529,11 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
         except KeyError:
             pdiff = 'n/a'
         else:
-            pdiff = (pval - target_pval) / target_pval
-            pdiff = '{:+7.2%}'.format(pdiff)
+            if pval is None or target_pval is None:
+                pdiff = 'n/a'
+            else:
+                pdiff = (pval - target_pval) / target_pval
+                pdiff = '{:+7.2%}'.format(pdiff)
 
         name, pvar, punit, *extras = key
         line = [name, pvar, pval, punit, pdiff, *extras]
diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index 3a9d6d80f5..51189b2811 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -578,7 +578,7 @@
         "general/resolve_module_conflicts": true,
         "general/save_log_files": false,
         "general/target_systems": ["*"],
-        "general/timestamp_dirs": "",
+        "general/timestamp_dirs": "%y%m%dT%H%M%S%z",
         "general/trap_job_errors": false,
         "general/unload_modules": [],
         "general/use_login_shell": false,

From 0266495cc33eb5664734c9bfcb2477b8ec7463c3 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 8 Jul 2024 19:39:19 +0300
Subject: [PATCH 034/128] Fix handling of `const` optional argument in
 `ArgumentParser`

---
 reframe/core/runtime.py      | 13 +++++++++----
 reframe/frontend/argparse.py | 25 +++++++++++++++++++++++--
 reframe/frontend/cli.py      |  7 +++++--
 unittests/test_argparser.py  | 19 +++++++------------
 4 files changed, 44 insertions(+), 20 deletions(-)

diff --git a/reframe/core/runtime.py b/reframe/core/runtime.py
index 784ba7c750..24d754b991 100644
--- a/reframe/core/runtime.py
+++ b/reframe/core/runtime.py
@@ -27,11 +27,12 @@ class RuntimeContext:
     .. versionadded:: 2.13
     '''
 
-    def __init__(self, site_config):
+    def __init__(self, site_config, *, use_timestamps=False):
         self._site_config = site_config
         self._system = System.create(site_config)
         self._current_run = 0
         self._timestamp = time.localtime()
+        self._use_timestamps = use_timestamps
 
     def _makedir(self, *dirs, wipeout=False):
         ret = os.path.join(*dirs)
@@ -110,7 +111,11 @@ def perflogdir(self):
 
     @property
     def timestamp(self):
-        timefmt = self.site_config.get('general/0/timestamp_dirs')
+        if self._use_timestamps:
+            timefmt = self.site_config.get('general/0/timestamp_dirs')
+        else:
+            timefmt = ''
+
         return time.strftime(timefmt, self._timestamp)
 
     @property
@@ -192,11 +197,11 @@ def get_default(self, option):
 _runtime_context = None
 
 
-def init_runtime(site_config):
+def init_runtime(site_config, **kwargs):
     global _runtime_context
 
     if _runtime_context is None:
-        _runtime_context = RuntimeContext(site_config)
+        _runtime_context = RuntimeContext(site_config, **kwargs)
 
 
 def runtime():
diff --git a/reframe/frontend/argparse.py b/reframe/frontend/argparse.py
index 82c86af1a3..ec6c667142 100644
--- a/reframe/frontend/argparse.py
+++ b/reframe/frontend/argparse.py
@@ -40,6 +40,24 @@
 # that essentially associate environment variables with configuration
 # arguments, without having to define a corresponding command line option.
 
+class _Undefined:
+    pass
+
+
+# We use a special value for denoting const values that are to be set from the
+# configuration default. This placeholder must be used as the `const` argument
+#  for options with `nargs='?'`. The underlying `ArugmentParser` will use the
+# `const` value as if it were supplied from the command-line thus fooling our
+# machinery of environment variables and configuration options overriding any
+# defaults. For this reason, we use a unique placeholder so that we can
+# distinguish whether this value is a default or actually supplied from the
+# command-line.
+CONST_DEFAULT = _Undefined()
+
+
+def _undefined(val):
+    return val is None or val is CONST_DEFAULT
+
 
 class _Namespace:
     def __init__(self, namespace, option_map):
@@ -76,7 +94,10 @@ def __getattr__(self, name):
             return ret
 
         envvar, _, action, arg_type, default = self.__option_map[name]
-        if ret is None and envvar is not None:
+        if ret is CONST_DEFAULT:
+            default = CONST_DEFAULT
+
+        if _undefined(ret) and envvar is not None:
             # Try the environment variable
             envvar, *delim = envvar.split(maxsplit=2)
             delim = delim[0] if delim else ','
@@ -120,7 +141,7 @@ def update_config(self, site_config):
                 errors.append(e)
                 continue
 
-            if value is not None:
+            if not _undefined(value):
                 site_config.add_sticky_option(confvar, value)
 
         return errors
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 8c0058bc09..faa0a40674 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -294,8 +294,9 @@ def main():
     )
     output_options.add_argument(
         '--timestamp', action='store', nargs='?', metavar='TIMEFMT',
+        const=argparse.CONST_DEFAULT,
         help=('Append a timestamp to the output and stage directory prefixes '
-              r'(default: "%y%m%dT%H%M%S%z")'),
+              '(default: "%%y%%m%%dT%%H%%M%%S%%z")'),
         envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs'
     )
 
@@ -543,6 +544,7 @@ def main():
     )
     reporting_options.add_argument(
         '--performance-report', action='store', nargs='?',
+        const=argparse.CONST_DEFAULT,
         configvar='general/perf_report_spec',
         envvar='RFM_PERF_REPORT_SPEC',
         help=('Print a report for performance tests '
@@ -856,7 +858,8 @@ def restrict_logging():
 
     try:
         printer.debug('Initializing runtime')
-        runtime.init_runtime(site_config)
+        runtime.init_runtime(site_config,
+                             use_timestamps=options.timestamp is not None)
     except errors.ConfigError as e:
         printer.error(f'failed to initialize runtime: {e}')
         printer.info(logfiles_message())
diff --git a/unittests/test_argparser.py b/unittests/test_argparser.py
index 6647ed1e19..cd6fffea21 100644
--- a/unittests/test_argparser.py
+++ b/unittests/test_argparser.py
@@ -7,7 +7,7 @@
 
 import reframe.core.runtime as rt
 import unittests.utility as test_util
-from reframe.frontend.argparse import ArgumentParser
+from reframe.frontend.argparse import ArgumentParser, CONST_DEFAULT
 
 
 @pytest.fixture
@@ -118,7 +118,7 @@ def extended_parser():
         default='bar'
     )
     foo_options.add_argument(
-        '--timestamp', action='store',
+        '--timestamp', action='store', nargs='?', const=CONST_DEFAULT,
         envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs'
     )
     foo_options.add_argument(
@@ -143,17 +143,14 @@ def extended_parser():
 
 def test_option_precedence(default_exec_ctx, extended_parser):
     with rt.temp_environment(env_vars={
-            'RFM_TIMESTAMP': '%F',
             'RFM_NON_DEFAULT_CRAYPE': 'yes',
             'RFM_MODULES_PRELOAD': 'a,b,c',
             'RFM_CHECK_SEARCH_PATH': 'x:y:z'
 
     }):
-        options = extended_parser.parse_args(
-            ['--timestamp=%FT%T', '--nocolor']
-        )
+        options = extended_parser.parse_args(['--nocolor', '--timestamp'])
         assert options.recursive is None
-        assert options.timestamp == '%FT%T'
+        assert options.timestamp is CONST_DEFAULT
         assert options.non_default_craype is True
         assert options.config_file is None
         assert options.prefix is None
@@ -165,19 +162,17 @@ def test_option_precedence(default_exec_ctx, extended_parser):
 
 def test_option_with_config(default_exec_ctx, extended_parser, tmp_path):
     with rt.temp_environment(env_vars={
-            'RFM_TIMESTAMP': '%F',
+            'RFM_TIMESTAMP_DIRS': r'%F',
             'RFM_NON_DEFAULT_CRAYPE': 'yes',
             'RFM_MODULES_PRELOAD': 'a,b,c',
             'RFM_KEEP_STAGE_FILES': 'no',
             'RFM_GIT_TIMEOUT': '0.3'
     }):
         site_config = rt.runtime().site_config
-        options = extended_parser.parse_args(
-            ['--timestamp=%FT%T', '--nocolor']
-        )
+        options = extended_parser.parse_args(['--nocolor', '--timestamp'])
         options.update_config(site_config)
         assert site_config.get('general/0/check_search_recursive') is False
-        assert site_config.get('general/0/timestamp_dirs') == '%FT%T'
+        assert site_config.get('general/0/timestamp_dirs') == r'%F'
         assert site_config.get('general/0/non_default_craype') is True
         assert site_config.get('systems/0/prefix') == str(tmp_path)
         assert site_config.get('general/0/colorize') is False

From 9b68376fb8747b8446b5cbbb069adf5fe82191c7 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 10 Jul 2024 15:46:47 +0300
Subject: [PATCH 035/128] Synchronize storing to the DB and saving the report
 file

---
 reframe/frontend/reporting/__init__.py | 12 +++++++++--
 reframe/frontend/reporting/storage.py  | 28 +++++++++++++-------------
 requirements.txt                       |  3 +++
 3 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index e808dd4bf6..b8e0d63538 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -15,6 +15,7 @@
 import socket
 import time
 from collections.abc import Hashable
+from filelock import FileLock
 
 import reframe as rfm
 import reframe.utility.jsonext as jsonext
@@ -362,7 +363,7 @@ def update_run_stats(self, stats):
             'num_skipped': self.__report['runs'][-1]['num_skipped']
         })
 
-    def save(self, filename, compress=False, link_to_last=True):
+    def _save(self, filename, compress, link_to_last):
         filename = _expand_report_filename(filename, newfile=True)
         with open(filename, 'w') as fp:
             if compress:
@@ -389,10 +390,17 @@ def save(self, filename, compress=False, link_to_last=True):
                 else:
                     raise ReframeError('path exists and is not a symlink')
 
+    def save(self, filename, compress=False, link_to_last=True):
+        prefix = os.path.dirname(filename) or '.'
+        with FileLock(os.path.join(prefix, '.report.lock')):
+            self._save(filename, compress, link_to_last)
+
     def store(self):
         '''Store the report in the results storage.'''
 
-        return StorageBackend.default().store(self, self.filename)
+        prefix = os.path.dirname(self.filename) or '.'
+        with FileLock(os.path.join(prefix, '.db.lock')):
+            return StorageBackend.default().store(self, self.filename)
 
     def generate_xml_report(self):
         '''Generate a JUnit report from a standard ReFrame JSON report.'''
diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 0422539713..af99b6ff7b 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -93,23 +93,23 @@ def _db_store_report(self, conn, report, report_file_path):
         session_uuid = datetime.fromtimestamp(session_start_unix).strftime(
             r'%Y%m%dT%H%M%S%z'
         )
+        cursor = conn.execute(
+            'INSERT INTO sessions VALUES('
+            ':session_id, :session_uuid, '
+            ':session_start_unix, :session_end_unix, '
+            ':json_blob, :report_file)',
+            {
+                'session_id': None,
+                'session_uuid': session_uuid,
+                'session_start_unix': session_start_unix,
+                'session_end_unix': session_end_unix,
+                'json_blob': jsonext.dumps(report),
+                'report_file': report_file_path
+            }
+        )
         for run_idx, run in enumerate(report['runs']):
             for test_idx, testcase in enumerate(run['testcases']):
                 sys, part = testcase['system'], testcase['partition']
-                cursor = conn.execute(
-                    'INSERT INTO sessions VALUES('
-                    ':session_id, :session_uuid, '
-                    ':session_start_unix, :session_end_unix, '
-                    ':json_blob, :report_file)',
-                    {
-                        'session_id': None,
-                        'session_uuid': session_uuid,
-                        'session_start_unix': session_start_unix,
-                        'session_end_unix': session_end_unix,
-                        'json_blob': jsonext.dumps(report),
-                        'report_file': report_file_path
-                    }
-                )
                 conn.execute(
                     'INSERT INTO testcases VALUES('
                     ':name, :system, :partition, :environ, '
diff --git a/requirements.txt b/requirements.txt
index e6df345eb9..790dd8cc60 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,9 @@
 archspec==0.2.4
 argcomplete==3.1.2; python_version < '3.8'
 argcomplete==3.4.0; python_version >= '3.8'
+filelock==3.4.1; python_version == '3.6'
+filelock==3.12.2; python_version == '3.7'
+filelock==3.15.4; python_version >= '3.8'
 importlib_metadata==4.0.1; python_version < '3.8'
 jsonschema==3.2.0
 lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64'

From 1108fcf97aedd47b509144daf11a28cfacaa59b5 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 10 Jul 2024 16:28:22 +0300
Subject: [PATCH 036/128] Use session ID as UUID

---
 reframe/frontend/reporting/storage.py | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index af99b6ff7b..c626ff8f1a 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -70,7 +70,6 @@ def _db_create(self):
         with sqlite3.connect(self.__db_file) as conn:
             conn.execute('CREATE TABLE IF NOT EXISTS sessions('
                          'id INTEGER PRIMARY KEY, '
-                         'session_uuid TEXT, '
                          'session_start_unix REAL, '
                          'session_end_unix REAL, '
                          'json_blob TEXT, '
@@ -90,23 +89,19 @@ def _db_create(self):
     def _db_store_report(self, conn, report, report_file_path):
         session_start_unix = report['session_info']['time_start_unix']
         session_end_unix = report['session_info']['time_end_unix']
-        session_uuid = datetime.fromtimestamp(session_start_unix).strftime(
-            r'%Y%m%dT%H%M%S%z'
-        )
         cursor = conn.execute(
             'INSERT INTO sessions VALUES('
-            ':session_id, :session_uuid, '
-            ':session_start_unix, :session_end_unix, '
+            ':session_id, :session_start_unix, :session_end_unix, '
             ':json_blob, :report_file)',
             {
                 'session_id': None,
-                'session_uuid': session_uuid,
                 'session_start_unix': session_start_unix,
                 'session_end_unix': session_end_unix,
                 'json_blob': jsonext.dumps(report),
                 'report_file': report_file_path
             }
         )
+        session_id = cursor.lastrowid
         for run_idx, run in enumerate(report['runs']):
             for test_idx, testcase in enumerate(run['testcases']):
                 sys, part = testcase['system'], testcase['partition']
@@ -123,13 +118,13 @@ def _db_store_report(self, conn, report, report_file_path):
                         'job_completion_time_unix': testcase[
                             'job_completion_time_unix'
                         ],
-                        'session_id': cursor.lastrowid,
+                        'session_id': session_id,
                         'run_index': run_idx,
                         'test_index': test_idx
                     }
                 )
 
-        return session_uuid
+        return session_id
 
     def store(self, report, report_file):
         with sqlite3.connect(self._db_file()) as conn:
@@ -157,7 +152,7 @@ def _fetch_testcases_raw(self, condition):
     def fetch_session_time_period(self, session_uuid):
         with sqlite3.connect(self._db_file()) as conn:
             query = ('SELECT session_start_unix, session_end_unix '
-                     f'FROM sessions WHERE session_uuid == "{session_uuid}" '
+                     f'FROM sessions WHERE id == "{session_uuid}" '
                      'LIMIT 1')
             getlogger().debug(query)
             results = conn.execute(query).fetchall()

From 3df63f23d3bf7aeda376ceb136b2e84ab615a699 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 10 Jul 2024 16:36:49 +0300
Subject: [PATCH 037/128] Remove unused import + fix `setup.cfg`

---
 reframe/frontend/reporting/storage.py | 1 -
 setup.cfg                             | 3 +++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index c626ff8f1a..089b19c55f 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -6,7 +6,6 @@
 import abc
 import os
 import sqlite3
-from datetime import datetime
 
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
diff --git a/setup.cfg b/setup.cfg
index 974e38855d..3dc33868e7 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -29,6 +29,9 @@ install_requires =
     archspec >= 0.2.4
     argcomplete
     argcomplete <= 3.1.2; python_version < '3.8'
+    filelock
+    filelock<=3.12.2; python_version == '3.7'
+    filelock<=3.4.1; python_version == '3.6'
     jsonschema
     lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64'
     lxml==5.2.2; python_version >= '3.8' or platform_machine != 'aarch64'

From aa75c377a1f43ac45162b4a08f4bfba41bcbcd63 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 10 Jul 2024 20:47:00 +0300
Subject: [PATCH 038/128] Group all actions in a required mutually exclusive
 option group

---
 reframe/frontend/argparse.py | 44 +++++++++++++++++++++++++++++++---
 reframe/frontend/cli.py      | 46 ++++++++++++------------------------
 unittests/test_argparser.py  |  7 ++++++
 unittests/test_cli.py        | 13 ++++++----
 4 files changed, 72 insertions(+), 38 deletions(-)

diff --git a/reframe/frontend/argparse.py b/reframe/frontend/argparse.py
index ec6c667142..93a4070f36 100644
--- a/reframe/frontend/argparse.py
+++ b/reframe/frontend/argparse.py
@@ -170,6 +170,12 @@ def __getattr__(self, name):
 
         return getattr(self._holder, name)
 
+    def __setattr__(self, name, value):
+        if name.startswith('_'):
+            super().__setattr__(name, value)
+        else:
+            setattr(self._holder, name, value)
+
     def add_argument(self, *flags, **kwargs):
         try:
             opt_name = kwargs['dest']
@@ -254,6 +260,14 @@ def add_argument_group(self, *args, **kwargs):
         self._groups.append(group)
         return group
 
+    def add_mutually_exclusive_group(self, *args, **kwargs):
+        group = _ArgumentGroup(
+            self._holder.add_mutually_exclusive_group(*args, **kwargs),
+            self._option_map
+        )
+        self._groups.append(group)
+        return group
+
     def _resolve_attr(self, attr, namespaces):
         for ns in namespaces:
             if ns is None:
@@ -269,7 +283,7 @@ def _update_defaults(self):
         for g in self._groups:
             self._defaults.__dict__.update(g._defaults.__dict__)
 
-    def parse_args(self, args=None, namespace=None):
+    def parse_args(self, args=None, namespace=None, suppress_required=False):
         '''Convert argument strings to objects and return them as attributes of
         a namespace.
 
@@ -281,7 +295,30 @@ def parse_args(self, args=None, namespace=None):
         for it will be looked up first in `namespace` and if not found there,
         it will be assigned the default value as specified in its corresponding
         `add_argument()` call. If no default value was specified either, the
-        attribute will be set to `None`.'''
+        attribute will be set to `None`.
+
+        If `suppress_required` is true, required mutually-exclusive groups will
+        be treated as optional for this parsing operation.
+        '''
+
+        class suppress_required_groups:
+            '''Temporarily suppress required groups if `suppress_required`
+            is true.'''
+            def __init__(this):
+                this._changed_grp = []
+
+            def __enter__(this):
+                if suppress_required:
+                    for grp in self._groups:
+                        if hasattr(grp, 'required') and grp.required:
+                            this._changed_grp.append(grp)
+                            grp.required = False
+
+                return this
+
+            def __exit__(this, *args, **kwargs):
+                for grp in this._changed_grp:
+                    grp.required = True
 
         # Enable auto-completion
         argcomplete.autocomplete(self._holder)
@@ -291,7 +328,8 @@ def parse_args(self, args=None, namespace=None):
         # newly parsed options to completely override any options defined in
         # namespace. The implementation of `argparse.ArgumentParser` does not
         # do this in options with an 'append' action.
-        options = self._holder.parse_args(args, None)
+        with suppress_required_groups():
+            options = self._holder.parse_args(args, None)
 
         # Check if namespace refers to our namespace and take the cmd options
         # namespace suitable for ArgumentParser
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index faa0a40674..0c03ceccf7 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -211,6 +211,7 @@ def calc_verbosity(site_config, quiesce):
 def main():
     # Setup command line options
     argparser = argparse.ArgumentParser()
+    action_options = argparser.add_mutually_exclusive_group(required=True)
     output_options = argparser.add_argument_group(
         'Options controlling ReFrame output'
     )
@@ -220,9 +221,6 @@ def main():
     select_options = argparser.add_argument_group(
         'Options for selecting checks'
     )
-    action_options = argparser.add_argument_group(
-        'Options controlling actions'
-    )
     run_options = argparser.add_argument_group(
         'Options controlling the execution of checks'
     )
@@ -374,7 +372,6 @@ def main():
         help=('Generate into FILE a Gitlab CI pipeline '
               'for the selected tests and exit'),
     )
-
     action_options.add_argument(
         '--describe', action='store_true',
         help='Give full details on the selected tests'
@@ -400,6 +397,18 @@ def main():
         '--dry-run', action='store_true',
         help='Dry run the tests without submitting them for execution'
     )
+    action_options.add_argument(
+        '--performance-compare', metavar='CMPSPEC', action='store',
+        help='Compare past performance results'
+    )
+    action_options.add_argument(
+        '--show-config', action='store', nargs='?', const='all',
+        metavar='PARAM',
+        help='Print the value of configuration parameter PARAM and exit'
+    )
+    action_options.add_argument(
+        '-V', '--version', action='version', version=osext.reframe_version()
+    )
 
     # Run options
     run_options.add_argument(
@@ -538,10 +547,6 @@ def main():
     )
 
     # Reporting options
-    reporting_options.add_argument(
-        '--performance-compare', metavar='CMPSPEC', action='store',
-        help='Compare past performance results'
-    )
     reporting_options.add_argument(
         '--performance-report', action='store', nargs='?',
         const=argparse.CONST_DEFAULT,
@@ -572,22 +577,10 @@ def main():
         help='Disable coloring of output',
         envvar='RFM_COLORIZE', configvar='general/colorize'
     )
-    misc_options.add_argument(
-        '--show-config', action='store', nargs='?', const='all',
-        metavar='PARAM',
-        help='Print the value of configuration parameter PARAM and exit'
-    )
-    misc_options.add_argument(
-        '--index-db', action='store_true',
-        help='Index old job reports in the database',
-    )
     misc_options.add_argument(
         '--system', action='store', help='Load configuration for SYSTEM',
         envvar='RFM_SYSTEM'
     )
-    misc_options.add_argument(
-        '-V', '--version', action='version', version=osext.reframe_version()
-    )
     misc_options.add_argument(
         '-v', '--verbose', action='count',
         help='Increase verbosity level of output',
@@ -838,7 +831,8 @@ def restrict_logging():
                     itertools.chain.from_iterable(shlex.split(m)
                                                   for m in mode_args))
                 # Parse the mode's options and reparse the command-line
-                options = argparser.parse_args(mode_args)
+                options = argparser.parse_args(mode_args,
+                                               suppress_required=True)
                 options = argparser.parse_args(namespace=options.cmd_options)
                 options.update_config(site_config)
 
@@ -1310,16 +1304,6 @@ def _sort_testcases(testcases):
             else:
                 sys.exit(0)
 
-        if not options.run and not options.dry_run:
-            printer.error("No action option specified. Available options:\n"
-                          "  - `-l'/`-L' for listing\n"
-                          "  - `-r' for running\n"
-                          "  - `--dry-run' for dry running\n"
-                          "  - `--list-tags' for listing unique test tags\n"
-                          "  - `--ci-generate' for generating a CI pipeline\n"
-                          f"Try `{argparser.prog} -h' for more options.")
-            sys.exit(1)
-
         # Manipulate ReFrame's environment
         if site_config.get('general/0/purge_environment'):
             rt.modules_system.unload_all()
diff --git a/unittests/test_argparser.py b/unittests/test_argparser.py
index cd6fffea21..6be36f3836 100644
--- a/unittests/test_argparser.py
+++ b/unittests/test_argparser.py
@@ -203,3 +203,10 @@ def test_envvar_option(default_exec_ctx, extended_parser):
 def test_envvar_option_default_val(default_exec_ctx, extended_parser):
     options = extended_parser.parse_args([])
     assert options.env_option == 'bar'
+
+
+def test_suppress_required(argparser):
+    group = argparser.add_mutually_exclusive_group(required=True)
+    group.add_argument('--foo', action='store_true')
+    group.add_argument('--bar', action='store_true')
+    argparser.parse_args([], suppress_required=True)
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 0f9f10880d..15cc95ca7b 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -528,6 +528,7 @@ def test_execution_modes(run_reframe, run_action):
     returncode, stdout, stderr = run_reframe(
         mode='unittest', action=run_action
     )
+    assert returncode == 0
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
     assert 'FAILED' not in stdout
@@ -747,7 +748,8 @@ def test_show_config_all(run_reframe):
     # Just make sure that this option does not make the frontend crash
     returncode, stdout, stderr = run_reframe(
         more_options=['--show-config'],
-        system='testsys'
+        system='testsys',
+        action=None
     )
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
@@ -758,7 +760,8 @@ def test_show_config_param(run_reframe):
     # Just make sure that this option does not make the frontend crash
     returncode, stdout, stderr = run_reframe(
         more_options=['--show-config=systems'],
-        system='testsys'
+        system='testsys',
+        action=None
     )
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
@@ -769,7 +772,8 @@ def test_show_config_unknown_param(run_reframe):
     # Just make sure that this option does not make the frontend crash
     returncode, stdout, stderr = run_reframe(
         more_options=['--show-config=foo'],
-        system='testsys'
+        system='testsys',
+        action=None
     )
     assert 'no such configuration parameter found' in stdout
     assert 'Traceback' not in stdout
@@ -780,7 +784,8 @@ def test_show_config_unknown_param(run_reframe):
 def test_show_config_null_param(run_reframe):
     returncode, stdout, stderr = run_reframe(
         more_options=['--show-config=general/report_junit'],
-        system='testsys'
+        system='testsys',
+        action=None
     )
     assert 'null' in stdout
     assert 'Traceback' not in stdout

From 8df09cec51dac132ee590909ee51bc54f1a79cb2 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Wed, 10 Jul 2024 23:58:54 +0300
Subject: [PATCH 039/128] Add `--detect-host-topology` to actions

---
 reframe/frontend/cli.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 0c03ceccf7..342b167701 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -376,6 +376,16 @@ def main():
         '--describe', action='store_true',
         help='Give full details on the selected tests'
     )
+    action_options.add_argument(
+        '--detect-host-topology', metavar='FILE', action='store',
+        nargs='?', const='-',
+        help=('Detect the local host topology and exit, '
+              'optionally saving it in FILE')
+    )
+    action_options.add_argument(
+        '--dry-run', action='store_true',
+        help='Dry run the tests without submitting them for execution'
+    )
     action_options.add_argument(
         '-L', '--list-detailed', nargs='?', const='T', choices=['C', 'T'],
         help=('List the selected tests (T) or the concretized test cases (C) '
@@ -389,18 +399,14 @@ def main():
         '--list-tags', action='store_true',
         help='List the unique tags found in the selected tests and exit'
     )
-    action_options.add_argument(
-        '-r', '--run', action='store_true',
-        help='Run the selected checks'
-    )
-    action_options.add_argument(
-        '--dry-run', action='store_true',
-        help='Dry run the tests without submitting them for execution'
-    )
     action_options.add_argument(
         '--performance-compare', metavar='CMPSPEC', action='store',
         help='Compare past performance results'
     )
+    action_options.add_argument(
+        '-r', '--run', action='store_true',
+        help='Run the selected checks'
+    )
     action_options.add_argument(
         '--show-config', action='store', nargs='?', const='all',
         metavar='PARAM',
@@ -563,12 +569,6 @@ def main():
         help='Set configuration file',
         envvar='RFM_CONFIG_FILES :'
     )
-    misc_options.add_argument(
-        '--detect-host-topology', metavar='FILE', action='store',
-        nargs='?', const='-',
-        help=('Detect the local host topology and exit, '
-              'optionally saving it in FILE')
-    )
     misc_options.add_argument(
         '--failure-stats', action='store_true', help='Print failure statistics'
     )

From 81a9513b4bb83eb0cdbf4ebb107f855277009653 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Thu, 11 Jul 2024 00:36:10 +0300
Subject: [PATCH 040/128] Fix unit tests

---
 unittests/test_cli.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 15cc95ca7b..6585dae0f6 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -1116,7 +1116,8 @@ def test_detect_host_topology(run_reframe):
     from reframe.utility.cpuinfo import cpuinfo
 
     returncode, stdout, stderr = run_reframe(
-        more_options=['--detect-host-topology']
+        more_options=['--detect-host-topology'],
+        action=None
     )
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
@@ -1129,7 +1130,8 @@ def test_detect_host_topology_file(run_reframe, tmp_path):
 
     topo_file = tmp_path / 'topo.json'
     returncode, stdout, stderr = run_reframe(
-        more_options=[f'--detect-host-topology={topo_file}']
+        more_options=[f'--detect-host-topology={topo_file}'],
+        action=None
     )
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr

From 5f6e12d491f61589aa9b6eb15ced3ed6b4fab42a Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 11 Jul 2024 13:35:27 +0300
Subject: [PATCH 041/128] Fetch test cases from session exclusively

---
 reframe/frontend/reporting/__init__.py | 20 +++++++++++++++-----
 reframe/frontend/reporting/storage.py  | 16 ++++++++++++++++
 reframe/frontend/reporting/utility.py  | 23 ++++++++++++++++-------
 3 files changed, 47 insertions(+), 12 deletions(-)

diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index b8e0d63538..70c44372fd 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -555,7 +555,7 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
 
 def performance_compare(cmp, report=None):
     match = parse_cmp_spec(cmp)
-    if match.period_base is None:
+    if match.period_base is None and match.session_base is None:
         if report is None:
             raise ValueError('report cannot be `None` '
                              'for current run comparisons')
@@ -564,14 +564,24 @@ def performance_compare(cmp, report=None):
         except IndexError:
             tcs_base = []
 
-    else:
+    elif match.period_base is not None:
         tcs_base = StorageBackend.default().fetch_testcases_time_period(
             *match.period_base
         )
+    else:
+        tcs_base = StorageBackend.default().fetch_testcases_from_session(
+            match.session_base
+        )
+
+    if match.period_target:
+        tcs_target = StorageBackend.default().fetch_testcases_time_period(
+            *match.period_target
+        )
+    else:
+        tcs_target = StorageBackend.default().fetch_testcases_from_session(
+            match.session_target
+        )
 
-    tcs_target = StorageBackend.default().fetch_testcases_time_period(
-        *match.period_target
-    )
     return compare_testcase_data(tcs_base, tcs_target, match.aggregator,
                                  match.aggregator, match.extra_groups,
                                  match.extra_cols)
diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 089b19c55f..ba97c37e76 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -166,3 +166,19 @@ def fetch_testcases_time_period(self, ts_start, ts_end):
             f'job_completion_time_unix <= {ts_end}) '
             'ORDER BY job_completion_time_unix'
         )
+
+    def fetch_testcases_from_session(self, session_uuid):
+        with sqlite3.connect(self._db_file()) as conn:
+            query = f'SELECT json_blob from sessions WHERE id=={session_uuid}'
+            getlogger().debug(query)
+            results = conn.execute(query).fetchall()
+
+        if not results:
+            return []
+
+        testcases = []
+        session_info = jsonext.loads(results[0][0])
+        for run in session_info['runs']:
+            testcases += run['testcases']
+
+        return testcases
diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py
index 05cc0cf337..6f2ba477af 100644
--- a/reframe/frontend/reporting/utility.py
+++ b/reframe/frontend/reporting/utility.py
@@ -169,11 +169,22 @@ def _parse_aggregation(s):
     return Aggregator.create(op), _parse_extra_cols(extra_groups)
 
 
-_Match = namedtuple('_Match', ['period_base', 'period_target',
-                               'aggregator', 'extra_groups', 'extra_cols'])
+_Match = namedtuple('_Match',
+                    ['period_base', 'period_target',
+                     'session_base', 'session_target',
+                     'aggregator', 'extra_groups', 'extra_cols'])
 
 
 def parse_cmp_spec(spec):
+    def _parse_period_spec(s):
+        if s is None:
+            return None, None
+
+        if s.startswith('^'):
+            return s[1:], None
+
+        return None, _parse_time_period(s)
+
     parts = spec.split('/')
     if len(parts) == 3:
         period_base, period_target, aggr, cols = None, *parts
@@ -182,11 +193,9 @@ def parse_cmp_spec(spec):
     else:
         raise ValueError(f'invalid cmp spec: {spec}')
 
-    if period_base is not None:
-        period_base = _parse_time_period(period_base)
-
-    period_target = _parse_time_period(period_target)
+    session_base, period_base = _parse_period_spec(period_base)
+    session_target, period_target = _parse_period_spec(period_target)
     aggr_fn, extra_groups = _parse_aggregation(aggr)
     extra_cols = _parse_extra_cols(cols)
-    return _Match(period_base, period_target,
+    return _Match(period_base, period_target, session_base, session_target,
                   aggr_fn, extra_groups, extra_cols)

From 13504563bb61fe6d36fe0669ae8df00207fa6d91 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 11 Jul 2024 15:53:17 +0300
Subject: [PATCH 042/128] Allow grouping testcasesby runid and session_id`

---
 reframe/frontend/reporting/storage.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index ba97c37e76..abeb4e6b87 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -129,6 +129,13 @@ def store(self, report, report_file):
         with sqlite3.connect(self._db_file()) as conn:
             return self._db_store_report(conn, report, report_file)
 
+    def _enrich_testcase(self, tc, session_uuid, runid):
+        '''Enrich testcase record with more information'''
+
+        tc['runid'] = runid
+        tc['session_uuid'] = session_uuid
+        return tc
+
     def _fetch_testcases_raw(self, condition):
         with sqlite3.connect(self._db_file()) as conn:
             query = ('SELECT session_id, run_index, test_index, json_blob '
@@ -142,9 +149,10 @@ def _fetch_testcases_raw(self, condition):
         sessions = {}
         for session_id, run_index, test_index, json_blob in results:
             report = jsonext.loads(sessions.setdefault(session_id, json_blob))
-            testcases.append(
-                report['runs'][run_index]['testcases'][test_index]
-            )
+            testcases.append(self._enrich_testcase(
+                report['runs'][run_index]['testcases'][test_index],
+                session_id, run_index
+            ))
 
         return testcases
 
@@ -176,9 +184,9 @@ def fetch_testcases_from_session(self, session_uuid):
         if not results:
             return []
 
-        testcases = []
         session_info = jsonext.loads(results[0][0])
-        for run in session_info['runs']:
-            testcases += run['testcases']
-
-        return testcases
+        return [
+            self._enrich_testcase(tc, session_uuid, runid)
+            for runid, run in enumerate(session_info['runs'])
+            for tc in run['testcases']
+        ]

From 92f91362a03b52bbae65a10b34eed1d6984de934 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 11 Jul 2024 18:50:23 +0300
Subject: [PATCH 043/128] Use proper UUIDs as session IDs

---
 reframe/frontend/reporting/__init__.py |  7 +++-
 reframe/frontend/reporting/storage.py  | 54 +++++++++++---------------
 reframe/schemas/runreport.json         |  5 ++-
 3 files changed, 32 insertions(+), 34 deletions(-)

diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 70c44372fd..05a85c4f1b 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -14,6 +14,7 @@
 import re
 import socket
 import time
+import uuid
 from collections.abc import Hashable
 from filelock import FileLock
 
@@ -213,7 +214,8 @@ def __init__(self):
         self.__report = {
             'session_info': {
                 'data_version': DATA_VERSION,
-                'hostname': socket.gethostname()
+                'hostname': socket.gethostname(),
+                'uuid': str(uuid.uuid4())
             },
             'runs': [],
             'restored_cases': []
@@ -255,6 +257,7 @@ def update_timestamps(self, ts_start, ts_end):
         })
 
     def update_run_stats(self, stats):
+        session_uuid = self.__report['session_info']['uuid']
         for runid, tasks in stats.runs():
             testcases = []
             num_failures = 0
@@ -288,7 +291,9 @@ def update_run_stats(self, stats):
                     'job_stdout': None,
                     'partition': partition.name,
                     'result': t.result,
+                    'runid': runid,
                     'scheduler': partition.scheduler.registered_name,
+                    'session_uuid': session_uuid,
                     'time_compile': t.duration('compile_complete'),
                     'time_performance': t.duration('performance'),
                     'time_run': t.duration('run_complete'),
diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index abeb4e6b87..9b82f5c9d9 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -68,7 +68,7 @@ def _db_create(self):
         )
         with sqlite3.connect(self.__db_file) as conn:
             conn.execute('CREATE TABLE IF NOT EXISTS sessions('
-                         'id INTEGER PRIMARY KEY, '
+                         'uuid TEXT PRIMARY KEY, '
                          'session_start_unix REAL, '
                          'session_end_unix REAL, '
                          'json_blob TEXT, '
@@ -79,28 +79,28 @@ def _db_create(self):
                          'partition TEXT,'
                          'environ TEXT,'
                          'job_completion_time_unix REAL,'
-                         'session_id INTEGER,'
+                         'session_uuid TEXT,'
                          'run_index INTEGER,'
                          'test_index INTEGER,'
-                         'FOREIGN KEY(session_id) '
-                         'REFERENCES sessions(session_id))')
+                         'FOREIGN KEY(session_uuid) '
+                         'REFERENCES sessions(uuid))')
 
     def _db_store_report(self, conn, report, report_file_path):
         session_start_unix = report['session_info']['time_start_unix']
         session_end_unix = report['session_info']['time_end_unix']
-        cursor = conn.execute(
+        session_uuid = report['session_info']['uuid']
+        conn.execute(
             'INSERT INTO sessions VALUES('
-            ':session_id, :session_start_unix, :session_end_unix, '
+            ':uuid, :session_start_unix, :session_end_unix, '
             ':json_blob, :report_file)',
             {
-                'session_id': None,
+                'uuid': session_uuid,
                 'session_start_unix': session_start_unix,
                 'session_end_unix': session_end_unix,
                 'json_blob': jsonext.dumps(report),
                 'report_file': report_file_path
             }
         )
-        session_id = cursor.lastrowid
         for run_idx, run in enumerate(report['runs']):
             for test_idx, testcase in enumerate(run['testcases']):
                 sys, part = testcase['system'], testcase['partition']
@@ -108,7 +108,7 @@ def _db_store_report(self, conn, report, report_file_path):
                     'INSERT INTO testcases VALUES('
                     ':name, :system, :partition, :environ, '
                     ':job_completion_time_unix, '
-                    ':session_id, :run_index, :test_index)',
+                    ':session_uuid, :run_index, :test_index)',
                     {
                         'name': testcase['name'],
                         'system': sys,
@@ -117,29 +117,22 @@ def _db_store_report(self, conn, report, report_file_path):
                         'job_completion_time_unix': testcase[
                             'job_completion_time_unix'
                         ],
-                        'session_id': session_id,
+                        'session_uuid': session_uuid,
                         'run_index': run_idx,
                         'test_index': test_idx
                     }
                 )
 
-        return session_id
+        return session_uuid
 
     def store(self, report, report_file):
         with sqlite3.connect(self._db_file()) as conn:
             return self._db_store_report(conn, report, report_file)
 
-    def _enrich_testcase(self, tc, session_uuid, runid):
-        '''Enrich testcase record with more information'''
-
-        tc['runid'] = runid
-        tc['session_uuid'] = session_uuid
-        return tc
-
     def _fetch_testcases_raw(self, condition):
         with sqlite3.connect(self._db_file()) as conn:
-            query = ('SELECT session_id, run_index, test_index, json_blob '
-                     'FROM testcases JOIN sessions ON session_id==id '
+            query = ('SELECT session_uuid, run_index, test_index, json_blob '
+                     'FROM testcases JOIN sessions ON session_uuid == uuid '
                      f'WHERE {condition}')
             getlogger().debug(query)
             results = conn.execute(query).fetchall()
@@ -147,19 +140,19 @@ def _fetch_testcases_raw(self, condition):
         # Retrieve files
         testcases = []
         sessions = {}
-        for session_id, run_index, test_index, json_blob in results:
-            report = jsonext.loads(sessions.setdefault(session_id, json_blob))
-            testcases.append(self._enrich_testcase(
+        for session_uuid, run_index, test_index, json_blob in results:
+            report = jsonext.loads(sessions.setdefault(session_uuid,
+                                                       json_blob))
+            testcases.append(
                 report['runs'][run_index]['testcases'][test_index],
-                session_id, run_index
-            ))
+            )
 
         return testcases
 
     def fetch_session_time_period(self, session_uuid):
         with sqlite3.connect(self._db_file()) as conn:
             query = ('SELECT session_start_unix, session_end_unix '
-                     f'FROM sessions WHERE id == "{session_uuid}" '
+                     f'FROM sessions WHERE uuid == "{session_uuid}" '
                      'LIMIT 1')
             getlogger().debug(query)
             results = conn.execute(query).fetchall()
@@ -177,7 +170,8 @@ def fetch_testcases_time_period(self, ts_start, ts_end):
 
     def fetch_testcases_from_session(self, session_uuid):
         with sqlite3.connect(self._db_file()) as conn:
-            query = f'SELECT json_blob from sessions WHERE id=={session_uuid}'
+            query = ('SELECT json_blob from sessions '
+                     f'WHERE uuid == "{session_uuid}"')
             getlogger().debug(query)
             results = conn.execute(query).fetchall()
 
@@ -185,8 +179,4 @@ def fetch_testcases_from_session(self, session_uuid):
             return []
 
         session_info = jsonext.loads(results[0][0])
-        return [
-            self._enrich_testcase(tc, session_uuid, runid)
-            for runid, run in enumerate(session_info['runs'])
-            for tc in run['testcases']
-        ]
+        return [tc for run in session_info['runs'] for tc in run['testcases']]
diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json
index 85a902e6ec..9a857a14d5 100644
--- a/reframe/schemas/runreport.json
+++ b/reframe/schemas/runreport.json
@@ -87,17 +87,20 @@
                 "num_skipped": {"type": "number"},
                 "prefix_output": {"type": "string"},
                 "prefix_stage": {"type": "string"},
+                "runid": {"type": "number"},
+                "session_uuid": {"type": "string"},
                 "time_elapsed": {"type": "number"},
                 "time_end": {"type": "string"},
                 "time_end_unix": {"type": "number"},
                 "time_start": {"type": "string"},
                 "time_start_unix": {"type": "number"},
                 "user": {"type": "string"},
+                "uuid": {"type": "string"},
                 "version": {"type": "string"},
                 "workdir": {"type": "string"}
             },
             "required": ["data_version", "hostname",
-                         "time_elapsed", "time_end", "time_start"]
+                         "time_elapsed", "time_end", "time_start", "uuid"]
         },
         "restored_cases": {
             "type": "array",

From d000d632ff82f63584656c4eb0a7294ff566ca71 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 12 Jul 2024 19:04:07 +0300
Subject: [PATCH 044/128] New command-line options for interacting with stored
 sessions

---
 reframe/frontend/cli.py                |  98 +++++++++++++++++++++--
 reframe/frontend/printer.py            |  12 +--
 reframe/frontend/reporting/__init__.py | 103 ++++++++++++++++++++++---
 reframe/frontend/reporting/storage.py  |  48 +++++++++---
 reframe/frontend/reporting/utility.py  |   4 +-
 reframe/schemas/runreport.json         |   3 +-
 6 files changed, 234 insertions(+), 34 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 342b167701..b6f2f6e73a 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -207,6 +207,29 @@ def calc_verbosity(site_config, quiesce):
     return curr_verbosity - quiesce
 
 
+class exit_gracefuly_on_error:
+    def __init__(self, message, logger=None, exceptions=None, exitcode=1):
+        self.__message = message
+        self.__logger = logger or PrettyPrinter()
+        self.__exceptions = exceptions or (Exception,)
+        self.__exitcode = exitcode
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is SystemExit:
+            # Allow users to exit inside the context manager
+            return
+
+        if isinstance(exc_val, self.__exceptions):
+            self.__logger.error(f'{self.__message}: {exc_val}')
+            self.__logger.verbose(
+                ''.join(traceback.format_exception(exc_type, exc_val, exc_tb))
+            )
+            sys.exit(self.__exitcode)
+
+
 @logging.time_function_noexit
 def main():
     # Setup command line options
@@ -366,16 +389,28 @@ def main():
         help='Select checks that satisfy the expression EXPR'
     )
 
-    # Action options
     action_options.add_argument(
         '--ci-generate', action='store', metavar='FILE',
         help=('Generate into FILE a Gitlab CI pipeline '
               'for the selected tests and exit'),
     )
+    action_options.add_argument(
+        '--delete-stored-session', action='store', metavar='SESSION_UUID',
+        help='Delete stored session'
+    )
     action_options.add_argument(
         '--describe', action='store_true',
         help='Give full details on the selected tests'
     )
+    action_options.add_argument(
+        '--describe-stored-session', action='store', metavar='SESSION_UUID',
+        help='Get detailed session information in JSON'
+    )
+    action_options.add_argument(
+        '--describe-stored-testcases', action='store',
+        metavar='SESSION_UUID|PERIOD',
+        help='Get detailed test case information in JSON'
+    )
     action_options.add_argument(
         '--detect-host-topology', metavar='FILE', action='store',
         nargs='?', const='-',
@@ -391,6 +426,15 @@ def main():
         help=('List the selected tests (T) or the concretized test cases (C) '
               'providing more details')
     )
+    action_options.add_argument(
+        '--list-stored-sessions', action='store_true',
+        help='List stored session'
+    )
+    action_options.add_argument(
+        '--list-stored-testcases', action='store',
+        metavar='SESSION_UUID|PERIOD',
+        help='List stored testcases by session or time period'
+    )
     action_options.add_argument(
         '-l', '--list', nargs='?', const='T', choices=['C', 'T'],
         help='List the selected tests (T) or the concretized test cases (C)'
@@ -734,7 +778,10 @@ def restrict_logging():
         '''
 
         if (options.show_config or
-            options.detect_host_topology or options.describe):
+            options.detect_host_topology or
+            options.describe or
+            options.describe_stored_session or
+            options.describe_stored_testcases):
             logging.getlogger().setLevel(logging.ERROR)
             return True
         else:
@@ -882,6 +929,47 @@ def restrict_logging():
         printer.info(logfiles_message())
         sys.exit(1)
 
+    if options.list_stored_sessions:
+        with exit_gracefuly_on_error('failed to retrieve session data',
+                                     printer):
+            printer.table(reporting.session_data())
+            sys.exit(0)
+
+    if options.list_stored_testcases:
+        with exit_gracefuly_on_error('failed to retrieve test case data',
+                                     printer):
+            printer.table(reporting.testcase_data(
+                options.list_stored_testcases
+            ))
+            sys.exit(0)
+
+    if options.describe_stored_session:
+        # Restore logging level
+        printer.setLevel(logging.INFO)
+        with exit_gracefuly_on_error('failed to retrieve session data',
+                                     printer):
+            printer.info(jsonext.dumps(reporting.session_info(
+                options.describe_stored_session
+            ), indent=2))
+            sys.exit(0)
+
+    if options.describe_stored_testcases:
+        # Restore logging level
+        printer.setLevel(logging.INFO)
+        with exit_gracefuly_on_error('failed to retrieve test case data',
+                                     printer):
+            printer.info(jsonext.dumps(reporting.testcase_info(
+                options.describe_stored_testcases
+            ), indent=2))
+            sys.exit(0)
+
+    if options.delete_stored_session:
+        session_uuid = options.delete_stored_session
+        with exit_gracefuly_on_error('failed to delete session', printer):
+            reporting.delete_session(session_uuid)
+            printer.info(f'Session {session_uuid} deleted successfully.')
+            sys.exit(0)
+
     # Show configuration after everything is set up
     if options.show_config:
         # Restore logging level
@@ -1296,7 +1384,7 @@ def _sort_testcases(testcases):
         if options.performance_compare:
             try:
                 printer.table(
-                    *reporting.performance_compare(options.performance_compare)
+                    reporting.performance_compare(options.performance_compare)
                 )
             except (errors.ReframeError, ValueError) as err:
                 printer.error(f'failed to generate performance report: {err}')
@@ -1458,7 +1546,7 @@ def module_unuse(*paths):
 
             if options.performance_report and not options.dry_run:
                 try:
-                    data, header = reporting.performance_compare(
+                    data = reporting.performance_compare(
                         rt.get_option('general/0/perf_report_spec'), report
                     )
                 except (errors.ReframeError, ValueError) as err:
@@ -1466,7 +1554,7 @@ def module_unuse(*paths):
                         f'failed to generate performance report: {err}'
                     )
                 else:
-                    printer.performance_report(data, header)
+                    printer.performance_report(data)
 
             # Generate the report for this session
             report_file = os.path.normpath(
diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index 2409637a20..cf0badb6c6 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -247,15 +247,17 @@ def retry_report(self, report):
 
         self.info('\n'.join(lines))
 
-    def performance_report(self, data, header):
+    def performance_report(self, data, **kwargs):
         width = min(80, shutil.get_terminal_size()[0])
         self.info('')
         self.info(' PERFORMANCE REPORT '.center(width, '='))
         self.info('')
-        self.table(data, header)
+        self.table(data, **kwargs)
         self.info('')
 
-    def table(self, data, header):
-        '''Print a table'''
+    def table(self, data, **kwargs):
+        '''Print tabular data'''
 
-        self.info(tabulate(data, headers=header, tablefmt='mixed_grid'))
+        kwargs.setdefault('headers', 'firstrow')
+        kwargs.setdefault('tablefmt', 'mixed_grid')
+        self.info(tabulate(data, **kwargs))
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 05a85c4f1b..01121b555a 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -23,10 +23,11 @@
 import reframe.utility.osext as osext
 from reframe.core.exceptions import ReframeError, what, is_severe
 from reframe.core.logging import getlogger, _format_time_rfc3339
+from reframe.core.runtime import runtime
 from reframe.core.warnings import suppress_deprecations
 from reframe.utility import nodelist_abbrev
 from .storage import StorageBackend
-from .utility import Aggregator, parse_cmp_spec
+from .utility import Aggregator, parse_cmp_spec, parse_time_period
 
 # The schema data version
 # Major version bumps are expected to break the validation of previous schemas
@@ -247,7 +248,7 @@ def update_restored_cases(self, restored_cases, restored_session):
                                            for c in restored_cases]
 
     def update_timestamps(self, ts_start, ts_end):
-        fmt = r'%FT%T%z'
+        fmt = r'%Y%m%dT%H%M%S%z'
         self.__report['session_info'].update({
             'time_start': time.strftime(fmt, time.localtime(ts_start)),
             'time_start_unix': ts_start,
@@ -263,7 +264,7 @@ def update_run_stats(self, stats):
             num_failures = 0
             num_aborted = 0
             num_skipped = 0
-            for t in tasks:
+            for tidx, t in enumerate(tasks):
                 # We take partition and environment from the test case and not
                 # from the check, since if the test fails before `setup()`,
                 # these are not set inside the check.
@@ -299,7 +300,8 @@ def update_run_stats(self, stats):
                     'time_run': t.duration('run_complete'),
                     'time_sanity': t.duration('sanity'),
                     'time_setup': t.duration('setup'),
-                    'time_total': t.duration('total')
+                    'time_total': t.duration('total'),
+                    'uuid': f'{session_uuid}:{runid}:{tidx}'
                 }
                 if check.job:
                     entry['job_stderr'] = check.stderr.evaluate()
@@ -412,6 +414,10 @@ def generate_xml_report(self):
 
         report = self.__report
         xml_testsuites = etree.Element('testsuites')
+        # Create a XSD-friendly timestamp
+        session_ts = time.strftime(
+            r'%FT%T', time.localtime(report['session_info']['time_start_unix'])
+        )
         for run_id, rfm_run in enumerate(report['runs']):
             xml_testsuite = etree.SubElement(
                 xml_testsuites, 'testsuite',
@@ -424,9 +430,7 @@ def generate_xml_report(self):
                     'package': 'reframe',
                     'tests': str(rfm_run['num_cases']),
                     'time': str(report['session_info']['time_elapsed']),
-                    # XSD schema does not like the timezone format,
-                    # so we remove it
-                    'timestamp': report['session_info']['time_start'][:-5],
+                    'timestamp': session_ts
                 }
             )
             etree.SubElement(xml_testsuite, 'properties')
@@ -534,7 +538,8 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
     ptarget = _aggregate_perf(grouped_target, target_fn, [])
 
     # Build the final table data
-    data = []
+    data = [['name', 'pvar', 'pval',
+             'punit', 'pdiff'] + extra_group_by + extra_cols]
     for key, aggr_data in pbase.items():
         pval = aggr_data['pval']
         try:
@@ -554,8 +559,7 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
         line += [aggr_data[c] for c in extra_cols]
         data.append(line)
 
-    return (data, (['name', 'pvar', 'pval', 'punit', 'pdiff'] +
-                   extra_group_by + extra_cols))
+    return data
 
 
 def performance_compare(cmp, report=None):
@@ -590,3 +594,82 @@ def performance_compare(cmp, report=None):
     return compare_testcase_data(tcs_base, tcs_target, match.aggregator,
                                  match.aggregator, match.extra_groups,
                                  match.extra_cols)
+
+
+def session_data():
+    '''Retrieve all sessions'''
+
+    data = [['UUID', 'Start time', 'End time', 'Num runs', 'Num cases']]
+    for sess_data in StorageBackend.default().fetch_all_sessions():
+        session_info = sess_data['session_info']
+        data.append(
+            [session_info['uuid'],
+             session_info['time_start'],
+             session_info['time_end'],
+             len(sess_data['runs']),
+             len(sess_data['runs'][0]['testcases'])]
+        )
+
+    return data
+
+
+def testcase_data(spec):
+    storage = StorageBackend.default()
+    if spec.startswith('^'):
+        testcases = storage.fetch_testcases_from_session(spec[1:])
+    else:
+        testcases = storage.fetch_testcases_time_period(
+            *parse_time_period(spec)
+        )
+
+    data = [['Name', 'System', 'Partition', 'Environment',
+             'Nodelist', 'Result', 'UUID']]
+    for tc in testcases:
+        data.append([
+            tc['name'],
+            tc['system'], tc['partition'], tc['environ'],
+            nodelist_abbrev(tc['job_nodelist']), tc['result'],
+            tc['uuid']
+        ])
+
+    return data
+
+
+def session_info(uuid):
+    '''Retrieve session details as JSON'''
+
+    session = StorageBackend.default().fetch_session_json(uuid)
+    if not session:
+        raise ReframeError(f'no such session: {uuid}')
+
+    return session
+
+
+def testcase_info(spec):
+    '''Retrieve test case details as JSON'''
+    testcases = []
+    if spec.startswith('^'):
+        session_uuid, *tc_index = spec[1:].split(':')
+        session = session_info(session_uuid)
+        if not tc_index:
+            for run in session['runs']:
+                testcases += run['testcases']
+        else:
+            run_index, test_index = tc_index
+            testcases.append(
+                session['runs'][run_index]['testcases'][test_index]
+            )
+    else:
+        testcases = StorageBackend.default().fetch_testcases_time_period(
+            *parse_time_period(spec)
+        )
+
+    return testcases
+
+
+def delete_session(session_uuid):
+    prefix = os.path.dirname(
+        osext.expandvars(runtime().get_option('general/0/report_file'))
+    )
+    with FileLock(os.path.join(prefix, '.db.lock')):
+        StorageBackend.default().remove_session(session_uuid)
diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 9b82f5c9d9..2a32dcc0b1 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -80,10 +80,9 @@ def _db_create(self):
                          'environ TEXT,'
                          'job_completion_time_unix REAL,'
                          'session_uuid TEXT,'
-                         'run_index INTEGER,'
-                         'test_index INTEGER,'
+                         'uuid TEXT,'
                          'FOREIGN KEY(session_uuid) '
-                         'REFERENCES sessions(uuid))')
+                         'REFERENCES sessions(uuid) ON DELETE CASCADE)')
 
     def _db_store_report(self, conn, report, report_file_path):
         session_start_unix = report['session_info']['time_start_unix']
@@ -101,14 +100,14 @@ def _db_store_report(self, conn, report, report_file_path):
                 'report_file': report_file_path
             }
         )
-        for run_idx, run in enumerate(report['runs']):
-            for test_idx, testcase in enumerate(run['testcases']):
+        for run in report['runs']:
+            for testcase in run['testcases']:
                 sys, part = testcase['system'], testcase['partition']
                 conn.execute(
                     'INSERT INTO testcases VALUES('
                     ':name, :system, :partition, :environ, '
                     ':job_completion_time_unix, '
-                    ':session_uuid, :run_index, :test_index)',
+                    ':session_uuid, :uuid)',
                     {
                         'name': testcase['name'],
                         'system': sys,
@@ -118,8 +117,7 @@ def _db_store_report(self, conn, report, report_file_path):
                             'job_completion_time_unix'
                         ],
                         'session_uuid': session_uuid,
-                        'run_index': run_idx,
-                        'test_index': test_idx
+                        'uuid': testcase['uuid']
                     }
                 )
 
@@ -131,8 +129,9 @@ def store(self, report, report_file):
 
     def _fetch_testcases_raw(self, condition):
         with sqlite3.connect(self._db_file()) as conn:
-            query = ('SELECT session_uuid, run_index, test_index, json_blob '
-                     'FROM testcases JOIN sessions ON session_uuid == uuid '
+            query = ('SELECT session_uuid, testcases.uuid as uuid, json_blob '
+                     'FROM testcases '
+                     'JOIN sessions ON session_uuid == sessions.uuid '
                      f'WHERE {condition}')
             getlogger().debug(query)
             results = conn.execute(query).fetchall()
@@ -140,7 +139,8 @@ def _fetch_testcases_raw(self, condition):
         # Retrieve files
         testcases = []
         sessions = {}
-        for session_uuid, run_index, test_index, json_blob in results:
+        for session_uuid, uuid, json_blob in results:
+            run_index, test_index = [int(x) for x in uuid.split(':')[1:]]
             report = jsonext.loads(sessions.setdefault(session_uuid,
                                                        json_blob))
             testcases.append(
@@ -180,3 +180,29 @@ def fetch_testcases_from_session(self, session_uuid):
 
         session_info = jsonext.loads(results[0][0])
         return [tc for run in session_info['runs'] for tc in run['testcases']]
+
+    def fetch_all_sessions(self):
+        with sqlite3.connect(self._db_file()) as conn:
+            query = ('SELECT json_blob from sessions '
+                     'ORDER BY session_start_unix')
+            getlogger().debug(query)
+            results = conn.execute(query).fetchall()
+
+        if not results:
+            return []
+
+        return [jsonext.loads(json_blob) for json_blob, *_ in results]
+
+    def fetch_session_json(self, uuid):
+        with sqlite3.connect(self._db_file()) as conn:
+            query = f'SELECT json_blob FROM sessions WHERE uuid == "{uuid}"'
+            getlogger().debug(query)
+            results = conn.execute(query).fetchall()
+
+        return jsonext.loads(results[0][0]) if results else {}
+
+    def remove_session(self, uuid):
+        with sqlite3.connect(self._db_file()) as conn:
+            query = f'DELETE FROM sessions WHERE uuid == "{uuid}"'
+            getlogger().debug(query)
+            conn.execute(query)
diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py
index 6f2ba477af..8d0b8e4092 100644
--- a/reframe/frontend/reporting/utility.py
+++ b/reframe/frontend/reporting/utility.py
@@ -128,7 +128,7 @@ def _do_parse(s):
     return ts.timestamp()
 
 
-def _parse_time_period(s):
+def parse_time_period(s):
     if s.startswith('^'):
         # Retrieve the period of a full session
         try:
@@ -183,7 +183,7 @@ def _parse_period_spec(s):
         if s.startswith('^'):
             return s[1:], None
 
-        return None, _parse_time_period(s)
+        return None, parse_time_period(s)
 
     parts = spec.split('/')
     if len(parts) == 3:
diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json
index 9a857a14d5..006f39bc28 100644
--- a/reframe/schemas/runreport.json
+++ b/reframe/schemas/runreport.json
@@ -57,7 +57,8 @@
                 "time_sanity": {"type": ["number", "null"]},
                 "time_setup": {"type": ["number", "null"]},
                 "time_total": {"type": ["number", "null"]},
-                "unique_name": {"type": "string"}
+                "unique_name": {"type": "string"},
+                "uuid": {"type": "string"}
             },
             "required": ["environ", "fail_phase", "fail_reason", "filename",
                          "job_completion_time_unix", "name", "perfvalues",

From 50294e7d89b149fe1ac77ece71a463abac4cfdca Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 15 Jul 2024 17:17:29 +0300
Subject: [PATCH 045/128] Make report generation optional + add new
 configuration section for the results storage

---
 reframe/frontend/cli.py                | 45 +++++++++++++++-----------
 reframe/frontend/reporting/__init__.py | 11 ++-----
 reframe/frontend/reporting/storage.py  | 32 ++++++++++--------
 reframe/schemas/config.json            | 16 +++++++--
 4 files changed, 61 insertions(+), 43 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index b6f2f6e73a..869be7d66e 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -677,6 +677,13 @@ def main():
         action='append',
         help='Directories where ReFrame will look for base configuration'
     )
+    argparser.add_argument(
+        dest='generate_file_reports',
+        envvar='RFM_GENERATE_FILE_REPORTS',
+        configvar='general/generate_file_reports',
+        action='store_true',
+        help='Save session report in files'
+    )
     argparser.add_argument(
         dest='git_timeout',
         envvar='RFM_GIT_TIMEOUT',
@@ -1564,24 +1571,26 @@ def module_unuse(*paths):
             if basedir:
                 os.makedirs(basedir, exist_ok=True)
 
-            # Save the report file
-            try:
-                default_loc = os.path.dirname(
-                    osext.expandvars(rt.get_default('general/report_file'))
-                )
-                report.save(
-                    report_file,
-                    compress=rt.get_option('general/0/compress_report'),
-                    link_to_last=(default_loc == os.path.dirname(report_file))
-                )
-            except OSError as e:
-                printer.warning(
-                    f'failed to generate report in {report_file!r}: {e}'
-                )
-            except errors.ReframeError as e:
-                printer.warning(
-                    f'failed to create symlink to latest report: {e}'
-                )
+            if rt.get_option('general/0/generate_file_reports'):
+                # Save the report file
+                try:
+                    default_loc = os.path.dirname(
+                        osext.expandvars(rt.get_default('general/report_file'))
+                    )
+                    report.save(
+                        report_file,
+                        compress=rt.get_option('general/0/compress_report'),
+                        link_to_last=(default_loc ==
+                                      os.path.dirname(report_file))
+                    )
+                except OSError as e:
+                    printer.warning(
+                        f'failed to generate report in {report_file!r}: {e}'
+                    )
+                except errors.ReframeError as e:
+                    printer.warning(
+                        f'failed to create symlink to latest report: {e}'
+                    )
 
             # Store the generated report for analytics
             try:
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 01121b555a..f5938b5ef1 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -23,7 +23,6 @@
 import reframe.utility.osext as osext
 from reframe.core.exceptions import ReframeError, what, is_severe
 from reframe.core.logging import getlogger, _format_time_rfc3339
-from reframe.core.runtime import runtime
 from reframe.core.warnings import suppress_deprecations
 from reframe.utility import nodelist_abbrev
 from .storage import StorageBackend
@@ -405,9 +404,7 @@ def save(self, filename, compress=False, link_to_last=True):
     def store(self):
         '''Store the report in the results storage.'''
 
-        prefix = os.path.dirname(self.filename) or '.'
-        with FileLock(os.path.join(prefix, '.db.lock')):
-            return StorageBackend.default().store(self, self.filename)
+        return StorageBackend.default().store(self, self.filename)
 
     def generate_xml_report(self):
         '''Generate a JUnit report from a standard ReFrame JSON report.'''
@@ -668,8 +665,4 @@ def testcase_info(spec):
 
 
 def delete_session(session_uuid):
-    prefix = os.path.dirname(
-        osext.expandvars(runtime().get_option('general/0/report_file'))
-    )
-    with FileLock(os.path.join(prefix, '.db.lock')):
-        StorageBackend.default().remove_session(session_uuid)
+    StorageBackend.default().remove_session(session_uuid)
diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 2a32dcc0b1..67869ad3dd 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -6,9 +6,11 @@
 import abc
 import os
 import sqlite3
+from filelock import FileLock
 
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
+from reframe.core.exceptions import ReframeError
 from reframe.core.logging import getlogger
 from reframe.core.runtime import runtime
 
@@ -22,12 +24,12 @@ def create(cls, backend, *args, **kwargs):
         if backend == 'sqlite':
             return _SqliteStorage(*args, **kwargs)
         else:
-            raise NotImplementedError
+            raise ReframeError(f'no such storage backend: {backend}')
 
     @classmethod
     def default(cls):
         '''Return default storage backend'''
-        return cls.create('sqlite')
+        return cls.create(runtime().get_option('storage/0/backend'))
 
     @abc.abstractmethod
     def store(self, report, report_file):
@@ -44,11 +46,9 @@ def fetch_testcases_time_period(self, ts_start, ts_end):
 
 class _SqliteStorage(StorageBackend):
     def __init__(self):
-        site_config = runtime().site_config
-        prefix = os.path.dirname(osext.expandvars(
-            site_config.get('general/0/report_file')
-        ))
-        self.__db_file = os.path.join(prefix, 'results.db')
+        self.__db_file = os.path.join(
+            osext.expandvars(runtime().get_option('storage/0/sqlite_db_file'))
+        )
 
     def _db_file(self):
         prefix = os.path.dirname(self.__db_file)
@@ -123,9 +123,11 @@ def _db_store_report(self, conn, report, report_file_path):
 
         return session_uuid
 
-    def store(self, report, report_file):
-        with sqlite3.connect(self._db_file()) as conn:
-            return self._db_store_report(conn, report, report_file)
+    def store(self, report, report_file=None):
+        prefix = os.path.dirname(self.__db_file)
+        with FileLock(os.path.join(prefix, '.db.lock')):
+            with sqlite3.connect(self._db_file()) as conn:
+                return self._db_store_report(conn, report, report_file)
 
     def _fetch_testcases_raw(self, condition):
         with sqlite3.connect(self._db_file()) as conn:
@@ -202,7 +204,9 @@ def fetch_session_json(self, uuid):
         return jsonext.loads(results[0][0]) if results else {}
 
     def remove_session(self, uuid):
-        with sqlite3.connect(self._db_file()) as conn:
-            query = f'DELETE FROM sessions WHERE uuid == "{uuid}"'
-            getlogger().debug(query)
-            conn.execute(query)
+        prefix = os.path.dirname(self.__db_file)
+        with FileLock(os.path.join(prefix, '.db.lock')):
+            with sqlite3.connect(self._db_file()) as conn:
+                query = f'DELETE FROM sessions WHERE uuid == "{uuid}"'
+                getlogger().debug(query)
+                conn.execute(query)
diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index 51189b2811..8bb6a1c882 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -504,6 +504,7 @@
                     "clean_stagedir": {"type": "boolean"},
                     "colorize": {"type": "boolean"},
                     "compress_report": {"type": "boolean"},
+                    "generate_file_reports": {"type": "boolean"},
                     "git_timeout": {"type": "number"},
                     "keep_stage_files": {"type": "boolean"},
                     "module_map_file": {"type": "string"},
@@ -519,7 +520,6 @@
                     "purge_environment": {"type": "boolean"},
                     "remote_detect": {"type": "boolean"},
                     "remote_workdir": {"type": "string"},
-                    "report_database_file": {"type": "string"},
                     "report_file": {"type": "string"},
                     "report_junit": {"type": ["string", "null"]},
                     "resolve_module_conflicts": {"type": "boolean"},
@@ -534,6 +534,16 @@
                 },
                 "additionalProperties": false
             }
+        },
+        "storage": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "backend": {"type": "string"},
+                    "sqlite_db_file": {"type": "string"}
+                }
+            }
         }
     },
     "required": ["systems", "environments", "logging"],
@@ -562,6 +572,7 @@
         "general/clean_stagedir": true,
         "general/colorize": true,
         "general/compress_report": false,
+        "general/generate_file_reports": true,
         "general/git_timeout": 5,
         "general/keep_stage_files": false,
         "general/module_map_file": "",
@@ -572,7 +583,6 @@
         "general/purge_environment": false,
         "general/remote_detect": false,
         "general/remote_workdir": ".",
-        "general/report_database_file": "${HOME}/.reframe/reports_index.db",
         "general/report_file": "${HOME}/.reframe/reports/run-report-{sessionid}.json",
         "general/report_junit": null,
         "general/resolve_module_conflicts": true,
@@ -610,6 +620,8 @@
         "logging/handlers_perflog/httpjson_debug": false,
         "modes/options": [],
         "modes/target_systems": ["*"],
+        "storage/backend": "sqlite",
+        "storage/sqlite_db_file": "${HOME}/.reframe/reports/results.db",
         "systems/descr": "",
         "systems/max_local_jobs": 8,
         "systems/modules_system": "nomod",

From edf5d546eb6156572f5e137967d1645bc03dd62b Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Tue, 16 Jul 2024 15:18:33 +0300
Subject: [PATCH 046/128] Add Sqlite DB schema version check

---
 reframe/frontend/reporting/storage.py | 36 ++++++++++++++++++++++-----
 1 file changed, 30 insertions(+), 6 deletions(-)

diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 67869ad3dd..0fc89d5743 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -45,6 +45,8 @@ def fetch_testcases_time_period(self, ts_start, ts_end):
 
 
 class _SqliteStorage(StorageBackend):
+    SCHEMA_VERSION = '1.0'
+
     def __init__(self):
         self.__db_file = os.path.join(
             osext.expandvars(runtime().get_option('storage/0/sqlite_db_file'))
@@ -59,6 +61,7 @@ def _db_file(self):
 
             self._db_create()
 
+        self._db_schema_check()
         return self.__db_file
 
     def _db_create(self):
@@ -75,14 +78,35 @@ def _db_create(self):
                          'report_file TEXT)')
             conn.execute('CREATE TABLE IF NOT EXISTS testcases('
                          'name TEXT,'
-                         'system TEXT,'
-                         'partition TEXT,'
-                         'environ TEXT,'
-                         'job_completion_time_unix REAL,'
-                         'session_uuid TEXT,'
-                         'uuid TEXT,'
+                         'system TEXT, '
+                         'partition TEXT, '
+                         'environ TEXT, '
+                         'job_completion_time_unix REAL, '
+                         'session_uuid TEXT, '
+                         'uuid TEXT, '
                          'FOREIGN KEY(session_uuid) '
                          'REFERENCES sessions(uuid) ON DELETE CASCADE)')
+            conn.execute('CREATE TABLE IF NOT EXISTS metadata('
+                         'schema_version TEXT)')
+
+    def _db_schema_check(self):
+        with sqlite3.connect(self.__db_file) as conn:
+            results = conn.execute(
+                'SELECT schema_version FROM metadata').fetchall()
+
+        if not results:
+            # DB is new, insert the schema version
+            with sqlite3.connect(self.__db_file) as conn:
+                conn.execute('INSERT INTO metadata VALUES(:schema_version)',
+                             {'schema_version': self.SCHEMA_VERSION})
+        else:
+            found_ver = results[0][0]
+            if found_ver != self.SCHEMA_VERSION:
+                raise ReframeError(
+                    f'results DB in {self.__db_file!r} is '
+                    'of incompatible version: '
+                    f'found {found_ver}, required: {self.SCHEMA_VERSION}'
+                )
 
     def _db_store_report(self, conn, report, report_file_path):
         session_start_unix = report['session_info']['time_start_unix']

From addcde08f218820f79c3595f8c96901cd102cdb8 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Tue, 16 Jul 2024 15:27:58 +0300
Subject: [PATCH 047/128] Support `w` for denoting weeks in period specs

---
 reframe/frontend/reporting/utility.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py
index 8d0b8e4092..6d4433815b 100644
--- a/reframe/frontend/reporting/utility.py
+++ b/reframe/frontend/reporting/utility.py
@@ -109,14 +109,18 @@ def _do_parse(s):
         ts = _do_parse(s)
     except ValueError as err:
         # Try the relative timestamps
-        match = re.match(r'(?P<ts>.*)(?P<amount>[\+|-]\d+)(?P<unit>[hdms])', s)
+        match = re.match(
+            r'(?P<ts>.*)(?P<amount>[\+|-]\d+)(?P<unit>[hdmsw])', s
+        )
         if not match:
             raise err
 
         ts = _do_parse(match.group('ts'))
         amount = int(match.group('amount'))
         unit = match.group('unit')
-        if unit == 'd':
+        if unit == 'w':
+            ts += timedelta(weeks=amount)
+        elif unit == 'd':
             ts += timedelta(days=amount)
         elif unit == 'm':
             ts += timedelta(minutes=amount)

From efb33194c1faf0e1fe66381f919332a6422e2027 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Tue, 16 Jul 2024 19:35:45 +0300
Subject: [PATCH 048/128] Improve error handling

---
 reframe/core/exceptions.py             | 14 ++++++++++++++
 reframe/frontend/cli.py                | 24 ++++++++++++------------
 reframe/frontend/reporting/__init__.py | 22 ++++++++++++++--------
 3 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/reframe/core/exceptions.py b/reframe/core/exceptions.py
index af9acc5751..d63403b587 100644
--- a/reframe/core/exceptions.py
+++ b/reframe/core/exceptions.py
@@ -388,3 +388,17 @@ def what(exc_type, exc_value, tb):
             reason += f': {exc_value}'
 
     return reason
+
+
+class reraise_as:
+    def __init__(self, new_exc, exceptions=(Exception,), message=''):
+        self.__new_exc = new_exc
+        self.__exceptions = exceptions
+        self.__message = message
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if isinstance(exc_val, self.__exceptions):
+            raise self.__new_exc(self.__message) from exc_val
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 869be7d66e..20707c8c1c 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -207,7 +207,7 @@ def calc_verbosity(site_config, quiesce):
     return curr_verbosity - quiesce
 
 
-class exit_gracefuly_on_error:
+class exit_gracefully_on_error:
     def __init__(self, message, logger=None, exceptions=None, exitcode=1):
         self.__message = message
         self.__logger = logger or PrettyPrinter()
@@ -937,14 +937,14 @@ def restrict_logging():
         sys.exit(1)
 
     if options.list_stored_sessions:
-        with exit_gracefuly_on_error('failed to retrieve session data',
-                                     printer):
+        with exit_gracefully_on_error('failed to retrieve session data',
+                                      printer):
             printer.table(reporting.session_data())
             sys.exit(0)
 
     if options.list_stored_testcases:
-        with exit_gracefuly_on_error('failed to retrieve test case data',
-                                     printer):
+        with exit_gracefully_on_error('failed to retrieve test case data',
+                                      printer):
             printer.table(reporting.testcase_data(
                 options.list_stored_testcases
             ))
@@ -953,8 +953,8 @@ def restrict_logging():
     if options.describe_stored_session:
         # Restore logging level
         printer.setLevel(logging.INFO)
-        with exit_gracefuly_on_error('failed to retrieve session data',
-                                     printer):
+        with exit_gracefully_on_error('failed to retrieve session data',
+                                      printer):
             printer.info(jsonext.dumps(reporting.session_info(
                 options.describe_stored_session
             ), indent=2))
@@ -963,8 +963,8 @@ def restrict_logging():
     if options.describe_stored_testcases:
         # Restore logging level
         printer.setLevel(logging.INFO)
-        with exit_gracefuly_on_error('failed to retrieve test case data',
-                                     printer):
+        with exit_gracefully_on_error('failed to retrieve test case data',
+                                      printer):
             printer.info(jsonext.dumps(reporting.testcase_info(
                 options.describe_stored_testcases
             ), indent=2))
@@ -972,7 +972,7 @@ def restrict_logging():
 
     if options.delete_stored_session:
         session_uuid = options.delete_stored_session
-        with exit_gracefuly_on_error('failed to delete session', printer):
+        with exit_gracefully_on_error('failed to delete session', printer):
             reporting.delete_session(session_uuid)
             printer.info(f'Session {session_uuid} deleted successfully.')
             sys.exit(0)
@@ -1393,7 +1393,7 @@ def _sort_testcases(testcases):
                 printer.table(
                     reporting.performance_compare(options.performance_compare)
                 )
-            except (errors.ReframeError, ValueError) as err:
+            except errors.ReframeError as err:
                 printer.error(f'failed to generate performance report: {err}')
                 sys.exit(1)
             else:
@@ -1556,7 +1556,7 @@ def module_unuse(*paths):
                     data = reporting.performance_compare(
                         rt.get_option('general/0/perf_report_spec'), report
                     )
-                except (errors.ReframeError, ValueError) as err:
+                except errors.ReframeError as err:
                     printer.warning(
                         f'failed to generate performance report: {err}'
                     )
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index f5938b5ef1..c542652dd7 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -21,7 +21,7 @@
 import reframe as rfm
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
-from reframe.core.exceptions import ReframeError, what, is_severe
+from reframe.core.exceptions import ReframeError, what, is_severe, reraise_as
 from reframe.core.logging import getlogger, _format_time_rfc3339
 from reframe.core.warnings import suppress_deprecations
 from reframe.utility import nodelist_abbrev
@@ -473,7 +473,9 @@ def save_junit(self, filename):
 def _group_key(groups, testcase):
     key = []
     for grp in groups:
-        val = testcase[grp]
+        with reraise_as(ReframeError, (KeyError,), 'no such group'):
+            val = testcase[grp]
+
         if grp == 'job_nodelist':
             # Fold nodelist before adding as a key element
             key.append(nodelist_abbrev(val))
@@ -514,11 +516,12 @@ def _aggregate_perf(grouped_testcases, aggr_fn, cols):
     for key, seq in grouped_testcases.items():
         aggr_data.setdefault(key, {})
         aggr_data[key]['pval'] = aggr_fn(tc['pval'] for tc in seq)
-        for c in cols:
-            aggr_data[key][c] = other_aggr(
-                nodelist_abbrev(tc[c]) if c == 'job_nodelist' else tc[c]
-                for tc in seq
-            )
+        with reraise_as(ReframeError, (KeyError,), 'no such column'):
+            for c in cols:
+                aggr_data[key][c] = other_aggr(
+                    nodelist_abbrev(tc[c]) if c == 'job_nodelist' else tc[c]
+                    for tc in seq
+                )
 
     return aggr_data
 
@@ -560,7 +563,10 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
 
 
 def performance_compare(cmp, report=None):
-    match = parse_cmp_spec(cmp)
+    with reraise_as(ReframeError, (ValueError,),
+                    'could not parse comparison spec'):
+        match = parse_cmp_spec(cmp)
+
     if match.period_base is None and match.session_base is None:
         if report is None:
             raise ValueError('report cannot be `None` '

From 5b7412a916dfd8a55deda128277363bcb869c1b5 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Tue, 16 Jul 2024 20:47:08 +0300
Subject: [PATCH 049/128] Do not generate reports and DB entries if report is
 empty

---
 reframe/frontend/cli.py                | 28 +++++++++++++++-----------
 reframe/frontend/reporting/__init__.py | 17 ++++++++++++----
 reframe/schemas/config.json            |  2 +-
 unittests/test_cli.py                  |  2 +-
 4 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 20707c8c1c..14c3d435c6 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -317,7 +317,7 @@ def main():
         '--timestamp', action='store', nargs='?', metavar='TIMEFMT',
         const=argparse.CONST_DEFAULT,
         help=('Append a timestamp to the output and stage directory prefixes '
-              '(default: "%%y%%m%%dT%%H%%M%%S%%z")'),
+              '(default: "%%Y%%m%%dT%%H%%M%%S%%z")'),
         envvar='RFM_TIMESTAMP_DIRS', configvar='general/timestamp_dirs'
     )
 
@@ -1551,7 +1551,8 @@ def module_unuse(*paths):
                         report, global_stats=options.duration or options.reruns
                     )
 
-            if options.performance_report and not options.dry_run:
+            if (options.performance_report and
+                not options.dry_run and not report.is_empty()):
                 try:
                     data = reporting.performance_compare(
                         rt.get_option('general/0/perf_report_spec'), report
@@ -1571,7 +1572,8 @@ def module_unuse(*paths):
             if basedir:
                 os.makedirs(basedir, exist_ok=True)
 
-            if rt.get_option('general/0/generate_file_reports'):
+            if (rt.get_option('general/0/generate_file_reports') and
+                not report.is_empty()):
                 # Save the report file
                 try:
                     default_loc = os.path.dirname(
@@ -1593,18 +1595,20 @@ def module_unuse(*paths):
                     )
 
             # Store the generated report for analytics
-            try:
-                sess_uuid = report.store()
-            except Exception as e:
-                printer.warning(
-                    f'failed to store results in the database: {e}'
-                )
-            else:
-                printer.info(f'Current session stored with UUID: {sess_uuid}')
+            if not report.is_empty():
+                try:
+                    sess_uuid = report.store()
+                except Exception as e:
+                    printer.warning(
+                        f'failed to store results in the database: {e}'
+                    )
+                else:
+                    printer.info('Current session stored with UUID: '
+                                 f'{sess_uuid}')
 
             # Generate the junit xml report for this session
             junit_report_file = rt.get_option('general/0/report_junit')
-            if junit_report_file:
+            if junit_report_file and not report.is_empty():
                 # Expand variables in filename
                 junit_report_file = osext.expandvars(junit_report_file)
                 try:
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index c542652dd7..4194998ef3 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -396,6 +396,10 @@ def _save(self, filename, compress, link_to_last):
                 else:
                     raise ReframeError('path exists and is not a symlink')
 
+    def is_empty(self):
+        '''Return :obj:`True` is no test cases where run'''
+        return self.__report['session_info']['num_cases'] == 0
+
     def save(self, filename, compress=False, link_to_last=True):
         prefix = os.path.dirname(filename) or '.'
         with FileLock(os.path.join(prefix, '.report.lock')):
@@ -625,13 +629,18 @@ def testcase_data(spec):
             *parse_time_period(spec)
         )
 
-    data = [['Name', 'System', 'Partition', 'Environment',
-             'Nodelist', 'Result', 'UUID']]
+    data = [['Name', 'SysEnv',
+             'Nodelist', 'Completion Time', 'Result', 'UUID']]
     for tc in testcases:
         data.append([
             tc['name'],
-            tc['system'], tc['partition'], tc['environ'],
-            nodelist_abbrev(tc['job_nodelist']), tc['result'],
+            f'{tc["system"]}:{tc["partition"]}@{tc["environ"]}',
+            nodelist_abbrev(tc['job_nodelist']),
+            # Always format the completion time as users can set their own
+            # formatting in the log record
+            time.strftime(r'%Y%m%dT%H%M%S%z',
+                          time.localtime(tc['job_completion_time_unix'])),
+            tc['result'],
             tc['uuid']
         ])
 
diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index 8bb6a1c882..900ab0de14 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -588,7 +588,7 @@
         "general/resolve_module_conflicts": true,
         "general/save_log_files": false,
         "general/target_systems": ["*"],
-        "general/timestamp_dirs": "%y%m%dT%H%M%S%z",
+        "general/timestamp_dirs": "%Y%m%dT%H%M%S%z",
         "general/trap_job_errors": false,
         "general/unload_modules": [],
         "general/use_login_shell": false,
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 6585dae0f6..04051c7b36 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -571,7 +571,7 @@ def test_timestamp_option_default(run_reframe):
     assert returncode == 0
 
     matches = re.findall(
-        r'(stage|output) directory: .*\/(\d{6}T\d{6}\+\d{4})', stdout
+        r'(stage|output) directory: .*\/(\d{8}T\d{6}\+\d{4})', stdout
     )
     assert len(matches) == 2
 

From 240d72b58392e228b9cec38436070c356f9c95e0 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 17 Jul 2024 12:25:44 +0300
Subject: [PATCH 050/128] Address remaining MR comments

---
 reframe/frontend/reporting/__init__.py | 19 ++++++++++---------
 reframe/schemas/runreport.json         |  6 +++---
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 4194998ef3..e4440236e2 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -33,6 +33,7 @@
 
 DATA_VERSION = '4.0'
 _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json')
+_DATETIME_FMT = r'%Y%m%dT%H%M%S%z'
 
 
 def format_testcase(json, name='unique_name'):
@@ -247,18 +248,18 @@ def update_restored_cases(self, restored_cases, restored_session):
                                            for c in restored_cases]
 
     def update_timestamps(self, ts_start, ts_end):
-        fmt = r'%Y%m%dT%H%M%S%z'
         self.__report['session_info'].update({
-            'time_start': time.strftime(fmt, time.localtime(ts_start)),
+            'time_start': time.strftime(_DATETIME_FMT,
+                                        time.localtime(ts_start)),
             'time_start_unix': ts_start,
-            'time_end': time.strftime(fmt, time.localtime(ts_end)),
+            'time_end': time.strftime(_DATETIME_FMT, time.localtime(ts_end)),
             'time_end_unix': ts_end,
             'time_elapsed': ts_end - ts_start
         })
 
     def update_run_stats(self, stats):
         session_uuid = self.__report['session_info']['uuid']
-        for runid, tasks in stats.runs():
+        for runidx, tasks in stats.runs():
             testcases = []
             num_failures = 0
             num_aborted = 0
@@ -291,7 +292,7 @@ def update_run_stats(self, stats):
                     'job_stdout': None,
                     'partition': partition.name,
                     'result': t.result,
-                    'runid': runid,
+                    'run_index': runidx,
                     'scheduler': partition.scheduler.registered_name,
                     'session_uuid': session_uuid,
                     'time_compile': t.duration('compile_complete'),
@@ -300,7 +301,7 @@ def update_run_stats(self, stats):
                     'time_sanity': t.duration('sanity'),
                     'time_setup': t.duration('setup'),
                     'time_total': t.duration('total'),
-                    'uuid': f'{session_uuid}:{runid}:{tidx}'
+                    'uuid': f'{session_uuid}:{runidx}:{tidx}'
                 }
                 if check.job:
                     entry['job_stderr'] = check.stderr.evaluate()
@@ -357,7 +358,7 @@ def update_run_stats(self, stats):
                 'num_failures': num_failures,
                 'num_aborted': num_aborted,
                 'num_skipped': num_skipped,
-                'runid': runid,
+                'run_index': runidx,
                 'testcases': testcases
             })
 
@@ -515,7 +516,7 @@ def _group_testcases(testcases, group_by, extra_cols):
 
 
 def _aggregate_perf(grouped_testcases, aggr_fn, cols):
-    other_aggr = Aggregator.create('join_uniq', '|')
+    other_aggr = Aggregator.create('join_uniq', '\n')
     aggr_data = {}
     for key, seq in grouped_testcases.items():
         aggr_data.setdefault(key, {})
@@ -638,7 +639,7 @@ def testcase_data(spec):
             nodelist_abbrev(tc['job_nodelist']),
             # Always format the completion time as users can set their own
             # formatting in the log record
-            time.strftime(r'%Y%m%dT%H%M%S%z',
+            time.strftime(_DATETIME_FMT,
                           time.localtime(tc['job_completion_time_unix'])),
             tc['result'],
             tc['uuid']
diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json
index 006f39bc28..a216a5cd1b 100644
--- a/reframe/schemas/runreport.json
+++ b/reframe/schemas/runreport.json
@@ -88,7 +88,6 @@
                 "num_skipped": {"type": "number"},
                 "prefix_output": {"type": "string"},
                 "prefix_stage": {"type": "string"},
-                "runid": {"type": "number"},
                 "session_uuid": {"type": "string"},
                 "time_elapsed": {"type": "number"},
                 "time_end": {"type": "string"},
@@ -101,7 +100,8 @@
                 "workdir": {"type": "string"}
             },
             "required": ["data_version", "hostname",
-                         "time_elapsed", "time_end", "time_start", "uuid"]
+                         "time_elapsed", "time_end_unix", "time_start_unix",
+                         "uuid"]
         },
         "restored_cases": {
             "type": "array",
@@ -116,7 +116,7 @@
                     "num_cases": {"type": "number"},
                     "num_failures": {"type": "number"},
                     "num_skipped": {"type": "number"},
-                    "runid": {"typeid": "number"},
+                    "run_index": {"type": "number"},
                     "testcases": {
                         "type": "array",
                         "items": {"$ref": "#/defs/testcase_type"}

From b0548fc4b78d75ea911a0aff1ece32336a8966a8 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 17 Jul 2024 14:57:26 +0300
Subject: [PATCH 051/128] Group by also system, partition and environment

---
 reframe/frontend/cli.py                |  2 +-
 reframe/frontend/printer.py            |  4 +--
 reframe/frontend/reporting/__init__.py | 49 ++++++++++++++++----------
 3 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 14c3d435c6..8d27636ba3 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -1230,7 +1230,7 @@ def print_infoline(param, value):
                 sys.exit(1)
 
             def _case_failed(t):
-                rec = restored_session.case(*t)
+                rec = restored_session.case(t)
                 if not rec:
                     return False
 
diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index cf0badb6c6..33450f971f 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -13,7 +13,7 @@
 import reframe.core.runtime as rt
 import reframe.utility.color as color
 from reframe.core.exceptions import SanityError
-from reframe.frontend.reporting import format_testcase
+from reframe.frontend.reporting import format_testcase_from_json
 from reframe.utility import nodelist_abbrev
 
 
@@ -236,7 +236,7 @@ def retry_report(self, report):
         for i, run in enumerate(report['runs'][1:], start=1):
             for tc in run['testcases']:
                 # Overwrite entry from previous run if available
-                tc_info = format_testcase(tc)
+                tc_info = format_testcase_from_json(tc)
                 messages[tc_info] = (
                     f"  * Test {tc_info} was retried {i} time(s) and"
                     f" {'failed' if tc['result'] == 'fail' else 'passed'}."
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index e4440236e2..314bcd83fe 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -36,13 +36,24 @@
 _DATETIME_FMT = r'%Y%m%dT%H%M%S%z'
 
 
-def format_testcase(json, name='unique_name'):
+def _format_sysenv(system, partition, environ):
+    return f'{system}:{partition}+{environ}'
+
+
+def format_testcase_from_json(tc):
     '''Format test case from its json representation'''
-    name = json[name]
-    system = json['system']
-    partition = json['partition']
-    environ = json['environ']
-    return f'{name}@{system}:{partition}+{environ}'
+    name = tc['name']
+    system = tc['system']
+    partition = tc['partition']
+    environ = tc['environ']
+    return f'{name} @{_format_sysenv(system, partition, environ)}'
+
+
+def format_testcase(tc):
+    return format_testcase_from_json({'name': tc.check.name,
+                                      'system': tc.check.current_system.name,
+                                      'partition': tc.partition.name,
+                                      'environ': tc.environ.name})
 
 
 class _RestoredSessionInfo:
@@ -57,11 +68,11 @@ def __init__(self, report):
         self._cases_index = {}
         for run in self._report['runs']:
             for tc in run['testcases']:
-                self._cases_index[format_testcase(tc)] = tc
+                self._cases_index[format_testcase_from_json(tc)] = tc
 
         # Index also the restored cases
         for tc in self._report['restored_cases']:
-            self._cases_index[format_testcase(tc)] = tc
+            self._cases_index[format_testcase_from_json(tc)] = tc
 
     def __getitem__(self, key):
         return self._report[key]
@@ -95,8 +106,8 @@ def slice(self, prop, when=None, unique=False):
 
                 yield val
 
-    def case(self, check, part, env):
-        key = f'{check.unique_name}@{part.fullname}+{env.name}'
+    def case(self, tc):
+        key = format_testcase(tc)
         ret = self._cases_index.get(key)
         if ret is None:
             # Look up the case in the fallback reports
@@ -123,7 +134,7 @@ def restore_dangling(self, graph):
         return graph, restored
 
     def _do_restore(self, testcase):
-        tc = self.case(*testcase)
+        tc = self.case(testcase)
         if tc is None:
             raise ReframeError(
                 f'could not restore testcase {testcase!r}: '
@@ -244,7 +255,7 @@ def update_session_info(self, session_info):
                 self.__report['session_info'][key] = val
 
     def update_restored_cases(self, restored_cases, restored_session):
-        self.__report['restored_cases'] = [restored_session.case(*c)
+        self.__report['restored_cases'] = [restored_session.case(c)
                                            for c in restored_cases]
 
     def update_timestamps(self, ts_start, ts_end):
@@ -437,7 +448,7 @@ def generate_xml_report(self):
             )
             etree.SubElement(xml_testsuite, 'properties')
             for tc in rfm_run['testcases']:
-                casename = f'{format_testcase(tc, name="name")}'
+                casename = f'{format_testcase_from_json(tc)}'
                 testcase = etree.SubElement(
                     xml_testsuite, 'testcase',
                     attrib={
@@ -535,7 +546,8 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
                           extra_group_by=None, extra_cols=None):
     extra_group_by = extra_group_by or []
     extra_cols = extra_cols or []
-    group_by = ['name', 'pvar', 'punit'] + extra_group_by
+    group_by = (['name', 'system', 'partition', 'environ', 'pvar', 'punit'] +
+                extra_group_by)
 
     grouped_base = _group_testcases(base_testcases, group_by, extra_cols)
     grouped_target = _group_testcases(target_testcases, group_by, extra_cols)
@@ -543,7 +555,7 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
     ptarget = _aggregate_perf(grouped_target, target_fn, [])
 
     # Build the final table data
-    data = [['name', 'pvar', 'pval',
+    data = [['name', 'sysenv', 'pvar', 'pval',
              'punit', 'pdiff'] + extra_group_by + extra_cols]
     for key, aggr_data in pbase.items():
         pval = aggr_data['pval']
@@ -558,8 +570,9 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
                 pdiff = (pval - target_pval) / target_pval
                 pdiff = '{:+7.2%}'.format(pdiff)
 
-        name, pvar, punit, *extras = key
-        line = [name, pvar, pval, punit, pdiff, *extras]
+        name, system, partition, environ, pvar, punit, *extras = key
+        line = [name, _format_sysenv(system, partition, environ),
+                pvar, pval, punit, pdiff, *extras]
         # Add the extra columns
         line += [aggr_data[c] for c in extra_cols]
         data.append(line)
@@ -635,7 +648,7 @@ def testcase_data(spec):
     for tc in testcases:
         data.append([
             tc['name'],
-            f'{tc["system"]}:{tc["partition"]}@{tc["environ"]}',
+            _format_sysenv(tc['system'], tc['partition'], tc['environ']),
             nodelist_abbrev(tc['job_nodelist']),
             # Always format the completion time as users can set their own
             # formatting in the log record

From a85d1bf33eafac9202170089e8d2d80bc0cd92e7 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 17 Jul 2024 17:10:52 +0300
Subject: [PATCH 052/128] Use the last successful retry as the reference test
 case for the performance report

---
 reframe/frontend/printer.py            | 22 +++++++++++-----------
 reframe/frontend/reporting/__init__.py | 10 ++++++++--
 2 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index 33450f971f..0767ac4dfe 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -228,22 +228,22 @@ def retry_report(self, report):
             # Do nothing if no retries
             return
 
-        line_width = shutil.get_terminal_size()[0]
-        lines = [line_width * '=']
+        line_width = min(80, shutil.get_terminal_size()[0])
+        lines = ['', line_width * '=']
         lines.append('SUMMARY OF RETRIES')
         lines.append(line_width * '-')
-        messages = {}
-        for i, run in enumerate(report['runs'][1:], start=1):
+        retried_tc = set()
+        for run in reversed(report['runs'][1:]):
+            runidx = run['run_index']
             for tc in run['testcases']:
                 # Overwrite entry from previous run if available
                 tc_info = format_testcase_from_json(tc)
-                messages[tc_info] = (
-                    f"  * Test {tc_info} was retried {i} time(s) and"
-                    f" {'failed' if tc['result'] == 'fail' else 'passed'}."
-                )
-
-        for msg in sorted(messages):
-            lines.append(msg)
+                if tc_info not in retried_tc:
+                    lines.append(
+                        f"  * Test {tc_info} was retried {runidx} time(s) and "
+                        f" {'failed' if tc['result'] == 'fail' else 'passed'}."
+                    )
+                    retried_tc.add(tc_info)
 
         self.info('\n'.join(lines))
 
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 314bcd83fe..6139802cf2 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -590,10 +590,16 @@ def performance_compare(cmp, report=None):
             raise ValueError('report cannot be `None` '
                              'for current run comparisons')
         try:
-            tcs_base = report['runs'][0]['testcases']
+            # Get the last retry from every test case
+            num_runs = len(report['runs'])
+            tcs_base = []
+            for run in report['runs']:
+                run_idx = run['run_index']
+                for tc in run['testcases']:
+                    if tc['result'] != 'fail' or run_idx == num_runs - 1:
+                        tcs_base.append(tc)
         except IndexError:
             tcs_base = []
-
     elif match.period_base is not None:
         tcs_base = StorageBackend.default().fetch_testcases_time_period(
             *match.period_base

From e2b4c51cb9dc7ce74d1c1c2d438fe3f8479d7165 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 18 Jul 2024 11:39:34 +0300
Subject: [PATCH 053/128] Move perflog testing to a different file

---
 unittests/conftest.py         |  38 +++
 unittests/test_perflogging.py | 458 ++++++++++++++++++++++++++++++++
 unittests/test_policies.py    | 476 ----------------------------------
 3 files changed, 496 insertions(+), 476 deletions(-)
 create mode 100644 unittests/test_perflogging.py

diff --git a/unittests/conftest.py b/unittests/conftest.py
index 1a2034f560..62d6ca126d 100644
--- a/unittests/conftest.py
+++ b/unittests/conftest.py
@@ -14,7 +14,10 @@
 
 import reframe.core.settings as settings
 import reframe.core.runtime as rt
+import reframe.frontend.executors as executors
+import reframe.frontend.executors.policies as policies
 import reframe.utility as util
+from reframe.frontend.loader import RegressionCheckLoader
 
 from .utility import TEST_CONFIG_FILE
 
@@ -80,6 +83,41 @@ def _make_exec_ctx(*args, **kwargs):
     yield _make_exec_ctx
 
 
+@pytest.fixture
+def common_exec_ctx(make_exec_ctx_g):
+    '''Execution context for the default generic system.'''
+    yield from make_exec_ctx_g(system='generic')
+
+
+@pytest.fixture
+def testsys_exec_ctx(make_exec_ctx_g):
+    '''Execution context for the `testsys:gpu` system.'''
+    yield from make_exec_ctx_g(system='testsys:gpu')
+
+
+@pytest.fixture
+def make_loader():
+    '''Test loader'''
+    def _make_loader(check_search_path, *args, **kwargs):
+        return RegressionCheckLoader(check_search_path, *args, **kwargs)
+
+    return _make_loader
+
+
+@pytest.fixture(params=[policies.SerialExecutionPolicy,
+                        policies.AsynchronousExecutionPolicy])
+def make_runner(request):
+    '''Test runner with all the execution policies'''
+
+    def _make_runner(*args, **kwargs):
+        # Use a much higher poll rate for the unit tests
+        policy = request.param()
+        policy._pollctl.SLEEP_MIN = 0.001
+        return executors.Runner(policy, *args, **kwargs)
+
+    return _make_runner
+
+
 @pytest.fixture
 def make_config_file(tmp_path):
     '''Create a temporary configuration file from the given configuration.
diff --git a/unittests/test_perflogging.py b/unittests/test_perflogging.py
new file mode 100644
index 0000000000..afc5e26f5e
--- /dev/null
+++ b/unittests/test_perflogging.py
@@ -0,0 +1,458 @@
+# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# ReFrame Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+import contextlib
+import io
+import os
+import pytest
+
+import reframe as rfm
+import reframe.core.logging as logging
+import reframe.core.runtime as rt
+import reframe.frontend.executors as executors
+import reframe.utility.osext as osext
+import reframe.utility.sanity as sn
+
+
+class _MyPerfTest(rfm.RunOnlyRegressionTest):
+    valid_systems = ['*']
+    valid_prog_environs = ['*']
+    executable = 'echo perf0=100 && echo perf1=50'
+
+    @sanity_function
+    def validate(self):
+        return sn.assert_found(r'perf0', self.stdout)
+
+    @performance_function('unit0')
+    def perf0(self):
+        return sn.extractsingle(r'perf0=(\S+)', self.stdout, 1, float)
+
+    @performance_function('unit1')
+    def perf1(self):
+        return sn.extractsingle(r'perf1=(\S+)', self.stdout, 1, float)
+
+
+class _MyPerfParamTest(_MyPerfTest):
+    p = parameter([1, 2])
+
+
+class _MyFailingTest(rfm.RunOnlyRegressionTest):
+    valid_systems = ['*']
+    valid_prog_environs = ['*']
+    executable = 'echo perf0=100'
+
+    @sanity_function
+    def validate(self):
+        return False
+
+    @performance_function('unit0')
+    def perf0(self):
+        return sn.extractsingle(r'perf0=(\S+)', self.stdout, 1, float)
+
+
+class _LazyPerfTest(rfm.RunOnlyRegressionTest):
+    valid_systems = ['*']
+    valid_prog_environs = ['*']
+    executable = 'echo perf0=100'
+
+    @sanity_function
+    def validate(self):
+        return True
+
+    @run_before('performance')
+    def set_perf_vars(self):
+        self.perf_variables = {
+            'perf0': sn.make_performance_function(
+                sn.extractsingle(r'perf0=(\S+)', self.stdout, 1, float),
+                'unit0'
+            )
+        }
+
+
+@pytest.fixture
+def perf_test():
+    return _MyPerfTest()
+
+
+@pytest.fixture
+def perf_param_tests():
+    return [_MyPerfParamTest(variant_num=v)
+            for v in range(_MyPerfParamTest.num_variants)]
+
+
+@pytest.fixture
+def failing_perf_test():
+    return _MyFailingTest()
+
+
+@pytest.fixture
+def lazy_perf_test():
+    return _LazyPerfTest()
+
+
+@pytest.fixture
+def simple_test():
+    class _MySimpleTest(rfm.RunOnlyRegressionTest):
+        valid_systems = ['*']
+        valid_prog_environs = ['*']
+        executable = 'echo hello'
+
+        @sanity_function
+        def validate(self):
+            return sn.assert_found(r'hello', self.stdout)
+
+    return _MySimpleTest()
+
+
+@pytest.fixture
+def config_perflog(make_config_file):
+    def _config_perflog(fmt, perffmt=None, logging_opts=None):
+        logging_config = {
+            'level': 'debug2',
+            'handlers': [{
+                'type': 'stream',
+                'name': 'stdout',
+                'level': 'info',
+                'format': '%(message)s'
+            }],
+            'handlers_perflog': [{
+                'type': 'filelog',
+                'prefix': '%(check_system)s/%(check_partition)s',
+                'level': 'info',
+                'format': fmt
+            }]
+        }
+        if logging_opts:
+            logging_config.update(logging_opts)
+
+        if perffmt is not None:
+            logging_config['handlers_perflog'][0]['format_perfvars'] = perffmt
+
+        return make_config_file({'logging': [logging_config]})
+
+    return _config_perflog
+
+
+def _count_lines(filepath):
+    count = 0
+    with open(filepath) as fp:
+        for line in fp:
+            count += 1
+
+    return count
+
+
+def _assert_header(filepath, header):
+    with open(filepath) as fp:
+        assert fp.readline().strip() == header
+
+
+def _assert_no_logging_error(fn, *args, **kwargs):
+    captured_stderr = io.StringIO()
+    with contextlib.redirect_stderr(captured_stderr):
+        fn(*args, **kwargs)
+
+    assert 'Logging error' not in captured_stderr.getvalue()
+
+
+def test_perf_logging(make_runner, make_exec_ctx, perf_test,
+                      config_perflog, tmp_path):
+    make_exec_ctx(
+        config_perflog(
+            fmt=(
+                '%(check_job_completion_time)s,%(version)s,'
+                '%(check_display_name)s,%(check_system)s,'
+                '%(check_partition)s,%(check_environ)s,'
+                '%(check_jobid)s,%(check_result)s,%(check_perfvalues)s'
+            ),
+            perffmt=(
+                '%(check_perf_value)s,%(check_perf_unit)s,'
+                '%(check_perf_ref)s,%(check_perf_lower_thres)s,'
+                '%(check_perf_upper_thres)s,'
+            )
+        )
+    )
+    logging.configure_logging(rt.runtime().site_config)
+    runner = make_runner()
+    testcases = executors.generate_testcases([perf_test])
+    runner.runall(testcases)
+
+    logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_MyPerfTest.log'
+    assert os.path.exists(logfile)
+    assert _count_lines(logfile) == 2
+
+    # Rerun with the same configuration and check that new entry is appended
+    testcases = executors.generate_testcases([perf_test])
+    runner = make_runner()
+    _assert_no_logging_error(runner.runall, testcases)
+    assert _count_lines(logfile) == 3
+
+    # Change the configuration and rerun
+    make_exec_ctx(
+        config_perflog(
+            fmt=(
+                '%(check_job_completion_time)s,%(version)s,'
+                '%(check_display_name)s,%(check_system)s,'
+                '%(check_partition)s,%(check_environ)s,'
+                '%(check_jobid)s,%(check_result)s,%(check_perfvalues)s'
+            ),
+            perffmt='%(check_perf_value)s,%(check_perf_unit)s,'
+        )
+    )
+    logging.configure_logging(rt.runtime().site_config)
+    testcases = executors.generate_testcases([perf_test])
+    runner = make_runner()
+    _assert_no_logging_error(runner.runall, testcases)
+    assert _count_lines(logfile) == 2
+    _assert_header(logfile,
+                   'job_completion_time,version,display_name,system,partition,'
+                   'environ,jobid,result,perf0_value,perf0_unit,'
+                   'perf1_value,perf1_unit')
+
+    logfile_prev = [(str(logfile) + '.h0', 3)]
+    for f, num_lines in logfile_prev:
+        assert os.path.exists(f)
+        _count_lines(f) == num_lines
+
+    # Change the test and rerun
+    perf_test.perf_variables['perfN'] = perf_test.perf_variables['perf1']
+
+    # We reconfigure the logging in order for the filelog handler to start
+    # from a clean state
+    logging.configure_logging(rt.runtime().site_config)
+    testcases = executors.generate_testcases([perf_test])
+    runner = make_runner()
+    _assert_no_logging_error(runner.runall, testcases)
+    assert _count_lines(logfile) == 2
+    _assert_header(logfile,
+                   'job_completion_time,version,display_name,system,partition,'
+                   'environ,jobid,result,perf0_value,perf0_unit,'
+                   'perf1_value,perf1_unit,perfN_value,perfN_unit')
+
+    logfile_prev = [(str(logfile) + '.h0', 3), (str(logfile) + '.h1', 2)]
+    for f, num_lines in logfile_prev:
+        assert os.path.exists(f)
+        _count_lines(f) == num_lines
+
+
+def test_perf_logging_no_end_delim(make_runner, make_exec_ctx, perf_test,
+                                   config_perflog, tmp_path):
+    make_exec_ctx(
+        config_perflog(
+            fmt=(
+                '%(check_job_completion_time)s,%(version)s,'
+                '%(check_display_name)s,%(check_system)s,'
+                '%(check_partition)s,%(check_environ)s,'
+                '%(check_jobid)s,%(check_result)s,%(check_perfvalues)s'
+            ),
+            perffmt='%(check_perf_value)s,%(check_perf_unit)s'
+        )
+    )
+    logging.configure_logging(rt.runtime().site_config)
+    runner = make_runner()
+    testcases = executors.generate_testcases([perf_test])
+    _assert_no_logging_error(runner.runall, testcases)
+
+    logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_MyPerfTest.log'
+    assert os.path.exists(logfile)
+    assert _count_lines(logfile) == 2
+
+    with open(logfile) as fp:
+        lines = fp.readlines()
+
+    assert len(lines) == 2
+    assert lines[0] == (
+        'job_completion_time,version,display_name,system,partition,'
+        'environ,jobid,result,perf0_value,perf0_unitperf1_value,perf1_unit\n'
+    )
+    assert '<error formatting the performance record' in lines[1]
+
+
+def test_perf_logging_no_perfvars(make_runner, make_exec_ctx, perf_test,
+                                  config_perflog, tmp_path):
+    make_exec_ctx(
+        config_perflog(
+            fmt=(
+                '%(check_job_completion_time)s,%(version)s,'
+                '%(check_display_name)s,%(check_system)s,'
+                '%(check_partition)s,%(check_environ)s,'
+                '%(check_jobid)s,%(check_result)s,%(check_perfvalues)s'
+            )
+        )
+    )
+    logging.configure_logging(rt.runtime().site_config)
+    runner = make_runner()
+    testcases = executors.generate_testcases([perf_test])
+    _assert_no_logging_error(runner.runall, testcases)
+
+    logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_MyPerfTest.log'
+    assert os.path.exists(logfile)
+    assert _count_lines(logfile) == 2
+
+    with open(logfile) as fp:
+        lines = fp.readlines()
+
+    assert len(lines) == 2
+    assert lines[0] == (
+        'job_completion_time,version,display_name,system,partition,'
+        'environ,jobid,result,\n'
+    )
+    assert 'error' not in lines[1]
+
+
+def test_perf_logging_multiline(make_runner, make_exec_ctx, perf_test,
+                                simple_test, failing_perf_test,
+                                config_perflog, tmp_path):
+    make_exec_ctx(
+        config_perflog(
+            fmt=('%(check_job_completion_time)s|reframe %(version)s|'
+                 '%(check_name)s|%(check_perf_var)s=%(check_perf_value)s|'
+                 'ref=%(check_perf_ref)s (l=%(check_perf_lower_thres)s, '
+                 'u=%(check_perf_upper_thres)s)|%(check_perf_unit)s'),
+            logging_opts={'perflog_compat': True}
+        )
+    )
+    logging.configure_logging(rt.runtime().site_config)
+    runner = make_runner()
+    testcases = executors.generate_testcases(
+        [perf_test, simple_test, failing_perf_test]
+    )
+    _assert_no_logging_error(runner.runall, testcases)
+
+    logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_MyPerfTest.log'
+    assert os.path.exists(logfile)
+    assert _count_lines(logfile) == 3
+
+    # assert that the emitted lines are correct
+    with open(logfile) as fp:
+        lines = fp.readlines()
+
+    version = osext.reframe_version()
+    assert lines[0] == ('job_completion_time|reframe version|name|'
+                        'perf_var=perf_value|ref=perf_ref '
+                        '(l=perf_lower_thres, u=perf_upper_thres)|perf_unit\n')
+    assert lines[1].endswith(
+        f'|reframe {version}|_MyPerfTest|'
+        f'perf0=100.0|ref=0 (l=null, u=null)|unit0\n'
+    )
+    assert lines[2].endswith(
+        f'|reframe {version}|_MyPerfTest|'
+        f'perf1=50.0|ref=0 (l=null, u=null)|unit1\n'
+    )
+
+
+def test_perf_logging_lazy(make_runner, make_exec_ctx, lazy_perf_test,
+                           config_perflog, tmp_path):
+    make_exec_ctx(
+        config_perflog(
+            fmt=(
+                '%(check_job_completion_time)s,%(version)s,'
+                '%(check_display_name)s,%(check_system)s,'
+                '%(check_partition)s,%(check_environ)s,'
+                '%(check_jobid)s,%(check_result)s,%(check_perfvalues)s'
+            ),
+            perffmt=(
+                '%(check_perf_value)s,%(check_perf_unit)s,'
+                '%(check_perf_ref)s,%(check_perf_lower_thres)s,'
+                '%(check_perf_upper_thres)s,'
+            )
+        )
+    )
+    logging.configure_logging(rt.runtime().site_config)
+    runner = make_runner()
+    testcases = executors.generate_testcases([lazy_perf_test])
+    _assert_no_logging_error(runner.runall, testcases)
+
+    logfile = (tmp_path / 'perflogs' / 'generic' / 'default' /
+               '_LazyPerfTest.log')
+    assert os.path.exists(logfile)
+
+
+@pytest.fixture(params=['%(check_result)s|%(check_#ALL)s', '%(check_#ALL)s'])
+def perflog_fmt(request):
+    return request.param
+
+
+def test_perf_logging_all_attrs(make_runner, make_exec_ctx, perf_test,
+                                config_perflog, tmp_path, perflog_fmt):
+    make_exec_ctx(config_perflog(fmt=perflog_fmt))
+    logging.configure_logging(rt.runtime().site_config)
+    runner = make_runner()
+    testcases = executors.generate_testcases([perf_test])
+    _assert_no_logging_error(runner.runall, testcases)
+
+    logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_MyPerfTest.log'
+    assert os.path.exists(logfile)
+    with open(logfile) as fp:
+        header = fp.readline()
+
+    loggable_attrs = type(perf_test).loggable_attrs()
+    assert (len(header.split('|')) ==
+            len(loggable_attrs) + (perflog_fmt != '%(check_#ALL)s'))
+
+
+def test_perf_logging_custom_vars(make_runner, make_exec_ctx,
+                                  config_perflog, tmp_path):
+    # Create two tests with different loggable variables
+    class _X(_MyPerfTest):
+        x = variable(int, value=1, loggable=True)
+
+    class _Y(_MyPerfTest):
+        y = variable(int, value=2, loggable=True)
+
+    make_exec_ctx(config_perflog(fmt='%(check_result)s|%(check_#ALL)s'))
+    logging.configure_logging(rt.runtime().site_config)
+    runner = make_runner()
+    testcases = executors.generate_testcases([_X(), _Y()])
+    _assert_no_logging_error(runner.runall, testcases)
+
+    logfiles = [tmp_path / 'perflogs' / 'generic' / 'default' / '_X.log',
+                tmp_path / 'perflogs' / 'generic' / 'default' / '_Y.log']
+    for f in logfiles:
+        with open(f) as fp:
+            header = fp.readline().strip()
+            if os.path.basename(f).startswith('_X'):
+                assert 'x' in header.split('|')
+            else:
+                assert 'y' in header.split('|')
+
+
+def test_perf_logging_param_test(make_runner, make_exec_ctx, perf_param_tests,
+                                 config_perflog, tmp_path):
+    make_exec_ctx(config_perflog(fmt='%(check_result)s|%(check_#ALL)s'))
+    logging.configure_logging(rt.runtime().site_config)
+    runner = make_runner()
+    testcases = executors.generate_testcases(perf_param_tests)
+    _assert_no_logging_error(runner.runall, testcases)
+
+    logfile = (tmp_path / 'perflogs' / 'generic' /
+               'default' / '_MyPerfParamTest.log')
+    assert os.path.exists(logfile)
+    assert _count_lines(logfile) == 3
+
+
+def test_perf_logging_sanity_failure(make_runner, make_exec_ctx,
+                                     config_perflog, tmp_path):
+    class _X(_MyPerfTest):
+        @sanity_function
+        def validate(self):
+            return sn.assert_true(0, msg='no way')
+
+    make_exec_ctx(config_perflog(
+        fmt='%(check_result)s|%(check_fail_reason)s|%(check_perfvalues)s',
+        perffmt='%(check_perf_value)s|'
+    ))
+    logging.configure_logging(rt.runtime().site_config)
+    runner = make_runner()
+    testcases = executors.generate_testcases([_X()])
+    _assert_no_logging_error(runner.runall, testcases)
+
+    logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_X.log'
+    assert os.path.exists(logfile)
+    with open(logfile) as fp:
+        lines = fp.readlines()
+
+    assert len(lines) == 2
+    assert lines[1] == 'fail|sanity error: no way|None|None\n'
diff --git a/unittests/test_policies.py b/unittests/test_policies.py
index 66ed64389b..09c00bd338 100644
--- a/unittests/test_policies.py
+++ b/unittests/test_policies.py
@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import contextlib
-import io
 import json
 import jsonschema
 import os
@@ -14,7 +13,6 @@
 import time
 
 import reframe as rfm
-import reframe.core.logging as logging
 import reframe.core.runtime as rt
 import reframe.frontend.dependencies as dependencies
 import reframe.frontend.executors as executors
@@ -22,7 +20,6 @@
 import reframe.frontend.reporting as reporting
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
-import reframe.utility.sanity as sn
 import unittests.utility as test_util
 
 from lxml import etree
@@ -32,7 +29,6 @@
                                      ReframeError,
                                      RunSessionTimeout,
                                      TaskDependencyError)
-from reframe.frontend.loader import RegressionCheckLoader
 from reframe.frontend.reporting import RunReport
 from unittests.resources.checks.hellocheck import HelloTest
 from unittests.resources.checks.frontend_checks import (
@@ -66,24 +62,6 @@ def timestamps(self):
         return self._time_start, self._time_end
 
 
-@pytest.fixture
-def make_loader():
-    def _make_loader(check_search_path, *args, **kwargs):
-        return RegressionCheckLoader(check_search_path, *args, **kwargs)
-
-    return _make_loader
-
-
-@pytest.fixture
-def common_exec_ctx(make_exec_ctx_g):
-    yield from make_exec_ctx_g(system='generic')
-
-
-@pytest.fixture
-def testsys_exec_ctx(make_exec_ctx_g):
-    yield from make_exec_ctx_g(system='testsys:gpu')
-
-
 def make_kbd_check(phase='wait'):
     return test_util.make_check(KeyboardInterruptCheck, phase=phase)
 
@@ -104,18 +82,6 @@ def _do_make_check(sleep_time, poll_fail=None):
     return _do_make_check
 
 
-@pytest.fixture(params=[policies.SerialExecutionPolicy,
-                        policies.AsynchronousExecutionPolicy])
-def make_runner(request):
-    def _make_runner(*args, **kwargs):
-        # Use a much higher poll rate for the unit tests
-        policy = request.param()
-        policy._pollctl.SLEEP_MIN = 0.001
-        return executors.Runner(policy, *args, **kwargs)
-
-    return _make_runner
-
-
 @pytest.fixture
 def make_cases(make_loader):
     def _make_cases(checks=None, sort=False, *args, **kwargs):
@@ -971,445 +937,3 @@ def assert_git_timeout(self):
     runner.runall(testcases)
     assert runner.stats.num_cases() == 2
     assert not runner.stats.failed()
-
-
-class _MyPerfTest(rfm.RunOnlyRegressionTest):
-    valid_systems = ['*']
-    valid_prog_environs = ['*']
-    executable = 'echo perf0=100 && echo perf1=50'
-
-    @sanity_function
-    def validate(self):
-        return sn.assert_found(r'perf0', self.stdout)
-
-    @performance_function('unit0')
-    def perf0(self):
-        return sn.extractsingle(r'perf0=(\S+)', self.stdout, 1, float)
-
-    @performance_function('unit1')
-    def perf1(self):
-        return sn.extractsingle(r'perf1=(\S+)', self.stdout, 1, float)
-
-
-class _MyPerfParamTest(_MyPerfTest):
-    p = parameter([1, 2])
-
-
-class _MyFailingTest(rfm.RunOnlyRegressionTest):
-    valid_systems = ['*']
-    valid_prog_environs = ['*']
-    executable = 'echo perf0=100'
-
-    @sanity_function
-    def validate(self):
-        return False
-
-    @performance_function('unit0')
-    def perf0(self):
-        return sn.extractsingle(r'perf0=(\S+)', self.stdout, 1, float)
-
-
-class _LazyPerfTest(rfm.RunOnlyRegressionTest):
-    valid_systems = ['*']
-    valid_prog_environs = ['*']
-    executable = 'echo perf0=100'
-
-    @sanity_function
-    def validate(self):
-        return True
-
-    @run_before('performance')
-    def set_perf_vars(self):
-        self.perf_variables = {
-            'perf0': sn.make_performance_function(
-                sn.extractsingle(r'perf0=(\S+)', self.stdout, 1, float),
-                'unit0'
-            )
-        }
-
-
-@pytest.fixture
-def perf_test():
-    return _MyPerfTest()
-
-
-@pytest.fixture
-def perf_param_tests():
-    return [_MyPerfParamTest(variant_num=v)
-            for v in range(_MyPerfParamTest.num_variants)]
-
-
-@pytest.fixture
-def failing_perf_test():
-    return _MyFailingTest()
-
-
-@pytest.fixture
-def lazy_perf_test():
-    return _LazyPerfTest()
-
-
-@pytest.fixture
-def simple_test():
-    class _MySimpleTest(rfm.RunOnlyRegressionTest):
-        valid_systems = ['*']
-        valid_prog_environs = ['*']
-        executable = 'echo hello'
-
-        @sanity_function
-        def validate(self):
-            return sn.assert_found(r'hello', self.stdout)
-
-    return _MySimpleTest()
-
-
-@pytest.fixture
-def config_perflog(make_config_file):
-    def _config_perflog(fmt, perffmt=None, logging_opts=None):
-        logging_config = {
-            'level': 'debug2',
-            'handlers': [{
-                'type': 'stream',
-                'name': 'stdout',
-                'level': 'info',
-                'format': '%(message)s'
-            }],
-            'handlers_perflog': [{
-                'type': 'filelog',
-                'prefix': '%(check_system)s/%(check_partition)s',
-                'level': 'info',
-                'format': fmt
-            }]
-        }
-        if logging_opts:
-            logging_config.update(logging_opts)
-
-        if perffmt is not None:
-            logging_config['handlers_perflog'][0]['format_perfvars'] = perffmt
-
-        return make_config_file({'logging': [logging_config]})
-
-    return _config_perflog
-
-
-def _count_lines(filepath):
-    count = 0
-    with open(filepath) as fp:
-        for line in fp:
-            count += 1
-
-    return count
-
-
-def _assert_header(filepath, header):
-    with open(filepath) as fp:
-        assert fp.readline().strip() == header
-
-
-def _assert_no_logging_error(fn, *args, **kwargs):
-    captured_stderr = io.StringIO()
-    with contextlib.redirect_stderr(captured_stderr):
-        fn(*args, **kwargs)
-
-    assert 'Logging error' not in captured_stderr.getvalue()
-
-
-def test_perf_logging(make_runner, make_exec_ctx, perf_test,
-                      config_perflog, tmp_path):
-    make_exec_ctx(
-        config_perflog(
-            fmt=(
-                '%(check_job_completion_time)s,%(version)s,'
-                '%(check_display_name)s,%(check_system)s,'
-                '%(check_partition)s,%(check_environ)s,'
-                '%(check_jobid)s,%(check_result)s,%(check_perfvalues)s'
-            ),
-            perffmt=(
-                '%(check_perf_value)s,%(check_perf_unit)s,'
-                '%(check_perf_ref)s,%(check_perf_lower_thres)s,'
-                '%(check_perf_upper_thres)s,'
-            )
-        )
-    )
-    logging.configure_logging(rt.runtime().site_config)
-    runner = make_runner()
-    testcases = executors.generate_testcases([perf_test])
-    runner.runall(testcases)
-
-    logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_MyPerfTest.log'
-    assert os.path.exists(logfile)
-    assert _count_lines(logfile) == 2
-
-    # Rerun with the same configuration and check that new entry is appended
-    testcases = executors.generate_testcases([perf_test])
-    runner = make_runner()
-    _assert_no_logging_error(runner.runall, testcases)
-    assert _count_lines(logfile) == 3
-
-    # Change the configuration and rerun
-    make_exec_ctx(
-        config_perflog(
-            fmt=(
-                '%(check_job_completion_time)s,%(version)s,'
-                '%(check_display_name)s,%(check_system)s,'
-                '%(check_partition)s,%(check_environ)s,'
-                '%(check_jobid)s,%(check_result)s,%(check_perfvalues)s'
-            ),
-            perffmt='%(check_perf_value)s,%(check_perf_unit)s,'
-        )
-    )
-    logging.configure_logging(rt.runtime().site_config)
-    testcases = executors.generate_testcases([perf_test])
-    runner = make_runner()
-    _assert_no_logging_error(runner.runall, testcases)
-    assert _count_lines(logfile) == 2
-    _assert_header(logfile,
-                   'job_completion_time,version,display_name,system,partition,'
-                   'environ,jobid,result,perf0_value,perf0_unit,'
-                   'perf1_value,perf1_unit')
-
-    logfile_prev = [(str(logfile) + '.h0', 3)]
-    for f, num_lines in logfile_prev:
-        assert os.path.exists(f)
-        _count_lines(f) == num_lines
-
-    # Change the test and rerun
-    perf_test.perf_variables['perfN'] = perf_test.perf_variables['perf1']
-
-    # We reconfigure the logging in order for the filelog handler to start
-    # from a clean state
-    logging.configure_logging(rt.runtime().site_config)
-    testcases = executors.generate_testcases([perf_test])
-    runner = make_runner()
-    _assert_no_logging_error(runner.runall, testcases)
-    assert _count_lines(logfile) == 2
-    _assert_header(logfile,
-                   'job_completion_time,version,display_name,system,partition,'
-                   'environ,jobid,result,perf0_value,perf0_unit,'
-                   'perf1_value,perf1_unit,perfN_value,perfN_unit')
-
-    logfile_prev = [(str(logfile) + '.h0', 3), (str(logfile) + '.h1', 2)]
-    for f, num_lines in logfile_prev:
-        assert os.path.exists(f)
-        _count_lines(f) == num_lines
-
-
-def test_perf_logging_no_end_delim(make_runner, make_exec_ctx, perf_test,
-                                   config_perflog, tmp_path):
-    make_exec_ctx(
-        config_perflog(
-            fmt=(
-                '%(check_job_completion_time)s,%(version)s,'
-                '%(check_display_name)s,%(check_system)s,'
-                '%(check_partition)s,%(check_environ)s,'
-                '%(check_jobid)s,%(check_result)s,%(check_perfvalues)s'
-            ),
-            perffmt='%(check_perf_value)s,%(check_perf_unit)s'
-        )
-    )
-    logging.configure_logging(rt.runtime().site_config)
-    runner = make_runner()
-    testcases = executors.generate_testcases([perf_test])
-    _assert_no_logging_error(runner.runall, testcases)
-
-    logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_MyPerfTest.log'
-    assert os.path.exists(logfile)
-    assert _count_lines(logfile) == 2
-
-    with open(logfile) as fp:
-        lines = fp.readlines()
-
-    assert len(lines) == 2
-    assert lines[0] == (
-        'job_completion_time,version,display_name,system,partition,'
-        'environ,jobid,result,perf0_value,perf0_unitperf1_value,perf1_unit\n'
-    )
-    assert '<error formatting the performance record' in lines[1]
-
-
-def test_perf_logging_no_perfvars(make_runner, make_exec_ctx, perf_test,
-                                  config_perflog, tmp_path):
-    make_exec_ctx(
-        config_perflog(
-            fmt=(
-                '%(check_job_completion_time)s,%(version)s,'
-                '%(check_display_name)s,%(check_system)s,'
-                '%(check_partition)s,%(check_environ)s,'
-                '%(check_jobid)s,%(check_result)s,%(check_perfvalues)s'
-            )
-        )
-    )
-    logging.configure_logging(rt.runtime().site_config)
-    runner = make_runner()
-    testcases = executors.generate_testcases([perf_test])
-    _assert_no_logging_error(runner.runall, testcases)
-
-    logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_MyPerfTest.log'
-    assert os.path.exists(logfile)
-    assert _count_lines(logfile) == 2
-
-    with open(logfile) as fp:
-        lines = fp.readlines()
-
-    assert len(lines) == 2
-    assert lines[0] == (
-        'job_completion_time,version,display_name,system,partition,'
-        'environ,jobid,result,\n'
-    )
-    assert 'error' not in lines[1]
-
-
-def test_perf_logging_multiline(make_runner, make_exec_ctx, perf_test,
-                                simple_test, failing_perf_test,
-                                config_perflog, tmp_path):
-    make_exec_ctx(
-        config_perflog(
-            fmt=('%(check_job_completion_time)s|reframe %(version)s|'
-                 '%(check_name)s|%(check_perf_var)s=%(check_perf_value)s|'
-                 'ref=%(check_perf_ref)s (l=%(check_perf_lower_thres)s, '
-                 'u=%(check_perf_upper_thres)s)|%(check_perf_unit)s'),
-            logging_opts={'perflog_compat': True}
-        )
-    )
-    logging.configure_logging(rt.runtime().site_config)
-    runner = make_runner()
-    testcases = executors.generate_testcases(
-        [perf_test, simple_test, failing_perf_test]
-    )
-    _assert_no_logging_error(runner.runall, testcases)
-
-    logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_MyPerfTest.log'
-    assert os.path.exists(logfile)
-    assert _count_lines(logfile) == 3
-
-    # assert that the emitted lines are correct
-    with open(logfile) as fp:
-        lines = fp.readlines()
-
-    version = osext.reframe_version()
-    assert lines[0] == ('job_completion_time|reframe version|name|'
-                        'perf_var=perf_value|ref=perf_ref '
-                        '(l=perf_lower_thres, u=perf_upper_thres)|perf_unit\n')
-    assert lines[1].endswith(
-        f'|reframe {version}|_MyPerfTest|'
-        f'perf0=100.0|ref=0 (l=null, u=null)|unit0\n'
-    )
-    assert lines[2].endswith(
-        f'|reframe {version}|_MyPerfTest|'
-        f'perf1=50.0|ref=0 (l=null, u=null)|unit1\n'
-    )
-
-
-def test_perf_logging_lazy(make_runner, make_exec_ctx, lazy_perf_test,
-                           config_perflog, tmp_path):
-    make_exec_ctx(
-        config_perflog(
-            fmt=(
-                '%(check_job_completion_time)s,%(version)s,'
-                '%(check_display_name)s,%(check_system)s,'
-                '%(check_partition)s,%(check_environ)s,'
-                '%(check_jobid)s,%(check_result)s,%(check_perfvalues)s'
-            ),
-            perffmt=(
-                '%(check_perf_value)s,%(check_perf_unit)s,'
-                '%(check_perf_ref)s,%(check_perf_lower_thres)s,'
-                '%(check_perf_upper_thres)s,'
-            )
-        )
-    )
-    logging.configure_logging(rt.runtime().site_config)
-    runner = make_runner()
-    testcases = executors.generate_testcases([lazy_perf_test])
-    _assert_no_logging_error(runner.runall, testcases)
-
-    logfile = (tmp_path / 'perflogs' / 'generic' / 'default' /
-               '_LazyPerfTest.log')
-    assert os.path.exists(logfile)
-
-
-@pytest.fixture(params=['%(check_result)s|%(check_#ALL)s', '%(check_#ALL)s'])
-def perflog_fmt(request):
-    return request.param
-
-
-def test_perf_logging_all_attrs(make_runner, make_exec_ctx, perf_test,
-                                config_perflog, tmp_path, perflog_fmt):
-    make_exec_ctx(config_perflog(fmt=perflog_fmt))
-    logging.configure_logging(rt.runtime().site_config)
-    runner = make_runner()
-    testcases = executors.generate_testcases([perf_test])
-    _assert_no_logging_error(runner.runall, testcases)
-
-    logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_MyPerfTest.log'
-    assert os.path.exists(logfile)
-    with open(logfile) as fp:
-        header = fp.readline()
-
-    loggable_attrs = type(perf_test).loggable_attrs()
-    assert (len(header.split('|')) ==
-            len(loggable_attrs) + (perflog_fmt != '%(check_#ALL)s'))
-
-
-def test_perf_logging_custom_vars(make_runner, make_exec_ctx,
-                                  config_perflog, tmp_path):
-    # Create two tests with different loggable variables
-    class _X(_MyPerfTest):
-        x = variable(int, value=1, loggable=True)
-
-    class _Y(_MyPerfTest):
-        y = variable(int, value=2, loggable=True)
-
-    make_exec_ctx(config_perflog(fmt='%(check_result)s|%(check_#ALL)s'))
-    logging.configure_logging(rt.runtime().site_config)
-    runner = make_runner()
-    testcases = executors.generate_testcases([_X(), _Y()])
-    _assert_no_logging_error(runner.runall, testcases)
-
-    logfiles = [tmp_path / 'perflogs' / 'generic' / 'default' / '_X.log',
-                tmp_path / 'perflogs' / 'generic' / 'default' / '_Y.log']
-    for f in logfiles:
-        with open(f) as fp:
-            header = fp.readline().strip()
-            if os.path.basename(f).startswith('_X'):
-                assert 'x' in header.split('|')
-            else:
-                assert 'y' in header.split('|')
-
-
-def test_perf_logging_param_test(make_runner, make_exec_ctx, perf_param_tests,
-                                 config_perflog, tmp_path):
-    make_exec_ctx(config_perflog(fmt='%(check_result)s|%(check_#ALL)s'))
-    logging.configure_logging(rt.runtime().site_config)
-    runner = make_runner()
-    testcases = executors.generate_testcases(perf_param_tests)
-    _assert_no_logging_error(runner.runall, testcases)
-
-    logfile = (tmp_path / 'perflogs' / 'generic' /
-               'default' / '_MyPerfParamTest.log')
-    assert os.path.exists(logfile)
-    assert _count_lines(logfile) == 3
-
-
-def test_perf_logging_sanity_failure(make_runner, make_exec_ctx,
-                                     config_perflog, tmp_path):
-    class _X(_MyPerfTest):
-        @sanity_function
-        def validate(self):
-            return sn.assert_true(0, msg='no way')
-
-    make_exec_ctx(config_perflog(
-        fmt='%(check_result)s|%(check_fail_reason)s|%(check_perfvalues)s',
-        perffmt='%(check_perf_value)s|'
-    ))
-    logging.configure_logging(rt.runtime().site_config)
-    runner = make_runner()
-    testcases = executors.generate_testcases([_X()])
-    _assert_no_logging_error(runner.runall, testcases)
-
-    logfile = tmp_path / 'perflogs' / 'generic' / 'default' / '_X.log'
-    assert os.path.exists(logfile)
-    with open(logfile) as fp:
-        lines = fp.readlines()
-
-    assert len(lines) == 2
-    assert lines[1] == 'fail|sanity error: no way|None|None\n'

From 7d9b9da376e21c7e4301a7a39dc4c74c0aeeb6b8 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 18 Jul 2024 13:01:39 +0300
Subject: [PATCH 054/128] More report tests to a `test_reporting.py`

---
 unittests/conftest.py       |  28 +++++
 unittests/test_policies.py  | 214 +-----------------------------------
 unittests/test_reporting.py | 174 +++++++++++++++++++++++++++++
 3 files changed, 208 insertions(+), 208 deletions(-)
 create mode 100644 unittests/test_reporting.py

diff --git a/unittests/conftest.py b/unittests/conftest.py
index 62d6ca126d..69294e4100 100644
--- a/unittests/conftest.py
+++ b/unittests/conftest.py
@@ -14,6 +14,7 @@
 
 import reframe.core.settings as settings
 import reframe.core.runtime as rt
+import reframe.frontend.dependencies as dependencies
 import reframe.frontend.executors as executors
 import reframe.frontend.executors.policies as policies
 import reframe.utility as util
@@ -118,6 +119,33 @@ def _make_runner(*args, **kwargs):
     return _make_runner
 
 
+@pytest.fixture
+def make_cases(make_loader):
+    def _make_cases(checks=None, sort=False, *args, **kwargs):
+        if checks is None:
+            checks = make_loader(
+                ['unittests/resources/checks'], *args, **kwargs
+            ).load_all(force=True)
+
+        cases = executors.generate_testcases(checks)
+        if sort:
+            depgraph, _ = dependencies.build_deps(cases)
+            dependencies.validate_deps(depgraph)
+            cases = dependencies.toposort(depgraph)
+
+        return cases
+
+    return _make_cases
+
+
+@pytest.fixture
+def cases_with_deps(make_loader, make_cases):
+    checks = make_loader(
+        ['unittests/resources/checks_unlisted/deps_complex.py']
+    ).load_all()
+    return make_cases(checks, sort=True)
+
+
 @pytest.fixture
 def make_config_file(tmp_path):
     '''Create a temporary configuration file from the given configuration.
diff --git a/unittests/test_policies.py b/unittests/test_policies.py
index 09c00bd338..37e40df5e9 100644
--- a/unittests/test_policies.py
+++ b/unittests/test_policies.py
@@ -4,32 +4,22 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import contextlib
-import json
-import jsonschema
 import os
 import pytest
-import socket
-import sys
-import time
 
 import reframe as rfm
 import reframe.core.runtime as rt
 import reframe.frontend.dependencies as dependencies
 import reframe.frontend.executors as executors
 import reframe.frontend.executors.policies as policies
-import reframe.frontend.reporting as reporting
-import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
 import unittests.utility as test_util
 
-from lxml import etree
 from reframe.core.exceptions import (AbortTaskError,
                                      FailureLimitError,
                                      ForceExitError,
-                                     ReframeError,
                                      RunSessionTimeout,
                                      TaskDependencyError)
-from reframe.frontend.reporting import RunReport
 from unittests.resources.checks.hellocheck import HelloTest
 from unittests.resources.checks.frontend_checks import (
     BadSetupCheck,
@@ -43,25 +33,6 @@
 )
 
 
-# NOTE: We could move this to utility
-class timer:
-    '''Context manager for timing'''
-
-    def __init__(self):
-        self._time_start = None
-        self._time_end = None
-
-    def __enter__(self):
-        self._time_start = time.time()
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self._time_end = time.time()
-
-    def timestamps(self):
-        return self._time_start, self._time_end
-
-
 def make_kbd_check(phase='wait'):
     return test_util.make_check(KeyboardInterruptCheck, phase=phase)
 
@@ -82,25 +53,6 @@ def _do_make_check(sleep_time, poll_fail=None):
     return _do_make_check
 
 
-@pytest.fixture
-def make_cases(make_loader):
-    def _make_cases(checks=None, sort=False, *args, **kwargs):
-        if checks is None:
-            checks = make_loader(
-                ['unittests/resources/checks'], *args, **kwargs
-            ).load_all(force=True)
-
-        cases = executors.generate_testcases(checks)
-        if sort:
-            depgraph, _ = dependencies.build_deps(cases)
-            dependencies.validate_deps(depgraph)
-            cases = dependencies.toposort(depgraph)
-
-        return cases
-
-    return _make_cases
-
-
 @pytest.fixture(params=['pre_setup', 'post_setup',
                         'pre_compile', 'post_compile',
                         'pre_run', 'post_run',
@@ -163,50 +115,9 @@ def num_failures_stage(runner, stage):
     return len([t for t in stats.failed() if t.failed_stage == stage])
 
 
-def _validate_runreport(report):
-    schema_filename = 'reframe/schemas/runreport.json'
-    with open(schema_filename) as fp:
-        schema = json.loads(fp.read())
-
-    jsonschema.validate(json.loads(report), schema)
-
-
-def _validate_junit_report(report):
-    # Cloned from
-    # https://raw.githubusercontent.com/windyroad/JUnit-Schema/master/JUnit.xsd
-    schema_file = 'reframe/schemas/junit.xsd'
-    with open(schema_file, encoding='utf-8') as fp:
-        schema = etree.XMLSchema(etree.parse(fp))
-
-    schema.assert_(report)
-
-
-def _generate_runreport(run_stats, time_start=None, time_end=None):
-    report = RunReport()
-    report.update_session_info({
-        'cmdline': ' '.join(sys.argv),
-        'config_files': rt.runtime().site_config.sources,
-        'data_version': reporting.DATA_VERSION,
-        'hostname': socket.gethostname(),
-        'prefix_output': rt.runtime().output_prefix,
-        'prefix_stage': rt.runtime().stage_prefix,
-        'user': osext.osuser(),
-        'version': osext.reframe_version(),
-        'workdir': os.getcwd()
-    })
-    if time_start and time_end:
-        report.update_timestamps(time_start, time_end)
-
-    if run_stats:
-        report.update_run_stats(run_stats)
-
-    return report
-
-
 def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path):
     runner = make_runner()
-    with timer() as tm:
-        runner.runall(make_cases())
+    runner.runall(make_cases())
 
     assert 9 == runner.stats.num_cases()
     assert_runall(runner)
@@ -216,39 +127,6 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path):
     assert 1 == num_failures_stage(runner, 'performance')
     assert 1 == num_failures_stage(runner, 'cleanup')
 
-    # We dump the report first, in order to get any object conversions right
-    report = _generate_runreport(runner.stats, *tm.timestamps())
-    report.save(tmp_path / 'report.json')
-
-    # We explicitly set `time_total` to `None` in the last test case, in order
-    # to test the proper handling of `None`.
-    report['runs'][0]['testcases'][-1]['time_total'] = None
-
-    # Validate the junit report
-    _validate_junit_report(report.generate_xml_report())
-
-    # Read and validate the report using the `reporting` module
-    reporting.restore_session(tmp_path / 'report.json')
-
-    # Try to load a non-existent report
-    with pytest.raises(ReframeError, match='failed to load report file'):
-        reporting.restore_session(tmp_path / 'does_not_exist.json')
-
-    # Generate an invalid JSON
-    with open(tmp_path / 'invalid.json', 'w') as fp:
-        jsonext.dump(report, fp)
-        fp.write('invalid')
-
-    with pytest.raises(ReframeError, match=r'is not a valid JSON file'):
-        reporting.restore_session(tmp_path / 'invalid.json')
-
-    # Generate a report that does not comply to the schema
-    del report['session_info']['data_version']
-    report.save(tmp_path / 'invalid-version.json')
-    with pytest.raises(ReframeError,
-                       match=r'failed to validate report'):
-        reporting.restore_session(tmp_path / 'invalid-version.json')
-
 
 def test_runall_skip_system_check(make_runner, make_cases, common_exec_ctx):
     runner = make_runner()
@@ -339,18 +217,6 @@ def test_runall_skip_tests(make_runner, make_cases,
     more_cases, stage = make_cases_for_skipping()
     cases = make_cases() + more_cases
     runner.runall(cases)
-
-    def assert_reported_skipped(num_skipped):
-        run_report = _generate_runreport(runner.stats)['runs']
-        assert run_report[0]['num_skipped'] == num_skipped
-
-        num_reported = 0
-        for tc in run_report[0]['testcases']:
-            if tc['result'] == 'skip':
-                num_reported += 1
-
-        assert num_reported == num_skipped
-
     assert_runall(runner)
     assert 11 == runner.stats.num_cases(0)
     assert 5 == runner.stats.num_cases(1)
@@ -359,11 +225,9 @@ def assert_reported_skipped(num_skipped):
     if stage.endswith('cleanup'):
         assert 0 == len(runner.stats.skipped(0))
         assert 0 == len(runner.stats.skipped(1))
-        assert_reported_skipped(0)
     else:
         assert 2 == len(runner.stats.skipped(0))
         assert 0 == len(runner.stats.skipped(1))
-        assert_reported_skipped(2)
 
 
 # We explicitly ask for a system with a non-local scheduler here, to make sure
@@ -478,18 +342,6 @@ def test_duration_limit(make_runner, make_cases, common_exec_ctx):
             assert num_aborted == 1
 
 
-@pytest.fixture
-def dep_checks(make_loader):
-    return make_loader(
-        ['unittests/resources/checks_unlisted/deps_complex.py']
-    ).load_all()
-
-
-@pytest.fixture
-def dep_cases(dep_checks, make_cases):
-    return make_cases(dep_checks, sort=True)
-
-
 def assert_dependency_run(runner):
     assert_runall(runner)
     stats = runner.stats
@@ -511,15 +363,16 @@ def assert_dependency_run(runner):
             assert os.path.exists(os.path.join(check.outputdir, 'out.txt'))
 
 
-def test_dependencies(make_runner, dep_cases, common_exec_ctx):
+def test_dependencies(make_runner, cases_with_deps, common_exec_ctx):
     runner = make_runner()
-    runner.runall(dep_cases)
+    runner.runall(cases_with_deps)
     assert_dependency_run(runner)
 
 
-def test_dependencies_with_retries(make_runner, dep_cases, common_exec_ctx):
+def test_dependencies_with_retries(make_runner, cases_with_deps,
+                                   common_exec_ctx):
     runner = make_runner(max_retries=2)
-    runner.runall(dep_cases)
+    runner.runall(cases_with_deps)
     assert_dependency_run(runner)
 
 
@@ -856,61 +709,6 @@ def test_compile_fail_reschedule_busy_loop(make_async_runner, make_cases,
     assert num_checks == len(stats.failed())
 
 
-@pytest.fixture
-def report_file(make_runner, dep_cases, common_exec_ctx, tmp_path):
-    runner = make_runner()
-    runner.policy.keep_stage_files = True
-    with timer() as tm:
-        runner.runall(dep_cases)
-
-    filename = tmp_path / 'report.json'
-    report = _generate_runreport(runner.stats, *tm.timestamps())
-    report.save(filename)
-    return filename
-
-
-def test_restore_session(report_file, make_runner,
-                         dep_cases, common_exec_ctx, tmp_path):
-    # Select a single test to run and create the pruned graph
-    selected = [tc for tc in dep_cases if tc.check.name == 'T1']
-    testgraph = dependencies.prune_deps(
-        dependencies.build_deps(dep_cases)[0], selected, max_depth=1
-    )
-
-    # Restore the required test cases
-    report = reporting.restore_session(report_file)
-    testgraph, restored_cases = report.restore_dangling(testgraph)
-
-    assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'}
-
-    # Run the selected test cases
-    runner = make_runner()
-    with timer() as tm:
-        runner.runall(selected, restored_cases)
-
-    new_report = _generate_runreport(runner.stats, *tm.timestamps())
-    assert new_report['runs'][0]['num_cases'] == 1
-    assert new_report['runs'][0]['testcases'][0]['name'] == 'T1'
-
-    # Generate an empty report and load it as primary with the original report
-    # as a fallback, in order to test if the dependencies are still resolved
-    # correctly
-    empty_report = _generate_runreport(None, *tm.timestamps())
-    empty_report_file = tmp_path / 'empty.json'
-    empty_report.save(empty_report_file)
-
-    report2 = reporting.restore_session(empty_report_file, report_file)
-    restored_cases = report2.restore_dangling(testgraph)[1]
-    assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'}
-
-    # Remove the test case dump file and retry
-    os.remove(tmp_path / 'stage' / 'generic' / 'default' /
-              'builtin' / 'T4' / '.rfm_testcase.json')
-
-    with pytest.raises(ReframeError, match=r'could not restore testcase'):
-        report.restore_dangling(testgraph)
-
-
 def test_config_params(make_runner, make_exec_ctx):
     '''Test that configuration parameters are properly retrieved with the
     various execution policies.
diff --git a/unittests/test_reporting.py b/unittests/test_reporting.py
new file mode 100644
index 0000000000..2f809d0c64
--- /dev/null
+++ b/unittests/test_reporting.py
@@ -0,0 +1,174 @@
+# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# ReFrame Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+import json
+import jsonschema
+import os
+import pytest
+import socket
+import sys
+import time
+from lxml import etree
+
+import reframe.core.runtime as rt
+import reframe.utility.osext as osext
+import reframe.utility.jsonext as jsonext
+import reframe.frontend.dependencies as dependencies
+import reframe.frontend.reporting as reporting
+from reframe.core.exceptions import ReframeError
+from reframe.frontend.reporting import RunReport
+
+
+# NOTE: We could move this to utility
+class _timer:
+    '''Context manager for timing'''
+
+    def __init__(self):
+        self._time_start = None
+        self._time_end = None
+
+    def __enter__(self):
+        self._time_start = time.time()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self._time_end = time.time()
+
+    def timestamps(self):
+        return self._time_start, self._time_end
+
+
+def _validate_runreport(report):
+    schema_filename = 'reframe/schemas/runreport.json'
+    with open(schema_filename) as fp:
+        schema = json.loads(fp.read())
+
+    jsonschema.validate(json.loads(report), schema)
+
+
+def _validate_junit_report(report):
+    # Cloned from
+    # https://raw.githubusercontent.com/windyroad/JUnit-Schema/master/JUnit.xsd
+    schema_file = 'reframe/schemas/junit.xsd'
+    with open(schema_file, encoding='utf-8') as fp:
+        schema = etree.XMLSchema(etree.parse(fp))
+
+    schema.assert_(report)
+
+
+def _generate_runreport(run_stats, time_start=None, time_end=None):
+    report = RunReport()
+    report.update_session_info({
+        'cmdline': ' '.join(sys.argv),
+        'config_files': rt.runtime().site_config.sources,
+        'data_version': reporting.DATA_VERSION,
+        'hostname': socket.gethostname(),
+        'prefix_output': rt.runtime().output_prefix,
+        'prefix_stage': rt.runtime().stage_prefix,
+        'user': osext.osuser(),
+        'version': osext.reframe_version(),
+        'workdir': os.getcwd()
+    })
+    if time_start and time_end:
+        report.update_timestamps(time_start, time_end)
+
+    if run_stats:
+        report.update_run_stats(run_stats)
+
+    return report
+
+
+def test_run_report(make_runner, make_cases, common_exec_ctx, tmp_path):
+    runner = make_runner()
+    with _timer() as tm:
+        runner.runall(make_cases())
+
+    # We dump the report first, in order to get any object conversions right
+    report = _generate_runreport(runner.stats, *tm.timestamps())
+    report.save(tmp_path / 'report.json')
+
+    # We explicitly set `time_total` to `None` in the last test case, in order
+    # to test the proper handling of `None`.
+    report['runs'][0]['testcases'][-1]['time_total'] = None
+
+    # Validate the junit report
+    _validate_junit_report(report.generate_xml_report())
+
+    # Read and validate the report using the `reporting` module
+    reporting.restore_session(tmp_path / 'report.json')
+
+    # Try to load a non-existent report
+    with pytest.raises(ReframeError, match='failed to load report file'):
+        reporting.restore_session(tmp_path / 'does_not_exist.json')
+
+    # Generate an invalid JSON
+    with open(tmp_path / 'invalid.json', 'w') as fp:
+        jsonext.dump(report, fp)
+        fp.write('invalid')
+
+    with pytest.raises(ReframeError, match=r'is not a valid JSON file'):
+        reporting.restore_session(tmp_path / 'invalid.json')
+
+    # Generate a report that does not comply to the schema
+    del report['session_info']['data_version']
+    report.save(tmp_path / 'invalid-version.json')
+    with pytest.raises(ReframeError,
+                       match=r'failed to validate report'):
+        reporting.restore_session(tmp_path / 'invalid-version.json')
+
+
+@pytest.fixture
+def report_file(make_runner, cases_with_deps, common_exec_ctx, tmp_path):
+    runner = make_runner()
+    runner.policy.keep_stage_files = True
+    with _timer() as tm:
+        runner.runall(cases_with_deps)
+
+    filename = tmp_path / 'report.json'
+    report = _generate_runreport(runner.stats, *tm.timestamps())
+    report.save(filename)
+    return filename
+
+
+def test_restore_session(report_file, make_runner, cases_with_deps,
+                         common_exec_ctx, tmp_path):
+    # Select a single test to run and create the pruned graph
+    selected = [tc for tc in cases_with_deps if tc.check.name == 'T1']
+    testgraph = dependencies.prune_deps(
+        dependencies.build_deps(cases_with_deps)[0], selected, max_depth=1
+    )
+
+    # Restore the required test cases
+    report = reporting.restore_session(report_file)
+    testgraph, restored_cases = report.restore_dangling(testgraph)
+
+    assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'}
+
+    # Run the selected test cases
+    runner = make_runner()
+    with _timer() as tm:
+        runner.runall(selected, restored_cases)
+
+    new_report = _generate_runreport(runner.stats, *tm.timestamps())
+    assert new_report['runs'][0]['num_cases'] == 1
+    assert new_report['runs'][0]['testcases'][0]['name'] == 'T1'
+
+    # Generate an empty report and load it as primary with the original report
+    # as a fallback, in order to test if the dependencies are still resolved
+    # correctly
+    empty_report = _generate_runreport(None, *tm.timestamps())
+    empty_report_file = tmp_path / 'empty.json'
+    empty_report.save(empty_report_file)
+
+    report2 = reporting.restore_session(empty_report_file, report_file)
+    restored_cases = report2.restore_dangling(testgraph)[1]
+    assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'}
+
+    # Remove the test case dump file and retry
+    os.remove(tmp_path / 'stage' / 'generic' / 'default' /
+              'builtin' / 'T4' / '.rfm_testcase.json')
+
+    with pytest.raises(ReframeError, match=r'could not restore testcase'):
+        report.restore_dangling(testgraph)

From b8d8e767e6f271bbf25e644283469cb00e70957e Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 18 Jul 2024 17:54:15 +0300
Subject: [PATCH 055/128] Add unit tests for reporting backend

---
 reframe/frontend/executors/__init__.py |   3 +-
 reframe/frontend/reporting/__init__.py |   3 +
 reframe/frontend/reporting/storage.py  |  11 +-
 reframe/frontend/reporting/utility.py  |  17 +-
 unittests/conftest.py                  |  10 +
 unittests/test_reporting.py            | 278 ++++++++++++++++++++++++-
 6 files changed, 303 insertions(+), 19 deletions(-)

diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index 8bf3f58117..d96cbaeb72 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -314,7 +314,8 @@ def failed_stage(self):
 
     @property
     def succeeded(self):
-        return self._current_stage in {'finalize', 'cleanup'}
+        return (self._current_stage in {'finalize', 'cleanup'} and
+                not self._failed_stage == 'cleanup')
 
     @property
     def completed(self):
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 6139802cf2..3cca83792b 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -235,6 +235,9 @@ def __init__(self):
         now = time.time()
         self.update_timestamps(now, now)
 
+    def data(self):
+        return self.__report
+
     @property
     def filename(self):
         return self.__filename
diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 0fc89d5743..e4548de3fb 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -149,8 +149,8 @@ def _db_store_report(self, conn, report, report_file_path):
 
     def store(self, report, report_file=None):
         prefix = os.path.dirname(self.__db_file)
-        with FileLock(os.path.join(prefix, '.db.lock')):
-            with sqlite3.connect(self._db_file()) as conn:
+        with sqlite3.connect(self._db_file()) as conn:
+            with FileLock(os.path.join(prefix, '.db.lock')):
                 return self._db_store_report(conn, report, report_file)
 
     def _fetch_testcases_raw(self, condition):
@@ -231,6 +231,9 @@ def remove_session(self, uuid):
         prefix = os.path.dirname(self.__db_file)
         with FileLock(os.path.join(prefix, '.db.lock')):
             with sqlite3.connect(self._db_file()) as conn:
-                query = f'DELETE FROM sessions WHERE uuid == "{uuid}"'
+                query = (f'DELETE FROM sessions WHERE uuid == "{uuid}" '
+                         'RETURNING *')
                 getlogger().debug(query)
-                conn.execute(query)
+                deleted = conn.execute(query).fetchall()
+                if not deleted:
+                    raise ReframeError(f'no such session: {uuid}')
diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py
index 6d4433815b..2da8030f53 100644
--- a/reframe/frontend/reporting/utility.py
+++ b/reframe/frontend/reporting/utility.py
@@ -110,7 +110,7 @@ def _do_parse(s):
     except ValueError as err:
         # Try the relative timestamps
         match = re.match(
-            r'(?P<ts>.*)(?P<amount>[\+|-]\d+)(?P<unit>[hdmsw])', s
+            r'(?P<ts>.*)(?P<amount>[\+|-]\d+)(?P<unit>[mhdw])', s
         )
         if not match:
             raise err
@@ -122,12 +122,10 @@ def _do_parse(s):
             ts += timedelta(weeks=amount)
         elif unit == 'd':
             ts += timedelta(days=amount)
-        elif unit == 'm':
-            ts += timedelta(minutes=amount)
         elif unit == 'h':
             ts += timedelta(hours=amount)
-        elif unit == 's':
-            ts += timedelta(seconds=amount)
+        elif unit == 'm':
+            ts += timedelta(minutes=amount)
 
     return ts.timestamp()
 
@@ -156,12 +154,11 @@ def parse_time_period(s):
 
 
 def _parse_extra_cols(s):
-    try:
-        extra_cols = s.split('+')[1:]
-    except (ValueError, IndexError):
-        raise ValueError(f'invalid extra groups spec: {s}') from None
+    if s and not s.startswith('+'):
+        raise ValueError(f'invalid column spec: {s}')
 
-    return extra_cols
+    # Remove any empty columns
+    return [x for x in s.split('+')[1:] if x]
 
 
 def _parse_aggregation(s):
diff --git a/unittests/conftest.py b/unittests/conftest.py
index 69294e4100..711716d174 100644
--- a/unittests/conftest.py
+++ b/unittests/conftest.py
@@ -119,6 +119,16 @@ def _make_runner(*args, **kwargs):
     return _make_runner
 
 
+@pytest.fixture
+def make_async_runner():
+    def _make_runner(*args, **kwargs):
+        policy = policies.AsynchronousExecutionPolicy()
+        policy._pollctl.SLEEP_MIN = 0.001
+        return executors.Runner(policy, *args, **kwargs)
+
+    return _make_runner
+
+
 @pytest.fixture
 def make_cases(make_loader):
     def _make_cases(checks=None, sort=False, *args, **kwargs):
diff --git a/unittests/test_reporting.py b/unittests/test_reporting.py
index 2f809d0c64..a25d6410b4 100644
--- a/unittests/test_reporting.py
+++ b/unittests/test_reporting.py
@@ -7,7 +7,6 @@
 import jsonschema
 import os
 import pytest
-import socket
 import sys
 import time
 from lxml import etree
@@ -17,6 +16,8 @@
 import reframe.utility.jsonext as jsonext
 import reframe.frontend.dependencies as dependencies
 import reframe.frontend.reporting as reporting
+import reframe.frontend.reporting.storage as report_storage
+import reframe.frontend.reporting.utility as report_util
 from reframe.core.exceptions import ReframeError
 from reframe.frontend.reporting import RunReport
 
@@ -45,7 +46,7 @@ def _validate_runreport(report):
     with open(schema_filename) as fp:
         schema = json.loads(fp.read())
 
-    jsonschema.validate(json.loads(report), schema)
+    jsonschema.validate(jsonext.loads(jsonext.dumps(report)), schema)
 
 
 def _validate_junit_report(report):
@@ -63,8 +64,6 @@ def _generate_runreport(run_stats, time_start=None, time_end=None):
     report.update_session_info({
         'cmdline': ' '.join(sys.argv),
         'config_files': rt.runtime().site_config.sources,
-        'data_version': reporting.DATA_VERSION,
-        'hostname': socket.gethostname(),
         'prefix_output': rt.runtime().output_prefix,
         'prefix_stage': rt.runtime().stage_prefix,
         'user': osext.osuser(),
@@ -88,6 +87,7 @@ def test_run_report(make_runner, make_cases, common_exec_ctx, tmp_path):
     # We dump the report first, in order to get any object conversions right
     report = _generate_runreport(runner.stats, *tm.timestamps())
     report.save(tmp_path / 'report.json')
+    _validate_runreport(report)
 
     # We explicitly set `time_total` to `None` in the last test case, in order
     # to test the proper handling of `None`.
@@ -172,3 +172,273 @@ def test_restore_session(report_file, make_runner, cases_with_deps,
 
     with pytest.raises(ReframeError, match=r'could not restore testcase'):
         report.restore_dangling(testgraph)
+
+
+@pytest.fixture(params=[
+    ('20240701:20240701T0100', 3600),
+    ('20240701T0000:20240701T0010', 600),
+    ('20240701T000000:20240701T001010', 610),
+    ('20240701T000000+0000:20240701T000000+0100', -3600),
+    ('20240701T000000+0000:20240701T000000-0100', 3600),
+    ('20240701T0000:20240701T0000+1m', 60),
+    ('20240701T0000:20240701T0000-1m', -60),
+    ('20240701T0000:20240701T0000+1h', 3600),
+    ('20240701T0000:20240701T0000-1h', -3600),
+    ('20240701T0000:20240701T0000+1d', 86400),
+    ('20240701T0000:20240701T0000-1d', -86400),
+    ('20240701T0000:20240701T0000+1w', 604800),
+    ('20240701T0000:20240701T0000-1w', -604800),
+    ('now:now+1m', 60),
+    ('now:now-1m', -60),
+    ('now:now+1h', 3600),
+    ('now:now-1h', -3600),
+    ('now:now+1d', 86400),
+    ('now:now-1d', -86400),
+    ('now:now+1w', 604800),
+    ('now:now-1w', -604800)
+])
+def time_period(request):
+    return request.param
+
+
+def test_parse_cmp_spec_period(time_period):
+    spec, duration = time_period
+    duration = int(duration)
+    match = report_util.parse_cmp_spec(f'{spec}/{spec}/mean:/')
+    for period in ('period_base', 'period_target'):
+        ts_start, ts_end = getattr(match, period)
+        if 'now' in spec:
+            # Truncate splits of seconds if using `now` timestamps
+            ts_start = int(ts_start)
+            ts_end = int(ts_end)
+
+        assert ts_end - ts_start == duration
+
+    # Check variant without base period
+    match = report_util.parse_cmp_spec(f'{spec}/mean:/')
+    assert match.period_base is None
+
+
+@pytest.fixture(params=['first', 'last', 'mean', 'median',
+                        'min', 'max'])
+def aggregator(request):
+    return request.param
+
+
+def test_parse_cmp_spec_aggregations(aggregator):
+    match = report_util.parse_cmp_spec(f'now-1m:now/now-1d:now/{aggregator}:/')
+    data = [1, 2, 3, 4, 5]
+    if aggregator == 'first':
+        match.aggregator(data) == data[0]
+    elif aggregator == 'last':
+        match.aggregator(data) == data[-1]
+    elif aggregator == 'min':
+        match.aggregator(data) == 1
+    elif aggregator == 'max':
+        match.aggregator(data) == 5
+    elif aggregator == 'median':
+        match.aggregator(data) == 3
+    elif aggregator == 'mean':
+        match.aggregator(data) == sum(data) / len(data)
+
+    # Check variant without base period
+    match = report_util.parse_cmp_spec(f'now-1d:now/{aggregator}:/')
+    assert match.period_base is None
+
+
+@pytest.fixture(params=[('', []), ('+', []),
+                        ('+col1', ['col1']), ('+col1+', ['col1']),
+                        ('+col1+col2', ['col1', 'col2'])])
+def extra_cols(request):
+    return request.param
+
+
+def test_parse_cmp_spec_group_by(extra_cols):
+    spec, expected = extra_cols
+    match = report_util.parse_cmp_spec(
+        f'now-1m:now/now-1d:now/min:{spec}/'
+    )
+    assert match.extra_groups == expected
+
+    # Check variant without base period
+    match = report_util.parse_cmp_spec(f'now-1d:now/min:{spec}/')
+    assert match.period_base is None
+
+
+def test_parse_cmp_spec_extra_cols(extra_cols):
+    spec, expected = extra_cols
+    match = report_util.parse_cmp_spec(
+        f'now-1m:now/now-1d:now/min:/{spec}'
+    )
+    assert match.extra_cols == expected
+
+    # Check variant without base period
+    match = report_util.parse_cmp_spec(f'now-1d:now/min:/{spec}')
+    assert match.period_base is None
+
+
+@pytest.fixture(params=['^uuid/now-1d:now/min:/',
+                        'now-1d:now/^uuid/min:/',
+                        '^uuid0/^uuid1/min:/',
+                        'now-1m:now/now-1d:now/min:/'])
+def uuid_spec(request):
+    return request.param
+
+
+def test_parse_cmp_spec_with_uuid(uuid_spec):
+    def _uuids(s):
+        parts = s.split('/')
+        base, target = None, None
+        if len(parts) == 3:
+            base = None
+            target = parts[0][1:] if parts[0].startswith('^') else None
+        else:
+            base = parts[0][1:] if parts[0].startswith('^') else None
+            target = parts[1][1:] if parts[1].startswith('^') else None
+
+        return base, target
+
+    match = report_util.parse_cmp_spec(uuid_spec)
+    base_uuid, target_uuid = _uuids(uuid_spec)
+    assert match.session_base == base_uuid
+    assert match.session_target == target_uuid
+
+
+@pytest.fixture(params=['2024:07:01T12:34:56', '20240701', '20240701:',
+                        '20240701T:now', 'now-1v:now', 'now:then',
+                        '20240701:now:'])
+def invalid_time_period(request):
+    return request.param
+
+
+def test_parse_cmp_spec_invalid_period(invalid_time_period):
+    with pytest.raises(ValueError):
+        report_util.parse_cmp_spec(f'{invalid_time_period}/now-1d:now/min:/')
+
+    with pytest.raises(ValueError):
+        report_util.parse_cmp_spec(f'now-1d:now/{invalid_time_period}/min:/')
+
+
+@pytest.fixture(params=['mean', 'foo:', 'mean:col1+col2', 'mean:col1,col2'])
+def invalid_aggr_spec(request):
+    return request.param
+
+
+def test_parse_cmp_spec_invalid_aggregation(invalid_aggr_spec):
+    with pytest.raises(ValueError):
+        report_util.parse_cmp_spec(
+            f'now-1m:now/now-1d:now/{invalid_aggr_spec}/'
+        )
+
+
+@pytest.fixture(params=['col1+col2', 'col1,col2'])
+def invalid_col_spec(request):
+    return request.param
+
+
+def test_parse_cmp_spec_invalid_extra_cols(invalid_col_spec):
+    with pytest.raises(ValueError):
+        report_util.parse_cmp_spec(
+            f'now-1m:now/now-1d:now/mean:/{invalid_col_spec}'
+        )
+
+
+@pytest.fixture(params=['now-1d:now',
+                        'now-1m:now/now-1d:now',
+                        'now-1m:now/now-1d:now/mean',
+                        'now-1m:now/now-1d:now/mean:',
+                        'now-1m:now/now-1d:now/mean:',
+                        '/now-1d:now/mean:/',
+                        'now-1m:now//mean:'])
+def various_invalid_specs(request):
+    return request.param
+
+
+def test_parse_cmp_spec_various_invalid(various_invalid_specs):
+    with pytest.raises(ValueError):
+        report_util.parse_cmp_spec(various_invalid_specs)
+
+
+def test_storage_api(make_async_runner, make_cases, common_exec_ctx,
+                     monkeypatch, tmp_path):
+    def _count_failed(testcases):
+        count = 0
+        for tc in testcases:
+            if tc['result'] == 'fail':
+                count += 1
+
+        return count
+
+    monkeypatch.setenv('HOME', str(tmp_path))
+    uuids = []
+    timestamps = []
+    for _ in range(2):
+        runner = make_async_runner()
+        with _timer() as tm:
+            runner.runall(make_cases())
+
+        timestamps.append(tm.timestamps())
+        report = _generate_runreport(runner.stats, *tm.timestamps())
+        uuids.append(report.store())
+
+    backend = report_storage.StorageBackend.default()
+
+    # Test `fetch_all_sessions`
+    stored_sessions = backend.fetch_all_sessions()
+    assert len(stored_sessions) == 2
+    for i, sess in enumerate(stored_sessions):
+        assert sess['session_info']['uuid'] == uuids[i]
+
+    # Test `fetch_session_json`
+    for uuid in uuids:
+        stored_session = backend.fetch_session_json(uuid)
+        assert stored_session['session_info']['uuid'] == uuid
+
+    # Test an invalid uuid
+    assert backend.fetch_session_json(0) == {}
+
+    # Test `fetch_session_time_period`
+    for i, uuid in enumerate(uuids):
+        ts_session = backend.fetch_session_time_period(uuid)
+        assert ts_session == timestamps[i]
+
+    # Test an invalid uuid
+    assert backend.fetch_session_time_period(0) == (None, None)
+
+    # Test `fetch_testcases_time_period`
+    testcases = backend.fetch_testcases_time_period(timestamps[0][0],
+                                                    timestamps[1][1])
+
+    # NOTE: test cases without an associated (run) job are not fetched by
+    # `fetch_testcases_time_period`;  in
+    # this case 3 test cases per session are ignored: `BadSetupCheckEarly`,
+    # `BadSetupCheck`, `CompileOnlyHelloTest`, which requires us to adapt the
+    # expected counts below
+    assert len(testcases) == 12
+    assert _count_failed(testcases) == 6
+
+    # Test the inverted period
+    assert backend.fetch_testcases_time_period(timestamps[1][1],
+                                               timestamps[0][0]) == []
+
+    # Test `fetch_testcases_from_session`
+    for i, uuid in enumerate(uuids):
+        testcases = backend.fetch_testcases_from_session(uuid)
+        assert len(testcases) == 9
+        assert _count_failed(testcases) == 5
+
+    # Test an invalid uuid
+    assert backend.fetch_testcases_from_session(0) == []
+
+    # Test session removal
+    backend.remove_session(uuids[-1])
+    assert len(backend.fetch_all_sessions()) == 1
+
+    testcases = backend.fetch_testcases_time_period(timestamps[0][0],
+                                                    timestamps[1][1])
+    assert len(testcases) == 6
+    assert _count_failed(testcases) == 3
+
+    # Try an invalid uuid
+    with pytest.raises(ReframeError):
+        backend.remove_session(0)

From 88f68635d05b7f2b7bbae72a875af3dd4c710c47 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 7 Aug 2024 13:32:36 +0300
Subject: [PATCH 056/128] Add CLI unit tests for storage options

---
 reframe/frontend/cli.py |   4 +-
 unittests/test_cli.py   | 132 +++++++++++++++++++++++++++++++---------
 2 files changed, 104 insertions(+), 32 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 8d27636ba3..9acf2ae73e 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -408,7 +408,7 @@ def main():
     )
     action_options.add_argument(
         '--describe-stored-testcases', action='store',
-        metavar='SESSION_UUID|PERIOD',
+        metavar='^SESSION_UUID|PERIOD',
         help='Get detailed test case information in JSON'
     )
     action_options.add_argument(
@@ -428,7 +428,7 @@ def main():
     )
     action_options.add_argument(
         '--list-stored-sessions', action='store_true',
-        help='List stored session'
+        help='List stored sessions'
     )
     action_options.add_argument(
         '--list-stored-testcases', action='store',
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 04051c7b36..a5d02d6db4 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -107,6 +107,8 @@ def _run_reframe(system='generic:default',
             argv += ['-h']
         elif action == 'describe':
             argv += ['--describe']
+        else:
+            argv += [action]
 
         if perflogdir:
             argv += ['--perflogdir', perflogdir]
@@ -164,10 +166,7 @@ def test_check_restore_session_failed(run_reframe, tmp_path):
     run_reframe(
         checkpath=['unittests/resources/checks_unlisted/deps_complex.py']
     )
-    returncode, stdout, _ = run_reframe(
-        checkpath=[],
-        more_options=['--restore-session', '--failed']
-    )
+    run_reframe(checkpath=[],  more_options=['--restore-session', '--failed'])
     report = reporting.restore_session(
         f'{tmp_path}/.reframe/reports/latest.json'
     )
@@ -186,10 +185,7 @@ def test_check_restore_session_succeeded_test(run_reframe, tmp_path):
         checkpath=['unittests/resources/checks_unlisted/deps_complex.py'],
         more_options=['--keep-stage-files']
     )
-    returncode, stdout, _ = run_reframe(
-        checkpath=[],
-        more_options=['--restore-session', '-n', 'T1']
-    )
+    run_reframe(checkpath=[], more_options=['--restore-session', '-n', 'T1'])
     report = reporting.restore_session(
         f'{tmp_path}/.reframe/reports/latest.json'
     )
@@ -747,9 +743,8 @@ def test_filtering_by_expr(run_reframe):
 def test_show_config_all(run_reframe):
     # Just make sure that this option does not make the frontend crash
     returncode, stdout, stderr = run_reframe(
-        more_options=['--show-config'],
-        system='testsys',
-        action=None
+        action='--show-config',
+        system='testsys'
     )
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
@@ -759,9 +754,8 @@ def test_show_config_all(run_reframe):
 def test_show_config_param(run_reframe):
     # Just make sure that this option does not make the frontend crash
     returncode, stdout, stderr = run_reframe(
-        more_options=['--show-config=systems'],
-        system='testsys',
-        action=None
+        action='--show-config=systems',
+        system='testsys'
     )
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
@@ -771,9 +765,8 @@ def test_show_config_param(run_reframe):
 def test_show_config_unknown_param(run_reframe):
     # Just make sure that this option does not make the frontend crash
     returncode, stdout, stderr = run_reframe(
-        more_options=['--show-config=foo'],
-        system='testsys',
-        action=None
+        action='--show-config=foo',
+        system='testsys'
     )
     assert 'no such configuration parameter found' in stdout
     assert 'Traceback' not in stdout
@@ -783,9 +776,8 @@ def test_show_config_unknown_param(run_reframe):
 
 def test_show_config_null_param(run_reframe):
     returncode, stdout, stderr = run_reframe(
-        more_options=['--show-config=general/report_junit'],
-        system='testsys',
-        action=None
+        action='--show-config=general/report_junit',
+        system='testsys'
     )
     assert 'null' in stdout
     assert 'Traceback' not in stdout
@@ -1115,10 +1107,7 @@ def test_exec_order(run_reframe, exec_order):
 def test_detect_host_topology(run_reframe):
     from reframe.utility.cpuinfo import cpuinfo
 
-    returncode, stdout, stderr = run_reframe(
-        more_options=['--detect-host-topology'],
-        action=None
-    )
+    returncode, stdout, stderr = run_reframe(action='--detect-host-topology')
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
     assert returncode == 0
@@ -1130,8 +1119,7 @@ def test_detect_host_topology_file(run_reframe, tmp_path):
 
     topo_file = tmp_path / 'topo.json'
     returncode, stdout, stderr = run_reframe(
-        more_options=[f'--detect-host-topology={topo_file}'],
-        action=None
+        action=f'--detect-host-topology={topo_file}',
     )
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
@@ -1198,7 +1186,7 @@ def test_fixture_registry_env_sys(run_reframe):
     assert returncode == 0
     assert 'e1' in stdout
     assert 'sys1:p0' in stdout
-    returncode, stdout, stderr = run_reframe(
+    returncode, stdout, _ = run_reframe(
         system='sys1:p1',
         environs=['e1'],
         checkpath=['unittests/resources/checks_unlisted/fixtures_simple.py'],
@@ -1208,7 +1196,7 @@ def test_fixture_registry_env_sys(run_reframe):
     assert returncode == 0
     assert 'e1' in stdout
     assert 'sys1:p1' in stdout
-    returncode, stdout, stderr = run_reframe(
+    returncode, stdout, _ = run_reframe(
         system='sys1:p1',
         environs=['e2'],
         checkpath=['unittests/resources/checks_unlisted/fixtures_simple.py'],
@@ -1230,7 +1218,7 @@ def test_fixture_resolution(run_reframe, run_action):
     assert returncode == 0
 
 
-def test_dynamic_tests(run_reframe, tmp_path, run_action):
+def test_dynamic_tests(run_reframe, run_action):
     returncode, stdout, _ = run_reframe(
         system='sys0',
         environs=[],
@@ -1243,7 +1231,7 @@ def test_dynamic_tests(run_reframe, tmp_path, run_action):
     assert 'FAILED' not in stdout
 
 
-def test_dynamic_tests_filtering(run_reframe, tmp_path, run_action):
+def test_dynamic_tests_filtering(run_reframe, run_action):
     returncode, stdout, _ = run_reframe(
         system='sys1',
         environs=[],
@@ -1267,3 +1255,87 @@ def test_testlib_inherit_fixture_in_different_files(run_reframe):
     assert returncode == 0
     assert 'Ran 3/3 test case(s)' in stdout
     assert 'FAILED' not in stdout
+
+
+def test_storage_options(run_reframe, tmp_path):
+    def assert_no_crash(returncode, stdout, stderr, exitcode=0):
+        assert returncode == exitcode
+        assert 'Traceback' not in stdout
+        assert 'Traceback' not in stderr
+        return returncode, stdout, stderr
+
+    run_reframe2 = functools.partial(
+        run_reframe,
+        checkpath=['unittests/resources/checks/frontend_checks.py']
+    )
+
+    # Run first a normal run with a performance test to initialize the DB
+    run_reframe2(action='run')
+    assert os.path.exists(tmp_path / '.reframe' / 'reports' / 'results.db')
+
+    stdout = assert_no_crash(
+        *run_reframe(action='--list-stored-sessions')
+    )[1]
+
+    # Get the session uuid for later queries
+    uuid = re.search(r'(\S+-\S+)', stdout).group(1)
+
+    # Get details from the last session
+    stdout = assert_no_crash(
+        *run_reframe2(action=f'--describe-stored-session={uuid}')
+    )[1]
+    session_json = json.loads(stdout)
+
+    # List test cases by session
+    assert_no_crash(*run_reframe2(action=f'--list-stored-testcases=^{uuid}'))
+    assert_no_crash(
+        *run_reframe2(action=f'--describe-stored-testcases=^{uuid}')
+    )
+
+    # List test cases by time period
+    ts_start = session_json['session_info']['time_start']
+    assert_no_crash(
+        *run_reframe2(action=f'--list-stored-testcases={ts_start}:now')
+    )
+    assert_no_crash(
+        *run_reframe2(action=f'--describe-stored-testcases={ts_start}:now')
+    )
+
+    # Check that invalid argument do not crash CLI
+    assert_no_crash(*run_reframe2(action='--describe-stored-session=0'),
+                    exitcode=1)
+    assert_no_crash(*run_reframe2(action='--describe-stored-testcases=0'),
+                    exitcode=1)
+    assert_no_crash(*run_reframe2(action='--list-stored-testcases=0'),
+                    exitcode=1)
+
+    # Remove session
+    assert_no_crash(*run_reframe2(action=f'--delete-stored-session={uuid}'))
+
+
+def test_performance_compare(run_reframe):
+    def assert_no_crash(returncode, stdout, stderr, exitcode=0):
+        assert returncode == exitcode
+        assert 'Traceback' not in stdout
+        assert 'Traceback' not in stderr
+        return returncode, stdout, stderr
+
+    run_reframe2 = functools.partial(
+        run_reframe,
+        checkpath=['unittests/resources/checks/frontend_checks.py']
+    )
+    run_reframe2(action='run')
+
+    # Rerun with various arguments
+    assert_no_crash(
+        *run_reframe2(
+            action='--performance-compare=now-1m:now/now-1d:now/mean:/+result'
+        )
+    )
+
+    # Check that invalid arguments do not crash the CLI
+    assert_no_crash(
+        *run_reframe2(
+            action='--performance-compare=now-1m:now/now-1d:now/mean:+foo/+bar'
+        ), exitcode=1
+    )

From d05e0f4096c2efa9fe7e1818f8e4a60eec5e4ecc Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 7 Aug 2024 16:03:01 +0300
Subject: [PATCH 057/128] Use alternative method for session deletion with
 Sqlite < 3.35

---
 reframe/frontend/reporting/storage.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index e4548de3fb..940ed24840 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -227,7 +227,22 @@ def fetch_session_json(self, uuid):
 
         return jsonext.loads(results[0][0]) if results else {}
 
-    def remove_session(self, uuid):
+    def _do_remove(self, uuid):
+        prefix = os.path.dirname(self.__db_file)
+        with FileLock(os.path.join(prefix, '.db.lock')):
+            with sqlite3.connect(self._db_file()) as conn:
+                # Check first if the uuid exists
+                query = f'SELECT * FROM sessions WHERE uuid == "{uuid}"'
+                getlogger().debug(query)
+                if not conn.execute(query).fetchall():
+                    raise ReframeError(f'no such session: {uuid}')
+
+                query = f'DELETE FROM sessions WHERE uuid == "{uuid}"'
+                getlogger().debug(query)
+                conn.execute(query)
+
+    def _do_remove2(self, uuid):
+        '''Remove a session using the RETURNING keyword'''
         prefix = os.path.dirname(self.__db_file)
         with FileLock(os.path.join(prefix, '.db.lock')):
             with sqlite3.connect(self._db_file()) as conn:
@@ -237,3 +252,9 @@ def remove_session(self, uuid):
                 deleted = conn.execute(query).fetchall()
                 if not deleted:
                     raise ReframeError(f'no such session: {uuid}')
+
+    def remove_session(self, uuid):
+        if sqlite3.sqlite_version_info >= (3, 35, 0):
+            self._do_remove2(uuid)
+        else:
+            self._do_remove(uuid)

From 60b37919b1106655f167cb88440ec91f07789ac8 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 7 Aug 2024 16:21:06 +0300
Subject: [PATCH 058/128] Fix unit tests for Python 3.6

---
 unittests/test_cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index a5d02d6db4..b63590afbc 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -1278,7 +1278,7 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0):
     )[1]
 
     # Get the session uuid for later queries
-    uuid = re.search(r'(\S+-\S+)', stdout).group(1)
+    uuid = re.search(r'(\w{8}-\w{4}-\w{4}-\w{4}-\w{12})', stdout).group(1)
 
     # Get details from the last session
     stdout = assert_no_crash(

From c0296c51fd1bb65561a8b206c3b1d7fb68386e09 Mon Sep 17 00:00:00 2001
From: crivella <davide.grassano@epfl.ch>
Date: Thu, 8 Aug 2024 16:49:05 +0200
Subject: [PATCH 059/128] Not a performance function

---
 hpctestlib/sciapps/metalwalls/benchmarks.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hpctestlib/sciapps/metalwalls/benchmarks.py b/hpctestlib/sciapps/metalwalls/benchmarks.py
index 8d250bb7c1..dda4f39edd 100644
--- a/hpctestlib/sciapps/metalwalls/benchmarks.py
+++ b/hpctestlib/sciapps/metalwalls/benchmarks.py
@@ -228,7 +228,6 @@ def extract_temperature(self):
         app = sn.extractall(rgx, 'run.out', 'flag', float)
         return app[-1]
 
-    @performance_function('s')
     def extract_time(
         self, name: str = None, parent: str = None, kind: str = None
     ) -> float:

From 4fd81cad0815a9d6cc23b907741f05fae9429960 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 8 Aug 2024 17:56:04 +0300
Subject: [PATCH 060/128] Update manpage

---
 docs/manpage.rst        | 376 ++++++++++++++++++++++++++++++++--------
 reframe/frontend/cli.py |   6 +-
 2 files changed, 305 insertions(+), 77 deletions(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index 60b0f9d2d8..053abae1e0 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -17,8 +17,16 @@ The ``reframe`` command is part of ReFrame's frontend.
 This frontend is responsible for loading and running regression tests written in ReFrame.
 ReFrame executes tests by sending them down to a well defined pipeline.
 The implementation of the different stages of this pipeline is part of ReFrame's core architecture, but the frontend is responsible for driving this pipeline and executing tests through it.
-There are three basic phases that the frontend goes through, which are described briefly in the following.
+Usually, ReFrame processes tests in three phases:
 
+1. It :ref:`discovers and loads tests <test-discovery>` from the filesystem.
+2. It :ref:`filters <test-filtering>` the loaded tests based on the current system and any other criteria specified by the user.
+3. It :ref:`acts <commands>` upon the selected tests.
+
+There are also ReFrame commands that do not operate on a set of tests.
+
+
+.. _test-discovery:
 
 -------------------------------
 Test discovery and test loading
@@ -234,18 +242,24 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep
       The ``NAME`` pattern is matched anywhere in the test name and not at its beginning.
       If you want to match at the beginning of a test name, you should prepend ``^``.
 
-------------
-Test actions
-------------
 
-ReFrame will finally act upon the selected tests.
-There are currently two actions that can be performed on tests: (a) list the tests and (b) execute the tests.
-An action must always be specified.
+.. _commands:
+
+--------
+Commands
+--------
+
+ReFrame commands are mutually exclusive and one of them must always be specified.
+There are commands that act upon the selected tests and others that have a helper function, such as querying the configuration, querying the results database etc.
+
+.. versionchanged:: 4.7
+
+   ReFrame commands are now mutually exclusive and only one can be specified every time.
 
 
 .. option:: --ci-generate=FILE
 
-   Do not run the tests, but generate a Gitlab `child pipeline <https://docs.gitlab.com/ee/ci/parent_child_pipelines.html>`__ specification in ``FILE``.
+   Generate a Gitlab `child pipeline <https://docs.gitlab.com/ee/ci/parent_child_pipelines.html>`__ specification in ``FILE`` that will run the selected tests.
 
    You can set up your Gitlab CI to use the generated file to run every test as a separate Gitlab job respecting test dependencies.
    For more information, have a look in :ref:`generate-ci-pipeline`.
@@ -256,6 +270,12 @@ An action must always be specified.
 
    .. versionadded:: 3.4.1
 
+.. option:: --delete-stored-session=UUID
+
+   Delete the stored session with the specified UUID from the results database.
+
+   .. versionadded:: 4.7
+
 .. option:: --describe
 
    Print a detailed description of the `selected tests <#test-filtering>`__ in JSON format and exit.
@@ -267,6 +287,34 @@ An action must always be specified.
    .. versionadded:: 3.10.0
 
 
+.. option:: --describe-stored-session=UUID
+
+   Get detailed information of the session with the specified UUID.
+   The output is in JSON format.
+
+   .. versionadded:: 4.7
+
+.. option:: --describe-stored-testcases=^SESSION_UUID|TIME_PERIOD
+
+   Get detailed test case information of the session with the specified UUID or from the specified time period.
+
+   If a session UUID is provided only information about the test cases of this session will be provided.
+   Note that the session UUID must be prefixed with ``^``.
+
+   For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`.
+
+   .. versionadded:: 4.7
+
+.. _--detect-host-topology:
+
+.. option:: --detect-host-topology[=FILE]
+
+   Detect the local host processor topology, store it to ``FILE`` and exit.
+
+   If no ``FILE`` is specified, the standard output will be used.
+
+   .. versionadded:: 3.7.0
+
 .. option:: --dry-run
 
    Dry run the selected tests.
@@ -280,7 +328,6 @@ An action must always be specified.
 
    .. versionadded:: 4.1
 
-
 .. option:: -L, --list-detailed[=T|C]
 
    List selected tests providing more details for each test.
@@ -297,6 +344,23 @@ An action must always be specified.
       The variable names to which fixtures are bound are also listed.
       See :ref:`test_naming_scheme` for more information.
 
+.. option:: --list-stored-sessions
+
+   List all sessions stored in the results database.
+
+   .. versionadded:: 4.7
+
+.. option:: --list-stored-testcases=^SESSION_UUID|TIME_PERIOD
+
+   List all test cases from the session with the specified UUID or from the specified time period.
+
+   If a session UUID is provided only the test cases of this session will be listed.
+   Note that the session UUID must be prefixed with ``^``.
+
+   For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`.
+
+   .. versionadded:: 4.7
+
 .. option:: -l, --list[=T|C]
 
    List selected tests and their dependencies.
@@ -323,15 +387,50 @@ An action must always be specified.
 
    .. versionadded:: 3.6.0
 
+.. option:: --performance-compare=CMPSPEC
+
+   Compare the performance of test cases that have run in the past.
+
+   Check the :ref:`performance-comparisons` section for the exact syntax of ``CMPSPEC``.
+
+   .. versionadded:: 4.7
+
 .. option:: -r, --run
 
-   Execute the selected tests.
+   Run the selected tests.
+
+.. option:: --show-config [PARAM]
+
+   Show the value of configuration parameter ``PARAM`` as this is defined for the currently selected system and exit.
+
+   The parameter value is printed in JSON format.
+   If ``PARAM`` is not specified or if it set to ``all``, the whole configuration for the currently selected system will be shown.
+   Configuration parameters are formatted as a path navigating from the top-level configuration object to the actual parameter.
+   The ``/`` character acts as a selector of configuration object properties or an index in array objects.
+   The ``@`` character acts as a selector by name for configuration objects that have a ``name`` property.
+   Here are some example queries:
+
+   - Retrieve all the partitions of the current system:
+
+     .. code:: bash
+
+        reframe --show-config=systems/0/partitions
+
+   - Retrieve the job scheduler of the partition named ``default``:
+
+     .. code:: bash
 
-If more than one action options are specified, the precedence order is the following:
+        reframe --show-config=systems/0/partitions/@default/scheduler
 
-   .. code-block:: console
+   - Retrieve the check search path for system ``foo``:
 
-      --describe > --list-detailed > --list > --list-tags > --ci-generate
+     .. code:: bash
+
+        reframe --system=foo --show-config=general/0/check_search_path
+
+.. option:: -V, --version
+
+   Print version and exit.
 
 
 ----------------------------------
@@ -971,16 +1070,6 @@ Miscellaneous options
 
    .. versionchanged:: 4.0.0
 
-.. _--detect-host-topology:
-
-.. option:: --detect-host-topology[=FILE]
-
-   Detect the local host processor topology, store it to ``FILE`` and exit.
-
-   If no ``FILE`` is specified, the standard output will be used.
-
-   .. versionadded:: 3.7.0
-
 .. option:: --failure-stats
 
    Print failure statistics at the end of the run.
@@ -995,11 +1084,21 @@ Miscellaneous options
 
    This option can also be set using the :envvar:`RFM_COLORIZE` environment variable or the :attr:`~config.general.colorize` general configuration parameter.
 
-.. option:: --performance-report
+.. _--performance-report:
+
+.. option:: --performance-report[=CMPSPEC]
 
-   Print a performance report for all the performance tests that have been run.
+   Print a report summarizing the performance of all performance tests that have run in the current session.
 
-   The report shows the performance values retrieved for the different performance variables defined in the tests.
+   For each test all of their performance variables are reported and optionally compared to past results based on the ``CMPSPEC`` specified.
+
+   If not specified, the default ``CMPSPEC`` is ``19700101T0000+0000:now/last:+job_nodelist/+result``, meaning that the current performance will be compared to the last run oif the same test grouped additionally by the ``job_nodelist`` and showing also the obtained result (``pass`` or ``fail``).
+
+   For the exact syntax of ``CMPSPEC``, refer to :ref:`performance-comparisons`.
+
+   .. versionchanged:: 4.7
+
+      The format of the performance report has changed and the optional ``CMPSPEC`` argument is now added.
 
 .. option:: -q, --quiet
 
@@ -1014,35 +1113,6 @@ Miscellaneous options
    .. versionadded:: 3.9.3
 
 
-.. option:: --show-config [PARAM]
-
-   Show the value of configuration parameter ``PARAM`` as this is defined for the currently selected system and exit.
-
-   The parameter value is printed in JSON format.
-   If ``PARAM`` is not specified or if it set to ``all``, the whole configuration for the currently selected system will be shown.
-   Configuration parameters are formatted as a path navigating from the top-level configuration object to the actual parameter.
-   The ``/`` character acts as a selector of configuration object properties or an index in array objects.
-   The ``@`` character acts as a selector by name for configuration objects that have a ``name`` property.
-   Here are some example queries:
-
-   - Retrieve all the partitions of the current system:
-
-     .. code:: bash
-
-        reframe --show-config=systems/0/partitions
-
-   - Retrieve the job scheduler of the partition named ``default``:
-
-     .. code:: bash
-
-        reframe --show-config=systems/0/partitions/@default/scheduler
-
-   - Retrieve the check search path for system ``foo``:
-
-     .. code:: bash
-
-        reframe --system=foo --show-config=general/0/check_search_path
-
 .. option:: --system=NAME
 
    Load the configuration for system ``NAME``.
@@ -1062,10 +1132,6 @@ Miscellaneous options
 
    If a new file is not given, a file in the system temporary directory will be created.
 
-.. option:: -V, --version
-
-   Print version and exit.
-
 .. option:: -v, --verbose
 
    Increase verbosity level of output.
@@ -1210,6 +1276,139 @@ Very large test names meant also very large path names which could also lead to
 Fixtures followed a similar naming pattern making them hard to debug.
 
 
+Result storage
+--------------
+
+.. versionadded:: 4.7
+
+ReFrame stores the results of every session that has exectuted at least one test into a database.
+There is only one storage backend supported at the moment and this is Sqlite.
+The full session information as recorded in a run report file (see :option:`--report-file`) is stored in the database.
+The test cases of the session are indexed by their run job completion time for quick retrieval of all the test cases that have run in a certain period of time.
+
+The database file is controlled by the :attr:`~config.storage.sqlite_db_file` configuration parameter and multiple ReFrame processes can access it safely simultaneously.
+
+There are several command-line options that allow users to query the results database, such as the :option:`--list-stored-sessions`, :option:`--list-stored-testcases`, :option:`--describe-stored-session` etc.
+Other options that access the results database are the :option:`--performance-compare` and :option:`--performance-report` options which compare the performance results of the same test cases in different periods of time or from different sessions.
+Check the :ref:`commands` section for the complete list and details of each option related to the results database.
+
+Since the report file information is now kept in the results database, there is no need to keep the report files separately, although this remains the default behavior for backward compatibility.
+You can disable the report generation by turning off the :attr:`~config.general.generate_file_reports` configuration parameter.
+The file report of any session can be retrieved from the database with the :option:`--describe-stored-session` option.
+
+
+.. _performance-comparisons:
+
+Performance comparisons
+-----------------------
+
+.. versionadded:: 4.7
+
+The :option:`--performance-compare` and :option:`--performance-report` options accept a ``CMPSPEC`` argument that specifies how to select and compare test cases.
+The full syntax of ``CMPSPEC`` is the following:
+
+.. code-block:: console
+
+   <base_cases>/<target_cases>/<aggr>/<extra_cols>
+
+The ``<base_cases>`` and ``<target_cases>`` subspecs specify how the base and target test cases will be retrieved.
+The base test cases will be compared against those from the target period.
+
+.. note::
+
+   The ``<base_cases>`` subspec is ommitted from the ``CMPSPEC`` of the :option:`--performance-report` option as the base test cases are always the test cases from the current session.
+
+The test cases for comparison can either be retrieved from an existing past session or a past time period.
+A past session is denoted with the ``^<session_uuid>`` syntax and only the test cases of that particular session will be selected.
+To view the UUIDs of all stored sessions, use the :option:`--list-stored-sessions` option.
+
+To retrieve results from a time period, check the :ref:`time period syntax <time-period-syntax>` below.
+
+The ``<aggr>`` subspec specifies how the performance of both the base and target cases should be grouped and aggregated.
+The syntax is the following:
+
+.. code-block:: console
+
+   <aggr_fn>:[+<groupby>]*
+
+The ``<aggr_fn>`` is a symbolic name for a function to aggregate the grouped test cases.
+It can take one of the following values:
+
+- ``first``: retrieve the performance data of the first test case only
+- ``last``: retrieve the performance data of the last test case only
+- ``max``: retrieve the maximum of all test cases
+- ``mean``: calculate the mean over all test cases
+- ``median``: retrieve the median of all test cases
+- ``min``: retrieve the minimum of all test cases
+
+The test cases are always grouped by the following attributes:
+
+- The test :attr:`~reframe.core.pipeline.RegressionTest.name`
+- The system name
+- The partition name
+- The environment name
+- The performance variable name (see :func:`@performance_function <reframe.core.builtins.performance_function>` and :attr:`~reframe.core.pipeline.RegressionTest.perf_variables`)
+- The performance variable unit
+
+The ``+<groupby>`` subspec specifies additional attributes to group the test cases by.
+Any loggable test attribute can be selected.
+
+.. note::
+
+   The loggable attributes of a test are the same as the ones list in the logging :attr:`~config.logging.handlers_perflog.format` option but without the ``check_`` prefix.
+
+Finally, the ``<extra_cols>`` subspec specifies additional test attributes to list as columns in the resulting comparison table.
+The syntax is the following:
+
+.. code-block:: console
+
+   [+<col>]*
+
+``col`` refers to any loggable attribute of the test.
+If these attributes have different values across the aggregated test cases,
+the unique values will be joined using the ``|`` separator.
+
+Here are some examples of performance comparison specs:
+
+- Compare the test cases of the session ``7a70b2da-1544-4ac4-baf4-0fcddd30b672`` with the mean performance of the last 10 days:
+
+  .. code-block:: console
+
+     ^7a70b2da-1544-4ac4-baf4-0fcddd30b672/now-10d:now/mean:/
+
+- Compare the best performance of the test cases run on two specific days, group by the node list and report also the test result:
+
+  .. code-block:: console
+
+     20240701:20240701+1d/20240705:20240705+1d/mean:+job_nodelist/+result
+
+.. _time-period-syntax:
+
+Time periods
+------------
+
+A time period needs to be specified as part of the ``CMPSPEC`` of the :option:`--performance-compare` and :option:`--performance-report` options or as an argument to options that request past results from results database.
+
+The general syntax of time period subspec is the following:
+
+.. code-block:: console
+
+   <ts_start>:<ts_end>
+
+``<ts_start>`` and ``<ts_end>`` are timestamp denoting the start and end of the requested period.
+More specifically, the syntax of each timestamp is the following:
+
+.. code-block:: console
+
+   <abs_timestamp>[+|-<amount>w|d|h|m]
+
+The ``<abs_timestamp>`` is an absolute timestamp in one of the following ``strptime``-compatible formats or the special value ``now``: ``%Y%m%d``, ``%Y%m%dT%H%M``, ``%Y%m%dT%H%M%S``, ``%Y%m%dT%H%M%S%z``.
+
+Optionally, a shift argument can be appended with ``+`` or ``-`` signs, followed by an amount of weeks (``w``), days (``d``), hours (``h``) or minutes (``m``).
+
+For example, the period of the last 10 days can be specified as ``now:now-10d``.
+Similarly, the period of the week starting on August 5, 2024 will be specified as ``20240805:20240805+1w``.
+
 Environment
 -----------
 
@@ -1226,21 +1425,6 @@ Here is an alphabetical list of the environment variables recognized by ReFrame.
 Whenever an environment variable is associated with a configuration option, its default value is omitted as it is the same.
 
 
-.. envvar:: RFM_SCHED_ACCESS_IN_SUBMIT
-
-   Pass access options in the submission command (relevant for LSF, OAR, PBS and Slurm).
-
-   .. table::
-      :align: left
-
-      ================================== ==================
-      Associated command line option     N/A
-      Associated configuration parameter :attr::attr:`~config.systems.partitions.sched_options.sched_access_in_submit`
-      ================================== ==================
-
-.. versionadded:: 4.7
-
-
 .. envvar:: RFM_AUTODETECT_FQDN
 
    Use the fully qualified domain name as the hostname.
@@ -1452,6 +1636,20 @@ Whenever an environment variable is associated with a configuration option, its
    .. versionadded:: 4.7
 
 
+.. envvar:: RFM_GENERATE_FILE_REPORTS
+
+   Store session reports also in files.
+
+   .. table::
+      :align: left
+
+      ================================== ==================
+      Associated command line option     n/a
+      Associated configuration parameter :attr:`~config.general.generate_file_reports`
+      ================================== ==================
+
+   .. versionadded:: 4.7
+
 .. envvar:: RFM_GIT_TIMEOUT
 
    Timeout value in seconds used when checking if a git repository exists.
@@ -1600,6 +1798,21 @@ Whenever an environment variable is associated with a configuration option, its
       ================================== ==================
 
 
+.. envvar:: RFM_PERF_REPORT_SPEC
+
+   The default ``CMPSPEC`` of the :option:`--performance-report` option.
+
+   .. table::
+      :align: left
+
+      ================================== ==================
+      Associated command line option     :option:`--performance-report`
+      Associated configuration parameter :attr:`~config.general.perf_report_spec`
+      ================================== ==================
+
+   .. versionadded:: 4.7
+
+
 .. envvar:: RFM_PERFLOG_DIR
 
    Directory prefix for logging performance data.
@@ -1744,6 +1957,21 @@ Whenever an environment variable is associated with a configuration option, its
       ================================== ==================
 
 
+.. envvar:: RFM_SCHED_ACCESS_IN_SUBMIT
+
+   Pass access options in the submission command (relevant for LSF, OAR, PBS and Slurm).
+
+   .. table::
+      :align: left
+
+      ================================== ==================
+      Associated command line option     N/A
+      Associated configuration parameter :attr::attr:`~config.systems.partitions.sched_options.sched_access_in_submit`
+      ================================== ==================
+
+.. versionadded:: 4.7
+
+
 .. envvar:: RFM_STAGE_DIR
 
    Directory prefix for staging test resources.
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 9acf2ae73e..6835ea797c 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -395,7 +395,7 @@ def main():
               'for the selected tests and exit'),
     )
     action_options.add_argument(
-        '--delete-stored-session', action='store', metavar='SESSION_UUID',
+        '--delete-stored-session', action='store', metavar='UUID',
         help='Delete stored session'
     )
     action_options.add_argument(
@@ -403,7 +403,7 @@ def main():
         help='Give full details on the selected tests'
     )
     action_options.add_argument(
-        '--describe-stored-session', action='store', metavar='SESSION_UUID',
+        '--describe-stored-session', action='store', metavar='UUID',
         help='Get detailed session information in JSON'
     )
     action_options.add_argument(
@@ -432,7 +432,7 @@ def main():
     )
     action_options.add_argument(
         '--list-stored-testcases', action='store',
-        metavar='SESSION_UUID|PERIOD',
+        metavar='^SESSION_UUID|PERIOD',
         help='List stored testcases by session or time period'
     )
     action_options.add_argument(

From 61c29fc113c0e5813118ef22e810af8efeed2017 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 9 Aug 2024 19:15:59 +0300
Subject: [PATCH 061/128] Adapt tutorial

---
 docs/config_reference.rst |  32 ++++++++
 docs/tutorial.rst         | 159 +++++++++++++++++++++++++++++++++++---
 2 files changed, 182 insertions(+), 9 deletions(-)

diff --git a/docs/config_reference.rst b/docs/config_reference.rst
index c8715dbd36..9f245dd89e 100644
--- a/docs/config_reference.rst
+++ b/docs/config_reference.rst
@@ -60,6 +60,13 @@ It consists of the following properties, which we also call conventionally *conf
 
    A list of `general configuration objects <#general-configuration>`__.
 
+.. py:data:: storage
+
+   :required: No
+
+   A list of :ref:`storage configuration objects <storage-configuration>`
+
+   .. versionadded:: 4.7
 
 .. py:data:: autodetect_methods
 
@@ -1602,6 +1609,31 @@ The options of an execution mode will be passed to ReFrame as if they were speci
    For a detailed description of this property, have a look at the :attr:`~environments.target_systems` definition for environments.
 
 
+.. _storage-configuration:
+
+Result storage configuration
+============================
+
+.. versionadded:: 4.7
+
+.. py:attribute:: storage.backend
+
+   :required: No
+   :default: ``"sqlite"``
+
+   The backend to use for storing the test results.
+
+   Currently, only Sqlite can be used as a storage backend.
+
+
+.. py:attribute:: storage.sqlite_db_file
+
+   :required: No
+   :default: ``"${HOME}/.reframe/reports/results.db"``
+
+   The Sqlite database file to use.
+
+
 General Configuration
 =====================
 
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index 78b1d3a672..5a27e9799d 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -168,24 +168,29 @@ This can be suppressed by increasing the level at which this information is logg
 Run reports and performance logging
 -----------------------------------
 
-Once a test session finishes, ReFrame generates a detailed JSON report under ``$HOME/.reframe/reports``.
-Every time ReFrame is run a new report will be generated automatically.
-The latest one is always symlinked by the ``latest.json`` name, unless the :option:`--report-file` option is given.
+Once a test session finishes, ReFrame stores the detailed session information in database file located under ``$HOME/.reframe/reports``.
+Past performance data can be retrieved from this database and compared with the current or another run.
+We detail handling of the results database in section :ref:`inspecting-past-results`.
+
+By default, the session information is also saved in a JSON report file under ``$HOME/.reframe/reports``.
+The latest report is always symlinked by the ``latest.json`` name, unless the :option:`--report-file` option is given.
 
 For performance tests, in particular, an additional CSV file is generated with all the relevant information.
 These files are located by default under ``perflogs/<system>/<partition>/<testname>.log``.
 In our example, this translates to ``perflogs/generic/default/stream_test.log``.
 The information that is being logged is fully configurable and we will cover this in the :ref:`logging` section.
 
-Finally, you can use also the :option:`--performance-report` option, which will print a summary of the results of the performance tests that have run in the current session.
+Finally, you can use also the :option:`--performance-report` option, which will print a summary of the results of the performance tests that have run in the current session and compare them (by default) with their last obtained performance.
 
 .. code-block:: console
 
-   [stream_test /2e15a047 @generic:default:builtin]
-     num_tasks: 1
-     performance:
-       - copy_bw: 22704.4 MB/s (r: 0 MB/s l: -inf% u: +inf%)
-       - triad_bw: 16040.9 MB/s (r: 0 MB/s l: -inf% u: +inf%)
+   ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━┑
+   │ name        │ sysenv                  │ pvar     │    pval │ punit   │ pdiff   │ job_nodelist   │ result   │
+   ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━━━━━━━━┿━━━━━━━━━━┥
+   │ stream_test │ generic:default+builtin │ copy_bw  │ 40304.2 │ MB/s    │ -0.08%  │ myhost         │ pass     │
+   ├─────────────┼─────────────────────────┼──────────┼─────────┼─────────┼─────────┼────────────────┼──────────┤
+   │ stream_test │ generic:default+builtin │ triad_bw │ 30550.3 │ MB/s    │ +0.04%  │ myhost         │ pass     │
+   ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━┙
 
 
 Inspecting the test artifacts
@@ -1969,3 +1974,139 @@ The format function takes the raw log record, the extras and the keys to ignore
 Since we can't know the exact log record attributes, we iterate over its :attr:`__dict__` items and format the record keys as we go.
 Also note that we ignore all private field of the record starting with ``_``.
 Rerunning the previous example with ``CUSTOM_JSON=1`` will generated the modified JSON record.
+
+
+.. _inspecting-past-results:
+
+Inspecting past results
+=======================
+
+.. versionadded:: 4.7
+
+For every session that has run at least one test case, ReFrame stores all its details, including the test cases, in a database.
+Essentially, the stored information is the same as the one found in the :ref:`report file <run-reports-and-performance-logging>`.
+
+To list all the stored sessions use the :option:`--list-stored-sessions` option:
+
+.. code-block:: bash
+
+   reframe --list-stored-sessions
+
+This produces a table where the most important information about a session is listed:
+its unique identifier, its start and end time and how many test cases have run:
+
+.. code-block:: console
+
+   ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━┑
+   │ UUID                                 │ Start time           │ End time             │   Num runs │   Num cases │
+   ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━┥
+   │ fddb6678-6de2-427c-96b5-d1c6b3215b0e │ 20240809T135331+0000 │ 20240809T135335+0000 │          1 │           1 │
+   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
+   │ d96a133c-e5a8-4ceb-88de-f8adfb393f28 │ 20240809T135342+0000 │ 20240809T135345+0000 │          1 │           1 │
+   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
+   │ c7508042-64be-406a-89f1-c5d31b90f838 │ 20240809T143710+0000 │ 20240809T143713+0000 │          1 │           1 │
+   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
+   │ 3bc5c067-42fa-4496-a5e4-50b92b3cc38e │ 20240809T144025+0000 │ 20240809T144026+0000 │          1 │           2 │
+   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
+   │ 53481b75-b98a-4668-b6ab-82b199cc2efe │ 20240809T144056+0000 │ 20240809T144057+0000 │          1 │           4 │
+   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
+   │
+   ...
+   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
+   │ bed145ca-0013-4b68-bfd4-620054121f91 │ 20240809T144459+0000 │ 20240809T144500+0000 │          1 │          10 │
+   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
+   │ a72c7536-274a-4a21-92c3-4116f38febd0 │ 20240809T144500+0000 │ 20240809T144500+0000 │          1 │           1 │
+   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
+   │ 8cb26ff4-7897-42fc-a993-fbdef57c8983 │ 20240809T144510+0000 │ 20240809T144511+0000 │          1 │           5 │
+   ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━┙
+
+You can use the :option:`--list-stored-testcases` to list the test cases of a specific session or those that have run within a certain period of time:
+
+.. code-block:: bash
+
+   reframe --list-stored-testcases=^53481b75-b98a-4668-b6ab-82b199cc2efe
+
+.. code-block:: console
+
+   ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑
+   │ Name                                    │ SysEnv                    │ Nodelist   │ Completion Time      │ Result   │ UUID                                     │
+   ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥
+   │ build_stream ~tutorialsys:default+gnu   │ tutorialsys:default+gnu   │            │ 20240809T145439+0000 │ pass     │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:0 │
+   ├─────────────────────────────────────────┼───────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
+   │ build_stream ~tutorialsys:default+clang │ tutorialsys:default+clang │            │ 20240809T145439+0000 │ pass     │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:1 │
+   ├─────────────────────────────────────────┼───────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
+   │ stream_test                             │ tutorialsys:default+gnu   │ myhost     │ 20240809T144057+0000 │ pass     │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:2 │
+   ├─────────────────────────────────────────┼───────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
+   │ stream_test                             │ tutorialsys:default+clang │ myhost     │ 20240809T144057+0000 │ pass     │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:3 │
+   ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
+
+.. note::
+
+   Note that you have to precede the session UUID with a ``^``.
+
+The test case UUID comprises the UUID of the session where this test case belongs to, its run index (which run inside the session) and its test case index inside the run.
+A session may have multiple runs if it has retried some failed test cases (see :option:`--max-retries`) or if it has run its tests repeatedly (see :option:`--reruns` and :option:`--duration`).
+
+You can also list the test cases that have run in a certain period of time use the :ref:`time period <time-period-syntax>` of :option:`--list-stored-testcases`:
+
+.. code-block:: bash
+
+   reframe --list-stored-testcases=20240809T144500+0000:now
+
+.. code-block:: console
+
+   ┍━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑
+   │ Name   │ SysEnv                  │ Nodelist   │ Completion Time      │ Result   │ UUID                                     │
+   ┝━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥
+   │ T2     │ generic:default+builtin │ myhost     │ 20240809T144500+0000 │ fail     │ bed145ca-0013-4b68-bfd4-620054121f91:0:7 │
+   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
+   │ T3     │ generic:default+builtin │ myhost     │ 20240809T144500+0000 │ pass     │ bed145ca-0013-4b68-bfd4-620054121f91:0:9 │
+   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
+   │ T6     │ generic:default+builtin │ myhost     │ 20240809T144500+0000 │ pass     │ a72c7536-274a-4a21-92c3-4116f38febd0:0:0 │
+   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
+   │ T0     │ generic:default+builtin │ myhost     │ 20240809T144510+0000 │ pass     │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:0 │
+   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
+   │ T4     │ generic:default+builtin │ myhost     │ 20240809T144510+0000 │ pass     │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:1 │
+   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
+   │ T5     │ generic:default+builtin │ myhost     │ 20240809T144510+0000 │ pass     │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:2 │
+   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
+   │ T1     │ generic:default+builtin │ myhost     │ 20240809T144510+0000 │ pass     │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:3 │
+   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
+   │ T6     │ generic:default+builtin │ myhost     │ 20240809T144510+0000 │ pass     │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:4 │
+   ┕━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
+
+To get all the details of a session or a set of test cases you can use the :option:`--describe-stored-session` and :option:`--describe-stored-testcases` options which will return a JSON record with all the details.
+
+Comparing performance of test cases
+-----------------------------------
+
+ReFrame can be used to compare the performance of the same test cases run in different time periods using the :option:`--performance-compare` option.
+The following will compare the performance of the test cases of the session ``a120b895-8fe9-4209-a742-997442e37c47`` with any other same test case that has run the last 24h:
+
+.. code-block:: bash
+
+   reframe --performance-compare=^a120b895-8fe9-4209-a742-997442e37c47/now-1d:now/mean:/
+
+.. code-block:: console
+
+   ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┑
+   │ name        │ sysenv                    │ pvar     │    pval │ punit   │ pdiff   │
+   ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┥
+   │ stream_test │ tutorialsys:default+gnu   │ copy_bw  │ 31274.5 │ MB/s    │ -22.47% │
+   ├─────────────┼───────────────────────────┼──────────┼─────────┼─────────┼─────────┤
+   │ stream_test │ tutorialsys:default+gnu   │ triad_bw │ 18993.4 │ MB/s    │ -42.02% │
+   ├─────────────┼───────────────────────────┼──────────┼─────────┼─────────┼─────────┤
+   │ stream_test │ tutorialsys:default+clang │ copy_bw  │ 38546.2 │ MB/s    │ -9.24%  │
+   ├─────────────┼───────────────────────────┼──────────┼─────────┼─────────┼─────────┤
+   │ stream_test │ tutorialsys:default+clang │ triad_bw │ 36866.3 │ MB/s    │ -4.72%  │
+   ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┙
+
+Similarly to the :option:`--performance-compare` option, the :option:`--performance-report` option can compare the performance of the current run with any arbitrary past session or past time period.
+
+Finally, you can delete complete a stored session using the :option:`--delete-stored-session` option:
+
+.. code-block:: bash
+
+   reframe --delete-stored-session=a120b895-8fe9-4209-a742-997442e37c47
+
+Deleting a session will also delete all its test cases from the database.

From 8e2cb61c78a0b2953e915bf681dd35b679f78197 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 9 Aug 2024 19:18:18 +0300
Subject: [PATCH 062/128] Fix comment typos

---
 reframe/frontend/argparse.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/reframe/frontend/argparse.py b/reframe/frontend/argparse.py
index 93a4070f36..ceec757b1a 100644
--- a/reframe/frontend/argparse.py
+++ b/reframe/frontend/argparse.py
@@ -46,7 +46,7 @@ class _Undefined:
 
 # We use a special value for denoting const values that are to be set from the
 # configuration default. This placeholder must be used as the `const` argument
-#  for options with `nargs='?'`. The underlying `ArugmentParser` will use the
+#  for options with `nargs='?'`. The underlying `ArgumentParser` will use the
 # `const` value as if it were supplied from the command-line thus fooling our
 # machinery of environment variables and configuration options overriding any
 # defaults. For this reason, we use a unique placeholder so that we can
@@ -244,7 +244,7 @@ class ArgumentParser(_ArgumentHolder):
     '''Reframe's extended argument parser.
 
     This argument parser behaves almost identical to the original
-    `argparse.ArgumenParser`. In fact, it uses such a parser internally,
+    `argparse.ArgumentParser`. In fact, it uses such a parser internally,
     delegating all the calls to it. The key difference is how newly parsed
     options are combined with existing namespaces in `parse_args()`.'''
 

From 71f8e5a2fdffa066caee7f7f9c52734b83c28c86 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 9 Aug 2024 19:57:22 +0300
Subject: [PATCH 063/128] Treat correctly invalid completion times in
 `--list-stored-testcases`

---
 .../tutorial/dockerfiles/singlenode.Dockerfile  |  8 ++++++++
 reframe/frontend/reporting/__init__.py          | 17 ++++++++++-------
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/examples/tutorial/dockerfiles/singlenode.Dockerfile b/examples/tutorial/dockerfiles/singlenode.Dockerfile
index 647f26d231..86750b20d8 100644
--- a/examples/tutorial/dockerfiles/singlenode.Dockerfile
+++ b/examples/tutorial/dockerfiles/singlenode.Dockerfile
@@ -10,8 +10,16 @@ RUN apt-get -y update && \
 ARG REFRAME_TAG=develop
 ARG REFRAME_REPO=reframe-hpc
 WORKDIR /usr/local/share
+
+# Clone reframe
 RUN git clone --depth 1 --branch $REFRAME_TAG https://github.com/$REFRAME_REPO/reframe.git && \
     cd reframe/ && ./bootstrap.sh
+
+# Comment the above line and uncomment the following two for development
+
+# COPY . /usr/local/share/reframe
+# RUN cd reframe && ./bootstrap.sh
+
 ENV PATH=/usr/local/share/reframe/bin:$PATH
 
 # Install stream
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 3cca83792b..b1ae4549d7 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -235,9 +235,6 @@ def __init__(self):
         now = time.time()
         self.update_timestamps(now, now)
 
-    def data(self):
-        return self.__report
-
     @property
     def filename(self):
         return self.__filename
@@ -655,14 +652,20 @@ def testcase_data(spec):
     data = [['Name', 'SysEnv',
              'Nodelist', 'Completion Time', 'Result', 'UUID']]
     for tc in testcases:
+        ts_completed = tc['job_completion_time_unix']
+        if not ts_completed:
+            completion_time = 'n/a'
+        else:
+            # Always format the completion time as users can set their own
+            # formatting in the log record
+            completion_time = time.strftime(_DATETIME_FMT,
+                                            time.localtime(ts_completed))
+
         data.append([
             tc['name'],
             _format_sysenv(tc['system'], tc['partition'], tc['environ']),
             nodelist_abbrev(tc['job_nodelist']),
-            # Always format the completion time as users can set their own
-            # formatting in the log record
-            time.strftime(_DATETIME_FMT,
-                          time.localtime(tc['job_completion_time_unix'])),
+            completion_time,
             tc['result'],
             tc['uuid']
         ])

From 7fa684bfe1e363e0b1d75a20614fe46dedb1167f Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 9 Aug 2024 20:28:43 +0300
Subject: [PATCH 064/128] Do not prefix UUIDs with `^`

---
 docs/manpage.rst                       | 10 ++++----
 docs/tutorial.rst                      |  7 ++----
 reframe/frontend/cli.py                |  4 ++--
 reframe/frontend/reporting/__init__.py | 10 ++++----
 reframe/frontend/reporting/utility.py  | 16 +++++++++----
 unittests/test_cli.py                  |  4 ++--
 unittests/test_reporting.py            | 32 ++++++++++++++++++++------
 7 files changed, 52 insertions(+), 31 deletions(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index 053abae1e0..7ed2c45192 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -294,12 +294,11 @@ There are commands that act upon the selected tests and others that have a helpe
 
    .. versionadded:: 4.7
 
-.. option:: --describe-stored-testcases=^SESSION_UUID|TIME_PERIOD
+.. option:: --describe-stored-testcases=SESSION_UUID|TIME_PERIOD
 
    Get detailed test case information of the session with the specified UUID or from the specified time period.
 
    If a session UUID is provided only information about the test cases of this session will be provided.
-   Note that the session UUID must be prefixed with ``^``.
 
    For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`.
 
@@ -350,12 +349,11 @@ There are commands that act upon the selected tests and others that have a helpe
 
    .. versionadded:: 4.7
 
-.. option:: --list-stored-testcases=^SESSION_UUID|TIME_PERIOD
+.. option:: --list-stored-testcases=SESSION_UUID|TIME_PERIOD
 
    List all test cases from the session with the specified UUID or from the specified time period.
 
    If a session UUID is provided only the test cases of this session will be listed.
-   Note that the session UUID must be prefixed with ``^``.
 
    For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`.
 
@@ -1319,7 +1317,7 @@ The base test cases will be compared against those from the target period.
    The ``<base_cases>`` subspec is ommitted from the ``CMPSPEC`` of the :option:`--performance-report` option as the base test cases are always the test cases from the current session.
 
 The test cases for comparison can either be retrieved from an existing past session or a past time period.
-A past session is denoted with the ``^<session_uuid>`` syntax and only the test cases of that particular session will be selected.
+A past session is denoted with the ``<session_uuid>`` syntax and only the test cases of that particular session will be selected.
 To view the UUIDs of all stored sessions, use the :option:`--list-stored-sessions` option.
 
 To retrieve results from a time period, check the :ref:`time period syntax <time-period-syntax>` below.
@@ -1374,7 +1372,7 @@ Here are some examples of performance comparison specs:
 
   .. code-block:: console
 
-     ^7a70b2da-1544-4ac4-baf4-0fcddd30b672/now-10d:now/mean:/
+     7a70b2da-1544-4ac4-baf4-0fcddd30b672/now-10d:now/mean:/
 
 - Compare the best performance of the test cases run on two specific days, group by the node list and report also the test result:
 
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index 5a27e9799d..419a6e0dc9 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -2024,7 +2024,7 @@ You can use the :option:`--list-stored-testcases` to list the test cases of a sp
 
 .. code-block:: bash
 
-   reframe --list-stored-testcases=^53481b75-b98a-4668-b6ab-82b199cc2efe
+   reframe --list-stored-testcases=53481b75-b98a-4668-b6ab-82b199cc2efe
 
 .. code-block:: console
 
@@ -2040,9 +2040,6 @@ You can use the :option:`--list-stored-testcases` to list the test cases of a sp
    │ stream_test                             │ tutorialsys:default+clang │ myhost     │ 20240809T144057+0000 │ pass     │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:3 │
    ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
 
-.. note::
-
-   Note that you have to precede the session UUID with a ``^``.
 
 The test case UUID comprises the UUID of the session where this test case belongs to, its run index (which run inside the session) and its test case index inside the run.
 A session may have multiple runs if it has retried some failed test cases (see :option:`--max-retries`) or if it has run its tests repeatedly (see :option:`--reruns` and :option:`--duration`).
@@ -2085,7 +2082,7 @@ The following will compare the performance of the test cases of the session ``a1
 
 .. code-block:: bash
 
-   reframe --performance-compare=^a120b895-8fe9-4209-a742-997442e37c47/now-1d:now/mean:/
+   reframe --performance-compare=a120b895-8fe9-4209-a742-997442e37c47/now-1d:now/mean:/
 
 .. code-block:: console
 
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 6835ea797c..516b085d38 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -408,7 +408,7 @@ def main():
     )
     action_options.add_argument(
         '--describe-stored-testcases', action='store',
-        metavar='^SESSION_UUID|PERIOD',
+        metavar='SESSION_UUID|PERIOD',
         help='Get detailed test case information in JSON'
     )
     action_options.add_argument(
@@ -432,7 +432,7 @@ def main():
     )
     action_options.add_argument(
         '--list-stored-testcases', action='store',
-        metavar='^SESSION_UUID|PERIOD',
+        metavar='SESSION_UUID|PERIOD',
         help='List stored testcases by session or time period'
     )
     action_options.add_argument(
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index b1ae4549d7..a4c860a28f 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -26,7 +26,7 @@
 from reframe.core.warnings import suppress_deprecations
 from reframe.utility import nodelist_abbrev
 from .storage import StorageBackend
-from .utility import Aggregator, parse_cmp_spec, parse_time_period
+from .utility import Aggregator, parse_cmp_spec, parse_time_period, is_uuid
 
 # The schema data version
 # Major version bumps are expected to break the validation of previous schemas
@@ -642,8 +642,8 @@ def session_data():
 
 def testcase_data(spec):
     storage = StorageBackend.default()
-    if spec.startswith('^'):
-        testcases = storage.fetch_testcases_from_session(spec[1:])
+    if is_uuid(spec):
+        testcases = storage.fetch_testcases_from_session(spec)
     else:
         testcases = storage.fetch_testcases_time_period(
             *parse_time_period(spec)
@@ -686,8 +686,8 @@ def session_info(uuid):
 def testcase_info(spec):
     '''Retrieve test case details as JSON'''
     testcases = []
-    if spec.startswith('^'):
-        session_uuid, *tc_index = spec[1:].split(':')
+    if is_uuid(spec):
+        session_uuid, *tc_index = spec.split(':')
         session = session_info(session_uuid)
         if not tc_index:
             for run in session['runs']:
diff --git a/reframe/frontend/reporting/utility.py b/reframe/frontend/reporting/utility.py
index 2da8030f53..fa98f39c20 100644
--- a/reframe/frontend/reporting/utility.py
+++ b/reframe/frontend/reporting/utility.py
@@ -130,11 +130,19 @@ def _do_parse(s):
     return ts.timestamp()
 
 
+_UUID_PATTERN = re.compile(r'^\w{8}-\w{4}-\w{4}-\w{4}-\w{12}(:\d+)?(:\d+)?$')
+
+
+def is_uuid(s):
+    '''Return true if `s` is a valid session, run or test case UUID'''
+    return _UUID_PATTERN.match(s) is not None
+
+
 def parse_time_period(s):
-    if s.startswith('^'):
+    if is_uuid(s):
         # Retrieve the period of a full session
         try:
-            session_uuid = s[1:]
+            session_uuid = s
         except IndexError:
             raise ValueError(f'invalid session uuid: {s}') from None
         else:
@@ -181,8 +189,8 @@ def _parse_period_spec(s):
         if s is None:
             return None, None
 
-        if s.startswith('^'):
-            return s[1:], None
+        if is_uuid(s):
+            return s, None
 
         return None, parse_time_period(s)
 
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index b63590afbc..21f71d355f 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -1287,9 +1287,9 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0):
     session_json = json.loads(stdout)
 
     # List test cases by session
-    assert_no_crash(*run_reframe2(action=f'--list-stored-testcases=^{uuid}'))
+    assert_no_crash(*run_reframe2(action=f'--list-stored-testcases={uuid}'))
     assert_no_crash(
-        *run_reframe2(action=f'--describe-stored-testcases=^{uuid}')
+        *run_reframe2(action=f'--describe-stored-testcases={uuid}')
     )
 
     # List test cases by time period
diff --git a/unittests/test_reporting.py b/unittests/test_reporting.py
index a25d6410b4..cff94f3cf8 100644
--- a/unittests/test_reporting.py
+++ b/unittests/test_reporting.py
@@ -277,10 +277,28 @@ def test_parse_cmp_spec_extra_cols(extra_cols):
     assert match.period_base is None
 
 
-@pytest.fixture(params=['^uuid/now-1d:now/min:/',
-                        'now-1d:now/^uuid/min:/',
-                        '^uuid0/^uuid1/min:/',
-                        'now-1m:now/now-1d:now/min:/'])
+def test_is_uuid():
+    # Test a standard UUID
+    assert report_util.is_uuid('7daf4a71-997b-4417-9bda-225c9cab96c2')
+
+    # Test a run UUID
+    assert report_util.is_uuid('7daf4a71-997b-4417-9bda-225c9cab96c2:0')
+
+    # Test a test case UUID
+    assert report_util.is_uuid('7daf4a71-997b-4417-9bda-225c9cab96c2:0:1')
+
+    # Test invalid UUIDs
+    assert not report_util.is_uuid('7daf4a71-997b-4417-9bda-225c9cab96c')
+    assert not report_util.is_uuid('7daf4a71-997b-4417-9bda-225c9cab96c2:')
+    assert not report_util.is_uuid('foo')
+
+
+@pytest.fixture(params=[
+    '7daf4a71-997b-4417-9bda-225c9cab96c2/now-1d:now/min:/',
+    'now-1d:now/7daf4a71-997b-4417-9bda-225c9cab96c2/min:/',
+    '7daf4a71-997b-4417-9bda-225c9cab96c2/7daf4a71-997b-4417-9bda-225c9cab96c2/min:/',  # noqa: E501
+    'now-1m:now/now-1d:now/min:/']
+)
 def uuid_spec(request):
     return request.param
 
@@ -291,10 +309,10 @@ def _uuids(s):
         base, target = None, None
         if len(parts) == 3:
             base = None
-            target = parts[0][1:] if parts[0].startswith('^') else None
+            target = parts[0] if report_util.is_uuid(parts[0]) else None
         else:
-            base = parts[0][1:] if parts[0].startswith('^') else None
-            target = parts[1][1:] if parts[1].startswith('^') else None
+            base = parts[0] if report_util.is_uuid(parts[0]) else None
+            target = parts[1] if report_util.is_uuid(parts[1]) else None
 
         return base, target
 

From d2e0844b5ec807150c2cd45ece68051d3fd46836 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 12 Aug 2024 12:30:15 +0300
Subject: [PATCH 065/128] Accept a time period argument in
 `--list-stored-sessions`

---
 docs/manpage.rst                       |  9 +++++++--
 docs/tutorial.rst                      |  2 +-
 reframe/frontend/cli.py                | 10 +++++++---
 reframe/frontend/reporting/__init__.py |  6 ++++--
 reframe/frontend/reporting/storage.py  | 16 +++++++++++++---
 unittests/test_reporting.py            | 13 ++++++++++---
 6 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index 7ed2c45192..d62e427136 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -343,9 +343,14 @@ There are commands that act upon the selected tests and others that have a helpe
       The variable names to which fixtures are bound are also listed.
       See :ref:`test_naming_scheme` for more information.
 
-.. option:: --list-stored-sessions
+.. _--list-stored-sessions:
 
-   List all sessions stored in the results database.
+.. option:: --list-stored-sessions[=TIME_PERIOD]
+
+   List sessions stored in the results database.
+
+   If ``TIME_PERIOD`` is not specified or if ``all`` is passed, all stored sessions will be listed.
+   For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`.
 
    .. versionadded:: 4.7
 
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index 419a6e0dc9..a6b46c1deb 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -1986,7 +1986,7 @@ Inspecting past results
 For every session that has run at least one test case, ReFrame stores all its details, including the test cases, in a database.
 Essentially, the stored information is the same as the one found in the :ref:`report file <run-reports-and-performance-logging>`.
 
-To list all the stored sessions use the :option:`--list-stored-sessions` option:
+To list the stored sessions use the :option:`--list-stored-sessions` option:
 
 .. code-block:: bash
 
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 516b085d38..8965467546 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -427,8 +427,8 @@ def main():
               'providing more details')
     )
     action_options.add_argument(
-        '--list-stored-sessions', action='store_true',
-        help='List stored sessions'
+        '--list-stored-sessions', nargs='?', action='store', const='all',
+        metavar='PERIOD', help='List stored sessions'
     )
     action_options.add_argument(
         '--list-stored-testcases', action='store',
@@ -939,7 +939,11 @@ def restrict_logging():
     if options.list_stored_sessions:
         with exit_gracefully_on_error('failed to retrieve session data',
                                       printer):
-            printer.table(reporting.session_data())
+            time_period = options.list_stored_sessions
+            if time_period == 'all':
+                time_period = None
+
+            printer.table(reporting.session_data(time_period))
             sys.exit(0)
 
     if options.list_stored_testcases:
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index a4c860a28f..07a98e885e 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -623,11 +623,13 @@ def performance_compare(cmp, report=None):
                                  match.extra_cols)
 
 
-def session_data():
+def session_data(time_period):
     '''Retrieve all sessions'''
 
     data = [['UUID', 'Start time', 'End time', 'Num runs', 'Num cases']]
-    for sess_data in StorageBackend.default().fetch_all_sessions():
+    for sess_data in StorageBackend.default().fetch_sessions_time_period(
+        *parse_time_period(time_period) if time_period else (None, None)
+    ):
         session_info = sess_data['session_info']
         data.append(
             [session_info['uuid'],
diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 940ed24840..434887ce7c 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -207,10 +207,20 @@ def fetch_testcases_from_session(self, session_uuid):
         session_info = jsonext.loads(results[0][0])
         return [tc for run in session_info['runs'] for tc in run['testcases']]
 
-    def fetch_all_sessions(self):
+    def fetch_sessions_time_period(self, ts_start=None, ts_end=None):
         with sqlite3.connect(self._db_file()) as conn:
-            query = ('SELECT json_blob from sessions '
-                     'ORDER BY session_start_unix')
+            query = 'SELECT json_blob from sessions'
+            if ts_start or ts_end:
+                query += ' WHERE ('
+                if ts_start:
+                    query += f'session_start_unix >= {ts_start}'
+
+                if ts_end:
+                    query += f' AND session_start_unix <= {ts_end}'
+
+                query += ')'
+
+            query += ' ORDER BY session_start_unix'
             getlogger().debug(query)
             results = conn.execute(query).fetchall()
 
diff --git a/unittests/test_reporting.py b/unittests/test_reporting.py
index cff94f3cf8..509e8d4d7a 100644
--- a/unittests/test_reporting.py
+++ b/unittests/test_reporting.py
@@ -401,8 +401,15 @@ def _count_failed(testcases):
 
     backend = report_storage.StorageBackend.default()
 
-    # Test `fetch_all_sessions`
-    stored_sessions = backend.fetch_all_sessions()
+    # Test `fetch_sessions_time_period`
+    stored_sessions = backend.fetch_sessions_time_period()
+    assert len(stored_sessions) == 2
+    for i, sess in enumerate(stored_sessions):
+        assert sess['session_info']['uuid'] == uuids[i]
+
+    # Test the time period version
+    now = time.time()
+    stored_sessions = backend.fetch_sessions_time_period(now - 60, now)
     assert len(stored_sessions) == 2
     for i, sess in enumerate(stored_sessions):
         assert sess['session_info']['uuid'] == uuids[i]
@@ -450,7 +457,7 @@ def _count_failed(testcases):
 
     # Test session removal
     backend.remove_session(uuids[-1])
-    assert len(backend.fetch_all_sessions()) == 1
+    assert len(backend.fetch_sessions_time_period()) == 1
 
     testcases = backend.fetch_testcases_time_period(timestamps[0][0],
                                                     timestamps[1][1])

From d08528f3028ee31a49faedd965c2c7ef899273ee Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Tue, 13 Aug 2024 20:19:12 +0300
Subject: [PATCH 066/128] Fix broken `nodelist` check for PBS-based backends

---
 reframe/core/schedulers/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe/core/schedulers/pbs.py b/reframe/core/schedulers/pbs.py
index a15bea5df9..86dbb6063d 100644
--- a/reframe/core/schedulers/pbs.py
+++ b/reframe/core/schedulers/pbs.py
@@ -185,7 +185,7 @@ def finished(self, job):
         return job.completed
 
     def _update_nodelist(self, job, nodespec):
-        if job.nodelist is not None:
+        if job.nodelist:
             return
 
         job._nodelist = [x.split('/')[0] for x in nodespec.split('+')]

From ddb8ad728d313cfc3f3234812f0d43b6fa17744d Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Tue, 13 Aug 2024 21:35:32 +0300
Subject: [PATCH 067/128] Allow setting the database file from an environment
 variable

---
 docs/manpage.rst        | 15 +++++++++++++++
 reframe/frontend/cli.py |  6 ++++++
 2 files changed, 21 insertions(+)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index d62e427136..cad06c510e 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -1988,6 +1988,21 @@ Whenever an environment variable is associated with a configuration option, its
       ================================== ==================
 
 
+.. envvar:: RFM_SQLITE_DB_FILE
+
+   The SQlite database file for storing test results.
+
+   .. table::
+      :align: left
+
+      ================================== ==================
+      Associated command line option     N/A
+      Associated configuration parameter :attr:`~config.storage.sqlite_db_file`
+      ================================== ==================
+
+   .. versionadded:: 4.7
+
+
 .. envvar:: RFM_SYSLOG_ADDRESS
 
    The address of the Syslog server to send performance logs.
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 8965467546..41b8a9c249 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -751,6 +751,12 @@ def main():
         action='store_true',
         help='Resolve module conflicts automatically'
     )
+    argparser.add_argument(
+        dest='sqlite_db_file',
+        envvar='RFM_SQLITE_DB_FILE',
+        configvar='storage/sqlite_db_file',
+        help='DB file where the results database resides'
+    )
     argparser.add_argument(
         dest='syslog_address',
         envvar='RFM_SYSLOG_ADDRESS',

From b1206ad6ae5b7234d017c4a5d90d6815c7ed005b Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 14 Aug 2024 11:43:25 +0300
Subject: [PATCH 068/128] Add CLI and config option to control the table format

---
 docs/config_reference.rst              | 14 +++++++++++++
 docs/manpage.rst                       | 27 ++++++++++++++++++++++++++
 reframe/frontend/cli.py                |  5 +++++
 reframe/frontend/printer.py            | 18 ++++++++++++++++-
 reframe/frontend/reporting/__init__.py |  9 ++++++++-
 reframe/schemas/config.json            |  2 ++
 unittests/test_cli.py                  | 15 ++++++++++----
 7 files changed, 84 insertions(+), 6 deletions(-)

diff --git a/docs/config_reference.rst b/docs/config_reference.rst
index 9f245dd89e..beeb9b457b 100644
--- a/docs/config_reference.rst
+++ b/docs/config_reference.rst
@@ -1891,6 +1891,20 @@ General Configuration
    For a detailed description of this property, have a look at the :attr:`~environments.target_systems` definition for environments.
 
 
+.. py:attribute:: general.table_format
+
+   :required: No
+   :default: ``"pretty"``
+
+   Set the formatting of tabular output.
+
+   The acceptable values are the following:
+
+   - ``csv``: Generate CSV output
+   - ``plain``: Generate a plain table without any lines
+   - ``pretty``: (default) Generate a pretty table
+
+
 .. py:attribute:: general.timestamp_dirs
 
    :required: No
diff --git a/docs/manpage.rst b/docs/manpage.rst
index cad06c510e..0ab2707771 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -1129,6 +1129,18 @@ Miscellaneous options
 
    This option can also be set using the :envvar:`RFM_SYSTEM` environment variable.
 
+.. option:: --table-format=csv|plain|pretty
+
+   Set the formatting of tabular output printed by options :option:`--performance-compare`, :option:`--performance-report` and the options controlling the stored sessions.
+
+   The acceptable values are the following:
+
+   - ``csv``: Generate CSV output
+   - ``plain``: Generate a plain table without any lines
+   - ``pretty``: (default) Generate a pretty table
+
+   .. versionadded:: 4.7
+
 .. option:: --upgrade-config-file=OLD[:NEW]
 
    Convert the old-style configuration file ``OLD``, place it into the new file ``NEW`` and exit.
@@ -2033,6 +2045,21 @@ Whenever an environment variable is associated with a configuration option, its
       ================================== ==================
 
 
+.. envvar:: RFM_TABLE_FORMAT
+
+   Set the format of the tables printed by various options accessing the results storage.
+
+   .. table::
+      :align: left
+
+      ================================== ==================
+      Associated command line option     :option:`--table-format`
+      Associated configuration parameter :attr:`~config.general.table_format`
+      ================================== ==================
+
+   .. versionadded:: 4.7
+
+
 .. envvar:: RFM_TIMESTAMP_DIRS
 
    Append a timestamp to the output and stage directory prefixes.
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 41b8a9c249..88f2df5567 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -625,6 +625,11 @@ def main():
         '--system', action='store', help='Load configuration for SYSTEM',
         envvar='RFM_SYSTEM'
     )
+    misc_options.add_argument(
+        '--table-format', choices=['csv', 'plain', 'pretty'],
+        help='Table formatting',
+        envvar='RFM_TABLE_FORMAT', configvar='general/table_format'
+    )
     misc_options.add_argument(
         '-v', '--verbose', action='count',
         help='Increase verbosity level of output',
diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index 0767ac4dfe..6613107df4 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -255,9 +255,25 @@ def performance_report(self, data, **kwargs):
         self.table(data, **kwargs)
         self.info('')
 
+    def _table_as_csv(self, data):
+        for line in data:
+            self.info(','.join(str(x) for x in line))
+
     def table(self, data, **kwargs):
         '''Print tabular data'''
 
+        table_format = rt.runtime().get_option('general/0/table_format')
+        if table_format == 'csv':
+            return self._table_as_csv(data)
+
+        # Map our options to tabulate
+        if table_format == 'plain':
+            tablefmt = 'plain'
+        elif table_format == 'pretty':
+            tablefmt = 'mixed_grid'
+        else:
+            raise ValueError(f'invalid table format: {table_format}')
+
         kwargs.setdefault('headers', 'firstrow')
-        kwargs.setdefault('tablefmt', 'mixed_grid')
+        kwargs.setdefault('tablefmt', tablefmt)
         self.info(tabulate(data, **kwargs))
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 07a98e885e..79831141ae 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -23,6 +23,7 @@
 import reframe.utility.osext as osext
 from reframe.core.exceptions import ReframeError, what, is_severe, reraise_as
 from reframe.core.logging import getlogger, _format_time_rfc3339
+from reframe.core.runtime import runtime
 from reframe.core.warnings import suppress_deprecations
 from reframe.utility import nodelist_abbrev
 from .storage import StorageBackend
@@ -527,7 +528,13 @@ def _group_testcases(testcases, group_by, extra_cols):
 
 
 def _aggregate_perf(grouped_testcases, aggr_fn, cols):
-    other_aggr = Aggregator.create('join_uniq', '\n')
+    if runtime().get_option('general/0/table_format') == 'csv':
+        # Use a csv friendly delimiter
+        delim = '|'
+    else:
+        delim = '\n'
+
+    other_aggr = Aggregator.create('join_uniq', delim)
     aggr_data = {}
     for key, seq in grouped_testcases.items():
         aggr_data.setdefault(key, {})
diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index 900ab0de14..06765f604a 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -525,6 +525,7 @@
                     "resolve_module_conflicts": {"type": "boolean"},
                     "save_log_files": {"type": "boolean"},
                     "target_systems": {"$ref": "#/defs/system_ref"},
+                    "table_format": {"enum": ["csv", "plain", "pretty"]},
                     "timestamp_dirs": {"type": "string"},
                     "trap_job_errors": {"type": "boolean"},
                     "unload_modules": {"$ref": "#/defs/modules_list"},
@@ -587,6 +588,7 @@
         "general/report_junit": null,
         "general/resolve_module_conflicts": true,
         "general/save_log_files": false,
+        "general/table_format": "pretty",
         "general/target_systems": ["*"],
         "general/timestamp_dirs": "%Y%m%dT%H%M%S%z",
         "general/trap_job_errors": false,
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 21f71d355f..c5f517819e 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -1257,7 +1257,12 @@ def test_testlib_inherit_fixture_in_different_files(run_reframe):
     assert 'FAILED' not in stdout
 
 
-def test_storage_options(run_reframe, tmp_path):
+@pytest.fixture(params=['csv', 'plain', 'pretty'])
+def table_format(request):
+    return request.param
+
+
+def test_storage_options(run_reframe, tmp_path, table_format):
     def assert_no_crash(returncode, stdout, stderr, exitcode=0):
         assert returncode == exitcode
         assert 'Traceback' not in stdout
@@ -1266,7 +1271,8 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0):
 
     run_reframe2 = functools.partial(
         run_reframe,
-        checkpath=['unittests/resources/checks/frontend_checks.py']
+        checkpath=['unittests/resources/checks/frontend_checks.py'],
+        more_options=[f'--table-format={table_format}']
     )
 
     # Run first a normal run with a performance test to initialize the DB
@@ -1313,7 +1319,7 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0):
     assert_no_crash(*run_reframe2(action=f'--delete-stored-session={uuid}'))
 
 
-def test_performance_compare(run_reframe):
+def test_performance_compare(run_reframe, table_format):
     def assert_no_crash(returncode, stdout, stderr, exitcode=0):
         assert returncode == exitcode
         assert 'Traceback' not in stdout
@@ -1322,7 +1328,8 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0):
 
     run_reframe2 = functools.partial(
         run_reframe,
-        checkpath=['unittests/resources/checks/frontend_checks.py']
+        checkpath=['unittests/resources/checks/frontend_checks.py'],
+        more_options=[f'--table-format={table_format}']
     )
     run_reframe2(action='run')
 

From db5b4bec772bf2f3d4143822423b0ef06789bdcf Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 14 Aug 2024 15:46:40 +0300
Subject: [PATCH 069/128] More compact pretty table output

---
 reframe/frontend/printer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index 6613107df4..a8a88b91cd 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -270,7 +270,7 @@ def table(self, data, **kwargs):
         if table_format == 'plain':
             tablefmt = 'plain'
         elif table_format == 'pretty':
-            tablefmt = 'mixed_grid'
+            tablefmt = 'mixed_outline'
         else:
             raise ValueError(f'invalid table format: {table_format}')
 

From 91c87c2be27de8141d33a599895cb28a096c78f3 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 14 Aug 2024 23:36:03 +0300
Subject: [PATCH 070/128] Add profiler calls to SQlite storage backend

---
 reframe/frontend/cli.py                |  1 +
 reframe/frontend/reporting/__init__.py | 11 ++++++++++-
 reframe/frontend/reporting/storage.py  | 14 +++++++++++++-
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 88f2df5567..72b8531934 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -218,6 +218,7 @@ def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
+        logging.getprofiler().print_report(self.__logger.debug)
         if exc_type is SystemExit:
             # Allow users to exit inside the context manager
             return
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 79831141ae..eb936ece38 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -22,7 +22,7 @@
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
 from reframe.core.exceptions import ReframeError, what, is_severe, reraise_as
-from reframe.core.logging import getlogger, _format_time_rfc3339
+from reframe.core.logging import getlogger, _format_time_rfc3339, time_function
 from reframe.core.runtime import runtime
 from reframe.core.warnings import suppress_deprecations
 from reframe.utility import nodelist_abbrev
@@ -504,6 +504,7 @@ def _group_key(groups, testcase):
     return tuple(key)
 
 
+@time_function
 def _group_testcases(testcases, group_by, extra_cols):
     grouped = {}
     for tc in testcases:
@@ -527,6 +528,7 @@ def _group_testcases(testcases, group_by, extra_cols):
     return grouped
 
 
+@time_function
 def _aggregate_perf(grouped_testcases, aggr_fn, cols):
     if runtime().get_option('general/0/table_format') == 'csv':
         # Use a csv friendly delimiter
@@ -549,6 +551,7 @@ def _aggregate_perf(grouped_testcases, aggr_fn, cols):
     return aggr_data
 
 
+@time_function
 def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
                           extra_group_by=None, extra_cols=None):
     extra_group_by = extra_group_by or []
@@ -587,6 +590,7 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
     return data
 
 
+@time_function
 def performance_compare(cmp, report=None):
     with reraise_as(ReframeError, (ValueError,),
                     'could not parse comparison spec'):
@@ -630,6 +634,7 @@ def performance_compare(cmp, report=None):
                                  match.extra_cols)
 
 
+@time_function
 def session_data(time_period):
     '''Retrieve all sessions'''
 
@@ -649,6 +654,7 @@ def session_data(time_period):
     return data
 
 
+@time_function
 def testcase_data(spec):
     storage = StorageBackend.default()
     if is_uuid(spec):
@@ -682,6 +688,7 @@ def testcase_data(spec):
     return data
 
 
+@time_function
 def session_info(uuid):
     '''Retrieve session details as JSON'''
 
@@ -692,6 +699,7 @@ def session_info(uuid):
     return session
 
 
+@time_function
 def testcase_info(spec):
     '''Retrieve test case details as JSON'''
     testcases = []
@@ -714,5 +722,6 @@ def testcase_info(spec):
     return testcases
 
 
+@time_function
 def delete_session(session_uuid):
     StorageBackend.default().remove_session(session_uuid)
diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 434887ce7c..b32833212e 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -11,7 +11,7 @@
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
 from reframe.core.exceptions import ReframeError
-from reframe.core.logging import getlogger
+from reframe.core.logging import getlogger, time_function, getprofiler
 from reframe.core.runtime import runtime
 
 
@@ -153,7 +153,9 @@ def store(self, report, report_file=None):
             with FileLock(os.path.join(prefix, '.db.lock')):
                 return self._db_store_report(conn, report, report_file)
 
+    @time_function
     def _fetch_testcases_raw(self, condition):
+        getprofiler().enter_region('sqlite query')
         with sqlite3.connect(self._db_file()) as conn:
             query = ('SELECT session_uuid, testcases.uuid as uuid, json_blob '
                      'FROM testcases '
@@ -162,19 +164,24 @@ def _fetch_testcases_raw(self, condition):
             getlogger().debug(query)
             results = conn.execute(query).fetchall()
 
+        getprofiler().exit_region()
+
         # Retrieve files
         testcases = []
         sessions = {}
         for session_uuid, uuid, json_blob in results:
             run_index, test_index = [int(x) for x in uuid.split(':')[1:]]
+            getprofiler().enter_region('json decode')
             report = jsonext.loads(sessions.setdefault(session_uuid,
                                                        json_blob))
+            getprofiler().exit_region()
             testcases.append(
                 report['runs'][run_index]['testcases'][test_index],
             )
 
         return testcases
 
+    @time_function
     def fetch_session_time_period(self, session_uuid):
         with sqlite3.connect(self._db_file()) as conn:
             query = ('SELECT session_start_unix, session_end_unix '
@@ -187,6 +194,7 @@ def fetch_session_time_period(self, session_uuid):
 
             return None, None
 
+    @time_function
     def fetch_testcases_time_period(self, ts_start, ts_end):
         return self._fetch_testcases_raw(
             f'(job_completion_time_unix >= {ts_start} AND '
@@ -194,6 +202,7 @@ def fetch_testcases_time_period(self, ts_start, ts_end):
             'ORDER BY job_completion_time_unix'
         )
 
+    @time_function
     def fetch_testcases_from_session(self, session_uuid):
         with sqlite3.connect(self._db_file()) as conn:
             query = ('SELECT json_blob from sessions '
@@ -207,6 +216,7 @@ def fetch_testcases_from_session(self, session_uuid):
         session_info = jsonext.loads(results[0][0])
         return [tc for run in session_info['runs'] for tc in run['testcases']]
 
+    @time_function
     def fetch_sessions_time_period(self, ts_start=None, ts_end=None):
         with sqlite3.connect(self._db_file()) as conn:
             query = 'SELECT json_blob from sessions'
@@ -229,6 +239,7 @@ def fetch_sessions_time_period(self, ts_start=None, ts_end=None):
 
         return [jsonext.loads(json_blob) for json_blob, *_ in results]
 
+    @time_function
     def fetch_session_json(self, uuid):
         with sqlite3.connect(self._db_file()) as conn:
             query = f'SELECT json_blob FROM sessions WHERE uuid == "{uuid}"'
@@ -263,6 +274,7 @@ def _do_remove2(self, uuid):
                 if not deleted:
                     raise ReframeError(f'no such session: {uuid}')
 
+    @time_function
     def remove_session(self, uuid):
         if sqlite3.sqlite_version_info >= (3, 35, 0):
             self._do_remove2(uuid)

From 34969e50b938c5e397d6cf4fd5cf45c0456744d8 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 15 Aug 2024 00:01:20 +0300
Subject: [PATCH 071/128] Optimize decoding of JSON blobs

---
 reframe/frontend/reporting/storage.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index b32833212e..9395dd8fe2 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -166,15 +166,26 @@ def _fetch_testcases_raw(self, condition):
 
         getprofiler().exit_region()
 
-        # Retrieve files
-        testcases = []
+        # Retrieve session info
         sessions = {}
+        for session_uuid, uuid, json_blob in results:
+            sessions.setdefault(session_uuid, json_blob)
+
+        # Join all sessions and decode them at once
+        reports_blob = '[' + ','.join(sessions.values()) + ']'
+        getprofiler().enter_region('json decode')
+        reports = jsonext.loads(reports_blob)
+        getprofiler().exit_region()
+
+        # Reindex sessions with their decoded data
+        for rpt in reports:
+            sessions[rpt['session_info']['uuid']] = rpt
+
+        # Extract the test case data
+        testcases = []
         for session_uuid, uuid, json_blob in results:
             run_index, test_index = [int(x) for x in uuid.split(':')[1:]]
-            getprofiler().enter_region('json decode')
-            report = jsonext.loads(sessions.setdefault(session_uuid,
-                                                       json_blob))
-            getprofiler().exit_region()
+            report = sessions[session_uuid]
             testcases.append(
                 report['runs'][run_index]['testcases'][test_index],
             )

From 5aa3da5a1cb0f854baddc19c1ffd816e3685f04c Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 22 Aug 2024 17:44:38 +0300
Subject: [PATCH 072/128] Create DB index for testcases table

---
 reframe/frontend/reporting/storage.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 9395dd8fe2..c3627dce33 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -86,6 +86,8 @@ def _db_create(self):
                          'uuid TEXT, '
                          'FOREIGN KEY(session_uuid) '
                          'REFERENCES sessions(uuid) ON DELETE CASCADE)')
+            conn.execute('CREATE INDEX IF NOT EXISTS index_testcases_time '
+                         'on testcases(job_completion_time_unix)')
             conn.execute('CREATE TABLE IF NOT EXISTS metadata('
                          'schema_version TEXT)')
 

From dd37f86c7e15779fb8f3fadade5ccd47aef380e8 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 22 Aug 2024 20:36:31 +0300
Subject: [PATCH 073/128] Support filtering by testcase query options

---
 reframe/frontend/cli.py                | 10 +++++++---
 reframe/frontend/reporting/__init__.py | 20 +++++++++----------
 reframe/frontend/reporting/storage.py  | 27 ++++++++++++++++++++------
 3 files changed, 38 insertions(+), 19 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 72b8531934..8cc57a0d63 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -959,10 +959,11 @@ def restrict_logging():
             sys.exit(0)
 
     if options.list_stored_testcases:
+        namepatt = '|'.join(options.names)
         with exit_gracefully_on_error('failed to retrieve test case data',
                                       printer):
             printer.table(reporting.testcase_data(
-                options.list_stored_testcases
+                options.list_stored_testcases, namepatt
             ))
             sys.exit(0)
 
@@ -979,10 +980,11 @@ def restrict_logging():
     if options.describe_stored_testcases:
         # Restore logging level
         printer.setLevel(logging.INFO)
+        namepatt = '|'.join(options.names)
         with exit_gracefully_on_error('failed to retrieve test case data',
                                       printer):
             printer.info(jsonext.dumps(reporting.testcase_info(
-                options.describe_stored_testcases
+                options.describe_stored_testcases, namepatt
             ), indent=2))
             sys.exit(0)
 
@@ -1405,9 +1407,11 @@ def _sort_testcases(testcases):
             sys.exit(0)
 
         if options.performance_compare:
+            namepatt = '|'.join(options.names)
             try:
                 printer.table(
-                    reporting.performance_compare(options.performance_compare)
+                    reporting.performance_compare(options.performance_compare,
+                                                  namepatt=namepatt)
                 )
             except errors.ReframeError as err:
                 printer.error(f'failed to generate performance report: {err}')
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index eb936ece38..9500c4ec13 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -591,7 +591,7 @@ def compare_testcase_data(base_testcases, target_testcases, base_fn, target_fn,
 
 
 @time_function
-def performance_compare(cmp, report=None):
+def performance_compare(cmp, report=None, namepatt=None):
     with reraise_as(ReframeError, (ValueError,),
                     'could not parse comparison spec'):
         match = parse_cmp_spec(cmp)
@@ -613,20 +613,20 @@ def performance_compare(cmp, report=None):
             tcs_base = []
     elif match.period_base is not None:
         tcs_base = StorageBackend.default().fetch_testcases_time_period(
-            *match.period_base
+            *match.period_base, namepatt
         )
     else:
         tcs_base = StorageBackend.default().fetch_testcases_from_session(
-            match.session_base
+            match.session_base, namepatt
         )
 
     if match.period_target:
         tcs_target = StorageBackend.default().fetch_testcases_time_period(
-            *match.period_target
+            *match.period_target, namepatt
         )
     else:
         tcs_target = StorageBackend.default().fetch_testcases_from_session(
-            match.session_target
+            match.session_target, namepatt
         )
 
     return compare_testcase_data(tcs_base, tcs_target, match.aggregator,
@@ -655,13 +655,13 @@ def session_data(time_period):
 
 
 @time_function
-def testcase_data(spec):
+def testcase_data(spec, namepatt=None):
     storage = StorageBackend.default()
     if is_uuid(spec):
-        testcases = storage.fetch_testcases_from_session(spec)
+        testcases = storage.fetch_testcases_from_session(spec, namepatt)
     else:
         testcases = storage.fetch_testcases_time_period(
-            *parse_time_period(spec)
+            *parse_time_period(spec), namepatt
         )
 
     data = [['Name', 'SysEnv',
@@ -700,7 +700,7 @@ def session_info(uuid):
 
 
 @time_function
-def testcase_info(spec):
+def testcase_info(spec, namepatt=None):
     '''Retrieve test case details as JSON'''
     testcases = []
     if is_uuid(spec):
@@ -716,7 +716,7 @@ def testcase_info(spec):
             )
     else:
         testcases = StorageBackend.default().fetch_testcases_time_period(
-            *parse_time_period(spec)
+            *parse_time_period(spec), namepatt
         )
 
     return testcases
diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index c3627dce33..87fed08720 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -5,6 +5,7 @@
 
 import abc
 import os
+import re
 import sqlite3
 from filelock import FileLock
 
@@ -64,6 +65,13 @@ def _db_file(self):
         self._db_schema_check()
         return self.__db_file
 
+    def _db_matches(self, patt, item):
+        if patt is None:
+            return True
+
+        regex = re.compile(patt)
+        return regex.match(item) is not None
+
     def _db_create(self):
         clsname = type(self).__name__
         getlogger().debug(
@@ -164,6 +172,9 @@ def _fetch_testcases_raw(self, condition):
                      'JOIN sessions ON session_uuid == sessions.uuid '
                      f'WHERE {condition}')
             getlogger().debug(query)
+
+            # Create SQLite function for filtering using name patterns
+            conn.create_function('REGEXP', 2, self._db_matches)
             results = conn.execute(query).fetchall()
 
         getprofiler().exit_region()
@@ -208,15 +219,18 @@ def fetch_session_time_period(self, session_uuid):
             return None, None
 
     @time_function
-    def fetch_testcases_time_period(self, ts_start, ts_end):
+    def fetch_testcases_time_period(self, ts_start, ts_end, name_pattern=None):
+        expr = (f'job_completion_time_unix >= {ts_start} AND '
+                f'job_completion_time_unix <= {ts_end}')
+        if name_pattern:
+            expr += f' AND name REGEXP "{name_pattern}"'
+
         return self._fetch_testcases_raw(
-            f'(job_completion_time_unix >= {ts_start} AND '
-            f'job_completion_time_unix <= {ts_end}) '
-            'ORDER BY job_completion_time_unix'
+            f'({expr}) ORDER BY job_completion_time_unix'
         )
 
     @time_function
-    def fetch_testcases_from_session(self, session_uuid):
+    def fetch_testcases_from_session(self, session_uuid, name_pattern=None):
         with sqlite3.connect(self._db_file()) as conn:
             query = ('SELECT json_blob from sessions '
                      f'WHERE uuid == "{session_uuid}"')
@@ -227,7 +241,8 @@ def fetch_testcases_from_session(self, session_uuid):
             return []
 
         session_info = jsonext.loads(results[0][0])
-        return [tc for run in session_info['runs'] for tc in run['testcases']]
+        return [tc for run in session_info['runs'] for tc in run['testcases']
+                if self._db_matches(name_pattern, tc['name'])]
 
     @time_function
     def fetch_sessions_time_period(self, ts_start=None, ts_end=None):

From e0f432008f296d1b6635a8c92508382a0f77e39a Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 23 Aug 2024 12:12:29 +0300
Subject: [PATCH 074/128] Add unit tests for test filtering in testcase query
 options

---
 unittests/test_reporting.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/unittests/test_reporting.py b/unittests/test_reporting.py
index 509e8d4d7a..3b602b0166 100644
--- a/unittests/test_reporting.py
+++ b/unittests/test_reporting.py
@@ -442,6 +442,13 @@ def _count_failed(testcases):
     assert len(testcases) == 12
     assert _count_failed(testcases) == 6
 
+    # Test name filtering
+    testcases = backend.fetch_testcases_time_period(timestamps[0][0],
+                                                    timestamps[1][1],
+                                                    '^HelloTest')
+    assert len(testcases) == 2
+    assert _count_failed(testcases) == 0
+
     # Test the inverted period
     assert backend.fetch_testcases_time_period(timestamps[1][1],
                                                timestamps[0][0]) == []
@@ -452,6 +459,11 @@ def _count_failed(testcases):
         assert len(testcases) == 9
         assert _count_failed(testcases) == 5
 
+        # Test name filtering
+        testcases = backend.fetch_testcases_from_session(uuid, '^HelloTest')
+        assert len(testcases) == 1
+        assert _count_failed(testcases) == 0
+
     # Test an invalid uuid
     assert backend.fetch_testcases_from_session(0) == []
 

From 159894dc377d68bf0d576aba81bc1e4d641663cf Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 23 Aug 2024 14:05:18 +0300
Subject: [PATCH 075/128] Do not print "ReFrame setup" header with
 `--performance-compare` option

---
 reframe/frontend/cli.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 8cc57a0d63..ec66aacf93 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -995,6 +995,19 @@ def restrict_logging():
             printer.info(f'Session {session_uuid} deleted successfully.')
             sys.exit(0)
 
+    if options.performance_compare:
+        namepatt = '|'.join(options.names)
+        try:
+            printer.table(
+                reporting.performance_compare(options.performance_compare,
+                                                namepatt=namepatt)
+            )
+        except errors.ReframeError as err:
+            printer.error(f'failed to generate performance report: {err}')
+            sys.exit(1)
+        else:
+            sys.exit(0)
+
     # Show configuration after everything is set up
     if options.show_config:
         # Restore logging level
@@ -1406,19 +1419,6 @@ def _sort_testcases(testcases):
             )
             sys.exit(0)
 
-        if options.performance_compare:
-            namepatt = '|'.join(options.names)
-            try:
-                printer.table(
-                    reporting.performance_compare(options.performance_compare,
-                                                  namepatt=namepatt)
-                )
-            except errors.ReframeError as err:
-                printer.error(f'failed to generate performance report: {err}')
-                sys.exit(1)
-            else:
-                sys.exit(0)
-
         # Manipulate ReFrame's environment
         if site_config.get('general/0/purge_environment'):
             rt.modules_system.unload_all()

From c4f6fc1baa448376ed3f7095d9effe627184b8ac Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 23 Aug 2024 13:35:16 +0300
Subject: [PATCH 076/128] Do not store session if dry run

---
 docs/tutorial.rst       | 5 ++---
 reframe/frontend/cli.py | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index a6b46c1deb..830da136d1 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -187,9 +187,8 @@ Finally, you can use also the :option:`--performance-report` option, which will
    ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━┑
    │ name        │ sysenv                  │ pvar     │    pval │ punit   │ pdiff   │ job_nodelist   │ result   │
    ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━━━━━━━━┿━━━━━━━━━━┥
-   │ stream_test │ generic:default+builtin │ copy_bw  │ 40304.2 │ MB/s    │ -0.08%  │ myhost         │ pass     │
-   ├─────────────┼─────────────────────────┼──────────┼─────────┼─────────┼─────────┼────────────────┼──────────┤
-   │ stream_test │ generic:default+builtin │ triad_bw │ 30550.3 │ MB/s    │ +0.04%  │ myhost         │ pass     │
+   │ stream_test │ generic:default+builtin │ copy_bw  │ 40310.2 │ MB/s    │ +0.05%  │ myhost         │ pass     │
+   │ stream_test │ generic:default+builtin │ triad_bw │ 30533.2 │ MB/s    │ -0.06%  │ myhost         │ pass     │
    ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━┙
 
 
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index ec66aacf93..6fa3c51b34 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -1615,7 +1615,7 @@ def module_unuse(*paths):
                     )
 
             # Store the generated report for analytics
-            if not report.is_empty():
+            if not report.is_empty() and not options.dry_run:
                 try:
                     sess_uuid = report.store()
                 except Exception as e:

From f2d3223abf51253fddaa9be1441b2b46f9d7d518 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 23 Aug 2024 13:34:04 +0300
Subject: [PATCH 077/128] Update docs and tutorial

---
 docs/manpage.rst                              |   9 +-
 docs/tutorial.rst                             | 120 +++++++++---------
 .../dockerfiles/singlenode.Dockerfile         |   8 +-
 3 files changed, 70 insertions(+), 67 deletions(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index 0ab2707771..50387abdbb 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -299,8 +299,8 @@ There are commands that act upon the selected tests and others that have a helpe
    Get detailed test case information of the session with the specified UUID or from the specified time period.
 
    If a session UUID is provided only information about the test cases of this session will be provided.
-
-   For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`.
+   This option can be combined with :option:`--name` to restrict the listing to specific tests.
+   For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax` section.
 
    .. versionadded:: 4.7
 
@@ -359,8 +359,8 @@ There are commands that act upon the selected tests and others that have a helpe
    List all test cases from the session with the specified UUID or from the specified time period.
 
    If a session UUID is provided only the test cases of this session will be listed.
-
-   For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`.
+   This option can be combined with :option:`--name` to restrict the listing to specific tests.
+   For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax` section.
 
    .. versionadded:: 4.7
 
@@ -394,6 +394,7 @@ There are commands that act upon the selected tests and others that have a helpe
 
    Compare the performance of test cases that have run in the past.
 
+   This option can be combined with :option:`--name` to restrict the comparison to specific tests.
    Check the :ref:`performance-comparisons` section for the exact syntax of ``CMPSPEC``.
 
    .. versionadded:: 4.7
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index 830da136d1..888671c660 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -187,8 +187,8 @@ Finally, you can use also the :option:`--performance-report` option, which will
    ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━┑
    │ name        │ sysenv                  │ pvar     │    pval │ punit   │ pdiff   │ job_nodelist   │ result   │
    ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━━━━━━━━┿━━━━━━━━━━┥
-   │ stream_test │ generic:default+builtin │ copy_bw  │ 40310.2 │ MB/s    │ +0.05%  │ myhost         │ pass     │
-   │ stream_test │ generic:default+builtin │ triad_bw │ 30533.2 │ MB/s    │ -0.06%  │ myhost         │ pass     │
+   │ stream_test │ generic:default+builtin │ copy_bw  │ 40292.1 │ MB/s    │ -0.04%  │ myhost         │ pass     │
+   │ stream_test │ generic:default+builtin │ triad_bw │ 30564.7 │ MB/s    │ +0.12%  │ myhost         │ pass     │
    ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━┙
 
 
@@ -1988,6 +1988,7 @@ Essentially, the stored information is the same as the one found in the :ref:`re
 To list the stored sessions use the :option:`--list-stored-sessions` option:
 
 .. code-block:: bash
+   :caption: Run in the single-node container.
 
    reframe --list-stored-sessions
 
@@ -1996,47 +1997,42 @@ its unique identifier, its start and end time and how many test cases have run:
 
 .. code-block:: console
 
-   ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━┑
+   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━┑
    │ UUID                                 │ Start time           │ End time             │   Num runs │   Num cases │
    ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━┥
-   │ fddb6678-6de2-427c-96b5-d1c6b3215b0e │ 20240809T135331+0000 │ 20240809T135335+0000 │          1 │           1 │
-   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
-   │ d96a133c-e5a8-4ceb-88de-f8adfb393f28 │ 20240809T135342+0000 │ 20240809T135345+0000 │          1 │           1 │
-   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
-   │ c7508042-64be-406a-89f1-c5d31b90f838 │ 20240809T143710+0000 │ 20240809T143713+0000 │          1 │           1 │
-   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
-   │ 3bc5c067-42fa-4496-a5e4-50b92b3cc38e │ 20240809T144025+0000 │ 20240809T144026+0000 │          1 │           2 │
-   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
-   │ 53481b75-b98a-4668-b6ab-82b199cc2efe │ 20240809T144056+0000 │ 20240809T144057+0000 │          1 │           4 │
-   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
-   │
-   ...
-   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
-   │ bed145ca-0013-4b68-bfd4-620054121f91 │ 20240809T144459+0000 │ 20240809T144500+0000 │          1 │          10 │
-   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
-   │ a72c7536-274a-4a21-92c3-4116f38febd0 │ 20240809T144500+0000 │ 20240809T144500+0000 │          1 │           1 │
-   ├──────────────────────────────────────┼──────────────────────┼──────────────────────┼────────────┼─────────────┤
-   │ 8cb26ff4-7897-42fc-a993-fbdef57c8983 │ 20240809T144510+0000 │ 20240809T144511+0000 │          1 │           5 │
+   │ fedb2cf8-6efa-43d8-a6dc-e72c868deba6 │ 20240823T104554+0000 │ 20240823T104557+0000 │          1 │           1 │
+   │ 4253d6b3-3926-4c4c-a7e8-3f7dffe9bf23 │ 20240823T104608+0000 │ 20240823T104612+0000 │          1 │           1 │
+   │ 453e64a2-f941-49e2-b628-bf50883a6387 │ 20240823T104721+0000 │ 20240823T104725+0000 │          1 │           1 │
+   │ d923cca2-a72b-43ca-aca1-de741b65088b │ 20240823T104753+0000 │ 20240823T104757+0000 │          1 │           1 │
+   │ 300b973b-84a6-4932-89eb-577a832fe357 │ 20240823T104814+0000 │ 20240823T104815+0000 │          1 │           2 │
+   │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e │ 20240823T104834+0000 │ 20240823T104835+0000 │          1 │           4 │
+   │ 2a00c55d-4492-498c-89f0-7cf821f308c1 │ 20240823T104843+0000 │ 20240823T104845+0000 │          1 │           4 │
+   │ 98fe5a68-2582-49ca-9c3c-6bfd9b877143 │ 20240823T104902+0000 │ 20240823T104903+0000 │          1 │           4 │
+   │ 4bbc27bc-be50-4cca-9d1b-c5fb4988a5c0 │ 20240823T104922+0000 │ 20240823T104933+0000 │          1 │          26 │
+   │ 200ea28f-6c3a-4973-a2b7-aa08408dbeec │ 20240823T104939+0000 │ 20240823T104943+0000 │          1 │          10 │
+   │ b756755b-3181-4bb4-9eaa-cc8c3a9d7a43 │ 20240823T104955+0000 │ 20240823T104956+0000 │          1 │          10 │
+   │ a8a99808-c22d-4b9c-83bc-164289fe6aa7 │ 20240823T105007+0000 │ 20240823T105007+0000 │          1 │           4 │
+   │ f9b63cdc-7dda-44c5-ab85-1e9752047834 │ 20240823T105019+0000 │ 20240823T105020+0000 │          1 │          10 │
+   │ 271fc2e7-b550-4325-b8bb-57bdf95f1d0d │ 20240823T105020+0000 │ 20240823T105020+0000 │          1 │           1 │
+   │ 50cdb774-f231-4f61-8472-7daaa5199d57 │ 20240823T105031+0000 │ 20240823T105032+0000 │          1 │           5 │
    ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━┙
 
 You can use the :option:`--list-stored-testcases` to list the test cases of a specific session or those that have run within a certain period of time:
 
 .. code-block:: bash
+   :caption: Run in the single-node container.
 
-   reframe --list-stored-testcases=53481b75-b98a-4668-b6ab-82b199cc2efe
+   reframe --list-stored-testcases=1fb8488e-c361-4355-b7df-c0dcf3cdcc1e
 
 .. code-block:: console
 
    ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑
    │ Name                                    │ SysEnv                    │ Nodelist   │ Completion Time      │ Result   │ UUID                                     │
    ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥
-   │ build_stream ~tutorialsys:default+gnu   │ tutorialsys:default+gnu   │            │ 20240809T145439+0000 │ pass     │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:0 │
-   ├─────────────────────────────────────────┼───────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
-   │ build_stream ~tutorialsys:default+clang │ tutorialsys:default+clang │            │ 20240809T145439+0000 │ pass     │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:1 │
-   ├─────────────────────────────────────────┼───────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
-   │ stream_test                             │ tutorialsys:default+gnu   │ myhost     │ 20240809T144057+0000 │ pass     │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:2 │
-   ├─────────────────────────────────────────┼───────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
-   │ stream_test                             │ tutorialsys:default+clang │ myhost     │ 20240809T144057+0000 │ pass     │ 53481b75-b98a-4668-b6ab-82b199cc2efe:0:3 │
+   │ build_stream ~tutorialsys:default+gnu   │ tutorialsys:default+gnu   │            │ n/a                  │ pass     │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:0 │
+   │ build_stream ~tutorialsys:default+clang │ tutorialsys:default+clang │            │ n/a                  │ pass     │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:1 │
+   │ stream_test                             │ tutorialsys:default+gnu   │ myhost     │ 20240823T104835+0000 │ pass     │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:2 │
+   │ stream_test                             │ tutorialsys:default+clang │ myhost     │ 20240823T104835+0000 │ pass     │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:3 │
    ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
 
 
@@ -2046,63 +2042,69 @@ A session may have multiple runs if it has retried some failed test cases (see :
 You can also list the test cases that have run in a certain period of time use the :ref:`time period <time-period-syntax>` of :option:`--list-stored-testcases`:
 
 .. code-block:: bash
+   :caption: Run in the single-node container.
 
-   reframe --list-stored-testcases=20240809T144500+0000:now
+   reframe --list-stored-testcases=20240823T104835+0000:now
 
 .. code-block:: console
 
-   ┍━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑
-   │ Name   │ SysEnv                  │ Nodelist   │ Completion Time      │ Result   │ UUID                                     │
-   ┝━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥
-   │ T2     │ generic:default+builtin │ myhost     │ 20240809T144500+0000 │ fail     │ bed145ca-0013-4b68-bfd4-620054121f91:0:7 │
-   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
-   │ T3     │ generic:default+builtin │ myhost     │ 20240809T144500+0000 │ pass     │ bed145ca-0013-4b68-bfd4-620054121f91:0:9 │
-   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
-   │ T6     │ generic:default+builtin │ myhost     │ 20240809T144500+0000 │ pass     │ a72c7536-274a-4a21-92c3-4116f38febd0:0:0 │
-   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
-   │ T0     │ generic:default+builtin │ myhost     │ 20240809T144510+0000 │ pass     │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:0 │
-   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
-   │ T4     │ generic:default+builtin │ myhost     │ 20240809T144510+0000 │ pass     │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:1 │
-   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
-   │ T5     │ generic:default+builtin │ myhost     │ 20240809T144510+0000 │ pass     │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:2 │
-   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
-   │ T1     │ generic:default+builtin │ myhost     │ 20240809T144510+0000 │ pass     │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:3 │
-   ├────────┼─────────────────────────┼────────────┼──────────────────────┼──────────┼──────────────────────────────────────────┤
-   │ T6     │ generic:default+builtin │ myhost     │ 20240809T144510+0000 │ pass     │ 8cb26ff4-7897-42fc-a993-fbdef57c8983:0:4 │
-   ┕━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
+   ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑
+   │ Name                                                │ SysEnv                    │ Nodelist   │ Completion Time      │ Result   │ UUID                                      │
+   ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥
+   │ stream_test                                         │ tutorialsys:default+gnu   │ myhost     │ 20240823T104835+0000 │ pass     │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:2  │
+   │ stream_test                                         │ tutorialsys:default+clang │ myhost     │ 20240823T104835+0000 │ pass     │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:3  │
+   │ stream_test                                         │ tutorialsys:default+gnu   │ myhost     │ 20240823T104844+0000 │ pass     │ 2a00c55d-4492-498c-89f0-7cf821f308c1:0:2  │
+   │ stream_test                                         │ tutorialsys:default+clang │ myhost     │ 20240823T104845+0000 │ pass     │ 2a00c55d-4492-498c-89f0-7cf821f308c1:0:3  │
+   │ stream_test                                         │ tutorialsys:default+gnu   │ myhost     │ 20240823T104903+0000 │ pass     │ 98fe5a68-2582-49ca-9c3c-6bfd9b877143:0:2  │
+   │ stream_test                                         │ tutorialsys:default+clang │ myhost     │ 20240823T104903+0000 │ pass     │ 98fe5a68-2582-49ca-9c3c-6bfd9b877143:0:3  │
+   ...
+   │ T6                                                  │ generic:default+builtin   │ myhost     │ 20240823T105020+0000 │ pass     │ 271fc2e7-b550-4325-b8bb-57bdf95f1d0d:0:0  │
+   │ T0                                                  │ generic:default+builtin   │ myhost     │ 20240823T105031+0000 │ pass     │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:0  │
+   │ T4                                                  │ generic:default+builtin   │ myhost     │ 20240823T105031+0000 │ pass     │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:1  │
+   │ T5                                                  │ generic:default+builtin   │ myhost     │ 20240823T105031+0000 │ pass     │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:2  │
+   │ T1                                                  │ generic:default+builtin   │ myhost     │ 20240823T105031+0000 │ pass     │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:3  │
+   │ T6                                                  │ generic:default+builtin   │ myhost     │ 20240823T105032+0000 │ pass     │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:4  │
+   ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
 
 To get all the details of a session or a set of test cases you can use the :option:`--describe-stored-session` and :option:`--describe-stored-testcases` options which will return a JSON record with all the details.
 
+You can also combine the :option:`-n` option with the :option:`--list-stored-testcases` and :option:`--describe-stored-testcases` options in order to restrict the listing to specific tests only:
+
+.. code-block:: bash
+   :caption: Run in the single-node container.
+
+   reframe --list-stored-testcases=20240823T104835+0000:now -n stream_test
+
+
 Comparing performance of test cases
 -----------------------------------
 
 ReFrame can be used to compare the performance of the same test cases run in different time periods using the :option:`--performance-compare` option.
-The following will compare the performance of the test cases of the session ``a120b895-8fe9-4209-a742-997442e37c47`` with any other same test case that has run the last 24h:
+The following will compare the performance of the test cases of the session ``1fb8488e-c361-4355-b7df-c0dcf3cdcc1e`` with any other same test case that has run the last 24h:
 
 .. code-block:: bash
+   :caption: Run in the single-node container.
 
-   reframe --performance-compare=a120b895-8fe9-4209-a742-997442e37c47/now-1d:now/mean:/
+   reframe --performance-compare=1fb8488e-c361-4355-b7df-c0dcf3cdcc1e/now-1d:now/mean:/
 
 .. code-block:: console
 
    ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┑
    │ name        │ sysenv                    │ pvar     │    pval │ punit   │ pdiff   │
    ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┥
-   │ stream_test │ tutorialsys:default+gnu   │ copy_bw  │ 31274.5 │ MB/s    │ -22.47% │
-   ├─────────────┼───────────────────────────┼──────────┼─────────┼─────────┼─────────┤
-   │ stream_test │ tutorialsys:default+gnu   │ triad_bw │ 18993.4 │ MB/s    │ -42.02% │
-   ├─────────────┼───────────────────────────┼──────────┼─────────┼─────────┼─────────┤
-   │ stream_test │ tutorialsys:default+clang │ copy_bw  │ 38546.2 │ MB/s    │ -9.24%  │
-   ├─────────────┼───────────────────────────┼──────────┼─────────┼─────────┼─────────┤
-   │ stream_test │ tutorialsys:default+clang │ triad_bw │ 36866.3 │ MB/s    │ -4.72%  │
+   │ stream_test │ tutorialsys:default+gnu   │ copy_bw  │ 44139   │ MB/s    │ +11.14% │
+   │ stream_test │ tutorialsys:default+gnu   │ triad_bw │ 39344.7 │ MB/s    │ +20.77% │
+   │ stream_test │ tutorialsys:default+clang │ copy_bw  │ 44979.1 │ MB/s    │ +10.81% │
+   │ stream_test │ tutorialsys:default+clang │ triad_bw │ 39330.8 │ MB/s    │ +8.28%  │
    ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┙
 
+The :option:`-n` option can also be combined with :option:`--performance-compare` to restrict the test cases listed.
 Similarly to the :option:`--performance-compare` option, the :option:`--performance-report` option can compare the performance of the current run with any arbitrary past session or past time period.
 
 Finally, you can delete complete a stored session using the :option:`--delete-stored-session` option:
 
 .. code-block:: bash
 
-   reframe --delete-stored-session=a120b895-8fe9-4209-a742-997442e37c47
+   reframe --delete-stored-session=1fb8488e-c361-4355-b7df-c0dcf3cdcc1e
 
 Deleting a session will also delete all its test cases from the database.
diff --git a/examples/tutorial/dockerfiles/singlenode.Dockerfile b/examples/tutorial/dockerfiles/singlenode.Dockerfile
index 86750b20d8..2c8fed9c78 100644
--- a/examples/tutorial/dockerfiles/singlenode.Dockerfile
+++ b/examples/tutorial/dockerfiles/singlenode.Dockerfile
@@ -12,13 +12,13 @@ ARG REFRAME_REPO=reframe-hpc
 WORKDIR /usr/local/share
 
 # Clone reframe
-RUN git clone --depth 1 --branch $REFRAME_TAG https://github.com/$REFRAME_REPO/reframe.git && \
-    cd reframe/ && ./bootstrap.sh
+# RUN git clone --depth 1 --branch $REFRAME_TAG https://github.com/$REFRAME_REPO/reframe.git && \
+#     cd reframe/ && ./bootstrap.sh
 
 # Comment the above line and uncomment the following two for development
 
-# COPY . /usr/local/share/reframe
-# RUN cd reframe && ./bootstrap.sh
+COPY . /usr/local/share/reframe
+RUN cd reframe && ./bootstrap.sh
 
 ENV PATH=/usr/local/share/reframe/bin:$PATH
 

From e2c7cb5e4f8705643a45fe355c1a1112b453a652 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 23 Aug 2024 15:09:14 +0300
Subject: [PATCH 078/128] Fix listing formatting in tutorial

---
 docs/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index 888671c660..f842125069 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -1997,7 +1997,7 @@ its unique identifier, its start and end time and how many test cases have run:
 
 .. code-block:: console
 
-   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━┑
+   ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━┑
    │ UUID                                 │ Start time           │ End time             │   Num runs │   Num cases │
    ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━┥
    │ fedb2cf8-6efa-43d8-a6dc-e72c868deba6 │ 20240823T104554+0000 │ 20240823T104557+0000 │          1 │           1 │

From d85690ca205aaf1f94f6a78e3bf2f62fefe57abb Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 23 Aug 2024 15:59:22 +0300
Subject: [PATCH 079/128] Support for using always unqualified hostnames in
 local scheduler

---
 docs/config_reference.rst        | 10 ++++++++++
 reframe/core/schedulers/local.py |  7 ++++++-
 reframe/schemas/config.json      |  2 ++
 unittests/test_schedulers.py     | 15 +++++++++++++++
 4 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/docs/config_reference.rst b/docs/config_reference.rst
index beeb9b457b..57fbcd4daa 100644
--- a/docs/config_reference.rst
+++ b/docs/config_reference.rst
@@ -439,6 +439,16 @@ System Partition Configuration
       No other test would be able to proceed.
 
 
+.. py:attribute:: systems.partitions.sched_options.unqualified_hostnames
+
+   :required: No
+   :default: ``false``
+
+   Use unqualified hostnames in the ``local`` scheduler backend.
+
+   .. versionadded:: 4.7
+
+
 .. py:attribute:: systems.partitions.sched_options.use_nodes_option
 
    :required: No
diff --git a/reframe/core/schedulers/local.py b/reframe/core/schedulers/local.py
index 5e41491f0f..87eead7530 100644
--- a/reframe/core/schedulers/local.py
+++ b/reframe/core/schedulers/local.py
@@ -70,7 +70,12 @@ def submit(self, job):
 
         # Update job info
         job._jobid = proc.pid
-        job._nodelist = [socket.gethostname()]
+        hostname = socket.gethostname()
+        if self.get_option('unqualified_hostnames'):
+            job._nodelist = [hostname.split('.')[0]]
+        else:
+            job._nodelist = [hostname]
+
         job._proc = proc
         job._f_stdout = f_stdout
         job._f_stderr = f_stderr
diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index 06765f604a..7183f787d0 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -120,6 +120,7 @@
                     "type": "array",
                     "items": {"type": "string"}
                 },
+                "unqualified_hostnames": {"type": "boolean"},
                 "use_nodes_option": {"type": "boolean"}
             }
         },
@@ -659,6 +660,7 @@
         "systems*/sched_options/ignore_reqnodenotavail": false,
         "systems*/sched_options/job_submit_timeout": 60,
         "systems*/sched_options/resubmit_on_errors": [],
+        "systems*/sched_options/unqualified_hostnames": false,
         "systems*/sched_options/use_nodes_option": false
     }
 }
diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py
index b2da80bcfb..a11223121f 100644
--- a/unittests/test_schedulers.py
+++ b/unittests/test_schedulers.py
@@ -522,6 +522,21 @@ def test_submit_timelimit(minimal_job, local_only):
         assert minimal_job.state == 'TIMEOUT'
 
 
+def test_submit_unqualified_hostnames(make_exec_ctx, make_job, local_only):
+    make_exec_ctx(
+        system='testsys',
+        options={
+            'systems/partitions/sched_options/unqualified_hostnames': True
+        }
+    )
+    hostname = socket.gethostname().split('.')[0]
+    minimal_job = make_job(sched_opts={'part_name': 'login'})
+    minimal_job.prepare('true')
+    minimal_job.submit()
+    minimal_job.wait()
+    assert minimal_job.nodelist == [hostname]
+
+
 def test_submit_job_array(make_job, slurm_only, exec_ctx):
     job = make_job(sched_access=exec_ctx.access)
     job.options = ['--array=0-1']

From 4088f2b7f1326fd109e32240a8ed31b5c3365813 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 23 Aug 2024 16:24:39 +0300
Subject: [PATCH 080/128] Make perf. reference selectable as a column

---
 reframe/frontend/reporting/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 9500c4ec13..3c952f8770 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -516,6 +516,7 @@ def _group_testcases(testcases, group_by, extra_cols):
             record = {
                 'pvar': pvar,
                 'pval': pval,
+                'pref': pref,
                 'plower': plower,
                 'pupper': pupper,
                 'punit': punit,

From 73af49fa7a31219608efc1bc19c062cfe17d7cb5 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Wed, 28 Aug 2024 12:01:28 +0200
Subject: [PATCH 081/128] Apply suggestions from code review

Co-authored-by: Theofilos Manitaras <manitaras@cscs.ch>
---
 docs/manpage.rst  | 8 ++++----
 docs/tutorial.rst | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index 50387abdbb..1265067a51 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -1096,7 +1096,7 @@ Miscellaneous options
 
    For each test all of their performance variables are reported and optionally compared to past results based on the ``CMPSPEC`` specified.
 
-   If not specified, the default ``CMPSPEC`` is ``19700101T0000+0000:now/last:+job_nodelist/+result``, meaning that the current performance will be compared to the last run oif the same test grouped additionally by the ``job_nodelist`` and showing also the obtained result (``pass`` or ``fail``).
+   If not specified, the default ``CMPSPEC`` is ``19700101T0000+0000:now/last:+job_nodelist/+result``, meaning that the current performance will be compared to the last run of the same test grouped additionally by the ``job_nodelist`` and showing also the obtained result (``pass`` or ``fail``).
 
    For the exact syntax of ``CMPSPEC``, refer to :ref:`performance-comparisons`.
 
@@ -1297,15 +1297,15 @@ Result storage
 
 .. versionadded:: 4.7
 
-ReFrame stores the results of every session that has exectuted at least one test into a database.
-There is only one storage backend supported at the moment and this is Sqlite.
+ReFrame stores the results of every session that has executed at least one test into a database.
+There is only one storage backend supported at the moment and this is SQLite.
 The full session information as recorded in a run report file (see :option:`--report-file`) is stored in the database.
 The test cases of the session are indexed by their run job completion time for quick retrieval of all the test cases that have run in a certain period of time.
 
 The database file is controlled by the :attr:`~config.storage.sqlite_db_file` configuration parameter and multiple ReFrame processes can access it safely simultaneously.
 
 There are several command-line options that allow users to query the results database, such as the :option:`--list-stored-sessions`, :option:`--list-stored-testcases`, :option:`--describe-stored-session` etc.
-Other options that access the results database are the :option:`--performance-compare` and :option:`--performance-report` options which compare the performance results of the same test cases in different periods of time or from different sessions.
+Other options that access the results database are the :option:`--performance-compare` and :option:`--performance-report` which compare the performance results of the same test cases in different periods of time or from different sessions.
 Check the :ref:`commands` section for the complete list and details of each option related to the results database.
 
 Since the report file information is now kept in the results database, there is no need to keep the report files separately, although this remains the default behavior for backward compatibility.
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index f842125069..f0b9540291 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -168,7 +168,7 @@ This can be suppressed by increasing the level at which this information is logg
 Run reports and performance logging
 -----------------------------------
 
-Once a test session finishes, ReFrame stores the detailed session information in database file located under ``$HOME/.reframe/reports``.
+Once a test session finishes, ReFrame stores the detailed session information in a database file located under ``$HOME/.reframe/reports``.
 Past performance data can be retrieved from this database and compared with the current or another run.
 We detail handling of the results database in section :ref:`inspecting-past-results`.
 
@@ -180,7 +180,7 @@ These files are located by default under ``perflogs/<system>/<partition>/<testna
 In our example, this translates to ``perflogs/generic/default/stream_test.log``.
 The information that is being logged is fully configurable and we will cover this in the :ref:`logging` section.
 
-Finally, you can use also the :option:`--performance-report` option, which will print a summary of the results of the performance tests that have run in the current session and compare them (by default) with their last obtained performance.
+Finally, you can use also the :option:`--performance-report` option, which will print a summary of the results of the performance tests that have run in the current session and compare them (by default) to their last obtained performance.
 
 .. code-block:: console
 
@@ -2101,7 +2101,7 @@ The following will compare the performance of the test cases of the session ``1f
 The :option:`-n` option can also be combined with :option:`--performance-compare` to restrict the test cases listed.
 Similarly to the :option:`--performance-compare` option, the :option:`--performance-report` option can compare the performance of the current run with any arbitrary past session or past time period.
 
-Finally, you can delete complete a stored session using the :option:`--delete-stored-session` option:
+Finally, you can delete completely a stored session using the :option:`--delete-stored-session` option:
 
 .. code-block:: bash
 

From d2612e6c96c1cf6c856026d4817a6f94956f3c58 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 28 Aug 2024 11:56:35 +0200
Subject: [PATCH 082/128] Print info line about results DB file

---
 reframe/frontend/cli.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 6fa3c51b34..da801dd453 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -1158,6 +1158,10 @@ def print_infoline(param, value):
     print_infoline('output directory', repr(session_info['prefix_output']))
     print_infoline('log files',
                    ', '.join(repr(s) for s in session_info['log_files']))
+    print_infoline(
+        'results database',
+        repr(osext.expandvars(rt.get_option('storage/0/sqlite_db_file')))
+    )
     printer.info('')
     try:
         logging.getprofiler().enter_region('test processing')

From 75763c901e85e87ef88d1ca53b8d766087edeb50 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 28 Aug 2024 11:57:37 +0200
Subject: [PATCH 083/128] Fix coding style

---
 reframe/frontend/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index da801dd453..e4e3bd1f27 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -1000,7 +1000,7 @@ def restrict_logging():
         try:
             printer.table(
                 reporting.performance_compare(options.performance_compare,
-                                                namepatt=namepatt)
+                                              namepatt=namepatt)
             )
         except errors.ReframeError as err:
             printer.error(f'failed to generate performance report: {err}')

From 2572d1fa0801e4ccce48deb3073102369a7dc220 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 28 Aug 2024 15:29:37 +0200
Subject: [PATCH 084/128] Address remaining PR comments

---
 docs/manpage.rst  | 2 +-
 docs/tutorial.rst | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index 1265067a51..5630174514 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -1422,7 +1422,7 @@ The ``<abs_timestamp>`` is an absolute timestamp in one of the following ``strpt
 
 Optionally, a shift argument can be appended with ``+`` or ``-`` signs, followed by an amount of weeks (``w``), days (``d``), hours (``h``) or minutes (``m``).
 
-For example, the period of the last 10 days can be specified as ``now:now-10d``.
+For example, the period of the last 10 days can be specified as ``now-10d:now``.
 Similarly, the period of the week starting on August 5, 2024 will be specified as ``20240805:20240805+1w``.
 
 Environment
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index f0b9540291..e2d8bd2a99 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -170,7 +170,7 @@ Run reports and performance logging
 
 Once a test session finishes, ReFrame stores the detailed session information in a database file located under ``$HOME/.reframe/reports``.
 Past performance data can be retrieved from this database and compared with the current or another run.
-We detail handling of the results database in section :ref:`inspecting-past-results`.
+We explain in detail the handling of the results database in section :ref:`inspecting-past-results`.
 
 By default, the session information is also saved in a JSON report file under ``$HOME/.reframe/reports``.
 The latest report is always symlinked by the ``latest.json`` name, unless the :option:`--report-file` option is given.

From aeb309e02f918e23510d2eff42c635728879a858 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Wed, 28 Aug 2024 16:24:24 +0200
Subject: [PATCH 085/128] Bump dev version

---
 reframe/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe/__init__.py b/reframe/__init__.py
index 4057cd289a..fa1a7aac91 100644
--- a/reframe/__init__.py
+++ b/reframe/__init__.py
@@ -6,7 +6,7 @@
 import os
 import sys
 
-VERSION = '4.7.0-dev.3'
+VERSION = '4.7.0-dev.4'
 INSTALL_PREFIX = os.path.normpath(
     os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 )

From 414e17b7b802140b0b80cbe1f3db510e81b4faab Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 29 Aug 2024 13:54:33 +0200
Subject: [PATCH 086/128] New `--table-hide-columns` option

---
 docs/manpage.rst            |  8 +++++++-
 reframe/frontend/cli.py     |  5 +++++
 reframe/frontend/printer.py | 12 +++++++++++-
 unittests/test_cli.py       |  9 +++++++++
 4 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index 5630174514..b5be5fd1f9 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -1132,7 +1132,7 @@ Miscellaneous options
 
 .. option:: --table-format=csv|plain|pretty
 
-   Set the formatting of tabular output printed by options :option:`--performance-compare`, :option:`--performance-report` and the options controlling the stored sessions.
+   Set the formatting of tabular output printed by the options :option:`--performance-compare`, :option:`--performance-report` and the options controlling the stored sessions.
 
    The acceptable values are the following:
 
@@ -1142,6 +1142,12 @@ Miscellaneous options
 
    .. versionadded:: 4.7
 
+.. option:: --table-hide-columns=COLUMNS
+
+   Hide the specified comma-separated list of columns from the tabular output printed by the options :option:`--performance-compare`, :option:`--performance-report` and the options controlling the stored sessions.
+
+   .. versionadded:: 4.7
+
 .. option:: --upgrade-config-file=OLD[:NEW]
 
    Convert the old-style configuration file ``OLD``, place it into the new file ``NEW`` and exit.
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index e4e3bd1f27..32eb96f808 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -631,6 +631,11 @@ def main():
         help='Table formatting',
         envvar='RFM_TABLE_FORMAT', configvar='general/table_format'
     )
+    misc_options.add_argument(
+        '--table-hide-columns', metavar='COLS', action='store',
+        help='Hide specific columns from the final table',
+        envvar='RFM_TABLE_HIDE_COLUMNS', configvar='general/table_hide_columns'
+    )
     misc_options.add_argument(
         '-v', '--verbose', action='count',
         help='Increase verbosity level of output',
diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index a8a88b91cd..6cd4200ed9 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -276,4 +276,14 @@ def table(self, data, **kwargs):
 
         kwargs.setdefault('headers', 'firstrow')
         kwargs.setdefault('tablefmt', tablefmt)
-        self.info(tabulate(data, **kwargs))
+        hide_columns = rt.runtime().get_option('general/0/table_hide_columns')
+        if hide_columns and kwargs['headers'] == 'firstrow' and data:
+            hide_columns = hide_columns.split(',')
+            colidx = [i for i, col in enumerate(data[0])
+                      if col not in hide_columns]
+
+            tab_data = [[rec[col] for col in colidx] for rec in data]
+        else:
+            tab_data = data
+
+        self.info(tabulate(tab_data, **kwargs))
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index c5f517819e..13339e8d9f 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -1298,6 +1298,15 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0):
         *run_reframe2(action=f'--describe-stored-testcases={uuid}')
     )
 
+    # Check hiding of table column
+    stdout = assert_no_crash(*run_reframe2(
+        action=f'--list-stored-testcases={uuid}',
+        more_options=['--table-hide-columns=SysEnv,Nodelist,UUID']
+    ))[1]
+    assert 'SysEnv' not in stdout
+    assert 'Nodelist' not in stdout
+    assert 'UUID' not in stdout
+
     # List test cases by time period
     ts_start = session_json['session_info']['time_start']
     assert_no_crash(

From 92c4aa7784b8e32d11199381748b20f56aafdbab Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 29 Aug 2024 13:58:13 +0200
Subject: [PATCH 087/128] Do not align decimals on the dot

---
 reframe/frontend/printer.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index 6cd4200ed9..e622ad77bc 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -276,6 +276,7 @@ def table(self, data, **kwargs):
 
         kwargs.setdefault('headers', 'firstrow')
         kwargs.setdefault('tablefmt', tablefmt)
+        kwargs.setdefault('numalign', 'right')
         hide_columns = rt.runtime().get_option('general/0/table_hide_columns')
         if hide_columns and kwargs['headers'] == 'firstrow' and data:
             hide_columns = hide_columns.split(',')

From a5a62eb34f2c35dc6122913b04a92d61e42cd6be Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 29 Aug 2024 14:34:12 +0200
Subject: [PATCH 088/128] Profile `performance_compare` + minor storage
 profiling improvements

---
 reframe/frontend/cli.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 32eb96f808..fd6af2e037 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -218,9 +218,10 @@ def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        logging.getprofiler().print_report(self.__logger.debug)
         if exc_type is SystemExit:
             # Allow users to exit inside the context manager
+            logging.getprofiler().exit_region()
+            logging.getprofiler().print_report(self.__logger.debug)
             return
 
         if isinstance(exc_val, self.__exceptions):
@@ -228,6 +229,8 @@ def __exit__(self, exc_type, exc_val, exc_tb):
             self.__logger.verbose(
                 ''.join(traceback.format_exception(exc_type, exc_val, exc_tb))
             )
+            logging.getprofiler().exit_region()
+            logging.getprofiler().print_report(self.__logger.debug)
             sys.exit(self.__exitcode)
 
 
@@ -1002,15 +1005,12 @@ def restrict_logging():
 
     if options.performance_compare:
         namepatt = '|'.join(options.names)
-        try:
+        with exit_gracefully_on_error('failed to generate performance report',
+                                      printer):
             printer.table(
                 reporting.performance_compare(options.performance_compare,
                                               namepatt=namepatt)
             )
-        except errors.ReframeError as err:
-            printer.error(f'failed to generate performance report: {err}')
-            sys.exit(1)
-        else:
             sys.exit(0)
 
     # Show configuration after everything is set up

From 47ab768fce4f09e02a9033fa5318cbf6dd436c1c Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 30 Aug 2024 14:31:07 +0200
Subject: [PATCH 089/128] Ignore records with unset performance when grouping
 test cases

---
 reframe/frontend/reporting/__init__.py        | 5 +++++
 unittests/resources/checks/frontend_checks.py | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 3c952f8770..3c23ae16f0 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -511,6 +511,11 @@ def _group_testcases(testcases, group_by, extra_cols):
         for pvar, reftuple in tc['perfvalues'].items():
             pvar = pvar.split(':')[-1]
             pval, pref, plower, pupper, punit = reftuple
+            if pval is None:
+                # Ignore `None` performance values
+                # (performance tests that failed sanity)
+                continue
+
             plower = pref * (1 + plower) if plower is not None else -math.inf
             pupper = pref * (1 + pupper) if pupper is not None else math.inf
             record = {
diff --git a/unittests/resources/checks/frontend_checks.py b/unittests/resources/checks/frontend_checks.py
index 6896981d68..4e0304ce0c 100644
--- a/unittests/resources/checks/frontend_checks.py
+++ b/unittests/resources/checks/frontend_checks.py
@@ -63,10 +63,15 @@ class NoPrgEnvCheck(BaseFrontendCheck):
 
 @rfm.simple_test
 class SanityFailureCheck(BaseFrontendCheck):
+    '''A performance test that fails in sanity.'''
     @sanity_function
     def validate_output(self):
         return sn.assert_found('foo', self.stdout)
 
+    @performance_function('Gflop/s')
+    def perf(self):
+        return sn.extractsingle(r'perf: (\d+)', self.stdout, 1, int)
+
 
 @rfm.simple_test
 class PerformanceFailureCheck(BaseFrontendCheck):

From 45f764bab69e6310da85584236e3e3a46709b8f9 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 30 Aug 2024 16:03:44 +0200
Subject: [PATCH 090/128] Print stack traces for exit requests in `debug` level
 instead of `debug2`

---
 reframe/frontend/cli.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index fd6af2e037..0e0f9024da 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -1662,8 +1662,8 @@ def module_unuse(*paths):
             printer.error(message)
 
         if errors.is_exit_request(*exc_info):
-            # Print stack traces for exit requests only when TOO verbose
-            printer.debug2(tb)
+            # Print stack traces for exit requests when debugging
+            printer.debug(tb)
         elif errors.is_severe(*exc_info):
             printer.error(tb)
         else:

From 47af4ffac470d4544c27320bf58ccbca9dd02798 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Fri, 30 Aug 2024 18:39:06 +0200
Subject: [PATCH 091/128] Bump dev version

---
 reframe/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe/__init__.py b/reframe/__init__.py
index fa1a7aac91..fdc966b40b 100644
--- a/reframe/__init__.py
+++ b/reframe/__init__.py
@@ -6,7 +6,7 @@
 import os
 import sys
 
-VERSION = '4.7.0-dev.4'
+VERSION = '4.7.0-dev.5'
 INSTALL_PREFIX = os.path.normpath(
     os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 )

From 6c9fe226cc09ca53bc8b2ca0a1bbcb212bb07b76 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 1 Sep 2024 15:54:10 +0000
Subject: [PATCH 092/128] Bump argcomplete from 3.4.0 to 3.5.0

Bumps [argcomplete](https://github.com/kislyuk/argcomplete) from 3.4.0 to 3.5.0.
- [Release notes](https://github.com/kislyuk/argcomplete/releases)
- [Changelog](https://github.com/kislyuk/argcomplete/blob/develop/Changes.rst)
- [Commits](https://github.com/kislyuk/argcomplete/compare/v3.4.0...v3.5.0)

---
updated-dependencies:
- dependency-name: argcomplete
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 790dd8cc60..3b3aa844dc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 archspec==0.2.4
 argcomplete==3.1.2; python_version < '3.8'
-argcomplete==3.4.0; python_version >= '3.8'
+argcomplete==3.5.0; python_version >= '3.8'
 filelock==3.4.1; python_version == '3.6'
 filelock==3.12.2; python_version == '3.7'
 filelock==3.15.4; python_version >= '3.8'

From bb5a82831d6f65d80aee23bba58d326af4f9b71f Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 30 Aug 2024 18:44:51 +0200
Subject: [PATCH 093/128] Expose SQLite connection timeout as configuration
 parameter

---
 docs/config_reference.rst             | 10 +++++++++-
 docs/manpage.rst                      | 17 +++++++++++++++-
 reframe/core/settings.py              |  2 +-
 reframe/frontend/cli.py               |  8 +++++++-
 reframe/frontend/reporting/storage.py | 28 ++++++++++++++++-----------
 reframe/schemas/config.json           |  2 ++
 6 files changed, 52 insertions(+), 15 deletions(-)

diff --git a/docs/config_reference.rst b/docs/config_reference.rst
index 57fbcd4daa..e22017ff40 100644
--- a/docs/config_reference.rst
+++ b/docs/config_reference.rst
@@ -1636,12 +1636,20 @@ Result storage configuration
    Currently, only Sqlite can be used as a storage backend.
 
 
+.. py:attribute:: storage.sqlite_conn_timeout
+
+   :required: No
+   :default: ``60``
+
+   Timeout in seconds for SQLite database connections.
+
+
 .. py:attribute:: storage.sqlite_db_file
 
    :required: No
    :default: ``"${HOME}/.reframe/reports/results.db"``
 
-   The Sqlite database file to use.
+   The SQLite database file to use.
 
 
 General Configuration
diff --git a/docs/manpage.rst b/docs/manpage.rst
index b5be5fd1f9..c2e0f080db 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -2007,9 +2007,24 @@ Whenever an environment variable is associated with a configuration option, its
       ================================== ==================
 
 
+.. envvar:: RFM_SQLITE_CONN_TIMEOUT
+
+   Timeout for SQLite database connections.
+
+   .. table::
+      :align: left
+
+      ================================== ==================
+      Associated command line option     N/A
+      Associated configuration parameter :attr:`~config.storage.sqlite_conn_timeout`
+      ================================== ==================
+
+   .. versionadded:: 4.7
+
+
 .. envvar:: RFM_SQLITE_DB_FILE
 
-   The SQlite database file for storing test results.
+   The SQLite database file for storing test results.
 
    .. table::
       :align: left
diff --git a/reframe/core/settings.py b/reframe/core/settings.py
index 471fac4435..6cdd0599e9 100644
--- a/reframe/core/settings.py
+++ b/reframe/core/settings.py
@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 #
-# Generic fallback configuration
+# Builtin configuration
 #
 
 site_configuration = {
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 0e0f9024da..19ad08890d 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -765,11 +765,17 @@ def main():
         action='store_true',
         help='Resolve module conflicts automatically'
     )
+    argparser.add_argument(
+        dest='sqlite_conn_timeout',
+        envvar='RFM_SQLITE_CONN_TIMEOUT',
+        configvar='storage/sqlite_conn_timeout',
+        help='Timeout for DB connections (SQLite backend)'
+    )
     argparser.add_argument(
         dest='sqlite_db_file',
         envvar='RFM_SQLITE_DB_FILE',
         configvar='storage/sqlite_db_file',
-        help='DB file where the results database resides'
+        help='DB file where the results database resides (SQLite backend)'
     )
     argparser.add_argument(
         dest='syslog_address',
diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 87fed08720..a25791a424 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -72,12 +72,18 @@ def _db_matches(self, patt, item):
         regex = re.compile(patt)
         return regex.match(item) is not None
 
+    def _db_connect(self, *args, **kwargs):
+        timeout = runtime().get_option('storage/0/sqlite_conn_timeout')
+        kwargs.setdefault('timeout', timeout)
+        with getprofiler().time_region('sqlite connect'):
+            return sqlite3.connect(*args, **kwargs)
+
     def _db_create(self):
         clsname = type(self).__name__
         getlogger().debug(
             f'{clsname}: creating results database in {self.__db_file}...'
         )
-        with sqlite3.connect(self.__db_file) as conn:
+        with self._db_connect(self.__db_file) as conn:
             conn.execute('CREATE TABLE IF NOT EXISTS sessions('
                          'uuid TEXT PRIMARY KEY, '
                          'session_start_unix REAL, '
@@ -100,13 +106,13 @@ def _db_create(self):
                          'schema_version TEXT)')
 
     def _db_schema_check(self):
-        with sqlite3.connect(self.__db_file) as conn:
+        with self._db_connect(self.__db_file) as conn:
             results = conn.execute(
                 'SELECT schema_version FROM metadata').fetchall()
 
         if not results:
             # DB is new, insert the schema version
-            with sqlite3.connect(self.__db_file) as conn:
+            with self._db_connect(self.__db_file) as conn:
                 conn.execute('INSERT INTO metadata VALUES(:schema_version)',
                              {'schema_version': self.SCHEMA_VERSION})
         else:
@@ -159,14 +165,14 @@ def _db_store_report(self, conn, report, report_file_path):
 
     def store(self, report, report_file=None):
         prefix = os.path.dirname(self.__db_file)
-        with sqlite3.connect(self._db_file()) as conn:
+        with self._db_connect(self._db_file()) as conn:
             with FileLock(os.path.join(prefix, '.db.lock')):
                 return self._db_store_report(conn, report, report_file)
 
     @time_function
     def _fetch_testcases_raw(self, condition):
         getprofiler().enter_region('sqlite query')
-        with sqlite3.connect(self._db_file()) as conn:
+        with self._db_connect(self._db_file()) as conn:
             query = ('SELECT session_uuid, testcases.uuid as uuid, json_blob '
                      'FROM testcases '
                      'JOIN sessions ON session_uuid == sessions.uuid '
@@ -207,7 +213,7 @@ def _fetch_testcases_raw(self, condition):
 
     @time_function
     def fetch_session_time_period(self, session_uuid):
-        with sqlite3.connect(self._db_file()) as conn:
+        with self._db_connect(self._db_file()) as conn:
             query = ('SELECT session_start_unix, session_end_unix '
                      f'FROM sessions WHERE uuid == "{session_uuid}" '
                      'LIMIT 1')
@@ -231,7 +237,7 @@ def fetch_testcases_time_period(self, ts_start, ts_end, name_pattern=None):
 
     @time_function
     def fetch_testcases_from_session(self, session_uuid, name_pattern=None):
-        with sqlite3.connect(self._db_file()) as conn:
+        with self._db_connect(self._db_file()) as conn:
             query = ('SELECT json_blob from sessions '
                      f'WHERE uuid == "{session_uuid}"')
             getlogger().debug(query)
@@ -246,7 +252,7 @@ def fetch_testcases_from_session(self, session_uuid, name_pattern=None):
 
     @time_function
     def fetch_sessions_time_period(self, ts_start=None, ts_end=None):
-        with sqlite3.connect(self._db_file()) as conn:
+        with self._db_connect(self._db_file()) as conn:
             query = 'SELECT json_blob from sessions'
             if ts_start or ts_end:
                 query += ' WHERE ('
@@ -269,7 +275,7 @@ def fetch_sessions_time_period(self, ts_start=None, ts_end=None):
 
     @time_function
     def fetch_session_json(self, uuid):
-        with sqlite3.connect(self._db_file()) as conn:
+        with self._db_connect(self._db_file()) as conn:
             query = f'SELECT json_blob FROM sessions WHERE uuid == "{uuid}"'
             getlogger().debug(query)
             results = conn.execute(query).fetchall()
@@ -279,7 +285,7 @@ def fetch_session_json(self, uuid):
     def _do_remove(self, uuid):
         prefix = os.path.dirname(self.__db_file)
         with FileLock(os.path.join(prefix, '.db.lock')):
-            with sqlite3.connect(self._db_file()) as conn:
+            with self._db_connect(self._db_file()) as conn:
                 # Check first if the uuid exists
                 query = f'SELECT * FROM sessions WHERE uuid == "{uuid}"'
                 getlogger().debug(query)
@@ -294,7 +300,7 @@ def _do_remove2(self, uuid):
         '''Remove a session using the RETURNING keyword'''
         prefix = os.path.dirname(self.__db_file)
         with FileLock(os.path.join(prefix, '.db.lock')):
-            with sqlite3.connect(self._db_file()) as conn:
+            with self._db_connect(self._db_file()) as conn:
                 query = (f'DELETE FROM sessions WHERE uuid == "{uuid}" '
                          'RETURNING *')
                 getlogger().debug(query)
diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index 7183f787d0..a5717f05d7 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -543,6 +543,7 @@
                 "type": "object",
                 "properties": {
                     "backend": {"type": "string"},
+                    "sqlite_conn_timeout": {"type": "number"},
                     "sqlite_db_file": {"type": "string"}
                 }
             }
@@ -624,6 +625,7 @@
         "modes/options": [],
         "modes/target_systems": ["*"],
         "storage/backend": "sqlite",
+        "storage/sqlite_conn_timeout": 60,
         "storage/sqlite_db_file": "${HOME}/.reframe/reports/results.db",
         "systems/descr": "",
         "systems/max_local_jobs": 8,

From a48a9af7dc14b0ae91845cc4e6d0cae45f275e55 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Sun, 1 Sep 2024 21:57:39 +0200
Subject: [PATCH 094/128] Issue warning if performance report cannot be
 produced

---
 reframe/frontend/cli.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 19ad08890d..88755a86f7 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -1592,10 +1592,13 @@ def module_unuse(*paths):
                     data = reporting.performance_compare(
                         rt.get_option('general/0/perf_report_spec'), report
                     )
-                except errors.ReframeError as err:
+                except Exception as err:
                     printer.warning(
                         f'failed to generate performance report: {err}'
                     )
+                    printer.verbose(
+                        ''.join(traceback.format_exception(*sys.exc_info()))
+                    )
                 else:
                     printer.performance_report(data)
 
@@ -1637,6 +1640,9 @@ def module_unuse(*paths):
                     printer.warning(
                         f'failed to store results in the database: {e}'
                     )
+                    printer.verbose(
+                        ''.join(traceback.format_exception(*sys.exc_info()))
+                    )
                 else:
                     printer.info('Current session stored with UUID: '
                                  f'{sess_uuid}')

From a93d60c98d5a0059b423d5a884e37d96f2b06c6c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 1 Sep 2024 20:52:33 +0000
Subject: [PATCH 095/128] Bump setuptools from 59.6.0 to 74.0.0

Bumps [setuptools](https://github.com/pypa/setuptools) from 59.6.0 to 74.0.0.
- [Release notes](https://github.com/pypa/setuptools/releases)
- [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst)
- [Commits](https://github.com/pypa/setuptools/compare/v59.6.0...v74.0.0)

---
updated-dependencies:
- dependency-name: setuptools
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 3b3aa844dc..31b12758a9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,7 +24,7 @@ semver==2.13.0; python_version == '3.6'
 semver==3.0.2; python_version >= '3.7'
 setuptools==59.6.0; python_version == '3.6'
 setuptools==68.0.0; python_version == '3.7'
-setuptools==72.1.0; python_version >= '3.8'
+setuptools==74.0.0; python_version >= '3.8'
 tabulate==0.8.10; python_version == '3.6'
 tabulate==0.9.0; python_version >= '3.7'
 wcwidth==0.2.13

From d329b48d84b4ec013697425e6fd0c220b5587c29 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 2 Sep 2024 07:47:17 +0000
Subject: [PATCH 096/128] Bump lxml from 5.2.2 to 5.3.0

Bumps [lxml](https://github.com/lxml/lxml) from 5.2.2 to 5.3.0.
- [Release notes](https://github.com/lxml/lxml/releases)
- [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt)
- [Commits](https://github.com/lxml/lxml/compare/lxml-5.2.2...lxml-5.3.0)

---
updated-dependencies:
- dependency-name: lxml
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 setup.cfg        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 31b12758a9..a17ebe94a1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,7 +7,7 @@ filelock==3.15.4; python_version >= '3.8'
 importlib_metadata==4.0.1; python_version < '3.8'
 jsonschema==3.2.0
 lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64'
-lxml==5.2.2; python_version >= '3.8' or platform_machine != 'aarch64'
+lxml==5.3.0; python_version >= '3.8' or platform_machine != 'aarch64'
 pytest==7.0.1; python_version < '3.8'
 pytest==8.3.2; python_version >= '3.8'
 pytest-forked==1.4.0; python_version == '3.6'
diff --git a/setup.cfg b/setup.cfg
index 3dc33868e7..642a277bc0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -34,7 +34,7 @@ install_requires =
     filelock<=3.4.1; python_version == '3.6'
     jsonschema
     lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64'
-    lxml==5.2.2; python_version >= '3.8' or platform_machine != 'aarch64'
+    lxml==5.3.0; python_version >= '3.8' or platform_machine != 'aarch64'
     PyYAML
     requests
     requests <= 2.27.1; python_version == '3.6'

From 97c0bb1b3bea12b19ff2d3e1ff04ebd65a6a2787 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 2 Sep 2024 08:13:52 +0000
Subject: [PATCH 097/128] Bump pyyaml from 6.0.1 to 6.0.2

Bumps [pyyaml](https://github.com/yaml/pyyaml) from 6.0.1 to 6.0.2.
- [Release notes](https://github.com/yaml/pyyaml/releases)
- [Changelog](https://github.com/yaml/pyyaml/blob/main/CHANGES)
- [Commits](https://github.com/yaml/pyyaml/compare/6.0.1...6.0.2)

---
updated-dependencies:
- dependency-name: pyyaml
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index a17ebe94a1..b888a4c925 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -16,7 +16,7 @@ pytest-parallel==0.1.1
 pytest-rerunfailures==10.3; python_version == '3.6'
 pytest-rerunfailures==13.0; python_version == '3.7'
 pytest-rerunfailures==14.0; python_version >= '3.8'
-PyYAML==6.0.1
+PyYAML==6.0.2
 requests==2.27.1; python_version == '3.6'
 requests==2.31.0; python_version == '3.7'
 requests==2.32.2; python_version >= '3.8'

From 013b366701f3f4dac8117e9eeb409392b88e02c4 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Mon, 2 Sep 2024 20:29:23 +0200
Subject: [PATCH 098/128] Pin PyYAML version for Python < 3.8

---
 requirements.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index b888a4c925..08e952b9ef 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -16,7 +16,8 @@ pytest-parallel==0.1.1
 pytest-rerunfailures==10.3; python_version == '3.6'
 pytest-rerunfailures==13.0; python_version == '3.7'
 pytest-rerunfailures==14.0; python_version >= '3.8'
-PyYAML==6.0.2
+PyYAML==6.0.1; python_version < '3.8'
+PyYAML==6.0.2; python_version >= '3.8'
 requests==2.27.1; python_version == '3.6'
 requests==2.31.0; python_version == '3.7'
 requests==2.32.2; python_version >= '3.8'

From 706ac5f0e781e27e7baa086c97c4b46277fe0928 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Mon, 2 Sep 2024 20:44:26 +0200
Subject: [PATCH 099/128] Print selected system in `[ReFrame Setup]`
 information

---
 reframe/frontend/cli.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index de832fac09..0fe842c492 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -985,11 +985,11 @@ def print_infoline(param, value):
     }
 
     # Print command line
-    printer.info(f"[ReFrame Setup]")
+    printer.info('[ReFrame Setup]')
     print_infoline('version', session_info['version'])
     print_infoline('command', repr(session_info['cmdline']))
     print_infoline(
-        f"launched by",
+        'launched by',
         f"{session_info['user'] or '<unknown>'}@{session_info['hostname']}"
     )
     print_infoline('working directory', repr(session_info['workdir']))
@@ -997,6 +997,7 @@ def print_infoline(param, value):
         'settings files',
         ', '.join(repr(x) for x in session_info['config_files'])
     )
+    print_infoline('selected system', repr(rt.system.name))
     print_infoline('check search path',
                    f"{'(R) ' if loader.recurse else ''}"
                    f"{':'.join(loader.load_path)!r}")

From b12f61d10aa987a12487d9524ca7e89b5d799ac4 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Thu, 5 Sep 2024 12:04:31 +0200
Subject: [PATCH 100/128] Change default values for `--performance-report` and
 `--list-stored-sessions`

---
 docs/manpage.rst            | 5 +++--
 reframe/frontend/cli.py     | 6 +++---
 reframe/schemas/config.json | 2 +-
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index c2e0f080db..ebe4a11a5a 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -349,7 +349,8 @@ There are commands that act upon the selected tests and others that have a helpe
 
    List sessions stored in the results database.
 
-   If ``TIME_PERIOD`` is not specified or if ``all`` is passed, all stored sessions will be listed.
+   If ``TIME_PERIOD`` is ``all``, all stored sessions will be listed.
+   If not specified, only the sessions of last week will be listed.
    For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`.
 
    .. versionadded:: 4.7
@@ -1096,7 +1097,7 @@ Miscellaneous options
 
    For each test all of their performance variables are reported and optionally compared to past results based on the ``CMPSPEC`` specified.
 
-   If not specified, the default ``CMPSPEC`` is ``19700101T0000+0000:now/last:+job_nodelist/+result``, meaning that the current performance will be compared to the last run of the same test grouped additionally by the ``job_nodelist`` and showing also the obtained result (``pass`` or ``fail``).
+   If not specified, the default ``CMPSPEC`` is ``now:now/last:/+job_nodelist+result``, meaning that the current performance will not be compared to any past run and, additionally, the ``job_nodelist`` and the test result (``pass`` or ``fail``) will be listed.
 
    For the exact syntax of ``CMPSPEC``, refer to :ref:`performance-comparisons`.
 
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 88755a86f7..2ccecfb700 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -431,8 +431,8 @@ def main():
               'providing more details')
     )
     action_options.add_argument(
-        '--list-stored-sessions', nargs='?', action='store', const='all',
-        metavar='PERIOD', help='List stored sessions'
+        '--list-stored-sessions', nargs='?', action='store',
+        const='now-1w:now', metavar='PERIOD', help='List stored sessions'
     )
     action_options.add_argument(
         '--list-stored-testcases', action='store',
@@ -607,7 +607,7 @@ def main():
         configvar='general/perf_report_spec',
         envvar='RFM_PERF_REPORT_SPEC',
         help=('Print a report for performance tests '
-              '(default: "19700101T0000+0000:now/last:+job_nodelist/+result")')
+              '(default: "now:now/last:+job_nodelist/+result")')
     )
 
     # Miscellaneous options
diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index a5717f05d7..4e270338af 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -582,7 +582,7 @@
         "general/module_mappings": [],
         "general/non_default_craype": false,
         "general/perf_info_level": "info",
-        "general/perf_report_spec": "19700101T0000+0000:now/last:+job_nodelist/+result",
+        "general/perf_report_spec": "now:now/last:/+job_nodelist+result",
         "general/purge_environment": false,
         "general/remote_detect": false,
         "general/remote_workdir": ".",

From 254eb05e3a920ea67d86185f9ae486568b1cd9f1 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Tue, 3 Sep 2024 17:28:13 +0200
Subject: [PATCH 101/128] Optimize testcase queries by time period

---
 reframe/frontend/reporting/storage.py | 36 +++++++++++++++++++--------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index a25791a424..745b728cb9 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -171,12 +171,12 @@ def store(self, report, report_file=None):
 
     @time_function
     def _fetch_testcases_raw(self, condition):
-        getprofiler().enter_region('sqlite query')
+        # Retrieve relevant session info and index it in Python
+        getprofiler().enter_region('sqlite session query')
         with self._db_connect(self._db_file()) as conn:
-            query = ('SELECT session_uuid, testcases.uuid as uuid, json_blob '
-                     'FROM testcases '
-                     'JOIN sessions ON session_uuid == sessions.uuid '
-                     f'WHERE {condition}')
+            query = ('SELECT uuid, json_blob FROM sessions WHERE uuid IN '
+                     '(SELECT DISTINCT session_uuid FROM testcases '
+                     f'WHERE {condition})')
             getlogger().debug(query)
 
             # Create SQLite function for filtering using name patterns
@@ -185,10 +185,9 @@ def _fetch_testcases_raw(self, condition):
 
         getprofiler().exit_region()
 
-        # Retrieve session info
         sessions = {}
-        for session_uuid, uuid, json_blob in results:
-            sessions.setdefault(session_uuid, json_blob)
+        for uuid, json_blob in results:
+            sessions.setdefault(uuid, json_blob)
 
         # Join all sessions and decode them at once
         reports_blob = '[' + ','.join(sessions.values()) + ']'
@@ -200,10 +199,20 @@ def _fetch_testcases_raw(self, condition):
         for rpt in reports:
             sessions[rpt['session_info']['uuid']] = rpt
 
-        # Extract the test case data
+        # Extract the test case data by extracting their UUIDs
+        getprofiler().enter_region('sqlite testcase query')
+        with self._db_connect(self._db_file()) as conn:
+            query = f'SELECT uuid FROM testcases WHERE {condition}'
+            getlogger().debug(query)
+            conn.create_function('REGEXP', 2, self._db_matches)
+            results = conn.execute(query).fetchall()
+
+        getprofiler().exit_region()
         testcases = []
-        for session_uuid, uuid, json_blob in results:
-            run_index, test_index = [int(x) for x in uuid.split(':')[1:]]
+        for uuid, *_ in results:
+            session_uuid, run_index, test_index = uuid.split(':')
+            run_index = int(run_index)
+            test_index = int(test_index)
             report = sessions[session_uuid]
             testcases.append(
                 report['runs'][run_index]['testcases'][test_index],
@@ -286,6 +295,9 @@ def _do_remove(self, uuid):
         prefix = os.path.dirname(self.__db_file)
         with FileLock(os.path.join(prefix, '.db.lock')):
             with self._db_connect(self._db_file()) as conn:
+                # Enable foreign keys for delete action to have cascade effect
+                conn.execute('PRAGMA foreign_keys = ON')
+
                 # Check first if the uuid exists
                 query = f'SELECT * FROM sessions WHERE uuid == "{uuid}"'
                 getlogger().debug(query)
@@ -301,6 +313,8 @@ def _do_remove2(self, uuid):
         prefix = os.path.dirname(self.__db_file)
         with FileLock(os.path.join(prefix, '.db.lock')):
             with self._db_connect(self._db_file()) as conn:
+                # Enable foreign keys for delete action to have cascade effect
+                conn.execute('PRAGMA foreign_keys = ON')
                 query = (f'DELETE FROM sessions WHERE uuid == "{uuid}" '
                          'RETURNING *')
                 getlogger().debug(query)

From e23a5e48e98be296012e9a110e1d93b3940873e7 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Wed, 4 Sep 2024 16:29:02 +0200
Subject: [PATCH 102/128] Fix `KeyError` problems in `_fetch_testcases_raw()`

---
 reframe/frontend/reporting/storage.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 745b728cb9..7e7cdedaee 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -213,10 +213,16 @@ def _fetch_testcases_raw(self, condition):
             session_uuid, run_index, test_index = uuid.split(':')
             run_index = int(run_index)
             test_index = int(test_index)
-            report = sessions[session_uuid]
-            testcases.append(
-                report['runs'][run_index]['testcases'][test_index],
-            )
+            try:
+                report = sessions[session_uuid]
+            except KeyError:
+                # Since we do two separate queries, new testcases may have been
+                # inserted to the DB meanwhile, so we ignore unknown sessions
+                continue
+            else:
+                testcases.append(
+                    report['runs'][run_index]['testcases'][test_index],
+                )
 
         return testcases
 

From 0bd216315be8d8916339f69209e9430f34170853 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Fri, 6 Sep 2024 10:12:06 +0200
Subject: [PATCH 103/128] Bump dev version

---
 reframe/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe/__init__.py b/reframe/__init__.py
index fdc966b40b..8cb41ff7bb 100644
--- a/reframe/__init__.py
+++ b/reframe/__init__.py
@@ -6,7 +6,7 @@
 import os
 import sys
 
-VERSION = '4.7.0-dev.5'
+VERSION = '4.7.0-dev.6'
 INSTALL_PREFIX = os.path.normpath(
     os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 )

From 2445e54afb30564741f08fc3a636e1cb49cce075 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Fri, 6 Sep 2024 23:58:51 +0200
Subject: [PATCH 104/128] Add ReFrame paper citation information

---
 README.md         | 34 ++++++++++++++++++++++++++++++++++
 README_minimal.md | 34 ++++++++++++++++++++++++++++++++++
 docs/index.rst    | 17 ++++++++++++++---
 3 files changed, 82 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 1007efe4f4..4eb351f853 100644
--- a/README.md
+++ b/README.md
@@ -97,3 +97,37 @@ Please join the community's [Slack channel](https://join.slack.com/t/reframetalk
 
 ReFrame is an open-source project and we welcome and encourage contributions!
 Check out our Contribution Guide [here](https://github.com/reframe-hpc/reframe/wiki/contributing-to-reframe).
+
+## Citing ReFrame
+
+You can cite ReFrame in publications as follows:
+
+> Vasileios Karakasis et al. "Enabling Continuous Testing of HPC Systems Using ReFrame". In: *Tools and Techniques for High Performance Computing. HUST - Annual Workshop on HPC User Support Tools* (Denver, Colorado, USA, Nov. 17–18, 2019). Ed. by Guido Juckeland and Sunita Chandrasekaran. Vol. 1190. Communications in Computer and Information Science. Cham, Switzerland: Springer International Publishing, Mar. 2020, pp. 49–68. isbn: 978-3-030-44728-1. doi: 10.1007/978-3-030-44728-1_3.
+
+The corresponding BibTeX entry is the following:
+
+```bibtex
+@InProceedings{karakasis20a,
+  author     = {Karakasis, Vasileios and Manitaras, Theofilos and Rusu, Victor Holanda and
+                Sarmiento-P{\'e}rez, Rafael and Bignamini, Christopher and Kraushaar, Matthias and
+                Jocksch, Andreas and Omlin, Samuel and Peretti-Pezzi, Guilherme and
+                Augusto, Jo{\~a}o P. S. C. and Friesen, Brian and He, Yun and Gerhardt, Lisa and
+                Cook, Brandon and You, Zhi-Qiang and Khuvis, Samuel and Tomko, Karen},
+  title      = {Enabling Continuous Testing of {HPC} Systems Using {ReFrame}},
+  booktitle  = {Tools and Techniques for High Performance Computing},
+  editor     = {Juckeland, Guido and Chandrasekaran, Sunita},
+  year       = {2020},
+  month      = mar,
+  series     = {Communications in Computer and Information Science},
+  volume     = {1190},
+  pages      = {49--68},
+  address    = {Cham, Switzerland},
+  publisher  = {Springer International Publishing},
+  doi        = {10.1007/978-3-030-44728-1_3},
+  venue      = {Denver, Colorado, USA},
+  eventdate  = {2019-11-17/2019-11-18},
+  eventtitle = {{HUST} - Annual Workshop on {HPC} User Support Tools},
+  isbn       = {978-3-030-44728-1},
+  issn       = {1865-0937},
+}
+```
diff --git a/README_minimal.md b/README_minimal.md
index 027200d2dc..90044a6c97 100644
--- a/README_minimal.md
+++ b/README_minimal.md
@@ -39,3 +39,37 @@ Please join the community's [Slack channel](https://reframe-slack.herokuapp.com)
 
 ReFrame is an open-source project and we welcome and encourage contributions!
 Check out our Contribution Guide [here](https://github.com/reframe-hpc/reframe/wiki/contributing-to-reframe).
+
+## Citing ReFrame
+
+You can cite ReFrame in publications as follows:
+
+> Vasileios Karakasis et al. “Enabling Continuous Testing of HPC Systems Using ReFrame”. In: *Tools and Techniques for High Performance Computing. HUST - Annual Workshop on HPC User Support Tools* (Denver, Colorado, USA, Nov. 17–18, 2019). Ed. by Guido Juckeland and Sunita Chandrasekaran. Vol. 1190. Communications in Computer and Information Science. Cham, Switzerland: Springer International Publishing, Mar. 2020, pp. 49–68. isbn: 978-3-030-44728-1. doi: 10.1007/978-3-030-44728-1_3.
+
+The corresponding BibTeX entry is the following:
+
+```bibtex
+@InProceedings{karakasis20a,
+  author     = {Karakasis, Vasileios and Manitaras, Theofilos and Rusu, Victor Holanda and
+                Sarmiento-P{\'e}rez, Rafael and Bignamini, Christopher and Kraushaar, Matthias and
+                Jocksch, Andreas and Omlin, Samuel and Peretti-Pezzi, Guilherme and
+                Augusto, Jo{\~a}o P. S. C. and Friesen, Brian and He, Yun and Gerhardt, Lisa and
+                Cook, Brandon and You, Zhi-Qiang and Khuvis, Samuel and Tomko, Karen},
+  title      = {Enabling Continuous Testing of {HPC} Systems Using {ReFrame}},
+  booktitle  = {Tools and Techniques for High Performance Computing},
+  editor     = {Juckeland, Guido and Chandrasekaran, Sunita},
+  year       = {2020},
+  month      = mar,
+  series     = {Communications in Computer and Information Science},
+  volume     = {1190},
+  pages      = {49--68},
+  address    = {Cham, Switzerland},
+  publisher  = {Springer International Publishing},
+  doi        = {10.1007/978-3-030-44728-1_3},
+  venue      = {Denver, Colorado, USA},
+  eventdate  = {2019-11-17/2019-11-18},
+  eventtitle = {{HUST} - Annual Workshop on {HPC} User Support Tools},
+  isbn       = {978-3-030-44728-1},
+  issn       = {1865-0937},
+}
+```
diff --git a/docs/index.rst b/docs/index.rst
index 6d35d027a8..0d6811cc5c 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -18,6 +18,9 @@ Finally, ReFrame offers a powerful and efficient runtime for running and managin
 Publications
 ============
 
+Presentations & Talks
+---------------------
+
 * Slides [`pdf <https://users.ugent.be/~kehoste/eum24/008_eum24_ReFrame.pdf>`__][`talk <https://www.youtube.com/live/uSEeU-VJf6k?si=YB19mFpG6aAEBOgH>`__] @ `9th EasyBuild User Meeting 2024 <https://easybuild.io/eum24/>`__.
 * Slides [`part 1 <https://docs.google.com/presentation/d/1GmO2Uf29SaLg36bPB9g9eeaKMN-bLlDJ5IvLGLQJfD8/edit?usp=share_link>`__][`part 2 <https://drive.google.com/file/d/1gZwch0BPc1wDEkMwbM4vxCpMzWIx-Lo1/view?usp=sharing>`__][`talk <https://youtu.be/0ApEKc185Bw>`__] @ `8th EasyBuild User Meeting 2023 <https://easybuild.io/eum23/>`__.
 * Slides [`pdf <https://drive.google.com/file/d/1vmaWyRHgtq3DrYhSCVBzR8U5ErKbxGNf/view?usp=sharing>`__] @ `7th EasyBuild User Meeting 2022 <https://easybuild.io/eum22/>`__.
@@ -32,18 +35,26 @@ Publications
 * Slides [`pdf <https://github.com/eth-cscs/UserLabDay/blob/master/2018/slides/ci_and_regression/ReFrame_CI.pdf>`__] @ `CSCS User Lab Day 2018 <https://github.com/eth-cscs/UserLabDay>`__.
 * Slides [`pdf <https://drive.google.com/open?id=1sZhibvUlGlT670aOHPdMlWFffWptYzLX>`__] @ `HPC Advisory Council 2018 <http://www.hpcadvisorycouncil.com/events/2018/swiss-workshop/>`__.
 * Slides [`pdf <https://drive.google.com/open?id=1EyJ-siupkgLeVT54A4WlFpQtrJaU0xOy>`__] @ `SC17 <https://sc17.supercomputing.org/>`__.
-* Slides [`pdf <https://drive.google.com/open?id=18VrCy0MTplGo67uxVbzYZicQChor9VSY>`__] @ `CUG 2017 <https://cug.org/cug-2017/>`__.
+* Slides [`pdf <https://cug.org/proceedings/cug2017_proceedings/includes/files/pap122s2-file2.pdf>`__] @ `CUG 2017 <https://cug.org/cug-2017/>`__.
 
 
-Webinars and Tutorials
-======================
+Webinars & Tutorials
+--------------------
 
 * "ReFrame – Efficient System and Application Performance Testing," CSCS Webinar, Aug. 29, 2022 [`slides <https://drive.google.com/file/d/1nOS_daleR79ZB1IaToVdW5mDpJQYRcY2/view?usp=sharing>`__] [`recording <https://youtu.be/NDxlKATEcQk>`__] [`demo run <https://asciinema.org/a/517693>`__].
 * Tutorial at 6th EasyBuild User Meeting 2021 [`YouTube <https://youtube.com/playlist?list=PLhnGtSmEGEQjySVEPTUSLpewpOWwX5mjb>`__]
 
 
+Papers
+------
+
+- Vasileios Karakasis et al. "A regression framework for checking the health of large HPC systems". In: *Cray User Group 2017* (Redmond, Washington, USA, May 8--11, 2017). [`pdf <https://cug.org/proceedings/cug2017_proceedings/includes/files/pap122s2-file1.pdf>`__]
+
+- Vasileios Karakasis et al. "Enabling Continuous Testing of HPC Systems Using ReFrame". In: *Tools and Techniques for High Performance Computing. HUST -- Annual Workshop on HPC User Support Tools* (Denver, Colorado, USA, Nov. 17--18, 2019). Ed. by Guido Juckeland and Sunita Chandrasekaran. Vol. 1190. Communications in Computer and Information Science. Cham, Switzerland: Springer International Publishing, Mar. 2020, pp. 49--68. isbn: 978-3-030-44728-1. doi: `10.1007/978-3-030-44728-1_3 <https://doi.org/10.1007/978-3-030-44728-1_3>`__.
+
 
 .. toctree::
+   :maxdepth: 2
    :caption: Table of Contents
 
    started

From 5b7535084c2f1d395cc29de291ea67dc7368ea4e Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Fri, 13 Sep 2024 13:46:19 +0200
Subject: [PATCH 105/128] Allow setting the file mode of the SQLite database

---
 docs/config_reference.rst             | 10 ++++++++++
 docs/manpage.rst                      | 15 +++++++++++++++
 reframe/frontend/cli.py               |  8 ++++++++
 reframe/frontend/reporting/storage.py | 19 +++++++++++++------
 reframe/schemas/config.json           |  4 +++-
 5 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/docs/config_reference.rst b/docs/config_reference.rst
index e22017ff40..8b84563be0 100644
--- a/docs/config_reference.rst
+++ b/docs/config_reference.rst
@@ -1651,6 +1651,16 @@ Result storage configuration
 
    The SQLite database file to use.
 
+.. py:attribute:: storage.sqlite_db_file_mode
+
+   :required: No
+   :default: ``"644"``
+
+   The permissions of the SQLite database file in octal form.
+
+   The mode will only taken into account upon creation of the DB file.
+   Permissions of an existing DB file have to be changed manually.
+
 
 General Configuration
 =====================
diff --git a/docs/manpage.rst b/docs/manpage.rst
index ebe4a11a5a..b33242b8a4 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -2038,6 +2038,21 @@ Whenever an environment variable is associated with a configuration option, its
    .. versionadded:: 4.7
 
 
+.. envvar:: RFM_SQLITE_DB_FILE_MODE
+
+   The permissions of the SQLite database file in octal form.
+
+   .. table::
+      :align: left
+
+      ================================== ==================
+      Associated command line option     N/A
+      Associated configuration parameter :attr:`~config.storage.sqlite_db_file_mode`
+      ================================== ==================
+
+   .. versionadded:: 4.7
+
+
 .. envvar:: RFM_SYSLOG_ADDRESS
 
    The address of the Syslog server to send performance logs.
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 2ccecfb700..19d4ffec6e 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
+import functools
 import inspect
 import itertools
 import json
@@ -777,6 +778,13 @@ def main():
         configvar='storage/sqlite_db_file',
         help='DB file where the results database resides (SQLite backend)'
     )
+    argparser.add_argument(
+        dest='sqlite_db_file_mode',
+        envvar='RFM_SQLITE_DB_FILE_MODE',
+        configvar='storage/sqlite_db_file_mode',
+        help='DB file permissions (SQLite backend)',
+        type=functools.partial(int, base=8)
+    )
     argparser.add_argument(
         dest='syslog_address',
         envvar='RFM_SYSLOG_ADDRESS',
diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 7e7cdedaee..4e78889852 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -52,6 +52,9 @@ def __init__(self):
         self.__db_file = os.path.join(
             osext.expandvars(runtime().get_option('storage/0/sqlite_db_file'))
         )
+        self.__db_file_mode = runtime().get_option(
+            'storage/0/sqlite_db_file_mode'
+        )
 
     def _db_file(self):
         prefix = os.path.dirname(self.__db_file)
@@ -78,6 +81,11 @@ def _db_connect(self, *args, **kwargs):
         with getprofiler().time_region('sqlite connect'):
             return sqlite3.connect(*args, **kwargs)
 
+    def _db_lock(self):
+        prefix = os.path.dirname(self.__db_file)
+        return FileLock(os.path.join(prefix, '.db.lock'),
+                        mode=self.__db_file_mode)
+
     def _db_create(self):
         clsname = type(self).__name__
         getlogger().debug(
@@ -104,6 +112,8 @@ def _db_create(self):
                          'on testcases(job_completion_time_unix)')
             conn.execute('CREATE TABLE IF NOT EXISTS metadata('
                          'schema_version TEXT)')
+        # Update DB file mode
+        os.chmod(self.__db_file, self.__db_file_mode)
 
     def _db_schema_check(self):
         with self._db_connect(self.__db_file) as conn:
@@ -164,9 +174,8 @@ def _db_store_report(self, conn, report, report_file_path):
         return session_uuid
 
     def store(self, report, report_file=None):
-        prefix = os.path.dirname(self.__db_file)
         with self._db_connect(self._db_file()) as conn:
-            with FileLock(os.path.join(prefix, '.db.lock')):
+            with self._db_lock():
                 return self._db_store_report(conn, report, report_file)
 
     @time_function
@@ -298,8 +307,7 @@ def fetch_session_json(self, uuid):
         return jsonext.loads(results[0][0]) if results else {}
 
     def _do_remove(self, uuid):
-        prefix = os.path.dirname(self.__db_file)
-        with FileLock(os.path.join(prefix, '.db.lock')):
+        with self._db_lock():
             with self._db_connect(self._db_file()) as conn:
                 # Enable foreign keys for delete action to have cascade effect
                 conn.execute('PRAGMA foreign_keys = ON')
@@ -316,8 +324,7 @@ def _do_remove(self, uuid):
 
     def _do_remove2(self, uuid):
         '''Remove a session using the RETURNING keyword'''
-        prefix = os.path.dirname(self.__db_file)
-        with FileLock(os.path.join(prefix, '.db.lock')):
+        with self._db_lock():
             with self._db_connect(self._db_file()) as conn:
                 # Enable foreign keys for delete action to have cascade effect
                 conn.execute('PRAGMA foreign_keys = ON')
diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index 4e270338af..a3b9eccd1a 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -544,7 +544,8 @@
                 "properties": {
                     "backend": {"type": "string"},
                     "sqlite_conn_timeout": {"type": "number"},
-                    "sqlite_db_file": {"type": "string"}
+                    "sqlite_db_file": {"type": "string"},
+                    "sqlite_db_file_mode": {"type": "string"}
                 }
             }
         }
@@ -627,6 +628,7 @@
         "storage/backend": "sqlite",
         "storage/sqlite_conn_timeout": 60,
         "storage/sqlite_db_file": "${HOME}/.reframe/reports/results.db",
+        "storage/sqlite_db_file_mode": "644",
         "systems/descr": "",
         "systems/max_local_jobs": 8,
         "systems/modules_system": "nomod",

From dd8b4e9a897d4e7025f5500ffee18f39426fecbd Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 16 Sep 2024 13:37:21 +0200
Subject: [PATCH 106/128] Fix failing unit tests for DB file mode

---
 reframe/frontend/reporting/storage.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index 4e78889852..ab0af5771c 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -52,9 +52,13 @@ def __init__(self):
         self.__db_file = os.path.join(
             osext.expandvars(runtime().get_option('storage/0/sqlite_db_file'))
         )
-        self.__db_file_mode = runtime().get_option(
+        mode = runtime().get_option(
             'storage/0/sqlite_db_file_mode'
         )
+        if not isinstance(mode, int):
+            self.__db_file_mode = int(mode, base=8)
+        else:
+            self.__db_file_mode = mode
 
     def _db_file(self):
         prefix = os.path.dirname(self.__db_file)

From 2b97cb7124f7867740f1094dc49b7437bfcf8ad4 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Tue, 17 Sep 2024 17:11:42 +0200
Subject: [PATCH 107/128] Fix failing unit tests for DB file mode on Python 3.6

---
 reframe/frontend/reporting/storage.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/reframe/frontend/reporting/storage.py b/reframe/frontend/reporting/storage.py
index ab0af5771c..686332e2da 100644
--- a/reframe/frontend/reporting/storage.py
+++ b/reframe/frontend/reporting/storage.py
@@ -7,6 +7,7 @@
 import os
 import re
 import sqlite3
+import sys
 from filelock import FileLock
 
 import reframe.utility.jsonext as jsonext
@@ -87,8 +88,15 @@ def _db_connect(self, *args, **kwargs):
 
     def _db_lock(self):
         prefix = os.path.dirname(self.__db_file)
-        return FileLock(os.path.join(prefix, '.db.lock'),
-                        mode=self.__db_file_mode)
+        if sys.version_info >= (3, 7):
+            kwargs = {'mode': self.__db_file_mode}
+        else:
+            # Python 3.6 forces us to use an older filelock version that does
+            # not support file modes. File modes where introduced in
+            # filelock 3.10
+            kwargs = {}
+
+        return FileLock(os.path.join(prefix, '.db.lock'), **kwargs)
 
     def _db_create(self):
         clsname = type(self).__name__

From fee92c34814d6cd0957eef23faeeab728b604463 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Tue, 17 Sep 2024 17:40:47 +0200
Subject: [PATCH 108/128] Bump dev version

---
 reframe/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe/__init__.py b/reframe/__init__.py
index 8cb41ff7bb..b935398b41 100644
--- a/reframe/__init__.py
+++ b/reframe/__init__.py
@@ -6,7 +6,7 @@
 import os
 import sys
 
-VERSION = '4.7.0-dev.6'
+VERSION = '4.7.0-dev.7'
 INSTALL_PREFIX = os.path.normpath(
     os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 )

From 8a7648be0804f8fdf66d0b8c45fc482c7c03e275 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Wed, 18 Sep 2024 23:09:04 +0200
Subject: [PATCH 109/128] Add new `--retries-threshold` option

---
 docs/manpage.rst                       | 11 +++++++++++
 reframe/frontend/cli.py                | 15 ++++++++++++++-
 reframe/frontend/executors/__init__.py |  7 +++++--
 unittests/test_policies.py             |  7 +++++++
 4 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index b33242b8a4..3f1f13cdf7 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -694,6 +694,17 @@ Options controlling ReFrame execution
    .. versionchanged:: 3.6.1
       Multiple report files are now accepted.
 
+
+.. option:: --retries-threshold=VALUE[%]
+
+   Skip retries (see :option:`--max-retries`) if failures exceed the given threshold.
+
+   Threshold can be specified either as an absolute value or as a percentage using the ``%`` character, e.g., ``--retries-threshold=30%``.
+   Note that in certain shells the ``%`` character may need to be escaped.
+
+   .. versionadded:: 4.7
+
+
 .. option:: -S, --setvar=[TEST.]VAR=VAL
 
    Set variable ``VAR`` in all tests or optionally only in test ``TEST`` to ``VAL``.
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 19d4ffec6e..7aafa78ddb 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -521,6 +521,11 @@ def main():
         metavar='REPORT',
         help='Restore a testing session from REPORT file'
     )
+    run_options.add_argument(
+        '--retries-threshold', action='store', default='1000%',
+        metavar='VALUE[%]',
+        help='Retry tests only if failures do not exceed threshold'
+    )
     run_options.add_argument(
         '-S', '--setvar', action='append', metavar='[TEST.]VAR=VAL',
         dest='vars', default=[],
@@ -1563,8 +1568,16 @@ def module_unuse(*paths):
                 f"{options.reruns}"
             )
 
+        # Parse retries threshold
+        if options.retries_threshold[-1] == '%':
+            ratio = int(options.retries_threshold[:-1]) / 100.
+            retries_threshold = int(len(testcases)*ratio)
+        else:
+            retries_threshold = int(options.retries_threshold)
+
         runner = Runner(exec_policy, printer, options.max_retries,
-                        options.maxfail, options.reruns, options.duration)
+                        options.maxfail, options.reruns, options.duration,
+                        retries_threshold)
         try:
             time_start = time.time()
             runner.runall(testcases, restored_cases)
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index d96cbaeb72..0714b63b92 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -573,10 +573,12 @@ class Runner:
     _timeout = fields.TypedField(typ.Duration, type(None), allow_implicit=True)
 
     def __init__(self, policy, printer=None, max_retries=0,
-                 max_failures=sys.maxsize, reruns=0, timeout=None):
+                 max_failures=sys.maxsize, reruns=0, timeout=None,
+                 retries_threshold=sys.maxsize):
         self._policy = policy
         self._printer = printer or PrettyPrinter()
         self._max_retries = max_retries
+        self._retries_threshold = retries_threshold
         self._num_reruns = reruns
         self._timeout = timeout
         self._t_init = timeout
@@ -620,7 +622,8 @@ def runall(self, testcases, restored_cases=None):
                 self._policy.set_expiry(self._t_init + self._timeout)
 
             self._runall(testcases)
-            if self._max_retries:
+            if (self._max_retries and
+                len(self._stats.failed()) <= self._retries_threshold):
                 restored_cases = restored_cases or []
                 self._retry_failed(testcases + restored_cases)
 
diff --git a/unittests/test_policies.py b/unittests/test_policies.py
index 37e40df5e9..19ac559606 100644
--- a/unittests/test_policies.py
+++ b/unittests/test_policies.py
@@ -274,6 +274,13 @@ def test_retries_bad_check(make_runner, make_cases, common_exec_ctx):
     assert_runall(runner)
     assert runner.max_retries == rt.runtime().current_run
     assert 2 == len(runner.stats.failed())
+    assert 3 == runner.stats.num_runs
+
+
+def test_retries_threshold(make_runner, make_cases, common_exec_ctx):
+    runner = make_runner(max_retries=2, retries_threshold=1)
+    runner.runall(make_cases([BadSetupCheck(), BadSetupCheckEarly()]))
+    assert 1 == runner.stats.num_runs
 
 
 def test_retries_good_check(make_runner, make_cases, common_exec_ctx):

From a3de0cd696e11d8c2251c139fd8ff91d7a9e8d1b Mon Sep 17 00:00:00 2001
From: Brandon Cook <bgcook@lbl.gov>
Date: Thu, 19 Sep 2024 17:35:29 -0700
Subject: [PATCH 110/128] Cache conflicted_modules results

---
 reframe/core/modules.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/reframe/core/modules.py b/reframe/core/modules.py
index d29a47affd..70001faee9 100644
--- a/reframe/core/modules.py
+++ b/reframe/core/modules.py
@@ -8,6 +8,7 @@
 #
 
 import abc
+import functools
 import os
 import re
 from collections import OrderedDict
@@ -1030,6 +1031,7 @@ def available_modules(self, substr):
 
         return ret
 
+    @functools.lru_cache
     def conflicted_modules(self, module):
         if module.collection:
             # Conflicts have no meaning in module collection. The modules

From 32e50c20390373617e31010bae557d06fc8a4a35 Mon Sep 17 00:00:00 2001
From: Brandon Cook <bgcook@lbl.gov>
Date: Fri, 20 Sep 2024 16:22:33 -0700
Subject: [PATCH 111/128] fix: use decorator variant with arguments

The variant of the decorator without arguments was added in Python 3.8.
---
 reframe/core/modules.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe/core/modules.py b/reframe/core/modules.py
index 70001faee9..e8c9c504b1 100644
--- a/reframe/core/modules.py
+++ b/reframe/core/modules.py
@@ -1031,7 +1031,7 @@ def available_modules(self, substr):
 
         return ret
 
-    @functools.lru_cache
+    @functools.lru_cache(maxsize=128)
     def conflicted_modules(self, module):
         if module.collection:
             # Conflicts have no meaning in module collection. The modules

From aa5472b37a205e0badc75b40fc2de72e46b421a3 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Sat, 21 Sep 2024 00:02:14 +0200
Subject: [PATCH 112/128] Improve naming of generated tests

---
 docs/tutorial.rst                             |  2 +-
 .../slurm-cluster/docker-compose.yml          |  6 ++++-
 reframe/core/meta.py                          |  4 ++--
 reframe/core/pipeline.py                      | 21 ++++++++++++++--
 reframe/frontend/cli.py                       |  5 ++--
 reframe/frontend/filters.py                   |  3 ---
 reframe/frontend/testgenerators.py            | 24 +++++++++++--------
 7 files changed, 44 insertions(+), 21 deletions(-)

diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index e2d8bd2a99..55c52bd1df 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -1672,7 +1672,7 @@ By inspecting the generated script files, you will notice that ReFrame emits the
 .. code-block:: bash
    :caption: Run with the Docker compose setup.
 
-   cat /scratch/rfm-stage/output/pseudo-cluster/compute/gnu/stream_test_pseudo-cluster_compute_8de19aca/rfm_job.sh
+   cat /scratch/rfm-stage/output/pseudo-cluster/compute/gnu/stream_test_584fea5f/rfm_job.sh
 
 .. code-block:: bash
 
diff --git a/examples/tutorial/dockerfiles/slurm-cluster/docker-compose.yml b/examples/tutorial/dockerfiles/slurm-cluster/docker-compose.yml
index 13ce87e9ff..d6ef75df9f 100644
--- a/examples/tutorial/dockerfiles/slurm-cluster/docker-compose.yml
+++ b/examples/tutorial/dockerfiles/slurm-cluster/docker-compose.yml
@@ -12,7 +12,11 @@ services:
 
   frontend:
     image: slurm-reframe
-    build: examples/tutorial/dockerfiles/slurm-cluster/reframe
+    build:
+      context: examples/tutorial/dockerfiles/slurm-cluster/reframe
+      args:
+        REFRAME_REPO: reframe-hpc
+        REFRAME_TAG: develop
     container_name: frontend
     hostname: login
     user: admin
diff --git a/reframe/core/meta.py b/reframe/core/meta.py
index 1b3faed419..b57de01c14 100644
--- a/reframe/core/meta.py
+++ b/reframe/core/meta.py
@@ -305,7 +305,6 @@ def __new__(metacls, name, bases, namespace, **kwargs):
         '''
 
         # Collect the loggable properties
-        loggable_props = []
         namespace['_rfm_loggable_props'] = [
             v.fget._rfm_loggable for v in namespace.values()
             if hasattr(v, 'fget') and hasattr(v.fget, '_rfm_loggable')
@@ -816,7 +815,8 @@ def is_abstract(cls):
     def variant_name(cls, variant_num=None):
         '''Return the name of the test variant with a specific variant number.
 
-        :param variant_num: An integer in the range of ``[0, cls.num_variants)``.
+        :param variant_num: An integer in the range of
+            ``[0, cls.num_variants)``.
         '''
 
         name = cls.__name__
diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
index f928a406f5..79c8d6a916 100644
--- a/reframe/core/pipeline.py
+++ b/reframe/core/pipeline.py
@@ -1194,9 +1194,26 @@ def unique_name(self):
     def name(self):
         '''The name of the test.
 
-        This is an alias of :attr:`display_name`.
+        This is an alias of :attr:`display_name` but omitting any implicit
+        parameters starting with ``$`` that are inserted by the
+        :option:`--repeat`, :option:`--distribute` and other similar options.
+
+        .. versionchanged:: 4.7
+
+           The implicit parameters starting with ``$`` are now omitted.
         '''
-        return self.display_name
+        if hasattr(self, '_rfm_name'):
+            return self._rfm_name
+
+        # Remove any special parameters starting with `$`
+        basename, *params = self.display_name.split(' ')
+        name = basename
+        for p in params:
+            if not p.startswith('%$'):
+                name += f' {p}'
+
+        self._rfm_name = name
+        return self._rfm_name
 
     @loggable
     @property
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 19d4ffec6e..ff88f5f856 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -141,14 +141,15 @@ def describe_checks(testcases, printer):
         if tc.check.is_fixture():
             continue
 
-        if tc.check.name not in unique_names:
-            unique_names.add(tc.check.name)
+        if tc.check.display_name not in unique_names:
+            unique_names.add(tc.check.display_name)
             rec = json.loads(jsonext.dumps(tc.check))
 
             # Now manipulate the record to be more user-friendly
             #
             # 1. Add other fields that are relevant for users
             # 2. Remove all private fields
+            rec['name'] = tc.check.name
             rec['unique_name'] = tc.check.unique_name
             rec['display_name'] = tc.check.display_name
             rec['pipeline_hooks'] = {}
diff --git a/reframe/frontend/filters.py b/reframe/frontend/filters.py
index 3b47ad5737..77d90b9b6a 100644
--- a/reframe/frontend/filters.py
+++ b/reframe/frontend/filters.py
@@ -6,7 +6,6 @@
 import re
 
 from reframe.core.exceptions import ReframeError
-from reframe.core.runtime import runtime
 
 
 def re_compile(patt):
@@ -22,7 +21,6 @@ def _have_name(patt):
     def _fn(case):
         # Match pattern, but remove spaces from the `display_name`
         display_name = case.check.display_name.replace(' ', '')
-        rt = runtime()
         if '@' in patt:
             # Do an exact match on the unique name
             return patt.replace('@', '_') == case.check.unique_name
@@ -43,7 +41,6 @@ def _fn(case):
 
 
 def have_any_name(names):
-    rt = runtime()
     variant_matches = []
     hash_matches = []
     regex_matches = []
diff --git a/reframe/frontend/testgenerators.py b/reframe/frontend/testgenerators.py
index bae4cc4f16..b5f597d48a 100644
--- a/reframe/frontend/testgenerators.py
+++ b/reframe/frontend/testgenerators.py
@@ -94,13 +94,16 @@ def _rfm_set_valid_systems(obj):
             obj.valid_systems = [partition.fullname]
 
         return make_test(
-            f'{cls.__name__}_{partition.fullname.replace(":", "_")}',
-            (cls,),
+            cls.__name__, (cls,),
             {
                 'valid_systems': [partition.fullname],
+                # We add a partition parameter so as to differentiate the test
+                # in case another test has the same nodes in another partition
+                '$part': builtins.parameter([partition.fullname],
+                                            loggable=False),
                 '$nid': builtins.parameter(
                     [[n] for n in node_map[partition.fullname]],
-                    fmt=util.nodelist_abbrev
+                    fmt=util.nodelist_abbrev, loggable=False
                 )
             },
             methods=[
@@ -110,7 +113,7 @@ def _rfm_set_valid_systems(obj):
                 # will not be overwritten by a parent post-init hook
                 builtins.run_after('init')(_rfm_set_valid_systems),
             ]
-        ), ['$nid']
+        ), ['$part', '$nid']
 
     return _generate_tests(testcases, _make_dist_test)
 
@@ -122,9 +125,10 @@ def repeat_tests(testcases, num_repeats):
     def _make_repeat_test(testcase):
         cls = type(testcase.check)
         return make_test(
-            f'{cls.__name__}', (cls,),
+            cls.__name__, (cls,),
             {
-                '$repeat_no': builtins.parameter(range(num_repeats))
+                '$repeat_no': builtins.parameter(range(num_repeats),
+                                                 loggable=False)
             }
         ), ['$repeat_no']
 
@@ -150,17 +154,17 @@ def _make_parameterized_test(testcase):
                 if var_check != cls.__name__:
                     continue
             else:
-                getlogger().warning(f'cannot set a variable in a fixture')
+                getlogger().warning('cannot set a variable in a fixture')
                 continue
 
-            if not var in cls.var_space:
+            if var not in cls.var_space:
                 getlogger().warning(
                     f'variable {var!r} not defined for test '
                     f'{check.display_name!r}; ignoring parameterization'
                 )
                 continue
 
-            body[f'${var}'] = builtins.parameter(values)
+            body[f'${var}'] = builtins.parameter(values, loggable=False)
 
         def _set_vars(self):
             for var in body.keys():
@@ -168,7 +172,7 @@ def _set_vars(self):
                         make_convertible(getattr(self, f'{var}')))
 
         return make_test(
-            f'{cls.__name__}', (cls,),
+            cls.__name__, (cls,),
             body=body,
             methods=[builtins.run_after('init')(_set_vars)]
         ), body.keys()

From 048049fc1be00ac4db58dd03650785057cc63c19 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 23 Sep 2024 18:31:21 +0200
Subject: [PATCH 113/128] Support session annotations

---
 docs/manpage.rst                       |  9 ++++++
 reframe/frontend/cli.py                | 13 ++++++++
 reframe/frontend/printer.py            |  9 +++++-
 reframe/frontend/reporting/__init__.py | 45 +++++++++++++++++++++-----
 unittests/test_cli.py                  | 26 +++++++++++----
 5 files changed, 87 insertions(+), 15 deletions(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index 3f1f13cdf7..cecb2176f0 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -1129,6 +1129,15 @@ Miscellaneous options
    .. versionadded:: 3.9.3
 
 
+.. option:: --session-extras KV_DATA
+
+   Annotate the current session with custom key/value metadata.
+
+   The key/value data is specified as a comma-separated list of `key=value` pairs.
+   When listing stored sessions with the :option:`--list-stored-sessions` option, any associated custom metadata will be presented by default.
+
+   .. versionadded:: 4.7
+
 .. option:: --system=NAME
 
    Load the configuration for system ``NAME``.
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 9594f5fae2..7602ae4edb 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -616,6 +616,10 @@ def main():
         help=('Print a report for performance tests '
               '(default: "now:now/last:+job_nodelist/+result")')
     )
+    reporting_options.add_argument(
+        '--session-extras', action='store', metavar='KV_DATA',
+        help='Annotate session with custom key/value data'
+    )
 
     # Miscellaneous options
     misc_options.add_argument(
@@ -1590,6 +1594,15 @@ def module_unuse(*paths):
             if options.restore_session is not None:
                 report.update_restored_cases(restored_cases, restored_session)
 
+            if options.session_extras:
+                # Update report's extras
+                extras = {}
+                for arg in options.session_extras.split(','):
+                    k, v = arg.split('=', maxsplit=1)
+                    extras[k] = v
+
+                report.update_extras(extras)
+
             # Print a retry report if we did any retries
             if options.max_retries and runner.stats.failed(run=0):
                 printer.retry_report(report)
diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index e622ad77bc..a54c333ec5 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -283,7 +283,14 @@ def table(self, data, **kwargs):
             colidx = [i for i, col in enumerate(data[0])
                       if col not in hide_columns]
 
-            tab_data = [[rec[col] for col in colidx] for rec in data]
+            def _access(seq, i, default=None):
+                # Safe access of i-th element of a sequence
+                try:
+                    return seq[i]
+                except IndexError:
+                    return default
+
+            tab_data = [[_access(rec, col) for col in colidx] for rec in data]
         else:
             tab_data = data
 
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 3c23ae16f0..1c7254f83e 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -25,7 +25,7 @@
 from reframe.core.logging import getlogger, _format_time_rfc3339, time_function
 from reframe.core.runtime import runtime
 from reframe.core.warnings import suppress_deprecations
-from reframe.utility import nodelist_abbrev
+from reframe.utility import nodelist_abbrev, OrderedSet
 from .storage import StorageBackend
 from .utility import Aggregator, parse_cmp_spec, parse_time_period, is_uuid
 
@@ -269,6 +269,14 @@ def update_timestamps(self, ts_start, ts_end):
             'time_elapsed': ts_end - ts_start
         })
 
+    def update_extras(self, extras):
+        '''Attach user-specific metadata to the session'''
+
+        # We prepend a special character to the user extras in order to avoid
+        # possible conflicts with existing keys
+        for k, v in extras.items():
+            self.__report['session_info'][f'${k}'] = v
+
     def update_run_stats(self, stats):
         session_uuid = self.__report['session_info']['uuid']
         for runidx, tasks in stats.runs():
@@ -645,17 +653,38 @@ def session_data(time_period):
     '''Retrieve all sessions'''
 
     data = [['UUID', 'Start time', 'End time', 'Num runs', 'Num cases']]
+    extra_cols = OrderedSet()
     for sess_data in StorageBackend.default().fetch_sessions_time_period(
         *parse_time_period(time_period) if time_period else (None, None)
     ):
         session_info = sess_data['session_info']
-        data.append(
-            [session_info['uuid'],
-             session_info['time_start'],
-             session_info['time_end'],
-             len(sess_data['runs']),
-             len(sess_data['runs'][0]['testcases'])]
-        )
+        record = [session_info['uuid'],
+                  session_info['time_start'],
+                  session_info['time_end'],
+                  len(sess_data['runs']),
+                  len(sess_data['runs'][0]['testcases'])]
+
+        # Expand output with any user metadata
+        for k in session_info:
+            if k.startswith('$'):
+                extra_cols.add(k[1:])
+
+        # Add any extras recorded so far
+        for key in extra_cols:
+            record.append(session_info.get(f'${key}', ''))
+
+        data.append(record)
+
+    # Do a final grooming pass of the data to expand short records
+    if extra_cols:
+        data[0] += extra_cols
+
+    for rec in data:
+        diff = len(extra_cols) - len(rec)
+        if diff == 0:
+            break
+
+        rec += ['n/a' for _ in range(diff)]
 
     return data
 
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 13339e8d9f..a8800b9b83 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -1262,13 +1262,14 @@ def table_format(request):
     return request.param
 
 
-def test_storage_options(run_reframe, tmp_path, table_format):
-    def assert_no_crash(returncode, stdout, stderr, exitcode=0):
-        assert returncode == exitcode
-        assert 'Traceback' not in stdout
-        assert 'Traceback' not in stderr
-        return returncode, stdout, stderr
+def assert_no_crash(returncode, stdout, stderr, exitcode=0):
+    assert returncode == exitcode
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    return returncode, stdout, stderr
+
 
+def test_storage_options(run_reframe, tmp_path, table_format):
     run_reframe2 = functools.partial(
         run_reframe,
         checkpath=['unittests/resources/checks/frontend_checks.py'],
@@ -1328,6 +1329,19 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0):
     assert_no_crash(*run_reframe2(action=f'--delete-stored-session={uuid}'))
 
 
+def test_session_annotations(run_reframe):
+    assert_no_crash(*run_reframe(
+        checkpath=['unittests/resources/checks/frontend_checks.py'],
+        action='-r',
+        more_options=['--session-extras', 'key1=val1,key2=val2',
+                      '-n', '^PerformanceFailureCheck']
+    ), exitcode=1)
+
+    stdout = assert_no_crash(*run_reframe(action='--list-stored-sessions'))[1]
+    for text in ['key1', 'key2', 'val1', 'val2']:
+        assert text in stdout
+
+
 def test_performance_compare(run_reframe, table_format):
     def assert_no_crash(returncode, stdout, stderr, exitcode=0):
         assert returncode == exitcode

From 7b760005756786e37d3599c9d61f82aff05ef92c Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Tue, 24 Sep 2024 13:49:25 +0200
Subject: [PATCH 114/128] Reorganize manpage

---
 docs/manpage.rst | 434 +++++++++++++++++++++++------------------------
 1 file changed, 217 insertions(+), 217 deletions(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index cecb2176f0..51d0686bcc 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -4,13 +4,13 @@ Command Line Reference
 
 
 Synopsis
---------
+========
 
 .. option:: reframe [OPTION]... ACTION
 
 
 Description
------------
+===========
 
 ReFrame provides both a :doc:`programming interface <regression_test_api>` for writing regression tests and a command-line interface for managing and running the tests, which is detailed here.
 The ``reframe`` command is part of ReFrame's frontend.
@@ -26,9 +26,212 @@ Usually, ReFrame processes tests in three phases:
 There are also ReFrame commands that do not operate on a set of tests.
 
 
+.. _commands:
+
+Commands
+--------
+
+ReFrame commands are mutually exclusive and one of them must always be specified.
+There are commands that act upon the selected tests and others that have a helper function, such as querying the configuration, querying the results database etc.
+
+.. versionchanged:: 4.7
+
+   ReFrame commands are now mutually exclusive and only one can be specified every time.
+
+
+Test commands
+^^^^^^^^^^^^^
+
+.. option:: --ci-generate=FILE
+
+   Generate a Gitlab `child pipeline <https://docs.gitlab.com/ee/ci/parent_child_pipelines.html>`__ specification in ``FILE`` that will run the selected tests.
+
+   You can set up your Gitlab CI to use the generated file to run every test as a separate Gitlab job respecting test dependencies.
+   For more information, have a look in :ref:`generate-ci-pipeline`.
+
+   .. note::
+      This option will not work with the :ref:`test generation options <test-generators>`.
+
+
+   .. versionadded:: 3.4.1
+
+.. option:: --describe
+
+   Print a detailed description of the `selected tests <#test-filtering>`__ in JSON format and exit.
+
+   .. note::
+      The generated test description corresponds to its state after it has been initialized.
+      If any of its attributes are changed or set during its execution, their updated values will not be shown by this listing.
+
+   .. versionadded:: 3.10.0
+
+
+.. option:: --dry-run
+
+   Dry run the selected tests.
+
+   The dry-run mode will try to execute as much of the test pipeline as possible.
+   More specifically, the tests will not be submitted and will not be run for real,
+   but their stage directory will be prepared and the corresponding job script will be emitted.
+   Similarly, the sanity and performance functions will not be evaluated but all the preparation will happen.
+   Tests run in dry-run mode will not fail unless there is a programming error in the test or if the test tries to use a resource that is not produced in dry run mode (e.g., access the standard output or a resource produced by a dependency outside any sanity or performance function).
+   In this case, users can call the :func:`~reframe.core.pipeline.RegressionTest.is_dry_run` method in their test and take a specific action if the test is run in dry-run mode.
+
+   .. versionadded:: 4.1
+
+.. option:: -L, --list-detailed[=T|C]
+
+   List selected tests providing more details for each test.
+
+   The unique id of each test (see also :attr:`~reframe.core.pipeline.RegressionTest.unique_name`) as well as the file where each test is defined are printed.
+
+   This option accepts optionally a single argument denoting what type of listing is requested.
+   Please refer to :option:`-l` for an explanation of this argument.
+
+   .. versionadded:: 3.10.0
+      Support for different types of listing is added.
+
+   .. versionchanged:: 4.0.5
+      The variable names to which fixtures are bound are also listed.
+      See :ref:`test_naming_scheme` for more information.
+
+.. option:: -l, --list[=T|C]
+
+   List selected tests and their dependencies.
+
+   This option accepts optionally a single argument denoting what type of listing is requested.
+   There are two types of possible listings:
+
+   - *Regular test listing* (``T``, the default): This type of listing lists the tests and their dependencies or fixtures using their :attr:`~reframe.core.pipeline.RegressionTest.display_name`. A test that is listed as a dependency of another test will not be listed separately.
+   - *Concretized test case listing* (``C``): This type of listing lists the exact test cases and their dependencies as they have been concretized for the current system and environment combinations.
+     This listing shows practically the exact test DAG that will be executed.
+
+   .. versionadded:: 3.10.0
+      Support for different types of listing is added.
+
+   .. versionchanged:: 4.0.5
+      The variable names to which fixtures are bound are also listed.
+      See :ref:`test_naming_scheme` for more information.
+
+.. option:: --list-tags
+
+   List the unique tags of the selected tests.
+
+   The tags are printed in alphabetical order.
+
+   .. versionadded:: 3.6.0
+
+.. option:: -r, --run
+
+   Run the selected tests.
+
+
+Result storage commands
+^^^^^^^^^^^^^^^^^^^^^^^
+
+.. option:: --delete-stored-session=UUID
+
+   Delete the stored session with the specified UUID from the results database.
+
+   .. versionadded:: 4.7
+
+.. option:: --describe-stored-session=UUID
+
+   Get detailed information of the session with the specified UUID.
+   The output is in JSON format.
+
+   .. versionadded:: 4.7
+
+.. option:: --describe-stored-testcases=SESSION_UUID|TIME_PERIOD
+
+   Get detailed test case information of the session with the specified UUID or from the specified time period.
+
+   If a session UUID is provided only information about the test cases of this session will be provided.
+   This option can be combined with :option:`--name` to restrict the listing to specific tests.
+   For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax` section.
+
+   .. versionadded:: 4.7
+
+.. _--list-stored-sessions:
+
+.. option:: --list-stored-sessions[=TIME_PERIOD]
+
+   List sessions stored in the results database.
+
+   If ``TIME_PERIOD`` is ``all``, all stored sessions will be listed.
+   If not specified, only the sessions of last week will be listed.
+   For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`.
+
+   .. versionadded:: 4.7
+
+.. option:: --list-stored-testcases=SESSION_UUID|TIME_PERIOD
+
+   List all test cases from the session with the specified UUID or from the specified time period.
+
+   If a session UUID is provided only the test cases of this session will be listed.
+   This option can be combined with :option:`--name` to restrict the listing to specific tests.
+   For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax` section.
+
+   .. versionadded:: 4.7
+
+.. option:: --performance-compare=CMPSPEC
+
+   Compare the performance of test cases that have run in the past.
+
+   This option can be combined with :option:`--name` to restrict the comparison to specific tests.
+   Check the :ref:`performance-comparisons` section for the exact syntax of ``CMPSPEC``.
+
+   .. versionadded:: 4.7
+
+Other commands
+^^^^^^^^^^^^^^
+
+.. _--detect-host-topology:
+
+.. option:: --detect-host-topology[=FILE]
+
+   Detect the local host processor topology, store it to ``FILE`` and exit.
+
+   If no ``FILE`` is specified, the standard output will be used.
+
+   .. versionadded:: 3.7.0
+
+.. option:: --show-config [PARAM]
+
+   Show the value of configuration parameter ``PARAM`` as this is defined for the currently selected system and exit.
+
+   The parameter value is printed in JSON format.
+   If ``PARAM`` is not specified or if it set to ``all``, the whole configuration for the currently selected system will be shown.
+   Configuration parameters are formatted as a path navigating from the top-level configuration object to the actual parameter.
+   The ``/`` character acts as a selector of configuration object properties or an index in array objects.
+   The ``@`` character acts as a selector by name for configuration objects that have a ``name`` property.
+   Here are some example queries:
+
+   - Retrieve all the partitions of the current system:
+
+     .. code:: bash
+
+        reframe --show-config=systems/0/partitions
+
+   - Retrieve the job scheduler of the partition named ``default``:
+
+     .. code:: bash
+
+        reframe --show-config=systems/0/partitions/@default/scheduler
+
+   - Retrieve the check search path for system ``foo``:
+
+     .. code:: bash
+
+        reframe --system=foo --show-config=general/0/check_search_path
+
+.. option:: -V, --version
+
+   Print version and exit.
+
+
 .. _test-discovery:
 
--------------------------------
 Test discovery and test loading
 -------------------------------
 
@@ -64,7 +267,6 @@ This is something that test developers should bear in mind.
 
 .. _test-filtering:
 
---------------
 Test filtering
 --------------
 
@@ -243,202 +445,6 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep
       If you want to match at the beginning of a test name, you should prepend ``^``.
 
 
-.. _commands:
-
---------
-Commands
---------
-
-ReFrame commands are mutually exclusive and one of them must always be specified.
-There are commands that act upon the selected tests and others that have a helper function, such as querying the configuration, querying the results database etc.
-
-.. versionchanged:: 4.7
-
-   ReFrame commands are now mutually exclusive and only one can be specified every time.
-
-
-.. option:: --ci-generate=FILE
-
-   Generate a Gitlab `child pipeline <https://docs.gitlab.com/ee/ci/parent_child_pipelines.html>`__ specification in ``FILE`` that will run the selected tests.
-
-   You can set up your Gitlab CI to use the generated file to run every test as a separate Gitlab job respecting test dependencies.
-   For more information, have a look in :ref:`generate-ci-pipeline`.
-
-   .. note::
-      This option will not work with the :ref:`test generation options <test-generators>`.
-
-
-   .. versionadded:: 3.4.1
-
-.. option:: --delete-stored-session=UUID
-
-   Delete the stored session with the specified UUID from the results database.
-
-   .. versionadded:: 4.7
-
-.. option:: --describe
-
-   Print a detailed description of the `selected tests <#test-filtering>`__ in JSON format and exit.
-
-   .. note::
-      The generated test description corresponds to its state after it has been initialized.
-      If any of its attributes are changed or set during its execution, their updated values will not be shown by this listing.
-
-   .. versionadded:: 3.10.0
-
-
-.. option:: --describe-stored-session=UUID
-
-   Get detailed information of the session with the specified UUID.
-   The output is in JSON format.
-
-   .. versionadded:: 4.7
-
-.. option:: --describe-stored-testcases=SESSION_UUID|TIME_PERIOD
-
-   Get detailed test case information of the session with the specified UUID or from the specified time period.
-
-   If a session UUID is provided only information about the test cases of this session will be provided.
-   This option can be combined with :option:`--name` to restrict the listing to specific tests.
-   For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax` section.
-
-   .. versionadded:: 4.7
-
-.. _--detect-host-topology:
-
-.. option:: --detect-host-topology[=FILE]
-
-   Detect the local host processor topology, store it to ``FILE`` and exit.
-
-   If no ``FILE`` is specified, the standard output will be used.
-
-   .. versionadded:: 3.7.0
-
-.. option:: --dry-run
-
-   Dry run the selected tests.
-
-   The dry-run mode will try to execute as much of the test pipeline as possible.
-   More specifically, the tests will not be submitted and will not be run for real,
-   but their stage directory will be prepared and the corresponding job script will be emitted.
-   Similarly, the sanity and performance functions will not be evaluated but all the preparation will happen.
-   Tests run in dry-run mode will not fail unless there is a programming error in the test or if the test tries to use a resource that is not produced in dry run mode (e.g., access the standard output or a resource produced by a dependency outside any sanity or performance function).
-   In this case, users can call the :func:`~reframe.core.pipeline.RegressionTest.is_dry_run` method in their test and take a specific action if the test is run in dry-run mode.
-
-   .. versionadded:: 4.1
-
-.. option:: -L, --list-detailed[=T|C]
-
-   List selected tests providing more details for each test.
-
-   The unique id of each test (see also :attr:`~reframe.core.pipeline.RegressionTest.unique_name`) as well as the file where each test is defined are printed.
-
-   This option accepts optionally a single argument denoting what type of listing is requested.
-   Please refer to :option:`-l` for an explanation of this argument.
-
-   .. versionadded:: 3.10.0
-      Support for different types of listing is added.
-
-   .. versionchanged:: 4.0.5
-      The variable names to which fixtures are bound are also listed.
-      See :ref:`test_naming_scheme` for more information.
-
-.. _--list-stored-sessions:
-
-.. option:: --list-stored-sessions[=TIME_PERIOD]
-
-   List sessions stored in the results database.
-
-   If ``TIME_PERIOD`` is ``all``, all stored sessions will be listed.
-   If not specified, only the sessions of last week will be listed.
-   For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`.
-
-   .. versionadded:: 4.7
-
-.. option:: --list-stored-testcases=SESSION_UUID|TIME_PERIOD
-
-   List all test cases from the session with the specified UUID or from the specified time period.
-
-   If a session UUID is provided only the test cases of this session will be listed.
-   This option can be combined with :option:`--name` to restrict the listing to specific tests.
-   For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax` section.
-
-   .. versionadded:: 4.7
-
-.. option:: -l, --list[=T|C]
-
-   List selected tests and their dependencies.
-
-   This option accepts optionally a single argument denoting what type of listing is requested.
-   There are two types of possible listings:
-
-   - *Regular test listing* (``T``, the default): This type of listing lists the tests and their dependencies or fixtures using their :attr:`~reframe.core.pipeline.RegressionTest.display_name`. A test that is listed as a dependency of another test will not be listed separately.
-   - *Concretized test case listing* (``C``): This type of listing lists the exact test cases and their dependencies as they have been concretized for the current system and environment combinations.
-     This listing shows practically the exact test DAG that will be executed.
-
-   .. versionadded:: 3.10.0
-      Support for different types of listing is added.
-
-   .. versionchanged:: 4.0.5
-      The variable names to which fixtures are bound are also listed.
-      See :ref:`test_naming_scheme` for more information.
-
-.. option:: --list-tags
-
-   List the unique tags of the selected tests.
-
-   The tags are printed in alphabetical order.
-
-   .. versionadded:: 3.6.0
-
-.. option:: --performance-compare=CMPSPEC
-
-   Compare the performance of test cases that have run in the past.
-
-   This option can be combined with :option:`--name` to restrict the comparison to specific tests.
-   Check the :ref:`performance-comparisons` section for the exact syntax of ``CMPSPEC``.
-
-   .. versionadded:: 4.7
-
-.. option:: -r, --run
-
-   Run the selected tests.
-
-.. option:: --show-config [PARAM]
-
-   Show the value of configuration parameter ``PARAM`` as this is defined for the currently selected system and exit.
-
-   The parameter value is printed in JSON format.
-   If ``PARAM`` is not specified or if it set to ``all``, the whole configuration for the currently selected system will be shown.
-   Configuration parameters are formatted as a path navigating from the top-level configuration object to the actual parameter.
-   The ``/`` character acts as a selector of configuration object properties or an index in array objects.
-   The ``@`` character acts as a selector by name for configuration objects that have a ``name`` property.
-   Here are some example queries:
-
-   - Retrieve all the partitions of the current system:
-
-     .. code:: bash
-
-        reframe --show-config=systems/0/partitions
-
-   - Retrieve the job scheduler of the partition named ``default``:
-
-     .. code:: bash
-
-        reframe --show-config=systems/0/partitions/@default/scheduler
-
-   - Retrieve the check search path for system ``foo``:
-
-     .. code:: bash
-
-        reframe --system=foo --show-config=general/0/check_search_path
-
-.. option:: -V, --version
-
-   Print version and exit.
-
-
-----------------------------------
 Options controlling ReFrame output
 ----------------------------------
 
@@ -559,7 +565,6 @@ Options controlling ReFrame output
    This option can also be set using the :envvar:`RFM_TIMESTAMP_DIRS` environment variable or the :attr:`~config.general.timestamp_dirs` general configuration parameter.
 
 
--------------------------------------
 Options controlling ReFrame execution
 -------------------------------------
 
@@ -811,7 +816,6 @@ Options controlling ReFrame execution
    Skip sanity checking phase.
 
 
-----------------------------------
 Options controlling job submission
 ----------------------------------
 
@@ -836,9 +840,9 @@ Options controlling job submission
    .. versionchanged:: 3.1
       Use ``&`` to combine constraints.
 
-------------------------
-Flexible node allocation
-------------------------
+
+Options controlling flexible node allocation
+--------------------------------------------
 
 ReFrame can automatically set the number of tasks of a test, if its :attr:`num_tasks <reframe.core.pipeline.RegressionTest.num_tasks>` attribute is set to a value less than or equal to zero.
 This scheme is conveniently called *flexible node allocation* and is valid only for the Slurm backend.
@@ -879,7 +883,6 @@ If no node can be selected, the test will be marked as a failure with an appropr
    .. versionadded:: 4.7
 
 
----------------------------------------
 Options controlling ReFrame environment
 ---------------------------------------
 
@@ -973,7 +976,6 @@ It does so by leveraging the selected system's environment modules system.
 
 .. _test-generators:
 
-----------------------------------------
 Options for generating tests dynamically
 ----------------------------------------
 
@@ -1062,7 +1064,6 @@ The way the tests are generated and how they interact with the test filtering op
    .. versionadded:: 3.12.0
 
 
----------------------
 Miscellaneous options
 ---------------------
 
@@ -1191,7 +1192,7 @@ Miscellaneous options
 .. _test_naming_scheme:
 
 Test Naming Scheme
-------------------
+==================
 
 .. versionadded:: 3.10.0
 
@@ -1308,7 +1309,6 @@ Also users should not rely on how the variant numbers are assigned to a test, as
    A hash code is associated with each test.
 
 
---------------------------------------
 Differences from the old naming scheme
 --------------------------------------
 
@@ -1320,7 +1320,7 @@ Fixtures followed a similar naming pattern making them hard to debug.
 
 
 Result storage
---------------
+==============
 
 .. versionadded:: 4.7
 
@@ -1343,7 +1343,7 @@ The file report of any session can be retrieved from the database with the :opti
 .. _performance-comparisons:
 
 Performance comparisons
------------------------
+=======================
 
 .. versionadded:: 4.7
 
@@ -1428,7 +1428,7 @@ Here are some examples of performance comparison specs:
 .. _time-period-syntax:
 
 Time periods
-------------
+============
 
 A time period needs to be specified as part of the ``CMPSPEC`` of the :option:`--performance-compare` and :option:`--performance-report` options or as an argument to options that request past results from results database.
 
@@ -1453,7 +1453,7 @@ For example, the period of the last 10 days can be specified as ``now-10d:now``.
 Similarly, the period of the week starting on August 5, 2024 will be specified as ``20240805:20240805+1w``.
 
 Environment
------------
+===========
 
 Several aspects of ReFrame can be controlled through environment variables.
 Usually environment variables have counterparts in command line options or configuration parameters.
@@ -2203,7 +2203,7 @@ Whenever an environment variable is associated with a configuration option, its
 
 
 Configuration File
-------------------
+==================
 
 The configuration file of ReFrame defines the systems and environments to test as well as parameters controlling the framework's behavior.
 
@@ -2216,12 +2216,12 @@ For a complete reference of the available configuration options, please refer to
 
 
 Reporting Bugs
---------------
+==============
 
 For bugs, feature request, help, please open an issue on Github: <https://github.com/reframe-hpc/reframe>
 
 
 See Also
---------
+========
 
 See full documentation online: <https://reframe-hpc.readthedocs.io/>

From 38c972c6638711512c4c8522a5058d3b2872c6e0 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarakasis@nvidia.com>
Date: Mon, 23 Sep 2024 18:31:21 +0200
Subject: [PATCH 115/128] Support session annotations

---
 docs/manpage.rst                       |  9 ++++++
 reframe/frontend/cli.py                | 13 ++++++++
 reframe/frontend/printer.py            |  9 +++++-
 reframe/frontend/reporting/__init__.py | 45 +++++++++++++++++++++-----
 unittests/test_cli.py                  | 26 +++++++++++----
 5 files changed, 87 insertions(+), 15 deletions(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index 3f1f13cdf7..cecb2176f0 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -1129,6 +1129,15 @@ Miscellaneous options
    .. versionadded:: 3.9.3
 
 
+.. option:: --session-extras KV_DATA
+
+   Annotate the current session with custom key/value metadata.
+
+   The key/value data is specified as a comma-separated list of `key=value` pairs.
+   When listing stored sessions with the :option:`--list-stored-sessions` option, any associated custom metadata will be presented by default.
+
+   .. versionadded:: 4.7
+
 .. option:: --system=NAME
 
    Load the configuration for system ``NAME``.
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 9594f5fae2..7602ae4edb 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -616,6 +616,10 @@ def main():
         help=('Print a report for performance tests '
               '(default: "now:now/last:+job_nodelist/+result")')
     )
+    reporting_options.add_argument(
+        '--session-extras', action='store', metavar='KV_DATA',
+        help='Annotate session with custom key/value data'
+    )
 
     # Miscellaneous options
     misc_options.add_argument(
@@ -1590,6 +1594,15 @@ def module_unuse(*paths):
             if options.restore_session is not None:
                 report.update_restored_cases(restored_cases, restored_session)
 
+            if options.session_extras:
+                # Update report's extras
+                extras = {}
+                for arg in options.session_extras.split(','):
+                    k, v = arg.split('=', maxsplit=1)
+                    extras[k] = v
+
+                report.update_extras(extras)
+
             # Print a retry report if we did any retries
             if options.max_retries and runner.stats.failed(run=0):
                 printer.retry_report(report)
diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index e622ad77bc..a54c333ec5 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -283,7 +283,14 @@ def table(self, data, **kwargs):
             colidx = [i for i, col in enumerate(data[0])
                       if col not in hide_columns]
 
-            tab_data = [[rec[col] for col in colidx] for rec in data]
+            def _access(seq, i, default=None):
+                # Safe access of i-th element of a sequence
+                try:
+                    return seq[i]
+                except IndexError:
+                    return default
+
+            tab_data = [[_access(rec, col) for col in colidx] for rec in data]
         else:
             tab_data = data
 
diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py
index 3c23ae16f0..1c7254f83e 100644
--- a/reframe/frontend/reporting/__init__.py
+++ b/reframe/frontend/reporting/__init__.py
@@ -25,7 +25,7 @@
 from reframe.core.logging import getlogger, _format_time_rfc3339, time_function
 from reframe.core.runtime import runtime
 from reframe.core.warnings import suppress_deprecations
-from reframe.utility import nodelist_abbrev
+from reframe.utility import nodelist_abbrev, OrderedSet
 from .storage import StorageBackend
 from .utility import Aggregator, parse_cmp_spec, parse_time_period, is_uuid
 
@@ -269,6 +269,14 @@ def update_timestamps(self, ts_start, ts_end):
             'time_elapsed': ts_end - ts_start
         })
 
+    def update_extras(self, extras):
+        '''Attach user-specific metadata to the session'''
+
+        # We prepend a special character to the user extras in order to avoid
+        # possible conflicts with existing keys
+        for k, v in extras.items():
+            self.__report['session_info'][f'${k}'] = v
+
     def update_run_stats(self, stats):
         session_uuid = self.__report['session_info']['uuid']
         for runidx, tasks in stats.runs():
@@ -645,17 +653,38 @@ def session_data(time_period):
     '''Retrieve all sessions'''
 
     data = [['UUID', 'Start time', 'End time', 'Num runs', 'Num cases']]
+    extra_cols = OrderedSet()
     for sess_data in StorageBackend.default().fetch_sessions_time_period(
         *parse_time_period(time_period) if time_period else (None, None)
     ):
         session_info = sess_data['session_info']
-        data.append(
-            [session_info['uuid'],
-             session_info['time_start'],
-             session_info['time_end'],
-             len(sess_data['runs']),
-             len(sess_data['runs'][0]['testcases'])]
-        )
+        record = [session_info['uuid'],
+                  session_info['time_start'],
+                  session_info['time_end'],
+                  len(sess_data['runs']),
+                  len(sess_data['runs'][0]['testcases'])]
+
+        # Expand output with any user metadata
+        for k in session_info:
+            if k.startswith('$'):
+                extra_cols.add(k[1:])
+
+        # Add any extras recorded so far
+        for key in extra_cols:
+            record.append(session_info.get(f'${key}', ''))
+
+        data.append(record)
+
+    # Do a final grooming pass of the data to expand short records
+    if extra_cols:
+        data[0] += extra_cols
+
+    for rec in data:
+        diff = len(extra_cols) - len(rec)
+        if diff == 0:
+            break
+
+        rec += ['n/a' for _ in range(diff)]
 
     return data
 
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 13339e8d9f..a8800b9b83 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -1262,13 +1262,14 @@ def table_format(request):
     return request.param
 
 
-def test_storage_options(run_reframe, tmp_path, table_format):
-    def assert_no_crash(returncode, stdout, stderr, exitcode=0):
-        assert returncode == exitcode
-        assert 'Traceback' not in stdout
-        assert 'Traceback' not in stderr
-        return returncode, stdout, stderr
+def assert_no_crash(returncode, stdout, stderr, exitcode=0):
+    assert returncode == exitcode
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    return returncode, stdout, stderr
+
 
+def test_storage_options(run_reframe, tmp_path, table_format):
     run_reframe2 = functools.partial(
         run_reframe,
         checkpath=['unittests/resources/checks/frontend_checks.py'],
@@ -1328,6 +1329,19 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0):
     assert_no_crash(*run_reframe2(action=f'--delete-stored-session={uuid}'))
 
 
+def test_session_annotations(run_reframe):
+    assert_no_crash(*run_reframe(
+        checkpath=['unittests/resources/checks/frontend_checks.py'],
+        action='-r',
+        more_options=['--session-extras', 'key1=val1,key2=val2',
+                      '-n', '^PerformanceFailureCheck']
+    ), exitcode=1)
+
+    stdout = assert_no_crash(*run_reframe(action='--list-stored-sessions'))[1]
+    for text in ['key1', 'key2', 'val1', 'val2']:
+        assert text in stdout
+
+
 def test_performance_compare(run_reframe, table_format):
     def assert_no_crash(returncode, stdout, stderr, exitcode=0):
         assert returncode == exitcode

From 2601af9362f506066734e39885c1b3cd5ce8eb86 Mon Sep 17 00:00:00 2001
From: Theofilos Manitaras <manitaras@cscs.ch>
Date: Tue, 24 Sep 2024 15:56:30 +0200
Subject: [PATCH 116/128] First attempt to test squeue scheduler

Signed-off-by: Theofilos Manitaras <manitaras@cscs.ch>
---
 .github/slurm-cluster/docker-compose.yml      | 113 +++++++++++++
 .github/slurm-cluster/reframe/Dockerfile      |  42 +++++
 .github/slurm-cluster/reframe/cgroup.conf     |   6 +
 .../reframe/docker-entrypoint.sh              |  20 +++
 .github/slurm-cluster/reframe/slurm.conf      | 153 ++++++++++++++++++
 .github/workflows/test-schedulers.yaml        |  21 +++
 6 files changed, 355 insertions(+)
 create mode 100644 .github/slurm-cluster/docker-compose.yml
 create mode 100644 .github/slurm-cluster/reframe/Dockerfile
 create mode 100644 .github/slurm-cluster/reframe/cgroup.conf
 create mode 100755 .github/slurm-cluster/reframe/docker-entrypoint.sh
 create mode 100644 .github/slurm-cluster/reframe/slurm.conf
 create mode 100644 .github/workflows/test-schedulers.yaml

diff --git a/.github/slurm-cluster/docker-compose.yml b/.github/slurm-cluster/docker-compose.yml
new file mode 100644
index 0000000000..8290a74ec9
--- /dev/null
+++ b/.github/slurm-cluster/docker-compose.yml
@@ -0,0 +1,113 @@
+services:
+  munge-key-generator:
+    image: ghcr.io/reframe-hpc/munge-ubuntu:20.04
+    hostname: munge-host
+    healthcheck:
+        test: ["CMD-SHELL", "test -f /scratch/munge.key"]
+        interval: 10s
+        timeout: 10s
+        retries: 5
+    volumes:
+      - shared-scratch:/scratch
+
+  frontend:
+    image: slurm-reframe
+    container_name: frontend
+    build:
+      dockerfile: .github/slurm-cluster/reframe/Dockerfile
+      context: ../../
+    hostname: login
+    user: admin
+    init: True
+    volumes:
+      - shared-home:/home/admin:rw
+      - shared-scratch:/scratch:rw
+    links:
+      - slurm-master
+    depends_on:
+      munge-key-generator:
+        condition: service_healthy
+      slurm-master:
+        condition: service_started
+      node0:
+        condition: service_started
+      node1:
+        condition: service_started
+      node2:
+        condition: service_started
+    environment:
+      - SLURM_CPUS_ON_NODE=1
+
+  slurm-master:
+    image: ghcr.io/reframe-hpc/slurm-master-ubuntu:20.04
+    hostname: slurm-master
+    user: admin
+    volumes:
+      - shared-home:/home/admin
+      - shared-scratch:/scratch:rw
+    depends_on:
+      munge-key-generator:
+          condition: service_healthy
+    environment:
+      - SLURM_CPUS_ON_NODE=1
+
+  node0:
+    image: ghcr.io/reframe-hpc/slurm-node-ubuntu:20.04
+    hostname: nid00
+    container_name: slurm-node0
+    user: admin
+    volumes:
+      - shared-home:/home/admin
+      - shared-scratch:/scratch:rw
+    environment:
+      - SLURM_NODENAME=nid00
+      - SLURM_CPUS_ON_NODE=1
+    depends_on:
+      munge-key-generator:
+          condition: service_healthy
+      slurm-master:
+          condition: service_started
+    links:
+      - slurm-master
+
+  node1:
+    image: ghcr.io/reframe-hpc/slurm-node-ubuntu:20.04
+    hostname: nid01
+    container_name: slurm-node1
+    user: admin
+    volumes:
+      - shared-home:/home/admin
+      - shared-scratch:/scratch:rw
+    environment:
+      - SLURM_NODENAME=nid01
+      - SLURM_CPUS_ON_NODE=1
+    depends_on:
+      munge-key-generator:
+          condition: service_healthy
+      slurm-master:
+          condition: service_started
+    links:
+      - slurm-master
+
+  node2:
+    image: ghcr.io/reframe-hpc/slurm-node-ubuntu:20.04
+    hostname: nid02
+    container_name: slurm-node2
+    user: admin
+    volumes:
+      - shared-home:/home/admin
+      - shared-scratch:/scratch:rw
+    environment:
+      - SLURM_NODENAME=nid02
+      - SLURM_CPUS_ON_NODE=1
+    depends_on:
+      munge-key-generator:
+          condition: service_healthy
+      slurm-master:
+          condition: service_started
+    links:
+      - slurm-master
+
+volumes:
+  shared-home:
+  shared-scratch:
diff --git a/.github/slurm-cluster/reframe/Dockerfile b/.github/slurm-cluster/reframe/Dockerfile
new file mode 100644
index 0000000000..53fc075d44
--- /dev/null
+++ b/.github/slurm-cluster/reframe/Dockerfile
@@ -0,0 +1,42 @@
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt update -y && \
+    apt install -y \
+    build-essential \
+    clang jq libomp-dev tree vim \
+    git \
+    mariadb-client \
+    munge \
+    slurm-client \
+    slurm-wlm-torque \
+    sudo \
+    python3 \
+    python3-pip \
+    wget \
+    curl \
+    mpich \
+    libmpich-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN useradd -m admin -s /usr/bin/bash -d /home/admin && \
+    echo "admin:admin" | chpasswd && adduser admin sudo && \
+    echo "admin     ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+COPY .github/slurm-cluster/reframe/slurm.conf /etc/slurm-llnl/
+COPY .github/slurm-cluster/reframe/cgroup.conf /etc/slurm-llnl/
+COPY .github/slurm-cluster/reframe/docker-entrypoint.sh /etc/slurm-llnl/
+COPY . /usr/local/share/reframe
+
+RUN mkdir /scratch && \
+    chown -R admin:admin /scratch
+
+RUN chmod +rx /etc/slurm-llnl/docker-entrypoint.sh
+
+WORKDIR /home/admin
+
+ENV USER admin
+ENV SHELL bash
+
+ENTRYPOINT ["/etc/slurm-llnl/docker-entrypoint.sh"]
diff --git a/.github/slurm-cluster/reframe/cgroup.conf b/.github/slurm-cluster/reframe/cgroup.conf
new file mode 100644
index 0000000000..0b634635a4
--- /dev/null
+++ b/.github/slurm-cluster/reframe/cgroup.conf
@@ -0,0 +1,6 @@
+CgroupAutomount=yes
+CgroupReleaseAgentDir="/etc/slurm/cgroup"
+ConstrainCores=yes
+ConstrainDevices=yes
+ConstrainRAMSpace=yes
+
diff --git a/.github/slurm-cluster/reframe/docker-entrypoint.sh b/.github/slurm-cluster/reframe/docker-entrypoint.sh
new file mode 100755
index 0000000000..48f9a9452c
--- /dev/null
+++ b/.github/slurm-cluster/reframe/docker-entrypoint.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+trap exit 0 INT
+
+while [ ! -f /scratch/munge.key ]
+do
+  sleep 1
+done
+
+sudo cp /scratch/munge.key /etc/munge/munge.key
+sudo service munge start
+sudo sed -i "s/REPLACE_IT/CPUs=${SLURM_CPUS_ON_NODE}/g" /etc/slurm-llnl/slurm.conf
+
+# Needs to be copied in the shared home directory
+cp -r /usr/local/share/reframe .
+cd reframe
+./bootstrap.sh
+
+tempdir=$(mktemp -d -p /scratch)
+TMPDIR=$tempdir ./test_reframe.py -v --rfm-user-config=examples/tutorial/config/cluster.py
diff --git a/.github/slurm-cluster/reframe/slurm.conf b/.github/slurm-cluster/reframe/slurm.conf
new file mode 100644
index 0000000000..9e2c782156
--- /dev/null
+++ b/.github/slurm-cluster/reframe/slurm.conf
@@ -0,0 +1,153 @@
+# slurm.conf file generated by configurator.html.
+# Put this file on all nodes of your cluster.
+# See the slurm.conf man page for more information.
+#
+SlurmctldHost=slurm-master
+#
+#DisableRootJobs=NO
+#EnforcePartLimits=NO
+#Epilog=
+#EpilogSlurmctld=
+#FirstJobId=1
+#MaxJobId=999999
+#GresTypes=
+#GroupUpdateForce=0
+#GroupUpdateTime=600
+#JobFileAppend=0
+#JobRequeue=1
+#JobSubmitPlugins=1
+#KillOnBadExit=0
+#LaunchType=launch/slurm
+#Licenses=foo*4,bar
+#MailProg=/bin/mail
+#MaxJobCount=5000
+#MaxStepCount=40000
+#MaxTasksPerNode=128
+MpiDefault=pmi2
+#MpiParams=ports=#-#
+#PluginDir=
+#PlugStackConfig=
+#PrivateData=jobs
+#ProctrackType=proctrack/cgroup
+ProctrackType=proctrack/linuxproc
+#Prolog=
+#PrologFlags=
+#PrologSlurmctld=
+#PropagatePrioProcess=0
+#PropagateResourceLimits=
+#PropagateResourceLimitsExcept=
+#RebootProgram=
+ReturnToService=1
+#SallocDefaultCommand=
+SlurmctldPidFile=/var/run/slurmctld.pid
+SlurmctldPort=6817
+SlurmdPidFile=/var/run/slurmd.pid
+SlurmdPort=6818
+SlurmdSpoolDir=/var/spool/slurmd
+SlurmUser=root
+#SlurmdUser=root
+#SrunEpilog=
+#SrunProlog=
+StateSaveLocation=/var/spool
+SwitchType=switch/none
+#TaskEpilog=
+TaskPlugin=task/affinity
+TaskPluginParam=Sched
+#TaskProlog=
+#TopologyPlugin=topology/tree
+#TmpFS=/tmp
+#TrackWCKey=no
+#TreeWidth=
+#UnkillableStepProgram=
+#UsePAM=0
+#
+#
+# TIMERS
+#BatchStartTimeout=10
+#CompleteWait=0
+#EpilogMsgTime=2000
+#GetEnvTimeout=2
+#HealthCheckInterval=0
+#HealthCheckProgram=
+InactiveLimit=0
+KillWait=30
+#MessageTimeout=10
+#ResvOverRun=0
+MinJobAge=300
+#OverTimeLimit=0
+SlurmctldTimeout=120
+SlurmdTimeout=300
+#UnkillableStepTimeout=60
+#VSizeFactor=0
+Waittime=0
+#
+#
+# SCHEDULING
+#DefMemPerCPU=0
+#MaxMemPerCPU=0
+#SchedulerTimeSlice=30
+SchedulerType=sched/backfill
+SelectType=select/cons_res
+SelectTypeParameters=CR_Core
+#
+#
+# JOB PRIORITY
+#PriorityFlags=
+#PriorityType=priority/basic
+#PriorityDecayHalfLife=
+#PriorityCalcPeriod=
+#PriorityFavorSmall=
+#PriorityMaxAge=
+#PriorityUsageResetPeriod=
+#PriorityWeightAge=
+#PriorityWeightFairshare=
+#PriorityWeightJobSize=
+#PriorityWeightPartition=
+#PriorityWeightQOS=
+#
+#
+# LOGGING AND ACCOUNTING
+#AccountingStorageEnforce=0
+#AccountingStorageHost=
+#AccountingStorageLoc=
+#AccountingStoragePass=
+#AccountingStoragePort=
+AccountingStorageType=accounting_storage/none
+#AccountingStorageUser=
+AccountingStoreJobComment=YES
+ClusterName=cluster
+#DebugFlags=
+#JobCompHost=
+#JobCompLoc=
+#JobCompPass=
+#JobCompPort=
+JobCompType=jobcomp/none
+#JobCompUser=
+#JobContainerType=job_container/none
+JobAcctGatherFrequency=30
+JobAcctGatherType=jobacct_gather/none
+SlurmctldDebug=error
+SlurmctldLogFile=/var/log/slurm-llnl/slurmctld.log
+SlurmdDebug=error
+SlurmdLogFile=/var/log/slurm-llnl/slurmd.log
+#SlurmSchedLogFile=
+#SlurmSchedLogLevel=
+#
+#
+# POWER SAVE SUPPORT FOR IDLE NODES (optional)
+#SuspendProgram=
+#ResumeProgram=
+#SuspendTimeout=
+#ResumeTimeout=
+#ResumeRate=
+#SuspendExcNodes=
+#SuspendExcParts=
+#SuspendRate=
+#SuspendTime=
+#
+#
+# COMPUTE NODES
+#
+NodeName=nid0[0-2] REPLACE_IT State=UNKNOWN
+PartitionName=all Nodes=ALL Default=YES MaxTime=INFINITE State=UP
+
diff --git a/.github/workflows/test-schedulers.yaml b/.github/workflows/test-schedulers.yaml
new file mode 100644
index 0000000000..585e8f2be9
--- /dev/null
+++ b/.github/workflows/test-schedulers.yaml
@@ -0,0 +1,21 @@
+name: Test Schedulers
+on:
+  pull_request: []
+
+jobs:
+  squeue-test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Build Images
+      run: |
+        docker compose -f ./github/slurm-cluster/docker-compose.yaml build
+    - name: Run Unittests
+      run: |
+        docker compose -f ./github/slurm-cluster/docker-compose.yml up --abort-on-container-exit --exit-code-from frontend

From 0007b2dcd2b5a1e9fa249851df03be0c4736251d Mon Sep 17 00:00:00 2001
From: Theofilos Manitaras <manitaras@cscs.ch>
Date: Tue, 24 Sep 2024 15:58:58 +0200
Subject: [PATCH 117/128] Fix path to yaml file

Signed-off-by: Theofilos Manitaras <manitaras@cscs.ch>
---
 .github/workflows/test-schedulers.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-schedulers.yaml b/.github/workflows/test-schedulers.yaml
index 585e8f2be9..5038d48582 100644
--- a/.github/workflows/test-schedulers.yaml
+++ b/.github/workflows/test-schedulers.yaml
@@ -15,7 +15,7 @@ jobs:
         password: ${{ secrets.GITHUB_TOKEN }}
     - name: Build Images
       run: |
-        docker compose -f ./github/slurm-cluster/docker-compose.yaml build
+        docker compose -f .github/slurm-cluster/docker-compose.yaml build
     - name: Run Unittests
       run: |
-        docker compose -f ./github/slurm-cluster/docker-compose.yml up --abort-on-container-exit --exit-code-from frontend
+        docker compose -f .github/slurm-cluster/docker-compose.yml up --abort-on-container-exit --exit-code-from frontend

From ff2eea3fb35154fe71126da2bb7dd00065a7392b Mon Sep 17 00:00:00 2001
From: Theofilos Manitaras <manitaras@cscs.ch>
Date: Tue, 24 Sep 2024 16:03:47 +0200
Subject: [PATCH 118/128] Fix .yaml file suffix

---
 .github/workflows/test-schedulers.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-schedulers.yaml b/.github/workflows/test-schedulers.yaml
index 5038d48582..4884073a17 100644
--- a/.github/workflows/test-schedulers.yaml
+++ b/.github/workflows/test-schedulers.yaml
@@ -15,7 +15,7 @@ jobs:
         password: ${{ secrets.GITHUB_TOKEN }}
     - name: Build Images
       run: |
-        docker compose -f .github/slurm-cluster/docker-compose.yaml build
+        docker compose -f .github/slurm-cluster/docker-compose.yml build
     - name: Run Unittests
       run: |
         docker compose -f .github/slurm-cluster/docker-compose.yml up --abort-on-container-exit --exit-code-from frontend

From d9fd79b5d077dead08aea8c9bcb86c185d2af81c Mon Sep 17 00:00:00 2001
From: Theofilos Manitaras <manitaras@cscs.ch>
Date: Wed, 25 Sep 2024 11:10:16 +0200
Subject: [PATCH 119/128] Run unittests with multiple schedulers

Signed-off-by: Theofilos Manitaras <manitaras@cscs.ch>
---
 .../docker-compose.yml                        |  3 +-
 .../reframe/Dockerfile                        |  6 +-
 .../reframe/cgroup.conf                       |  0
 .../reframe/docker-entrypoint.sh              |  4 +-
 .../reframe/slurm.conf                        |  0
 .github/workflows/test-schedulers.yaml        | 11 ++--
 config/ci-cluster.py                          | 66 +++++++++++++++++++
 7 files changed, 81 insertions(+), 9 deletions(-)
 rename .github/{slurm-cluster => pseudo-cluster}/docker-compose.yml (96%)
 rename .github/{slurm-cluster => pseudo-cluster}/reframe/Dockerfile (79%)
 rename .github/{slurm-cluster => pseudo-cluster}/reframe/cgroup.conf (100%)
 rename .github/{slurm-cluster => pseudo-cluster}/reframe/docker-entrypoint.sh (72%)
 rename .github/{slurm-cluster => pseudo-cluster}/reframe/slurm.conf (100%)
 create mode 100644 config/ci-cluster.py

diff --git a/.github/slurm-cluster/docker-compose.yml b/.github/pseudo-cluster/docker-compose.yml
similarity index 96%
rename from .github/slurm-cluster/docker-compose.yml
rename to .github/pseudo-cluster/docker-compose.yml
index 8290a74ec9..a6adb767f4 100644
--- a/.github/slurm-cluster/docker-compose.yml
+++ b/.github/pseudo-cluster/docker-compose.yml
@@ -14,7 +14,7 @@ services:
     image: slurm-reframe
     container_name: frontend
     build:
-      dockerfile: .github/slurm-cluster/reframe/Dockerfile
+      dockerfile: .github/pseudo-cluster/reframe/Dockerfile
       context: ../../
     hostname: login
     user: admin
@@ -37,6 +37,7 @@ services:
         condition: service_started
     environment:
       - SLURM_CPUS_ON_NODE=1
+      - BACKEND=${BACKEND:-squeue}
 
   slurm-master:
     image: ghcr.io/reframe-hpc/slurm-master-ubuntu:20.04
diff --git a/.github/slurm-cluster/reframe/Dockerfile b/.github/pseudo-cluster/reframe/Dockerfile
similarity index 79%
rename from .github/slurm-cluster/reframe/Dockerfile
rename to .github/pseudo-cluster/reframe/Dockerfile
index 53fc075d44..afeac49f98 100644
--- a/.github/slurm-cluster/reframe/Dockerfile
+++ b/.github/pseudo-cluster/reframe/Dockerfile
@@ -24,9 +24,9 @@ RUN useradd -m admin -s /usr/bin/bash -d /home/admin && \
     echo "admin:admin" | chpasswd && adduser admin sudo && \
     echo "admin     ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 
-COPY .github/slurm-cluster/reframe/slurm.conf /etc/slurm-llnl/
-COPY .github/slurm-cluster/reframe/cgroup.conf /etc/slurm-llnl/
-COPY .github/slurm-cluster/reframe/docker-entrypoint.sh /etc/slurm-llnl/
+COPY .github/pseudo-cluster/reframe/slurm.conf /etc/slurm-llnl/
+COPY .github/pseudo-cluster/reframe/cgroup.conf /etc/slurm-llnl/
+COPY .github/pseudo-cluster/reframe/docker-entrypoint.sh /etc/slurm-llnl/
 COPY . /usr/local/share/reframe
 
 RUN mkdir /scratch && \
diff --git a/.github/slurm-cluster/reframe/cgroup.conf b/.github/pseudo-cluster/reframe/cgroup.conf
similarity index 100%
rename from .github/slurm-cluster/reframe/cgroup.conf
rename to .github/pseudo-cluster/reframe/cgroup.conf
diff --git a/.github/slurm-cluster/reframe/docker-entrypoint.sh b/.github/pseudo-cluster/reframe/docker-entrypoint.sh
similarity index 72%
rename from .github/slurm-cluster/reframe/docker-entrypoint.sh
rename to .github/pseudo-cluster/reframe/docker-entrypoint.sh
index 48f9a9452c..9509ccb508 100755
--- a/.github/slurm-cluster/reframe/docker-entrypoint.sh
+++ b/.github/pseudo-cluster/reframe/docker-entrypoint.sh
@@ -17,4 +17,6 @@ cd reframe
 ./bootstrap.sh
 
 tempdir=$(mktemp -d -p /scratch)
-TMPDIR=$tempdir ./test_reframe.py -v --rfm-user-config=examples/tutorial/config/cluster.py
+TMPDIR=$tempdir ./test_reframe.py -v \
+    --rfm-user-config=config/ci-cluster.py \
+    --rfm-user-system=pseudo-cluster:compute-${BACKEND:-squeue}
diff --git a/.github/slurm-cluster/reframe/slurm.conf b/.github/pseudo-cluster/reframe/slurm.conf
similarity index 100%
rename from .github/slurm-cluster/reframe/slurm.conf
rename to .github/pseudo-cluster/reframe/slurm.conf
diff --git a/.github/workflows/test-schedulers.yaml b/.github/workflows/test-schedulers.yaml
index 4884073a17..5b35709f97 100644
--- a/.github/workflows/test-schedulers.yaml
+++ b/.github/workflows/test-schedulers.yaml
@@ -3,8 +3,11 @@ on:
   pull_request: []
 
 jobs:
-  squeue-test:
+  scheduler-test:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        scheduler: ['squeue', 'torque']
     steps:
     - uses: actions/checkout@v4
     - name: Login to GitHub Container Registry
@@ -15,7 +18,7 @@ jobs:
         password: ${{ secrets.GITHUB_TOKEN }}
     - name: Build Images
       run: |
-        docker compose -f .github/slurm-cluster/docker-compose.yml build
-    - name: Run Unittests
+        docker compose -f .github/pseudo-cluster/docker-compose.yml build
+    - name: Run Unittests with ${{ matrix.scheduler }} sceduler
       run: |
-        docker compose -f .github/slurm-cluster/docker-compose.yml up --abort-on-container-exit --exit-code-from frontend
+        BACKEND=${{ matrix.scheduler }}docker compose -f .github/pseudo-cluster/docker-compose.yml up --abort-on-container-exit --exit-code-from frontend
diff --git a/config/ci-cluster.py b/config/ci-cluster.py
new file mode 100644
index 0000000000..32487114ad
--- /dev/null
+++ b/config/ci-cluster.py
@@ -0,0 +1,66 @@
+# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# ReFrame Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+site_configuration = {
+    'systems': [
+        {
+            'name': 'pseudo-cluster',
+            'descr': 'CI Slurm-based pseudo cluster',
+            'hostnames': ['login'],
+            'partitions': [
+                {
+                    'name': 'login',
+                    'descr': 'Login nodes',
+                    'scheduler': 'local',
+                    'launcher': 'local',
+                    'environs': ['gnu', 'clang']
+                },
+                {
+                    'name': 'compute-squeue',
+                    'descr': 'Squeue compute nodes',
+                    'scheduler': 'squeue',
+                    'launcher': 'srun',
+                    'access': ['-p all'],
+                    'environs': ['gnu', 'clang']
+                },
+                {
+                    'name': 'compute-torque',
+                    'descr': 'Torque compute nodes',
+                    'scheduler': 'squeue',
+                    'launcher': 'mpiexec',
+                    'access': ['-p all'],
+                    'environs': ['gnu', 'clang']
+                }
+
+            ]
+        },
+    ],
+    'environments': [
+        {
+            'name': 'baseline',
+            'features': ['stream']
+        },
+        {
+            'name': 'gnu',
+            'cc': 'gcc',
+            'cxx': 'g++',
+            'features': ['openmp'],
+            'extras': {'omp_flag': '-fopenmp'}
+        },
+        {
+            'name': 'clang',
+            'cc': 'clang',
+            'cxx': 'clang++',
+            'features': ['openmp'],
+            'extras': {'omp_flag': '-fopenmp'}
+        }
+    ],
+    'modes': [
+        {
+            'name': 'singlethread',
+            'options': ['-E num_threads==1']
+        }
+    ]
+}

From 297d2e679622d30fd2cc57e4b1b72292dba0aec0 Mon Sep 17 00:00:00 2001
From: Theofilos Manitaras <manitaras@cscs.ch>
Date: Wed, 25 Sep 2024 11:13:50 +0200
Subject: [PATCH 120/128] Add missing space

Signed-off-by: Theofilos Manitaras <manitaras@cscs.ch>
---
 .github/workflows/test-schedulers.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-schedulers.yaml b/.github/workflows/test-schedulers.yaml
index 5b35709f97..e496179547 100644
--- a/.github/workflows/test-schedulers.yaml
+++ b/.github/workflows/test-schedulers.yaml
@@ -21,4 +21,4 @@ jobs:
         docker compose -f .github/pseudo-cluster/docker-compose.yml build
     - name: Run Unittests with ${{ matrix.scheduler }} sceduler
       run: |
-        BACKEND=${{ matrix.scheduler }}docker compose -f .github/pseudo-cluster/docker-compose.yml up --abort-on-container-exit --exit-code-from frontend
+        BACKEND=${{ matrix.scheduler }} docker compose -f .github/pseudo-cluster/docker-compose.yml up --abort-on-container-exit --exit-code-from frontend

From 95baa8a37f2a54416ae48229092092e135dc0c8c Mon Sep 17 00:00:00 2001
From: Theofilos Manitaras <manitaras@cscs.ch>
Date: Wed, 25 Sep 2024 11:39:41 +0200
Subject: [PATCH 121/128] Add pbs scheduler testing

Signed-off-by: Theofilos Manitaras <manitaras@cscs.ch>
---
 .github/pseudo-cluster/reframe/docker-entrypoint.sh |  2 ++
 .github/workflows/test-schedulers.yaml              |  2 +-
 config/ci-cluster.py                                | 11 +++++++++--
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/.github/pseudo-cluster/reframe/docker-entrypoint.sh b/.github/pseudo-cluster/reframe/docker-entrypoint.sh
index 9509ccb508..c49b721388 100755
--- a/.github/pseudo-cluster/reframe/docker-entrypoint.sh
+++ b/.github/pseudo-cluster/reframe/docker-entrypoint.sh
@@ -16,6 +16,8 @@ cp -r /usr/local/share/reframe .
 cd reframe
 ./bootstrap.sh
 
+echo "Running unittests with backend scheduler: ${BACKEND}"
+
 tempdir=$(mktemp -d -p /scratch)
 TMPDIR=$tempdir ./test_reframe.py -v \
     --rfm-user-config=config/ci-cluster.py \
diff --git a/.github/workflows/test-schedulers.yaml b/.github/workflows/test-schedulers.yaml
index e496179547..13613eb5b0 100644
--- a/.github/workflows/test-schedulers.yaml
+++ b/.github/workflows/test-schedulers.yaml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        scheduler: ['squeue', 'torque']
+        scheduler: ['pbs', 'squeue', 'torque']
     steps:
     - uses: actions/checkout@v4
     - name: Login to GitHub Container Registry
diff --git a/config/ci-cluster.py b/config/ci-cluster.py
index 32487114ad..3d30203fbd 100644
--- a/config/ci-cluster.py
+++ b/config/ci-cluster.py
@@ -28,12 +28,19 @@
                 {
                     'name': 'compute-torque',
                     'descr': 'Torque compute nodes',
-                    'scheduler': 'squeue',
+                    'scheduler': 'torque',
+                    'launcher': 'mpiexec',
+                    'access': ['-p all'],
+                    'environs': ['gnu', 'clang']
+                },
+                {
+                    'name': 'compute-pbs',
+                    'descr': 'PBS compute nodes',
+                    'scheduler': 'pbs',
                     'launcher': 'mpiexec',
                     'access': ['-p all'],
                     'environs': ['gnu', 'clang']
                 }
-
             ]
         },
     ],

From 45541fb2da9fcc66e18b077f6e1a612743376835 Mon Sep 17 00:00:00 2001
From: Theofilos Manitaras <manitaras@cscs.ch>
Date: Thu, 26 Sep 2024 12:02:14 +0200
Subject: [PATCH 122/128] Address PR comments

Signed-off-by: Theofilos Manitaras <manitaras@cscs.ch>
---
 .github/pseudo-cluster/reframe/cgroup.conf    |   7 +-
 .../reframe/docker-entrypoint.sh              |   2 +-
 .github/pseudo-cluster/reframe/slurm.conf     | 154 +---------------
 .github/workflows/test-schedulers.yaml        |   2 +-
 Jenkinsfile                                   | 169 ------------------
 ci-scripts/ci-runner.bash                     |   4 +-
 {config => ci-scripts/configs}/ci-cluster.py  |   0
 {config => ci-scripts/configs}/cscs-ci.py     |   0
 8 files changed, 6 insertions(+), 332 deletions(-)
 mode change 100644 => 120000 .github/pseudo-cluster/reframe/cgroup.conf
 mode change 100644 => 120000 .github/pseudo-cluster/reframe/slurm.conf
 delete mode 100644 Jenkinsfile
 rename {config => ci-scripts/configs}/ci-cluster.py (100%)
 rename {config => ci-scripts/configs}/cscs-ci.py (100%)

diff --git a/.github/pseudo-cluster/reframe/cgroup.conf b/.github/pseudo-cluster/reframe/cgroup.conf
deleted file mode 100644
index 0b634635a4..0000000000
--- a/.github/pseudo-cluster/reframe/cgroup.conf
+++ /dev/null
@@ -1,6 +0,0 @@
-CgroupAutomount=yes
-CgroupReleaseAgentDir="/etc/slurm/cgroup"
-ConstrainCores=yes
-ConstrainDevices=yes
-ConstrainRAMSpace=yes
-
diff --git a/.github/pseudo-cluster/reframe/cgroup.conf b/.github/pseudo-cluster/reframe/cgroup.conf
new file mode 120000
index 0000000000..c99f23956f
--- /dev/null
+++ b/.github/pseudo-cluster/reframe/cgroup.conf
@@ -0,0 +1 @@
+../../../examples/tutorial/dockerfiles/slurm-cluster/reframe/cgroup.conf
\ No newline at end of file
diff --git a/.github/pseudo-cluster/reframe/docker-entrypoint.sh b/.github/pseudo-cluster/reframe/docker-entrypoint.sh
index c49b721388..665f23ff8d 100755
--- a/.github/pseudo-cluster/reframe/docker-entrypoint.sh
+++ b/.github/pseudo-cluster/reframe/docker-entrypoint.sh
@@ -20,5 +20,5 @@ echo "Running unittests with backend scheduler: ${BACKEND}"
 
 tempdir=$(mktemp -d -p /scratch)
 TMPDIR=$tempdir ./test_reframe.py -v \
-    --rfm-user-config=config/ci-cluster.py \
+    --rfm-user-config=ci-scripts/configs/ci-cluster.py \
     --rfm-user-system=pseudo-cluster:compute-${BACKEND:-squeue}
diff --git a/.github/pseudo-cluster/reframe/slurm.conf b/.github/pseudo-cluster/reframe/slurm.conf
deleted file mode 100644
index 9e2c782156..0000000000
--- a/.github/pseudo-cluster/reframe/slurm.conf
+++ /dev/null
@@ -1,153 +0,0 @@
-# slurm.conf file generated by configurator.html.
-# Put this file on all nodes of your cluster.
-# See the slurm.conf man page for more information.
-#
-SlurmctldHost=slurm-master
-#
-#DisableRootJobs=NO
-#EnforcePartLimits=NO
-#Epilog=
-#EpilogSlurmctld=
-#FirstJobId=1
-#MaxJobId=999999
-#GresTypes=
-#GroupUpdateForce=0
-#GroupUpdateTime=600
-#JobFileAppend=0
-#JobRequeue=1
-#JobSubmitPlugins=1
-#KillOnBadExit=0
-#LaunchType=launch/slurm
-#Licenses=foo*4,bar
-#MailProg=/bin/mail
-#MaxJobCount=5000
-#MaxStepCount=40000
-#MaxTasksPerNode=128
-MpiDefault=pmi2
-#MpiParams=ports=#-#
-#PluginDir=
-#PlugStackConfig=
-#PrivateData=jobs
-#ProctrackType=proctrack/cgroup
-ProctrackType=proctrack/linuxproc
-#Prolog=
-#PrologFlags=
-#PrologSlurmctld=
-#PropagatePrioProcess=0
-#PropagateResourceLimits=
-#PropagateResourceLimitsExcept=
-#RebootProgram=
-ReturnToService=1
-#SallocDefaultCommand=
-SlurmctldPidFile=/var/run/slurmctld.pid
-SlurmctldPort=6817
-SlurmdPidFile=/var/run/slurmd.pid
-SlurmdPort=6818
-SlurmdSpoolDir=/var/spool/slurmd
-SlurmUser=root
-#SlurmdUser=root
-#SrunEpilog=
-#SrunProlog=
-StateSaveLocation=/var/spool
-SwitchType=switch/none
-#TaskEpilog=
-TaskPlugin=task/affinity
-TaskPluginParam=Sched
-#TaskProlog=
-#TopologyPlugin=topology/tree
-#TmpFS=/tmp
-#TrackWCKey=no
-#TreeWidth=
-#UnkillableStepProgram=
-#UsePAM=0
-#
-#
-# TIMERS
-#BatchStartTimeout=10
-#CompleteWait=0
-#EpilogMsgTime=2000
-#GetEnvTimeout=2
-#HealthCheckInterval=0
-#HealthCheckProgram=
-InactiveLimit=0
-KillWait=30
-#MessageTimeout=10
-#ResvOverRun=0
-MinJobAge=300
-#OverTimeLimit=0
-SlurmctldTimeout=120
-SlurmdTimeout=300
-#UnkillableStepTimeout=60
-#VSizeFactor=0
-Waittime=0
-#
-#
-# SCHEDULING
-#DefMemPerCPU=0
-#MaxMemPerCPU=0
-#SchedulerTimeSlice=30
-SchedulerType=sched/backfill
-SelectType=select/cons_res
-SelectTypeParameters=CR_Core
-#
-#
-# JOB PRIORITY
-#PriorityFlags=
-#PriorityType=priority/basic
-#PriorityDecayHalfLife=
-#PriorityCalcPeriod=
-#PriorityFavorSmall=
-#PriorityMaxAge=
-#PriorityUsageResetPeriod=
-#PriorityWeightAge=
-#PriorityWeightFairshare=
-#PriorityWeightJobSize=
-#PriorityWeightPartition=
-#PriorityWeightQOS=
-#
-#
-# LOGGING AND ACCOUNTING
-#AccountingStorageEnforce=0
-#AccountingStorageHost=
-#AccountingStorageLoc=
-#AccountingStoragePass=
-#AccountingStoragePort=
-AccountingStorageType=accounting_storage/none
-#AccountingStorageUser=
-AccountingStoreJobComment=YES
-ClusterName=cluster
-#DebugFlags=
-#JobCompHost=
-#JobCompLoc=
-#JobCompPass=
-#JobCompPort=
-JobCompType=jobcomp/none
-#JobCompUser=
-#JobContainerType=job_container/none
-JobAcctGatherFrequency=30
-JobAcctGatherType=jobacct_gather/none
-SlurmctldDebug=error
-SlurmctldLogFile=/var/log/slurm-llnl/slurmctld.log
-SlurmdDebug=error
-SlurmdLogFile=/var/log/slurm-llnl/slurmd.log
-#SlurmSchedLogFile=
-#SlurmSchedLogLevel=
-#
-#
-# POWER SAVE SUPPORT FOR IDLE NODES (optional)
-#SuspendProgram=
-#ResumeProgram=
-#SuspendTimeout=
-#ResumeTimeout=
-#ResumeRate=
-#SuspendExcNodes=
-#SuspendExcParts=
-#SuspendRate=
-#SuspendTime=
-#
-#
-# COMPUTE NODES
-#
-NodeName=nid0[0-2] REPLACE_IT State=UNKNOWN
-PartitionName=all Nodes=ALL Default=YES MaxTime=INFINITE State=UP
-
diff --git a/.github/pseudo-cluster/reframe/slurm.conf b/.github/pseudo-cluster/reframe/slurm.conf
new file mode 120000
index 0000000000..39eb5d7b6f
--- /dev/null
+++ b/.github/pseudo-cluster/reframe/slurm.conf
@@ -0,0 +1 @@
+../../../examples/tutorial/dockerfiles/slurm-cluster/reframe/slurm.conf
\ No newline at end of file
diff --git a/.github/workflows/test-schedulers.yaml b/.github/workflows/test-schedulers.yaml
index 13613eb5b0..83285e0988 100644
--- a/.github/workflows/test-schedulers.yaml
+++ b/.github/workflows/test-schedulers.yaml
@@ -1,4 +1,4 @@
-name: Test Schedulers
+name: ReFrame CI / Scheduler backend tests
 on:
   pull_request: []
 
diff --git a/Jenkinsfile b/Jenkinsfile
deleted file mode 100644
index 1553a9b1b1..0000000000
--- a/Jenkinsfile
+++ /dev/null
@@ -1,169 +0,0 @@
-#!/usr/bin/env groovy
-
-def dirPrefix = 'reframe-ci'
-def loginBash = '#!/bin/bash -l'
-def bashScript = 'ci-scripts/ci-runner.bash'
-def machinesList = params.machines.split()
-def machinesToRun = machinesList
-def runTests = true
-def uniqueID
-
-stage('Initialization') {
-    node('master') {
-        catchError(stageResult: 'FAILURE') {
-            uniqueID = "${env.ghprbActualCommit[0..6]}-${env.BUILD_ID}"
-            echo 'Environment Variables:'
-            echo sh(script: 'env|sort', returnStdout: true)
-
-            def githubComment = env.ghprbCommentBody
-            if (githubComment == 'null' || !githubComment.trim().startsWith('@jenkins-cscs')) {
-                machinesToRun = machinesList
-                currentBuild.result = 'SUCCESS'
-                return
-            }
-
-            def splittedComment = githubComment.split()
-            if (splittedComment.size() < 3) {
-                println 'No machines were found. Aborting...'
-                currentBuild.result = 'ABORTED'
-                return
-            }
-            if (splittedComment[1] != 'retry') {
-                println "Invalid command ${splittedComment[1]}. Aborting..."
-                currentBuild.result = 'ABORTED'
-                return
-            }
-            if (splittedComment[2] == 'all') {
-                machinesToRun = machinesList
-                currentBuild.result = 'SUCCESS'
-                return
-            }
-            else if (splittedComment[2] == 'none') {
-                runTests = false
-                currentBuild.result = 'SUCCESS'
-                return
-            }
-
-            machinesRequested = []
-            for (i = 2; i < splittedComment.size(); i++) {
-                machinesRequested.add(splittedComment[i])
-            }
-
-            machinesToRun = machinesRequested.findAll({it in machinesList})
-            if (!machinesToRun) {
-                println 'No machines were found. Aborting...'
-                currentBuild.result = 'ABORTED'
-                return
-            }
-            currentBuild.result = 'SUCCESS'
-        }
-    }
-}
-
-if (!runTests) {
-    println "Won't execute any test (${currentBuild.result}). Exiting..."
-    return
-}
-
-if (currentBuild.result != 'SUCCESS') {
-    println "Initialization failed (${currentBuild.result}). Exiting..."
-    return
-}
-
-def builds = [:]
-stage('Unittest') {
-    for (mach in machinesToRun) {
-        def machineName = mach
-        builds[machineName] = {
-            node(machineName) {
-                def scratch = sh(returnStdout: true,
-                                 script: """${loginBash}
-                                            echo \$SCRATCH""").trim()
-                def reframeDir = "${scratch}/${dirPrefix}-${machineName}-${uniqueID}"
-                dir(reframeDir) {
-                    checkout scm
-                    sh("""${loginBash}
-                          bash ${reframeDir}/${bashScript} -f ${reframeDir} -i ''""")
-                }
-            }
-        }
-    }
-
-    catchError(stageResult: 'FAILURE') {
-        parallel builds
-    }
-}
-
-builds = [:]
-stage('Tutorial Check') {
-    if (currentBuild.result != 'SUCCESS') {
-        println 'Not executing "Tutorial Check" Stage'
-        return
-    }
-    else {
-        catchError(stageResult: 'FAILURE') {
-            if (!('daint' in machinesToRun)) {
-                return
-            }
-            node('daint') {
-                def scratch = sh(returnStdout: true,
-                                 script: """${loginBash}
-                                            echo \$SCRATCH""").trim()
-                def reframeDir = "${scratch}/${dirPrefix}-daint-${uniqueID}"
-                dir(reframeDir) {
-                    sh("""${loginBash}
-                          bash ${reframeDir}/${bashScript} -f ${reframeDir} -i '' -t""")
-                }
-            }
-        }
-    }
-}
-
-builds = [:]
-stage('Cleanup') {
-    if (currentBuild.result != 'SUCCESS') {
-        println 'Not executing "Cleanup" Stage'
-        return
-    }
-    else {
-        for (mach in machinesToRun) {
-            def machineName = mach
-            builds[machineName] = {
-                node(machineName) {
-                    def scratch = sh(returnStdout: true,
-                                     script: """$loginBash
-                                                echo \$SCRATCH""").trim()
-                    def reframeDir = "${scratch}/${dirPrefix}-${machineName}-${uniqueID}"
-                    sh("""${loginBash}
-                          rm -rf ${reframeDir}
-                          date""")
-
-                }
-            }
-        }
-        catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
-            parallel builds
-        }
-    }
-}
-
-def staleCleanupInterval = 3
-builds = [:]
-stage('Cleanup Stale') {
-     for (mach in machinesToRun) {
-        def machineName = mach
-        builds[machineName] = {
-            node(machineName) {
-                def scratch = sh(returnStdout: true,
-                                 script: """${loginBash}
-                                            echo \$SCRATCH""").trim()
-                sh("""${loginBash}
-                      find ${scratch} -maxdepth 1 -name 'reframe-ci*' -ctime +${staleCleanupInterval} -type d -exec printf 'Removing  %s\\n' {} +
-                      find ${scratch} -maxdepth 1 -name 'reframe-ci*' -ctime +${staleCleanupInterval} -type d -exec rm -rf {} +""")
-            }
-        }
-    }
-    catchError(buildResult: 'SUCCESS',  stageResult: 'FAILURE') {
-        parallel builds
-    }
-}
diff --git a/ci-scripts/ci-runner.bash b/ci-scripts/ci-runner.bash
index 50130481b9..08e0062c9d 100644
--- a/ci-scripts/ci-runner.bash
+++ b/ci-scripts/ci-runner.bash
@@ -170,7 +170,7 @@ else
         for backend in slurm pbs torque; do
             echo "[INFO] Running unit tests with ${backend}"
             TMPDIR=$tempdir checked_exec ./test_reframe.py ${parallel_opts} \
-                         --rfm-user-config=config/cscs-ci.py \
+                         --rfm-user-config=ci-scripts/configs/cscs-ci.py \
                          -W=error::reframe.core.warnings.ReframeDeprecationWarning \
                          --rfm-user-system=dom:${backend} -ra
         done
@@ -178,7 +178,7 @@ else
     else
         echo "[INFO] Running unit tests"
         TMPDIR=$tempdir checked_exec ./test_reframe.py ${parallel_opts} \
-                     --rfm-user-config=config/cscs-ci.py \
+                     --rfm-user-config=ci-scripts/configs/cscs-ci.py \
                      -W=error::reframe.core.warnings.ReframeDeprecationWarning -ra
     fi
 
diff --git a/config/ci-cluster.py b/ci-scripts/configs/ci-cluster.py
similarity index 100%
rename from config/ci-cluster.py
rename to ci-scripts/configs/ci-cluster.py
diff --git a/config/cscs-ci.py b/ci-scripts/configs/cscs-ci.py
similarity index 100%
rename from config/cscs-ci.py
rename to ci-scripts/configs/cscs-ci.py

From ea6fbdc3c043c543c5c0df8644349ce531ae09ef Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Thu, 26 Sep 2024 14:40:22 +0200
Subject: [PATCH 123/128] Bump patch level

---
 reframe/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe/__init__.py b/reframe/__init__.py
index 1f7a3b4d10..5bc08e7aae 100644
--- a/reframe/__init__.py
+++ b/reframe/__init__.py
@@ -6,7 +6,7 @@
 import os
 import sys
 
-VERSION = '4.6.2'
+VERSION = '4.6.3'
 INSTALL_PREFIX = os.path.normpath(
     os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 )

From e2d6a6c9a54dba88b5ebfb3bcef22d37ab64e980 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Thu, 26 Sep 2024 15:14:49 +0200
Subject: [PATCH 124/128] Bump dev version

---
 reframe/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe/__init__.py b/reframe/__init__.py
index b935398b41..5c108f4362 100644
--- a/reframe/__init__.py
+++ b/reframe/__init__.py
@@ -6,7 +6,7 @@
 import os
 import sys
 
-VERSION = '4.7.0-dev.7'
+VERSION = '4.7.0-dev.8'
 INSTALL_PREFIX = os.path.normpath(
     os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 )

From d1ca38618040317467e410a004251e37b496dd07 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 1 Oct 2024 15:40:00 +0000
Subject: [PATCH 125/128] Bump setuptools from 59.6.0 to 75.1.0

Bumps [setuptools](https://github.com/pypa/setuptools) from 59.6.0 to 75.1.0.
- [Release notes](https://github.com/pypa/setuptools/releases)
- [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst)
- [Commits](https://github.com/pypa/setuptools/compare/v59.6.0...v75.1.0)

---
updated-dependencies:
- dependency-name: setuptools
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 08e952b9ef..9df75f6696 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -25,7 +25,7 @@ semver==2.13.0; python_version == '3.6'
 semver==3.0.2; python_version >= '3.7'
 setuptools==59.6.0; python_version == '3.6'
 setuptools==68.0.0; python_version == '3.7'
-setuptools==74.0.0; python_version >= '3.8'
+setuptools==75.1.0; python_version >= '3.8'
 tabulate==0.8.10; python_version == '3.6'
 tabulate==0.9.0; python_version >= '3.7'
 wcwidth==0.2.13

From 84c1bcb7c7d955658fe6fcce18c0f3c8970b4a71 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 1 Oct 2024 20:55:59 +0000
Subject: [PATCH 126/128] Bump pytest from 8.3.2 to 8.3.3

Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.3.2 to 8.3.3.
- [Release notes](https://github.com/pytest-dev/pytest/releases)
- [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest/compare/8.3.2...8.3.3)

---
updated-dependencies:
- dependency-name: pytest
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 9df75f6696..be5394ab4f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,7 +9,7 @@ jsonschema==3.2.0
 lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64'
 lxml==5.3.0; python_version >= '3.8' or platform_machine != 'aarch64'
 pytest==7.0.1; python_version < '3.8'
-pytest==8.3.2; python_version >= '3.8'
+pytest==8.3.3; python_version >= '3.8'
 pytest-forked==1.4.0; python_version == '3.6'
 pytest-forked==1.6.0; python_version >= '3.7'
 pytest-parallel==0.1.1

From 9c641464478e99d96f2129fd6a8706881d7c5574 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 1 Oct 2024 20:56:04 +0000
Subject: [PATCH 127/128] Bump filelock from 3.4.1 to 3.16.1

Bumps [filelock](https://github.com/tox-dev/py-filelock) from 3.4.1 to 3.16.1.
- [Release notes](https://github.com/tox-dev/py-filelock/releases)
- [Changelog](https://github.com/tox-dev/filelock/blob/main/docs/changelog.rst)
- [Commits](https://github.com/tox-dev/py-filelock/compare/3.4.1...3.16.1)

---
updated-dependencies:
- dependency-name: filelock
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 setup.cfg        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 9df75f6696..df3a63e6a5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ argcomplete==3.1.2; python_version < '3.8'
 argcomplete==3.5.0; python_version >= '3.8'
 filelock==3.4.1; python_version == '3.6'
 filelock==3.12.2; python_version == '3.7'
-filelock==3.15.4; python_version >= '3.8'
+filelock==3.16.1; python_version >= '3.8'
 importlib_metadata==4.0.1; python_version < '3.8'
 jsonschema==3.2.0
 lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64'
diff --git a/setup.cfg b/setup.cfg
index 642a277bc0..2f10ded97f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -31,7 +31,7 @@ install_requires =
     argcomplete <= 3.1.2; python_version < '3.8'
     filelock
     filelock<=3.12.2; python_version == '3.7'
-    filelock<=3.4.1; python_version == '3.6'
+    filelock<=3.16.1; python_version == '3.6'
     jsonschema
     lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64'
     lxml==5.3.0; python_version >= '3.8' or platform_machine != 'aarch64'

From 471c5408d9362c0e6115e8c3fe236da936bb108f Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <vkarak@gmail.com>
Date: Tue, 1 Oct 2024 22:58:08 +0200
Subject: [PATCH 128/128] Revert changes in `setup.cfg`

---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 2f10ded97f..642a277bc0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -31,7 +31,7 @@ install_requires =
     argcomplete <= 3.1.2; python_version < '3.8'
     filelock
     filelock<=3.12.2; python_version == '3.7'
-    filelock<=3.16.1; python_version == '3.6'
+    filelock<=3.4.1; python_version == '3.6'
     jsonschema
     lxml==5.2.0; python_version < '3.8' and platform_machine == 'aarch64'
     lxml==5.3.0; python_version >= '3.8' or platform_machine != 'aarch64'