diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 04d6959d6..18e9046d3 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1651,7 +1651,7 @@ Result storage configuration .. py:attribute:: storage.enable :required: No - :default: ``true`` + :default: ``false`` Enable results storage. diff --git a/docs/manpage.rst b/docs/manpage.rst index ea3f90339..1047ac369 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -129,6 +129,9 @@ Test commands Result storage commands ^^^^^^^^^^^^^^^^^^^^^^^ +All these commands require the results storage feature to be enabled. +See :ref:`querying-past-results` for more information. + .. option:: --delete-stored-sessions=SELECT_SPEC Delete the stored sessions matching the given selection criteria. @@ -1116,8 +1119,13 @@ Miscellaneous options Print a report summarizing the performance of all performance tests that have run in the current session. - For each test all of their performance variables are reported and optionally compared to past results based on the ``CMPSPEC`` specified. - If not specified, ``CMPSPEC`` defaults to ``now:now/last:/+job_nodelist+result``, meaning that the current performance will not be compared to any past run and, additionally, the ``job_nodelist`` and the test result (``pass`` or ``fail``) will be listed. + The behaviour of this option is different if the :ref:`results storage feature ` is enabled or not. + + If not enabled, a table will be printed summarizing the performance of all test cases that have run in this session. + If a test is retried multiple times, only its last result will be reported. + + If results storage is enabled, then this option can perform automatic comparisons with the results of the same tests in the past. + This is contolled by the optinal ``CMPSPEC`` argument, which defaults to ``now-1d:now/last:+job_nodelist/+result``, meaning that the current performance will be compared to the last run of the same test on the same nodes from the past 24 hours. For the exact syntax of ``CMPSPEC``, refer to :ref:`querying-past-results`. @@ -1356,8 +1364,10 @@ Querying past results .. versionadded:: 4.7 -ReFrame provides several options for querying and inspecting past sessions and test case results. -All those options follow a common syntax that builds on top of the following elements: +ReFrame offers the possibility of querying and inspecting past sessions and test case results. +This feature must be explicitly enabled either by setting the :envvar:`RFM_ENABLE_RESULTS_STORAGE` environment variable or the :attr:`storage.enable ` configuration parameter. + +All command-line options that interact with the results database follow a common syntax that builds on top of the following elements: 1. Selection of sessions and test cases 2. Grouping of test cases and performance aggregations @@ -1723,6 +1733,21 @@ Whenever an environment variable is associated with a configuration option, its .. versionadded:: 4.0.0 +.. envvar:: RFM_ENABLE_RESULTS_STORAGE + + Enable results storage. + + .. table:: + :align: left + + ================================== ================== + Associated command line option N/A + Associated configuration parameter :attr:`storage.enable ` + ================================== ================== + + .. versionadded:: 4.7 + + .. envvar:: RFM_FLEX_ALLOC_STRICT Fail flexible tests if their minimum task requirement is not satisfied. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index b3eead5eb..842aa0725 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -168,12 +168,10 @@ This can be suppressed by increasing the level at which this information is logg Run reports and performance logging ----------------------------------- -Once a test session finishes, ReFrame stores the detailed session information in a database file located under ``$HOME/.reframe/reports``. -Past performance data can be retrieved from this database and compared with the current or another run. -We explain in detail the handling of the results database in section :ref:`inspecting-past-results`. - -By default, the session information is also saved in a JSON report file under ``$HOME/.reframe/reports``. +By default, the session information is saved in a JSON report file under ``$HOME/.reframe/reports``. The latest report is always symlinked by the ``latest.json`` name, unless the :option:`--report-file` option is given. +The detailed session information can also be saved in a database file located under ``$HOME/.reframe/reports`` to allow for inspection and comparison of past results. +We will cover this capability in section :ref:`inspecting-past-results`. For performance tests, in particular, an additional CSV file is generated with all the relevant information. These files are located by default under ``perflogs///.log``. @@ -1982,8 +1980,18 @@ Inspecting past results .. versionadded:: 4.7 -For every session that has run at least one test case, ReFrame stores all its details, including the test cases, in a database. -Essentially, the stored information is the same as the one found in the :ref:`report file `. +ReFrame offers the ability to store each run session in a database which you can later query to inspect and compare past results. +You can enable this feature by either setting the environment variable ``RFM_ENABLE_RESULTS_STORAGE=y`` or by setting the :attr:`storage.enable ` parameter in the configuration file. + +To demonstrate the use of this feature, we will rerun all experiments with the results storage enabled. + +.. code-block:: bash + + export RFM_ENABLE_RESULTS_STORAGE=y + ./reframe-examples/tutorial/scripts/runall.sh + +For every session that has run at least one test case, ReFrame stores all its details in a database. +Essentially, the stored information is the same as the one found in the :ref:`report file ` but stored in a way that allows fast queries. To list the stored sessions use the :option:`--list-stored-sessions` option: diff --git a/examples/tutorial/scripts/runall.sh b/examples/tutorial/scripts/runall.sh new file mode 100755 index 000000000..07091dbc8 --- /dev/null +++ b/examples/tutorial/scripts/runall.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -x + +pushd reframe-examples/tutorial +reframe -c stream/stream_runonly.py -r +reframe -c stream/stream_runonly.py -r +reframe -C config/baseline.py -c stream/stream_runonly.py -r +reframe -C config/baseline_environs.py -c stream/stream_build_run.py --exec-policy=serial -r +reframe -C config/baseline_environs.py -c stream/stream_fixtures.py -l +reframe -C config/baseline_environs.py -c stream/stream_fixtures.py -r +reframe -C config/baseline_environs.py -c stream/stream_variables.py -S num_threads=2 -r +reframe -C config/baseline_environs.py -c stream/stream_variables_fixtures.py --exec-policy=serial -S stream_test.stream_binary.array_size=50000000 -r +reframe -C config/baseline_environs.py -c stream/stream_parameters.py --exec-policy=serial -r +reframe -C config/baseline_environs.py -c stream/stream_variables_fixtures.py -P num_threads=1,2,4,8 --exec-policy=serial -r +reframe -c deps/deps_complex.py -r +reframe --restore-session --failed -r +reframe -c deps/deps_complex.py --keep-stage-files -r +reframe --restore-session --keep-stage-files -n T6 -r +reframe -c deps/deps_complex.py -n T6 -r +popd diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 92d9a1e01..fc4af30f0 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -628,7 +628,7 @@ def main(): configvar='general/perf_report_spec', envvar='RFM_PERF_REPORT_SPEC', help=('Print a report for performance tests ' - '(default: "now:now/last:+job_nodelist/+result")') + '(default: "now-1d:now/last:+job_nodelist/+result")') ) reporting_options.add_argument( '--session-extras', action='append', metavar='KV_DATA', @@ -979,8 +979,7 @@ def restrict_logging(): '--describe-stored-testcases', '--list-stored-sessions', '--list-stored-testcases', - '--performance-compare', - '--performance-report']): + '--performance-compare']): sys.exit(1) rt = runtime.runtime() @@ -1655,9 +1654,12 @@ def module_unuse(*paths): if (options.performance_report and not options.dry_run and not report.is_empty()): try: - data = reporting.performance_compare( - rt.get_option('general/0/perf_report_spec'), report - ) + if rt.get_option('storage/0/enable'): + data = reporting.performance_compare( + rt.get_option('general/0/perf_report_spec'), report + ) + else: + data = report.report_data() except Exception as err: printer.warning( f'failed to generate performance report: {err}' @@ -1699,7 +1701,8 @@ def module_unuse(*paths): ) # Store the generated report for analytics - if not report.is_empty() and not options.dry_run: + if (rt.get_option('storage/0/enable') and + not report.is_empty() and not options.dry_run): try: sess_uuid = report.store() except Exception as e: diff --git a/reframe/frontend/reporting/__init__.py b/reframe/frontend/reporting/__init__.py index c1ff5745c..5662e2441 100644 --- a/reframe/frontend/reporting/__init__.py +++ b/reframe/frontend/reporting/__init__.py @@ -460,6 +460,42 @@ def store(self): return StorageBackend.default().store(self, self.filename) + def report_data(self): + '''Get tabular data from this report''' + + columns = ['name', 'sysenv', 'job_nodelist', + 'pvar', 'punit', 'pval', 'result'] + data = [columns] + num_runs = len(self.__report['runs']) + for runid, runinfo in enumerate(self.__report['runs']): + for tc in map(_TCProxy, runinfo['testcases']): + if tc['result'] != 'success' and runid != num_runs - 1: + # Skip this testcase until its last retry + continue + + for pvar, reftuple in tc['perfvalues'].items(): + pvar = pvar.split(':')[-1] + pval, _, _, _, punit = reftuple + if pval is None: + # Ignore `None` performance values + # (performance tests that failed sanity) + continue + + line = [] + for c in columns: + if c == 'pvar': + line.append(pvar) + elif c == 'pval': + line.append(pval) + elif c == 'punit': + line.append(punit) + else: + line.append(tc[c]) + + data.append(line) + + return data + def generate_xml_report(self): '''Generate a JUnit report from a standard ReFrame JSON report.''' diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index cb560b67f..4e36d1a51 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -589,7 +589,7 @@ "general/module_mappings": [], "general/non_default_craype": false, "general/perf_info_level": "info", - "general/perf_report_spec": "now:now/last:/+job_nodelist+result", + "general/perf_report_spec": "now-1d:now/last:/+job_nodelist+result", "general/pipeline_timeout": 3, "general/purge_environment": false, "general/remote_detect": false, @@ -633,7 +633,7 @@ "logging/handlers_perflog/httpjson_debug": false, "modes/options": [], "modes/target_systems": ["*"], - "storage/enable": true, + "storage/enable": false, "storage/backend": "sqlite", "storage/sqlite_conn_timeout": 60, "storage/sqlite_db_file": "${HOME}/.reframe/reports/results.db", diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 29bbb3457..191cd68d3 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -420,7 +420,14 @@ def test_perflogdir_from_env(run_reframe, tmp_path, monkeypatch): 'default' / 'PerformanceFailureCheck.log') -def test_performance_report(run_reframe, run_action): +@pytest.fixture(params=['storage=yes', 'storage=no']) +def storage_enabled(request, monkeypatch): + value = request.param.split('=')[1] + monkeypatch.setenv('RFM_ENABLE_RESULTS_STORAGE', value) + return value == 'yes' + + +def test_performance_report(run_reframe, run_action, storage_enabled): returncode, stdout, stderr = run_reframe( checkpath=['unittests/resources/checks/frontend_checks.py'], more_options=['-n', '^PerformanceFailureCheck', @@ -433,6 +440,9 @@ def test_performance_report(run_reframe, run_action): else: assert returncode == 0 + if run_action != 'dry_run': + assert 'PERFORMANCE REPORT' in stdout + assert 'Traceback' not in stdout assert 'Traceback' not in stderr @@ -1269,7 +1279,8 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0): return returncode, stdout, stderr -def test_storage_options(run_reframe, tmp_path, table_format): +def test_storage_options(run_reframe, tmp_path, table_format, monkeypatch): + monkeypatch.setenv('RFM_ENABLE_RESULTS_STORAGE', 'yes') run_reframe2 = functools.partial( run_reframe, checkpath=['unittests/resources/checks/frontend_checks.py'], @@ -1335,8 +1346,7 @@ def test_storage_options(run_reframe, tmp_path, table_format): '--describe-stored-testcases=now-1d:now', '--list-stored-sessions', '--list-stored-testcases=now-1d:now/mean:/', - '--performance-compare=now-1d:now/now-1d/mean:/', - '--performance-report=now-1d:now/mean:/' + '--performance-compare=now-1d:now/now-1d/mean:/' ]) def storage_option(request): return request.param @@ -1359,7 +1369,8 @@ def test_disabled_results_storage(run_reframe, storage_option, monkeypatch): assert 'requires results storage' in stdout -def test_session_annotations(run_reframe): +def test_session_annotations(run_reframe, monkeypatch): + monkeypatch.setenv('RFM_ENABLE_RESULTS_STORAGE', 'yes') assert_no_crash(*run_reframe( checkpath=['unittests/resources/checks/frontend_checks.py'], action='-r', @@ -1373,13 +1384,14 @@ def test_session_annotations(run_reframe): assert text in stdout -def test_performance_compare(run_reframe, table_format): +def test_performance_compare(run_reframe, table_format, monkeypatch): def assert_no_crash(returncode, stdout, stderr, exitcode=0): assert returncode == exitcode assert 'Traceback' not in stdout assert 'Traceback' not in stderr return returncode, stdout, stderr + monkeypatch.setenv('RFM_ENABLE_RESULTS_STORAGE', 'yes') run_reframe2 = functools.partial( run_reframe, checkpath=['unittests/resources/checks/frontend_checks.py'],