Skip to content

Commit

Permalink
Fixed corruption in checkpoint file after an exception (fitbenchmarki…
Browse files Browse the repository at this point in the history 
…ng#1271)

* Finalise checkpoint file if there's an exception

* Add test for valid cp after failed run

* Use default parser in test

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Fix line length

---------

Co-authored-by: Tyrone Rees <[email protected]>
Co-authored-by: Jessica Huntley <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
@AndrewLister-STFC @tyronerees
@jess-farmer @pre-commit-ci
4 people authored Nov 29, 2024
1 parent 98e3393 commit a700baf
Show file tree
Hide file tree
Showing 2 changed files with 111 additions and 72 deletions.
143 changes: 76 additions & 67 deletions fitbenchmarking/cli/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -383,76 +383,85 @@ def run(problem_sets, additional_options=None, options_file="", debug=False):
pp_dfs_all_prob_sets = {}
cp = Checkpoint(options=options)

for sub_dir in problem_sets:
# Create full path for the directory that holds a group of
# problem definition files
data_dir = os.path.join(current_path, sub_dir)

test_data = glob.glob(data_dir + "/*.*")

if not test_data:
LOGGER.warning("Problem set %s not found", data_dir)
continue

# generate group label/name used for problem set
try:
with open(
os.path.join(data_dir, "META.txt"), encoding="utf-8"
) as f:
label = f.readline().strip("\n")
except OSError:
label = sub_dir.replace("/", "_")

LOGGER.info("Running the benchmarking on the %s problem set", label)
fit = Fit(
options=options, data_dir=data_dir, label=label, checkpointer=cp
)
results, failed_problems, unselected_minimizers = fit.benchmark()

# If a result has error flag 4 then the result contains dummy values,
# if this is the case for all results then output should not be
# produced as results tables won't show meaningful values.
all_dummy_results_flag = True
for result in results:
if result.error_flag != 4:
all_dummy_results_flag = False
break

# If the results are an empty list then this means that all minimizers
# raise an exception and the tables will produce errors if they run
# for that problem set.
if not results or all_dummy_results_flag:
message = (
"\nWARNING: \nThe user chosen options and/or problem "
" setup resulted in all minimizers and/or parsers "
"raising an exception. Because of this, results for "
f"the {label} problem set will not be displayed. "
"Please see the logs for more detail on why this is "
"the case."
try:
for sub_dir in problem_sets:
# Create full path for the directory that holds a group of
# problem definition files
data_dir = os.path.join(current_path, sub_dir)

test_data = glob.glob(data_dir + "/*.*")

if not test_data:
LOGGER.warning("Problem set %s not found", data_dir)
continue

# generate group label/name used for problem set
try:
with open(
os.path.join(data_dir, "META.txt"), encoding="utf-8"
) as f:
label = f.readline().strip("\n")
except OSError:
label = sub_dir.replace("/", "_")

LOGGER.info(
"Running the benchmarking on the %s problem set", label
)
LOGGER.warning(message)
else:
LOGGER.info("Producing output for the %s problem set", label)
# Display the runtime and accuracy results in a table
group_results_dir, pp_dfs = save_results(
group_name=label,
results=results,
fit = Fit(
options=options,
failed_problems=failed_problems,
unselected_minimizers=unselected_minimizers,
config=cp.config,
data_dir=data_dir,
label=label,
checkpointer=cp,
)

pp_dfs_all_prob_sets[label] = pp_dfs

LOGGER.info("Completed benchmarking for %s problem set", sub_dir)
group_results_dir = os.path.relpath(
path=group_results_dir, start=options.results_dir
)
result_dir.append(group_results_dir)
group_labels.append(label)

cp.finalise()
results, failed_problems, unselected_minimizers = fit.benchmark()

# If a result has error flag 4 then the result contains dummy
# values, if this is the case for all results then output should
# not be produced as results tables won't show meaningful values.
all_dummy_results_flag = True
for result in results:
if result.error_flag != 4:
all_dummy_results_flag = False
break

# If the results are an empty list then this means that all
# minimizers raise an exception and the tables will produce
# errors if they run for that problem set.
if not results or all_dummy_results_flag:
message = (
"\nWARNING: \nThe user chosen options and/or problem "
" setup resulted in all minimizers and/or parsers "
"raising an exception. Because of this, results for "
f"the {label} problem set will not be displayed. "
"Please see the logs for more detail on why this is "
"the case."
)
LOGGER.warning(message)
else:
LOGGER.info("Producing output for the %s problem set", label)
# Display the runtime and accuracy results in a table
group_results_dir, pp_dfs = save_results(
group_name=label,
results=results,
options=options,
failed_problems=failed_problems,
unselected_minimizers=unselected_minimizers,
config=cp.config,
)

pp_dfs_all_prob_sets[label] = pp_dfs

LOGGER.info(
"Completed benchmarking for %s problem set", sub_dir
)
group_results_dir = os.path.relpath(
path=group_results_dir, start=options.results_dir
)
result_dir.append(group_results_dir)
group_labels.append(label)

finally:
cp.finalise()

# Check result_dir is non empty before producing output
if not result_dir:
Expand Down
40 changes: 35 additions & 5 deletions fitbenchmarking/cli/tests/test_main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@

import inspect
import os
from json import load
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest import TestCase
from unittest.mock import patch

Expand All @@ -14,6 +16,7 @@
from fitbenchmarking.cost_func.nlls_cost_func import NLLSCostFunc
from fitbenchmarking.parsing.parser_factory import parse_problem_file
from fitbenchmarking.utils import exceptions, fitbm_result
from fitbenchmarking.utils.misc import get_problem_files
from fitbenchmarking.utils.options import Options


Expand Down Expand Up @@ -67,11 +70,7 @@ def test_check_no_results_produced(self, benchmark):
benchmark.return_value = ([], [], {})

with self.assertRaises(exceptions.NoResultsError):
main.run(
["examples/benchmark_problems/simple_tests"],
os.path.dirname(__file__),
debug=True,
)
main.run(["examples/benchmark_problems/simple_tests"], debug=True)

@patch("fitbenchmarking.cli.main.Fit.benchmark")
def test_all_dummy_results_produced(self, benchmark):
Expand All @@ -98,3 +97,34 @@ def test_file_path_exception_raised(self, mock):
with self.assertRaises(SystemExit) as exp:
main.main()
self.assertEqual(exp.exception.code, 1)

@patch("fitbenchmarking.cli.main.save_results")
@patch("fitbenchmarking.utils.misc.get_problem_files")
def test_checkpoint_file_on_fail(self, get_problems, save_results):
"""
Checks that the checkpoint file is valid json if there's a crash.
"""
get_problems.side_effect = lambda path: [get_problem_files(path)[0]]
save_results.side_effect = RuntimeError(
"Exception raised during save..."
)

with TemporaryDirectory() as results_dir:
with self.assertRaises(RuntimeError):
main.run(
["examples/benchmark_problems/NIST/low_difficulty"],
additional_options={
"scipy_ls": ["lm-scipy"],
"software": ["scipy_ls"],
"num_runs": 1,
"results_dir": results_dir,
},
debug=True,
)

with open(f"{results_dir}/checkpoint.json", encoding="utf8") as f:
# This will fail if the json is invalid
contents = load(f)

# Check that it's not empty
self.assertTrue(contents)

0 comments on commit a700baf

Please sign in to comment.