From 37a625865eb34580aebd9f2c3329ebdd364cabf3 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 24 Jan 2022 08:59:32 +0100 Subject: [PATCH 01/57] Explain time limit for add_command(). --- lab/experiment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lab/experiment.py b/lab/experiment.py index 4a1acea42..75fb5dd27 100644 --- a/lab/experiment.py +++ b/lab/experiment.py @@ -203,7 +203,9 @@ def add_command( After *time_limit* seconds the signal SIGXCPU is sent to the command. The process can catch this signal and exit gracefully. If it doesn't catch the SIGXCPU signal, the command is aborted - with SIGKILL after five additional seconds. + with SIGKILL after five additional seconds. The time spent by a + command is the sum of time spent across all threads of the + process. The command is aborted with SIGKILL when it uses more than *memory_limit* MiB. From 4096f604619c6d40c543df23bc73809307b52652 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 24 Jan 2022 09:04:53 +0100 Subject: [PATCH 02/57] Fix link to Singularity debian file. --- .github/workflows/ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 6df3c5456..330a8bab1 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -86,7 +86,7 @@ jobs: - name: Install Singularity working-directory: ../deps run: | - wget --no-verbose http://ftp.se.debian.org/debian/pool/main/s/singularity-container/singularity-container_3.5.2+ds1-1_amd64.deb -O singularity.deb + wget --no-verbose http://ftp.se.debian.org/debian/pool/main/s/singularity-container/singularity-container_3.5.2+ds2-1_amd64.deb -O singularity.deb sudo dpkg -i singularity.deb rm singularity.deb From 76c4fb7e2ccd745119b4daf38d6a976a65b912b5 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 24 Jan 2022 11:33:24 +0100 Subject: [PATCH 03/57] Use old Singularity debian file. --- .github/workflows/ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 330a8bab1..8b03918e4 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -86,7 +86,7 @@ jobs: - name: Install Singularity working-directory: ../deps run: | - wget --no-verbose http://ftp.se.debian.org/debian/pool/main/s/singularity-container/singularity-container_3.5.2+ds2-1_amd64.deb -O singularity.deb + wget --no-verbose https://www.ida.liu.se/~jense56/files/singularity-container_3.5.2+ds1-1_amd64.deb -O singularity.deb sudo dpkg -i singularity.deb rm singularity.deb From 148281cc7262e6766684263779b2d447437435e8 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Thu, 10 Mar 2022 21:24:47 +0100 Subject: [PATCH 04/57] Simplify example Fast Downward experiment. --- examples/downward/2020-09-11-A-cg-vs-ff.py | 22 +++++++- examples/downward/project.py | 64 ++++------------------ 2 files changed, 30 insertions(+), 56 deletions(-) diff --git a/examples/downward/2020-09-11-A-cg-vs-ff.py b/examples/downward/2020-09-11-A-cg-vs-ff.py index 11efc9724..b022ed033 100755 --- a/examples/downward/2020-09-11-A-cg-vs-ff.py +++ b/examples/downward/2020-09-11-A-cg-vs-ff.py @@ -1,15 +1,18 @@ #! /usr/bin/env python import os +import shutil import project REPO = project.get_repo_base() BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] +SCP_LOGIN = "myname@myserver.com" +REMOTE_REPOS_DIR = "/infai/seipp/projects" if project.REMOTE: SUITE = project.SUITE_SATISFICING - ENV = project.BaselSlurmEnvironment(email="my.name@unibas.ch") + ENV = project.BaselSlurmEnvironment(email="my.name@myhost.ch") else: SUITE = ["depot:p01.pddl", "grid:prob01.pddl", "gripper:prob01.pddl"] ENV = project.LocalEnvironment(processes=2) @@ -35,7 +38,6 @@ "search_start_time", "search_start_memory", "total_time", - "initial_h_value", "h_values", "coverage", "expansions", @@ -43,7 +45,7 @@ project.EVALUATIONS_PER_TIME, ] -exp = project.CommonExperiment(environment=ENV) +exp = project.FastDownwardExperiment(environment=ENV) for config_nick, config in CONFIGS: for rev, rev_nick in REVS: algo_name = f"{rev_nick}:{config_nick}" if rev_nick else config_nick @@ -57,6 +59,20 @@ ) exp.add_suite(BENCHMARKS_DIR, SUITE) +exp.add_parser(exp.EXITCODE_PARSER) +exp.add_parser(exp.TRANSLATOR_PARSER) +exp.add_parser(exp.SINGLE_SEARCH_PARSER) +exp.add_parser(project.DIR / "parser.py") +exp.add_parser(exp.PLANNER_PARSER) + +exp.add_step("build", exp.build) +exp.add_step("start", exp.start_runs) +exp.add_fetcher(name="fetch") + +if not project.REMOTE: + exp.add_step("remove-eval-dir", shutil.rmtree, exp.eval_dir, ignore_errors=True) + project.add_scp_step(exp, SCP_LOGIN, REMOTE_REPOS_DIR) + project.add_absolute_report( exp, attributes=ATTRIBUTES, filter=[project.add_evaluations_per_time] ) diff --git a/examples/downward/project.py b/examples/downward/project.py index 764b69bdf..c080783ca 100644 --- a/examples/downward/project.py +++ b/examples/downward/project.py @@ -1,36 +1,29 @@ -from collections import namedtuple -import getpass from pathlib import Path import platform -import shutil import subprocess import sys from downward.experiment import FastDownwardExperiment from downward.reports.absolute import AbsoluteReport from downward.reports.scatter import ScatterPlotReport +from downward.reports.taskwise import TaskwiseReport from lab import tools -from lab.environments import BaselSlurmEnvironment, LocalEnvironment +from lab.environments import ( + BaselSlurmEnvironment, + LocalEnvironment, + TetralithEnvironment, +) from lab.experiment import ARGPARSER from lab.reports import Attribute, geometric_mean # Silence import-unused messages. Experiment scripts may use these imports. -assert BaselSlurmEnvironment and LocalEnvironment and ScatterPlotReport +assert LocalEnvironment and ScatterPlotReport and TaskwiseReport DIR = Path(__file__).resolve().parent NODE = platform.node() -REMOTE = NODE.endswith(".scicore.unibas.ch") or NODE.endswith(".cluster.bc2.ch") - -User = namedtuple("User", ["scp_login", "remote_repos"]) -USERS = { - "jendrik": User( - scp_login="seipp@login.scicore.unibas.ch", - remote_repos="/infai/seipp/projects", - ), -} -USER = USERS.get(getpass.getuser()) +REMOTE = NODE.endswith((".scicore.unibas.ch", ".cluster.bc2.ch")) def parse_args(): @@ -121,8 +114,8 @@ def _get_exp_dir_relative_to_repo(): return repo_name / rel_script_dir / "data" / expname -def add_scp_step(exp): - remote_exp = Path(USER.remote_repos) / _get_exp_dir_relative_to_repo() +def add_scp_step(exp, login, repos_dir): + remote_exp = Path(repos_dir) / _get_exp_dir_relative_to_repo() exp.add_step( "scp-eval-dir", subprocess.call, @@ -130,7 +123,7 @@ def add_scp_step(exp): "scp", "-r", # Copy recursively. "-C", # Compress files. - f"{USER.scp_login}:{remote_exp}-eval", + f"{login}:{remote_exp}-eval", f"{exp.path}-eval", ], ) @@ -172,38 +165,3 @@ def add_absolute_report(exp, *, name=None, outfile=None, **kwargs): if not REMOTE: exp.add_step(f"open-{name}", subprocess.call, ["xdg-open", outfile]) exp.add_step(f"publish-{name}", subprocess.call, ["publish", outfile]) - - -class CommonExperiment(FastDownwardExperiment): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - self.add_step("build", self.build) - self.add_step("start", self.start_runs) - self.add_fetcher(name="fetch") - - if not REMOTE: - self.add_step( - "remove-eval-dir", shutil.rmtree, self.eval_dir, ignore_errors=True - ) - add_scp_step(self) - - self.add_parser(self.EXITCODE_PARSER) - self.add_parser(self.TRANSLATOR_PARSER) - self.add_parser(self.SINGLE_SEARCH_PARSER) - self.add_parser(self.PLANNER_PARSER) - self.add_parser(DIR / "parser.py") - - def _add_runs(self): - """ - Example showing how to modify the automatically generated runs. - - This uses private members, so it might break between different - versions of Lab. - - """ - FastDownwardExperiment._add_runs(self) - for run in self.runs: - command = run.commands["planner"] - # Slightly raise soft limit for output to stdout. - command[1]["soft_stdout_limit"] = 1.5 * 1024 From 2e2a53a8462e84499135080af5fa41bef18ed34e Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Thu, 10 Mar 2022 21:34:50 +0100 Subject: [PATCH 05/57] Fix style. --- examples/downward/project.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/downward/project.py b/examples/downward/project.py index c080783ca..296026a1c 100644 --- a/examples/downward/project.py +++ b/examples/downward/project.py @@ -18,7 +18,14 @@ # Silence import-unused messages. Experiment scripts may use these imports. -assert LocalEnvironment and ScatterPlotReport and TaskwiseReport +assert ( + BaselSlurmEnvironment + and FastDownwardExperiment + and LocalEnvironment + and ScatterPlotReport + and TaskwiseReport + and TetralithEnvironment +) DIR = Path(__file__).resolve().parent From 7b15953d066ce6de5ebf16b324ae97bceb66500a Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 11 Mar 2022 12:34:03 +0100 Subject: [PATCH 06/57] Fix header sizes in HTML reports. --- docs/news.rst | 4 ++-- downward/reports/absolute.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 29c372846..1bab65756 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,7 +1,7 @@ Changelog ========= -next (unreleased) +v7.1 (unreleased) ----------------- Lab @@ -10,7 +10,7 @@ Lab Downward Lab ^^^^^^^^^^^^ -* No changes so far. +* Fix header sizes in HTML reports (Jendrik Seipp). v7.0 (2021-10-24) diff --git a/downward/reports/absolute.py b/downward/reports/absolute.py index 169d0c5e7..9a2cb32f7 100644 --- a/downward/reports/absolute.py +++ b/downward/reports/absolute.py @@ -157,7 +157,7 @@ def get_markup(self): if domain: assert table toc_line.append(f"[''{domain}'' #{attribute}-{domain}]") - parts.append(f"== {domain} ==[{attribute}-{domain}]\n{table}\n") + parts.append(f"=== {domain} ===[{attribute}-{domain}]\n{table}\n") else: if table: parts.append(f"{table}\n") @@ -179,7 +179,7 @@ def get_markup(self): toc = "\n".join(toc_lines) content = "\n".join( - f"= {attr} =[{attr}]\n\n{section}" for (attr, section) in sections + f"== {attr} ==[{attr}]\n\n{section}" for (attr, section) in sections ) return f"{toc}\n\n\n{content}" From 7aececbc5c6f797eeaec9e87681bd195e5f3ca1f Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 11 Mar 2022 12:45:26 +0100 Subject: [PATCH 07/57] domain-wise --> per-domain --- downward/reports/absolute.py | 4 ++-- lab/reports/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/downward/reports/absolute.py b/downward/reports/absolute.py index 9a2cb32f7..45eac0453 100644 --- a/downward/reports/absolute.py +++ b/downward/reports/absolute.py @@ -143,7 +143,7 @@ def get_markup(self): tables.append( ( "", - f"Domain-wise reports only support numeric " + f"Per-domain reports only support numeric " f"attributes, but {attribute} has type " f"{self._all_attributes[attribute].__name__}.", ) @@ -165,7 +165,7 @@ def get_markup(self): parts.append( f"No task was found where all algorithms " f'have a value for "{attribute}". Therefore no ' - f"domain-wise table can be generated.\n" + f"per-domain table can be generated.\n" ) toc_lines.append(f"- **[''{attribute}'' #{attribute}]**") diff --git a/lab/reports/__init__.py b/lab/reports/__init__.py index 1a7fcff8e..ad0b29c70 100644 --- a/lab/reports/__init__.py +++ b/lab/reports/__init__.py @@ -96,7 +96,7 @@ def __init__( value for: * *absolute*: if False, only include tasks for which all task - runs have values in a domain-wise table (e.g. ``coverage`` is + runs have values in a per-domain table (e.g. ``coverage`` is absolute, whereas ``expansions`` is not, because we can't compare algorithms A and B for task X if B has no value for ``expansions``). @@ -371,7 +371,7 @@ def get_text(self): "This happens when no significant changes occured or " "if for all attributes and all problems never all " "algorithms had a value for this attribute in a " - "domain-wise report." + "per-domain report." ) return doc.render(self.output_format, {"toc": self.toc}) From 4705e06608d0fcf8ff3969d536193dfc5beeb512 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 11 Mar 2022 14:54:07 +0100 Subject: [PATCH 08/57] Include domains in attribute overview tables even if none of their tasks has an attribute value for all algorithms. --- docs/news.rst | 5 ++++- downward/reports/absolute.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 1bab65756..8685cc115 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,11 +6,14 @@ v7.1 (unreleased) Lab ^^^ -* Revamp Singularity example experiment: use runsolver to limit resource usage (Silvan Sievers and Jendrik Seipp). +* Revamp Singularity example experiment: use runsolver to limit resource usage + (Silvan Sievers and Jendrik Seipp). Downward Lab ^^^^^^^^^^^^ * Fix header sizes in HTML reports (Jendrik Seipp). +* Include domains in attribute overview tables even if none of their tasks has an + attribute value for all algorithms (Jendrik Seipp). v7.0 (2021-10-24) diff --git a/downward/reports/absolute.py b/downward/reports/absolute.py index 45eac0453..38f3186c7 100644 --- a/downward/reports/absolute.py +++ b/downward/reports/absolute.py @@ -211,7 +211,7 @@ def _add_table_info(self, attribute, func_name, table): """ if not attribute.absolute: table.info.append( - f"Only instances where all algorithms have a " + f"Only tasks where all algorithms have a " f'value for "{attribute}" are considered.' ) table.info.append( @@ -235,7 +235,10 @@ def _get_suite_table(self, attribute): func_name, func = self._get_aggregation_function(attribute) num_probs = 0 self._add_table_info(attribute, func_name, table) - domain_algo_values = defaultdict(list) + domain_algo_values = {} + for domain in self.domains: + for algorithm in self.algorithms: + domain_algo_values[(domain, algorithm)] = [] for (domain, _), runs in self.problem_runs.items(): # If the attribute is absolute, no runs must have been filtered and # no values must be missing. @@ -258,8 +261,7 @@ def _get_suite_table(self, attribute): # different problem numbers. for domain in self.domains: task_counts = [ - str(len(domain_algo_values.get((domain, algo), []))) - for algo in self.algorithms + str(len(domain_algo_values[(domain, algo)])) for algo in self.algorithms ] if len(set(task_counts)) == 1: count = task_counts[0] @@ -272,7 +274,8 @@ def _get_suite_table(self, attribute): table.cell_formatters[domain][table.header_column] = formatter for (domain, algo), values in domain_algo_values.items(): - table.add_cell(domain, algo, func(values)) + domain_value = func(values) if values else None + table.add_cell(domain, algo, domain_value) table.num_values = num_probs return table From 5e5f2349c1a43341604c1ac1e30020a6c30a150c Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Thu, 21 Apr 2022 22:33:07 +0200 Subject: [PATCH 09/57] Compute planner scores. --- downward/scripts/planner-parser.py | 54 ++++++++++++++++++++++++++++-- examples/lmcut.py | 2 +- examples/showcase-options.py | 4 +-- lab/tools.py | 16 +++++++++ 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/downward/scripts/planner-parser.py b/downward/scripts/planner-parser.py index 839c33b29..27f72d1ce 100755 --- a/downward/scripts/planner-parser.py +++ b/downward/scripts/planner-parser.py @@ -1,5 +1,6 @@ #! /usr/bin/env python +from lab import tools from lab.parser import Parser @@ -11,21 +12,66 @@ def add_planner_memory(content, props): def add_planner_time(content, props): + # Newer planner versions print the planner time and we parse it below. Don't overwrite it. + if "planner_time" not in props: + return try: props["planner_time"] = props["translator_time_done"] + props["total_time"] except KeyError: pass +def add_planner_scores(content, props): + """ + Compute scores for overall planner runtime and memory usage. + + Best possible performance in a task is counted as 1, while failure to solve + a task and worst performance are counted as 0. + + """ + success = props["coverage"] or props["unsolvable"] + + try: + time_limit = props["planner_time_limit"] + except KeyError: + print("planner_time_limit missing -> can't compute planner time score") + else: + props["score_planner_time"] = tools.compute_log_score( + success, props.get("planner_time"), lower_bound=1.0, upper_bound=time_limit + ) + + try: + memory_limit_kb = props["planner_memory_limit"] * 1024 + except KeyError: + print("planner_memory_limit missing -> can't compute planner memory score") + else: + props["score_planner_memory"] = tools.compute_log_score( + success, props.get("planner_memory"), lower_bound=2000, upper_bound=memory_limit_kb + ) + + class PlannerParser(Parser): def __init__(self): Parser.__init__(self) - self.add_function(add_planner_memory) - self.add_function(add_planner_time) + self.add_pattern( + "planner_time_limit", + r"planner time limit: (.+)s", + type=float, + ) + self.add_pattern( + "planner_memory_limit", + r"planner memory limit: (.+) MB", + type=int, + ) self.add_pattern( "node", r"node: (.+)\n", type=str, file="driver.log", required=True ) + self.add_pattern( + "planner_time", + r"Planner time: (.+)s", + type=float, + ) self.add_pattern( "planner_wall_clock_time", r"planner wall-clock time: (.+)s", @@ -34,6 +80,10 @@ def __init__(self): required=True, ) + self.add_function(add_planner_memory) + self.add_function(add_planner_time) + self.add_function(add_planner_scores) + def main(): parser = PlannerParser() diff --git a/examples/lmcut.py b/examples/lmcut.py index 29df28564..965950fa5 100755 --- a/examples/lmcut.py +++ b/examples/lmcut.py @@ -12,7 +12,7 @@ from lab.environments import BaselSlurmEnvironment, LocalEnvironment -ATTRIBUTES = ["coverage", "error", "expansions", "total_time"] +ATTRIBUTES = ["coverage", "error", "expansions", "planner_memory", "planner_time"] NODE = platform.node() if NODE.endswith((".cluster.bc2.ch", ".scicore.unibas.ch")): diff --git a/examples/showcase-options.py b/examples/showcase-options.py index c71e8b651..c898dcdc7 100755 --- a/examples/showcase-options.py +++ b/examples/showcase-options.py @@ -172,11 +172,11 @@ def eval_dir(num): name="report-abs-p-filter", ) exp.add_report( - AbsoluteReport(attributes=["coverage", "error"], format="tex"), + AbsoluteReport(attributes=["coverage", "error", "score_planner_time"], format="tex"), outfile="report-abs-combined.tex", ) exp.add_report( - AbsoluteReport(attributes=["coverage", "error"], format="html"), + AbsoluteReport(attributes=["coverage", "error", "score_planner_memory"], format="html"), outfile="report-abs-combined.html", ) exp.add_report( diff --git a/lab/tools.py b/lab/tools.py index b0347662d..37ae7156a 100644 --- a/lab/tools.py +++ b/lab/tools.py @@ -2,6 +2,7 @@ import colorsys import functools import logging +import math import os from pathlib import Path import pkgutil @@ -233,6 +234,21 @@ def add_unexplained_error(dictionary, error): dictionary[key].append(error) +def compute_log_score(success, value, lower_bound, upper_bound): + """Compute score between 0 and 1. + + Best possible performance (value <= lower_bound) counts as 1, while failed + runs (!success) and worst performance (value >= upper_bound) counts as 0. + """ + if value is None or not success: + return 0. + value = max(value, lower_bound) + value = min(value, upper_bound) + raw_score = math.log(value) - math.log(upper_bound) + best_raw_score = math.log(lower_bound) - math.log(upper_bound) + return raw_score / best_raw_score + + class Properties(dict): class _PropertiesEncoder(json.JSONEncoder): def default(self, o): From d219d4bb63a7030be2a42825b5e227d504e9464f Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Thu, 21 Apr 2022 22:36:39 +0200 Subject: [PATCH 10/57] Reuse compute_log_score() function. --- downward/scripts/single-search-parser.py | 27 +++++++++--------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/downward/scripts/single-search-parser.py b/downward/scripts/single-search-parser.py index 113d36630..d38bf7675 100755 --- a/downward/scripts/single-search-parser.py +++ b/downward/scripts/single-search-parser.py @@ -8,6 +8,7 @@ import re import sys +from lab import tools from lab.parser import Parser @@ -108,19 +109,11 @@ def add_scores(content, props): to solve a task and worst performance are counted as 0. """ - - def log_score(value, min_bound, max_bound): - if value is None or not props["coverage"]: - return 0 - value = max(value, min_bound) - value = min(value, max_bound) - raw_score = math.log(value) - math.log(max_bound) - best_raw_score = math.log(min_bound) - math.log(max_bound) - return raw_score / best_raw_score + success = props["coverage"] or props["unsolvable"] for attr in ("expansions", "evaluations", "generated"): - props["score_" + attr] = log_score( - props.get(attr), min_bound=100, max_bound=1e6 + props["score_" + attr] = tools.compute_log_score( + success, props.get(attr), lower_bound=100, upper_bound=1e6 ) try: @@ -128,11 +121,11 @@ def log_score(value, min_bound, max_bound): except KeyError: print("search time limit missing -> can't compute time scores") else: - props["score_total_time"] = log_score( - props.get("total_time"), min_bound=1.0, max_bound=max_time + props["score_total_time"] = tools.compute_log_score( + success, props.get("total_time"), lower_bound=1.0, upper_bound=max_time ) - props["score_search_time"] = log_score( - props.get("search_time"), min_bound=1.0, max_bound=max_time + props["score_search_time"] = tools.compute_log_score( + success, props.get("search_time"), lower_bound=1.0, upper_bound=max_time ) try: @@ -140,8 +133,8 @@ def log_score(value, min_bound, max_bound): except KeyError: print("search memory limit missing -> can't compute memory score") else: - props["score_memory"] = log_score( - props.get("memory"), min_bound=2000, max_bound=max_memory_kb + props["score_memory"] = tools.compute_log_score( + success, props.get("memory"), lower_bound=2000, upper_bound=max_memory_kb ) From bca9b8bf2bc8fb791fa12b019b39066cbc0c971c Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Thu, 21 Apr 2022 23:20:24 +0200 Subject: [PATCH 11/57] Upgrade black version. --- dev/make-release-notes.py | 2 +- downward/reports/scatter_matplotlib.py | 2 +- downward/scripts/planner-parser.py | 5 ++++- examples/showcase-options.py | 8 ++++++-- lab/reports/__init__.py | 2 +- lab/tools.py | 2 +- tox.ini | 4 ++-- 7 files changed, 16 insertions(+), 9 deletions(-) diff --git a/dev/make-release-notes.py b/dev/make-release-notes.py index c79160287..474b70735 100755 --- a/dev/make-release-notes.py +++ b/dev/make-release-notes.py @@ -5,7 +5,7 @@ _, VERSION, CHANGELOG, LIST = sys.argv -REGEX = fr""" +REGEX = rf""" Changelog\n =========\n \n diff --git a/downward/reports/scatter_matplotlib.py b/downward/reports/scatter_matplotlib.py index f77c1bfc2..f69e58c53 100644 --- a/downward/reports/scatter_matplotlib.py +++ b/downward/reports/scatter_matplotlib.py @@ -33,7 +33,7 @@ def create_legend(self): @staticmethod def _get_max_supported_value(scale): if scale == "linear": - return 10 ** 12 # Larger values cause numerical problems. + return 10**12 # Larger values cause numerical problems. else: assert scale in {"log", "symlog"}, scale return sys.maxsize diff --git a/downward/scripts/planner-parser.py b/downward/scripts/planner-parser.py index 27f72d1ce..db8c363c3 100755 --- a/downward/scripts/planner-parser.py +++ b/downward/scripts/planner-parser.py @@ -46,7 +46,10 @@ def add_planner_scores(content, props): print("planner_memory_limit missing -> can't compute planner memory score") else: props["score_planner_memory"] = tools.compute_log_score( - success, props.get("planner_memory"), lower_bound=2000, upper_bound=memory_limit_kb + success, + props.get("planner_memory"), + lower_bound=2000, + upper_bound=memory_limit_kb, ) diff --git a/examples/showcase-options.py b/examples/showcase-options.py index c898dcdc7..6c6dbf5cc 100755 --- a/examples/showcase-options.py +++ b/examples/showcase-options.py @@ -172,11 +172,15 @@ def eval_dir(num): name="report-abs-p-filter", ) exp.add_report( - AbsoluteReport(attributes=["coverage", "error", "score_planner_time"], format="tex"), + AbsoluteReport( + attributes=["coverage", "error", "score_planner_time"], format="tex" + ), outfile="report-abs-combined.tex", ) exp.add_report( - AbsoluteReport(attributes=["coverage", "error", "score_planner_memory"], format="html"), + AbsoluteReport( + attributes=["coverage", "error", "score_planner_memory"], format="html" + ), outfile="report-abs-combined.html", ) exp.add_report( diff --git a/lab/reports/__init__.py b/lab/reports/__init__.py index ad0b29c70..a854d6b5a 100644 --- a/lab/reports/__init__.py +++ b/lab/reports/__init__.py @@ -36,7 +36,7 @@ def geometric_mean(values): """ assert None not in values exp = 1.0 / len(values) - return tools.product([val ** exp for val in values]) + return tools.product([val**exp for val in values]) def finite_sum(values): diff --git a/lab/tools.py b/lab/tools.py index 37ae7156a..14c707262 100644 --- a/lab/tools.py +++ b/lab/tools.py @@ -241,7 +241,7 @@ def compute_log_score(success, value, lower_bound, upper_bound): runs (!success) and worst performance (value >= upper_bound) counts as 0. """ if value is None or not success: - return 0. + return 0.0 value = max(value, lower_bound) value = min(value, upper_bound) raw_score = math.log(value) - math.log(upper_bound) diff --git a/tox.ini b/tox.ini index a65b626d0..01fd77b5f 100644 --- a/tox.ini +++ b/tox.ini @@ -48,7 +48,7 @@ commands = [testenv:style] skipsdist = true deps = - black==20.8b0 + black==22.3.0 blackdoc==0.1.2 flake8 flake8-2020 @@ -64,7 +64,7 @@ commands = [testenv:fix-style] skipsdist = true deps = - black==20.8b0 + black==22.3.0 blackdoc==0.1.2 isort>=5.0,<5.1 pyupgrade==2.18.3 From 117422d8724c88064591d4992a9a9fcb71289941 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 22 Apr 2022 09:44:30 +0200 Subject: [PATCH 12/57] Upgrade blackdoc. --- docs/autobuild.sh | 1 + downward/reports/__init__.py | 2 ++ downward/reports/scatter.py | 2 ++ lab/cached_revision.py | 1 + lab/reports/__init__.py | 4 ++++ lab/reports/filter.py | 1 + tox.ini | 4 ++-- 7 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/autobuild.sh b/docs/autobuild.sh index a4e13a5e3..38c02b21a 100755 --- a/docs/autobuild.sh +++ b/docs/autobuild.sh @@ -2,6 +2,7 @@ # Automatically rebuild Sphinx documentation when files change. DOCS="$( dirname "$0" )" +DOCS="$( realpath "$DOCS" )" REPO="$( realpath "$DOCS/../" )" cd "$REPO/docs" diff --git a/downward/reports/__init__.py b/downward/reports/__init__.py index 5c25007db..409c99132 100644 --- a/downward/reports/__init__.py +++ b/downward/reports/__init__.py @@ -75,6 +75,7 @@ def __init__(self, **kwargs): >>> # Use a filter function to select algorithms. >>> def only_blind_and_lmcut(run): ... return run["algorithm"] in ["blind", "lmcut"] + ... >>> report = PlanningReport(filter=only_blind_and_lmcut) >>> # Use "filter_algorithm" to select and *order* algorithms. @@ -96,6 +97,7 @@ def __init__(self, **kwargs): ... times = [t for t in times if t is not None] ... map[(domain, problem)] = min(times) if times else None ... return str(map) + ... """ # Set non-default options for some attributes. diff --git a/downward/reports/scatter.py b/downward/reports/scatter.py index 395cbbca1..6474b4139 100644 --- a/downward/reports/scatter.py +++ b/downward/reports/scatter.py @@ -57,6 +57,7 @@ def __init__( ... # run2['domain'] has the same value, because we always ... # compare two runs of the same problem. ... return run1["domain"] + ... Example grouping by difficulty: @@ -68,6 +69,7 @@ def __init__( ... if time1 == time2: ... return "equal" ... return "worse" + ... >>> from downward.experiment import FastDownwardExperiment >>> exp = FastDownwardExperiment() diff --git a/lab/cached_revision.py b/lab/cached_revision.py index 1d49c36d6..8c5992f79 100644 --- a/lab/cached_revision.py +++ b/lab/cached_revision.py @@ -78,6 +78,7 @@ def __init__(self, repo, rev, build_cmd, exclude=None): ... rev = "main" ... cr = CachedRevision(repo, rev, ["./build.py"], exclude=["experiments"]) ... # cr.cache(revision_cache) # Uncomment to actually cache the code. + ... You can now copy the cached repo to your experiment: diff --git a/lab/reports/__init__.py b/lab/reports/__init__.py index a854d6b5a..ea27ab3af 100644 --- a/lab/reports/__init__.py +++ b/lab/reports/__init__.py @@ -200,6 +200,7 @@ def __init__(self, attributes=None, format="html", filter=None, **kwargs): >>> def low_init_h(run): ... return run["initial_h_value"] <= 100 + ... >>> report = Report(filter=low_init_h) Only include runs from "blocks" and "barman" with a timeout: @@ -214,6 +215,7 @@ def __init__(self, attributes=None, format="html", filter=None, **kwargs): ... if expansions is not None and time: ... run["expansions_per_time"] = expansions / time ... return run + ... >>> report = Report( ... attributes=["expansions_per_time"], filter=[add_expansions_per_time] ... ) @@ -225,6 +227,7 @@ def __init__(self, attributes=None, format="html", filter=None, **kwargs): ... paper_names = {"lama11": "LAMA 2011", "fdss_sat1": "FDSS 1"} ... run["algorithm"] = paper_names[name] ... return run + ... >>> # We want LAMA 2011 to be the leftmost column. >>> # filter_* filters are evaluated last, so we use the updated @@ -471,6 +474,7 @@ def __init__(self, title="", min_wins=None, colored=False, digits=2): >>> t.add_row("prob2", {"cfg1": 15, "cfg2": 25}) >>> def remove_quotes(s): ... return s.replace('""', "") + ... >>> print(remove_quotes(str(t))) || expansions | cfg1 | cfg2 | | prob1 | 10 | 20 | diff --git a/lab/reports/filter.py b/lab/reports/filter.py index 961d688cc..0fedfc1cd 100644 --- a/lab/reports/filter.py +++ b/lab/reports/filter.py @@ -9,6 +9,7 @@ class FilterReport(Report): >>> def remove_openstacks(run): ... return "openstacks" not in run["domain"] + ... >>> from lab.experiment import Experiment >>> report = FilterReport(filter=remove_openstacks) diff --git a/tox.ini b/tox.ini index 01fd77b5f..7616104bb 100644 --- a/tox.ini +++ b/tox.ini @@ -49,7 +49,7 @@ commands = skipsdist = true deps = black==22.3.0 - blackdoc==0.1.2 + blackdoc==0.3.4 flake8 flake8-2020 flake8-bugbear @@ -65,7 +65,7 @@ commands = skipsdist = true deps = black==22.3.0 - blackdoc==0.1.2 + blackdoc==0.3.4 isort>=5.0,<5.1 pyupgrade==2.18.3 commands = From c09a50ff5ae1c1c104c6f817b7f7855d54658228 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 22 Apr 2022 10:05:18 +0200 Subject: [PATCH 13/57] Fix style. --- downward/scripts/planner-parser.py | 2 +- downward/scripts/single-search-parser.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/downward/scripts/planner-parser.py b/downward/scripts/planner-parser.py index db8c363c3..a32c7afb4 100755 --- a/downward/scripts/planner-parser.py +++ b/downward/scripts/planner-parser.py @@ -12,7 +12,7 @@ def add_planner_memory(content, props): def add_planner_time(content, props): - # Newer planner versions print the planner time and we parse it below. Don't overwrite it. + # Newer planner versions print planner time and we parse it below. Don't overwrite it. if "planner_time" not in props: return try: diff --git a/downward/scripts/single-search-parser.py b/downward/scripts/single-search-parser.py index d38bf7675..9421e632c 100755 --- a/downward/scripts/single-search-parser.py +++ b/downward/scripts/single-search-parser.py @@ -4,7 +4,6 @@ Regular expressions and functions for parsing single-search runs of Fast Downward. """ -import math import re import sys From 17e4841a25fe0f0d96d294d4117e4b6bacb56b01 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 22 Apr 2022 10:20:57 +0200 Subject: [PATCH 14/57] Update docs to account for new scores. --- docs/news.rst | 2 ++ downward/experiment.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 8685cc115..2341251f7 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -14,6 +14,8 @@ Downward Lab * Fix header sizes in HTML reports (Jendrik Seipp). * Include domains in attribute overview tables even if none of their tasks has an attribute value for all algorithms (Jendrik Seipp). +* Compute "score_planner_time" and "score_planner_memory" attributes in planner + parser (Jendrik Seipp). v7.0 (2021-10-24) diff --git a/downward/experiment.py b/downward/experiment.py index 6c88b1441..0e73e8488 100644 --- a/downward/experiment.py +++ b/downward/experiment.py @@ -125,7 +125,7 @@ class FastDownwardExperiment(Experiment): #: "translator_peak_memory", "translator_time_done". #: #: Parsed attributes: "node", "planner_memory", "planner_time", - #: "planner_wall_clock_time". + #: "planner_wall_clock_time", "score_planner_memory", "score_planner_time". PLANNER_PARSER = os.path.join(DOWNWARD_SCRIPTS_DIR, "planner-parser.py") def __init__(self, path=None, environment=None, revision_cache=None): From 7226ce50307fca0a6caf43fa5e445c4dbceff3d6 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 16 May 2022 21:35:19 +0200 Subject: [PATCH 15/57] Build suite only once. --- downward/experiment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/downward/experiment.py b/downward/experiment.py index 0e73e8488..eb24cea8d 100644 --- a/downward/experiment.py +++ b/downward/experiment.py @@ -362,6 +362,7 @@ def _add_code(self): ) def _add_runs(self): + tasks = self._get_tasks() for algo in self._algorithms.values(): - for task in self._get_tasks(): + for task in tasks: self.add_run(FastDownwardRun(self, algo, task)) From 2f83212febaa152124d474f85b43b2680cb38047 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 16 May 2022 21:38:31 +0200 Subject: [PATCH 16/57] Only consider files ending with .pddl and .sas when building suites. --- docs/news.rst | 1 + downward/experiment.py | 19 ++++++++++--------- downward/suites.py | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 2341251f7..2b2f2b8fc 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -16,6 +16,7 @@ Downward Lab attribute value for all algorithms (Jendrik Seipp). * Compute "score_planner_time" and "score_planner_memory" attributes in planner parser (Jendrik Seipp). +* Only consider files ending with ".pddl" and ".sas" when building suites (Jendrik Seipp). v7.0 (2021-10-24) diff --git a/downward/experiment.py b/downward/experiment.py index eb24cea8d..9d7e75918 100644 --- a/downward/experiment.py +++ b/downward/experiment.py @@ -176,7 +176,8 @@ def add_suite(self, benchmarks_dir, suite): """Add PDDL or SAS+ benchmarks to the experiment. *benchmarks_dir* must be a path to a benchmark directory. It must - contain domain directories, which in turn hold PDDL or SAS+ files. + contain domain directories, which in turn hold PDDL or SAS+ files + (ending with ".pddl" or ".sas"). *suite* must be a list of domain or domain:task names. :: @@ -187,14 +188,14 @@ def add_suite(self, benchmarks_dir, suite): >>> exp.add_suite(benchmarks_dir, ["rubiks-cube:p01.sas"]) One source for benchmarks is - https://github.com/aibasel/downward-benchmarks. After cloning the - repo, you can generate suites with the ``suites.py`` script. We - recommend using the suite ``optimal_strips`` for optimal STRIPS planners - and ``satisficing`` for satisficing planners:: - - # Create standard optimal planning suite. - $ path/to/downward-benchmarks/suites.py optimal_strips - ['airport', ..., 'zenotravel'] + https://github.com/aibasel/downward-benchmarks. After cloning the repo, + you can generate suites with the ``suites.py`` script. We recommend + using the suite ``optimal_strips`` for optimal STRIPS planners and + ``satisficing`` for satisficing planners:: + + # Create standard optimal planning suite. $ + path/to/downward-benchmarks/suites.py optimal_strips ['airport', + ..., 'zenotravel'] Then you can copy the generated list into your experiment script:: diff --git a/downward/suites.py b/downward/suites.py index 95ea9c286..39156726c 100644 --- a/downward/suites.py +++ b/downward/suites.py @@ -44,7 +44,7 @@ def __init__(self, benchmarks_dir, domain): [ p for p in os.listdir(directory) - if "domain" not in p and not p.endswith(".py") + if "domain" not in p and p.endswith((".pddl", ".sas")) ] ) self.problems = [ From 62bd54bd61208d3f4e8a1936ec79bd21e1417fb1 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 16 May 2022 21:59:23 +0200 Subject: [PATCH 17/57] Explicitly left-align non-numeric cells to avoid \multicolumn entries in Latex output. --- docs/news.rst | 2 ++ examples/lmcut.py | 2 +- lab/reports/__init__.py | 47 +++++++++++++++++++++-------------------- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 2b2f2b8fc..db932e6cc 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -17,6 +17,8 @@ Downward Lab * Compute "score_planner_time" and "score_planner_memory" attributes in planner parser (Jendrik Seipp). * Only consider files ending with ".pddl" and ".sas" when building suites (Jendrik Seipp). +* Explicitly left-align non-numeric cells to avoid \\multicolumn entries in Latex output + (Jendrik Seipp). v7.0 (2021-10-24) diff --git a/examples/lmcut.py b/examples/lmcut.py index 965950fa5..7dec2282b 100755 --- a/examples/lmcut.py +++ b/examples/lmcut.py @@ -52,7 +52,7 @@ exp.add_fetcher(name="fetch") # Add report step (AbsoluteReport is the standard report). -exp.add_report(AbsoluteReport(attributes=ATTRIBUTES), outfile="report.html") +exp.add_report(AbsoluteReport(attributes=ATTRIBUTES, format="html"), outfile="report.html") # Add scatter plot report step. exp.add_report( diff --git a/lab/reports/__init__.py b/lab/reports/__init__.py index ea27ab3af..01d076657 100644 --- a/lab/reports/__init__.py +++ b/lab/reports/__init__.py @@ -433,10 +433,12 @@ def _apply_filter(self): class CellFormatter: """Formating information for one cell in a table.""" - def __init__(self, bold=False, count=None, link=None): + def __init__(self, bold=False, count=None, link=None, color=None, align_right=False): self.bold = bold self.count = count self.link = link + self.color = color + self.align_right = align_right def format_value(self, value): result = str(value) @@ -446,6 +448,14 @@ def format_value(self, value): result = f"{result} ({self.count})" if self.bold: result = f"**{result}**" + if self.color: + result = f"{{{result}|color:{self.color}}}" + + if self.align_right: + result = " " + result + else: + result += " " + return result @@ -477,8 +487,8 @@ def __init__(self, title="", min_wins=None, colored=False, digits=2): ... >>> print(remove_quotes(str(t))) || expansions | cfg1 | cfg2 | - | prob1 | 10 | 20 | - | prob2 | 15 | 25 | + | prob1 | 10 | 20 | + | prob2 | 15 | 25 | >>> t.row_names ['prob1', 'prob2'] >>> t.col_names @@ -490,15 +500,15 @@ def __init__(self, title="", min_wins=None, colored=False, digits=2): >>> t.add_summary_function("SUM", sum) >>> print(remove_quotes(str(t))) || expansions | cfg1 | cfg2 | - | prob1 | 10 | 20 | - | prob2 | 15 | 25 | - | **SUM** | 25 | 45 | + | prob1 | 10 | 20 | + | prob2 | 15 | 25 | + | **SUM** | 25 | 45 | >>> t.set_column_order(["cfg2", "cfg1"]) >>> print(remove_quotes(str(t))) || expansions | cfg2 | cfg1 | - | prob1 | 20 | 10 | - | prob2 | 25 | 15 | - | **SUM** | 45 | 25 | + | prob1 | 20 | 10 | + | prob2 | 25 | 15 | + | **SUM** | 45 | 25 | """ collections.defaultdict.__init__(self, dict) @@ -768,20 +778,11 @@ def _format_cell(self, row_name, col_name, value, color=None, bold=False): default format. """ formatter = self.cell_formatters.get(row_name, {}).get(col_name) - if formatter: - return formatter.format_value(value) - - justify_right = isinstance(value, (float, int)) - - value_text = self._format_value(value) - - if color is not None: - value_text = f"{{{value_text}|color:{color}}}" - if bold: - value_text = f"**{value_text}**" - if justify_right: - value_text = " " + value_text - return value_text + if not formatter: + align_right = isinstance(value, (float, int)) or value is None or value == "?" + value = self._format_value(value) + formatter = CellFormatter(bold=bold, color=color, align_right=align_right) + return formatter.format_value(value) def _get_markup(self, cells): """ From 8acff50255aff532a274536ea058d50421f5706b Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 23 May 2022 09:28:44 +0200 Subject: [PATCH 18/57] Fix style. --- examples/lmcut.py | 4 +++- lab/reports/__init__.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/examples/lmcut.py b/examples/lmcut.py index 7dec2282b..bc65d0917 100755 --- a/examples/lmcut.py +++ b/examples/lmcut.py @@ -52,7 +52,9 @@ exp.add_fetcher(name="fetch") # Add report step (AbsoluteReport is the standard report). -exp.add_report(AbsoluteReport(attributes=ATTRIBUTES, format="html"), outfile="report.html") +exp.add_report( + AbsoluteReport(attributes=ATTRIBUTES, format="html"), outfile="report.html" +) # Add scatter plot report step. exp.add_report( diff --git a/lab/reports/__init__.py b/lab/reports/__init__.py index 01d076657..7f574c405 100644 --- a/lab/reports/__init__.py +++ b/lab/reports/__init__.py @@ -433,7 +433,9 @@ def _apply_filter(self): class CellFormatter: """Formating information for one cell in a table.""" - def __init__(self, bold=False, count=None, link=None, color=None, align_right=False): + def __init__( + self, bold=False, count=None, link=None, color=None, align_right=False + ): self.bold = bold self.count = count self.link = link @@ -779,7 +781,9 @@ def _format_cell(self, row_name, col_name, value, color=None, bold=False): """ formatter = self.cell_formatters.get(row_name, {}).get(col_name) if not formatter: - align_right = isinstance(value, (float, int)) or value is None or value == "?" + align_right = ( + isinstance(value, (float, int)) or value is None or value == "?" + ) value = self._format_value(value) formatter = CellFormatter(bold=bold, color=color, align_right=align_right) return formatter.format_value(value) From c4be4f575f81fc77b9aeadbc33311af5dd001e1a Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 20 Jun 2022 14:05:11 +0200 Subject: [PATCH 19/57] Add release date. --- docs/news.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index db932e6cc..87129763d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,12 +1,12 @@ Changelog ========= -v7.1 (unreleased) +v7.1 (2022-06-20) ----------------- Lab ^^^ -* Revamp Singularity example experiment: use runsolver to limit resource usage +* Revamp Singularity example experiment: use ``runsolver`` to limit resource usage (Silvan Sievers and Jendrik Seipp). Downward Lab From 6cac5f4aef842b793399665c86057f8faf466b39 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 20 Jun 2022 14:06:31 +0200 Subject: [PATCH 20/57] Fix release script. --- dev/release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/release.sh b/dev/release.sh index 2b8e7e44d..37e0ac189 100755 --- a/dev/release.sh +++ b/dev/release.sh @@ -23,8 +23,8 @@ if [[ $retcode != 0 ]]; then exit 1 fi -if [[ $(git rev-parse --abbrev-ref HEAD) != master ]]; then - echo "Must be on master for release" +if [[ $(git rev-parse --abbrev-ref HEAD) != main ]]; then + echo "Must be on main branch for release" exit 1 fi From 1277d8da4ea34fa8aa7b4ddaa93f8fb95f0aad04 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 20 Jun 2022 14:15:40 +0200 Subject: [PATCH 21/57] Update version number to 7.1 for release. --- lab/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lab/__init__.py b/lab/__init__.py index 238ae3865..c3ff4b00e 100644 --- a/lab/__init__.py +++ b/lab/__init__.py @@ -1,2 +1,2 @@ #: Lab version number. A "+" is appended to all non-tagged revisions. -__version__ = "7.0+" +__version__ = "7.1" From d1755816a237d9d5cbfe10bd3a2f8ba9c6544da4 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 20 Jun 2022 14:15:47 +0200 Subject: [PATCH 22/57] Update version number to 7.1+ after release. --- lab/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lab/__init__.py b/lab/__init__.py index c3ff4b00e..c99c0123e 100644 --- a/lab/__init__.py +++ b/lab/__init__.py @@ -1,2 +1,2 @@ #: Lab version number. A "+" is appended to all non-tagged revisions. -__version__ = "7.1" +__version__ = "7.1+" From 89f421656fd9f58ff55f7fc737e14f49446afedc Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sun, 3 Jul 2022 19:07:37 +0200 Subject: [PATCH 23/57] Parse peak memory in anytime search parser. --- docs/news.rst | 12 ++++++++++++ downward/scripts/anytime-search-parser.py | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 87129763d..1ea693b52 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,6 +1,18 @@ Changelog ========= +v7.2 (unreleased) +----------------- + +Lab +^^^ +* No changes so far. + +Downward Lab +^^^^^^^^^^^^ +* Parse peak memory in anytime search parser (Jendrik Seipp). + + v7.1 (2022-06-20) ----------------- diff --git a/downward/scripts/anytime-search-parser.py b/downward/scripts/anytime-search-parser.py index d312ba364..c3f59ec56 100755 --- a/downward/scripts/anytime-search-parser.py +++ b/downward/scripts/anytime-search-parser.py @@ -37,8 +37,26 @@ def coverage(content, props): props["coverage"] = int("cost" in props) +def add_memory(content, props): + """Add "memory" attribute if the run was not aborted. + + Peak memory usage is printed even for runs that are terminated + abnormally. For these runs we do not take the reported value into + account since the value is censored: it only takes into account the + memory usage until termination. + + """ + raw_memory = props.get("raw_memory") + if raw_memory is not None: + if raw_memory < 0: + props.add_unexplained_error("planner failed to log peak memory") + elif props["coverage"]: + props["memory"] = raw_memory + + def main(): parser = Parser() + parser.add_pattern("raw_memory", r"Peak memory: (.+) KB", type=int), parser.add_function(find_all_matches("cost:all", r"Plan cost: (.+)\n", type=float)) parser.add_function( find_all_matches("steps:all", r"Plan length: (.+) step\(s\).\n", type=float) From 2a34e97cfff3dc8db79c35c57885e1e3ead62ff7 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sun, 3 Jul 2022 19:13:14 +0200 Subject: [PATCH 24/57] Don't overwrite planner_time attribute. --- downward/scripts/planner-parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/downward/scripts/planner-parser.py b/downward/scripts/planner-parser.py index a32c7afb4..6acae4244 100755 --- a/downward/scripts/planner-parser.py +++ b/downward/scripts/planner-parser.py @@ -13,7 +13,7 @@ def add_planner_memory(content, props): def add_planner_time(content, props): # Newer planner versions print planner time and we parse it below. Don't overwrite it. - if "planner_time" not in props: + if "planner_time" in props: return try: props["planner_time"] = props["translator_time_done"] + props["total_time"] From 2a3d7c116ad5aeba8a2bcfd12ff2b4dffc527404 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sun, 3 Jul 2022 19:21:33 +0200 Subject: [PATCH 25/57] Only store planner_time attribute for successful planner runs. --- docs/news.rst | 1 + downward/scripts/planner-parser.py | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 1ea693b52..b4a892a45 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -11,6 +11,7 @@ Lab Downward Lab ^^^^^^^^^^^^ * Parse peak memory in anytime search parser (Jendrik Seipp). +* Only store "planner_time" attribute for successful planner runs (Jendrik Seipp). v7.1 (2022-06-20) diff --git a/downward/scripts/planner-parser.py b/downward/scripts/planner-parser.py index 6acae4244..42c4fd0ba 100755 --- a/downward/scripts/planner-parser.py +++ b/downward/scripts/planner-parser.py @@ -7,18 +7,24 @@ def add_planner_memory(content, props): try: props["planner_memory"] = max(props["translator_peak_memory"], props["memory"]) + assert props["coverage"] except KeyError: pass def add_planner_time(content, props): - # Newer planner versions print planner time and we parse it below. Don't overwrite it. - if "planner_time" in props: - return - try: - props["planner_time"] = props["translator_time_done"] + props["total_time"] - except KeyError: - pass + # Only add planner_time for successful runs. + if props["coverage"]: + # Newer planner versions print planner time and we parse it below. Don't overwrite it. + if "planner_time" not in props: + try: + props["planner_time"] = ( + props["translator_time_done"] + props["total_time"] + ) + except KeyError: + pass + elif "planner_time" in props: + del props["planner_time"] def add_planner_scores(content, props): From bb967dec17986bd8d9c18c72bf2f1ea1d41ce9fa Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sun, 3 Jul 2022 19:26:48 +0200 Subject: [PATCH 26/57] Add pre-commit file. --- .pre-commit-config.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..1a2bd6706 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +repos: + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + + - repo: https://github.com/asottile/pyupgrade + rev: v2.18.3 + hooks: + - id: pyupgrade From a4f8b60ecbe647baedfc1f5a36b78defff28b660 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sun, 3 Jul 2022 21:41:42 +0200 Subject: [PATCH 27/57] Execute add_memory() function. --- downward/scripts/anytime-search-parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/downward/scripts/anytime-search-parser.py b/downward/scripts/anytime-search-parser.py index c3f59ec56..ea11262ec 100755 --- a/downward/scripts/anytime-search-parser.py +++ b/downward/scripts/anytime-search-parser.py @@ -64,6 +64,7 @@ def main(): parser.add_function(reduce_to_min("cost:all", "cost")) parser.add_function(reduce_to_min("steps:all", "steps")) parser.add_function(coverage) + parser.add_function(add_memory) parser.parse() From 81bf276c0050e9c7c9329897546dec99dc84b8cd Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sun, 3 Jul 2022 21:53:30 +0200 Subject: [PATCH 28/57] Add flake8 to pre-commit file. --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1a2bd6706..8829e523e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,12 @@ repos: hooks: - id: black + - repo: https://github.com/pycqa/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + args: ['--extend-ignore=E203', '--exclude=build,data,revision-cache,conf.py,.git,.tox,.venv', '--max-line-length=90'] + - repo: https://github.com/asottile/pyupgrade rev: v2.18.3 hooks: From d44d6f97950283b1d570cec4dbf6fbe2cb83e312 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sun, 3 Jul 2022 21:53:38 +0200 Subject: [PATCH 29/57] Fix style. --- downward/scripts/planner-parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/downward/scripts/planner-parser.py b/downward/scripts/planner-parser.py index 42c4fd0ba..388aabe45 100755 --- a/downward/scripts/planner-parser.py +++ b/downward/scripts/planner-parser.py @@ -15,7 +15,8 @@ def add_planner_memory(content, props): def add_planner_time(content, props): # Only add planner_time for successful runs. if props["coverage"]: - # Newer planner versions print planner time and we parse it below. Don't overwrite it. + # Newer planner versions print planner time and we parse it below. + # Don't overwrite it. if "planner_time" not in props: try: props["planner_time"] = ( From b4d5599d9ceb9f62fcf0354722c0eb98192677dd Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sun, 3 Jul 2022 22:14:14 +0200 Subject: [PATCH 30/57] Only store planner_memory attribute for successful planner runs. --- docs/news.rst | 3 ++- downward/scripts/planner-parser.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index b4a892a45..af9b5cd92 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -11,7 +11,8 @@ Lab Downward Lab ^^^^^^^^^^^^ * Parse peak memory in anytime search parser (Jendrik Seipp). -* Only store "planner_time" attribute for successful planner runs (Jendrik Seipp). +* Only store "planner_memory" and "planner_time" attributes for successful planner + runs (Jendrik Seipp). v7.1 (2022-06-20) diff --git a/downward/scripts/planner-parser.py b/downward/scripts/planner-parser.py index 388aabe45..89ff157b6 100755 --- a/downward/scripts/planner-parser.py +++ b/downward/scripts/planner-parser.py @@ -5,11 +5,14 @@ def add_planner_memory(content, props): - try: - props["planner_memory"] = max(props["translator_peak_memory"], props["memory"]) - assert props["coverage"] - except KeyError: - pass + # Only add planner_memory for successful runs. + if props["coverage"]: + try: + props["planner_memory"] = max( + props["translator_peak_memory"], props["memory"] + ) + except KeyError: + pass def add_planner_time(content, props): From ae70fd793a79e9cb2665a5b6aef38990bee4a65e Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Tue, 26 Jul 2022 16:14:50 +0200 Subject: [PATCH 31/57] Add flexible downward example experiment. --- .github/workflows/ubuntu.yml | 2 +- docs/news.rst | 1 + examples/downward/2020-09-11-A-cg-vs-ff.py | 6 +- .../downward/2020-09-11-B-bounded-cost.py | 114 ++++++++++++++++++ examples/downward/bounds.json | 5 + tests/run-downward-experiment | 15 +++ tests/run-example-experiment | 6 +- tox.ini | 12 +- 8 files changed, 156 insertions(+), 5 deletions(-) create mode 100755 examples/downward/2020-09-11-B-bounded-cost.py create mode 100644 examples/downward/bounds.json create mode 100755 tests/run-downward-experiment diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 8b03918e4..473f30054 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -134,7 +134,7 @@ jobs: echo CACHE: ${DOWNWARD_REVISION_CACHE} export DOWNWARD_REPO=${DOWNWARD_REPO} - time tox -e py,ff,singularity + time tox -e py,downward,ff,singularity - name: Test installation with pip run: | diff --git a/docs/news.rst b/docs/news.rst index af9b5cd92..fd366e3df 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -13,6 +13,7 @@ Downward Lab * Parse peak memory in anytime search parser (Jendrik Seipp). * Only store "planner_memory" and "planner_time" attributes for successful planner runs (Jendrik Seipp). +* Add flexible example experiment for planners based on Fast Downward (Jendrik Seipp). v7.1 (2022-06-20) diff --git a/examples/downward/2020-09-11-A-cg-vs-ff.py b/examples/downward/2020-09-11-A-cg-vs-ff.py index b022ed033..0fd007bac 100755 --- a/examples/downward/2020-09-11-A-cg-vs-ff.py +++ b/examples/downward/2020-09-11-A-cg-vs-ff.py @@ -10,6 +10,8 @@ BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] SCP_LOGIN = "myname@myserver.com" REMOTE_REPOS_DIR = "/infai/seipp/projects" +# If REVISION_CACHE is None, the default ./data/revision-cache is used. +REVISION_CACHE = os.environ.get("DOWNWARD_REVISION_CACHE") if project.REMOTE: SUITE = project.SUITE_SATISFICING ENV = project.BaselSlurmEnvironment(email="my.name@myhost.ch") @@ -30,7 +32,7 @@ BUILD_OPTIONS = [] DRIVER_OPTIONS = ["--overall-time-limit", "5m"] REVS = [ - ("release-20.06.0", "20.06"), + ("main", "main"), ] ATTRIBUTES = [ "error", @@ -45,7 +47,7 @@ project.EVALUATIONS_PER_TIME, ] -exp = project.FastDownwardExperiment(environment=ENV) +exp = project.FastDownwardExperiment(environment=ENV, revision_cache=REVISION_CACHE) for config_nick, config in CONFIGS: for rev, rev_nick in REVS: algo_name = f"{rev_nick}:{config_nick}" if rev_nick else config_nick diff --git a/examples/downward/2020-09-11-B-bounded-cost.py b/examples/downward/2020-09-11-B-bounded-cost.py new file mode 100755 index 000000000..fd3efa504 --- /dev/null +++ b/examples/downward/2020-09-11-B-bounded-cost.py @@ -0,0 +1,114 @@ +#! /usr/bin/env python + +import json +import os +import shutil + +from downward import suites +from downward.cached_revision import CachedFastDownwardRevision +from downward.experiment import ( + _DownwardAlgorithm, + _get_solver_resource_name, + FastDownwardRun, +) +from lab.experiment import Experiment + +import project + + +REPO = project.get_repo_base() +BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] +SCP_LOGIN = "myname@myserver.com" +REMOTE_REPOS_DIR = "/infai/seipp/projects" +BOUNDS_FILE = "bounds.json" +SUITE = ["depot:p01.pddl", "grid:prob01.pddl", "gripper:prob01.pddl"] +# If REVISION_CACHE is None, the default ./data/revision-cache is used. +REVISION_CACHE = os.environ.get("DOWNWARD_REVISION_CACHE") +if project.REMOTE: + ENV = project.BaselSlurmEnvironment(email="my.name@myhost.ch") +else: + ENV = project.LocalEnvironment(processes=2) + +CONFIGS = [ + ("ff", ["--search", "lazy_greedy([ff()], bound=BOUND)"]), +] +BUILD_OPTIONS = [] +DRIVER_OPTIONS = [ + "--validate", + "--overall-time-limit", + "5m", + "--overall-memory-limit", + "3584M", +] +# Pairs of revision identifier and revision nick. +REVS = [ + ("main", "main"), +] +ATTRIBUTES = [ + "error", + "run_dir", + "search_start_time", + "search_start_memory", + "total_time", + "h_values", + "coverage", + "expansions", + "memory", + project.EVALUATIONS_PER_TIME, +] + +exp = Experiment(environment=ENV) +for rev, rev_nick in REVS: + cached_rev = CachedFastDownwardRevision(REPO, rev, BUILD_OPTIONS) + cached_rev.cache(REVISION_CACHE) + cache_path = os.path.join(REVISION_CACHE, cached_rev.name) + dest_path = "code-" + cached_rev.name + exp.add_resource("", cache_path, dest_path) + # Overwrite the script to set an environment variable. + exp.add_resource( + _get_solver_resource_name(cached_rev), + os.path.join(cache_path, "fast-downward.py"), + os.path.join(dest_path, "fast-downward.py"), + ) + for config_nick, config in CONFIGS: + algo_name = f"{rev_nick}-{config_nick}" + + bounds = {} + with open(BOUNDS_FILE) as f: + bounds = json.load(f) + for task in suites.build_suite(BENCHMARKS_DIR, SUITE): + upper_bound = bounds[f"{task.domain}:{task.problem}"] + if upper_bound is None: + upper_bound = "infinity" + config_with_bound = config.copy() + config_with_bound[-1] = config_with_bound[-1].replace( + "bound=BOUND", f"bound={upper_bound}" + ) + algo = _DownwardAlgorithm( + algo_name, + cached_rev, + DRIVER_OPTIONS, + config_with_bound, + ) + run = FastDownwardRun(exp, algo, task) + exp.add_run(run) + +exp.add_parser(project.FastDownwardExperiment.EXITCODE_PARSER) +exp.add_parser(project.FastDownwardExperiment.TRANSLATOR_PARSER) +exp.add_parser(project.FastDownwardExperiment.SINGLE_SEARCH_PARSER) +exp.add_parser(project.DIR / "parser.py") +exp.add_parser(project.FastDownwardExperiment.PLANNER_PARSER) + +exp.add_step("build", exp.build) +exp.add_step("start", exp.start_runs) +exp.add_fetcher(name="fetch") + +if not project.REMOTE: + exp.add_step("remove-eval-dir", shutil.rmtree, exp.eval_dir, ignore_errors=True) + project.add_scp_step(exp, SCP_LOGIN, REMOTE_REPOS_DIR) + +project.add_absolute_report( + exp, attributes=ATTRIBUTES, filter=[project.add_evaluations_per_time] +) + +exp.run_steps() diff --git a/examples/downward/bounds.json b/examples/downward/bounds.json new file mode 100644 index 000000000..f2e49019e --- /dev/null +++ b/examples/downward/bounds.json @@ -0,0 +1,5 @@ +{ + "depot:p01.pddl": 16, + "grid:prob01.pddl": 20, + "gripper:prob01.pddl": 25 +} diff --git a/tests/run-downward-experiment b/tests/run-downward-experiment new file mode 100755 index 000000000..10a077de8 --- /dev/null +++ b/tests/run-downward-experiment @@ -0,0 +1,15 @@ +#! /bin/bash + +set -euo pipefail + +DIR=$(dirname ${0}) +DIR=$(realpath ${DIR}) +REPO=$(dirname ${DIR}) +EXPDIR=${DOWNWARD_REPO}/experiments/tmp-downward-lab-project + +rm -rf ${EXPDIR}/data/2020-09-11-* +mkdir -p ${EXPDIR} +cp ${REPO}/examples/downward/*.py ${EXPDIR} +cp ${REPO}/examples/downward/bounds.json ${EXPDIR} +${DIR}/run-example-experiment ${EXPDIR}/2020-09-11-A-cg-vs-ff.py +${DIR}/run-example-experiment ${EXPDIR}/2020-09-11-B-bounded-cost.py diff --git a/tests/run-example-experiment b/tests/run-example-experiment index dd4099042..4805e1e9a 100755 --- a/tests/run-example-experiment +++ b/tests/run-example-experiment @@ -4,7 +4,11 @@ set -euo pipefail check () { expname="$1" - "./${expname}.py" --all + if [[ "${expname}" == *"2020-09-11-"* ]]; then + "./${expname}.py" 1 2 3 6 + else + "./${expname}.py" --all + fi properties="data/$expname-eval/properties" if [[ ! -f "$properties" ]]; then echo "File not found: $properties" diff --git a/tox.ini b/tox.ini index 7616104bb..1071848e5 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,16 @@ passenv = whitelist_externals = bash +[testenv:downward] +commands = + bash {toxinidir}/tests/run-downward-experiment +passenv = + DOWNWARD_BENCHMARKS + DOWNWARD_REPO + DOWNWARD_REVISION_CACHE +whitelist_externals = + bash + [testenv:singularity] commands = bash {toxinidir}/tests/run-example-experiment singularity/singularity-exp.py @@ -50,7 +60,7 @@ skipsdist = true deps = black==22.3.0 blackdoc==0.3.4 - flake8 + flake8==4.0.1 flake8-2020 flake8-bugbear flake8-comprehensions From 9791571005d70b0190a2b149fc0d96e449886d01 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Wed, 27 Jul 2022 23:59:06 +0200 Subject: [PATCH 32/57] Document pre-commit hook. --- CONTRIBUTING.md | 17 ++++++++++++----- docs/news.rst | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 705656480..fc28f9886 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,6 +23,17 @@ from GitHub in editable mode: For details on how to set everything up, please see the [GitHub actions file](.github/workflows/ubuntu.yml). +# Setting up a pre-commit hook for style checks + + python3 -m pip install pre-commit + pre-commit install + +Now the most important style checks are run for the changed files before each commit. + +# Fixing the code style + + tox -e fix-style + # Running tests cd lab @@ -59,8 +70,4 @@ Now you can run the example Singularity experiment with `tox -e singularity`. ## Run all tests -Once you have installed all dependecies, you can run all tests by executing `tox` without any options. - -# Fixing the code style - - tox -e fix-style +Once you have installed all dependencies, you can run all tests by executing `tox` without any options. diff --git a/docs/news.rst b/docs/news.rst index fd366e3df..1bfdb2c04 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,7 +6,7 @@ v7.2 (unreleased) Lab ^^^ -* No changes so far. +* For developers: document pre-commit hook in ``CONTRIBUTING.md`` file (Jendrik Seipp). Downward Lab ^^^^^^^^^^^^ From 12e786096d65ae87eb523df1e7e4c81d439817bc Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 22 Aug 2022 21:49:27 +0200 Subject: [PATCH 33/57] Touch up example downward experiment. --- docs/news.rst | 2 +- .../downward/2020-09-11-B-bounded-cost.py | 16 +++++++---- examples/downward/project.py | 28 ++++++++++++++++++- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 1bfdb2c04..0df4ca6e2 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,7 +6,7 @@ v7.2 (unreleased) Lab ^^^ -* For developers: document pre-commit hook in ``CONTRIBUTING.md`` file (Jendrik Seipp). +* For contributors: document pre-commit hook in ``CONTRIBUTING.md`` file (Jendrik Seipp). Downward Lab ^^^^^^^^^^^^ diff --git a/examples/downward/2020-09-11-B-bounded-cost.py b/examples/downward/2020-09-11-B-bounded-cost.py index fd3efa504..43391f8ed 100755 --- a/examples/downward/2020-09-11-B-bounded-cost.py +++ b/examples/downward/2020-09-11-B-bounded-cost.py @@ -2,6 +2,7 @@ import json import os +import os.path import shutil from downward import suites @@ -11,7 +12,7 @@ _get_solver_resource_name, FastDownwardRun, ) -from lab.experiment import Experiment +from lab.experiment import Experiment, get_default_data_dir import project @@ -22,10 +23,15 @@ REMOTE_REPOS_DIR = "/infai/seipp/projects" BOUNDS_FILE = "bounds.json" SUITE = ["depot:p01.pddl", "grid:prob01.pddl", "gripper:prob01.pddl"] -# If REVISION_CACHE is None, the default ./data/revision-cache is used. -REVISION_CACHE = os.environ.get("DOWNWARD_REVISION_CACHE") +REVISION_CACHE = os.environ.get("DOWNWARD_REVISION_CACHE") or os.path.join( + get_default_data_dir(), "revision-cache" +) if project.REMOTE: - ENV = project.BaselSlurmEnvironment(email="my.name@myhost.ch") + # ENV = project.BaselSlurmEnvironment(email="my.name@myhost.ch") + ENV = project.TetralithEnvironment( + email="first.last@liu.se", extra_options="#SBATCH --account=snic2022-5-341" + ) + SUITE = project.SUITE_OPTIMAL_STRIPS else: ENV = project.LocalEnvironment(processes=2) @@ -71,7 +77,7 @@ os.path.join(dest_path, "fast-downward.py"), ) for config_nick, config in CONFIGS: - algo_name = f"{rev_nick}-{config_nick}" + algo_name = f"{rev_nick}-{config_nick}" if rev_nick else config_nick bounds = {} with open(BOUNDS_FILE) as f: diff --git a/examples/downward/project.py b/examples/downward/project.py index 296026a1c..e4b4e2139 100644 --- a/examples/downward/project.py +++ b/examples/downward/project.py @@ -1,5 +1,6 @@ from pathlib import Path import platform +import re import subprocess import sys @@ -30,7 +31,10 @@ DIR = Path(__file__).resolve().parent NODE = platform.node() -REMOTE = NODE.endswith((".scicore.unibas.ch", ".cluster.bc2.ch")) +# Cover both the Basel and Linköping clusters for simplicity. +REMOTE = NODE.endswith((".scicore.unibas.ch", ".cluster.bc2.ch")) or re.match( + r"tetralith\d+\.nsc\.liu\.se|n\d+", NODE +) def parse_args(): @@ -79,6 +83,28 @@ def parse_args(): "visitall-sat11-strips", "visitall-sat14-strips", "woodworking-sat08-strips", "woodworking-sat11-strips", "zenotravel", ] + +SUITE_OPTIMAL_STRIPS = [ + "agricola-opt18-strips", "airport", "barman-opt11-strips", + "barman-opt14-strips", "blocks", "childsnack-opt14-strips", + "data-network-opt18-strips", "depot", "driverlog", "elevators-opt08-strips", + "elevators-opt11-strips", "floortile-opt11-strips", "floortile-opt14-strips", + "freecell", "ged-opt14-strips", "grid", "gripper", "hiking-opt14-strips", + "logistics00", "logistics98", "miconic", "movie", "mprime", "mystery", + "nomystery-opt11-strips", "openstacks-opt08-strips", "openstacks-opt11-strips", + "openstacks-opt14-strips", "openstacks-strips", "organic-synthesis-opt18-strips", + "organic-synthesis-split-opt18-strips", "parcprinter-08-strips", + "parcprinter-opt11-strips", "parking-opt11-strips", "parking-opt14-strips", + "pathways", "pegsol-08-strips", "pegsol-opt11-strips", + "petri-net-alignment-opt18-strips", "pipesworld-notankage", "pipesworld-tankage", + "psr-small", "rovers", "satellite", "scanalyzer-08-strips", + "scanalyzer-opt11-strips", "snake-opt18-strips", "sokoban-opt08-strips", + "sokoban-opt11-strips", "spider-opt18-strips", "storage", "termes-opt18-strips", + "tetris-opt14-strips", "tidybot-opt11-strips", "tidybot-opt14-strips", "tpp", + "transport-opt08-strips", "transport-opt11-strips", "transport-opt14-strips", + "trucks-strips", "visitall-opt11-strips", "visitall-opt14-strips", + "woodworking-opt08-strips", "woodworking-opt11-strips", "zenotravel", +] # fmt: on From 5af66d9319eb4a378d001d5dfb778e4ab565fba6 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Wed, 5 Oct 2022 18:28:42 +0200 Subject: [PATCH 34/57] Show how to group domain directories in example Fast Downward experiment --- docs/news.rst | 1 + .../downward/2020-09-11-B-bounded-cost.py | 4 +- examples/downward/project.py | 116 ++++++++++++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 0df4ca6e2..8b721b7b1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -14,6 +14,7 @@ Downward Lab * Only store "planner_memory" and "planner_time" attributes for successful planner runs (Jendrik Seipp). * Add flexible example experiment for planners based on Fast Downward (Jendrik Seipp). +* Show how to group domain directories in example Fast Downward experiment (Jendrik Seipp). v7.1 (2022-06-20) diff --git a/examples/downward/2020-09-11-B-bounded-cost.py b/examples/downward/2020-09-11-B-bounded-cost.py index 43391f8ed..95d20934d 100755 --- a/examples/downward/2020-09-11-B-bounded-cost.py +++ b/examples/downward/2020-09-11-B-bounded-cost.py @@ -114,7 +114,9 @@ project.add_scp_step(exp, SCP_LOGIN, REMOTE_REPOS_DIR) project.add_absolute_report( - exp, attributes=ATTRIBUTES, filter=[project.add_evaluations_per_time] + exp, + attributes=ATTRIBUTES, + filter=[project.add_evaluations_per_time, project.group_domains], ) exp.run_steps() diff --git a/examples/downward/project.py b/examples/downward/project.py index e4b4e2139..0d2076d26 100644 --- a/examples/downward/project.py +++ b/examples/downward/project.py @@ -105,9 +105,125 @@ def parse_args(): "trucks-strips", "visitall-opt11-strips", "visitall-opt14-strips", "woodworking-opt08-strips", "woodworking-opt11-strips", "zenotravel", ] + +DOMAIN_GROUPS = { + "airport": ["airport"], + "assembly": ["assembly"], + "barman": [ + "barman", "barman-opt11-strips", "barman-opt14-strips", + "barman-sat11-strips", "barman-sat14-strips"], + "blocksworld": ["blocks", "blocksworld"], + "cavediving": ["cavediving-14-adl"], + "childsnack": ["childsnack-opt14-strips", "childsnack-sat14-strips"], + "citycar": ["citycar-opt14-adl", "citycar-sat14-adl"], + "depots": ["depot", "depots"], + "driverlog": ["driverlog"], + "elevators": [ + "elevators-opt08-strips", "elevators-opt11-strips", + "elevators-sat08-strips", "elevators-sat11-strips"], + "floortile": [ + "floortile-opt11-strips", "floortile-opt14-strips", + "floortile-sat11-strips", "floortile-sat14-strips"], + "freecell": ["freecell"], + "ged": ["ged-opt14-strips", "ged-sat14-strips"], + "grid": ["grid"], + "gripper": ["gripper"], + "hiking": ["hiking-opt14-strips", "hiking-sat14-strips"], + "logistics": ["logistics98", "logistics00"], + "maintenance": ["maintenance-opt14-adl", "maintenance-sat14-adl"], + "miconic": ["miconic", "miconic-strips"], + "miconic-fulladl": ["miconic-fulladl"], + "miconic-simpleadl": ["miconic-simpleadl"], + "movie": ["movie"], + "mprime": ["mprime"], + "mystery": ["mystery"], + "nomystery": ["nomystery-opt11-strips", "nomystery-sat11-strips"], + "openstacks": [ + "openstacks", "openstacks-strips", "openstacks-opt08-strips", + "openstacks-opt11-strips", "openstacks-opt14-strips", + "openstacks-sat08-adl", "openstacks-sat08-strips", + "openstacks-sat11-strips", "openstacks-sat14-strips", + "openstacks-opt08-adl", "openstacks-sat08-adl"], + "optical-telegraphs": ["optical-telegraphs"], + "parcprinter": [ + "parcprinter-08-strips", "parcprinter-opt11-strips", "parcprinter-sat11-strips"], + "parking": [ + "parking-opt11-strips", "parking-opt14-strips", + "parking-sat11-strips", "parking-sat14-strips"], + "pathways": ["pathways"], + "pathways-noneg": ["pathways-noneg"], + "pegsol": ["pegsol-08-strips", "pegsol-opt11-strips", "pegsol-sat11-strips"], + "philosophers": ["philosophers"], + "pipes-nt": ["pipesworld-notankage"], + "pipes-t": ["pipesworld-tankage"], + "psr": ["psr-middle", "psr-large", "psr-small"], + "rovers": ["rover", "rovers"], + "satellite": ["satellite"], + "scanalyzer": [ + "scanalyzer-08-strips", "scanalyzer-opt11-strips", "scanalyzer-sat11-strips"], + "schedule": ["schedule"], + "sokoban": [ + "sokoban-opt08-strips", "sokoban-opt11-strips", + "sokoban-sat08-strips", "sokoban-sat11-strips"], + "storage": ["storage"], + "tetris": ["tetris-opt14-strips", "tetris-sat14-strips"], + "thoughtful": ["thoughtful-sat14-strips"], + "tidybot": [ + "tidybot-opt11-strips", "tidybot-opt14-strips", + "tidybot-sat11-strips", "tidybot-sat14-strips"], + "tpp": ["tpp"], + "transport": [ + "transport-opt08-strips", "transport-opt11-strips", "transport-opt14-strips", + "transport-sat08-strips", "transport-sat11-strips", "transport-sat14-strips"], + "trucks": ["trucks", "trucks-strips"], + "visitall": [ + "visitall-opt11-strips", "visitall-opt14-strips", + "visitall-sat11-strips", "visitall-sat14-strips"], + "woodworking": [ + "woodworking-opt08-strips", "woodworking-opt11-strips", + "woodworking-sat08-strips", "woodworking-sat11-strips"], + "zenotravel": ["zenotravel"], + # IPC 2018: + "agricola": ["agricola", "agricola-opt18-strips", "agricola-sat18-strips"], + "caldera": ["caldera-opt18-adl", "caldera-sat18-adl"], + "caldera-split": ["caldera-split-opt18-adl", "caldera-split-sat18-adl"], + "data-network": [ + "data-network", "data-network-opt18-strips", "data-network-sat18-strips"], + "flashfill": ["flashfill-sat18-adl"], + "nurikabe": ["nurikabe-opt18-adl", "nurikabe-sat18-adl"], + "organic-split": [ + "organic-synthesis-split", "organic-synthesis-split-opt18-strips", + "organic-synthesis-split-sat18-strips"], + "organic" : [ + "organic-synthesis", "organic-synthesis-opt18-strips", + "organic-synthesis-sat18-strips"], + "petri-net": [ + "petri-net-alignment", "petri-net-alignment-opt18-strips", + "petri-net-alignment-sat18-strips"], + "settlers": ["settlers-opt18-adl", "settlers-sat18-adl"], + "snake": ["snake", "snake-opt18-strips", "snake-sat18-strips"], + "spider": ["spider", "spider-opt18-strips", "spider-sat18-strips"], + "termes": ["termes", "termes-opt18-strips", "termes-sat18-strips"], +} # fmt: on +DOMAIN_RENAMINGS = {} +for group_name, domains in DOMAIN_GROUPS.items(): + for domain in domains: + DOMAIN_RENAMINGS[domain] = group_name +for group_name in DOMAIN_GROUPS: + DOMAIN_RENAMINGS[group_name] = group_name + + +def group_domains(run): + old_domain = run["domain"] + run["domain"] = DOMAIN_RENAMINGS[old_domain] + run["problem"] = old_domain + "-" + run["problem"] + run["id"][2] = run["problem"] + return run + + def get_repo_base() -> Path: """Get base directory of the repository, as an absolute path. From 822def7189440e7a4c4405e0cb8e940b78a21734 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Wed, 5 Oct 2022 18:40:09 +0200 Subject: [PATCH 35/57] Run continuous integration tests for Python 3.10 and Ubuntu 22.04. (#105) --- .github/workflows/ubuntu.yml | 10 ++-------- docs/news.rst | 1 + lab/tools.py | 5 +---- setup.py | 1 + tox.ini | 2 +- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 473f30054..3cb9a7b07 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -1,7 +1,5 @@ name: Ubuntu -# see https://github.community/t5/GitHub-Actions/How-to-trigger-an-action-on-push-or-pull-request-but-not-both/m-p/35805 -# and https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/6 on: [push, pull_request] jobs: @@ -9,14 +7,10 @@ jobs: runs-on: ${{ matrix.os }} - # We want to run on external PRs, but not on our own internal PRs as they'll be run - # by the push to the branch. - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - strategy: matrix: - os: [ubuntu-18.04, ubuntu-20.04] - python-version: [3.6, 3.7, 3.8, 3.9] + os: [ubuntu-20.04, ubuntu-22.04] + python-version: [3.6, 3.7, 3.8, 3.9, "3.10"] steps: - uses: actions/checkout@v2 diff --git a/docs/news.rst b/docs/news.rst index 8b721b7b1..57979cff1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -24,6 +24,7 @@ Lab ^^^ * Revamp Singularity example experiment: use ``runsolver`` to limit resource usage (Silvan Sievers and Jendrik Seipp). +* Run continuous integration tests for Python 3.10 (Jendrik Seipp). Downward Lab ^^^^^^^^^^^^ diff --git a/lab/tools.py b/lab/tools.py index 14c707262..5f7529966 100644 --- a/lab/tools.py +++ b/lab/tools.py @@ -503,10 +503,7 @@ def get_unexplained_errors_message(run): if not unexplained_errors or unexplained_errors == ["output-to-slurm.err"]: return "" else: - return ( - f"Unexplained error(s) in {run['run_dir']}: please inspect" - f" output and error logs." - ) + return f"Unexplained error(s) in {run['run_dir']}: {unexplained_errors}" def get_slurm_err_content(src_dir): diff --git a/setup.py b/setup.py index 391ecbb36..e57e8dc83 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Topic :: Scientific/Engineering", ], install_requires=[ diff --git a/tox.ini b/tox.ini index 1071848e5..923b90f17 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, py38, py39, ff, singularity, style, docs +envlist = py36, py37, py38, py39, py310, ff, singularity, style, docs basepython = python3 skip_missing_interpreters = true From d0cc29007b0bfa4490fa30515cb81f02103053f7 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Wed, 5 Oct 2022 18:43:22 +0200 Subject: [PATCH 36/57] Fix news file. --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 57979cff1..8a053e166 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,6 +6,7 @@ v7.2 (unreleased) Lab ^^^ +* Run continuous integration tests for Python 3.10 (Jendrik Seipp). * For contributors: document pre-commit hook in ``CONTRIBUTING.md`` file (Jendrik Seipp). Downward Lab @@ -24,7 +25,6 @@ Lab ^^^ * Revamp Singularity example experiment: use ``runsolver`` to limit resource usage (Silvan Sievers and Jendrik Seipp). -* Run continuous integration tests for Python 3.10 (Jendrik Seipp). Downward Lab ^^^^^^^^^^^^ From 20ab71917a5ddcdd42d0f7059ffdde1037f9077c Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 7 Oct 2022 10:14:35 +0200 Subject: [PATCH 37/57] Raise minimum Python version to 3.7. --- .github/workflows/ubuntu.yml | 2 +- INSTALL.rst | 4 ++-- docs/downward.tutorial.rst | 2 +- docs/news.rst | 3 ++- setup.py | 3 +-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 3cb9a7b07..a88c8fdcb 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04, ubuntu-22.04] - python-version: [3.6, 3.7, 3.8, 3.9, "3.10"] + python-version: [3.7, 3.8, 3.9, "3.10"] steps: - uses: actions/checkout@v2 diff --git a/INSTALL.rst b/INSTALL.rst index a6cd62938..05ba9d737 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -1,7 +1,7 @@ Install Lab ----------- -Lab requires Python 3.6+ and Linux (e.g., Ubuntu). We recommend installing +Lab requires Python 3.7+ and Linux (e.g., Ubuntu). We recommend installing Lab in a `Python virtual environment `_. This has the advantage that there are no modifications to the system-wide configuration, and that @@ -18,7 +18,7 @@ for different papers) without conflicts:: # If PYTHONPATH is set, unset it to obtain a clean environment. unset PYTHONPATH - # Create and activate a Python 3 virtual environment for Lab. + # Create and activate a Python virtual environment for Lab. python3 -m venv --prompt my-paper .venv source .venv/bin/activate diff --git a/docs/downward.tutorial.rst b/docs/downward.tutorial.rst index 999aadfed..4e1879ed8 100644 --- a/docs/downward.tutorial.rst +++ b/docs/downward.tutorial.rst @@ -22,7 +22,7 @@ alternative that has proven to work well in practice. Installation ------------ -Lab requires **Python 3.6+** and **Linux**. To run Fast Downward +Lab requires **Python 3.7+** and **Linux**. To run Fast Downward experiments, you'll need a **Fast Downward** repository, planning **benchmarks** and a plan **validator**. :: diff --git a/docs/news.rst b/docs/news.rst index 8a053e166..1f43e6150 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,7 +6,8 @@ v7.2 (unreleased) Lab ^^^ -* Run continuous integration tests for Python 3.10 (Jendrik Seipp). +* Raise minimum supported Python version to 3.7 (Jendrik Seipp). +* Add support for Python 3.10 (Jendrik Seipp). * For contributors: document pre-commit hook in ``CONTRIBUTING.md`` file (Jendrik Seipp). Downward Lab diff --git a/setup.py b/setup.py index e57e8dc83..a31f98977 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ "Intended Audience :: Science/Research", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -40,5 +39,5 @@ "simplejson", # optional, speeds up reading properties files "txt2tags>=3.6", # for HTML and Latex reports ], - python_requires=">=3.6", + python_requires=">=3.7", ) From d764cc50e624573ff361212a291527498f936adf Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 7 Oct 2022 18:25:53 +0200 Subject: [PATCH 38/57] Fix CI (#107) * Use precompiled FF binary. * Bump pyupgrade version and remove py36 mentions. --- .github/workflows/ubuntu.yml | 20 ++++++++++++-------- pyproject.toml | 2 +- tests/check-style | 2 +- tox.ini | 8 ++++---- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index a88c8fdcb..749e2763b 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -57,14 +57,18 @@ jobs: - name: Compile FF working-directory: ../deps run: | - sudo apt-get -y install g++ make flex bison - wget http://fai.cs.uni-saarland.de/hoffmann/ff/FF-v2.3.tgz - tar -xzvf FF-v2.3.tgz - pushd FF-v2.3/ - make -j - cp ff ../ - popd - rm -r FF-v2.3/ FF-v2.3.tgz + # GCC 10 fails to compile FF 2.3, so we use a precompiled FF binary. + # sudo apt-get -y install g++ make flex bison + # wget http://fai.cs.uni-saarland.de/hoffmann/ff/FF-v2.3.tgz + # tar -xzvf FF-v2.3.tgz + # pushd FF-v2.3/ + # make -j + # cp ff ../ + # popd + # rm -r FF-v2.3/ FF-v2.3.tgz + wget 'https://github.com/hectorpal/fast-forward-linux-binaries/raw/main/ff.gz' + gunzip ff.gz + chmod +x ff - name: Compile runsolver working-directory: ../deps diff --git a/pyproject.toml b/pyproject.toml index a2f6a5ffd..e5f3f7ff9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.black] line-length = 88 -target-version = ['py36', 'py37', 'py38'] +target-version = ['py37', 'py38', 'py39', 'py310'] include = '\.pyi?$' exclude = ''' /( diff --git a/tests/check-style b/tests/check-style index e1526669b..b2d6206cd 100755 --- a/tests/check-style +++ b/tests/check-style @@ -7,4 +7,4 @@ python -m blackdoc --check . # E203: whitespace before ':' (not compliant with PEP 8) python -m flake8 --extend-ignore E203 --exclude=build,data,revision-cache,conf.py,.git,.tox,.venv --max-line-length=90 python -m isort --check-only downward/ examples/ lab/ tests/ setup.py -python -m pyupgrade --py36-plus `find downward lab tests -name "*.py"` +python -m pyupgrade --py37-plus `find downward lab tests -name "*.py"` diff --git a/tox.ini b/tox.ini index 923b90f17..4d51a80a8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, py38, py39, py310, ff, singularity, style, docs +envlist = py37, py38, py39, py310, ff, singularity, style, docs basepython = python3 skip_missing_interpreters = true @@ -65,7 +65,7 @@ deps = flake8-bugbear flake8-comprehensions isort>=5.0,<5.1 - pyupgrade==2.18.3 + pyupgrade==3.0.0 vulture commands = bash {toxinidir}/tests/find-dead-code @@ -77,11 +77,11 @@ deps = black==22.3.0 blackdoc==0.3.4 isort>=5.0,<5.1 - pyupgrade==2.18.3 + pyupgrade==3.0.0 commands = black . blackdoc . isort downward/ examples/ lab/ tests/ setup.py - bash -c 'pyupgrade --py36-plus --exit-zero `find downward lab tests -name "*.py"`' + bash -c 'pyupgrade --py37-plus --exit-zero `find downward lab tests -name "*.py"`' whitelist_externals = bash From d5332684c46f4f21cc5506c8cd38578c5d9ebf15 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 7 Oct 2022 18:51:55 +0200 Subject: [PATCH 39/57] Apply parsing functions in the order in which they were added. --- docs/news.rst | 1 + lab/parser.py | 29 ++++++++++++++--------------- tox.ini | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 1f43e6150..aa296ad68 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,6 +8,7 @@ Lab ^^^ * Raise minimum supported Python version to 3.7 (Jendrik Seipp). * Add support for Python 3.10 (Jendrik Seipp). +* Apply parsing functions in the order in which they were added (Jendrik Seipp). * For contributors: document pre-commit hook in ``CONTRIBUTING.md`` file (Jendrik Seipp). Downward Lab diff --git a/lab/parser.py b/lab/parser.py index 7b594b810..5bd9b1dc4 100644 --- a/lab/parser.py +++ b/lab/parser.py @@ -47,6 +47,12 @@ def _get_pattern_flags(s): return flags +class _Function: + def __init__(self, function, filename): + self.function = function + self.filename = filename + + class _Pattern: def __init__(self, attribute, regex, required, type_, flags): self.attribute = attribute @@ -81,15 +87,13 @@ def __str__(self): class _FileParser: """ - Private class that parses a given file according to the added patterns - and functions. + Private class that parses a given file according to the added patterns. """ def __init__(self): self.filename = None self.content = None self.patterns = [] - self.functions = [] def load_file(self, filename): self.filename = filename @@ -99,9 +103,6 @@ def load_file(self, filename): def add_pattern(self, pattern): self.patterns.append(pattern) - def add_function(self, function): - self.functions.append(function) - def search_patterns(self): assert self.content is not None found_props = {} @@ -109,11 +110,6 @@ def search_patterns(self): found_props.update(pattern.search(self.content, self.filename)) return found_props - def apply_functions(self, props): - assert self.content is not None - for function in self.functions: - function(self.content, props) - class Parser: """ @@ -124,6 +120,7 @@ class Parser: def __init__(self): tools.configure_logging() self.file_parsers = defaultdict(_FileParser) + self.functions = [] def add_pattern( self, attribute, regex, file="run.log", type=int, flags="", required=False @@ -163,7 +160,7 @@ def add_function(self, function, file="run.log"): r"""Call ``function(open(file).read(), properties)`` during parsing. Functions are applied **after** all patterns have been - evaluated. + evaluated and in the order in which they are added to the parser. The function is passed the file contents and the properties dictionary. It must manipulate the passed properties @@ -185,7 +182,7 @@ def add_function(self, function, file="run.log"): run. """ - self.file_parsers[file].add_function(function) + self.functions.append(_Function(function, file)) def parse(self): """Search all patterns and apply all functions. @@ -212,7 +209,9 @@ def parse(self): for file_parser in self.file_parsers.values(): self.props.update(file_parser.search_patterns()) - for file_parser in self.file_parsers.values(): - file_parser.apply_functions(self.props) + for function in self.functions: + with open(function.filename) as f: + content = f.read() + function.function(content, self.props) self.props.write() diff --git a/tox.ini b/tox.ini index 4d51a80a8..d64b28cdc 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ skip_missing_interpreters = true deps = pytest commands = - pytest --doctest-modules downward lab tests examples/showcase-options.py + pytest --doctest-modules --ignore=downward/scripts downward lab tests examples/showcase-options.py bash {toxinidir}/tests/run-example-experiment vertex-cover/exp.py bash {toxinidir}/tests/run-example-experiment lmcut.py bash {toxinidir}/tests/run-example-experiment showcase-options.py From 49791028de071fa30e833c5124cde716a9379a31 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 7 Oct 2022 19:56:19 +0200 Subject: [PATCH 40/57] Use Ubuntu version in revision cache key. --- .github/workflows/ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 749e2763b..1827c7b15 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -26,7 +26,7 @@ jobs: uses: actions/cache@v1 with: path: revision-cache - key: ${{ runner.os }}-revision-cache-${{ env.GIT_DOWNWARD_REV }} + key: ${{ matrix.os }}-revision-cache-${{ env.GIT_DOWNWARD_REV }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 From f3c4e25f4c9c80d8f42f5ea84a8d202eabea0f30 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sun, 9 Oct 2022 11:43:09 +0200 Subject: [PATCH 41/57] Revise news file. --- docs/news.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index aa296ad68..f7cd31aad 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,7 +1,7 @@ Changelog ========= -v7.2 (unreleased) +v7.2 (2022-10-09) ----------------- Lab @@ -16,7 +16,7 @@ Downward Lab * Parse peak memory in anytime search parser (Jendrik Seipp). * Only store "planner_memory" and "planner_time" attributes for successful planner runs (Jendrik Seipp). -* Add flexible example experiment for planners based on Fast Downward (Jendrik Seipp). +* Add fully customizable example planner experiment without `FastDownwardExperiment` class (Jendrik Seipp). * Show how to group domain directories in example Fast Downward experiment (Jendrik Seipp). From 515bdae953d057ca7577d6a9a801862c345cd606 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sun, 9 Oct 2022 11:44:21 +0200 Subject: [PATCH 42/57] Update version number to 7.2 for release. --- lab/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lab/__init__.py b/lab/__init__.py index c99c0123e..1d78a3b3c 100644 --- a/lab/__init__.py +++ b/lab/__init__.py @@ -1,2 +1,2 @@ #: Lab version number. A "+" is appended to all non-tagged revisions. -__version__ = "7.1+" +__version__ = "7.2" From f655b6282a36b997d14f76e28a0d850c040c16c0 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sun, 9 Oct 2022 11:44:34 +0200 Subject: [PATCH 43/57] Update version number to 7.2+ after release. --- lab/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lab/__init__.py b/lab/__init__.py index 1d78a3b3c..24002ae02 100644 --- a/lab/__init__.py +++ b/lab/__init__.py @@ -1,2 +1,2 @@ #: Lab version number. A "+" is appended to all non-tagged revisions. -__version__ = "7.2" +__version__ = "7.2+" From dfa67faa867e0835c871c082c6e0128bb976e7cf Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sun, 9 Oct 2022 11:45:53 +0200 Subject: [PATCH 44/57] Fix syntax. --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index f7cd31aad..e167bcf4f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -16,7 +16,7 @@ Downward Lab * Parse peak memory in anytime search parser (Jendrik Seipp). * Only store "planner_memory" and "planner_time" attributes for successful planner runs (Jendrik Seipp). -* Add fully customizable example planner experiment without `FastDownwardExperiment` class (Jendrik Seipp). +* Add fully customizable example planner experiment without ``FastDownwardExperiment`` class (Jendrik Seipp). * Show how to group domain directories in example Fast Downward experiment (Jendrik Seipp). From ee1cf96f047b6475634d318a88b5687cfc082d21 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 30 Jan 2023 22:37:38 +0100 Subject: [PATCH 45/57] Remove deprecated names from tox.ini file. --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index d64b28cdc..1b6259470 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ passenv = DOWNWARD_BENCHMARKS DOWNWARD_REPO DOWNWARD_REVISION_CACHE -whitelist_externals = +allowlist_externals = bash [testenv:ff] @@ -25,7 +25,7 @@ commands = bash {toxinidir}/tests/run-example-experiment ff/ff.py passenv = DOWNWARD_BENCHMARKS -whitelist_externals = +allowlist_externals = bash [testenv:downward] @@ -35,7 +35,7 @@ passenv = DOWNWARD_BENCHMARKS DOWNWARD_REPO DOWNWARD_REVISION_CACHE -whitelist_externals = +allowlist_externals = bash [testenv:singularity] @@ -44,7 +44,7 @@ commands = passenv = DOWNWARD_BENCHMARKS SINGULARITY_IMAGES -whitelist_externals = +allowlist_externals = bash [testenv:docs] @@ -83,5 +83,5 @@ commands = blackdoc . isort downward/ examples/ lab/ tests/ setup.py bash -c 'pyupgrade --py37-plus --exit-zero `find downward lab tests -name "*.py"`' -whitelist_externals = +allowlist_externals = bash From cf30c3b1f3a0d38a92cad7384edeaa0dcb253d66 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 30 Jan 2023 22:53:43 +0100 Subject: [PATCH 46/57] Upgrade to newer pyupgrade version. --- .pre-commit-config.yaml | 2 +- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8829e523e..ba3df0f49 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,6 @@ repos: args: ['--extend-ignore=E203', '--exclude=build,data,revision-cache,conf.py,.git,.tox,.venv', '--max-line-length=90'] - repo: https://github.com/asottile/pyupgrade - rev: v2.18.3 + rev: v3.3.1 hooks: - id: pyupgrade diff --git a/tox.ini b/tox.ini index 1b6259470..4bd019919 100644 --- a/tox.ini +++ b/tox.ini @@ -65,7 +65,7 @@ deps = flake8-bugbear flake8-comprehensions isort>=5.0,<5.1 - pyupgrade==3.0.0 + pyupgrade==3.3.1 vulture commands = bash {toxinidir}/tests/find-dead-code @@ -77,7 +77,7 @@ deps = black==22.3.0 blackdoc==0.3.4 isort>=5.0,<5.1 - pyupgrade==3.0.0 + pyupgrade==3.3.1 commands = black . blackdoc . From cf80f6532a2f4a73e29132b1d50fb14a89b653e9 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Mon, 30 Jan 2023 22:55:54 +0100 Subject: [PATCH 47/57] Use pathlib in example experiment. --- examples/downward/2020-09-11-B-bounded-cost.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/examples/downward/2020-09-11-B-bounded-cost.py b/examples/downward/2020-09-11-B-bounded-cost.py index 95d20934d..260c3ec6f 100755 --- a/examples/downward/2020-09-11-B-bounded-cost.py +++ b/examples/downward/2020-09-11-B-bounded-cost.py @@ -2,7 +2,7 @@ import json import os -import os.path +from pathlib import Path import shutil from downward import suites @@ -23,9 +23,10 @@ REMOTE_REPOS_DIR = "/infai/seipp/projects" BOUNDS_FILE = "bounds.json" SUITE = ["depot:p01.pddl", "grid:prob01.pddl", "gripper:prob01.pddl"] -REVISION_CACHE = os.environ.get("DOWNWARD_REVISION_CACHE") or os.path.join( - get_default_data_dir(), "revision-cache" -) +try: + REVISION_CACHE = Path(os.environ["DOWNWARD_REVISION_CACHE"]) +except KeyError: + REVISION_CACHE = Path(get_default_data_dir()) / "revision-cache" if project.REMOTE: # ENV = project.BaselSlurmEnvironment(email="my.name@myhost.ch") ENV = project.TetralithEnvironment( @@ -67,14 +68,14 @@ for rev, rev_nick in REVS: cached_rev = CachedFastDownwardRevision(REPO, rev, BUILD_OPTIONS) cached_rev.cache(REVISION_CACHE) - cache_path = os.path.join(REVISION_CACHE, cached_rev.name) - dest_path = "code-" + cached_rev.name + cache_path = REVISION_CACHE / cached_rev.name + dest_path = Path(f"code-{cached_rev.name}") exp.add_resource("", cache_path, dest_path) # Overwrite the script to set an environment variable. exp.add_resource( _get_solver_resource_name(cached_rev), - os.path.join(cache_path, "fast-downward.py"), - os.path.join(dest_path, "fast-downward.py"), + cache_path / "fast-downward.py", + dest_path / "fast-downward.py", ) for config_nick, config in CONFIGS: algo_name = f"{rev_nick}-{config_nick}" if rev_nick else config_nick From cbee020340c4dc6c5bd80c4de4f8bbe71f15ee85 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Tue, 31 Jan 2023 10:17:59 +0100 Subject: [PATCH 48/57] Tox: skip py39 since it chokes on distutils. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4bd019919..296cfb90d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, py310, ff, singularity, style, docs +envlist = py37, py38, py310, ff, singularity, style, docs # Skip py39 since it chokes on distutils. basepython = python3 skip_missing_interpreters = true From b38e2558dc7d02c8d88e38ed47afd84e966b7625 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Thu, 23 Feb 2023 19:02:48 +0100 Subject: [PATCH 49/57] Add README file for example downward experiment. --- examples/downward/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 examples/downward/README.md diff --git a/examples/downward/README.md b/examples/downward/README.md new file mode 100644 index 000000000..4c7e52dc5 --- /dev/null +++ b/examples/downward/README.md @@ -0,0 +1,23 @@ +# Setup instructions + +Create a [virtual environment](https://docs.python.org/3/tutorial/venv.html), +activate it and install all dependencies: + + sudo apt install python3 python3-venv + python3 -m venv --prompt myvenv .venv + source .venv/bin/activate + pip install --upgrade pip wheel + pip install -r requirements.txt + +If the last step fails, try regenerating a new `requirements.txt` from +`requirements.in` for your Python version: + + source .venv/bin/activate + pip install pip-tools + pip-compile + pip install -r requirements.txt + +Please note that before running an experiment script you need to +activate the virtual environment with + + source .venv/bin/activate From f32383ecdd49692fd70205aeb50388cc94cd727d Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Thu, 2 Mar 2023 22:04:54 +0100 Subject: [PATCH 50/57] Matplotlib changed name b to visible. Remove name to be backwards compatible. --- downward/reports/scatter_matplotlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/downward/reports/scatter_matplotlib.py b/downward/reports/scatter_matplotlib.py index f69e58c53..0f51bc697 100644 --- a/downward/reports/scatter_matplotlib.py +++ b/downward/reports/scatter_matplotlib.py @@ -71,7 +71,7 @@ class ScatterMatplotlib: @classmethod def _plot(cls, report, axes): - axes.grid(b=True, linestyle="-", color="0.75") + axes.grid(True, linestyle="-", color="0.75") for category, coords in sorted(report.categories.items()): x_vals, y_vals = zip(*coords) From 3f02eb45014e3a90a047000cbf69e67710cbfcf2 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Thu, 2 Mar 2023 22:50:21 +0100 Subject: [PATCH 51/57] Test Python 3.11, use Apptainer, test ubuntu-latest image. (#112) --- .github/workflows/ubuntu.yml | 20 ++++++++++---------- examples/singularity/filter-stderr.py | 3 ++- setup.py | 1 + tox.ini | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 1827c7b15..71a7a5c15 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -9,11 +9,11 @@ jobs: strategy: matrix: - os: [ubuntu-20.04, ubuntu-22.04] - python-version: [3.7, 3.8, 3.9, "3.10"] + os: [ubuntu-latest] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set revision run: | @@ -23,13 +23,13 @@ jobs: # Compile each Fast Downward revision only once and cache the results. - name: Cache revisions id: cache-revisions - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: revision-cache key: ${{ matrix.os }}-revision-cache-${{ env.GIT_DOWNWARD_REV }} - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -43,7 +43,7 @@ jobs: python -m pip install --upgrade pip tox - name: Check style - if: matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8' + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' run: | tox -e docs,style @@ -81,12 +81,12 @@ jobs: popd rm -r runsolver-dir/ - - name: Install Singularity + - name: Install Apptainer working-directory: ../deps run: | - wget --no-verbose https://www.ida.liu.se/~jense56/files/singularity-container_3.5.2+ds1-1_amd64.deb -O singularity.deb - sudo dpkg -i singularity.deb - rm singularity.deb + sudo add-apt-repository -y ppa:apptainer/ppa + sudo apt-get update + sudo apt-get install -y apptainer - name: Compile VAL working-directory: ../deps diff --git a/examples/singularity/filter-stderr.py b/examples/singularity/filter-stderr.py index 45cfd7db3..a5eb43d6b 100755 --- a/examples/singularity/filter-stderr.py +++ b/examples/singularity/filter-stderr.py @@ -13,6 +13,7 @@ "differs from the one in the portfolio file", "Terminated", "Killed", + "underlay of /etc/localtime required more than", ] @@ -22,7 +23,7 @@ def main(): if stderr.is_file(): need_to_filter = False filtered_content = [] - with open(stderr, "r") as f: + with open(stderr) as f: for line in f: if any(pattern in line for pattern in IGNORE_PATTERNS): need_to_filter = True diff --git a/setup.py b/setup.py index a31f98977..daae36a17 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering", ], install_requires=[ diff --git a/tox.ini b/tox.ini index 296cfb90d..7ac39b138 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py310, ff, singularity, style, docs # Skip py39 since it chokes on distutils. +envlist = py37, py38, py310, py311, ff, singularity, style, docs # Skip py39 since it chokes on distutils. basepython = python3 skip_missing_interpreters = true From 75274a348f95893af198a216f82b273ee297605e Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 3 Mar 2023 21:55:22 +0100 Subject: [PATCH 52/57] Transparently handle xz-compressed properties files. (#111) --- .github/workflows/ubuntu.yml | 1 + docs/news.rst | 12 ++++++++++ lab/fetcher.py | 5 +++- lab/reports/__init__.py | 7 ++---- lab/tools.py | 46 +++++++++++++++++++++++------------- 5 files changed, 49 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 71a7a5c15..43ae9d96c 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -8,6 +8,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] diff --git a/docs/news.rst b/docs/news.rst index e167bcf4f..4a37acfe8 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,6 +1,18 @@ Changelog ========= +v7.3 (unreleased) +----------------- + +Lab +^^^ +* Transparently handle xz-compressed properties files (Jendrik Seipp). + +Downward Lab +^^^^^^^^^^^^ +* No changes yet. + + v7.2 (2022-10-09) ----------------- diff --git a/lab/fetcher.py b/lab/fetcher.py index b29b2ce9a..07d3fa3c2 100644 --- a/lab/fetcher.py +++ b/lab/fetcher.py @@ -107,7 +107,10 @@ def __call__(self, src_dir, eval_dir=None, merge=None, filter=None, **kwargs): os.path.join(src_dir, "runs-00001-00100") ) if fetch_from_eval_dir: - src_props = tools.Properties(filename=os.path.join(src_dir, "properties")) + src_path = os.path.join(src_dir, "properties") + src_props = tools.Properties(filename=src_path) + if not src_props: + logging.critical(f"No properties found in {src_dir}") run_filter.apply(src_props) combined_props.update(src_props) logging.info(f"Fetched properties of {len(src_props)} runs.") diff --git a/lab/reports/__init__.py b/lab/reports/__init__.py index 7f574c405..e1ab09c33 100644 --- a/lab/reports/__init__.py +++ b/lab/reports/__init__.py @@ -415,14 +415,11 @@ def _scan_data(self): def _load_data(self): props_file = os.path.join(self.eval_dir, "properties") - if not os.path.exists(props_file): - logging.critical(f"Properties file not found at {props_file}") - logging.info("Reading properties file") self.props = tools.Properties(filename=props_file) - logging.info("Reading properties file finished") if not self.props: - logging.critical("properties file in evaluation dir is empty.") + logging.critical(f"No properties found in {self.eval_dir}") + logging.info("Reading properties file finished") def _apply_filter(self): self.run_filter.apply(self.props) diff --git a/lab/tools.py b/lab/tools.py index 5f7529966..8e557b269 100644 --- a/lab/tools.py +++ b/lab/tools.py @@ -2,6 +2,7 @@ import colorsys import functools import logging +import lzma import math import os from pathlib import Path @@ -257,37 +258,50 @@ def default(self, o): else: return super().default(o) + JSON_ARGS = { + "cls": _PropertiesEncoder, + "indent": 2, + "separators": (",", ": "), + "sort_keys": True, + } + + """Transparently handle properties files compressed with xz.""" + def __init__(self, filename=None): - self.filename = filename - self.load(filename) + self.path = filename + if self.path: + self.path = Path(self.path).resolve() + xz_path = self.path.with_suffix(".xz") + if self.path.is_file() and xz_path.is_file(): + logging.critical(f"Only one of {self.path} and {xz_path} may exist") + if not self.path.is_file() and xz_path.is_file(): + self.path = xz_path + if self.path.is_file(): + self.load(self.path) dict.__init__(self) def __str__(self): - return json.dumps( - self, - cls=self._PropertiesEncoder, - indent=2, - separators=(",", ": "), - sort_keys=True, - ) + return json.dumps(self, **self.JSON_ARGS) def load(self, filename): - if not filename or not os.path.exists(filename): - return - with open(filename) as f: + path = Path(filename) + open_func = lzma.open if path.suffix == ".xz" else open + with open_func(path) as f: try: self.update(json.load(f)) except ValueError as e: - logging.critical(f"JSON parse error in file '{filename}': {e}") + logging.critical(f"JSON parse error in file '{path}': {e}") def add_unexplained_error(self, error): add_unexplained_error(self, error) def write(self): """Write the properties to disk.""" - assert self.filename - makedirs(os.path.dirname(self.filename)) - write_file(self.filename, str(self)) + assert self.path + self.path.parent.mkdir(parents=True, exist_ok=True) + open_func = lzma.open if self.path.suffix == ".xz" else open + with open_func(self.path, "w") as f: + json.dump(self, f, **self.JSON_ARGS) class RunFilter: From 5f0616e08234f78cc7f02ba86d437075ac787b3d Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 3 Mar 2023 21:59:24 +0100 Subject: [PATCH 53/57] Add fetch_algorithms() function to downward example project. --- examples/downward/project.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/examples/downward/project.py b/examples/downward/project.py index 0d2076d26..bfb366344 100644 --- a/examples/downward/project.py +++ b/examples/downward/project.py @@ -297,6 +297,28 @@ def rename_and_filter(run): ) +def fetch_algorithms(exp, expname, *, algos=None, name=None, filters=None): + """ + Fetch multiple or all algorithms. + """ + assert not expname.rstrip("/").endswith("-eval") + algos = set(algos or []) + filters = filters or [] + if algos: + + def algo_filter(run): + return run["algorithm"] in algos + + filters.append(algo_filter) + + exp.add_fetcher( + f"data/{expname}-eval", + filter=filters, + name=name or f"fetch-from-{expname}", + merge=True, + ) + + def add_absolute_report(exp, *, name=None, outfile=None, **kwargs): report = AbsoluteReport(**kwargs) if name and not outfile: From 3b6f5a90e21e6a75faaf345ce165b1a70853a917 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 3 Mar 2023 22:01:10 +0100 Subject: [PATCH 54/57] Update changelog. --- docs/news.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 4a37acfe8..03d57e833 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,16 +1,17 @@ Changelog ========= -v7.3 (unreleased) +v7.3 (2023-03-03) ----------------- Lab ^^^ * Transparently handle xz-compressed properties files (Jendrik Seipp). +* Add CI tests for Python 3.11 (Jendrik Seipp). Downward Lab ^^^^^^^^^^^^ -* No changes yet. +* Adapt code for Matplotlib version 3.7 (Jendrik Seipp). v7.2 (2022-10-09) From 271d9db7c19c22209c4b2be3f1829834223770a2 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 3 Mar 2023 22:03:25 +0100 Subject: [PATCH 55/57] Use new function. --- examples/downward/01-evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/downward/01-evaluation.py b/examples/downward/01-evaluation.py index facf91317..0683a5829 100755 --- a/examples/downward/01-evaluation.py +++ b/examples/downward/01-evaluation.py @@ -25,7 +25,7 @@ ) project.fetch_algorithm(exp, "2020-09-11-A-cg-vs-ff", "20.06:01-cg", new_algo="cg") -project.fetch_algorithm(exp, "2020-09-11-A-cg-vs-ff", "20.06:02-ff", new_algo="ff") +project.fetch_algorithms(exp, "2020-09-11-B-bounded-cost") filters = [project.add_evaluations_per_time] From 21c4aaeeda36972751f7db81e69afaef583a4e48 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 3 Mar 2023 22:03:50 +0100 Subject: [PATCH 56/57] Update version number to 7.3 for release. --- lab/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lab/__init__.py b/lab/__init__.py index 24002ae02..534f16565 100644 --- a/lab/__init__.py +++ b/lab/__init__.py @@ -1,2 +1,2 @@ #: Lab version number. A "+" is appended to all non-tagged revisions. -__version__ = "7.2+" +__version__ = "7.3" From 5a9d12dec4284a2c345e9412cb45629c2842676f Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 3 Mar 2023 22:03:55 +0100 Subject: [PATCH 57/57] Update version number to 7.3+ after release. --- lab/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lab/__init__.py b/lab/__init__.py index 534f16565..1f451fafe 100644 --- a/lab/__init__.py +++ b/lab/__init__.py @@ -1,2 +1,2 @@ #: Lab version number. A "+" is appended to all non-tagged revisions. -__version__ = "7.3" +__version__ = "7.3+"