diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 6df3c5456..43ae9d96c 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -1,7 +1,5 @@ name: Ubuntu -# see https://github.community/t5/GitHub-Actions/How-to-trigger-an-action-on-push-or-pull-request-but-not-both/m-p/35805 -# and https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/6 on: [push, pull_request] jobs: @@ -9,17 +7,14 @@ jobs: runs-on: ${{ matrix.os }} - # We want to run on external PRs, but not on our own internal PRs as they'll be run - # by the push to the branch. - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - strategy: + fail-fast: false matrix: - os: [ubuntu-18.04, ubuntu-20.04] - python-version: [3.6, 3.7, 3.8, 3.9] + os: [ubuntu-latest] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set revision run: | @@ -29,13 +24,13 @@ jobs: # Compile each Fast Downward revision only once and cache the results. - name: Cache revisions id: cache-revisions - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: revision-cache - key: ${{ runner.os }}-revision-cache-${{ env.GIT_DOWNWARD_REV }} + key: ${{ matrix.os }}-revision-cache-${{ env.GIT_DOWNWARD_REV }} - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -49,7 +44,7 @@ jobs: python -m pip install --upgrade pip tox - name: Check style - if: matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.8' + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' run: | tox -e docs,style @@ -63,14 +58,18 @@ jobs: - name: Compile FF working-directory: ../deps run: | - sudo apt-get -y install g++ make flex bison - wget http://fai.cs.uni-saarland.de/hoffmann/ff/FF-v2.3.tgz - tar -xzvf FF-v2.3.tgz - pushd FF-v2.3/ - make -j - cp ff ../ - popd - rm -r FF-v2.3/ FF-v2.3.tgz + # GCC 10 fails to compile FF 2.3, so we use a precompiled FF binary. + # sudo apt-get -y install g++ make flex bison + # wget http://fai.cs.uni-saarland.de/hoffmann/ff/FF-v2.3.tgz + # tar -xzvf FF-v2.3.tgz + # pushd FF-v2.3/ + # make -j + # cp ff ../ + # popd + # rm -r FF-v2.3/ FF-v2.3.tgz + wget 'https://github.com/hectorpal/fast-forward-linux-binaries/raw/main/ff.gz' + gunzip ff.gz + chmod +x ff - name: Compile runsolver working-directory: ../deps @@ -83,12 +82,12 @@ jobs: popd rm -r runsolver-dir/ - - name: Install Singularity + - name: Install Apptainer working-directory: ../deps run: | - wget --no-verbose http://ftp.se.debian.org/debian/pool/main/s/singularity-container/singularity-container_3.5.2+ds1-1_amd64.deb -O singularity.deb - sudo dpkg -i singularity.deb - rm singularity.deb + sudo add-apt-repository -y ppa:apptainer/ppa + sudo apt-get update + sudo apt-get install -y apptainer - name: Compile VAL working-directory: ../deps @@ -134,7 +133,7 @@ jobs: echo CACHE: ${DOWNWARD_REVISION_CACHE} export DOWNWARD_REPO=${DOWNWARD_REPO} - time tox -e py,ff,singularity + time tox -e py,downward,ff,singularity - name: Test installation with pip run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..ba3df0f49 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +repos: + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + + - repo: https://github.com/pycqa/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + args: ['--extend-ignore=E203', '--exclude=build,data,revision-cache,conf.py,.git,.tox,.venv', '--max-line-length=90'] + + - repo: https://github.com/asottile/pyupgrade + rev: v3.3.1 + hooks: + - id: pyupgrade diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 705656480..fc28f9886 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,6 +23,17 @@ from GitHub in editable mode: For details on how to set everything up, please see the [GitHub actions file](.github/workflows/ubuntu.yml). +# Setting up a pre-commit hook for style checks + + python3 -m pip install pre-commit + pre-commit install + +Now the most important style checks are run for the changed files before each commit. + +# Fixing the code style + + tox -e fix-style + # Running tests cd lab @@ -59,8 +70,4 @@ Now you can run the example Singularity experiment with `tox -e singularity`. ## Run all tests -Once you have installed all dependecies, you can run all tests by executing `tox` without any options. - -# Fixing the code style - - tox -e fix-style +Once you have installed all dependencies, you can run all tests by executing `tox` without any options. diff --git a/INSTALL.rst b/INSTALL.rst index a6cd62938..05ba9d737 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -1,7 +1,7 @@ Install Lab ----------- -Lab requires Python 3.6+ and Linux (e.g., Ubuntu). We recommend installing +Lab requires Python 3.7+ and Linux (e.g., Ubuntu). We recommend installing Lab in a `Python virtual environment `_. This has the advantage that there are no modifications to the system-wide configuration, and that @@ -18,7 +18,7 @@ for different papers) without conflicts:: # If PYTHONPATH is set, unset it to obtain a clean environment. unset PYTHONPATH - # Create and activate a Python 3 virtual environment for Lab. + # Create and activate a Python virtual environment for Lab. python3 -m venv --prompt my-paper .venv source .venv/bin/activate diff --git a/dev/make-release-notes.py b/dev/make-release-notes.py index c79160287..474b70735 100755 --- a/dev/make-release-notes.py +++ b/dev/make-release-notes.py @@ -5,7 +5,7 @@ _, VERSION, CHANGELOG, LIST = sys.argv -REGEX = fr""" +REGEX = rf""" Changelog\n =========\n \n diff --git a/dev/release.sh b/dev/release.sh index 2b8e7e44d..37e0ac189 100755 --- a/dev/release.sh +++ b/dev/release.sh @@ -23,8 +23,8 @@ if [[ $retcode != 0 ]]; then exit 1 fi -if [[ $(git rev-parse --abbrev-ref HEAD) != master ]]; then - echo "Must be on master for release" +if [[ $(git rev-parse --abbrev-ref HEAD) != main ]]; then + echo "Must be on main branch for release" exit 1 fi diff --git a/docs/autobuild.sh b/docs/autobuild.sh index a4e13a5e3..38c02b21a 100755 --- a/docs/autobuild.sh +++ b/docs/autobuild.sh @@ -2,6 +2,7 @@ # Automatically rebuild Sphinx documentation when files change. DOCS="$( dirname "$0" )" +DOCS="$( realpath "$DOCS" )" REPO="$( realpath "$DOCS/../" )" cd "$REPO/docs" diff --git a/docs/downward.tutorial.rst b/docs/downward.tutorial.rst index 999aadfed..4e1879ed8 100644 --- a/docs/downward.tutorial.rst +++ b/docs/downward.tutorial.rst @@ -22,7 +22,7 @@ alternative that has proven to work well in practice. Installation ------------ -Lab requires **Python 3.6+** and **Linux**. To run Fast Downward +Lab requires **Python 3.7+** and **Linux**. To run Fast Downward experiments, you'll need a **Fast Downward** repository, planning **benchmarks** and a plan **validator**. :: diff --git a/docs/news.rst b/docs/news.rst index 29c372846..03d57e833 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,16 +1,56 @@ Changelog ========= -next (unreleased) +v7.3 (2023-03-03) ----------------- Lab ^^^ -* Revamp Singularity example experiment: use runsolver to limit resource usage (Silvan Sievers and Jendrik Seipp). +* Transparently handle xz-compressed properties files (Jendrik Seipp). +* Add CI tests for Python 3.11 (Jendrik Seipp). Downward Lab ^^^^^^^^^^^^ -* No changes so far. +* Adapt code for Matplotlib version 3.7 (Jendrik Seipp). + + +v7.2 (2022-10-09) +----------------- + +Lab +^^^ +* Raise minimum supported Python version to 3.7 (Jendrik Seipp). +* Add support for Python 3.10 (Jendrik Seipp). +* Apply parsing functions in the order in which they were added (Jendrik Seipp). +* For contributors: document pre-commit hook in ``CONTRIBUTING.md`` file (Jendrik Seipp). + +Downward Lab +^^^^^^^^^^^^ +* Parse peak memory in anytime search parser (Jendrik Seipp). +* Only store "planner_memory" and "planner_time" attributes for successful planner + runs (Jendrik Seipp). +* Add fully customizable example planner experiment without ``FastDownwardExperiment`` class (Jendrik Seipp). +* Show how to group domain directories in example Fast Downward experiment (Jendrik Seipp). + + +v7.1 (2022-06-20) +----------------- + +Lab +^^^ +* Revamp Singularity example experiment: use ``runsolver`` to limit resource usage + (Silvan Sievers and Jendrik Seipp). + +Downward Lab +^^^^^^^^^^^^ +* Fix header sizes in HTML reports (Jendrik Seipp). +* Include domains in attribute overview tables even if none of their tasks has an + attribute value for all algorithms (Jendrik Seipp). +* Compute "score_planner_time" and "score_planner_memory" attributes in planner + parser (Jendrik Seipp). +* Only consider files ending with ".pddl" and ".sas" when building suites (Jendrik Seipp). +* Explicitly left-align non-numeric cells to avoid \\multicolumn entries in Latex output + (Jendrik Seipp). v7.0 (2021-10-24) diff --git a/downward/experiment.py b/downward/experiment.py index 6c88b1441..9d7e75918 100644 --- a/downward/experiment.py +++ b/downward/experiment.py @@ -125,7 +125,7 @@ class FastDownwardExperiment(Experiment): #: "translator_peak_memory", "translator_time_done". #: #: Parsed attributes: "node", "planner_memory", "planner_time", - #: "planner_wall_clock_time". + #: "planner_wall_clock_time", "score_planner_memory", "score_planner_time". PLANNER_PARSER = os.path.join(DOWNWARD_SCRIPTS_DIR, "planner-parser.py") def __init__(self, path=None, environment=None, revision_cache=None): @@ -176,7 +176,8 @@ def add_suite(self, benchmarks_dir, suite): """Add PDDL or SAS+ benchmarks to the experiment. *benchmarks_dir* must be a path to a benchmark directory. It must - contain domain directories, which in turn hold PDDL or SAS+ files. + contain domain directories, which in turn hold PDDL or SAS+ files + (ending with ".pddl" or ".sas"). *suite* must be a list of domain or domain:task names. :: @@ -187,14 +188,14 @@ def add_suite(self, benchmarks_dir, suite): >>> exp.add_suite(benchmarks_dir, ["rubiks-cube:p01.sas"]) One source for benchmarks is - https://github.com/aibasel/downward-benchmarks. After cloning the - repo, you can generate suites with the ``suites.py`` script. We - recommend using the suite ``optimal_strips`` for optimal STRIPS planners - and ``satisficing`` for satisficing planners:: + https://github.com/aibasel/downward-benchmarks. After cloning the repo, + you can generate suites with the ``suites.py`` script. We recommend + using the suite ``optimal_strips`` for optimal STRIPS planners and + ``satisficing`` for satisficing planners:: - # Create standard optimal planning suite. - $ path/to/downward-benchmarks/suites.py optimal_strips - ['airport', ..., 'zenotravel'] + # Create standard optimal planning suite. $ + path/to/downward-benchmarks/suites.py optimal_strips ['airport', + ..., 'zenotravel'] Then you can copy the generated list into your experiment script:: @@ -362,6 +363,7 @@ def _add_code(self): ) def _add_runs(self): + tasks = self._get_tasks() for algo in self._algorithms.values(): - for task in self._get_tasks(): + for task in tasks: self.add_run(FastDownwardRun(self, algo, task)) diff --git a/downward/reports/__init__.py b/downward/reports/__init__.py index 408a0533c..ec20a1d5c 100644 --- a/downward/reports/__init__.py +++ b/downward/reports/__init__.py @@ -76,6 +76,7 @@ def __init__(self, **kwargs): >>> # Use a filter function to select algorithms. >>> def only_blind_and_lmcut(run): ... return run["algorithm"] in ["blind", "lmcut"] + ... >>> report = PlanningReport(filter=only_blind_and_lmcut) >>> # Use "filter_algorithm" to select and *order* algorithms. @@ -97,6 +98,7 @@ def __init__(self, **kwargs): ... times = [t for t in times if t is not None] ... map[(domain, problem)] = min(times) if times else None ... return str(map) + ... """ # Set non-default options for some attributes. diff --git a/downward/reports/absolute.py b/downward/reports/absolute.py index 169d0c5e7..38f3186c7 100644 --- a/downward/reports/absolute.py +++ b/downward/reports/absolute.py @@ -143,7 +143,7 @@ def get_markup(self): tables.append( ( "", - f"Domain-wise reports only support numeric " + f"Per-domain reports only support numeric " f"attributes, but {attribute} has type " f"{self._all_attributes[attribute].__name__}.", ) @@ -157,7 +157,7 @@ def get_markup(self): if domain: assert table toc_line.append(f"[''{domain}'' #{attribute}-{domain}]") - parts.append(f"== {domain} ==[{attribute}-{domain}]\n{table}\n") + parts.append(f"=== {domain} ===[{attribute}-{domain}]\n{table}\n") else: if table: parts.append(f"{table}\n") @@ -165,7 +165,7 @@ def get_markup(self): parts.append( f"No task was found where all algorithms " f'have a value for "{attribute}". Therefore no ' - f"domain-wise table can be generated.\n" + f"per-domain table can be generated.\n" ) toc_lines.append(f"- **[''{attribute}'' #{attribute}]**") @@ -179,7 +179,7 @@ def get_markup(self): toc = "\n".join(toc_lines) content = "\n".join( - f"= {attr} =[{attr}]\n\n{section}" for (attr, section) in sections + f"== {attr} ==[{attr}]\n\n{section}" for (attr, section) in sections ) return f"{toc}\n\n\n{content}" @@ -211,7 +211,7 @@ def _add_table_info(self, attribute, func_name, table): """ if not attribute.absolute: table.info.append( - f"Only instances where all algorithms have a " + f"Only tasks where all algorithms have a " f'value for "{attribute}" are considered.' ) table.info.append( @@ -235,7 +235,10 @@ def _get_suite_table(self, attribute): func_name, func = self._get_aggregation_function(attribute) num_probs = 0 self._add_table_info(attribute, func_name, table) - domain_algo_values = defaultdict(list) + domain_algo_values = {} + for domain in self.domains: + for algorithm in self.algorithms: + domain_algo_values[(domain, algorithm)] = [] for (domain, _), runs in self.problem_runs.items(): # If the attribute is absolute, no runs must have been filtered and # no values must be missing. @@ -258,8 +261,7 @@ def _get_suite_table(self, attribute): # different problem numbers. for domain in self.domains: task_counts = [ - str(len(domain_algo_values.get((domain, algo), []))) - for algo in self.algorithms + str(len(domain_algo_values[(domain, algo)])) for algo in self.algorithms ] if len(set(task_counts)) == 1: count = task_counts[0] @@ -272,7 +274,8 @@ def _get_suite_table(self, attribute): table.cell_formatters[domain][table.header_column] = formatter for (domain, algo), values in domain_algo_values.items(): - table.add_cell(domain, algo, func(values)) + domain_value = func(values) if values else None + table.add_cell(domain, algo, domain_value) table.num_values = num_probs return table diff --git a/downward/reports/scatter.py b/downward/reports/scatter.py index 464810f6b..8175069da 100644 --- a/downward/reports/scatter.py +++ b/downward/reports/scatter.py @@ -57,6 +57,7 @@ def __init__( ... # run2['domain'] has the same value, because we always ... # compare two runs of the same problem. ... return run1["domain"] + ... Example grouping by difficulty: @@ -68,6 +69,7 @@ def __init__( ... if time1 == time2: ... return "equal" ... return "worse" + ... >>> from downward.experiment import FastDownwardExperiment >>> exp = FastDownwardExperiment() diff --git a/downward/reports/scatter_matplotlib.py b/downward/reports/scatter_matplotlib.py index f77c1bfc2..0f51bc697 100644 --- a/downward/reports/scatter_matplotlib.py +++ b/downward/reports/scatter_matplotlib.py @@ -33,7 +33,7 @@ def create_legend(self): @staticmethod def _get_max_supported_value(scale): if scale == "linear": - return 10 ** 12 # Larger values cause numerical problems. + return 10**12 # Larger values cause numerical problems. else: assert scale in {"log", "symlog"}, scale return sys.maxsize @@ -71,7 +71,7 @@ class ScatterMatplotlib: @classmethod def _plot(cls, report, axes): - axes.grid(b=True, linestyle="-", color="0.75") + axes.grid(True, linestyle="-", color="0.75") for category, coords in sorted(report.categories.items()): x_vals, y_vals = zip(*coords) diff --git a/downward/scripts/anytime-search-parser.py b/downward/scripts/anytime-search-parser.py index d312ba364..ea11262ec 100755 --- a/downward/scripts/anytime-search-parser.py +++ b/downward/scripts/anytime-search-parser.py @@ -37,8 +37,26 @@ def coverage(content, props): props["coverage"] = int("cost" in props) +def add_memory(content, props): + """Add "memory" attribute if the run was not aborted. + + Peak memory usage is printed even for runs that are terminated + abnormally. For these runs we do not take the reported value into + account since the value is censored: it only takes into account the + memory usage until termination. + + """ + raw_memory = props.get("raw_memory") + if raw_memory is not None: + if raw_memory < 0: + props.add_unexplained_error("planner failed to log peak memory") + elif props["coverage"]: + props["memory"] = raw_memory + + def main(): parser = Parser() + parser.add_pattern("raw_memory", r"Peak memory: (.+) KB", type=int), parser.add_function(find_all_matches("cost:all", r"Plan cost: (.+)\n", type=float)) parser.add_function( find_all_matches("steps:all", r"Plan length: (.+) step\(s\).\n", type=float) @@ -46,6 +64,7 @@ def main(): parser.add_function(reduce_to_min("cost:all", "cost")) parser.add_function(reduce_to_min("steps:all", "steps")) parser.add_function(coverage) + parser.add_function(add_memory) parser.parse() diff --git a/downward/scripts/planner-parser.py b/downward/scripts/planner-parser.py index 839c33b29..89ff157b6 100755 --- a/downward/scripts/planner-parser.py +++ b/downward/scripts/planner-parser.py @@ -1,31 +1,90 @@ #! /usr/bin/env python +from lab import tools from lab.parser import Parser def add_planner_memory(content, props): - try: - props["planner_memory"] = max(props["translator_peak_memory"], props["memory"]) - except KeyError: - pass + # Only add planner_memory for successful runs. + if props["coverage"]: + try: + props["planner_memory"] = max( + props["translator_peak_memory"], props["memory"] + ) + except KeyError: + pass def add_planner_time(content, props): + # Only add planner_time for successful runs. + if props["coverage"]: + # Newer planner versions print planner time and we parse it below. + # Don't overwrite it. + if "planner_time" not in props: + try: + props["planner_time"] = ( + props["translator_time_done"] + props["total_time"] + ) + except KeyError: + pass + elif "planner_time" in props: + del props["planner_time"] + + +def add_planner_scores(content, props): + """ + Compute scores for overall planner runtime and memory usage. + + Best possible performance in a task is counted as 1, while failure to solve + a task and worst performance are counted as 0. + + """ + success = props["coverage"] or props["unsolvable"] + try: - props["planner_time"] = props["translator_time_done"] + props["total_time"] + time_limit = props["planner_time_limit"] except KeyError: - pass + print("planner_time_limit missing -> can't compute planner time score") + else: + props["score_planner_time"] = tools.compute_log_score( + success, props.get("planner_time"), lower_bound=1.0, upper_bound=time_limit + ) + + try: + memory_limit_kb = props["planner_memory_limit"] * 1024 + except KeyError: + print("planner_memory_limit missing -> can't compute planner memory score") + else: + props["score_planner_memory"] = tools.compute_log_score( + success, + props.get("planner_memory"), + lower_bound=2000, + upper_bound=memory_limit_kb, + ) class PlannerParser(Parser): def __init__(self): Parser.__init__(self) - self.add_function(add_planner_memory) - self.add_function(add_planner_time) + self.add_pattern( + "planner_time_limit", + r"planner time limit: (.+)s", + type=float, + ) + self.add_pattern( + "planner_memory_limit", + r"planner memory limit: (.+) MB", + type=int, + ) self.add_pattern( "node", r"node: (.+)\n", type=str, file="driver.log", required=True ) + self.add_pattern( + "planner_time", + r"Planner time: (.+)s", + type=float, + ) self.add_pattern( "planner_wall_clock_time", r"planner wall-clock time: (.+)s", @@ -34,6 +93,10 @@ def __init__(self): required=True, ) + self.add_function(add_planner_memory) + self.add_function(add_planner_time) + self.add_function(add_planner_scores) + def main(): parser = PlannerParser() diff --git a/downward/scripts/single-search-parser.py b/downward/scripts/single-search-parser.py index 113d36630..9421e632c 100755 --- a/downward/scripts/single-search-parser.py +++ b/downward/scripts/single-search-parser.py @@ -4,10 +4,10 @@ Regular expressions and functions for parsing single-search runs of Fast Downward. """ -import math import re import sys +from lab import tools from lab.parser import Parser @@ -108,19 +108,11 @@ def add_scores(content, props): to solve a task and worst performance are counted as 0. """ - - def log_score(value, min_bound, max_bound): - if value is None or not props["coverage"]: - return 0 - value = max(value, min_bound) - value = min(value, max_bound) - raw_score = math.log(value) - math.log(max_bound) - best_raw_score = math.log(min_bound) - math.log(max_bound) - return raw_score / best_raw_score + success = props["coverage"] or props["unsolvable"] for attr in ("expansions", "evaluations", "generated"): - props["score_" + attr] = log_score( - props.get(attr), min_bound=100, max_bound=1e6 + props["score_" + attr] = tools.compute_log_score( + success, props.get(attr), lower_bound=100, upper_bound=1e6 ) try: @@ -128,11 +120,11 @@ def log_score(value, min_bound, max_bound): except KeyError: print("search time limit missing -> can't compute time scores") else: - props["score_total_time"] = log_score( - props.get("total_time"), min_bound=1.0, max_bound=max_time + props["score_total_time"] = tools.compute_log_score( + success, props.get("total_time"), lower_bound=1.0, upper_bound=max_time ) - props["score_search_time"] = log_score( - props.get("search_time"), min_bound=1.0, max_bound=max_time + props["score_search_time"] = tools.compute_log_score( + success, props.get("search_time"), lower_bound=1.0, upper_bound=max_time ) try: @@ -140,8 +132,8 @@ def log_score(value, min_bound, max_bound): except KeyError: print("search memory limit missing -> can't compute memory score") else: - props["score_memory"] = log_score( - props.get("memory"), min_bound=2000, max_bound=max_memory_kb + props["score_memory"] = tools.compute_log_score( + success, props.get("memory"), lower_bound=2000, upper_bound=max_memory_kb ) diff --git a/downward/suites.py b/downward/suites.py index 95ea9c286..39156726c 100644 --- a/downward/suites.py +++ b/downward/suites.py @@ -44,7 +44,7 @@ def __init__(self, benchmarks_dir, domain): [ p for p in os.listdir(directory) - if "domain" not in p and not p.endswith(".py") + if "domain" not in p and p.endswith((".pddl", ".sas")) ] ) self.problems = [ diff --git a/examples/downward/01-evaluation.py b/examples/downward/01-evaluation.py index facf91317..0683a5829 100755 --- a/examples/downward/01-evaluation.py +++ b/examples/downward/01-evaluation.py @@ -25,7 +25,7 @@ ) project.fetch_algorithm(exp, "2020-09-11-A-cg-vs-ff", "20.06:01-cg", new_algo="cg") -project.fetch_algorithm(exp, "2020-09-11-A-cg-vs-ff", "20.06:02-ff", new_algo="ff") +project.fetch_algorithms(exp, "2020-09-11-B-bounded-cost") filters = [project.add_evaluations_per_time] diff --git a/examples/downward/2020-09-11-A-cg-vs-ff.py b/examples/downward/2020-09-11-A-cg-vs-ff.py index 11efc9724..0fd007bac 100755 --- a/examples/downward/2020-09-11-A-cg-vs-ff.py +++ b/examples/downward/2020-09-11-A-cg-vs-ff.py @@ -1,15 +1,20 @@ #! /usr/bin/env python import os +import shutil import project REPO = project.get_repo_base() BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] +SCP_LOGIN = "myname@myserver.com" +REMOTE_REPOS_DIR = "/infai/seipp/projects" +# If REVISION_CACHE is None, the default ./data/revision-cache is used. +REVISION_CACHE = os.environ.get("DOWNWARD_REVISION_CACHE") if project.REMOTE: SUITE = project.SUITE_SATISFICING - ENV = project.BaselSlurmEnvironment(email="my.name@unibas.ch") + ENV = project.BaselSlurmEnvironment(email="my.name@myhost.ch") else: SUITE = ["depot:p01.pddl", "grid:prob01.pddl", "gripper:prob01.pddl"] ENV = project.LocalEnvironment(processes=2) @@ -27,7 +32,7 @@ BUILD_OPTIONS = [] DRIVER_OPTIONS = ["--overall-time-limit", "5m"] REVS = [ - ("release-20.06.0", "20.06"), + ("main", "main"), ] ATTRIBUTES = [ "error", @@ -35,7 +40,6 @@ "search_start_time", "search_start_memory", "total_time", - "initial_h_value", "h_values", "coverage", "expansions", @@ -43,7 +47,7 @@ project.EVALUATIONS_PER_TIME, ] -exp = project.CommonExperiment(environment=ENV) +exp = project.FastDownwardExperiment(environment=ENV, revision_cache=REVISION_CACHE) for config_nick, config in CONFIGS: for rev, rev_nick in REVS: algo_name = f"{rev_nick}:{config_nick}" if rev_nick else config_nick @@ -57,6 +61,20 @@ ) exp.add_suite(BENCHMARKS_DIR, SUITE) +exp.add_parser(exp.EXITCODE_PARSER) +exp.add_parser(exp.TRANSLATOR_PARSER) +exp.add_parser(exp.SINGLE_SEARCH_PARSER) +exp.add_parser(project.DIR / "parser.py") +exp.add_parser(exp.PLANNER_PARSER) + +exp.add_step("build", exp.build) +exp.add_step("start", exp.start_runs) +exp.add_fetcher(name="fetch") + +if not project.REMOTE: + exp.add_step("remove-eval-dir", shutil.rmtree, exp.eval_dir, ignore_errors=True) + project.add_scp_step(exp, SCP_LOGIN, REMOTE_REPOS_DIR) + project.add_absolute_report( exp, attributes=ATTRIBUTES, filter=[project.add_evaluations_per_time] ) diff --git a/examples/downward/2020-09-11-B-bounded-cost.py b/examples/downward/2020-09-11-B-bounded-cost.py new file mode 100755 index 000000000..260c3ec6f --- /dev/null +++ b/examples/downward/2020-09-11-B-bounded-cost.py @@ -0,0 +1,123 @@ +#! /usr/bin/env python + +import json +import os +from pathlib import Path +import shutil + +from downward import suites +from downward.cached_revision import CachedFastDownwardRevision +from downward.experiment import ( + _DownwardAlgorithm, + _get_solver_resource_name, + FastDownwardRun, +) +from lab.experiment import Experiment, get_default_data_dir + +import project + + +REPO = project.get_repo_base() +BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] +SCP_LOGIN = "myname@myserver.com" +REMOTE_REPOS_DIR = "/infai/seipp/projects" +BOUNDS_FILE = "bounds.json" +SUITE = ["depot:p01.pddl", "grid:prob01.pddl", "gripper:prob01.pddl"] +try: + REVISION_CACHE = Path(os.environ["DOWNWARD_REVISION_CACHE"]) +except KeyError: + REVISION_CACHE = Path(get_default_data_dir()) / "revision-cache" +if project.REMOTE: + # ENV = project.BaselSlurmEnvironment(email="my.name@myhost.ch") + ENV = project.TetralithEnvironment( + email="first.last@liu.se", extra_options="#SBATCH --account=snic2022-5-341" + ) + SUITE = project.SUITE_OPTIMAL_STRIPS +else: + ENV = project.LocalEnvironment(processes=2) + +CONFIGS = [ + ("ff", ["--search", "lazy_greedy([ff()], bound=BOUND)"]), +] +BUILD_OPTIONS = [] +DRIVER_OPTIONS = [ + "--validate", + "--overall-time-limit", + "5m", + "--overall-memory-limit", + "3584M", +] +# Pairs of revision identifier and revision nick. +REVS = [ + ("main", "main"), +] +ATTRIBUTES = [ + "error", + "run_dir", + "search_start_time", + "search_start_memory", + "total_time", + "h_values", + "coverage", + "expansions", + "memory", + project.EVALUATIONS_PER_TIME, +] + +exp = Experiment(environment=ENV) +for rev, rev_nick in REVS: + cached_rev = CachedFastDownwardRevision(REPO, rev, BUILD_OPTIONS) + cached_rev.cache(REVISION_CACHE) + cache_path = REVISION_CACHE / cached_rev.name + dest_path = Path(f"code-{cached_rev.name}") + exp.add_resource("", cache_path, dest_path) + # Overwrite the script to set an environment variable. + exp.add_resource( + _get_solver_resource_name(cached_rev), + cache_path / "fast-downward.py", + dest_path / "fast-downward.py", + ) + for config_nick, config in CONFIGS: + algo_name = f"{rev_nick}-{config_nick}" if rev_nick else config_nick + + bounds = {} + with open(BOUNDS_FILE) as f: + bounds = json.load(f) + for task in suites.build_suite(BENCHMARKS_DIR, SUITE): + upper_bound = bounds[f"{task.domain}:{task.problem}"] + if upper_bound is None: + upper_bound = "infinity" + config_with_bound = config.copy() + config_with_bound[-1] = config_with_bound[-1].replace( + "bound=BOUND", f"bound={upper_bound}" + ) + algo = _DownwardAlgorithm( + algo_name, + cached_rev, + DRIVER_OPTIONS, + config_with_bound, + ) + run = FastDownwardRun(exp, algo, task) + exp.add_run(run) + +exp.add_parser(project.FastDownwardExperiment.EXITCODE_PARSER) +exp.add_parser(project.FastDownwardExperiment.TRANSLATOR_PARSER) +exp.add_parser(project.FastDownwardExperiment.SINGLE_SEARCH_PARSER) +exp.add_parser(project.DIR / "parser.py") +exp.add_parser(project.FastDownwardExperiment.PLANNER_PARSER) + +exp.add_step("build", exp.build) +exp.add_step("start", exp.start_runs) +exp.add_fetcher(name="fetch") + +if not project.REMOTE: + exp.add_step("remove-eval-dir", shutil.rmtree, exp.eval_dir, ignore_errors=True) + project.add_scp_step(exp, SCP_LOGIN, REMOTE_REPOS_DIR) + +project.add_absolute_report( + exp, + attributes=ATTRIBUTES, + filter=[project.add_evaluations_per_time, project.group_domains], +) + +exp.run_steps() diff --git a/examples/downward/README.md b/examples/downward/README.md new file mode 100644 index 000000000..4c7e52dc5 --- /dev/null +++ b/examples/downward/README.md @@ -0,0 +1,23 @@ +# Setup instructions + +Create a [virtual environment](https://docs.python.org/3/tutorial/venv.html), +activate it and install all dependencies: + + sudo apt install python3 python3-venv + python3 -m venv --prompt myvenv .venv + source .venv/bin/activate + pip install --upgrade pip wheel + pip install -r requirements.txt + +If the last step fails, try regenerating a new `requirements.txt` from +`requirements.in` for your Python version: + + source .venv/bin/activate + pip install pip-tools + pip-compile + pip install -r requirements.txt + +Please note that before running an experiment script you need to +activate the virtual environment with + + source .venv/bin/activate diff --git a/examples/downward/bounds.json b/examples/downward/bounds.json new file mode 100644 index 000000000..f2e49019e --- /dev/null +++ b/examples/downward/bounds.json @@ -0,0 +1,5 @@ +{ + "depot:p01.pddl": 16, + "grid:prob01.pddl": 20, + "gripper:prob01.pddl": 25 +} diff --git a/examples/downward/project.py b/examples/downward/project.py index 764b69bdf..bfb366344 100644 --- a/examples/downward/project.py +++ b/examples/downward/project.py @@ -1,36 +1,40 @@ -from collections import namedtuple -import getpass from pathlib import Path import platform -import shutil +import re import subprocess import sys from downward.experiment import FastDownwardExperiment from downward.reports.absolute import AbsoluteReport from downward.reports.scatter import ScatterPlotReport +from downward.reports.taskwise import TaskwiseReport from lab import tools -from lab.environments import BaselSlurmEnvironment, LocalEnvironment +from lab.environments import ( + BaselSlurmEnvironment, + LocalEnvironment, + TetralithEnvironment, +) from lab.experiment import ARGPARSER from lab.reports import Attribute, geometric_mean # Silence import-unused messages. Experiment scripts may use these imports. -assert BaselSlurmEnvironment and LocalEnvironment and ScatterPlotReport +assert ( + BaselSlurmEnvironment + and FastDownwardExperiment + and LocalEnvironment + and ScatterPlotReport + and TaskwiseReport + and TetralithEnvironment +) DIR = Path(__file__).resolve().parent NODE = platform.node() -REMOTE = NODE.endswith(".scicore.unibas.ch") or NODE.endswith(".cluster.bc2.ch") - -User = namedtuple("User", ["scp_login", "remote_repos"]) -USERS = { - "jendrik": User( - scp_login="seipp@login.scicore.unibas.ch", - remote_repos="/infai/seipp/projects", - ), -} -USER = USERS.get(getpass.getuser()) +# Cover both the Basel and Linköping clusters for simplicity. +REMOTE = NODE.endswith((".scicore.unibas.ch", ".cluster.bc2.ch")) or re.match( + r"tetralith\d+\.nsc\.liu\.se|n\d+", NODE +) def parse_args(): @@ -79,9 +83,147 @@ def parse_args(): "visitall-sat11-strips", "visitall-sat14-strips", "woodworking-sat08-strips", "woodworking-sat11-strips", "zenotravel", ] + +SUITE_OPTIMAL_STRIPS = [ + "agricola-opt18-strips", "airport", "barman-opt11-strips", + "barman-opt14-strips", "blocks", "childsnack-opt14-strips", + "data-network-opt18-strips", "depot", "driverlog", "elevators-opt08-strips", + "elevators-opt11-strips", "floortile-opt11-strips", "floortile-opt14-strips", + "freecell", "ged-opt14-strips", "grid", "gripper", "hiking-opt14-strips", + "logistics00", "logistics98", "miconic", "movie", "mprime", "mystery", + "nomystery-opt11-strips", "openstacks-opt08-strips", "openstacks-opt11-strips", + "openstacks-opt14-strips", "openstacks-strips", "organic-synthesis-opt18-strips", + "organic-synthesis-split-opt18-strips", "parcprinter-08-strips", + "parcprinter-opt11-strips", "parking-opt11-strips", "parking-opt14-strips", + "pathways", "pegsol-08-strips", "pegsol-opt11-strips", + "petri-net-alignment-opt18-strips", "pipesworld-notankage", "pipesworld-tankage", + "psr-small", "rovers", "satellite", "scanalyzer-08-strips", + "scanalyzer-opt11-strips", "snake-opt18-strips", "sokoban-opt08-strips", + "sokoban-opt11-strips", "spider-opt18-strips", "storage", "termes-opt18-strips", + "tetris-opt14-strips", "tidybot-opt11-strips", "tidybot-opt14-strips", "tpp", + "transport-opt08-strips", "transport-opt11-strips", "transport-opt14-strips", + "trucks-strips", "visitall-opt11-strips", "visitall-opt14-strips", + "woodworking-opt08-strips", "woodworking-opt11-strips", "zenotravel", +] + +DOMAIN_GROUPS = { + "airport": ["airport"], + "assembly": ["assembly"], + "barman": [ + "barman", "barman-opt11-strips", "barman-opt14-strips", + "barman-sat11-strips", "barman-sat14-strips"], + "blocksworld": ["blocks", "blocksworld"], + "cavediving": ["cavediving-14-adl"], + "childsnack": ["childsnack-opt14-strips", "childsnack-sat14-strips"], + "citycar": ["citycar-opt14-adl", "citycar-sat14-adl"], + "depots": ["depot", "depots"], + "driverlog": ["driverlog"], + "elevators": [ + "elevators-opt08-strips", "elevators-opt11-strips", + "elevators-sat08-strips", "elevators-sat11-strips"], + "floortile": [ + "floortile-opt11-strips", "floortile-opt14-strips", + "floortile-sat11-strips", "floortile-sat14-strips"], + "freecell": ["freecell"], + "ged": ["ged-opt14-strips", "ged-sat14-strips"], + "grid": ["grid"], + "gripper": ["gripper"], + "hiking": ["hiking-opt14-strips", "hiking-sat14-strips"], + "logistics": ["logistics98", "logistics00"], + "maintenance": ["maintenance-opt14-adl", "maintenance-sat14-adl"], + "miconic": ["miconic", "miconic-strips"], + "miconic-fulladl": ["miconic-fulladl"], + "miconic-simpleadl": ["miconic-simpleadl"], + "movie": ["movie"], + "mprime": ["mprime"], + "mystery": ["mystery"], + "nomystery": ["nomystery-opt11-strips", "nomystery-sat11-strips"], + "openstacks": [ + "openstacks", "openstacks-strips", "openstacks-opt08-strips", + "openstacks-opt11-strips", "openstacks-opt14-strips", + "openstacks-sat08-adl", "openstacks-sat08-strips", + "openstacks-sat11-strips", "openstacks-sat14-strips", + "openstacks-opt08-adl", "openstacks-sat08-adl"], + "optical-telegraphs": ["optical-telegraphs"], + "parcprinter": [ + "parcprinter-08-strips", "parcprinter-opt11-strips", "parcprinter-sat11-strips"], + "parking": [ + "parking-opt11-strips", "parking-opt14-strips", + "parking-sat11-strips", "parking-sat14-strips"], + "pathways": ["pathways"], + "pathways-noneg": ["pathways-noneg"], + "pegsol": ["pegsol-08-strips", "pegsol-opt11-strips", "pegsol-sat11-strips"], + "philosophers": ["philosophers"], + "pipes-nt": ["pipesworld-notankage"], + "pipes-t": ["pipesworld-tankage"], + "psr": ["psr-middle", "psr-large", "psr-small"], + "rovers": ["rover", "rovers"], + "satellite": ["satellite"], + "scanalyzer": [ + "scanalyzer-08-strips", "scanalyzer-opt11-strips", "scanalyzer-sat11-strips"], + "schedule": ["schedule"], + "sokoban": [ + "sokoban-opt08-strips", "sokoban-opt11-strips", + "sokoban-sat08-strips", "sokoban-sat11-strips"], + "storage": ["storage"], + "tetris": ["tetris-opt14-strips", "tetris-sat14-strips"], + "thoughtful": ["thoughtful-sat14-strips"], + "tidybot": [ + "tidybot-opt11-strips", "tidybot-opt14-strips", + "tidybot-sat11-strips", "tidybot-sat14-strips"], + "tpp": ["tpp"], + "transport": [ + "transport-opt08-strips", "transport-opt11-strips", "transport-opt14-strips", + "transport-sat08-strips", "transport-sat11-strips", "transport-sat14-strips"], + "trucks": ["trucks", "trucks-strips"], + "visitall": [ + "visitall-opt11-strips", "visitall-opt14-strips", + "visitall-sat11-strips", "visitall-sat14-strips"], + "woodworking": [ + "woodworking-opt08-strips", "woodworking-opt11-strips", + "woodworking-sat08-strips", "woodworking-sat11-strips"], + "zenotravel": ["zenotravel"], + # IPC 2018: + "agricola": ["agricola", "agricola-opt18-strips", "agricola-sat18-strips"], + "caldera": ["caldera-opt18-adl", "caldera-sat18-adl"], + "caldera-split": ["caldera-split-opt18-adl", "caldera-split-sat18-adl"], + "data-network": [ + "data-network", "data-network-opt18-strips", "data-network-sat18-strips"], + "flashfill": ["flashfill-sat18-adl"], + "nurikabe": ["nurikabe-opt18-adl", "nurikabe-sat18-adl"], + "organic-split": [ + "organic-synthesis-split", "organic-synthesis-split-opt18-strips", + "organic-synthesis-split-sat18-strips"], + "organic" : [ + "organic-synthesis", "organic-synthesis-opt18-strips", + "organic-synthesis-sat18-strips"], + "petri-net": [ + "petri-net-alignment", "petri-net-alignment-opt18-strips", + "petri-net-alignment-sat18-strips"], + "settlers": ["settlers-opt18-adl", "settlers-sat18-adl"], + "snake": ["snake", "snake-opt18-strips", "snake-sat18-strips"], + "spider": ["spider", "spider-opt18-strips", "spider-sat18-strips"], + "termes": ["termes", "termes-opt18-strips", "termes-sat18-strips"], +} # fmt: on +DOMAIN_RENAMINGS = {} +for group_name, domains in DOMAIN_GROUPS.items(): + for domain in domains: + DOMAIN_RENAMINGS[domain] = group_name +for group_name in DOMAIN_GROUPS: + DOMAIN_RENAMINGS[group_name] = group_name + + +def group_domains(run): + old_domain = run["domain"] + run["domain"] = DOMAIN_RENAMINGS[old_domain] + run["problem"] = old_domain + "-" + run["problem"] + run["id"][2] = run["problem"] + return run + + def get_repo_base() -> Path: """Get base directory of the repository, as an absolute path. @@ -121,8 +263,8 @@ def _get_exp_dir_relative_to_repo(): return repo_name / rel_script_dir / "data" / expname -def add_scp_step(exp): - remote_exp = Path(USER.remote_repos) / _get_exp_dir_relative_to_repo() +def add_scp_step(exp, login, repos_dir): + remote_exp = Path(repos_dir) / _get_exp_dir_relative_to_repo() exp.add_step( "scp-eval-dir", subprocess.call, @@ -130,7 +272,7 @@ def add_scp_step(exp): "scp", "-r", # Copy recursively. "-C", # Compress files. - f"{USER.scp_login}:{remote_exp}-eval", + f"{login}:{remote_exp}-eval", f"{exp.path}-eval", ], ) @@ -155,6 +297,28 @@ def rename_and_filter(run): ) +def fetch_algorithms(exp, expname, *, algos=None, name=None, filters=None): + """ + Fetch multiple or all algorithms. + """ + assert not expname.rstrip("/").endswith("-eval") + algos = set(algos or []) + filters = filters or [] + if algos: + + def algo_filter(run): + return run["algorithm"] in algos + + filters.append(algo_filter) + + exp.add_fetcher( + f"data/{expname}-eval", + filter=filters, + name=name or f"fetch-from-{expname}", + merge=True, + ) + + def add_absolute_report(exp, *, name=None, outfile=None, **kwargs): report = AbsoluteReport(**kwargs) if name and not outfile: @@ -172,38 +336,3 @@ def add_absolute_report(exp, *, name=None, outfile=None, **kwargs): if not REMOTE: exp.add_step(f"open-{name}", subprocess.call, ["xdg-open", outfile]) exp.add_step(f"publish-{name}", subprocess.call, ["publish", outfile]) - - -class CommonExperiment(FastDownwardExperiment): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - self.add_step("build", self.build) - self.add_step("start", self.start_runs) - self.add_fetcher(name="fetch") - - if not REMOTE: - self.add_step( - "remove-eval-dir", shutil.rmtree, self.eval_dir, ignore_errors=True - ) - add_scp_step(self) - - self.add_parser(self.EXITCODE_PARSER) - self.add_parser(self.TRANSLATOR_PARSER) - self.add_parser(self.SINGLE_SEARCH_PARSER) - self.add_parser(self.PLANNER_PARSER) - self.add_parser(DIR / "parser.py") - - def _add_runs(self): - """ - Example showing how to modify the automatically generated runs. - - This uses private members, so it might break between different - versions of Lab. - - """ - FastDownwardExperiment._add_runs(self) - for run in self.runs: - command = run.commands["planner"] - # Slightly raise soft limit for output to stdout. - command[1]["soft_stdout_limit"] = 1.5 * 1024 diff --git a/examples/lmcut.py b/examples/lmcut.py index 29df28564..bc65d0917 100755 --- a/examples/lmcut.py +++ b/examples/lmcut.py @@ -12,7 +12,7 @@ from lab.environments import BaselSlurmEnvironment, LocalEnvironment -ATTRIBUTES = ["coverage", "error", "expansions", "total_time"] +ATTRIBUTES = ["coverage", "error", "expansions", "planner_memory", "planner_time"] NODE = platform.node() if NODE.endswith((".cluster.bc2.ch", ".scicore.unibas.ch")): @@ -52,7 +52,9 @@ exp.add_fetcher(name="fetch") # Add report step (AbsoluteReport is the standard report). -exp.add_report(AbsoluteReport(attributes=ATTRIBUTES), outfile="report.html") +exp.add_report( + AbsoluteReport(attributes=ATTRIBUTES, format="html"), outfile="report.html" +) # Add scatter plot report step. exp.add_report( diff --git a/examples/showcase-options.py b/examples/showcase-options.py index c71e8b651..6c6dbf5cc 100755 --- a/examples/showcase-options.py +++ b/examples/showcase-options.py @@ -172,11 +172,15 @@ def eval_dir(num): name="report-abs-p-filter", ) exp.add_report( - AbsoluteReport(attributes=["coverage", "error"], format="tex"), + AbsoluteReport( + attributes=["coverage", "error", "score_planner_time"], format="tex" + ), outfile="report-abs-combined.tex", ) exp.add_report( - AbsoluteReport(attributes=["coverage", "error"], format="html"), + AbsoluteReport( + attributes=["coverage", "error", "score_planner_memory"], format="html" + ), outfile="report-abs-combined.html", ) exp.add_report( diff --git a/examples/singularity/filter-stderr.py b/examples/singularity/filter-stderr.py index 45cfd7db3..a5eb43d6b 100755 --- a/examples/singularity/filter-stderr.py +++ b/examples/singularity/filter-stderr.py @@ -13,6 +13,7 @@ "differs from the one in the portfolio file", "Terminated", "Killed", + "underlay of /etc/localtime required more than", ] @@ -22,7 +23,7 @@ def main(): if stderr.is_file(): need_to_filter = False filtered_content = [] - with open(stderr, "r") as f: + with open(stderr) as f: for line in f: if any(pattern in line for pattern in IGNORE_PATTERNS): need_to_filter = True diff --git a/lab/__init__.py b/lab/__init__.py index 238ae3865..1f451fafe 100644 --- a/lab/__init__.py +++ b/lab/__init__.py @@ -1,2 +1,2 @@ #: Lab version number. A "+" is appended to all non-tagged revisions. -__version__ = "7.0+" +__version__ = "7.3+" diff --git a/lab/cached_revision.py b/lab/cached_revision.py index 1d49c36d6..8c5992f79 100644 --- a/lab/cached_revision.py +++ b/lab/cached_revision.py @@ -78,6 +78,7 @@ def __init__(self, repo, rev, build_cmd, exclude=None): ... rev = "main" ... cr = CachedRevision(repo, rev, ["./build.py"], exclude=["experiments"]) ... # cr.cache(revision_cache) # Uncomment to actually cache the code. + ... You can now copy the cached repo to your experiment: diff --git a/lab/experiment.py b/lab/experiment.py index 4a1acea42..75fb5dd27 100644 --- a/lab/experiment.py +++ b/lab/experiment.py @@ -203,7 +203,9 @@ def add_command( After *time_limit* seconds the signal SIGXCPU is sent to the command. The process can catch this signal and exit gracefully. If it doesn't catch the SIGXCPU signal, the command is aborted - with SIGKILL after five additional seconds. + with SIGKILL after five additional seconds. The time spent by a + command is the sum of time spent across all threads of the + process. The command is aborted with SIGKILL when it uses more than *memory_limit* MiB. diff --git a/lab/fetcher.py b/lab/fetcher.py index b29b2ce9a..07d3fa3c2 100644 --- a/lab/fetcher.py +++ b/lab/fetcher.py @@ -107,7 +107,10 @@ def __call__(self, src_dir, eval_dir=None, merge=None, filter=None, **kwargs): os.path.join(src_dir, "runs-00001-00100") ) if fetch_from_eval_dir: - src_props = tools.Properties(filename=os.path.join(src_dir, "properties")) + src_path = os.path.join(src_dir, "properties") + src_props = tools.Properties(filename=src_path) + if not src_props: + logging.critical(f"No properties found in {src_dir}") run_filter.apply(src_props) combined_props.update(src_props) logging.info(f"Fetched properties of {len(src_props)} runs.") diff --git a/lab/parser.py b/lab/parser.py index 7b594b810..5bd9b1dc4 100644 --- a/lab/parser.py +++ b/lab/parser.py @@ -47,6 +47,12 @@ def _get_pattern_flags(s): return flags +class _Function: + def __init__(self, function, filename): + self.function = function + self.filename = filename + + class _Pattern: def __init__(self, attribute, regex, required, type_, flags): self.attribute = attribute @@ -81,15 +87,13 @@ def __str__(self): class _FileParser: """ - Private class that parses a given file according to the added patterns - and functions. + Private class that parses a given file according to the added patterns. """ def __init__(self): self.filename = None self.content = None self.patterns = [] - self.functions = [] def load_file(self, filename): self.filename = filename @@ -99,9 +103,6 @@ def load_file(self, filename): def add_pattern(self, pattern): self.patterns.append(pattern) - def add_function(self, function): - self.functions.append(function) - def search_patterns(self): assert self.content is not None found_props = {} @@ -109,11 +110,6 @@ def search_patterns(self): found_props.update(pattern.search(self.content, self.filename)) return found_props - def apply_functions(self, props): - assert self.content is not None - for function in self.functions: - function(self.content, props) - class Parser: """ @@ -124,6 +120,7 @@ class Parser: def __init__(self): tools.configure_logging() self.file_parsers = defaultdict(_FileParser) + self.functions = [] def add_pattern( self, attribute, regex, file="run.log", type=int, flags="", required=False @@ -163,7 +160,7 @@ def add_function(self, function, file="run.log"): r"""Call ``function(open(file).read(), properties)`` during parsing. Functions are applied **after** all patterns have been - evaluated. + evaluated and in the order in which they are added to the parser. The function is passed the file contents and the properties dictionary. It must manipulate the passed properties @@ -185,7 +182,7 @@ def add_function(self, function, file="run.log"): run. """ - self.file_parsers[file].add_function(function) + self.functions.append(_Function(function, file)) def parse(self): """Search all patterns and apply all functions. @@ -212,7 +209,9 @@ def parse(self): for file_parser in self.file_parsers.values(): self.props.update(file_parser.search_patterns()) - for file_parser in self.file_parsers.values(): - file_parser.apply_functions(self.props) + for function in self.functions: + with open(function.filename) as f: + content = f.read() + function.function(content, self.props) self.props.write() diff --git a/lab/reports/__init__.py b/lab/reports/__init__.py index 0fadb5638..ac5a0e464 100644 --- a/lab/reports/__init__.py +++ b/lab/reports/__init__.py @@ -40,7 +40,7 @@ def geometric_mean(values): return 0 assert None not in values exp = 1.0 / len(values) - return tools.product([val ** exp for val in values]) + return tools.product([val**exp for val in values]) def finite_sum(values): @@ -126,7 +126,7 @@ def __init__( value for: * *absolute*: if False, only include tasks for which all task - runs have values in a domain-wise table (e.g. ``coverage`` is + runs have values in a per-domain table (e.g. ``coverage`` is absolute, whereas ``expansions`` is not, because we can't compare algorithms A and B for task X if B has no value for ``expansions``). @@ -230,6 +230,7 @@ def __init__(self, attributes=None, format="html", filter=None, **kwargs): >>> def low_init_h(run): ... return run["initial_h_value"] <= 100 + ... >>> report = Report(filter=low_init_h) Only include runs from "blocks" and "barman" with a timeout: @@ -244,6 +245,7 @@ def __init__(self, attributes=None, format="html", filter=None, **kwargs): ... if expansions is not None and time: ... run["expansions_per_time"] = expansions / time ... return run + ... >>> report = Report( ... attributes=["expansions_per_time"], filter=[add_expansions_per_time] ... ) @@ -255,6 +257,7 @@ def __init__(self, attributes=None, format="html", filter=None, **kwargs): ... paper_names = {"lama11": "LAMA 2011", "fdss_sat1": "FDSS 1"} ... run["algorithm"] = paper_names[name] ... return run + ... >>> # We want LAMA 2011 to be the leftmost column. >>> # filter_* filters are evaluated last, so we use the updated @@ -401,7 +404,7 @@ def get_text(self): "This happens when no significant changes occured or " "if for all attributes and all problems never all " "algorithms had a value for this attribute in a " - "domain-wise report." + "per-domain report." ) return doc.render(self.output_format, {"toc": self.toc}) @@ -442,14 +445,11 @@ def _scan_data(self): def _load_data(self): props_file = os.path.join(self.eval_dir, "properties") - if not os.path.exists(props_file): - logging.critical(f"Properties file not found at {props_file}") - logging.info("Reading properties file") self.props = tools.Properties(filename=props_file) - logging.info("Reading properties file finished") if not self.props: - logging.critical("properties file in evaluation dir is empty.") + logging.critical(f"No properties found in {self.eval_dir}") + logging.info("Reading properties file finished") def _apply_filter(self): self.run_filter.apply(self.props) @@ -460,10 +460,14 @@ def _apply_filter(self): class CellFormatter: """Formating information for one cell in a table.""" - def __init__(self, bold=False, count=None, link=None): + def __init__( + self, bold=False, count=None, link=None, color=None, align_right=False + ): self.bold = bold self.count = count self.link = link + self.color = color + self.align_right = align_right def format_value(self, value): result = str(value) @@ -473,6 +477,14 @@ def format_value(self, value): result = f"{result} ({self.count})" if self.bold: result = f"**{result}**" + if self.color: + result = f"{{{result}|color:{self.color}}}" + + if self.align_right: + result = " " + result + else: + result += " " + return result @@ -501,10 +513,11 @@ def __init__(self, title="", min_wins=None, colored=False, digits=2): >>> t.add_row("prob2", {"cfg1": 15, "cfg2": 25}) >>> def remove_quotes(s): ... return s.replace('""', "") + ... >>> print(remove_quotes(str(t))) || expansions | cfg1 | cfg2 | - | prob1 | 10 | 20 | - | prob2 | 15 | 25 | + | prob1 | 10 | 20 | + | prob2 | 15 | 25 | >>> t.row_names ['prob1', 'prob2'] >>> t.col_names @@ -516,15 +529,15 @@ def __init__(self, title="", min_wins=None, colored=False, digits=2): >>> t.add_summary_function("SUM", sum) >>> print(remove_quotes(str(t))) || expansions | cfg1 | cfg2 | - | prob1 | 10 | 20 | - | prob2 | 15 | 25 | - | **SUM** | 25 | 45 | + | prob1 | 10 | 20 | + | prob2 | 15 | 25 | + | **SUM** | 25 | 45 | >>> t.set_column_order(["cfg2", "cfg1"]) >>> print(remove_quotes(str(t))) || expansions | cfg2 | cfg1 | - | prob1 | 20 | 10 | - | prob2 | 25 | 15 | - | **SUM** | 45 | 25 | + | prob1 | 20 | 10 | + | prob2 | 25 | 15 | + | **SUM** | 45 | 25 | """ collections.defaultdict.__init__(self, dict) @@ -794,20 +807,13 @@ def _format_cell(self, row_name, col_name, value, color=None, bold=False): default format. """ formatter = self.cell_formatters.get(row_name, {}).get(col_name) - if formatter: - return formatter.format_value(value) - - justify_right = isinstance(value, (float, int)) - - value_text = self._format_value(value) - - if color is not None: - value_text = f"{{{value_text}|color:{color}}}" - if bold: - value_text = f"**{value_text}**" - if justify_right: - value_text = " " + value_text - return value_text + if not formatter: + align_right = ( + isinstance(value, (float, int)) or value is None or value == "?" + ) + value = self._format_value(value) + formatter = CellFormatter(bold=bold, color=color, align_right=align_right) + return formatter.format_value(value) def _get_markup(self, cells): """ diff --git a/lab/reports/filter.py b/lab/reports/filter.py index 961d688cc..0fedfc1cd 100644 --- a/lab/reports/filter.py +++ b/lab/reports/filter.py @@ -9,6 +9,7 @@ class FilterReport(Report): >>> def remove_openstacks(run): ... return "openstacks" not in run["domain"] + ... >>> from lab.experiment import Experiment >>> report = FilterReport(filter=remove_openstacks) diff --git a/lab/tools.py b/lab/tools.py index 03d56e1ba..6a484b634 100644 --- a/lab/tools.py +++ b/lab/tools.py @@ -2,6 +2,8 @@ import colorsys import functools import logging +import lzma +import math import os from pathlib import Path import pkgutil @@ -233,6 +235,21 @@ def add_unexplained_error(dictionary, error): dictionary[key].append(error) +def compute_log_score(success, value, lower_bound, upper_bound): + """Compute score between 0 and 1. + + Best possible performance (value <= lower_bound) counts as 1, while failed + runs (!success) and worst performance (value >= upper_bound) counts as 0. + """ + if value is None or not success: + return 0.0 + value = max(value, lower_bound) + value = min(value, upper_bound) + raw_score = math.log(value) - math.log(upper_bound) + best_raw_score = math.log(lower_bound) - math.log(upper_bound) + return raw_score / best_raw_score + + class Properties(dict): class _PropertiesEncoder(json.JSONEncoder): def default(self, o): @@ -241,37 +258,50 @@ def default(self, o): else: return super().default(o) + JSON_ARGS = { + "cls": _PropertiesEncoder, + "indent": 2, + "separators": (",", ": "), + "sort_keys": True, + } + + """Transparently handle properties files compressed with xz.""" + def __init__(self, filename=None): - self.filename = filename - self.load(filename) + self.path = filename + if self.path: + self.path = Path(self.path).resolve() + xz_path = self.path.with_suffix(".xz") + if self.path.is_file() and xz_path.is_file(): + logging.critical(f"Only one of {self.path} and {xz_path} may exist") + if not self.path.is_file() and xz_path.is_file(): + self.path = xz_path + if self.path.is_file(): + self.load(self.path) dict.__init__(self) def __str__(self): - return json.dumps( - self, - cls=self._PropertiesEncoder, - indent=2, - separators=(",", ": "), - sort_keys=True, - ) + return json.dumps(self, **self.JSON_ARGS) def load(self, filename): - if not filename or not os.path.exists(filename): - return - with open(filename) as f: + path = Path(filename) + open_func = lzma.open if path.suffix == ".xz" else open + with open_func(path) as f: try: self.update(json.load(f)) except ValueError as e: - logging.critical(f"JSON parse error in file '{filename}': {e}") + logging.critical(f"JSON parse error in file '{path}': {e}") def add_unexplained_error(self, error): add_unexplained_error(self, error) def write(self): """Write the properties to disk.""" - assert self.filename - makedirs(os.path.dirname(self.filename)) - write_file(self.filename, str(self)) + assert self.path + self.path.parent.mkdir(parents=True, exist_ok=True) + open_func = lzma.open if self.path.suffix == ".xz" else open + with open_func(self.path, "w") as f: + json.dump(self, f, **self.JSON_ARGS) class RunFilter: @@ -488,10 +518,7 @@ def get_unexplained_errors_message(run): if not unexplained_errors or unexplained_errors == ["output-to-slurm.err"]: return "" else: - return ( - f"Unexplained error(s) in {run['run_dir']}: please inspect" - f" output and error logs." - ) + return f"Unexplained error(s) in {run['run_dir']}: {unexplained_errors}" def get_slurm_err_content(src_dir): diff --git a/pyproject.toml b/pyproject.toml index a2f6a5ffd..e5f3f7ff9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.black] line-length = 88 -target-version = ['py36', 'py37', 'py38'] +target-version = ['py37', 'py38', 'py39', 'py310'] include = '\.pyi?$' exclude = ''' /( diff --git a/setup.py b/setup.py index 391ecbb36..daae36a17 100644 --- a/setup.py +++ b/setup.py @@ -28,10 +28,11 @@ "Intended Audience :: Science/Research", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering", ], install_requires=[ @@ -39,5 +40,5 @@ "simplejson", # optional, speeds up reading properties files "txt2tags>=3.6", # for HTML and Latex reports ], - python_requires=">=3.6", + python_requires=">=3.7", ) diff --git a/tests/check-style b/tests/check-style index e1526669b..b2d6206cd 100755 --- a/tests/check-style +++ b/tests/check-style @@ -7,4 +7,4 @@ python -m blackdoc --check . # E203: whitespace before ':' (not compliant with PEP 8) python -m flake8 --extend-ignore E203 --exclude=build,data,revision-cache,conf.py,.git,.tox,.venv --max-line-length=90 python -m isort --check-only downward/ examples/ lab/ tests/ setup.py -python -m pyupgrade --py36-plus `find downward lab tests -name "*.py"` +python -m pyupgrade --py37-plus `find downward lab tests -name "*.py"` diff --git a/tests/run-downward-experiment b/tests/run-downward-experiment new file mode 100755 index 000000000..10a077de8 --- /dev/null +++ b/tests/run-downward-experiment @@ -0,0 +1,15 @@ +#! /bin/bash + +set -euo pipefail + +DIR=$(dirname ${0}) +DIR=$(realpath ${DIR}) +REPO=$(dirname ${DIR}) +EXPDIR=${DOWNWARD_REPO}/experiments/tmp-downward-lab-project + +rm -rf ${EXPDIR}/data/2020-09-11-* +mkdir -p ${EXPDIR} +cp ${REPO}/examples/downward/*.py ${EXPDIR} +cp ${REPO}/examples/downward/bounds.json ${EXPDIR} +${DIR}/run-example-experiment ${EXPDIR}/2020-09-11-A-cg-vs-ff.py +${DIR}/run-example-experiment ${EXPDIR}/2020-09-11-B-bounded-cost.py diff --git a/tests/run-example-experiment b/tests/run-example-experiment index dd4099042..4805e1e9a 100755 --- a/tests/run-example-experiment +++ b/tests/run-example-experiment @@ -4,7 +4,11 @@ set -euo pipefail check () { expname="$1" - "./${expname}.py" --all + if [[ "${expname}" == *"2020-09-11-"* ]]; then + "./${expname}.py" 1 2 3 6 + else + "./${expname}.py" --all + fi properties="data/$expname-eval/properties" if [[ ! -f "$properties" ]]; then echo "File not found: $properties" diff --git a/tox.ini b/tox.ini index a65b626d0..7ac39b138 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, py38, py39, ff, singularity, style, docs +envlist = py37, py38, py310, py311, ff, singularity, style, docs # Skip py39 since it chokes on distutils. basepython = python3 skip_missing_interpreters = true @@ -7,7 +7,7 @@ skip_missing_interpreters = true deps = pytest commands = - pytest --doctest-modules downward lab tests examples/showcase-options.py + pytest --doctest-modules --ignore=downward/scripts downward lab tests examples/showcase-options.py bash {toxinidir}/tests/run-example-experiment vertex-cover/exp.py bash {toxinidir}/tests/run-example-experiment lmcut.py bash {toxinidir}/tests/run-example-experiment showcase-options.py @@ -17,7 +17,7 @@ passenv = DOWNWARD_BENCHMARKS DOWNWARD_REPO DOWNWARD_REVISION_CACHE -whitelist_externals = +allowlist_externals = bash [testenv:ff] @@ -25,7 +25,17 @@ commands = bash {toxinidir}/tests/run-example-experiment ff/ff.py passenv = DOWNWARD_BENCHMARKS -whitelist_externals = +allowlist_externals = + bash + +[testenv:downward] +commands = + bash {toxinidir}/tests/run-downward-experiment +passenv = + DOWNWARD_BENCHMARKS + DOWNWARD_REPO + DOWNWARD_REVISION_CACHE +allowlist_externals = bash [testenv:singularity] @@ -34,7 +44,7 @@ commands = passenv = DOWNWARD_BENCHMARKS SINGULARITY_IMAGES -whitelist_externals = +allowlist_externals = bash [testenv:docs] @@ -48,14 +58,14 @@ commands = [testenv:style] skipsdist = true deps = - black==20.8b0 - blackdoc==0.1.2 - flake8 + black==22.3.0 + blackdoc==0.3.4 + flake8==4.0.1 flake8-2020 flake8-bugbear flake8-comprehensions isort>=5.0,<5.1 - pyupgrade==2.18.3 + pyupgrade==3.3.1 vulture commands = bash {toxinidir}/tests/find-dead-code @@ -64,14 +74,14 @@ commands = [testenv:fix-style] skipsdist = true deps = - black==20.8b0 - blackdoc==0.1.2 + black==22.3.0 + blackdoc==0.3.4 isort>=5.0,<5.1 - pyupgrade==2.18.3 + pyupgrade==3.3.1 commands = black . blackdoc . isort downward/ examples/ lab/ tests/ setup.py - bash -c 'pyupgrade --py36-plus --exit-zero `find downward lab tests -name "*.py"`' -whitelist_externals = + bash -c 'pyupgrade --py37-plus --exit-zero `find downward lab tests -name "*.py"`' +allowlist_externals = bash