diff --git a/.gitignore b/.gitignore index 7c508dbdd..3b6280a29 100644 --- a/.gitignore +++ b/.gitignore @@ -104,6 +104,11 @@ csp/lib/ *.so *.tsbuildinfo +# Benchmarks +.asv +ci/benchmarks/* +!ci/benchmarks/benchmarks.json + # Jupyter / Editors .ipynb_checkpoints .autoversion diff --git a/Makefile b/Makefile index d37aa8f64..752f7b8e8 100644 --- a/Makefile +++ b/Makefile @@ -118,6 +118,28 @@ dockerps: ## spin up docker compose services for adapter testing dockerdown: ## spin up docker compose services for adapter testing $(DOCKER) compose -f ci/$(ADAPTER)/docker-compose.yml down +############## +# BENCHMARKS # +############## +.PHONY: benchmark benchmarks benchmark-regen benchmark-view benchmarks-regen benchmarks-view +benchmark: ## run benchmarks + python -m asv run --config csp/benchmarks/asv.conf.jsonc --verbose `git rev-parse --abbrev-ref HEAD`^! + +# https://github.com/airspeed-velocity/asv/issues/1027 +# https://github.com/airspeed-velocity/asv/issues/488 +benchmark-regen: + python -m asv run --config csp/benchmarks/asv.conf.jsonc --verbose v0.0.4^! + python -m asv run --config csp/benchmarks/asv.conf.jsonc --verbose v0.0.5^! + +benchmark-view: ## generate viewable website of benchmark results + python -m asv publish --config csp/benchmarks/asv.conf.jsonc + python -m asv preview --config csp/benchmarks/asv.conf.jsonc + +# Alias +benchmarks: benchmark +benchmarks-regen: benchmark-regen +benchmarks-view: benchmark-view + ########### # VERSION # ########### diff --git a/ci/benchmarks/benchmarks.json b/ci/benchmarks/benchmarks.json new file mode 100644 index 000000000..e70ce4253 --- /dev/null +++ b/ci/benchmarks/benchmarks.json @@ -0,0 +1,25 @@ +{ + "stats.basic.StatsBenchmarkSuite.time_stats": { + "code": "class StatsBenchmarkSuite:\n def time_stats(self, function):\n def g():\n data = csp.curve(typ=np.ndarray, data=self.data)\n value = getattr(csp.stats, function)(data, interval=self.interval, **self.function_args.get(function, {}))\n csp.add_graph_output(\"final_value\", value, tick_count=1)\n \n timer = Timer(\n lambda: csp.run(g, realtime=False, starttime=self.start_date, endtime=timedelta(seconds=self.num_rows))\n )\n elapsed = timer.timeit(1)\n return elapsed\n\n def setup(self, _):\n self.start_date = datetime(2020, 1, 1)\n self.num_rows = 1_000\n self.array_size = 100\n self.test_times = [self.start_date + timedelta(seconds=i) for i in range(self.num_rows)]\n self.random_values = [\n np.random.normal(size=(self.array_size,)) for i in range(self.num_rows)\n ] # 100 element np array\n self.data = list(zip(self.test_times, self.random_values))\n self.interval = 500", + "min_run_count": 2, + "name": "stats.basic.StatsBenchmarkSuite.time_stats", + "number": 0, + "param_names": [ + "function" + ], + "params": [ + [ + "'median'", + "'quantile'", + "'rank'" + ] + ], + "rounds": 2, + "sample_time": 0.01, + "type": "time", + "unit": "seconds", + "version": "f57f3ee288b0805597f9edee91b4d1dddf41046d34fbd46cfbd7135f459e62e3", + "warmup_time": -1 + }, + "version": 2 +} \ No newline at end of file diff --git a/conda/dev-environment-unix.yml b/conda/dev-environment-unix.yml index fa3740a5b..618210d28 100644 --- a/conda/dev-environment-unix.yml +++ b/conda/dev-environment-unix.yml @@ -3,6 +3,7 @@ channels: - conda-forge - nodefaults dependencies: + - asv - bison - brotli - build diff --git a/conda/dev-environment-win.yml b/conda/dev-environment-win.yml index 8ca2482d8..89f8cb98c 100644 --- a/conda/dev-environment-win.yml +++ b/conda/dev-environment-win.yml @@ -3,6 +3,7 @@ channels: - conda-forge - nodefaults dependencies: + - asv - brotli - build - bump2version>=1 diff --git a/csp/benchmarks/__init__.py b/csp/benchmarks/__init__.py new file mode 100644 index 000000000..55e5f844b --- /dev/null +++ b/csp/benchmarks/__init__.py @@ -0,0 +1 @@ +from .common import * diff --git a/csp/benchmarks/asv.conf.jsonc b/csp/benchmarks/asv.conf.jsonc new file mode 100644 index 000000000..da4f196a7 --- /dev/null +++ b/csp/benchmarks/asv.conf.jsonc @@ -0,0 +1,33 @@ +// https://asv.readthedocs.io/en/v0.6.3/asv.conf.json.html +{ + "version": 1, + "project": "csp", + "project_url": "https://github.com/Point72/csp", + "repo": "../..", + "branches": ["main", "tkp/bm"], + "dvcs": "git", + + "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], + "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], + "build_command": [ + "python -m pip install build", + "python -m build --wheel -o {build_cache_dir} {build_dir}" + ], + "environment_type": "virtualenv", + "install_timeout": 600, + "show_commit_url": "http://github.com/point72/csp/commit/", + + "pythons": ["3.11"], + + // "environment_type": "mamba", + // "conda_channels": ["conda-forge"], + // "conda_environment_file": "conda/dev-environment-unix.yml", + + "benchmark_dir": "../../csp/benchmarks", + "env_dir": "../../.asv/env", + "results_dir": "../../ci/benchmarks", + "html_dir": "../../.asv/html", + + "hash_length": 8, + "build_cache_size": 2 +} diff --git a/csp/benchmarks/common.py b/csp/benchmarks/common.py new file mode 100644 index 000000000..90844a455 --- /dev/null +++ b/csp/benchmarks/common.py @@ -0,0 +1,63 @@ +from asv_runner.benchmarks import benchmark_types +from asv_runner.benchmarks.mark import SkipNotImplemented +from logging import getLogger + +__all__ = ("ASVBenchmarkHelper",) + + +class ASVBenchmarkHelper: + """A helper base class to mimic some of what ASV does when running benchmarks, to + test them outside of ASV. + + NOTE: should be removed in favor of calling ASV itself from python, if possible. + """ + + def __init__(self, *args, **kwargs): + self.log = getLogger(self.__class__.__name__) + + def run_all(self): + # https://asv.readthedocs.io/en/v0.6.3/writing_benchmarks.html#benchmark-types + benchmarks = {} + + for method in dir(self): + for cls in benchmark_types: + if cls.name_regex.match(method): + benchmark_type = cls.__name__.replace("Benchmark", "") + if benchmark_type not in benchmarks: + benchmarks[benchmark_type] = [] + + name = f"{self.__class__.__qualname__}.{method}" + func = getattr(self, method) + benchmarks[benchmark_type].append(cls(name, func, (func, self))) + + def run_benchmark(benchmark): + skip = benchmark.do_setup() + try: + if skip: + return + try: + benchmark.do_run() + except SkipNotImplemented: + pass + finally: + benchmark.do_teardown() + + for type, benchmarks_to_run in benchmarks.items(): + if benchmarks_to_run: + self.log.warn(f"Running benchmarks for {type}") + for benchmark in benchmarks_to_run: + if len(getattr(self, "params", [])): + # TODO: cleaner + param_count = 0 + while param_count < 100: + try: + benchmark.set_param_idx(param_count) + params = benchmark._current_params + self.log.warn(f"[{type}][{benchmark.name}][{'.'.join(str(_) for _ in params)}]") + run_benchmark(benchmark=benchmark) + param_count += 1 + except ValueError: + break + else: + self.log.warn(f"Running [{type}][{benchmark.func.__name__}]") + run_benchmark(benchmark=benchmark) diff --git a/csp/benchmarks/stats/__init__.py b/csp/benchmarks/stats/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/csp/benchmarks/stats/basic.py b/csp/benchmarks/stats/basic.py new file mode 100644 index 000000000..9b212ba2d --- /dev/null +++ b/csp/benchmarks/stats/basic.py @@ -0,0 +1,50 @@ +import numpy as np +from datetime import datetime, timedelta +from timeit import Timer + +import csp +from csp.benchmarks import ASVBenchmarkHelper + +__all__ = ("StatsBenchmarkSuite",) + + +class StatsBenchmarkSuite(ASVBenchmarkHelper): + """ + python -m csp.benchmarks.stats.basic + """ + + params = (("median", "quantile", "rank"),) + param_names = ("function",) + + rounds = 5 + repeat = (100, 200, 60.0) + + function_args = {"quantile": {"quant": 0.95}} + + def setup(self, _): + self.start_date = datetime(2020, 1, 1) + self.num_rows = 1_000 + self.array_size = 100 + self.test_times = [self.start_date + timedelta(seconds=i) for i in range(self.num_rows)] + self.random_values = [ + np.random.normal(size=(self.array_size,)) for i in range(self.num_rows) + ] # 100 element np array + self.data = list(zip(self.test_times, self.random_values)) + self.interval = 500 + + def time_stats(self, function): + def g(): + data = csp.curve(typ=np.ndarray, data=self.data) + value = getattr(csp.stats, function)(data, interval=self.interval, **self.function_args.get(function, {})) + csp.add_graph_output("final_value", value, tick_count=1) + + timer = Timer( + lambda: csp.run(g, realtime=False, starttime=self.start_date, endtime=timedelta(seconds=self.num_rows)) + ) + elapsed = timer.timeit(1) + return elapsed + + +if __name__ == "__main__": + sbs = StatsBenchmarkSuite() + sbs.run_all() diff --git a/pyproject.toml b/pyproject.toml index 511405e8e..35be4b300 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,8 @@ develop = [ "sqlalchemy", # db "threadpoolctl", # test_random "tornado", # profiler, perspective, websocket + # benchmarks + "asv", ] showgraph = [ "graphviz",