From 02487003ea6c73f450791eef8b359dab78e9a31e Mon Sep 17 00:00:00 2001 From: Rick Ratzel <3039903+rlratzel@users.noreply.github.com> Date: Thu, 21 Nov 2024 09:19:51 -0600 Subject: [PATCH] Includes all deferred conversion costs in benchmarks (#34) This refactors the helper that returns the callable to benchmark to fully expand Mapping return types. This is needed in case those return types lazily convert values from device to host, so the complete cost can be included in the measured/benchmarked run. This is similar to exhausting returned generators that deferred computations in NetworkX, and doing this keeps us consistent with benchmarking the full compute and/or conversion costs for both libraries. I changed the option name from `exhaust_returned_iterator` to `force_unlazy_eval` since this is no longer being used only for iterators, but I'm taking suggestions for a better name. Authors: - Rick Ratzel (https://github.com/rlratzel) - Erik Welch (https://github.com/eriknw) Approvers: - Erik Welch (https://github.com/eriknw) - Ralph Liu (https://github.com/nv-rliu) URL: https://github.com/rapidsai/nx-cugraph/pull/34 --- .../nx-cugraph/pytest-based/bench_algos.py | 43 +++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/benchmarks/nx-cugraph/pytest-based/bench_algos.py b/benchmarks/nx-cugraph/pytest-based/bench_algos.py index 015318c5a..d5257e2ef 100644 --- a/benchmarks/nx-cugraph/pytest-based/bench_algos.py +++ b/benchmarks/nx-cugraph/pytest-based/bench_algos.py @@ -130,12 +130,21 @@ def get_backend_wrapper(backend_name): "backend" kwarg on it. """ - def wrap_callable_for_dispatch(func, exhaust_returned_iterator=False): + # FIXME: consider a pytest param to run force_unlazy_eval=False for + # benchmarks that set it to True in order to measure the time spent for any + # deferred computation/conversions. + def wrap_callable_for_dispatch(func, force_unlazy_eval=False): + # force_unlazy_eval=True forces iterators and other containers to + # generate a complete set of results in order to include any deferred + # compute or conversion in the benchmark def wrapper(*args, **kwargs): kwargs["backend"] = backend_name retval = func(*args, **kwargs) - if exhaust_returned_iterator: - retval = list(retval) + if force_unlazy_eval: + if isinstance(retval, Mapping): + retval = dict(retval) + else: + retval = list(retval) return retval return wrapper @@ -440,7 +449,7 @@ def bench_shortest_path(benchmark, graph_obj, backend_wrapper): node = get_highest_degree_node(graph_obj) result = benchmark.pedantic( - target=backend_wrapper(nx.shortest_path), + target=backend_wrapper(nx.shortest_path, force_unlazy_eval=True), args=(G,), kwargs=dict( source=node, @@ -474,7 +483,7 @@ def bench_single_target_shortest_path_length(benchmark, graph_obj, backend_wrapp node = get_highest_degree_node(graph_obj) result = benchmark.pedantic( target=backend_wrapper( - nx.single_target_shortest_path_length, exhaust_returned_iterator=True + nx.single_target_shortest_path_length, force_unlazy_eval=True ), args=(G,), kwargs=dict( @@ -484,9 +493,11 @@ def bench_single_target_shortest_path_length(benchmark, graph_obj, backend_wrapp iterations=iterations, warmup_rounds=warmup_rounds, ) - # exhaust_returned_iterator=True forces the result to a list, but is not - # needed for this algo in NX 3.3+ which returns a dict instead of an - # iterator. Forcing to a list does not change the benchmark timing. + # force_unlazy_eval=True forces iterators and other containers to generate + # a complete set of results (in order to include any deferred compute or + # conversion in the benchmark), but is not needed for this algo in NX 3.3+ + # since it returns a dict instead of an iterator. Forcing eval does not + # change the benchmark timing. assert type(result) is list @@ -525,7 +536,7 @@ def bench_generic_bfs_edges(benchmark, graph_obj, backend_wrapper): G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper) node = get_highest_degree_node(graph_obj) result = benchmark.pedantic( - target=backend_wrapper(nx.generic_bfs_edges, exhaust_returned_iterator=True), + target=backend_wrapper(nx.generic_bfs_edges, force_unlazy_eval=True), args=(G,), kwargs=dict( source=node, @@ -541,7 +552,7 @@ def bench_bfs_edges(benchmark, graph_obj, backend_wrapper): G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper) node = get_highest_degree_node(graph_obj) result = benchmark.pedantic( - target=backend_wrapper(nx.bfs_edges, exhaust_returned_iterator=True), + target=backend_wrapper(nx.bfs_edges, force_unlazy_eval=True), args=(G,), kwargs=dict( source=node, @@ -557,7 +568,7 @@ def bench_bfs_layers(benchmark, graph_obj, backend_wrapper): G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper) node = get_highest_degree_node(graph_obj) result = benchmark.pedantic( - target=backend_wrapper(nx.bfs_layers, exhaust_returned_iterator=True), + target=backend_wrapper(nx.bfs_layers, force_unlazy_eval=True), args=(G,), kwargs=dict( sources=node, @@ -573,7 +584,7 @@ def bench_bfs_predecessors(benchmark, graph_obj, backend_wrapper): G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper) node = get_highest_degree_node(graph_obj) result = benchmark.pedantic( - target=backend_wrapper(nx.bfs_predecessors, exhaust_returned_iterator=True), + target=backend_wrapper(nx.bfs_predecessors, force_unlazy_eval=True), args=(G,), kwargs=dict( source=node, @@ -589,7 +600,7 @@ def bench_bfs_successors(benchmark, graph_obj, backend_wrapper): G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper) node = get_highest_degree_node(graph_obj) result = benchmark.pedantic( - target=backend_wrapper(nx.bfs_successors, exhaust_returned_iterator=True), + target=backend_wrapper(nx.bfs_successors, force_unlazy_eval=True), args=(G,), kwargs=dict( source=node, @@ -773,7 +784,7 @@ def bench_strongly_connected_components(benchmark, graph_obj, backend_wrapper): G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper) result = benchmark.pedantic( target=backend_wrapper( - nx.strongly_connected_components, exhaust_returned_iterator=True + nx.strongly_connected_components, force_unlazy_eval=True ), args=(G,), rounds=rounds, @@ -816,9 +827,7 @@ def bench_triangles(benchmark, graph_obj, backend_wrapper): def bench_weakly_connected_components(benchmark, graph_obj, backend_wrapper): G = get_graph_obj_for_benchmark(graph_obj, backend_wrapper) result = benchmark.pedantic( - target=backend_wrapper( - nx.weakly_connected_components, exhaust_returned_iterator=True - ), + target=backend_wrapper(nx.weakly_connected_components, force_unlazy_eval=True), args=(G,), rounds=rounds, iterations=iterations,