From 92cb3e145ccf27ca56133be7bcd7b2dee0253004 Mon Sep 17 00:00:00 2001 From: Adam Siemieniuk Date: Tue, 6 Aug 2024 12:41:59 +0200 Subject: [PATCH] Named ops benchmarks (#948) Adds MLP model benchmarks generated by mlir-gen with named ops. The goal is to establish baseline performance of the current pipeline on IR using only Linalg named operations. For ease of comparison, the MLP models from base benchmarks are used. --- CMakeLists.txt | 1 + benchmarks/config/base/named-ops.json | 33 +++++++++++++++++++++++++++ scripts/buildkite/benchmark.sh | 1 + 3 files changed, 35 insertions(+) create mode 100644 benchmarks/config/base/named-ops.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 9584802e2..c1d22eaf3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,7 @@ set(BENCH_CFGS ${CONFIG_DIR}/base/base.json ${CONFIG_DIR}/base/pack.json ${CONFIG_DIR}/base/mha.json + ${CONFIG_DIR}/base/named-ops.json ) string(JOIN ',' BENCH_CFGS_STR ${BENCH_CFGS}) # Run a small set of benchmarks with small iterations to test the benchmarks and run locally on small machines diff --git a/benchmarks/config/base/named-ops.json b/benchmarks/config/base/named-ops.json new file mode 100644 index 000000000..0145c554e --- /dev/null +++ b/benchmarks/config/base/named-ops.json @@ -0,0 +1,33 @@ +[ + { + "mlp_named_ops": { + "fp32_3x1024_const_mlir": { + "type": "IR-GEN", + "benchmark": [ "mlir-gen", "--output=named --kernel=const --float-type=f32 --batch=256 --layers=1024,1024,1024,1024" ], + "environment": {}, + "flags": [ "-n", "100" ], + "extensions": [ "(avx2|asimd)" ] + }, + "fp32_3x1024_args_mlir": { + "type": "IR-GEN", + "benchmark": [ "mlir-gen", "--output=named --kernel=args --bias --relu --float-type=f32 --batch=256 --layers=1024,1024,1024,1024" ], + "environment": {}, + "flags": [ "-n", "100" ], + "extensions": [ "(avx2|asimd)" ] + }, + "bf16_3x1024_const_mlir": { + "type": "IR-GEN", + "benchmark": [ "mlir-gen", "--output=named --kernel=const --bias --relu --float-type=bf16 --batch=256 --layers=1024,1024,1024,1024" ], + "environment": {}, + "flags": [ "-n", "100"], + "extensions": [ "(avx2|asimd)" ] + }, + "bf16_3x1024_args_mlir": { + "type": "IR-GEN", + "benchmark": [ "mlir-gen", "--output=named --kernel=args --bias --relu --float-type=bf16 --batch=256 --layers=1024,1024,1024,1024" ], + "environment": {}, + "flags": [ "-n", "100"], + "extensions": [ "(avx2|asimd)" ] + } + }} +] diff --git a/scripts/buildkite/benchmark.sh b/scripts/buildkite/benchmark.sh index 17d834e59..0ea2e70c9 100755 --- a/scripts/buildkite/benchmark.sh +++ b/scripts/buildkite/benchmark.sh @@ -98,6 +98,7 @@ if [ "$BENCH_BASE" ]; then benchmark base/base.json "Base Benchmarks" benchmark base/pack.json "Pack Benchmarks" benchmark base/mha.json "MHA Benchmarks" + benchmark base/named-ops.json "Named Ops Benchmarks" fi # PyTorch model benchmarks