diff --git a/benchmark/Project.toml b/benchmark/Project.toml new file mode 100644 index 0000000..cd75249 --- /dev/null +++ b/benchmark/Project.toml @@ -0,0 +1,9 @@ +[deps] +AirspeedVelocity = "1c8270ee-6884-45cc-9545-60fa71ec23e4" +BenchmarkPlots = "ab8c0f59-4072-4e0d-8f91-a91e1495eb26" +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471" +Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" +Functors = "d9f16b24-f501-4c13-a1f2-28368ffc5196" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +StatsPlots = "f3b207a7-027a-5e70-b257-86293d7955fd" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl new file mode 100644 index 0000000..9ec3d97 --- /dev/null +++ b/benchmark/benchmarks.jl @@ -0,0 +1,55 @@ +using BenchmarkTools: BenchmarkTools, BenchmarkGroup, @benchmarkable, @btime, @benchmark, judge +using ConcreteStructs: @concrete +using Flux: Dense, Chain +using LinearAlgebra: BLAS +using Functors +using Statistics: median + +const SUITE = BenchmarkGroup() +const BENCHMARK_CPU_THREADS = Threads.nthreads() +BLAS.set_num_threads(BENCHMARK_CPU_THREADS) + + +@concrete struct A + w + b + σ +end + +struct B + w + b + σ +end + +function setup_fmap_bench!(suite) + a = A(rand(5,5), rand(5), tanh) + suite["fmap"]["concrete struct"] = @benchmarkable fmap(identity, $a) + + a = B(rand(5,5), rand(5), tanh) + suite["fmap"]["non-concrete struct"] = @benchmarkable fmap(identity, $a) + + a = Dense(5, 5, tanh) + suite["fmap"]["flux dense"] = @benchmarkable fmap(identity, $a) + + a = Chain(Dense(5, 5, tanh), Dense(5, 5, tanh)) + suite["fmap"]["flux dense chain"] = @benchmarkable fmap(identity, $a) + + return suite +end + +setup_fmap_bench!(SUITE) + +# results = BenchmarkTools.run(SUITE; verbose=true) + +# filename = joinpath(@__DIR__, "benchmarks_old.json") +# BenchmarkTools.save(filename, median(results)) + + +# # Plot +# using StatsPlots, BenchmarkTools +# plot(results["fmap"], yaxis=:log10, st=:violin) + +# # Compare +# old_results = BenchmarkTools.load("benchmarks.json")[1] +# judge(median(results), old_results)