benchamark

benchmark/Project.toml

@@ -0,0 +1,9 @@
+[deps]
+AirspeedVelocity = "1c8270ee-6884-45cc-9545-60fa71ec23e4"
+BenchmarkPlots = "ab8c0f59-4072-4e0d-8f91-a91e1495eb26"
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
+Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
+Functors = "d9f16b24-f501-4c13-a1f2-28368ffc5196"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+StatsPlots = "f3b207a7-027a-5e70-b257-86293d7955fd"

benchmark/benchmarks.jl

@@ -0,0 +1,55 @@
+using BenchmarkTools: BenchmarkTools, BenchmarkGroup, @benchmarkable, @btime, @benchmark, judge
+using ConcreteStructs: @concrete
+using Flux: Dense, Chain
+using LinearAlgebra: BLAS
+using Functors
+using Statistics: median
+
+const SUITE = BenchmarkGroup()
+const BENCHMARK_CPU_THREADS = Threads.nthreads()
+BLAS.set_num_threads(BENCHMARK_CPU_THREADS)
+
+
+@concrete struct A
+    w
+    b
+    σ
+end
+
+struct B
+    w
+    b
+    σ
+end
+
+function setup_fmap_bench!(suite)
+    a = A(rand(5,5), rand(5), tanh)
+    suite["fmap"]["concrete struct"] = @benchmarkable fmap(identity, $a)
+
+    a = B(rand(5,5), rand(5), tanh)
+    suite["fmap"]["non-concrete struct"] = @benchmarkable fmap(identity, $a)
+
+    a = Dense(5, 5, tanh)
+    suite["fmap"]["flux dense"] = @benchmarkable fmap(identity, $a)
+
+    a = Chain(Dense(5, 5, tanh), Dense(5, 5, tanh))
+    suite["fmap"]["flux dense chain"] = @benchmarkable fmap(identity, $a)
+
+    return suite
+end
+
+setup_fmap_bench!(SUITE)
+
+# results = BenchmarkTools.run(SUITE; verbose=true)
+
+# filename = joinpath(@__DIR__, "benchmarks_old.json")
+# BenchmarkTools.save(filename, median(results))
+
+
+# # Plot
+# using StatsPlots, BenchmarkTools
+# plot(results["fmap"], yaxis=:log10, st=:violin)
+
+# # Compare
+# old_results = BenchmarkTools.load("benchmarks.json")[1]
+# judge(median(results), old_results)