diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
new file mode 100644
index 0000000..1fdd088
--- /dev/null
+++ b/.github/workflows/benchmarks.yml
@@ -0,0 +1,106 @@
+name: Benchmarks
+
+on:
+ pull_request:
+ push:
+ branches:
+ - master
+ tags: '*'
+ workflow_dispatch:
+
+concurrency:
+ # Skip intermediate builds: all builds except for builds on the `master` or `release-*` branches
+ # Cancel intermediate builds: only pull request builds
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref != 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-') || github.run_number }}
+ cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+permissions:
+ contents: read
+
+jobs:
+ test:
+ runs-on: ${{ matrix.os }}
+ defaults:
+ run:
+ shell: bash
+ strategy:
+ matrix:
+ os: [ubuntu-latest]
+ java-version: ['17']
+ julia-version: ['1.7.1']
+ python-version: ['3.9']
+ numpy-version: ['1.22']
+ gfortran-version: ['9'] # Note: unused since is built-in.
+ rust-version: ['1.42.0'] # Note: unused since controlled by `rust/rust-toolchain`
+ js-version: ['16']
+ r-version: ['4.1.2']
+ lua-version: ['2.0.5'] # Note: Not used as benchmark is broken.
+ go-version: ['1.17.4'] # Note: Not used as benchmark is broken.
+
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ persist-credentials: false
+ - name: "Cache Julia"
+ id: cache-julia
+ uses: actions/cache@v2
+ with:
+ path: ~/julia
+ key: ${{ runner.os }}-v${{ matrix.julia-version }}
+ - name: "Build Julia"
+ if: steps.cache-julia.outputs.cache-hit != 'true'
+ uses: julia-actions/build-julia@v1
+ with:
+ ref: v${{ matrix.julia-version }}
+ - name: "Install Julia packages"
+ run: |
+ julia -e 'using Pkg; Pkg.add("Compat")'
+ - name: "Set up dSFMT"
+ run: |
+ cd ~/
+ mkdir -p dSFMT
+ cd dSFMT
+ wget https://github.com/MersenneTwister-Lab/dSFMT/archive/refs/tags/v2.2.4.tar.gz
+ echo "39682961ecfba621a98dbb6610b6ae2b7d6add450d4f08d8d4edd0e10abd8174 v2.2.4.tar.gz" | sha256sum --check --status
+ tar -xzf v2.2.4.tar.gz
+ mv dSFMT-*/* ./
+ - name: "Set up OpenBLAS"
+ run: |
+ sudo apt-get install -y libopenblas-dev
+ - name: "Set up Python"
+ uses: actions/setup-python@v1
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: "Set up NumPy"
+ run: pip install numpy==${{ matrix.numpy-version }}
+ - name: "Set up Rust"
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: ${{ matrix.rust-version }}
+ - name: "Set up Java"
+ uses: actions/setup-java@v2
+ with:
+ distribution: 'temurin'
+ java-version: ${{ matrix.java-version }}
+ cache: 'maven'
+ - name: "Set up JavaScript"
+ uses: actions/setup-node@v2
+ with:
+ node-version: ${{ matrix.js-version }}
+ - name: "Set up R"
+ uses: r-lib/actions/setup-r@v2
+ with:
+ r-version: ${{ matrix.r-version }}
+ - name: "Set up LuaJit"
+ uses: leafo/gh-actions-lua@v8.0.0
+ with:
+ luaVersion: luajit-${{ matrix.lua-version }}
+ - name: "Set up Go"
+ uses: actions/setup-go@v2
+ with:
+ go-version: ${{ matrix.go-version }}
+ - name: "Run benchmark"
+ run: |
+ JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.html
+ - name: "Print benchmark data"
+ run: cat gh_action_benchmarks.csv
diff --git a/Makefile b/Makefile
index 319fe5f..6f8988d 100644
--- a/Makefile
+++ b/Makefile
@@ -1,10 +1,21 @@
ifndef JULIAHOME
$(error JULIAHOME not defined. Set value to the root of the Julia source tree.)
endif
+ifndef DSFMTDIR
+$(error DSFMTDIR not defined. Set value to the root of the dSFMT source tree.)
+endif
+
+
+# Will make multi-line targets work
+# (so we can use @for on the second line)
+.ONESHELL:
+
include $(JULIAHOME)/Make.inc
include $(JULIAHOME)/deps/Versions.make
-NODEJSBIN = node8
+NODEJSBIN = node
+
+ITERATIONS=$(shell seq 1 5)
#Use python2 for Python 2.x
PYTHON = python3
@@ -19,13 +30,6 @@ else
MATHEMATICABIN = math
endif
-#Which BLAS library am I using?
-ifeq ($(USE_SYSTEM_BLAS), 0)
-BLASMANIFEST=$(shell cat $(JULIAHOME)/usr/manifest/openblas)
-BLASDIR=$(JULIAHOME)/deps/scratch/$(BLASMANIFEST)/
-LIBBLAS=$(BLASDIR)$(LIBBLASNAME).a
-endif
-
FFLAGS=-fexternal-blas
#gfortran cannot multiply matrices using 64-bit external BLAS.
ifeq ($(findstring gfortran, $(FC)), gfortran)
@@ -43,9 +47,6 @@ LIBM = $(LIBMDIR)libopenlibm.a
endif
endif
-DSFMTDIR = $(JULIAHOME)/deps/scratch/dsfmt-$(DSFMT_VER)
-RMATHDIR = $(JULIAHOME)/deps/scratch/Rmath-julia-$(RMATH_JULIA_VER)
-
default: benchmarks.html
export OMP_NUM_THREADS=1
@@ -53,15 +54,14 @@ export GOTO_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
perf.h: $(JULIAHOME)/deps/Versions.make
- echo '#include "$(BLASDIR)cblas.h"' > $@
+ echo '#include "cblas.h"' > $@
echo '#include "$(DSFMTDIR)/dSFMT.c"' >> $@
bin/perf%: perf.c perf.h
- $(CC) -std=c99 -O$* $< -o $@ -I$(DSFMTDIR) $(LIBBLAS) -L$(LIBMDIR) $(LIBM) $(CFLAGS) -lpthread
+ $(CC) -std=c99 -O$* $< -o $@ -I$(DSFMTDIR) -lopenblas -L$(LIBMDIR) $(LIBM) $(CFLAGS) -lpthread
bin/fperf%: perf.f90
mkdir -p mods/$@ #Modules for each binary go in separate directories
-# $(FC) $(FFLAGS) -Jmods/$@ -O$* $< -o $@ $(LIBBLAS) -L$(LIBMDIR) $(LIBM) -lpthread
$(FC) $(FFLAGS) -Jmods/$@ -O$* $< -o $@ -lopenblas -L$(LIBMDIR) $(LIBM) -lpthread
benchmarks/c.csv: \
@@ -80,73 +80,90 @@ benchmarks/fortran.csv: \
benchmarks/c%.csv: bin/perf%
- for t in 1 2 3 4 5; do $<; done >$@
+ @for t in $(ITERATIONS); do $<; done >$@
benchmarks/fortran%.csv: bin/fperf%
- for t in 1 2 3 4 5; do $<; done >$@
+ @for t in $(ITERATIONS); do $<; done >$@
benchmarks/go.csv: export GOPATH=$(abspath gopath)
benchmarks/go.csv: perf.go
- #CGO_LDFLAGS="$(LIBBLAS) $(LIBM)" go get github.com/gonum/blas/cgo
+ #CGO_LDFLAGS="-lopenblas $(LIBM)" go get github.com/gonum/blas/cgo
go get github.com/gonum/blas/blas64
go get github.com/gonum/blas/cgo
go get github.com/gonum/matrix/mat64
go get github.com/gonum/stat
- for t in 1 2 3 4 5; do go run $<; done >$@
+ @for t in $(ITERATIONS); do go run $<; done >$@
benchmarks/julia.csv: perf.jl
- for t in 1 2 3 4 5; do $(JULIAHOME)/usr/bin/julia $<; done >$@
+ @for t in $(ITERATIONS); do $(JULIAHOME)/usr/bin/julia $<; done >$@
benchmarks/python.csv: perf.py
- for t in 1 2 3 4 5; do $(PYTHON) $<; done >$@
+ @for t in $(ITERATIONS); do $(PYTHON) $<; done >$@
benchmarks/matlab.csv: perf.m
- for t in 1 2 3 4 5; do matlab -nojvm -singleCompThread -r 'perf; perf; exit' | grep ^matlab | tail -8; done >$@
+ @for t in $(ITERATIONS); do matlab -nojvm -singleCompThread -r 'perf; perf; exit' | grep ^matlab | tail -8; done >$@
benchmarks/octave.csv: perf.m
- for t in 1 2 3 4 5; do $(OCTAVE) -q --eval perf 2>/dev/null; done >$@
+ @for t in $(ITERATIONS); do $(OCTAVE) -q --eval perf 2>/dev/null; done >$@
benchmarks/r.csv: perf.R
- for t in 1 2 3 4 5; do cat $< | R --vanilla --slave 2>/dev/null; done >$@
+ @for t in $(ITERATIONS); do cat $< | R --vanilla --slave 2>/dev/null; done >$@
benchmarks/javascript.csv: perf.js
- for t in 1 2 3 4 5; do $(NODEJSBIN) $<; done >$@
+ @for t in $(ITERATIONS); do $(NODEJSBIN) $<; done >$@
benchmarks/mathematica.csv: perf.nb
- for t in 1 2 3 4 5; do $(MATHEMATICABIN) -noprompt -run "<<$<; Exit[]"; done >$@
+ @for t in $(ITERATIONS); do $(MATHEMATICABIN) -noprompt -run "<<$<; Exit[]"; done >$@
benchmarks/stata.csv: perf.do
- for t in 1 2 3 4 5; do stata -b do $^ $@; done
+ @for t in $(ITERATIONS); do stata -b do $^ $@; done
benchmarks/lua.csv: perf.lua
- for t in 1 2 3 4 5; do scilua $<; done >$@
+ @for t in $(ITERATIONS); do luajit $<; done >$@
benchmarks/java.csv: java/src/main/java/PerfBLAS.java
- cd java; sh setup.sh; for t in 1 2 3 4 5; do mvn -q exec:java; done >../$@
+ cd java
+ sh setup.sh
+ @for t in $(ITERATIONS); do mvn -q exec:java; done >../$@
benchmarks/scala.csv: scala/src/main/scala/perf.scala scala/build.sbt
- cd scala; for t in 1 2 3 4 5; do sbt run; done >../$@
+ cd scala
+ @for t in $(ITERATIONS); do sbt run; done >../$@
benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock
- cd rust; for t in 1 2 3 4 5; do cargo run --release -q; done >../$@
+ cd rust
+ @for t in $(ITERATIONS); do cargo run --release -q; done >../$@
LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust
+GH_ACTION_LANGUAGES = c fortran java javascript julia python r rust
# These were formerly listed in LANGUAGES, but I can't get them to run
# 2017-09-27 johnfgibson
# scala, stata
BENCHMARKS = $(foreach lang,$(LANGUAGES),benchmarks/$(lang).csv)
+GH_ACTION_BENCHMARKS = $(foreach lang,$(GH_ACTION_LANGUAGES),benchmarks/$(lang).csv)
+
+COLON_SEPARATED_GHA_LANGUAGES = $(shell echo $(GH_ACTION_LANGUAGES) | sed 's/ /:/g')
versions.csv: bin/versions.sh
$^ >$@
+gh_action_versions.csv: bin/versions.sh
+ $^ $(COLON_SEPARATED_GHA_LANGUAGES) >$@
+
benchmarks.csv: bin/collect.jl $(BENCHMARKS)
@$(call PRINT_JULIA, $^ >$@)
+gh_action_benchmarks.csv: bin/collect.jl $(GH_ACTION_BENCHMARKS)
+ @$(call PRINT_JULIA, $^ >$@)
+
benchmarks.html: bin/table.jl versions.csv benchmarks.csv
@$(call PRINT_JULIA, $^ >$@)
+gh_action_benchmarks.html: bin/table.jl gh_action_versions.csv gh_action_benchmarks.csv
+ @$(call PRINT_JULIA, $^ >$@)
+
clean:
@rm -rf perf.h bin/perf* bin/fperf* benchmarks/*.csv benchmarks.csv mods *~ octave-core perf.log gopath/*
diff --git a/bin/versions.sh b/bin/versions.sh
index 542c3a4..5964303 100755
--- a/bin/versions.sh
+++ b/bin/versions.sh
@@ -1,41 +1,74 @@
#!/usr/bin/env bash
-echo -n "c,gcc "
-gcc -v 2>&1 | grep "gcc version" | cut -f3 -d" "
+# User argument declaring what languages to query:
+DEFAULT_LANGUAGES="c:fortran:go:java:javascript:julia:lua:mathematica:matlab:octave:python:r:rust"
+LANGUAGES=${1:-DEFAULT_LANGUAGES}
-echo -n "fortran,gcc "
-gfortran -v 2>&1 | grep "gcc version" | cut -f3 -d" "
+LANGUAGES=":${LANGUAGES}:"
-echo -n go,
-go version | cut -f3 -d" "
+# Check if ":c:" in languages:
+if [[ $LANGUAGES == *":c:"* ]]; then
+ echo -n "c,gcc "
+ gcc -v 2>&1 | grep "gcc version" | cut -f3 -d" "
+fi
-echo -n java,
-java -version 2>&1 |grep "version" | cut -f3 -d " " | cut -c 2-9
+if [[ $LANGUAGES == *":fortran:"* ]]; then
+ echo -n "fortran,gcc "
+ gfortran -v 2>&1 | grep "gcc version" | cut -f3 -d" "
+fi
-echo -n "javascript,V8 "
-node8 -e "console.log(process.versions.v8)"
+if [[ $LANGUAGES == *":go:"* ]]; then
+ echo -n go,
+ go version | cut -f3 -d" "
+fi
-echo -n "julia,"
-$JULIAHOME/usr/bin/julia -v | cut -f3 -d" "
+if [[ $LANGUAGES == *":java:"* ]]; then
+ echo -n java,
+ java -version 2>&1 |grep "version" | cut -f3 -d " " | cut -c 2-9
+fi
-echo -n "lua,"
-# scilua -v 2>&1 | grep Shell | cut -f3 -d" " | cut -f1 -d,
-echo scilua v1.0.0-b12
+if [[ $LANGUAGES == *":javascript:"* ]]; then
+ echo -n "javascript,V8 "
+ node -e "console.log(process.versions.v8)"
+fi
-echo -n "mathematica,"
-echo quit | math -version | head -n 1 | cut -f2 -d" "
+if [[ $LANGUAGES == *":julia:"* ]]; then
+ echo -n "julia,"
+ $JULIAHOME/usr/bin/julia -v | cut -f3 -d" "
+fi
-echo -n "matlab,R"
-matlab -nodisplay -nojvm -nosplash -r "version -release, quit" | tail -n3 | head -n1 | cut -f5 -d" " | sed "s/'//g"
+if [[ $LANGUAGES == *":lua:"* ]]; then
+ echo -n "lua,"
+ # scilua -v 2>&1 | grep Shell | cut -f3 -d" " | cut -f1 -d,
+ echo scilua v1.0.0-b12
+fi
-echo -n "octave,"
-octave-cli -v | grep version | cut -f4 -d" "
+if [[ $LANGUAGES == *":mathematica:"* ]]; then
+ echo -n "mathematica,"
+ echo quit | math -version | head -n 1 | cut -f2 -d" "
+fi
-echo -n "python,"
-python3 -V 2>&1 | cut -f2 -d" "
+if [[ $LANGUAGES == *":matlab:"* ]]; then
+ echo -n "matlab,R"
+ matlab -nodisplay -nojvm -nosplash -r "version -release, quit" | tail -n3 | head -n1 | cut -f5 -d" " | sed "s/'//g"
+fi
-echo -n "r,"
-R --version | grep "R version" | cut -f3 -d" "
+if [[ $LANGUAGES == *":octave:"* ]]; then
+ echo -n "octave,"
+ octave-cli -v | grep version | cut -f4 -d" "
+fi
-echo -n "rust,"
-(cd rust; rustc --version | cut -c 7- | sed 's/ ([0-9a-f]* /
(/g')
+if [[ $LANGUAGES == *":python:"* ]]; then
+ echo -n "python,"
+ python3 -V 2>&1 | cut -f2 -d" "
+fi
+
+if [[ $LANGUAGES == *":r:"* ]]; then
+ echo -n "r,"
+ R --version | grep "R version" | cut -f3 -d" "
+fi
+
+if [[ $LANGUAGES == *":rust:"* ]]; then
+ echo -n "rust,"
+ (cd rust; rustc --version | cut -c 7- | sed 's/ ([0-9a-f]* /
(/g')
+fi