Create GitHub workflow to run benchmark suite (#51)

Co-authored-by: Dilum Aluthge <[email protected]>

Co-authored-by: Mosè Giordano <[email protected]>

Author: MilesCranmer

.github/workflows/benchmarks.yml

@@ -0,0 +1,106 @@
+name: Benchmarks
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+    tags: '*'
+  workflow_dispatch:
+
+concurrency:
+  # Skip intermediate builds: all builds except for builds on the `master` or `release-*` branches
+  # Cancel intermediate builds: only pull request builds
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref != 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-') || github.run_number }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        java-version: ['17']
+        julia-version: ['1.7.1']
+        python-version: ['3.9']
+        numpy-version: ['1.22']
+        gfortran-version: ['9']  # Note: unused since is built-in.
+        rust-version: ['1.42.0']  # Note: unused since controlled by `rust/rust-toolchain`
+        js-version: ['16']
+        r-version: ['4.1.2']
+        lua-version: ['2.0.5']  # Note: Not used as benchmark is broken.
+        go-version: ['1.17.4']  # Note: Not used as benchmark is broken.
+    
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+      - name: "Cache Julia"
+        id: cache-julia
+        uses: actions/cache@v2
+        with:
+          path: ~/julia
+          key: ${{ runner.os }}-v${{ matrix.julia-version }}
+      - name: "Build Julia"
+        if: steps.cache-julia.outputs.cache-hit != 'true'
+        uses: julia-actions/build-julia@v1
+        with:
+          ref: v${{ matrix.julia-version }}
+      - name: "Install Julia packages"
+        run: |
+          julia -e 'using Pkg; Pkg.add("Compat")'
+      - name: "Set up dSFMT"
+        run: |
+          cd ~/
+          mkdir -p dSFMT
+          cd dSFMT
+          wget https://github.com/MersenneTwister-Lab/dSFMT/archive/refs/tags/v2.2.4.tar.gz
+          echo "39682961ecfba621a98dbb6610b6ae2b7d6add450d4f08d8d4edd0e10abd8174 v2.2.4.tar.gz" | sha256sum --check --status
+          tar -xzf v2.2.4.tar.gz
+          mv dSFMT-*/* ./
+      - name: "Set up OpenBLAS"
+        run: |
+          sudo apt-get install -y libopenblas-dev
+      - name: "Set up Python"
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: "Set up NumPy"
+        run: pip install numpy==${{ matrix.numpy-version }}
+      - name: "Set up Rust"
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ matrix.rust-version }}
+      - name: "Set up Java"
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java-version }}
+          cache: 'maven'
+      - name: "Set up JavaScript"
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.js-version }}
+      - name: "Set up R"
+        uses: r-lib/actions/setup-r@v2
+        with:
+          r-version: ${{ matrix.r-version }}
+      - name: "Set up LuaJit"
+        uses: leafo/[email protected]
+        with:
+          luaVersion: luajit-${{ matrix.lua-version }}
+      - name: "Set up Go"
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: "Run benchmark"
+        run: |
+          JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.html
+      - name: "Print benchmark data"
+        run: cat gh_action_benchmarks.csv

Makefile

@@ -1,10 +1,21 @@
 ifndef JULIAHOME
 $(error JULIAHOME not defined. Set value to the root of the Julia source tree.)
 endif
+ifndef DSFMTDIR
+$(error DSFMTDIR not defined. Set value to the root of the dSFMT source tree.)
+endif
+
+
+# Will make multi-line targets work
+# (so we can use @for on the second line)
+.ONESHELL:
+
 include $(JULIAHOME)/Make.inc
 include $(JULIAHOME)/deps/Versions.make
 
-NODEJSBIN = node8
+NODEJSBIN = node
+
+ITERATIONS=$(shell seq 1 5)
 
 #Use python2 for Python 2.x
 PYTHON = python3
@@ -19,13 +30,6 @@ else
 MATHEMATICABIN = math
 endif
 
-#Which BLAS library am I using?
-ifeq ($(USE_SYSTEM_BLAS), 0)
-BLASMANIFEST=$(shell cat $(JULIAHOME)/usr/manifest/openblas)
-BLASDIR=$(JULIAHOME)/deps/scratch/$(BLASMANIFEST)/
-LIBBLAS=$(BLASDIR)$(LIBBLASNAME).a
-endif
-
 FFLAGS=-fexternal-blas
 #gfortran cannot multiply matrices using 64-bit external BLAS.
 ifeq ($(findstring gfortran, $(FC)), gfortran)
@@ -43,25 +47,21 @@ LIBM = $(LIBMDIR)libopenlibm.a
 endif
 endif
 
-DSFMTDIR = $(JULIAHOME)/deps/scratch/dsfmt-$(DSFMT_VER)
-RMATHDIR = $(JULIAHOME)/deps/scratch/Rmath-julia-$(RMATH_JULIA_VER)
-
 default: benchmarks.html
 
 export OMP_NUM_THREADS=1
 export GOTO_NUM_THREADS=1
 export OPENBLAS_NUM_THREADS=1
 
 perf.h: $(JULIAHOME)/deps/Versions.make
-	echo '#include "$(BLASDIR)cblas.h"' > $@
+	echo '#include "cblas.h"' > $@
 	echo '#include "$(DSFMTDIR)/dSFMT.c"' >> $@
 
 bin/perf%: perf.c perf.h
-	$(CC) -std=c99 -O$* $< -o $@  -I$(DSFMTDIR) $(LIBBLAS) -L$(LIBMDIR) $(LIBM) $(CFLAGS) -lpthread
+	$(CC) -std=c99 -O$* $< -o $@  -I$(DSFMTDIR) -lopenblas -L$(LIBMDIR) $(LIBM) $(CFLAGS) -lpthread
 
 bin/fperf%: perf.f90
 	mkdir -p mods/$@ #Modules for each binary go in separate directories
-#	$(FC) $(FFLAGS) -Jmods/$@ -O$* $< -o $@ $(LIBBLAS) -L$(LIBMDIR) $(LIBM) -lpthread
 	$(FC) $(FFLAGS) -Jmods/$@ -O$* $< -o $@ -lopenblas -L$(LIBMDIR) $(LIBM) -lpthread
 
 benchmarks/c.csv: \
@@ -80,73 +80,90 @@ benchmarks/fortran.csv: \
 
 
 benchmarks/c%.csv: bin/perf%
-	for t in 1 2 3 4 5; do $<; done >$@
+	@for t in $(ITERATIONS); do $<; done >$@
 
 benchmarks/fortran%.csv: bin/fperf%
-	for t in 1 2 3 4 5; do $<; done >$@
+	@for t in $(ITERATIONS); do $<; done >$@
 
 benchmarks/go.csv: export GOPATH=$(abspath gopath)
 benchmarks/go.csv: perf.go
-	#CGO_LDFLAGS="$(LIBBLAS) $(LIBM)" go get github.com/gonum/blas/cgo
+	#CGO_LDFLAGS="-lopenblas $(LIBM)" go get github.com/gonum/blas/cgo
 	go get github.com/gonum/blas/blas64
 	go get github.com/gonum/blas/cgo
 	go get github.com/gonum/matrix/mat64
 	go get github.com/gonum/stat
-	for t in 1 2 3 4 5; do go run $<; done >$@
+	@for t in $(ITERATIONS); do go run $<; done >$@
 
 benchmarks/julia.csv: perf.jl
-	for t in 1 2 3 4 5; do $(JULIAHOME)/usr/bin/julia $<; done >$@
+	@for t in $(ITERATIONS); do $(JULIAHOME)/usr/bin/julia $<; done >$@
 
 benchmarks/python.csv: perf.py
-	for t in 1 2 3 4 5; do $(PYTHON) $<; done >$@
+	@for t in $(ITERATIONS); do $(PYTHON) $<; done >$@
 
 benchmarks/matlab.csv: perf.m
-	for t in 1 2 3 4 5; do matlab -nojvm -singleCompThread -r 'perf; perf; exit' | grep ^matlab | tail -8; done >$@
+	@for t in $(ITERATIONS); do matlab -nojvm -singleCompThread -r 'perf; perf; exit' | grep ^matlab | tail -8; done >$@
 
 benchmarks/octave.csv: perf.m
-	for t in 1 2 3 4 5; do $(OCTAVE) -q --eval perf 2>/dev/null; done >$@
+	@for t in $(ITERATIONS); do $(OCTAVE) -q --eval perf 2>/dev/null; done >$@
 
 benchmarks/r.csv: perf.R
-	for t in 1 2 3 4 5; do cat $< | R --vanilla --slave 2>/dev/null; done >$@
+	@for t in $(ITERATIONS); do cat $< | R --vanilla --slave 2>/dev/null; done >$@
 
 benchmarks/javascript.csv: perf.js
-	for t in 1 2 3 4 5; do $(NODEJSBIN) $<; done >$@
+	@for t in $(ITERATIONS); do $(NODEJSBIN) $<; done >$@
 
 benchmarks/mathematica.csv: perf.nb
-	for t in 1 2 3 4 5; do $(MATHEMATICABIN) -noprompt -run "<<$<; Exit[]"; done >$@
+	@for t in $(ITERATIONS); do $(MATHEMATICABIN) -noprompt -run "<<$<; Exit[]"; done >$@
 
 benchmarks/stata.csv: perf.do
-	for t in 1 2 3 4 5; do stata -b do $^ $@; done
+	@for t in $(ITERATIONS); do stata -b do $^ $@; done
 
 benchmarks/lua.csv: perf.lua
-	for t in 1 2 3 4 5; do scilua $<; done >$@
+	@for t in $(ITERATIONS); do luajit $<; done >$@
 
 benchmarks/java.csv: java/src/main/java/PerfBLAS.java
-	cd java; sh setup.sh; for t in 1 2 3 4 5; do mvn -q exec:java; done >../$@
+	cd java
+	sh setup.sh
+	@for t in $(ITERATIONS); do mvn -q exec:java; done >../$@
 
 benchmarks/scala.csv: scala/src/main/scala/perf.scala scala/build.sbt
-	cd scala; for t in 1 2 3 4 5; do sbt run; done >../$@
+	cd scala
+	@for t in $(ITERATIONS); do sbt run; done >../$@
 
 benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock
-	cd rust; for t in 1 2 3 4 5; do cargo run --release -q; done >../$@
+	cd rust
+	@for t in $(ITERATIONS); do cargo run --release -q; done >../$@
 
 LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust
+GH_ACTION_LANGUAGES = c fortran java javascript julia python r rust
 
 # These were formerly listed in LANGUAGES, but I can't get them to run
 # 2017-09-27 johnfgibson
 #	scala, stata
 
 BENCHMARKS = $(foreach lang,$(LANGUAGES),benchmarks/$(lang).csv)
+GH_ACTION_BENCHMARKS = $(foreach lang,$(GH_ACTION_LANGUAGES),benchmarks/$(lang).csv)
+
+COLON_SEPARATED_GHA_LANGUAGES = $(shell echo $(GH_ACTION_LANGUAGES) | sed 's/ /:/g')
 
 versions.csv: bin/versions.sh
 	$^ >$@
 
+gh_action_versions.csv: bin/versions.sh
+	$^ $(COLON_SEPARATED_GHA_LANGUAGES) >$@
+
 benchmarks.csv: bin/collect.jl $(BENCHMARKS)
 	@$(call PRINT_JULIA, $^ >$@)
 
+gh_action_benchmarks.csv: bin/collect.jl $(GH_ACTION_BENCHMARKS)
+	@$(call PRINT_JULIA, $^ >$@)
+
 benchmarks.html: bin/table.jl versions.csv benchmarks.csv
 	@$(call PRINT_JULIA, $^ >$@)
 
+gh_action_benchmarks.html: bin/table.jl gh_action_versions.csv gh_action_benchmarks.csv
+	@$(call PRINT_JULIA, $^ >$@)
+
 clean:
 	@rm -rf perf.h bin/perf* bin/fperf* benchmarks/*.csv benchmarks.csv mods *~ octave-core perf.log gopath/*
 

bin/versions.sh

@@ -1,41 +1,74 @@
 #!/usr/bin/env bash
 
-echo -n "c,gcc "
-gcc -v 2>&1 | grep "gcc version" | cut -f3 -d" "
+# User argument declaring what languages to query:
+DEFAULT_LANGUAGES="c:fortran:go:java:javascript:julia:lua:mathematica:matlab:octave:python:r:rust"
+LANGUAGES=${1:-DEFAULT_LANGUAGES}
 
-echo -n "fortran,gcc "
-gfortran -v 2>&1 | grep "gcc version" | cut -f3 -d" "
+LANGUAGES=":${LANGUAGES}:"
 
-echo -n go,
-go version | cut -f3 -d" "
+# Check if ":c:" in languages:
+if [[ $LANGUAGES == *":c:"* ]]; then
+    echo -n "c,gcc "
+    gcc -v 2>&1 | grep "gcc version" | cut -f3 -d" "
+fi
 
-echo -n java,
-java -version 2>&1 |grep "version" | cut -f3 -d " " | cut -c 2-9
+if [[ $LANGUAGES == *":fortran:"* ]]; then
+    echo -n "fortran,gcc "
+    gfortran -v 2>&1 | grep "gcc version" | cut -f3 -d" "
+fi
 
-echo -n "javascript,V8 "
-node8 -e "console.log(process.versions.v8)"
+if [[ $LANGUAGES == *":go:"* ]]; then
+    echo -n go,
+    go version | cut -f3 -d" "
+fi
 
-echo -n "julia,"
-$JULIAHOME/usr/bin/julia -v | cut -f3 -d" "
+if [[ $LANGUAGES == *":java:"* ]]; then
+    echo -n java,
+    java -version 2>&1 |grep "version" | cut -f3 -d " " | cut -c 2-9
+fi
 
-echo -n "lua,"
-# scilua -v 2>&1 | grep Shell | cut -f3 -d" " | cut -f1 -d,
-echo scilua v1.0.0-b12
+if [[ $LANGUAGES == *":javascript:"* ]]; then
+    echo -n "javascript,V8 "
+    node -e "console.log(process.versions.v8)"
+fi
 
-echo -n "mathematica,"
-echo quit | math -version | head -n 1 | cut -f2 -d" "
+if [[ $LANGUAGES == *":julia:"* ]]; then
+    echo -n "julia,"
+    $JULIAHOME/usr/bin/julia -v | cut -f3 -d" "
+fi
 
-echo -n "matlab,R"
-matlab -nodisplay -nojvm -nosplash -r "version -release, quit" | tail -n3 | head -n1 | cut -f5 -d" " | sed "s/'//g"
+if [[ $LANGUAGES == *":lua:"* ]]; then
+    echo -n "lua,"
+    # scilua -v 2>&1 | grep Shell | cut -f3 -d" " | cut -f1 -d,
+    echo scilua v1.0.0-b12
+fi
 
-echo -n "octave,"
-octave-cli -v | grep version | cut -f4 -d" "
+if [[ $LANGUAGES == *":mathematica:"* ]]; then
+    echo -n "mathematica,"
+    echo quit | math -version | head -n 1 | cut -f2 -d" "
+fi
 
-echo -n "python,"
-python3 -V 2>&1 | cut -f2 -d" "
+if [[ $LANGUAGES == *":matlab:"* ]]; then
+    echo -n "matlab,R"
+    matlab -nodisplay -nojvm -nosplash -r "version -release, quit" | tail -n3 | head -n1 | cut -f5 -d" " | sed "s/'//g"
+fi
 
-echo -n "r,"
-R --version | grep "R version" | cut -f3 -d" "
+if [[ $LANGUAGES == *":octave:"* ]]; then
+    echo -n "octave,"
+    octave-cli -v | grep version | cut -f4 -d" "
+fi
 
-echo -n "rust,"
-(cd rust; rustc --version | cut -c 7- | sed 's/ ([0-9a-f]* /<br>(/g')
+if [[ $LANGUAGES == *":python:"* ]]; then
+    echo -n "python,"
+    python3 -V 2>&1 | cut -f2 -d" "
+fi
+
+if [[ $LANGUAGES == *":r:"* ]]; then
+    echo -n "r,"
+    R --version | grep "R version" | cut -f3 -d" "
+fi
+
+if [[ $LANGUAGES == *":rust:"* ]]; then
+    echo -n "rust,"
+    (cd rust; rustc --version | cut -c 7- | sed 's/ ([0-9a-f]* /<br>(/g')
+fi