diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000..1fdd088 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,106 @@ +name: Benchmarks + +on: + pull_request: + push: + branches: + - master + tags: '*' + workflow_dispatch: + +concurrency: + # Skip intermediate builds: all builds except for builds on the `master` or `release-*` branches + # Cancel intermediate builds: only pull request builds + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref != 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-') || github.run_number }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +permissions: + contents: read + +jobs: + test: + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + strategy: + matrix: + os: [ubuntu-latest] + java-version: ['17'] + julia-version: ['1.7.1'] + python-version: ['3.9'] + numpy-version: ['1.22'] + gfortran-version: ['9'] # Note: unused since is built-in. + rust-version: ['1.42.0'] # Note: unused since controlled by `rust/rust-toolchain` + js-version: ['16'] + r-version: ['4.1.2'] + lua-version: ['2.0.5'] # Note: Not used as benchmark is broken. + go-version: ['1.17.4'] # Note: Not used as benchmark is broken. + + steps: + - uses: actions/checkout@v3 + with: + persist-credentials: false + - name: "Cache Julia" + id: cache-julia + uses: actions/cache@v2 + with: + path: ~/julia + key: ${{ runner.os }}-v${{ matrix.julia-version }} + - name: "Build Julia" + if: steps.cache-julia.outputs.cache-hit != 'true' + uses: julia-actions/build-julia@v1 + with: + ref: v${{ matrix.julia-version }} + - name: "Install Julia packages" + run: | + julia -e 'using Pkg; Pkg.add("Compat")' + - name: "Set up dSFMT" + run: | + cd ~/ + mkdir -p dSFMT + cd dSFMT + wget https://github.com/MersenneTwister-Lab/dSFMT/archive/refs/tags/v2.2.4.tar.gz + echo "39682961ecfba621a98dbb6610b6ae2b7d6add450d4f08d8d4edd0e10abd8174 v2.2.4.tar.gz" | sha256sum --check --status + tar -xzf v2.2.4.tar.gz + mv dSFMT-*/* ./ + - name: "Set up OpenBLAS" + run: | + sudo apt-get install -y libopenblas-dev + - name: "Set up Python" + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: "Set up NumPy" + run: pip install numpy==${{ matrix.numpy-version }} + - name: "Set up Rust" + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust-version }} + - name: "Set up Java" + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: ${{ matrix.java-version }} + cache: 'maven' + - name: "Set up JavaScript" + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.js-version }} + - name: "Set up R" + uses: r-lib/actions/setup-r@v2 + with: + r-version: ${{ matrix.r-version }} + - name: "Set up LuaJit" + uses: leafo/gh-actions-lua@v8.0.0 + with: + luaVersion: luajit-${{ matrix.lua-version }} + - name: "Set up Go" + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + - name: "Run benchmark" + run: | + JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.html + - name: "Print benchmark data" + run: cat gh_action_benchmarks.csv diff --git a/Makefile b/Makefile index 319fe5f..6f8988d 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,21 @@ ifndef JULIAHOME $(error JULIAHOME not defined. Set value to the root of the Julia source tree.) endif +ifndef DSFMTDIR +$(error DSFMTDIR not defined. Set value to the root of the dSFMT source tree.) +endif + + +# Will make multi-line targets work +# (so we can use @for on the second line) +.ONESHELL: + include $(JULIAHOME)/Make.inc include $(JULIAHOME)/deps/Versions.make -NODEJSBIN = node8 +NODEJSBIN = node + +ITERATIONS=$(shell seq 1 5) #Use python2 for Python 2.x PYTHON = python3 @@ -19,13 +30,6 @@ else MATHEMATICABIN = math endif -#Which BLAS library am I using? -ifeq ($(USE_SYSTEM_BLAS), 0) -BLASMANIFEST=$(shell cat $(JULIAHOME)/usr/manifest/openblas) -BLASDIR=$(JULIAHOME)/deps/scratch/$(BLASMANIFEST)/ -LIBBLAS=$(BLASDIR)$(LIBBLASNAME).a -endif - FFLAGS=-fexternal-blas #gfortran cannot multiply matrices using 64-bit external BLAS. ifeq ($(findstring gfortran, $(FC)), gfortran) @@ -43,9 +47,6 @@ LIBM = $(LIBMDIR)libopenlibm.a endif endif -DSFMTDIR = $(JULIAHOME)/deps/scratch/dsfmt-$(DSFMT_VER) -RMATHDIR = $(JULIAHOME)/deps/scratch/Rmath-julia-$(RMATH_JULIA_VER) - default: benchmarks.html export OMP_NUM_THREADS=1 @@ -53,15 +54,14 @@ export GOTO_NUM_THREADS=1 export OPENBLAS_NUM_THREADS=1 perf.h: $(JULIAHOME)/deps/Versions.make - echo '#include "$(BLASDIR)cblas.h"' > $@ + echo '#include "cblas.h"' > $@ echo '#include "$(DSFMTDIR)/dSFMT.c"' >> $@ bin/perf%: perf.c perf.h - $(CC) -std=c99 -O$* $< -o $@ -I$(DSFMTDIR) $(LIBBLAS) -L$(LIBMDIR) $(LIBM) $(CFLAGS) -lpthread + $(CC) -std=c99 -O$* $< -o $@ -I$(DSFMTDIR) -lopenblas -L$(LIBMDIR) $(LIBM) $(CFLAGS) -lpthread bin/fperf%: perf.f90 mkdir -p mods/$@ #Modules for each binary go in separate directories -# $(FC) $(FFLAGS) -Jmods/$@ -O$* $< -o $@ $(LIBBLAS) -L$(LIBMDIR) $(LIBM) -lpthread $(FC) $(FFLAGS) -Jmods/$@ -O$* $< -o $@ -lopenblas -L$(LIBMDIR) $(LIBM) -lpthread benchmarks/c.csv: \ @@ -80,73 +80,90 @@ benchmarks/fortran.csv: \ benchmarks/c%.csv: bin/perf% - for t in 1 2 3 4 5; do $<; done >$@ + @for t in $(ITERATIONS); do $<; done >$@ benchmarks/fortran%.csv: bin/fperf% - for t in 1 2 3 4 5; do $<; done >$@ + @for t in $(ITERATIONS); do $<; done >$@ benchmarks/go.csv: export GOPATH=$(abspath gopath) benchmarks/go.csv: perf.go - #CGO_LDFLAGS="$(LIBBLAS) $(LIBM)" go get github.com/gonum/blas/cgo + #CGO_LDFLAGS="-lopenblas $(LIBM)" go get github.com/gonum/blas/cgo go get github.com/gonum/blas/blas64 go get github.com/gonum/blas/cgo go get github.com/gonum/matrix/mat64 go get github.com/gonum/stat - for t in 1 2 3 4 5; do go run $<; done >$@ + @for t in $(ITERATIONS); do go run $<; done >$@ benchmarks/julia.csv: perf.jl - for t in 1 2 3 4 5; do $(JULIAHOME)/usr/bin/julia $<; done >$@ + @for t in $(ITERATIONS); do $(JULIAHOME)/usr/bin/julia $<; done >$@ benchmarks/python.csv: perf.py - for t in 1 2 3 4 5; do $(PYTHON) $<; done >$@ + @for t in $(ITERATIONS); do $(PYTHON) $<; done >$@ benchmarks/matlab.csv: perf.m - for t in 1 2 3 4 5; do matlab -nojvm -singleCompThread -r 'perf; perf; exit' | grep ^matlab | tail -8; done >$@ + @for t in $(ITERATIONS); do matlab -nojvm -singleCompThread -r 'perf; perf; exit' | grep ^matlab | tail -8; done >$@ benchmarks/octave.csv: perf.m - for t in 1 2 3 4 5; do $(OCTAVE) -q --eval perf 2>/dev/null; done >$@ + @for t in $(ITERATIONS); do $(OCTAVE) -q --eval perf 2>/dev/null; done >$@ benchmarks/r.csv: perf.R - for t in 1 2 3 4 5; do cat $< | R --vanilla --slave 2>/dev/null; done >$@ + @for t in $(ITERATIONS); do cat $< | R --vanilla --slave 2>/dev/null; done >$@ benchmarks/javascript.csv: perf.js - for t in 1 2 3 4 5; do $(NODEJSBIN) $<; done >$@ + @for t in $(ITERATIONS); do $(NODEJSBIN) $<; done >$@ benchmarks/mathematica.csv: perf.nb - for t in 1 2 3 4 5; do $(MATHEMATICABIN) -noprompt -run "<<$<; Exit[]"; done >$@ + @for t in $(ITERATIONS); do $(MATHEMATICABIN) -noprompt -run "<<$<; Exit[]"; done >$@ benchmarks/stata.csv: perf.do - for t in 1 2 3 4 5; do stata -b do $^ $@; done + @for t in $(ITERATIONS); do stata -b do $^ $@; done benchmarks/lua.csv: perf.lua - for t in 1 2 3 4 5; do scilua $<; done >$@ + @for t in $(ITERATIONS); do luajit $<; done >$@ benchmarks/java.csv: java/src/main/java/PerfBLAS.java - cd java; sh setup.sh; for t in 1 2 3 4 5; do mvn -q exec:java; done >../$@ + cd java + sh setup.sh + @for t in $(ITERATIONS); do mvn -q exec:java; done >../$@ benchmarks/scala.csv: scala/src/main/scala/perf.scala scala/build.sbt - cd scala; for t in 1 2 3 4 5; do sbt run; done >../$@ + cd scala + @for t in $(ITERATIONS); do sbt run; done >../$@ benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock - cd rust; for t in 1 2 3 4 5; do cargo run --release -q; done >../$@ + cd rust + @for t in $(ITERATIONS); do cargo run --release -q; done >../$@ LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust +GH_ACTION_LANGUAGES = c fortran java javascript julia python r rust # These were formerly listed in LANGUAGES, but I can't get them to run # 2017-09-27 johnfgibson # scala, stata BENCHMARKS = $(foreach lang,$(LANGUAGES),benchmarks/$(lang).csv) +GH_ACTION_BENCHMARKS = $(foreach lang,$(GH_ACTION_LANGUAGES),benchmarks/$(lang).csv) + +COLON_SEPARATED_GHA_LANGUAGES = $(shell echo $(GH_ACTION_LANGUAGES) | sed 's/ /:/g') versions.csv: bin/versions.sh $^ >$@ +gh_action_versions.csv: bin/versions.sh + $^ $(COLON_SEPARATED_GHA_LANGUAGES) >$@ + benchmarks.csv: bin/collect.jl $(BENCHMARKS) @$(call PRINT_JULIA, $^ >$@) +gh_action_benchmarks.csv: bin/collect.jl $(GH_ACTION_BENCHMARKS) + @$(call PRINT_JULIA, $^ >$@) + benchmarks.html: bin/table.jl versions.csv benchmarks.csv @$(call PRINT_JULIA, $^ >$@) +gh_action_benchmarks.html: bin/table.jl gh_action_versions.csv gh_action_benchmarks.csv + @$(call PRINT_JULIA, $^ >$@) + clean: @rm -rf perf.h bin/perf* bin/fperf* benchmarks/*.csv benchmarks.csv mods *~ octave-core perf.log gopath/* diff --git a/bin/versions.sh b/bin/versions.sh index 542c3a4..5964303 100755 --- a/bin/versions.sh +++ b/bin/versions.sh @@ -1,41 +1,74 @@ #!/usr/bin/env bash -echo -n "c,gcc " -gcc -v 2>&1 | grep "gcc version" | cut -f3 -d" " +# User argument declaring what languages to query: +DEFAULT_LANGUAGES="c:fortran:go:java:javascript:julia:lua:mathematica:matlab:octave:python:r:rust" +LANGUAGES=${1:-DEFAULT_LANGUAGES} -echo -n "fortran,gcc " -gfortran -v 2>&1 | grep "gcc version" | cut -f3 -d" " +LANGUAGES=":${LANGUAGES}:" -echo -n go, -go version | cut -f3 -d" " +# Check if ":c:" in languages: +if [[ $LANGUAGES == *":c:"* ]]; then + echo -n "c,gcc " + gcc -v 2>&1 | grep "gcc version" | cut -f3 -d" " +fi -echo -n java, -java -version 2>&1 |grep "version" | cut -f3 -d " " | cut -c 2-9 +if [[ $LANGUAGES == *":fortran:"* ]]; then + echo -n "fortran,gcc " + gfortran -v 2>&1 | grep "gcc version" | cut -f3 -d" " +fi -echo -n "javascript,V8 " -node8 -e "console.log(process.versions.v8)" +if [[ $LANGUAGES == *":go:"* ]]; then + echo -n go, + go version | cut -f3 -d" " +fi -echo -n "julia," -$JULIAHOME/usr/bin/julia -v | cut -f3 -d" " +if [[ $LANGUAGES == *":java:"* ]]; then + echo -n java, + java -version 2>&1 |grep "version" | cut -f3 -d " " | cut -c 2-9 +fi -echo -n "lua," -# scilua -v 2>&1 | grep Shell | cut -f3 -d" " | cut -f1 -d, -echo scilua v1.0.0-b12 +if [[ $LANGUAGES == *":javascript:"* ]]; then + echo -n "javascript,V8 " + node -e "console.log(process.versions.v8)" +fi -echo -n "mathematica," -echo quit | math -version | head -n 1 | cut -f2 -d" " +if [[ $LANGUAGES == *":julia:"* ]]; then + echo -n "julia," + $JULIAHOME/usr/bin/julia -v | cut -f3 -d" " +fi -echo -n "matlab,R" -matlab -nodisplay -nojvm -nosplash -r "version -release, quit" | tail -n3 | head -n1 | cut -f5 -d" " | sed "s/'//g" +if [[ $LANGUAGES == *":lua:"* ]]; then + echo -n "lua," + # scilua -v 2>&1 | grep Shell | cut -f3 -d" " | cut -f1 -d, + echo scilua v1.0.0-b12 +fi -echo -n "octave," -octave-cli -v | grep version | cut -f4 -d" " +if [[ $LANGUAGES == *":mathematica:"* ]]; then + echo -n "mathematica," + echo quit | math -version | head -n 1 | cut -f2 -d" " +fi -echo -n "python," -python3 -V 2>&1 | cut -f2 -d" " +if [[ $LANGUAGES == *":matlab:"* ]]; then + echo -n "matlab,R" + matlab -nodisplay -nojvm -nosplash -r "version -release, quit" | tail -n3 | head -n1 | cut -f5 -d" " | sed "s/'//g" +fi -echo -n "r," -R --version | grep "R version" | cut -f3 -d" " +if [[ $LANGUAGES == *":octave:"* ]]; then + echo -n "octave," + octave-cli -v | grep version | cut -f4 -d" " +fi -echo -n "rust," -(cd rust; rustc --version | cut -c 7- | sed 's/ ([0-9a-f]* /
(/g') +if [[ $LANGUAGES == *":python:"* ]]; then + echo -n "python," + python3 -V 2>&1 | cut -f2 -d" " +fi + +if [[ $LANGUAGES == *":r:"* ]]; then + echo -n "r," + R --version | grep "R version" | cut -f3 -d" " +fi + +if [[ $LANGUAGES == *":rust:"* ]]; then + echo -n "rust," + (cd rust; rustc --version | cut -c 7- | sed 's/ ([0-9a-f]* /
(/g') +fi