From 7999801a81fe07df6e4063cc3cbd5ff48431cb8d Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 13:58:26 -0500
Subject: [PATCH 01/46] Allow querying arbitrary languages in versions.sh

---
 bin/versions.sh | 85 +++++++++++++++++++++++++++++++++----------------
 1 file changed, 58 insertions(+), 27 deletions(-)

diff --git a/bin/versions.sh b/bin/versions.sh
index 542c3a4..12016a9 100755
--- a/bin/versions.sh
+++ b/bin/versions.sh
@@ -1,41 +1,72 @@
 #!/usr/bin/env bash
 
-echo -n "c,gcc "
-gcc -v 2>&1 | grep "gcc version" | cut -f3 -d" "
+# User argument declaring what languages to query:
+DEFAULT_LANGUAGES=",c,fortran,go,java,javascript,julia,lua,mathematica,matlab,octave,python,r,rust,"
+LANGUAGES=${1:-DEFAULT_LANGUAGES}
 
-echo -n "fortran,gcc "
-gfortran -v 2>&1 | grep "gcc version" | cut -f3 -d" "
+# Check if ",c," in languages:
+if [[ $LANGUAGES == *",c,"* ]]; then
+    echo -n "c,gcc "
+    gcc -v 2>&1 | grep "gcc version" | cut -f3 -d" "
+fi
 
-echo -n go,
-go version | cut -f3 -d" "
+if [[ $LANGUAGES == *",fortran,"* ]]; then
+    echo -n "fortran,gcc "
+    gfortran -v 2>&1 | grep "gcc version" | cut -f3 -d" "
+fi
 
-echo -n java,
-java -version 2>&1 |grep "version" | cut -f3 -d " " | cut -c 2-9
+if [[ $LANGUAGES == *",go,"* ]]; then
+    echo -n go,
+    go version | cut -f3 -d" "
+fi
 
-echo -n "javascript,V8 "
-node8 -e "console.log(process.versions.v8)"
+if [[ $LANGUAGES == *",java,"* ]]; then
+    echo -n java,
+    java -version 2>&1 |grep "version" | cut -f3 -d " " | cut -c 2-9
+fi
 
-echo -n "julia,"
-$JULIAHOME/usr/bin/julia -v | cut -f3 -d" "
+if [[ $LANGUAGES == *",javascript,"* ]]; then
+    echo -n "javascript,V8 "
+    node8 -e "console.log(process.versions.v8)"
+fi
 
-echo -n "lua,"
-# scilua -v 2>&1 | grep Shell | cut -f3 -d" " | cut -f1 -d,
-echo scilua v1.0.0-b12
+if [[ $LANGUAGES == *",julia,"* ]]; then
+    echo -n "julia,"
+    $JULIAHOME/usr/bin/julia -v | cut -f3 -d" "
+fi
 
-echo -n "mathematica,"
-echo quit | math -version | head -n 1 | cut -f2 -d" "
+if [[ $LANGUAGES == *",lua,"* ]]; then
+    echo -n "lua,"
+    # scilua -v 2>&1 | grep Shell | cut -f3 -d" " | cut -f1 -d,
+    echo scilua v1.0.0-b12
+fi
 
-echo -n "matlab,R"
-matlab -nodisplay -nojvm -nosplash -r "version -release, quit" | tail -n3 | head -n1 | cut -f5 -d" " | sed "s/'//g"
+if [[ $LANGUAGES == *",mathematica,"* ]]; then
+    echo -n "mathematica,"
+    echo quit | math -version | head -n 1 | cut -f2 -d" "
+fi
 
-echo -n "octave,"
-octave-cli -v | grep version | cut -f4 -d" "
+if [[ $LANGUAGES == *",matlab,"* ]]; then
+    echo -n "matlab,R"
+    matlab -nodisplay -nojvm -nosplash -r "version -release, quit" | tail -n3 | head -n1 | cut -f5 -d" " | sed "s/'//g"
+fi
 
-echo -n "python,"
-python3 -V 2>&1 | cut -f2 -d" "
+if [[ $LANGUAGES == *",octave,"* ]]; then
+    echo -n "octave,"
+    octave-cli -v | grep version | cut -f4 -d" "
+fi
 
-echo -n "r,"
-R --version | grep "R version" | cut -f3 -d" "
+if [[ $LANGUAGES == *",python,"* ]]; then
+    echo -n "python,"
+    python3 -V 2>&1 | cut -f2 -d" "
+fi
 
-echo -n "rust,"
-(cd rust; rustc --version | cut -c 7- | sed 's/ ([0-9a-f]* /<br>(/g')
+if [[ $LANGUAGES == *",r,"* ]]; then
+    echo -n "r,"
+    R --version | grep "R version" | cut -f3 -d" "
+fi
+
+if [[ $LANGUAGES == *",rust,"* ]]; then
+    echo -n "rust,"
+    (cd rust; rustc --version | cut -c 7- | sed 's/ ([0-9a-f]* /<br>(/g')
+fi

From be74ccf370d635ae48b801d818577d9efebef67e Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 13:58:55 -0500
Subject: [PATCH 02/46] Modify Makefile to only run some languages for GH
 actions

---
 Makefile | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/Makefile b/Makefile
index 319fe5f..eed5d12 100644
--- a/Makefile
+++ b/Makefile
@@ -131,22 +131,33 @@ benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock
 	cd rust; for t in 1 2 3 4 5; do cargo run --release -q; done >../$@
 
 LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust
+GH_ACTION_LANGUAGES = c fortran julia python rust
 
 # These were formerly listed in LANGUAGES, but I can't get them to run
 # 2017-09-27 johnfgibson
 #	scala, stata
 
 BENCHMARKS = $(foreach lang,$(LANGUAGES),benchmarks/$(lang).csv)
+GH_ACTION_BENCHMARKS = $(foreach lang,$(GH_ACTION_LANGUAGES),benchmarks/$(lang).csv)
 
 versions.csv: bin/versions.sh
 	$^ >$@
 
+gh_action_versions.csv: bin/versions.sh
+	bin/versions.sh ",c,fortran,julia,python,rust," >$@
+
 benchmarks.csv: bin/collect.jl $(BENCHMARKS)
 	@$(call PRINT_JULIA, $^ >$@)
 
+gh_action_benchmarks.csv: bin/collect.jl $(GH_ACTION_BENCHMARKS)
+	@$(call PRINT_JULIA, $^ >$@)
+
 benchmarks.html: bin/table.jl versions.csv benchmarks.csv
 	@$(call PRINT_JULIA, $^ >$@)
 
+gh_action_benchmarks.html: bin/table.jl gh_action_versions.csv gh_action_benchmarks.csv
+	@$(call PRINT_JULIA, $^ >$@)
+
 clean:
 	@rm -rf perf.h bin/perf* bin/fperf* benchmarks/*.csv benchmarks.csv mods *~ octave-core perf.log gopath/*
 

From 4a234c2105d191bf634e79e502e8169637aed241 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 13:59:15 -0500
Subject: [PATCH 03/46] Create draft workflow for running benchmarks

---
 .github/workflows/benchmarks.yml | 55 ++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 .github/workflows/benchmarks.yml

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
new file mode 100644
index 0000000..6ee63aa
--- /dev/null
+++ b/.github/workflows/benchmarks.yml
@@ -0,0 +1,55 @@
+name: Benchmarks
+
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    branches:
+      - '*'
+  workflow_dispatch:
+    branches:
+      - '*'
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        julia-version: ['1.7.1']
+        python-version: ['3.9']
+        # TODO: Add numpy version
+        gfortran-version: ['9']
+        rust-version: ['1.42.0']
+    
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Update Ubuntu packages"
+        run: |
+          apt-get update
+          apt-get install -y aptitude
+          add-apt-repository ppa:ubuntu-toolchain-r/test
+          apt-get update
+      - name: "Set up Julia"
+        uses: julia-actions/setup-julia@v1.6.0
+        with:
+          version: ${{ matrix.julia-version }}
+      - name: "Install Julia source tree"
+        run: git clone https://github.com/JuliaLang/Julia $HOME/julia-src
+      - name: "Set up Python and NumPy"
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+          pip-packages: numpy
+      - name: "Set up fortran"
+        run: apt-get install -y gfortran-${{ matrix.gfortran-version }}
+      - name: "Set up Rust"
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ matrix.rust-version }}
+      - name: "Run benchmark"
+        run: JULIAHOME=$HOME/julia-src make gh_action_benchmarks.csv

From 046ad04d7d8cfc81625e3a3c01885d955d88df82 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 14:00:51 -0500
Subject: [PATCH 04/46] Add sudo

---
 .github/workflows/benchmarks.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 6ee63aa..6c2379b 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -30,10 +30,10 @@ jobs:
       - uses: actions/checkout@v2
       - name: "Update Ubuntu packages"
         run: |
-          apt-get update
-          apt-get install -y aptitude
-          add-apt-repository ppa:ubuntu-toolchain-r/test
-          apt-get update
+          sudo apt-get update
+          sudo apt-get install -y aptitude
+          sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+          sudo apt-get update
       - name: "Set up Julia"
         uses: julia-actions/setup-julia@v1.6.0
         with:

From 287df3e91aa95a99d91a2d11bbdbad42fd17c19a Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 14:02:37 -0500
Subject: [PATCH 05/46] Add missing sudo for fortran

---
 .github/workflows/benchmarks.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 6c2379b..60422a5 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -46,7 +46,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
           pip-packages: numpy
       - name: "Set up fortran"
-        run: apt-get install -y gfortran-${{ matrix.gfortran-version }}
+        run: sudo apt-get install -y gfortran-${{ matrix.gfortran-version }}
       - name: "Set up Rust"
         uses: actions-rs/toolchain@v1
         with:

From b762cf6a069ec0bad202d4e474d13cf9d7cd84c4 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 14:03:53 -0500
Subject: [PATCH 06/46] Fix numpy install

---
 .github/workflows/benchmarks.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 60422a5..07e9c2d 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -22,6 +22,7 @@ jobs:
         os: [ubuntu-latest]
         julia-version: ['1.7.1']
         python-version: ['3.9']
+        numpy-version: ['1.22']
         # TODO: Add numpy version
         gfortran-version: ['9']
         rust-version: ['1.42.0']
@@ -40,11 +41,12 @@ jobs:
           version: ${{ matrix.julia-version }}
       - name: "Install Julia source tree"
         run: git clone https://github.com/JuliaLang/Julia $HOME/julia-src
-      - name: "Set up Python and NumPy"
+      - name: "Set up Python"
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
-          pip-packages: numpy
+      - name: "Set up NumPy"
+        run: pip install numpy==${{ matrix.numpy-version }}
       - name: "Set up fortran"
         run: sudo apt-get install -y gfortran-${{ matrix.gfortran-version }}
       - name: "Set up Rust"

From 4fa9d4ada8b82a92c82f979c2f776bbb0bd78a4c Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 14:07:50 -0500
Subject: [PATCH 07/46] Attempt to fix JULIAHOME

---
 .github/workflows/benchmarks.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 07e9c2d..429e3b2 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -39,8 +39,6 @@ jobs:
         uses: julia-actions/setup-julia@v1.6.0
         with:
           version: ${{ matrix.julia-version }}
-      - name: "Install Julia source tree"
-        run: git clone https://github.com/JuliaLang/Julia $HOME/julia-src
       - name: "Set up Python"
         uses: actions/setup-python@v1
         with:
@@ -54,4 +52,6 @@ jobs:
         with:
           toolchain: ${{ matrix.rust-version }}
       - name: "Run benchmark"
-        run: JULIAHOME=$HOME/julia-src make gh_action_benchmarks.csv
+        run: |
+          find / -name 'openblas' 2>/dev/null
+          JULIAHOME=$HOME/usr make gh_action_benchmarks.csv

From 5fe74343f17fc6f5971be92fbe0903026057fa12 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 14:15:14 -0500
Subject: [PATCH 08/46] Fix JULIAHOME

---
 .github/workflows/benchmarks.yml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 429e3b2..f84ff19 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -23,7 +23,6 @@ jobs:
         julia-version: ['1.7.1']
         python-version: ['3.9']
         numpy-version: ['1.22']
-        # TODO: Add numpy version
         gfortran-version: ['9']
         rust-version: ['1.42.0']
     
@@ -33,7 +32,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y aptitude
-          sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+          sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
           sudo apt-get update
       - name: "Set up Julia"
         uses: julia-actions/setup-julia@v1.6.0
@@ -52,6 +51,4 @@ jobs:
         with:
           toolchain: ${{ matrix.rust-version }}
       - name: "Run benchmark"
-        run: |
-          find / -name 'openblas' 2>/dev/null
-          JULIAHOME=$HOME/usr make gh_action_benchmarks.csv
+        run: JULIAHOME=/ make gh_action_benchmarks.csv

From 6d13d883575e29bfb7adddb6ce818853fef0e9eb Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 14:20:38 -0500
Subject: [PATCH 09/46] Make action build julia from source

---
 .github/workflows/benchmarks.yml | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index f84ff19..20ff619 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -34,10 +34,17 @@ jobs:
           sudo apt-get install -y aptitude
           sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
           sudo apt-get update
-      - name: "Set up Julia"
-        uses: julia-actions/setup-julia@v1.6.0
+      - name: "Cache Julia"
+        id: cache-julia
+        uses: actions/cache@v2
         with:
-          version: ${{ matrix.julia-version }}
+          path: ~/julia
+          key: ${{ runner.os }}-${{ matrix.julia-version }}
+      - name: Build Julia
+        if: steps.cache-julia.outputs.cache-hit != 'true'
+        uses: julia-actions/build-julia@v1
+        with:
+          ref: v${{ matrix.julia-version }}
       - name: "Set up Python"
         uses: actions/setup-python@v1
         with:
@@ -51,4 +58,4 @@ jobs:
         with:
           toolchain: ${{ matrix.rust-version }}
       - name: "Run benchmark"
-        run: JULIAHOME=/ make gh_action_benchmarks.csv
+        run: JULIAHOME=~/julia make gh_action_benchmarks.csv

From d6b67210fe86b4c710bd12a3df613ff314ca3c63 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 14:34:22 -0500
Subject: [PATCH 10/46] Add go to github workflow

---
 .github/workflows/benchmarks.yml | 4 ++++
 Makefile                         | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 20ff619..cd45ae0 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -25,6 +25,7 @@ jobs:
         numpy-version: ['1.22']
         gfortran-version: ['9']
         rust-version: ['1.42.0']
+        go-version: ['1.17.4']
     
     steps:
       - uses: actions/checkout@v2
@@ -57,5 +58,8 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           toolchain: ${{ matrix.rust-version }}
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
       - name: "Run benchmark"
         run: JULIAHOME=~/julia make gh_action_benchmarks.csv
diff --git a/Makefile b/Makefile
index eed5d12..0849ede 100644
--- a/Makefile
+++ b/Makefile
@@ -131,7 +131,7 @@ benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock
 	cd rust; for t in 1 2 3 4 5; do cargo run --release -q; done >../$@
 
 LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust
-GH_ACTION_LANGUAGES = c fortran julia python rust
+GH_ACTION_LANGUAGES = c fortran go julia python rust
 
 # These were formerly listed in LANGUAGES, but I can't get them to run
 # 2017-09-27 johnfgibson
@@ -144,7 +144,7 @@ versions.csv: bin/versions.sh
 	$^ >$@
 
 gh_action_versions.csv: bin/versions.sh
-	bin/versions.sh ",c,fortran,julia,python,rust," >$@
+	bin/versions.sh ",c,fortran,go,julia,python,rust," >$@
 
 benchmarks.csv: bin/collect.jl $(BENCHMARKS)
 	@$(call PRINT_JULIA, $^ >$@)

From b3f0c13e0edf52ed724dae8641a4d60a0bb169f2 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 14:42:04 -0500
Subject: [PATCH 11/46] Fix Julia build caching

---
 .github/workflows/benchmarks.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index cd45ae0..c450b42 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -40,7 +40,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/julia
-          key: ${{ runner.os }}-${{ matrix.julia-version }}
+          key: ${{ runner.os }}-v${{ matrix.julia-version }}
       - name: Build Julia
         if: steps.cache-julia.outputs.cache-hit != 'true'
         uses: julia-actions/build-julia@v1

From de589803b318268d508f527f074ba2a5cd0d9caa Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 17:01:16 -0500
Subject: [PATCH 12/46] Fix spaces in BLASDIR

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 0849ede..4f95af2 100644
--- a/Makefile
+++ b/Makefile
@@ -22,8 +22,8 @@ endif
 #Which BLAS library am I using?
 ifeq ($(USE_SYSTEM_BLAS), 0)
 BLASMANIFEST=$(shell cat $(JULIAHOME)/usr/manifest/openblas)
-BLASDIR=$(JULIAHOME)/deps/scratch/$(BLASMANIFEST)/
-LIBBLAS=$(BLASDIR)$(LIBBLASNAME).a
+BLASDIR=$(shell echo $(JULIAHOME)/deps/scratch/$(BLASMANIFEST)/ | sed 's! !\\ !g'))
+LIBBLAS=$(shell echo $(BLASDIR)$(LIBBLASNAME).a | sed 's! !\\ !g')
 endif
 
 FFLAGS=-fexternal-blas

From 2b6a3d44b75c57b000236460637ca176d23744e8 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 18:14:36 -0500
Subject: [PATCH 13/46] Run empty benchmark to cache Julia build

---
 .github/workflows/benchmarks.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index c450b42..1941d8c 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -62,4 +62,5 @@ jobs:
         with:
           go-version: ${{ matrix.go-version }}
       - name: "Run benchmark"
-        run: JULIAHOME=~/julia make gh_action_benchmarks.csv
+        # run: JULIAHOME=~/julia make gh_action_benchmarks.csv
+        run: echo "Empty."

From 72b7d447a3b985e21b927da0cc5cb616a5fd476e Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 18:14:59 -0500
Subject: [PATCH 14/46] Turn back on benchmark

---
 .github/workflows/benchmarks.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 1941d8c..c450b42 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -62,5 +62,4 @@ jobs:
         with:
           go-version: ${{ matrix.go-version }}
       - name: "Run benchmark"
-        # run: JULIAHOME=~/julia make gh_action_benchmarks.csv
-        run: echo "Empty."
+        run: JULIAHOME=~/julia make gh_action_benchmarks.csv

From 90c55e87a8562ec267732b16ee5d746876ff6380 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 18:15:15 -0500
Subject: [PATCH 15/46] Fix syntax error in BLASDIR

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 4f95af2..12d8527 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ endif
 #Which BLAS library am I using?
 ifeq ($(USE_SYSTEM_BLAS), 0)
 BLASMANIFEST=$(shell cat $(JULIAHOME)/usr/manifest/openblas)
-BLASDIR=$(shell echo $(JULIAHOME)/deps/scratch/$(BLASMANIFEST)/ | sed 's! !\\ !g'))
+BLASDIR=$(shell echo $(JULIAHOME)/deps/scratch/$(BLASMANIFEST)/ | sed 's! !\\ !g')
 LIBBLAS=$(shell echo $(BLASDIR)$(LIBBLASNAME).a | sed 's! !\\ !g')
 endif
 

From 6b5da95cee817cd4dfce09d30289ca29fb754ffd Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 18:30:16 -0500
Subject: [PATCH 16/46] Allow variable number of iterations in tests

---
 Makefile | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/Makefile b/Makefile
index 12d8527..bf69bdf 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,8 @@ include $(JULIAHOME)/deps/Versions.make
 
 NODEJSBIN = node8
 
+ITERATIONS=$(shell seq 1 5)
+
 #Use python2 for Python 2.x
 PYTHON = python3
 
@@ -80,10 +82,10 @@ benchmarks/fortran.csv: \
 
 
 benchmarks/c%.csv: bin/perf%
-	for t in 1 2 3 4 5; do $<; done >$@
+	$(foreach t,$(ITERATIONS), $<;) >$@
 
 benchmarks/fortran%.csv: bin/fperf%
-	for t in 1 2 3 4 5; do $<; done >$@
+	$(foreach t,$(ITERATIONS), $<;) >$@
 
 benchmarks/go.csv: export GOPATH=$(abspath gopath)
 benchmarks/go.csv: perf.go
@@ -92,43 +94,43 @@ benchmarks/go.csv: perf.go
 	go get github.com/gonum/blas/cgo
 	go get github.com/gonum/matrix/mat64
 	go get github.com/gonum/stat
-	for t in 1 2 3 4 5; do go run $<; done >$@
+	$(foreach t,$(ITERATIONS), go run $<;) >$@
 
 benchmarks/julia.csv: perf.jl
-	for t in 1 2 3 4 5; do $(JULIAHOME)/usr/bin/julia $<; done >$@
+	$(foreach t,$(ITERATIONS), $(JULIAHOME)/usr/bin/julia $<;) >$@
 
 benchmarks/python.csv: perf.py
-	for t in 1 2 3 4 5; do $(PYTHON) $<; done >$@
+	$(foreach t,$(ITERATIONS), $(PYTHON) $<;) >$@
 
 benchmarks/matlab.csv: perf.m
-	for t in 1 2 3 4 5; do matlab -nojvm -singleCompThread -r 'perf; perf; exit' | grep ^matlab | tail -8; done >$@
+	$(foreach t,$(ITERATIONS), matlab -nojvm -singleCompThread -r 'perf; perf; exit' | grep ^matlab | tail -8;) >$@
 
 benchmarks/octave.csv: perf.m
-	for t in 1 2 3 4 5; do $(OCTAVE) -q --eval perf 2>/dev/null; done >$@
+	$(foreach t,$(ITERATIONS), $(OCTAVE) -q --eval perf 2>/dev/null;) >$@
 
 benchmarks/r.csv: perf.R
-	for t in 1 2 3 4 5; do cat $< | R --vanilla --slave 2>/dev/null; done >$@
+	$(foreach t,$(ITERATIONS), cat $< | R --vanilla --slave 2>/dev/null;) >$@
 
 benchmarks/javascript.csv: perf.js
-	for t in 1 2 3 4 5; do $(NODEJSBIN) $<; done >$@
+	$(foreach t,$(ITERATIONS), $(NODEJSBIN) $<;) >$@
 
 benchmarks/mathematica.csv: perf.nb
-	for t in 1 2 3 4 5; do $(MATHEMATICABIN) -noprompt -run "<<$<; Exit[]"; done >$@
+	$(foreach t,$(ITERATIONS), $(MATHEMATICABIN) -noprompt -run "<<$<; Exit[]";) >$@
 
 benchmarks/stata.csv: perf.do
-	for t in 1 2 3 4 5; do stata -b do $^ $@; done
+	$(foreach t,$(ITERATIONS), stata -b do $^ $@;)
 
 benchmarks/lua.csv: perf.lua
-	for t in 1 2 3 4 5; do scilua $<; done >$@
+	$(foreach t,$(ITERATIONS), scilua $<;) >$@
 
 benchmarks/java.csv: java/src/main/java/PerfBLAS.java
-	cd java; sh setup.sh; for t in 1 2 3 4 5; do mvn -q exec:java; done >../$@
+	cd java; sh setup.sh; $(foreach t,$(ITERATIONS), mvn -q exec:java;) >../$@
 
 benchmarks/scala.csv: scala/src/main/scala/perf.scala scala/build.sbt
-	cd scala; for t in 1 2 3 4 5; do sbt run; done >../$@
+	cd scala; $(foreach t,$(ITERATIONS), sbt run;) >../$@
 
 benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock
-	cd rust; for t in 1 2 3 4 5; do cargo run --release -q; done >../$@
+	cd rust; $(foreach t,$(ITERATIONS), cargo run --release -q;) >../$@
 
 LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust
 GH_ACTION_LANGUAGES = c fortran go julia python rust

From e16181398340432895200985ba7784ff0c35a48f Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 18:59:46 -0500
Subject: [PATCH 17/46] cut first part of openblas manifest

---
 .github/workflows/benchmarks.yml | 2 +-
 Makefile                         | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index c450b42..8619615 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -41,7 +41,7 @@ jobs:
         with:
           path: ~/julia
           key: ${{ runner.os }}-v${{ matrix.julia-version }}
-      - name: Build Julia
+      - name: "Build Julia"
         if: steps.cache-julia.outputs.cache-hit != 'true'
         uses: julia-actions/build-julia@v1
         with:
diff --git a/Makefile b/Makefile
index bf69bdf..04ea37b 100644
--- a/Makefile
+++ b/Makefile
@@ -23,9 +23,10 @@ endif
 
 #Which BLAS library am I using?
 ifeq ($(USE_SYSTEM_BLAS), 0)
-BLASMANIFEST=$(shell cat $(JULIAHOME)/usr/manifest/openblas)
-BLASDIR=$(shell echo $(JULIAHOME)/deps/scratch/$(BLASMANIFEST)/ | sed 's! !\\ !g')
-LIBBLAS=$(shell echo $(BLASDIR)$(LIBBLASNAME).a | sed 's! !\\ !g')
+# manifest contains `blasmanifest bb-uninstaller`, so we need to cut the second part:
+BLASMANIFEST=$(shell cat $(JULIAHOME)/usr/manifest/openblas | cut -d' ' -f1)
+BLASDIR=$(JULIAHOME)/deps/scratch/$(BLASMANIFEST)/
+LIBBLAS=$(BLASDIR)$(LIBBLASNAME).a
 endif
 
 FFLAGS=-fexternal-blas

From be3ae824295483bb96ba6d0a8e53b4ef0b125052 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 19:04:26 -0500
Subject: [PATCH 18/46] Find openblas library

---
 .github/workflows/benchmarks.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 8619615..b250b68 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -62,4 +62,6 @@ jobs:
         with:
           go-version: ${{ matrix.go-version }}
       - name: "Run benchmark"
-        run: JULIAHOME=~/julia make gh_action_benchmarks.csv
+        run: |
+          find ~/julia -name 'libopenblas*'
+          JULIAHOME=~/julia make gh_action_benchmarks.csv

From 80ee2d954d8ef82c00d8f62358d8d5b25ea3d9b1 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 19:42:08 -0500
Subject: [PATCH 19/46] Fix BLAS linking

---
 Makefile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 04ea37b..9164b3e 100644
--- a/Makefile
+++ b/Makefile
@@ -24,9 +24,8 @@ endif
 #Which BLAS library am I using?
 ifeq ($(USE_SYSTEM_BLAS), 0)
 # manifest contains `blasmanifest bb-uninstaller`, so we need to cut the second part:
-BLASMANIFEST=$(shell cat $(JULIAHOME)/usr/manifest/openblas | cut -d' ' -f1)
-BLASDIR=$(JULIAHOME)/deps/scratch/$(BLASMANIFEST)/
-LIBBLAS=$(BLASDIR)$(LIBBLASNAME).a
+BLASDIR=$(JULIAHOME)/deps/srccache/
+LIBBLAS=$(BLASDIR)/lib/$(LIBBLASNAME).dylib
 endif
 
 FFLAGS=-fexternal-blas
@@ -56,7 +55,7 @@ export GOTO_NUM_THREADS=1
 export OPENBLAS_NUM_THREADS=1
 
 perf.h: $(JULIAHOME)/deps/Versions.make
-	echo '#include "$(BLASDIR)cblas.h"' > $@
+	echo '#include "$(BLASDIR)/include/cblas.h"' > $@
 	echo '#include "$(DSFMTDIR)/dSFMT.c"' >> $@
 
 bin/perf%: perf.c perf.h

From 0ccc6c852e2aa98ffabfddd6cabcaffb59f3303e Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 19:43:25 -0500
Subject: [PATCH 20/46] Force user to set up dSFMT

---
 .github/workflows/benchmarks.yml | 11 ++++++++++-
 Makefile                         |  7 ++++---
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index b250b68..3d76a31 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -46,6 +46,15 @@ jobs:
         uses: julia-actions/build-julia@v1
         with:
           ref: v${{ matrix.julia-version }}
+      - name: "Download dSFMT"
+        run: |
+          cd ~/
+          mkdir -p dSFMT
+          cd dSFMT
+          wget https://github.com/MersenneTwister-Lab/dSFMT/archive/refs/tags/v2.2.4.tar.gz
+          echo "39682961ecfba621a98dbb6610b6ae2b7d6add450d4f08d8d4edd0e10abd8174 v2.2.4.tar.gz" | sha256sum --check --status
+          tar -xzf v2.2.4.tar.gz
+          mv dSFMT-*/* ./
       - name: "Set up Python"
         uses: actions/setup-python@v1
         with:
@@ -64,4 +73,4 @@ jobs:
       - name: "Run benchmark"
         run: |
           find ~/julia -name 'libopenblas*'
-          JULIAHOME=~/julia make gh_action_benchmarks.csv
+          JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.csv
diff --git a/Makefile b/Makefile
index 9164b3e..33f7a99 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,10 @@
 ifndef JULIAHOME
 $(error JULIAHOME not defined. Set value to the root of the Julia source tree.)
 endif
+ifndef DSFMTDIR
+$(error DSFMTDIR not defined. Set value to the root of the dSFMT source tree.)
+endif
+
 include $(JULIAHOME)/Make.inc
 include $(JULIAHOME)/deps/Versions.make
 
@@ -45,9 +49,6 @@ LIBM = $(LIBMDIR)libopenlibm.a
 endif
 endif
 
-DSFMTDIR = $(JULIAHOME)/deps/scratch/dsfmt-$(DSFMT_VER)
-RMATHDIR = $(JULIAHOME)/deps/scratch/Rmath-julia-$(RMATH_JULIA_VER)
-
 default: benchmarks.html
 
 export OMP_NUM_THREADS=1

From 89fc84aeeb0728b35489d3ff2a6aca619f200449 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 19:46:31 -0500
Subject: [PATCH 21/46] Fix blas dir on GitHub actions

---
 .github/workflows/benchmarks.yml | 1 -
 Makefile                         | 3 +--
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 3d76a31..309df94 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -72,5 +72,4 @@ jobs:
           go-version: ${{ matrix.go-version }}
       - name: "Run benchmark"
         run: |
-          find ~/julia -name 'libopenblas*'
           JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.csv
diff --git a/Makefile b/Makefile
index 33f7a99..9a7281d 100644
--- a/Makefile
+++ b/Makefile
@@ -27,8 +27,7 @@ endif
 
 #Which BLAS library am I using?
 ifeq ($(USE_SYSTEM_BLAS), 0)
-# manifest contains `blasmanifest bb-uninstaller`, so we need to cut the second part:
-BLASDIR=$(JULIAHOME)/deps/srccache/
+BLASDIR=$(JULIAHOME)/usr/
 LIBBLAS=$(BLASDIR)/lib/$(LIBBLASNAME).dylib
 endif
 

From 2055303f9de47a48372c5145c19b80b7170806b9 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 19:58:31 -0500
Subject: [PATCH 22/46] Unpack blas manually

---
 .github/workflows/benchmarks.yml | 8 ++++++--
 Makefile                         | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 309df94..e1b0980 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -46,7 +46,7 @@ jobs:
         uses: julia-actions/build-julia@v1
         with:
           ref: v${{ matrix.julia-version }}
-      - name: "Download dSFMT"
+      - name: "Setup dSFMT"
         run: |
           cd ~/
           mkdir -p dSFMT
@@ -55,6 +55,10 @@ jobs:
           echo "39682961ecfba621a98dbb6610b6ae2b7d6add450d4f08d8d4edd0e10abd8174 v2.2.4.tar.gz" | sha256sum --check --status
           tar -xzf v2.2.4.tar.gz
           mv dSFMT-*/* ./
+      - name: "Setup OpenBLAS"
+        run: |
+          cd ~/julia/deps/srccache
+          tar -xzf OpenBLAS*
       - name: "Set up Python"
         uses: actions/setup-python@v1
         with:
@@ -72,4 +76,4 @@ jobs:
           go-version: ${{ matrix.go-version }}
       - name: "Run benchmark"
         run: |
-          JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.csv
+          JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.csv
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 9a7281d..9281f04 100644
--- a/Makefile
+++ b/Makefile
@@ -27,7 +27,7 @@ endif
 
 #Which BLAS library am I using?
 ifeq ($(USE_SYSTEM_BLAS), 0)
-BLASDIR=$(JULIAHOME)/usr/
+BLASDIR=$(JULIAHOME)/deps/srccache/
 LIBBLAS=$(BLASDIR)/lib/$(LIBBLASNAME).dylib
 endif
 

From 991eae93db8416cef4ae0899350556fc7406350b Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 20:32:08 -0500
Subject: [PATCH 23/46] Fix openblas import

---
 .github/workflows/benchmarks.yml | 6 +++---
 Makefile                         | 9 ++++-----
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index e1b0980..a008e1f 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -57,8 +57,7 @@ jobs:
           mv dSFMT-*/* ./
       - name: "Setup OpenBLAS"
         run: |
-          cd ~/julia/deps/srccache
-          tar -xzf OpenBLAS*
+          sudo apt-get install -y libopenblas-dev
       - name: "Set up Python"
         uses: actions/setup-python@v1
         with:
@@ -76,4 +75,5 @@ jobs:
           go-version: ${{ matrix.go-version }}
       - name: "Run benchmark"
         run: |
-          JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.csv
\ No newline at end of file
+          find / -name 'libopenblas*'
+          JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ BLASDIR=/usr/ make gh_action_benchmarks.csv
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 9281f04..132544b 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,9 @@ endif
 ifndef DSFMTDIR
 $(error DSFMTDIR not defined. Set value to the root of the dSFMT source tree.)
 endif
+ifndef BLASDIR
+$(error BLASDIR not defined. Set value to the root of the OpenBLAS source tree.)
+endif
 
 include $(JULIAHOME)/Make.inc
 include $(JULIAHOME)/deps/Versions.make
@@ -25,11 +28,7 @@ else
 MATHEMATICABIN = math
 endif
 
-#Which BLAS library am I using?
-ifeq ($(USE_SYSTEM_BLAS), 0)
-BLASDIR=$(JULIAHOME)/deps/srccache/
-LIBBLAS=$(BLASDIR)/lib/$(LIBBLASNAME).dylib
-endif
+LIBBLAS=$(BLASDIR)/lib/$(LIBBLASNAME).so
 
 FFLAGS=-fexternal-blas
 #gfortran cannot multiply matrices using 64-bit external BLAS.

From 88570dda9cf5026d4ce608c5af886a5de151bb9b Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 20:48:14 -0500
Subject: [PATCH 24/46] Switch to dynamic blas library

---
 .github/workflows/benchmarks.yml |  3 +--
 Makefile                         | 12 +++---------
 2 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index a008e1f..03fa4e7 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -75,5 +75,4 @@ jobs:
           go-version: ${{ matrix.go-version }}
       - name: "Run benchmark"
         run: |
-          find / -name 'libopenblas*'
-          JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ BLASDIR=/usr/ make gh_action_benchmarks.csv
\ No newline at end of file
+          JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.csv
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 132544b..d2dc168 100644
--- a/Makefile
+++ b/Makefile
@@ -4,9 +4,6 @@ endif
 ifndef DSFMTDIR
 $(error DSFMTDIR not defined. Set value to the root of the dSFMT source tree.)
 endif
-ifndef BLASDIR
-$(error BLASDIR not defined. Set value to the root of the OpenBLAS source tree.)
-endif
 
 include $(JULIAHOME)/Make.inc
 include $(JULIAHOME)/deps/Versions.make
@@ -28,8 +25,6 @@ else
 MATHEMATICABIN = math
 endif
 
-LIBBLAS=$(BLASDIR)/lib/$(LIBBLASNAME).so
-
 FFLAGS=-fexternal-blas
 #gfortran cannot multiply matrices using 64-bit external BLAS.
 ifeq ($(findstring gfortran, $(FC)), gfortran)
@@ -54,15 +49,14 @@ export GOTO_NUM_THREADS=1
 export OPENBLAS_NUM_THREADS=1
 
 perf.h: $(JULIAHOME)/deps/Versions.make
-	echo '#include "$(BLASDIR)/include/cblas.h"' > $@
+	echo '#include "cblas.h"' > $@
 	echo '#include "$(DSFMTDIR)/dSFMT.c"' >> $@
 
 bin/perf%: perf.c perf.h
-	$(CC) -std=c99 -O$* $< -o $@  -I$(DSFMTDIR) $(LIBBLAS) -L$(LIBMDIR) $(LIBM) $(CFLAGS) -lpthread
+	$(CC) -std=c99 -O$* $< -o $@  -I$(DSFMTDIR) -lopenblas -L$(LIBMDIR) $(LIBM) $(CFLAGS) -lpthread
 
 bin/fperf%: perf.f90
 	mkdir -p mods/$@ #Modules for each binary go in separate directories
-#	$(FC) $(FFLAGS) -Jmods/$@ -O$* $< -o $@ $(LIBBLAS) -L$(LIBMDIR) $(LIBM) -lpthread
 	$(FC) $(FFLAGS) -Jmods/$@ -O$* $< -o $@ -lopenblas -L$(LIBMDIR) $(LIBM) -lpthread
 
 benchmarks/c.csv: \
@@ -88,7 +82,7 @@ benchmarks/fortran%.csv: bin/fperf%
 
 benchmarks/go.csv: export GOPATH=$(abspath gopath)
 benchmarks/go.csv: perf.go
-	#CGO_LDFLAGS="$(LIBBLAS) $(LIBM)" go get github.com/gonum/blas/cgo
+	#CGO_LDFLAGS="-lopenblas $(LIBM)" go get github.com/gonum/blas/cgo
 	go get github.com/gonum/blas/blas64
 	go get github.com/gonum/blas/cgo
 	go get github.com/gonum/matrix/mat64

From 5002a952afdaf81436a06fe98e818d50efb912b1 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 21:03:51 -0500
Subject: [PATCH 25/46] Try fixing Go benchmark

---
 Makefile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index d2dc168..6f2473e 100644
--- a/Makefile
+++ b/Makefile
@@ -82,9 +82,9 @@ benchmarks/fortran%.csv: bin/fperf%
 
 benchmarks/go.csv: export GOPATH=$(abspath gopath)
 benchmarks/go.csv: perf.go
-	#CGO_LDFLAGS="-lopenblas $(LIBM)" go get github.com/gonum/blas/cgo
+	CGO_LDFLAGS="-lopenblas $(LIBM)" go get github.com/gonum/blas/cgo
 	go get github.com/gonum/blas/blas64
-	go get github.com/gonum/blas/cgo
+	# go get github.com/gonum/blas/cgo
 	go get github.com/gonum/matrix/mat64
 	go get github.com/gonum/stat
 	$(foreach t,$(ITERATIONS), go run $<;) >$@
@@ -126,7 +126,8 @@ benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock
 	cd rust; $(foreach t,$(ITERATIONS), cargo run --release -q;) >../$@
 
 LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust
-GH_ACTION_LANGUAGES = c fortran go julia python rust
+# GH_ACTION_LANGUAGES = c fortran go julia python rust
+GH_ACTION_LANGUAGES = go julia python rust # Just for testing.
 
 # These were formerly listed in LANGUAGES, but I can't get them to run
 # 2017-09-27 johnfgibson

From 547881cfd6e616257e5baf4fe4ccc22b34000ac5 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 21:08:31 -0500
Subject: [PATCH 26/46] Update golang numerical package

---
 Makefile | 6 +-----
 perf.go  | 9 ++-------
 2 files changed, 3 insertions(+), 12 deletions(-)

diff --git a/Makefile b/Makefile
index 6f2473e..52e9c47 100644
--- a/Makefile
+++ b/Makefile
@@ -82,11 +82,7 @@ benchmarks/fortran%.csv: bin/fperf%
 
 benchmarks/go.csv: export GOPATH=$(abspath gopath)
 benchmarks/go.csv: perf.go
-	CGO_LDFLAGS="-lopenblas $(LIBM)" go get github.com/gonum/blas/cgo
-	go get github.com/gonum/blas/blas64
-	# go get github.com/gonum/blas/cgo
-	go get github.com/gonum/matrix/mat64
-	go get github.com/gonum/stat
+	go get gonum.org/v1/gonum@v0.9.3
 	$(foreach t,$(ITERATIONS), go run $<;) >$@
 
 benchmarks/julia.csv: perf.jl
diff --git a/perf.go b/perf.go
index 077a0bc..bebffc6 100644
--- a/perf.go
+++ b/perf.go
@@ -3,9 +3,7 @@
 // Three gonum packages must be installed, and then an additional environment
 // variable must be set to use the BLAS installation.
 // To install the gonum packages, run:
-// 		go get github.com/gonum/blas
-//		go get github.com/gonum/matrix/mat64
-//		go get github.com/gonum/stat
+// 		go get gonum.org/v1/gonum
 // The cgo ldflags must then be set to use the BLAS implementation. As an example,
 // download OpenBLAS to ~/software
 //		git clone https://github.com/xianyi/OpenBLAS
@@ -27,10 +25,7 @@ import (
 	"os"
 	"bufio"
 
-	"github.com/gonum/blas/blas64"
-	"github.com/gonum/blas/cgo"
-	"github.com/gonum/matrix/mat64"
-	"github.com/gonum/stat"
+	"gonum.org/v1/gonum"
 )
 
 func init() {

From 426f69a210f9ad3f47a1a68229babd2896532205 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 21:13:48 -0500
Subject: [PATCH 27/46] Install required Compat.jl

---
 .github/workflows/benchmarks.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 03fa4e7..ae50ed2 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -46,6 +46,9 @@ jobs:
         uses: julia-actions/build-julia@v1
         with:
           ref: v${{ matrix.julia-version }}
+      - name: "Install Julia packages"
+        run: |
+          julia -e 'using Pkg; Pkg.add("Compat")'
       - name: "Setup dSFMT"
         run: |
           cd ~/

From 3a339cc26776c62a4c42dc571fe95b5c4d4330e8 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 21:37:03 -0500
Subject: [PATCH 28/46] Revert golang edits and remove from GH actions

---
 .github/workflows/benchmarks.yml | 4 ----
 Makefile                         | 9 ++++++---
 perf.go                          | 9 +++++++--
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index ae50ed2..f149b84 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -25,7 +25,6 @@ jobs:
         numpy-version: ['1.22']
         gfortran-version: ['9']
         rust-version: ['1.42.0']
-        go-version: ['1.17.4']
     
     steps:
       - uses: actions/checkout@v2
@@ -73,9 +72,6 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           toolchain: ${{ matrix.rust-version }}
-      - uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
       - name: "Run benchmark"
         run: |
           JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.csv
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 52e9c47..0b0b3ef 100644
--- a/Makefile
+++ b/Makefile
@@ -82,7 +82,11 @@ benchmarks/fortran%.csv: bin/fperf%
 
 benchmarks/go.csv: export GOPATH=$(abspath gopath)
 benchmarks/go.csv: perf.go
-	go get gonum.org/v1/gonum@v0.9.3
+	#CGO_LDFLAGS="-lopenblas $(LIBM)" go get github.com/gonum/blas/cgo
+	go get github.com/gonum/blas/blas64
+	go get github.com/gonum/blas/cgo
+	go get github.com/gonum/matrix/mat64
+	go get github.com/gonum/stat
 	$(foreach t,$(ITERATIONS), go run $<;) >$@
 
 benchmarks/julia.csv: perf.jl
@@ -122,8 +126,7 @@ benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock
 	cd rust; $(foreach t,$(ITERATIONS), cargo run --release -q;) >../$@
 
 LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust
-# GH_ACTION_LANGUAGES = c fortran go julia python rust
-GH_ACTION_LANGUAGES = go julia python rust # Just for testing.
+GH_ACTION_LANGUAGES = c fortran julia python rust
 
 # These were formerly listed in LANGUAGES, but I can't get them to run
 # 2017-09-27 johnfgibson
diff --git a/perf.go b/perf.go
index bebffc6..077a0bc 100644
--- a/perf.go
+++ b/perf.go
@@ -3,7 +3,9 @@
 // Three gonum packages must be installed, and then an additional environment
 // variable must be set to use the BLAS installation.
 // To install the gonum packages, run:
-// 		go get gonum.org/v1/gonum
+// 		go get github.com/gonum/blas
+//		go get github.com/gonum/matrix/mat64
+//		go get github.com/gonum/stat
 // The cgo ldflags must then be set to use the BLAS implementation. As an example,
 // download OpenBLAS to ~/software
 //		git clone https://github.com/xianyi/OpenBLAS
@@ -25,7 +27,10 @@ import (
 	"os"
 	"bufio"
 
-	"gonum.org/v1/gonum"
+	"github.com/gonum/blas/blas64"
+	"github.com/gonum/blas/cgo"
+	"github.com/gonum/matrix/mat64"
+	"github.com/gonum/stat"
 )
 
 func init() {

From 25762d46d59eb7e14ff360fae3b247498c155abf Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 21:41:51 -0500
Subject: [PATCH 29/46] Add Java to github actions

---
 .github/workflows/benchmarks.yml | 7 +++++++
 Makefile                         | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index f149b84..4563459 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -20,6 +20,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
+        java-version: ['17']
         julia-version: ['1.7.1']
         python-version: ['3.9']
         numpy-version: ['1.22']
@@ -72,6 +73,12 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           toolchain: ${{ matrix.rust-version }}
+      - name: "Set up Java"
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java-version }}
+          cache: 'maven'
       - name: "Run benchmark"
         run: |
           JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.csv
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 0b0b3ef..bf08e92 100644
--- a/Makefile
+++ b/Makefile
@@ -126,7 +126,7 @@ benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock
 	cd rust; $(foreach t,$(ITERATIONS), cargo run --release -q;) >../$@
 
 LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust
-GH_ACTION_LANGUAGES = c fortran julia python rust
+GH_ACTION_LANGUAGES = c fortran java julia python rust
 
 # These were formerly listed in LANGUAGES, but I can't get them to run
 # 2017-09-27 johnfgibson

From f0bfea72575f2f1cef7e60c5cf84b295ad7f2d9c Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 21:47:41 -0500
Subject: [PATCH 30/46] Nitpick naming in workflow

---
 .github/workflows/benchmarks.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 4563459..cd7b67f 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -49,7 +49,7 @@ jobs:
       - name: "Install Julia packages"
         run: |
           julia -e 'using Pkg; Pkg.add("Compat")'
-      - name: "Setup dSFMT"
+      - name: "Set up dSFMT"
         run: |
           cd ~/
           mkdir -p dSFMT
@@ -58,7 +58,7 @@ jobs:
           echo "39682961ecfba621a98dbb6610b6ae2b7d6add450d4f08d8d4edd0e10abd8174 v2.2.4.tar.gz" | sha256sum --check --status
           tar -xzf v2.2.4.tar.gz
           mv dSFMT-*/* ./
-      - name: "Setup OpenBLAS"
+      - name: "Set up OpenBLAS"
         run: |
           sudo apt-get install -y libopenblas-dev
       - name: "Set up Python"

From d8d07d8a055453674bfc1ef824d05034e395717e Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 21:50:24 -0500
Subject: [PATCH 31/46] Note that fortran version is unused

---
 .github/workflows/benchmarks.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index cd7b67f..d89000e 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -24,7 +24,7 @@ jobs:
         julia-version: ['1.7.1']
         python-version: ['3.9']
         numpy-version: ['1.22']
-        gfortran-version: ['9']
+        gfortran-version: ['9'] # Note: unused since is built-in.
         rust-version: ['1.42.0']
     
     steps:
@@ -67,8 +67,6 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: "Set up NumPy"
         run: pip install numpy==${{ matrix.numpy-version }}
-      - name: "Set up fortran"
-        run: sudo apt-get install -y gfortran-${{ matrix.gfortran-version }}
       - name: "Set up Rust"
         uses: actions-rs/toolchain@v1
         with:

From 18e6396e07268fe6b7db3a475857076135a25269 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 21:55:13 -0500
Subject: [PATCH 32/46] Set up javascript

---
 .github/workflows/benchmarks.yml | 5 +++++
 Makefile                         | 5 +++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index d89000e..e84ba8d 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -26,6 +26,7 @@ jobs:
         numpy-version: ['1.22']
         gfortran-version: ['9'] # Note: unused since is built-in.
         rust-version: ['1.42.0']
+        js-version: ['16.14.10']
     
     steps:
       - uses: actions/checkout@v2
@@ -77,6 +78,10 @@ jobs:
           distribution: 'temurin'
           java-version: ${{ matrix.java-version }}
           cache: 'maven'
+      - name: "Set up JavaScript"
+        uses: actions/setup-node@v2
+        with:
+          nodejs-version: ${{ matrix.js-version }}
       - name: "Run benchmark"
         run: |
           JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.csv
\ No newline at end of file
diff --git a/Makefile b/Makefile
index bf08e92..c506d6c 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,7 @@ endif
 include $(JULIAHOME)/Make.inc
 include $(JULIAHOME)/deps/Versions.make
 
-NODEJSBIN = node8
+NODEJSBIN = node
 
 ITERATIONS=$(shell seq 1 5)
 
@@ -126,7 +126,8 @@ benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock
 	cd rust; $(foreach t,$(ITERATIONS), cargo run --release -q;) >../$@
 
 LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust
-GH_ACTION_LANGUAGES = c fortran java julia python rust
+# GH_ACTION_LANGUAGES = c fortran java javascript julia python rust
+GH_ACTION_LANGUAGES = javascript
 
 # These were formerly listed in LANGUAGES, but I can't get them to run
 # 2017-09-27 johnfgibson

From 52b6a4a1802ab4bfb703ab722c10695b12cc4c14 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 21:56:49 -0500
Subject: [PATCH 33/46] Set up R lang

---
 .github/workflows/benchmarks.yml | 5 +++++
 Makefile                         | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index e84ba8d..54278c5 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -27,6 +27,7 @@ jobs:
         gfortran-version: ['9'] # Note: unused since is built-in.
         rust-version: ['1.42.0']
         js-version: ['16.14.10']
+        r-version: ['4.1.2']
     
     steps:
       - uses: actions/checkout@v2
@@ -82,6 +83,10 @@ jobs:
         uses: actions/setup-node@v2
         with:
           nodejs-version: ${{ matrix.js-version }}
+      - name: "Set up R"
+        uses: r-lib/actions/setup-r@v2
+        with:
+          r-version: ${{ matrix.r-version }}
       - name: "Run benchmark"
         run: |
           JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.csv
\ No newline at end of file
diff --git a/Makefile b/Makefile
index c506d6c..5c92fe1 100644
--- a/Makefile
+++ b/Makefile
@@ -127,7 +127,7 @@ benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock
 
 LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust
 # GH_ACTION_LANGUAGES = c fortran java javascript julia python rust
-GH_ACTION_LANGUAGES = javascript
+GH_ACTION_LANGUAGES = javascript r
 
 # These were formerly listed in LANGUAGES, but I can't get them to run
 # 2017-09-27 johnfgibson

From 46e5aad80da0e3942bb49677d62406aa9ecead7f Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 21:58:57 -0500
Subject: [PATCH 34/46] Fix node-version argument to action

---
 .github/workflows/benchmarks.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 54278c5..f75b758 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -82,7 +82,7 @@ jobs:
       - name: "Set up JavaScript"
         uses: actions/setup-node@v2
         with:
-          nodejs-version: ${{ matrix.js-version }}
+          node-version: ${{ matrix.js-version }}
       - name: "Set up R"
         uses: r-lib/actions/setup-r@v2
         with:

From 90e171d70aa29f41b0a4a3d7baa043b3b119d5e9 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 22:04:55 -0500
Subject: [PATCH 35/46] Add LuaJIT to github action

---
 .github/workflows/benchmarks.yml | 5 +++++
 Makefile                         | 6 +++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index f75b758..368c9da 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -28,6 +28,7 @@ jobs:
         rust-version: ['1.42.0']
         js-version: ['16.14.10']
         r-version: ['4.1.2']
+        lua-version: ['2.0.5']
     
     steps:
       - uses: actions/checkout@v2
@@ -87,6 +88,10 @@ jobs:
         uses: r-lib/actions/setup-r@v2
         with:
           r-version: ${{ matrix.r-version }}
+      - name: "Set up LuaJit"
+        uses: leafo/gh-actions-lua@v8.0.0
+        with:
+          luaVersion: luajit-${{ matrix.lua-version }}
       - name: "Run benchmark"
         run: |
           JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.csv
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 5c92fe1..0e65d24 100644
--- a/Makefile
+++ b/Makefile
@@ -126,8 +126,8 @@ benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock
 	cd rust; $(foreach t,$(ITERATIONS), cargo run --release -q;) >../$@
 
 LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust
-# GH_ACTION_LANGUAGES = c fortran java javascript julia python rust
-GH_ACTION_LANGUAGES = javascript r
+# GH_ACTION_LANGUAGES = c fortran java javascript julia python r rust
+GH_ACTION_LANGUAGES = lua r
 
 # These were formerly listed in LANGUAGES, but I can't get them to run
 # 2017-09-27 johnfgibson
@@ -140,7 +140,7 @@ versions.csv: bin/versions.sh
 	$^ >$@
 
 gh_action_versions.csv: bin/versions.sh
-	bin/versions.sh ",c,fortran,go,julia,python,rust," >$@
+	bin/versions.sh ",c,fortran,java,javascript,julia,lua,python,r,rust," >$@
 
 benchmarks.csv: bin/collect.jl $(BENCHMARKS)
 	@$(call PRINT_JULIA, $^ >$@)

From df6de3e12f1f10177530b082d2e6574f689d6248 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 22:05:40 -0500
Subject: [PATCH 36/46] Coarsen choice of nodejs

---
 .github/workflows/benchmarks.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 368c9da..d975872 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -26,7 +26,7 @@ jobs:
         numpy-version: ['1.22']
         gfortran-version: ['9'] # Note: unused since is built-in.
         rust-version: ['1.42.0']
-        js-version: ['16.14.10']
+        js-version: ['16']
         r-version: ['4.1.2']
         lua-version: ['2.0.5']
     

From ea741a07456ec54e6c7c959106f4cb0055496785 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 22:10:59 -0500
Subject: [PATCH 37/46] Change luajit command

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 0e65d24..1baccac 100644
--- a/Makefile
+++ b/Makefile
@@ -114,7 +114,7 @@ benchmarks/stata.csv: perf.do
 	$(foreach t,$(ITERATIONS), stata -b do $^ $@;)
 
 benchmarks/lua.csv: perf.lua
-	$(foreach t,$(ITERATIONS), scilua $<;) >$@
+	$(foreach t,$(ITERATIONS), luajit $<;) >$@
 
 benchmarks/java.csv: java/src/main/java/PerfBLAS.java
 	cd java; sh setup.sh; $(foreach t,$(ITERATIONS), mvn -q exec:java;) >../$@

From 7f3401dfc34877649f1d4e88bf1585dc2b9a490c Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 22:17:44 -0500
Subject: [PATCH 38/46] Turn off lua benchmark as it is broken

---
 .github/workflows/benchmarks.yml | 11 ++++++++---
 Makefile                         |  5 ++---
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index d975872..b3a2899 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -24,11 +24,12 @@ jobs:
         julia-version: ['1.7.1']
         python-version: ['3.9']
         numpy-version: ['1.22']
-        gfortran-version: ['9'] # Note: unused since is built-in.
+        gfortran-version: ['9']  # Note: unused since is built-in.
         rust-version: ['1.42.0']
         js-version: ['16']
         r-version: ['4.1.2']
-        lua-version: ['2.0.5']
+        lua-version: ['2.0.5']  # Note: Not used as benchmark is broken.
+        go-version: ['1.17.4']  # Note: Not used as benchmark is broken.
     
     steps:
       - uses: actions/checkout@v2
@@ -92,6 +93,10 @@ jobs:
         uses: leafo/gh-actions-lua@v8.0.0
         with:
           luaVersion: luajit-${{ matrix.lua-version }}
+      - name: "Set up Go"
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
       - name: "Run benchmark"
         run: |
-          JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.csv
\ No newline at end of file
+          JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.html
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 1baccac..96dbf23 100644
--- a/Makefile
+++ b/Makefile
@@ -126,8 +126,7 @@ benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock
 	cd rust; $(foreach t,$(ITERATIONS), cargo run --release -q;) >../$@
 
 LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust
-# GH_ACTION_LANGUAGES = c fortran java javascript julia python r rust
-GH_ACTION_LANGUAGES = lua r
+GH_ACTION_LANGUAGES = c fortran java javascript julia python r rust
 
 # These were formerly listed in LANGUAGES, but I can't get them to run
 # 2017-09-27 johnfgibson
@@ -140,7 +139,7 @@ versions.csv: bin/versions.sh
 	$^ >$@
 
 gh_action_versions.csv: bin/versions.sh
-	bin/versions.sh ",c,fortran,java,javascript,julia,lua,python,r,rust," >$@
+	bin/versions.sh ",c,fortran,java,javascript,julia,python,r,rust," >$@
 
 benchmarks.csv: bin/collect.jl $(BENCHMARKS)
 	@$(call PRINT_JULIA, $^ >$@)

From 5ecbcbb5cc714eb8f297a8ce27f1cc0d308226b0 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 22:58:16 -0500
Subject: [PATCH 39/46] Change to using @for syntax

---
 Makefile | 39 ++++++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/Makefile b/Makefile
index 96dbf23..136694d 100644
--- a/Makefile
+++ b/Makefile
@@ -5,6 +5,11 @@ ifndef DSFMTDIR
 $(error DSFMTDIR not defined. Set value to the root of the dSFMT source tree.)
 endif
 
+
+# Will make multi-line targets work
+# (so we can use @for on the second line)
+.ONESHELL:
+
 include $(JULIAHOME)/Make.inc
 include $(JULIAHOME)/deps/Versions.make
 
@@ -75,10 +80,10 @@ benchmarks/fortran.csv: \
 
 
 benchmarks/c%.csv: bin/perf%
-	$(foreach t,$(ITERATIONS), $<;) >$@
+	@for t in $(ITERATIONS); do $<; done >$@
 
 benchmarks/fortran%.csv: bin/fperf%
-	$(foreach t,$(ITERATIONS), $<;) >$@
+	@for t in $(ITERATIONS); do $<; done >$@
 
 benchmarks/go.csv: export GOPATH=$(abspath gopath)
 benchmarks/go.csv: perf.go
@@ -87,43 +92,47 @@ benchmarks/go.csv: perf.go
 	go get github.com/gonum/blas/cgo
 	go get github.com/gonum/matrix/mat64
 	go get github.com/gonum/stat
-	$(foreach t,$(ITERATIONS), go run $<;) >$@
+	@for t in $(ITERATIONS); do go run $<; done >$@
 
 benchmarks/julia.csv: perf.jl
-	$(foreach t,$(ITERATIONS), $(JULIAHOME)/usr/bin/julia $<;) >$@
+	@for t in $(ITERATIONS); do $(JULIAHOME)/usr/bin/julia $<; done >$@
 
 benchmarks/python.csv: perf.py
-	$(foreach t,$(ITERATIONS), $(PYTHON) $<;) >$@
+	@for t in $(ITERATIONS); do $(PYTHON) $<; done >$@
 
 benchmarks/matlab.csv: perf.m
-	$(foreach t,$(ITERATIONS), matlab -nojvm -singleCompThread -r 'perf; perf; exit' | grep ^matlab | tail -8;) >$@
+	@for t in $(ITERATIONS); do matlab -nojvm -singleCompThread -r 'perf; perf; exit' | grep ^matlab | tail -8; done >$@
 
 benchmarks/octave.csv: perf.m
-	$(foreach t,$(ITERATIONS), $(OCTAVE) -q --eval perf 2>/dev/null;) >$@
+	@for t in $(ITERATIONS); do $(OCTAVE) -q --eval perf 2>/dev/null; done >$@
 
 benchmarks/r.csv: perf.R
-	$(foreach t,$(ITERATIONS), cat $< | R --vanilla --slave 2>/dev/null;) >$@
+	@for t in $(ITERATIONS); do cat $< | R --vanilla --slave 2>/dev/null; done >$@
 
 benchmarks/javascript.csv: perf.js
-	$(foreach t,$(ITERATIONS), $(NODEJSBIN) $<;) >$@
+	@for t in $(ITERATIONS); do $(NODEJSBIN) $<; done >$@
 
 benchmarks/mathematica.csv: perf.nb
-	$(foreach t,$(ITERATIONS), $(MATHEMATICABIN) -noprompt -run "<<$<; Exit[]";) >$@
+	@for t in $(ITERATIONS); do $(MATHEMATICABIN) -noprompt -run "<<$<; Exit[]"; done >$@
 
 benchmarks/stata.csv: perf.do
-	$(foreach t,$(ITERATIONS), stata -b do $^ $@;)
+	@for t in $(ITERATIONS); do stata -b do $^ $@; done
 
 benchmarks/lua.csv: perf.lua
-	$(foreach t,$(ITERATIONS), luajit $<;) >$@
+	@for t in $(ITERATIONS); do luajit $<; done >$@
 
 benchmarks/java.csv: java/src/main/java/PerfBLAS.java
-	cd java; sh setup.sh; $(foreach t,$(ITERATIONS), mvn -q exec:java;) >../$@
+	cd java
+	sh setup.sh
+	@for t in $(ITERATIONS); do mvn -q exec:java; done >../$@
 
 benchmarks/scala.csv: scala/src/main/scala/perf.scala scala/build.sbt
-	cd scala; $(foreach t,$(ITERATIONS), sbt run;) >../$@
+	cd scala
+	@for t in $(ITERATIONS); do sbt run; done >../$@
 
 benchmarks/rust.csv: rust/src/main.rs rust/src/util.rs rust/Cargo.lock
-	cd rust; $(foreach t,$(ITERATIONS), cargo run --release -q;) >../$@
+	cd rust
+	@for t in $(ITERATIONS); do cargo run --release -q; done >../$@
 
 LANGUAGES = c fortran go java javascript julia lua mathematica matlab octave python r rust
 GH_ACTION_LANGUAGES = c fortran java javascript julia python r rust

From 10be12bd26403d34f640cd4ab4de450c0283f7f9 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Fri, 18 Feb 2022 23:12:38 -0500
Subject: [PATCH 40/46] Correctly store javascript version

---
 bin/versions.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/versions.sh b/bin/versions.sh
index 12016a9..4d2cc2d 100755
--- a/bin/versions.sh
+++ b/bin/versions.sh
@@ -27,7 +27,7 @@ fi
 
 if [[ $LANGUAGES == *",javascript,"* ]]; then
     echo -n "javascript,V8 "
-    node8 -e "console.log(process.versions.v8)"
+    node -e "console.log(process.versions.v8)"
 fi
 
 if [[ $LANGUAGES == *",julia,"* ]]; then

From e922655016a0ea7a301e9f1e6e497ed55c1561f6 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Sat, 19 Feb 2022 00:46:06 -0500
Subject: [PATCH 41/46] Remove unnecessary apt update

---
 .github/workflows/benchmarks.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index b3a2899..8f6747b 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -33,12 +33,6 @@ jobs:
     
     steps:
       - uses: actions/checkout@v2
-      - name: "Update Ubuntu packages"
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y aptitude
-          sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-          sudo apt-get update
       - name: "Cache Julia"
         id: cache-julia
         uses: actions/cache@v2

From 3ca28e0f440364c28d6fb7f5e02beaa0809737f8 Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Sat, 19 Feb 2022 00:59:49 -0500
Subject: [PATCH 42/46] Switch from comma to colon to avoid ambiguity

---
 Makefile        |  4 +++-
 bin/versions.sh | 32 +++++++++++++++++---------------
 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/Makefile b/Makefile
index 136694d..6f8988d 100644
--- a/Makefile
+++ b/Makefile
@@ -144,11 +144,13 @@ GH_ACTION_LANGUAGES = c fortran java javascript julia python r rust
 BENCHMARKS = $(foreach lang,$(LANGUAGES),benchmarks/$(lang).csv)
 GH_ACTION_BENCHMARKS = $(foreach lang,$(GH_ACTION_LANGUAGES),benchmarks/$(lang).csv)
 
+COLON_SEPARATED_GHA_LANGUAGES = $(shell echo $(GH_ACTION_LANGUAGES) | sed 's/ /:/g')
+
 versions.csv: bin/versions.sh
 	$^ >$@
 
 gh_action_versions.csv: bin/versions.sh
-	bin/versions.sh ",c,fortran,java,javascript,julia,python,r,rust," >$@
+	$^ $(COLON_SEPARATED_GHA_LANGUAGES) >$@
 
 benchmarks.csv: bin/collect.jl $(BENCHMARKS)
 	@$(call PRINT_JULIA, $^ >$@)
diff --git a/bin/versions.sh b/bin/versions.sh
index 4d2cc2d..5964303 100755
--- a/bin/versions.sh
+++ b/bin/versions.sh
@@ -1,72 +1,74 @@
 #!/usr/bin/env bash
 
 # User argument declaring what languages to query:
-DEFAULT_LANGUAGES=",c,fortran,go,java,javascript,julia,lua,mathematica,matlab,octave,python,r,rust,"
+DEFAULT_LANGUAGES="c:fortran:go:java:javascript:julia:lua:mathematica:matlab:octave:python:r:rust"
 LANGUAGES=${1:-DEFAULT_LANGUAGES}
 
-# Check if ",c," in languages:
-if [[ $LANGUAGES == *",c,"* ]]; then
+LANGUAGES=":${LANGUAGES}:"
+
+# Check if ":c:" in languages:
+if [[ $LANGUAGES == *":c:"* ]]; then
     echo -n "c,gcc "
     gcc -v 2>&1 | grep "gcc version" | cut -f3 -d" "
 fi
 
-if [[ $LANGUAGES == *",fortran,"* ]]; then
+if [[ $LANGUAGES == *":fortran:"* ]]; then
     echo -n "fortran,gcc "
     gfortran -v 2>&1 | grep "gcc version" | cut -f3 -d" "
 fi
 
-if [[ $LANGUAGES == *",go,"* ]]; then
+if [[ $LANGUAGES == *":go:"* ]]; then
     echo -n go,
     go version | cut -f3 -d" "
 fi
 
-if [[ $LANGUAGES == *",java,"* ]]; then
+if [[ $LANGUAGES == *":java:"* ]]; then
     echo -n java,
     java -version 2>&1 |grep "version" | cut -f3 -d " " | cut -c 2-9
 fi
 
-if [[ $LANGUAGES == *",javascript,"* ]]; then
+if [[ $LANGUAGES == *":javascript:"* ]]; then
     echo -n "javascript,V8 "
     node -e "console.log(process.versions.v8)"
 fi
 
-if [[ $LANGUAGES == *",julia,"* ]]; then
+if [[ $LANGUAGES == *":julia:"* ]]; then
     echo -n "julia,"
     $JULIAHOME/usr/bin/julia -v | cut -f3 -d" "
 fi
 
-if [[ $LANGUAGES == *",lua,"* ]]; then
+if [[ $LANGUAGES == *":lua:"* ]]; then
     echo -n "lua,"
     # scilua -v 2>&1 | grep Shell | cut -f3 -d" " | cut -f1 -d,
     echo scilua v1.0.0-b12
 fi
 
-if [[ $LANGUAGES == *",mathematica,"* ]]; then
+if [[ $LANGUAGES == *":mathematica:"* ]]; then
     echo -n "mathematica,"
     echo quit | math -version | head -n 1 | cut -f2 -d" "
 fi
 
-if [[ $LANGUAGES == *",matlab,"* ]]; then
+if [[ $LANGUAGES == *":matlab:"* ]]; then
     echo -n "matlab,R"
     matlab -nodisplay -nojvm -nosplash -r "version -release, quit" | tail -n3 | head -n1 | cut -f5 -d" " | sed "s/'//g"
 fi
 
-if [[ $LANGUAGES == *",octave,"* ]]; then
+if [[ $LANGUAGES == *":octave:"* ]]; then
     echo -n "octave,"
     octave-cli -v | grep version | cut -f4 -d" "
 fi
 
-if [[ $LANGUAGES == *",python,"* ]]; then
+if [[ $LANGUAGES == *":python:"* ]]; then
     echo -n "python,"
     python3 -V 2>&1 | cut -f2 -d" "
 fi
 
-if [[ $LANGUAGES == *",r,"* ]]; then
+if [[ $LANGUAGES == *":r:"* ]]; then
     echo -n "r,"
     R --version | grep "R version" | cut -f3 -d" "
 fi
 
-if [[ $LANGUAGES == *",rust,"* ]]; then
+if [[ $LANGUAGES == *":rust:"* ]]; then
     echo -n "rust,"
     (cd rust; rustc --version | cut -c 7- | sed 's/ ([0-9a-f]* /<br>(/g')
 fi

From 99c3b082fb83055d7bcfe2f043232c3ffb5771ab Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Sat, 19 Feb 2022 01:22:21 -0500
Subject: [PATCH 43/46] Get github action to print csv at end

---
 .github/workflows/benchmarks.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 8f6747b..51801f1 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -93,4 +93,6 @@ jobs:
           go-version: ${{ matrix.go-version }}
       - name: "Run benchmark"
         run: |
-          JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.html
\ No newline at end of file
+          JULIAHOME=~/julia DSFMTDIR=~/dSFMT/ make gh_action_benchmarks.html
+      - name: "Print benchmark data"
+        run: cat gh_action_benchmarks.csv

From 44c6b3bc495a7d3cbd4ac24d6093b3040d3d4d2f Mon Sep 17 00:00:00 2001
From: MilesCranmer <miles.cranmer@gmail.com>
Date: Tue, 22 Feb 2022 16:02:39 -0500
Subject: [PATCH 44/46] Add comment about rust version being ignored

---
 .github/workflows/benchmarks.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 51801f1..9bf3bfc 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -25,7 +25,7 @@ jobs:
         python-version: ['3.9']
         numpy-version: ['1.22']
         gfortran-version: ['9']  # Note: unused since is built-in.
-        rust-version: ['1.42.0']
+        rust-version: ['1.42.0']  # Note: unused since controlled by `rust/rust-toolchain`
         js-version: ['16']
         r-version: ['4.1.2']
         lua-version: ['2.0.5']  # Note: Not used as benchmark is broken.

From 5084231156945c9a307c6af48769ed15d1d80537 Mon Sep 17 00:00:00 2001
From: Miles Cranmer <miles.cranmer@gmail.com>
Date: Thu, 10 Mar 2022 16:55:09 -0500
Subject: [PATCH 45/46] Apply suggestions from code review

Co-authored-by: Dilum Aluthge <dilum@aluthge.com>
---
 .github/workflows/benchmarks.yml | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 9bf3bfc..b303f3a 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -1,15 +1,21 @@
 name: Benchmarks
 
 on:
-  push:
-    branches:
-      - '*'
   pull_request:
+  push:
     branches:
-      - '*'
+      - master
+    tags: '*'
   workflow_dispatch:
-    branches:
-      - '*'
+
+concurrency:
+  # Skip intermediate builds: all builds except for builds on the `master` or `release-*` branches
+  # Cancel intermediate builds: only pull request builds
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref != 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-') || github.run_number }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+permissions:
+  contents: read
 
 jobs:
   test:
@@ -33,6 +39,8 @@ jobs:
     
     steps:
       - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
       - name: "Cache Julia"
         id: cache-julia
         uses: actions/cache@v2

From 0ec2b43572b00b2048353f0114d8f5270ecc52f2 Mon Sep 17 00:00:00 2001
From: Miles Cranmer <miles.cranmer@gmail.com>
Date: Thu, 10 Mar 2022 16:59:56 -0500
Subject: [PATCH 46/46] Update .github/workflows/benchmarks.yml
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Mosè Giordano <giordano@users.noreply.github.com>
---
 .github/workflows/benchmarks.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index b303f3a..1fdd088 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -38,7 +38,7 @@ jobs:
         go-version: ['1.17.4']  # Note: Not used as benchmark is broken.
     
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           persist-credentials: false
       - name: "Cache Julia"