src/hevm/run-blockchain-tests

#!/usr/bin/env bash
set -e

# Invoke with hevm e.g.
# hevm compliance --tests ~/ethereum-tests --skip modexp --timeout 20 --html

HEVM=${HEVM:-hevm}

if [[ "$#" -lt 1 ]]; then
  echo >&2 "usage: $(basename "$0") <tests-dir>"
  exit 1
fi

tests=$1
html=$2
match=$3
skip=$4
timeout=${5:-10}

_html () {
cat <<.
<!doctype html>
<title>hevm test results</title>
<style>
* { font-family:
"latin modern mono", "fantasque sans mono",
inconsolata, menlo, monospace;
font-size: 22px;
line-height: 26px; }
body { margin: 2rem; }
header { text-align: center; margin: 4rem 0; }
table { border-collapse: collapse; width: 100%; }
tr:nth-child(even) { background: rgba(0, 0, 0, 0.05); }
td:not(:first-child):not(:last-child) { padding: 0 1rem; }
.category { opacity: 0.6; text-align: right }
a { color: darkblue; text-decoration: none; }
h1, h2 { text-align: center; margin-top: 2rem  }
.testcase { font-weight: bold }
#failed .testcase { color: rgb(200, 0, 0) }
#balancefailed .testcase { color: rgb(200, 200, 0) }
#passed .testcase { color: rgb(0, 150, 0) }
#skipped .testcase { color: rgb(0, 100, 250) }
#timeout .testcase { color: rgb(0, 0, 250) }
</style>
<header>
<h1>hevm consensus test report</h1>
<p>$(date +%Y-%m-%d)</p>
<p>$(echo "$npass passed, $nbal bad-balance, $nnon bad-nonce, $nstr bad-storage, $nfail failed, $nskip skipped, $ntime timeout")</p>
(Test suite: <span class=GeneralStateTests</span>GeneralStateTests</span> for Berlin)
</header>
<h2>Failed tests</h2>
<table id=failed>
<tbody>
$(echo $noncefailed)
$(echo $storagefailed)
$(echo $failed)
</table>
<h2>Failed tests (due to balance only)</h2>
<table id=balancefailed>
<tbody>
$(echo $balancefailed)
</table>
<h2>Timeout tests</h2>
<table id=timeout>
<tbody>
$(echo $timeouts)
</table>
<h2>Skipped tests</h2>
<table id=skipped>
<tbody>
$(echo $skipped)
</table>
<h2>Passed tests</h2>
<table id=passed>
<tbody>
$(echo $passed)
</table>
.
}

shopt -s nocasematch
{
  cd "$tests"
  for x in BlockchainTests/GeneralStateTests/*/*; do
    if [ -d $x ]; then
      for y in $x/*; do
        if [[ $y =~ .*$match.* ]] && [[ -n $skip && $y =~ .*$skip.* ]]; then
          for job in $(<$y jq '.|keys[]' -r); do
            echo -n "$job " ; echo "skip"
          done
        elif [[ $y =~ .*$match.* ]]; then
          set +e
          "$HEVM" bc-test --file $y --timeout $timeout 2>&1
          set -e
        fi
      done
    else
      if [[ $x =~ .*$match.* ]] && [[ -n $skip && $x =~ .*$skip.* ]]; then
        for job in $(<$x jq '.|keys[]' -r); do
          echo -n "$job " ; echo "skip"
        done
      elif [[ $x =~ .*$match.* ]]; then
        set +e
        "$HEVM" bc-test --file $x --timeout $timeout 2>&1
        set -e
      fi
    fi
  done
} | {
  while read test outcome; do
    echo >&2 "$test $outcome"
    row="<tr><td class=testcase>$test<td>$outcome"
    row+=$'\n'
    case $outcome in
      ok)          passed+=$row ;;
      bad-balance) balancefailed+=$row ;;
      bad-nonce)   noncefailed+=$row ;;
      bad-storage) storagefailed+=$row ;;
      timeout)     timeouts+=$row ;;
      skip)        skipped+=$row ;;
      *)           failed+=$row ;;
    esac
  done

  sum () { echo -ne "$1" | wc -l | awk '{print $1}'; }

  npass=$(sum "$passed")
  nbal=$(sum "$balancefailed")
  nnon=$(sum "$noncefailed")
  nstr=$(sum "$storagefailed")
  nfail=$(sum "$failed")
  ntime=$(sum "$timeouts")
  nskip=$(sum "$skipped")

  echo >&2 "passed: $npass"
  echo >&2 "bad-balance: $nbal"
  echo >&2 "bad-nonce: $nnon"
  echo >&2 "bad-storage: $nstr"
  echo >&2 "failed: $nfail"
  echo >&2 "timeout: $ntime"
  echo >&2 "skipped: $nskip"

  if [[ $html == "True" ]]; then
    _html
  fi

  nbad=$(($nbal + $nnon + $nstr + $nfail))

  [[ $nbad -gt 0 ]] && exit 1 || exit 0
}