From fdf8fc3476137e77346ed4fd28ca1e7b92aac239 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 26 Sep 2024 19:35:17 -0400 Subject: [PATCH] adding new unit tests --- .github/workflows/run-docker-unit-tests.yml | 117 ++++++++++++++++++++ docker-compose.yml | 21 ++++ run-all-unit-tests.sh | 79 +++++++++++++ run-unit-tests.sh | 37 ++++++- 4 files changed, 249 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/run-docker-unit-tests.yml create mode 100755 run-all-unit-tests.sh diff --git a/.github/workflows/run-docker-unit-tests.yml b/.github/workflows/run-docker-unit-tests.yml new file mode 100644 index 0000000000..e6d28097fc --- /dev/null +++ b/.github/workflows/run-docker-unit-tests.yml @@ -0,0 +1,117 @@ +name: run Perf Test and collect data +env: + default_image: "ghcr.io/meta-introspector/o1js/o1js-perf-recording:latest" + container_name: "unit-tests" +on: + workflow_dispatch: + inputs: + image_url: + description: 'Docker url to execute' + default: "ghcr.io/meta-introspector/o1js/o1js-perf-recording:latest" + push: + branches: + # add your branch here to auto trigger on push + #- "feature/just_test" + #- "feature/rebase" + - "feature/unit-tests" + pull_request: + branches: [ "collect-perf" ] +jobs: + build: + strategy: + matrix: + tests: + - "/app/dist/node/lib/util/base58.unit-test.js" + - "/app/dist/node/lib/ml/consistency.unit-test.js" + - "/app/dist/node/lib/mina/account-update.unit-test.js" + - "/app/dist/node/lib/mina/hash-input.unit-test.js" + - "/app/dist/node/lib/mina/mina.unit-test.js" + - "/app/dist/node/lib/mina/actions/offchain-contract.unit-test.js" + - "/app/dist/node/lib/mina/actions/batch-reducer.unit-test.js" + - "/app/dist/node/lib/mina/actions/batch-reducer-program.unit-test.js" + - "/app/dist/node/lib/mina/test/dynamic-call.unit-test.js" + - "/app/dist/node/lib/mina/token/forest-iterator.unit-test.js" + - "/app/dist/node/lib/mina/token/token-contract.unit-test.js" + - "/app/dist/node/lib/mina/fetch.unit-test.js" + - "/app/dist/node/lib/mina/account-update-layout.unit-test.js" + - "/app/dist/node/lib/proof-system/proof-system.unit-test.js" + - "/app/dist/node/lib/proof-system/sideloaded.unit-test.js" + - "/app/dist/node/lib/provable/test/bitwise.unit-test.js" + - "/app/dist/node/lib/provable/test/base64.unit-test.js" + - "/app/dist/node/lib/provable/test/field.unit-test.js" + - "/app/dist/node/lib/provable/test/nullifier.unit-test.js" + - "/app/dist/node/lib/provable/test/provable.unit-test.js" + - "/app/dist/node/lib/provable/test/sha256.unit-test.js" + - "/app/dist/node/lib/provable/test/string.unit-test.js" + - "/app/dist/node/lib/provable/test/range-check.unit-test.js" + - "/app/dist/node/lib/provable/test/foreign-field.unit-test.js" + - "/app/dist/node/lib/provable/test/group.unit-test.js" + - "/app/dist/node/lib/provable/test/custom-gates-recursion.unit-test.js" + - "/app/dist/node/lib/provable/test/foreign-curve.unit-test.js" + - "/app/dist/node/lib/provable/test/foreign-field-gadgets.unit-test.js" + - "/app/dist/node/lib/provable/test/elliptic-curve.unit-test.js" + - "/app/dist/node/lib/provable/test/lookup.unit-test.js" + - "/app/dist/node/lib/provable/test/ecdsa.unit-test.js" + - "/app/dist/node/lib/provable/test/arithmetic.unit-test.js" + - "/app/dist/node/lib/provable/test/primitives.unit-test.js" + - "/app/dist/node/lib/provable/test/keccak.unit-test.js" + - "/app/dist/node/lib/provable/test/struct.unit-test.js" + - "/app/dist/node/lib/provable/test/merkle-tree.unit-test.js" + - "/app/dist/node/lib/testing/testing.unit-test.js" + - "/app/dist/node/mina-signer/tests/verify-in-snark.unit-test.js" + - "/app/dist/node/mina-signer/tests/zkapp.unit-test.js" + - "/app/dist/node/mina-signer/src/sign-legacy.unit-test.js" + - "/app/dist/node/mina-signer/src/transaction-hash.unit-test.js" + - "/app/dist/node/mina-signer/src/signature.unit-test.js" + - "/app/dist/node/mina-signer/src/sign-zkapp-command.unit-test.js" + - "/app/dist/node/bindings/lib/binable.unit-test.js" + - "/app/dist/node/bindings/crypto/bigint.unit-test.js" + - "/app/dist/node/bindings/crypto/finite-field.unit-test.js" + - "/app/dist/node/bindings/crypto/glv.unit-test.js" + - "/app/dist/node/bindings/crypto/poseidon.unit-test.js" + - "/app/dist/node/bindings/crypto/bindings/bindings.unit-test.js" + - "/app/dist/node/bindings/crypto/elliptic-curve.unit-test.js" + + runs-on: ubuntu-latest + steps: + - name: Sets NAME + env: + name: "${{matrix.tests}}" + run: | + TEST_NAME1=`echo $name | sed -e 's!/!-!g' ` + echo "TEST_NAME=test${TEST_NAME1}" >> $GITHUB_ENV + + - uses: meta-introspector/checkout@v4 + #with: + # submodules: recursive + + - name: Login to GHCR + uses: meta-introspector/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: pull the image + run: docker pull ${{ inputs.image_url || env.default_image }} + + - name: run the Docker tests + run: TESTS=${{matrix.tests}} docker compose up ${{ env.container_name }} + env: + DOCKER_IMAGE_URL: ${{ inputs.image_url || env.default_image}} + + - name: docker cp results + run: docker compose cp ${{ env.container_name }}:/tmp/perf.data.tar.gz perf.data.tar.gz + + - name: Archive results + uses: meta-introspector/upload-artifact@v4 + with: + name: ${{ env.TEST_NAME }}.perf.data.tar.gz + path: perf.data.tar.gz + + - name: delete the results + run: rm -rf /tmp/perf* + + - name: delete the container + run: | + docker compose down diff --git a/docker-compose.yml b/docker-compose.yml index f586cb2de5..207e81e440 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,27 @@ services: command: "bash -x /opt/introspector/test/run-all-tests.sh" + unit-tests: + restart: no + image: ghcr.io/meta-introspector/o1js/o1js-perf-recording:latest + privileged: true + environment: + - PROOF_LEVEL=full + - LOG_LEVEL=Debug + - TESTS=${UNIT_TESTS} + working_dir: /app + # mount the source + volumes: + - type: bind + source: "/home/runner/work/o1js/o1js/" + target: "/opt/introspector/test/" + cap_add: + - SYS_PTRACE + - SYS_ADMIN + # run the test from the mounted source + command: "bash -x /opt/introspector/test/run-all-tests.sh" + + reporting: restart: no image: ghcr.io/meta-introspector/o1js/o1js-perf-reporting:latest diff --git a/run-all-unit-tests.sh b/run-all-unit-tests.sh new file mode 100755 index 0000000000..0f7b076dce --- /dev/null +++ b/run-all-unit-tests.sh @@ -0,0 +1,79 @@ +#!/bin/bash +set +e +set -x +export OUTPUT_DIR="/tmp/perf" +export NODE_OUTPUT_DIR="${OUTPUT_DIR}/node" +export CLINIC_OUTPUT_DIR="${OUTPUT_DIR}/clinic" +export NO_INSIGHT=true +export NODE_OPTIONS="--experimental-vm-modules --perf-basic-prof --enable-source-maps --stack-trace-limit=1000 --report-dir $NODE_OUTPUT_DIR --perf-prof-unwinding-info --perf-prof" +export NO_INSIGHT=true +export JESTOPTS=--collectCoverage +export NODEOPT1="--prof --expose-gc" + + +# setup perf permissions +echo 2 > /proc/sys/kernel/perf_event_paranoid + +mkdir "${OUTPUT_DIR}" +mkdir "${NODE_OUTPUT_DIR}" +mkdir "${CLINIC_OUTPUT_DIR}" +pnpm install -g clinic + +cd /app/ +export SOURCE_DIR=/app/src +# limit to only working tests +#TESTS="${SOURCE_DIR}/lib/provable/test/merkle-list.test.ts ${SOURCE_DIR}/lib/provable/test/merkle-tree.test.ts ${SOURCE_DIR}/lib/provable/test/scalar.test.ts ${SOURCE_DIR}/lib/provable/test/merkle-map.test.ts ${SOURCE_DIR}/lib/provable/test/provable.test.ts ${SOURCE_DIR}/lib/provable/test/primitives.test.ts ${SOURCE_DIR}/lib/provable/test/group.test.ts ${SOURCE_DIR}/lib/provable/test/int.test.ts ${SOURCE_DIR}/lib/mina/precondition.test.ts" +#${SOURCE_DIR}/lib/mina/token.test.ts" +# FIXME this all tests is not used because many of them fail. +ALL_TESTS=`ls -b ${SOURCE_DIR}/lib/provable/test/*.test.ts ${SOURCE_DIR}/lib/mina/*.test.ts ` + + +run_test() { + testname=$1 + export MULTIPLE="${testname}" + export perfdata="${testname}.perf.data" + OUTPUT_DIR2="${OUTPUT_DIR}$testname/" + mkdir -p "${OUTPUT_DIR2}clinic/" + mkdir -p "${OUTPUT_DIR2}log/" + mkdir -p "${OUTPUT_DIR2}coverage/" + + clinic flame -- node $NODEOPT1 ./node_modules/.bin/../jest/bin/jest.js "${JESTOPTS}" "${MULTIPLE}" | tee "${OUTPUT_DIR2}clinic-flame.txt" 2>&1 + clinic doctor -- node $NODEOPT1 ./node_modules/.bin/../jest/bin/jest.js "${JESTOPTS}" "${MULTIPLE}" | tee "${OUTPUT_DIR2}clinic-doctor.txt" 2>&1 + clinic bubbleprof -- node $NODEOPT1 ./node_modules/.bin/../jest/bin/jest.js "${JESTOPTS}" "${MULTIPLE}" | tee "${OUTPUT_DIR2}clinic-bubble.txt" 2>&1 + clinic heapprofiler -- node $NODEOPT1 ./node_modules/.bin/../jest/bin/jest.js "${JESTOPTS}" "${MULTIPLE}" | tee "${OUTPUT_DIR2}clinic-heap.txt" 2>&1 + perf record -g -o "${testname}.perf.data" -F 999 --call-graph dwarf node $NODEOPT1 ./node_modules/.bin/../jest/bin/jest.js "${JESTOPTS}" "${MULTIPLE}" > "${testname}.reportout.txt" 2>&1 + perf archive ${testname}.perf.data + perf report -i "${perfdata}" --verbose --stdio --header > "${perfdata}.header.txt" 2>&1 + perf report -i "${perfdata}" --verbose --stdio --stats > "${perfdata}.stats.txt" 2>&1 + perf script -F +pid -i "${perfdata}" > "${perfdata}.script.txt" 2>&1 + perf report --stdio -i "${perfdata}" > "${perfdata}.report.txt" 2>&1 + ls -latr jit*.dump + rm -r jit*.dump + + mv ${testname}.perf.data "${OUTPUT_DIR2}/perf_data" + mv "${testname}.*.txt" "${OUTPUT_DIR2}/perf_data" + mv .clinic/* "${OUTPUT_DIR2}clinic/" + mv coverage/* "${OUTPUT_DIR2}coverage/" + mv *.log "${OUTPUT_DIR2}log/" + du -s "${OUTPUT_DIR2}" + find "${OUTPUT_DIR2}" + ls -latr "${OUTPUT_DIR2}" +} + +for testname in $TESTS; +do + export MULTIPLE1="${testname}" + export perfdata1="${testname}.perf.data" + export OUTPUT_DIR3="${OUTPUT_DIR}$testname/" + results=$(run_test $testname) + echo $results > "${OUTPUT_DIR3}stdout.txt" + tar -czvf "${OUTPUT_DIR}${testname}.tgz" "${OUTPUT_DIR3}" + # remove the intermediates for space saving. + rm -rf "${OUTPUT_DIR3}/" + ls -latr "${OUTPUT_DIR}" +done + +tar -czf /tmp/perf.data.tar.gz /tmp/perf/* + + + diff --git a/run-unit-tests.sh b/run-unit-tests.sh index 08eaf3e4a3..9af06eac03 100755 --- a/run-unit-tests.sh +++ b/run-unit-tests.sh @@ -2,13 +2,40 @@ set -e shopt -s globstar # to expand '**' into nested directories./ -npm run build +#npm run build + +export NODEOPT1="--prof --expose-gc --enable-source-maps --stack-trace-limit=1000" # find all unit tests in dist/node and run them # TODO it would be nice to make this work on Mac # here is a first attempt which has the problem of not failing if one of the tests fails # find ./dist/node -name "*.unit-test.js" -print -exec node {} \; -for f in ./dist/node/**/*.unit-test.js; do - echo "Running $f" - node --enable-source-maps --stack-trace-limit=1000 $f; -done +#for f in ./dist/node/**/*.unit-test.js; do + # echo "Running $f" +testname=$TESTS +export MULTIPLE="${testname}" +export perfdata="${testname}.perf.data" +OUTPUT_DIR2="${OUTPUT_DIR}$testname/" +mkdir -p "${OUTPUT_DIR2}clinic/" +mkdir -p "${OUTPUT_DIR2}log/" +mkdir -p "${OUTPUT_DIR2}coverage/" + +#node $f; +clinic flame -- node $NODEOPT1 ./node_modules/.bin/../jest/bin/jest.js "${JESTOPTS}" "${MULTIPLE}" | tee "${OUTPUT_DIR2}clinic-flame.txt" 2>&1 +clinic doctor -- node $NODEOPT1 ./node_modules/.bin/../jest/bin/jest.js "${JESTOPTS}" "${MULTIPLE}" | tee "${OUTPUT_DIR2}clinic-doctor.txt" 2>&1 +clinic bubbleprof -- node $NODEOPT1 ./node_modules/.bin/../jest/bin/jest.js "${JESTOPTS}" "${MULTIPLE}" | tee "${OUTPUT_DIR2}clinic-bubble.txt" 2>&1 +clinic heapprofiler -- node $NODEOPT1 ./node_modules/.bin/../jest/bin/jest.js "${JESTOPTS}" "${MULTIPLE}" | tee "${OUTPUT_DIR2}clinic-heap.txt" 2>&1 +perf record -g -o "${testname}.perf.data" -F 999 --call-graph dwarf node $NODEOPT1 ./node_modules/.bin/../jest/bin/jest.js "${JESTOPTS}" "${MULTIPLE}" > "${testname}.reportout.txt" 2>&1 +perf archive ${testname}.perf.data +perf report -i "${perfdata}" --verbose --stdio --header > "${perfdata}.header.txt" 2>&1 +perf report -i "${perfdata}" --verbose --stdio --stats > "${perfdata}.stats.txt" 2>&1 +perf script -F +pid -i "${perfdata}" > "${perfdata}.script.txt" 2>&1 +perf report --stdio -i "${perfdata}" > "${perfdata}.report.txt" 2>&1 +ls -latr jit*.dump +rm -r jit*.dump + +tar -czvf "${OUTPUT_DIR}${testname}.tgz" "${OUTPUT_DIR}/*" +ls -latr "${OUTPUT_DIR}" +#done + +tar -czf /tmp/perf.data.tar.gz /tmp/perf/*