diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3e22eacc..7648ad62 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -221,10 +221,15 @@ jobs: run: make test_server - - name: Get unit test coverage - id: coverage + - name: Check test metrics + id: test_metrics run: - make coverage_server + make metrics_test_server + + - name: Check coverage metrics + id: coverage_metrics + run: + make metrics_coverage_server - name: Setup Kosli CLI if: ${{ github.ref == 'refs/heads/main' && (success() || failure()) }} @@ -241,16 +246,25 @@ jobs: --name=saver.unit-test --results-dir=./reports/server/junit - - name: Attest coverage evidence to Kosli + - name: Attest test metrics to Kosli + if: ${{ github.ref == 'refs/heads/main' && (success() || failure()) }} + env: + KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }} + run: | + KOSLI_COMPLIANT=$([ "${{ steps.test_metrics.outcome }}" == 'success' ] && echo true || echo false) + kosli attest generic \ + --name=saver.unit-test-metrics \ + --user-data="./reports/server/test_metrics.json" + + - name: Attest coverage metrics to Kosli if: ${{ github.ref == 'refs/heads/main' && (success() || failure()) }} env: KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }} run: | - KOSLI_COMPLIANT=$([ "${{ steps.coverage.outcome }}" == 'success' ] && echo true || echo false) + KOSLI_COMPLIANT=$([ "${{ steps.coverage_metrics.outcome }}" == 'success' ] && echo true || echo false) kosli attest generic \ - --description="unit-test branch-coverage and metrics" \ - --name=saver.unit-test-branch-coverage \ - --user-data="./reports/server/coverage_metrics.json" + --name=saver.unit-test-coverage-metrics \ + --user-data="./reports/server/coverage_metrics.json" integration-tests: @@ -275,10 +289,15 @@ jobs: run: make image_client test_client - - name: Get integration test coverage - id: coverage + - name: Check test metrics + id: test_metrics run: - make coverage_client + make metrics_test_client + + - name: Check coverage metrics + id: coverage_metrics + run: + make metrics_coverage_client - name: Setup Kosli CLI if: ${{ github.ref == 'refs/heads/main' && (success() || failure()) }} @@ -295,15 +314,24 @@ jobs: --name=saver.integration-test --results-dir=./reports/client/junit - - name: Attest coverage evidence to Kosli + - name: Attest test metrics to Kosli + if: ${{ github.ref == 'refs/heads/main' && (success() || failure()) }} + env: + KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }} + run: | + KOSLI_COMPLIANT=$([ "${{ steps.test_metrics.outcome }}" == 'success' ] && echo true || echo false) + kosli attest generic \ + --name=saver.integration-test-metrics \ + --user-data="./reports/client/test_metrics.json" + + - name: Attest coverage metrics to Kosli if: ${{ github.ref == 'refs/heads/main' && (success() || failure()) }} env: KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }} run: | - KOSLI_COMPLIANT=$([ "${{ steps.coverage.outcome }}" == 'success' ] && echo true || echo false) + KOSLI_COMPLIANT=$([ "${{ steps.coverage_metrics.outcome }}" == 'success' ] && echo true || echo false) kosli attest generic \ - --description="integration-test branch-coverage and metrics" \ - --name=saver.integration-test-branch-coverage \ + --name=saver.integration-test-coverage-metrics \ --user-data="./reports/client/coverage_metrics.json" diff --git a/.kosli.yml b/.kosli.yml index 4513771c..4c716632 100644 --- a/.kosli.yml +++ b/.kosli.yml @@ -13,10 +13,14 @@ trail: type: snyk - name: unit-test type: junit - - name: unit-test-branch-coverage + - name: unit-test-metrics + type: generic + - name: unit-test-coverage-metrics type: generic - name: integration-test type: junit - - name: integration-test-branch-coverage + - name: integration-test-metrics + type: generic + - name: integration-coverage-metrics type: generic diff --git a/Makefile b/Makefile index f6e45e79..5ccfccc5 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -all_server: image_server test_server coverage_server +all_server: image_server test_server metrics_test_server metrics_coverage_server image_server: @${PWD}/bin/build_image.sh server @@ -7,11 +7,14 @@ image_server: test_server: @${PWD}/bin/run_tests.sh server -coverage_server: - @${PWD}/bin/check_coverage.sh server +metrics_test_server: + @${PWD}/bin/check_test_metrics.sh server +metrics_coverage_server: + @${PWD}/bin/check_coverage_metrics.sh server -all_client: image_client test_client coverage_client + +all_client: image_client test_client metrics_test_client metrics_coverage_client image_client: @${PWD}/bin/build_image.sh client @@ -19,8 +22,11 @@ image_client: test_client: @${PWD}/bin/run_tests.sh client -coverage_client: - @${PWD}/bin/check_coverage.sh client +metrics_test_client: + @${PWD}/bin/check_test_metrics.sh client + +metrics_coverage_client: + @${PWD}/bin/check_coverage_metrics.sh client rubocop-lint: diff --git a/README.md b/README.md index bd24598d..de8c03ea 100644 --- a/README.md +++ b/README.md @@ -13,21 +13,25 @@ There are two sets of tests: - client: these run from outside the saver container, making api calls only ```bash -# To build the images +# Build the images $ make {image_server|image_client} -# To run all tests +# Run all tests $ make {test_server|test_client} -# To run only specific tests +# Run only specific tests $ ./bin/run_tests.sh {-h|--help} $ ./bin/run_tests.sh server Ks366 -# To check coverage metrics -$ make {coverage_server|coverage_client} +# Check test metrics +$ make {metrics_test_server|metrics_test_client} + +# Check coverage metrics +$ make {metrics_coverage_server|metrics_coverage_client} ``` # API + ## Group * [POST group_create(manifest)](docs/api.md#post-group_createmanifest) @@ -61,5 +65,6 @@ $ make {coverage_server|coverage_client} - [GET ready?](docs/api.md#get-ready) - [GET sha](docs/api.md#get-sha) +## Screenshots ![cyber-dojo.org home page](https://github.com/cyber-dojo/cyber-dojo/blob/master/shared/home_page_snapshot.png) diff --git a/bin/check_coverage.sh b/bin/check_coverage_metrics.sh similarity index 66% rename from bin/check_coverage.sh rename to bin/check_coverage_metrics.sh index 6cc0e7df..98690375 100755 --- a/bin/check_coverage.sh +++ b/bin/check_coverage_metrics.sh @@ -12,7 +12,7 @@ show_help() Use: ${MY_NAME} {server|client} - Check test coverage (and other metrics) for tests run from inside the client or server container only + Check test coverage metrics for tests run from inside the client or server container only EOF } @@ -49,30 +49,26 @@ check_coverage() local -r HOST_REPORTS_DIR="${ROOT_DIR}/reports/${TYPE}" # where report json files have been written to local -r CONTAINER_TMP_DIR=/tmp - exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/${TEST_LOG}" - exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/test_metrics.json" - exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/coverage_metrics.json" - exit_non_zero_unless_file_exists "${HOST_TEST_DIR}/config/check_test_metrics.rb" + exit_non_zero_unless_file_exists "${HOST_TEST_DIR}/config/check_metrics.rb" # evaluator + exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/coverage_metrics.json" # data from test run + exit_non_zero_unless_file_exists "${HOST_TEST_DIR}/config/coverage_metrics_limits.rb" # metric limits set +e docker run \ --read-only \ --rm \ --entrypoint="" \ - --env COVERAGE_ROOT="${CONTAINER_TMP_DIR}" \ - --env COVERAGE_CODE_TAB_NAME=app \ - --env COVERAGE_TEST_TAB_NAME=test \ - --volume ${HOST_REPORTS_DIR}/test_metrics.json:${CONTAINER_TMP_DIR}/test_metrics.json:ro \ + --volume ${HOST_TEST_DIR}/config/check_metrics.rb:${CONTAINER_TMP_DIR}/check_metrics.rb:ro \ --volume ${HOST_REPORTS_DIR}/coverage_metrics.json:${CONTAINER_TMP_DIR}/coverage_metrics.json:ro \ - --volume ${HOST_TEST_DIR}/config/check_test_metrics.rb:${CONTAINER_TMP_DIR}/check_test_metrics.rb:ro \ + --volume ${HOST_TEST_DIR}/config/coverage_metrics_limits.rb:${CONTAINER_TMP_DIR}/coverage_metrics_limits.rb:ro \ "${CYBER_DOJO_SAVER_IMAGE}:${CYBER_DOJO_SAVER_TAG}" \ - sh -c "ruby ${CONTAINER_TMP_DIR}/check_test_metrics.rb" \ + sh -c "ruby ${CONTAINER_TMP_DIR}/check_metrics.rb ${CONTAINER_TMP_DIR}/coverage_metrics.json coverage_metrics_limits" \ | tee -a "${HOST_REPORTS_DIR}/${TEST_LOG}" local -r STATUS=${PIPESTATUS[0]} set -e - echo "${TYPE} coverage status == ${STATUS}" + echo "${TYPE} coverage metrics status == ${STATUS}" echo return "${STATUS}" } diff --git a/bin/check_test_metrics.sh b/bin/check_test_metrics.sh new file mode 100755 index 00000000..f6fb2fd6 --- /dev/null +++ b/bin/check_test_metrics.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -Eeu + +export ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +source "${ROOT_DIR}/bin/lib.sh" + +show_help() +{ + local -r MY_NAME=$(basename "${BASH_SOURCE[0]}") + cat <<- EOF + + Use: ${MY_NAME} {server|client} + + Check test metrics for tests run from inside the client or server container only + +EOF +} + +check_args() +{ + case "${1:-}" in + '-h' | '--help') + show_help + exit 0 + ;; + 'server' | 'client') + ;; + '') + show_help + stderr "no argument - must be 'client' or 'server'" + exit 42 + ;; + *) + show_help + stderr "argument is '${1:-}' - must be 'client' or 'server'" + exit 42 + esac +} + +check_coverage() +{ + check_args "$@" + export $(echo_versioner_env_vars) + + local -r TYPE="${1}" # {server|client} + local -r TEST_LOG=test.log + local -r HOST_TEST_DIR="${ROOT_DIR}/test/${TYPE}" + local -r HOST_REPORTS_DIR="${ROOT_DIR}/reports/${TYPE}" # where report json files have been written to + local -r CONTAINER_TMP_DIR=/tmp + + exit_non_zero_unless_file_exists "${HOST_TEST_DIR}/config/check_metrics.rb" # evaluator + exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/test_metrics.json" # data from test run + exit_non_zero_unless_file_exists "${HOST_TEST_DIR}/config/test_metrics_limits.rb" # metric limits + + set +e + docker run \ + --read-only \ + --rm \ + --entrypoint="" \ + --volume ${HOST_TEST_DIR}/config/check_metrics.rb:${CONTAINER_TMP_DIR}/check_metrics.rb:ro \ + --volume ${HOST_REPORTS_DIR}/test_metrics.json:${CONTAINER_TMP_DIR}/test_metrics.json:ro \ + --volume ${HOST_TEST_DIR}/config/test_metrics_limits.rb:${CONTAINER_TMP_DIR}/test_metrics_limits.rb:ro \ + "${CYBER_DOJO_SAVER_IMAGE}:${CYBER_DOJO_SAVER_TAG}" \ + sh -c "ruby ${CONTAINER_TMP_DIR}/check_metrics.rb ${CONTAINER_TMP_DIR}/test_metrics.json test_metrics_limits" \ + | tee -a "${HOST_REPORTS_DIR}/${TEST_LOG}" + + local -r STATUS=${PIPESTATUS[0]} + set -e + + echo "${TYPE} test metrics status == ${STATUS}" + echo + return "${STATUS}" +} + +check_coverage "$@" diff --git a/bin/run_tests.sh b/bin/run_tests.sh index 072678d8..36bb8aac 100755 --- a/bin/run_tests.sh +++ b/bin/run_tests.sh @@ -61,7 +61,7 @@ run_tests_in_container() set +e docker exec \ - --env COVERAGE_CODE_TAB_NAME=app \ + --env COVERAGE_CODE_TAB_NAME=code \ --env COVERAGE_TEST_TAB_NAME=test \ --user "${USER}" \ "${CONTAINER_NAME}" \ diff --git a/test/client/config/check_metrics.rb b/test/client/config/check_metrics.rb new file mode 100644 index 00000000..f398b7e4 --- /dev/null +++ b/test/client/config/check_metrics.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'json' + +def coloured(arg) + red = 31 + green = 32 + colourize(arg ? green : red, arg) +end + +def colourize(code, word) + "\e[#{code}m #{word} \e[0m" +end + +data = JSON.parse(File.read(ARGV[0])) +require_relative ARGV[1] # metrics + +results = [] +metrics.each do |paths, op, limit| + if paths.nil? + puts + next + end + value = data + paths.split('.').each { |path| value = value[path] } + + # puts "value=#{value}, op=#{op}, limit=#{limit}" # debug + result = eval("#{value} #{op} #{limit}") + puts '%s | %s %s %s | %s' % [ + paths.rjust(35), value.to_s.rjust(5), " #{op}", limit.to_s.rjust(5), coloured(result) + ] + results << result +end +puts +exit results.all? diff --git a/test/client/config/check_test_metrics.rb b/test/client/config/check_test_metrics.rb deleted file mode 100644 index 846b94d8..00000000 --- a/test/client/config/check_test_metrics.rb +++ /dev/null @@ -1,62 +0,0 @@ -# frozen_string_literal: true - -# Uses data from two json files: -# - reports/client/test_metrics.json generated in slim_json_reporter.rb by minitest. See id58_test_base.rb -# - reports/client/coverage_metrics.json generated in simplecov_formatter_json.rb by simplecov. See coverage.rb - -require 'json' - -def coloured(arg) - red = 31 - green = 32 - colourize(arg ? green : red, arg) -end - -def colourize(code, word) - "\e[#{code}m #{word} \e[0m" -end - -def table_data - cov_root = ENV.fetch('COVERAGE_ROOT') - stats = JSON.parse(File.read("#{cov_root}/test_metrics.json")) - - cov_json = JSON.parse(File.read("#{cov_root}/coverage_metrics.json")) - test_cov = cov_json['groups'][ENV.fetch('COVERAGE_TEST_TAB_NAME')] - code_cov = cov_json['groups'][ENV.fetch('COVERAGE_CODE_TAB_NAME')] - - [ - [ nil ], - [ 'test.count', stats['test_count'], '>=', 133 ], - [ 'test.duration', stats['total_time'], '<=', 30 ], - [ nil ], - [ 'test.failures', stats['failure_count'], '<=', 0 ], - [ 'test.errors', stats['error_count' ], '<=', 0 ], - [ 'test.skips', stats['skip_count' ], '<=', 0 ], - [ nil ], - [ 'test.lines.total', test_cov['lines' ]['total' ], '<=', 590 ], - [ 'test.lines.missed', test_cov['lines' ]['missed'], '<=', 0 ], - [ 'test.branches.total', test_cov['branches']['total' ], '<=', 2 ], - [ 'test.branches.missed', test_cov['branches']['missed'], '<=', 0 ], - [ nil ], - [ 'code.lines.total', code_cov['lines' ]['total' ], '<=', 127 ], - [ 'code.lines.missed', code_cov['lines' ]['missed'], '<=', 0 ], - [ 'code.branches.total', code_cov['branches']['total' ], '<=', 2 ], - [ 'code.branches.missed', code_cov['branches']['missed'], '<=', 0 ], - ] -end - -results = [] -table_data.each do |name, value, op, limit| - if name.nil? - puts - next - end - # puts "name=#{name}, value=#{value}, op=#{op}, limit=#{limit}" # debug - result = eval("#{value} #{op} #{limit}") - puts '%s | %s %s %s | %s' % [ - name.rjust(25), value.to_s.rjust(5), " #{op}", limit.to_s.rjust(5), coloured(result) - ] - results << result -end -puts -exit results.all? diff --git a/test/client/config/coverage_metrics_limits.rb b/test/client/config/coverage_metrics_limits.rb new file mode 100644 index 00000000..59893b3e --- /dev/null +++ b/test/client/config/coverage_metrics_limits.rb @@ -0,0 +1,15 @@ + +def metrics + [ + [ nil ], + [ 'test.lines.total' , '<=', 590 ], + [ 'test.lines.missed' , '<=', 0 ], + [ 'test.branches.total' , '<=', 2 ], + [ 'test.branches.missed', '<=', 0 ], + [ nil ], + [ 'code.lines.total' , '<=', 127 ], + [ 'code.lines.missed' , '<=', 0 ], + [ 'code.branches.total' , '<=', 2 ], + [ 'code.branches.missed', '<=', 0 ], + ] +end diff --git a/test/client/config/simplecov_formatter_json.rb b/test/client/config/simplecov_formatter_json.rb index df5800ba..02b0e2dc 100644 --- a/test/client/config/simplecov_formatter_json.rb +++ b/test/client/config/simplecov_formatter_json.rb @@ -5,9 +5,12 @@ class SimpleCov::Formatter::JSONFormatter # based on https://github.com/vicentllongo/simplecov-json def format(result) - groups = {} + data = { + timestamp: result.created_at.to_i, + command_name: result.command_name, + } result.groups.each do |name,file_list| - groups[name] = { + data[name] = { lines: { total: file_list.lines_of_code, covered: file_list.covered_lines, @@ -20,11 +23,6 @@ def format(result) } } end - data = { - timestamp: result.created_at.to_i, - command_name: result.command_name, - groups: groups, - } File.open(output_filepath, "w+") do |file| file.print(JSON.pretty_generate(data)) end diff --git a/test/client/config/test_metrics_limits.rb b/test/client/config/test_metrics_limits.rb new file mode 100644 index 00000000..a61105eb --- /dev/null +++ b/test/client/config/test_metrics_limits.rb @@ -0,0 +1,12 @@ + +def metrics + [ + [ nil ], + [ 'test_count', '>=', 133 ], + [ 'total_time', '<=', 30 ], + [ nil ], + [ 'failure_count', '<=', 0 ], + [ 'error_count' , '<=', 0 ], + [ 'skip_count' , '<=', 0 ], + ] +end diff --git a/test/server/config/check_metrics.rb b/test/server/config/check_metrics.rb new file mode 100644 index 00000000..f398b7e4 --- /dev/null +++ b/test/server/config/check_metrics.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'json' + +def coloured(arg) + red = 31 + green = 32 + colourize(arg ? green : red, arg) +end + +def colourize(code, word) + "\e[#{code}m #{word} \e[0m" +end + +data = JSON.parse(File.read(ARGV[0])) +require_relative ARGV[1] # metrics + +results = [] +metrics.each do |paths, op, limit| + if paths.nil? + puts + next + end + value = data + paths.split('.').each { |path| value = value[path] } + + # puts "value=#{value}, op=#{op}, limit=#{limit}" # debug + result = eval("#{value} #{op} #{limit}") + puts '%s | %s %s %s | %s' % [ + paths.rjust(35), value.to_s.rjust(5), " #{op}", limit.to_s.rjust(5), coloured(result) + ] + results << result +end +puts +exit results.all? diff --git a/test/server/config/check_test_metrics.rb b/test/server/config/check_test_metrics.rb deleted file mode 100644 index b669147b..00000000 --- a/test/server/config/check_test_metrics.rb +++ /dev/null @@ -1,62 +0,0 @@ -# frozen_string_literal: true - -# Uses data from two json files: -# - reports/server/test_metrics.json generated in slim_json_reporter.rb by minitest. See id58_test_base.rb -# - reports/server/coverage_metrics.json generated in simplecov_formatter_json.rb by simplecov. See coverage.rb - -require 'json' - -def coloured(arg) - red = 31 - green = 32 - colourize(arg ? green : red, arg) -end - -def colourize(code, word) - "\e[#{code}m #{word} \e[0m" -end - -def table_data - cov_root = ENV.fetch('COVERAGE_ROOT') - stats = JSON.parse(File.read("#{cov_root}/test_metrics.json")) - - cov_json = JSON.parse(File.read("#{cov_root}/coverage_metrics.json")) - test_cov = cov_json['groups'][ENV.fetch('COVERAGE_TEST_TAB_NAME')] - code_cov = cov_json['groups'][ENV.fetch('COVERAGE_CODE_TAB_NAME')] - - [ - [ nil ], - [ 'test.count', stats['test_count'], '>=', 310 ], - [ 'test.duration', stats['total_time'], '<=', 100 ], - [ nil ], - [ 'test.failures', stats['failure_count'], '<=', 0 ], - [ 'test.errors', stats['error_count' ], '<=', 0 ], - [ 'test.skips', stats['skip_count' ], '<=', 0 ], - [ nil ], - [ 'test.lines.total', test_cov['lines' ]['total' ], '<=', 1781 ], - [ 'test.lines.missed', test_cov['lines' ]['missed'], '<=', 0 ], - [ 'test.branches.total', test_cov['branches']['total' ], '<=', 12 ], - [ 'test.branches.missed', test_cov['branches']['missed'], '<=', 0 ], - [ nil ], - [ 'code.lines.total', code_cov['lines' ]['total' ], '<=', 1249 ], - [ 'code.lines.missed', code_cov['lines' ]['missed'], '<=', 10 ], - [ 'code.branches.total', code_cov['branches']['total' ], '<=', 141 ], - [ 'code.branches.missed', code_cov['branches']['missed'], '<=', 2 ], - ] -end - -results = [] -table_data.each do |name, value, op, limit| - if name.nil? - puts - next - end - # puts "name=#{name}, value=#{value}, op=#{op}, limit=#{limit}" # debug - result = eval("#{value} #{op} #{limit}") - puts '%s | %s %s %s | %s' % [ - name.rjust(25), value.to_s.rjust(5), " #{op}", limit.to_s.rjust(5), coloured(result) - ] - results << result -end -puts -exit results.all? diff --git a/test/server/config/coverage_metrics_limits.rb b/test/server/config/coverage_metrics_limits.rb new file mode 100644 index 00000000..785e3e5a --- /dev/null +++ b/test/server/config/coverage_metrics_limits.rb @@ -0,0 +1,15 @@ + +def metrics + [ + [ nil ], + [ 'test.lines.total' , '<=', 1781 ], + [ 'test.lines.missed' , '<=', 0 ], + [ 'test.branches.total' , '<=', 12 ], + [ 'test.branches.missed', '<=', 0 ], + [ nil ], + [ 'code.lines.total' , '<=', 1249 ], + [ 'code.lines.missed' , '<=', 10 ], + [ 'code.branches.total' , '<=', 141 ], + [ 'code.branches.missed', '<=', 2 ], + ] +end diff --git a/test/server/config/simplecov_formatter_json.rb b/test/server/config/simplecov_formatter_json.rb index df5800ba..02b0e2dc 100644 --- a/test/server/config/simplecov_formatter_json.rb +++ b/test/server/config/simplecov_formatter_json.rb @@ -5,9 +5,12 @@ class SimpleCov::Formatter::JSONFormatter # based on https://github.com/vicentllongo/simplecov-json def format(result) - groups = {} + data = { + timestamp: result.created_at.to_i, + command_name: result.command_name, + } result.groups.each do |name,file_list| - groups[name] = { + data[name] = { lines: { total: file_list.lines_of_code, covered: file_list.covered_lines, @@ -20,11 +23,6 @@ def format(result) } } end - data = { - timestamp: result.created_at.to_i, - command_name: result.command_name, - groups: groups, - } File.open(output_filepath, "w+") do |file| file.print(JSON.pretty_generate(data)) end diff --git a/test/server/config/test_metrics_limits.rb b/test/server/config/test_metrics_limits.rb new file mode 100644 index 00000000..0ac5da68 --- /dev/null +++ b/test/server/config/test_metrics_limits.rb @@ -0,0 +1,12 @@ + +def metrics + [ + [ nil ], + [ 'test_count', '>=', 310 ], + [ 'total_time', '<=', 100 ], + [ nil ], + [ 'failure_count', '<=', 0 ], + [ 'error_count' , '<=', 0 ], + [ 'skip_count' , '<=', 0 ], + ] +end