tests: add new runner (koreader#1989)

.ci/test_script.sh

@@ -16,7 +16,7 @@ if [ "${EMULATE_READER}" = "1" ]; then
         # install test data
         travis_retry make test-data
         # finally make test
-        travis_retry make --assume-old=all test
+        travis_retry make --assume-old=all test T="-o '${PWD}/test-results.xml'"
     ) || exit_code=1
 else
     if [ "${TARGET}" = "android" ] && [ -f build/luajit ]; then

.circleci/config.yml

@@ -128,6 +128,9 @@ jobs:
       ccache_maxsize:
         type: string
         default: "128M"
+      save_tests:
+        type: boolean
+        default: false
     executor: << parameters.executor >>
     resource_class: medium
     environment:
@@ -201,6 +204,15 @@ jobs:
       - run:
           name: Test
           command: source .ci/test_script.sh
+      - when:
+          condition: << parameters.save_tests >>
+          steps:
+              - store_test_results:
+                  name: Store tests results
+                  path: test-results.xml
+              - store_artifacts:
+                  name: Upload tests results to artifacts
+                  path: test-results.xml
       # }}}
 
   # }}}
@@ -229,19 +241,22 @@ workflows:
               only: master
           requires:
             - lint
+          save_tests: true
 
       - build_and_test:
           name: emu_gcc_ninja_debug
           executor: emu_gcc_ninja_debug
           requires:
             - lint
           ccache_maxsize: "512M"
+          save_tests: true
 
       - build_and_test:
           name: emu_clang_ninja
           executor: emu_clang_ninja
           requires:
             - lint
+          save_tests: true
 
       # }}}
 

.github/workflows/build.yml

@@ -69,6 +69,7 @@ jobs:
           packages=(
             autoconf
             automake
+            bash
             binutils
             cmake
             coreutils

Makefile

@@ -105,11 +105,6 @@ $(OUTPUT_DIR)/ffi
 $(OUTPUT_DIR)/fonts/
 $(STAGING_DIR)/
 endef
-ifneq (,$(EMULATE_READER))
-define SKELETON +=
-$(OUTPUT_DIR)/spec/base
-endef
-endif
 
 skeleton: $(strip $(SKELETON))
 
@@ -136,16 +131,36 @@ ifneq (,$(EMULATE_READER))
 
 download-all: test-data
 
-$(OUTPUT_DIR)/.busted: | $(OUTPUT_DIR)/
-	$(SYMLINK) $(KOR_BASE)/.busted $@
-
 $(OUTPUT_DIR)/spec/base: | $(OUTPUT_DIR)/spec/
 	$(SYMLINK) $(KOR_BASE)/spec $@
 
+$(OUTPUT_DIR)/spec/config.lua: | $(OUTPUT_DIR)/spec/
+	$(SYMLINK) $(KOR_BASE)/test-runner/busted_config.lua $@
+
+$(OUTPUT_DIR)/spec/meson.build: | $(OUTPUT_DIR)/spec/
+	$(SYMLINK) $(KOR_BASE)/test-runner/meson.build $@
+
+$(OUTPUT_DIR)/spec/runtests: | $(OUTPUT_DIR)/spec/
+	$(SYMLINK) $(KOR_BASE)/test-runner/runtests $@
+
 $(BASE_PREFIX)test: $(BASE_PREFIX)all test-data
-	cd $(OUTPUT_DIR) && $(BUSTED_LUAJIT) $(BUSTED_OVERRIDES)
+	$(RUNTESTS) $(OUTPUT_DIR) base $T
+
+define test_data_common
+$(OUTPUT_DIR)/spec/config.lua
+$(OUTPUT_DIR)/spec/meson.build
+$(OUTPUT_DIR)/spec/runtests
+endef
+
+skeleton: $(strip $(test_data_common))
+
+define test_data_base
+$(OUTPUT_DIR)/data/tessdata/eng.traineddata
+$(OUTPUT_DIR)/fonts/droid/DroidSansMono.ttf
+$(OUTPUT_DIR)/spec/base
+endef
 
-test-data: $(OUTPUT_DIR)/.busted $(OUTPUT_DIR)/data/tessdata/eng.traineddata $(OUTPUT_DIR)/spec/base $(OUTPUT_DIR)/fonts/droid/DroidSansMono.ttf
+test-data: $(strip $(test_data_common) $(test_data_base))
 
 TESSDATA_FILE = $(THIRDPARTY_DIR)/tesseract/build/downloads/eng.traineddata
 TESSDATA_FILE_URL = https://github.com/tesseract-ocr/tessdata/raw/4.1.0/$(notdir $(TESSDATA_FILE))

Makefile.defs

@@ -1076,21 +1076,18 @@ endif
 
 ifneq (,$(EMULATE_READER))
 
-define busted_fn
-busted() {(
+define runtests_fn
+runtests() {(
+kodir="$$1";
+shift 1;
 export LUA_CPATH='./?.so;$(abspath $(SPEC_ROCKS_DIR)/lib/lua/5.1/?.so)';
 export LUA_PATH='./?.lua;$(abspath $(SPEC_ROCKS_DIR)/share/lua/5.1/?.lua);$(abspath $(SPEC_ROCKS_DIR)/share/lua/5.1/?/init.lua)';
 export TESSDATA_DIR="$$PWD/data";
-./luajit -e 'require "busted.runner" {standalone = false}' /dev/null
---exclude-tags=notest
---helper=ffi/loadlib.lua
---output=gtest
---sort-files
-"$$@";
+"$$kodir/spec/runtests" $(PARALLEL_JOBS:%=-j%) "$$@";
 )}
 endef
 
-BUSTED_LUAJIT = $(strip $(busted_fn)); busted
+RUNTESTS = $(strip $(runtests_fn)); runtests
 
 endif
 

cache-key.base

@@ -14,6 +14,7 @@
 # Excludes.
 !*.lua
 !/ffi-cdecl/**
+!/test-runner/**
 !/utils/bincheck/*
 
 # Exceptions.

test-runner/busted_config.lua

@@ -0,0 +1,23 @@
+local lfs = require("lfs")
+
+local testsuites = {}
+local roots = {}
+local lpaths = {}
+for entry in lfs.dir("spec") do
+    if not string.match(entry, "^[.]") then
+        local testroot = "spec/" .. entry .. "/unit"
+        local testpath = testroot .. "/?.lua"
+        testsuites[entry] = {}
+        testsuites[entry].ROOT = {testroot}
+        testsuites[entry].lpath = testpath
+        table.insert(roots, testroot)
+        table.insert(lpaths, testpath)
+    end
+end
+
+testsuites.all = {
+    ROOT = roots,
+    lpath = table.concat(lpaths, ";"),
+}
+
+return testsuites

test-runner/meson.build

@@ -0,0 +1,68 @@
+project('koreader-testrunner', version: '0.1')
+
+kodir = get_option('prefix')
+testdir = 'spec'
+
+fs = import('fs')
+
+bash = find_program('bash')
+find = find_program('find')
+sort = find_program('sort')
+luajit = find_program(kodir / 'luajit')
+
+_testroot = kodir / testdir
+foreach _testpath : run_command(
+  bash, '-o', 'pipefail', '-c',
+  '"$1" -L "$2" -type f "(" -name "*_bench.lua" -o -name "*_spec.lua" ")" | "$3"',
+  '--', find, _testroot, sort,
+  check: true,
+).stdout().strip('\n').split('\n')
+  assert(_testpath.startswith(_testroot))
+  # Fugly, but no `str.length()`…
+  _testpath = _testpath.replace(_testroot + '/', '')
+  _parts = _testpath.split('/')
+  _suite = _parts[0]
+  assert(_suite != '')
+  _is_bench = _parts[-1].endswith('_bench.lua')
+  if _is_bench
+    _test = _parts[-1].substring(0, -10)
+    # message('benchmark:', _suite, _test)
+  else
+    _test = _parts[-1].substring(0, -9)
+    _id = _suite / _test
+    # message('test:', _suite, _test)
+  endif
+  _args = [
+    '-e', 'require "busted.runner" {standalone = false}', '/dev/null',
+    '--output=gtest', '-Xoutput=--color',
+    '--run=' + _suite,
+    testdir / _testpath,
+  ]
+  _env = environment()
+  _env.set('KO_HOME', meson.current_build_dir() / '_'.join(_suite, _test))
+  # Don't fail the testsuite on ASAN detected leaks.
+  _env.prepend('LSAN_OPTIONS', 'exitcode=0', separator: ' ')
+  if _is_bench
+    benchmark(
+      _test,
+      luajit,
+      args: _args,
+      env: _env,
+      protocol: 'exitcode',
+      suite: _suite,
+      timeout: 0,
+      workdir: fs.parent(luajit.full_path()),
+    )
+  else
+    test(
+      _test,
+      luajit,
+      args: _args,
+      env: _env,
+      protocol: 'exitcode',
+      suite: _suite,
+      timeout: 5 * 60,
+      workdir: fs.parent(luajit.full_path()),
+    )
+  endif
+endforeach