From 300f8ad66985ec059c5228c1b335f4037399de49 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 2 Apr 2024 23:50:24 +0200 Subject: [PATCH] added progress bar monitor to the single threaded test runner --- .../DefaultTestResultListener.java | 2 +- .../rascalmpl/interpreter/TestEvaluator.java | 100 ++++++------ src/org/rascalmpl/library/util/Progress.rsc | 1 + src/org/rascalmpl/library/util/Random.rsc | 11 +- src/org/rascalmpl/library/util/RunTests.java | 3 +- .../infrastructure/RascalJUnitTestRunner.java | 146 ++++++++++-------- 6 files changed, 146 insertions(+), 117 deletions(-) diff --git a/src/org/rascalmpl/interpreter/DefaultTestResultListener.java b/src/org/rascalmpl/interpreter/DefaultTestResultListener.java index e1753d20748..f899c7f45b9 100644 --- a/src/org/rascalmpl/interpreter/DefaultTestResultListener.java +++ b/src/org/rascalmpl/interpreter/DefaultTestResultListener.java @@ -20,7 +20,7 @@ import io.usethesource.vallang.ISourceLocation; -public class DefaultTestResultListener implements ITestResultListener{ +public class DefaultTestResultListener implements ITestResultListener { private int successes; private int failures; private int errors; diff --git a/src/org/rascalmpl/interpreter/TestEvaluator.java b/src/org/rascalmpl/interpreter/TestEvaluator.java index bd0fb404449..5fba294fb59 100644 --- a/src/org/rascalmpl/interpreter/TestEvaluator.java +++ b/src/org/rascalmpl/interpreter/TestEvaluator.java @@ -87,61 +87,67 @@ public static int readIntTag(AbstractFunction test, String key, int defaultVal) private void runTests(ModuleEnvironment env, List tests) { testResultListener.start(env.getName(), tests.size()); - // first, let's shuffle the tests - tests = new ArrayList<>(tests); // just to be sure, clone the list - Collections.shuffle(tests); - - QuickCheck qc = new QuickCheck(new Random(), eval.__getVf()); - for (AbstractFunction test: tests) { - if (test.hasTag("ignore") || test.hasTag("Ignore") || test.hasTag("ignoreInterpreter") || test.hasTag("IgnoreInterpreter")) { - testResultListener.ignored(test.getName(), test.getAst().getLocation()); - continue; - } + eval.job("Testing " + env.getName(), tests.size(), (String jn) -> { + // first, let's shuffle the tests + var theTests = new ArrayList<>(tests); // just to be sure, clone the list + Collections.shuffle(theTests); + + QuickCheck qc = new QuickCheck(new Random(), eval.__getVf()); + for (AbstractFunction test: theTests) { + eval.jobStep(jn, test.getName()); + + if (test.hasTag("ignore") || test.hasTag("Ignore") || test.hasTag("ignoreInterpreter") || test.hasTag("IgnoreInterpreter")) { + testResultListener.ignored(test.getName(), test.getAst().getLocation()); + continue; + } - try{ - int maxDepth = readIntTag(test, QuickCheck.MAXDEPTH, 5); - int maxWidth = readIntTag(test, QuickCheck.MAXWIDTH, 5); - int tries = readIntTag(test, QuickCheck.TRIES, 500); - String expected = null; - if(test.hasTag(QuickCheck.EXPECT_TAG)){ - expected = ((IString) test.getTag(QuickCheck.EXPECT_TAG)).getValue(); - } - TestResult result = qc.test(test.getEnv().getName() + "::" + test.getName(), test.getFormals(), expected, (Type[] actuals, IValue[] args) -> { - try { - IValue testResult = test.call(actuals, args, null).getValue(); - if ((testResult instanceof IBool) && ((IBool)testResult).getValue()) { - return QuickCheck.SUCCESS; + try{ + int maxDepth = readIntTag(test, QuickCheck.MAXDEPTH, 5); + int maxWidth = readIntTag(test, QuickCheck.MAXWIDTH, 5); + int tries = readIntTag(test, QuickCheck.TRIES, 500); + String expected = null; + if(test.hasTag(QuickCheck.EXPECT_TAG)){ + expected = ((IString) test.getTag(QuickCheck.EXPECT_TAG)).getValue(); + } + TestResult result = qc.test(test.getEnv().getName() + "::" + test.getName(), test.getFormals(), expected, (Type[] actuals, IValue[] args) -> { + try { + IValue testResult = test.call(actuals, args, null).getValue(); + if ((testResult instanceof IBool) && ((IBool)testResult).getValue()) { + return QuickCheck.SUCCESS; + } + else { + return new TestResult(false, null); + } } - else { - return new TestResult(false, null); + catch (Throwable e) { + // TODO: add bound type parameters + return new UnExpectedExceptionThrownResult(test.getEnv().getName() + "::" + test.getName(), actuals, Map.of(), args, e); } + }, env.getRoot().getStore(), tries, maxDepth, maxWidth); + + eval.getOutPrinter().flush(); + eval.getErrorPrinter().flush(); + + if (!result.succeeded()) { + StringWriter sw = new StringWriter(); + PrintWriter out = new PrintWriter(sw); + result.writeMessage(out); + out.flush(); + testResultListener.report(false, test.getName(), test.getAst().getLocation(), sw.getBuffer().toString(), result.thrownException()); + } else { + testResultListener.report(true, test.getName(), test.getAst().getLocation(), "test succeeded", null); } - catch (Throwable e) { - // TODO: add bound type parameters - return new UnExpectedExceptionThrownResult(test.getEnv().getName() + "::" + test.getName(), actuals, Map.of(), args, e); - } - }, env.getRoot().getStore(), tries, maxDepth, maxWidth); + } + catch (Throwable e) { + testResultListener.report(false, test.getName(), test.getAst().getLocation(), e.getMessage(), e); + } eval.getOutPrinter().flush(); eval.getErrorPrinter().flush(); - - if (!result.succeeded()) { - StringWriter sw = new StringWriter(); - PrintWriter out = new PrintWriter(sw); - result.writeMessage(out); - out.flush(); - testResultListener.report(false, test.getName(), test.getAst().getLocation(), sw.getBuffer().toString(), result.thrownException()); - } else { - testResultListener.report(true, test.getName(), test.getAst().getLocation(), "test succeeded", null); - } - } - catch (Throwable e) { - testResultListener.report(false, test.getName(), test.getAst().getLocation(), e.getMessage(), e); } - - eval.getOutPrinter().flush(); - eval.getErrorPrinter().flush(); - } + + return true; + }); testResultListener.done(); } } diff --git a/src/org/rascalmpl/library/util/Progress.rsc b/src/org/rascalmpl/library/util/Progress.rsc index 362134c1150..0ddf88701d8 100644 --- a/src/org/rascalmpl/library/util/Progress.rsc +++ b/src/org/rascalmpl/library/util/Progress.rsc @@ -6,6 +6,7 @@ http://www.eclipse.org/legal/epl-v10.html } @contributor{Jouke Stoel - jouke.stoel@cwi.nl - CWI} +@deprecated{Use util::Monitor for the same effect with more support for different IDEs and commandline environments.} module util::Progress import String; diff --git a/src/org/rascalmpl/library/util/Random.rsc b/src/org/rascalmpl/library/util/Random.rsc index e9e70b375e8..254b72472f6 100644 --- a/src/org/rascalmpl/library/util/Random.rsc +++ b/src/org/rascalmpl/library/util/Random.rsc @@ -1,11 +1,18 @@ +@license{ + Copyright (c) 2019 CWI + All rights reserved. This program and the accompanying materials + are made available under the terms of the Eclipse Public License v1.0 + which accompanies this distribution, and is available at + http://www.eclipse.org/legal/epl-v10.html +} +@contributor{Davy Landman} +@contributor{Jurgen J. Vinju} module util::Random - @synopsis{Get a random value of a certain type} @javaClass{org.rascalmpl.library.Prelude} java &T randomValue(type[&T] ofType, int depth = 5, int width = 5); - @synopsis{Get a random value of a certain type} @javaClass{org.rascalmpl.library.Prelude} java &T randomValue(type[&T] ofType, int seed, int depth = 5, int width = 5); diff --git a/src/org/rascalmpl/library/util/RunTests.java b/src/org/rascalmpl/library/util/RunTests.java index 2646545d377..4ad5ba81bca 100644 --- a/src/org/rascalmpl/library/util/RunTests.java +++ b/src/org/rascalmpl/library/util/RunTests.java @@ -43,12 +43,11 @@ public class RunTests { public RunTests(IValueFactory vf) { } - public IList runTests(IString moduleName, IEvaluatorContext ctx) { return runTests(moduleName.getValue(), (Evaluator) ctx.getEvaluator()); } - // TODO: this has to be rewritten after we finish the compiler, using some Java reflection. + // TODO: this has to be rewritten after we finish the compiler, using some Java reflection. public static IList runTests(String module, Evaluator eval) { IValueFactory vf = eval.getValueFactory(); ModuleEnvironment root = new ModuleEnvironment("***testroot***", eval.getHeap()); diff --git a/src/org/rascalmpl/test/infrastructure/RascalJUnitTestRunner.java b/src/org/rascalmpl/test/infrastructure/RascalJUnitTestRunner.java index 43acea0b68b..6bc3ca4049e 100644 --- a/src/org/rascalmpl/test/infrastructure/RascalJUnitTestRunner.java +++ b/src/org/rascalmpl/test/infrastructure/RascalJUnitTestRunner.java @@ -36,6 +36,7 @@ import org.rascalmpl.interpreter.utils.RascalManifest; import org.rascalmpl.library.util.PathConfig; import org.rascalmpl.library.util.PathConfig.RascalConfigMode; +import org.rascalmpl.repl.TerminalProgressBarMonitor; import org.rascalmpl.shell.ShellEvaluatorFactory; import org.rascalmpl.uri.URIResolverRegistry; import org.rascalmpl.uri.URIUtil; @@ -46,6 +47,8 @@ import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IValue; +import jline.Terminal; +import jline.TerminalFactory; public class RascalJUnitTestRunner extends Runner { private static Evaluator evaluator; @@ -62,7 +65,11 @@ public class RascalJUnitTestRunner extends Runner { heap = new GlobalEnvironment(); root = heap.addModule(new ModuleEnvironment("___junit_test___", heap)); - evaluator = new Evaluator(ValueFactoryFactory.getValueFactory(), System.in, System.err, System.out, root, heap); + Terminal tm = TerminalFactory.get(); + TerminalProgressBarMonitor monitor = new TerminalProgressBarMonitor(System.out, tm); + + evaluator = new Evaluator(ValueFactoryFactory.getValueFactory(), System.in, System.err, monitor, root, heap); + evaluator.setMonitor(monitor); evaluator.addRascalSearchPathContributor(StandardLibraryContributor.getInstance()); evaluator.getConfiguration().setErrors(true); } @@ -165,7 +172,7 @@ public static List getRecursiveModuleList(ISourceLocation root, List getRecursiveModuleList(ISourceLocation root, List modules = new ArrayList<>(10); - for (String src : new RascalManifest().getSourceRoots(projectRoot)) { - getRecursiveModuleList(URIUtil.getChildLocation(projectRoot, src + "/" + prefix.replaceAll("::", "/")), modules); - } - - Collections.shuffle(modules); // make sure the import order is different, not just the reported modules - - for (String module : modules) { - String name = prefix + "::" + module; - Description modDesc = Description.createSuiteDescription(name); - - try { - System.err.println("Loading module:" + name); - evaluator.doImport(new NullRascalMonitor(), name); - List tests = heap.getModule(name.replaceAll("\\\\","")).getTests(); + evaluator.job("Loading test modules", 1, (jobName) -> { + try { + evaluator.jobTodo(jobName, 1); + List modules = new ArrayList<>(10); + for (String src : new RascalManifest().getSourceRoots(projectRoot)) { + getRecursiveModuleList(URIUtil.getChildLocation(projectRoot, src + "/" + prefix.replaceAll("::", "/")), modules); + } - if (tests.isEmpty()) { - System.err.println("\tskipping. Module has no tests."); - continue; - } + Collections.shuffle(modules); // make sure the import order is different, not just the reported modules + evaluator.jobStep(jobName, "detected " + modules.size() + " modules"); + evaluator.jobTodo(jobName, modules.size()); + for (String module : modules) { + String name = prefix + "::" + module; + Description modDesc = Description.createSuiteDescription(name); + evaluator.jobStep(jobName, "Preparing " + name); + + try { + evaluator.doImport(new NullRascalMonitor(), name); + List tests = heap.getModule(name.replaceAll("\\\\","")).getTests(); - System.err.println("\t adding " + tests.size() + " tests for " + name); - desc.addChild(modDesc); + if (tests.isEmpty()) { + continue; + } + + desc.addChild(modDesc); - // the order of the tests aren't decided by this list so no need to randomly order them. - for (AbstractFunction f : tests) { - modDesc.addChild(Description.createTestDescription(clazz, computeTestName(f.getName(), f.getAst().getLocation()))); + // the order of the tests aren't decided by this list so no need to randomly order them. + for (AbstractFunction f : tests) { + modDesc.addChild(Description.createTestDescription(clazz, computeTestName(f.getName(), f.getAst().getLocation()))); + } } - } - catch (Throwable e) { - - desc.addChild(modDesc); + catch (Throwable e) { + desc.addChild(modDesc); - Description testDesc = Description.createTestDescription(clazz, name + " compilation failed", new CompilationFailed() { - @Override - public Class annotationType() { - return getClass(); - } - }); + Description testDesc = Description.createTestDescription(clazz, name + " compilation failed", new CompilationFailed() { + @Override + public Class annotationType() { + return getClass(); + } + }); - modDesc.addChild(testDesc); + modDesc.addChild(testDesc); + } } - } - return desc; - } catch (IOException e) { - Description testDesc = Description.createTestDescription(clazz, prefix + " compilation failed: " + e.getMessage(), new CompilationFailed() { - @Override - public Class annotationType() { - return getClass(); - } - }); + return true; + } catch (IOException e) { + Description testDesc = Description.createTestDescription(clazz, prefix + " compilation failed: " + e.getMessage(), new CompilationFailed() { + @Override + public Class annotationType() { + return getClass(); + } + }); - desc.addChild(testDesc); + desc.addChild(testDesc); - System.err.println("[ERROR] Could not create tests suite: " + e); - - return desc; - } + evaluator.warning("Could not create tests suite: " + e, URIUtil.rootLocation("unknown")); + + return false; + } + }); + + return desc; } @Override @@ -263,20 +273,26 @@ public void run(final RunNotifier notifier) { } notifier.fireTestRunStarted(desc); - for (Description mod : desc.getChildren()) { - // TODO: this will never match because we are on the level of module descriptions now. - // This the reason that modules with errors in them silently succeed with 0 tests run. - if (mod.getAnnotations().stream().anyMatch(t -> t instanceof CompilationFailed)) { - notifier.fireTestFailure(new Failure(desc, new IllegalArgumentException(mod.getDisplayName() + " had importing errors"))); - break; + evaluator.job("Running module tests", desc.getChildren().size(), (String jn) -> { + for (Description mod : desc.getChildren()) { + evaluator.jobStep(jn, "Running " + mod.getDisplayName()); + + // TODO: this will never match because we are on the level of module descriptions now. + // This the reason that modules with errors in them silently succeed with 0 tests run. + if (mod.getAnnotations().stream().anyMatch(t -> t instanceof CompilationFailed)) { + notifier.fireTestFailure(new Failure(desc, new IllegalArgumentException(mod.getDisplayName() + " had importing errors"))); + break; + } + + // TODO: we lose the link here with the test Descriptors for specific test functions + // and this impacts the accuracy of the reporting (only module name, not test name which failed) + Listener listener = new Listener(notifier, mod); + TestEvaluator runner = new TestEvaluator(evaluator, listener); + runner.test(mod.getDisplayName()); } - // TODO: we lose the link here with the test Descriptors for specific test functions - // and this impacts the accuracy of the reporting (only module name, not test name which failed) - Listener listener = new Listener(notifier, mod); - TestEvaluator runner = new TestEvaluator(evaluator, listener); - runner.test(mod.getDisplayName()); - } + return true; + }); notifier.fireTestRunFinished(new Result()); }