Skip to content

Commit

Permalink
added progress bar monitor to the single threaded test runner
Browse files Browse the repository at this point in the history
  • Loading branch information
@jurgenvinju
jurgenvinju committed Apr 2, 2024
1 parent 0ca2819 commit 300f8ad
Show file tree
Hide file tree
Showing 6 changed files with 146 additions and 117 deletions.
2 changes: 1 addition & 1 deletion src/org/rascalmpl/interpreter/DefaultTestResultListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import io.usethesource.vallang.ISourceLocation;

public class DefaultTestResultListener implements ITestResultListener{
public class DefaultTestResultListener implements ITestResultListener {
private int successes;
private int failures;
private int errors;
Expand Down
100 changes: 53 additions & 47 deletions src/org/rascalmpl/interpreter/TestEvaluator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,61 +87,67 @@ public static int readIntTag(AbstractFunction test, String key, int defaultVal)

private void runTests(ModuleEnvironment env, List<AbstractFunction> tests) {
testResultListener.start(env.getName(), tests.size());
// first, let's shuffle the tests
tests = new ArrayList<>(tests); // just to be sure, clone the list
Collections.shuffle(tests);

QuickCheck qc = new QuickCheck(new Random(), eval.__getVf());
for (AbstractFunction test: tests) {
if (test.hasTag("ignore") || test.hasTag("Ignore") || test.hasTag("ignoreInterpreter") || test.hasTag("IgnoreInterpreter")) {
testResultListener.ignored(test.getName(), test.getAst().getLocation());
continue;
}
eval.job("Testing " + env.getName(), tests.size(), (String jn) -> {
// first, let's shuffle the tests
var theTests = new ArrayList<>(tests); // just to be sure, clone the list
Collections.shuffle(theTests);

QuickCheck qc = new QuickCheck(new Random(), eval.__getVf());
for (AbstractFunction test: theTests) {
eval.jobStep(jn, test.getName());

if (test.hasTag("ignore") || test.hasTag("Ignore") || test.hasTag("ignoreInterpreter") || test.hasTag("IgnoreInterpreter")) {
testResultListener.ignored(test.getName(), test.getAst().getLocation());
continue;
}

try{
int maxDepth = readIntTag(test, QuickCheck.MAXDEPTH, 5);
int maxWidth = readIntTag(test, QuickCheck.MAXWIDTH, 5);
int tries = readIntTag(test, QuickCheck.TRIES, 500);
String expected = null;
if(test.hasTag(QuickCheck.EXPECT_TAG)){
expected = ((IString) test.getTag(QuickCheck.EXPECT_TAG)).getValue();
}
TestResult result = qc.test(test.getEnv().getName() + "::" + test.getName(), test.getFormals(), expected, (Type[] actuals, IValue[] args) -> {
try {
IValue testResult = test.call(actuals, args, null).getValue();
if ((testResult instanceof IBool) && ((IBool)testResult).getValue()) {
return QuickCheck.SUCCESS;
try{
int maxDepth = readIntTag(test, QuickCheck.MAXDEPTH, 5);
int maxWidth = readIntTag(test, QuickCheck.MAXWIDTH, 5);
int tries = readIntTag(test, QuickCheck.TRIES, 500);
String expected = null;
if(test.hasTag(QuickCheck.EXPECT_TAG)){
expected = ((IString) test.getTag(QuickCheck.EXPECT_TAG)).getValue();
}
TestResult result = qc.test(test.getEnv().getName() + "::" + test.getName(), test.getFormals(), expected, (Type[] actuals, IValue[] args) -> {
try {
IValue testResult = test.call(actuals, args, null).getValue();
if ((testResult instanceof IBool) && ((IBool)testResult).getValue()) {
return QuickCheck.SUCCESS;
}
else {
return new TestResult(false, null);
}
}
else {
return new TestResult(false, null);
catch (Throwable e) {
// TODO: add bound type parameters
return new UnExpectedExceptionThrownResult(test.getEnv().getName() + "::" + test.getName(), actuals, Map.of(), args, e);
}
}, env.getRoot().getStore(), tries, maxDepth, maxWidth);

eval.getOutPrinter().flush();
eval.getErrorPrinter().flush();

if (!result.succeeded()) {
StringWriter sw = new StringWriter();
PrintWriter out = new PrintWriter(sw);
result.writeMessage(out);
out.flush();
testResultListener.report(false, test.getName(), test.getAst().getLocation(), sw.getBuffer().toString(), result.thrownException());
} else {
testResultListener.report(true, test.getName(), test.getAst().getLocation(), "test succeeded", null);
}
catch (Throwable e) {
// TODO: add bound type parameters
return new UnExpectedExceptionThrownResult(test.getEnv().getName() + "::" + test.getName(), actuals, Map.of(), args, e);
}
}, env.getRoot().getStore(), tries, maxDepth, maxWidth);
}
catch (Throwable e) {
testResultListener.report(false, test.getName(), test.getAst().getLocation(), e.getMessage(), e);
}

eval.getOutPrinter().flush();
eval.getErrorPrinter().flush();

if (!result.succeeded()) {
StringWriter sw = new StringWriter();
PrintWriter out = new PrintWriter(sw);
result.writeMessage(out);
out.flush();
testResultListener.report(false, test.getName(), test.getAst().getLocation(), sw.getBuffer().toString(), result.thrownException());
} else {
testResultListener.report(true, test.getName(), test.getAst().getLocation(), "test succeeded", null);
}
}
catch (Throwable e) {
testResultListener.report(false, test.getName(), test.getAst().getLocation(), e.getMessage(), e);
}

eval.getOutPrinter().flush();
eval.getErrorPrinter().flush();
}

return true;
});
testResultListener.done();
}
}
1 change: 1 addition & 0 deletions src/org/rascalmpl/library/util/Progress.rsc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
http://www.eclipse.org/legal/epl-v10.html
}
@contributor{Jouke Stoel - [email protected] - CWI}
@deprecated{Use util::Monitor for the same effect with more support for different IDEs and commandline environments.}
module util::Progress

import String;
Expand Down
11 changes: 9 additions & 2 deletions src/org/rascalmpl/library/util/Random.rsc
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
@license{
Copyright (c) 2019 CWI
All rights reserved. This program and the accompanying materials
are made available under the terms of the Eclipse Public License v1.0
which accompanies this distribution, and is available at
http://www.eclipse.org/legal/epl-v10.html
}
@contributor{Davy Landman}
@contributor{Jurgen J. Vinju}
module util::Random


@synopsis{Get a random value of a certain type}
@javaClass{org.rascalmpl.library.Prelude}
java &T randomValue(type[&T] ofType, int depth = 5, int width = 5);


@synopsis{Get a random value of a certain type}
@javaClass{org.rascalmpl.library.Prelude}
java &T randomValue(type[&T] ofType, int seed, int depth = 5, int width = 5);
Expand Down
3 changes: 1 addition & 2 deletions src/org/rascalmpl/library/util/RunTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,11 @@ public class RunTests {

public RunTests(IValueFactory vf) { }


public IList runTests(IString moduleName, IEvaluatorContext ctx) {
return runTests(moduleName.getValue(), (Evaluator) ctx.getEvaluator());
}

// TODO: this has to be rewritten after we finish the compiler, using some Java reflection.
// TODO: this has to be rewritten after we finish the compiler, using some Java reflection.
public static IList runTests(String module, Evaluator eval) {
IValueFactory vf = eval.getValueFactory();
ModuleEnvironment root = new ModuleEnvironment("***testroot***", eval.getHeap());
Expand Down
146 changes: 81 additions & 65 deletions src/org/rascalmpl/test/infrastructure/RascalJUnitTestRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.rascalmpl.interpreter.utils.RascalManifest;
import org.rascalmpl.library.util.PathConfig;
import org.rascalmpl.library.util.PathConfig.RascalConfigMode;
import org.rascalmpl.repl.TerminalProgressBarMonitor;
import org.rascalmpl.shell.ShellEvaluatorFactory;
import org.rascalmpl.uri.URIResolverRegistry;
import org.rascalmpl.uri.URIUtil;
Expand All @@ -46,6 +47,8 @@

import io.usethesource.vallang.ISourceLocation;
import io.usethesource.vallang.IValue;
import jline.Terminal;
import jline.TerminalFactory;

public class RascalJUnitTestRunner extends Runner {
private static Evaluator evaluator;
Expand All @@ -62,7 +65,11 @@ public class RascalJUnitTestRunner extends Runner {
heap = new GlobalEnvironment();
root = heap.addModule(new ModuleEnvironment("___junit_test___", heap));

evaluator = new Evaluator(ValueFactoryFactory.getValueFactory(), System.in, System.err, System.out, root, heap);
Terminal tm = TerminalFactory.get();
TerminalProgressBarMonitor monitor = new TerminalProgressBarMonitor(System.out, tm);

evaluator = new Evaluator(ValueFactoryFactory.getValueFactory(), System.in, System.err, monitor, root, heap);
evaluator.setMonitor(monitor);
evaluator.addRascalSearchPathContributor(StandardLibraryContributor.getInstance());
evaluator.getConfiguration().setErrors(true);
}
Expand Down Expand Up @@ -165,7 +172,7 @@ public static List<String> getRecursiveModuleList(ISourceLocation root, List<Str
ISourceLocation currentDir = todo.poll();

if (!URIResolverRegistry.getInstance().exists(currentDir)) {
System.err.println("[INFO] skipping " + currentDir + ", does not exist.");
evaluator.warning("Skipping " + currentDir + ", does not exist.", currentDir);
continue;
}

Expand All @@ -190,70 +197,73 @@ public static List<String> getRecursiveModuleList(ISourceLocation root, List<Str

}
@Override
public Description getDescription() {
public Description getDescription() {
Description desc = Description.createSuiteDescription(prefix);
this.desc = desc;

try {
List<String> modules = new ArrayList<>(10);
for (String src : new RascalManifest().getSourceRoots(projectRoot)) {
getRecursiveModuleList(URIUtil.getChildLocation(projectRoot, src + "/" + prefix.replaceAll("::", "/")), modules);
}

Collections.shuffle(modules); // make sure the import order is different, not just the reported modules

for (String module : modules) {
String name = prefix + "::" + module;
Description modDesc = Description.createSuiteDescription(name);

try {
System.err.println("Loading module:" + name);
evaluator.doImport(new NullRascalMonitor(), name);
List<AbstractFunction> tests = heap.getModule(name.replaceAll("\\\\","")).getTests();
evaluator.job("Loading test modules", 1, (jobName) -> {
try {
evaluator.jobTodo(jobName, 1);
List<String> modules = new ArrayList<>(10);
for (String src : new RascalManifest().getSourceRoots(projectRoot)) {
getRecursiveModuleList(URIUtil.getChildLocation(projectRoot, src + "/" + prefix.replaceAll("::", "/")), modules);
}

if (tests.isEmpty()) {
System.err.println("\tskipping. Module has no tests.");
continue;
}
Collections.shuffle(modules); // make sure the import order is different, not just the reported modules
evaluator.jobStep(jobName, "detected " + modules.size() + " modules");
evaluator.jobTodo(jobName, modules.size());
for (String module : modules) {
String name = prefix + "::" + module;
Description modDesc = Description.createSuiteDescription(name);
evaluator.jobStep(jobName, "Preparing " + name);

try {
evaluator.doImport(new NullRascalMonitor(), name);
List<AbstractFunction> tests = heap.getModule(name.replaceAll("\\\\","")).getTests();

System.err.println("\t adding " + tests.size() + " tests for " + name);
desc.addChild(modDesc);
if (tests.isEmpty()) {
continue;
}

desc.addChild(modDesc);

// the order of the tests aren't decided by this list so no need to randomly order them.
for (AbstractFunction f : tests) {
modDesc.addChild(Description.createTestDescription(clazz, computeTestName(f.getName(), f.getAst().getLocation())));
// the order of the tests aren't decided by this list so no need to randomly order them.
for (AbstractFunction f : tests) {
modDesc.addChild(Description.createTestDescription(clazz, computeTestName(f.getName(), f.getAst().getLocation())));
}
}
}
catch (Throwable e) {

desc.addChild(modDesc);
catch (Throwable e) {
desc.addChild(modDesc);

Description testDesc = Description.createTestDescription(clazz, name + " compilation failed", new CompilationFailed() {
@Override
public Class<? extends Annotation> annotationType() {
return getClass();
}
});
Description testDesc = Description.createTestDescription(clazz, name + " compilation failed", new CompilationFailed() {
@Override
public Class<? extends Annotation> annotationType() {
return getClass();
}
});

modDesc.addChild(testDesc);
modDesc.addChild(testDesc);
}
}
}

return desc;
} catch (IOException e) {
Description testDesc = Description.createTestDescription(clazz, prefix + " compilation failed: " + e.getMessage(), new CompilationFailed() {
@Override
public Class<? extends Annotation> annotationType() {
return getClass();
}
});
return true;
} catch (IOException e) {
Description testDesc = Description.createTestDescription(clazz, prefix + " compilation failed: " + e.getMessage(), new CompilationFailed() {
@Override
public Class<? extends Annotation> annotationType() {
return getClass();
}
});

desc.addChild(testDesc);
desc.addChild(testDesc);

System.err.println("[ERROR] Could not create tests suite: " + e);

return desc;
}
evaluator.warning("Could not create tests suite: " + e, URIUtil.rootLocation("unknown"));

return false;
}
});

return desc;
}

@Override
Expand All @@ -263,20 +273,26 @@ public void run(final RunNotifier notifier) {
}
notifier.fireTestRunStarted(desc);

for (Description mod : desc.getChildren()) {
// TODO: this will never match because we are on the level of module descriptions now.
// This the reason that modules with errors in them silently succeed with 0 tests run.
if (mod.getAnnotations().stream().anyMatch(t -> t instanceof CompilationFailed)) {
notifier.fireTestFailure(new Failure(desc, new IllegalArgumentException(mod.getDisplayName() + " had importing errors")));
break;
evaluator.job("Running module tests", desc.getChildren().size(), (String jn) -> {
for (Description mod : desc.getChildren()) {
evaluator.jobStep(jn, "Running " + mod.getDisplayName());

// TODO: this will never match because we are on the level of module descriptions now.
// This the reason that modules with errors in them silently succeed with 0 tests run.
if (mod.getAnnotations().stream().anyMatch(t -> t instanceof CompilationFailed)) {
notifier.fireTestFailure(new Failure(desc, new IllegalArgumentException(mod.getDisplayName() + " had importing errors")));
break;
}

// TODO: we lose the link here with the test Descriptors for specific test functions
// and this impacts the accuracy of the reporting (only module name, not test name which failed)
Listener listener = new Listener(notifier, mod);
TestEvaluator runner = new TestEvaluator(evaluator, listener);
runner.test(mod.getDisplayName());
}

// TODO: we lose the link here with the test Descriptors for specific test functions
// and this impacts the accuracy of the reporting (only module name, not test name which failed)
Listener listener = new Listener(notifier, mod);
TestEvaluator runner = new TestEvaluator(evaluator, listener);
runner.test(mod.getDisplayName());
}
return true;
});

notifier.fireTestRunFinished(new Result());
}
Expand Down

0 comments on commit 300f8ad

Please sign in to comment.