Skip to content

Commit

Permalink
Added timing ratio column and replaced ',' with ':' in end-of-line uri
Browse files Browse the repository at this point in the history
  • Loading branch information
PieterOlivier committed Nov 16, 2024
1 parent c861e93 commit e8bed22
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ module lang::rascal::tests::concrete::recovery::ErrorRecoveryBenchmark
import lang::rascal::tests::concrete::recovery::RecoveryTestSupport;

import IO;
import ValueIO;
import util::Benchmark;
import util::SystemAPI;
import String;
import List;

Expand Down Expand Up @@ -35,7 +37,13 @@ void runLanguageTests() {

void runRascalBatchTest(int maxFiles=1000, int minFileSize=0, int maxFileSize=4000, int fromFile=0) {
int startTime = realTime();
TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", maxFiles, minFileSize, maxFileSize, fromFile, |cwd:///rascal-recovery-stats.csv|);

map[str,str] env = getSystemEnvironment();
loc statFile = "STATFILE" in env ? readTextValueString(#loc, env["STATFILE"]) : |cwd:///dev/null|;

println("Writing stats to <statFile>");

TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", maxFiles, minFileSize, maxFileSize, fromFile, statFile);
int duration = realTime() - startTime;
println();
println("================================================================");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public data TestStats = testStats(
FrequencyTable errorCounts=(),
FrequencyTable errorSizes=());

private TestMeasurement testRecovery(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, loc source, loc statFile) {
private TestMeasurement testRecovery(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, loc source, loc statFile, int referenceParseTime) {
int startTime = 0;
int duration = 0;
int disambDuration = -1;
Expand Down Expand Up @@ -94,7 +94,8 @@ private TestMeasurement testRecovery(&T (value input, loc origin) standardParser
}

if (statFile != |unknown:///|) {
appendToFile(statFile, "<source>,<size(input)>,<result>,<duration>,<disambDuration>,<errorCount>,<errorSize>\n");
int ratio = percent(duration, referenceParseTime);
appendToFile(statFile, "<source>,<size(input)>,<result>,<duration>,<ratio>,<disambDuration>,<errorCount>,<errorSize>\n");
}

return measurement;
Expand Down Expand Up @@ -223,7 +224,7 @@ FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, &
while (i < len && (end == -1 || i<=end)) {
str modifiedInput = substring(input, 0, i) + substring(input, i+1);
source.query = "deletedChar=<i>";
TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, source, statFile);
TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, source, statFile, referenceParseTime);
stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit);
if (i < len && substring(input, i, i+1) == "\n") {
println();
Expand Down Expand Up @@ -251,8 +252,8 @@ FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (va
continue;
}
modifiedInput = substring(input, 0, pos) + substring(input, lineEnd);
source.query = "deletedUntilEol=<line>,<pos>,<lineEnd>";
TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, source, statFile);
source.query = "deletedUntilEol=<line>:<pos>:<lineEnd>";
TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, source, statFile, referenceParseTime);
stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit);
}
lineStart = lineEnd+1;
Expand Down Expand Up @@ -433,7 +434,7 @@ TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int m
TestStats runBatchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int maxFiles, int minFileSize, int maxFileSize, loc statFile, TestStats cumulativeStats) {
println("Batch testing in directory <dir> (maxFiles=<maxFiles>, maxFileSize=<maxFileSize>, fromFile=<fromFile>)");
writeFile(statFile, "source,size,result,duration,disambiguationDuration,errorCount,errorSize\n");
writeFile(statFile, "source,size,result,duration,ratio,disambiguationDuration,errorCount,errorSize\n");
for (entry <- listEntries(dir)) {
loc file = dir + entry;
if (isFile(file)) {
Expand Down

0 comments on commit e8bed22

Please sign in to comment.