From 161689827f9dd7fc0a15ca520308d6b6eef28606 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Fri, 18 Oct 2024 07:57:55 +0200 Subject: [PATCH 01/12] Started working on multi-error support Experimented with max skip length --- .../concrete/recovery/bugs/MultiErrorBug.rsc | 33 +++++++++++++++++++ .../recovery/bugs/MultiErrorBugInput.txt | 17 ++++++++++ .../concrete/recovery/bugs/MultiErrorPico.rsc | 32 ++++++++++++++++++ .../recovery/bugs/MultiErrorPicoInput.pico | 1 + .../CaseInsensitiveLiteralMatcher.java | 5 +-- .../parser/uptr/recovery/FailingMatcher.java | 2 +- .../parser/uptr/recovery/InputMatcher.java | 2 +- .../parser/uptr/recovery/LiteralMatcher.java | 5 +-- .../uptr/recovery/ToTokenRecoverer.java | 8 ++--- 9 files changed, 95 insertions(+), 10 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBug.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBugInput.txt create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPico.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPicoInput.pico diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBug.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBug.rsc new file mode 100644 index 00000000000..6f95984a751 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBug.rsc @@ -0,0 +1,33 @@ +module lang::rascal::tests::concrete::recovery::bugs::MultiErrorBug + + +import lang::rascal::\syntax::Rascal; +import ParseTree; +import IO; +import util::ErrorRecovery; +import List; +import vis::Text; + +bool multiErrorBug() { + str input = readFile(|std:///lang/rascal/tests/concrete/recovery/bugs/MultiErrorBugInput.txt|); + Tree t = parser(#start[Module], allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=false|); + + println(prettyTree(t)); + + list[Tree] errors = findAllErrors(t); + println(" Errors"); + for (Tree error <- errors) { + Tree skipped = getSkipped(error); + println(" : "); + } + Tree disambiguated = disambiguateErrors(t); + list[Tree] disambiguatedErrors = findAllErrors(disambiguated); + println("After disambiguating:"); + println(" Errors"); + for (Tree error <- disambiguatedErrors) { + Tree skipped = getSkipped(error); + println(" : "); + } + return true; +} + diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBugInput.txt b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBugInput.txt new file mode 100644 index 00000000000..7d6c255752f --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBugInput.txt @@ -0,0 +1,17 @@ +module sandbox::SimpleSyntax + +start syntax Start = S; + +syntax S = A | B | C | D | E | F; + +syntax A = "a"; + +syntax B = #"b"; + +syntax C = "c"; + +syntax D = #"d"; + +syntax E = "e"; + +syntax F = "f"; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPico.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPico.rsc new file mode 100644 index 00000000000..bcdbe125b42 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPico.rsc @@ -0,0 +1,32 @@ +module lang::rascal::tests::concrete::recovery::bugs::MultiErrorPico + +import lang::pico::\syntax::Main; +import ParseTree; +import IO; +import util::ErrorRecovery; +import List; +import vis::Text; + +bool multiErrorPico() { + str input = readFile(|std:///lang/rascal/tests/concrete/recovery/bugs/MultiErrorPicoInput.pico|); + Tree t = parser(#start[Program], allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=true|); + + println(prettyTree(t)); + + list[Tree] errors = findAllErrors(t); + println(" Errors"); + for (Tree error <- errors) { + Tree skipped = getSkipped(error); + println(" : "); + } + Tree disambiguated = disambiguateErrors(t); + list[Tree] disambiguatedErrors = findAllErrors(disambiguated); + println("After disambiguating:"); + println(" Errors"); + for (Tree error <- disambiguatedErrors) { + Tree skipped = getSkipped(error); + println(" : "); + } + return true; +} + diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPicoInput.pico b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPicoInput.pico new file mode 100644 index 00000000000..d94083a5603 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPicoInput.pico @@ -0,0 +1 @@ +begindeclare;i:=#1;j:=#2;k:=3end \ No newline at end of file diff --git a/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java index f1dda955e94..406f06ca140 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java @@ -46,10 +46,11 @@ public CaseInsensitiveLiteralMatcher(int[][] ciLiteral) { } @Override - public MatchResult findMatch(int[] input, int startLocation) { + public MatchResult findMatch(int[] input, int startLocation, int maxLength) { int length = chars.length; - for (int start=startLocation; start < input.length - length; start++) { + int limit = input.length - length + 1; + for (int start=startLocation; start < limit; start++) { boolean matches = true; for (int i=0; i, AbstractNode> reviveStac // For now we ignore unmatchable leaf nodes and filtered nodes. At some point we might use those to // improve error recovery. - + ArrayList> failedNodes = new ArrayList<>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); @@ -154,7 +154,7 @@ private List> findSkippingNodes(int[] input, int // Find the last token of this production and skip until after that List endMatchers = findEndMatchers(recoveryNode); for (InputMatcher endMatcher : endMatchers) { - MatchResult endMatch = endMatcher.findMatch(input, startLocation); + MatchResult endMatch = endMatcher.findMatch(input, startLocation, 5); if (endMatch != null) { result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, endMatch.getEnd()); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); @@ -164,7 +164,7 @@ private List> findSkippingNodes(int[] input, int // Find the first token of the next production and skip until before that List nextMatchers = findNextMatchers(recoveryNode); for (InputMatcher nextMatcher : nextMatchers) { - MatchResult nextMatch = nextMatcher.findMatch(input, startLocation+1); + MatchResult nextMatch = nextMatcher.findMatch(input, startLocation+1, 5); if (nextMatch != null) { result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, nextMatch.getStart()); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); @@ -396,7 +396,7 @@ private static void collectUnmatchableMidProductionNodes(int location, AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference // updates of the static version - + // Merge the information on the predecessors into the failed node. for(int j = failedNodePredecessors.size() - 1; j >= 0; --j) { AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); From 93c1277c2e8226aec70ef11e998f1625ac4f0cc8 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Fri, 18 Oct 2024 07:58:53 +0200 Subject: [PATCH 02/12] Fixed formatting of main parse function --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index fea89a68099..695486f9043 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -1347,12 +1347,10 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] initTime(); try { - if(invoked){ throw new RuntimeException("Can only invoke 'parse' once."); } - invoked = true; // Initialize. @@ -1395,6 +1393,7 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] debugListener.shifting(location, input, positionStore); } + // Reduce-expand loop. do { debugListener.iterating(); @@ -1410,10 +1409,10 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] shiftedLevel = true; if (!findStacksToReduce()) { - if(location == input.length){ + if(location == input.length) { EdgesSet

startNodeEdgesSet = startNode.getIncomingEdges(); int resultStoreId = getResultStoreId(startNode.getId()); - if(startNodeEdgesSet != null && startNodeEdgesSet.getLastVisitedLevel(resultStoreId) == input.length){ + if(startNodeEdgesSet != null && startNodeEdgesSet.getLastVisitedLevel(resultStoreId) == input.length) { result = startNodeEdgesSet.getLastResult(resultStoreId); // Success. break; } From 9df7f4af2b4d8d12c195f43fb64a9ea4b9b2c5ff Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Sat, 19 Oct 2024 10:42:21 +0200 Subject: [PATCH 03/12] Trigger error recovery when only recovery stacks are left --- .../concrete/recovery/RecoveryTestSupport.rsc | 2 +- .../recovery/bugs/InfiniteLoopMultiError.rsc | 15 +++++++++++ .../recovery/bugs/MinimalMultiError.rsc | 17 ++++++++++++ src/org/rascalmpl/parser/gtd/SGTDBF.java | 27 +++++++++++++++++++ .../uptr/recovery/ToTokenRecoverer.java | 4 +-- 5 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopMultiError.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MinimalMultiError.rsc diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 847345d35a1..49cbc602f5f 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -433,7 +433,7 @@ TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int m TestStats runBatchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int maxFiles, int minFileSize, int maxFileSize, loc statFile, TestStats cumulativeStats) { println("Batch testing in directory

(maxFiles=, maxFileSize=, fromFile=)"); - writeFile(statFile, "source,size,result,duration,disambiguationDuration,errorSize\n"); + writeFile(statFile, "source,size,result,duration,disambiguationDuration,errorCount,errorSize\n"); for (entry <- listEntries(dir)) { loc file = dir + entry; if (isFile(file)) { diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopMultiError.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopMultiError.rsc new file mode 100644 index 00000000000..6fa4525a051 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopMultiError.rsc @@ -0,0 +1,15 @@ +module lang::rascal::tests::concrete::recovery::bugs::InfiniteLoopMultiError + + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +import lang::rascal::\syntax::Rascal; +import ParseTree; +import IO; + +void testInfiniteLoopMultiError() { + standardParser = parser(#start[Module], allowRecovery=false, allowAmbiguity=true); + recoveryParser = parser(#start[Module], allowRecovery=true, allowAmbiguity=true); + source = |std:///Exception.rsc|; + input = readFile(source); + testDeleteUntilEol(standardParser, recoveryParser, source, input, 200, 100, begin=662, end=664); +} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MinimalMultiError.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MinimalMultiError.rsc new file mode 100644 index 00000000000..75797f226f2 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MinimalMultiError.rsc @@ -0,0 +1,17 @@ +module lang::rascal::tests::concrete::recovery::bugs::MinimalMultiError + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; + +layout Layout = [\ ]* !>> [\ ]; + +syntax S = T; + +syntax T = AB AB AB End; +syntax AB = 'a' 'b'; +syntax End = "$"; + +test bool multiOk() = checkRecovery(#S, "ababab$", []); + +test bool multiOneError() = checkRecovery(#S, "acabab$", ["c"]); + +test bool multiTwoError() = checkRecovery(#S, "acacab$", ["c","c"]); diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 695486f9043..2c249b0cc3f 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -32,6 +32,7 @@ import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.EpsilonStackNode; import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; +import org.rascalmpl.parser.gtd.stack.SkippingStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.stack.filter.ICompletionFilter; import org.rascalmpl.parser.gtd.stack.filter.IEnterFilter; @@ -1408,6 +1409,10 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] shiftedLevel = true; + if (onlyRecoveredStacksLeft() && attemptRecovery()) { + continue; + } + if (!findStacksToReduce()) { if(location == input.length) { EdgesSet

startNodeEdgesSet = startNode.getIncomingEdges(); @@ -1463,6 +1468,28 @@ private void initTime() { timestamp = System.nanoTime(); } + private boolean onlyRecoveredStacksLeft() { + int recoveredStacksFound = 0; + + for (int i=0; i, AbstractNode> todoList = todoLists[i]; + if (todoList != null) { + int size = todoList.getSize(); + for (int j=0; j 50) { + return false; + } + } + } + } + + return recoveredStacksFound > 0; + } + private void checkTime(String msg) { long newStamp = System.nanoTime(); long duration = newStamp - timestamp; diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 3fa24c7dfd2..092b2f9dcf8 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -154,7 +154,7 @@ private List> findSkippingNodes(int[] input, int // Find the last token of this production and skip until after that List endMatchers = findEndMatchers(recoveryNode); for (InputMatcher endMatcher : endMatchers) { - MatchResult endMatch = endMatcher.findMatch(input, startLocation, 5); + MatchResult endMatch = endMatcher.findMatch(input, startLocation, Integer.MAX_VALUE/2); if (endMatch != null) { result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, endMatch.getEnd()); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); @@ -164,7 +164,7 @@ private List> findSkippingNodes(int[] input, int // Find the first token of the next production and skip until before that List nextMatchers = findNextMatchers(recoveryNode); for (InputMatcher nextMatcher : nextMatchers) { - MatchResult nextMatch = nextMatcher.findMatch(input, startLocation+1, 5); + MatchResult nextMatch = nextMatcher.findMatch(input, startLocation+1, Integer.MAX_VALUE/2); if (nextMatch != null) { result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, nextMatch.getStart()); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); From 0637b148bcf909396db35a646d1455f28a81c61f Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Sat, 19 Oct 2024 10:43:45 +0200 Subject: [PATCH 04/12] Ignoring generated file --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index b5d6e93bca0..e93bbee77b0 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ dependency-reduced-pom.xml *.iml .idea/ + +# generated by error recovery benchmark +rascal-recovery-stats.csv From 1700d1b38bb3e910a87519343cddd632539bc49d Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 21 Oct 2024 07:23:40 +0200 Subject: [PATCH 05/12] Filter skipped nodes that skip the same chars and have the same result sort --- .../recovery/bugs/MultiProdRecovery.rsc | 20 ++++++++++++++++++ .../uptr/recovery/ToTokenRecoverer.java | 21 ++++++++++++++++--- 2 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiProdRecovery.rsc diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiProdRecovery.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiProdRecovery.rsc new file mode 100644 index 00000000000..3715d8125a3 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiProdRecovery.rsc @@ -0,0 +1,20 @@ +module lang::rascal::tests::concrete::recovery::bugs::MultiProdRecovery + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; + +layout Layout = [\ ]* !>> [\ ]; + +syntax Prog = Stat*; + +syntax Stat + = Expr ";" + | "{" Stat* "}"; + +syntax Expr + = Expr "+" Expr + | Expr "-" Expr + | "e"; + +test bool multiProdOk() = checkRecovery(#Prog, "{ e + e; }", []); + +test bool multiProdOperatorError() = checkRecovery(#Prog, "{ e * e; }", []); diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 092b2f9dcf8..30d04a8705d 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -13,10 +13,13 @@ package org.rascalmpl.parser.uptr.recovery; import java.net.URI; +import java.util.BitSet; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; +import org.apache.commons.lang3.tuple.Triple; import org.rascalmpl.parser.gtd.ExpectsProvider; import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.AbstractNode; @@ -42,6 +45,7 @@ import org.rascalmpl.parser.uptr.recovery.InputMatcher.MatchResult; import org.rascalmpl.parser.util.ParseStateVisualizer; import org.rascalmpl.values.parsetrees.ProductionAdapter; +import org.rascalmpl.values.parsetrees.TreeAdapter; import io.usethesource.vallang.IConstructor; @@ -81,13 +85,15 @@ private DoubleArrayList, AbstractNode> reviveNod DoubleArrayList, ArrayList> recoveryNodes) { DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList<>(); + Set> skippedIds = new HashSet>(); + // Sort nodes by start location - recoveryNodes + recoveryNodes .sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); if (VISUALIZE_RECOVERY_NODES) { - ParseStateVisualizer visualizer = new ParseStateVisualizer("Recovery"); - visualizer.visualizeRecoveryNodes(recoveryNodes); + ParseStateVisualizer visualizer = new ParseStateVisualizer("Recovery"); + visualizer.visualizeRecoveryNodes(recoveryNodes); } for (int i = 0; i, AbstractNode> reviveNod for (int j = prods.size() - 1; j >= 0; --j) { IConstructor prod = prods.get(j); + IConstructor type = ProductionAdapter.getType(prod); + List> skippingNodes = findSkippingNodes(input, location, recoveryNode, prod, startLocation); for (SkippingStackNode skippingNode : skippingNodes) { + int skipLength = skippingNode.getLength(); + + if (!skippedIds.add(Triple.ofNonNull(startLocation, type, skipLength))) { + // Do not add this skipped node if a node with the same startLocation, type, and skipLength has been added already + continue; + } + AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); From af5ab3a5e9cfeb413ab4042ad7eb3aac95078b75 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 21 Oct 2024 09:35:57 +0200 Subject: [PATCH 06/12] Fixed tests that failed because error recovery has been improved --- .../lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc | 2 +- .../rascal/tests/concrete/recovery/bugs/MultiProdRecovery.rsc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc index da52a010466..1d54ed8ab92 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc @@ -54,7 +54,7 @@ test bool rascalFunctionDeclarationMissingCloseParen() = checkRecovery(#Function test bool rascalIfMissingExpr() = checkRecovery(#FunctionDeclaration, "void f(){if(){1;}}", [")"]); -test bool rascalIfBodyEmpty() = checkRecovery(#start[Module], "module A void f(){1;} void g(){if(1){}} void h(){1;}", ["} void h(){1"]); +test bool rascalIfBodyEmpty() = checkRecovery(#start[Module], "module A void f(){1;} void g(){if(1){}} void h(){1;}", ["{", "} "]); // Not working yet: /* diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiProdRecovery.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiProdRecovery.rsc index 3715d8125a3..ed6d77fe645 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiProdRecovery.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiProdRecovery.rsc @@ -17,4 +17,4 @@ syntax Expr test bool multiProdOk() = checkRecovery(#Prog, "{ e + e; }", []); -test bool multiProdOperatorError() = checkRecovery(#Prog, "{ e * e; }", []); +test bool multiProdOperatorError() = checkRecovery(#Prog, "{ e * e; }", ["* "]); From 5c7398f6d8d28873df6f6742b0caca55886ed66d Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 22 Oct 2024 07:35:10 +0200 Subject: [PATCH 07/12] Simplified MultiErrorPico test --- .../concrete/recovery/bugs/MultiErrorPico.rsc | 33 ++++--------------- .../recovery/bugs/MultiErrorPicoInput.pico | 7 +++- 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPico.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPico.rsc index bcdbe125b42..2a81b5f6af3 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPico.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPico.rsc @@ -1,32 +1,13 @@ module lang::rascal::tests::concrete::recovery::bugs::MultiErrorPico import lang::pico::\syntax::Main; -import ParseTree; -import IO; -import util::ErrorRecovery; -import List; -import vis::Text; +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; bool multiErrorPico() { - str input = readFile(|std:///lang/rascal/tests/concrete/recovery/bugs/MultiErrorPicoInput.pico|); - Tree t = parser(#start[Program], allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=true|); - - println(prettyTree(t)); - - list[Tree] errors = findAllErrors(t); - println(" Errors"); - for (Tree error <- errors) { - Tree skipped = getSkipped(error); - println(" : "); - } - Tree disambiguated = disambiguateErrors(t); - list[Tree] disambiguatedErrors = findAllErrors(disambiguated); - println("After disambiguating:"); - println(" Errors"); - for (Tree error <- disambiguatedErrors) { - Tree skipped = getSkipped(error); - println(" : "); - } - return true; + return checkRecovery(#start[Program], "begin + declare; + i := #1; + j := #2; + k := 3 +end" , ["#1", "#2"]); } - diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPicoInput.pico b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPicoInput.pico index d94083a5603..a7393e9f705 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPicoInput.pico +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorPicoInput.pico @@ -1 +1,6 @@ -begindeclare;i:=#1;j:=#2;k:=3end \ No newline at end of file +begin + declare; + i := #1; + j := #2; + k := 3 +end \ No newline at end of file From fa0ed1d183d4afadcfbeb7faed0185b144d23a93 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 22 Oct 2024 14:31:58 +0200 Subject: [PATCH 08/12] Fixed indentation --- src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 30d04a8705d..6dafd03b356 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -45,7 +45,6 @@ import org.rascalmpl.parser.uptr.recovery.InputMatcher.MatchResult; import org.rascalmpl.parser.util.ParseStateVisualizer; import org.rascalmpl.values.parsetrees.ProductionAdapter; -import org.rascalmpl.values.parsetrees.TreeAdapter; import io.usethesource.vallang.IConstructor; @@ -85,7 +84,7 @@ private DoubleArrayList, AbstractNode> reviveNod DoubleArrayList, ArrayList> recoveryNodes) { DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList<>(); - Set> skippedIds = new HashSet>(); + Set> skippedIds = new HashSet<>(); // Sort nodes by start location recoveryNodes @@ -295,9 +294,7 @@ public Void visit(LiteralStackNode literal) { return null; } - @Override - public Void visit(CaseInsensitiveLiteralStackNode literal) { matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); return null; From 90c3f23ba4e8fe2d6c76d30262e74f18fd45a4ad Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 23 Oct 2024 10:21:34 +0200 Subject: [PATCH 09/12] Added comment on the use of "max match length" --- src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 6dafd03b356..7c9651a9ab4 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -13,9 +13,7 @@ package org.rascalmpl.parser.uptr.recovery; import java.net.URI; -import java.util.BitSet; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Set; @@ -168,6 +166,7 @@ private List> findSkippingNodes(int[] input, int // Find the last token of this production and skip until after that List endMatchers = findEndMatchers(recoveryNode); for (InputMatcher endMatcher : endMatchers) { + // For now take a very large (basically unlimited) "max match length", experiment with smaller limit later MatchResult endMatch = endMatcher.findMatch(input, startLocation, Integer.MAX_VALUE/2); if (endMatch != null) { result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, endMatch.getEnd()); @@ -178,6 +177,7 @@ private List> findSkippingNodes(int[] input, int // Find the first token of the next production and skip until before that List nextMatchers = findNextMatchers(recoveryNode); for (InputMatcher nextMatcher : nextMatchers) { + // For now take a very large (basically unlimited) "max match length", experiment with smaller limit later MatchResult nextMatch = nextMatcher.findMatch(input, startLocation+1, Integer.MAX_VALUE/2); if (nextMatch != null) { result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, nextMatch.getStart()); From b1622c0219adf52f9d5acba3e2ed1cb93b37727d Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 23 Oct 2024 10:26:30 +0200 Subject: [PATCH 10/12] Consolidated multi-error issue encountered in Rascal. MultiErrorBug condenses the Rascal syntax to only cover the problematic issue. The issue itself is of yet unsolved. --- .../concrete/recovery/bugs/MultiErrorBug.rsc | 24 ++++++++++++++++--- .../recovery/bugs/MultiErrorBugInput.txt | 17 ------------- 2 files changed, 21 insertions(+), 20 deletions(-) delete mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBugInput.txt diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBug.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBug.rsc index 6f95984a751..0b199a923d7 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBug.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBug.rsc @@ -1,15 +1,33 @@ module lang::rascal::tests::concrete::recovery::bugs::MultiErrorBug - -import lang::rascal::\syntax::Rascal; import ParseTree; import IO; import util::ErrorRecovery; import List; import vis::Text; +start syntax Module = SyntaxDefinition* ; + +syntax SyntaxDefinition = Prod ";" ; + +lexical Name = [A-Z]; + +syntax Sym + = Sym NonterminalLabel + | StringConstant + ; + +lexical StringConstant = "\"" StringCharacter* "\"" ; + +lexical StringCharacter = ![\"] ; + +lexical NonterminalLabel = [a-z] ; + +syntax Prod = Sym* ; + +// This is an open issue: instead of two small errors, one big error tree is returned. bool multiErrorBug() { - str input = readFile(|std:///lang/rascal/tests/concrete/recovery/bugs/MultiErrorBugInput.txt|); + str input = "#\"a\";#\"b\";"; Tree t = parser(#start[Module], allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=false|); println(prettyTree(t)); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBugInput.txt b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBugInput.txt deleted file mode 100644 index 7d6c255752f..00000000000 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/MultiErrorBugInput.txt +++ /dev/null @@ -1,17 +0,0 @@ -module sandbox::SimpleSyntax - -start syntax Start = S; - -syntax S = A | B | C | D | E | F; - -syntax A = "a"; - -syntax B = #"b"; - -syntax C = "c"; - -syntax D = #"d"; - -syntax E = "e"; - -syntax F = "f"; From 68c7538bf6d3c00a2d3b56eb44c4182afcd3b4d9 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 23 Oct 2024 12:05:09 +0200 Subject: [PATCH 11/12] Fixed limit calculation in CaseInsenstiveLiteralMatcher --- .../parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java index 406f06ca140..c72e1fa1387 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java @@ -49,7 +49,7 @@ public CaseInsensitiveLiteralMatcher(int[][] ciLiteral) { public MatchResult findMatch(int[] input, int startLocation, int maxLength) { int length = chars.length; - int limit = input.length - length + 1; + int limit = Math.min(startLocation + maxLength - length, input.length - length + 1); for (int start=startLocation; start < limit; start++) { boolean matches = true; for (int i=0; i Date: Wed, 23 Oct 2024 12:07:09 +0200 Subject: [PATCH 12/12] Removed obsolete comment --- .../rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 7c9651a9ab4..cb6e9f9725a 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -154,15 +154,6 @@ private List> findSkippingNodes(int[] input, int return nodes; // No other nodes would be useful } - // Try to find whitespace to skip to - // This often creates hopeless recovery attempts, but it might help in some cases. - // Further experimentation should quantify this statement. - /* - * result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, - * dot); if (result != null) { nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), - * prod, result, startLocation)); } - */ - // Find the last token of this production and skip until after that List endMatchers = findEndMatchers(recoveryNode); for (InputMatcher endMatcher : endMatchers) {