From 9deefa361bbc0b4c68c2ff405b057001c8d0b45c Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 30 Mar 2022 16:45:59 +0200 Subject: [PATCH 001/190] recovered Recoverer from git history --- .../parser/uptr/recovery/Recoverer.java | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 src/org/rascalmpl/parser/uptr/recovery/Recoverer.java diff --git a/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java b/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java new file mode 100644 index 00000000000..0a69c83636b --- /dev/null +++ b/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java @@ -0,0 +1,205 @@ +/******************************************************************************* + * Copyright (c) 2009-2013 CWI + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + + * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI + * * Arnold Lankamp - Arnold.Lankamp@cwi.nl +*******************************************************************************/ +package org.rascalmpl.parser.uptr.recovery; + +import org.eclipse.imp.pdb.facts.IConstructor; +import org.rascalmpl.parser.gtd.recovery.IRecoverer; +import org.rascalmpl.parser.gtd.result.AbstractNode; +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; +import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; +import org.rascalmpl.parser.gtd.stack.SkippingStackNode; +import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; +import org.rascalmpl.parser.gtd.util.ArrayList; +import org.rascalmpl.parser.gtd.util.DoubleArrayList; +import org.rascalmpl.parser.gtd.util.DoubleStack; +import org.rascalmpl.parser.gtd.util.IntegerObjectList; +import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; +import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.values.uptr.ProductionAdapter; + +public class Recoverer implements IRecoverer{ + // TODO: its a magic constant, and it may clash with other generated constants + // should generate implementation of static int getLastId() in generated parser to fix this. + private int recoveryId = 100000; + + private final int[][] continuationCharactersList; + + private final ObjectKeyedIntegerMap robust; + + public Recoverer(IConstructor[] robustProds, int[][] continuationCharactersList){ + super(); + + this.continuationCharactersList = continuationCharactersList; + + this.robust = new ObjectKeyedIntegerMap(); + + for (int i = robustProds.length - 1; i >= 0; --i) { + robust.put(robustProds[i], i); + } + } + + private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ + DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); + + for(int i = recoveryNodes.size() - 1; i >= 0; --i) { + AbstractStackNode recoveryNode = recoveryNodes.getFirst(i); + ArrayList prods = recoveryNodes.getSecond(i); + + // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). + for(int j = prods.size() - 1; j >= 0; --j){ + IConstructor prod = prods.get(j); + + AbstractStackNode continuer = new RecoveryPointStackNode(recoveryId++, prod, recoveryNode); + + int startLocation = recoveryNode.getStartLocation(); + + int[] until = continuationCharactersList[robust.get(prod)]; + AbstractStackNode recoverLiteral = new SkippingStackNode(recoveryId++, until, input, startLocation, prod); + recoverLiteral = recoverLiteral.getCleanCopy(startLocation); + recoverLiteral.initEdges(); + EdgesSet edges = new EdgesSet(1); + edges.add(continuer); + + recoverLiteral.addEdges(edges, startLocation); + + continuer.setIncomingEdges(edges); + + recoveredNodes.add(recoverLiteral, recoverLiteral.getResult()); + } + } + + return recoveredNodes; + } + + private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { + DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList, ArrayList>(); + + for(int i = failedNodes.size() - 1; i >= 0; --i){ + findRecoveryNodes(failedNodes.get(i), recoveryNodes); + } + + return reviveNodes(input, location, recoveryNodes); + } + + private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { + for(int i = unexpandableNodes.getSize() - 1; i >= 0; --i){ + failedNodes.add(unexpandableNodes.get(i)); + } + } + + private static void collectUnmatchableMidProductionNodes(int location, DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ + for(int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i){ + DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); + AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference updates of the static version + + // Merge the information on the predecessors into the failed node. + for(int j = failedNodePredecessors.size() - 1; j >= 0; --j){ + AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); + AbstractNode predecessorResult = failedNodePredecessors.getSecond(j); + failedNode.updateNode(predecessor, predecessorResult); + } + + failedNodes.add(failedNode); + } + } + + private boolean isRobust(IConstructor prod) { + return robust.contains(prod); + } + + /** + * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. + */ + private void findRecoveryNodes(AbstractStackNode failer, DoubleArrayList, ArrayList> recoveryNodes) { + ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap>(); + Stack> todo = new Stack>(); + + todo.push(failer); + + while (!todo.isEmpty()) { + AbstractStackNode node = todo.pop(); + if(visited.contains(node)) continue; // Don't follow cycles + + visited.put(node, 0); + + ArrayList recoveryProductions = new ArrayList(); + collectProductions(node, recoveryProductions); + + if(recoveryProductions.size() > 0){ + recoveryNodes.add(node, recoveryProductions); + } + + IntegerObjectList> edges = node.getEdges(); + + for(int i = edges.size() - 1; i >= 0; --i){ + EdgesSet edgesList = edges.getValue(i); + + if (edgesList != null) { + for(int j = edgesList.size() - 1; j >= 0; --j){ + AbstractStackNode parent = edgesList.get(j); + + todo.push(parent); + } + } + } + } + } + + // Gathers all productions that are marked for recovery (the given node can be part of a prefix shared production) + private void collectProductions(AbstractStackNode node, ArrayList productions) { + AbstractStackNode[] production = node.getProduction(); + if(production == null) return; // The root node does not have a production, so ignore it. + + int dot = node.getDot(); + + if(node.isEndNode()){ + IConstructor parentProduction = node.getParentProduction(); + if(isRobust(parentProduction)){ + productions.add(parentProduction); + + if(ProductionAdapter.isList(parentProduction)) return; // Don't follow productions in lists productions, since they are 'cyclic'. + } + } + + for(int i = dot + 1; i < production.length; ++i){ + AbstractStackNode currentNode = production[i]; + if(currentNode.isEndNode()){ + IConstructor parentProduction = currentNode.getParentProduction(); + if(isRobust(parentProduction)){ + productions.add(parentProduction); + } + } + + AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); + if(alternateProductions != null){ + for(int j = alternateProductions.length - 1; j >= 0; --j){ + collectProductions(alternateProductions[j][i], productions); + } + } + } + } + + public DoubleArrayList, AbstractNode> reviveStacks(int[] input, + int location, + Stack> unexpandableNodes, + Stack> unmatchableLeafNodes, + DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, + DoubleStack, AbstractNode> filteredNodes) { + ArrayList> failedNodes = new ArrayList>(); + collectUnexpandableNodes(unexpandableNodes, failedNodes); + collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); + //collectFilteredNodes(filteredNodes, failedNodes); + + return reviveFailedNodes(input, location, failedNodes); + } +} From 9ae895c908e87fa8f152d8fc50a24a03745a542a Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 09:58:30 +0200 Subject: [PATCH 002/190] fixed imports and whitespace --- .../parser/uptr/recovery/Recoverer.java | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java b/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java index 0a69c83636b..76f1f6485c2 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2009-2013 CWI + * Copyright (c) 2009-2022 NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -12,7 +12,6 @@ *******************************************************************************/ package org.rascalmpl.parser.uptr.recovery; -import org.eclipse.imp.pdb.facts.IConstructor; import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; @@ -25,7 +24,9 @@ import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; -import org.rascalmpl.values.uptr.ProductionAdapter; +import org.rascalmpl.values.parsetrees.ProductionAdapter; + +import io.usethesource.vallang.IConstructor; public class Recoverer implements IRecoverer{ // TODO: its a magic constant, and it may clash with other generated constants @@ -51,12 +52,12 @@ public Recoverer(IConstructor[] robustProds, int[][] continuationCharactersList) private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); - for(int i = recoveryNodes.size() - 1; i >= 0; --i) { + for (int i = recoveryNodes.size() - 1; i >= 0; --i) { AbstractStackNode recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). - for(int j = prods.size() - 1; j >= 0; --j){ + for (int j = prods.size() - 1; j >= 0; --j) { IConstructor prod = prods.get(j); AbstractStackNode continuer = new RecoveryPointStackNode(recoveryId++, prod, recoveryNode); @@ -84,7 +85,7 @@ private DoubleArrayList, AbstractNode> reviveNod private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList, ArrayList>(); - for(int i = failedNodes.size() - 1; i >= 0; --i){ + for (int i = failedNodes.size() - 1; i >= 0; --i) { findRecoveryNodes(failedNodes.get(i), recoveryNodes); } @@ -92,18 +93,18 @@ private DoubleArrayList, AbstractNode> reviveFai } private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { - for(int i = unexpandableNodes.getSize() - 1; i >= 0; --i){ + for (int i = unexpandableNodes.getSize() - 1; i >= 0; --i) { failedNodes.add(unexpandableNodes.get(i)); } } private static void collectUnmatchableMidProductionNodes(int location, DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ - for(int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i){ + for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference updates of the static version // Merge the information on the predecessors into the failed node. - for(int j = failedNodePredecessors.size() - 1; j >= 0; --j){ + for(int j = failedNodePredecessors.size() - 1; j >= 0; --j) { AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); AbstractNode predecessorResult = failedNodePredecessors.getSecond(j); failedNode.updateNode(predecessor, predecessorResult); @@ -135,13 +136,13 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr ArrayList recoveryProductions = new ArrayList(); collectProductions(node, recoveryProductions); - if(recoveryProductions.size() > 0){ + if (recoveryProductions.size() > 0) { recoveryNodes.add(node, recoveryProductions); } IntegerObjectList> edges = node.getEdges(); - for(int i = edges.size() - 1; i >= 0; --i){ + for (int i = edges.size() - 1; i >= 0; --i) { EdgesSet edgesList = edges.getValue(i); if (edgesList != null) { @@ -162,27 +163,29 @@ private void collectProductions(AbstractStackNode node, ArrayList< int dot = node.getDot(); - if(node.isEndNode()){ + if (node.isEndNode()) { IConstructor parentProduction = node.getParentProduction(); if(isRobust(parentProduction)){ productions.add(parentProduction); - if(ProductionAdapter.isList(parentProduction)) return; // Don't follow productions in lists productions, since they are 'cyclic'. + if (ProductionAdapter.isList(parentProduction)) { + return; // Don't follow productions in lists productions, since they are 'cyclic'. + } } } - for(int i = dot + 1; i < production.length; ++i){ + for (int i = dot + 1; i < production.length; ++i) { AbstractStackNode currentNode = production[i]; - if(currentNode.isEndNode()){ + if (currentNode.isEndNode()) { IConstructor parentProduction = currentNode.getParentProduction(); - if(isRobust(parentProduction)){ + if (isRobust(parentProduction)) { productions.add(parentProduction); } } AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); - if(alternateProductions != null){ - for(int j = alternateProductions.length - 1; j >= 0; --j){ + if (alternateProductions != null) { + for (int j = alternateProductions.length - 1; j >= 0; --j) { collectProductions(alternateProductions[j][i], productions); } } From 4e43070c15d65ff35de6df74c7787254bab81cbe Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 10:15:31 +0200 Subject: [PATCH 003/190] added documentation --- .../parser/gtd/recovery/IRecoverer.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/parser/gtd/recovery/IRecoverer.java b/src/org/rascalmpl/parser/gtd/recovery/IRecoverer.java index f7db2b352ab..40dc4d52257 100644 --- a/src/org/rascalmpl/parser/gtd/recovery/IRecoverer.java +++ b/src/org/rascalmpl/parser/gtd/recovery/IRecoverer.java @@ -17,7 +17,22 @@ import org.rascalmpl.parser.gtd.util.DoubleStack; import org.rascalmpl.parser.gtd.util.Stack; -public interface IRecoverer

{ +public interface IRecoverer

{ + /** + * reviveStacks is called when the parser is unable to make more progress and the end-of-input + * has not been reached. The parameters provide insight into the current state of the parser + * and some of its history. With this information new stack nodes may be generated for the parser + * to continue with. It is up to the reviveStacks method to make sure the parser still generates + * derivation trees that cover the entire input. + * + * @param input the 24-bit unicode input character array + * @param location the current character offset in the input that the parser got stuck on + * @param unexpandableNodes these are non-terminals that were predicted at this location but did not fly + * @param unmatchableLeafNodes these are the terminals that were predicted but did not fly + * @param unmatchableMidProductionNodes these are quasi-non-terminals due to prefix sharing that did not fly + * @param filteredNodes these are non-terminals nodes that did not fly due to a disambiguation filter + * @return a list of new predictions for the parser to continue with + */ DoubleArrayList, AbstractNode> reviveStacks(int[] input, int location, Stack> unexpandableNodes, From ff99ac9bc7eae6dfeb257914319628270f1e0a56 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 10:16:05 +0200 Subject: [PATCH 004/190] renamed recoverer and simplified to whitespace only and all context-free non-terminals --- ...er.java => ToNextWhitespaceRecoverer.java} | 29 ++++--------------- 1 file changed, 5 insertions(+), 24 deletions(-) rename src/org/rascalmpl/parser/uptr/recovery/{Recoverer.java => ToNextWhitespaceRecoverer.java} (91%) diff --git a/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java similarity index 91% rename from src/org/rascalmpl/parser/uptr/recovery/Recoverer.java rename to src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 76f1f6485c2..7d2ceddbb79 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -28,26 +28,12 @@ import io.usethesource.vallang.IConstructor; -public class Recoverer implements IRecoverer{ +public class ToNextWhitespaceRecoverer implements IRecoverer{ // TODO: its a magic constant, and it may clash with other generated constants // should generate implementation of static int getLastId() in generated parser to fix this. private int recoveryId = 100000; + private static final int[] WHITESPACE = {' ', '\r', '\n', '\t' }; - private final int[][] continuationCharactersList; - - private final ObjectKeyedIntegerMap robust; - - public Recoverer(IConstructor[] robustProds, int[][] continuationCharactersList){ - super(); - - this.continuationCharactersList = continuationCharactersList; - - this.robust = new ObjectKeyedIntegerMap(); - - for (int i = robustProds.length - 1; i >= 0; --i) { - robust.put(robustProds[i], i); - } - } private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); @@ -64,8 +50,7 @@ private DoubleArrayList, AbstractNode> reviveNod int startLocation = recoveryNode.getStartLocation(); - int[] until = continuationCharactersList[robust.get(prod)]; - AbstractStackNode recoverLiteral = new SkippingStackNode(recoveryId++, until, input, startLocation, prod); + AbstractStackNode recoverLiteral = new SkippingStackNode(recoveryId++, WHITESPACE, input, startLocation, prod); recoverLiteral = recoverLiteral.getCleanCopy(startLocation); recoverLiteral.initEdges(); EdgesSet edges = new EdgesSet(1); @@ -114,10 +99,6 @@ private static void collectUnmatchableMidProductionNodes(int location, DoubleSta } } - private boolean isRobust(IConstructor prod) { - return robust.contains(prod); - } - /** * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. */ @@ -165,7 +146,7 @@ private void collectProductions(AbstractStackNode node, ArrayList< if (node.isEndNode()) { IConstructor parentProduction = node.getParentProduction(); - if(isRobust(parentProduction)){ + if(ProductionAdapter.isContextFree(parentProduction)){ productions.add(parentProduction); if (ProductionAdapter.isList(parentProduction)) { @@ -178,7 +159,7 @@ private void collectProductions(AbstractStackNode node, ArrayList< AbstractStackNode currentNode = production[i]; if (currentNode.isEndNode()) { IConstructor parentProduction = currentNode.getParentProduction(); - if (isRobust(parentProduction)) { + if (ProductionAdapter.isContextFree(parentProduction)) { productions.add(parentProduction); } } From d2a0b668e1c584f76b91ee35b296554b9eda946d Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 12:20:55 +0200 Subject: [PATCH 005/190] removed unused javadoc plugin that produced warnings --- pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pom.xml b/pom.xml index 3d285611905..ae8b8ea02a2 100644 --- a/pom.xml +++ b/pom.xml @@ -92,14 +92,6 @@ - - org.apache.maven.plugins - maven-javadoc-plugin - 3.2.0 - - -Xdoclint:none - - org.codehaus.mojo buildnumber-maven-plugin From d9439a450020942fe80617379277c75c5020be63 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 12:21:07 +0200 Subject: [PATCH 006/190] fixed compiler warnings --- .../library/lang/java/m3/internal/JarConverter.java | 7 ------- src/org/rascalmpl/semantics/dynamic/Tree.java | 11 ++++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/org/rascalmpl/library/lang/java/m3/internal/JarConverter.java b/src/org/rascalmpl/library/lang/java/m3/internal/JarConverter.java index a202cf5087a..0061d3ee329 100644 --- a/src/org/rascalmpl/library/lang/java/m3/internal/JarConverter.java +++ b/src/org/rascalmpl/library/lang/java/m3/internal/JarConverter.java @@ -266,7 +266,6 @@ private void setClassRelations(ClassReader classReader, String compUnitRelative) ISourceLocation classLogical = resolver.resolveBinding(classNode, null); ISourceLocation classPhysical = M3LocationUtil.makeLocation(compUnitPhysical, classReader.header, classReader.b.length); IConstructor cons = resolver.resolveType(classNode, null); - @SuppressWarnings("unchecked") List annotations = composeAnnotations(classNode.visibleAnnotations, classNode.invisibleAnnotations); addToContainment(compUnitLogical, classLogical); @@ -291,7 +290,6 @@ private void setClassRelations(ClassReader classReader, String compUnitRelative) */ private void setInnerClassRelations(ClassNode classNode, ISourceLocation classLogical) { // cn.innerClasses and cn.outerClass are not providing consistent information. - @SuppressWarnings("unchecked") List innerClasses = classNode.innerClasses; if (innerClasses != null) { @@ -322,7 +320,6 @@ private void setInnerClassRelations(ClassNode classNode, ISourceLocation classLo * @param classNode - class node where fields are declared * @param classLogical - class location */ - @SuppressWarnings("unchecked") private void setFieldRelations(ClassNode classNode, ISourceLocation classLogical) { List fields = classNode.fields; @@ -356,7 +353,6 @@ private void setFieldRelations(ClassNode classNode, ISourceLocation classLogical * @param classNode - class node where fields are declared * @param classLogical - class location */ - @SuppressWarnings("unchecked") private void setMethodRelations(ClassNode classNode, ISourceLocation classLogical) { List methods = classNode.methods; @@ -400,7 +396,6 @@ private void setMethodOverridesRelation(String superClass, MethodNode methodNode if (classReader != null) { ClassNode classNode = new ClassNode(); classReader.accept(classNode, ClassReader.SKIP_DEBUG); - @SuppressWarnings("unchecked") List superMethods = classNode.methods; if (superMethods != null) { @@ -425,7 +420,6 @@ private void setMethodOverridesRelation(String superClass, MethodNode methodNode * @param methodLogical - logical location of the method */ private void setExceptionRelations(MethodNode methodNode, ISourceLocation methodLogical) { - @SuppressWarnings("unchecked") List exceptions = methodNode.exceptions; for (String exception : exceptions) { @@ -468,7 +462,6 @@ private void setInstructionRelations(MethodNode methodNode, ISourceLocation meth InsnList instructions = methodNode.instructions; if(instructions != null) { - @SuppressWarnings("unchecked") ListIterator iterator = instructions.iterator(); while (iterator.hasNext()) { diff --git a/src/org/rascalmpl/semantics/dynamic/Tree.java b/src/org/rascalmpl/semantics/dynamic/Tree.java index 51b36ffa629..f69f8c750c5 100644 --- a/src/org/rascalmpl/semantics/dynamic/Tree.java +++ b/src/org/rascalmpl/semantics/dynamic/Tree.java @@ -27,15 +27,17 @@ import org.rascalmpl.interpreter.matching.IMatchingResult; import org.rascalmpl.interpreter.matching.LiteralPattern; import org.rascalmpl.interpreter.matching.NodePattern; -import org.rascalmpl.interpreter.matching.QualifiedNamePattern; import org.rascalmpl.interpreter.matching.SetPattern; import org.rascalmpl.interpreter.matching.TypedVariablePattern; import org.rascalmpl.interpreter.result.Result; import org.rascalmpl.interpreter.staticErrors.UndeclaredVariable; import org.rascalmpl.interpreter.staticErrors.UninitializedVariable; -import org.rascalmpl.interpreter.utils.Names; import org.rascalmpl.types.NonTerminalType; import org.rascalmpl.types.RascalTypeFactory; +import org.rascalmpl.values.RascalValueFactory; +import org.rascalmpl.values.parsetrees.ProductionAdapter; +import org.rascalmpl.values.parsetrees.SymbolAdapter; +import org.rascalmpl.values.parsetrees.TreeAdapter; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IList; @@ -45,11 +47,6 @@ import io.usethesource.vallang.IValue; import io.usethesource.vallang.type.Type; -import org.rascalmpl.values.RascalValueFactory; -import org.rascalmpl.values.parsetrees.ProductionAdapter; -import org.rascalmpl.values.parsetrees.SymbolAdapter; -import org.rascalmpl.values.parsetrees.TreeAdapter; - /** * These classes special case Expression.CallOrTree for concrete syntax patterns */ From d87b4b85291cd6f0a219a3b2e11be5ba36d45afa Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 12:29:11 +0200 Subject: [PATCH 007/190] fixed warnings --- .../rascalmpl/interpreter/matching/VariableBecomesPattern.java | 1 - src/org/rascalmpl/interpreter/result/ResultFactory.java | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/org/rascalmpl/interpreter/matching/VariableBecomesPattern.java b/src/org/rascalmpl/interpreter/matching/VariableBecomesPattern.java index 66e2530f716..0370efb1b42 100644 --- a/src/org/rascalmpl/interpreter/matching/VariableBecomesPattern.java +++ b/src/org/rascalmpl/interpreter/matching/VariableBecomesPattern.java @@ -17,7 +17,6 @@ import java.util.HashMap; import java.util.List; -import org.rascalmpl.ast.AbstractAST; import org.rascalmpl.ast.Expression; import org.rascalmpl.interpreter.IEvaluatorContext; import org.rascalmpl.interpreter.env.Environment; diff --git a/src/org/rascalmpl/interpreter/result/ResultFactory.java b/src/org/rascalmpl/interpreter/result/ResultFactory.java index 27917622418..4607b274ce0 100644 --- a/src/org/rascalmpl/interpreter/result/ResultFactory.java +++ b/src/org/rascalmpl/interpreter/result/ResultFactory.java @@ -13,10 +13,8 @@ import org.rascalmpl.exceptions.ImplementationError; import org.rascalmpl.interpreter.IEvaluatorContext; -import org.rascalmpl.interpreter.utils.TreeAsNode; import org.rascalmpl.types.RascalType; import org.rascalmpl.values.RascalValueFactory; -import org.rascalmpl.values.parsetrees.ITree; import io.usethesource.vallang.IBool; import io.usethesource.vallang.IConstructor; From 4897d88fbb5b285ea5a0a5b0f3fb575365387aec Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 12:29:26 +0200 Subject: [PATCH 008/190] fixed warnings --- src/org/rascalmpl/util/HeapDumper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/org/rascalmpl/util/HeapDumper.java b/src/org/rascalmpl/util/HeapDumper.java index d2376434353..4af965e9ded 100644 --- a/src/org/rascalmpl/util/HeapDumper.java +++ b/src/org/rascalmpl/util/HeapDumper.java @@ -25,7 +25,6 @@ * * Use it to dump the heap to a file for further analysis. */ -@SuppressWarnings("restriction") public class HeapDumper { private static final class InstanceKeeper { /*package*/ From 58034115b6d59f342b01a1c62e8321771b385d7e Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 12:29:57 +0200 Subject: [PATCH 009/190] added boolean parameter \'robust\' to parsing API --- src/org/rascalmpl/library/ParseTree.rsc | 16 ++++++++-------- src/org/rascalmpl/library/Prelude.java | 8 ++++---- .../rascalmpl/values/IRascalValueFactory.java | 9 +++++---- .../values/RascalFunctionValueFactory.java | 18 ++++++++++-------- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index cedafe73c61..2f2385d6ab1 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -449,14 +449,14 @@ catch ParseError(loc l): { } ---- } -public &T<:Tree parse(type[&T<:Tree] begin, str input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) - = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, |unknown:///|); +public &T<:Tree parse(type[&T<:Tree] begin, str input, bool allowAmbiguity=false, bool hasSideEffects=false, bool robust=false, set[Tree(Tree)] filters={}) + = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, robust=robust, filters=filters)(input, |unknown:///|); -public &T<:Tree parse(type[&T<:Tree] begin, str input, loc origin, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) - = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, origin); +public &T<:Tree parse(type[&T<:Tree] begin, str input, loc origin, bool allowAmbiguity=false, bool hasSideEffects=false, bool robust=false, set[Tree(Tree)] filters={}) + = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, robust=robust, filters=filters)(input, origin); -public &T<:Tree parse(type[&T<:Tree] begin, loc input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) - = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, input); +public &T<:Tree parse(type[&T<:Tree] begin, loc input, bool allowAmbiguity=false, bool hasSideEffects=false, bool robust=false, set[Tree(Tree)] filters={}) + = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, robust=robust, filters=filters)(input, input); @doc{ .Synopsis @@ -487,7 +487,7 @@ The parse function behaves differently depending of the given keyword parameters * parse forest to be constructed in polynomial time. } @javaClass{org.rascalmpl.library.Prelude} -public java &T (value input, loc origin) parser(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, set[Tree(Tree)] filters={}); +public java &T (value input, loc origin) parser(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, bool robust=false, set[Tree(Tree)] filters={}); @doc{ .Synopsis @@ -499,7 +499,7 @@ This parser generator behaves the same as the `parser` function, but it produces nonterminal parameter. This can be used to select a specific non-terminal from the grammar to use as start-symbol for parsing. } @javaClass{org.rascalmpl.library.Prelude} -public java &U (type[&U] nonterminal, value input, loc origin) parsers(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, set[Tree(Tree)] filters={}); +public java &U (type[&U] nonterminal, value input, loc origin) parsers(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, bool robust=false, set[Tree(Tree)] filters={}); @doc{ .Synopsis parse the input but instead of returning the entire tree, return the trees for the first ambiguous substring. diff --git a/src/org/rascalmpl/library/Prelude.java b/src/org/rascalmpl/library/Prelude.java index e096c3b0e5c..258d54c4aef 100644 --- a/src/org/rascalmpl/library/Prelude.java +++ b/src/org/rascalmpl/library/Prelude.java @@ -2237,12 +2237,12 @@ public INode arbNode() { protected final TypeReifier tr; - public IFunction parser(IValue start, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { - return rascalValues.parser(start, allowAmbiguity, hasSideEffects, firstAmbiguity, filters); + public IFunction parser(IValue start, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, IBool robust, ISet filters) { + return rascalValues.parser(start, allowAmbiguity, hasSideEffects, firstAmbiguity, robust, filters); } - public IFunction parsers(IValue start, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { - return rascalValues.parsers(start, allowAmbiguity, hasSideEffects, firstAmbiguity, filters); + public IFunction parsers(IValue start, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, IBool robust, ISet filters) { + return rascalValues.parsers(start, allowAmbiguity, hasSideEffects, firstAmbiguity, robust, filters); } // REFLECT -- copy in {@link PreludeCompiled} diff --git a/src/org/rascalmpl/values/IRascalValueFactory.java b/src/org/rascalmpl/values/IRascalValueFactory.java index 405924792bf..f43d662b1ef 100644 --- a/src/org/rascalmpl/values/IRascalValueFactory.java +++ b/src/org/rascalmpl/values/IRascalValueFactory.java @@ -87,12 +87,15 @@ default IFunction function(Type functionType, BiFunction Date: Thu, 31 Mar 2022 12:37:15 +0200 Subject: [PATCH 010/190] wired boolean parameter for robustness from Rascal function down to parser implementation --- src/org/rascalmpl/interpreter/Evaluator.java | 18 +++++++++-------- src/org/rascalmpl/interpreter/IEvaluator.java | 8 ++++---- .../semantics/dynamic/Expression.java | 4 ++-- .../values/RascalFunctionValueFactory.java | 20 +++++++++---------- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/org/rascalmpl/interpreter/Evaluator.java b/src/org/rascalmpl/interpreter/Evaluator.java index 2069f732aba..4a2929545b4 100755 --- a/src/org/rascalmpl/interpreter/Evaluator.java +++ b/src/org/rascalmpl/interpreter/Evaluator.java @@ -103,6 +103,7 @@ import org.rascalmpl.parser.uptr.UPTRNodeFactory; import org.rascalmpl.parser.uptr.action.NoActionExecutor; import org.rascalmpl.parser.uptr.action.RascalFunctionActionExecutor; +import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; import org.rascalmpl.uri.URIResolverRegistry; import org.rascalmpl.uri.URIUtil; import org.rascalmpl.values.parsetrees.ITree; @@ -714,7 +715,7 @@ public IValue call(QualifiedName qualifiedName, Map kwArgs, IValu } @Override - public ITree parseObject(IConstructor grammar, ISet filters, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean hasSideEffects) { + public ITree parseObject(IConstructor grammar, ISet filters, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean hasSideEffects, boolean robust) { IConstructor startSort = (IConstructor) grammar.get("symbol"); IGTD parser; String name; @@ -728,16 +729,17 @@ public ITree parseObject(IConstructor grammar, ISet filters, ISourceLocation loc ? new RascalFunctionActionExecutor(filters, !hasSideEffects) : new NoActionExecutor(); - return (ITree) parser.parse(name, location.getURI(), input, exec, new DefaultNodeFlattener(), new UPTRNodeFactory(allowAmbiguity), (IRecoverer) null); + IRecoverer recoverer = robust ? new ToNextWhitespaceRecoverer() : null; + return (ITree) parser.parse(name, location.getURI(), input, exec, new DefaultNodeFlattener(), new UPTRNodeFactory(allowAmbiguity), recoverer); } @Override - public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, ISet filters, ISourceLocation location, boolean allowAmbiguity, boolean hasSideEffects){ + public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, ISet filters, ISourceLocation location, boolean allowAmbiguity, boolean hasSideEffects, boolean robust){ IRascalMonitor old = setMonitor(monitor); try { char[] input = getResourceContent(location); - return parseObject(startSort, filters, location, input, allowAmbiguity, hasSideEffects); + return parseObject(startSort, filters, location, input, allowAmbiguity, hasSideEffects, robust); } catch(IOException ioex){ throw RuntimeExceptionFactory.io(vf.string(ioex.getMessage()), getCurrentAST(), getStackTrace()); @@ -748,10 +750,10 @@ public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, } @Override - public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, ISet filters, String input, boolean allowAmbiguity, boolean hasSideEffects) { + public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, ISet filters, String input, boolean allowAmbiguity, boolean hasSideEffects, boolean robust) { IRascalMonitor old = setMonitor(monitor); try { - return parseObject(startSort, filters, URIUtil.invalidLocation(), input.toCharArray(), allowAmbiguity, hasSideEffects); + return parseObject(startSort, filters, URIUtil.invalidLocation(), input.toCharArray(), allowAmbiguity, hasSideEffects, robust); } finally { setMonitor(old); @@ -759,10 +761,10 @@ public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, } @Override - public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, ISet filters, String input, ISourceLocation loc, boolean allowAmbiguity, boolean hasSideEffects) { + public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, ISet filters, String input, ISourceLocation loc, boolean allowAmbiguity, boolean hasSideEffects, boolean robust) { IRascalMonitor old = setMonitor(monitor); try{ - return parseObject(startSort, filters, loc, input.toCharArray(), allowAmbiguity, hasSideEffects); + return parseObject(startSort, filters, loc, input.toCharArray(), allowAmbiguity, hasSideEffects, robust); }finally{ setMonitor(old); } diff --git a/src/org/rascalmpl/interpreter/IEvaluator.java b/src/org/rascalmpl/interpreter/IEvaluator.java index 35f8065cfb1..2bb5fc69db6 100644 --- a/src/org/rascalmpl/interpreter/IEvaluator.java +++ b/src/org/rascalmpl/interpreter/IEvaluator.java @@ -144,16 +144,16 @@ public Result evalMore(IRascalMonitor monitor, String commands, */ public IValue call(String adt, String name, IValue... args); - public IConstructor parseObject(IConstructor startSort, ISet filters, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean hasSideEffects); + public IConstructor parseObject(IConstructor startSort, ISet filters, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean hasSideEffects, boolean robust); public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, - ISet filters, String input, ISourceLocation loc, boolean allowAmbiguity, boolean hasSideEffects); + ISet filters, String input, ISourceLocation loc, boolean allowAmbiguity, boolean hasSideEffects, boolean robust); public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, - ISet filters, String input, boolean allowAmbiguity, boolean hasSideEffects); + ISet filters, String input, boolean allowAmbiguity, boolean hasSideEffects, boolean robust); public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, - ISet filters, ISourceLocation location, boolean allowAmbiguity, boolean hasSideEffects); + ISet filters, ISourceLocation location, boolean allowAmbiguity, boolean hasSideEffects, boolean robust); /** * Freeze the global state of this evaluator so that it can no longer be updated. diff --git a/src/org/rascalmpl/semantics/dynamic/Expression.java b/src/org/rascalmpl/semantics/dynamic/Expression.java index 95adc37aedc..78117d90e3b 100644 --- a/src/org/rascalmpl/semantics/dynamic/Expression.java +++ b/src/org/rascalmpl/semantics/dynamic/Expression.java @@ -1089,11 +1089,11 @@ public Result interpret(IEvaluator> __eval) { if (result.getStaticType().isString()) { tree = __eval.parseObject(value, VF.set(), this.getLocation(), - ((IString) result.getValue()).getValue().toCharArray(), true, false); + ((IString) result.getValue()).getValue().toCharArray(), true, false, false); } else if (result.getStaticType().isSourceLocation()) { tree = __eval.parseObject(__eval, value, VF.set(), - ((ISourceLocation) result.getValue()), true, false); + ((ISourceLocation) result.getValue()), true, false, false); } assert tree != null; // because we checked earlier diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 616f4f57877..76a150ddd7d 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -229,10 +229,10 @@ else if (parameters[0].getType().isSourceLocation()) { } if (parameters[0].getType().isString()) { - return parse(grammar, filters, (IString) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, hasSideEffects, ctx); + return parse(grammar, filters, (IString) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, hasSideEffects, robust, ctx); } else if (parameters[0].getType().isSourceLocation()) { - return parse(grammar, filters, (ISourceLocation) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, hasSideEffects, ctx); + return parse(grammar, filters, (ISourceLocation) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, hasSideEffects, robust, ctx); } } @@ -243,7 +243,7 @@ protected Throw fail(IValue... parameters) { return RuntimeExceptionFactory.callFailed(URIUtil.rootLocation("unknown"), Arrays.stream(parameters).collect(ctx.getValueFactory().listWriter())); } - protected IValue parse(IValue start, ISet filters, IString input, ISourceLocation origin, boolean allowAmbiguity, boolean hasSideEffects, IEvaluatorContext ctx) { + protected IValue parse(IValue start, ISet filters, IString input, ISourceLocation origin, boolean allowAmbiguity, boolean hasSideEffects, boolean robust, IEvaluatorContext ctx) { try { Type reified = start.getType(); if (origin == null) { @@ -251,7 +251,7 @@ protected IValue parse(IValue start, ISet filters, IString input, ISourceLocati } IConstructor grammar = checkPreconditions(start, reified); - return ctx.getEvaluator().parseObject(grammar, filters, origin, input.getValue().toCharArray(), allowAmbiguity, hasSideEffects); + return ctx.getEvaluator().parseObject(grammar, filters, origin, input.getValue().toCharArray(), allowAmbiguity, hasSideEffects, robust); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -271,7 +271,7 @@ protected IValue firstAmbiguity(IValue start, IString input, IEvaluatorContext c IConstructor grammar = checkPreconditions(start, reified); try { - return ctx.getEvaluator().parseObject(grammar, vf.set(), URIUtil.invalidLocation(), input.getValue().toCharArray(), false, false); + return ctx.getEvaluator().parseObject(grammar, vf.set(), URIUtil.invalidLocation(), input.getValue().toCharArray(), false, false, false); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -290,7 +290,7 @@ protected IValue firstAmbiguity(IValue start, ISourceLocation input, IEvaluatorC IConstructor grammar = checkPreconditions(start, reified); try { - return ctx.getEvaluator().parseObject(grammar, vf.set(), input, readAll(input), false, false); + return ctx.getEvaluator().parseObject(grammar, vf.set(), input, readAll(input), false, false, false); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -317,7 +317,7 @@ private IString printSymbol(IConstructor symbol) { return vf.string(SymbolAdapter.toString(symbol, false)); } - protected IValue parse(IValue start, ISet filters, ISourceLocation input, ISourceLocation origin, boolean allowAmbiguity, boolean hasSideEffects, IEvaluatorContext ctx) { + protected IValue parse(IValue start, ISet filters, ISourceLocation input, ISourceLocation origin, boolean allowAmbiguity, boolean hasSideEffects, boolean robust, IEvaluatorContext ctx) { Type reified = start.getType(); IConstructor grammar = checkPreconditions(start, reified); @@ -326,7 +326,7 @@ protected IValue parse(IValue start, ISet filters, ISourceLocation input, ISourc } try { - return ctx.getEvaluator().parseObject(grammar, vf.set(), input, readAll(input), allowAmbiguity, hasSideEffects); + return ctx.getEvaluator().parseObject(grammar, vf.set(), input, readAll(input), allowAmbiguity, hasSideEffects, robust); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -397,10 +397,10 @@ else if (parameters[1].getType().isSourceLocation()) { } if (parameters[1].getType().isString()) { - return parse(parameters[0], filters, (IString) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, hasSideEffects, ctx); + return parse(parameters[0], filters, (IString) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, hasSideEffects, robust, ctx); } else if (parameters[1].getType().isSourceLocation()) { - return parse(parameters[0], filters, (ISourceLocation) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, hasSideEffects, ctx); + return parse(parameters[0], filters, (ISourceLocation) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, hasSideEffects, robust, ctx); } } From 7318e8f4d11a565e289cfb21c8c56852469109dd Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 12:49:17 +0200 Subject: [PATCH 011/190] added override --- .../parser/uptr/recovery/ToNextWhitespaceRecoverer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 7d2ceddbb79..bdb6c8665e1 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -173,6 +173,7 @@ private void collectProductions(AbstractStackNode node, ArrayList< } } + @Override public DoubleArrayList, AbstractNode> reviveStacks(int[] input, int location, Stack> unexpandableNodes, From f6e13d88bf8ebf965497298c6b3fb0d5e6e59dfc Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 11 Apr 2022 17:19:24 +0200 Subject: [PATCH 012/190] minor additions to make recovery work and removed dead code --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 2 +- .../parser/gtd/stack/SkippingStackNode.java | 4 +- .../recovery/ToNextWhitespaceRecoverer.java | 51 ++----------------- 3 files changed, 9 insertions(+), 48 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 07e35861917..6e0c8e31845 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -815,7 +815,7 @@ private boolean findStacksToReduce(){ } } - if (recoverer != null) { + if (recoverer != null && location < input.length) { DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (recoveredNodes.size() > 0) { for (int i = 0; i < recoveredNodes.size(); i++) { diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index eb243b3b6c0..63164ab3bc7 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -91,7 +91,9 @@ public int hashCode(){ } public boolean isEqual(AbstractStackNode

stackNode){ - if(!(stackNode instanceof SkippingStackNode)) return false; + if ( !(stackNode instanceof SkippingStackNode)) { + return false; + } SkippingStackNode

otherNode = (SkippingStackNode

) stackNode; diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index bdb6c8665e1..f195e71a0b3 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -24,7 +24,6 @@ import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; -import org.rascalmpl.values.parsetrees.ProductionAdapter; import io.usethesource.vallang.IConstructor; @@ -110,24 +109,20 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr while (!todo.isEmpty()) { AbstractStackNode node = todo.pop(); - if(visited.contains(node)) continue; // Don't follow cycles - visited.put(node, 0); - - ArrayList recoveryProductions = new ArrayList(); - collectProductions(node, recoveryProductions); - - if (recoveryProductions.size() > 0) { - recoveryNodes.add(node, recoveryProductions); + if (visited.contains(node)) { + continue; // Don't follow cycles } + visited.put(node, 0); + IntegerObjectList> edges = node.getEdges(); for (int i = edges.size() - 1; i >= 0; --i) { EdgesSet edgesList = edges.getValue(i); if (edgesList != null) { - for(int j = edgesList.size() - 1; j >= 0; --j){ + for (int j = edgesList.size() - 1; j >= 0; --j) { AbstractStackNode parent = edgesList.get(j); todo.push(parent); @@ -137,42 +132,6 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr } } - // Gathers all productions that are marked for recovery (the given node can be part of a prefix shared production) - private void collectProductions(AbstractStackNode node, ArrayList productions) { - AbstractStackNode[] production = node.getProduction(); - if(production == null) return; // The root node does not have a production, so ignore it. - - int dot = node.getDot(); - - if (node.isEndNode()) { - IConstructor parentProduction = node.getParentProduction(); - if(ProductionAdapter.isContextFree(parentProduction)){ - productions.add(parentProduction); - - if (ProductionAdapter.isList(parentProduction)) { - return; // Don't follow productions in lists productions, since they are 'cyclic'. - } - } - } - - for (int i = dot + 1; i < production.length; ++i) { - AbstractStackNode currentNode = production[i]; - if (currentNode.isEndNode()) { - IConstructor parentProduction = currentNode.getParentProduction(); - if (ProductionAdapter.isContextFree(parentProduction)) { - productions.add(parentProduction); - } - } - - AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); - if (alternateProductions != null) { - for (int j = alternateProductions.length - 1; j >= 0; --j) { - collectProductions(alternateProductions[j][i], productions); - } - } - } - } - @Override public DoubleArrayList, AbstractNode> reviveStacks(int[] input, int location, From ba0fb43fc8207d12999924ac3c72aff398aa1cfc Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 13 Apr 2022 16:28:46 +0200 Subject: [PATCH 013/190] recovered a missing piece from the recovery code --- .../recovery/ToNextWhitespaceRecoverer.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index f195e71a0b3..bd94e061868 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -24,6 +24,7 @@ import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.values.parsetrees.ProductionAdapter; import io.usethesource.vallang.IConstructor; @@ -116,6 +117,12 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr visited.put(node, 0); + ArrayList recoveryProductions = new ArrayList(); + collectProductions(node, recoveryProductions); + if (recoveryProductions.size() > 0) { + recoveryNodes.add(node, recoveryProductions); + } + IntegerObjectList> edges = node.getEdges(); for (int i = edges.size() - 1; i >= 0; --i) { @@ -132,6 +139,45 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr } } + // Gathers all productions that are marked for recovery (the given node can be part of a prefix shared production) + private void collectProductions(AbstractStackNode node, ArrayList productions) { + AbstractStackNode[] production = node.getProduction(); + if (production == null) { + return; // The root node does not have a production, so ignore it. + } + + int dot = node.getDot(); + + if (node.isEndNode()) { + IConstructor parentProduction = node.getParentProduction(); + if (ProductionAdapter.isContextFree(parentProduction)){ + productions.add(parentProduction); + + if (ProductionAdapter.isList(parentProduction)) { + return; // Don't follow productions in lists productions, since they are 'cyclic'. + } + } + } + + for (int i = dot + 1; i < production.length; ++i) { + AbstractStackNode currentNode = production[i]; + if (currentNode.isEndNode()) { + IConstructor parentProduction = currentNode.getParentProduction(); + if (ProductionAdapter.isContextFree(parentProduction)) { + productions.add(parentProduction); + } + } + + AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); + if (alternateProductions != null) { + for (int j = alternateProductions.length - 1; j >= 0; --j) { + collectProductions(alternateProductions[j][i], productions); + } + } + } + } + + @Override public DoubleArrayList, AbstractNode> reviveStacks(int[] input, int location, From 650224af21b441a874f0e9241f85474470c8bcd9 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 14 Apr 2022 17:23:15 +0200 Subject: [PATCH 014/190] Debugging error recovery Sometimes recovery nodes start before the current location where the parser failed to continue. Since the parser works with a short queue of schedulede TODO's around the current cursor, we might end up outside of this queue when recovering. This breaks several unspecified invariants of the SGTBF implementations. For now I added a detection that a recovery node is to be planned before the currently retained history and filter that recovery node. The next step will be to make sure backtracking over the current location is made possible. --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 6e0c8e31845..83b7578afb7 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -823,7 +823,9 @@ private boolean findStacksToReduce(){ int levelsFromHere = recovered.getLength() - (location - recovered.getStartLocation()); - addTodo(recovered, levelsFromHere, recoveredNodes.getSecond(i)); +// if (levelsFromHere >= 0) { // TODO experien + addTodo(recovered, levelsFromHere, recoveredNodes.getSecond(i)); +// } } return findStacksToReduce(); } @@ -853,6 +855,13 @@ private void addTodo(AbstractStackNode

node, int length, AbstractNode result) queueDepth = length + 1; queueIndex = 0; } + else if (length < 0) { + if (length + queueIndex < 0) { + // the queue is not long enough back into the past locations + // to cover this recovery node, so we must skip it + return; + } + } int insertLocation = (queueIndex + length) % queueDepth; DoubleStack, AbstractNode> terminalsTodo = todoLists[insertLocation]; From 6d5d49b64a2a166dd19c0da9f7e7786e3e29634b Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 15 Apr 2022 10:54:51 +0200 Subject: [PATCH 015/190] added possibility for recovered nodes to start back in time (at earlier input locations) --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 81 +++++++++++++++++++----- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 83b7578afb7..c2777fbea8f 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -786,7 +786,7 @@ private boolean findFirstStacksToReduce(){ if (recoveredNodes.size() > 0) { // TODO Do something with the revived node. Is this the right location to do this? for (int i = 0; i < recoveredNodes.size(); i++) { AbstractStackNode

recovered = recoveredNodes.getFirst(i); - addTodo(recovered, recovered.getLength(), recoveredNodes.getSecond(i)); + queueMatchableNode(recovered, recovered.getLength(), recoveredNodes.getSecond(i)); } return findStacksToReduce(); } @@ -821,11 +821,9 @@ private boolean findStacksToReduce(){ for (int i = 0; i < recoveredNodes.size(); i++) { AbstractStackNode

recovered = recoveredNodes.getFirst(i); - int levelsFromHere = recovered.getLength() - (location - recovered.getStartLocation()); +// int levelsFromHere = recovered.getLength() - (location - recovered.getStartLocation()); -// if (levelsFromHere >= 0) { // TODO experien - addTodo(recovered, levelsFromHere, recoveredNodes.getSecond(i)); -// } + queueRecoveryNode(recovered, recovered.getStartLocation(), recovered.getLength(), recoveredNodes.getSecond(i)); } return findStacksToReduce(); } @@ -844,8 +842,9 @@ public boolean parseErrorHasOccurred(){ * Inserts a stack bottom into the todo-list. */ @SuppressWarnings("unchecked") - private void addTodo(AbstractStackNode

node, int length, AbstractNode result){ - if(result == null) throw new RuntimeException(); + private void queueMatchableNode(AbstractStackNode

node, int length, AbstractNode result){ + assert result != null; + int queueDepth = todoLists.length; if(length >= queueDepth){ DoubleStack, AbstractNode>[] oldTodoLists = todoLists; @@ -855,13 +854,6 @@ private void addTodo(AbstractStackNode

node, int length, AbstractNode result) queueDepth = length + 1; queueIndex = 0; } - else if (length < 0) { - if (length + queueIndex < 0) { - // the queue is not long enough back into the past locations - // to cover this recovery node, so we must skip it - return; - } - } int insertLocation = (queueIndex + length) % queueDepth; DoubleStack, AbstractNode> terminalsTodo = todoLists[insertLocation]; @@ -872,6 +864,61 @@ else if (length < 0) { terminalsTodo.push(node, result); } + /** + * Inserts a recovery node into the todo-list, and possibly + * rewinds the parser to an earlier location in the input + */ + @SuppressWarnings("unchecked") + private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int length, AbstractNode result){ + assert result != null; + + int queueDepth = todoLists.length; + + if (startPosition < location) { + // Have to reset the parser to an earlier location to at least + // be able to process the new node. Cannot throw away the queue, + // because there are possibly already other recovery tokens in the queue. + // However, we may assume that the queue before the current index is + // done, based on the way we cycle the queue now. The queue is + // looking forward to the future and we never re-use past entries. + + int negativeOffset = location - startPosition; + + DoubleStack, AbstractNode>[] oldTodoLists = todoLists; + todoLists = new DoubleStack[negativeOffset + Math.max(queueDepth, length) + 1]; + System.arraycopy(oldTodoLists, queueIndex, todoLists, negativeOffset, queueDepth - queueIndex); + System.arraycopy(oldTodoLists, 0, todoLists, queueDepth - queueIndex + negativeOffset, queueIndex); + + // reset the parser! + queueIndex = 0; + location = startPosition; + + DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; + if (terminalsTodo == null){ + terminalsTodo = new DoubleStack, AbstractNode>(); + todoLists[queueIndex] = terminalsTodo; + } + else { + assert false: "this should never happen"; + } + + terminalsTodo.push(node, result); + } + else if (startPosition == location) { + // this is the normal case where new matchable nodes are discovered + // for the current parsing location, so reuse the code for queuing + queueMatchableNode(node, length, result); + } + else { + // This would mean we have discovered a recovery node for a location + // we have not been yet. That would be odd because then there would + // not have been a parse error and we wouldn't need recovery... + throw new RuntimeException("discovered a future recovery? " + node); + } + } + + + /** * Handles the retrieved alternatives for the given stack. */ @@ -913,7 +960,7 @@ private boolean handleExpects(AbstractStackNode

[] expects, EdgesSet

cached first = first.getCleanCopyWithResult(location, result); - addTodo(first, length, result); + queueMatchableNode(first, length, result); }else{ first = first.getCleanCopy(location); stacksToExpand.push(first); @@ -974,7 +1021,7 @@ private void expandStack(AbstractStackNode

stack){ } if(stack.isMatchable()){ // Eager matching optimization related. - addTodo(stack, stack.getLength(), stack.getResult()); + queueMatchableNode(stack, stack.getLength(), stack.getResult()); }else if(!stack.isExpandable()){ // A 'normal' non-terminal. EdgesSet

cachedEdges = cachedEdgesForExpect.get(stack.getName()); if(cachedEdges == null){ @@ -1051,7 +1098,7 @@ private void expandStack(AbstractStackNode

stack){ } child = child.getCleanCopyWithResult(location, result); - addTodo(child, length, result); + queueMatchableNode(child, length, result); }else{ child = child.getCleanCopy(location); stacksToExpand.push(child); From 78ee76be767a99c77256e38dce44618af01d4f5f Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 15 Apr 2022 11:02:26 +0200 Subject: [PATCH 016/190] fixed off-by-one: error nodes should be scheduled one character ahead because the next parser loop iteration always wants to advance one character --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index c2777fbea8f..1ac29bbd8db 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -886,17 +886,17 @@ private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int DoubleStack, AbstractNode>[] oldTodoLists = todoLists; todoLists = new DoubleStack[negativeOffset + Math.max(queueDepth, length) + 1]; - System.arraycopy(oldTodoLists, queueIndex, todoLists, negativeOffset, queueDepth - queueIndex); - System.arraycopy(oldTodoLists, 0, todoLists, queueDepth - queueIndex + negativeOffset, queueIndex); + System.arraycopy(oldTodoLists, queueIndex, todoLists, negativeOffset + 1, queueDepth - queueIndex); + System.arraycopy(oldTodoLists, 0, todoLists, queueDepth - queueIndex + negativeOffset + 1, queueIndex); // reset the parser! queueIndex = 0; location = startPosition; - DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; + DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex + 1]; if (terminalsTodo == null){ terminalsTodo = new DoubleStack, AbstractNode>(); - todoLists[queueIndex] = terminalsTodo; + todoLists[queueIndex + 1] = terminalsTodo; } else { assert false: "this should never happen"; From d9f8fc971dfdc9b01324a46a0a5e8543b8efadeb Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 15 Apr 2022 11:12:30 +0200 Subject: [PATCH 017/190] fixed another off-by-one --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 1ac29bbd8db..08a60529fd4 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -891,7 +891,7 @@ private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int // reset the parser! queueIndex = 0; - location = startPosition; + location = startPosition + 1; DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex + 1]; if (terminalsTodo == null){ From 685198dadd38c6b4114f42877782fc70413da47c Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 15 Apr 2022 11:28:08 +0200 Subject: [PATCH 018/190] added initial construction of skipped nodes --- src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java | 4 +++- src/org/rascalmpl/values/RascalValueFactory.java | 1 + src/org/rascalmpl/values/parsetrees/ProductionAdapter.java | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index 8a6199fa2a0..3bb39dbda0d 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -1,6 +1,7 @@ package org.rascalmpl.parser.uptr; import java.net.URI; +import java.util.Arrays; import java.util.IdentityHashMap; import java.util.Map; @@ -142,6 +143,7 @@ public Object getProductionFromNode(ITree node){ @Override public ITree createRecoveryNode(int[] characters) { - throw new UnsupportedOperationException(); + IList chars = Arrays.stream(characters).mapToObj(ch -> VF.character(ch)).collect(VF.listWriter()); + return VF.appl(VF.constructor(RascalValueFactory.Production_Skipped), chars); } } diff --git a/src/org/rascalmpl/values/RascalValueFactory.java b/src/org/rascalmpl/values/RascalValueFactory.java index c7bca997e1b..34f57cf0b29 100644 --- a/src/org/rascalmpl/values/RascalValueFactory.java +++ b/src/org/rascalmpl/values/RascalValueFactory.java @@ -152,6 +152,7 @@ public class RascalValueFactory extends AbstractValueFactoryAdapter implements I public static final Type Production_Priority = tf.constructor(uptr, Production, "priority", Symbol, "def", tf.listType(Production), "choices"); public static final Type Production_Composition = tf.constructor(uptr, Production, "composition", Production, "lhs", Production, "rhs"); public static final Type Production_Associativity = tf.constructor(uptr, Production, "associativity", Symbol, "def", Associativity, "assoc", tf.setType(Production), "alternatives"); + public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped"); /* Constructors for Attr */ public static final Type Attr_Assoc = tf.constructor(uptr, Attr, "assoc", Associativity, "assoc"); diff --git a/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java b/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java index 591d781a674..be88043ebb0 100644 --- a/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java +++ b/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java @@ -138,6 +138,10 @@ public static boolean isDefault(IConstructor tree) { public static boolean isRegular(IConstructor tree) { return tree.getConstructorType() == RascalValueFactory.Production_Regular; } + + public static boolean isSkipped(IConstructor tree) { + return tree.getConstructorType() == RascalValueFactory.Production_Skipped; + } public static boolean isSeparatedList(IConstructor tree) { IConstructor rhs = getType(tree); From 1d5435bac11fba6eb32451213cbe1b73fb99cc97 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 15 Apr 2022 11:37:35 +0200 Subject: [PATCH 019/190] gave skipped productions a type such that all trees have a type --- src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java | 3 ++- src/org/rascalmpl/values/RascalValueFactory.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index 3bb39dbda0d..5988650baaa 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -23,6 +23,7 @@ public class UPTRNodeFactory implements INodeConstructorFactory{ private final static RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory(); + private final static IConstructor SKIPPED = VF.constructor(RascalValueFactory.Production_Skipped, VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Range, VF.integer(1), VF.integer(Character.MAX_CODE_POINT))))); private boolean allowAmb; public UPTRNodeFactory(boolean allowAmbiguity){ @@ -144,6 +145,6 @@ public Object getProductionFromNode(ITree node){ @Override public ITree createRecoveryNode(int[] characters) { IList chars = Arrays.stream(characters).mapToObj(ch -> VF.character(ch)).collect(VF.listWriter()); - return VF.appl(VF.constructor(RascalValueFactory.Production_Skipped), chars); + return VF.appl(SKIPPED, chars); } } diff --git a/src/org/rascalmpl/values/RascalValueFactory.java b/src/org/rascalmpl/values/RascalValueFactory.java index 34f57cf0b29..832eebc7800 100644 --- a/src/org/rascalmpl/values/RascalValueFactory.java +++ b/src/org/rascalmpl/values/RascalValueFactory.java @@ -152,7 +152,7 @@ public class RascalValueFactory extends AbstractValueFactoryAdapter implements I public static final Type Production_Priority = tf.constructor(uptr, Production, "priority", Symbol, "def", tf.listType(Production), "choices"); public static final Type Production_Composition = tf.constructor(uptr, Production, "composition", Production, "lhs", Production, "rhs"); public static final Type Production_Associativity = tf.constructor(uptr, Production, "associativity", Symbol, "def", Associativity, "assoc", tf.setType(Production), "alternatives"); - public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped"); + public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped", Symbol, "def"); /* Constructors for Attr */ public static final Type Attr_Assoc = tf.constructor(uptr, Attr, "assoc", Associativity, "assoc"); From 5265ef09cbacb8426a6cb96827e5bf1803617c47 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 20 Apr 2022 11:56:30 +0200 Subject: [PATCH 020/190] updated template --- .../ISSUE_TEMPLATE/stable-release-manual-testing-template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/stable-release-manual-testing-template.md b/.github/ISSUE_TEMPLATE/stable-release-manual-testing-template.md index 645e4465a8e..2bd1728f945 100644 --- a/.github/ISSUE_TEMPLATE/stable-release-manual-testing-template.md +++ b/.github/ISSUE_TEMPLATE/stable-release-manual-testing-template.md @@ -34,7 +34,7 @@ First a "pre-release" of the supporting compiler/typechecker tools must be done, # Manual feature tests -- [ ] Eclipse download and install latest unstable release from update site https://update.rascal-mpl.org/unstable +- [ ] Eclipse download and install latest unstable release from update site https://releases.usethesource.io/maven/org/rascalmpl/rascal-update-site/ - [ ] Open a Rascal REPL using the toolbar button - [ ] Can create new Rascal project using the wizard - [ ] Can create new Rascal module using the wizard From a2162c9316a437fc6d53cc543779ae74cf33b39e Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 20 Apr 2022 15:38:00 +0200 Subject: [PATCH 021/190] bumped rascal-maven-plugin to 0.8.0 to see if we can benefit from optimizations --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8c16b33c736..a03c136c879 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,7 @@ org.rascalmpl rascal-maven-plugin - 0.7.4 + 0.8.0 true ${project.build.outputDirectory} From 91fb32bb77df44314ba693713f9739886e67f95a Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 21 Apr 2022 17:23:53 +0200 Subject: [PATCH 022/190] upgraded to rascal-maven-plugin 0.8.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a03c136c879..12667de5e3a 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,7 @@ org.rascalmpl rascal-maven-plugin - 0.8.0 + 0.8.1 true ${project.build.outputDirectory} From 018826fc305dde4f9a079a1772d45b569ea9d205 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 21 Apr 2022 18:31:28 +0200 Subject: [PATCH 023/190] [maven-release-plugin] prepare release v0.23.1 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 12667de5e3a..9d2872e64b8 100644 --- a/pom.xml +++ b/pom.xml @@ -3,12 +3,12 @@ org.rascalmpl rascal - 0.23.1-SNAPSHOT + 0.23.1 jar scm:git:ssh://git@github.com/usethesource/rascal.git - HEAD + v0.23.1 From 7b6e8b82a6dc89d456385ff561c49087691cf084 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 21 Apr 2022 18:31:33 +0200 Subject: [PATCH 024/190] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 9d2872e64b8..03de13d9e2f 100644 --- a/pom.xml +++ b/pom.xml @@ -3,12 +3,12 @@ org.rascalmpl rascal - 0.23.1 + 0.23.2-SNAPSHOT jar scm:git:ssh://git@github.com/usethesource/rascal.git - v0.23.1 + HEAD From 4aefd529d5d21dc2d7c0ff21c5b3469612a241ed Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 22 Apr 2022 15:56:45 +0200 Subject: [PATCH 025/190] make sure that (a) BasicIDEServices are registered for the commandline version of Rascal and (b) the edit command is wired to the edit IDEService --- .../rascal/tutor/TutorCommandExecutor.java | 7 +++++-- src/org/rascalmpl/library/util/TermREPL.java | 5 ++++- src/org/rascalmpl/repl/BaseREPL.java | 6 +++--- src/org/rascalmpl/repl/ILanguageProtocol.java | 4 +++- .../rascalmpl/repl/RascalInterpreterREPL.java | 7 ++++--- .../semantics/dynamic/ShellCommand.java | 19 +++++++++++++++++++ src/org/rascalmpl/shell/REPLRunner.java | 13 +++++++++---- 7 files changed, 47 insertions(+), 14 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tutor/TutorCommandExecutor.java b/src/org/rascalmpl/library/lang/rascal/tutor/TutorCommandExecutor.java index 533245d5f2c..4261a9a8de3 100644 --- a/src/org/rascalmpl/library/lang/rascal/tutor/TutorCommandExecutor.java +++ b/src/org/rascalmpl/library/lang/rascal/tutor/TutorCommandExecutor.java @@ -16,6 +16,8 @@ import java.util.Map; import java.util.UUID; +import org.rascalmpl.ideservices.BasicIDEServices; +import org.rascalmpl.ideservices.IDEServices; import org.rascalmpl.interpreter.Evaluator; import org.rascalmpl.library.Prelude; import org.rascalmpl.library.util.PathConfig; @@ -41,14 +43,15 @@ public TutorCommandExecutor(PathConfig pcfg) throws IOException, URISyntaxExcept repl = new RascalInterpreterREPL(false, false, null) { @Override - protected Evaluator constructEvaluator(InputStream input, OutputStream stdout, OutputStream stderr) { + protected Evaluator constructEvaluator(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices services) { Evaluator eval = ShellEvaluatorFactory.getDefaultEvaluator(input, stdout, stderr); eval.getConfiguration().setRascalJavaClassPathProperty(javaCompilerPathAsString(pcfg.getJavaCompilerPath())); + eval.setMonitor(services); return eval; } }; - repl.initialize(shellInputNotUsed, shellStandardOutput, shellErrorOutput); + repl.initialize(shellInputNotUsed, shellStandardOutput, shellErrorOutput, null); repl.setMeasureCommandTime(false); } diff --git a/src/org/rascalmpl/library/util/TermREPL.java b/src/org/rascalmpl/library/util/TermREPL.java index fa9f98907d8..75becc5460d 100644 --- a/src/org/rascalmpl/library/util/TermREPL.java +++ b/src/org/rascalmpl/library/util/TermREPL.java @@ -14,6 +14,7 @@ import java.util.function.Function; import org.rascalmpl.exceptions.RuntimeExceptionFactory; +import org.rascalmpl.ideservices.IDEServices; import org.rascalmpl.interpreter.Evaluator; import org.rascalmpl.interpreter.IEvaluatorContext; import org.rascalmpl.interpreter.result.AbstractFunction; @@ -102,6 +103,7 @@ public static class TheREPL implements ILanguageProtocol { private final AbstractFunction completor; private final IValueFactory vf; private final AbstractFunction stacktrace; + private IDEServices services; public TheREPL(IValueFactory vf, IString title, IString welcome, IString prompt, IString quit, ISourceLocation history, IFunction handler, IFunction completor, IValue stacktrace, InputStream input, OutputStream stderr, OutputStream stdout) { @@ -144,10 +146,11 @@ public void stackTraceRequested() { } @Override - public void initialize(InputStream input, OutputStream stdout, OutputStream stderr) { + public void initialize(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices services) { this.stdout = stdout; this.stderr = stderr; this.input = input; + this.services = services; } @Override diff --git a/src/org/rascalmpl/repl/BaseREPL.java b/src/org/rascalmpl/repl/BaseREPL.java index 459df65964f..74217675357 100755 --- a/src/org/rascalmpl/repl/BaseREPL.java +++ b/src/org/rascalmpl/repl/BaseREPL.java @@ -110,7 +110,7 @@ else if (prettyPrompt) { else { this.errorWriter = new FilterWriter(reader.getOutput()) { }; // create a basic wrapper to avoid locking on stdout and stderr } - initialize(stdin, terminal.wrapOutIfNeeded(stdout) /*JURGEN LET OP reader.getOutput()*/, terminal.wrapOutIfNeeded(stderr)); + initialize(stdin, terminal.wrapOutIfNeeded(stdout) /*JURGEN LET OP reader.getOutput()*/, terminal.wrapOutIfNeeded(stderr), ideServices); if (supportsCompletion()) { reader.addCompleter(new Completer(){ @Override @@ -155,8 +155,8 @@ public static char ctrl(char ch) { * @throws IOException * @throws URISyntaxException */ - protected void initialize(InputStream input, OutputStream stdout, OutputStream stderr) throws IOException, URISyntaxException { - language.initialize(input, stdout, stderr); + protected void initialize(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices services) throws IOException, URISyntaxException { + language.initialize(input, stdout, stderr, services); } /** diff --git a/src/org/rascalmpl/repl/ILanguageProtocol.java b/src/org/rascalmpl/repl/ILanguageProtocol.java index 627ef161199..9a32ca88762 100755 --- a/src/org/rascalmpl/repl/ILanguageProtocol.java +++ b/src/org/rascalmpl/repl/ILanguageProtocol.java @@ -27,6 +27,8 @@ import java.io.OutputStream; import java.util.Map; +import org.rascalmpl.ideservices.IDEServices; + public interface ILanguageProtocol { @@ -35,7 +37,7 @@ public interface ILanguageProtocol { * @param stdout the output stream to write normal output to. * @param stderr the error stream to write error messages on, depending on the environment and options passed, will print in red. */ - void initialize(InputStream input, OutputStream stdout, OutputStream stderr); + void initialize(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices services); /** * Will be called everytime a new prompt is printed. diff --git a/src/org/rascalmpl/repl/RascalInterpreterREPL.java b/src/org/rascalmpl/repl/RascalInterpreterREPL.java index 5b9b6a0a9bf..5333e75a9b6 100644 --- a/src/org/rascalmpl/repl/RascalInterpreterREPL.java +++ b/src/org/rascalmpl/repl/RascalInterpreterREPL.java @@ -21,6 +21,7 @@ import org.rascalmpl.exceptions.StackTrace; import org.rascalmpl.exceptions.Throw; +import org.rascalmpl.ideservices.IDEServices; import org.rascalmpl.interpreter.Configuration; import org.rascalmpl.interpreter.Evaluator; import org.rascalmpl.interpreter.control_exceptions.InterruptException; @@ -70,11 +71,11 @@ public boolean getMeasureCommandTime() { } @Override - public void initialize(InputStream input, OutputStream stdout, OutputStream stderr) { - eval = constructEvaluator(input, stdout, stderr); + public void initialize(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices ideServices) { + eval = constructEvaluator(input, stdout, stderr, ideServices); } - protected abstract Evaluator constructEvaluator(InputStream input, OutputStream stdout, OutputStream stderr); + protected abstract Evaluator constructEvaluator(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices ideServices); @Override public PrintWriter getErrorWriter() { diff --git a/src/org/rascalmpl/semantics/dynamic/ShellCommand.java b/src/org/rascalmpl/semantics/dynamic/ShellCommand.java index 56f1e03a75d..d1e9ed311e1 100644 --- a/src/org/rascalmpl/semantics/dynamic/ShellCommand.java +++ b/src/org/rascalmpl/semantics/dynamic/ShellCommand.java @@ -15,6 +15,8 @@ import org.rascalmpl.ast.Expression; import org.rascalmpl.ast.QualifiedName; +import org.rascalmpl.debug.IRascalMonitor; +import org.rascalmpl.ideservices.IDEServices; import org.rascalmpl.interpreter.Configuration; import org.rascalmpl.interpreter.IEvaluator; import org.rascalmpl.interpreter.control_exceptions.QuitException; @@ -34,6 +36,23 @@ public Edit(ISourceLocation __param1, IConstructor tree, QualifiedName __param2) @Override public Result interpret(IEvaluator> __eval) { + IRascalMonitor monitor = __eval.getMonitor(); + + if (monitor instanceof IDEServices) { + IDEServices services = (IDEServices) monitor; + String name = Names.fullName(getName()); + + ISourceLocation uri = __eval.getRascalResolver().resolveModule(name); + if (uri == null) { + __eval.getOutPrinter().println("module " + name + " can not be found in the search path."); + } + + services.edit(uri); + } + else { + __eval.getOutPrinter().println("The current Rascal execution environment does not know how to start an editor."); + } + return org.rascalmpl.interpreter.result.ResultFactory.nothing(); } } diff --git a/src/org/rascalmpl/shell/REPLRunner.java b/src/org/rascalmpl/shell/REPLRunner.java index 708e7259978..0e82f29a4ce 100755 --- a/src/org/rascalmpl/shell/REPLRunner.java +++ b/src/org/rascalmpl/shell/REPLRunner.java @@ -5,9 +5,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.PrintWriter; import java.net.URISyntaxException; import java.util.Map; +import org.rascalmpl.ideservices.BasicIDEServices; +import org.rascalmpl.ideservices.IDEServices; import org.rascalmpl.interpreter.Evaluator; import org.rascalmpl.repl.BaseREPL; import org.rascalmpl.repl.ILanguageProtocol; @@ -33,11 +36,11 @@ private static File getHistoryFile() throws IOException { public REPLRunner(InputStream stdin, OutputStream stderr, OutputStream stdout, Terminal term) throws IOException, URISyntaxException { super(makeInterpreter(stdin, stderr, stdout, true, term.isAnsiSupported(), getHistoryFile(), term), null, - stdin, stderr, stdout, true, term.isAnsiSupported(), getHistoryFile(), term, null); + stdin, stderr, stdout, true, term.isAnsiSupported(), getHistoryFile(), term, new BasicIDEServices(new PrintWriter(stderr))); } public REPLRunner(ILanguageProtocol language) throws IOException, URISyntaxException { - super(language, null, null, null, null, true, true, new File(""), null, null); + super(language, null, null, null, null, true, true, new File(""), null, new BasicIDEServices(new PrintWriter(System.err))); } private static ILanguageProtocol makeInterpreter(InputStream stdin, OutputStream stderr, OutputStream stdout, @@ -46,8 +49,10 @@ private static ILanguageProtocol makeInterpreter(InputStream stdin, OutputStream RascalInterpreterREPL repl = new RascalInterpreterREPL(prettyPrompt, allowColors, getHistoryFile()) { @Override - protected Evaluator constructEvaluator(InputStream input, OutputStream stdout, OutputStream stderr) { - return ShellEvaluatorFactory.getDefaultEvaluator(input, stdout, stderr); + protected Evaluator constructEvaluator(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices services) { + Evaluator eval = ShellEvaluatorFactory.getDefaultEvaluator(input, stdout, stderr); + eval.setMonitor(services); + return eval; } @Override From dde9bd95da711fd53a620b2e0e5dcf1eed7cf98f Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 25 Apr 2022 09:46:50 +0200 Subject: [PATCH 026/190] basic IDE services can now also browse contents of files which are not in the scheme by copyinhthe contents to a tmp file --- .../ideservices/BasicIDEServices.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/org/rascalmpl/ideservices/BasicIDEServices.java b/src/org/rascalmpl/ideservices/BasicIDEServices.java index 8334de5bda2..8215b0b0a72 100644 --- a/src/org/rascalmpl/ideservices/BasicIDEServices.java +++ b/src/org/rascalmpl/ideservices/BasicIDEServices.java @@ -21,6 +21,8 @@ import java.nio.file.Paths; import org.rascalmpl.interpreter.ConsoleRascalMonitor; +import org.rascalmpl.uri.URIResolverRegistry; +import org.rascalmpl.uri.URIUtil; import io.usethesource.vallang.ISourceLocation; @@ -65,12 +67,21 @@ public void browse(URI uri){ } @Override - public void edit(ISourceLocation loc){ - if(loc.getScheme() != "file"){ - stderr.println("Can only edit files using the \"file\" scheme"); - return; + public void edit(ISourceLocation loc) { + try { + if (loc.getScheme() != "file") { + ISourceLocation tmp = URIUtil.correctLocation("tmp", "", "rascal-edits"); + tmp = URIUtil.getChildLocation(tmp, loc.getScheme()); + tmp = URIUtil.getChildLocation(tmp, loc.getPath()); + URIResolverRegistry.getInstance().copy(loc, tmp, false, true); + loc = URIResolverRegistry.getInstance().logicalToPhysical(tmp); } + edit(Paths.get(loc.getURI())); + } + catch (IOException e) { + stderr.println("Can not edit " + loc + " because: " + e); + } } /* (non-Javadoc) @@ -86,7 +97,7 @@ public void edit(Path path) { stderr.println(e.getMessage()); } } else { - stderr.println("Desktop not supported, cannout open editor"); + stderr.println("Desktop not supported, cannot open editor"); } } From 517df9cfb368ca58ec78f49f08acc14cd570f09b Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 10 Jun 2024 11:45:32 +0200 Subject: [PATCH 027/190] Started working on a simple recovery test --- src/org/rascalmpl/library/ParseTree.rsc | 1 + .../parser/gtd/util/IdDispenser.java | 5 + .../recovery/ToNextWhitespaceRecoverer.java | 14 +- .../rascalmpl/test/parser/IParserTest.java | 20 +++ .../rascalmpl/test/parser/RecoveryTests.java | 137 ++++++++++++++++++ 5 files changed, 171 insertions(+), 6 deletions(-) create mode 100644 src/org/rascalmpl/parser/gtd/util/IdDispenser.java create mode 100644 test/org/rascalmpl/test/parser/RecoveryTests.java diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 3289f88fd2c..2813f58b661 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -189,6 +189,7 @@ data Production | \reference(Symbol def, str cons) // <5> ; +data Production = skipped(Symbol def, Production prod, int validPrefix); @synopsis{Attributes in productions.} @description{ diff --git a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java new file mode 100644 index 00000000000..3952d7fa3c0 --- /dev/null +++ b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java @@ -0,0 +1,5 @@ +package org.rascalmpl.parser.gtd.util; + +public interface IdDispenser { + int dispenseId(); +} diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index bd94e061868..70730462e27 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -21,6 +21,7 @@ import org.rascalmpl.parser.gtd.util.ArrayList; import org.rascalmpl.parser.gtd.util.DoubleArrayList; import org.rascalmpl.parser.gtd.util.DoubleStack; +import org.rascalmpl.parser.gtd.util.IdDispenser; import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; @@ -28,12 +29,10 @@ import io.usethesource.vallang.IConstructor; -public class ToNextWhitespaceRecoverer implements IRecoverer{ - // TODO: its a magic constant, and it may clash with other generated constants - // should generate implementation of static int getLastId() in generated parser to fix this. - private int recoveryId = 100000; +public class ToNextWhitespaceRecoverer implements IRecoverer { private static final int[] WHITESPACE = {' ', '\r', '\n', '\t' }; + private IdDispenser stackNodeIdDispenser; private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); @@ -46,11 +45,11 @@ private DoubleArrayList, AbstractNode> reviveNod for (int j = prods.size() - 1; j >= 0; --j) { IConstructor prod = prods.get(j); - AbstractStackNode continuer = new RecoveryPointStackNode(recoveryId++, prod, recoveryNode); + AbstractStackNode continuer = new RecoveryPointStackNode(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); int startLocation = recoveryNode.getStartLocation(); - AbstractStackNode recoverLiteral = new SkippingStackNode(recoveryId++, WHITESPACE, input, startLocation, prod); + AbstractStackNode recoverLiteral = new SkippingStackNode(stackNodeIdDispenser.dispenseId(), WHITESPACE, input, startLocation, prod); recoverLiteral = recoverLiteral.getCleanCopy(startLocation); recoverLiteral.initEdges(); EdgesSet edges = new EdgesSet(1); @@ -177,6 +176,9 @@ private void collectProductions(AbstractStackNode node, ArrayList< } } + public ToNextWhitespaceRecoverer(IdDispenser stackNodeIdDispenser) { + this.stackNodeIdDispenser = stackNodeIdDispenser; + } @Override public DoubleArrayList, AbstractNode> reviveStacks(int[] input, diff --git a/test/org/rascalmpl/test/parser/IParserTest.java b/test/org/rascalmpl/test/parser/IParserTest.java index 9cf98df7cb2..a9598a8239e 100644 --- a/test/org/rascalmpl/test/parser/IParserTest.java +++ b/test/org/rascalmpl/test/parser/IParserTest.java @@ -13,13 +13,33 @@ import java.io.IOException; +import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IValue; import io.usethesource.vallang.IValueFactory; + +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.values.ValueFactoryFactory; import org.rascalmpl.values.parsetrees.ITree; public interface IParserTest{ public final static IValueFactory VF = ValueFactoryFactory.getValueFactory(); + + @SafeVarargs + public static AbstractStackNode[] createExpectArray(IConstructor prod, AbstractStackNode... nodes) { + @SuppressWarnings({"unchecked", "cast"}) + AbstractStackNode[] expectArray = (AbstractStackNode[]) new AbstractStackNode[nodes.length]; + + int index = 0; + for (AbstractStackNode node : nodes) { + expectArray[index] = node; + node.setProduction(expectArray); + index++; + } + + expectArray[index-1].setAlternativeProduction(prod); + + return expectArray; + } ITree executeParser(); diff --git a/test/org/rascalmpl/test/parser/RecoveryTests.java b/test/org/rascalmpl/test/parser/RecoveryTests.java new file mode 100644 index 00000000000..869b74bca6e --- /dev/null +++ b/test/org/rascalmpl/test/parser/RecoveryTests.java @@ -0,0 +1,137 @@ +package org.rascalmpl.test.parser; + +import java.io.IOException; +import java.io.StringReader; + +import org.junit.Assert; +import org.junit.Test; +import org.rascalmpl.parser.gtd.SGTDBF; +import org.rascalmpl.parser.gtd.recovery.IRecoverer; +import org.rascalmpl.parser.gtd.result.out.DefaultNodeFlattener; +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; +import org.rascalmpl.parser.gtd.stack.LiteralStackNode; +import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; +import org.rascalmpl.parser.uptr.UPTRNodeFactory; +import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; +import org.rascalmpl.values.RascalValueFactory; +import org.rascalmpl.values.ValueFactoryFactory; +import org.rascalmpl.values.parsetrees.ITree; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.ISourceLocation; +import io.usethesource.vallang.IValue; +import io.usethesource.vallang.io.StandardTextReader; + +/** + * S -> A ws B ws C + * A -> [a] ws [a] + * B -> [b] ws [b] + * C -> [c] ws [c] + * + * ws -> [\ ] + */ +public class RecoveryTests extends SGTDBF implements IParserTest{ + private final static IConstructor SYMBOL_START_S = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("S")); + + private final static IConstructor SYMBOL_ws = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("ws")); + private final static IConstructor SYMBOL_char_space = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(32)))); + + private final static IConstructor SYMBOL_A = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("A")); + private final static IConstructor SYMBOL_a = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("a")); + private final static IConstructor SYMBOL_char_a = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(97)))); + private final static IConstructor PROD_A_a_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_A, VF.list(SYMBOL_a, SYMBOL_ws, SYMBOL_a), VF.set()); + + private final static IConstructor SYMBOL_B = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("B")); + private final static IConstructor SYMBOL_b = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("b")); + private final static IConstructor SYMBOL_char_b = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(98)))); + private final static IConstructor PROD_B_b_b = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_B, VF.list(SYMBOL_b, SYMBOL_ws, SYMBOL_b), VF.set()); + + private final static IConstructor SYMBOL_C = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("C")); + private final static IConstructor SYMBOL_c = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("c")); + private final static IConstructor SYMBOL_char_c = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(99)))); + private final static IConstructor PROD_C_c_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_C, VF.list(SYMBOL_c, SYMBOL_ws, SYMBOL_c), VF.set()); + + private final static IConstructor PROD_S_A_B_C = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_START_S, VF.list(SYMBOL_A, SYMBOL_ws, SYMBOL_B, SYMBOL_ws, SYMBOL_C), VF.set()); + + private final static IConstructor PROD_ws = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_ws, VF.list(SYMBOL_char_space), VF.set()); + private final static IConstructor PROD_a_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_a, VF.list(SYMBOL_char_a), VF.set()); + private final static IConstructor PROD_b_b = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_b, VF.list(SYMBOL_char_b), VF.set()); + private final static IConstructor PROD_c_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_c, VF.list(SYMBOL_char_c), VF.set()); + + public AbstractStackNode[] S(){ + return IParserTest.createExpectArray(PROD_S_A_B_C, + new NonTerminalStackNode(1, 0, "A"), + new LiteralStackNode(2, 1, PROD_ws, new int[]{' '}), + new NonTerminalStackNode(3, 2, "B"), + new LiteralStackNode(4, 3, PROD_ws, new int[]{' '}), + new NonTerminalStackNode(5, 4, "C") + ); + } + + public AbstractStackNode[] A(){ + return IParserTest.createExpectArray(PROD_A_a_a, + new LiteralStackNode(6, 0, PROD_a_a, new int[]{'a'}), + new LiteralStackNode(7, 1, PROD_ws, new int[]{' '}), + new LiteralStackNode(8, 2, PROD_a_a, new int[]{'a'}) + ); + } + + public AbstractStackNode[] B(){ + return IParserTest.createExpectArray(PROD_B_b_b, + new LiteralStackNode(9, 0, PROD_b_b, new int[]{'b'}), + new LiteralStackNode(10, 1, PROD_ws, new int[]{' '}), + new LiteralStackNode(11, 2, PROD_b_b, new int[]{'b'}) + ); + } + + public AbstractStackNode[] C(){ + return IParserTest.createExpectArray(PROD_C_c_c, + new LiteralStackNode(12, 0, PROD_c_c, new int[]{'c'}), + new LiteralStackNode(13, 1, PROD_ws, new int[]{' '}), + new LiteralStackNode(14, 2, PROD_c_c, new int[]{'c'}) + ); + } + + private int nextFreeStackNodeId = 100; + + @Override + protected int getFreeStackNodeId() { + return nextFreeStackNodeId++; + } + + private ITree parse(String s) { + IRecoverer recoverer = new ToNextWhitespaceRecoverer(() -> nextFreeStackNodeId++); + return parse("S" /* NONTERMINAL_START_S */, null, s.toCharArray(), + new DefaultNodeFlattener(), new UPTRNodeFactory(false), recoverer, null); + } + + private ITree toTree(String s) { + try { + return (ITree) new StandardTextReader().read(ValueFactoryFactory.getValueFactory(), RascalValueFactory.uptr, RascalValueFactory.Tree, new StringReader(s)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public ITree executeParser() { + return parse("a a b b c c"); + } + + @Override + public IValue getExpectedResult() { + return toTree("appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\"),lit(\"ws\"),lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\"),lit(\"ws\"),lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"); + } + + @Test + public void testOk() { + Assert.assertEquals(getExpectedResult(), executeParser()); + } + + @Test + public void testMissingCharRecovery() { + String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\"),lit(\"ws\"),lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\"),lit(\"ws\"),lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; + Assert.assertEquals(toTree(expected), parse("a a b c c")); + } + +} From c779820cb9332ed3b56623056c796829cf96c098 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 17 Jun 2024 09:33:56 +0200 Subject: [PATCH 028/190] Added 'toString' methods and fixed issue in recoverer. - Added a lot of toString methods to help understading when debugging as VSCode uses toString to display the value of objects - Fixed iteration order in recoverer. I am not sure if the incoming nodes are always ordered by location, if not we might need to sort them in the future. --- .vscode/launch.json | 9 +- src/org/rascalmpl/parser/gtd/SGTDBF.java | 57 ++++++- .../parser/gtd/debug/IDebugListener.java | 13 ++ .../gtd/result/AbstractContainerNode.java | 36 +++++ .../rascalmpl/parser/gtd/result/CharNode.java | 6 + .../parser/gtd/result/EpsilonNode.java | 5 + .../gtd/result/ExpandableContainerNode.java | 5 + .../parser/gtd/result/LiteralNode.java | 11 ++ .../parser/gtd/result/RecoveredNode.java | 5 + .../parser/gtd/result/SkippedNode.java | 9 ++ .../parser/gtd/result/SortContainerNode.java | 5 + .../result/error/ErrorListContainerNode.java | 5 + .../result/error/ErrorSortContainerNode.java | 6 + .../parser/gtd/result/error/ExpectedNode.java | 8 + .../parser/gtd/result/struct/Link.java | 4 + .../gtd/stack/AbstractMatchableStackNode.java | 2 +- .../parser/gtd/stack/AbstractStackNode.java | 73 ++++++++- .../parser/gtd/stack/LiteralStackNode.java | 15 +- .../gtd/stack/NonTerminalStackNode.java | 9 +- .../parser/gtd/stack/SkippingStackNode.java | 22 ++- .../parser/gtd/stack/edge/EdgesSet.java | 33 +++- .../rascalmpl/parser/gtd/util/ArrayList.java | 13 ++ .../parser/gtd/util/DoubleArrayList.java | 16 ++ .../parser/gtd/util/DoubleStack.java | 15 ++ .../parser/gtd/util/IntegerObjectList.java | 16 ++ .../parser/uptr/debug/DebugLogger.java | 56 +++++++ .../recovery/ToNextWhitespaceRecoverer.java | 51 +++--- src/org/rascalmpl/parser/util/DebugUtil.java | 39 +++++ .../rascalmpl/unicode/UnicodeConverter.java | 12 ++ .../rascalmpl/test/parser/IParserTest.java | 2 +- .../rascalmpl/test/parser/RecoveryTests2.java | 148 ++++++++++++++++++ 31 files changed, 655 insertions(+), 51 deletions(-) create mode 100644 src/org/rascalmpl/parser/util/DebugUtil.java create mode 100644 src/org/rascalmpl/unicode/UnicodeConverter.java create mode 100644 test/org/rascalmpl/test/parser/RecoveryTests2.java diff --git a/.vscode/launch.json b/.vscode/launch.json index aa506d012ad..8d7a2f6a52c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,13 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + "type": "java", + "name": "Simple2", + "request": "launch", + "mainClass": "org.rascalmpl.test.parser.Simple2", + "projectName": "rascal" + }, { "type": "java", "name": "Launch DocRunner", @@ -38,7 +45,7 @@ "request": "launch", "mainClass": "org.rascalmpl.shell.RascalShell", "projectName": "rascal", - "cwd" : "${workspaceFolder}/../rascal-tutor", + "cwd": "${workspaceFolder}/../rascal-tutor", "vmArgs": "-Xss80m -Xmx2g -ea" }, { diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 771f3e6157c..458568f5643 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -54,24 +54,53 @@ public abstract class SGTDBF implements IGTD{ private URI inputURI; private int[] input; - + private int location; + protected int lookAheadChar; + + // A mapping between character location and line/column. private final PositionStore positionStore; + // Terminals that matched. Circular buffer indexed by length of the terminal. Each entry contains the node to reduce and the result node + // This is a circular buffer where queueIndex determines the start of the buffer. + // At each position, a stack is maintained of all terminals to reduce of a certain length. + // So at queueIndex+3, all terminals of length 3 that need reducing are stored. private DoubleStack, AbstractNode>[] todoLists; private int queueIndex; + // Stack of non-terminal nodes to expand + // - Nodes are removed in expand, which pops and expands all stack nodes on this stack + // - Nodes are added in: + // - parse: the root node is pushed + // - updateNextNode: next node of the production is pushed + // - updateAlternativeNextNode: next node of a prefix-shared production is pushed + // - handleExpects: non-matchable first element of each alternative is pushed + // - expandStack: when an expandable stack is expanded, all non-matchable children are pushed private final Stack> stacksToExpand; + + // The current stack of non-terminals to reduce. Each stack has a container node to accumulate results. + // - Nodes are removed in `reduce` where all productions are advanced one dot past the non-terminal node + // - Nodes are added in: + // - handleEdgeList: result container node is created and all edges are pushed with the same result container node + // - handleEdgeListWithRestrictions: result container node is created and all edges are pushed with the same result container node + // - expandStack: non-matchable, non-expandable nodes (and their existing result container node) are added if their name can be found in `cachedEdgesForExpect`. + // - expandStack: expandable nodes that are nullable? Might be a cycle thing private final DoubleStack, AbstractContainerNode

> stacksWithNonTerminalsToReduce; + + // The current stack of non-terminals to reduce: it contains the matchable node with the smallest length from todoLists. + // - Nodes are removed in `reduce` where all productions are advanced one dot past the matchable node + // - Variable is assigned in: + // - findFirstStacksToReduce: the first non-empty `todoList` is assigned to this variable + // - findStacksToReduce: again the first non-empty `todoList` is assigned to this variable + // - parse: variable is used in main reduce/expand loop to determine when it is time to look for more `stacksToReduce`. private DoubleStack, AbstractNode> stacksWithTerminalsToReduce; private final HashMap> cachedEdgesForExpect; private final IntegerKeyedDoubleValueHashMap, DoubleArrayList, AbstractNode>> sharedNextNodes; - private int location; - - protected int lookAheadChar; + // Reflection is used to get the expects for each non-terminal. + // This cache is used so the reflection call is only needed once. private final HashMap[]> expectCache; private final IntegerObjectList> sharedLastExpects; @@ -81,7 +110,7 @@ public abstract class SGTDBF implements IGTD{ // Error reporting private final Stack> unexpandableNodes; - private final Stack> unmatchableLeafNodes; + private final Stack> unmatchableLeafNodes; // Leaf nodes (for instance literals) that failed to match private final DoubleStack, AbstractNode>, AbstractStackNode

> unmatchableMidProductionNodes; private final DoubleStack, AbstractNode> filteredNodes; @@ -791,7 +820,13 @@ private boolean findFirstStacksToReduce(){ } if (recoverer != null) { + if (debugListener != null) { + debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + } DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + if (debugListener != null) { + debugListener.revived(recoveredNodes); + } if (recoveredNodes.size() > 0) { // TODO Do something with the revived node. Is this the right location to do this? for (int i = 0; i < recoveredNodes.size(); i++) { AbstractStackNode

recovered = recoveredNodes.getFirst(i); @@ -825,10 +860,16 @@ private boolean findStacksToReduce(){ } if (recoverer != null && location < input.length) { + if (debugListener != null) { + debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + } DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + debugListener.revived(recoveredNodes); if (recoveredNodes.size() > 0) { + + // for (int i = recoveredNodes.size()-1; i>= 0; i--) { for (int i = 0; i < recoveredNodes.size(); i++) { - AbstractStackNode

recovered = recoveredNodes.getFirst(i); + AbstractStackNode

recovered = recoveredNodes.getFirst(i); // int levelsFromHere = recovered.getLength() - (location - recovered.getStartLocation()); @@ -900,12 +941,12 @@ private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int // reset the parser! queueIndex = 0; - location = startPosition + 1; + location = startPosition + 1; DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex + 1]; if (terminalsTodo == null){ terminalsTodo = new DoubleStack, AbstractNode>(); - todoLists[queueIndex + 1] = terminalsTodo; + todoLists[queueIndex + 1] = terminalsTodo; // Why the +1 and not length? To get the recovered node to be processed first? } else { assert false: "this should never happen"; diff --git a/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java b/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java index ae412bde870..fc6a8a2bec3 100644 --- a/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java +++ b/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java @@ -16,6 +16,9 @@ import org.rascalmpl.parser.gtd.result.struct.Link; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; +import org.rascalmpl.parser.gtd.util.DoubleArrayList; +import org.rascalmpl.parser.gtd.util.DoubleStack; +import org.rascalmpl.parser.gtd.util.Stack; public interface IDebugListener

{ void shifting(int offset, int[] input, PositionStore positionStore); @@ -47,4 +50,14 @@ public interface IDebugListener

{ void filteredByEnterFilter(AbstractStackNode

node); void filteredByCompletionFilter(AbstractStackNode

node, AbstractNode result); + + void reviving(int[] input, + int location, + Stack> unexpandableNodes, + Stack> unmatchableLeafNodes, + DoubleStack, AbstractNode>, + AbstractStackNode

> unmatchableMidProductionNodes, + DoubleStack, AbstractNode> filteredNodes); + + void revived(DoubleArrayList, AbstractNode> recoveredNodes); } diff --git a/src/org/rascalmpl/parser/gtd/result/AbstractContainerNode.java b/src/org/rascalmpl/parser/gtd/result/AbstractContainerNode.java index 1a9d8373f59..f57f543ec94 100644 --- a/src/org/rascalmpl/parser/gtd/result/AbstractContainerNode.java +++ b/src/org/rascalmpl/parser/gtd/result/AbstractContainerNode.java @@ -15,6 +15,9 @@ import org.rascalmpl.parser.gtd.result.struct.Link; import org.rascalmpl.parser.gtd.util.ArrayList; +import org.rascalmpl.parser.util.DebugUtil; + +import io.usethesource.vallang.IConstructor; /** * All nodes in the resulting tree that can contain other nodes are a subtype @@ -147,4 +150,37 @@ public ArrayList

getAdditionalProductions(){ public ArrayList getAdditionalAlternatives(){ return alternatives; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("offset=" + offset); + builder.append(",endOffset=" + endOffset); + if (isNullable) { + builder.append(",nullable"); + } + if (isSeparator) { + builder.append(",separator"); + } + if (isLayout) { + builder.append(",layout"); + } + if (firstAlternative != null) { + builder.append(",alternatives=["); + builder.append(firstAlternative); + builder.append(":"); + builder.append(DebugUtil.prodToString((IConstructor) firstProduction)); + + if (alternatives != null) { + for (int i=0; i getPrefixes(){ public AbstractNode getNode(){ return node; } + + public String toString() { + return "Link[node=" + node + ", prefixes=" + (prefixes == null ? 0 : prefixes.size()) + "]"; + } } diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java index 41951569714..3afee7e1894 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java @@ -65,5 +65,5 @@ public AbstractStackNode

getEmptyChild(){ public final boolean isMatchable(){ return true; } - + } diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index 54e7fb5d70d..f0df8513e42 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -11,6 +11,8 @@ *******************************************************************************/ package org.rascalmpl.parser.gtd.stack; +import java.util.Arrays; + import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.result.struct.Link; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; @@ -20,6 +22,9 @@ import org.rascalmpl.parser.gtd.util.BitSet; import org.rascalmpl.parser.gtd.util.IntegerList; import org.rascalmpl.parser.gtd.util.IntegerObjectList; +import org.rascalmpl.parser.util.DebugUtil; + +import io.usethesource.vallang.IConstructor; @SuppressWarnings({"unchecked", "cast"}) public abstract class AbstractStackNode

{ @@ -28,8 +33,8 @@ public abstract class AbstractStackNode

{ protected AbstractStackNode

[] production; protected AbstractStackNode

[][] alternateProductions; - - protected IntegerObjectList> edgesMap; + + protected IntegerObjectList> edgesMap; // : key=startLocation, value=EdgesSet a that location protected ArrayList[] prefixesMap; protected EdgesSet

incomingEdges; @@ -732,7 +737,69 @@ public IntegerList getPropagatedReductions(){ return propagatedReductions; } - + + public String toShortString() { + return "id=" + id + ",dot=" + dot + ",start=" + startLocation; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append(id); + builder.append('.'); + builder.append(dot); + builder.append('@'); + builder.append(startLocation); + if (production != null) { + builder.append(",prod=["); + boolean first = true; + for (AbstractStackNode

prodElem : production) { + if (first) { + first = false; + } else { + builder.append(","); + } + builder.append(prodElem.getId()); + } + builder.append("]"); + } + if (isEndNode) { + builder.append(",endNode"); + } + if (isSeparator) { + builder.append(",separator"); + } + if (isLayout) { + builder.append(",layout"); + } + + if (alternateProductions != null && alternateProductions.length != 0) { + builder.append(",alternateProductions=" + Arrays.toString(alternateProductions)); + } + if (edgesMap != null && edgesMap.size() != 0) { + builder.append(",edges=" + edgesMap); + } + if (prefixesMap != null && prefixesMap.length != 0) { + builder.append(",prefixes=" + Arrays.toString(prefixesMap)); + } + if (incomingEdges != null && incomingEdges.size() != 0) { + builder.append(",incomingEdges=" + incomingEdges); + } + if (alternativeProduction != null) { + builder.append(",alternativeProduction=" + DebugUtil.prodToString((IConstructor)alternativeProduction)); + } + if (propagatedPrefixes != null) { + builder.append(",propagatedPrefixes=" + propagatedPrefixes); + } + if (propagatedReductions != null) { + builder.append(",propagatedReductions=" + propagatedReductions); + } + + // Do not print filters for now. + + return builder.toString(); + } + // Matchables. /** * Matches the symbol associated with this node to the input at the specified location. diff --git a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java index 0dafc11f539..cd0db6bf2f2 100644 --- a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java @@ -15,6 +15,7 @@ import org.rascalmpl.parser.gtd.result.LiteralNode; import org.rascalmpl.parser.gtd.stack.filter.ICompletionFilter; import org.rascalmpl.parser.gtd.stack.filter.IEnterFilter; +import org.rascalmpl.unicode.UnicodeConverter; public final class LiteralStackNode

extends AbstractMatchableStackNode

{ private final int[] literal; @@ -78,15 +79,13 @@ public AbstractNode getResult(){ } public String toString(){ - StringBuilder sb = new StringBuilder(); - for (int i : literal) { - sb.appendCodePoint(i); - } - sb.append(getId()); - sb.append('('); - sb.append(startLocation); - sb.append(')'); + StringBuilder sb = new StringBuilder("lit['"); + sb.append(UnicodeConverter.unicodeArrayToString(literal)); + sb.append("',"); + sb.append(super.toString()); + sb.append(']'); + return sb.toString(); } diff --git a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java index 7f00fe8397b..247562026f7 100644 --- a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java @@ -77,12 +77,11 @@ public AbstractNode getResult(){ } public String toString(){ - StringBuilder sb = new StringBuilder(); + StringBuilder sb = new StringBuilder("NonTerminal["); sb.append(expectIdentifier); - sb.append(getId()); - sb.append('('); - sb.append(startLocation); - sb.append(')'); + sb.append(","); + sb.append(super.toString()); + sb.append("]"); return sb.toString(); } diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 63164ab3bc7..84590af3d54 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -76,7 +76,7 @@ public AbstractNode getResult(){ return result; } - public String toString(){ + /*Original: public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(getId()); sb.append('('); @@ -84,12 +84,28 @@ public String toString(){ sb.append(')'); return sb.toString(); - } + }*/ + @Override + public String toString() { + return "SkippingStackNode[result=" + result + "," + super.toString() + "]"; + } + + @Override public int hashCode(){ return getParentProduction().hashCode(); } - + + @SuppressWarnings("unchecked") + @Override + public boolean equals(Object rhs) { + if (rhs instanceof AbstractStackNode) { + return isEqual((AbstractStackNode

)rhs); + } + + return false; + } + public boolean isEqual(AbstractStackNode

stackNode){ if ( !(stackNode instanceof SkippingStackNode)) { return false; diff --git a/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java b/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java index b778f6677a3..f95daf557c7 100644 --- a/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java +++ b/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java @@ -28,11 +28,13 @@ public class EdgesSet

{ private AbstractStackNode

[] edges; private int size; - + private int lastVisitedLevel = -1; private IntegerMap lastVisitedFilteredLevel; private AbstractContainerNode

lastResults; + + // Indexed by `resultStoreId`. private IntegerObjectList> lastFilteredResults; public EdgesSet(){ @@ -140,4 +142,33 @@ public int size(){ public void clear(){ size = 0; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("EdgesSet[{"); + + for (int i=0; i 0) { + builder.append(","); + } + builder.append(edges[i].getName() + "=" + edges[i].getId() + "." + edges[i].getDot() + "@" + edges[i].getStartLocation()); + } + + builder.append("}"); + + if (lastVisitedLevel >= 0) { + builder.append(",lastVisitedLevel=" + lastVisitedLevel); + } + if (lastResults != null) { + builder.append(",lastResults=" + lastResults); + } + + // Skip 'filtered' fields for now + + builder.append("]"); + + return builder.toString(); + } + + } diff --git a/src/org/rascalmpl/parser/gtd/util/ArrayList.java b/src/org/rascalmpl/parser/gtd/util/ArrayList.java index d93c1740981..899fcdbb0d4 100644 --- a/src/org/rascalmpl/parser/gtd/util/ArrayList.java +++ b/src/org/rascalmpl/parser/gtd/util/ArrayList.java @@ -104,4 +104,17 @@ public int size(){ public Object[] getBackingArray(){ return data; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("["); + for (int i=0; i 0) { + builder.append(","); + } + builder.append(data[i]); + } + builder.append("]"); + return builder.toString(); + } } diff --git a/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java b/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java index e1e6d4ea9ea..69b9e5ef0ef 100644 --- a/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java +++ b/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java @@ -102,4 +102,20 @@ public void ditryClear(){ public int size(){ return size; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("list["); + for (int i=0; i"); + } + builder.append("]\n"); + return builder.toString(); + } + } diff --git a/src/org/rascalmpl/parser/gtd/util/DoubleStack.java b/src/org/rascalmpl/parser/gtd/util/DoubleStack.java index 9251e89a519..3c2c6f3a6d7 100644 --- a/src/org/rascalmpl/parser/gtd/util/DoubleStack.java +++ b/src/org/rascalmpl/parser/gtd/util/DoubleStack.java @@ -171,4 +171,19 @@ public void clear(){ public void dirtyClear(){ size = 0; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("stack["); + for (int i=0; i"); + } + builder.append("]\n"); + return builder.toString(); + } } diff --git a/src/org/rascalmpl/parser/gtd/util/IntegerObjectList.java b/src/org/rascalmpl/parser/gtd/util/IntegerObjectList.java index 92d1009b02a..30b63fd46db 100644 --- a/src/org/rascalmpl/parser/gtd/util/IntegerObjectList.java +++ b/src/org/rascalmpl/parser/gtd/util/IntegerObjectList.java @@ -125,4 +125,20 @@ public void clear(){ public void dirtyClear(){ size = 0; } + + public String toString() { + StringBuilder builder = new StringBuilder("["); + + for (int i=0; i 0) { + builder.append(","); + } + builder.append(keys[i]); + builder.append("="); + builder.append(values[i]); + } + + builder.append("]"); + return builder.toString(); + } } diff --git a/src/org/rascalmpl/parser/uptr/debug/DebugLogger.java b/src/org/rascalmpl/parser/uptr/debug/DebugLogger.java index 89ba535a3bc..fb05ba5d95d 100644 --- a/src/org/rascalmpl/parser/uptr/debug/DebugLogger.java +++ b/src/org/rascalmpl/parser/uptr/debug/DebugLogger.java @@ -9,6 +9,9 @@ import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.util.ArrayList; +import org.rascalmpl.parser.gtd.util.DoubleArrayList; +import org.rascalmpl.parser.gtd.util.DoubleStack; +import org.rascalmpl.parser.gtd.util.Stack; import org.rascalmpl.values.parsetrees.ProductionAdapter; import io.usethesource.vallang.IConstructor; @@ -153,4 +156,57 @@ public void filteredByEnterFilter(AbstractStackNode node){ public void filteredByCompletionFilter(AbstractStackNode node, AbstractNode result){ out.println(String.format("Filtered by completion filter: %s", node)); } + + @Override + public void reviving(int[] input, int location, Stack> unexpandableNodes, + Stack> unmatchableLeafNodes, + DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, + DoubleStack, AbstractNode> filteredNodes) { + out.print("Reviving at "); + out.print(location); + out.print(": input='"); + for (int i=0; i<8 && location+i < input.length; i++) { + out.print((char) input[location+i]); + } + out.print("', unexpandable="); + + boolean first = true; + for (int i=0; i 0) { + out.print(", unmatchableLeafNodes="); + out.print(unmatchableLeafNodes.getSize()); + } + + if (unmatchableMidProductionNodes.getSize() > 0) { + out.print(", unmatchableMidProductionNodes="); + out.print(unmatchableMidProductionNodes.toString()); + } + + if (filteredNodes.getSize() > 0) { + out.print(", filteredNodes="); + out.print(filteredNodes.getSize()); + } + + out.println(); + } + + @Override + public void revived(DoubleArrayList, AbstractNode> recoveredNodes) { + out.println("Revived nodes:"); + for (int i=0; i { private static final int[] WHITESPACE = {' ', '\r', '\n', '\t' }; private IdDispenser stackNodeIdDispenser; - + + public ToNextWhitespaceRecoverer(IdDispenser stackNodeIdDispenser) { + this.stackNodeIdDispenser = stackNodeIdDispenser; + } + + @Override + public DoubleArrayList, AbstractNode> reviveStacks(int[] input, + int location, + Stack> unexpandableNodes, + Stack> unmatchableLeafNodes, + DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, + DoubleStack, AbstractNode> filteredNodes) { + + ArrayList> failedNodes = new ArrayList>(); + collectUnexpandableNodes(unexpandableNodes, failedNodes); + collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); + //collectFilteredNodes(filteredNodes, failedNodes); + + return reviveFailedNodes(input, location, failedNodes); + } + private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); - for (int i = recoveryNodes.size() - 1; i >= 0; --i) { + // original: for (int i = recoveryNodes.size() - 1; i >= 0; --i) { + // But this caused problems because recovery nodes with a later position + // where queued before nodes with an earlier position which the parser cannot handle. + for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); @@ -82,6 +105,11 @@ private static void collectUnexpandableNodes(Stack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); @@ -119,7 +147,7 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr ArrayList recoveryProductions = new ArrayList(); collectProductions(node, recoveryProductions); if (recoveryProductions.size() > 0) { - recoveryNodes.add(node, recoveryProductions); + recoveryNodes.add(node, recoveryProductions); } IntegerObjectList> edges = node.getEdges(); @@ -176,22 +204,5 @@ private void collectProductions(AbstractStackNode node, ArrayList< } } - public ToNextWhitespaceRecoverer(IdDispenser stackNodeIdDispenser) { - this.stackNodeIdDispenser = stackNodeIdDispenser; - } - @Override - public DoubleArrayList, AbstractNode> reviveStacks(int[] input, - int location, - Stack> unexpandableNodes, - Stack> unmatchableLeafNodes, - DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, - DoubleStack, AbstractNode> filteredNodes) { - ArrayList> failedNodes = new ArrayList>(); - collectUnexpandableNodes(unexpandableNodes, failedNodes); - collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); - //collectFilteredNodes(filteredNodes, failedNodes); - - return reviveFailedNodes(input, location, failedNodes); - } } diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java new file mode 100644 index 00000000000..8f0391caafa --- /dev/null +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -0,0 +1,39 @@ +package org.rascalmpl.parser.util; + +import java.util.Iterator; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IList; +import io.usethesource.vallang.IValue; + +public class DebugUtil { + /** + * Turn a production IConstructor into a string of the form "S -> E1 E2 ..." + */ + public static String prodToString(IConstructor prod) { + StringBuilder builder = new StringBuilder("'"); + + IConstructor sort = (IConstructor) prod.get(0); + builder.append(stripQuotes(String.valueOf(sort.get(0)))); + + builder.append(" ->"); + + IList children = (IList) prod.get(1); + for (IValue child : children) { + builder.append(" "); + IConstructor conChild = (IConstructor) child; + builder.append(stripQuotes(String.valueOf((conChild).get(0)))); + } + builder.append("'"); + + return builder.toString(); + } + + private static String stripQuotes(String s) { + if (s.charAt(0) == '"' && s.charAt(s.length()-1) == '"') { + return s.substring(1, s.length()-1); + } + + return s; + } +} diff --git a/src/org/rascalmpl/unicode/UnicodeConverter.java b/src/org/rascalmpl/unicode/UnicodeConverter.java new file mode 100644 index 00000000000..b98980301d7 --- /dev/null +++ b/src/org/rascalmpl/unicode/UnicodeConverter.java @@ -0,0 +1,12 @@ +package org.rascalmpl.unicode; + +public class UnicodeConverter { + public static String unicodeArrayToString(int[] chars) { + StringBuilder builder = new StringBuilder(); + for (int c : chars) { + builder.appendCodePoint(c); + } + + return builder.toString(); + } +} diff --git a/test/org/rascalmpl/test/parser/IParserTest.java b/test/org/rascalmpl/test/parser/IParserTest.java index a9598a8239e..ccf5cd8c5bb 100644 --- a/test/org/rascalmpl/test/parser/IParserTest.java +++ b/test/org/rascalmpl/test/parser/IParserTest.java @@ -38,7 +38,7 @@ public static AbstractStackNode[] createExpectArray(IConstructor p expectArray[index-1].setAlternativeProduction(prod); - return expectArray; + return (AbstractStackNode[]) new AbstractStackNode[]{expectArray[0]}; } ITree executeParser(); diff --git a/test/org/rascalmpl/test/parser/RecoveryTests2.java b/test/org/rascalmpl/test/parser/RecoveryTests2.java new file mode 100644 index 00000000000..6156cdc8529 --- /dev/null +++ b/test/org/rascalmpl/test/parser/RecoveryTests2.java @@ -0,0 +1,148 @@ +package org.rascalmpl.test.parser; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringReader; + +import org.junit.Assert; +import org.junit.Test; +import org.rascalmpl.parser.gtd.SGTDBF; +import org.rascalmpl.parser.gtd.recovery.IRecoverer; +import org.rascalmpl.parser.gtd.result.out.DefaultNodeFlattener; +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; +import org.rascalmpl.parser.gtd.stack.LiteralStackNode; +import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; +import org.rascalmpl.parser.uptr.UPTRNodeFactory; +import org.rascalmpl.parser.uptr.debug.DebugLogger; +import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; +import org.rascalmpl.values.RascalValueFactory; +import org.rascalmpl.values.ValueFactoryFactory; +import org.rascalmpl.values.parsetrees.ITree; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.ISourceLocation; +import io.usethesource.vallang.IValue; +import io.usethesource.vallang.io.StandardTextReader; + +/** + * S -> A ws B ws C + * A -> [a] + * B -> [b] ws [b] + * C -> [c] + * + * ws -> [\ ] + */ +public class RecoveryTests2 extends SGTDBF implements IParserTest{ + private final static IConstructor SYMBOL_START_S = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("S")); + + private final static IConstructor SYMBOL_ws = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("ws")); + private final static IConstructor SYMBOL_char_space = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(32)))); + + private final static IConstructor SYMBOL_A = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("A")); + private final static IConstructor SYMBOL_a = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("a")); + private final static IConstructor SYMBOL_char_a = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(97)))); + private final static IConstructor PROD_A_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_A, VF.list(SYMBOL_a), VF.set()); + + private final static IConstructor SYMBOL_B = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("B")); + private final static IConstructor SYMBOL_b = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("b")); + private final static IConstructor SYMBOL_char_b = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(98)))); + private final static IConstructor PROD_B_b_b = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_B, VF.list(SYMBOL_b, SYMBOL_ws, SYMBOL_b), VF.set()); + + private final static IConstructor SYMBOL_C = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("C")); + private final static IConstructor SYMBOL_c = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("c")); + private final static IConstructor SYMBOL_char_c = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(99)))); + private final static IConstructor PROD_C_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_C, VF.list(SYMBOL_c), VF.set()); + + private final static IConstructor PROD_S_A_B_C = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_START_S, VF.list(SYMBOL_A, SYMBOL_ws, SYMBOL_B, SYMBOL_ws, SYMBOL_C), VF.set()); + + private final static IConstructor PROD_ws = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_ws, VF.list(SYMBOL_char_space), VF.set()); + private final static IConstructor PROD_a_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_a, VF.list(SYMBOL_char_a), VF.set()); + private final static IConstructor PROD_b_b = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_b, VF.list(SYMBOL_char_b), VF.set()); + private final static IConstructor PROD_c_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_c, VF.list(SYMBOL_char_c), VF.set()); + + public AbstractStackNode[] S(){ + return IParserTest.createExpectArray(PROD_S_A_B_C, + new NonTerminalStackNode(1, 0, "A"), + new LiteralStackNode(2, 1, PROD_ws, new int[]{' '}), + new NonTerminalStackNode(3, 2, "B"), + new LiteralStackNode(4, 3, PROD_ws, new int[]{' '}), + new NonTerminalStackNode(5, 4, "C") + ); + } + + public AbstractStackNode[] A(){ + return IParserTest.createExpectArray(PROD_A_a, + new LiteralStackNode(6, 0, PROD_a_a, new int[]{'a'}) + ); + } + + public AbstractStackNode[] B(){ + return IParserTest.createExpectArray(PROD_B_b_b, + new LiteralStackNode(9, 0, PROD_b_b, new int[]{'b'}), + new LiteralStackNode(10, 1, PROD_ws, new int[]{' '}), + new LiteralStackNode(11, 2, PROD_b_b, new int[]{'b'}) + ); + } + + public AbstractStackNode[] C(){ + return IParserTest.createExpectArray(PROD_C_c, + new LiteralStackNode(12, 0, PROD_c_c, new int[]{'c'}) + ); + } + + private int nextFreeStackNodeId = 100; + + @Override + protected int getFreeStackNodeId() { + return nextFreeStackNodeId++; + } + + private ITree parse(String s) { + DebugLogger debugLogger = new DebugLogger(new PrintWriter(System.out)); + IRecoverer recoverer = new ToNextWhitespaceRecoverer(() -> nextFreeStackNodeId++); + return parse("S" /* NONTERMINAL_START_S */, null, s.toCharArray(), + new DefaultNodeFlattener(), new UPTRNodeFactory(false), recoverer, debugLogger); + } + + private ITree parseB(String s) { + return parse("B", null, s.toCharArray(), + new DefaultNodeFlattener(), new UPTRNodeFactory(false), null, null); + } + + + private ITree toTree(String s) { + try { + return (ITree) new StandardTextReader().read(ValueFactoryFactory.getValueFactory(), RascalValueFactory.uptr, RascalValueFactory.Tree, new StringReader(s)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public ITree executeParser() { + return parse("a b b c"); + } + + @Override + public IValue getExpectedResult() { + return toTree("appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"); + } + + @Test + public void testOk() { + Assert.assertEquals(getExpectedResult(), executeParser()); + } + + @Test + public void testB() { + String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\"),lit(\"ws\"),lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\"),lit(\"ws\"),lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; + Assert.assertEquals(toTree(expected), parseB("b")); + } + + @Test + public void testMissingCharRecovery() { + String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(skipped(\\char-class([range(1,1114111)])),[char(120)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; + Assert.assertEquals(toTree(expected), parse("a b x c")); + } + +} From 13ffcefa1c7e62e62e80f63fe0eff1b26c67b154 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 18 Jun 2024 11:45:51 +0200 Subject: [PATCH 029/190] Added 'allowRecovery` keyword parameter in parsing functions. --- src/org/rascalmpl/interpreter/Evaluator.java | 2 +- src/org/rascalmpl/library/ParseTree.rsc | 29 +++++----- src/org/rascalmpl/library/Prelude.java | 24 ++++---- src/org/rascalmpl/parser/gtd/SGTDBF.java | 7 ++- .../parser/gtd/util/IdDispenser.java | 2 +- .../util/ReflectiveStackNodeIdDispenser.java | 53 +++++++++++++++++ .../semantics/dynamic/Expression.java | 14 +++-- .../rascalmpl/semantics/dynamic/Import.java | 4 +- .../rascalmpl/values/IRascalValueFactory.java | 8 +-- .../values/RascalFunctionValueFactory.java | 58 +++++++++++-------- 10 files changed, 133 insertions(+), 68 deletions(-) create mode 100644 src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java diff --git a/src/org/rascalmpl/interpreter/Evaluator.java b/src/org/rascalmpl/interpreter/Evaluator.java index 4ed8cd191ce..5f662043c28 100755 --- a/src/org/rascalmpl/interpreter/Evaluator.java +++ b/src/org/rascalmpl/interpreter/Evaluator.java @@ -990,7 +990,7 @@ private IFunction parserForCurrentModule(RascalFunctionValueFactory vf, ModuleEn IMap syntaxDefinition = curMod.getSyntaxDefinition(); IMap grammar = (IMap) getParserGenerator().getGrammarFromModules(getMonitor(), curMod.getName(), syntaxDefinition).get("rules"); IConstructor reifiedType = vf.reifiedType(dummy, grammar); - return vf.parsers(reifiedType, vf.bool(false), vf.bool(false), vf.bool(false), vf.set()); + return vf.parsers(reifiedType, vf.bool(false), vf.bool(false), vf.bool(false), vf.bool(false), vf.set()); } private Result evalMore(String command, ISourceLocation location) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 2813f58b661..733680858ed 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -393,14 +393,14 @@ catch ParseError(loc l): { } ``` } -&T<:Tree parse(type[&T<:Tree] begin, str input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) - = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, |unknown:///|); +&T<:Tree parse(type[&T<:Tree] begin, str input, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) + = parser(begin, allowAmbiguity=allowAmbiguity, allowRecovery=allowRecovery, hasSideEffects=hasSideEffects, filters=filters)(input, |unknown:///|); -&T<:Tree parse(type[&T<:Tree] begin, str input, loc origin, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) - = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, origin); +&T<:Tree parse(type[&T<:Tree] begin, str input, loc origin, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) + = parser(begin, allowAmbiguity=allowAmbiguity, allowRecovery=allowRecovery, hasSideEffects=hasSideEffects, filters=filters)(input, origin); -&T<:Tree parse(type[&T<:Tree] begin, loc input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) - = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, input); +&T<:Tree parse(type[&T<:Tree] begin, loc input, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) + = parser(begin, allowAmbiguity=allowAmbiguity, allowRecovery=allowRecovery, hasSideEffects=hasSideEffects, filters=filters)(input, input); @synopsis{Generates a parser from an input grammar.} @@ -416,15 +416,16 @@ So the parse function reads either directly from a str or via the contents of a which leads to the prefix of the `src` fields of the resulting tree. The parse function behaves differently depending of the given keyword parameters: - * `allowAmbiguity`: if true then no exception is thrown in case of ambiguity and a parse forest is returned. if false, + * `allowAmbiguity`: if true then no exception is thrown in case of ambiguity and a parse forest is returned. if false, the parser throws an exception during tree building and produces only the first ambiguous subtree in its message. if set to `false`, the parse constructs trees in linear time. if set to `true` the parser constructs trees in polynomial time. - * + * 'allowRecovery`: ***experimental*** if true, the parser tries to recover on a parse error. if a parse error is encountered that can be recovered from, special `skipped` nodes + are included in the resulting parse tree. More documentation will be added here when this feature matures. * `hasSideEffects`: if false then the parser is a lot faster when constructing trees, since it does not execute the parse _actions_ in an interpreted environment to make side effects (like a symbol table) and it can share more intermediate results as a result. } @javaClass{org.rascalmpl.library.Prelude} -java &T (value input, loc origin) parser(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); +java &T (value input, loc origin) parser(type[&T] grammar, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); @javaClass{org.rascalmpl.library.Prelude} @synopsis{Generates a parser function that can be used to find the left-most deepest ambiguous sub-sentence.} @@ -437,7 +438,7 @@ the tree that exhibits ambiguity. This can be done very quickly, while the whole * The returned sub-tree usually has a different type than the parameter of the type[] symbol that was passed in. The reason is that sub-trees typically have a different non-terminal than the start non-terminal of a grammar. } -java Tree (value input, loc origin) firstAmbiguityFinder(type[Tree] grammar, bool hasSideEffects=false, set[Tree(Tree)] filters={}); +java Tree (value input, loc origin) firstAmbiguityFinder(type[Tree] grammar, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); @synopsis{Generates parsers from a grammar (reified type), where all non-terminals in the grammar can be used as start-symbol.} @description{ @@ -445,7 +446,7 @@ This parser generator behaves the same as the `parser` function, but it produces nonterminal parameter. This can be used to select a specific non-terminal from the grammar to use as start-symbol for parsing. } @javaClass{org.rascalmpl.library.Prelude} -java &U (type[&U] nonterminal, value input, loc origin) parsers(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); +java &U (type[&U] nonterminal, value input, loc origin) parsers(type[&T] grammar, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); @javaClass{org.rascalmpl.library.Prelude} @synopsis{Generates a parser function that can be used to find the left-most deepest ambiguous sub-sentence.} @@ -458,7 +459,7 @@ the tree that exhibits ambiguity. This can be done very quickly, while the whole * The returned sub-tree usually has a different type than the parameter of the type[] symbol that was passed in. The reason is that sub-trees typically have a different non-terminal than the start non-terminal of a grammar. } -java Tree (type[Tree] nonterminal, value input, loc origin) firstAmbiguityFinders(type[Tree] grammar, bool hasSideEffects=false, set[Tree(Tree)] filters={}); +java Tree (type[Tree] nonterminal, value input, loc origin) firstAmbiguityFinders(type[Tree] grammar, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); @synopsis{Parse the input but instead of returning the entire tree, return the trees for the first ambiguous substring.} @description{ @@ -536,7 +537,7 @@ p(type(sort("E"), ()), "e+e", |src:///|); * reifiying types (use of `#`) will trigger the loading of a parser generator anyway. You have to use this notation for types to avoid that: `type(\start(sort("MySort")), ())` to avoid the computation for `#start[A]` } -java &U (type[&U] nonterminal, value input, loc origin) loadParsers(loc savedParsers, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); +java &U (type[&U] nonterminal, value input, loc origin) loadParsers(loc savedParsers, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); @synopsis{Load a previously serialized parser, for a specific non-terminal, from disk for usage} @description{ @@ -544,7 +545,7 @@ This loader behaves just like ((loadParsers)), except that the resulting parser bound to a specific non-terminal. } @javaClass{org.rascalmpl.library.Prelude} -java &U (value input, loc origin) loadParser(type[&U] nonterminal, loc savedParsers, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); +java &U (value input, loc origin) loadParser(type[&U] nonterminal, loc savedParsers, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); @synopsis{Yield the string of characters that form the leafs of the given parse tree.} @description{ diff --git a/src/org/rascalmpl/library/Prelude.java b/src/org/rascalmpl/library/Prelude.java index 33036ee01ab..242137d8aa5 100644 --- a/src/org/rascalmpl/library/Prelude.java +++ b/src/org/rascalmpl/library/Prelude.java @@ -2349,20 +2349,20 @@ public INode arbNode() { protected final TypeReifier tr; - public IFunction parser(IValue start, IBool allowAmbiguity, IBool hasSideEffects, ISet filters) { - return rascalValues.parser(start, allowAmbiguity, hasSideEffects, values.bool(false), filters); + public IFunction parser(IValue start, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, ISet filters) { + return rascalValues.parser(start, allowAmbiguity, allowRecovery, hasSideEffects, values.bool(false), filters); } - public IFunction firstAmbiguityFinder(IValue start, IBool hasSideEffects, ISet filters) { - return rascalValues.parser(start, values.bool(true), hasSideEffects, values.bool(true), filters); + public IFunction firstAmbiguityFinder(IValue start, IBool allowRecovery, IBool hasSideEffects, ISet filters) { + return rascalValues.parser(start, values.bool(true), allowRecovery, hasSideEffects, values.bool(true), filters); } - public IFunction parsers(IValue start, IBool allowAmbiguity, IBool hasSideEffects, ISet filters) { - return rascalValues.parsers(start, allowAmbiguity, hasSideEffects, values.bool(false), filters); + public IFunction parsers(IValue start, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, ISet filters) { + return rascalValues.parsers(start, allowAmbiguity, allowRecovery, hasSideEffects, values.bool(false), filters); } - public IFunction firstAmbiguityFinders(IValue start, IBool hasSideEffects, ISet filters) { - return rascalValues.parsers(start, values.bool(true), hasSideEffects, values.bool(true), filters); + public IFunction firstAmbiguityFinders(IValue start, IBool allowRecovery, IBool hasSideEffects, ISet filters) { + return rascalValues.parsers(start, values.bool(true), allowRecovery, hasSideEffects, values.bool(true), filters); } public void storeParsers(IValue start, ISourceLocation saveLocation) { @@ -2377,18 +2377,18 @@ public void storeParsers(IValue start, ISourceLocation saveLocation) { } } - public IFunction loadParsers(ISourceLocation savedLocation, IBool allowAmbiguity, IBool hasSideEffects, ISet filters) { + public IFunction loadParsers(ISourceLocation savedLocation, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, ISet filters) { try { - return rascalValues.loadParsers(savedLocation, allowAmbiguity, hasSideEffects, values.bool(false), filters); + return rascalValues.loadParsers(savedLocation, allowAmbiguity, allowRecovery, hasSideEffects, values.bool(false), filters); } catch (IOException | ClassNotFoundException e) { throw RuntimeExceptionFactory.io(e.getMessage()); } } - public IFunction loadParser(IValue grammar, ISourceLocation savedLocation, IBool allowAmbiguity, IBool hasSideEffects, ISet filters) { + public IFunction loadParser(IValue grammar, ISourceLocation savedLocation, IBool allowRecovery, IBool allowAmbiguity, IBool hasSideEffects, ISet filters) { try { - return rascalValues.loadParser(grammar, savedLocation, allowAmbiguity, hasSideEffects, values.bool(false), filters); + return rascalValues.loadParser(grammar, savedLocation, allowAmbiguity, allowRecovery, hasSideEffects, values.bool(false), filters); } catch (IOException | ClassNotFoundException e) { throw RuntimeExceptionFactory.io(e.getMessage()); diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 458568f5643..15940442b81 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -864,10 +864,11 @@ private boolean findStacksToReduce(){ debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); } DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); - debugListener.revived(recoveredNodes); + if (debugListener != null) { + debugListener.revived(recoveredNodes); + } if (recoveredNodes.size() > 0) { - - // for (int i = recoveredNodes.size()-1; i>= 0; i--) { + // was: for (int i = recoveredNodes.size()-1; i>= 0; i--) { for (int i = 0; i < recoveredNodes.size(); i++) { AbstractStackNode

recovered = recoveredNodes.getFirst(i); diff --git a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java index 3952d7fa3c0..fe9aa091406 100644 --- a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java +++ b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java @@ -1,5 +1,5 @@ package org.rascalmpl.parser.gtd.util; public interface IdDispenser { - int dispenseId(); + int dispenseId(); } diff --git a/src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java b/src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java new file mode 100644 index 00000000000..dfbafab9d84 --- /dev/null +++ b/src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java @@ -0,0 +1,53 @@ +package org.rascalmpl.parser.gtd.util; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +import org.rascalmpl.parser.gtd.IGTD; +import org.rascalmpl.values.parsetrees.ITree; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.ISourceLocation; + +/** + * To offer backwards compatibility for generated parsers that do not yet have the "getFreeStackNodeId" method yet, + * this class uses reflection to find that method and otherwise improvises by just using a ridiculously high starting number. + */ +public class ReflectiveStackNodeIdDispenser implements IdDispenser { + private IGTD parser; + private Method dispenseMethod; + private int nextNodeIdBackup = (Integer.MAX_VALUE/4)*3; + + public ReflectiveStackNodeIdDispenser(IGTD parser) { + try { + dispenseMethod = parser.getClass().getMethod("getFreeStackNodeId"); + } catch (NoSuchMethodException e) { + // Custom IGTB implementation without "getFreeStackNodeId" method. No biggy, we just use nextNodeIdBackup. + } + } + + @Override + public int dispenseId() { + if (dispenseMethod != null) { + try { + return (Integer)dispenseMethod.invoke(parser); + } + catch (InvocationTargetException e) { + if (e.getTargetException() instanceof UnsupportedOperationException) { + // We are dealing with a parser class that has no generated "getFreeStackNodeId" method (yet), + // for backwards compatibility we fall back on "nextNodeIdBackup". + dispenseMethod = null; // No reason to try again. + } else { + throw new RuntimeException(e); + } + } + catch (IllegalAccessException | IllegalArgumentException e) { + throw new RuntimeException(e); + } + } + + return nextNodeIdBackup++; + } + +} + diff --git a/src/org/rascalmpl/semantics/dynamic/Expression.java b/src/org/rascalmpl/semantics/dynamic/Expression.java index ba5f629189d..4d7ed5c1c3f 100644 --- a/src/org/rascalmpl/semantics/dynamic/Expression.java +++ b/src/org/rascalmpl/semantics/dynamic/Expression.java @@ -1070,7 +1070,7 @@ private boolean isBootstrapped(IEvaluatorContext eval) { return false; } - private ITree parseObject(IEvaluatorContext eval, IConstructor grammar, ISet filters, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean hasSideEffects) { + private ITree parseObject(IEvaluatorContext eval, IConstructor grammar, ISet filters, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean allowRecovery, boolean hasSideEffects) { RascalFunctionValueFactory vf = eval.getFunctionValueFactory(); IString str = vf.string(new String(input)); @@ -1078,19 +1078,19 @@ private ITree parseObject(IEvaluatorContext eval, IConstructor grammar, ISet fil return (ITree) vf.bootstrapParsers().call(grammar, str, location); } else { - IFunction parser = vf.parser(grammar, vf.bool(allowAmbiguity), vf.bool(hasSideEffects), vf.bool(false), filters); + IFunction parser = vf.parser(grammar, vf.bool(allowAmbiguity), vf.bool(allowRecovery), vf.bool(hasSideEffects), vf.bool(false), filters); return (ITree) parser.call(vf.string(new String(input)), location); } } - private ITree parseObject(IEvaluatorContext eval, IConstructor grammar, ISet filters, ISourceLocation location, boolean allowAmbiguity, boolean hasSideEffects) { + private ITree parseObject(IEvaluatorContext eval, IConstructor grammar, ISet filters, ISourceLocation location, boolean allowAmbiguity, boolean allowRecovery, boolean hasSideEffects) { RascalFunctionValueFactory vf = eval.getFunctionValueFactory(); if (isBootstrapped(eval)) { return (ITree) vf.bootstrapParsers().call(grammar, location, location); } else { - IFunction parser = vf.parser(grammar, vf.bool(allowAmbiguity), vf.bool(hasSideEffects), vf.bool(false), filters); + IFunction parser = vf.parser(grammar, vf.bool(allowAmbiguity), vf.bool(allowRecovery), vf.bool(hasSideEffects), vf.bool(false), filters); return (ITree) parser.call(location, location); } } @@ -1125,11 +1125,13 @@ public Result interpret(IEvaluator> __eval) { IConstructor value = ((IRascalValueFactory) __eval.getValueFactory()).reifiedType(symbol, gr); if (result.getStaticType().isString()) { + // TODO: discuss if we want to allow recovery here tree = parseObject(__eval, value, VF.set(), this.getLocation(), - ((IString) result.getValue()).getValue().toCharArray(), true, false); + ((IString) result.getValue()).getValue().toCharArray(), true, false, false); } else if (result.getStaticType().isSourceLocation()) { - tree = parseObject(__eval, value, VF.set(), (ISourceLocation) result.getValue(), true, false); + // TODO: discuss if we want to allow recovery here + tree = parseObject(__eval, value, VF.set(), (ISourceLocation) result.getValue(), true, false, false); } assert tree != null; // because we checked earlier diff --git a/src/org/rascalmpl/semantics/dynamic/Import.java b/src/org/rascalmpl/semantics/dynamic/Import.java index ebb4656494a..35088776167 100644 --- a/src/org/rascalmpl/semantics/dynamic/Import.java +++ b/src/org/rascalmpl/semantics/dynamic/Import.java @@ -487,7 +487,7 @@ public static ITree parseModuleAndFragments(char[] data, ISourceLocation locatio } else if (reg.exists(parserCacheFile)) { // if we cached a ModuleFile.parsers file, we will use the parser from that (typically after deployment time) - parsers = vf.loadParsers(parserCacheFile, vf.bool(false),vf.bool(false),vf.bool(false), vf.set()); + parsers = vf.loadParsers(parserCacheFile, vf.bool(false), vf.bool(false), vf.bool(false), vf.bool(false), vf.set()); } else { // otherwise we have to generate a fresh parser for this module now @@ -495,7 +495,7 @@ else if (reg.exists(parserCacheFile)) { IMap syntaxDefinition = env.getSyntaxDefinition(); IMap grammar = (IMap) eval.getParserGenerator().getGrammarFromModules(eval.getMonitor(),env.getName(), syntaxDefinition).get("rules"); IConstructor reifiedType = vf.reifiedType(dummy, grammar); - parsers = vf.parsers(reifiedType, vf.bool(false), vf.bool(false), vf.bool(false), vf.set()); + parsers = vf.parsers(reifiedType, vf.bool(false), vf.bool(false), vf.bool(false), vf.bool(false), vf.set()); } try { diff --git a/src/org/rascalmpl/values/IRascalValueFactory.java b/src/org/rascalmpl/values/IRascalValueFactory.java index 4770b97e701..407e88e4972 100644 --- a/src/org/rascalmpl/values/IRascalValueFactory.java +++ b/src/org/rascalmpl/values/IRascalValueFactory.java @@ -94,7 +94,7 @@ default IFunction function(Type functionType, BiFunction getParserGenerator(), this, caller, parser, allowAmbiguity, hasSideEffects, firstAmbiguity, filters)); + return function(functionType, new ParametrizedParseFunction(() -> getParserGenerator(), this, caller, parser, allowAmbiguity, allowRecovery, hasSideEffects, firstAmbiguity, filters)); } @Override @@ -292,7 +294,7 @@ public void storeParsers(IValue reifiedGrammar, ISourceLocation saveLocation) th } @Override - public IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) throws IOException, ClassNotFoundException { + public IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) throws IOException, ClassNotFoundException { RascalTypeFactory rtf = RascalTypeFactory.getInstance(); TypeFactory tf = TypeFactory.getInstance(); @@ -318,12 +320,12 @@ public IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, this, caller, parser, - allowAmbiguity, hasSideEffects, firstAmbiguity, filters) + allowAmbiguity, allowRecovery, hasSideEffects, firstAmbiguity, filters) ); } @Override - public IFunction loadParser(IValue reifiedGrammar, ISourceLocation saveLocation, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) throws IOException, ClassNotFoundException { + public IFunction loadParser(IValue reifiedGrammar, ISourceLocation saveLocation, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) throws IOException, ClassNotFoundException { TypeFactory tf = TypeFactory.getInstance(); Type functionType = tf.functionType(reifiedGrammar.getType().getTypeParameters().getFieldType(0), @@ -347,7 +349,7 @@ public IFunction loadParser(IValue reifiedGrammar, ISourceLocation saveLocation, name = generator.getParserMethodName(startSort); } - return function(functionType, new ParseFunction(ctx.getValueFactory(), caller, parser, name, allowAmbiguity, hasSideEffects, firstAmbiguity, filters)); + return function(functionType, new ParseFunction(ctx.getValueFactory(), caller, parser, name, allowAmbiguity, allowRecovery, hasSideEffects, firstAmbiguity, filters)); } /** @@ -372,7 +374,7 @@ public IFunction bootstrapParsers() { AbstractAST current = ctx.getCurrentAST(); ISourceLocation caller = current != null ? current.getLocation() : URIUtil.rootLocation("unknown"); - return function(functionType, new ParametrizedParseFunction(() -> getParserGenerator(), this, caller, parser, vf.bool(false), vf.bool(false), vf.bool(false), ctx.getValueFactory().set())); + return function(functionType, new ParametrizedParseFunction(() -> getParserGenerator(), this, caller, parser, vf.bool(false), vf.bool(false), vf.bool(false), vf.bool(false), ctx.getValueFactory().set())); } public IString createHole(ITree part, IInteger index) { @@ -417,19 +419,21 @@ static private class ParseFunction implements BiFunction> parser; protected final String methodName; protected final ISourceLocation caller; - public ParseFunction(IValueFactory vf, ISourceLocation caller, Class> parser, String methodName, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { + public ParseFunction(IValueFactory vf, ISourceLocation caller, Class> parser, String methodName, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { this.vf = vf; this.caller = caller; this.parser = parser; this.methodName = methodName; this.filters = filters; this.allowAmbiguity = allowAmbiguity.getValue() || firstAmbiguity.getValue(); + this.allowRecovery = allowRecovery.getValue(); this.hasSideEffects = hasSideEffects.getValue(); this.firstAmbiguity = firstAmbiguity.getValue(); } @@ -454,10 +458,10 @@ else if (parameters[0].getType().isSourceLocation()) { } if (parameters[0].getType().isString()) { - return parse(methodName, filters, (IString) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, hasSideEffects); + return parse(methodName, filters, (IString) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, allowRecovery, hasSideEffects); } else if (parameters[0].getType().isSourceLocation()) { - return parse(methodName, filters, (ISourceLocation) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, hasSideEffects); + return parse(methodName, filters, (ISourceLocation) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, allowRecovery, hasSideEffects); } } @@ -477,13 +481,13 @@ private IGTD getParser() { } } - protected IValue parse(String methodName, ISet filters, IString input, ISourceLocation origin, boolean allowAmbiguity, boolean hasSideEffects) { + protected IValue parse(String methodName, ISet filters, IString input, ISourceLocation origin, boolean allowAmbiguity, boolean allowRecovery, boolean hasSideEffects) { try { if (origin == null) { origin = URIUtil.rootLocation("unknown"); } - return parseObject(methodName, origin, input.getValue().toCharArray(), allowAmbiguity, hasSideEffects, filters); + return parseObject(methodName, origin, input.getValue().toCharArray(), allowAmbiguity, allowRecovery, hasSideEffects, filters); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -500,7 +504,7 @@ protected IValue parse(String methodName, ISet filters, IString input, ISourceL protected IValue firstAmbiguity(String methodName, IString input) { try { - return parseObject(methodName, URIUtil.invalidLocation(), input.getValue().toCharArray(), false, false, vf.set()); + return parseObject(methodName, URIUtil.invalidLocation(), input.getValue().toCharArray(), false, false, false, vf.set()); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -516,7 +520,7 @@ protected IValue firstAmbiguity(String methodName, IString input) { protected IValue firstAmbiguity(String methodName, ISourceLocation input) { try { - return parseObject(methodName, input, readAll(input), false, false, vf.set()); + return parseObject(methodName, input, readAll(input), false, false, false, vf.set()); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -543,13 +547,13 @@ private IString printSymbol(IConstructor symbol) { return vf.string(SymbolAdapter.toString(symbol, false)); } - protected IValue parse(String methodName, ISet filters, ISourceLocation input, ISourceLocation origin, boolean allowAmbiguity, boolean hasSideEffects) { + protected IValue parse(String methodName, ISet filters, ISourceLocation input, ISourceLocation origin, boolean allowAmbiguity, boolean allowRecovery, boolean hasSideEffects) { if (origin == null) { origin = input; } try { - return parseObject(methodName, input, readAll(input), allowAmbiguity, hasSideEffects, filters); + return parseObject(methodName, input, readAll(input), allowAmbiguity, allowRecovery, hasSideEffects, filters); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -567,10 +571,14 @@ protected IValue parse(String methodName, ISet filters, ISourceLocation input, I } } - private ITree parseObject(String methodName, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean hasSideEffects, ISet filters) { + private ITree parseObject(String methodName, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean allowRecovery, boolean hasSideEffects, ISet filters) { IActionExecutor exec = filters.isEmpty() ? new NoActionExecutor() : new RascalFunctionActionExecutor(filters, !hasSideEffects); - - return (ITree) getParser().parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener(), new UPTRNodeFactory(allowAmbiguity), (IRecoverer) null); + IGTD parserInstance = getParser(); + IRecoverer recoverer = null; + if (allowRecovery) { + recoverer = new ToNextWhitespaceRecoverer(new ReflectiveStackNodeIdDispenser(parserInstance)); + } + return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer); } } @@ -585,8 +593,8 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in static private class ParametrizedParseFunction extends ParseFunction { private Supplier generator; - public ParametrizedParseFunction(Supplier generator, IValueFactory vf, ISourceLocation caller, Class> parser, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { - super(vf, caller, parser, null, allowAmbiguity, hasSideEffects, firstAmbiguity, filters); + public ParametrizedParseFunction(Supplier generator, IValueFactory vf, ISourceLocation caller, Class> parser, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { + super(vf, caller, parser, null, allowAmbiguity, allowRecovery, hasSideEffects, firstAmbiguity, filters); this.generator = generator; } @@ -621,10 +629,10 @@ else if (parameters[1].getType().isSourceLocation()) { } if (parameters[1].getType().isString()) { - return parse(name, filters, (IString) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, hasSideEffects); + return parse(name, filters, (IString) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, allowRecovery, hasSideEffects); } else if (parameters[1].getType().isSourceLocation()) { - return parse(name, filters, (ISourceLocation) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, hasSideEffects); + return parse(name, filters, (ISourceLocation) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, allowRecovery, hasSideEffects); } } From b8eb72e807ce6b8ed93081809f8f6b050e809adb Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 18 Jun 2024 13:46:51 +0200 Subject: [PATCH 030/190] Removed 'Reflective' prefix from StackNodeIdDispenser class name Reflective is just the way the dispenser is implemented. No need for users of this class to be aware of this. --- ...iveStackNodeIdDispenser.java => StackNodeIdDispenser.java} | 4 ++-- .../parser/uptr/recovery/ToNextWhitespaceRecoverer.java | 2 +- src/org/rascalmpl/values/RascalFunctionValueFactory.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename src/org/rascalmpl/parser/gtd/util/{ReflectiveStackNodeIdDispenser.java => StackNodeIdDispenser.java} (91%) diff --git a/src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java similarity index 91% rename from src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java rename to src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java index dfbafab9d84..403bfbe9247 100644 --- a/src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java +++ b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java @@ -13,12 +13,12 @@ * To offer backwards compatibility for generated parsers that do not yet have the "getFreeStackNodeId" method yet, * this class uses reflection to find that method and otherwise improvises by just using a ridiculously high starting number. */ -public class ReflectiveStackNodeIdDispenser implements IdDispenser { +public class StackNodeIdDispenser implements IdDispenser { private IGTD parser; private Method dispenseMethod; private int nextNodeIdBackup = (Integer.MAX_VALUE/4)*3; - public ReflectiveStackNodeIdDispenser(IGTD parser) { + public StackNodeIdDispenser(IGTD parser) { try { dispenseMethod = parser.getClass().getMethod("getFreeStackNodeId"); } catch (NoSuchMethodException e) { diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 382662ffc4c..daebd1eca8e 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -63,7 +63,7 @@ private DoubleArrayList, AbstractNode> reviveNod for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); - + // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). for (int j = prods.size() - 1; j >= 0; --j) { IConstructor prod = prods.get(j); diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 918148c87e8..8278d4dce93 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -47,7 +47,7 @@ import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.action.IActionExecutor; import org.rascalmpl.parser.gtd.result.out.DefaultNodeFlattener; -import org.rascalmpl.parser.gtd.util.ReflectiveStackNodeIdDispenser; +import org.rascalmpl.parser.gtd.util.StackNodeIdDispenser; import org.rascalmpl.parser.uptr.UPTRNodeFactory; import org.rascalmpl.parser.uptr.action.NoActionExecutor; import org.rascalmpl.parser.uptr.action.RascalFunctionActionExecutor; @@ -576,7 +576,7 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in IGTD parserInstance = getParser(); IRecoverer recoverer = null; if (allowRecovery) { - recoverer = new ToNextWhitespaceRecoverer(new ReflectiveStackNodeIdDispenser(parserInstance)); + recoverer = new ToNextWhitespaceRecoverer(new StackNodeIdDispenser(parserInstance)); } return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer); } From f82006c9a0bf89c772dcea677b6115bf324c0b11 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 27 Jun 2024 15:02:08 +0200 Subject: [PATCH 031/190] WIP: recovery starting to work --- .../tests/recovery/BasicRecoveryTests.rsc | 18 +++ src/org/rascalmpl/parser/gtd/SGTDBF.java | 19 +-- .../parser/gtd/result/SkippedNode.java | 19 ++- .../result/out/INodeConstructorFactory.java | 4 +- .../gtd/result/out/RecoveryNodeFlattener.java | 3 +- .../parser/gtd/stack/AbstractStackNode.java | 2 + .../parser/gtd/stack/SkippingStackNode.java | 10 +- .../parser/gtd/util/DoubleArrayList.java | 19 +++ .../parser/uptr/UPTRNodeFactory.java | 22 ++- .../recovery/ToNextWhitespaceRecoverer.java | 22 ++- .../rascalmpl/values/RascalValueFactory.java | 2 +- .../rascalmpl/test/parser/RecoveryTests.java | 35 ++--- .../rascalmpl/test/parser/RecoveryTests2.java | 148 ------------------ .../test/recovery/ErrorRecoveryModules.java | 11 ++ 14 files changed, 133 insertions(+), 201 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc delete mode 100644 test/org/rascalmpl/test/parser/RecoveryTests2.java create mode 100644 test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc new file mode 100644 index 00000000000..de6194dcc99 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -0,0 +1,18 @@ +module lang::rascal::tests::recovery::BasicRecoveryTests + + +import ParseTree; +import IO; + +layout Layout = [\ ]* !>> [\ ]; + +syntax S = A B C; +syntax A = "a"; +syntax B = "b" "b"; +syntax C = "c"; + +test bool parseTest() { + Tree t = parse(#S, "a b x c", allowRecovery=true); + iprintln(t); + return true; +} diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 15940442b81..f7a4ddb5588 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -919,6 +919,8 @@ private void queueMatchableNode(AbstractStackNode

node, int length, AbstractN * Inserts a recovery node into the todo-list, and possibly * rewinds the parser to an earlier location in the input */ + // TODO: look for original code: + // - queueMatchableNode (just above here) @SuppressWarnings("unchecked") private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int length, AbstractNode result){ assert result != null; @@ -937,20 +939,19 @@ private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int DoubleStack, AbstractNode>[] oldTodoLists = todoLists; todoLists = new DoubleStack[negativeOffset + Math.max(queueDepth, length) + 1]; - System.arraycopy(oldTodoLists, queueIndex, todoLists, negativeOffset + 1, queueDepth - queueIndex); - System.arraycopy(oldTodoLists, 0, todoLists, queueDepth - queueIndex + negativeOffset + 1, queueIndex); + System.arraycopy(oldTodoLists, queueIndex, todoLists, negativeOffset, queueDepth - queueIndex); + System.arraycopy(oldTodoLists, 0, todoLists, queueDepth - queueIndex + negativeOffset , queueIndex); // reset the parser! queueIndex = 0; - location = startPosition + 1; + location = startPosition; - DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex + 1]; - if (terminalsTodo == null){ + // was: DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex + 1]; + DoubleStack, AbstractNode> terminalsTodo = todoLists[length]; + if (terminalsTodo == null) { terminalsTodo = new DoubleStack, AbstractNode>(); - todoLists[queueIndex + 1] = terminalsTodo; // Why the +1 and not length? To get the recovered node to be processed first? - } - else { - assert false: "this should never happen"; + // was: todoLists[queueIndex + 1] = terminalsTodo; // Why the +1 and not length? To get the recovered node to be processed first? + todoLists[length] = terminalsTodo; } terminalsTodo.push(node, result); diff --git a/src/org/rascalmpl/parser/gtd/result/SkippedNode.java b/src/org/rascalmpl/parser/gtd/result/SkippedNode.java index 8fd8adacfc2..48882533b2b 100644 --- a/src/org/rascalmpl/parser/gtd/result/SkippedNode.java +++ b/src/org/rascalmpl/parser/gtd/result/SkippedNode.java @@ -11,8 +11,6 @@ *******************************************************************************/ package org.rascalmpl.parser.gtd.result; -import java.util.Arrays; - import org.rascalmpl.unicode.UnicodeConverter; /** @@ -21,13 +19,16 @@ public class SkippedNode extends AbstractNode { public final static int ID = 9; + private final Object production; + private final int dot; private final int[] skippedChars; - private final int offset; - public SkippedNode(int[] skippedChars, int offset){ + public SkippedNode(Object production, int dot, int[] skippedChars, int offset) { super(); + this.production = production; + this.dot = dot; this.skippedChars = skippedChars; this.offset = offset; } @@ -35,7 +36,15 @@ public SkippedNode(int[] skippedChars, int offset){ public int getTypeIdentifier(){ return ID; } - + + public Object getProduction() { + return production; + } + + public int getDot() { + return dot; + } + public int[] getSkippedChars(){ return skippedChars; } diff --git a/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java b/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java index e79ae43757b..5a94f7c3b64 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java +++ b/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java @@ -38,8 +38,8 @@ public interface INodeConstructorFactory { T createListAmbiguityNode(ArrayList alternatives); - T createRecoveryNode(int[] characters); - + T createRecoveryNode(int dot, ArrayList recognizedPrefix, int[] unrecognizedCharacters, Object production); + ArrayList getChildren(T node); P createPositionInformation(URI input, int offset, int endOffset, PositionStore positionStore); diff --git a/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java b/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java index 833da06dcd4..b7a2605bfa9 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java +++ b/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java @@ -12,6 +12,7 @@ package org.rascalmpl.parser.gtd.result.out; import org.rascalmpl.parser.gtd.result.SkippedNode; +import org.rascalmpl.parser.gtd.util.ArrayList; /** * A converter for result nodes that contain skipped characters for error recovery @@ -23,6 +24,6 @@ public RecoveryNodeFlattener(){ } public T convertToUPTR(INodeConstructorFactory nodeConstructorFactory, SkippedNode node){ - return nodeConstructorFactory.createRecoveryNode(node.getSkippedChars()); + return nodeConstructorFactory.createRecoveryNode(node.getDot(), new ArrayList<>(), node.getSkippedChars(), node.getProduction()); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index f0df8513e42..4f8259690df 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -34,7 +34,9 @@ public abstract class AbstractStackNode

{ protected AbstractStackNode

[] production; protected AbstractStackNode

[][] alternateProductions; + // Our edges protected IntegerObjectList> edgesMap; // : key=startLocation, value=EdgesSet a that location + // Edges of our children protected ArrayList[] prefixesMap; protected EdgesSet

incomingEdges; diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 84590af3d54..2ef909c5684 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -17,10 +17,10 @@ public final class SkippingStackNode

extends AbstractMatchableStackNode

{ private final SkippedNode result; - public SkippingStackNode(int id, int[] until, int[] input, int startLocation, P parentProduction){ + public SkippingStackNode(int id, int[] until, int[] input, int startLocation, P parentProduction, int dot){ super(id, 0); - this.result = buildResult(input, until, startLocation); + this.result = buildResult(input, until, startLocation, parentProduction, dot); setAlternativeProduction(parentProduction); } @@ -36,7 +36,7 @@ private SkippingStackNode(SkippingStackNode

original, SkippedNode result, int this.result = result; } - private static SkippedNode buildResult(int[] input, int[] until, int startLocation){ + private SkippedNode buildResult(int[] input, int[] until, int startLocation, P production, int dot){ for (int to = startLocation ; to < input.length; ++to) { for (int i = 0; i < until.length; ++i) { if (input[to] == until[i]) { @@ -44,12 +44,12 @@ private static SkippedNode buildResult(int[] input, int[] until, int startLocati int[] chars = new int[length]; System.arraycopy(input, startLocation, chars, 0, length); - return new SkippedNode(chars, startLocation); + return new SkippedNode(production, dot, chars, startLocation); } } } - return new SkippedNode(new int[0], startLocation); + return new SkippedNode(production, dot, new int[0], startLocation); } public boolean isEmptyLeafNode(){ diff --git a/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java b/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java index 69b9e5ef0ef..655adb7a879 100644 --- a/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java +++ b/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java @@ -11,6 +11,10 @@ *******************************************************************************/ package org.rascalmpl.parser.gtd.util; +import java.util.Comparator; + +import org.apache.commons.lang3.tuple.Pair; + @SuppressWarnings("unchecked") public class DoubleArrayList{ private final static int DEFAULT_SIZE = 8; @@ -103,6 +107,21 @@ public int size(){ return size; } + public void sort(Comparator> comparator) { + java.util.List> elems = new java.util.ArrayList<>(size); + for (int i=0; i elem = elems.get(i); + first[i] = elem.getLeft(); + second[i] = elem.getRight(); + } + } + @Override public String toString() { StringBuilder builder = new StringBuilder("list["); diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index 5988650baaa..b971764ccfd 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -1,7 +1,6 @@ package org.rascalmpl.parser.uptr; import java.net.URI; -import java.util.Arrays; import java.util.IdentityHashMap; import java.util.Map; @@ -14,6 +13,7 @@ import io.usethesource.vallang.IListWriter; import io.usethesource.vallang.ISetWriter; import io.usethesource.vallang.ISourceLocation; +import io.usethesource.vallang.IValue; import org.rascalmpl.values.RascalValueFactory; import org.rascalmpl.values.ValueFactoryFactory; @@ -22,8 +22,7 @@ import org.rascalmpl.values.parsetrees.TreeAdapter; public class UPTRNodeFactory implements INodeConstructorFactory{ - private final static RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory(); - private final static IConstructor SKIPPED = VF.constructor(RascalValueFactory.Production_Skipped, VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Range, VF.integer(1), VF.integer(Character.MAX_CODE_POINT))))); + private static final RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory(); private boolean allowAmb; public UPTRNodeFactory(boolean allowAmbiguity){ @@ -143,8 +142,19 @@ public Object getProductionFromNode(ITree node){ } @Override - public ITree createRecoveryNode(int[] characters) { - IList chars = Arrays.stream(characters).mapToObj(ch -> VF.character(ch)).collect(VF.listWriter()); - return VF.appl(SKIPPED, chars); + public ITree createRecoveryNode(int dot, ArrayList recognizedPrefix, int[] unrecognizedCharacters, Object production) { + IListWriter args = VF.listWriter(); + for (int i=0; i, AbstractNode> reviveStac ArrayList> failedNodes = new ArrayList>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); + // TODO: handle unmatchableLeafNodes //collectFilteredNodes(filteredNodes, failedNodes); return reviveFailedNodes(input, location, failedNodes); @@ -60,22 +62,28 @@ private DoubleArrayList, AbstractNode> reviveNod // original: for (int i = recoveryNodes.size() - 1; i >= 0; --i) { // But this caused problems because recovery nodes with a later position // where queued before nodes with an earlier position which the parser cannot handle. - for (int i = 0; i Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); + + for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); + //Pair, ArrayList> elem = elems.get(i); + //AbstractStackNode recoveryNode = elem.getLeft(); + //ArrayList prods = elem.getRight(); // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). for (int j = prods.size() - 1; j >= 0; --j) { IConstructor prod = prods.get(j); - AbstractStackNode continuer = new RecoveryPointStackNode(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); + AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); int startLocation = recoveryNode.getStartLocation(); - AbstractStackNode recoverLiteral = new SkippingStackNode(stackNodeIdDispenser.dispenseId(), WHITESPACE, input, startLocation, prod); + AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), WHITESPACE, input, startLocation, prod, recoveryNode.getDot()); recoverLiteral = recoverLiteral.getCleanCopy(startLocation); recoverLiteral.initEdges(); - EdgesSet edges = new EdgesSet(1); + EdgesSet edges = new EdgesSet<>(1); edges.add(continuer); recoverLiteral.addEdges(edges, startLocation); @@ -106,9 +114,11 @@ private static void collectUnexpandableNodes(Stack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { @@ -152,7 +162,7 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr IntegerObjectList> edges = node.getEdges(); - for (int i = edges.size() - 1; i >= 0; --i) { + for (int i = edges.size() - 1; i >= 0; --i) { // Rewind EdgesSet edgesList = edges.getValue(i); if (edgesList != null) { diff --git a/src/org/rascalmpl/values/RascalValueFactory.java b/src/org/rascalmpl/values/RascalValueFactory.java index cbfba6611b4..ad3b5f4d668 100644 --- a/src/org/rascalmpl/values/RascalValueFactory.java +++ b/src/org/rascalmpl/values/RascalValueFactory.java @@ -152,7 +152,7 @@ public class RascalValueFactory extends AbstractValueFactoryAdapter implements I public static final Type Production_Priority = tf.constructor(uptr, Production, "priority", Symbol, "def", tf.listType(Production), "choices"); public static final Type Production_Composition = tf.constructor(uptr, Production, "composition", Production, "lhs", Production, "rhs"); public static final Type Production_Associativity = tf.constructor(uptr, Production, "associativity", Symbol, "def", Associativity, "assoc", tf.setType(Production), "alternatives"); - public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped", Symbol, "def"); + public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped", Symbol, "def", Production, "prod", tf.integerType(), "dot"); /* Constructors for Attr */ public static final Type Attr_Assoc = tf.constructor(uptr, Attr, "assoc", Associativity, "assoc"); diff --git a/test/org/rascalmpl/test/parser/RecoveryTests.java b/test/org/rascalmpl/test/parser/RecoveryTests.java index 869b74bca6e..ce3030b8223 100644 --- a/test/org/rascalmpl/test/parser/RecoveryTests.java +++ b/test/org/rascalmpl/test/parser/RecoveryTests.java @@ -1,6 +1,7 @@ package org.rascalmpl.test.parser; import java.io.IOException; +import java.io.PrintWriter; import java.io.StringReader; import org.junit.Assert; @@ -12,6 +13,7 @@ import org.rascalmpl.parser.gtd.stack.LiteralStackNode; import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; import org.rascalmpl.parser.uptr.UPTRNodeFactory; +import org.rascalmpl.parser.uptr.debug.DebugLogger; import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; import org.rascalmpl.values.RascalValueFactory; import org.rascalmpl.values.ValueFactoryFactory; @@ -24,9 +26,9 @@ /** * S -> A ws B ws C - * A -> [a] ws [a] + * A -> [a] * B -> [b] ws [b] - * C -> [c] ws [c] + * C -> [c] * * ws -> [\ ] */ @@ -39,7 +41,7 @@ public class RecoveryTests extends SGTDBF private final static IConstructor SYMBOL_A = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("A")); private final static IConstructor SYMBOL_a = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("a")); private final static IConstructor SYMBOL_char_a = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(97)))); - private final static IConstructor PROD_A_a_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_A, VF.list(SYMBOL_a, SYMBOL_ws, SYMBOL_a), VF.set()); + private final static IConstructor PROD_A_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_A, VF.list(SYMBOL_a), VF.set()); private final static IConstructor SYMBOL_B = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("B")); private final static IConstructor SYMBOL_b = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("b")); @@ -49,7 +51,7 @@ public class RecoveryTests extends SGTDBF private final static IConstructor SYMBOL_C = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("C")); private final static IConstructor SYMBOL_c = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("c")); private final static IConstructor SYMBOL_char_c = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(99)))); - private final static IConstructor PROD_C_c_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_C, VF.list(SYMBOL_c, SYMBOL_ws, SYMBOL_c), VF.set()); + private final static IConstructor PROD_C_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_C, VF.list(SYMBOL_c), VF.set()); private final static IConstructor PROD_S_A_B_C = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_START_S, VF.list(SYMBOL_A, SYMBOL_ws, SYMBOL_B, SYMBOL_ws, SYMBOL_C), VF.set()); @@ -69,10 +71,8 @@ public AbstractStackNode[] S(){ } public AbstractStackNode[] A(){ - return IParserTest.createExpectArray(PROD_A_a_a, - new LiteralStackNode(6, 0, PROD_a_a, new int[]{'a'}), - new LiteralStackNode(7, 1, PROD_ws, new int[]{' '}), - new LiteralStackNode(8, 2, PROD_a_a, new int[]{'a'}) + return IParserTest.createExpectArray(PROD_A_a, + new LiteralStackNode(6, 0, PROD_a_a, new int[]{'a'}) ); } @@ -85,10 +85,8 @@ public AbstractStackNode[] B(){ } public AbstractStackNode[] C(){ - return IParserTest.createExpectArray(PROD_C_c_c, - new LiteralStackNode(12, 0, PROD_c_c, new int[]{'c'}), - new LiteralStackNode(13, 1, PROD_ws, new int[]{' '}), - new LiteralStackNode(14, 2, PROD_c_c, new int[]{'c'}) + return IParserTest.createExpectArray(PROD_C_c, + new LiteralStackNode(12, 0, PROD_c_c, new int[]{'c'}) ); } @@ -100,9 +98,10 @@ protected int getFreeStackNodeId() { } private ITree parse(String s) { + DebugLogger debugLogger = new DebugLogger(new PrintWriter(System.out)); IRecoverer recoverer = new ToNextWhitespaceRecoverer(() -> nextFreeStackNodeId++); return parse("S" /* NONTERMINAL_START_S */, null, s.toCharArray(), - new DefaultNodeFlattener(), new UPTRNodeFactory(false), recoverer, null); + new DefaultNodeFlattener(), new UPTRNodeFactory(false), recoverer, debugLogger); } private ITree toTree(String s) { @@ -115,12 +114,12 @@ private ITree toTree(String s) { @Override public ITree executeParser() { - return parse("a a b b c c"); + return parse("a b b c"); } @Override public IValue getExpectedResult() { - return toTree("appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\"),lit(\"ws\"),lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\"),lit(\"ws\"),lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"); + return toTree("appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"); } @Test @@ -129,9 +128,9 @@ public void testOk() { } @Test - public void testMissingCharRecovery() { - String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\"),lit(\"ws\"),lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\"),lit(\"ws\"),lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; - Assert.assertEquals(toTree(expected), parse("a a b c c")); + public void testIncorrectCharacter() { + String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(skipped(sort(\"B\"),prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),2),[char(120)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; + Assert.assertEquals(toTree(expected), parse("a b x c")); } } diff --git a/test/org/rascalmpl/test/parser/RecoveryTests2.java b/test/org/rascalmpl/test/parser/RecoveryTests2.java deleted file mode 100644 index 6156cdc8529..00000000000 --- a/test/org/rascalmpl/test/parser/RecoveryTests2.java +++ /dev/null @@ -1,148 +0,0 @@ -package org.rascalmpl.test.parser; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringReader; - -import org.junit.Assert; -import org.junit.Test; -import org.rascalmpl.parser.gtd.SGTDBF; -import org.rascalmpl.parser.gtd.recovery.IRecoverer; -import org.rascalmpl.parser.gtd.result.out.DefaultNodeFlattener; -import org.rascalmpl.parser.gtd.stack.AbstractStackNode; -import org.rascalmpl.parser.gtd.stack.LiteralStackNode; -import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; -import org.rascalmpl.parser.uptr.UPTRNodeFactory; -import org.rascalmpl.parser.uptr.debug.DebugLogger; -import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; -import org.rascalmpl.values.RascalValueFactory; -import org.rascalmpl.values.ValueFactoryFactory; -import org.rascalmpl.values.parsetrees.ITree; - -import io.usethesource.vallang.IConstructor; -import io.usethesource.vallang.ISourceLocation; -import io.usethesource.vallang.IValue; -import io.usethesource.vallang.io.StandardTextReader; - -/** - * S -> A ws B ws C - * A -> [a] - * B -> [b] ws [b] - * C -> [c] - * - * ws -> [\ ] - */ -public class RecoveryTests2 extends SGTDBF implements IParserTest{ - private final static IConstructor SYMBOL_START_S = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("S")); - - private final static IConstructor SYMBOL_ws = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("ws")); - private final static IConstructor SYMBOL_char_space = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(32)))); - - private final static IConstructor SYMBOL_A = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("A")); - private final static IConstructor SYMBOL_a = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("a")); - private final static IConstructor SYMBOL_char_a = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(97)))); - private final static IConstructor PROD_A_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_A, VF.list(SYMBOL_a), VF.set()); - - private final static IConstructor SYMBOL_B = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("B")); - private final static IConstructor SYMBOL_b = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("b")); - private final static IConstructor SYMBOL_char_b = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(98)))); - private final static IConstructor PROD_B_b_b = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_B, VF.list(SYMBOL_b, SYMBOL_ws, SYMBOL_b), VF.set()); - - private final static IConstructor SYMBOL_C = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("C")); - private final static IConstructor SYMBOL_c = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("c")); - private final static IConstructor SYMBOL_char_c = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(99)))); - private final static IConstructor PROD_C_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_C, VF.list(SYMBOL_c), VF.set()); - - private final static IConstructor PROD_S_A_B_C = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_START_S, VF.list(SYMBOL_A, SYMBOL_ws, SYMBOL_B, SYMBOL_ws, SYMBOL_C), VF.set()); - - private final static IConstructor PROD_ws = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_ws, VF.list(SYMBOL_char_space), VF.set()); - private final static IConstructor PROD_a_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_a, VF.list(SYMBOL_char_a), VF.set()); - private final static IConstructor PROD_b_b = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_b, VF.list(SYMBOL_char_b), VF.set()); - private final static IConstructor PROD_c_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_c, VF.list(SYMBOL_char_c), VF.set()); - - public AbstractStackNode[] S(){ - return IParserTest.createExpectArray(PROD_S_A_B_C, - new NonTerminalStackNode(1, 0, "A"), - new LiteralStackNode(2, 1, PROD_ws, new int[]{' '}), - new NonTerminalStackNode(3, 2, "B"), - new LiteralStackNode(4, 3, PROD_ws, new int[]{' '}), - new NonTerminalStackNode(5, 4, "C") - ); - } - - public AbstractStackNode[] A(){ - return IParserTest.createExpectArray(PROD_A_a, - new LiteralStackNode(6, 0, PROD_a_a, new int[]{'a'}) - ); - } - - public AbstractStackNode[] B(){ - return IParserTest.createExpectArray(PROD_B_b_b, - new LiteralStackNode(9, 0, PROD_b_b, new int[]{'b'}), - new LiteralStackNode(10, 1, PROD_ws, new int[]{' '}), - new LiteralStackNode(11, 2, PROD_b_b, new int[]{'b'}) - ); - } - - public AbstractStackNode[] C(){ - return IParserTest.createExpectArray(PROD_C_c, - new LiteralStackNode(12, 0, PROD_c_c, new int[]{'c'}) - ); - } - - private int nextFreeStackNodeId = 100; - - @Override - protected int getFreeStackNodeId() { - return nextFreeStackNodeId++; - } - - private ITree parse(String s) { - DebugLogger debugLogger = new DebugLogger(new PrintWriter(System.out)); - IRecoverer recoverer = new ToNextWhitespaceRecoverer(() -> nextFreeStackNodeId++); - return parse("S" /* NONTERMINAL_START_S */, null, s.toCharArray(), - new DefaultNodeFlattener(), new UPTRNodeFactory(false), recoverer, debugLogger); - } - - private ITree parseB(String s) { - return parse("B", null, s.toCharArray(), - new DefaultNodeFlattener(), new UPTRNodeFactory(false), null, null); - } - - - private ITree toTree(String s) { - try { - return (ITree) new StandardTextReader().read(ValueFactoryFactory.getValueFactory(), RascalValueFactory.uptr, RascalValueFactory.Tree, new StringReader(s)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public ITree executeParser() { - return parse("a b b c"); - } - - @Override - public IValue getExpectedResult() { - return toTree("appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"); - } - - @Test - public void testOk() { - Assert.assertEquals(getExpectedResult(), executeParser()); - } - - @Test - public void testB() { - String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\"),lit(\"ws\"),lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\"),lit(\"ws\"),lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; - Assert.assertEquals(toTree(expected), parseB("b")); - } - - @Test - public void testMissingCharRecovery() { - String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(skipped(\\char-class([range(1,1114111)])),[char(120)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; - Assert.assertEquals(toTree(expected), parse("a b x c")); - } - -} diff --git a/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java b/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java new file mode 100644 index 00000000000..abf446cf843 --- /dev/null +++ b/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java @@ -0,0 +1,11 @@ +package org.rascalmpl.test.recovery; + +import org.junit.runner.RunWith; +import org.rascalmpl.test.infrastructure.RascalJUnitTestPrefix; +import org.rascalmpl.test.infrastructure.RascalJUnitTestRunner; + +@RunWith(RascalJUnitTestRunner.class) +@RascalJUnitTestPrefix("lang::rascal::tests::recovery") +public class ErrorRecoveryModules { + +} From 88cddc347c2d8b0d7f0623b3770b6068517eba2d Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 10 Jul 2024 12:21:28 +0200 Subject: [PATCH 032/190] Added some methods to analyze recovered parse errors in parse trees --- src/org/rascalmpl/library/ParseTree.rsc | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 733680858ed..010655cf502 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -189,7 +189,7 @@ data Production | \reference(Symbol def, str cons) // <5> ; -data Production = skipped(Symbol def, Production prod, int validPrefix); +data Production = skipped(Symbol def, Production prod, int dot); @synopsis{Attributes in productions.} @description{ @@ -766,3 +766,25 @@ bool isNonTerminalType(Symbol::\parameterized-sort(str _, list[Symbol] _)) = tru bool isNonTerminalType(Symbol::\parameterized-lex(str _, list[Symbol] _)) = true; bool isNonTerminalType(Symbol::\start(Symbol s)) = isNonTerminalType(s); default bool isNonTerminalType(Symbol s) = false; + +@synopsis{Check if a parse tree contains any skipped nodes, the result of error recovery.} +bool hasErrors(Tree tree) = /appl(skipped(_, _, _), _) := tree; + +@synopsis{Find all skipped nodes in a parse tree.} +list[Tree] findAllErrors(Tree tree) { + return for (/error:appl(_, [*_, appl(skipped(_, _, _), _)]) := tree) append(error); +} + +@synopsis{Find the first skipped node in a parse tree.} +Tree findFirstError(Tree tree) { + if (/error:appl(_, [*_, appl(skipped(_, _, _), _)]) := tree) return error; + fail; +} + +Symbol getErrorSymbol(appl(skipped(Symbol symbol, _, _), _)) = symbol; + +Production getErrorProduction(appl(skipped(_, Production production, _), _)) = production; + +int getErrorPosition(appl(skipped(_, _, int dot), _)) = dot; + +str getErrorString(appl(skipped(_, _, _), list chars)) = ""; From 6d192810a989ab1589398b444c0248b241ca80b2 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 10 Jul 2024 12:26:26 +0200 Subject: [PATCH 033/190] Added comment and removed unused import --- .../parser/uptr/recovery/ToNextWhitespaceRecoverer.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 97c4941ca4c..24566530f12 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -12,7 +12,6 @@ *******************************************************************************/ package org.rascalmpl.parser.uptr.recovery; -import org.apache.commons.lang3.tuple.Pair; import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; @@ -79,9 +78,8 @@ private DoubleArrayList, AbstractNode> reviveNod AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); int startLocation = recoveryNode.getStartLocation(); - AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), WHITESPACE, input, startLocation, prod, recoveryNode.getDot()); - recoverLiteral = recoverLiteral.getCleanCopy(startLocation); + recoverLiteral = recoverLiteral.getCleanCopy(startLocation); // This copy might not be needed recoverLiteral.initEdges(); EdgesSet edges = new EdgesSet<>(1); edges.add(continuer); From bcd59147860023708de6e39e556346d7c6d7c288 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 10 Jul 2024 12:26:59 +0200 Subject: [PATCH 034/190] Enabled logging when error recovery is in effect --- src/org/rascalmpl/values/RascalFunctionValueFactory.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 8278d4dce93..0a4cffc46a5 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -13,6 +13,7 @@ package org.rascalmpl.values; import java.io.IOException; +import java.io.PrintWriter; import java.io.Reader; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; @@ -41,6 +42,7 @@ import org.rascalmpl.library.lang.rascal.syntax.RascalParser; import org.rascalmpl.parser.ParserGenerator; import org.rascalmpl.parser.gtd.IGTD; +import org.rascalmpl.parser.gtd.debug.IDebugListener; import org.rascalmpl.parser.gtd.exception.ParseError; import org.rascalmpl.parser.gtd.exception.UndeclaredNonTerminalException; import org.rascalmpl.parser.gtd.io.InputConverter; @@ -51,6 +53,7 @@ import org.rascalmpl.parser.uptr.UPTRNodeFactory; import org.rascalmpl.parser.uptr.action.NoActionExecutor; import org.rascalmpl.parser.uptr.action.RascalFunctionActionExecutor; +import org.rascalmpl.parser.uptr.debug.DebugLogger; import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; import org.rascalmpl.types.NonTerminalType; import org.rascalmpl.types.RascalTypeFactory; @@ -575,10 +578,12 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in IActionExecutor exec = filters.isEmpty() ? new NoActionExecutor() : new RascalFunctionActionExecutor(filters, !hasSideEffects); IGTD parserInstance = getParser(); IRecoverer recoverer = null; + IDebugListener debugListener = null; if (allowRecovery) { recoverer = new ToNextWhitespaceRecoverer(new StackNodeIdDispenser(parserInstance)); + debugListener = new DebugLogger(new PrintWriter(System.out, true)); } - return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer); + return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer, debugListener); } } From fb6ffe56484f43111c04fb2b387f04ce40ba4965 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 10 Jul 2024 12:39:15 +0200 Subject: [PATCH 035/190] Made printing of productions more robust (it crashed on 'regular' productions) --- src/org/rascalmpl/parser/util/DebugUtil.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index 8f0391caafa..a99a0fa4606 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -1,7 +1,5 @@ package org.rascalmpl.parser.util; -import java.util.Iterator; - import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IList; import io.usethesource.vallang.IValue; @@ -18,12 +16,18 @@ public static String prodToString(IConstructor prod) { builder.append(" ->"); - IList children = (IList) prod.get(1); - for (IValue child : children) { + if (prod.getName() == "prod") { + IList children = (IList) prod.get(1); + for (IValue child : children) { + builder.append(" "); + IConstructor conChild = (IConstructor) child; + builder.append(stripQuotes(String.valueOf((conChild).get(0)))); + } + } else { builder.append(" "); - IConstructor conChild = (IConstructor) child; - builder.append(stripQuotes(String.valueOf((conChild).get(0)))); + builder.append(prod.toString()); } + builder.append("'"); return builder.toString(); From 27f1afce2e41adb79c78a92e6d48c8903cad1957 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 10 Jul 2024 12:40:38 +0200 Subject: [PATCH 036/190] Added some basic recovery tests --- .../tests/recovery/BasicRecoveryTests.rsc | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index de6194dcc99..9370fe68cdb 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -6,13 +6,15 @@ import IO; layout Layout = [\ ]* !>> [\ ]; -syntax S = A B C; -syntax A = "a"; -syntax B = "b" "b"; -syntax C = "c"; +syntax S = A End; +syntax A = "1" "2" "3"; +syntax End = "$"; -test bool parseTest() { - Tree t = parse(#S, "a b x c", allowRecovery=true); - iprintln(t); - return true; +test bool parseOk() { + return !hasErrors(parse(#S, "1 2 3 $", allowRecovery=true)); +} + +test bool simpleRecovery() { + Tree t = parse(#S, "1 2 x $", allowRecovery=true); + return hasErrors(t) && size(findAllErrors(t)) == 1; } From 15fedbf07a6acdfe15f4e95805b9691f065c3b80 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 18 Jul 2024 08:50:57 +0200 Subject: [PATCH 037/190] Implemented parser datastructure visualization using graphviz --- .../tests/recovery/BasicRecoveryTests.rsc | 26 +- src/org/rascalmpl/parser/gtd/SGTDBF.java | 136 ++++- .../parser/gtd/stack/AbstractStackNode.java | 11 +- .../gtd/stack/AlternativeStackNode.java | 2 +- .../parser/gtd/stack/CharStackNode.java | 9 +- .../parser/gtd/stack/LiteralStackNode.java | 5 + .../recovery/ToNextWhitespaceRecoverer.java | 7 +- .../util/visualize/DebugVisualizer.java | 521 ++++++++++++++++++ .../visualize/dot/AttributeStatement.java | 28 + .../util/visualize/dot/CompassPoint.java | 5 + .../util/visualize/dot/DotAttribute.java | 104 ++++ .../rascalmpl/util/visualize/dot/DotEdge.java | 66 +++ .../util/visualize/dot/DotField.java | 23 + .../util/visualize/dot/DotGraph.java | 107 ++++ .../rascalmpl/util/visualize/dot/DotNode.java | 69 +++ .../util/visualize/dot/DotRecord.java | 37 ++ .../util/visualize/dot/DotRecordEntry.java | 7 + .../util/visualize/dot/DotStatement.java | 7 + .../util/visualize/dot/DotSubgraph.java | 40 ++ .../rascalmpl/util/visualize/dot/NodeId.java | 126 +++++ 20 files changed, 1300 insertions(+), 36 deletions(-) create mode 100644 src/org/rascalmpl/util/visualize/DebugVisualizer.java create mode 100644 src/org/rascalmpl/util/visualize/dot/AttributeStatement.java create mode 100644 src/org/rascalmpl/util/visualize/dot/CompassPoint.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotAttribute.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotEdge.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotField.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotGraph.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotNode.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotRecord.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotStatement.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotSubgraph.java create mode 100644 src/org/rascalmpl/util/visualize/dot/NodeId.java diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index 9370fe68cdb..ec7ff7270a1 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -1,20 +1,28 @@ module lang::rascal::tests::recovery::BasicRecoveryTests - import ParseTree; -import IO; -layout Layout = [\ ]* !>> [\ ]; +layout Layout = [\ ]; //* !>> [\ ]; -syntax S = A End; -syntax A = "1" "2" "3"; +syntax S = ABC End; +syntax ABC = "a" "b" "c"; syntax End = "$"; -test bool parseOk() { - return !hasErrors(parse(#S, "1 2 3 $", allowRecovery=true)); +test bool ok() { + return !hasErrors(parse(#S, "a b c $", allowRecovery=true)); +} + +test bool abx() { + Tree t = parse(#S, "a b x $", allowRecovery=true); + return hasErrors(t) && size(findAllErrors(t)) == 1; +} + +test bool axc() { + Tree t = parse(#S, "a x c $", allowRecovery=true); + return hasErrors(t) && size(findAllErrors(t)) == 1; } -test bool simpleRecovery() { - Tree t = parse(#S, "1 2 x $", allowRecovery=true); +test bool ax() { + Tree t = parse(#S, "a x $", allowRecovery=true); return hasErrors(t) && size(findAllErrors(t)) == 1; } diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index f7a4ddb5588..b7f7aaa7a5d 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -34,6 +34,7 @@ import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.EpsilonStackNode; import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; +import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.stack.filter.ICompletionFilter; import org.rascalmpl.parser.gtd.stack.filter.IEnterFilter; @@ -45,6 +46,8 @@ import org.rascalmpl.parser.gtd.util.IntegerList; import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.util.visualize.DebugVisualizer; +import org.rascalmpl.util.visualize.dot.NodeId; /** * This is the core of the parser; it drives the parse process. @@ -122,14 +125,16 @@ public abstract class SGTDBF implements IGTD{ // Debugging private IDebugListener

debugListener; + private DebugVisualizer visualizer; // Temporary instrumentation for accurate profiling private long timestamp; private boolean printTimes = false; + public SGTDBF(){ super(); - + positionStore = new PositionStore(); stacksToExpand = new Stack>(); @@ -149,6 +154,8 @@ public SGTDBF(){ unmatchableLeafNodes = new Stack>(); unmatchableMidProductionNodes = new DoubleStack, AbstractNode>, AbstractStackNode

>(); filteredNodes = new DoubleStack, AbstractNode>(); + + visualizer = new DebugVisualizer("Parser"); } /** @@ -758,7 +765,9 @@ private void moveToNext(AbstractStackNode

node, AbstractNode result){ */ private void move(AbstractStackNode

node, AbstractNode result){ if(debugListener != null) debugListener.moving(node, result); - + if (node instanceof RecoveryPointStackNode) { + opportunityToBreak(); + } // Handle filtering. ICompletionFilter[] completionFilters = node.getCompletionFilters(); if(completionFilters != null){ @@ -790,13 +799,17 @@ private void move(AbstractStackNode

node, AbstractNode result){ /** * Initiate the handling of stacks. */ - private void reduce(){ + private void reduceTerminals() { // Reduce terminals + visualize("Reducing terminals", DebugVisualizer.TERMINALS_TO_REDUCE_ID); while(!stacksWithTerminalsToReduce.isEmpty()){ move(stacksWithTerminalsToReduce.peekFirst(), stacksWithTerminalsToReduce.popSecond()); } - + } + + private void reduceNonTerminals() { // Reduce non-terminals + visualize("Reducing non-terminals", DebugVisualizer.NON_TERMINALS_TO_REDUCE_ID); while(!stacksWithNonTerminalsToReduce.isEmpty()){ move(stacksWithNonTerminalsToReduce.peekFirst(), stacksWithNonTerminalsToReduce.popSecond()); } @@ -822,6 +835,7 @@ private boolean findFirstStacksToReduce(){ if (recoverer != null) { if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); } DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { @@ -845,12 +859,18 @@ private boolean findFirstStacksToReduce(){ * Locates the set of stacks that is queued for handling, for which the least amount of characters needs to be shifted. */ private boolean findStacksToReduce(){ + visualize("Finding stacks to reduce", DebugVisualizer.TODO_LISTS_ID); int queueDepth = todoLists.length; for(int i = 1; i < queueDepth; ++i){ queueIndex = (queueIndex + 1) % queueDepth; DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; if(!(terminalsTodo == null || terminalsTodo.isEmpty())){ + if (debugListener != null) { + NodeId reduceNodeId = new NodeId("todo-" + i); + visualize("Found stack to reduce", reduceNodeId); + } + stacksWithTerminalsToReduce = terminalsTodo; location += i; @@ -861,19 +881,29 @@ private boolean findStacksToReduce(){ if (recoverer != null && location < input.length) { if (debugListener != null) { + visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); } DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); + // Visualize state and include recovered nodes + visualizer.createGraph(this, "Reviving"); + visualizer.addRecoveredNodes(recoveredNodes); + visualizer.writeGraph(); + opportunityToBreak(); } if (recoveredNodes.size() > 0) { // was: for (int i = recoveredNodes.size()-1; i>= 0; i--) { for (int i = 0; i < recoveredNodes.size(); i++) { - AbstractStackNode

recovered = recoveredNodes.getFirst(i); + AbstractStackNode

recovered = recoveredNodes.getFirst(i); // int levelsFromHere = recovered.getLength() - (location - recovered.getStartLocation()); + if (debugListener != null) { + debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + visualize("Queue recovery node", DebugVisualizer.getNodeId(recovered)); + } queueRecoveryNode(recovered, recovered.getStartLocation(), recovered.getLength(), recoveredNodes.getSecond(i)); } return findStacksToReduce(); @@ -884,7 +914,7 @@ private boolean findStacksToReduce(){ return false; } - + public boolean parseErrorHasOccurred(){ return parseErrorOccured; } @@ -1201,6 +1231,7 @@ private void expandStack(AbstractStackNode

stack){ * Initiate stack expansion for all queued stacks. */ private void expand(){ + visualize("Expanding", DebugVisualizer.STACKS_TO_EXPAND_ID); while(!stacksToExpand.isEmpty()){ expandStack(stacksToExpand.pop()); } @@ -1244,14 +1275,16 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] stacksToExpand.push(rootNode); lookAheadChar = (input.length > 0) ? input[0] : 0; - if(debugListener != null) debugListener.shifting(location, input, positionStore); + if(debugListener != null) { + debugListener.shifting(location, input, positionStore); + } expand(); if(findFirstStacksToReduce()){ boolean shiftedLevel = (location != 0); - do{ + do { lookAheadChar = (location < input.length) ? input[location] : 0; if(shiftedLevel){ // Nullable fix for the first level. sharedNextNodes.clear(); @@ -1266,18 +1299,22 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] } // Reduce-expand loop. - do{ + do { if(debugListener != null) debugListener.iterating(); + + reduceTerminals(); - reduce(); + reduceNonTerminals(); expand(); - }while(!stacksWithNonTerminalsToReduce.isEmpty() || !stacksWithTerminalsToReduce.isEmpty()); + } while(!stacksWithNonTerminalsToReduce.isEmpty() || !stacksWithTerminalsToReduce.isEmpty()); shiftedLevel = true; - }while(findStacksToReduce()); + } while(findStacksToReduce()); } + visualize("Done", DebugVisualizer.PARSER_ID); + // Check if we were successful. if(location == input.length){ EdgesSet

startNodeEdgesSet = startNode.getIncomingEdges(); @@ -1424,4 +1461,79 @@ protected T buildResult(AbstractNode result, INodeFlattener converter, INo checkTime("Unbinarizing, post-parse filtering, and mapping to UPTR"); } } + + /** + * Datastructure visualization for debugging purposes + */ + + private void visualize(String step, NodeId highlight) { + // Only visualize when debugging + if (debugListener != null) { + visualizer.createGraph(this, step); + if (highlight != null) { + visualizer.highlight(highlight); + } + visualizer.writeGraph(); + + opportunityToBreak(); + } + } + + private void opportunityToBreak() { + } + + + /** + * Getters used for graph generation/debugging + */ + + public int[] getInput() { + return input; + } + + public int getLocation() { + return location; + } + + public int getLookAheadChar() { + return lookAheadChar; + } + + public DoubleStack, AbstractNode>[] getTodoLists() { + return todoLists; + } + + public int getQueueIndex() { + return queueIndex; + } + + public Stack> getStacksToExpand() { + return stacksToExpand; + } + + public DoubleStack, AbstractContainerNode

> getStacksWithNonTerminalsToReduce() { + return stacksWithNonTerminalsToReduce; + } + + public DoubleStack, AbstractNode> getStacksWithTerminalsToReduce() { + return stacksWithTerminalsToReduce; + } + + public Stack> getUnexpandableNodes() { + return unexpandableNodes; + } + + public Stack> getUnmatchableLeafNodes() { + return unmatchableLeafNodes; + } + + public DoubleStack, AbstractNode>, AbstractStackNode

> getUnmatchableMidProductionNodes() { + return unmatchableMidProductionNodes; + } + + public DoubleStack, AbstractNode> getFilteredNodes() { + return filteredNodes; + } + + } diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index 005b1e99455..e431402f300 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -35,7 +35,7 @@ public abstract class AbstractStackNode

{ protected AbstractStackNode

[][] alternateProductions; // Our edges - protected IntegerObjectList> edgesMap; // : key=startLocation, value=EdgesSet a that location + protected IntegerObjectList> edgesMap; // : key=startLocation, value=EdgesSet at that location // Edges of our children protected ArrayList[] prefixesMap; @@ -187,7 +187,7 @@ public boolean isExpandable(){ * Returns the name associated with the symbol in this node (optional operation). */ public abstract String getName(); - + /** * Check whether of this this node is equal to the given node. */ @@ -745,7 +745,7 @@ public IntegerList getPropagatedReductions(){ } public String toShortString() { - return "id=" + id + ",dot=" + dot + ",start=" + startLocation; + return "." + dot + "@" + startLocation; } @Override @@ -765,7 +765,7 @@ public String toString() { } else { builder.append(","); } - builder.append(prodElem.getId()); + builder.append(prodElem.toShortString()); } builder.append("]"); } @@ -782,9 +782,10 @@ public String toString() { if (alternateProductions != null && alternateProductions.length != 0) { builder.append(",alternateProductions=" + Arrays.toString(alternateProductions)); } + /* if (edgesMap != null && edgesMap.size() != 0) { builder.append(",edges=" + edgesMap); - } + }*/ if (prefixesMap != null && prefixesMap.length != 0) { builder.append(",prefixes=" + Arrays.toString(prefixesMap)); } diff --git a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java index 041e9273ce9..6412f806d25 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java @@ -47,7 +47,7 @@ private AlternativeStackNode(AlternativeStackNode

original, int startLocation children = original.children; } - + /** * Generates and initializes the alternatives for this alternative. */ diff --git a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java index ff2d8b2e326..94274125fc2 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java @@ -89,7 +89,7 @@ public AbstractNode getResult(){ public String toString(){ StringBuilder sb = new StringBuilder(); - sb.append('['); + sb.append("CharStackNode[class="); int[] range = ranges[0]; sb.append(range[0]); sb.append('-'); @@ -101,13 +101,10 @@ public String toString(){ sb.append('-'); sb.append(range[1]); } + sb.append(","); + sb.append(super.toString()); sb.append(']'); - sb.append(getId()); - sb.append('('); - sb.append(startLocation); - sb.append(')'); - return sb.toString(); } diff --git a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java index cd0db6bf2f2..f83bd73703b 100644 --- a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java @@ -77,6 +77,11 @@ public int getLength(){ public AbstractNode getResult(){ return result; } + + @Override + public String toShortString() { + return "'" + UnicodeConverter.unicodeArrayToString(literal) + "'"; + } public String toString(){ StringBuilder sb = new StringBuilder("lit['"); diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 24566530f12..65add6461bf 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -25,6 +25,7 @@ import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.util.visualize.DebugVisualizer; import org.rascalmpl.values.parsetrees.ProductionAdapter; import io.usethesource.vallang.IConstructor; @@ -67,9 +68,6 @@ private DoubleArrayList, AbstractNode> reviveNod for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); - //Pair, ArrayList> elem = elems.get(i); - //AbstractStackNode recoveryNode = elem.getLeft(); - //ArrayList prods = elem.getRight(); // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). for (int j = prods.size() - 1; j >= 0; --j) { @@ -183,10 +181,12 @@ private void collectProductions(AbstractStackNode node, ArrayList< int dot = node.getDot(); + System.err.println("collect productions for node: " + node); if (node.isEndNode()) { IConstructor parentProduction = node.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)){ productions.add(parentProduction); + System.err.println("adding production: " + parentProduction); if (ProductionAdapter.isList(parentProduction)) { return; // Don't follow productions in lists productions, since they are 'cyclic'. @@ -200,6 +200,7 @@ private void collectProductions(AbstractStackNode node, ArrayList< IConstructor parentProduction = currentNode.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)) { productions.add(parentProduction); + System.err.println("adding production at " + i + ": " + parentProduction); } } diff --git a/src/org/rascalmpl/util/visualize/DebugVisualizer.java b/src/org/rascalmpl/util/visualize/DebugVisualizer.java new file mode 100644 index 00000000000..b83f12976d0 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/DebugVisualizer.java @@ -0,0 +1,521 @@ +package org.rascalmpl.util.visualize; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; + +import org.apache.commons.io.FileUtils; +import org.rascalmpl.parser.gtd.SGTDBF; +import org.rascalmpl.parser.gtd.result.AbstractNode; +import org.rascalmpl.parser.gtd.result.CharNode; +import org.rascalmpl.parser.gtd.result.EpsilonNode; +import org.rascalmpl.parser.gtd.result.LiteralNode; +import org.rascalmpl.parser.gtd.result.RecoveredNode; +import org.rascalmpl.parser.gtd.result.SkippedNode; +import org.rascalmpl.parser.gtd.result.SortContainerNode; +import org.rascalmpl.parser.gtd.result.struct.Link; +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; +import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; +import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; +import org.rascalmpl.parser.gtd.util.DoubleArrayList; +import org.rascalmpl.parser.gtd.util.DoubleStack; +import org.rascalmpl.parser.gtd.util.IntegerObjectList; +import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.parser.util.DebugUtil; +import org.rascalmpl.unicode.UnicodeConverter; +import org.rascalmpl.util.visualize.dot.CompassPoint; +import org.rascalmpl.util.visualize.dot.DotAttribute; +import org.rascalmpl.util.visualize.dot.DotEdge; +import org.rascalmpl.util.visualize.dot.DotField; +import org.rascalmpl.util.visualize.dot.DotGraph; +import org.rascalmpl.util.visualize.dot.DotNode; +import org.rascalmpl.util.visualize.dot.DotRecord; +import org.rascalmpl.util.visualize.dot.NodeId; + +import io.usethesource.vallang.IConstructor; + +public class DebugVisualizer { + static final String BASE_DIR = "D:/debug/parser-traces/docs/"; + private static final boolean VISUALIZATION_ENABLED = true; + + public static final NodeId PARSER_ID = new NodeId("Parser"); + public static final NodeId TODO_LISTS_ID= new NodeId("todoLists"); + public static final NodeId STACKS_TO_EXPAND_ID = new NodeId("stacksToExpand"); + public static final NodeId TERMINALS_TO_REDUCE_ID = new NodeId("terminalsToReduce"); + public static final NodeId NON_TERMINALS_TO_REDUCE_ID = new NodeId("nonTerminalsToReduce"); + + public static final NodeId ERROR_TRACKING_ID = new NodeId("error"); + public static final NodeId UNEXPANDABLE_NODES_ID = new NodeId("unexpandableNodes"); + public static final NodeId UNMATCHABLE_LEAF_NODES_ID = new NodeId("unmatchableLeafNodes"); + public static final NodeId UNMATCHABLE_MID_PRODUCTION_NODES_ID = new NodeId("unmatchableMidProductionNodes"); + public static final NodeId FILTERED_NODES_ID = new NodeId("filteredNodes"); + + private static final NodeId RECOVERED_NODES_ID = new NodeId("recoveredNodes"); + + /*static public class GraphObject { + static public class Kind { + private boolean dotGraph = true; + } + + private Kind kind = new Kind(); + private String text = "\ndigraph G {\n a -> b; \n}\n"; + }*/ + + private static class StreamGobbler implements Runnable { + private InputStream inputStream; + private Consumer consumer; + + public StreamGobbler(InputStream inputStream, Consumer consumer) { + this.inputStream = inputStream; + this.consumer = consumer; + } + + @Override + public void run() { + new BufferedReader(new InputStreamReader(inputStream)).lines() + .forEach(consumer); + } + } + + private String name; + private Map stackNodeNodes; + private DotGraph graph; + private int frame; + + public DebugVisualizer(String name) { + this.name = name; + stackNodeNodes = new HashMap<>(); + File frameDir = new File(BASE_DIR + "/frames/" + name); + if (frameDir.exists()) { + try { + FileUtils.deleteDirectory(frameDir); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + frameDir.mkdirs(); + } + + public void visualize(AbstractStackNode node) { + if (VISUALIZATION_ENABLED) { + writeGraph(createGraph(node)); + } + } + + public int getFrame() { + return frame; + } + + private void reset() { + stackNodeNodes.clear(); + graph = null; + frame++; + } + + public void highlight(NodeId id) { + if (graph != null) { + graph.highlight(id); + } + } + + public void highlightStack(AbstractStackNode stack) { + if (graph != null) { + DotNode dotNode = stackNodeNodes.get(stack.getId()); + highlight(dotNode.getId()); + } + } + + private synchronized DotGraph createGraph(AbstractStackNode stackNode) { + reset(); + graph = new DotGraph(name, true); + addStack(graph, stackNode); + return graph; + } + + public synchronized

void addRecoveredNodes(DoubleArrayList, AbstractNode> recoveredNodes) { + addStackAndNodeDoubleList(graph, RECOVERED_NODES_ID, recoveredNodes); + graph.addEdge(ERROR_TRACKING_ID, RECOVERED_NODES_ID, "Nodes to revive"); + highlight(RECOVERED_NODES_ID); + } + + private

DotNode addStack(DotGraph graph, AbstractStackNode

stackNode) { + DotNode node = stackNodeNodes.get(stackNode.getId()); + if (node != null) { + return node; + } + + node = createDotNode(stackNode); + + stackNodeNodes.put(stackNode.getId(), node); + + graph.addNode(node); + + IntegerObjectList> edges = stackNode.getEdges(); + if (edges != null) { + for (int i = edges.size() - 1; i >= 0; --i) { + EdgesSet

edgesList = edges.getValue(i); + if (edgesList != null) { + for (int j = edgesList.size() - 1; j >= 0; --j) { + AbstractStackNode

parentStackNode = edgesList.get(j); + DotNode parentDotNode = addStack(graph, parentStackNode); + graph.addEdge(node.getId(), parentDotNode.getId()); + } + } + } + } + + return node; + } + + private

DotNode createDotNode(AbstractStackNode

stackNode) { + String type = stackNode.getClass().getSimpleName(); + if (type.endsWith("StackNode")) { + type = type.substring(0, type.length() - "StackNode".length()); + } + + String nodeName; + + if (stackNode instanceof RecoveryPointStackNode) { + RecoveryPointStackNode

recoveryNode = (RecoveryPointStackNode

) stackNode; + nodeName = String.valueOf(recoveryNode.getId()); + } else { + try { + nodeName = stackNode.getName(); + } catch (UnsupportedOperationException e) { + nodeName = ""; + } + + if (nodeName.startsWith("layouts_")) { + nodeName = nodeName.substring("layouts_".length()); + } + } + + DotNode node = new DotNode(getNodeId(stackNode)); + String label = String.format("%s: %s\n.%d@%d", + type, nodeName, stackNode.getDot(), stackNode.getStartLocation()); + P parentProduction = stackNode.getParentProduction(); + if (parentProduction instanceof IConstructor) { + label += "\nin: " + DebugUtil.prodToString((IConstructor) parentProduction); + } + node.addAttribute(DotAttribute.ATTR_LABEL, label); + + // TODO: add prefixes + + return node; + } + + private void addParserNode(DotGraph graph, AbstractNode parserNode) { + NodeId id = getNodeId(parserNode); + DotNode dotNode = new DotNode(id); + dotNode.addAttribute(DotAttribute.ATTR_NODE_SHAPE, "octagon"); + + String nodeName = parserNode.getClass().getSimpleName(); + if (nodeName.endsWith("Node")) { + nodeName = nodeName.substring(0, nodeName.length() - "Node".length()); + } + + dotNode.addAttribute(DotAttribute.ATTR_LABEL, nodeName); + + switch (parserNode.getTypeIdentifier()) { + case EpsilonNode.ID: + break; + case CharNode.ID: + enrichCharNode(dotNode, (CharNode) parserNode); + break; + case LiteralNode.ID: + enrichLiteralNode(dotNode, (LiteralNode) parserNode); + break; + case SortContainerNode.ID: + case RecoveredNode.ID: + enrichSortContainerNode(dotNode, (SortContainerNode) parserNode); + break; + case SkippedNode.ID: + enrichSkippedNode(dotNode, (SkippedNode) parserNode); + break; + default: + enrichUnknownParserNode(dotNode, parserNode); + break; + } + + graph.addNode(dotNode); + } + + private void enrichCharNode(DotNode dotNode, CharNode charNode) { + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); + int c = charNode.getCharacter(); + label += "\nchar=" + c + "('" + (char) c + "')"; + dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); + } + + private void enrichLiteralNode(DotNode dotNode, LiteralNode literalNode) { + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); + int[] content = literalNode.getContent(); + label += " \"" + UnicodeConverter.unicodeArrayToString(content) + "\""; + /* Maybe include production? + label += "\nprod=" + DebugUtil.prodToString((IConstructor)literalNode.getProduction()); + */ + dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); + } + + private void enrichSkippedNode(DotNode dotNode, SkippedNode skippedNode) { + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); + label += "\n." + skippedNode.getDot() + "@" + skippedNode.getOffset() + ": " + " \"" + UnicodeConverter.unicodeArrayToString(skippedNode.getSkippedChars()) + "\""; + label += "\nin: " + DebugUtil.prodToString((IConstructor) skippedNode.getProduction()); + + dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); + } + + private void enrichSortContainerNode(DotNode dotNode, SortContainerNode sortNode) { + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); + label += " " + sortNode.getOffset() + "-" + sortNode.getEndOffset(); + label += "\n" + DebugUtil.prodToString(sortNode.getFirstProduction()); + dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); + + // TODO: add links + } + + private void enrichUnknownParserNode(DotNode dotNode, AbstractNode parserNode) { + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); + label += "\ntype=" + parserNode.getTypeIdentifier(); + dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); + } + + public static

NodeId getNodeId(AbstractStackNode

stackNode) { + return new NodeId(String.valueOf(stackNode.getId())); + } + + private static NodeId getNodeId(Object node) { + return new NodeId(String.valueOf(System.identityHashCode(node))); + } + + public void writeGraph() { + if (graph != null) { + writeGraph(graph); + } + } + + public void createGraph(SGTDBF parser, String step) { + if (!VISUALIZATION_ENABLED) { + return; + } + reset(); + + graph = new DotGraph(name, true); + + int location = parser.getLocation(); + + DotNode parserNode = new DotNode(PARSER_ID); + + String input = UnicodeConverter.unicodeArrayToString(parser.getInput()); + + char lookahead = (char) parser.getLookAheadChar(); + if (lookahead == '\0') { + lookahead = '$'; + } + + String label = String.format("Parser\nInput: \"%s\"\nLocation: %d ('%c')\nStep %d: %s", + input, location, lookahead, frame, step); + parserNode.setAttribute(DotAttribute.ATTR_LABEL, label); + graph.addNode(parserNode); + + addTodoLists(parser, graph); + addStacksToExpand(parser, graph); + addTerminalsToReduce(parser, graph); + addNonTerminalsToReduce(parser, graph); + + addErrorNodes(parser, graph); + } + + private void addTodoLists(SGTDBF parser, DotGraph graph) { + DoubleStack, AbstractNode>[] todoLists = parser.getTodoLists(); + int start = parser.getQueueIndex(); + + DotNode todoListsNode = DotNode.createArrayNode(TODO_LISTS_ID, todoLists.length); + + for (int tokenLength=1; tokenLength<=todoLists.length; tokenLength++) { + int index = (start + tokenLength - 1) % todoLists.length; + DoubleStack, AbstractNode> todoList = todoLists[index]; + if (todoList != null && !todoList.isEmpty()) { + NodeId todoListId = new NodeId("todo-" + tokenLength); + addStackAndNodeDoubleStack(graph, todoListId, todoList); + graph.addEdge(DotEdge.createArrayEdge(TODO_LISTS_ID, tokenLength, todoListId)); + } + } + + graph.addNode(todoListsNode); + graph.addEdge(PARSER_ID, TODO_LISTS_ID, "todo lists"); + } + + private void addStacksToExpand(SGTDBF parser, DotGraph graph) { + Stack> stacksToExpand = parser.getStacksToExpand(); + addStackNodeStack(graph, STACKS_TO_EXPAND_ID, stacksToExpand); + graph.addEdge(PARSER_ID, STACKS_TO_EXPAND_ID, "stacks to expand"); + } + + private void addTerminalsToReduce(SGTDBF parser, DotGraph graph) { + addStackAndNodeDoubleStack(graph, TERMINALS_TO_REDUCE_ID, parser.getStacksWithTerminalsToReduce()); + graph.addEdge(PARSER_ID, TERMINALS_TO_REDUCE_ID, "terminals to reduce"); + } + + private void addNonTerminalsToReduce(SGTDBF parser, DotGraph graph) { + addStackAndNodeDoubleStack(graph, NON_TERMINALS_TO_REDUCE_ID, parser.getStacksWithNonTerminalsToReduce()); + graph.addEdge(PARSER_ID, NON_TERMINALS_TO_REDUCE_ID, "non-terminals to reduce"); + } + + private void addErrorNodes(SGTDBF parser, DotGraph graph) { + addUnexpandableNodes(parser, graph); + addUnmatchableLeafNodes(parser, graph); + addUnmatchableMidProductionNodes(parser, graph); + addFilteredNodes(parser, graph); + + graph.addNode(ERROR_TRACKING_ID, "Errors"); + + graph.addEdge(PARSER_ID, ERROR_TRACKING_ID, "error tracking"); + graph.addEdge(ERROR_TRACKING_ID, UNEXPANDABLE_NODES_ID, "unexpandable"); + graph.addEdge(ERROR_TRACKING_ID, UNMATCHABLE_LEAF_NODES_ID, "unmatchable leafs"); + graph.addEdge(ERROR_TRACKING_ID, UNMATCHABLE_MID_PRODUCTION_NODES_ID, "unmatchable mid-prod"); + graph.addEdge(ERROR_TRACKING_ID, FILTERED_NODES_ID, "filtered"); + } + + private void addUnexpandableNodes(SGTDBF parser, DotGraph graph) { + addStackNodeStack(graph, UNEXPANDABLE_NODES_ID, parser.getUnexpandableNodes()); + } + + private void addUnmatchableLeafNodes(SGTDBF parser, DotGraph graph) { + addStackNodeStack(graph, UNMATCHABLE_LEAF_NODES_ID, parser.getUnmatchableLeafNodes()); + } + + private void addUnmatchableMidProductionNodes(SGTDBF parser, DotGraph graph) { + DoubleStack, AbstractNode>, AbstractStackNode

> unmatchableMidProductionNodes = parser.getUnmatchableMidProductionNodes(); + + graph.addArrayNode(UNMATCHABLE_MID_PRODUCTION_NODES_ID, unmatchableMidProductionNodes.getSize()); + for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { + NodeId failureId = new NodeId("unmatchable-mid-production-" + i); + DotRecord failureRecord = new DotRecord(); + failureRecord.addEntry(new DotField("Failed Node", "failedNode")); + failureRecord.addEntry(new DotField("Predecessors", "predecessors")); + graph.addRecordNode(failureId, failureRecord); + graph.addEdge(new NodeId(UNMATCHABLE_MID_PRODUCTION_NODES_ID, String.valueOf(i)), failureId); + + DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); + AbstractStackNode

failedNode = unmatchableMidProductionNodes.getSecond(i); + + DotNode node = addStack(graph, failedNode); + NodeId predecessorsId = new NodeId("unmatchable-mid-production-predecessors-" + i); + addStackAndNodeDoubleList(graph, predecessorsId, failedNodePredecessors); + + graph.addEdge(new NodeId(failureId, "failedNode"), node.getId()); + graph.addEdge(new NodeId(failureId, "predecessors"), predecessorsId); + } + } + + private void addFilteredNodes(SGTDBF parser, DotGraph graph) { + addStackAndNodeDoubleStack(graph, FILTERED_NODES_ID, parser.getFilteredNodes()); + } + /* + * private final Stack> unexpandableNodes; + private final Stack> unmatchableLeafNodes; // Leaf nodes (for instance literals) that failed to match + private final DoubleStack, AbstractNode>, AbstractStackNode

> unmatchableMidProductionNodes; + private final DoubleStack, AbstractNode> filteredNodes; + + */ + + private void addStackAndNodeDoubleStack(DotGraph graph, NodeId nodeId, DoubleStack, N> doubleStack) { + DotNode arrayNode = DotNode.createArrayNode(nodeId, doubleStack == null ? 0 : doubleStack.getSize()); + graph.addNode(arrayNode); + + if (doubleStack == null) { + return; + } + + for (int j=0; j stack = doubleStack.getFirst(j); + DotNode stackDotNode = addStack(graph, stack); + graph.addEdge(new NodeId(nodeId, String.valueOf(j), CompassPoint.SW), stackDotNode.getId(), "Stack"); + + AbstractNode node = doubleStack.getSecond(j); + addParserNode(graph, node); + graph.addEdge(new NodeId(nodeId, String.valueOf(j), CompassPoint.SE), getNodeId(node), "Node"); + } + } + + private void addStackAndNodeDoubleList(DotGraph graph, NodeId nodeId, DoubleArrayList, N> doubleList) { + DotNode arrayNode = DotNode.createArrayNode(nodeId, doubleList.size()); + graph.addNode(arrayNode); + + for (int i=0; i stack = doubleList.getFirst(i); + DotNode stackDotNode = addStack(graph, stack); + graph.addEdge(new NodeId(nodeId, String.valueOf(i), CompassPoint.SW), stackDotNode.getId(), "Stack"); + + AbstractNode node = doubleList.getSecond(i); + addParserNode(graph, node); + graph.addEdge(new NodeId(nodeId, String.valueOf(i), CompassPoint.SE), getNodeId(node), "Node"); + } + } + + private

void addStackNodeStack(DotGraph graph, NodeId nodeId, Stack> stack) { + if (stack == null) { + return; + } + + DotNode arrayNode = DotNode.createArrayNode(nodeId, stack.getSize()); + + for (int j=0; j stackNode = stack.get(j); + addStack(graph, stackNode); + + graph.addEdge(DotEdge.createArrayEdge(nodeId, j, getNodeId(stackNode))); + } + + graph.addNode(arrayNode); + } + + private void writeGraph(DotGraph graph) { + try { + String dotFile = BASE_DIR + name + ".dot"; + String svgFile = BASE_DIR + name + ".svg"; + String frameDir = BASE_DIR + "/frames/" + name + "/"; + String frameSvgFile = frameDir + String.format("%04d", frame) + ".svg"; + String frameDotFile = frameDir + String.format("%04d", frame) + ".dot"; + FileWriter writer = new FileWriter(dotFile); + writer.write(graph.toString()); + writer.close(); + + String cmd = String.format("dot -Tsvg %s -o %s", dotFile, svgFile); + Process process = Runtime.getRuntime().exec(cmd); + StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), System.out::println); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + Future future = executorService.submit(streamGobbler); + + process.waitFor(); + future.get(10, TimeUnit.SECONDS); + + Path svgPath = Paths.get(svgFile); + Path dotPath = Paths.get(dotFile); + Path frameSvgPath = Paths.get(frameSvgFile); + Path frameDotPath = Paths.get(frameDotFile); + Files.copy(svgPath, frameSvgPath, StandardCopyOption.REPLACE_EXISTING); + Files.copy(dotPath, frameDotPath, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException | InterruptedException | ExecutionException | TimeoutException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java b/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java new file mode 100644 index 00000000000..87d745a5e5c --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java @@ -0,0 +1,28 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; + +public class AttributeStatement { + enum Scope { GRAPH, NODE, EDGE } + + private Scope scope; + private DotAttribute attribute; + + void writeSource(PrintWriter writer) { + if (scope != null) { + switch (scope) { + case GRAPH: + writer.write("graph "); + break; + case NODE: + writer.write("node "); + break; + case EDGE: + writer.write("edge "); + break; + } + } + + attribute.writeSource(writer); + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/CompassPoint.java b/src/org/rascalmpl/util/visualize/dot/CompassPoint.java new file mode 100644 index 00000000000..3b31af24fd1 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/CompassPoint.java @@ -0,0 +1,5 @@ +package org.rascalmpl.util.visualize.dot; + +public enum CompassPoint { + N, NE, E, SE, S, SW, W, NW, C, X; +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotAttribute.java b/src/org/rascalmpl/util/visualize/dot/DotAttribute.java new file mode 100644 index 00000000000..761218c6e29 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotAttribute.java @@ -0,0 +1,104 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.List; + +public class DotAttribute { + public static final String ATTR_LABEL = "label"; + public static final String ATTR_COLOR = "color"; + public static final String ATTR_NODE_SHAPE = "shape"; + public static final String NODE_SHAPE_RECORD = "record"; + + String property; + String value; + + public DotAttribute(String property, String value) { + this.property = property; + this.value = value; + } + + public String getProperty() { + return property; + } + + public String getValue() { + return value; + } + + void setValue(String value) { + this.value = value; + } + + void writeSource(PrintWriter writer) { + NodeId.writeId(writer, property); + writer.write("="); + NodeId.writeId(writer, value); + } + + static void writeAttributes(PrintWriter writer, List attributes) { + if (!attributes.isEmpty()) { + writer.write("["); + boolean first = true; + for (DotAttribute attribute : attributes) { + if (first) { + first = false; + } else { + writer.write(", "); + } + attribute.writeSource(writer); + } + writer.write("]"); + } + } + + public static DotAttribute createRecordLabel(List elements) { + StringBuilder value = new StringBuilder(); + + boolean first = true; + for (String element : elements) { + if (first) { + first = false; + } else { + value.append("| "); + } + + value.append('<'); + value.append(element); + value.append('>'); + value.append(' '); + value.append(element); + } + + return new DotAttribute(ATTR_LABEL, value.toString()); + } + + public static DotAttribute createArrayLabel(int size) { + StringBuilder value = new StringBuilder(); + + boolean first = true; + for (int i=0; i'); + value.append(' '); + + value.append(i); + } + + return new DotAttribute(ATTR_LABEL, value.toString()); + } + + public static DotAttribute createRecordLabel(DotRecord rec) { + StringWriter writer = new StringWriter(); + rec.writeSource(new PrintWriter(writer, true), true); + return new DotAttribute(ATTR_LABEL, writer.toString()); + } + +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotEdge.java b/src/org/rascalmpl/util/visualize/dot/DotEdge.java new file mode 100644 index 00000000000..7faf1ee1259 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotEdge.java @@ -0,0 +1,66 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +public class DotEdge implements DotStatement { + private NodeId from; + private List to; + private boolean directed; + private List attributes; + + public DotEdge(NodeId from) { + this(from, true); + } + + public DotEdge(NodeId from, boolean directed) { + this.from = from; + this.directed = directed; + to = new ArrayList<>(); + attributes = new ArrayList<>(); + } + + public DotEdge(NodeId from, NodeId... to) { + this(from, true, to); + } + + public DotEdge(NodeId from, boolean directed, NodeId... to) { + this(from, directed); + for (NodeId node : to) { + addTo(node); + } + } + + public void addTo(NodeId node) { + to.add(node); + } + + public void addAttribute(String property, String value) { + attributes.add(new DotAttribute(property, value)); + } + + @Override + public void writeSource(PrintWriter writer) { + from.writeSource(writer); + for (NodeId node : to) { + if (directed) { + writer.write(" -> "); + } else { + writer.write(" -- "); + } + + node.writeSource(writer); + } + DotAttribute.writeAttributes(writer, attributes); + } + + public static DotEdge createArrayEdge(NodeId array, int index, NodeId element) { + return new DotEdge(new NodeId(array, String.valueOf(index)), true, element); + } + + public static DotEdge createArrayEdge(NodeId array, int index, CompassPoint direction, NodeId element) { + return new DotEdge(new NodeId(array.getId(), String.valueOf(index), direction), true, element); + } +} + diff --git a/src/org/rascalmpl/util/visualize/dot/DotField.java b/src/org/rascalmpl/util/visualize/dot/DotField.java new file mode 100644 index 00000000000..70a25aad7b8 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotField.java @@ -0,0 +1,23 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; + +public class DotField implements DotRecordEntry { + private String label; + private String portId; + + public DotField(String label, String portId) { + this.label = label; + this.portId = portId; + } + + @Override + public void writeSource(PrintWriter writer, boolean topLevel) { + if (portId != null) { + writer.write('<'); + writer.write(portId); + writer.write("> "); + } + NodeId.writeString(writer, label); + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotGraph.java b/src/org/rascalmpl/util/visualize/dot/DotGraph.java new file mode 100644 index 00000000000..27293e2f1e5 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotGraph.java @@ -0,0 +1,107 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/* +Loosely follow (a subset of) the DotGraph syntax definition: +https://graphviz.org/doc/info/lang.html +*/ + +public class DotGraph { + private static final String COLOR_HIGHLIGHT = "red"; + + private String id; + private boolean digraph; + private List statements; + private Map nodes; + + public DotGraph(String id, boolean digraph) { + this.id = id; + this.digraph = digraph; + statements = new ArrayList<>(); + nodes = new HashMap<>(); + } + + private void addStatement(DotStatement statement) { + statements.add(statement); + } + + public void addNode(DotNode node) { + addStatement(node); + nodes.put(node.getId(), node); + } + + public void addNode(String id, String label) { + addNode(new NodeId(id), label); + } + + public void addNode(NodeId id, String label) { + DotNode node = new DotNode(id); + node.addAttribute(DotAttribute.ATTR_LABEL, label); + addNode(node); + } + + public void addArrayNode(NodeId id, int size) { + DotNode node = new DotNode(id); + node.addAttribute(DotAttribute.ATTR_NODE_SHAPE, DotAttribute.NODE_SHAPE_RECORD); + node.addAttribute(DotAttribute.createArrayLabel(size)); + addNode(node); + } + + public void addRecordNode(NodeId id, DotRecord dotRecord) { + DotNode node = new DotNode(id); + node.addAttribute(DotAttribute.ATTR_NODE_SHAPE, DotAttribute.NODE_SHAPE_RECORD); + node.addAttribute(DotAttribute.createRecordLabel(dotRecord)); + addNode(node); + } + + public void highlight(NodeId id) { + DotNode node = nodes.get(id); + if (node != null) { + node.addAttribute(DotAttribute.ATTR_COLOR, COLOR_HIGHLIGHT); + } + } + + public void addEdge(String from, String to) { + addEdge(new NodeId(from), new NodeId(to)); + } + + public void addEdge(NodeId from, NodeId to) { + addEdge(new DotEdge(from, true, to)); + } + + public void addEdge(NodeId from, NodeId to, String label) { + DotEdge edge = new DotEdge(from, true, to); + edge.addAttribute(DotAttribute.ATTR_LABEL, label); + addEdge(edge); + } + + public void addEdge(DotEdge edge) { + addStatement(edge); + } + + public void writeSource(PrintWriter writer) { + writer.write(digraph ? "digraph" : "graph"); + if (id != null) { + writer.write(" "); + writer.write(id); + } + writer.println(" {"); + for (DotStatement statement : statements) { + statement.writeSource(writer); + writer.println(";"); + } + writer.println("}"); + } + + public String toString() { + StringWriter writer = new StringWriter(); + writeSource(new PrintWriter(writer)); + return writer.toString(); + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotNode.java b/src/org/rascalmpl/util/visualize/dot/DotNode.java new file mode 100644 index 00000000000..118d1c00c96 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotNode.java @@ -0,0 +1,69 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +public class DotNode implements DotStatement { + public static DotNode createArrayNode(NodeId id, int size) { + DotNode node = new DotNode(id); + node.addAttribute(DotAttribute.createArrayLabel(size)); + node.addAttribute(DotAttribute.ATTR_NODE_SHAPE, DotAttribute.NODE_SHAPE_RECORD); + return node; + } + + private NodeId id; + private List attributes; + + public DotNode(String id) { + this(new NodeId(id)); + } + + public DotNode(NodeId id) { + this.id = id; + attributes = new ArrayList<>(); + } + + public NodeId getId() { + return id; + } + + public void addAttribute(String property, String value) { + addAttribute(new DotAttribute(property, value)); + } + + public void addAttribute(DotAttribute attribute) { + attributes.add(attribute); + } + + public void setAttribute(String property, String value) { + for (DotAttribute attribute : attributes) { + if (attribute.getProperty().equals(property)) { + attribute.setValue(value); + return; + } + } + + addAttribute(property, value); + } + + public String getAttributeValue(String property) { + for (DotAttribute attribute : attributes) { + if (attribute.getProperty().equals(property)) { + return attribute.getValue(); + } + } + + return null; + } + + public void setLabel(String label) { + setAttribute(DotAttribute.ATTR_LABEL, label); + } + + @Override + public void writeSource(PrintWriter writer) { + id.writeSource(writer); + DotAttribute.writeAttributes(writer, attributes); + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotRecord.java b/src/org/rascalmpl/util/visualize/dot/DotRecord.java new file mode 100644 index 00000000000..73a5cdd73b8 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotRecord.java @@ -0,0 +1,37 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +public class DotRecord implements DotRecordEntry { + private List entries; + + public DotRecord() { + entries = new ArrayList<>(); + } + + public void addEntry(DotRecordEntry entry) { + entries.add(entry); + } + + @Override + public void writeSource(PrintWriter writer, boolean topLevel) { + if (!topLevel) { + writer.write("{ "); + } + boolean first = true; + for (DotRecordEntry entry : entries) { + if (first) { + first = false; + } else { + writer.write(" | "); + } + + entry.writeSource(writer, false); + } + if (!topLevel) { + writer.write("} "); + } + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java b/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java new file mode 100644 index 00000000000..a0ed741e9f0 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java @@ -0,0 +1,7 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; + +public interface DotRecordEntry { + void writeSource(PrintWriter writer, boolean topLevel); +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotStatement.java b/src/org/rascalmpl/util/visualize/dot/DotStatement.java new file mode 100644 index 00000000000..83919814059 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotStatement.java @@ -0,0 +1,7 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; + +public interface DotStatement { + void writeSource(PrintWriter writer); +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java b/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java new file mode 100644 index 00000000000..23e7fb6b6a6 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java @@ -0,0 +1,40 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +public class DotSubgraph implements DotStatement { + private String id; + private List statements; + + public DotSubgraph(String id) { + this.id = id; + statements = new ArrayList<>(); + } + + public void addStatement(DotStatement statement) { + statements.add(statement); + } + + public String getId() { + return id; + } + + @Override + public void writeSource(PrintWriter writer) { + writer.write("subgraph"); + if (id != null) { + NodeId.writeId(writer, id); + writer.write(" "); + } + writer.println(" {"); + for (DotStatement statement : statements) { + writer.write(" "); + statement.writeSource(writer); + writer.println(";"); + } + + writer.println(" }"); + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/NodeId.java b/src/org/rascalmpl/util/visualize/dot/NodeId.java new file mode 100644 index 00000000000..fe2c3849ea3 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/NodeId.java @@ -0,0 +1,126 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; + +public class NodeId { + public static void writeId(PrintWriter stream, String id) { + stream.write("\""); + writeString(stream, id); + stream.write("\""); + } + + public static void writeString(PrintWriter stream, String s) { + for (int i=0; i Date: Thu, 18 Jul 2024 13:01:46 +0200 Subject: [PATCH 038/190] Implemented skipping of remaining characters when top-level production is recovered --- .../tests/recovery/BasicRecoveryTests.rsc | 11 ++++- src/org/rascalmpl/parser/gtd/SGTDBF.java | 4 +- .../parser/gtd/stack/SkippingStackNode.java | 46 +++++++++++-------- .../recovery/ToNextWhitespaceRecoverer.java | 41 ++++++++++++++++- 4 files changed, 79 insertions(+), 23 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index ec7ff7270a1..2dc8c2fbdec 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -1,8 +1,9 @@ module lang::rascal::tests::recovery::BasicRecoveryTests import ParseTree; +import IO; -layout Layout = [\ ]; //* !>> [\ ]; +layout Layout = [\ ]* !>> [\ ]; syntax S = ABC End; syntax ABC = "a" "b" "c"; @@ -13,7 +14,8 @@ test bool ok() { } test bool abx() { - Tree t = parse(#S, "a b x $", allowRecovery=true); + Tree t = parse(#S, "a b x $", allowRecovery=true, allowAmbiguity=true); + iprintln(t); return hasErrors(t) && size(findAllErrors(t)) == 1; } @@ -26,3 +28,8 @@ test bool ax() { Tree t = parse(#S, "a x $", allowRecovery=true); return hasErrors(t) && size(findAllErrors(t)) == 1; } + +test bool missingEnd() { + Tree t = parse(#S, "a b c", allowRecovery=true); + return hasErrors(t) && size(findAllErrors(t)) == 1; +} diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index b7f7aaa7a5d..f3094d2e4e0 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -861,7 +861,7 @@ private boolean findFirstStacksToReduce(){ private boolean findStacksToReduce(){ visualize("Finding stacks to reduce", DebugVisualizer.TODO_LISTS_ID); int queueDepth = todoLists.length; - for(int i = 1; i < queueDepth; ++i){ + for(int i = 1; i < queueDepth-1; ++i){ queueIndex = (queueIndex + 1) % queueDepth; DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; @@ -879,7 +879,7 @@ private boolean findStacksToReduce(){ } } - if (recoverer != null && location < input.length) { + if (recoverer != null) { if (debugListener != null) { visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 2ef909c5684..7030149d130 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -14,13 +14,39 @@ import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.result.SkippedNode; +import io.usethesource.vallang.IConstructor; + public final class SkippingStackNode

extends AbstractMatchableStackNode

{ private final SkippedNode result; - public SkippingStackNode(int id, int[] until, int[] input, int startLocation, P parentProduction, int dot){ + public static SkippedNode createResultUntilCharClass(int[] until, int[] input, int startLocation, IConstructor production, int dot) { + for (int to = startLocation ; to < input.length; ++to) { + for (int i = 0; i < until.length; ++i) { + if (input[to] == until[i]) { + int length = to - startLocation; + return new SkippedNode(production, dot, createSkippedToken(input, startLocation, length), startLocation); + } + } + } + + return new SkippedNode(production, dot, new int[0], startLocation); + } + + public static SkippedNode createResultUntilEndOfInput(int[] input, int startLocation, IConstructor production, int dot) { + int length = input.length - startLocation; + return new SkippedNode(production, dot, createSkippedToken(input, startLocation, length), startLocation); + } + + private static int[] createSkippedToken(int[] input, int startLocation, int length) { + int[] token = new int[length]; + System.arraycopy(input, startLocation, token, 0, length); + return token; + } + + public SkippingStackNode(int id, P parentProduction, SkippedNode result) { super(id, 0); - this.result = buildResult(input, until, startLocation, parentProduction, dot); + this.result = result; setAlternativeProduction(parentProduction); } @@ -35,23 +61,7 @@ private SkippingStackNode(SkippingStackNode

original, SkippedNode result, int this.result = result; } - - private SkippedNode buildResult(int[] input, int[] until, int startLocation, P production, int dot){ - for (int to = startLocation ; to < input.length; ++to) { - for (int i = 0; i < until.length; ++i) { - if (input[to] == until[i]) { - int length = to - startLocation; - int[] chars = new int[length]; - System.arraycopy(input, startLocation, chars, 0, length); - - return new SkippedNode(production, dot, chars, startLocation); - } - } - } - return new SkippedNode(production, dot, new int[0], startLocation); - } - public boolean isEmptyLeafNode(){ return result.isEmpty(); } diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 65add6461bf..e799a05b42e 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -14,6 +14,7 @@ import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.AbstractNode; +import org.rascalmpl.parser.gtd.result.SkippedNode; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; import org.rascalmpl.parser.gtd.stack.SkippingStackNode; @@ -75,8 +76,17 @@ private DoubleArrayList, AbstractNode> reviveNod AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); + int dot = recoveryNode.getDot(); int startLocation = recoveryNode.getStartLocation(); - AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), WHITESPACE, input, startLocation, prod, recoveryNode.getDot()); + + SkippedNode result; + if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { + result = SkippingStackNode.createResultUntilEndOfInput(input, startLocation, prod, dot); + } else { + result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, dot); + } + + AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result); recoverLiteral = recoverLiteral.getCleanCopy(startLocation); // This copy might not be needed recoverLiteral.initEdges(); EdgesSet edges = new EdgesSet<>(1); @@ -92,6 +102,35 @@ private DoubleArrayList, AbstractNode> reviveNod return recoveredNodes; } + + // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) + // As this is experimental code, this method is extremely conservative. + // Any sharing will result in returning 'false'. + // We will need to change this strategy in the future to improve error recovery. + private boolean isTopLevelProduction(AbstractStackNode node) { + while (node != null && node.getDot() != 0) { + node = getSinglePredecessor(node); + } + + if (node != null) { + node = getSinglePredecessor(node); + return node != null && node.getStartLocation() == -1; + } + + return false; + } + + private AbstractStackNode getSinglePredecessor(AbstractStackNode node) { + IntegerObjectList> edgeMap = node.getEdges(); + if (edgeMap.size() == 1) { + EdgesSet edges = edgeMap.getValue(0); + if (edges.size() == 1) { + return edges.get(0); + } + } + + return null; + } private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList, ArrayList>(); From 2b044fdd0ca49eb0785c23c351a6ff6066351c4b Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 18 Jul 2024 13:25:56 +0200 Subject: [PATCH 039/190] Reinstated end-of-input check before recovering (for now) --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index f3094d2e4e0..6743876f9b2 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -879,7 +879,7 @@ private boolean findStacksToReduce(){ } } - if (recoverer != null) { + if (recoverer != null && location < input.length) { if (debugListener != null) { visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); From d5dbe8c2cf92543fdddb558e827eca1677b029a0 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 6 Aug 2024 09:32:21 +0200 Subject: [PATCH 040/190] Minor comment and test changes --- .../library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc | 4 ++++ src/org/rascalmpl/parser/gtd/SGTDBF.java | 2 -- .../parser/uptr/recovery/ToNextWhitespaceRecoverer.java | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index 2dc8c2fbdec..c9078463389 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -21,15 +21,19 @@ test bool abx() { test bool axc() { Tree t = parse(#S, "a x c $", allowRecovery=true); + iprintln(t); return hasErrors(t) && size(findAllErrors(t)) == 1; } test bool ax() { Tree t = parse(#S, "a x $", allowRecovery=true); + iprintln(t); return hasErrors(t) && size(findAllErrors(t)) == 1; } +/* test bool missingEnd() { Tree t = parse(#S, "a b c", allowRecovery=true); return hasErrors(t) && size(findAllErrors(t)) == 1; } +*/ diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 6743876f9b2..2bf7774ed65 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -949,8 +949,6 @@ private void queueMatchableNode(AbstractStackNode

node, int length, AbstractN * Inserts a recovery node into the todo-list, and possibly * rewinds the parser to an earlier location in the input */ - // TODO: look for original code: - // - queueMatchableNode (just above here) @SuppressWarnings("unchecked") private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int length, AbstractNode result){ assert result != null; diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index e799a05b42e..7ca1a20a46f 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -26,7 +26,6 @@ import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; -import org.rascalmpl.util.visualize.DebugVisualizer; import org.rascalmpl.values.parsetrees.ProductionAdapter; import io.usethesource.vallang.IConstructor; From 2246e06b8f33e013bbcb26008f6f72f4371666bf Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 8 Aug 2024 10:34:20 +0200 Subject: [PATCH 041/190] In addition to using whitespace we now also skip until after the last token in the production --- src/org/rascalmpl/library/ParseTree.rsc | 30 +- .../tests/recovery/BasicRecoveryTests2.rsc | 40 +++ .../gtd/stack/AbstractMatchableStackNode.java | 6 +- .../parser/gtd/stack/AbstractStackNode.java | 10 +- .../parser/gtd/stack/SkippingStackNode.java | 31 +- .../uptr/recovery/ToTokenRecoverer.java | 305 ++++++++++++++++++ .../util/visualize/DebugVisualizer.java | 19 +- .../values/RascalFunctionValueFactory.java | 3 +- 8 files changed, 430 insertions(+), 14 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc create mode 100644 src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 010655cf502..1f88edb3ef1 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -143,6 +143,7 @@ extend Type; extend Message; extend List; +import String; @synopsis{The Tree data type as produced by the parser.} @description{ @@ -770,21 +771,38 @@ default bool isNonTerminalType(Symbol s) = false; @synopsis{Check if a parse tree contains any skipped nodes, the result of error recovery.} bool hasErrors(Tree tree) = /appl(skipped(_, _, _), _) := tree; -@synopsis{Find all skipped nodes in a parse tree.} +@synopsis{Find all error productions in a parse tree.} list[Tree] findAllErrors(Tree tree) { return for (/error:appl(_, [*_, appl(skipped(_, _, _), _)]) := tree) append(error); } -@synopsis{Find the first skipped node in a parse tree.} +@synopsis{Find the first production containing an error.} Tree findFirstError(Tree tree) { if (/error:appl(_, [*_, appl(skipped(_, _, _), _)]) := tree) return error; fail; } -Symbol getErrorSymbol(appl(skipped(Symbol symbol, _, _), _)) = symbol; +private Production getSkipped(appl(_, [*_, appl(skip:skipped(_, _, _), _)])) = skip; -Production getErrorProduction(appl(skipped(_, Production production, _), _)) = production; +Symbol getErrorSymbol(Tree error) { + if (skipped(Symbol s, _, _) := getSkipped(error)) { + return s; + } + fail; +} + +Production getErrorProduction(Tree error) { + if (skipped(_, Production p, _) := getSkipped(error)) { + return p; + } + fail; +} -int getErrorPosition(appl(skipped(_, _, int dot), _)) = dot; +int getErrorPosition(Tree error) { + if (skipped(_, _, int dot) := getSkipped(error)) { + return dot; + } + fail; +} -str getErrorString(appl(skipped(_, _, _), list chars)) = ""; +str getErrorText(appl(_, [*_, appl(skipped(_, _, _), chars)])) = stringChars([c | ch <- chars, char(c) := ch]); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc new file mode 100644 index 00000000000..5a08df51ec7 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc @@ -0,0 +1,40 @@ +module lang::rascal::tests::recovery::BasicRecoveryTests2 + +import ParseTree; +import IO; + +layout Layout = [\ ]* !>> [\ ]; + +syntax S = T; + +syntax T = ABC End; +syntax ABC = "a" "b" "c"; +syntax End = "$"; + +test bool ok() { + return !hasErrors(parse(#S, "a b c $", allowRecovery=true)); +} + +test bool abx() { + Tree t = parse(#S, "a b x $", allowRecovery=true); + return getErrorText(findFirstError(t)) == "x"; +} + +test bool axc() { + Tree t = parse(#S, "a x c $", allowRecovery=true); + iprintln(t); + return getErrorText(findFirstError(t)) == "x c"; +} + +test bool ax() { + Tree t = parse(#S, "a x $", allowRecovery=true); + return getErrorText(findFirstError(t)) == "x"; +} + +/* +test bool missingEnd() { + Tree t = parse(#S, "a b c", allowRecovery=true); + println("Error text: "); + return hasErrors(t) && size(findAllErrors(t)) == 1; +} +*/ diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java index 3afee7e1894..3943c6dca5f 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java @@ -25,7 +25,11 @@ public abstract class AbstractMatchableStackNode

extends AbstractStackNode

protected AbstractMatchableStackNode(int id, int dot){ super(id, dot); } - + + protected AbstractMatchableStackNode(int id, int dot, int startLocation){ + super(id, dot, startLocation); + } + protected AbstractMatchableStackNode(int id, int dot, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); } diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index e431402f300..b477301e954 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -63,17 +63,21 @@ public abstract class AbstractStackNode

{ private IntegerList propagatedReductions; protected AbstractStackNode(int id, int dot){ + this(id, dot, DEFAULT_START_LOCATION); + } + + protected AbstractStackNode(int id, int dot, int startLocation) { super(); this.id = id; this.dot = dot; - this.startLocation = DEFAULT_START_LOCATION; + this.startLocation = startLocation; this.enterFilters = null; this.completionFilters = null; } - + protected AbstractStackNode(int id, int dot, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(); @@ -745,7 +749,7 @@ public IntegerList getPropagatedReductions(){ } public String toShortString() { - return "." + dot + "@" + startLocation; + return null; } @Override diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 7030149d130..ae0bcf41be0 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -37,6 +37,28 @@ public static SkippedNode createResultUntilEndOfInput(int[] input, int startLoca return new SkippedNode(production, dot, createSkippedToken(input, startLocation, length), startLocation); } + public static SkippedNode createResultUntilToken(String token, int[] input, int startLocation, IConstructor production, int dot) { + int length = token.length(); + for (int start=startLocation; start+length < input.length; start++) { + boolean match = true; + for (int j=0; j original, int startLocation){ super(original, startLocation); diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java new file mode 100644 index 00000000000..858f2d7cdf3 --- /dev/null +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -0,0 +1,305 @@ +/******************************************************************************* + * Copyright (c) 2009-2022 NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + + * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI + * * Arnold Lankamp - Arnold.Lankamp@cwi.nl +*******************************************************************************/ +package org.rascalmpl.parser.uptr.recovery; + +import java.util.List; + +import org.rascalmpl.parser.gtd.recovery.IRecoverer; +import org.rascalmpl.parser.gtd.result.AbstractNode; +import org.rascalmpl.parser.gtd.result.SkippedNode; +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; +import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; +import org.rascalmpl.parser.gtd.stack.SkippingStackNode; +import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; +import org.rascalmpl.parser.gtd.util.ArrayList; +import org.rascalmpl.parser.gtd.util.DoubleArrayList; +import org.rascalmpl.parser.gtd.util.DoubleStack; +import org.rascalmpl.parser.gtd.util.IdDispenser; +import org.rascalmpl.parser.gtd.util.IntegerObjectList; +import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; +import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.values.RascalValueFactory; +import org.rascalmpl.values.parsetrees.ProductionAdapter; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IList; +import io.usethesource.vallang.IString; +import io.usethesource.vallang.IValue; + +public class ToTokenRecoverer implements IRecoverer { + private static final int[] WHITESPACE = {' ', '\r', '\n', '\t' }; + + private IdDispenser stackNodeIdDispenser; + + public ToTokenRecoverer(IdDispenser stackNodeIdDispenser) { + this.stackNodeIdDispenser = stackNodeIdDispenser; + } + + @Override + public DoubleArrayList, AbstractNode> reviveStacks(int[] input, + int location, + Stack> unexpandableNodes, + Stack> unmatchableLeafNodes, + DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, + DoubleStack, AbstractNode> filteredNodes) { + + ArrayList> failedNodes = new ArrayList>(); + collectUnexpandableNodes(unexpandableNodes, failedNodes); + collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); + // TODO: handle unmatchableLeafNodes + //collectFilteredNodes(filteredNodes, failedNodes); + + return reviveFailedNodes(input, location, failedNodes); + } + + private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ + DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); + + // original: for (int i = recoveryNodes.size() - 1; i >= 0; --i) { + // But this caused problems because recovery nodes with a later position + // where queued before nodes with an earlier position which the parser cannot handle. + + recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); + + for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); + ArrayList prods = recoveryNodes.getSecond(i); + + int dot = recoveryNode.getDot(); + int startLocation = recoveryNode.getStartLocation(); + + // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). + for (int j = prods.size() - 1; j >= 0; --j) { + IConstructor prod = prods.get(j); + + List> skippingNodes = findSkippingNodes(input, recoveryNode, prod, dot, startLocation); + for (SkippingStackNode skippingNode : skippingNodes) { + AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); + + EdgesSet edges = new EdgesSet<>(1); + edges.add(continuer); + + continuer.setIncomingEdges(edges); + + skippingNode.initEdges(); + skippingNode.addEdges(edges, startLocation); + recoveredNodes.add(skippingNode, skippingNode.getResult()); + } + } + } + + return recoveredNodes; + } + + List> findSkippingNodes(int[] input, AbstractStackNode recoveryNode, IConstructor prod, int dot, int startLocation) { + List> nodes = new java.util.ArrayList<>(); + + SkippedNode result; + + // If we are the top-level node, just skip the rest of the input + if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { + result = SkippingStackNode.createResultUntilEndOfInput(input, startLocation, prod, dot); + nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); + return nodes; // No other nodes would be useful + } + + // Try to find whitespace to skip to + result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, dot); + if (result != null) { + nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); + } + + // Do something more clever: find the last token of this production and skip until after that + String lastLiteral = findLastLiteral(prod); + if (lastLiteral != null) { + result = SkippingStackNode.createResultUntilToken(lastLiteral, input, startLocation, prod, dot); + if (result != null) { + nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); + } + } + + // TODO: or find the first token of the next production and skip until before that + + return nodes; + } + + String findLastLiteral(IConstructor prod) { + IList args = (IList) prod.get(1); + if (args.isEmpty()) { + return null; + } + + IValue last = args.get(args.length()-1); + if (last instanceof IConstructor) { + IConstructor cons = (IConstructor) last; + if (cons.getConstructorType() == RascalValueFactory.Symbol_Lit) { + return ((IString) cons.get(0)).getValue(); + } + } + + return null; + } + + // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) + // As this is experimental code, this method is extremely conservative. + // Any sharing will result in returning 'false'. + // We will need to change this strategy in the future to improve error recovery. + private boolean isTopLevelProduction(AbstractStackNode node) { + while (node != null && node.getDot() != 0) { + node = getSinglePredecessor(node); + } + + if (node != null) { + node = getSinglePredecessor(node); + return node != null && node.getStartLocation() == -1; + } + + return false; + } + + private AbstractStackNode getSinglePredecessor(AbstractStackNode node) { + IntegerObjectList> edgeMap = node.getEdges(); + if (edgeMap.size() == 1) { + EdgesSet edges = edgeMap.getValue(0); + if (edges.size() == 1) { + return edges.get(0); + } + } + + return null; + } + + private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { + DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList, ArrayList>(); + + for (int i = failedNodes.size() - 1; i >= 0; --i) { + findRecoveryNodes(failedNodes.get(i), recoveryNodes); + } + + return reviveNodes(input, location, recoveryNodes); + } + + private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { + for (int i = unexpandableNodes.getSize() - 1; i >= 0; --i) { + failedNodes.add(unexpandableNodes.get(i)); + } + } + + /** + * Make a fresh copy of each unmatchable mid-production node and link in the predecessors of the original node. + * The new copies are added to `failedNodes` + * @param location the location where the failure occurs + * @param unmatchableMidProductionNodes each pair consists of a list of predecessors and a node that failed to match + * @param failedNodes the list to which failed nodes must be added + */ + private static void collectUnmatchableMidProductionNodes(int location, DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ + for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { + DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); + AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference updates of the static version + + // Merge the information on the predecessors into the failed node. + for(int j = failedNodePredecessors.size() - 1; j >= 0; --j) { + AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); + AbstractNode predecessorResult = failedNodePredecessors.getSecond(j); + failedNode.updateNode(predecessor, predecessorResult); + } + + failedNodes.add(failedNode); + } + } + + /** + * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. + */ + private void findRecoveryNodes(AbstractStackNode failer, DoubleArrayList, ArrayList> recoveryNodes) { + ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap>(); + Stack> todo = new Stack>(); + + todo.push(failer); + + while (!todo.isEmpty()) { + AbstractStackNode node = todo.pop(); + + if (visited.contains(node)) { + continue; // Don't follow cycles + } + + visited.put(node, 0); + + if (node == failer || node.getDot() == 0) { + ArrayList recoveryProductions = new ArrayList(); + collectProductions(node, recoveryProductions); + if (recoveryProductions.size() > 0) { + recoveryNodes.add(node, recoveryProductions); + } + } + + IntegerObjectList> edges = node.getEdges(); + + for (int i = edges.size() - 1; i >= 0; --i) { // Rewind + EdgesSet edgesList = edges.getValue(i); + + if (edgesList != null) { + for (int j = edgesList.size() - 1; j >= 0; --j) { + AbstractStackNode parent = edgesList.get(j); + + todo.push(parent); + } + } + } + } + } + + // Gathers all productions that are marked for recovery (the given node can be part of a prefix shared production) + private void collectProductions(AbstractStackNode node, ArrayList productions) { + AbstractStackNode[] production = node.getProduction(); + if (production == null) { + return; // The root node does not have a production, so ignore it. + } + + int dot = node.getDot(); + + System.err.println("collect productions for node: " + node); + if (node.isEndNode()) { + IConstructor parentProduction = node.getParentProduction(); + if (ProductionAdapter.isContextFree(parentProduction)){ + productions.add(parentProduction); + System.err.println("adding production: " + parentProduction); + + if (ProductionAdapter.isList(parentProduction)) { + return; // Don't follow productions in lists productions, since they are 'cyclic'. + } + } + } + + for (int i = dot + 1; i < production.length; ++i) { + AbstractStackNode currentNode = production[i]; + if (currentNode.isEndNode()) { + IConstructor parentProduction = currentNode.getParentProduction(); + if (ProductionAdapter.isContextFree(parentProduction)) { + productions.add(parentProduction); + System.err.println("adding production at " + i + ": " + parentProduction); + } + } + + AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); + if (alternateProductions != null) { + for (int j = alternateProductions.length - 1; j >= 0; --j) { + collectProductions(alternateProductions[j][i], productions); + } + } + } + } + + +} diff --git a/src/org/rascalmpl/util/visualize/DebugVisualizer.java b/src/org/rascalmpl/util/visualize/DebugVisualizer.java index b83f12976d0..b0fb64f41ca 100644 --- a/src/org/rascalmpl/util/visualize/DebugVisualizer.java +++ b/src/org/rascalmpl/util/visualize/DebugVisualizer.java @@ -210,6 +210,12 @@ private

DotNode createDotNode(AbstractStackNode

stackNode) { DotNode node = new DotNode(getNodeId(stackNode)); String label = String.format("%s: %s\n.%d@%d", type, nodeName, stackNode.getDot(), stackNode.getStartLocation()); + + String shortString = stackNode.toShortString(); + if (shortString != null) { + label += "\n" + shortString; + } + P parentProduction = stackNode.getParentProduction(); if (parentProduction instanceof IConstructor) { label += "\nin: " + DebugUtil.prodToString((IConstructor) parentProduction); @@ -461,13 +467,22 @@ private void addStackAndNodeDoubleList(DotGraph grap graph.addNode(arrayNode); for (int i=0; i stack = doubleList.getFirst(i); DotNode stackDotNode = addStack(graph, stack); - graph.addEdge(new NodeId(nodeId, String.valueOf(i), CompassPoint.SW), stackDotNode.getId(), "Stack"); + graph.addEdge(new NodeId(entryId, "stack", CompassPoint.SW), stackDotNode.getId(), "Stack"); AbstractNode node = doubleList.getSecond(i); addParserNode(graph, node); - graph.addEdge(new NodeId(nodeId, String.valueOf(i), CompassPoint.SE), getNodeId(node), "Node"); + graph.addEdge(new NodeId(entryId, "node", CompassPoint.SE), getNodeId(node), "Node"); + + + graph.addEdge(new NodeId(nodeId, String.valueOf(i)), entryId); } } diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 0a4cffc46a5..ca7d5ed073a 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -55,6 +55,7 @@ import org.rascalmpl.parser.uptr.action.RascalFunctionActionExecutor; import org.rascalmpl.parser.uptr.debug.DebugLogger; import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; +import org.rascalmpl.parser.uptr.recovery.ToTokenRecoverer; import org.rascalmpl.types.NonTerminalType; import org.rascalmpl.types.RascalTypeFactory; import org.rascalmpl.types.ReifiedType; @@ -580,7 +581,7 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in IRecoverer recoverer = null; IDebugListener debugListener = null; if (allowRecovery) { - recoverer = new ToNextWhitespaceRecoverer(new StackNodeIdDispenser(parserInstance)); + recoverer = new ToTokenRecoverer(new StackNodeIdDispenser(parserInstance)); debugListener = new DebugLogger(new PrintWriter(System.out, true)); } return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer, debugListener); From 4b11450c03f676c12d32048c94a23e639484a6f1 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 19 Aug 2024 10:32:54 +0200 Subject: [PATCH 042/190] Implemented dynamic last/follow calculations --- .vscode/launch.json | 9 + .../tests/recovery/BasicRecoveryTests.rsc | 34 ++-- .../tests/recovery/BasicRecoveryTests2.rsc | 40 ----- .../tests/recovery/ListRecoveryTests.rsc | 32 ++++ .../tests/recovery/NestedRecoveryTests.rsc | 28 ++++ .../tests/recovery/PicoRecoveryTests.rsc | 122 ++++++++++++++ .../tests/recovery/RunRecoveryTests.rsc | 9 + src/org/rascalmpl/parser/gtd/SGTDBF.java | 20 ++- .../CaseInsensitiveLiteralStackNode.java | 13 ++ .../parser/gtd/stack/CharStackNode.java | 30 +++- .../parser/gtd/stack/LiteralStackNode.java | 4 + .../gtd/stack/SeparatedListStackNode.java | 5 +- .../parser/gtd/stack/SkippingStackNode.java | 4 +- .../CaseInsensitiveLiteralMatcher.java | 66 ++++++++ .../parser/uptr/recovery/FailingMatcher.java | 8 + .../parser/uptr/recovery/InputMatcher.java | 66 ++++++++ .../parser/uptr/recovery/LiteralMatcher.java | 41 +++++ .../uptr/recovery/ToTokenRecoverer.java | 158 +++++++++++++++--- src/org/rascalmpl/parser/util/DebugUtil.java | 4 +- .../util/visualize/DebugVisualizer.java | 112 ++++++++++++- .../util/visualize/dot/DotGraph.java | 6 +- .../values/RascalFunctionValueFactory.java | 2 +- 22 files changed, 716 insertions(+), 97 deletions(-) delete mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc create mode 100644 src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java create mode 100644 src/org/rascalmpl/parser/uptr/recovery/FailingMatcher.java create mode 100644 src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java create mode 100644 src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java diff --git a/.vscode/launch.json b/.vscode/launch.json index 8d7a2f6a52c..e16d6d8230e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -39,6 +39,15 @@ "projectName": "rascal", "vmArgs": "-Xss80m -Xmx2g -ea" }, + { + "type": "java", + "name": "Recovery tests", + "request": "launch", + "mainClass": "org.rascalmpl.shell.RascalShell", + "projectName": "rascal", + "vmArgs": "-Xss80m -Xmx2g -ea", + "args": "lang::rascal::tests::recovery::RunRecoveryTests" + }, { "type": "java", "name": "Launch RascalShell Tutor", diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index c9078463389..e84ba513118 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -5,35 +5,35 @@ import IO; layout Layout = [\ ]* !>> [\ ]; -syntax S = ABC End; -syntax ABC = "a" "b" "c"; +syntax S = T; + +syntax T = ABC End; +syntax ABC = 'a' 'b' 'c'; syntax End = "$"; -test bool ok() { - return !hasErrors(parse(#S, "a b c $", allowRecovery=true)); +private Tree parseS(str input, bool visualize=false) + = parser(#S, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); + +test bool basicOk() { + return !hasErrors(parseS("a b c $", visualize=true)); } test bool abx() { - Tree t = parse(#S, "a b x $", allowRecovery=true, allowAmbiguity=true); - iprintln(t); - return hasErrors(t) && size(findAllErrors(t)) == 1; + Tree t = parseS("a b x $", visualize=true); + return getErrorText(findFirstError(t)) == "x"; } test bool axc() { - Tree t = parse(#S, "a x c $", allowRecovery=true); + Tree t = parseS("a x c $", visualize=true); iprintln(t); - return hasErrors(t) && size(findAllErrors(t)) == 1; + return getErrorText(findFirstError(t)) == "x c"; } test bool ax() { - Tree t = parse(#S, "a x $", allowRecovery=true); - iprintln(t); - return hasErrors(t) && size(findAllErrors(t)) == 1; + Tree t = parseS("a x $", visualize=true); + return getErrorText(findFirstError(t)) = "x"; } -/* -test bool missingEnd() { - Tree t = parse(#S, "a b c", allowRecovery=true); - return hasErrors(t) && size(findAllErrors(t)) == 1; +int main(list[str] args){ + startRepl(REPL()); } -*/ diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc deleted file mode 100644 index 5a08df51ec7..00000000000 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc +++ /dev/null @@ -1,40 +0,0 @@ -module lang::rascal::tests::recovery::BasicRecoveryTests2 - -import ParseTree; -import IO; - -layout Layout = [\ ]* !>> [\ ]; - -syntax S = T; - -syntax T = ABC End; -syntax ABC = "a" "b" "c"; -syntax End = "$"; - -test bool ok() { - return !hasErrors(parse(#S, "a b c $", allowRecovery=true)); -} - -test bool abx() { - Tree t = parse(#S, "a b x $", allowRecovery=true); - return getErrorText(findFirstError(t)) == "x"; -} - -test bool axc() { - Tree t = parse(#S, "a x c $", allowRecovery=true); - iprintln(t); - return getErrorText(findFirstError(t)) == "x c"; -} - -test bool ax() { - Tree t = parse(#S, "a x $", allowRecovery=true); - return getErrorText(findFirstError(t)) == "x"; -} - -/* -test bool missingEnd() { - Tree t = parse(#S, "a b c", allowRecovery=true); - println("Error text: "); - return hasErrors(t) && size(findAllErrors(t)) == 1; -} -*/ diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc new file mode 100644 index 00000000000..184fd2b2eb8 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc @@ -0,0 +1,32 @@ +module lang::rascal::tests::recovery::ListRecoveryTests + +import ParseTree; +import IO; + +layout Layout = [\ ]* !>> [\ ]; + +syntax S = T End; + +syntax T = { AB "," }*; +syntax AB = "a" "b"; +syntax End = "$"; + +Tree parseList(str s, bool visualize=false) { + return parser(#S, allowRecovery=true, allowAmbiguity=true)(s, |unknown:///?visualize=<"">|); +} + +test bool listOk() { + return !hasErrors(parseList("a b , a b , a b $", visualize=true)); +} + +test bool listTypo() { + Tree t = parseList("a b, a x, ab $", visualize=true); + iprintln(t); + return hasErrors(t); +} + +test bool listTypoWs() { + Tree t = parseList("a b , a x , a b $", visualize=true); + iprintln(t); + return hasErrors(t); +} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc new file mode 100644 index 00000000000..292043f00a4 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc @@ -0,0 +1,28 @@ +module lang::rascal::tests::recovery::NestedRecoveryTests + +import ParseTree; +import IO; + +layout Layout = [\ ]* !>> [\ ]; + +syntax S = T; + +syntax T = A B "c"; + +syntax A = "a"; +syntax B = "b" "b"; +//syntax C = "c"; + +private Tree parseS(str input, bool visualize=false) + = parser(#S, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); + +test bool nestedOk() { + return !hasErrors(parseS("a b b c", visualize=true)); +} + +test bool nestedTypo() { + Tree t = parseS("a b x c", visualize=true); + iprintln(t); + println("Error text: \'\'"); + return getErrorText(findFirstError(t)) == "x "; +} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc new file mode 100644 index 00000000000..c4683085179 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc @@ -0,0 +1,122 @@ +module lang::rascal::tests::recovery::PicoRecoveryTests + +import lang::pico::\syntax::Main; + +import ParseTree; +import IO; + +private Tree parsePico(str input, bool visualize=false) + = parser(#Program, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); + +test bool picoOk() { + t = parsePico("begin declare input : natural, + output : natural, + repnr : natural, + rep : natural; + input := 14; + output := 0; + while input - 1 do + rep := output; + repnr := input; + while repnr - 1 do + output := output + rep; + repnr := repnr - 1 + od; + input := input - 1 + od +end"); + return !hasErrors(t); +} + +test bool picoTypo() { + t = parsePico("begin declare input : natural, + output : natural, + repnr : natural, + rep : natural; + input := 14; + output := 0; + while input - 1 do + rep := output; + repnr := input; + while repnr - 1 do + output := output x rep; + repnr := repnr - 1 + od; + input := input - 1 + od +end"); + iprintln(findFirstError(t)); + return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "x rep; + repnr := repnr - 1 + od"; +} + +test bool picoMissingSemi() { + t = parsePico("begin declare input : natural, + output : natural, + repnr : natural, + rep : natural; + input := 14; + output := 0; + while input - 1 do + rep := output; + repnr := input; + while repnr - 1 do + output := output + rep; + repnr := repnr - 1 + od + input := input - 1 + od +end"); + str errorText = getErrorText(findFirstError(t)); + println("error count: "); + println("error text: "); + + for (error <- findAllErrors(t)) { + println(" error: "); + } + + return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 + od"; +} + +test bool picoMissingTypoMinimal() { + t = parsePico( +"begin declare; + while input do + input x= 14; + output := 0 + od +end", visualize=true); + + for (error <- findAllErrors(t)) { + println(" error: "); + } + +return !hasErrors(t); + /*str errorText = getErrorText(findFirstError(t)); + println("error text: "); + return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 + od"; + */ +} +test bool picoMissingSemiMinimal() { + t = parsePico( +"begin declare; + while input do + input := 14 + output := 0 + od +end", visualize=true); + + for (error <- findAllErrors(t)) { + println(" error: "); + } + +return !hasErrors(t); + /*str errorText = getErrorText(findFirstError(t)); + println("error text: "); + return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 + od"; + */ +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc new file mode 100644 index 00000000000..3a5e589a2dd --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc @@ -0,0 +1,9 @@ +module lang::rascal::tests::recovery::RunRecoveryTests + +import lang::rascal::tests::recovery::BasicRecoveryTests; +import util::REPL; + +int main(list[str] args){ + Terminal terminal = newREPL(repl()); + return 0; +} diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 2bf7774ed65..c8e37cf7bf4 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -154,8 +154,6 @@ public SGTDBF(){ unmatchableLeafNodes = new Stack>(); unmatchableMidProductionNodes = new DoubleStack, AbstractNode>, AbstractStackNode

>(); filteredNodes = new DoubleStack, AbstractNode>(); - - visualizer = new DebugVisualizer("Parser"); } /** @@ -835,8 +833,8 @@ private boolean findFirstStacksToReduce(){ if (recoverer != null) { if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); - visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); } + visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); @@ -866,7 +864,7 @@ private boolean findStacksToReduce(){ DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; if(!(terminalsTodo == null || terminalsTodo.isEmpty())){ - if (debugListener != null) { + if (DebugVisualizer.VISUALIZATION_ENABLED) { NodeId reduceNodeId = new NodeId("todo-" + i); visualize("Found stack to reduce", reduceNodeId); } @@ -881,18 +879,21 @@ private boolean findStacksToReduce(){ if (recoverer != null && location < input.length) { if (debugListener != null) { - visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); } + visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); + } + if (DebugVisualizer.VISUALIZATION_ENABLED && visualizer != null) { // Visualize state and include recovered nodes visualizer.createGraph(this, "Reviving"); visualizer.addRecoveredNodes(recoveredNodes); visualizer.writeGraph(); opportunityToBreak(); } + if (recoveredNodes.size() > 0) { // was: for (int i = recoveredNodes.size()-1; i>= 0; i--) { for (int i = 0; i < recoveredNodes.size(); i++) { @@ -902,8 +903,8 @@ private boolean findStacksToReduce(){ if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); - visualize("Queue recovery node", DebugVisualizer.getNodeId(recovered)); } + visualize("Queue recovery node", DebugVisualizer.getNodeId(recovered)); queueRecoveryNode(recovered, recovered.getStartLocation(), recovered.getLength(), recoveredNodes.getSecond(i)); } return findStacksToReduce(); @@ -1261,6 +1262,9 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] this.recoverer = recoverer; this.debugListener = debugListener; + + String query = inputURI.getQuery(); + visualizer = query != null && query.contains("visualize=true") ? new DebugVisualizer("Parser") : null; // Initialzed the position store. positionStore.index(input); @@ -1466,7 +1470,7 @@ protected T buildResult(AbstractNode result, INodeFlattener converter, INo private void visualize(String step, NodeId highlight) { // Only visualize when debugging - if (debugListener != null) { + if (DebugVisualizer.VISUALIZATION_ENABLED && visualizer != null) { visualizer.createGraph(this, step); if (highlight != null) { visualizer.highlight(highlight); @@ -1477,7 +1481,7 @@ private void visualize(String step, NodeId highlight) { } } - private void opportunityToBreak() { + public static void opportunityToBreak() { } diff --git a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java index 9bf625a33fb..169a5e2fd66 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java @@ -117,7 +117,20 @@ public int getLength(){ public AbstractNode getResult(){ return result; } + + public int[][] getLiteral() { + return ciLiteral; + } + @Override + public String toShortString() { + int[] codePoints = new int[ciLiteral.length]; + for (int i=0; i extends AbstractMatchableStackNode

{ private final int[][] ranges; @@ -86,6 +87,33 @@ public AbstractNode getResult(){ return result; } + @Override + public String toShortString() { + StringBuilder sb = new StringBuilder(); + for (int i=0; i 0) { + sb.append(','); + } + + int[] range = ranges[i]; + sb.append(codePointToString(range[0])); + if (range[0] != range[1]) { + sb.append('-'); + sb.append(codePointToString(range[1])); + } + } + + return sb.toString(); + } + + private String codePointToString(int codePoint) { + if (Character.isLetterOrDigit(codePoint)) { + return new String(Character.toChars(codePoint)); + } + + return String.valueOf(codePoint); + } + public String toString(){ StringBuilder sb = new StringBuilder(); @@ -94,7 +122,7 @@ public String toString(){ sb.append(range[0]); sb.append('-'); sb.append(range[1]); - for(int i = ranges.length - 2; i >= 0; --i){ + for(int i = 1; i original, int startLocation){ super(original, startLocation); diff --git a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java index 38adc9cf619..15418e849c5 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java @@ -31,8 +31,11 @@ public SeparatedListStackNode(int id, int dot, P production, AbstractStackNode

child, AbstractStackNode

[] separators, boolean isPlusList, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index ae0bcf41be0..e6edd477d0e 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -55,8 +55,8 @@ public static SkippedNode createResultUntilToken(String token, int[] input, int return null; } - public static SkippedNode createResultUntilChar(int[] input, int startLocation, int length, IConstructor production, int dot) { - return new SkippedNode(production, dot, createSkippedToken(input, startLocation, length), startLocation); + public static SkippedNode createResultUntilChar(int[] input, int startLocation, int endLocation, IConstructor production, int dot) { + return new SkippedNode(production, dot, createSkippedToken(input, startLocation, endLocation - startLocation), startLocation); } private static int[] createSkippedToken(int[] input, int startLocation, int length) { diff --git a/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java new file mode 100644 index 00000000000..587bd419748 --- /dev/null +++ b/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java @@ -0,0 +1,66 @@ +package org.rascalmpl.parser.uptr.recovery; + +public class CaseInsensitiveLiteralMatcher implements InputMatcher { + private final int[] chars; + private final int[] altChars; + + public CaseInsensitiveLiteralMatcher(String literal) { + int length = literal.length(); + chars = new int[length]; + altChars = new int[length]; + for(int i = 0; i InputMatcher createMatcher(AbstractStackNode

stackNode) { + if (stackNode instanceof LiteralStackNode) { + return new LiteralMatcher(((LiteralStackNode

) stackNode).getLiteral()); + } + + if (stackNode instanceof CaseInsensitiveLiteralStackNode) { + return new CaseInsensitiveLiteralMatcher(((CaseInsensitiveLiteralStackNode

) stackNode).getLiteral()); + } + + return null; + } +} diff --git a/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java new file mode 100644 index 00000000000..b5531cff372 --- /dev/null +++ b/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java @@ -0,0 +1,41 @@ +package org.rascalmpl.parser.uptr.recovery; + +public class LiteralMatcher implements InputMatcher { + private int[] chars; + + public LiteralMatcher(String literal) { + chars = new int[literal.length()]; + for (int i=0; i { - private static final int[] WHITESPACE = {' ', '\r', '\n', '\t' }; - private IdDispenser stackNodeIdDispenser; public ToTokenRecoverer(IdDispenser stackNodeIdDispenser) { @@ -71,6 +78,9 @@ private DoubleArrayList, AbstractNode> reviveNod recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); + DebugVisualizer visualizer = new DebugVisualizer("Recovery"); + visualizer.visualizeRecoveryNodes(recoveryNodes); + for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); @@ -114,42 +124,130 @@ List> findSkippingNodes(int[] input, AbstractSta } // Try to find whitespace to skip to + /* result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, dot); if (result != null) { nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); - } + }*/ // Do something more clever: find the last token of this production and skip until after that - String lastLiteral = findLastLiteral(prod); - if (lastLiteral != null) { - result = SkippingStackNode.createResultUntilToken(lastLiteral, input, startLocation, prod, dot); - if (result != null) { + List endMatchers = findEndMatchers(prod); + for (InputMatcher endMatcher : endMatchers) { + MatchResult endMatch = endMatcher.findMatch(input, startLocation); + if (endMatch != null) { + result = SkippingStackNode.createResultUntilChar(input, startLocation, endMatch.getEnd(), prod, dot); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); } } - // TODO: or find the first token of the next production and skip until before that + // Find the first token of the next production and skip until before that + List nextMatchers = findNextMatchers(recoveryNode); + for (InputMatcher nextMatcher : nextMatchers) { + MatchResult nextMatch = nextMatcher.findMatch(input, startLocation); + if (nextMatch != null) { + result = SkippingStackNode.createResultUntilChar(input, startLocation, nextMatch.getStart(), prod, dot); + nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); + } + } return nodes; } - String findLastLiteral(IConstructor prod) { + // Gather matchers for the last token of a production + List findEndMatchers(IConstructor prod) { IList args = (IList) prod.get(1); if (args.isEmpty()) { - return null; + return Collections.emptyList(); } IValue last = args.get(args.length()-1); if (last instanceof IConstructor) { - IConstructor cons = (IConstructor) last; - if (cons.getConstructorType() == RascalValueFactory.Symbol_Lit) { - return ((IString) cons.get(0)).getValue(); + return Collections.singletonList(InputMatcher.createMatcher((IConstructor) last)); + } + + return Collections.emptyList(); + } + + void forAllParents(AbstractStackNode stackNode, Consumer> consumer) { + IntegerObjectList> edges = stackNode.getEdges(); + if (edges != null) { + for (int i = edges.size() - 1; i >= 0; --i) { + EdgesSet edgesList = edges.getValue(i); + if (edgesList != null) { + for (int j = edgesList.size() - 1; j >= 0; --j) { + consumer.accept(edgesList.get(j)); + } + } + } + } + } + + AbstractStackNode getSingleParentStack(AbstractStackNode stackNode) { + if (stackNode == null) { + return null; + } + + IntegerObjectList> edges = stackNode.getEdges(); + if (edges != null) { + EdgesSet edgesList = edges.getValue(0); + if (edgesList != null) { + return edgesList.get(0); } } return null; } + // Find matchers for the first token after the current stack node + List findNextMatchers(AbstractStackNode stackNode) { + DebugVisualizer visualizer = new DebugVisualizer("findNextMatcher"); + visualizer.visualize(stackNode); + + final List matchers = new java.util.ArrayList<>(); + + AbstractStackNode parent = getSingleParentStack(stackNode); + if (parent == null) { + return matchers; + } + + AbstractStackNode[] prod = parent.getProduction(); + if (prod == null) { + return matchers; + } + + int nextDot = parent.getDot() + 1; + if (nextDot >= prod.length) { + return matchers; + } + + AbstractStackNode next = prod[nextDot]; + if (next instanceof NonTerminalStackNode && next.getName().startsWith("layouts_")) { + // Look "through" layout for now, should be more general to look through any node that can be empty + nextDot++; + if (nextDot >= prod.length) { + return matchers; + } + next = prod[nextDot]; + } + + if (next instanceof LiteralStackNode) { + LiteralStackNode nextLiteral = (LiteralStackNode) next; + matchers.add(new LiteralMatcher(nextLiteral.getLiteral())); + } + + if (next instanceof CaseInsensitiveLiteralStackNode) { + CaseInsensitiveLiteralStackNode nextLiteral = (CaseInsensitiveLiteralStackNode) next; + matchers.add(new CaseInsensitiveLiteralMatcher(nextLiteral.getLiteral())); + } + + if (next instanceof NonTerminalStackNode) { + NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) next; + SGTDBF.opportunityToBreak(); + } + + return matchers; + } + // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) // As this is experimental code, this method is extremely conservative. // Any sharing will result in returning 'false'. @@ -227,6 +325,7 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr todo.push(failer); + boolean useNext = true; // Use this to only use the last active element from a production while (!todo.isEmpty()) { AbstractStackNode node = todo.pop(); @@ -236,13 +335,14 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr visited.put(node, 0); - if (node == failer || node.getDot() == 0) { + if (useNext) { ArrayList recoveryProductions = new ArrayList(); collectProductions(node, recoveryProductions); if (recoveryProductions.size() > 0) { - recoveryNodes.add(node, recoveryProductions); + addRecoveryNode(node, recoveryProductions, recoveryNodes); } } + useNext = node.getDot() == 0; IntegerObjectList> edges = node.getEdges(); @@ -259,6 +359,28 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr } } } + + // Only add recovery nodes that are not already present. + private void addRecoveryNode(AbstractStackNode node, ArrayList productions, DoubleArrayList, ArrayList> recoveryNodes) { + for (int i=0; i prods = recoveryNodes.getSecond(i); + if (prods.size() == productions.size()) { + boolean equal = true; + for (int j=0; equal && j node, ArrayList productions) { @@ -269,12 +391,10 @@ private void collectProductions(AbstractStackNode node, ArrayList< int dot = node.getDot(); - System.err.println("collect productions for node: " + node); if (node.isEndNode()) { IConstructor parentProduction = node.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)){ productions.add(parentProduction); - System.err.println("adding production: " + parentProduction); if (ProductionAdapter.isList(parentProduction)) { return; // Don't follow productions in lists productions, since they are 'cyclic'. diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index a99a0fa4606..11a3808482f 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -16,7 +16,7 @@ public static String prodToString(IConstructor prod) { builder.append(" ->"); - if (prod.getName() == "prod") { + if (prod.getName().equals("prod")) { IList children = (IList) prod.get(1); for (IValue child : children) { builder.append(" "); @@ -35,7 +35,7 @@ public static String prodToString(IConstructor prod) { private static String stripQuotes(String s) { if (s.charAt(0) == '"' && s.charAt(s.length()-1) == '"') { - return s.substring(1, s.length()-1); + return s.substring(1, s.length()-1).replace("\\", ""); } return s; diff --git a/src/org/rascalmpl/util/visualize/DebugVisualizer.java b/src/org/rascalmpl/util/visualize/DebugVisualizer.java index b0fb64f41ca..c944242d5a3 100644 --- a/src/org/rascalmpl/util/visualize/DebugVisualizer.java +++ b/src/org/rascalmpl/util/visualize/DebugVisualizer.java @@ -12,7 +12,9 @@ import java.nio.file.StandardCopyOption; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -30,10 +32,10 @@ import org.rascalmpl.parser.gtd.result.RecoveredNode; import org.rascalmpl.parser.gtd.result.SkippedNode; import org.rascalmpl.parser.gtd.result.SortContainerNode; -import org.rascalmpl.parser.gtd.result.struct.Link; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; +import org.rascalmpl.parser.gtd.util.ArrayList; import org.rascalmpl.parser.gtd.util.DoubleArrayList; import org.rascalmpl.parser.gtd.util.DoubleStack; import org.rascalmpl.parser.gtd.util.IntegerObjectList; @@ -53,7 +55,8 @@ public class DebugVisualizer { static final String BASE_DIR = "D:/debug/parser-traces/docs/"; - private static final boolean VISUALIZATION_ENABLED = true; + public static final boolean VISUALIZATION_ENABLED = true; + private static final boolean INCLUDE_PRODUCTIONS = false; public static final NodeId PARSER_ID = new NodeId("Parser"); public static final NodeId TODO_LISTS_ID= new NodeId("todoLists"); @@ -119,6 +122,14 @@ public void visualize(AbstractStackNode node) { } } + public void visualizeRecoveryNodes(DoubleArrayList, ArrayList> recoveryNodes) { + writeGraph(createGraph(recoveryNodes)); + } + + public void visualizeProductionTrees(AbstractStackNode[] nodes) { + writeGraph(createProductionGraph(nodes)); + } + public int getFrame() { return frame; } @@ -149,6 +160,76 @@ private synchronized DotGraph createGraph(AbstractStackNode stackN return graph; } + private DotGraph createGraph(DoubleArrayList, ArrayList> recoveryNodes) { + reset(); + graph = new DotGraph(name, true); + final NodeId RECOVERY_NODES_ID = new NodeId("recovery-nodes"); + + DotNode arrayNode = DotNode.createArrayNode(RECOVERY_NODES_ID, recoveryNodes.size()); + graph.addNode(arrayNode); + + for (int i=0; i[] stackNodes) { + reset(); + graph = new DotGraph(name, true); + for (AbstractStackNode stackNode : stackNodes) { + addProductionNodes(graph, stackNode); + } + return graph; + } + + private

NodeId addProductionNodes(DotGraph graph, AbstractStackNode

stackNode) { + DotNode node = createDotNode(stackNode); + graph.addNode(node); + + AbstractStackNode

[] prods = stackNode.getProduction(); + if (prods != null) { + NodeId prodArrayId = new NodeId(node.getId() + "-prod"); + graph.addArrayNode(prodArrayId, prods.length); + for (int i=0; i child = prods[i]; + DotNode childNode = createDotNode(child); + graph.addNode(childNode); + graph.addEdge(new NodeId(prodArrayId, String.valueOf(i)), childNode.getId()); + } + + graph.addEdge(node.getId(), prodArrayId, "Production"); + } + + return node.getId(); + } + + private void addProductionArray(DotGraph graph, NodeId nodeId, ArrayList productions) { + DotNode arrayNode = DotNode.createArrayNode(nodeId, productions.size()); + graph.addNode(arrayNode); + for (int i=0; i void addRecoveredNodes(DoubleArrayList, AbstractNode> recoveredNodes) { addStackAndNodeDoubleList(graph, RECOVERED_NODES_ID, recoveredNodes); graph.addEdge(ERROR_TRACKING_ID, RECOVERED_NODES_ID, "Nodes to revive"); @@ -167,6 +248,10 @@ private

DotNode addStack(DotGraph graph, AbstractStackNode

stackNode) { graph.addNode(node); + if (INCLUDE_PRODUCTIONS) { + addProductionNodes(graph, stackNode); + } + IntegerObjectList> edges = stackNode.getEdges(); if (edges != null) { for (int i = edges.size() - 1; i >= 0; --i) { @@ -207,9 +292,28 @@ private

DotNode createDotNode(AbstractStackNode

stackNode) { } } + int dot = stackNode.getDot(); + + String extraInfo = ""; + if (stackNode.isMatchable()) { + extraInfo += ",matchable"; + } + if (stackNode.isSeparator()) { + extraInfo += ",sep"; + } + if (stackNode.isExpandable()) { + extraInfo += ",expandable"; + } + if (stackNode.isLayout()) { + extraInfo += ",layout"; + } + if (stackNode.isEndNode()) { + extraInfo += ",end"; + } + DotNode node = new DotNode(getNodeId(stackNode)); - String label = String.format("%s: %s\n.%d@%d", - type, nodeName, stackNode.getDot(), stackNode.getStartLocation()); + String label = String.format("%s: %s\n.%d@%d %s", + type, nodeName, dot, stackNode.getStartLocation(), extraInfo); String shortString = stackNode.toShortString(); if (shortString != null) { diff --git a/src/org/rascalmpl/util/visualize/dot/DotGraph.java b/src/org/rascalmpl/util/visualize/dot/DotGraph.java index 27293e2f1e5..dc11db9202d 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotGraph.java +++ b/src/org/rascalmpl/util/visualize/dot/DotGraph.java @@ -32,8 +32,10 @@ private void addStatement(DotStatement statement) { } public void addNode(DotNode node) { - addStatement(node); - nodes.put(node.getId(), node); + if (!nodes.containsKey(node.getId())) { + addStatement(node); + nodes.put(node.getId(), node); + } } public void addNode(String id, String label) { diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index ca7d5ed073a..bac01801576 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -582,7 +582,7 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in IDebugListener debugListener = null; if (allowRecovery) { recoverer = new ToTokenRecoverer(new StackNodeIdDispenser(parserInstance)); - debugListener = new DebugLogger(new PrintWriter(System.out, true)); + //debugListener = new DebugLogger(new PrintWriter(System.out, true)); } return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer, debugListener); } From 8ef03b400b659c32ca727ee73e3ca5f18f1f26e7 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 20 Aug 2024 08:55:32 +0200 Subject: [PATCH 043/190] Minor cleanup of parser tests and debug support --- .../rascal/tests/recovery/PicoRecoveryTests.rsc | 4 ++-- src/org/rascalmpl/parser/gtd/SGTDBF.java | 14 +++++--------- src/org/rascalmpl/parser/util/DebugUtil.java | 7 +++++++ 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc index c4683085179..7c2b4aad77e 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc @@ -93,7 +93,7 @@ end", visualize=true); println(" error: "); } -return !hasErrors(t); +return hasErrors(t); /*str errorText = getErrorText(findFirstError(t)); println("error text: "); return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 @@ -113,7 +113,7 @@ end", visualize=true); println(" error: "); } -return !hasErrors(t); +return hasErrors(t); /*str errorText = getErrorText(findFirstError(t)); println("error text: "); return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index c8e37cf7bf4..a3dd982bbc5 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -46,6 +46,7 @@ import org.rascalmpl.parser.gtd.util.IntegerList; import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.parser.util.DebugUtil; import org.rascalmpl.util.visualize.DebugVisualizer; import org.rascalmpl.util.visualize.dot.NodeId; @@ -763,9 +764,7 @@ private void moveToNext(AbstractStackNode

node, AbstractNode result){ */ private void move(AbstractStackNode

node, AbstractNode result){ if(debugListener != null) debugListener.moving(node, result); - if (node instanceof RecoveryPointStackNode) { - opportunityToBreak(); - } + // Handle filtering. ICompletionFilter[] completionFilters = node.getCompletionFilters(); if(completionFilters != null){ @@ -891,7 +890,7 @@ private boolean findStacksToReduce(){ visualizer.createGraph(this, "Reviving"); visualizer.addRecoveredNodes(recoveredNodes); visualizer.writeGraph(); - opportunityToBreak(); + DebugUtil.opportunityToBreak(); } if (recoveredNodes.size() > 0) { @@ -1477,14 +1476,11 @@ private void visualize(String step, NodeId highlight) { } visualizer.writeGraph(); - opportunityToBreak(); + DebugUtil.opportunityToBreak(); } } - public static void opportunityToBreak() { - } - - + /** * Getters used for graph generation/debugging */ diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index 11a3808482f..4e821916fea 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -40,4 +40,11 @@ private static String stripQuotes(String s) { return s; } + + /* + * Nop method that allows breakpoints to be set at the call site even if originally there is no code to break on + */ + public static void opportunityToBreak() { + } + } From 7d82f20aed3ae4fe71d3d706071ad42d6421c02c Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 21 Aug 2024 09:08:31 +0200 Subject: [PATCH 044/190] Implemented error tree fixup (including source locations) --- src/org/rascalmpl/library/ParseTree.rsc | 43 +++---- .../tests/recovery/BasicRecoveryTests.rsc | 8 +- src/org/rascalmpl/parser/gtd/SGTDBF.java | 107 +++++++++++++++++- .../parser/gtd/result/SkippedNode.java | 13 ++- .../gtd/result/out/DefaultNodeFlattener.java | 2 +- .../result/out/INodeConstructorFactory.java | 4 +- .../gtd/result/out/RecoveryNodeFlattener.java | 14 ++- .../parser/gtd/stack/SkippingStackNode.java | 20 ++-- .../parser/uptr/UPTRNodeFactory.java | 21 +++- .../recovery/ToNextWhitespaceRecoverer.java | 4 +- .../uptr/recovery/ToTokenRecoverer.java | 44 ++++--- src/org/rascalmpl/parser/util/DebugUtil.java | 6 +- .../values/RascalFunctionValueFactory.java | 6 +- .../rascalmpl/values/RascalValueFactory.java | 3 +- 14 files changed, 211 insertions(+), 84 deletions(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 1f88edb3ef1..efb1b386534 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -190,7 +190,8 @@ data Production | \reference(Symbol def, str cons) // <5> ; -data Production = skipped(Symbol def, Production prod, int dot); +data Production = error(Symbol def, Production prod, int dot) + | skipped(Symbol symbol); @synopsis{Attributes in productions.} @description{ @@ -769,40 +770,32 @@ bool isNonTerminalType(Symbol::\start(Symbol s)) = isNonTerminalType(s); default bool isNonTerminalType(Symbol s) = false; @synopsis{Check if a parse tree contains any skipped nodes, the result of error recovery.} -bool hasErrors(Tree tree) = /appl(skipped(_, _, _), _) := tree; +bool hasErrors(Tree tree) = /appl(error(_, _, _), _) := tree; @synopsis{Find all error productions in a parse tree.} -list[Tree] findAllErrors(Tree tree) { - return for (/error:appl(_, [*_, appl(skipped(_, _, _), _)]) := tree) append(error); -} +list[Tree] findAllErrors(Tree tree) = [err | /err:appl(error(_, _, _), _) := tree]; @synopsis{Find the first production containing an error.} Tree findFirstError(Tree tree) { - if (/error:appl(_, [*_, appl(skipped(_, _, _), _)]) := tree) return error; + if (/err:appl(error(_, _, _), _) := tree) return err; fail; } -private Production getSkipped(appl(_, [*_, appl(skip:skipped(_, _, _), _)])) = skip; +@synopsis{Get the symbol (sort) of the failing production} +Symbol getErrorSymbol(appl(error(Symbol sym, _, _), _)) = sym; -Symbol getErrorSymbol(Tree error) { - if (skipped(Symbol s, _, _) := getSkipped(error)) { - return s; - } - fail; -} +@synopsis{Get the production that failed} +Production getErrorProduction(appl(error(_, Production prod, _), _)) = prod; -Production getErrorProduction(Tree error) { - if (skipped(_, Production p, _) := getSkipped(error)) { - return p; - } - fail; -} +@synopsis{Get the dot (position in the production) of the failing element in a production} +int getErrorDot(appl(error(_, _, int dot), _)) = dot; -int getErrorPosition(Tree error) { - if (skipped(_, _, int dot) := getSkipped(error)) { - return dot; - } - fail; +@synopsis{Get the skipped tree} +Tree getSkipped(appl(error(_, _, _), [*_, skip:appl(skipped(_), _)])) { + return skip; } -str getErrorText(appl(_, [*_, appl(skipped(_, _, _), chars)])) = stringChars([c | ch <- chars, char(c) := ch]); +@synopsis{Get the text that failed to parse} +str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) { + return stringChars([c | ch <- chars, char(c) := ch]); +} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index e84ba513118..8be50b9704a 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -25,15 +25,13 @@ test bool abx() { test bool axc() { Tree t = parseS("a x c $", visualize=true); - iprintln(t); + iprintln(getSkipped(findFirstError(t))@\loc); return getErrorText(findFirstError(t)) == "x c"; } +/* We need to look through non-terminals to make this test work test bool ax() { Tree t = parseS("a x $", visualize=true); return getErrorText(findFirstError(t)) = "x"; } - -int main(list[str] args){ - startRepl(REPL()); -} +*/ diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index a3dd982bbc5..7d5a2659227 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -13,6 +13,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URI; +import java.util.concurrent.atomic.AtomicBoolean; import org.rascalmpl.parser.gtd.debug.IDebugListener; import org.rascalmpl.parser.gtd.exception.ParseError; @@ -34,7 +35,6 @@ import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.EpsilonStackNode; import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; -import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.stack.filter.ICompletionFilter; import org.rascalmpl.parser.gtd.stack.filter.IEnterFilter; @@ -49,6 +49,13 @@ import org.rascalmpl.parser.util.DebugUtil; import org.rascalmpl.util.visualize.DebugVisualizer; import org.rascalmpl.util.visualize.dot.NodeId; +import org.rascalmpl.values.RascalValueFactory; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IList; +import io.usethesource.vallang.ISet; +import io.usethesource.vallang.IValue; +import io.usethesource.vallang.type.Type; /** * This is the core of the parser; it drives the parse process. @@ -1445,8 +1452,11 @@ protected T buildResult(AbstractNode result, INodeFlattener converter, INo finally { actionExecutor.completed(rootEnvironment, (parseResult == null)); } - if(parseResult != null){ - return parseResult; // Success. + if(parseResult != null) { + if (recoverer != null) { + parseResult = fixErrorNodes(parseResult, nodeConstructorFactory); + } + return parseResult; // Success. } int offset = filteringTracker.getOffset(); @@ -1463,6 +1473,97 @@ protected T buildResult(AbstractNode result, INodeFlattener converter, INo } } + /** + * Error nodes end up in a n inconvenient form because of the parser algorithm. + * This post-processing step transforms the original tree into a more useful form. + * In essence, error subtrees look like this after parsing: + * `appl(prod(S,[]), [,,...,appl(skipped(S,prod(S,[]),),[)]])` + * This method pulls up the skipped node, transforming these trees into: + * `appl(skipped(S,prod(S,[]),), [,,...,[])])` + * This means productions that failed to parse can be recognized at the top level. + * Note that this can only be done when we know the actual type of T is IConstructor. + */ + @SuppressWarnings("unchecked") + T fixErrorNodes(T tree, INodeConstructorFactory nodeConstructorFactory) { + if (!(tree instanceof IConstructor)) { + return tree; + } + + return (T) fixErrorNodes((IConstructor) tree, (INodeConstructorFactory) nodeConstructorFactory); + } + + IConstructor fixErrorNodes(IConstructor tree, INodeConstructorFactory nodeConstructorFactory) { + IConstructor result; + Type type = tree.getConstructorType(); + if (type == RascalValueFactory.Tree_Appl) { + IValue prod = tree.get(0); + IList childList = (IList) tree.get(1); + int childCount = childList.length(); + + ArrayList children = new ArrayList<>(childCount); + boolean anyChanges = false; + boolean errorTree = false; + for (int i=0; i alternatives = new ArrayList<>(alternativeSet.size()); + final AtomicBoolean anyChanges = new AtomicBoolean(false); + alternativeSet.forEach(alt -> { + IConstructor newAlt = fixErrorNodes((IConstructor) alt, nodeConstructorFactory); + if (newAlt != alt) { + anyChanges.setPlain(true); + } + alternatives.add(newAlt); + }); + if (anyChanges.getPlain()) { + result = nodeConstructorFactory.createAmbiguityNode(alternatives); + } else { + result = tree; + } + } else if (type == RascalValueFactory.Tree_Cycle) { + result = tree; + } else { + throw new RuntimeException("Unrecognized tree type: " + type); + } + + if (result != tree) { + IValue loc = tree.asWithKeywordParameters().getParameter(RascalValueFactory.Location); + result = result.asWithKeywordParameters().setParameter(RascalValueFactory.Location, loc); + } + + return result; + } + /** * Datastructure visualization for debugging purposes */ diff --git a/src/org/rascalmpl/parser/gtd/result/SkippedNode.java b/src/org/rascalmpl/parser/gtd/result/SkippedNode.java index 48882533b2b..53578520ce2 100644 --- a/src/org/rascalmpl/parser/gtd/result/SkippedNode.java +++ b/src/org/rascalmpl/parser/gtd/result/SkippedNode.java @@ -11,22 +11,27 @@ *******************************************************************************/ package org.rascalmpl.parser.gtd.result; +import java.net.URI; + import org.rascalmpl.unicode.UnicodeConverter; /** * Result tree node that represents a skipped portion of the input sentence. */ public class SkippedNode extends AbstractNode { - public final static int ID = 9; + public static final int ID = 9; + // TODO: only "skippedChars" is actually needed + private final URI input; private final Object production; private final int dot; private final int[] skippedChars; private final int offset; - public SkippedNode(Object production, int dot, int[] skippedChars, int offset) { + public SkippedNode(URI input, Object production, int dot, int[] skippedChars, int offset) { super(); + this.input = input; this.production = production; this.dot = dot; this.skippedChars = skippedChars; @@ -37,6 +42,10 @@ public int getTypeIdentifier(){ return ID; } + public URI getInput() { + return input; + } + public Object getProduction() { return production; } diff --git a/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java b/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java index ad16a892f8a..4f1655d800f 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java +++ b/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java @@ -66,7 +66,7 @@ public T convert(INodeConstructorFactory nodeConstructorFactory, AbstractN case RecoveredNode.ID: return convert(nodeConstructorFactory, ((SortContainerNode) node).getFirstAlternative().getNode(), stack, depth, cycleMark, positionStore, filteringTracker, actionExecutor, environment); case SkippedNode.ID: - return recoveryNodeConverter.convertToUPTR(nodeConstructorFactory, (SkippedNode) node); + return recoveryNodeConverter.convertToUPTR(nodeConstructorFactory, (SkippedNode) node, positionStore); default: throw new RuntimeException("Incorrect result node id: "+node.getTypeIdentifier()); } diff --git a/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java b/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java index 5a94f7c3b64..1de4b441a58 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java +++ b/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java @@ -38,7 +38,9 @@ public interface INodeConstructorFactory { T createListAmbiguityNode(ArrayList alternatives); - T createRecoveryNode(int dot, ArrayList recognizedPrefix, int[] unrecognizedCharacters, Object production); + T createSkippedNode(int[] unrecognizedCharacters); + + T createErrorNode(ArrayList children, Object production); ArrayList getChildren(T node); diff --git a/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java b/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java index b7a2605bfa9..973e39e120c 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java +++ b/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java @@ -11,8 +11,8 @@ *******************************************************************************/ package org.rascalmpl.parser.gtd.result.out; +import org.rascalmpl.parser.gtd.location.PositionStore; import org.rascalmpl.parser.gtd.result.SkippedNode; -import org.rascalmpl.parser.gtd.util.ArrayList; /** * A converter for result nodes that contain skipped characters for error recovery @@ -23,7 +23,15 @@ public RecoveryNodeFlattener(){ super(); } - public T convertToUPTR(INodeConstructorFactory nodeConstructorFactory, SkippedNode node){ - return nodeConstructorFactory.createRecoveryNode(node.getDot(), new ArrayList<>(), node.getSkippedChars(), node.getProduction()); + public T convertToUPTR(INodeConstructorFactory nodeConstructorFactory, SkippedNode node, PositionStore positionStore){ + T result = nodeConstructorFactory.createSkippedNode(node.getSkippedChars()); + + // Add source location + int startOffset = node.getOffset(); + int endOffset = startOffset + node.getLength(); + P sourceLocation = nodeConstructorFactory.createPositionInformation(node.getInput(), startOffset, endOffset, positionStore); + result = nodeConstructorFactory.addPositionInformation(result, sourceLocation); + + return result; } } diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index e6edd477d0e..b4ddf1e8ec3 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -11,6 +11,8 @@ *******************************************************************************/ package org.rascalmpl.parser.gtd.stack; +import java.net.URI; + import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.result.SkippedNode; @@ -19,25 +21,25 @@ public final class SkippingStackNode

extends AbstractMatchableStackNode

{ private final SkippedNode result; - public static SkippedNode createResultUntilCharClass(int[] until, int[] input, int startLocation, IConstructor production, int dot) { + public static SkippedNode createResultUntilCharClass(URI uri, int[] until, int[] input, int startLocation, IConstructor production, int dot) { for (int to = startLocation ; to < input.length; ++to) { for (int i = 0; i < until.length; ++i) { if (input[to] == until[i]) { int length = to - startLocation; - return new SkippedNode(production, dot, createSkippedToken(input, startLocation, length), startLocation); + return new SkippedNode(uri, production, dot, createSkippedToken(input, startLocation, length), startLocation); } } } - return new SkippedNode(production, dot, new int[0], startLocation); + return new SkippedNode(uri, production, dot, new int[0], startLocation); } - public static SkippedNode createResultUntilEndOfInput(int[] input, int startLocation, IConstructor production, int dot) { + public static SkippedNode createResultUntilEndOfInput(URI uri, int[] input, int startLocation, IConstructor production, int dot) { int length = input.length - startLocation; - return new SkippedNode(production, dot, createSkippedToken(input, startLocation, length), startLocation); + return new SkippedNode(uri, production, dot, createSkippedToken(input, startLocation, length), startLocation); } - public static SkippedNode createResultUntilToken(String token, int[] input, int startLocation, IConstructor production, int dot) { + public static SkippedNode createResultUntilToken(URI uri, String token, int[] input, int startLocation, IConstructor production, int dot) { int length = token.length(); for (int start=startLocation; start+length < input.length; start++) { boolean match = true; @@ -47,7 +49,7 @@ public static SkippedNode createResultUntilToken(String token, int[] input, int } if (match) { - return createResultUntilChar(input, startLocation, start+length-startLocation, production, dot); + return createResultUntilChar(uri, input, startLocation, start+length-startLocation, production, dot); } } } @@ -55,8 +57,8 @@ public static SkippedNode createResultUntilToken(String token, int[] input, int return null; } - public static SkippedNode createResultUntilChar(int[] input, int startLocation, int endLocation, IConstructor production, int dot) { - return new SkippedNode(production, dot, createSkippedToken(input, startLocation, endLocation - startLocation), startLocation); + public static SkippedNode createResultUntilChar(URI uri, int[] input, int startLocation, int endLocation, IConstructor production, int dot) { + return new SkippedNode(uri, production, dot, createSkippedToken(input, startLocation, endLocation - startLocation), startLocation); } private static int[] createSkippedToken(int[] input, int startLocation, int length) { diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index b971764ccfd..d3aa2a63a7e 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -1,6 +1,7 @@ package org.rascalmpl.parser.uptr; import java.net.URI; +import java.util.Arrays; import java.util.IdentityHashMap; import java.util.Map; @@ -13,7 +14,6 @@ import io.usethesource.vallang.IListWriter; import io.usethesource.vallang.ISetWriter; import io.usethesource.vallang.ISourceLocation; -import io.usethesource.vallang.IValue; import org.rascalmpl.values.RascalValueFactory; import org.rascalmpl.values.ValueFactoryFactory; @@ -23,6 +23,8 @@ public class UPTRNodeFactory implements INodeConstructorFactory{ private static final RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory(); + private static final IConstructor SKIPPED = VF.constructor(RascalValueFactory.Production_Skipped, VF.constructor(RascalValueFactory.Symbol_IterStar, VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Range, VF.integer(1), VF.integer(Character.MAX_CODE_POINT)))))); + private boolean allowAmb; public UPTRNodeFactory(boolean allowAmbiguity){ @@ -142,7 +144,21 @@ public Object getProductionFromNode(ITree node){ } @Override - public ITree createRecoveryNode(int dot, ArrayList recognizedPrefix, int[] unrecognizedCharacters, Object production) { + public ITree createSkippedNode(int[] characters) { + IList chars = Arrays.stream(characters).mapToObj(VF::character).collect(VF.listWriter()); + return VF.appl(SKIPPED, chars); + } + + public ITree createErrorNode(ArrayList children, Object production) { + IConstructor prod = (IConstructor) production; + IConstructor errorProd = VF.constructor(RascalValueFactory.Production_Error, prod.get(0), prod, VF.integer(children.size()-1)); + return buildAppl(children, errorProd); + } + + /* + @Override + public ITree createRecoveryNode(int[] characters) { + } IListWriter args = VF.listWriter(); for (int i=0; i recognizedPrefix, int[ IConstructor skipped = VF.constructor(RascalValueFactory.Production_Skipped, prod.get(0), (IValue) production, VF.integer(dot)); return VF.appl(skipped, args.done()); } + */ } diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 7ca1a20a46f..7b1e9439af8 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -80,9 +80,9 @@ private DoubleArrayList, AbstractNode> reviveNod SkippedNode result; if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { - result = SkippingStackNode.createResultUntilEndOfInput(input, startLocation, prod, dot); + result = SkippingStackNode.createResultUntilEndOfInput(null, input, startLocation, prod, dot); } else { - result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, dot); + result = SkippingStackNode.createResultUntilCharClass(null, WHITESPACE, input, startLocation, prod, dot); } AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result); diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 2c779efdef1..1815b5d4448 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -12,13 +12,11 @@ *******************************************************************************/ package org.rascalmpl.parser.uptr.recovery; +import java.net.URI; import java.util.Collections; import java.util.List; -import java.util.Set; -import java.util.TreeSet; import java.util.function.Consumer; -import org.rascalmpl.parser.gtd.SGTDBF; import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.result.SkippedNode; @@ -27,7 +25,6 @@ import org.rascalmpl.parser.gtd.stack.LiteralStackNode; import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; -import org.rascalmpl.parser.gtd.stack.SeparatedListStackNode; import org.rascalmpl.parser.gtd.stack.SkippingStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.util.ArrayList; @@ -38,6 +35,7 @@ import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; import org.rascalmpl.parser.uptr.recovery.InputMatcher.MatchResult; +import org.rascalmpl.parser.util.DebugUtil; import org.rascalmpl.util.visualize.DebugVisualizer; import org.rascalmpl.values.parsetrees.ProductionAdapter; @@ -46,9 +44,11 @@ import io.usethesource.vallang.IValue; public class ToTokenRecoverer implements IRecoverer { + private URI uri; private IdDispenser stackNodeIdDispenser; - public ToTokenRecoverer(IdDispenser stackNodeIdDispenser) { + public ToTokenRecoverer(URI uri, IdDispenser stackNodeIdDispenser) { + this.uri = uri; this.stackNodeIdDispenser = stackNodeIdDispenser; } @@ -60,17 +60,17 @@ public DoubleArrayList, AbstractNode> reviveStac DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, DoubleStack, AbstractNode> filteredNodes) { - ArrayList> failedNodes = new ArrayList>(); + ArrayList> failedNodes = new ArrayList<>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); - // TODO: handle unmatchableLeafNodes - //collectFilteredNodes(filteredNodes, failedNodes); + // handle unmatchableLeafNodes + // collectFilteredNodes(filteredNodes, failedNodes); return reviveFailedNodes(input, location, failedNodes); } private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ - DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); + DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList<>(); // original: for (int i = recoveryNodes.size() - 1; i >= 0; --i) { // But this caused problems because recovery nodes with a later position @@ -118,7 +118,7 @@ List> findSkippingNodes(int[] input, AbstractSta // If we are the top-level node, just skip the rest of the input if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { - result = SkippingStackNode.createResultUntilEndOfInput(input, startLocation, prod, dot); + result = SkippingStackNode.createResultUntilEndOfInput(uri, input, startLocation, prod, dot); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); return nodes; // No other nodes would be useful } @@ -135,7 +135,7 @@ List> findSkippingNodes(int[] input, AbstractSta for (InputMatcher endMatcher : endMatchers) { MatchResult endMatch = endMatcher.findMatch(input, startLocation); if (endMatch != null) { - result = SkippingStackNode.createResultUntilChar(input, startLocation, endMatch.getEnd(), prod, dot); + result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, endMatch.getEnd(), prod, dot); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); } } @@ -145,7 +145,7 @@ List> findSkippingNodes(int[] input, AbstractSta for (InputMatcher nextMatcher : nextMatchers) { MatchResult nextMatch = nextMatcher.findMatch(input, startLocation); if (nextMatch != null) { - result = SkippingStackNode.createResultUntilChar(input, startLocation, nextMatch.getStart(), prod, dot); + result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, nextMatch.getStart(), prod, dot); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); } } @@ -241,8 +241,8 @@ List findNextMatchers(AbstractStackNode stackNode) { } if (next instanceof NonTerminalStackNode) { - NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) next; - SGTDBF.opportunityToBreak(); + //NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) next; + DebugUtil.opportunityToBreak(); } return matchers; @@ -278,7 +278,7 @@ private AbstractStackNode getSinglePredecessor(AbstractStackNode, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { - DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList, ArrayList>(); + DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList<>(); for (int i = failedNodes.size() - 1; i >= 0; --i) { findRecoveryNodes(failedNodes.get(i), recoveryNodes); @@ -320,8 +320,8 @@ private static void collectUnmatchableMidProductionNodes(int location, DoubleSta * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. */ private void findRecoveryNodes(AbstractStackNode failer, DoubleArrayList, ArrayList> recoveryNodes) { - ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap>(); - Stack> todo = new Stack>(); + ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap<>(); + Stack> todo = new Stack<>(); todo.push(failer); @@ -336,7 +336,7 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr visited.put(node, 0); if (useNext) { - ArrayList recoveryProductions = new ArrayList(); + ArrayList recoveryProductions = new ArrayList<>(); collectProductions(node, recoveryProductions); if (recoveryProductions.size() > 0) { addRecoveryNode(node, recoveryProductions, recoveryNodes); @@ -389,8 +389,6 @@ private void collectProductions(AbstractStackNode node, ArrayList< return; // The root node does not have a production, so ignore it. } - int dot = node.getDot(); - if (node.isEndNode()) { IConstructor parentProduction = node.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)){ @@ -402,13 +400,13 @@ private void collectProductions(AbstractStackNode node, ArrayList< } } + int dot = node.getDot(); for (int i = dot + 1; i < production.length; ++i) { AbstractStackNode currentNode = production[i]; if (currentNode.isEndNode()) { IConstructor parentProduction = currentNode.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)) { productions.add(parentProduction); - System.err.println("adding production at " + i + ": " + parentProduction); } } @@ -419,7 +417,5 @@ private void collectProductions(AbstractStackNode node, ArrayList< } } } - } - - + } } diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index 4e821916fea..e1c29c6b084 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -41,10 +41,8 @@ private static String stripQuotes(String s) { return s; } - /* - * Nop method that allows breakpoints to be set at the call site even if originally there is no code to break on - */ public static void opportunityToBreak() { + // Nop method that allows breakpoints to be set at the call site even if originally there is no code to break on } - + } diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index bac01801576..b9fa5e44989 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -16,6 +16,7 @@ import java.io.PrintWriter; import java.io.Reader; import java.lang.reflect.InvocationTargetException; +import java.net.URI; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -580,11 +581,12 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in IGTD parserInstance = getParser(); IRecoverer recoverer = null; IDebugListener debugListener = null; + URI uri = location.getURI(); if (allowRecovery) { - recoverer = new ToTokenRecoverer(new StackNodeIdDispenser(parserInstance)); + recoverer = new ToTokenRecoverer(uri, new StackNodeIdDispenser(parserInstance)); //debugListener = new DebugLogger(new PrintWriter(System.out, true)); } - return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer, debugListener); + return (ITree) parserInstance.parse(methodName, uri, input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer, debugListener); } } diff --git a/src/org/rascalmpl/values/RascalValueFactory.java b/src/org/rascalmpl/values/RascalValueFactory.java index ad3b5f4d668..3c4a7218b5d 100644 --- a/src/org/rascalmpl/values/RascalValueFactory.java +++ b/src/org/rascalmpl/values/RascalValueFactory.java @@ -152,7 +152,8 @@ public class RascalValueFactory extends AbstractValueFactoryAdapter implements I public static final Type Production_Priority = tf.constructor(uptr, Production, "priority", Symbol, "def", tf.listType(Production), "choices"); public static final Type Production_Composition = tf.constructor(uptr, Production, "composition", Production, "lhs", Production, "rhs"); public static final Type Production_Associativity = tf.constructor(uptr, Production, "associativity", Symbol, "def", Associativity, "assoc", tf.setType(Production), "alternatives"); - public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped", Symbol, "def", Production, "prod", tf.integerType(), "dot"); + public static final Type Production_Error = tf.constructor(uptr, Production, "error", Symbol, "def", Production, "prod", tf.integerType(), "dot"); + public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped", Symbol, "def"); /* Constructors for Attr */ public static final Type Attr_Assoc = tf.constructor(uptr, Attr, "assoc", Associativity, "assoc"); From 706662bb2af4bee91e81f67dfccc01bb15392a92 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 21 Aug 2024 09:19:42 +0200 Subject: [PATCH 045/190] Renamed RecoveryNodeFlattener to SkippedNodeFlattener --- .../lang/rascal/tests/recovery/BasicRecoveryTests.rsc | 2 +- .../parser/gtd/result/out/DefaultNodeFlattener.java | 10 +++++----- ...eryNodeFlattener.java => SkippedNodeFlattener.java} | 5 ++--- 3 files changed, 8 insertions(+), 9 deletions(-) rename src/org/rascalmpl/parser/gtd/result/out/{RecoveryNodeFlattener.java => SkippedNodeFlattener.java} (94%) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index 8be50b9704a..bfc309359ce 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -25,7 +25,7 @@ test bool abx() { test bool axc() { Tree t = parseS("a x c $", visualize=true); - iprintln(getSkipped(findFirstError(t))@\loc); + iprintln(getSkipped(findFirstError(t)).src); return getErrorText(findFirstError(t)) == "x c"; } diff --git a/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java b/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java index 4f1655d800f..12ca785fb41 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java +++ b/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java @@ -25,13 +25,12 @@ /** * Converter for parse trees that produces trees in UPTR format. */ -@SuppressWarnings("unchecked") public class DefaultNodeFlattener implements INodeFlattener{ private final CharNodeFlattener charNodeConverter; private final LiteralNodeFlattener literalNodeConverter; private final SortContainerNodeFlattener sortContainerNodeConverter; private final ListContainerNodeFlattener listContainerNodeConverter; - private final RecoveryNodeFlattener recoveryNodeConverter; + private final SkippedNodeFlattener skippedNodeConverter; public DefaultNodeFlattener(){ super(); @@ -40,7 +39,7 @@ public DefaultNodeFlattener(){ literalNodeConverter = new LiteralNodeFlattener(); sortContainerNodeConverter = new SortContainerNodeFlattener(); listContainerNodeConverter = new ListContainerNodeFlattener(); - recoveryNodeConverter = new RecoveryNodeFlattener(); + skippedNodeConverter = new SkippedNodeFlattener(); } /** @@ -53,6 +52,7 @@ protected static class IsInError{ /** * Convert the given node. */ + @SuppressWarnings("unchecked") public T convert(INodeConstructorFactory nodeConstructorFactory, AbstractNode node, IndexedStack stack, int depth, CycleMark cycleMark, PositionStore positionStore, FilteringTracker filteringTracker, IActionExecutor actionExecutor, Object environment){ switch(node.getTypeIdentifier()){ case CharNode.ID: @@ -66,7 +66,7 @@ public T convert(INodeConstructorFactory nodeConstructorFactory, AbstractN case RecoveredNode.ID: return convert(nodeConstructorFactory, ((SortContainerNode) node).getFirstAlternative().getNode(), stack, depth, cycleMark, positionStore, filteringTracker, actionExecutor, environment); case SkippedNode.ID: - return recoveryNodeConverter.convertToUPTR(nodeConstructorFactory, (SkippedNode) node, positionStore); + return skippedNodeConverter.convertToUPTR(nodeConstructorFactory, (SkippedNode) node, positionStore); default: throw new RuntimeException("Incorrect result node id: "+node.getTypeIdentifier()); } @@ -76,6 +76,6 @@ public T convert(INodeConstructorFactory nodeConstructorFactory, AbstractN * Converts the given parse tree to a tree in UPTR format. */ public T convert(INodeConstructorFactory nodeConstructorFactory, AbstractNode parseTree, PositionStore positionStore, FilteringTracker filteringTracker, IActionExecutor actionExecutor, Object rootEnvironment){ - return convert(nodeConstructorFactory, parseTree, new IndexedStack(), 0, new CycleMark(), positionStore, filteringTracker, actionExecutor, rootEnvironment); + return convert(nodeConstructorFactory, parseTree, new IndexedStack<>(), 0, new CycleMark(), positionStore, filteringTracker, actionExecutor, rootEnvironment); } } diff --git a/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java b/src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java similarity index 94% rename from src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java rename to src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java index 973e39e120c..d0b5877e7c8 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java +++ b/src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java @@ -17,9 +17,8 @@ /** * A converter for result nodes that contain skipped characters for error recovery */ -public class RecoveryNodeFlattener{ - - public RecoveryNodeFlattener(){ +public class SkippedNodeFlattener{ + public SkippedNodeFlattener(){ super(); } From 7d6812965d2761e9626285003603f8ca95b9a71f Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 22 Aug 2024 10:36:00 +0200 Subject: [PATCH 046/190] Implemented basic error disambiguation --- src/org/rascalmpl/library/ParseTree.rsc | 63 +++++++++++++++++++ .../tests/recovery/BasicRecoveryTests.rsc | 4 +- .../tests/recovery/PicoRecoveryTests.rsc | 14 ++++- 3 files changed, 77 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index efb1b386534..74ab80f4a7c 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -142,8 +142,11 @@ module ParseTree extend Type; extend Message; extend List; +extend Set; import String; +import IO; +import Node; @synopsis{The Tree data type as produced by the parser.} @description{ @@ -178,6 +181,8 @@ construct ordered and un-ordered compositions, and associativity groups. <4> `assoc` means all alternatives are acceptable, but nested on the declared side; <5> `reference` means a reference to another production rule which should be substituted there, for extending priority chains and such. +<6> `error` means a node produced by error recovery. +<7> `skipped` means characters skipped during error recovery, always the last child of an `appl` with a `error` production. } data Production = prod(Symbol def, list[Symbol] symbols, set[Attr] attributes) // <1> @@ -799,3 +804,61 @@ Tree getSkipped(appl(error(_, _, _), [*_, skip:appl(skipped(_), _)])) { str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) { return stringChars([c | ch <- chars, char(c) := ch]); } + +@synopsis{Error recovery often produces ambiguous trees where errors can be recovered in multiple ways. +This filter removes error trees until no ambiguities caused by error recovery are left. +Note that regular ambiguous trees remain in the parse tree. +} +Tree defaultErrorDisambiguationFilter(t: appl(Production prod, args)) { + Tree result = appl(prod, [defaultErrorDisambiguationFilter(arg) | arg <- args]); + return setKeywordParameters(result, getKeywordParameters(t)); +} + +Tree defaultErrorDisambiguationFilter(amb(set[Tree] alternatives)) { + // Go depth-first + set[Tree] disambiguatedAlts = { defaultErrorDisambiguationFilter(alt) | Tree alt <- alternatives }; + + set[Tree] errorTrees = { alt | Tree alt <- disambiguatedAlts, appl(error(_,_,_), _) := alt }; + set[Tree] nonErrorTrees = { alt | Tree alt <- disambiguatedAlts, appl(error(_,_,_), _) !:= alt }; + + if (nonErrorTrees == {}) { + return getBestErrorTree(errorTrees); + } else if ({Tree single} := nonErrorTrees) { + // One ambiguity left, no ambiguity concerns here + return single; + } + + // Multiple non-error trees left, return an ambiguity node with just the non-error trees + return amb(nonErrorTrees); +} + +private Tree getBestErrorTree(set[Tree] trees) { + Tree best = char(0); + int bestErrorCount = -1; + int bestErrorLength = 0; + + for (tree <- trees) { + list[Tree] errors = findAllErrors(tree); + int errorCount = size(errors); + int errorLength = 0; + for (err <- errors) { + errorLength += getSkipped(err).src.length; + } + + if (bestErrorCount == -1 || errorCount < bestErrorCount || (errorCount == bestErrorCount && errorLength < bestErrorLength)) { + best = tree; + bestErrorCount = errorCount; + bestErrorLength = errorLength; + } + } + + if (bestErrorCount != -1) { + return best; + } + + // trees must have been empty + fail; +} + +// Handle char and cycle nodes +default Tree defaultErrorDisambiguationFilter(Tree t) = t; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index bfc309359ce..f2e1e129787 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -25,7 +25,9 @@ test bool abx() { test bool axc() { Tree t = parseS("a x c $", visualize=true); - iprintln(getSkipped(findFirstError(t)).src); + iprintln(t); + println("after disambiguation:"); + iprintln(defaultErrorDisambiguationFilter(t)); return getErrorText(findFirstError(t)) == "x c"; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc index 7c2b4aad77e..5d3236ba653 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc @@ -5,7 +5,7 @@ import lang::pico::\syntax::Main; import ParseTree; import IO; -private Tree parsePico(str input, bool visualize=false) +Tree parsePico(str input, bool visualize=false) = parser(#Program, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); test bool picoOk() { @@ -80,19 +80,27 @@ end"); od"; } -test bool picoMissingTypoMinimal() { +test bool picoTypoMinimal() { t = parsePico( "begin declare; while input do input x= 14; output := 0 od -end", visualize=true); +end", visualize=false); + + iprintln(t); for (error <- findAllErrors(t)) { println(" error: "); } + disambiguated = defaultErrorDisambiguationFilter(t); + println("after disambiguation:"); + for (error <- findAllErrors(disambiguated)) { + println(" error: "); + } + return hasErrors(t); /*str errorText = getErrorText(findFirstError(t)); println("error text: "); From 6b2939c447fbae9bb76b12d92ed5f03db741c081 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 22 Aug 2024 11:31:22 +0200 Subject: [PATCH 047/190] Added support for error and skipped productions in vis::Text --- src/org/rascalmpl/library/vis/Text.rsc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/org/rascalmpl/library/vis/Text.rsc b/src/org/rascalmpl/library/vis/Text.rsc index 566a229c92d..5c8c6846925 100644 --- a/src/org/rascalmpl/library/vis/Text.rsc +++ b/src/org/rascalmpl/library/vis/Text.rsc @@ -41,6 +41,8 @@ str prettyTree(Tree t, bool src=false, bool characters=true, bool \layout=false, str nodeLabel(appl(prod(label(str l, Symbol nt), _, _), _)) = " = : "; str nodeLabel(appl(prod(Symbol nt, as, _), _)) = " = <}>"; + str nodeLabel(appl(error(Symbol nt, Production p, int dot), _)) = "!error dot=: "; + str nodeLabel(appl(skipped(Symbol s), chars)) = "skipped"; str nodeLabel(appl(regular(Symbol nt), _)) = ""; str nodeLabel(char(32)) = "⎵"; str nodeLabel(char(10)) = "\\r"; From ff350caf1c06fcde38de223b9236e4914d77486a Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 26 Aug 2024 13:50:58 +0200 Subject: [PATCH 048/190] Look "through" non-terminals when finding end/next matchers --- src/org/rascalmpl/library/ParseTree.rsc | 4 +- .../tests/recovery/RascalRecoveryTests.rsc | 163 ++++++++++++++++++ .../lang/rascal/tests/recovery/Test.rsc | 1 + .../lang/rascal/tests/recovery/ToyRascal.rsc | 17 ++ .../tests/recovery/ToyRascalRecoveryTests.rsc | 37 ++++ .../rascalmpl/parser/gtd/ExpectsProvider.java | 7 + src/org/rascalmpl/parser/gtd/IGTD.java | 2 +- src/org/rascalmpl/parser/gtd/SGTDBF.java | 39 +++-- .../gtd/stack/AlternativeStackNode.java | 4 + .../parser/gtd/stack/EmptyStackNode.java | 4 + .../parser/gtd/stack/ListStackNode.java | 4 + .../gtd/stack/MultiCharacterStackNode.java | 4 + .../gtd/stack/NonTerminalStackNode.java | 4 + .../parser/gtd/stack/OptionalStackNode.java | 4 + .../gtd/stack/SeparatedListStackNode.java | 4 + .../parser/gtd/stack/SequenceStackNode.java | 4 + .../parser/gtd/stack/SkippingStackNode.java | 4 + .../uptr/recovery/ToTokenRecoverer.java | 142 ++++++++++----- .../values/RascalFunctionValueFactory.java | 2 +- 19 files changed, 383 insertions(+), 67 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc create mode 100644 src/org/rascalmpl/parser/gtd/ExpectsProvider.java diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 74ab80f4a7c..2e4c22af8a1 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -800,7 +800,9 @@ Tree getSkipped(appl(error(_, _, _), [*_, skip:appl(skipped(_), _)])) { return skip; } -@synopsis{Get the text that failed to parse} +@synopsis{Get the text that failed to parse. This is only the text of the part that has been skipped to be able to continue parsing. +If you want the text of the whole error tree, you can just use string interpolation: "". +} str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) { return stringChars([c | ch <- chars, char(c) := ch]); } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc new file mode 100644 index 00000000000..7c17abd899d --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc @@ -0,0 +1,163 @@ +module lang::rascal::tests::recovery::RascalRecoveryTests + +import lang::rascal::\syntax::Rascal; + +import ParseTree; +import IO; + +Tree parseRascal(str input, bool visualize=false) { + Tree result = parser(#start[Module], allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); + list[Tree] errors = findAllErrors(result); + if (errors != []) { + println("Tree has errors"); + for (error <- errors) { + println("- "); + } + + Tree disambiguated = defaultErrorDisambiguationFilter(result); + println("Best error: "); + } + + return result; +} + +Tree parseFunctionDeclaration(str input, bool visualize=false) { + Tree result = parser(#FunctionDeclaration, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); + list[Tree] errors = findAllErrors(result); + if (errors != []) { + println("Tree has errors"); + for (error <- errors) { + println("- "); + } + + Tree disambiguated = defaultErrorDisambiguationFilter(result); + println("Best error: "); + } + + return result; +} + +test bool rascalOk() { + Tree t = parseRascal(" + module A + + int inc(int i) { + return i+1; + } + "); + return !hasErrors(t); +} + +test bool rascalFunctionDeclarationOk() { + Tree t = parseFunctionDeclaration("void f(){}"); + return !hasErrors(t); +} + + +test bool rascalModuleFollowedBySemi() { + Tree t = parseRascal(" + module A + ; + "); + + // There are a lot of productions in Rascal that have a ; as terminator. + // The parser assumes the user has only entered the ; on one of them, + // so the error list contains them all. + list[Tree] errors = findAllErrors(t); + assert size(errors) == 10; + + return getErrorText(findFirstError(t)) == ";"; +} + +test bool rascalOperatorTypo() { + Tree t = parseRascal(" + module A + + int f() = 1 x 1; + "); + + println("error text: "); + return getErrorText(findFirstError(t)) == "x 1"; +} + +test bool rascalIllegalStatement() { + Tree t = parseRascal("module A void f(){a}"); + + println("error text: "); + return getErrorText(findFirstError(t)) == "a}"; +} + +test bool rascalMissingCloseParen() { + Tree t = parseRascal("module A void f({} void g(){}"); + + assert getErrorText(findFirstError(t)) == "void g("; + assert getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "("; + + return true; +} + +test bool rascalFunctionDeclarationMissingCloseParen() { + Tree t = parseFunctionDeclaration("void f({} void g() {}", visualize=false); + + assert getErrorText(findFirstError(t)) == "void g("; + + Tree error = findFirstError(defaultErrorDisambiguationFilter(t)); + assert getErrorText(error) == "("; + Tree skipped = getSkipped(error); + loc location = getSkipped(error).src; + assert location.begin.column == 16 && location.length == 1; + + return true; +} + +// Not working yet: +/* +test bool rascalMissingOpeningParen() { + Tree t = parseRascal("module A void f){} void g() { }"); + + println("error text: "); + return getErrorText(findFirstError(t)) == "a}"; +} + +test bool rascalFunFunMissingCloseParen() { + Tree t = parseRascal("module A void f(){void g({}} void h(){}"); + + println("error text: "); + return getErrorText(findFirstError(t)) == "a}"; +} + +test bool rascalIfMissingExpr() { + Tree t = parseRascal("module A void f(){if(){}} 1;", visualize=false); + + println("error text: "); + return getErrorText(findFirstError(t)) == ";"; +} + +test bool rascalIfMissingOpeningParen() { + Tree t = parseRascal("module A void f(){if 1){}}", visualize=false); + + println("error text: "); + return getErrorText(findFirstError(t)) == ";"; +} + +test bool rascalIfMissingCloseParen() { + Tree t = parseRascal("module A void f(){if(1{}}", visualize=false); + + println("error text: "); + return getErrorText(findFirstError(t)) == ";"; +} + +test bool rascalIfEmptyBody() { + Tree t = parseRascal("module A void f(){if(1){}} 1;"); + + println("error text: "); + return getErrorText(findFirstError(t)) == ";"; +} + +test bool rascalIfMissingSemi() { + Tree t = parseRascal("module A void f(){if (true) {a}}"); + + println("error text: "); + return getErrorText(findFirstError(t)) == ";"; +} +*/ diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc new file mode 100644 index 00000000000..455e3661677 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc @@ -0,0 +1 @@ +module A void f(){ void g() {}} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc new file mode 100644 index 00000000000..81a9fd1c5b2 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc @@ -0,0 +1,17 @@ +module lang::rascal::tests::recovery::ToyRascal + +start syntax FunctionDeclaration = Signature FunctionBody; + +syntax Signature = Name Parameters; + +syntax Name = "f" | "g"; + +syntax Parameters = "(" ")"; + +syntax FunctionBody = "{" Statement* statements "}" ; + +syntax Statement + = "{" Statement+ statements "}" + | "s" ";"; + +layout Layout = [ \n\r\t]*; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc new file mode 100644 index 00000000000..455197c1f35 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc @@ -0,0 +1,37 @@ +module lang::rascal::tests::recovery::ToyRascalRecoveryTests + +import lang::rascal::tests::recovery::ToyRascal; + +import ParseTree; +import IO; + +Tree parseToyRascal(str input, bool visualize=false) { + Tree result = parser(#start[FunctionDeclaration], allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); + list[Tree] errors = findAllErrors(result); + if (errors != []) { + println("Tree has errors"); + for (error <- errors) { + println("- "); + } + + Tree disambiguated = defaultErrorDisambiguationFilter(result); + println("Best error: "); + } + + return result; +} + +test bool toyRascalOk() { + Tree t = parseToyRascal("f(){s;}"); + return !hasErrors(t); +} + +test bool toyRascalMissingOpenParen() { + Tree t = parseToyRascal("f){}", visualize=true); + return hasErrors(t) && getErrorText(findFirstError(t)) == ")"; +} + +test bool toyRascalMissingCloseParen() { + Tree t = parseToyRascal("f({}", visualize=true); + return hasErrors(t) && getErrorText(findFirstError(t)) == ")"; +} diff --git a/src/org/rascalmpl/parser/gtd/ExpectsProvider.java b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java new file mode 100644 index 00000000000..18266a7d10d --- /dev/null +++ b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java @@ -0,0 +1,7 @@ +package org.rascalmpl.parser.gtd; + +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; + +public interface ExpectsProvider

{ + AbstractStackNode

[] getExpects(String nonTerminal); +} diff --git a/src/org/rascalmpl/parser/gtd/IGTD.java b/src/org/rascalmpl/parser/gtd/IGTD.java index bb4e9b5c656..0e61d06b213 100644 --- a/src/org/rascalmpl/parser/gtd/IGTD.java +++ b/src/org/rascalmpl/parser/gtd/IGTD.java @@ -22,7 +22,7 @@ /** * Parser interface. */ -public interface IGTD{ +public interface IGTD extends ExpectsProvider

{ /** * Parse the input string, using the given non-terminal as start node. If * the parse process successfully completes a result will be constructed diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 7d5a2659227..1ed1b597447 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -60,7 +60,7 @@ /** * This is the core of the parser; it drives the parse process. */ -public abstract class SGTDBF implements IGTD{ +public abstract class SGTDBF implements IGTD { private final static int DEFAULT_TODOLIST_CAPACITY = 16; private URI inputURI; @@ -173,35 +173,38 @@ protected int getFreeStackNodeId() { throw new UnsupportedOperationException(); } - /** - * Triggers the gathering of alternatives for the given non-terminal. - */ @SuppressWarnings("unchecked") - protected AbstractStackNode

[] invokeExpects(AbstractStackNode

nonTerminal){ - String name = nonTerminal.getName(); - AbstractStackNode

[] expects = expectCache.get(name); - if(expects == null){ + public AbstractStackNode

[] getExpects(String nonTerminal) { + AbstractStackNode

[] expects = expectCache.get(nonTerminal); + if (expects == null) { try{ - Method method = getClass().getMethod(name); - try{ + Method method = getClass().getMethod(nonTerminal); + try { method.setAccessible(true); // Try to bypass the 'isAccessible' check to save time. - }catch(SecurityException sex){ + } catch (SecurityException sex) { // Ignore this if it happens. } expects = (AbstractStackNode

[]) method.invoke(this); - }catch(NoSuchMethodException nsmex){ - throw new UndeclaredNonTerminalException(name, getClass()); - }catch(IllegalAccessException iaex){ + } catch (NoSuchMethodException nsmex) { + throw new UndeclaredNonTerminalException(nonTerminal, getClass()); + } catch (IllegalAccessException iaex) { throw new RuntimeException(iaex); - }catch(InvocationTargetException itex){ + } catch (InvocationTargetException itex) { throw new RuntimeException(itex.getTargetException()); } - expectCache.putUnsafe(name, expects); + expectCache.putUnsafe(nonTerminal, expects); } - - return expects; + + return expects; + } + + /** + * Triggers the gathering of alternatives for the given non-terminal. + */ + protected AbstractStackNode

[] invokeExpects(AbstractStackNode

nonTerminal){ + return getExpects(nonTerminal.getName()); } /** diff --git a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java index 6412f806d25..dceb05517ec 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java @@ -88,6 +88,10 @@ public AbstractStackNode

getEmptyChild(){ throw new UnsupportedOperationException(); } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); sb.append("alt"); diff --git a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java index 79297837d49..685c839065d 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java @@ -80,6 +80,10 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); diff --git a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java index 7f5734b2a10..a513a9472c9 100644 --- a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java @@ -93,6 +93,10 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); diff --git a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java index 6543b14f9e1..3b0ed831748 100644 --- a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java @@ -104,6 +104,10 @@ public AbstractNode getResult(){ return result; } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); diff --git a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java index 247562026f7..3114bc97baf 100644 --- a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java @@ -76,6 +76,10 @@ public AbstractNode getResult(){ throw new UnsupportedOperationException(); } + public String toShortString() { + return expectIdentifier; + } + public String toString(){ StringBuilder sb = new StringBuilder("NonTerminal["); sb.append(expectIdentifier); diff --git a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java index 4b916f2a3be..c4694769941 100644 --- a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java @@ -92,6 +92,10 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); diff --git a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java index 15418e849c5..3310d11076b 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java @@ -110,6 +110,10 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); diff --git a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java index 638d978d00b..6d25c37750e 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java @@ -86,6 +86,10 @@ public AbstractStackNode

getEmptyChild(){ throw new UnsupportedOperationException(); } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); sb.append("seq"); diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index b4ddf1e8ec3..4acefaf19d3 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -127,6 +127,10 @@ public AbstractNode getResult(){ return sb.toString(); }*/ + public String toShortString() { + return "skip(" + result.toString() + ")"; + } + @Override public String toString() { return "SkippingStackNode[result=" + result + "," + super.toString() + "]"; diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 1815b5d4448..9ef73497c44 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -9,14 +9,18 @@ * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI * * Arnold Lankamp - Arnold.Lankamp@cwi.nl + * * Pieter Olviier - Pieter.Olivier@swat.engineering *******************************************************************************/ package org.rascalmpl.parser.uptr.recovery; import java.net.URI; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.function.Consumer; +import org.rascalmpl.parser.gtd.ExpectsProvider; import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.result.SkippedNode; @@ -35,7 +39,6 @@ import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; import org.rascalmpl.parser.uptr.recovery.InputMatcher.MatchResult; -import org.rascalmpl.parser.util.DebugUtil; import org.rascalmpl.util.visualize.DebugVisualizer; import org.rascalmpl.values.parsetrees.ProductionAdapter; @@ -46,9 +49,11 @@ public class ToTokenRecoverer implements IRecoverer { private URI uri; private IdDispenser stackNodeIdDispenser; + private ExpectsProvider expectsProvider; - public ToTokenRecoverer(URI uri, IdDispenser stackNodeIdDispenser) { + public ToTokenRecoverer(URI uri, ExpectsProvider expectsProvider, IdDispenser stackNodeIdDispenser) { this.uri = uri; + this.expectsProvider = expectsProvider; this.stackNodeIdDispenser = stackNodeIdDispenser; } @@ -60,22 +65,19 @@ public DoubleArrayList, AbstractNode> reviveStac DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, DoubleStack, AbstractNode> filteredNodes) { + // For now we ignore unmatchable leaf nodes and filtered nodes. At some point we might use those to improve error recovery. + ArrayList> failedNodes = new ArrayList<>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); - // handle unmatchableLeafNodes - // collectFilteredNodes(filteredNodes, failedNodes); - return reviveFailedNodes(input, location, failedNodes); + return reviveFailedNodes(input, failedNodes); } - private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ + private DoubleArrayList, AbstractNode> reviveNodes(int[] input, DoubleArrayList, ArrayList> recoveryNodes){ DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList<>(); - - // original: for (int i = recoveryNodes.size() - 1; i >= 0; --i) { - // But this caused problems because recovery nodes with a later position - // where queued before nodes with an earlier position which the parser cannot handle. + // Sort nodes by start location recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); DebugVisualizer visualizer = new DebugVisualizer("Recovery"); @@ -124,14 +126,16 @@ List> findSkippingNodes(int[] input, AbstractSta } // Try to find whitespace to skip to + // This often creates hopeless recovery attempts, but it might help in some cases. + // Further experimentation should quantify this statement. /* result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, dot); if (result != null) { nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); }*/ - // Do something more clever: find the last token of this production and skip until after that - List endMatchers = findEndMatchers(prod); + // Find the last token of this production and skip until after that + List endMatchers = findEndMatchers(recoveryNode); for (InputMatcher endMatcher : endMatchers) { MatchResult endMatch = endMatcher.findMatch(input, startLocation); if (endMatch != null) { @@ -143,7 +147,7 @@ List> findSkippingNodes(int[] input, AbstractSta // Find the first token of the next production and skip until before that List nextMatchers = findNextMatchers(recoveryNode); for (InputMatcher nextMatcher : nextMatchers) { - MatchResult nextMatch = nextMatcher.findMatch(input, startLocation); + MatchResult nextMatch = nextMatcher.findMatch(input, startLocation+1); if (nextMatch != null) { result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, nextMatch.getStart(), prod, dot); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); @@ -154,7 +158,7 @@ List> findSkippingNodes(int[] input, AbstractSta } // Gather matchers for the last token of a production - List findEndMatchers(IConstructor prod) { + List findEndMatchersBasedOnProduction(IConstructor prod) { IList args = (IList) prod.get(1); if (args.isEmpty()) { return Collections.emptyList(); @@ -168,6 +172,50 @@ List findEndMatchers(IConstructor prod) { return Collections.emptyList(); } + // Find matchers for the last token of the current stack node + List findEndMatchers(AbstractStackNode stackNode) { + final List matchers = new java.util.ArrayList<>(); + + AbstractStackNode[] prod = stackNode.getProduction(); + addEndMatchers(prod, prod.length-1, matchers, new HashSet<>()); + + return matchers; + } + + void addEndMatchers(AbstractStackNode[] prod, int dot, List matchers, Set visitedNodes) { + if (prod == null || dot < 0) { + return; + } + + AbstractStackNode last = prod[dot]; + if (visitedNodes.contains(last.getId())) { + return; + } + visitedNodes.add(last.getId()); + + // Future improvement: while (isNullable(last) addEndMatchers(prod, dot-1, matchers); + + if (last instanceof LiteralStackNode) { + LiteralStackNode lastLiteral = (LiteralStackNode) last; + matchers.add(new LiteralMatcher(lastLiteral.getLiteral())); + } + + if (last instanceof CaseInsensitiveLiteralStackNode) { + CaseInsensitiveLiteralStackNode lastLiteral = (CaseInsensitiveLiteralStackNode) last; + matchers.add(new CaseInsensitiveLiteralMatcher(lastLiteral.getLiteral())); + } + + if (last instanceof NonTerminalStackNode) { + NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) last; + String name = nextNonTerminal.getName(); + AbstractStackNode[] alternatives = expectsProvider.getExpects(name); + for (AbstractStackNode alternative : alternatives) { + // In the future we might want to use all prefix-sharing alternatives here + addEndMatchers(alternative.getProduction(), 0, matchers, visitedNodes); + } + } + } + void forAllParents(AbstractStackNode stackNode, Consumer> consumer) { IntegerObjectList> edges = stackNode.getEdges(); if (edges != null) { @@ -182,7 +230,7 @@ void forAllParents(AbstractStackNode stackNode, Consumer getSingleParentStack(AbstractStackNode stackNode) { + private AbstractStackNode getSingleParentStack(AbstractStackNode stackNode) { if (stackNode == null) { return null; } @@ -200,35 +248,39 @@ AbstractStackNode getSingleParentStack(AbstractStackNode findNextMatchers(AbstractStackNode stackNode) { - DebugVisualizer visualizer = new DebugVisualizer("findNextMatcher"); - visualizer.visualize(stackNode); - final List matchers = new java.util.ArrayList<>(); + // Future improvement: use all parents instead of just one AbstractStackNode parent = getSingleParentStack(stackNode); if (parent == null) { return matchers; } - AbstractStackNode[] prod = parent.getProduction(); - if (prod == null) { - return matchers; - } + addNextMatchers(parent.getProduction(), parent.getDot()+1, matchers, new HashSet<>()); - int nextDot = parent.getDot() + 1; - if (nextDot >= prod.length) { - return matchers; + return matchers; + } + + private void addNextMatchers(AbstractStackNode[] prod, int dot, List matchers, Set visitedNodes) { + if (prod == null || dot >= prod.length) { + return; } - AbstractStackNode next = prod[nextDot]; - if (next instanceof NonTerminalStackNode && next.getName().startsWith("layouts_")) { - // Look "through" layout for now, should be more general to look through any node that can be empty - nextDot++; - if (nextDot >= prod.length) { - return matchers; + AbstractStackNode next = prod[dot]; + while (next instanceof NonTerminalStackNode && next.getName().startsWith("layouts_")) { + // Look "through" layout for now, this should really be more general and look through any node that can be empty + // When a node can be empty, we should also consider all prefix-shared alternatives. + dot++; + if (dot >= prod.length) { + return; } - next = prod[nextDot]; + next = prod[dot]; + } + + if (visitedNodes.contains(next.getId())) { + return; } + visitedNodes.add(next.getId()); if (next instanceof LiteralStackNode) { LiteralStackNode nextLiteral = (LiteralStackNode) next; @@ -241,11 +293,13 @@ List findNextMatchers(AbstractStackNode stackNode) { } if (next instanceof NonTerminalStackNode) { - //NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) next; - DebugUtil.opportunityToBreak(); + NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) next; + String name = nextNonTerminal.getName(); + AbstractStackNode[] alternatives = expectsProvider.getExpects(name); + for (AbstractStackNode alternative : alternatives) { + addNextMatchers(alternative.getProduction(), 0, matchers, visitedNodes); + } } - - return matchers; } // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) @@ -277,14 +331,14 @@ private AbstractStackNode getSinglePredecessor(AbstractStackNode, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { + private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, ArrayList> failedNodes) { DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList<>(); for (int i = failedNodes.size() - 1; i >= 0; --i) { findRecoveryNodes(failedNodes.get(i), recoveryNodes); } - return reviveNodes(input, location, recoveryNodes); + return reviveNodes(input, recoveryNodes); } private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { @@ -325,7 +379,6 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr todo.push(failer); - boolean useNext = true; // Use this to only use the last active element from a production while (!todo.isEmpty()) { AbstractStackNode node = todo.pop(); @@ -335,14 +388,11 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr visited.put(node, 0); - if (useNext) { - ArrayList recoveryProductions = new ArrayList<>(); - collectProductions(node, recoveryProductions); - if (recoveryProductions.size() > 0) { - addRecoveryNode(node, recoveryProductions, recoveryNodes); - } + ArrayList recoveryProductions = new ArrayList<>(); + collectProductions(node, recoveryProductions); + if (recoveryProductions.size() > 0) { + addRecoveryNode(node, recoveryProductions, recoveryNodes); } - useNext = node.getDot() == 0; IntegerObjectList> edges = node.getEdges(); diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index b9fa5e44989..f7ff0997972 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -583,7 +583,7 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in IDebugListener debugListener = null; URI uri = location.getURI(); if (allowRecovery) { - recoverer = new ToTokenRecoverer(uri, new StackNodeIdDispenser(parserInstance)); + recoverer = new ToTokenRecoverer(uri, parserInstance, new StackNodeIdDispenser(parserInstance)); //debugListener = new DebugLogger(new PrintWriter(System.out, true)); } return (ITree) parserInstance.parse(methodName, uri, input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer, debugListener); From d765aaada08bc8fedb87fa6abee88d491989caa1 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 07:45:20 +0200 Subject: [PATCH 049/190] Miscellaneous cleanup of error recovery code --- .vscode/launch.json | 16 ------ .../tests/recovery/BasicRecoveryTests.rsc | 4 +- .../tests/recovery/NestedRecoveryTests.rsc | 12 ++-- .../tests/recovery/PicoRecoveryTests.rsc | 57 ++++--------------- .../tests/recovery/RunRecoveryTests.rsc | 9 --- src/org/rascalmpl/library/util/TermREPL.java | 2 - src/org/rascalmpl/parser/gtd/SGTDBF.java | 21 +++---- .../parser/gtd/stack/SkippingStackNode.java | 12 +--- .../parser/gtd/stack/edge/EdgesSet.java | 4 +- 9 files changed, 29 insertions(+), 108 deletions(-) delete mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc diff --git a/.vscode/launch.json b/.vscode/launch.json index e16d6d8230e..833fb54b4da 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,13 +4,6 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ - { - "type": "java", - "name": "Simple2", - "request": "launch", - "mainClass": "org.rascalmpl.test.parser.Simple2", - "projectName": "rascal" - }, { "type": "java", "name": "Launch DocRunner", @@ -39,15 +32,6 @@ "projectName": "rascal", "vmArgs": "-Xss80m -Xmx2g -ea" }, - { - "type": "java", - "name": "Recovery tests", - "request": "launch", - "mainClass": "org.rascalmpl.shell.RascalShell", - "projectName": "rascal", - "vmArgs": "-Xss80m -Xmx2g -ea", - "args": "lang::rascal::tests::recovery::RunRecoveryTests" - }, { "type": "java", "name": "Launch RascalShell Tutor", diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index f2e1e129787..287ad093e71 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -31,9 +31,7 @@ test bool axc() { return getErrorText(findFirstError(t)) == "x c"; } -/* We need to look through non-terminals to make this test work test bool ax() { Tree t = parseS("a x $", visualize=true); - return getErrorText(findFirstError(t)) = "x"; + return getErrorText(findFirstError(t)) == "x "; } -*/ diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc index 292043f00a4..f335dcd3b12 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc @@ -7,22 +7,20 @@ layout Layout = [\ ]* !>> [\ ]; syntax S = T; -syntax T = A B "c"; +syntax T = A B C; syntax A = "a"; syntax B = "b" "b"; -//syntax C = "c"; +syntax C = "c"; private Tree parseS(str input, bool visualize=false) = parser(#S, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); test bool nestedOk() { - return !hasErrors(parseS("a b b c", visualize=true)); + return !hasErrors(parseS("a b b c")); } test bool nestedTypo() { - Tree t = parseS("a b x c", visualize=true); - iprintln(t); - println("Error text: \'\'"); - return getErrorText(findFirstError(t)) == "x "; + Tree t = parseS("a b x c"); + return getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "x "; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc index 5d3236ba653..8634cca3e29 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc @@ -45,10 +45,8 @@ test bool picoTypo() { input := input - 1 od end"); - iprintln(findFirstError(t)); - return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "x rep; - repnr := repnr - 1 - od"; + Tree error = findFirstError(defaultErrorDisambiguationFilter(t)); + return getErrorText(error) == "output x rep"; } test bool picoMissingSemi() { @@ -68,63 +66,32 @@ test bool picoMissingSemi() { input := input - 1 od end"); - str errorText = getErrorText(findFirstError(t)); - println("error count: "); - println("error text: "); - - for (error <- findAllErrors(t)) { - println(" error: "); - } - - return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 - od"; + Tree error = findFirstError(defaultErrorDisambiguationFilter(t)); + return getErrorText(error) == "input := input - 1\n od"; } -test bool picoTypoMinimal() { +test bool picoTypoSmall() { t = parsePico( "begin declare; while input do input x= 14; output := 0 od -end", visualize=false); - - iprintln(t); - - for (error <- findAllErrors(t)) { - println(" error: "); - } +end"); - disambiguated = defaultErrorDisambiguationFilter(t); - println("after disambiguation:"); - for (error <- findAllErrors(disambiguated)) { - println(" error: "); + Tree error = findFirstError(defaultErrorDisambiguationFilter(t)); + return getErrorText(error) == "x= 14"; } -return hasErrors(t); - /*str errorText = getErrorText(findFirstError(t)); - println("error text: "); - return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 - od"; - */ -} -test bool picoMissingSemiMinimal() { +test bool picoMissingSemiSmall() { t = parsePico( "begin declare; while input do input := 14 output := 0 od -end", visualize=true); - - for (error <- findAllErrors(t)) { - println(" error: "); - } +end"); -return hasErrors(t); - /*str errorText = getErrorText(findFirstError(t)); - println("error text: "); - return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 - od"; - */ + Tree error = findFirstError(defaultErrorDisambiguationFilter(t)); + return getErrorText(error) == "output := 0\n od"; } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc deleted file mode 100644 index 3a5e589a2dd..00000000000 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc +++ /dev/null @@ -1,9 +0,0 @@ -module lang::rascal::tests::recovery::RunRecoveryTests - -import lang::rascal::tests::recovery::BasicRecoveryTests; -import util::REPL; - -int main(list[str] args){ - Terminal terminal = newREPL(repl()); - return 0; -} diff --git a/src/org/rascalmpl/library/util/TermREPL.java b/src/org/rascalmpl/library/util/TermREPL.java index 75ab0f0398b..f149bb36340 100644 --- a/src/org/rascalmpl/library/util/TermREPL.java +++ b/src/org/rascalmpl/library/util/TermREPL.java @@ -103,7 +103,6 @@ public static class TheREPL implements ILanguageProtocol { private final AbstractFunction completor; private final IValueFactory vf; private final AbstractFunction stacktrace; - private IDEServices services; public TheREPL(IValueFactory vf, IString title, IString welcome, IString prompt, IString quit, ISourceLocation history, IFunction handler, IFunction completor, IValue stacktrace, InputStream input, OutputStream stderr, OutputStream stdout) { @@ -150,7 +149,6 @@ public void initialize(InputStream input, OutputStream stdout, OutputStream stde this.stdout = stdout; this.stderr = stderr; this.input = input; - this.services = services; } @Override diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 1ed1b597447..7f8f8e540b1 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -111,7 +111,7 @@ public abstract class SGTDBF implements IGTD { // Reflection is used to get the expects for each non-terminal. - // This cache is used so the reflection call is only needed once. + // This cache is used so the reflection call is only needed once per non-terminal. private final HashMap[]> expectCache; private final IntegerObjectList> sharedLastExpects; @@ -984,11 +984,9 @@ private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int queueIndex = 0; location = startPosition; - // was: DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex + 1]; DoubleStack, AbstractNode> terminalsTodo = todoLists[length]; if (terminalsTodo == null) { terminalsTodo = new DoubleStack, AbstractNode>(); - // was: todoLists[queueIndex + 1] = terminalsTodo; // Why the +1 and not length? To get the recovered node to be processed first? todoLists[length] = terminalsTodo; } @@ -1007,8 +1005,6 @@ else if (startPosition == location) { } } - - /** * Handles the retrieved alternatives for the given stack. */ @@ -1480,14 +1476,14 @@ protected T buildResult(AbstractNode result, INodeFlattener converter, INo * Error nodes end up in a n inconvenient form because of the parser algorithm. * This post-processing step transforms the original tree into a more useful form. * In essence, error subtrees look like this after parsing: - * `appl(prod(S,[]), [,,...,appl(skipped(S,prod(S,[]),),[)]])` - * This method pulls up the skipped node, transforming these trees into: - * `appl(skipped(S,prod(S,[]),), [,,...,[])])` + * `appl(prod(S,[]), [,,...,appl(skipped([]))])` + * This method transforms these trees into: + * `appl(error(S,prod(S,[]),), [,,...,appl(skipped([]))])` * This means productions that failed to parse can be recognized at the top level. * Note that this can only be done when we know the actual type of T is IConstructor. */ @SuppressWarnings("unchecked") - T fixErrorNodes(T tree, INodeConstructorFactory nodeConstructorFactory) { + private T fixErrorNodes(T tree, INodeConstructorFactory nodeConstructorFactory) { if (!(tree instanceof IConstructor)) { return tree; } @@ -1495,7 +1491,7 @@ T fixErrorNodes(T tree, INodeConstructorFactory nodeConstructorFactory) { return (T) fixErrorNodes((IConstructor) tree, (INodeConstructorFactory) nodeConstructorFactory); } - IConstructor fixErrorNodes(IConstructor tree, INodeConstructorFactory nodeConstructorFactory) { + private IConstructor fixErrorNodes(IConstructor tree, INodeConstructorFactory nodeConstructorFactory) { IConstructor result; Type type = tree.getConstructorType(); if (type == RascalValueFactory.Tree_Appl) { @@ -1584,9 +1580,8 @@ private void visualize(String step, NodeId highlight) { } } - /** - * Getters used for graph generation/debugging + * Getters used for graph generation for debugging (see DebugVisualizer) */ public int[] getInput() { @@ -1636,6 +1631,4 @@ public DoubleStack, AbstractNode>, Abstract public DoubleStack, AbstractNode> getFilteredNodes() { return filteredNodes; } - - } diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 4acefaf19d3..8ab46ea51b2 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -8,6 +8,7 @@ * Contributors: * * Arnold Lankamp - Arnold.Lankamp@cwi.nl + * * Pieter Olivier - Pieter.Olivier@swat.engineering *******************************************************************************/ package org.rascalmpl.parser.gtd.stack; @@ -117,16 +118,7 @@ public AbstractNode getResult(){ return result; } - /*Original: public String toString(){ - StringBuilder sb = new StringBuilder(); - sb.append(getId()); - sb.append('('); - sb.append(startLocation); - sb.append(')'); - - return sb.toString(); - }*/ - + @Override public String toShortString() { return "skip(" + result.toString() + ")"; } diff --git a/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java b/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java index f95daf557c7..351cd048f7b 100644 --- a/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java +++ b/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java @@ -22,9 +22,9 @@ @SuppressWarnings({"unchecked", "cast"}) public class EdgesSet

{ - public final static int DEFAULT_RESULT_STORE_ID = -1; + public static final int DEFAULT_RESULT_STORE_ID = -1; - private final static int DEFAULT_SIZE = 8; + private static final int DEFAULT_SIZE = 8; private AbstractStackNode

[] edges; private int size; From a4c4494ff786dca029a8f51e8976a2f9d63a8585 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 09:09:03 +0200 Subject: [PATCH 050/190] Removed unused fields from SkippedNode --- .../parser/gtd/result/SkippedNode.java | 18 +++--------------- .../parser/gtd/stack/SkippingStackNode.java | 2 +- .../util/visualize/DebugVisualizer.java | 10 ++-------- 3 files changed, 6 insertions(+), 24 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/result/SkippedNode.java b/src/org/rascalmpl/parser/gtd/result/SkippedNode.java index 53578520ce2..2f7541abb2a 100644 --- a/src/org/rascalmpl/parser/gtd/result/SkippedNode.java +++ b/src/org/rascalmpl/parser/gtd/result/SkippedNode.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2009-2013 CWI + * Copyright (c) 2009-2024 CWI * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -8,6 +8,7 @@ * Contributors: * * Arnold Lankamp - Arnold.Lankamp@cwi.nl + * * Pieter Olivier - Pieter.Olivier@swat.engineering *******************************************************************************/ package org.rascalmpl.parser.gtd.result; @@ -21,19 +22,14 @@ public class SkippedNode extends AbstractNode { public static final int ID = 9; - // TODO: only "skippedChars" is actually needed private final URI input; - private final Object production; - private final int dot; private final int[] skippedChars; private final int offset; - public SkippedNode(URI input, Object production, int dot, int[] skippedChars, int offset) { + public SkippedNode(URI input, int[] skippedChars, int offset) { super(); this.input = input; - this.production = production; - this.dot = dot; this.skippedChars = skippedChars; this.offset = offset; } @@ -46,14 +42,6 @@ public URI getInput() { return input; } - public Object getProduction() { - return production; - } - - public int getDot() { - return dot; - } - public int[] getSkippedChars(){ return skippedChars; } diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 8ab46ea51b2..375f5a47944 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2009-2013 CWI + * Copyright (c) 2009-2024 CWI * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at diff --git a/src/org/rascalmpl/util/visualize/DebugVisualizer.java b/src/org/rascalmpl/util/visualize/DebugVisualizer.java index c944242d5a3..de1abef0230 100644 --- a/src/org/rascalmpl/util/visualize/DebugVisualizer.java +++ b/src/org/rascalmpl/util/visualize/DebugVisualizer.java @@ -10,11 +10,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -386,8 +383,7 @@ private void enrichLiteralNode(DotNode dotNode, LiteralNode literalNode) { private void enrichSkippedNode(DotNode dotNode, SkippedNode skippedNode) { String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); - label += "\n." + skippedNode.getDot() + "@" + skippedNode.getOffset() + ": " + " \"" + UnicodeConverter.unicodeArrayToString(skippedNode.getSkippedChars()) + "\""; - label += "\nin: " + DebugUtil.prodToString((IConstructor) skippedNode.getProduction()); + label += "\n@" + skippedNode.getOffset() + ": " + " \"" + UnicodeConverter.unicodeArrayToString(skippedNode.getSkippedChars()) + "\""; dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); } @@ -397,8 +393,6 @@ private void enrichSortContainerNode(DotNode dotNode, SortContainerNode void writeGraph() { + public void writeGraph() { if (graph != null) { writeGraph(graph); } From 34dccde4decda522bb7d8c3b1689b2fc4d931a78 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 09:59:34 +0200 Subject: [PATCH 051/190] Switched from hardcoding to environment variable for the parser visualization path --- .../parser/gtd/stack/SkippingStackNode.java | 24 ++++----- .../recovery/ToNextWhitespaceRecoverer.java | 36 +++++-------- .../uptr/recovery/ToTokenRecoverer.java | 13 +++-- .../util/visualize/DebugVisualizer.java | 54 ++++++++++--------- src/org/rascalmpl/util/visualize/replay.html | 36 +++++++++++++ 5 files changed, 96 insertions(+), 67 deletions(-) create mode 100644 src/org/rascalmpl/util/visualize/replay.html diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 375f5a47944..5c3ca844518 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -17,30 +17,28 @@ import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.result.SkippedNode; -import io.usethesource.vallang.IConstructor; - public final class SkippingStackNode

extends AbstractMatchableStackNode

{ private final SkippedNode result; - public static SkippedNode createResultUntilCharClass(URI uri, int[] until, int[] input, int startLocation, IConstructor production, int dot) { + public static SkippedNode createResultUntilCharClass(URI uri, int[] until, int[] input, int startLocation) { for (int to = startLocation ; to < input.length; ++to) { for (int i = 0; i < until.length; ++i) { if (input[to] == until[i]) { int length = to - startLocation; - return new SkippedNode(uri, production, dot, createSkippedToken(input, startLocation, length), startLocation); + return new SkippedNode(uri, createSkippedToken(input, startLocation, length), startLocation); } } } - return new SkippedNode(uri, production, dot, new int[0], startLocation); + return new SkippedNode(uri, new int[0], startLocation); } - public static SkippedNode createResultUntilEndOfInput(URI uri, int[] input, int startLocation, IConstructor production, int dot) { + public static SkippedNode createResultUntilEndOfInput(URI uri, int[] input, int startLocation) { int length = input.length - startLocation; - return new SkippedNode(uri, production, dot, createSkippedToken(input, startLocation, length), startLocation); + return new SkippedNode(uri, createSkippedToken(input, startLocation, length), startLocation); } - public static SkippedNode createResultUntilToken(URI uri, String token, int[] input, int startLocation, IConstructor production, int dot) { + public static SkippedNode createResultUntilToken(URI uri, String token, int[] input, int startLocation) { int length = token.length(); for (int start=startLocation; start+length < input.length; start++) { boolean match = true; @@ -50,7 +48,7 @@ public static SkippedNode createResultUntilToken(URI uri, String token, int[] in } if (match) { - return createResultUntilChar(uri, input, startLocation, start+length-startLocation, production, dot); + return createResultUntilChar(uri, input, startLocation, start+length-startLocation); } } } @@ -58,8 +56,8 @@ public static SkippedNode createResultUntilToken(URI uri, String token, int[] in return null; } - public static SkippedNode createResultUntilChar(URI uri, int[] input, int startLocation, int endLocation, IConstructor production, int dot) { - return new SkippedNode(uri, production, dot, createSkippedToken(input, startLocation, endLocation - startLocation), startLocation); + public static SkippedNode createResultUntilChar(URI uri, int[] input, int startLocation, int endLocation) { + return new SkippedNode(uri, createSkippedToken(input, startLocation, endLocation - startLocation), startLocation); } private static int[] createSkippedToken(int[] input, int startLocation, int length) { @@ -103,11 +101,11 @@ public AbstractNode match(int[] input, int location){ } public AbstractStackNode

getCleanCopy(int startLocation){ - return new SkippingStackNode

(this, startLocation); + return new SkippingStackNode<>(this, startLocation); } public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ - return new SkippingStackNode

(this, (SkippedNode) result, startLocation); + return new SkippingStackNode<>(this, (SkippedNode) result, startLocation); } public int getLength(){ diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 7b1e9439af8..78fb79ca821 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -47,21 +47,17 @@ public DoubleArrayList, AbstractNode> reviveStac DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, DoubleStack, AbstractNode> filteredNodes) { - ArrayList> failedNodes = new ArrayList>(); + ArrayList> failedNodes = new ArrayList<>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); - // TODO: handle unmatchableLeafNodes + // handle unmatchableLeafNodes? //collectFilteredNodes(filteredNodes, failedNodes); - return reviveFailedNodes(input, location, failedNodes); + return reviveFailedNodes(input, failedNodes); } - private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ - DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); - - // original: for (int i = recoveryNodes.size() - 1; i >= 0; --i) { - // But this caused problems because recovery nodes with a later position - // where queued before nodes with an earlier position which the parser cannot handle. + private DoubleArrayList, AbstractNode> reviveNodes(int[] input, DoubleArrayList, ArrayList> recoveryNodes){ + DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList<>(); recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); @@ -75,14 +71,13 @@ private DoubleArrayList, AbstractNode> reviveNod AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); - int dot = recoveryNode.getDot(); int startLocation = recoveryNode.getStartLocation(); SkippedNode result; if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { - result = SkippingStackNode.createResultUntilEndOfInput(null, input, startLocation, prod, dot); + result = SkippingStackNode.createResultUntilEndOfInput(null, input, startLocation); } else { - result = SkippingStackNode.createResultUntilCharClass(null, WHITESPACE, input, startLocation, prod, dot); + result = SkippingStackNode.createResultUntilCharClass(null, WHITESPACE, input, startLocation); } AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result); @@ -131,14 +126,14 @@ private AbstractStackNode getSinglePredecessor(AbstractStackNode, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { - DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList, ArrayList>(); + private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, ArrayList> failedNodes) { + DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList<>(); for (int i = failedNodes.size() - 1; i >= 0; --i) { findRecoveryNodes(failedNodes.get(i), recoveryNodes); } - return reviveNodes(input, location, recoveryNodes); + return reviveNodes(input, recoveryNodes); } private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { @@ -174,8 +169,8 @@ private static void collectUnmatchableMidProductionNodes(int location, DoubleSta * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. */ private void findRecoveryNodes(AbstractStackNode failer, DoubleArrayList, ArrayList> recoveryNodes) { - ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap>(); - Stack> todo = new Stack>(); + ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap<>(); + Stack> todo = new Stack<>(); todo.push(failer); @@ -188,7 +183,7 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr visited.put(node, 0); - ArrayList recoveryProductions = new ArrayList(); + ArrayList recoveryProductions = new ArrayList<>(); collectProductions(node, recoveryProductions); if (recoveryProductions.size() > 0) { recoveryNodes.add(node, recoveryProductions); @@ -219,12 +214,10 @@ private void collectProductions(AbstractStackNode node, ArrayList< int dot = node.getDot(); - System.err.println("collect productions for node: " + node); if (node.isEndNode()) { IConstructor parentProduction = node.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)){ productions.add(parentProduction); - System.err.println("adding production: " + parentProduction); if (ProductionAdapter.isList(parentProduction)) { return; // Don't follow productions in lists productions, since they are 'cyclic'. @@ -238,7 +231,6 @@ private void collectProductions(AbstractStackNode node, ArrayList< IConstructor parentProduction = currentNode.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)) { productions.add(parentProduction); - System.err.println("adding production at " + i + ": " + parentProduction); } } @@ -250,6 +242,4 @@ private void collectProductions(AbstractStackNode node, ArrayList< } } } - - } diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 9ef73497c44..fd028a1dc30 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2009-2022 NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2009-2024 NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -87,14 +87,13 @@ private DoubleArrayList, AbstractNode> reviveNod AbstractStackNode recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); - int dot = recoveryNode.getDot(); int startLocation = recoveryNode.getStartLocation(); // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). for (int j = prods.size() - 1; j >= 0; --j) { IConstructor prod = prods.get(j); - List> skippingNodes = findSkippingNodes(input, recoveryNode, prod, dot, startLocation); + List> skippingNodes = findSkippingNodes(input, recoveryNode, prod, startLocation); for (SkippingStackNode skippingNode : skippingNodes) { AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); @@ -113,14 +112,14 @@ private DoubleArrayList, AbstractNode> reviveNod return recoveredNodes; } - List> findSkippingNodes(int[] input, AbstractStackNode recoveryNode, IConstructor prod, int dot, int startLocation) { + List> findSkippingNodes(int[] input, AbstractStackNode recoveryNode, IConstructor prod, int startLocation) { List> nodes = new java.util.ArrayList<>(); SkippedNode result; // If we are the top-level node, just skip the rest of the input if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { - result = SkippingStackNode.createResultUntilEndOfInput(uri, input, startLocation, prod, dot); + result = SkippingStackNode.createResultUntilEndOfInput(uri, input, startLocation); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); return nodes; // No other nodes would be useful } @@ -139,7 +138,7 @@ List> findSkippingNodes(int[] input, AbstractSta for (InputMatcher endMatcher : endMatchers) { MatchResult endMatch = endMatcher.findMatch(input, startLocation); if (endMatch != null) { - result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, endMatch.getEnd(), prod, dot); + result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, endMatch.getEnd()); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); } } @@ -149,7 +148,7 @@ List> findSkippingNodes(int[] input, AbstractSta for (InputMatcher nextMatcher : nextMatchers) { MatchResult nextMatch = nextMatcher.findMatch(input, startLocation+1); if (nextMatch != null) { - result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, nextMatch.getStart(), prod, dot); + result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, nextMatch.getStart()); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); } } diff --git a/src/org/rascalmpl/util/visualize/DebugVisualizer.java b/src/org/rascalmpl/util/visualize/DebugVisualizer.java index de1abef0230..f2c941044dd 100644 --- a/src/org/rascalmpl/util/visualize/DebugVisualizer.java +++ b/src/org/rascalmpl/util/visualize/DebugVisualizer.java @@ -7,8 +7,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.HashMap; import java.util.Map; @@ -50,9 +48,20 @@ import io.usethesource.vallang.IConstructor; +/** + * The parser uses quite complex datastructures. + * In order to understand what is going on when parsing, this class can generate graphs (as dot files) + * representing the internal datastructurs of the parser. + * + * These graphs are written to files that are relative to a directory specified in the environment + * variable PARSER_VISUALIZATION_PATH. + * + * The parser can generate a large number of snapshots of the parser state during a single parse. + * The file 'replay.html' contains an simple example of a html file to navigate through these snapshots. + */ public class DebugVisualizer { - static final String BASE_DIR = "D:/debug/parser-traces/docs/"; public static final boolean VISUALIZATION_ENABLED = true; + private static final String PARSER_VISUALIZATION_PATH_ENV = "PARSER_VISUALIZATION_PATH"; private static final boolean INCLUDE_PRODUCTIONS = false; public static final NodeId PARSER_ID = new NodeId("Parser"); @@ -69,15 +78,6 @@ public class DebugVisualizer { private static final NodeId RECOVERED_NODES_ID = new NodeId("recoveredNodes"); - /*static public class GraphObject { - static public class Kind { - private boolean dotGraph = true; - } - - private Kind kind = new Kind(); - private String text = "\ndigraph G {\n a -> b; \n}\n"; - }*/ - private static class StreamGobbler implements Runnable { private InputStream inputStream; private Consumer consumer; @@ -95,14 +95,24 @@ public void run() { } private String name; + private File basePath; + private File frameDir; private Map stackNodeNodes; private DotGraph graph; private int frame; public DebugVisualizer(String name) { + // In the future we might want to offer some way to control the path from within Rascal. + String path = System.getenv(PARSER_VISUALIZATION_PATH_ENV); + if (path == null) { + throw new RuntimeException("The environment variable '" + PARSER_VISUALIZATION_PATH_ENV + "' is not set."); + } + basePath = new File(System.getenv(PARSER_VISUALIZATION_PATH_ENV)); + this.name = name; stackNodeNodes = new HashMap<>(); - File frameDir = new File(BASE_DIR + "/frames/" + name); + + frameDir = new File(new File(basePath, "frames"), name); if (frameDir.exists()) { try { FileUtils.deleteDirectory(frameDir); @@ -603,11 +613,11 @@ private

void addStackNodeStack(DotGraph graph, NodeId nodeId, Stack + + + + From 7eceab22d24b6b8cda704aec094bca5800c213ff Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 10:03:01 +0200 Subject: [PATCH 052/190] Fixed some compiler warnings in the DebugVisualizer --- .../util/visualize/DebugVisualizer.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/org/rascalmpl/util/visualize/DebugVisualizer.java b/src/org/rascalmpl/util/visualize/DebugVisualizer.java index f2c941044dd..d2b1c28d359 100644 --- a/src/org/rascalmpl/util/visualize/DebugVisualizer.java +++ b/src/org/rascalmpl/util/visualize/DebugVisualizer.java @@ -170,9 +170,9 @@ private synchronized DotGraph createGraph(AbstractStackNode stackN private DotGraph createGraph(DoubleArrayList, ArrayList> recoveryNodes) { reset(); graph = new DotGraph(name, true); - final NodeId RECOVERY_NODES_ID = new NodeId("recovery-nodes"); + final NodeId recoveryNodesId = new NodeId("recovery-nodes"); - DotNode arrayNode = DotNode.createArrayNode(RECOVERY_NODES_ID, recoveryNodes.size()); + DotNode arrayNode = DotNode.createArrayNode(recoveryNodesId, recoveryNodes.size()); graph.addNode(arrayNode); for (int i=0; i, Ar recoveryRecord.addEntry(new DotField("Productions", "productions")); graph.addRecordNode(pairId, recoveryRecord); - graph.addEdge(new NodeId(RECOVERY_NODES_ID, String.valueOf(i)), pairId); + graph.addEdge(new NodeId(recoveryNodesId, String.valueOf(i)), pairId); DotNode node = addStack(graph, recoveryNodes.getFirst(i)); @@ -333,11 +333,10 @@ private

DotNode createDotNode(AbstractStackNode

stackNode) { } node.addAttribute(DotAttribute.ATTR_LABEL, label); - // TODO: add prefixes - return node; } + @SuppressWarnings("unchecked") private void addParserNode(DotGraph graph, AbstractNode parserNode) { NodeId id = getNodeId(parserNode); DotNode dotNode = new DotNode(id); @@ -543,13 +542,6 @@ private void addUnmatchableMidProductionNodes(SGTDBF parser, private void addFilteredNodes(SGTDBF parser, DotGraph graph) { addStackAndNodeDoubleStack(graph, FILTERED_NODES_ID, parser.getFilteredNodes()); } - /* - * private final Stack> unexpandableNodes; - private final Stack> unmatchableLeafNodes; // Leaf nodes (for instance literals) that failed to match - private final DoubleStack, AbstractNode>, AbstractStackNode

> unmatchableMidProductionNodes; - private final DoubleStack, AbstractNode> filteredNodes; - - */ private void addStackAndNodeDoubleStack(DotGraph graph, NodeId nodeId, DoubleStack, N> doubleStack) { DotNode arrayNode = DotNode.createArrayNode(nodeId, doubleStack == null ? 0 : doubleStack.getSize()); From 8e8f4cb66133b0bfc706cf6c3dd7c29d442ff302 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 11:16:31 +0200 Subject: [PATCH 053/190] Fixed some compiler warnings --- .../gtd/stack/AbstractMatchableStackNode.java | 1 + .../parser/gtd/stack/AbstractStackNode.java | 2 ++ .../parser/gtd/stack/AlternativeStackNode.java | 13 +++++++++++++ .../stack/CaseInsensitiveLiteralStackNode.java | 17 +++++++++++++++-- .../parser/gtd/stack/CharStackNode.java | 16 +++++++++++++--- .../parser/gtd/stack/EmptyStackNode.java | 15 +++++++++++++++ .../parser/gtd/stack/EpsilonStackNode.java | 16 ++++++++++++++-- .../parser/gtd/stack/ListStackNode.java | 15 ++++++++++++++- .../parser/gtd/stack/LiteralStackNode.java | 12 ++++++++++++ .../gtd/stack/MultiCharacterStackNode.java | 15 ++++++++++++++- .../parser/gtd/stack/NonTerminalStackNode.java | 5 +++++ .../parser/gtd/stack/OptionalStackNode.java | 5 +++++ .../gtd/stack/RecoveryPointStackNode.java | 5 +++++ .../gtd/stack/SeparatedListStackNode.java | 5 +++++ .../parser/gtd/stack/SequenceStackNode.java | 5 +++++ .../parser/gtd/stack/SkippingStackNode.java | 4 ++++ 16 files changed, 142 insertions(+), 9 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java index 3943c6dca5f..fddba15db90 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java @@ -66,6 +66,7 @@ public AbstractStackNode

getEmptyChild(){ throw new UnsupportedOperationException(); } + @Override public final boolean isMatchable(){ return true; } diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index b477301e954..f156e70fab7 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -811,6 +811,8 @@ public String toString() { return builder.toString(); } + abstract void accept(StackNodeVisitor

visitor); + // Matchables. /** * Matches the symbol associated with this node to the input at the specified location. diff --git a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java index dceb05517ec..a16aa931335 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java @@ -88,10 +88,12 @@ public AbstractStackNode

getEmptyChild(){ throw new UnsupportedOperationException(); } + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append("alt"); @@ -103,10 +105,16 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return production.hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof AlternativeStackNode)) return false; @@ -116,4 +124,9 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java index 169a5e2fd66..e52e41031ad 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java @@ -103,11 +103,11 @@ public AbstractNode match(int[] input, int location){ } public AbstractStackNode

getCleanCopy(int startLocation){ - return new CaseInsensitiveLiteralStackNode

(this, startLocation); + return new CaseInsensitiveLiteralStackNode<>(this, startLocation); } public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ - return new CaseInsensitiveLiteralStackNode

(this, startLocation, result); + return new CaseInsensitiveLiteralStackNode<>(this, startLocation, result); } public int getLength(){ @@ -131,6 +131,7 @@ public String toShortString() { return new String(codePoints, 0, codePoints.length); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); for(int i = 0; i < ciLiteral.length; ++i){ @@ -144,10 +145,17 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return production.hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + + @Override public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof CaseInsensitiveLiteralStackNode)) return false; @@ -157,4 +165,9 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java index 8f2bd794944..7ffcddcaf3a 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java @@ -15,7 +15,6 @@ import org.rascalmpl.parser.gtd.result.CharNode; import org.rascalmpl.parser.gtd.stack.filter.ICompletionFilter; import org.rascalmpl.parser.gtd.stack.filter.IEnterFilter; -import org.rascalmpl.unicode.UnicodeConverter; public final class CharStackNode

extends AbstractMatchableStackNode

{ private final int[][] ranges; @@ -72,11 +71,11 @@ public AbstractNode match(int[] input, int location){ } public AbstractStackNode

getCleanCopy(int startLocation){ - return new CharStackNode

(this, startLocation); + return new CharStackNode<>(this, startLocation); } public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ - return new CharStackNode

(this, startLocation, result); + return new CharStackNode<>(this, startLocation, result); } public int getLength(){ @@ -114,6 +113,7 @@ private String codePointToString(int codePoint) { return String.valueOf(codePoint); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); @@ -148,6 +148,11 @@ public int hashCode(){ return hash; } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof CharStackNode)) return false; @@ -168,4 +173,9 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java index 685c839065d..34b7bc4d973 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java @@ -80,10 +80,12 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); @@ -94,13 +96,26 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return 1; } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + + @Override public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof EmptyStackNode)) return false; return hasEqualFilters(stackNode); } + + @Override + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java index dfa16c05fed..2c5f0758a55 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java @@ -54,11 +54,11 @@ public AbstractNode match(int[] input, int location){ } public AbstractStackNode

getCleanCopy(int startLocation){ - return new EpsilonStackNode

(this, startLocation); + return new EpsilonStackNode<>(this, startLocation); } public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ - return new EpsilonStackNode

(this, startLocation, result); + return new EpsilonStackNode<>(this, startLocation, result); } public int getLength(){ @@ -69,6 +69,7 @@ public AbstractNode getResult(){ return result; } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append('('); @@ -78,13 +79,24 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return 0; } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof EpsilonStackNode)) return false; return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java index a513a9472c9..7262820a538 100644 --- a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java @@ -78,7 +78,7 @@ public String getName(){ } public AbstractStackNode

getCleanCopy(int startLocation){ - return new ListStackNode

(this, startLocation); + return new ListStackNode<>(this, startLocation); } public AbstractStackNode

[] getChildren(){ @@ -93,10 +93,12 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); @@ -107,10 +109,16 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return production.hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof ListStackNode)) return false; @@ -120,4 +128,9 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java index 770a84a8a2e..32fc6a86eb8 100644 --- a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java @@ -87,6 +87,7 @@ public String toShortString() { return "'" + UnicodeConverter.unicodeArrayToString(literal) + "'"; } + @Override public String toString(){ StringBuilder sb = new StringBuilder("lit['"); sb.append(UnicodeConverter.unicodeArrayToString(literal)); @@ -98,10 +99,16 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return production.hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof LiteralStackNode)) return false; @@ -111,4 +118,9 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java index 3b0ed831748..b0f3e7f8179 100644 --- a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java @@ -104,10 +104,12 @@ public AbstractNode getResult(){ return result; } + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); @@ -128,12 +130,13 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ int hash = 0; for(int i = characters.length - 1; i >= 0; --i){ int[] chars = characters[i]; - for(int j = chars.length - 1; j <= 0; --j){ + for(int j = chars.length - 1; j >= 0; --j){ hash = hash << 3 + hash >> 5; hash ^= chars[0] + (chars[1] << 2); } @@ -142,6 +145,11 @@ public int hashCode(){ return hash; } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof MultiCharacterStackNode)) return false; @@ -167,4 +175,9 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java index 3114bc97baf..8c87075059c 100644 --- a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java @@ -103,4 +103,9 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java index c4694769941..8782c226fe9 100644 --- a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java @@ -119,4 +119,9 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java index 60daa28bc93..dbe3331121f 100644 --- a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java @@ -116,4 +116,9 @@ public boolean isEqual(AbstractStackNode

stackNode){ return otherNode.name.equals(name) && otherNode.startLocation == startLocation; } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java index 3310d11076b..48ce1e873ca 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java @@ -137,4 +137,9 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java index 6d25c37750e..95393e73b0e 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java @@ -114,4 +114,9 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 5c3ca844518..56ef3a54a30 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -150,4 +150,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return otherNode.id == id; } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } } From 70d04cc8ee0a74a785b1cfd1711213a96e70a7bb Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 11:17:40 +0200 Subject: [PATCH 054/190] INtroduced StackNodeVisitor --- .../parser/gtd/stack/StackNodeVisitor.java | 18 +++++ .../gtd/stack/StackNodeVisitorAdapter.java | 69 +++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java create mode 100644 src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java new file mode 100644 index 00000000000..1dd2620c983 --- /dev/null +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java @@ -0,0 +1,18 @@ +package org.rascalmpl.parser.gtd.stack; + +public interface StackNodeVisitor

{ + void visit(AlternativeStackNode

node); + void visit(CaseInsensitiveLiteralStackNode

node); + void visit(CharStackNode

node); + void visit(EmptyStackNode

node); + void visit(EpsilonStackNode

node); + void visit(ListStackNode

node); + void visit(LiteralStackNode

node); + void visit(MultiCharacterStackNode

node); + void visit(NonTerminalStackNode

node); + void visit(OptionalStackNode

node); + void visit(RecoveryPointStackNode

node); + void visit(SeparatedListStackNode

node); + void visit(SequenceStackNode

node); + void visit(SkippingStackNode

node); +} diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java new file mode 100644 index 00000000000..d4e10e8874f --- /dev/null +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java @@ -0,0 +1,69 @@ +package org.rascalmpl.parser.gtd.stack; + +public class StackNodeVisitorAdapter

implements StackNodeVisitor

{ + + @Override + public void visit(AlternativeStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(CaseInsensitiveLiteralStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(CharStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(EmptyStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(EpsilonStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(ListStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(LiteralStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(MultiCharacterStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(NonTerminalStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(OptionalStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(SeparatedListStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(SequenceStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(SkippingStackNode

node) { + // Do nothing fallback + } +} From defa9aeac9c36ee4deda64edbf0be7cbea7c4785 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 11:20:55 +0200 Subject: [PATCH 055/190] Fixed more warnings --- .../parser/gtd/stack/NonTerminalStackNode.java | 10 +++++++++- .../rascalmpl/parser/gtd/stack/OptionalStackNode.java | 8 ++++++++ .../parser/gtd/stack/RecoveryPointStackNode.java | 7 +++++++ .../parser/gtd/stack/SeparatedListStackNode.java | 8 ++++++++ .../rascalmpl/parser/gtd/stack/SequenceStackNode.java | 10 +++++++++- .../rascalmpl/parser/gtd/stack/SkippingStackNode.java | 7 +------ 6 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java index 8c87075059c..0fafc7e43e6 100644 --- a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java @@ -49,7 +49,7 @@ public AbstractNode match(int[] input, int location){ } public AbstractStackNode

getCleanCopy(int startLocation){ - return new NonTerminalStackNode

(this, startLocation); + return new NonTerminalStackNode<>(this, startLocation); } public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ @@ -76,10 +76,12 @@ public AbstractNode getResult(){ throw new UnsupportedOperationException(); } + @Override public String toShortString() { return expectIdentifier; } + @Override public String toString(){ StringBuilder sb = new StringBuilder("NonTerminal["); sb.append(expectIdentifier); @@ -90,10 +92,16 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return expectIdentifier.hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof NonTerminalStackNode)) return false; diff --git a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java index 8782c226fe9..28ff7500370 100644 --- a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java @@ -92,10 +92,12 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); @@ -106,10 +108,16 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return production.hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof OptionalStackNode)) return false; diff --git a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java index dbe3331121f..d426deae943 100644 --- a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java @@ -93,6 +93,7 @@ public AbstractNode getResult(){ throw new UnsupportedOperationException(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(getName()); @@ -105,10 +106,16 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return getName().hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof RecoveryPointStackNode)) return false; diff --git a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java index 48ce1e873ca..0f9c23e2e17 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java @@ -110,10 +110,12 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); @@ -124,10 +126,16 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return production.hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof SeparatedListStackNode)) return false; diff --git a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java index 95393e73b0e..bb408c81c9a 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java @@ -71,7 +71,7 @@ public String getName(){ } public AbstractStackNode

getCleanCopy(int startLocation){ - return new SequenceStackNode

(this, startLocation); + return new SequenceStackNode<>(this, startLocation); } public AbstractStackNode

[] getChildren(){ @@ -86,10 +86,12 @@ public AbstractStackNode

getEmptyChild(){ throw new UnsupportedOperationException(); } + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append("seq"); @@ -101,10 +103,16 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return production.hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof SequenceStackNode)) return false; diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 56ef3a54a30..35e6957f5ed 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -131,14 +131,9 @@ public int hashCode(){ return getParentProduction().hashCode(); } - @SuppressWarnings("unchecked") @Override public boolean equals(Object rhs) { - if (rhs instanceof AbstractStackNode) { - return isEqual((AbstractStackNode

)rhs); - } - - return false; + return super.equals(rhs); } public boolean isEqual(AbstractStackNode

stackNode){ From a8194e61725f5b763be4f0df819d5c10b93e1181 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 11:24:27 +0200 Subject: [PATCH 056/190] Madde AbstractStackNode.visit public --- src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java | 2 +- src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java | 3 ++- .../parser/gtd/stack/CaseInsensitiveLiteralStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/CharStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java | 2 +- src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/ListStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java | 3 ++- .../rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java | 3 ++- .../rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java | 3 ++- .../rascalmpl/parser/gtd/stack/SeparatedListStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java | 3 ++- .../rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java | 5 +++++ 16 files changed, 33 insertions(+), 15 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index f156e70fab7..e6f24555ae2 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -811,7 +811,7 @@ public String toString() { return builder.toString(); } - abstract void accept(StackNodeVisitor

visitor); + public abstract void accept(StackNodeVisitor

visitor); // Matchables. /** diff --git a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java index a16aa931335..2347322e8c2 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java @@ -125,7 +125,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java index e52e41031ad..12b61d087f1 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java @@ -166,7 +166,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java index 7ffcddcaf3a..2968bcd27c3 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java @@ -174,7 +174,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java index 34b7bc4d973..8accaaa2dbe 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java @@ -114,7 +114,7 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - void accept(StackNodeVisitor

visitor) { + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java index 2c5f0758a55..e0c92afef9e 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java @@ -95,7 +95,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java index 7262820a538..d4c8384d369 100644 --- a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java @@ -129,7 +129,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java index 32fc6a86eb8..578f9651b14 100644 --- a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java @@ -119,7 +119,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java index b0f3e7f8179..f1cb484ab3c 100644 --- a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java @@ -176,7 +176,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java index 0fafc7e43e6..8cb886190e2 100644 --- a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java @@ -112,7 +112,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java index 28ff7500370..d2c318c784a 100644 --- a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java @@ -128,7 +128,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java index d426deae943..7cedde9b78a 100644 --- a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java @@ -124,7 +124,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return otherNode.name.equals(name) && otherNode.startLocation == startLocation; } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java index 0f9c23e2e17..2bac1eae426 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java @@ -146,7 +146,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java index bb408c81c9a..877eacaa76a 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java @@ -123,7 +123,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 35e6957f5ed..10136783df0 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -146,7 +146,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return otherNode.id == id; } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java index d4e10e8874f..73d14ad3da7 100644 --- a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java @@ -52,6 +52,11 @@ public void visit(OptionalStackNode

node) { // Do nothing fallback } + @Override + public void visit(RecoveryPointStackNode

node) { + // Do nothing fallback + } + @Override public void visit(SeparatedListStackNode

node) { // Do nothing fallback From 6feaf4da327ce5f59a6ae5ad618276ad45f6846c Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 11:35:07 +0200 Subject: [PATCH 057/190] Use visitor pattern instead of instanceof in ToTokenRecoverer --- .../uptr/recovery/ToTokenRecoverer.java | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index fd028a1dc30..088ad252d46 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -30,6 +30,7 @@ import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; import org.rascalmpl.parser.gtd.stack.SkippingStackNode; +import org.rascalmpl.parser.gtd.stack.StackNodeVisitorAdapter; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.util.ArrayList; import org.rascalmpl.parser.gtd.util.DoubleArrayList; @@ -194,25 +195,26 @@ void addEndMatchers(AbstractStackNode[] prod, int dot, List lastLiteral = (LiteralStackNode) last; - matchers.add(new LiteralMatcher(lastLiteral.getLiteral())); + last.accept(new StackNodeVisitorAdapter() { + @Override + public void visit(LiteralStackNode literal) { + matchers.add(new LiteralMatcher(literal.getLiteral())); } - if (last instanceof CaseInsensitiveLiteralStackNode) { - CaseInsensitiveLiteralStackNode lastLiteral = (CaseInsensitiveLiteralStackNode) last; - matchers.add(new CaseInsensitiveLiteralMatcher(lastLiteral.getLiteral())); + @Override + public void visit(CaseInsensitiveLiteralStackNode literal) { + matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); } - if (last instanceof NonTerminalStackNode) { - NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) last; - String name = nextNonTerminal.getName(); + @Override + public void visit(NonTerminalStackNode nonTerminal) { + String name = nonTerminal.getName(); AbstractStackNode[] alternatives = expectsProvider.getExpects(name); for (AbstractStackNode alternative : alternatives) { - // In the future we might want to use all prefix-sharing alternatives here addEndMatchers(alternative.getProduction(), 0, matchers, visitedNodes); } } + }); } void forAllParents(AbstractStackNode stackNode, Consumer> consumer) { @@ -281,24 +283,26 @@ private void addNextMatchers(AbstractStackNode[] prod, int dot, Li } visitedNodes.add(next.getId()); - if (next instanceof LiteralStackNode) { - LiteralStackNode nextLiteral = (LiteralStackNode) next; - matchers.add(new LiteralMatcher(nextLiteral.getLiteral())); + next.accept(new StackNodeVisitorAdapter() { + @Override + public void visit(LiteralStackNode literal) { + matchers.add(new LiteralMatcher(literal.getLiteral())); } - if (next instanceof CaseInsensitiveLiteralStackNode) { - CaseInsensitiveLiteralStackNode nextLiteral = (CaseInsensitiveLiteralStackNode) next; - matchers.add(new CaseInsensitiveLiteralMatcher(nextLiteral.getLiteral())); + @Override + public void visit(CaseInsensitiveLiteralStackNode literal) { + matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); } - if (next instanceof NonTerminalStackNode) { - NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) next; - String name = nextNonTerminal.getName(); + @Override + public void visit(NonTerminalStackNode nonTerminal) { + String name = nonTerminal.getName(); AbstractStackNode[] alternatives = expectsProvider.getExpects(name); for (AbstractStackNode alternative : alternatives) { addNextMatchers(alternative.getProduction(), 0, matchers, visitedNodes); } } + }); } // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) From b95e14ed100a037e4b0546ed5a8d917fecc573f1 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 11:49:34 +0200 Subject: [PATCH 058/190] Renamed DebugVisualizer to ParseStateVisualizer --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 28 +++++++------- .../gtd/stack/RecoveryPointStackNode.java | 2 +- .../uptr/recovery/ToTokenRecoverer.java | 5 +-- ...ualizer.java => ParseStateVisualizer.java} | 38 +++++++++---------- 4 files changed, 34 insertions(+), 39 deletions(-) rename src/org/rascalmpl/util/visualize/{DebugVisualizer.java => ParseStateVisualizer.java} (97%) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 7f8f8e540b1..fbce93e0154 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -47,7 +47,7 @@ import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.Stack; import org.rascalmpl.parser.util.DebugUtil; -import org.rascalmpl.util.visualize.DebugVisualizer; +import org.rascalmpl.util.visualize.ParseStateVisualizer; import org.rascalmpl.util.visualize.dot.NodeId; import org.rascalmpl.values.RascalValueFactory; @@ -133,7 +133,7 @@ public abstract class SGTDBF implements IGTD { // Debugging private IDebugListener

debugListener; - private DebugVisualizer visualizer; + private ParseStateVisualizer visualizer; // Temporary instrumentation for accurate profiling private long timestamp; @@ -808,7 +808,7 @@ private void move(AbstractStackNode

node, AbstractNode result){ */ private void reduceTerminals() { // Reduce terminals - visualize("Reducing terminals", DebugVisualizer.TERMINALS_TO_REDUCE_ID); + visualize("Reducing terminals", ParseStateVisualizer.TERMINALS_TO_REDUCE_ID); while(!stacksWithTerminalsToReduce.isEmpty()){ move(stacksWithTerminalsToReduce.peekFirst(), stacksWithTerminalsToReduce.popSecond()); } @@ -816,7 +816,7 @@ private void reduceTerminals() { private void reduceNonTerminals() { // Reduce non-terminals - visualize("Reducing non-terminals", DebugVisualizer.NON_TERMINALS_TO_REDUCE_ID); + visualize("Reducing non-terminals", ParseStateVisualizer.NON_TERMINALS_TO_REDUCE_ID); while(!stacksWithNonTerminalsToReduce.isEmpty()){ move(stacksWithNonTerminalsToReduce.peekFirst(), stacksWithNonTerminalsToReduce.popSecond()); } @@ -843,7 +843,7 @@ private boolean findFirstStacksToReduce(){ if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); } - visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); + visualize("Recovering", ParseStateVisualizer.ERROR_TRACKING_ID); DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); @@ -866,14 +866,14 @@ private boolean findFirstStacksToReduce(){ * Locates the set of stacks that is queued for handling, for which the least amount of characters needs to be shifted. */ private boolean findStacksToReduce(){ - visualize("Finding stacks to reduce", DebugVisualizer.TODO_LISTS_ID); + visualize("Finding stacks to reduce", ParseStateVisualizer.TODO_LISTS_ID); int queueDepth = todoLists.length; for(int i = 1; i < queueDepth-1; ++i){ queueIndex = (queueIndex + 1) % queueDepth; DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; if(!(terminalsTodo == null || terminalsTodo.isEmpty())){ - if (DebugVisualizer.VISUALIZATION_ENABLED) { + if (ParseStateVisualizer.VISUALIZATION_ENABLED) { NodeId reduceNodeId = new NodeId("todo-" + i); visualize("Found stack to reduce", reduceNodeId); } @@ -890,12 +890,12 @@ private boolean findStacksToReduce(){ if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); } - visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); + visualize("Recovering", ParseStateVisualizer.ERROR_TRACKING_ID); DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); } - if (DebugVisualizer.VISUALIZATION_ENABLED && visualizer != null) { + if (ParseStateVisualizer.VISUALIZATION_ENABLED && visualizer != null) { // Visualize state and include recovered nodes visualizer.createGraph(this, "Reviving"); visualizer.addRecoveredNodes(recoveredNodes); @@ -913,7 +913,7 @@ private boolean findStacksToReduce(){ if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); } - visualize("Queue recovery node", DebugVisualizer.getNodeId(recovered)); + visualize("Queue recovery node", ParseStateVisualizer.getNodeId(recovered)); queueRecoveryNode(recovered, recovered.getStartLocation(), recovered.getLength(), recoveredNodes.getSecond(i)); } return findStacksToReduce(); @@ -1235,7 +1235,7 @@ private void expandStack(AbstractStackNode

stack){ * Initiate stack expansion for all queued stacks. */ private void expand(){ - visualize("Expanding", DebugVisualizer.STACKS_TO_EXPAND_ID); + visualize("Expanding", ParseStateVisualizer.STACKS_TO_EXPAND_ID); while(!stacksToExpand.isEmpty()){ expandStack(stacksToExpand.pop()); } @@ -1269,7 +1269,7 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] this.debugListener = debugListener; String query = inputURI.getQuery(); - visualizer = query != null && query.contains("visualize=true") ? new DebugVisualizer("Parser") : null; + visualizer = query != null && query.contains("visualize=true") ? new ParseStateVisualizer("Parser") : null; // Initialzed the position store. positionStore.index(input); @@ -1320,7 +1320,7 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] } while(findStacksToReduce()); } - visualize("Done", DebugVisualizer.PARSER_ID); + visualize("Done", ParseStateVisualizer.PARSER_ID); // Check if we were successful. if(location == input.length){ @@ -1569,7 +1569,7 @@ private IConstructor fixErrorNodes(IConstructor tree, INodeConstructorFactory, AbstractNode> reviveNod // Sort nodes by start location recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); - DebugVisualizer visualizer = new DebugVisualizer("Recovery"); + ParseStateVisualizer visualizer = new ParseStateVisualizer("Recovery"); visualizer.visualizeRecoveryNodes(recoveryNodes); for (int i = 0; i, Ar addProductionArray(graph, productionsId, recoveryNodes.getSecond(i)); graph.addEdge(new NodeId(pairId, "productions"), productionsId); } - + return graph; } @@ -248,7 +249,7 @@ private

DotNode addStack(DotGraph graph, AbstractStackNode

stackNode) { if (node != null) { return node; } - + node = createDotNode(stackNode); stackNodeNodes.put(stackNode.getId(), node); @@ -284,19 +285,14 @@ private

DotNode createDotNode(AbstractStackNode

stackNode) { String nodeName; - if (stackNode instanceof RecoveryPointStackNode) { - RecoveryPointStackNode

recoveryNode = (RecoveryPointStackNode

) stackNode; - nodeName = String.valueOf(recoveryNode.getId()); - } else { - try { - nodeName = stackNode.getName(); - } catch (UnsupportedOperationException e) { - nodeName = ""; - } + try { + nodeName = stackNode.getName(); + } catch (UnsupportedOperationException e) { + nodeName = ""; + } - if (nodeName.startsWith("layouts_")) { - nodeName = nodeName.substring("layouts_".length()); - } + if (nodeName.startsWith("layouts_")) { + nodeName = nodeName.substring("layouts_".length()); } int dot = stackNode.getDot(); @@ -319,7 +315,7 @@ private

DotNode createDotNode(AbstractStackNode

stackNode) { } DotNode node = new DotNode(getNodeId(stackNode)); - String label = String.format("%s: %s\n.%d@%d %s", + String label = String.format("%s: %s\n.%d@%d %s", type, nodeName, dot, stackNode.getStartLocation(), extraInfo); String shortString = stackNode.toShortString(); @@ -384,7 +380,7 @@ private void enrichLiteralNode(DotNode dotNode, LiteralNode literalNode) { String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); int[] content = literalNode.getContent(); label += " \"" + UnicodeConverter.unicodeArrayToString(content) + "\""; - /* Maybe include production? + /* Maybe include production? label += "\nprod=" + DebugUtil.prodToString((IConstructor)literalNode.getProduction()); */ dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); @@ -403,7 +399,7 @@ private void enrichSortContainerNode(DotNode dotNode, SortContainerNode void createGraph(SGTDBF parser, String step) { lookahead = '$'; } - String label = String.format("Parser\nInput: \"%s\"\nLocation: %d ('%c')\nStep %d: %s", + String label = String.format("Parser\nInput: \"%s\"\nLocation: %d ('%c')\nStep %d: %s", input, location, lookahead, frame, step); parserNode.setAttribute(DotAttribute.ATTR_LABEL, label); graph.addNode(parserNode); @@ -538,7 +534,7 @@ private void addUnmatchableMidProductionNodes(SGTDBF parser, graph.addEdge(new NodeId(failureId, "predecessors"), predecessorsId); } } - + private void addFilteredNodes(SGTDBF parser, DotGraph graph) { addStackAndNodeDoubleStack(graph, FILTERED_NODES_ID, parser.getFilteredNodes()); } From f185e70f56d1ad6fc4317e426114d588f94fa03e Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 12:11:18 +0200 Subject: [PATCH 059/190] Generalized AbstractStackNode visitor to be able to return values --- .../parser/gtd/stack/AbstractStackNode.java | 2 +- .../gtd/stack/AlternativeStackNode.java | 4 +- .../CaseInsensitiveLiteralStackNode.java | 4 +- .../parser/gtd/stack/CharStackNode.java | 4 +- .../parser/gtd/stack/EmptyStackNode.java | 4 +- .../parser/gtd/stack/EpsilonStackNode.java | 4 +- .../parser/gtd/stack/ListStackNode.java | 4 +- .../parser/gtd/stack/LiteralStackNode.java | 4 +- .../gtd/stack/MultiCharacterStackNode.java | 4 +- .../gtd/stack/NonTerminalStackNode.java | 4 +- .../parser/gtd/stack/OptionalStackNode.java | 4 +- .../gtd/stack/RecoveryPointStackNode.java | 4 +- .../gtd/stack/SeparatedListStackNode.java | 4 +- .../parser/gtd/stack/SequenceStackNode.java | 4 +- .../parser/gtd/stack/SkippingStackNode.java | 4 +- .../parser/gtd/stack/StackNodeVisitor.java | 30 ++++++------- .../gtd/stack/StackNodeVisitorAdapter.java | 44 ++++++++++++------- .../parser/uptr/recovery/InputMatcher.java | 15 ++++--- .../uptr/recovery/ToTokenRecoverer.java | 22 ++++++---- 19 files changed, 96 insertions(+), 73 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index e6f24555ae2..3b25784f36d 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -811,7 +811,7 @@ public String toString() { return builder.toString(); } - public abstract void accept(StackNodeVisitor

visitor); + public abstract R accept(StackNodeVisitor visitor); // Matchables. /** diff --git a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java index 2347322e8c2..c4943a1d663 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java @@ -126,8 +126,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java index 12b61d087f1..9f62087c1ab 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java @@ -167,8 +167,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java index 2968bcd27c3..de4a8b25198 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java @@ -175,8 +175,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java index 8accaaa2dbe..65def9c7346 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java @@ -114,8 +114,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java index e0c92afef9e..b641a4ed413 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java @@ -96,8 +96,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java index d4c8384d369..48402479888 100644 --- a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java @@ -130,8 +130,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java index 578f9651b14..b51342d730d 100644 --- a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java @@ -120,8 +120,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java index f1cb484ab3c..768672aac7f 100644 --- a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java @@ -177,8 +177,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java index 8cb886190e2..3d971f8112f 100644 --- a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java @@ -113,8 +113,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java index d2c318c784a..772a7d7e632 100644 --- a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java @@ -129,8 +129,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java index 0b3104f9aee..8188887f4e7 100644 --- a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java @@ -125,8 +125,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java index 2bac1eae426..db5c658ec14 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java @@ -147,8 +147,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java index 877eacaa76a..6d50af066a4 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java @@ -124,8 +124,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 10136783df0..0409c5b4e03 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -147,7 +147,7 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java index 1dd2620c983..1171c62fd0b 100644 --- a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java @@ -1,18 +1,18 @@ package org.rascalmpl.parser.gtd.stack; -public interface StackNodeVisitor

{ - void visit(AlternativeStackNode

node); - void visit(CaseInsensitiveLiteralStackNode

node); - void visit(CharStackNode

node); - void visit(EmptyStackNode

node); - void visit(EpsilonStackNode

node); - void visit(ListStackNode

node); - void visit(LiteralStackNode

node); - void visit(MultiCharacterStackNode

node); - void visit(NonTerminalStackNode

node); - void visit(OptionalStackNode

node); - void visit(RecoveryPointStackNode

node); - void visit(SeparatedListStackNode

node); - void visit(SequenceStackNode

node); - void visit(SkippingStackNode

node); +public interface StackNodeVisitor { + R visit(AlternativeStackNode

node); + R visit(CaseInsensitiveLiteralStackNode

node); + R visit(CharStackNode

node); + R visit(EmptyStackNode

node); + R visit(EpsilonStackNode

node); + R visit(ListStackNode

node); + R visit(LiteralStackNode

node); + R visit(MultiCharacterStackNode

node); + R visit(NonTerminalStackNode

node); + R visit(OptionalStackNode

node); + R visit(RecoveryPointStackNode

node); + R visit(SeparatedListStackNode

node); + R visit(SequenceStackNode

node); + R visit(SkippingStackNode

node); } diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java index 73d14ad3da7..bcd6dd2fec9 100644 --- a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java @@ -1,74 +1,88 @@ package org.rascalmpl.parser.gtd.stack; -public class StackNodeVisitorAdapter

implements StackNodeVisitor

{ +public class StackNodeVisitorAdapter implements StackNodeVisitor { @Override - public void visit(AlternativeStackNode

node) { + public R visit(AlternativeStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(CaseInsensitiveLiteralStackNode

node) { + public R visit(CaseInsensitiveLiteralStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(CharStackNode

node) { + public R visit(CharStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(EmptyStackNode

node) { + public R visit(EmptyStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(EpsilonStackNode

node) { + public R visit(EpsilonStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(ListStackNode

node) { + public R visit(ListStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(LiteralStackNode

node) { + public R visit(LiteralStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(MultiCharacterStackNode

node) { + public R visit(MultiCharacterStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(NonTerminalStackNode

node) { + public R visit(NonTerminalStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(OptionalStackNode

node) { + public R visit(OptionalStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(RecoveryPointStackNode

node) { + public R visit(RecoveryPointStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(SeparatedListStackNode

node) { + public R visit(SeparatedListStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(SequenceStackNode

node) { + public R visit(SequenceStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(SkippingStackNode

node) { + public R visit(SkippingStackNode

node) { // Do nothing fallback + return null; } } diff --git a/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java index 614df9acd32..48f5b869a0c 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java @@ -3,6 +3,7 @@ import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.CaseInsensitiveLiteralStackNode; import org.rascalmpl.parser.gtd.stack.LiteralStackNode; +import org.rascalmpl.parser.gtd.stack.StackNodeVisitorAdapter; import org.rascalmpl.values.RascalValueFactory; import io.usethesource.vallang.IConstructor; @@ -53,14 +54,16 @@ public static InputMatcher createMatcher(IConstructor constructor) { } public static

InputMatcher createMatcher(AbstractStackNode

stackNode) { - if (stackNode instanceof LiteralStackNode) { - return new LiteralMatcher(((LiteralStackNode

) stackNode).getLiteral()); + return stackNode.accept(new StackNodeVisitorAdapter() { + @Override + public InputMatcher visit(LiteralStackNode

literal) { + return new LiteralMatcher(literal.getLiteral()); } - if (stackNode instanceof CaseInsensitiveLiteralStackNode) { - return new CaseInsensitiveLiteralMatcher(((CaseInsensitiveLiteralStackNode

) stackNode).getLiteral()); + @Override + public InputMatcher visit(CaseInsensitiveLiteralStackNode

literal) { + return new CaseInsensitiveLiteralMatcher(literal.getLiteral()); } - - return null; + }); } } diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index f1f83e2c794..0c66f239929 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -194,24 +194,27 @@ void addEndMatchers(AbstractStackNode[] prod, int dot, List() { + last.accept(new StackNodeVisitorAdapter() { @Override - public void visit(LiteralStackNode literal) { + public Void visit(LiteralStackNode literal) { matchers.add(new LiteralMatcher(literal.getLiteral())); + return null; } @Override - public void visit(CaseInsensitiveLiteralStackNode literal) { + public Void visit(CaseInsensitiveLiteralStackNode literal) { matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); + return null; } @Override - public void visit(NonTerminalStackNode nonTerminal) { + public Void visit(NonTerminalStackNode nonTerminal) { String name = nonTerminal.getName(); AbstractStackNode[] alternatives = expectsProvider.getExpects(name); for (AbstractStackNode alternative : alternatives) { addEndMatchers(alternative.getProduction(), 0, matchers, visitedNodes); } + return null; } }); } @@ -282,24 +285,27 @@ private void addNextMatchers(AbstractStackNode[] prod, int dot, Li } visitedNodes.add(next.getId()); - next.accept(new StackNodeVisitorAdapter() { + next.accept(new StackNodeVisitorAdapter() { @Override - public void visit(LiteralStackNode literal) { + public Void visit(LiteralStackNode literal) { matchers.add(new LiteralMatcher(literal.getLiteral())); + return null; } @Override - public void visit(CaseInsensitiveLiteralStackNode literal) { + public Void visit(CaseInsensitiveLiteralStackNode literal) { matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); + return null; } @Override - public void visit(NonTerminalStackNode nonTerminal) { + public Void visit(NonTerminalStackNode nonTerminal) { String name = nonTerminal.getName(); AbstractStackNode[] alternatives = expectsProvider.getExpects(name); for (AbstractStackNode alternative : alternatives) { addNextMatchers(alternative.getProduction(), 0, matchers, visitedNodes); } + return null; } }); } From 4c9664fd4da334280841930df7568af9e0c07525 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 12:12:48 +0200 Subject: [PATCH 060/190] Removed debug print --- .../library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index 287ad093e71..ae67890bd18 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -25,9 +25,6 @@ test bool abx() { test bool axc() { Tree t = parseS("a x c $", visualize=true); - iprintln(t); - println("after disambiguation:"); - iprintln(defaultErrorDisambiguationFilter(t)); return getErrorText(findFirstError(t)) == "x c"; } From 154a0b8d3153e6c8fbaf7dde65478f32c6fd95e9 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 13:29:23 +0200 Subject: [PATCH 061/190] Added license header to files introduced for error recovery --- .../tests/recovery/BasicRecoveryTests.rsc | 15 ++++++++++++++- .../tests/recovery/ListRecoveryTests.rsc | 14 ++++++++++++++ .../tests/recovery/NestedRecoveryTests.rsc | 14 ++++++++++++++ .../tests/recovery/PicoRecoveryTests.rsc | 14 ++++++++++++++ .../tests/recovery/RascalRecoveryTests.rsc | 14 ++++++++++++++ .../lang/rascal/tests/recovery/Test.rsc | 1 - .../lang/rascal/tests/recovery/ToyRascal.rsc | 14 ++++++++++++++ .../tests/recovery/ToyRascalRecoveryTests.rsc | 14 ++++++++++++++ .../rascalmpl/parser/gtd/ExpectsProvider.java | 14 ++++++++++++++ .../parser/gtd/stack/StackNodeVisitor.java | 14 ++++++++++++++ .../gtd/stack/StackNodeVisitorAdapter.java | 14 ++++++++++++++ .../rascalmpl/parser/gtd/util/IdDispenser.java | 14 ++++++++++++++ .../parser/gtd/util/StackNodeIdDispenser.java | 14 ++++++++++++++ .../CaseInsensitiveLiteralMatcher.java | 14 ++++++++++++++ .../parser/uptr/recovery/FailingMatcher.java | 14 ++++++++++++++ .../parser/uptr/recovery/InputMatcher.java | 14 ++++++++++++++ .../parser/uptr/recovery/LiteralMatcher.java | 14 ++++++++++++++ src/org/rascalmpl/parser/util/DebugUtil.java | 18 ++++++++++++++++++ .../util/visualize/ParseStateVisualizer.java | 16 ++++++++++++++-- src/org/rascalmpl/util/visualize/replay.html | 12 ++++++++++++ .../test/recovery/ErrorRecoveryModules.java | 14 ++++++++++++++ 21 files changed, 282 insertions(+), 4 deletions(-) delete mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index ae67890bd18..385eab384f1 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -1,7 +1,20 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::BasicRecoveryTests import ParseTree; -import IO; layout Layout = [\ ]* !>> [\ ]; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc index 184fd2b2eb8..1ef55f7172d 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::ListRecoveryTests import ParseTree; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc index f335dcd3b12..40359414817 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::NestedRecoveryTests import ParseTree; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc index 8634cca3e29..0cf2472ef94 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::PicoRecoveryTests import lang::pico::\syntax::Main; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc index 7c17abd899d..82e94caedae 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::RascalRecoveryTests import lang::rascal::\syntax::Rascal; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc deleted file mode 100644 index 455e3661677..00000000000 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc +++ /dev/null @@ -1 +0,0 @@ -module A void f(){ void g() {}} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc index 81a9fd1c5b2..db9b28ac98a 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::ToyRascal start syntax FunctionDeclaration = Signature FunctionBody; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc index 455197c1f35..d3900bda115 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::ToyRascalRecoveryTests import lang::rascal::tests::recovery::ToyRascal; diff --git a/src/org/rascalmpl/parser/gtd/ExpectsProvider.java b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java index 18266a7d10d..5233abc0928 100644 --- a/src/org/rascalmpl/parser/gtd/ExpectsProvider.java +++ b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.parser.gtd; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java index 1171c62fd0b..e842ef3ab27 100644 --- a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.parser.gtd.stack; public interface StackNodeVisitor { diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java index bcd6dd2fec9..3cee5991caa 100644 --- a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.parser.gtd.stack; public class StackNodeVisitorAdapter implements StackNodeVisitor { diff --git a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java index fe9aa091406..572861a5455 100644 --- a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java +++ b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.parser.gtd.util; public interface IdDispenser { diff --git a/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java index 403bfbe9247..e3bf7204af5 100644 --- a/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java +++ b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.parser.gtd.util; import java.lang.reflect.InvocationTargetException; diff --git a/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java index 587bd419748..b943b87792b 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.parser.uptr.recovery; public class CaseInsensitiveLiteralMatcher implements InputMatcher { diff --git a/src/org/rascalmpl/parser/uptr/recovery/FailingMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/FailingMatcher.java index 1e4eb6b8b95..f69dac24ad8 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/FailingMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/FailingMatcher.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.parser.uptr.recovery; public class FailingMatcher implements InputMatcher { diff --git a/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java index 48f5b869a0c..11ea11a242e 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.parser.uptr.recovery; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; diff --git a/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java index b5531cff372..2a5b2d3a188 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.parser.uptr.recovery; public class LiteralMatcher implements InputMatcher { diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index e1c29c6b084..680a40046f0 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.parser.util; import io.usethesource.vallang.IConstructor; @@ -8,6 +22,10 @@ public class DebugUtil { /** * Turn a production IConstructor into a string of the form "S -> E1 E2 ..." */ + + private DebugUtil() { + } + public static String prodToString(IConstructor prod) { StringBuilder builder = new StringBuilder("'"); diff --git a/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java b/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java index a9040131dba..7086cd1654e 100644 --- a/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java +++ b/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.util.visualize; import java.io.BufferedReader; @@ -28,8 +42,6 @@ import org.rascalmpl.parser.gtd.result.SkippedNode; import org.rascalmpl.parser.gtd.result.SortContainerNode; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; -import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; -import org.rascalmpl.parser.gtd.stack.StackNodeVisitorAdapter; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.util.ArrayList; import org.rascalmpl.parser.gtd.util.DoubleArrayList; diff --git a/src/org/rascalmpl/util/visualize/replay.html b/src/org/rascalmpl/util/visualize/replay.html index ff8c16daeed..409874049aa 100644 --- a/src/org/rascalmpl/util/visualize/replay.html +++ b/src/org/rascalmpl/util/visualize/replay.html @@ -1,3 +1,15 @@ + diff --git a/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java b/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java index abf446cf843..a3ee64e99db 100644 --- a/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java +++ b/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.test.recovery; import org.junit.runner.RunWith; From 6e19cd936403c044e4d31bd124d85cd991070842 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 13:49:55 +0200 Subject: [PATCH 062/190] Added license header to more files and fixed years. --- .../rascal/tests/recovery/BasicRecoveryTests.rsc | 2 +- .../rascal/tests/recovery/ListRecoveryTests.rsc | 2 +- .../rascal/tests/recovery/NestedRecoveryTests.rsc | 2 +- .../rascal/tests/recovery/PicoRecoveryTests.rsc | 2 +- .../rascal/tests/recovery/RascalRecoveryTests.rsc | 2 +- .../lang/rascal/tests/recovery/ToyRascal.rsc | 2 +- .../tests/recovery/ToyRascalRecoveryTests.rsc | 2 +- src/org/rascalmpl/library/util/TermREPL.java | 13 +++++++++++++ src/org/rascalmpl/parser/gtd/ExpectsProvider.java | 2 +- .../parser/gtd/debug/IDebugListener.java | 2 +- .../gtd/result/out/SkippedNodeFlattener.java | 2 +- .../parser/gtd/stack/StackNodeVisitor.java | 2 +- .../parser/gtd/util/StackNodeIdDispenser.java | 2 +- .../parser/uptr/recovery/InputMatcher.java | 2 +- src/org/rascalmpl/parser/util/DebugUtil.java | 2 +- .../util/visualize/ParseStateVisualizer.java | 2 +- .../util/visualize/dot/AttributeStatement.java | 13 +++++++++++++ .../util/visualize/dot/CompassPoint.java | 14 ++++++++++++++ .../util/visualize/dot/DotAttribute.java | 14 ++++++++++++++ src/org/rascalmpl/util/visualize/dot/DotEdge.java | 14 ++++++++++++++ .../rascalmpl/util/visualize/dot/DotField.java | 14 ++++++++++++++ .../rascalmpl/util/visualize/dot/DotGraph.java | 14 ++++++++++++++ src/org/rascalmpl/util/visualize/dot/DotNode.java | 14 ++++++++++++++ .../rascalmpl/util/visualize/dot/DotRecord.java | 14 ++++++++++++++ .../util/visualize/dot/DotRecordEntry.java | 14 ++++++++++++++ .../util/visualize/dot/DotStatement.java | 14 ++++++++++++++ .../rascalmpl/util/visualize/dot/DotSubgraph.java | 14 ++++++++++++++ src/org/rascalmpl/util/visualize/dot/NodeId.java | 14 ++++++++++++++ src/org/rascalmpl/util/visualize/replay.html | 2 +- test/org/rascalmpl/test/parser/RecoveryTests.java | 14 ++++++++++++++ test/org/rascalmpl/test/parser/StackNodeTest.java | 15 ++++++++++++++- 31 files changed, 224 insertions(+), 17 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index 385eab384f1..7067e1d048e 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc index 1ef55f7172d..9e4ca7e12f2 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc index 40359414817..47f93523051 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc index 0cf2472ef94..0be49d061f6 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc index 82e94caedae..ccc2918d1d8 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc index db9b28ac98a..35c466c90c4 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc index d3900bda115..7e56e3369a4 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/util/TermREPL.java b/src/org/rascalmpl/library/util/TermREPL.java index f149bb36340..59d3177aaf7 100644 --- a/src/org/rascalmpl/library/util/TermREPL.java +++ b/src/org/rascalmpl/library/util/TermREPL.java @@ -1,3 +1,16 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ package org.rascalmpl.library.util; import java.io.ByteArrayInputStream; diff --git a/src/org/rascalmpl/parser/gtd/ExpectsProvider.java b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java index 5233abc0928..fbbd7970c4e 100644 --- a/src/org/rascalmpl/parser/gtd/ExpectsProvider.java +++ b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java b/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java index fc6a8a2bec3..3a8f18a07c7 100644 --- a/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java +++ b/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2012-2013 CWI + * Copyright (c) 2012-2024 CWI * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at diff --git a/src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java b/src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java index d0b5877e7c8..72af911f3e0 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java +++ b/src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2009-2013 CWI + * Copyright (c) 2009-2024 CWI * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java index e842ef3ab27..fbe9b7939d8 100644 --- a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java index e3bf7204af5..944da1d99a4 100644 --- a/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java +++ b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java index 11ea11a242e..659ccb9d25c 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index 680a40046f0..9b7049efec2 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java b/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java index 7086cd1654e..faf6e762a0b 100644 --- a/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java +++ b/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java b/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java index 87d745a5e5c..34cd55e08ab 100644 --- a/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java +++ b/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java @@ -1,3 +1,16 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/CompassPoint.java b/src/org/rascalmpl/util/visualize/dot/CompassPoint.java index 3b31af24fd1..06b85c814ab 100644 --- a/src/org/rascalmpl/util/visualize/dot/CompassPoint.java +++ b/src/org/rascalmpl/util/visualize/dot/CompassPoint.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.util.visualize.dot; public enum CompassPoint { diff --git a/src/org/rascalmpl/util/visualize/dot/DotAttribute.java b/src/org/rascalmpl/util/visualize/dot/DotAttribute.java index 761218c6e29..b9420fa598b 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotAttribute.java +++ b/src/org/rascalmpl/util/visualize/dot/DotAttribute.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/DotEdge.java b/src/org/rascalmpl/util/visualize/dot/DotEdge.java index 7faf1ee1259..d8fb169e29b 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotEdge.java +++ b/src/org/rascalmpl/util/visualize/dot/DotEdge.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/DotField.java b/src/org/rascalmpl/util/visualize/dot/DotField.java index 70a25aad7b8..c9c947a5503 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotField.java +++ b/src/org/rascalmpl/util/visualize/dot/DotField.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/DotGraph.java b/src/org/rascalmpl/util/visualize/dot/DotGraph.java index dc11db9202d..1553491c368 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotGraph.java +++ b/src/org/rascalmpl/util/visualize/dot/DotGraph.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/DotNode.java b/src/org/rascalmpl/util/visualize/dot/DotNode.java index 118d1c00c96..027c404e4a6 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotNode.java +++ b/src/org/rascalmpl/util/visualize/dot/DotNode.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/DotRecord.java b/src/org/rascalmpl/util/visualize/dot/DotRecord.java index 73a5cdd73b8..f4ef5045c6b 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotRecord.java +++ b/src/org/rascalmpl/util/visualize/dot/DotRecord.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java b/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java index a0ed741e9f0..318485024a6 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java +++ b/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/DotStatement.java b/src/org/rascalmpl/util/visualize/dot/DotStatement.java index 83919814059..a27780d2aba 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotStatement.java +++ b/src/org/rascalmpl/util/visualize/dot/DotStatement.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java b/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java index 23e7fb6b6a6..fa09078eb9d 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java +++ b/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/NodeId.java b/src/org/rascalmpl/util/visualize/dot/NodeId.java index fe2c3849ea3..c3f4ec711db 100644 --- a/src/org/rascalmpl/util/visualize/dot/NodeId.java +++ b/src/org/rascalmpl/util/visualize/dot/NodeId.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/replay.html b/src/org/rascalmpl/util/visualize/replay.html index 409874049aa..891778454b5 100644 --- a/src/org/rascalmpl/util/visualize/replay.html +++ b/src/org/rascalmpl/util/visualize/replay.html @@ -1,5 +1,5 @@ From 36752014c13f4dc52af002b231b6db2329a830c4 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Fri, 6 Sep 2024 11:21:59 +0200 Subject: [PATCH 077/190] Various changes based on an excellent code review by Sung. --- src/org/rascalmpl/library/ParseTree.rsc | 29 +- .../tests/recovery/BasicRecoveryTests.rsc | 8 +- .../tests/recovery/RascalRecoveryTests.rsc | 3 +- src/org/rascalmpl/parser/gtd/SGTDBF.java | 653 ++++++++++++------ .../parser/gtd/stack/AbstractStackNode.java | 8 +- .../parser/gtd/stack/EpsilonStackNode.java | 5 + .../gtd/stack/RecoveryPointStackNode.java | 5 + .../parser/gtd/stack/SkippingStackNode.java | 18 - .../gtd/stack/StackNodeVisitorAdapter.java | 2 +- .../parser/gtd/util/DoubleArrayList.java | 15 +- .../parser/gtd/util/IdDispenser.java | 2 +- .../parser/uptr/UPTRNodeFactory.java | 19 - .../CaseInsensitiveLiteralMatcher.java | 2 +- .../parser/uptr/recovery/InputMatcher.java | 5 +- .../recovery/ToNextWhitespaceRecoverer.java | 245 ------- .../uptr/recovery/ToTokenRecoverer.java | 164 +++-- src/org/rascalmpl/parser/util/DebugUtil.java | 6 +- .../util/visualize/ParseStateVisualizer.java | 17 +- .../util/visualize/dot/CompassPoint.java | 2 + .../rascalmpl/util/visualize/dot/NodeId.java | 20 +- src/org/rascalmpl/util/visualize/replay.html | 20 +- .../values/RascalFunctionValueFactory.java | 3 - .../rascalmpl/test/parser/RecoveryTests.java | 150 ---- 23 files changed, 580 insertions(+), 821 deletions(-) delete mode 100644 src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java delete mode 100644 test/org/rascalmpl/test/parser/RecoveryTests.java diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 72fc6ee8caa..e9a30f0771a 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -195,8 +195,8 @@ data Production ; data Production - = error(Symbol def, Production prod, int dot) - | skipped(Symbol symbol); + = \error(Symbol def, Production prod, int dot) + | \skipped(Symbol symbol); @synopsis{Attributes in productions.} @description{ @@ -775,22 +775,17 @@ bool isNonTerminalType(Symbol::\parameterized-lex(str _, list[Symbol] _)) = true bool isNonTerminalType(Symbol::\start(Symbol s)) = isNonTerminalType(s); default bool isNonTerminalType(Symbol s) = false; -@synopsis{Check if a parse tree contains any skipped nodes, the result of error recovery.} +@synopsis{Check if a parse tree contains any error nodes, the result of error recovery.} bool hasErrors(Tree tree) = /appl(error(_, _, _), _) := tree; @synopsis{Find all error productions in a parse tree.} list[Tree] findAllErrors(Tree tree) = [err | /err:appl(error(_, _, _), _) := tree]; @synopsis{Find the first production containing an error.} -Tree findFirstError(Tree tree) { - if (/err:appl(error(_, _, _), _) := tree) return err; - fail; -} +Tree findFirstError(/err:appl(error(_, _, _), _)) = err; -@synopsis{Find the best error from a tree containing errors.} -Tree findBestError(Tree tree) { - return findFirstError(defaultErrorDisambiguationFilter(tree)); -} +@synopsis{Find the best error from a tree containing errors. This function will fail if `tree` does not contain an error.} +Tree findBestError(Tree tree) = findFirstError(defaultErrorDisambiguationFilter(tree)); @synopsis{Get the symbol (sort) of the failing production} Symbol getErrorSymbol(appl(error(Symbol sym, _, _), _)) = sym; @@ -802,16 +797,12 @@ Production getErrorProduction(appl(error(_, Production prod, _), _)) = prod; int getErrorDot(appl(error(_, _, int dot), _)) = dot; @synopsis{Get the skipped tree} -Tree getSkipped(appl(error(_, _, _), [*_, skip:appl(skipped(_), _)])) { - return skip; -} +Tree getSkipped(appl(error(_, _, _), [*_, skip:appl(skipped(_), _)])) = skip; @synopsis{Get the text that failed to parse. This is only the text of the part that has been skipped to be able to continue parsing. If you want the text of the whole error tree, you can just use string interpolation: "". } -str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) { - return stringChars([c | ch <- chars, char(c) := ch]); -} +str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) = stringChars([c | char(c) <- chars]); @synopsis{Error recovery often produces ambiguous trees where errors can be recovered in multiple ways. This filter removes error trees until no ambiguities caused by error recovery are left. @@ -831,7 +822,9 @@ Tree defaultErrorDisambiguationFilter(amb(set[Tree] alternatives)) { if (nonErrorTrees == {}) { return getBestErrorTree(errorTrees); - } else if ({Tree single} := nonErrorTrees) { + } + + if ({Tree single} := nonErrorTrees) { // One ambiguity left, no ambiguity concerns here return single; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index c872a1ce7b0..410f392e446 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -28,20 +28,20 @@ private Tree parseS(str input, bool visualize=false) = parser(#S, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); test bool basicOk() { - return !hasErrors(parseS("a b c $", visualize=true)); + return !hasErrors(parseS("a b c $")); } test bool abx() { - Tree t = parseS("a b x $", visualize=true); + Tree t = parseS("a b x $"); return getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "x "; } test bool axc() { - Tree t = parseS("a x c $", visualize=true); + Tree t = parseS("a x c $"); return getErrorText(findFirstError(t)) == "x c"; } test bool ax() { - Tree t = parseS("a x $", visualize=true); + Tree t = parseS("a x $"); return getErrorText(findFirstError(t)) == "x "; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc index 3c71c2f7e80..8a461911cfb 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc @@ -108,13 +108,12 @@ test bool rascalMissingCloseParen() { } test bool rascalFunctionDeclarationMissingCloseParen() { - Tree t = parseFunctionDeclaration("void f({} void g() {}", visualize=false); + Tree t = parseFunctionDeclaration("void f({} void g() {}"); assert getErrorText(findFirstError(t)) == "void g("; Tree error = findFirstError(defaultErrorDisambiguationFilter(t)); assert getErrorText(error) == "("; - Tree skipped = getSkipped(error); loc location = getSkipped(error).src; assert location.begin.column == 16 && location.length == 1; diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 28d6d3536fc..a23e80bfad8 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -1,12 +1,9 @@ /******************************************************************************* - * Copyright (c) 2009-2013 CWI - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html + * Copyright (c) 2009-2013 CWI All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v1.0 which accompanies this + * distribution, and is available at http://www.eclipse.org/legal/epl-v10.html * - * Contributors: - * * Arnold Lankamp - Arnold.Lankamp@cwi.nl + * Contributors: * Arnold Lankamp - Arnold.Lankamp@cwi.nl *******************************************************************************/ package org.rascalmpl.parser.gtd; @@ -71,7 +68,8 @@ public abstract class SGTDBF implements IGTD { // A mapping between character location and line/column. private final PositionStore positionStore; - // Terminals that matched. Circular buffer indexed by length of the terminal. Each entry contains the node to reduce and the result node + // Terminals that matched. Circular buffer indexed by length of the terminal. Each entry contains + // the node to reduce and the result node // This is a circular buffer where queueIndex determines the start of the buffer. // At each position, a stack is maintained of all terminals to reduce of a certain length. // So at queueIndex+3, all terminals of length 3 that need reducing are stored. @@ -88,21 +86,29 @@ public abstract class SGTDBF implements IGTD { // - expandStack: when an expandable stack is expanded, all non-matchable children are pushed private final Stack> stacksToExpand; - // The current stack of non-terminals to reduce. Each stack has a container node to accumulate results. - // - Nodes are removed in `reduce` where all productions are advanced one dot past the non-terminal node + // The current stack of non-terminals to reduce. Each stack has a container node to accumulate + // results. + // - Nodes are removed in `reduceNonTerminals` where all productions are advanced one dot past the + // non-terminal node // - Nodes are added in: - // - handleEdgeList: result container node is created and all edges are pushed with the same result container node - // - handleEdgeListWithRestrictions: result container node is created and all edges are pushed with the same result container node - // - expandStack: non-matchable, non-expandable nodes (and their existing result container node) are added if their name can be found in `cachedEdgesForExpect`. + // - handleEdgeList: result container node is created and all edges are pushed with the same result + // container node + // - handleEdgeListWithRestrictions: result container node is created and all edges are pushed with + // the same result container node + // - expandStack: non-matchable, non-expandable nodes (and their existing result container node) are + // added if their name can be found in `cachedEdgesForExpect`. // - expandStack: expandable nodes that are nullable? Might be a cycle thing private final DoubleStack, AbstractContainerNode

> stacksWithNonTerminalsToReduce; - // The current stack of non-terminals to reduce: it contains the matchable node with the smallest length from todoLists. - // - Nodes are removed in `reduce` where all productions are advanced one dot past the matchable node + // The current stack of non-terminals to reduce: it contains the matchable node with the smallest + // length from todoLists. + // - Nodes are removed in `reduceTerminals` where all productions are advanced one dot past the + // matchable node // - Variable is assigned in: // - findFirstStacksToReduce: the first non-empty `todoList` is assigned to this variable // - findStacksToReduce: again the first non-empty `todoList` is assigned to this variable - // - parse: variable is used in main reduce/expand loop to determine when it is time to look for more `stacksToReduce`. + // - parse: variable is used in main reduce/expand loop to determine when it is time to look for + // more `stacksToReduce`. private DoubleStack, AbstractNode> stacksWithTerminalsToReduce; private final HashMap> cachedEdgesForExpect; @@ -121,7 +127,8 @@ public abstract class SGTDBF implements IGTD { // Error reporting private final Stack> unexpandableNodes; - private final Stack> unmatchableLeafNodes; // Leaf nodes (for instance literals) that failed to match + private final Stack> unmatchableLeafNodes; // Leaf nodes (for instance literals) that failed to + // match private final DoubleStack, AbstractNode>, AbstractStackNode

> unmatchableMidProductionNodes; private final DoubleStack, AbstractNode> filteredNodes; @@ -150,7 +157,8 @@ public SGTDBF(){ cachedEdgesForExpect = new HashMap>(); - sharedNextNodes = new IntegerKeyedDoubleValueHashMap, DoubleArrayList, AbstractNode>>(); + sharedNextNodes = + new IntegerKeyedDoubleValueHashMap, DoubleArrayList, AbstractNode>>(); location = 0; @@ -160,14 +168,15 @@ public SGTDBF(){ unexpandableNodes = new Stack>(); unmatchableLeafNodes = new Stack>(); - unmatchableMidProductionNodes = new DoubleStack, AbstractNode>, AbstractStackNode

>(); + unmatchableMidProductionNodes = + new DoubleStack, AbstractNode>, AbstractStackNode

>(); filteredNodes = new DoubleStack, AbstractNode>(); } /** - * Return a stack node id that is guaranteed not to be in use. - * The parser generator generates an override for this method as it knows which ids have been dispensed. - * Tests that need this should override this method, probably using a common base class. + * Return a stack node id that is guaranteed not to be in use. The parser generator generates an + * override for this method as it knows which ids have been dispensed. Tests that need this should + * override this method, probably using a common base class. */ protected int getFreeStackNodeId() { throw new UnsupportedOperationException(); @@ -181,16 +190,20 @@ public AbstractStackNode

[] getExpects(String nonTerminal) { Method method = getClass().getMethod(nonTerminal); try { method.setAccessible(true); // Try to bypass the 'isAccessible' check to save time. - } catch (SecurityException sex) { + } + catch (SecurityException sex) { // Ignore this if it happens. } expects = (AbstractStackNode

[]) method.invoke(this); - } catch (NoSuchMethodException nsmex) { + } + catch (NoSuchMethodException nsmex) { throw new UndeclaredNonTerminalException(nonTerminal, getClass()); - } catch (IllegalAccessException iaex) { + } + catch (IllegalAccessException iaex) { throw new RuntimeException(iaex); - } catch (InvocationTargetException itex) { + } + catch (InvocationTargetException itex) { throw new RuntimeException(itex.getTargetException()); } @@ -210,8 +223,10 @@ protected AbstractStackNode

[] invokeExpects(AbstractStackNode

nonTerminal) /** * Moves to the next symbol in the production. */ - private AbstractStackNode

updateNextNode(AbstractStackNode

next, AbstractStackNode

node, AbstractNode result){ - IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> alternativeEntry = sharedNextNodes.get(next.getId()); + private AbstractStackNode

updateNextNode(AbstractStackNode

next, AbstractStackNode

node, + AbstractNode result) { + IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> alternativeEntry = + sharedNextNodes.get(next.getId()); if(alternativeEntry != null){ // Sharing check. DoubleArrayList, AbstractNode> predecessors = alternativeEntry.value2; if(predecessors != null){ @@ -226,20 +241,27 @@ private AbstractStackNode

updateNextNode(AbstractStackNode

next, AbstractS // Encountered a possible stack 'overtake'. if(node.getStartLocation() != location){ propagateEdgesAndPrefixes(node, result, alternative, alternative.getResult()); - }else{ - propagateEdgesAndPrefixesForNullable(node, result, alternative, alternative.getResult(), node.getEdges().size()); + } + else { + propagateEdgesAndPrefixesForNullable(node, result, alternative, alternative.getResult(), + node.getEdges().size()); } return alternative; } - }else{ + } + else { EdgesSet

alternativeEdgesSet = alternative.getIncomingEdges(); int resultStoreId = getResultStoreId(alternative.getId()); - if(alternativeEdgesSet != null && alternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location){ + if (alternativeEdgesSet != null + && alternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location) { // Encountered a possible stack 'overtake'. if(node.getStartLocation() != location){ - propagateEdgesAndPrefixes(node, result, alternative, alternativeEdgesSet.getLastResult(resultStoreId)); - }else{ - propagateEdgesAndPrefixesForNullable(node, result, alternative, alternativeEdgesSet.getLastResult(resultStoreId), node.getEdges().size()); + propagateEdgesAndPrefixes(node, result, alternative, + alternativeEdgesSet.getLastResult(resultStoreId)); + } + else { + propagateEdgesAndPrefixesForNullable(node, result, alternative, + alternativeEdgesSet.getLastResult(resultStoreId), node.getEdges().size()); } return alternative; } @@ -248,42 +270,51 @@ private AbstractStackNode

updateNextNode(AbstractStackNode

next, AbstractS alternative.updateNode(node, result); - if(debugListener != null) debugListener.progressed(node, result, alternative); + if (debugListener != null) + debugListener.progressed(node, result, alternative); return alternative; } if(next.isMatchable()){ // Eager matching optimization. - if((location + next.getLength()) > input.length) return null; + if ((location + next.getLength()) > input.length) + return null; AbstractNode nextResult = next.match(input, location); if(nextResult == null){ - // Push the node including it's predecessor to the appropriate error tracking collection (and take care of merging when necessary). - DoubleArrayList, AbstractNode> predecessors = new DoubleArrayList, AbstractNode>(); + // Push the node including it's predecessor to the appropriate error tracking collection (and take + // care of merging when necessary). + DoubleArrayList, AbstractNode> predecessors = + new DoubleArrayList, AbstractNode>(); predecessors.add(node, result); unmatchableMidProductionNodes.push(predecessors, next); sharedNextNodes.putUnsafe(next.getId(), next, predecessors); - if(debugListener != null) debugListener.failedToMatch(next); + if (debugListener != null) + debugListener.failedToMatch(next); return null; } - if(debugListener != null) debugListener.matched(next, nextResult); + if (debugListener != null) + debugListener.matched(next, nextResult); next = next.getCleanCopyWithResult(location, nextResult); - }else{ + } + else { next = next.getCleanCopy(location); } if(!node.isMatchable() || result.isEmpty()){ next.updateNode(node, result); - }else{ // Non-nullable terminal specific edge set sharing optimization. + } + else { // Non-nullable terminal specific edge set sharing optimization. next.updateNodeAfterNonEmptyMatchable(node, result); } - if(debugListener != null) debugListener.progressed(node, result, next); + if (debugListener != null) + debugListener.progressed(node, result, next); sharedNextNodes.putUnsafe(next.getId(), next, null); stacksToExpand.push(next); @@ -294,9 +325,11 @@ private AbstractStackNode

updateNextNode(AbstractStackNode

next, AbstractS /** * Moves to the next symbol in an alternative continuation of a prefix-shared production. */ - private boolean updateAlternativeNextNode(AbstractStackNode

next, AbstractStackNode

node, AbstractNode result, IntegerObjectList> edgesMap, ArrayList[] prefixesMap){ + private boolean updateAlternativeNextNode(AbstractStackNode

next, AbstractStackNode

node, AbstractNode result, + IntegerObjectList> edgesMap, ArrayList[] prefixesMap) { int id = next.getId(); - IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> alternativeEntry = sharedNextNodes.get(id); + IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> alternativeEntry = + sharedNextNodes.get(id); if(alternativeEntry != null){ // Sharing check. DoubleArrayList, AbstractNode> predecessors = alternativeEntry.value2; if(predecessors != null){ @@ -309,55 +342,68 @@ private boolean updateAlternativeNextNode(AbstractStackNode

next, AbstractSta if(alternative.isMatchable()){ if(alternative.isEmptyLeafNode()){ // Encountered a possible stack 'overtake'. - propagateAlternativeEdgesAndPrefixes(node, result, alternative, alternative.getResult(), node.getEdges().size(), edgesMap, prefixesMap); + propagateAlternativeEdgesAndPrefixes(node, result, alternative, alternative.getResult(), + node.getEdges().size(), edgesMap, prefixesMap); return true; } - }else{ + } + else { EdgesSet

alternativeEdgesSet = alternative.getIncomingEdges(); int resultStoreId = getResultStoreId(alternative.getId()); - if(alternativeEdgesSet != null && alternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location){ + if (alternativeEdgesSet != null + && alternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location) { AbstractContainerNode

nextResult = alternativeEdgesSet.getLastResult(resultStoreId); // Encountered a possible stack 'overtake'. - propagateAlternativeEdgesAndPrefixes(node, result, alternative, nextResult, node.getEdges().size(), edgesMap, prefixesMap); + propagateAlternativeEdgesAndPrefixes(node, result, alternative, nextResult, + node.getEdges().size(), edgesMap, prefixesMap); return true; } } } - alternative.updatePrefixSharedNode(edgesMap, prefixesMap); // Prevent unnecessary overhead; share whenever possible. + alternative.updatePrefixSharedNode(edgesMap, prefixesMap); // Prevent unnecessary overhead; share whenever + // possible. - if(debugListener != null) debugListener.progressed(node, result, alternative); + if (debugListener != null) + debugListener.progressed(node, result, alternative); return true; } if(next.isMatchable()){ // Eager matching optimization. - if((location + next.getLength()) > input.length) return false; + if ((location + next.getLength()) > input.length) + return false; AbstractNode nextResult = next.match(input, location); if(nextResult == null){ - // Push the node including it's predecessor to the appropriate error tracking collection (and take care of merging when necessary). - DoubleArrayList, AbstractNode> predecessors = new DoubleArrayList, AbstractNode>(); + // Push the node including it's predecessor to the appropriate error tracking collection (and take + // care of merging when necessary). + DoubleArrayList, AbstractNode> predecessors = + new DoubleArrayList, AbstractNode>(); predecessors.add(node, result); unmatchableMidProductionNodes.push(predecessors, next); sharedNextNodes.putUnsafe(id, next, predecessors); - if(debugListener != null) debugListener.failedToMatch(next); + if (debugListener != null) + debugListener.failedToMatch(next); return false; } - if(debugListener != null) debugListener.matched(next, nextResult); + if (debugListener != null) + debugListener.matched(next, nextResult); next = next.getCleanCopyWithResult(location, nextResult); - }else{ + } + else { next = next.getCleanCopy(location); } next.updatePrefixSharedNode(edgesMap, prefixesMap); // Prevent unnecessary overhead; share whenever possible. - if(debugListener != null) debugListener.progressed(node, result, next); + if (debugListener != null) + debugListener.progressed(node, result, next); sharedNextNodes.putUnsafe(id, next, null); stacksToExpand.push(next); @@ -366,10 +412,10 @@ private boolean updateAlternativeNextNode(AbstractStackNode

next, AbstractSta } /** - * Part of the hidden-right-recursion fix. - * Executes absent reductions. + * Part of the hidden-right-recursion fix. Executes absent reductions. */ - private void propagateReductions(AbstractStackNode

node, AbstractNode nodeResultStore, AbstractStackNode

next, AbstractNode nextResultStore, int potentialNewEdges){ + private void propagateReductions(AbstractStackNode

node, AbstractNode nodeResultStore, AbstractStackNode

next, + AbstractNode nextResultStore, int potentialNewEdges) { IntegerList propagatedReductions = next.getPropagatedReductions(); IntegerObjectList> edgesMap = node.getEdges(); @@ -399,26 +445,28 @@ private void propagateReductions(AbstractStackNode

node, AbstractNode nodeRes EdgesSet

edgeSet = edgesMap.getValue(i); - if(debugListener != null) debugListener.reducing(node, resultLink, edgeSet); + if (debugListener != null) + debugListener.reducing(node, resultLink, edgeSet); if(!hasNestingRestrictions){ handleEdgeList(edgeSet, name, production, resultLink, startLocation); - }else{ + } + else { handleEdgeListWithRestrictions(edgeSet, name, production, resultLink, startLocation, filteredParents); } } } /** - * Part of the hidden-right-recursion fix. - * Propagates absent prefixes. + * Part of the hidden-right-recursion fix. Propagates absent prefixes. */ private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResult, int nrOfAddedEdges){ // Proceed with the tail of the production. int nextDot = next.getDot() + 1; AbstractStackNode

[] prod = next.getProduction(); AbstractStackNode

nextNext = prod[nextDot]; - IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> nextNextAlternativeEntry = sharedNextNodes.get(nextNext.getId()); + IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> nextNextAlternativeEntry = + sharedNextNodes.get(nextNext.getId()); AbstractStackNode

nextNextAlternative = null; if(nextNextAlternativeEntry != null){ // Valid continuation. DoubleArrayList, AbstractNode> predecessors = nextNextAlternativeEntry.value2; @@ -426,24 +474,33 @@ private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResul nextNextAlternative = nextNextAlternativeEntry.value1; if(nextNextAlternative.isMatchable()){ if(nextNextAlternative.isEmptyLeafNode()){ - propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative, nextNextAlternative.getResult(), nrOfAddedEdges); - }else{ + propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative, + nextNextAlternative.getResult(), nrOfAddedEdges); + } + else { nextNextAlternative.updateNode(next, nextResult); - if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAlternative); + if (debugListener != null) + debugListener.propagated(next, nextResult, nextNextAlternative); } - }else{ + } + else { EdgesSet

nextNextAlternativeEdgesSet = nextNextAlternative.getIncomingEdges(); int resultStoreId = getResultStoreId(nextNextAlternative.getId()); - if(nextNextAlternativeEdgesSet != null && nextNextAlternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location){ - propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative, nextNextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges); - }else{ + if (nextNextAlternativeEdgesSet != null + && nextNextAlternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location) { + propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative, + nextNextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges); + } + else { nextNextAlternative.updateNode(next, nextResult); - if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAlternative); + if (debugListener != null) + debugListener.propagated(next, nextResult, nextNextAlternative); + } } } - }else{ + else { predecessors.add(next, nextResult); } } @@ -451,12 +508,16 @@ private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResul // Handle alternative continuations (related to prefix sharing). AbstractStackNode

[][] alternateProds = next.getAlternateProductions(); if(alternateProds != null){ - if(nextNextAlternative == null){ // If the first continuation has not been initialized yet (it may be a matchable that didn't match), create a dummy version to construct the necessary edges and prefixes. - if(!nextNext.isMatchable()) return; // Matchable, abort. + if (nextNextAlternative == null) { // If the first continuation has not been initialized yet (it may be a + // matchable that didn't match), create a dummy version to construct the + // necessary edges and prefixes. + if (!nextNext.isMatchable()) + return; // Matchable, abort. nextNextAlternative = nextNext.getCleanCopy(location); nextNextAlternative.updateNode(next, nextResult); - if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAlternative); + if (debugListener != null) + debugListener.propagated(next, nextResult, nextNextAlternative); } IntegerObjectList> nextNextEdgesMap = nextNextAlternative.getEdges(); @@ -464,35 +525,49 @@ private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResul for(int i = alternateProds.length - 1; i >= 0; --i){ prod = alternateProds[i]; - if(nextDot == prod.length) continue; + if (nextDot == prod.length) + continue; AbstractStackNode

alternativeNextNext = prod[nextDot]; - IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> nextNextAltAlternativeEntry = sharedNextNodes.get(alternativeNextNext.getId()); + IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> nextNextAltAlternativeEntry = + sharedNextNodes.get(alternativeNextNext.getId()); AbstractStackNode

nextNextAltAlternative = null; if(nextNextAltAlternativeEntry != null){ - DoubleArrayList, AbstractNode> predecessors = nextNextAltAlternativeEntry.value2; + DoubleArrayList, AbstractNode> predecessors = + nextNextAltAlternativeEntry.value2; if(predecessors == null){ nextNextAltAlternative = nextNextAltAlternativeEntry.value1; if(nextNextAltAlternative.isMatchable()){ if(nextNextAltAlternative.isEmptyLeafNode()){ - propagateAlternativeEdgesAndPrefixes(next, nextResult, nextNextAltAlternative, nextNextAltAlternative.getResult(), nrOfAddedEdges, nextNextEdgesMap, nextNextPrefixesMap); - }else{ + propagateAlternativeEdgesAndPrefixes(next, nextResult, nextNextAltAlternative, + nextNextAltAlternative.getResult(), nrOfAddedEdges, nextNextEdgesMap, + nextNextPrefixesMap); + } + else { nextNextAltAlternative.updatePrefixSharedNode(nextNextEdgesMap, nextNextPrefixesMap); - if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAltAlternative); + if (debugListener != null) + debugListener.propagated(next, nextResult, nextNextAltAlternative); } - }else{ + } + else { EdgesSet

nextAlternativeEdgesSet = nextNextAltAlternative.getIncomingEdges(); int resultStoreId = getResultStoreId(nextNextAltAlternative.getId()); - if(nextAlternativeEdgesSet != null && nextAlternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location){ - propagateAlternativeEdgesAndPrefixes(next, nextResult, nextNextAltAlternative, nextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges, nextNextEdgesMap, nextNextPrefixesMap); - }else{ + if (nextAlternativeEdgesSet != null + && nextAlternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location) { + propagateAlternativeEdgesAndPrefixes(next, nextResult, nextNextAltAlternative, + nextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges, + nextNextEdgesMap, nextNextPrefixesMap); + } + else { nextNextAltAlternative.updatePrefixSharedNode(nextNextEdgesMap, nextNextPrefixesMap); - if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAltAlternative); + if (debugListener != null) + debugListener.propagated(next, nextResult, nextNextAltAlternative); } } - }else{ + } + else { predecessors.add(next, nextResult); } } @@ -501,15 +576,18 @@ private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResul } /** - * Part of the hidden-right-recursion fix. - * Inserts missing prefixes and triggers reductions where necessary. + * Part of the hidden-right-recursion fix. Inserts missing prefixes and triggers reductions where + * necessary. */ - private void propagateEdgesAndPrefixes(AbstractStackNode

node, AbstractNode nodeResult, AbstractStackNode

next, AbstractNode nextResult){ + private void propagateEdgesAndPrefixes(AbstractStackNode

node, AbstractNode nodeResult, + AbstractStackNode

next, AbstractNode nextResult) { int nrOfAddedEdges = next.updateOvertakenNode(node, nodeResult); - if(debugListener != null) debugListener.propagated(node, nodeResult, next); + if (debugListener != null) + debugListener.propagated(node, nodeResult, next); - if(nrOfAddedEdges == 0) return; + if (nrOfAddedEdges == 0) + return; if(next.isEndNode()){ propagateReductions(node, nodeResult, next, nextResult, nrOfAddedEdges); @@ -521,15 +599,18 @@ private void propagateEdgesAndPrefixes(AbstractStackNode

node, AbstractNode n } /** - * Part of the hidden-right-recursion fix. - * Inserts missing prefixes and triggers reductions where necessary (specific for nullable nodes). + * Part of the hidden-right-recursion fix. Inserts missing prefixes and triggers reductions where + * necessary (specific for nullable nodes). */ - private void propagateEdgesAndPrefixesForNullable(AbstractStackNode

node, AbstractNode nodeResult, AbstractStackNode

next, AbstractNode nextResult, int potentialNewEdges){ + private void propagateEdgesAndPrefixesForNullable(AbstractStackNode

node, AbstractNode nodeResult, + AbstractStackNode

next, AbstractNode nextResult, int potentialNewEdges) { int nrOfAddedEdges = next.updateOvertakenNullableNode(node, nodeResult, potentialNewEdges); - if(debugListener != null) debugListener.propagated(node, nodeResult, next); + if (debugListener != null) + debugListener.propagated(node, nodeResult, next); - if(nrOfAddedEdges == 0) return; + if (nrOfAddedEdges == 0) + return; if(next.isEndNode()){ propagateReductions(node, nodeResult, next, nextResult, nrOfAddedEdges); @@ -541,15 +622,19 @@ private void propagateEdgesAndPrefixesForNullable(AbstractStackNode

node, Abs } /** - * Part of the hidden-right-recursion fix. - * Inserts missing prefixes and triggers reductions where necessary (specifically for alternative continuations of prefix-shared productions). + * Part of the hidden-right-recursion fix. Inserts missing prefixes and triggers reductions where + * necessary (specifically for alternative continuations of prefix-shared productions). */ - private void propagateAlternativeEdgesAndPrefixes(AbstractStackNode

node, AbstractNode nodeResult, AbstractStackNode

next, AbstractNode nextResult, int potentialNewEdges, IntegerObjectList> edgesMap, ArrayList[] prefixesMap){ + private void propagateAlternativeEdgesAndPrefixes(AbstractStackNode

node, AbstractNode nodeResult, + AbstractStackNode

next, AbstractNode nextResult, int potentialNewEdges, + IntegerObjectList> edgesMap, ArrayList[] prefixesMap) { next.updatePrefixSharedNode(edgesMap, prefixesMap); - if(debugListener != null) debugListener.propagated(node, nodeResult, next); + if (debugListener != null) + debugListener.propagated(node, nodeResult, next); - if(potentialNewEdges == 0) return; + if (potentialNewEdges == 0) + return; if(next.isEndNode()){ propagateReductions(node, nodeResult, next, nextResult, potentialNewEdges); @@ -582,12 +667,16 @@ private void updateEdges(AbstractStackNode

node, AbstractNode result){ EdgesSet

edgeSet = edgesMap.getValue(i); - if(debugListener != null) debugListener.reducing(node, resultLink, edgeSet); + if (debugListener != null) + debugListener.reducing(node, resultLink, edgeSet); - if(!hasNestingRestrictions){ // Select the optimized path for handling edge sets that don't have nesting restrictions associated with them. + if (!hasNestingRestrictions) { // Select the optimized path for handling edge sets that don't have nesting + // restrictions associated with them. handleEdgeList(edgeSet, name, production, resultLink, edgesMap.getKey(i)); - }else{ - handleEdgeListWithRestrictions(edgeSet, name, production, resultLink, edgesMap.getKey(i), filteredParents); + } + else { + handleEdgeListWithRestrictions(edgeSet, name, production, resultLink, edgesMap.getKey(i), + filteredParents); } } } @@ -616,18 +705,22 @@ private void updateNullableEdges(AbstractStackNode

node, AbstractNode result) for(int i = edgesMap.size() - 1; i >= 0; --i){ int startLocation = edgesMap.getKey(i); - if(propagatedReductions.containsBefore(startLocation, initialSize)) continue; // Prevent duplicate reductions (artifact of the hidden-right-recursion fix). + if (propagatedReductions.containsBefore(startLocation, initialSize)) + continue; // Prevent duplicate reductions (artifact of the hidden-right-recursion fix). propagatedReductions.add(startLocation); Link resultLink = new Link((prefixesMap != null) ? prefixesMap[i] : null, result); EdgesSet

edgeSet = edgesMap.getValue(i); - if(debugListener != null) debugListener.reducing(node, resultLink, edgeSet); + if (debugListener != null) + debugListener.reducing(node, resultLink, edgeSet); - if(!hasNestingRestrictions){ // Select the optimized path for handling edge sets that don't have nesting restrictions associated with them. + if (!hasNestingRestrictions) { // Select the optimized path for handling edge sets that don't have nesting + // restrictions associated with them. handleEdgeList(edgeSet, name, production, resultLink, startLocation); - }else{ + } + else { handleEdgeListWithRestrictions(edgeSet, name, production, resultLink, startLocation, filteredParents); } } @@ -644,26 +737,33 @@ private void handleEdgeList(EdgesSet

edgeSet, String name, P production, Link if(edge.isRecovered()){ resultStore = new RecoveredNode

(inputURI, startLocation, location); - }else if(edge.isExpandable()){ - resultStore = new ExpandableContainerNode

(inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout()); - }else{ - resultStore = new SortContainerNode

(inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout()); + } + else if (edge.isExpandable()) { + resultStore = new ExpandableContainerNode

(inputURI, startLocation, location, + startLocation == location, edge.isSeparator(), edge.isLayout()); + } + else { + resultStore = new SortContainerNode

(inputURI, startLocation, location, startLocation == location, + edge.isSeparator(), edge.isLayout()); } stacksWithNonTerminalsToReduce.push(edge, resultStore); - if(debugListener != null) debugListener.reduced(edge); + if (debugListener != null) + debugListener.reduced(edge); for(int j = edgeSet.size() - 1; j >= 1; --j){ edge = edgeSet.get(j); stacksWithNonTerminalsToReduce.push(edge, resultStore); - if(debugListener != null) debugListener.reduced(edge); + if (debugListener != null) + debugListener.reduced(edge); } edgeSet.setLastVisitedLevel(location, resultStoreId); edgeSet.setLastResult(resultStore, resultStoreId); - }else{ + } + else { resultStore = edgeSet.getLastResult(resultStoreId); } @@ -677,9 +777,11 @@ private void handleEdgeList(EdgesSet

edgeSet, String name, P production, Link /** * Handles reductions which may be associated with nesting restrictions. */ - private void handleEdgeListWithRestrictions(EdgesSet

edgeSet, String name, P production, Link resultLink, int startLocation, IntegerList filteredParents){ + private void handleEdgeListWithRestrictions(EdgesSet

edgeSet, String name, P production, Link resultLink, + int startLocation, IntegerList filteredParents) { // Only add the result to each resultstore once. - // Make sure each edge only gets added to the non-terminal reduction list once per level, by keeping track of them. + // Make sure each edge only gets added to the non-terminal reduction list once per level, by keeping + // track of them. firstTimeRegistration.clear(); firstTimeReductions.clear(); for(int j = edgeSet.size() - 1; j >= 0; --j){ @@ -688,7 +790,8 @@ private void handleEdgeListWithRestrictions(EdgesSet

edgeSet, String name, P if(!firstTimeReductions.contains(resultStoreId)){ if(firstTimeRegistration.contains(resultStoreId)){ - if(debugListener != null) debugListener.filteredByNestingRestriction(edge); + if (debugListener != null) + debugListener.filteredByNestingRestriction(edge); continue; } @@ -703,10 +806,14 @@ private void handleEdgeListWithRestrictions(EdgesSet

edgeSet, String name, P if(resultStore == null){ if (edge.isRecovered()) { resultStore = new RecoveredNode

(inputURI, startLocation, location); - }else if (edge.isExpandable()) { - resultStore = new ExpandableContainerNode

(inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout()); - }else { - resultStore = new SortContainerNode

(inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout()); + } + else if (edge.isExpandable()) { + resultStore = new ExpandableContainerNode

(inputURI, startLocation, location, + startLocation == location, edge.isSeparator(), edge.isLayout()); + } + else { + resultStore = new SortContainerNode

(inputURI, startLocation, location, + startLocation == location, edge.isSeparator(), edge.isLayout()); } edgeSet.setLastVisitedLevel(location, resultStoreId); @@ -718,11 +825,15 @@ private void handleEdgeListWithRestrictions(EdgesSet

edgeSet, String name, P resultStore.addAlternative(production, resultLink); - if(debugListener != null) debugListener.reduced(edge); - }else{ - if(debugListener != null) debugListener.filteredByNestingRestriction(edge); + if (debugListener != null) + debugListener.reduced(edge); } - }else{ + else { + if (debugListener != null) + debugListener.filteredByNestingRestriction(edge); + } + } + else { AbstractContainerNode

resultStore = edgeSet.getLastResult(resultStoreId); stacksWithNonTerminalsToReduce.push(edge, resultStore); } @@ -751,12 +862,14 @@ private void moveToNext(AbstractStackNode

node, AbstractNode result){ for(int i = alternateProds.length - 1; i >= 0; --i){ prod = alternateProds[i]; - if(nextDot == prod.length) continue; + if (nextDot == prod.length) + continue; AbstractStackNode

newAlternativeNext = prod[nextDot]; if(edgesMap != null){ updateAlternativeNextNode(newAlternativeNext, node, result, edgesMap, prefixesMap); - }else{ + } + else { AbstractStackNode

alternativeNext = updateNextNode(newAlternativeNext, node, result); if(alternativeNext != null){ @@ -769,11 +882,12 @@ private void moveToNext(AbstractStackNode

node, AbstractNode result){ } /** - * Progress to the next 'states' associated with the given node. - * I.e. move to the next symbol(s) in the production (if available) and executed reductions if necessary. + * Progress to the next 'states' associated with the given node. I.e. move to the next symbol(s) in + * the production (if available) and executed reductions if necessary. */ private void move(AbstractStackNode

node, AbstractNode result){ - if(debugListener != null) debugListener.moving(node, result); + if (debugListener != null) + debugListener.moving(node, result); // Handle filtering. ICompletionFilter[] completionFilters = node.getCompletionFilters(); @@ -783,7 +897,8 @@ private void move(AbstractStackNode

node, AbstractNode result){ if(completionFilters[i].isFiltered(input, startLocation, location, positionStore)){ filteredNodes.push(node, result); - if(debugListener != null) debugListener.filteredByCompletionFilter(node, result); + if (debugListener != null) + debugListener.filteredByCompletionFilter(node, result); return; } @@ -791,9 +906,21 @@ private void move(AbstractStackNode

node, AbstractNode result){ } if(node.isEndNode()){ - if(!result.isEmpty() || node.getId() == AbstractExpandableStackNode.DEFAULT_LIST_EPSILON_ID){ // Only go into the nullable fix path for nullables (special list epsilons can be ignored as well). + if (!result.isEmpty() || node.getId() == AbstractExpandableStackNode.DEFAULT_LIST_EPSILON_ID) { // Only go + // into the + // nullable + // fix path + // for + // nullables + // (special + // list + // epsilons + // can be + // ignored + // as well). updateEdges(node, result); - }else{ + } + else { updateNullableEdges(node, result); } } @@ -823,7 +950,8 @@ private void reduceNonTerminals() { } /** - * Locates the initial set of stacks that is queued for handling, for which the least amount of characters needs to be shifted. + * Locates the initial set of stacks that is queued for handling, for which the least amount of + * characters needs to be shifted. */ private boolean findFirstStacksToReduce(){ for(int i = 0; i < todoLists.length; ++i){ @@ -841,14 +969,17 @@ private boolean findFirstStacksToReduce(){ if (recoverer != null) { if (debugListener != null) { - debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, + unmatchableMidProductionNodes, filteredNodes); } visualize("Recovering", ParseStateVisualizer.ERROR_TRACKING_ID); - DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, + unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); } - if (recoveredNodes.size() > 0) { // TODO Do something with the revived node. Is this the right location to do this? + if (recoveredNodes.size() > 0) { // TODO Do something with the revived node. Is this the right location to + // do this? for (int i = 0; i < recoveredNodes.size(); i++) { AbstractStackNode

recovered = recoveredNodes.getFirst(i); queueMatchableNode(recovered, recovered.getLength(), recoveredNodes.getSecond(i)); @@ -863,7 +994,8 @@ private boolean findFirstStacksToReduce(){ } /** - * Locates the set of stacks that is queued for handling, for which the least amount of characters needs to be shifted. + * Locates the set of stacks that is queued for handling, for which the least amount of characters + * needs to be shifted. */ private boolean findStacksToReduce(){ visualize("Finding stacks to reduce", ParseStateVisualizer.TODO_LISTS_ID); @@ -888,10 +1020,12 @@ private boolean findStacksToReduce(){ if (recoverer != null && location < input.length) { if (debugListener != null) { - debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, + unmatchableMidProductionNodes, filteredNodes); } visualize("Recovering", ParseStateVisualizer.ERROR_TRACKING_ID); - DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, + unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); } @@ -907,10 +1041,12 @@ private boolean findStacksToReduce(){ for (int i = 0; i < recoveredNodes.size(); i++) { AbstractStackNode

recovered = recoveredNodes.getFirst(i); if (debugListener != null) { - debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, + unmatchableMidProductionNodes, filteredNodes); } visualize("Queue recovery node", ParseStateVisualizer.getNodeId(recovered)); - queueRecoveryNode(recovered, recovered.getStartLocation(), recovered.getLength(), recoveredNodes.getSecond(i)); + queueRecoveryNode(recovered, recovered.getStartLocation(), recovered.getLength(), + recoveredNodes.getSecond(i)); } return findStacksToReduce(); } @@ -952,8 +1088,8 @@ private void queueMatchableNode(AbstractStackNode

node, int length, AbstractN } /** - * Inserts a recovery node into the todoList, and possibly - * rewinds the parser to an earlier location in the input + * Inserts a recovery node into the todoList, and possibly rewinds the parser to an earlier location + * in the input */ @SuppressWarnings("unchecked") private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int length, AbstractNode result){ @@ -1004,7 +1140,8 @@ else if (startPosition == location) { /** * Handles the retrieved alternatives for the given stack. */ - private boolean handleExpects(AbstractStackNode

[] expects, EdgesSet

cachedEdges, AbstractStackNode

stackBeingWorkedOn){ + private boolean handleExpects(AbstractStackNode

[] expects, EdgesSet

cachedEdges, + AbstractStackNode

stackBeingWorkedOn) { boolean hasValidAlternatives = false; sharedLastExpects.dirtyClear(); @@ -1015,25 +1152,29 @@ private boolean handleExpects(AbstractStackNode

[] expects, EdgesSet

cached if(first.isMatchable()){ // Eager matching optimization. int length = first.getLength(); int endLocation = location + length; - if(endLocation > input.length) continue; + if (endLocation > input.length) + continue; AbstractNode result = first.match(input, location); if(result == null){ unmatchableLeafNodes.push(first); - if(debugListener != null) debugListener.failedToMatch(first); + if (debugListener != null) + debugListener.failedToMatch(first); continue; } - if(debugListener != null) debugListener.matched(first, result); + if (debugListener != null) + debugListener.matched(first, result); // Handle filtering. IEnterFilter[] enterFilters = first.getEnterFilters(); if(enterFilters != null){ for(int j = enterFilters.length - 1; j >= 0; --j){ if(enterFilters[j].isFiltered(input, location, positionStore)){ - if(debugListener != null) debugListener.filteredByEnterFilter(first); + if (debugListener != null) + debugListener.filteredByEnterFilter(first); continue EXPECTS; } @@ -1043,7 +1184,8 @@ private boolean handleExpects(AbstractStackNode

[] expects, EdgesSet

cached first = first.getCleanCopyWithResult(location, result); queueMatchableNode(first, length, result); - }else{ + } + else { first = first.getCleanCopy(location); stacksToExpand.push(first); } @@ -1055,7 +1197,8 @@ private boolean handleExpects(AbstractStackNode

[] expects, EdgesSet

cached hasValidAlternatives = true; - if(debugListener != null) debugListener.expanded(stackBeingWorkedOn, first); + if (debugListener != null) + debugListener.expanded(stackBeingWorkedOn, first); } return hasValidAlternatives; @@ -1086,7 +1229,8 @@ protected int getResultStoreId(int id){ * Expands the given stack node. */ private void expandStack(AbstractStackNode

stack){ - if(debugListener != null) debugListener.expanding(stack); + if (debugListener != null) + debugListener.expanding(stack); // Handle filtering. IEnterFilter[] enterFilters = stack.getEnterFilters(); @@ -1095,7 +1239,8 @@ private void expandStack(AbstractStackNode

stack){ if(enterFilters[i].isFiltered(input, location, positionStore)){ unexpandableNodes.push(stack); - if(debugListener != null) debugListener.filteredByEnterFilter(stack); + if (debugListener != null) + debugListener.filteredByEnterFilter(stack); return; } @@ -1104,7 +1249,8 @@ private void expandStack(AbstractStackNode

stack){ if(stack.isMatchable()){ // Eager matching optimization related. queueMatchableNode(stack, stack.getLength(), stack.getResult()); - }else if(!stack.isExpandable()){ // A 'normal' non-terminal. + } + else if (!stack.isExpandable()) { // A 'normal' non-terminal. EdgesSet

cachedEdges = cachedEdgesForExpect.get(stack.getName()); if(cachedEdges == null){ cachedEdges = new EdgesSet

(1); @@ -1120,19 +1266,22 @@ private void expandStack(AbstractStackNode

stack){ unexpandableNodes.push(stack); return; } - }else{ + } + else { int resultStoreId = getResultStoreId(stack.getId()); if(cachedEdges.getLastVisitedLevel(resultStoreId) == location){ // Is nullable, add the known results. stacksWithNonTerminalsToReduce.push(stack, cachedEdges.getLastResult(resultStoreId)); - if(debugListener != null) debugListener.foundIterationCachedNullableResult(stack); + if (debugListener != null) + debugListener.foundIterationCachedNullableResult(stack); } } cachedEdges.add(stack); stack.setIncomingEdges(cachedEdges); - }else{ // Expandable + } + else { // Expandable EdgesSet

cachedEdges = cachedEdgesForExpect.get(stack.getName()); if(cachedEdges == null){ boolean expanded = false; @@ -1146,33 +1295,39 @@ private void expandStack(AbstractStackNode

stack){ AbstractStackNode

child = listChildren[i]; int childId = child.getId(); - IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> sharedChildEntry = sharedNextNodes.get(childId); + IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> sharedChildEntry = + sharedNextNodes.get(childId); if(sharedChildEntry != null && sharedChildEntry.value2 == null){ AbstractStackNode

sharedChild = sharedChildEntry.value1; sharedChild.setEdgesSetWithPrefix(cachedEdges, null, location); - }else{ + } + else { if(child.isMatchable()){ int length = child.getLength(); int endLocation = location + length; - if(endLocation > input.length) continue; + if (endLocation > input.length) + continue; AbstractNode result = child.match(input, location); if(result == null){ unmatchableLeafNodes.push(child); - if(debugListener != null) debugListener.failedToMatch(child); + if (debugListener != null) + debugListener.failedToMatch(child); continue; } - if(debugListener != null) debugListener.matched(child, result); + if (debugListener != null) + debugListener.matched(child, result); // Handle filtering IEnterFilter[] childEnterFilters = child.getEnterFilters(); if(childEnterFilters != null){ for(int j = childEnterFilters.length - 1; j >= 0; --j){ if(childEnterFilters[j].isFiltered(input, location, positionStore)) { - if(debugListener != null) debugListener.filteredByEnterFilter(child); + if (debugListener != null) + debugListener.filteredByEnterFilter(child); continue CHILDREN; } @@ -1181,7 +1336,8 @@ private void expandStack(AbstractStackNode

stack){ child = child.getCleanCopyWithResult(location, result); queueMatchableNode(child, length, result); - }else{ + } + else { child = child.getCleanCopy(location); stacksToExpand.push(child); } @@ -1191,20 +1347,23 @@ private void expandStack(AbstractStackNode

stack){ sharedNextNodes.putUnsafe(childId, child, null); - if(debugListener != null) debugListener.expanded(stack, child); + if (debugListener != null) + debugListener.expanded(stack, child); } expanded = true; } if(stack.canBeEmpty()){ // Star list or optional. - AbstractStackNode

empty = stack.getEmptyChild().getCleanCopyWithResult(location, EpsilonStackNode.EPSILON_RESULT); + AbstractStackNode

empty = + stack.getEmptyChild().getCleanCopyWithResult(location, EpsilonStackNode.EPSILON_RESULT); empty.initEdges(); empty.addEdges(cachedEdges, location); stacksToExpand.push(empty); - if(debugListener != null) debugListener.expanded(stack, empty); + if (debugListener != null) + debugListener.expanded(stack, empty); expanded = true; } @@ -1218,7 +1377,8 @@ private void expandStack(AbstractStackNode

stack){ if(cachedEdges.getLastVisitedLevel(resultStoreId) == location){ // Is nullable, add the known results. stacksWithNonTerminalsToReduce.push(stack, cachedEdges.getLastResult(resultStoreId)); - if(debugListener != null) debugListener.foundIterationCachedNullableResult(stack); + if (debugListener != null) + debugListener.foundIterationCachedNullableResult(stack); } cachedEdges.add(stack); @@ -1245,7 +1405,8 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] * Initiates parsing. */ @SuppressWarnings("unchecked") - protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] input, IRecoverer

recoverer, IDebugListener

debugListener){ + protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] input, IRecoverer

recoverer, + IDebugListener

debugListener) { initTime(); try { @@ -1264,9 +1425,10 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] this.recoverer = recoverer; this.debugListener = debugListener; - if (inputURI != null) { + if (ParseStateVisualizer.VISUALIZATION_ENABLED && inputURI != null) { String query = inputURI.getQuery(); - visualizer = query != null && query.contains("visualize=true") ? new ParseStateVisualizer("Parser") : null; + visualizer = + query != null && query.contains("visualize=true") ? new ParseStateVisualizer("Parser") : null; } // Initialzed the position store. @@ -1300,22 +1462,27 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] unmatchableMidProductionNodes.dirtyClear(); filteredNodes.dirtyClear(); - if(debugListener != null) debugListener.shifting(location, input, positionStore); + if (debugListener != null) + debugListener.shifting(location, input, positionStore); } // Reduce-expand loop. do { - if(debugListener != null) debugListener.iterating(); + if (debugListener != null) { + debugListener.iterating(); + } reduceTerminals(); reduceNonTerminals(); expand(); - } while(!stacksWithNonTerminalsToReduce.isEmpty() || !stacksWithTerminalsToReduce.isEmpty()); + } + while (!stacksWithNonTerminalsToReduce.isEmpty() || !stacksWithTerminalsToReduce.isEmpty()); shiftedLevel = true; - } while(findStacksToReduce()); + } + while (findStacksToReduce()); } visualize("Done", ParseStateVisualizer.PARSER_ID); @@ -1342,9 +1509,17 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] int line = positionStore.findLine(errorLocation); int column = positionStore.getColumn(errorLocation, line); if (location == input.length) { - throw new ParseError("Parse error", inputURI, errorLocation, 0, line + 1, line + 1, column, column, (Stack>) (Stack) unexpandableNodes, (Stack>) (Stack) unmatchableLeafNodes, (DoubleStack>, AbstractStackNode>) (DoubleStack) unmatchableMidProductionNodes, (DoubleStack, AbstractNode>) (DoubleStack) filteredNodes); + throw new ParseError("Parse error", inputURI, errorLocation, 0, line + 1, line + 1, column, column, + (Stack>) (Stack) unexpandableNodes, + (Stack>) (Stack) unmatchableLeafNodes, + (DoubleStack>, AbstractStackNode>) (DoubleStack) unmatchableMidProductionNodes, + (DoubleStack, AbstractNode>) (DoubleStack) filteredNodes); } - throw new ParseError("Parse error", inputURI, errorLocation, 1, line + 1, line + 1, column, column + 1, (Stack>) (Stack) unexpandableNodes, (Stack>) (Stack) unmatchableLeafNodes, (DoubleStack>, AbstractStackNode>) (DoubleStack) unmatchableMidProductionNodes, (DoubleStack, AbstractNode>) (DoubleStack) filteredNodes); + throw new ParseError("Parse error", inputURI, errorLocation, 1, line + 1, line + 1, column, column + 1, + (Stack>) (Stack) unexpandableNodes, + (Stack>) (Stack) unmatchableLeafNodes, + (DoubleStack>, AbstractStackNode>) (DoubleStack) unmatchableMidProductionNodes, + (DoubleStack, AbstractNode>) (DoubleStack) filteredNodes); } finally { checkTime("Error handling"); @@ -1381,52 +1556,73 @@ private static int[] charsToInts(char[] input){ /** * Parses with post parse filtering. */ - private T parse(String nonterminal, URI inputURI, int[] input, IActionExecutor actionExecutor, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, IDebugListener

debugListener){ - AbstractNode result = parse(new NonTerminalStackNode

(AbstractStackNode.START_SYMBOL_ID, 0, nonterminal), inputURI, input, recoverer, debugListener); + private T parse(String nonterminal, URI inputURI, int[] input, IActionExecutor actionExecutor, + INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, + IDebugListener

debugListener) { + AbstractNode result = parse(new NonTerminalStackNode

(AbstractStackNode.START_SYMBOL_ID, 0, nonterminal), + inputURI, input, recoverer, debugListener); return buildResult(result, converter, nodeConstructorFactory, actionExecutor); } - public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, IDebugListener

debugListener){ - return parse(nonterminal, inputURI, charsToInts(input), actionExecutor, converter, nodeConstructorFactory, recoverer, debugListener); + public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, + INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, + IDebugListener

debugListener) { + return parse(nonterminal, inputURI, charsToInts(input), actionExecutor, converter, nodeConstructorFactory, + recoverer, debugListener); } - public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer){ + public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, + INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer) { return parse(nonterminal, inputURI, input, actionExecutor, converter, nodeConstructorFactory, recoverer, null); } - public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IDebugListener

debugListener){ - return parse(nonterminal, inputURI, input, actionExecutor, converter, nodeConstructorFactory, null, debugListener); + public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, + INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, + IDebugListener

debugListener) { + return parse(nonterminal, inputURI, input, actionExecutor, converter, nodeConstructorFactory, null, + debugListener); } - public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory){ + public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, + INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory) { return parse(nonterminal, inputURI, input, actionExecutor, converter, nodeConstructorFactory, null, null); } /** * Parses without post parse filtering. */ - private T parse(String nonterminal, URI inputURI, int[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, IDebugListener

debugListener){ - AbstractNode result = parse(new NonTerminalStackNode

(AbstractStackNode.START_SYMBOL_ID, 0, nonterminal), inputURI, input, recoverer, debugListener); + private T parse(String nonterminal, URI inputURI, int[] input, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, + IDebugListener

debugListener) { + AbstractNode result = parse(new NonTerminalStackNode

(AbstractStackNode.START_SYMBOL_ID, 0, nonterminal), + inputURI, input, recoverer, debugListener); return buildResult(result, converter, nodeConstructorFactory, new VoidActionExecutor()); } - public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, IDebugListener

debugListener){ - return parse(nonterminal, inputURI, charsToInts(input), converter, nodeConstructorFactory, recoverer, debugListener); + public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, + IDebugListener

debugListener) { + return parse(nonterminal, inputURI, charsToInts(input), converter, nodeConstructorFactory, recoverer, + debugListener); } - public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer){ + public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer) { return parse(nonterminal, inputURI, input, converter, nodeConstructorFactory, recoverer, null); } - public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IDebugListener

debugListener){ + public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory, IDebugListener

debugListener) { return parse(nonterminal, inputURI, input, converter, nodeConstructorFactory, null, debugListener); } - public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory) { + public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory) { return parse(nonterminal, inputURI, charsToInts(input), converter, nodeConstructorFactory, null, null); } - protected T parse(AbstractStackNode

startNode, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory) { + protected T parse(AbstractStackNode

startNode, URI inputURI, char[] input, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory) { AbstractNode result = parse(startNode, inputURI, charsToInts(input), null, null); @@ -1436,7 +1632,8 @@ protected T parse(AbstractStackNode

startNode, URI inputURI, char[] input, IN /** * Constructed the final parse result using the given converter. */ - protected T buildResult(AbstractNode result, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IActionExecutor actionExecutor){ + protected T buildResult(AbstractNode result, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory, IActionExecutor actionExecutor) { initTime(); try { FilteringTracker filteringTracker = new FilteringTracker(); @@ -1444,14 +1641,15 @@ protected T buildResult(AbstractNode result, INodeFlattener converter, INo Object rootEnvironment = actionExecutor != null ? actionExecutor.createRootEnvironment() : null; T parseResult = null; try { - parseResult = converter.convert(nodeConstructorFactory, result, positionStore, filteringTracker, actionExecutor, rootEnvironment); + parseResult = converter.convert(nodeConstructorFactory, result, positionStore, filteringTracker, + actionExecutor, rootEnvironment); } finally { actionExecutor.completed(rootEnvironment, (parseResult == null)); } if(parseResult != null) { if (recoverer != null) { - parseResult = fixErrorNodes(parseResult, nodeConstructorFactory); + parseResult = introduceErrorNodes(parseResult, nodeConstructorFactory); } return parseResult; // Success. } @@ -1463,7 +1661,8 @@ protected T buildResult(AbstractNode result, INodeFlattener converter, INo int beginColumn = positionStore.getColumn(offset, beginLine); int endLine = positionStore.findLine(endOffset); int endColumn = positionStore.getColumn(endOffset, endLine); - throw new ParseError("All results were filtered", inputURI, offset, length, beginLine + 1, endLine + 1, beginColumn, endColumn); + throw new ParseError("All results were filtered", inputURI, offset, length, beginLine + 1, endLine + 1, + beginColumn, endColumn); } finally { checkTime("Unbinarizing, post-parse filtering, and mapping to UPTR"); @@ -1471,36 +1670,41 @@ protected T buildResult(AbstractNode result, INodeFlattener converter, INo } /** - * Error nodes end up in a n inconvenient form because of the parser algorithm. - * This post-processing step transforms the original tree into a more useful form. - * In essence, error subtrees look like this after parsing: - * `appl(prod(S,[]), [,,...,appl(skipped([]))])` - * This method transforms these trees into: + * After parsing, parse trees will only contain `skipped` nodes. This post-processing step + * transforms the original tree into a more useful form. In essence, subtrees containing errors look + * like this after parsing: `appl(prod(S,[]), + * [,,...,appl(skipped([]))])` This method transforms these trees into: * `appl(error(S,prod(S,[]),), [,,...,appl(skipped([]))])` - * This means productions that failed to parse can be recognized at the top level. - * Note that this can only be done when we know the actual type of T is IConstructor. + * This means productions that failed to parse can be recognized at the top level. Note that this + * can only be done when we know the actual type of T is IConstructor. */ @SuppressWarnings("unchecked") - private T fixErrorNodes(T tree, INodeConstructorFactory nodeConstructorFactory) { + private T introduceErrorNodes(T tree, INodeConstructorFactory nodeConstructorFactory) { if (!(tree instanceof IConstructor)) { return tree; } - return (T) fixErrorNodes((IConstructor) tree, (INodeConstructorFactory) nodeConstructorFactory); + return (T) introduceErrorNodes((IConstructor) tree, + (INodeConstructorFactory) nodeConstructorFactory); } - private IConstructor fixErrorNodes(IConstructor tree, INodeConstructorFactory nodeConstructorFactory) { + private IConstructor introduceErrorNodes(IConstructor tree, + INodeConstructorFactory nodeConstructorFactory) { IConstructor result; Type type = tree.getConstructorType(); if (type == RascalValueFactory.Tree_Appl) { result = fixErrorAppl(tree, nodeConstructorFactory); - } else if (type == RascalValueFactory.Tree_Char) { + } + else if (type == RascalValueFactory.Tree_Char) { result = tree; - } else if (type == RascalValueFactory.Tree_Amb) { + } + else if (type == RascalValueFactory.Tree_Amb) { result = fixErrorAmb(tree, nodeConstructorFactory); - } else if (type == RascalValueFactory.Tree_Cycle) { + } + else if (type == RascalValueFactory.Tree_Cycle) { result = tree; - } else { + } + else { throw new RuntimeException("Unrecognized tree type: " + type); } @@ -1512,7 +1716,8 @@ private IConstructor fixErrorNodes(IConstructor tree, INodeConstructorFactory nodeConstructorFactory) { + private IConstructor fixErrorAppl(IConstructor tree, + INodeConstructorFactory nodeConstructorFactory) { IValue prod = tree.get(0); IList childList = (IList) tree.get(1); int childCount = childList.length(); @@ -1535,7 +1740,7 @@ private IConstructor fixErrorAppl(IConstructor tree, INodeConstructorFactory nodeConstructorFactory) { + private IConstructor fixErrorAmb(IConstructor tree, + INodeConstructorFactory nodeConstructorFactory) { ISet alternativeSet = (ISet) tree.get(0); ArrayList alternatives = new ArrayList<>(alternativeSet.size()); final AtomicBoolean anyChanges = new AtomicBoolean(false); alternativeSet.forEach(alt -> { - IConstructor newAlt = fixErrorNodes((IConstructor) alt, nodeConstructorFactory); + IConstructor newAlt = introduceErrorNodes((IConstructor) alt, nodeConstructorFactory); if (newAlt != alt) { anyChanges.setPlain(true); } diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index 3b25784f36d..832ab244ca8 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -748,9 +748,7 @@ public IntegerList getPropagatedReductions(){ return propagatedReductions; } - public String toShortString() { - return null; - } + public abstract String toShortString(); @Override public String toString() { @@ -786,10 +784,6 @@ public String toString() { if (alternateProductions != null && alternateProductions.length != 0) { builder.append(",alternateProductions=" + Arrays.toString(alternateProductions)); } - /* - if (edgesMap != null && edgesMap.size() != 0) { - builder.append(",edges=" + edgesMap); - }*/ if (prefixesMap != null && prefixesMap.length != 0) { builder.append(",prefixes=" + Arrays.toString(prefixesMap)); } diff --git a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java index b641a4ed413..e7ff655bdcf 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java @@ -69,6 +69,11 @@ public AbstractNode getResult(){ return result; } + @Override + public String toShortString() { + return toString(); + } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); diff --git a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java index 8188887f4e7..34b8f2318a9 100644 --- a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java @@ -93,6 +93,11 @@ public AbstractNode getResult(){ throw new UnsupportedOperationException(); } + @Override + public String toShortString() { + return toString(); + } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 0409c5b4e03..06450f62c4d 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -38,24 +38,6 @@ public static SkippedNode createResultUntilEndOfInput(URI uri, int[] input, int return new SkippedNode(uri, createSkippedToken(input, startLocation, length), startLocation); } - public static SkippedNode createResultUntilToken(URI uri, String token, int[] input, int startLocation) { - int length = token.length(); - for (int start=startLocation; start+length < input.length; start++) { - boolean match = true; - for (int j=0; j> comparator) { - java.util.List> elems = new java.util.ArrayList<>(size); + private java.util.List> zip() { + java.util.List> entries = new java.util.ArrayList<>(size); for (int i=0; i> comparator) { + java.util.List> entries = zip(); + + entries.sort(comparator); for (int i=0; i elem = elems.get(i); + Pair elem = entries.get(i); first[i] = elem.getLeft(); second[i] = elem.getRight(); } diff --git a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java index 572861a5455..f83fe0c4bf6 100644 --- a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java +++ b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index d3aa2a63a7e..3b5e8a84c68 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -155,23 +155,4 @@ public ITree createErrorNode(ArrayList children, Object production) { return buildAppl(children, errorProd); } - /* - @Override - public ITree createRecoveryNode(int[] characters) { - } - IListWriter args = VF.listWriter(); - for (int i=0; i { - private static final int[] WHITESPACE = {' ', '\r', '\n', '\t' }; - - private IdDispenser stackNodeIdDispenser; - - public ToNextWhitespaceRecoverer(IdDispenser stackNodeIdDispenser) { - this.stackNodeIdDispenser = stackNodeIdDispenser; - } - - @Override - public DoubleArrayList, AbstractNode> reviveStacks(int[] input, - int location, - Stack> unexpandableNodes, - Stack> unmatchableLeafNodes, - DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, - DoubleStack, AbstractNode> filteredNodes) { - - ArrayList> failedNodes = new ArrayList<>(); - collectUnexpandableNodes(unexpandableNodes, failedNodes); - collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); - // handle unmatchableLeafNodes? - //collectFilteredNodes(filteredNodes, failedNodes); - - return reviveFailedNodes(input, failedNodes); - } - - private DoubleArrayList, AbstractNode> reviveNodes(int[] input, DoubleArrayList, ArrayList> recoveryNodes){ - DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList<>(); - - recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); - - for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); - ArrayList prods = recoveryNodes.getSecond(i); - - // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). - for (int j = prods.size() - 1; j >= 0; --j) { - IConstructor prod = prods.get(j); - - AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); - - int startLocation = recoveryNode.getStartLocation(); - - SkippedNode result; - if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { - result = SkippingStackNode.createResultUntilEndOfInput(null, input, startLocation); - } else { - result = SkippingStackNode.createResultUntilCharClass(null, WHITESPACE, input, startLocation); - } - - AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result); - recoverLiteral = recoverLiteral.getCleanCopy(startLocation); // This copy might not be needed - recoverLiteral.initEdges(); - EdgesSet edges = new EdgesSet<>(1); - edges.add(continuer); - - recoverLiteral.addEdges(edges, startLocation); - - continuer.setIncomingEdges(edges); - - recoveredNodes.add(recoverLiteral, recoverLiteral.getResult()); - } - } - - return recoveredNodes; - } - - // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) - // As this is experimental code, this method is extremely conservative. - // Any sharing will result in returning 'false'. - // We will need to change this strategy in the future to improve error recovery. - private boolean isTopLevelProduction(AbstractStackNode node) { - while (node != null && node.getDot() != 0) { - node = getSinglePredecessor(node); - } - - if (node != null) { - node = getSinglePredecessor(node); - return node != null && node.getStartLocation() == -1; - } - - return false; - } - - private AbstractStackNode getSinglePredecessor(AbstractStackNode node) { - IntegerObjectList> edgeMap = node.getEdges(); - if (edgeMap.size() == 1) { - EdgesSet edges = edgeMap.getValue(0); - if (edges.size() == 1) { - return edges.get(0); - } - } - - return null; - } - - private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, ArrayList> failedNodes) { - DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList<>(); - - for (int i = failedNodes.size() - 1; i >= 0; --i) { - findRecoveryNodes(failedNodes.get(i), recoveryNodes); - } - - return reviveNodes(input, recoveryNodes); - } - - private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { - for (int i = unexpandableNodes.getSize() - 1; i >= 0; --i) { - failedNodes.add(unexpandableNodes.get(i)); - } - } - - /** - * Make a fresh copy of each unmatchable mid-production node and link in the predecessors of the original node. - * The new copies are added to `failedNodes` - * @param location the location where the failure occurs - * @param unmatchableMidProductionNodes each pair consists of a list of predecessors and a node that failed to match - * @param failedNodes the list to which failed nodes must be added - */ - private static void collectUnmatchableMidProductionNodes(int location, DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ - for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { - DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); - AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference updates of the static version - - // Merge the information on the predecessors into the failed node. - for(int j = failedNodePredecessors.size() - 1; j >= 0; --j) { - AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); - AbstractNode predecessorResult = failedNodePredecessors.getSecond(j); - failedNode.updateNode(predecessor, predecessorResult); - } - - failedNodes.add(failedNode); - } - } - - /** - * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. - */ - private void findRecoveryNodes(AbstractStackNode failer, DoubleArrayList, ArrayList> recoveryNodes) { - ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap<>(); - Stack> todo = new Stack<>(); - - todo.push(failer); - - while (!todo.isEmpty()) { - AbstractStackNode node = todo.pop(); - - if (visited.contains(node)) { - continue; // Don't follow cycles - } - - visited.put(node, 0); - - ArrayList recoveryProductions = new ArrayList<>(); - collectProductions(node, recoveryProductions); - if (recoveryProductions.size() > 0) { - recoveryNodes.add(node, recoveryProductions); - } - - IntegerObjectList> edges = node.getEdges(); - - for (int i = edges.size() - 1; i >= 0; --i) { // Rewind - EdgesSet edgesList = edges.getValue(i); - - if (edgesList != null) { - for (int j = edgesList.size() - 1; j >= 0; --j) { - AbstractStackNode parent = edgesList.get(j); - - todo.push(parent); - } - } - } - } - } - - // Gathers all productions that are marked for recovery (the given node can be part of a prefix shared production) - private void collectProductions(AbstractStackNode node, ArrayList productions) { - AbstractStackNode[] production = node.getProduction(); - if (production == null) { - return; // The root node does not have a production, so ignore it. - } - - int dot = node.getDot(); - - if (node.isEndNode()) { - IConstructor parentProduction = node.getParentProduction(); - if (ProductionAdapter.isContextFree(parentProduction)){ - productions.add(parentProduction); - - if (ProductionAdapter.isList(parentProduction)) { - return; // Don't follow productions in lists productions, since they are 'cyclic'. - } - } - } - - for (int i = dot + 1; i < production.length; ++i) { - AbstractStackNode currentNode = production[i]; - if (currentNode.isEndNode()) { - IConstructor parentProduction = currentNode.getParentProduction(); - if (ProductionAdapter.isContextFree(parentProduction)) { - productions.add(parentProduction); - } - } - - AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); - if (alternateProductions != null) { - for (int j = alternateProductions.length - 1; j >= 0; --j) { - collectProductions(alternateProductions[j][i], productions); - } - } - } - } -} diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 0c66f239929..72d9188e1e2 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -1,23 +1,19 @@ /******************************************************************************* - * Copyright (c) 2009-2024 NWO-I Centrum Wiskunde & Informatica (CWI) - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at + * Copyright (c) 2009-2024 NWO-I Centrum Wiskunde & Informatica (CWI) All rights reserved. This + * program and the accompanying materials are made available under the terms of the Eclipse Public + * License v1.0 which accompanies this distribution, and is available at * http://www.eclipse.org/legal/epl-v10.html * * Contributors: - - * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI - * * Arnold Lankamp - Arnold.Lankamp@cwi.nl + * + * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI * Arnold Lankamp - Arnold.Lankamp@cwi.nl *******************************************************************************/ package org.rascalmpl.parser.uptr.recovery; import java.net.URI; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.function.Consumer; import org.rascalmpl.parser.gtd.ExpectsProvider; import org.rascalmpl.parser.gtd.recovery.IRecoverer; @@ -43,8 +39,6 @@ import org.rascalmpl.values.parsetrees.ProductionAdapter; import io.usethesource.vallang.IConstructor; -import io.usethesource.vallang.IList; -import io.usethesource.vallang.IValue; public class ToTokenRecoverer implements IRecoverer { private URI uri; @@ -58,14 +52,14 @@ public ToTokenRecoverer(URI uri, ExpectsProvider expectsProvider, } @Override - public DoubleArrayList, AbstractNode> reviveStacks(int[] input, - int location, + public DoubleArrayList, AbstractNode> reviveStacks(int[] input, int location, Stack> unexpandableNodes, Stack> unmatchableLeafNodes, DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, DoubleStack, AbstractNode> filteredNodes) { - // For now we ignore unmatchable leaf nodes and filtered nodes. At some point we might use those to improve error recovery. + // For now we ignore unmatchable leaf nodes and filtered nodes. At some point we might use those to + // improve error recovery. ArrayList> failedNodes = new ArrayList<>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); @@ -74,11 +68,13 @@ public DoubleArrayList, AbstractNode> reviveStac return reviveFailedNodes(input, failedNodes); } - private DoubleArrayList, AbstractNode> reviveNodes(int[] input, DoubleArrayList, ArrayList> recoveryNodes){ + private DoubleArrayList, AbstractNode> reviveNodes(int[] input, + DoubleArrayList, ArrayList> recoveryNodes) { DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList<>(); // Sort nodes by start location - recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); + recoveryNodes + .sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); ParseStateVisualizer visualizer = new ParseStateVisualizer("Recovery"); visualizer.visualizeRecoveryNodes(recoveryNodes); @@ -89,13 +85,16 @@ private DoubleArrayList, AbstractNode> reviveNod int startLocation = recoveryNode.getStartLocation(); - // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). + // Handle every possible continuation associated with the recovery node (there can be more then one + // because of prefix-sharing). for (int j = prods.size() - 1; j >= 0; --j) { IConstructor prod = prods.get(j); - List> skippingNodes = findSkippingNodes(input, recoveryNode, prod, startLocation); + List> skippingNodes = + findSkippingNodes(input, recoveryNode, prod, startLocation); for (SkippingStackNode skippingNode : skippingNodes) { - AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); + AbstractStackNode continuer = + new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); EdgesSet edges = new EdgesSet<>(1); edges.add(continuer); @@ -112,7 +111,8 @@ private DoubleArrayList, AbstractNode> reviveNod return recoveredNodes; } - List> findSkippingNodes(int[] input, AbstractStackNode recoveryNode, IConstructor prod, int startLocation) { + private List> findSkippingNodes(int[] input, + AbstractStackNode recoveryNode, IConstructor prod, int startLocation) { List> nodes = new java.util.ArrayList<>(); SkippedNode result; @@ -128,10 +128,10 @@ List> findSkippingNodes(int[] input, AbstractSta // This often creates hopeless recovery attempts, but it might help in some cases. // Further experimentation should quantify this statement. /* - result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, dot); - if (result != null) { - nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); - }*/ + * result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, + * dot); if (result != null) { nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), + * prod, result, startLocation)); } + */ // Find the last token of this production and skip until after that List endMatchers = findEndMatchers(recoveryNode); @@ -156,23 +156,8 @@ List> findSkippingNodes(int[] input, AbstractSta return nodes; } - // Gather matchers for the last token of a production - List findEndMatchersBasedOnProduction(IConstructor prod) { - IList args = (IList) prod.get(1); - if (args.isEmpty()) { - return Collections.emptyList(); - } - - IValue last = args.get(args.length()-1); - if (last instanceof IConstructor) { - return Collections.singletonList(InputMatcher.createMatcher((IConstructor) last)); - } - - return Collections.emptyList(); - } - // Find matchers for the last token of the current stack node - List findEndMatchers(AbstractStackNode stackNode) { + private List findEndMatchers(AbstractStackNode stackNode) { final List matchers = new java.util.ArrayList<>(); AbstractStackNode[] prod = stackNode.getProduction(); @@ -181,8 +166,9 @@ List findEndMatchers(AbstractStackNode stackNode) { return matchers; } - void addEndMatchers(AbstractStackNode[] prod, int dot, List matchers, Set visitedNodes) { - if (prod == null || dot < 0) { + private void addEndMatchers(AbstractStackNode[] prod, int dot, List matchers, + Set visitedNodes) { + if (prod == null || dot < 0 || dot >= prod.length) { return; } @@ -219,20 +205,6 @@ public Void visit(NonTerminalStackNode nonTerminal) { }); } - void forAllParents(AbstractStackNode stackNode, Consumer> consumer) { - IntegerObjectList> edges = stackNode.getEdges(); - if (edges != null) { - for (int i = edges.size() - 1; i >= 0; --i) { - EdgesSet edgesList = edges.getValue(i); - if (edgesList != null) { - for (int j = edgesList.size() - 1; j >= 0; --j) { - consumer.accept(edgesList.get(j)); - } - } - } - } - } - private AbstractStackNode getSingleParentStack(AbstractStackNode stackNode) { if (stackNode == null) { return null; @@ -250,7 +222,7 @@ private AbstractStackNode getSingleParentStack(AbstractStackNode findNextMatchers(AbstractStackNode stackNode) { + private List findNextMatchers(AbstractStackNode stackNode) { final List matchers = new java.util.ArrayList<>(); // Future improvement: use all parents instead of just one @@ -264,14 +236,16 @@ List findNextMatchers(AbstractStackNode stackNode) { return matchers; } - private void addNextMatchers(AbstractStackNode[] prod, int dot, List matchers, Set visitedNodes) { - if (prod == null || dot >= prod.length) { + private void addNextMatchers(AbstractStackNode[] prod, int dot, List matchers, + Set visitedNodes) { + if (prod == null || dot < 0 || dot >= prod.length) { return; } AbstractStackNode next = prod[dot]; while (next instanceof NonTerminalStackNode && next.getName().startsWith("layouts_")) { - // Look "through" layout for now, this should really be more general and look through any node that can be empty + // Look "through" layout for now, this should really be more general and look through any node that + // can be empty // When a node can be empty, we should also consider all prefix-shared alternatives. dot++; if (dot >= prod.length) { @@ -292,29 +266,36 @@ public Void visit(LiteralStackNode literal) { return null; } + @Override + public Void visit(CaseInsensitiveLiteralStackNode literal) { matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); return null; } @Override + public Void visit(NonTerminalStackNode nonTerminal) { String name = nonTerminal.getName(); AbstractStackNode[] alternatives = expectsProvider.getExpects(name); for (AbstractStackNode alternative : alternatives) { addNextMatchers(alternative.getProduction(), 0, matchers, visitedNodes); } + return null; } + }); } - // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) + // Check if a node is a top-level production (i.e., its parent production node has no parents and + // starts at position -1) // As this is experimental code, this method is extremely conservative. // Any sharing will result in returning 'false'. // We will need to change this strategy in the future to improve error recovery. private boolean isTopLevelProduction(AbstractStackNode node) { + while (node != null && node.getDot() != 0) { node = getSinglePredecessor(node); } @@ -339,8 +320,11 @@ private AbstractStackNode getSinglePredecessor(AbstractStackNode, AbstractNode> reviveFailedNodes(int[] input, ArrayList> failedNodes) { - DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList<>(); + + private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, + ArrayList> failedNodes) { + DoubleArrayList, ArrayList> recoveryNodes = + new DoubleArrayList<>(); for (int i = failedNodes.size() - 1; i >= 0; --i) { findRecoveryNodes(failedNodes.get(i), recoveryNodes); @@ -349,23 +333,31 @@ private DoubleArrayList, AbstractNode> reviveFai return reviveNodes(input, recoveryNodes); } - private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { + private static void collectUnexpandableNodes(Stack> unexpandableNodes, + ArrayList> failedNodes) { for (int i = unexpandableNodes.getSize() - 1; i >= 0; --i) { failedNodes.add(unexpandableNodes.get(i)); } } /** - * Make a fresh copy of each unmatchable mid-production node and link in the predecessors of the original node. - * The new copies are added to `failedNodes` + * Make a fresh copy of each unmatchable mid-production node and link in the predecessors of the + * original node. The new copies are added to `failedNodes` + * * @param location the location where the failure occurs - * @param unmatchableMidProductionNodes each pair consists of a list of predecessors and a node that failed to match + * @param unmatchableMidProductionNodes each pair consists of a list of predecessors and a node that + * failed to match * @param failedNodes the list to which failed nodes must be added */ - private static void collectUnmatchableMidProductionNodes(int location, DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ + private static void collectUnmatchableMidProductionNodes(int location, + DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, + ArrayList> failedNodes) { for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { - DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); - AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference updates of the static version + DoubleArrayList, AbstractNode> failedNodePredecessors = + unmatchableMidProductionNodes.getFirst(i); + AbstractStackNode failedNode = + unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference + // updates of the static version // Merge the information on the predecessors into the failed node. for(int j = failedNodePredecessors.size() - 1; j >= 0; --j) { @@ -381,7 +373,8 @@ private static void collectUnmatchableMidProductionNodes(int location, DoubleSta /** * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. */ - private void findRecoveryNodes(AbstractStackNode failer, DoubleArrayList, ArrayList> recoveryNodes) { + private void findRecoveryNodes(AbstractStackNode failer, + DoubleArrayList, ArrayList> recoveryNodes) { ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap<>(); Stack> todo = new Stack<>(); @@ -419,28 +412,33 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr } // Only add recovery nodes that are not already present. - private void addRecoveryNode(AbstractStackNode node, ArrayList productions, DoubleArrayList, ArrayList> recoveryNodes) { + private void addRecoveryNode(AbstractStackNode node, ArrayList productions, + DoubleArrayList, ArrayList> recoveryNodes) { for (int i=0; i prods = recoveryNodes.getSecond(i); - if (prods.size() == productions.size()) { - boolean equal = true; - for (int j=0; equal && j prods1, ArrayList prods2) { + if (prods1.size() != prods2.size()) { + return false; } + + for (int j = 0; j < prods1.size(); j++) { + if (prods1.get(j) != prods2.get(j)) { + return false; } } - recoveryNodes.add(node, productions); + return true; } - // Gathers all productions that are marked for recovery (the given node can be part of a prefix shared production) + // Gathers all productions that are marked for recovery (the given node can be part of a prefix + // shared production) private void collectProductions(AbstractStackNode node, ArrayList productions) { AbstractStackNode[] production = node.getProduction(); if (production == null) { diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index 9b7049efec2..64f8149fd2b 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -30,7 +30,7 @@ public static String prodToString(IConstructor prod) { StringBuilder builder = new StringBuilder("'"); IConstructor sort = (IConstructor) prod.get(0); - builder.append(stripQuotes(String.valueOf(sort.get(0)))); + builder.append(quotedStringToPlain(String.valueOf(sort.get(0)))); builder.append(" ->"); @@ -39,7 +39,7 @@ public static String prodToString(IConstructor prod) { for (IValue child : children) { builder.append(" "); IConstructor conChild = (IConstructor) child; - builder.append(stripQuotes(String.valueOf((conChild).get(0)))); + builder.append(quotedStringToPlain(String.valueOf((conChild).get(0)))); } } else { builder.append(" "); @@ -51,7 +51,7 @@ public static String prodToString(IConstructor prod) { return builder.toString(); } - private static String stripQuotes(String s) { + private static String quotedStringToPlain(String s) { if (s.charAt(0) == '"' && s.charAt(s.length()-1) == '"') { return s.substring(1, s.length()-1).replace("\\", ""); } diff --git a/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java b/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java index faf6e762a0b..0232737011a 100644 --- a/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java +++ b/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java @@ -107,10 +107,10 @@ public void run() { } } - private String name; - private File basePath; - private File frameDir; - private Map stackNodeNodes; + private final String name; + private final File basePath; + private final File frameDir; + private final Map stackNodeNodes; private DotGraph graph; private int frame; @@ -382,19 +382,14 @@ private void addParserNode(DotGraph graph, AbstractNode parserNode) { } private void enrichCharNode(DotNode dotNode, CharNode charNode) { - String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); int c = charNode.getCharacter(); - label += "\nchar=" + c + "('" + (char) c + "')"; + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL) + "\nchar=" + c + "('" + (char) c + "')"; dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); } private void enrichLiteralNode(DotNode dotNode, LiteralNode literalNode) { - String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); int[] content = literalNode.getContent(); - label += " \"" + UnicodeConverter.unicodeArrayToString(content) + "\""; - /* Maybe include production? - label += "\nprod=" + DebugUtil.prodToString((IConstructor)literalNode.getProduction()); - */ + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL) + " \"" + UnicodeConverter.unicodeArrayToString(content) + "\""; dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); } diff --git a/src/org/rascalmpl/util/visualize/dot/CompassPoint.java b/src/org/rascalmpl/util/visualize/dot/CompassPoint.java index 06b85c814ab..5f4c475c119 100644 --- a/src/org/rascalmpl/util/visualize/dot/CompassPoint.java +++ b/src/org/rascalmpl/util/visualize/dot/CompassPoint.java @@ -14,6 +14,8 @@ package org.rascalmpl.util.visualize.dot; + + // Note: C means Center, X means unspecified. public enum CompassPoint { N, NE, E, SE, S, SW, W, NW, C, X; } diff --git a/src/org/rascalmpl/util/visualize/dot/NodeId.java b/src/org/rascalmpl/util/visualize/dot/NodeId.java index c3f4ec711db..ff1c35811e8 100644 --- a/src/org/rascalmpl/util/visualize/dot/NodeId.java +++ b/src/org/rascalmpl/util/visualize/dot/NodeId.java @@ -15,6 +15,7 @@ package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; +import java.util.Objects; public class NodeId { public static void writeId(PrintWriter stream, String id) { @@ -104,21 +105,10 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) return false; NodeId other = (NodeId) obj; - if (id == null) { - if (other.id != null) - return false; - } - else if (!id.equals(other.id)) - return false; - if (portId == null) { - if (other.portId != null) - return false; - } - else if (!portId.equals(other.portId)) - return false; - if (direction != other.direction) - return false; - return true; + + return Objects.equals(id, other.id) + && Objects.equals(portId, other.portId) + && direction == other.direction; } @Override diff --git a/src/org/rascalmpl/util/visualize/replay.html b/src/org/rascalmpl/util/visualize/replay.html index 891778454b5..23e9f700ed0 100644 --- a/src/org/rascalmpl/util/visualize/replay.html +++ b/src/org/rascalmpl/util/visualize/replay.html @@ -11,7 +11,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> - +