{ + AbstractStackNode
[] getExpects(String nonTerminal);
+}
diff --git a/src/org/rascalmpl/parser/gtd/IGTD.java b/src/org/rascalmpl/parser/gtd/IGTD.java
index bb4e9b5c656..0e61d06b213 100644
--- a/src/org/rascalmpl/parser/gtd/IGTD.java
+++ b/src/org/rascalmpl/parser/gtd/IGTD.java
@@ -22,7 +22,7 @@
/**
* Parser interface.
*/
-public interface IGTD {
/**
* Parse the input string, using the given non-terminal as start node. If
* the parse process successfully completes a result will be constructed
diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java
index b0232bacf3e..9d38723078f 100755
--- a/src/org/rascalmpl/parser/gtd/SGTDBF.java
+++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java
@@ -1,18 +1,16 @@
/*******************************************************************************
- * Copyright (c) 2009-2013 CWI
- * All rights reserved. This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License v1.0
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/epl-v10.html
+ * Copyright (c) 2009-2013 CWI All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v1.0 which accompanies this
+ * distribution, and is available at http://www.eclipse.org/legal/epl-v10.html
*
- * Contributors:
- * * Arnold Lankamp - Arnold.Lankamp@cwi.nl
+ * Contributors: * Arnold Lankamp - Arnold.Lankamp@cwi.nl
*******************************************************************************/
package org.rascalmpl.parser.gtd;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URI;
+import java.util.Map;
import org.rascalmpl.parser.gtd.debug.IDebugListener;
import org.rascalmpl.parser.gtd.exception.ParseError;
@@ -34,6 +32,7 @@
import org.rascalmpl.parser.gtd.stack.AbstractStackNode;
import org.rascalmpl.parser.gtd.stack.EpsilonStackNode;
import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode;
+import org.rascalmpl.parser.gtd.stack.SkippingStackNode;
import org.rascalmpl.parser.gtd.stack.edge.EdgesSet;
import org.rascalmpl.parser.gtd.stack.filter.ICompletionFilter;
import org.rascalmpl.parser.gtd.stack.filter.IEnterFilter;
@@ -45,33 +44,84 @@
import org.rascalmpl.parser.gtd.util.IntegerList;
import org.rascalmpl.parser.gtd.util.IntegerObjectList;
import org.rascalmpl.parser.gtd.util.Stack;
+import org.rascalmpl.parser.uptr.debug.NopDebugListener;
+import org.rascalmpl.parser.util.DebugUtil;
+import org.rascalmpl.parser.util.ParseStateVisualizer;
+import org.rascalmpl.util.visualize.dot.NodeId;
+import org.rascalmpl.values.RascalValueFactory;
+import org.rascalmpl.values.parsetrees.ITree;
+import org.rascalmpl.values.parsetrees.TreeAdapter;
+
+import io.usethesource.vallang.IConstructor;
+import io.usethesource.vallang.IList;
+import io.usethesource.vallang.ISet;
+import io.usethesource.vallang.IValue;
+import io.usethesource.vallang.type.Type;
/**
* This is the core of the parser; it drives the parse process.
*/
-public abstract class SGTDBF > stacksWithNonTerminalsToReduce;
+
+ // The current stack of non-terminals to reduce: it contains the matchable node with the smallest
+ // length from todoLists.
+ // - Nodes are removed in `reduceTerminals` where all productions are advanced one dot past the
+ // matchable node
+ // - Variable is assigned in:
+ // - findFirstStacksToReduce: the first non-empty `todoList` is assigned to this variable
+ // - findStacksToReduce: again the first non-empty `todoList` is assigned to this variable
+ // - parse: variable is used in main reduce/expand loop to determine when it is time to look for
+ // more `stacksToReduce`.
private DoubleStack > unmatchableMidProductionNodes;
private final DoubleStack recoverer;
+ private Map debugListener;
+ private ParseStateVisualizer visualizer;
// Temporary instrumentation for accurate profiling
private long timestamp;
private boolean printTimes = false;
+
public SGTDBF(){
super();
-
+
positionStore = new PositionStore();
stacksToExpand = new Stack >();
+ unmatchableMidProductionNodes =
+ new DoubleStack >();
filteredNodes = new DoubleStack [] invokeExpects(AbstractStackNode nonTerminal){
- String name = nonTerminal.getName();
- AbstractStackNode [] expects = expectCache.get(name);
- if(expects == null){
+ public AbstractStackNode [] getExpects(String nonTerminal) {
+ AbstractStackNode [] expects = expectCache.get(nonTerminal);
+ if (expects == null) {
try{
- Method method = getClass().getMethod(name);
- try{
+ Method method = getClass().getMethod(nonTerminal);
+ try {
method.setAccessible(true); // Try to bypass the 'isAccessible' check to save time.
- }catch(SecurityException sex){
+ }
+ catch (SecurityException sex) {
// Ignore this if it happens.
}
expects = (AbstractStackNode []) method.invoke(this);
- }catch(NoSuchMethodException nsmex){
- throw new UndeclaredNonTerminalException(name, getClass());
- }catch(IllegalAccessException iaex){
+ }
+ catch (NoSuchMethodException nsmex) {
+ throw new UndeclaredNonTerminalException(nonTerminal, getClass());
+ }
+ catch (IllegalAccessException iaex) {
throw new RuntimeException(iaex);
- }catch(InvocationTargetException itex){
+ }
+ catch (InvocationTargetException itex) {
throw new RuntimeException(itex.getTargetException());
}
- expectCache.putUnsafe(name, expects);
+ expectCache.putUnsafe(nonTerminal, expects);
}
-
- return expects;
+
+ return expects;
+ }
+
+ /**
+ * Triggers the gathering of alternatives for the given non-terminal.
+ */
+ protected AbstractStackNode [] invokeExpects(AbstractStackNode nonTerminal){
+ return getExpects(nonTerminal.getName());
}
/**
* Moves to the next symbol in the production.
*/
- private AbstractStackNode updateNextNode(AbstractStackNode next, AbstractStackNode node, AbstractNode result){
- IntegerKeyedDoubleValueHashMap.Entry updateNextNode(AbstractStackNode next, AbstractStackNode node,
+ AbstractNode result) {
+ IntegerKeyedDoubleValueHashMap.Entry updateNextNode(AbstractStackNode next, AbstractS
// Encountered a possible stack 'overtake'.
if(node.getStartLocation() != location){
propagateEdgesAndPrefixes(node, result, alternative, alternative.getResult());
- }else{
- propagateEdgesAndPrefixesForNullable(node, result, alternative, alternative.getResult(), node.getEdges().size());
+ }
+ else {
+ propagateEdgesAndPrefixesForNullable(node, result, alternative, alternative.getResult(),
+ node.getEdges().size());
}
return alternative;
}
- }else{
+ }
+ else {
EdgesSet alternativeEdgesSet = alternative.getIncomingEdges();
int resultStoreId = getResultStoreId(alternative.getId());
- if(alternativeEdgesSet != null && alternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location){
+ if (alternativeEdgesSet != null
+ && alternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location) {
// Encountered a possible stack 'overtake'.
if(node.getStartLocation() != location){
- propagateEdgesAndPrefixes(node, result, alternative, alternativeEdgesSet.getLastResult(resultStoreId));
- }else{
- propagateEdgesAndPrefixesForNullable(node, result, alternative, alternativeEdgesSet.getLastResult(resultStoreId), node.getEdges().size());
+ propagateEdgesAndPrefixes(node, result, alternative,
+ alternativeEdgesSet.getLastResult(resultStoreId));
+ }
+ else {
+ propagateEdgesAndPrefixesForNullable(node, result, alternative,
+ alternativeEdgesSet.getLastResult(resultStoreId), node.getEdges().size());
}
return alternative;
}
@@ -203,42 +276,47 @@ private AbstractStackNode updateNextNode(AbstractStackNode next, AbstractS
alternative.updateNode(node, result);
- if(debugListener != null) debugListener.progressed(node, result, alternative);
+ debugListener.progressed(node, result, alternative);
return alternative;
}
if(next.isMatchable()){ // Eager matching optimization.
- if((location + next.getLength()) > input.length) return null;
+ if ((location + next.getLength()) > input.length)
+ return null;
AbstractNode nextResult = next.match(input, location);
if(nextResult == null){
- // Push the node including it's predecessor to the appropriate error tracking collection (and take care of merging when necessary).
- DoubleArrayList updateNextNode(AbstractStackNode next, AbstractS
/**
* Moves to the next symbol in an alternative continuation of a prefix-shared production.
*/
- private boolean updateAlternativeNextNode(AbstractStackNode next, AbstractStackNode node, AbstractNode result, IntegerObjectList next, AbstractStackNode node, AbstractNode result,
+ IntegerObjectList next, AbstractSta
if(alternative.isMatchable()){
if(alternative.isEmptyLeafNode()){
// Encountered a possible stack 'overtake'.
- propagateAlternativeEdgesAndPrefixes(node, result, alternative, alternative.getResult(), node.getEdges().size(), edgesMap, prefixesMap);
+ propagateAlternativeEdgesAndPrefixes(node, result, alternative, alternative.getResult(),
+ node.getEdges().size(), edgesMap, prefixesMap);
return true;
}
- }else{
+ }
+ else {
EdgesSet alternativeEdgesSet = alternative.getIncomingEdges();
int resultStoreId = getResultStoreId(alternative.getId());
- if(alternativeEdgesSet != null && alternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location){
+ if (alternativeEdgesSet != null
+ && alternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location) {
AbstractContainerNode nextResult = alternativeEdgesSet.getLastResult(resultStoreId);
// Encountered a possible stack 'overtake'.
- propagateAlternativeEdgesAndPrefixes(node, result, alternative, nextResult, node.getEdges().size(), edgesMap, prefixesMap);
+ propagateAlternativeEdgesAndPrefixes(node, result, alternative, nextResult,
+ node.getEdges().size(), edgesMap, prefixesMap);
return true;
}
}
}
- alternative.updatePrefixSharedNode(edgesMap, prefixesMap); // Prevent unnecessary overhead; share whenever possible.
+ alternative.updatePrefixSharedNode(edgesMap, prefixesMap); // Prevent unnecessary overhead; share whenever
+ // possible.
- if(debugListener != null) debugListener.progressed(node, result, alternative);
+ debugListener.progressed(node, result, alternative);
return true;
}
if(next.isMatchable()){ // Eager matching optimization.
- if((location + next.getLength()) > input.length) return false;
+ if ((location + next.getLength()) > input.length)
+ return false;
AbstractNode nextResult = next.match(input, location);
if(nextResult == null){
- // Push the node including it's predecessor to the appropriate error tracking collection (and take care of merging when necessary).
- DoubleArrayList next, AbstractSta
}
/**
- * Part of the hidden-right-recursion fix.
- * Executes absent reductions.
+ * Part of the hidden-right-recursion fix. Executes absent reductions.
*/
- private void propagateReductions(AbstractStackNode node, AbstractNode nodeResultStore, AbstractStackNode next, AbstractNode nextResultStore, int potentialNewEdges){
+ private void propagateReductions(AbstractStackNode node, AbstractNode nodeResultStore, AbstractStackNode next,
+ AbstractNode nextResultStore, int potentialNewEdges) {
IntegerList propagatedReductions = next.getPropagatedReductions();
IntegerObjectList node, AbstractNode nodeRes
EdgesSet edgeSet = edgesMap.getValue(i);
- if(debugListener != null) debugListener.reducing(node, resultLink, edgeSet);
+ debugListener.reducing(node, resultLink, edgeSet);
if(!hasNestingRestrictions){
handleEdgeList(edgeSet, name, production, resultLink, startLocation);
- }else{
+ }
+ else {
handleEdgeListWithRestrictions(edgeSet, name, production, resultLink, startLocation, filteredParents);
}
}
}
/**
- * Part of the hidden-right-recursion fix.
- * Propagates absent prefixes.
+ * Part of the hidden-right-recursion fix. Propagates absent prefixes.
*/
private void propagatePrefixes(AbstractStackNode next, AbstractNode nextResult, int nrOfAddedEdges){
// Proceed with the tail of the production.
int nextDot = next.getDot() + 1;
AbstractStackNode [] prod = next.getProduction();
AbstractStackNode nextNext = prod[nextDot];
- IntegerKeyedDoubleValueHashMap.Entry nextNextAlternative = null;
if(nextNextAlternativeEntry != null){ // Valid continuation.
DoubleArrayList next, AbstractNode nextResul
nextNextAlternative = nextNextAlternativeEntry.value1;
if(nextNextAlternative.isMatchable()){
if(nextNextAlternative.isEmptyLeafNode()){
- propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative, nextNextAlternative.getResult(), nrOfAddedEdges);
- }else{
+ propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative,
+ nextNextAlternative.getResult(), nrOfAddedEdges);
+ }
+ else {
nextNextAlternative.updateNode(next, nextResult);
- if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAlternative);
+ debugListener.propagated(next, nextResult, nextNextAlternative);
+ }
}
- }else{
+ else {
EdgesSet nextNextAlternativeEdgesSet = nextNextAlternative.getIncomingEdges();
int resultStoreId = getResultStoreId(nextNextAlternative.getId());
- if(nextNextAlternativeEdgesSet != null && nextNextAlternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location){
- propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative, nextNextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges);
- }else{
+ if (nextNextAlternativeEdgesSet != null
+ && nextNextAlternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location) {
+ propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative,
+ nextNextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges);
+ }
+ else {
nextNextAlternative.updateNode(next, nextResult);
- if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAlternative);
+ debugListener.propagated(next, nextResult, nextNextAlternative);
+ }
}
}
- }else{
+ else {
predecessors.add(next, nextResult);
}
}
@@ -406,12 +503,15 @@ private void propagatePrefixes(AbstractStackNode next, AbstractNode nextResul
// Handle alternative continuations (related to prefix sharing).
AbstractStackNode [][] alternateProds = next.getAlternateProductions();
if(alternateProds != null){
- if(nextNextAlternative == null){ // If the first continuation has not been initialized yet (it may be a matchable that didn't match), create a dummy version to construct the necessary edges and prefixes.
- if(!nextNext.isMatchable()) return; // Matchable, abort.
+ if (nextNextAlternative == null) { // If the first continuation has not been initialized yet (it may be a
+ // matchable that didn't match), create a dummy version to construct the
+ // necessary edges and prefixes.
+ if (!nextNext.isMatchable())
+ return; // Matchable, abort.
nextNextAlternative = nextNext.getCleanCopy(location);
nextNextAlternative.updateNode(next, nextResult);
- if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAlternative);
+ debugListener.propagated(next, nextResult, nextNextAlternative);
}
IntegerObjectList next, AbstractNode nextResul
for(int i = alternateProds.length - 1; i >= 0; --i){
prod = alternateProds[i];
- if(nextDot == prod.length) continue;
+ if (nextDot == prod.length)
+ continue;
AbstractStackNode alternativeNextNext = prod[nextDot];
- IntegerKeyedDoubleValueHashMap.Entry nextNextAltAlternative = null;
if(nextNextAltAlternativeEntry != null){
- DoubleArrayList nextAlternativeEdgesSet = nextNextAltAlternative.getIncomingEdges();
int resultStoreId = getResultStoreId(nextNextAltAlternative.getId());
- if(nextAlternativeEdgesSet != null && nextAlternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location){
- propagateAlternativeEdgesAndPrefixes(next, nextResult, nextNextAltAlternative, nextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges, nextNextEdgesMap, nextNextPrefixesMap);
- }else{
+ if (nextAlternativeEdgesSet != null
+ && nextAlternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location) {
+ propagateAlternativeEdgesAndPrefixes(next, nextResult, nextNextAltAlternative,
+ nextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges,
+ nextNextEdgesMap, nextNextPrefixesMap);
+ }
+ else {
nextNextAltAlternative.updatePrefixSharedNode(nextNextEdgesMap, nextNextPrefixesMap);
- if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAltAlternative);
+ debugListener.propagated(next, nextResult, nextNextAltAlternative);
}
}
- }else{
+ }
+ else {
predecessors.add(next, nextResult);
}
}
@@ -456,15 +568,17 @@ private void propagatePrefixes(AbstractStackNode next, AbstractNode nextResul
}
/**
- * Part of the hidden-right-recursion fix.
- * Inserts missing prefixes and triggers reductions where necessary.
+ * Part of the hidden-right-recursion fix. Inserts missing prefixes and triggers reductions where
+ * necessary.
*/
- private void propagateEdgesAndPrefixes(AbstractStackNode node, AbstractNode nodeResult, AbstractStackNode next, AbstractNode nextResult){
+ private void propagateEdgesAndPrefixes(AbstractStackNode node, AbstractNode nodeResult,
+ AbstractStackNode next, AbstractNode nextResult) {
int nrOfAddedEdges = next.updateOvertakenNode(node, nodeResult);
- if(debugListener != null) debugListener.propagated(node, nodeResult, next);
+ debugListener.propagated(node, nodeResult, next);
- if(nrOfAddedEdges == 0) return;
+ if (nrOfAddedEdges == 0)
+ return;
if(next.isEndNode()){
propagateReductions(node, nodeResult, next, nextResult, nrOfAddedEdges);
@@ -476,15 +590,17 @@ private void propagateEdgesAndPrefixes(AbstractStackNode node, AbstractNode n
}
/**
- * Part of the hidden-right-recursion fix.
- * Inserts missing prefixes and triggers reductions where necessary (specific for nullable nodes).
+ * Part of the hidden-right-recursion fix. Inserts missing prefixes and triggers reductions where
+ * necessary (specific for nullable nodes).
*/
- private void propagateEdgesAndPrefixesForNullable(AbstractStackNode node, AbstractNode nodeResult, AbstractStackNode next, AbstractNode nextResult, int potentialNewEdges){
+ private void propagateEdgesAndPrefixesForNullable(AbstractStackNode node, AbstractNode nodeResult,
+ AbstractStackNode next, AbstractNode nextResult, int potentialNewEdges) {
int nrOfAddedEdges = next.updateOvertakenNullableNode(node, nodeResult, potentialNewEdges);
- if(debugListener != null) debugListener.propagated(node, nodeResult, next);
+ debugListener.propagated(node, nodeResult, next);
- if(nrOfAddedEdges == 0) return;
+ if (nrOfAddedEdges == 0)
+ return;
if(next.isEndNode()){
propagateReductions(node, nodeResult, next, nextResult, nrOfAddedEdges);
@@ -496,15 +612,18 @@ private void propagateEdgesAndPrefixesForNullable(AbstractStackNode node, Abs
}
/**
- * Part of the hidden-right-recursion fix.
- * Inserts missing prefixes and triggers reductions where necessary (specifically for alternative continuations of prefix-shared productions).
+ * Part of the hidden-right-recursion fix. Inserts missing prefixes and triggers reductions where
+ * necessary (specifically for alternative continuations of prefix-shared productions).
*/
- private void propagateAlternativeEdgesAndPrefixes(AbstractStackNode node, AbstractNode nodeResult, AbstractStackNode next, AbstractNode nextResult, int potentialNewEdges, IntegerObjectList node, AbstractNode nodeResult,
+ AbstractStackNode next, AbstractNode nextResult, int potentialNewEdges,
+ IntegerObjectList node, AbstractNode result){
EdgesSet edgeSet = edgesMap.getValue(i);
- if(debugListener != null) debugListener.reducing(node, resultLink, edgeSet);
+ debugListener.reducing(node, resultLink, edgeSet);
- if(!hasNestingRestrictions){ // Select the optimized path for handling edge sets that don't have nesting restrictions associated with them.
+ if (!hasNestingRestrictions) { // Select the optimized path for handling edge sets that don't have nesting
+ // restrictions associated with them.
handleEdgeList(edgeSet, name, production, resultLink, edgesMap.getKey(i));
- }else{
- handleEdgeListWithRestrictions(edgeSet, name, production, resultLink, edgesMap.getKey(i), filteredParents);
+ }
+ else {
+ handleEdgeListWithRestrictions(edgeSet, name, production, resultLink, edgesMap.getKey(i),
+ filteredParents);
}
}
}
@@ -571,18 +693,21 @@ private void updateNullableEdges(AbstractStackNode node, AbstractNode result)
for(int i = edgesMap.size() - 1; i >= 0; --i){
int startLocation = edgesMap.getKey(i);
- if(propagatedReductions.containsBefore(startLocation, initialSize)) continue; // Prevent duplicate reductions (artifact of the hidden-right-recursion fix).
+ if (propagatedReductions.containsBefore(startLocation, initialSize))
+ continue; // Prevent duplicate reductions (artifact of the hidden-right-recursion fix).
propagatedReductions.add(startLocation);
Link resultLink = new Link((prefixesMap != null) ? prefixesMap[i] : null, result);
EdgesSet edgeSet = edgesMap.getValue(i);
- if(debugListener != null) debugListener.reducing(node, resultLink, edgeSet);
+ debugListener.reducing(node, resultLink, edgeSet);
- if(!hasNestingRestrictions){ // Select the optimized path for handling edge sets that don't have nesting restrictions associated with them.
+ if (!hasNestingRestrictions) { // Select the optimized path for handling edge sets that don't have nesting
+ // restrictions associated with them.
handleEdgeList(edgeSet, name, production, resultLink, startLocation);
- }else{
+ }
+ else {
handleEdgeListWithRestrictions(edgeSet, name, production, resultLink, startLocation, filteredParents);
}
}
@@ -599,26 +724,31 @@ private void handleEdgeList(EdgesSet edgeSet, String name, P production, Link
if(edge.isRecovered()){
resultStore = new RecoveredNode (inputURI, startLocation, location);
- }else if(edge.isExpandable()){
- resultStore = new ExpandableContainerNode (inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout());
- }else{
- resultStore = new SortContainerNode (inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout());
+ }
+ else if (edge.isExpandable()) {
+ resultStore = new ExpandableContainerNode (inputURI, startLocation, location,
+ startLocation == location, edge.isSeparator(), edge.isLayout());
+ }
+ else {
+ resultStore = new SortContainerNode (inputURI, startLocation, location, startLocation == location,
+ edge.isSeparator(), edge.isLayout());
}
stacksWithNonTerminalsToReduce.push(edge, resultStore);
- if(debugListener != null) debugListener.reduced(edge);
+ debugListener.reduced(edge);
for(int j = edgeSet.size() - 1; j >= 1; --j){
edge = edgeSet.get(j);
stacksWithNonTerminalsToReduce.push(edge, resultStore);
- if(debugListener != null) debugListener.reduced(edge);
+ debugListener.reduced(edge);
}
edgeSet.setLastVisitedLevel(location, resultStoreId);
edgeSet.setLastResult(resultStore, resultStoreId);
- }else{
+ }
+ else {
resultStore = edgeSet.getLastResult(resultStoreId);
}
@@ -632,9 +762,11 @@ private void handleEdgeList(EdgesSet edgeSet, String name, P production, Link
/**
* Handles reductions which may be associated with nesting restrictions.
*/
- private void handleEdgeListWithRestrictions(EdgesSet edgeSet, String name, P production, Link resultLink, int startLocation, IntegerList filteredParents){
+ private void handleEdgeListWithRestrictions(EdgesSet edgeSet, String name, P production, Link resultLink,
+ int startLocation, IntegerList filteredParents) {
// Only add the result to each resultstore once.
- // Make sure each edge only gets added to the non-terminal reduction list once per level, by keeping track of them.
+ // Make sure each edge only gets added to the non-terminal reduction list once per level, by keeping
+ // track of them.
firstTimeRegistration.clear();
firstTimeReductions.clear();
for(int j = edgeSet.size() - 1; j >= 0; --j){
@@ -643,7 +775,7 @@ private void handleEdgeListWithRestrictions(EdgesSet edgeSet, String name, P
if(!firstTimeReductions.contains(resultStoreId)){
if(firstTimeRegistration.contains(resultStoreId)){
- if(debugListener != null) debugListener.filteredByNestingRestriction(edge);
+ debugListener.filteredByNestingRestriction(edge);
continue;
}
@@ -658,10 +790,14 @@ private void handleEdgeListWithRestrictions(EdgesSet edgeSet, String name, P
if(resultStore == null){
if (edge.isRecovered()) {
resultStore = new RecoveredNode (inputURI, startLocation, location);
- }else if (edge.isExpandable()) {
- resultStore = new ExpandableContainerNode (inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout());
- }else {
- resultStore = new SortContainerNode (inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout());
+ }
+ else if (edge.isExpandable()) {
+ resultStore = new ExpandableContainerNode (inputURI, startLocation, location,
+ startLocation == location, edge.isSeparator(), edge.isLayout());
+ }
+ else {
+ resultStore = new SortContainerNode (inputURI, startLocation, location,
+ startLocation == location, edge.isSeparator(), edge.isLayout());
}
edgeSet.setLastVisitedLevel(location, resultStoreId);
@@ -673,11 +809,13 @@ private void handleEdgeListWithRestrictions(EdgesSet edgeSet, String name, P
resultStore.addAlternative(production, resultLink);
- if(debugListener != null) debugListener.reduced(edge);
- }else{
- if(debugListener != null) debugListener.filteredByNestingRestriction(edge);
+ debugListener.reduced(edge);
+ }
+ else {
+ debugListener.filteredByNestingRestriction(edge);
}
- }else{
+ }
+ else {
AbstractContainerNode resultStore = edgeSet.getLastResult(resultStoreId);
stacksWithNonTerminalsToReduce.push(edge, resultStore);
}
@@ -706,12 +844,14 @@ private void moveToNext(AbstractStackNode node, AbstractNode result){
for(int i = alternateProds.length - 1; i >= 0; --i){
prod = alternateProds[i];
- if(nextDot == prod.length) continue;
+ if (nextDot == prod.length)
+ continue;
AbstractStackNode newAlternativeNext = prod[nextDot];
if(edgesMap != null){
updateAlternativeNextNode(newAlternativeNext, node, result, edgesMap, prefixesMap);
- }else{
+ }
+ else {
AbstractStackNode alternativeNext = updateNextNode(newAlternativeNext, node, result);
if(alternativeNext != null){
@@ -724,12 +864,12 @@ private void moveToNext(AbstractStackNode node, AbstractNode result){
}
/**
- * Progress to the next 'states' associated with the given node.
- * I.e. move to the next symbol(s) in the production (if available) and executed reductions if necessary.
+ * Progress to the next 'states' associated with the given node. I.e. move to the next symbol(s) in
+ * the production (if available) and executed reductions if necessary.
*/
private void move(AbstractStackNode node, AbstractNode result){
- if(debugListener != null) debugListener.moving(node, result);
-
+ debugListener.moving(node, result);
+
// Handle filtering.
ICompletionFilter[] completionFilters = node.getCompletionFilters();
if(completionFilters != null){
@@ -738,7 +878,7 @@ private void move(AbstractStackNode node, AbstractNode result){
if(completionFilters[i].isFiltered(input, startLocation, location, positionStore)){
filteredNodes.push(node, result);
- if(debugListener != null) debugListener.filteredByCompletionFilter(node, result);
+ debugListener.filteredByCompletionFilter(node, result);
return;
}
@@ -746,9 +886,11 @@ private void move(AbstractStackNode node, AbstractNode result){
}
if(node.isEndNode()){
- if(!result.isEmpty() || node.getId() == AbstractExpandableStackNode.DEFAULT_LIST_EPSILON_ID){ // Only go into the nullable fix path for nullables (special list epsilons can be ignored as well).
+ // Only go into the nullable fix path for nullables (special list epsilons can be ignored as well).
+ if (!result.isEmpty() || node.getId() == AbstractExpandableStackNode.DEFAULT_LIST_EPSILON_ID) {
updateEdges(node, result);
- }else{
+ }
+ else {
updateNullableEdges(node, result);
}
}
@@ -761,20 +903,25 @@ private void move(AbstractStackNode node, AbstractNode result){
/**
* Initiate the handling of stacks.
*/
- private void reduce(){
+ private void reduceTerminals() {
// Reduce terminals
+ visualize("Reducing terminals", ParseStateVisualizer.TERMINALS_TO_REDUCE_ID);
while(!stacksWithTerminalsToReduce.isEmpty()){
move(stacksWithTerminalsToReduce.peekFirst(), stacksWithTerminalsToReduce.popSecond());
}
-
+ }
+
+ private void reduceNonTerminals() {
// Reduce non-terminals
+ visualize("Reducing non-terminals", ParseStateVisualizer.NON_TERMINALS_TO_REDUCE_ID);
while(!stacksWithNonTerminalsToReduce.isEmpty()){
move(stacksWithNonTerminalsToReduce.peekFirst(), stacksWithNonTerminalsToReduce.popSecond());
}
}
/**
- * Locates the initial set of stacks that is queued for handling, for which the least amount of characters needs to be shifted.
+ * Locates the initial set of stacks that is queued for handling, for which the least amount of
+ * characters needs to be shifted.
*/
private boolean findFirstStacksToReduce(){
for(int i = 0; i < todoLists.length; ++i){
@@ -790,32 +937,26 @@ private boolean findFirstStacksToReduce(){
}
}
- if (recoverer != null) {
- DoubleArrayList recovered = recoveredNodes.getFirst(i);
- addTodo(recovered, recovered.getLength(), recoveredNodes.getSecond(i));
- }
- return findStacksToReduce();
- }
-
- parseErrorOccured = true;
- }
-
- return false;
+ return attemptRecovery();
}
/**
- * Locates the set of stacks that is queued for handling, for which the least amount of characters needs to be shifted.
+ * Locates the set of stacks that is queued for handling, for which the least amount of characters
+ * needs to be shifted.
*/
private boolean findStacksToReduce(){
+ visualize("Finding stacks to reduce", ParseStateVisualizer.TODO_LISTS_ID);
int queueDepth = todoLists.length;
for(int i = 1; i < queueDepth; ++i){
queueIndex = (queueIndex + 1) % queueDepth;
DoubleStack recovered = recoveredNodes.getFirst(i);
-
- int levelsFromHere = recovered.getLength() - (location - recovered.getStartLocation());
-
- addTodo(recovered, levelsFromHere, recoveredNodes.getSecond(i));
+ debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes,
+ unmatchableMidProductionNodes, filteredNodes);
+ visualize("Queue recovery node", ParseStateVisualizer.getNodeId(recovered));
+ queueRecoveryNode(recovered, recovered.getStartLocation(), recovered.getLength(),
+ recoveredNodes.getSecond(i));
}
+ parseErrorRecovered = true;
return findStacksToReduce();
}
- parseErrorOccured = true;
+ parseErrorEncountered = true;
}
return false;
}
-
+
+
public boolean parseErrorHasOccurred(){
- return parseErrorOccured;
+ return parseErrorEncountered;
}
/**
- * Inserts a stack bottom into the todo-list.
+ * Inserts a stack bottom into the todoList.
*/
@SuppressWarnings("unchecked")
- private void addTodo(AbstractStackNode node, int length, AbstractNode result){
- if(result == null) throw new RuntimeException();
+ private void queueMatchableNode(AbstractStackNode node, int length, AbstractNode result){
+ assert result != null;
+
int queueDepth = todoLists.length;
if(length >= queueDepth){
DoubleStack node, int length, AbstractNode result)
int insertLocation = (queueIndex + length) % queueDepth;
DoubleStack node, int startPosition, int length, AbstractNode result){
+ assert result != null;
+
+ int queueDepth = todoLists.length;
+
+ if (startPosition < location) {
+ // Have to reset the parser to an earlier location to at least
+ // be able to process the new node. Cannot throw away the queue,
+ // because there are possibly already other recovery tokens in the queue.
+ // However, we may assume that the queue before the current index is
+ // done, based on the way we cycle the queue now. The queue is
+ // looking forward to the future and we never re-use past entries.
+
+ int negativeOffset = location - startPosition;
+
+ DoubleStack [] expects, EdgesSet cachedEdges, AbstractStackNode stackBeingWorkedOn){
+ private boolean handleExpects(AbstractStackNode [] expects, EdgesSet cachedEdges,
+ AbstractStackNode stackBeingWorkedOn) {
boolean hasValidAlternatives = false;
sharedLastExpects.dirtyClear();
@@ -886,25 +1099,26 @@ private boolean handleExpects(AbstractStackNode [] expects, EdgesSet cached
if(first.isMatchable()){ // Eager matching optimization.
int length = first.getLength();
int endLocation = location + length;
- if(endLocation > input.length) continue;
+ if (endLocation > input.length)
+ continue;
AbstractNode result = first.match(input, location);
if(result == null){
unmatchableLeafNodes.push(first);
- if(debugListener != null) debugListener.failedToMatch(first);
+ debugListener.failedToMatch(first);
continue;
}
- if(debugListener != null) debugListener.matched(first, result);
+ debugListener.matched(first, result);
// Handle filtering.
IEnterFilter[] enterFilters = first.getEnterFilters();
if(enterFilters != null){
for(int j = enterFilters.length - 1; j >= 0; --j){
if(enterFilters[j].isFiltered(input, location, positionStore)){
- if(debugListener != null) debugListener.filteredByEnterFilter(first);
+ debugListener.filteredByEnterFilter(first);
continue EXPECTS;
}
@@ -913,8 +1127,9 @@ private boolean handleExpects(AbstractStackNode [] expects, EdgesSet cached
first = first.getCleanCopyWithResult(location, result);
- addTodo(first, length, result);
- }else{
+ queueMatchableNode(first, length, result);
+ }
+ else {
first = first.getCleanCopy(location);
stacksToExpand.push(first);
}
@@ -926,7 +1141,7 @@ private boolean handleExpects(AbstractStackNode [] expects, EdgesSet cached
hasValidAlternatives = true;
- if(debugListener != null) debugListener.expanded(stackBeingWorkedOn, first);
+ debugListener.expanded(stackBeingWorkedOn, first);
}
return hasValidAlternatives;
@@ -957,7 +1172,7 @@ protected int getResultStoreId(int id){
* Expands the given stack node.
*/
private void expandStack(AbstractStackNode stack){
- if(debugListener != null) debugListener.expanding(stack);
+ debugListener.expanding(stack);
// Handle filtering.
IEnterFilter[] enterFilters = stack.getEnterFilters();
@@ -966,7 +1181,7 @@ private void expandStack(AbstractStackNode stack){
if(enterFilters[i].isFiltered(input, location, positionStore)){
unexpandableNodes.push(stack);
- if(debugListener != null) debugListener.filteredByEnterFilter(stack);
+ debugListener.filteredByEnterFilter(stack);
return;
}
@@ -974,8 +1189,9 @@ private void expandStack(AbstractStackNode stack){
}
if(stack.isMatchable()){ // Eager matching optimization related.
- addTodo(stack, stack.getLength(), stack.getResult());
- }else if(!stack.isExpandable()){ // A 'normal' non-terminal.
+ queueMatchableNode(stack, stack.getLength(), stack.getResult());
+ }
+ else if (!stack.isExpandable()) { // A 'normal' non-terminal.
EdgesSet cachedEdges = cachedEdgesForExpect.get(stack.getName());
if(cachedEdges == null){
cachedEdges = new EdgesSet (1);
@@ -991,19 +1207,21 @@ private void expandStack(AbstractStackNode stack){
unexpandableNodes.push(stack);
return;
}
- }else{
+ }
+ else {
int resultStoreId = getResultStoreId(stack.getId());
if(cachedEdges.getLastVisitedLevel(resultStoreId) == location){ // Is nullable, add the known results.
stacksWithNonTerminalsToReduce.push(stack, cachedEdges.getLastResult(resultStoreId));
- if(debugListener != null) debugListener.foundIterationCachedNullableResult(stack);
+ debugListener.foundIterationCachedNullableResult(stack);
}
}
cachedEdges.add(stack);
stack.setIncomingEdges(cachedEdges);
- }else{ // Expandable
+ }
+ else { // Expandable
EdgesSet cachedEdges = cachedEdgesForExpect.get(stack.getName());
if(cachedEdges == null){
boolean expanded = false;
@@ -1017,33 +1235,36 @@ private void expandStack(AbstractStackNode stack){
AbstractStackNode child = listChildren[i];
int childId = child.getId();
- IntegerKeyedDoubleValueHashMap.Entry sharedChild = sharedChildEntry.value1;
sharedChild.setEdgesSetWithPrefix(cachedEdges, null, location);
- }else{
+ }
+ else {
if(child.isMatchable()){
int length = child.getLength();
int endLocation = location + length;
- if(endLocation > input.length) continue;
+ if (endLocation > input.length)
+ continue;
AbstractNode result = child.match(input, location);
if(result == null){
unmatchableLeafNodes.push(child);
- if(debugListener != null) debugListener.failedToMatch(child);
+ debugListener.failedToMatch(child);
continue;
}
- if(debugListener != null) debugListener.matched(child, result);
+ debugListener.matched(child, result);
// Handle filtering
IEnterFilter[] childEnterFilters = child.getEnterFilters();
if(childEnterFilters != null){
for(int j = childEnterFilters.length - 1; j >= 0; --j){
if(childEnterFilters[j].isFiltered(input, location, positionStore)) {
- if(debugListener != null) debugListener.filteredByEnterFilter(child);
+ debugListener.filteredByEnterFilter(child);
continue CHILDREN;
}
@@ -1051,8 +1272,9 @@ private void expandStack(AbstractStackNode stack){
}
child = child.getCleanCopyWithResult(location, result);
- addTodo(child, length, result);
- }else{
+ queueMatchableNode(child, length, result);
+ }
+ else {
child = child.getCleanCopy(location);
stacksToExpand.push(child);
}
@@ -1062,20 +1284,21 @@ private void expandStack(AbstractStackNode stack){
sharedNextNodes.putUnsafe(childId, child, null);
- if(debugListener != null) debugListener.expanded(stack, child);
+ debugListener.expanded(stack, child);
}
expanded = true;
}
if(stack.canBeEmpty()){ // Star list or optional.
- AbstractStackNode empty = stack.getEmptyChild().getCleanCopyWithResult(location, EpsilonStackNode.EPSILON_RESULT);
+ AbstractStackNode empty =
+ stack.getEmptyChild().getCleanCopyWithResult(location, EpsilonStackNode.EPSILON_RESULT);
empty.initEdges();
empty.addEdges(cachedEdges, location);
stacksToExpand.push(empty);
- if(debugListener != null) debugListener.expanded(stack, empty);
+ debugListener.expanded(stack, empty);
expanded = true;
}
@@ -1089,7 +1312,7 @@ private void expandStack(AbstractStackNode stack){
if(cachedEdges.getLastVisitedLevel(resultStoreId) == location){ // Is nullable, add the known results.
stacksWithNonTerminalsToReduce.push(stack, cachedEdges.getLastResult(resultStoreId));
- if(debugListener != null) debugListener.foundIterationCachedNullableResult(stack);
+ debugListener.foundIterationCachedNullableResult(stack);
}
cachedEdges.add(stack);
@@ -1102,6 +1325,7 @@ private void expandStack(AbstractStackNode stack){
* Initiate stack expansion for all queued stacks.
*/
private void expand(){
+ visualize("Expanding", ParseStateVisualizer.STACKS_TO_EXPAND_ID);
while(!stacksToExpand.isEmpty()){
expandStack(stacksToExpand.pop());
}
@@ -1115,105 +1339,153 @@ protected AbstractNode parse(AbstractStackNode startNode, URI inputURI, int[]
* Initiates parsing.
*/
@SuppressWarnings("unchecked")
- protected AbstractNode parse(AbstractStackNode startNode, URI inputURI, int[] input, IRecoverer recoverer, IDebugListener debugListener){
- initTime();
+ protected AbstractNode parse(AbstractStackNode startNode, URI inputURI, int[] input,
+ IRecoverer recoverer, IDebugListener debugListener) {
+ if (debugListener == null) {
+ debugListener = new NopDebugListener<>();
+ }
- try {
+ initTime();
- if(invoked){
- throw new RuntimeException("Can only invoke 'parse' once.");
- }
+ try {
+ if(invoked){
+ throw new RuntimeException("Can only invoke 'parse' once.");
+ }
+
+ invoked = true;
+ // Initialize.
+ this.inputURI = inputURI;
+ this.input = input;
- invoked = true;
+ this.recoverer = recoverer;
+ this.debugListener = debugListener;
- // Initialize.
- this.inputURI = inputURI;
- this.input = input;
+ visualizer = ParseStateVisualizer.shouldVisualizeUri(inputURI) ? new ParseStateVisualizer("Parser") : null;
- this.recoverer = recoverer;
- this.debugListener = debugListener;
+ // Initialzed the position store.
+ positionStore.index(input);
- // Initialzed the position store.
- positionStore.index(input);
+ todoLists = new DoubleStack[DEFAULT_TODOLIST_CAPACITY];
- todoLists = new DoubleStack[DEFAULT_TODOLIST_CAPACITY];
+ // Handle the initial expansion of the root node.
+ AbstractStackNode rootNode = startNode;
+ rootNode.initEdges();
+ stacksToExpand.push(rootNode);
+ lookAheadChar = (input.length > 0) ? input[0] : 0;
- // Handle the initial expansion of the root node.
- AbstractStackNode rootNode = startNode;
- rootNode.initEdges();
- stacksToExpand.push(rootNode);
- lookAheadChar = (input.length > 0) ? input[0] : 0;
+ debugListener.shifting(location, input, positionStore);
- if(debugListener != null) debugListener.shifting(location, input, positionStore);
+ expand();
- expand();
+ AbstractContainerNode result = null;
+ if(findFirstStacksToReduce()) {
+ boolean shiftedLevel = (location != 0);
+ while (true) {
+ lookAheadChar = (location < input.length) ? input[location] : 0;
+ if(shiftedLevel){ // Nullable fix for the first level.
+ sharedNextNodes.clear();
+ cachedEdgesForExpect.clear();
- if(findFirstStacksToReduce()){
- boolean shiftedLevel = (location != 0);
+ unexpandableNodes.dirtyClear();
+ unmatchableLeafNodes.dirtyClear();
+ unmatchableMidProductionNodes.dirtyClear();
+ filteredNodes.dirtyClear();
- do{
- lookAheadChar = (location < input.length) ? input[location] : 0;
- if(shiftedLevel){ // Nullable fix for the first level.
- sharedNextNodes.clear();
- cachedEdgesForExpect.clear();
+ debugListener.shifting(location, input, positionStore);
+ }
- unexpandableNodes.dirtyClear();
- unmatchableLeafNodes.dirtyClear();
- unmatchableMidProductionNodes.dirtyClear();
- filteredNodes.dirtyClear();
+ // Reduce-expand loop.
+ do {
+ debugListener.iterating();
+ reduceTerminals();
+ reduceNonTerminals();
+ expand();
+ }
+ while (!stacksWithNonTerminalsToReduce.isEmpty() || !stacksWithTerminalsToReduce.isEmpty());
- if(debugListener != null) debugListener.shifting(location, input, positionStore);
- }
+ shiftedLevel = true;
- // Reduce-expand loop.
- do{
- if(debugListener != null) debugListener.iterating();
+ if (onlyRecoveredStacksLeft() && attemptRecovery()) {
+ continue;
+ }
- reduce();
+ if (!findStacksToReduce()) {
+ if(location == input.length) {
+ EdgesSet startNodeEdgesSet = startNode.getIncomingEdges();
+ int resultStoreId = getResultStoreId(startNode.getId());
+ if(startNodeEdgesSet != null && startNodeEdgesSet.getLastVisitedLevel(resultStoreId) == input.length) {
+ result = startNodeEdgesSet.getLastResult(resultStoreId); // Success.
+ break;
+ }
+ }
+ if (!attemptRecovery()) {
+ // Unsuccessful parse
+ break;
+ }
+ }
+ }
+ }
- expand();
- }while(!stacksWithNonTerminalsToReduce.isEmpty() || !stacksWithTerminalsToReduce.isEmpty());
+ visualize("Done", ParseStateVisualizer.PARSER_ID);
- shiftedLevel = true;
- }while(findStacksToReduce());
- }
+ if (result != null) {
+ return result;
+ }
+ } finally {
+ checkTime("Parsing");
+ }
- // Check if we were successful.
- if(location == input.length){
- EdgesSet startNodeEdgesSet = startNode.getIncomingEdges();
- int resultStoreId = getResultStoreId(startNode.getId());
- if(startNodeEdgesSet != null && startNodeEdgesSet.getLastVisitedLevel(resultStoreId) == input.length){
- // Parsing succeeded.
- return startNodeEdgesSet.getLastResult(resultStoreId); // Success.
- }
- }
- }
- finally {
- checkTime("Parsing");
- }
+ try {
+ // A parse error occured, and recovery failed as well
+ parseErrorEncountered = true;
- try {
- // A parse error occured, and recovery failed as well
- parseErrorOccured = true;
-
- int errorLocation = (location == Integer.MAX_VALUE ? 0 : location);
- int line = positionStore.findLine(errorLocation);
- int column = positionStore.getColumn(errorLocation, line);
- if (location == input.length) {
- throw new ParseError("Parse error", inputURI, errorLocation, 0, line + 1, line + 1, column, column, (Stack recoverer, IDebugListener debugListener){
- AbstractNode result = parse(new NonTerminalStackNode (AbstractStackNode.START_SYMBOL_ID, 0, nonterminal), inputURI, input, recoverer, debugListener);
+ private T parse(String nonterminal, URI inputURI, int[] input, IActionExecutor recoverer,
+ IDebugListener debugListener) {
+ AbstractNode result = parse(new NonTerminalStackNode (AbstractStackNode.START_SYMBOL_ID, 0, nonterminal),
+ inputURI, input, recoverer, debugListener);
return buildResult(result, converter, nodeConstructorFactory, actionExecutor);
}
- public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor recoverer, IDebugListener debugListener){
- return parse(nonterminal, inputURI, charsToInts(input), actionExecutor, converter, nodeConstructorFactory, recoverer, debugListener);
+ public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor recoverer,
+ IDebugListener debugListener) {
+ return parse(nonterminal, inputURI, charsToInts(input), actionExecutor, converter, nodeConstructorFactory,
+ recoverer, debugListener);
}
- public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor recoverer){
+ public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor recoverer) {
return parse(nonterminal, inputURI, input, actionExecutor, converter, nodeConstructorFactory, recoverer, null);
}
- public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor debugListener){
- return parse(nonterminal, inputURI, input, actionExecutor, converter, nodeConstructorFactory, null, debugListener);
+ public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor debugListener) {
+ return parse(nonterminal, inputURI, input, actionExecutor, converter, nodeConstructorFactory, null,
+ debugListener);
}
- public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor