Parser & Integration (#23)

* Comment the parsing tree generation

* Comment the parsing tree generation

* Minor Changes

* parser module

* rename `productionsVector`

* Add error handling and traces to parsing

* handle when the input is empty but the stack isn't while parsing

* handle when not all inputs are consumed while parsing

* parsing bug fixes

* add test coverage for parsing

* add printing stack trace only

* rename `ParserPhase` back to `SyntaxPhase`

* cleanup

* integration

* make printTrace output more readable

* add epsilon derivations to parsing tree print

* fix backslashes terminals bug

* small output adjustment

* merge final version

* print grammar

* merge

* comment some unit tests

---------

Co-authored-by: Ahmed Adel <[email protected]>
Co-authored-by: Meniem Hany <[email protected]>
Co-authored-by: Zyad Samy <[email protected]>
Co-authored-by: Deffo0 <[email protected]>

Google_tests/CMakeLists.txt

@@ -12,7 +12,7 @@ add_executable(Google_Tests_run UtilsTest.cpp
         SP_GrammarConverterTest.cpp
         SP_FollowSetGeneratorTest.cpp
         SP_FirstSetGeneratorTest.cpp
-        SP_FollowSetGeneratorTest.cpp)
+        SP_ParsingTest.cpp)
 
 target_link_libraries(Google_Tests_run gtest gtest_main)
 target_link_libraries(Google_Tests_run Util_lib)
@@ -21,5 +21,6 @@ target_link_libraries(Google_Tests_run NFAConverter_lib)
 target_link_libraries(Google_Tests_run DFAConverter_lib)
 target_link_libraries(Google_Tests_run SymbolTableGenerator_lib)
 target_link_libraries(Google_Tests_run GrammarParser_lib)
+target_link_libraries(Google_Tests_run PredictiveParser_lib)
 target_link_libraries(Google_Tests_run FirstAndFollowGenerator_lib)
 

Google_tests/GrammarExamples/1

@@ -0,0 +1,4 @@
+# Expr ::= Term ExprPrime
+# ExprPrime ::= '+' Term ExprPrime | '-' Term ExprPrime | \L
+# Term ::= Factor '*' Term | Factor '/' Term | Factor
+# Factor ::= '(' Expr ')' | 'num' | 'iden'

Google_tests/SP_ParsingTest.cpp

@@ -0,0 +1,210 @@
+//#include <gtest/gtest.h>
+//#include "../SyntaxPhase/Common/NonTerminal.h"
+//#include "../SyntaxPhase/GrammarParser/GrammarConverter.h"
+//#include "../SyntaxPhase/GrammarParser/Grammar.h"
+//#include "../SyntaxPhase/PredictiveParser/Token.h"
+//#include "../SyntaxPhase/PredictiveParser/Parser.h"
+//
+//std::string g1Path = "../../GrammarTest.txt";
+//
+//std::pair<Symbol*, std::unordered_map<std::pair<NonTerminal*,std::string>, ParsingTableEntry, PairHash, PairEqual>> MathExprGrammar() {
+//    // Lecture-Parsing-2 Example-2 p-26
+//    NonTerminal* E = new NonTerminal("E");
+//    NonTerminal* E_ = new NonTerminal("E'");
+//    NonTerminal* T = new NonTerminal("T");
+//    NonTerminal* T_ = new NonTerminal("T'");
+//    NonTerminal* F = new NonTerminal("F");
+//    Terminal* id = new Terminal("id");
+//    Terminal* plus = new Terminal("+");
+//    Terminal* mult = new Terminal("*");
+//    Terminal* openB = new Terminal("(");
+//    Terminal* closeB = new Terminal(")");
+//
+//    std::vector<Symbol*> Eprod = {T,E_};
+//    std::vector<Symbol*> E_prod = {plus,T,E_};
+//    std::vector<Symbol*> Tprod = {F,T_};
+//    std::vector<Symbol*> T_prod = {mult,F,T_};
+//    std::vector<Symbol*> Fprod1 = {openB,E,closeB};
+//    std::vector<Symbol*> Fprod2 = {id};
+//
+//    std::unordered_map<std::pair<NonTerminal*,std::string>, ParsingTableEntry, PairHash, PairEqual> parsingTable;
+//    parsingTable[{E,id->getName()}] = ParsingTableEntry(Eprod);
+//    parsingTable[{T,id->getName()}] = ParsingTableEntry(Tprod);
+//    parsingTable[{F,id->getName()}] = ParsingTableEntry(Fprod2);
+//    parsingTable[{E_,plus->getName()}] = ParsingTableEntry(E_prod);
+//    parsingTable[{T_,plus->getName()}] = ParsingTableEntry("epsilon");
+//    parsingTable[{T_,mult->getName()}] = ParsingTableEntry(T_prod);
+//    parsingTable[{E,openB->getName()}] = ParsingTableEntry(Eprod);
+//    parsingTable[{T,openB->getName()}] = ParsingTableEntry(Tprod);
+//    parsingTable[{F,openB->getName()}] = ParsingTableEntry(Fprod1);
+//    parsingTable[{E_,closeB->getName()}] = ParsingTableEntry("epsilon");
+//    parsingTable[{T_,closeB->getName()}] = ParsingTableEntry("epsilon");
+//    parsingTable[{E_,"$"}] = ParsingTableEntry("epsilon");
+//    parsingTable[{T_,"$"}] = ParsingTableEntry("epsilon");
+//
+//    return {E, parsingTable};
+//}
+//
+//TEST(ParserManualTest, ValidInput) {
+//    std::vector<Token> input = {
+//        Token("a", "a"),
+//        Token("b", "b"),
+//        Token("b", "b"),
+//        Token("a", "a"),
+//    };
+//
+//    // S -> aBa
+//    // B -> bB | eps
+//    NonTerminal *S = new NonTerminal("S");
+//    NonTerminal *B = new NonTerminal("B");
+//    Terminal *a = new Terminal("a");
+//    Terminal *b = new Terminal("b");
+//
+//    std::vector<Symbol *> S_production_1 = {a, B, a};
+//    std::vector<Symbol *> B_production_1 = {b, B};
+//
+//    std::unordered_map<std::pair<NonTerminal *, std::string>, ParsingTableEntry, PairHash, PairEqual> parsingTable;
+//    parsingTable[{S, a->getName()}] = ParsingTableEntry({a, B, a});
+//    parsingTable[{B, a->getName()}] = ParsingTableEntry("epsilon");
+//    parsingTable[{B, b->getName()}] = ParsingTableEntry({b, B});
+//
+//    Parser parser(S, parsingTable);
+//    ParsingResult result = parser.parse(input);
+//    result.tree.print();
+//    result.printTrace();
+//}
+//
+//TEST(ParserManualTest, ValidInput2) {
+//    std::vector<Token> input = {
+//        Token("id","x"),
+//        Token("+",""),
+//        Token("id","y"),
+//    };
+//
+//    auto g = MathExprGrammar();
+//    auto startSymbol = g.first;
+//    auto parsingTable = g.second;
+//
+//    Parser parser(startSymbol, parsingTable);
+//    ParsingResult result = parser.parse(input);
+//    result.tree.print();
+//    result.printTrace();
+//}
+//
+//TEST(ParserManualTest, InvalidInputExcessToken) {
+//    std::vector<Token> input = {
+//        Token("(","("),
+//        Token("id","x"),
+//        Token("+",""),
+//        Token("id","y"),
+//        Token(")",")"),
+//        Token("*","*"),
+//        Token("id","z"),
+//        Token("id","w"),
+//        Token("id","w"),
+//    };
+//
+//    auto g = MathExprGrammar();
+//    auto startSymbol = g.first;
+//    auto parsingTable = g.second;
+//
+//    Parser parser(startSymbol, parsingTable);
+//    ParsingResult result = parser.parse(input);
+//    result.tree.print();
+//    result.printTrace();
+//}
+//
+//TEST(ParserManualTest, InvalidInputMissingToken) {
+//    // P-46 left example
+//    NonTerminal* S = new NonTerminal("S");
+//    NonTerminal* A = new NonTerminal("A");
+//    Terminal* a = new Terminal("a");
+//    Terminal* b = new Terminal("b");
+//    Terminal* c = new Terminal("c");
+//    Terminal* d = new Terminal("d");
+//    Terminal* e = new Terminal("e");
+//
+//    std::vector<Symbol*> S_production_1 = {A,b,S};
+//    std::vector<Symbol*> S_production_2 = {e};
+//    std::vector<Symbol*> A_production_1 = {a};
+//    std::vector<Symbol*> A_production_2 = {c,A,d};
+//
+//    std::unordered_map<std::pair<NonTerminal*,std::string>, ParsingTableEntry, PairHash, PairEqual> parsingTable;
+//    parsingTable[{S,a->getName()}] = ParsingTableEntry(S_production_1);
+//    parsingTable[{A,a->getName()}] = ParsingTableEntry(A_production_1);
+//    parsingTable[{A,b->getName()}] = ParsingTableEntry("sync");
+//    parsingTable[{S,c->getName()}] = ParsingTableEntry(S_production_1);
+//    parsingTable[{A,c->getName()}] = ParsingTableEntry(A_production_2);
+//    parsingTable[{A,d->getName()}] = ParsingTableEntry("sync");
+//    parsingTable[{S,e->getName()}] = ParsingTableEntry(S_production_2);
+//    parsingTable[{S,"$"}] = ParsingTableEntry("epsilon");
+//
+//
+//    std::vector<Token> input = {
+//        Token("a","a"),
+//        Token("a","a"),
+//        Token("b","b")
+//    };
+//
+//    Parser parser(S, parsingTable);
+//    ParsingResult result = parser.parse(input);
+//    result.tree.print();
+//    result.printTrace();
+//}
+//
+//TEST(ParserManualTest, InvalidInputExcessToken2) {
+//    // P-46 left example
+//    NonTerminal* S = new NonTerminal("S");
+//    NonTerminal* A = new NonTerminal("A");
+//    Terminal* a = new Terminal("a");
+//    Terminal* b = new Terminal("b");
+//    Terminal* c = new Terminal("c");
+//    Terminal* d = new Terminal("d");
+//    Terminal* e = new Terminal("e");
+//
+//    std::vector<Symbol*> S_production_1 = {A,b,S};
+//    std::vector<Symbol*> S_production_2 = {e};
+//    std::vector<Symbol*> A_production_1 = {a};
+//    std::vector<Symbol*> A_production_2 = {c,A,d};
+//
+//    std::unordered_map<std::pair<NonTerminal*,std::string>, ParsingTableEntry, PairHash, PairEqual> parsingTable;
+//    parsingTable[{S,a->getName()}] = ParsingTableEntry(S_production_1);
+//    parsingTable[{A,a->getName()}] = ParsingTableEntry(A_production_1);
+//    parsingTable[{A,b->getName()}] = ParsingTableEntry("sync");
+//    parsingTable[{S,c->getName()}] = ParsingTableEntry(S_production_1);
+//    parsingTable[{A,c->getName()}] = ParsingTableEntry(A_production_2);
+//    parsingTable[{A,d->getName()}] = ParsingTableEntry("sync");
+//    parsingTable[{S,e->getName()}] = ParsingTableEntry(S_production_2);
+//    parsingTable[{S,"$"}] = ParsingTableEntry("epsilon");
+//
+//
+//    std::vector<Token> input = {
+//        Token("c","c"),
+//        Token("e","e"),
+//        Token("a","a"),
+//        Token("d","d"),
+//        Token("b","b"),
+//    };
+//
+//    Parser parser(S, parsingTable);
+//    ParsingResult result = parser.parse(input);
+//    result.tree.print();
+//    result.printTrace();
+//}
+//
+//TEST(ParserManualTest, InvalidInputEmptyWhileStackNot) {
+//    std::vector<Token> input = {
+//            Token("id", "x"),
+//            Token("+", ""),
+//            // Token("id","y"),
+//    };
+//
+//    auto g = MathExprGrammar();
+//    auto startSymbol = g.first;
+//    auto parsingTable = g.second;
+//
+//    Parser parser(startSymbol, parsingTable);
+//    ParsingResult result = parser.parse(input);
+//    result.tree.print();
+//    result.printTrace();
+//}

Grammar.txt

@@ -6,12 +6,12 @@
 | ASSIGNMENT
 # DECLARATION ::= PRIMITIVE_TYPE 'id' ';'
 # PRIMITIVE_TYPE ::= 'int' | 'float'
-# IF ::= 'if' '(' EXPRESSION ')' '{' STATEMENT '}' 'else' '{' STATEMENT '}' 
+# IF ::= 'if' '(' EXPRESSION ')' '{' STATEMENT '}' 'else' '{' STATEMENT '}'
 # WHILE ::= 'while' '(' EXPRESSION ')' '{' STATEMENT '}'
 # ASSIGNMENT ::= 'id' '=' EXPRESSION ';'
 # EXPRESSION ::= SIMPLE_EXPRESSION
 		     | SIMPLE_EXPRESSION 'relop' SIMPLE_EXPRESSION
-# SIMPLE_EXPRESSION ::= TERM | SIGN TERM | SIMPLE_EXPRESSION 'addop' TERM 
+# SIMPLE_EXPRESSION ::= TERM | SIGN TERM | SIMPLE_EXPRESSION 'addop' TERM
 # TERM ::= FACTOR | TERM 'mulop' FACTOR
 # FACTOR ::= 'id' | 'num' | '(' EXPRESSION ')'
 # SIGN ::= '+' | '-'

GrammarTest.txt

@@ -6,3 +6,4 @@
 # F` ::= '*' F | '\\L'
 # P ::= '(' E ')' | 'a' | 'b' | 'Em'
 
+pukk

LexicalPhase/RulesParser/RegularDefinition.cpp

@@ -17,7 +17,11 @@ const std::string &RegularDefinition::getRegex() const {
     return regex;
 }
 
-RegularDefinition::RegularDefinition(std::string name, std::string regex) : name(std::move(name)), regex(std::move(regex)) {}
+RegularDefinition::RegularDefinition(std::string name, std::string regex) : regex(std::move(regex)) {
+    std::string s = name;
+    replaceAll(s, "\\", "");
+    this->name = s;
+}
 
 int RegularDefinition::standardizeRegex(std::vector<RegularDefinition> regularDefinitions) {
     removeConsecutiveSpaces(regex);

Rules.txt

@@ -5,8 +5,7 @@ digits = digit+
 {boolean int float}
 num: digit+ | digit+ . digits ( \L | E digits)
 relop: \=\= | !\= | > | >\= | < | <\=
-assign:\=
 { if else while }
-[; , \( \) { }]
+[; , \( \) { } =]
 addop: \+ | \-
 mulop: \* | /

SyntaxPhase/Common/Symbol.h

@@ -6,6 +6,8 @@
 #define COMPILER_SYMBOL_H
 
 #include <string>
+#include <iostream>
+
 
 class Symbol {
 

SyntaxPhase/GrammarParser/Grammar.cpp

@@ -74,7 +74,7 @@ Grammar::Grammar(GrammarConverter modifiedGrammar) : modifiedGrammar(std::move(m
 
 std::ostream &operator<<(std::ostream &os, const Grammar &g) {
     for (const auto& r : g.standardizedNonTerminals)
-        os << r;
+        os << r.get();
     os << "\n\n\n";
     return os;
 }

SyntaxPhase/GrammarParser/Grammar.h

@@ -30,8 +30,10 @@ private :
     friend std::ostream& operator<<(std::ostream& os, const Grammar& g);
     void standardizeNonTerminals();
     explicit Grammar(GrammarConverter modifiedGrammar);
-
     static Terminal* epsilon;
+
+    // for debugging purposes
+    explicit Grammar(){};
 };
 
 

SyntaxPhase/PredictiveParser/CMakeLists.txt

@@ -4,13 +4,18 @@ set(HEADER_FILES
         Common.h
         Parser.h
         ParsingTree.h
+        ParsingTrace.h
+        ParsingResult.h
         Token.h
         ParsingTableEntry.h)
-
-
-
+        
+        
+        
 set(SOURCE_FILES
         Parser.cpp
+        ParsingTree.cpp
+        ParsingTrace.cpp
+        ParsingResult.cpp
         ParsingTableEntry.cpp)