Add Token_Stream class as new lexer and ast_link to every token

trlc/ast.py

@@ -112,6 +112,10 @@ def __init__(self, location):
         assert isinstance(location, Location)
         self.location = location
 
+    def set_ast_link(self, tok):
+        assert isinstance(tok, Token)
+        tok.ast_link = self
+
     def write_indent(self, indent, message):  # pragma: no cover
         # lobster-exclude: Debugging feature
         assert isinstance(indent, int)
@@ -266,7 +270,9 @@ def resolve_imports(self, mh, stab):
             # We can ignore errors here, because that just means we
             # generate more error later.
             try:
-                self.imports.add(stab.lookup(mh, t_import, Package))
+                a_import = stab.lookup(mh, t_import, Package)
+                self.imports.add(a_import)
+                a_import.set_ast_link(t_import)
             except TRLC_Error:
                 pass
 
@@ -2399,6 +2405,10 @@ def dump(self, indent=0):  # pragma: no cover
         self.write_indent(indent + 1, "Declared_Late: %s" % self.declared_late)
         self.symbols.dump(indent + 1, omit_heading=True)
 
+    def __repr__(self):
+        return "%s<%s>" % (self.__class__.__name__,
+                           self.name)
+
 
 class Composite_Type(Concrete_Type, metaclass=ABCMeta):
     """Abstract base for record and tuple types, as they share some
@@ -2495,6 +2505,11 @@ def dump(self, indent=0):  # pragma: no cover
         self.write_indent(indent + 1, "Optional: %s" % self.optional)
         self.write_indent(indent + 1, "Type: %s" % self.n_typ.name)
 
+    def __repr__(self):
+        return "%s<%s>" % (self.__class__.__name__,
+                           self.member_of.fully_qualified_name() + "." +
+                           self.name)
+
 
 class Record_Type(Composite_Type):
     """A user-defined record type.
@@ -2950,6 +2965,12 @@ def perform_checks(self, mh):
 
         return ok
 
+    def __repr__(self):
+        return "%s<%s>" % (self.__class__.__name__,
+                           self.n_package.name + "." +
+                           self.n_typ.name + "." +
+                           self.name)
+
 
 class Section(Entity):
     # lobster-trace: LRM.Section_Declaration

trlc/lexer.py

@@ -159,7 +159,7 @@ class Token(Token_Base):
         "STRING"     : "string literal",
     }
 
-    def __init__(self, location, kind, value=None):
+    def __init__(self, location, kind, value=None, ast_link=None):
         assert kind in Token.KIND
         if kind in ("COMMENT", "IDENTIFIER", "BUILTIN",
                     "KEYWORD", "OPERATOR", "STRING"):
@@ -171,6 +171,7 @@ def __init__(self, location, kind, value=None):
         else:
             assert value is None
         super().__init__(location, kind, value)
+        self.ast_link = ast_link
 
     def __repr__(self):
         if self.value is None:
@@ -186,6 +187,7 @@ def __init__(self, mh, content):
         self.mh        = mh
         self.content   = content
         self.length    = len(self.content)
+        self.tokens    = []
 
         self.lexpos  = -3
         self.line_no = 0
@@ -643,6 +645,15 @@ def token(self):
         return Token(sref, kind, value)
 
 
+class Token_Stream(TRLC_Lexer):
+
+    def token(self):
+        tok = super().token()
+        if tok is not None:
+            self.tokens.append(tok)
+        return tok
+
+
 def sanity_test():
     # lobster-exclude: Developer test function
     mh    = Message_Handler()