Add and improve docstrings (#1326)

lark/ast_utils.py

@@ -1,5 +1,5 @@
 """
-    Module of utilities for transforming a lark.Tree into a custom Abstract Syntax Tree
+    Module of utilities for transforming a lark.Tree into a custom Abstract Syntax Tree (AST defined in classes)
 """
 
 import inspect, re

lark/indenter.py

@@ -1,4 +1,4 @@
-"Provides Indentation services for languages with indentation similar to Python"
+"Provides a post-lexer for implementing Python-style indentation."
 
 from abc import ABC, abstractmethod
 from typing import List, Iterator

lark/lexer.py

@@ -30,6 +30,7 @@
     has_interegular = False
 
 class Pattern(Serialize, ABC):
+    "An abstraction over regular expressions."
 
     value: str
     flags: Collection[str]
@@ -112,6 +113,7 @@ def max_width(self) -> int:
 
 
 class TerminalDef(Serialize):
+    "A definition of a terminal"
     __serialize_fields__ = 'name', 'pattern', 'priority'
     __serialize_namespace__ = PatternStr, PatternRE
 
@@ -269,6 +271,8 @@ def __eq__(self, other):
 
 
 class LineCounter:
+    "A utility class for keeping track of line & column information"
+
     __slots__ = 'char_pos', 'line', 'column', 'line_start_pos', 'newline_char'
 
     def __init__(self, newline_char):

lark/load_grammar.py

@@ -1,4 +1,6 @@
-"""Parses and creates Grammar objects"""
+"""Parses and compiles Lark grammars into an internal representation.
+"""
+
 import hashlib
 import os.path
 import sys

lark/parse_tree_builder.py

@@ -1,3 +1,5 @@
+"""Provides functions for the automatic building and shaping of the parse-tree."""
+
 from typing import List
 
 from .exceptions import GrammarError, ConfigurationError

lark/parser_frontends.py

@@ -214,7 +214,7 @@ def create_earley_parser(lexer_conf: LexerConf, parser_conf: ParserConf, options
 
 class CYK_FrontEnd:
     def __init__(self, lexer_conf, parser_conf, options=None):
-        self._analysis = GrammarAnalyzer(parser_conf)
+        # self._analysis = GrammarAnalyzer(parser_conf)
         self.parser = cyk.Parser(parser_conf.rules)
 
         self.callbacks = parser_conf.callbacks

lark/parsers/grammar_analysis.py

@@ -1,3 +1,5 @@
+"Provides for superficial grammar analysis."
+
 from collections import Counter, defaultdict
 
 from ..utils import bfs, fzset, classify

lark/reconstruct.py

@@ -1,4 +1,5 @@
-"""Reconstruct text from a tree, based on Lark grammar"""
+"""This is an experimental tool for reconstructing text from a shaped tree, based on a Lark grammar.
+"""
 
 from typing import Dict, Callable, Iterable, Optional