more flexible tree structure and listener can have any lifetime now, more type safety

Author: rrevenantt

.gitignore

@@ -1,5 +1,7 @@
 .idea
 .vscode
 /target
+/tests/gen/*.tokens
+/tests/gen/*.interp
 **/*.rs.bk
 Cargo.lock

README.md

@@ -17,17 +17,18 @@ For now development is going on in this repository
 but eventually it will be merged to main ANTLR4 repo
 
 Currently requires nightly version of rust. 
-This likely will be the case until `specialization`,`try_blocks` and `unsize` features are stabilized. 
+This likely will be the case until `unsize` or some kind of `CoerceUnsized` is stabilized. 
+There are other unstable features in use but only `CoerceUnsized` is essential. 
 
 Remaining things before merge:
  - API stabilization
    - [ ] Rust api guidelines compliance  
    - [ ] more tests for API because it is quite different from Java
- - more generic `PredictionContext`
  - generate enum for labeled alternatives without redundant `Error` option
  - option to generate fields instead of getters by default
  - move useful exports to lib.rs for better documentation
  - reexport statics crate and move to once_cell
+ - support byte level parser
 
 Can be done after merge: 
  - profiling and performance optimizations

grammars/CSV.g4

@@ -1,7 +1,7 @@
 grammar CSV;
 
 @tokenfactory{
-pub type LocalTokenFactory<'input> = antlr_rust::common_token_factory::ArenaCommonFactory<'input>;
+pub type LocalTokenFactory<'input> = antlr_rust::token_factory::ArenaCommonFactory<'input>;
 }
 
 csvFile: hdr row+ ;

src/atn.rs

@@ -8,8 +8,9 @@ use crate::dfa::ScopeExt;
 use crate::interval_set::IntervalSet;
 use crate::lexer_action::LexerAction;
 use crate::ll1_analyzer::LL1Analyzer;
-use crate::parser_rule_context::{ParserRuleContext, ParserRuleContextType};
-use crate::rule_context::RuleContext;
+use crate::parser::ParserNodeType;
+use crate::parser_rule_context::ParserRuleContext;
+use crate::rule_context::{EmptyContextType, RuleContext};
 use crate::token::{TOKEN_EOF, TOKEN_EPSILON};
 use crate::token_factory::{CommonTokenFactory, TokenFactory};
 use crate::transition::RuleTransition;
@@ -59,7 +60,7 @@ impl ATN {
     ///rule.
     pub fn next_tokens<'a>(&self, s: &'a dyn ATNState) -> &'a IntervalSet {
         s.get_next_tokens_within_rule().get_or_init(|| {
-            self.next_tokens_in_ctx(s, None::<&dyn ParserRuleContext<TF=CommonTokenFactory>>)
+            self.next_tokens_in_ctx::<EmptyContextType<CommonTokenFactory>>(s, None)
                 .modify_with(|r| {
                     r.read_only = true
                 }
@@ -71,9 +72,9 @@ impl ATN {
     /// If `ctx` is null, the set of tokens will not include what can follow
     /// the rule surrounding `s`. In other words, the set will be
     /// restricted to tokens reachable staying within `s`'s rule.
-    pub fn next_tokens_in_ctx<'a, TF: TokenFactory<'a> + 'a>(&self, s: &dyn ATNState, _ctx: Option<&(dyn ParserRuleContext<'a, TF=TF> + 'a)>) -> IntervalSet {
+    pub fn next_tokens_in_ctx<'a, Ctx: ParserNodeType<'a>>(&self, s: &dyn ATNState, _ctx: Option<&Ctx::Type>) -> IntervalSet {
         let analyzer = LL1Analyzer::new(self);
-        analyzer.look(s, None, _ctx)
+        analyzer.look::<'a, Ctx>(s, None, _ctx)
     }
 
     pub(crate) fn add_state(&mut self, state: Box<dyn ATNState>) {
@@ -125,7 +126,7 @@ impl ATN {
     /// specified state in the specified context.
     /// @throws IllegalArgumentException if the ATN does not contain a state with
     /// number {@code stateNumber}
-    pub fn get_expected_tokens<'a, TF: TokenFactory<'a>>(&self, state_number: isize, _ctx: &ParserRuleContextType<'a, TF>) -> IntervalSet {
+    pub fn get_expected_tokens<'a, Ctx: ParserNodeType<'a>>(&self, state_number: isize, _ctx: &Rc<Ctx::Type>) -> IntervalSet {
         let s = self.states[state_number as usize].as_ref();
         let mut following = self.next_tokens(s);
         if !following.contains(TOKEN_EPSILON) {

src/atn_state.rs

@@ -80,7 +80,7 @@ pub trait ATNState: Sync + Send + Debug {
     fn get_state_type_id(&self) -> isize;
 
     fn get_state_number(&self) -> usize;
-    fn set_state_number(&self, stateNumber: isize);
+    fn set_state_number(&self, state_number: isize);
 
     fn get_transitions(&self) -> &Vec<Box<dyn Transition>>;
     fn set_transitions(&self, t: Vec<Box<dyn Transition>>);
@@ -156,7 +156,7 @@ impl ATNState for BaseATNState {
         self.state_number
     }
 
-    fn set_state_number(&self, _stateNumber: isize) {
+    fn set_state_number(&self, _state_number: isize) {
         unimplemented!()
     }
 

src/error_listener.rs

@@ -13,7 +13,7 @@ use crate::recognizer::Recognizer;
 use crate::token::Token;
 
 pub trait ErrorListener<T> {
-    fn syntax_error(&self, /*todo _recognizer: (Option<&dyn Lexer<'_>>,Option<&dyn Parser<'_>>),*/ _offending_symbol: Option<&dyn Token>,
+    fn syntax_error(&self, _recognizer: &T, _offending_symbol: Option<&dyn Token>,
                     _line: isize, _column: isize, _msg: &str, _e: Option<&ANTLRError>, ) {}
 
     fn report_ambiguity(&self, _recognizer: &T, _dfa: &DFA, _start_index: isize, _stop_index: isize,
@@ -30,7 +30,7 @@ pub trait ErrorListener<T> {
 pub struct ConsoleErrorListener {}
 
 impl<T> ErrorListener<T> for ConsoleErrorListener {
-    fn syntax_error(&self, _offending_symbol: Option<&dyn Token>,
+    fn syntax_error(&self, _recognizer: &T, _offending_symbol: Option<&dyn Token>,
                     line: isize, column: isize, msg: &str, _e: Option<&ANTLRError>) {
         eprintln!("line {}:{} {}", line, column, msg);
     }
@@ -41,9 +41,9 @@ pub(crate) struct ProxyErrorListener<'a, T> {
 }
 
 impl<'a, T> ErrorListener<T> for ProxyErrorListener<'a, T> {
-    fn syntax_error(&self, offending_symbol: Option<&dyn Token>, line: isize, column: isize, msg: &str, e: Option<&ANTLRError>) {
+    fn syntax_error(&self, _recognizer: &T, offending_symbol: Option<&dyn Token>, line: isize, column: isize, msg: &str, e: Option<&ANTLRError>) {
         for listener in self.delegates.deref() {
-            listener.syntax_error(offending_symbol, line, column, msg, e)
+            listener.syntax_error(_recognizer, offending_symbol, line, column, msg, e)
         }
     }
 

src/error_strategy.rs

@@ -11,11 +11,13 @@ use crate::char_stream::CharStream;
 use crate::dfa::ScopeExt;
 use crate::errors::{ANTLRError, FailedPredicateError, InputMisMatchError, NoViableAltError, RecognitionError};
 use crate::interval_set::IntervalSet;
-use crate::parser::Parser;
-use crate::parser_rule_context::{ParserRuleContext, ParserRuleContextType};
+use crate::parser::{Parser, ParserNodeType};
+use crate::parser_rule_context::ParserRuleContext;
+use crate::rule_context::{CustomRuleContext, RuleContext};
 use crate::token::{OwningToken, Token, TOKEN_DEFAULT_CHANNEL, TOKEN_EOF, TOKEN_EPSILON, TOKEN_INVALID_TYPE};
 use crate::token_factory::TokenFactory;
 use crate::transition::RuleTransition;
+use crate::tree::Tree;
 use crate::utils::escape_whitespaces;
 
 /// The interface for defining strategies to deal with syntax errors encountered
@@ -41,16 +43,16 @@ pub trait ErrorStrategy<'a, T: Parser<'a>> {
     fn report_match(&mut self, recognizer: &mut T);
 }
 
-pub struct DefaultErrorStrategy<'input, TF: TokenFactory<'input> + 'input> {
+pub struct DefaultErrorStrategy<'input, Ctx: ParserNodeType<'input>> {
     error_recovery_mode: bool,
     last_error_index: isize,
     last_error_states: Option<IntervalSet>,
     next_tokens_state: isize,
-    next_tokens_ctx: Option<ParserRuleContextType<'input, TF>>,
+    next_tokens_ctx: Option<Rc<Ctx::Type>>,
 }
 
 
-impl<'input, TF: TokenFactory<'input> + 'input> DefaultErrorStrategy<'input, TF> {
+impl<'input, Ctx: ParserNodeType<'input>> DefaultErrorStrategy<'input, Ctx> {
     pub fn new() -> Self {
         Self {
             error_recovery_mode: false,
@@ -61,17 +63,17 @@ impl<'input, TF: TokenFactory<'input> + 'input> DefaultErrorStrategy<'input, TF>
         }
     }
 
-    fn begin_error_condition<T: Parser<'input, TF=TF>>(&mut self, _recognizer: &T) {
+    fn begin_error_condition<T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&mut self, _recognizer: &T) {
         self.error_recovery_mode = true;
     }
 
-    fn end_error_condition<T: Parser<'input, TF=TF>>(&mut self, _recognizer: &T) {
+    fn end_error_condition<T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&mut self, _recognizer: &T) {
         self.error_recovery_mode = false;
         self.last_error_index = -1;
         self.last_error_states = None;
     }
 
-    fn report_no_viable_alternative<T: Parser<'input, TF=TF>>(&self, recognizer: &mut T, e: &NoViableAltError) -> String {
+    fn report_no_viable_alternative<T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&self, recognizer: &mut T, e: &NoViableAltError) -> String {
         let input = if e.start_token.token_type == TOKEN_EOF {
             "<EOF>".to_owned()
         } else {
@@ -81,19 +83,19 @@ impl<'input, TF: TokenFactory<'input> + 'input> DefaultErrorStrategy<'input, TF>
         format!("no viable alternative at input '{}'", input)
     }
 
-    fn report_input_mismatch<T: Parser<'input, TF=TF>>(&self, recognizer: &T, e: &InputMisMatchError) -> String {
+    fn report_input_mismatch<T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&self, recognizer: &T, e: &InputMisMatchError) -> String {
         format!("mismatched input {} expecting {}",
                 self.get_token_error_display(&e.base.offending_token),
                 e.base.get_expected_tokens(recognizer).to_token_string(recognizer.get_vocabulary()))
     }
 
-    fn report_failed_predicate<T: Parser<'input, TF=TF>>(&self, recognizer: &T, e: &FailedPredicateError) -> String {
+    fn report_failed_predicate<T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&self, recognizer: &T, e: &FailedPredicateError) -> String {
         format!("rule {} {}",
                 recognizer.get_rule_names()[recognizer.get_parser_rule_context().get_rule_index()],
                 e.base.message)
     }
 
-    fn report_unwanted_token<T: Parser<'input, TF=TF>>(&mut self, recognizer: &mut T) {
+    fn report_unwanted_token<T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&mut self, recognizer: &mut T) {
         if self.in_error_recovery_mode(recognizer) {
             return;
         }
@@ -108,7 +110,7 @@ impl<'input, TF: TokenFactory<'input> + 'input> DefaultErrorStrategy<'input, TF>
         recognizer.notify_error_listeners(msg, Some(t), None);
     }
 
-    fn report_missing_token<T: Parser<'input, TF=TF>>(&mut self, recognizer: &mut T) {
+    fn report_missing_token<T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&mut self, recognizer: &mut T) {
         if self.in_error_recovery_mode(recognizer) {
             return;
         }
@@ -126,13 +128,13 @@ impl<'input, TF: TokenFactory<'input> + 'input> DefaultErrorStrategy<'input, TF>
         recognizer.notify_error_listeners(msg, Some(t), None);
     }
 
-    fn single_token_insertion<T: Parser<'input, TF=TF>>(&mut self, recognizer: &mut T) -> bool {
+    fn single_token_insertion<T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&mut self, recognizer: &mut T) -> bool {
         let current_token = recognizer.get_input_stream_mut().la(1);
 
         let atn = recognizer.get_interpreter().atn();
         let current_state = atn.states[recognizer.get_state() as usize].as_ref();
         let next = current_state.get_transitions().first().unwrap().get_target();
-        let expect_at_ll2 = atn.next_tokens_in_ctx(
+        let expect_at_ll2 = atn.next_tokens_in_ctx::<'input, Ctx>(
             atn.states[next].as_ref(),
             Some(recognizer.get_parser_rule_context().deref()),
         );
@@ -143,7 +145,7 @@ impl<'input, TF: TokenFactory<'input> + 'input> DefaultErrorStrategy<'input, TF>
         false
     }
 
-    fn single_token_deletion<'a, T: Parser<'input, TF=TF>>(&mut self, recognizer: &'a mut T) -> Option<&'a (TF::Tok)> {
+    fn single_token_deletion<'a, T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&mut self, recognizer: &'a mut T) -> Option<&'a <T::TF as TokenFactory<'input>>::Tok> {
         let next_token_type = recognizer.get_input_stream_mut().la(2);
         let expecting = self.get_expected_tokens(recognizer);
 //        println!("expecting {}", expecting.to_token_string(recognizer.get_vocabulary()));
@@ -157,7 +159,7 @@ impl<'input, TF: TokenFactory<'input> + 'input> DefaultErrorStrategy<'input, TF>
         None
     }
 
-    fn get_missing_symbol<T: Parser<'input, TF=TF>>(&self, recognizer: &mut T) -> TF::Tok {
+    fn get_missing_symbol<T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&self, recognizer: &mut T) -> <T::TF as TokenFactory<'input>>::Tok {
         let expected = self.get_expected_tokens(recognizer);
         let expected_token_type = expected.get_min().unwrap_or(TOKEN_INVALID_TYPE);
         let token_text = if expected_token_type == TOKEN_EOF {
@@ -189,7 +191,7 @@ impl<'input, TF: TokenFactory<'input> + 'input> DefaultErrorStrategy<'input, TF>
         // .modify_with(|it| it.text = token_text)
     }
 
-    fn get_expected_tokens<T: Parser<'input, TF=TF>>(&self, recognizer: &T) -> IntervalSet {
+    fn get_expected_tokens<T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&self, recognizer: &T) -> IntervalSet {
         recognizer.get_expected_tokens()
     }
 
@@ -202,7 +204,7 @@ impl<'input, TF: TokenFactory<'input> + 'input> DefaultErrorStrategy<'input, TF>
         format!("'{}'", escape_whitespaces(s, false))
     }
 
-    fn get_error_recovery_set<T: Parser<'input, TF=TF>>(&self, recognizer: &T) -> IntervalSet {
+    fn get_error_recovery_set<T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&self, recognizer: &T) -> IntervalSet {
         let atn = recognizer.get_interpreter().atn();
         let mut ctx = Some(recognizer.get_parser_rule_context().clone());
         let mut recover_set = IntervalSet::new();
@@ -220,7 +222,7 @@ impl<'input, TF: TokenFactory<'input> + 'input> DefaultErrorStrategy<'input, TF>
         return recover_set;
     }
 
-    fn consume_until<T: Parser<'input, TF=TF>>(&mut self, recognizer: &mut T, set: &IntervalSet) {
+    fn consume_until<T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&mut self, recognizer: &mut T, set: &IntervalSet) {
         let mut ttype = recognizer.get_input_stream_mut().la(1);
         while ttype != TOKEN_EOF && !set.contains(ttype) {
             recognizer.consume(self);
@@ -229,7 +231,7 @@ impl<'input, TF: TokenFactory<'input> + 'input> DefaultErrorStrategy<'input, TF>
     }
 }
 
-impl<'a, T: Parser<'a>> ErrorStrategy<'a, T> for DefaultErrorStrategy<'a, T::TF> {
+impl<'a, T: Parser<'a>> ErrorStrategy<'a, T> for DefaultErrorStrategy<'a, T::Node> {
     fn reset(&mut self, _recognizer: &mut T) {
         unimplemented!()
     }
@@ -367,12 +369,12 @@ myparser.err_handler = BailErrorStrategy::new();
 
 [`ParserRuleContext.exception`]: todo
 */
-pub struct BailErrorStrategy<'input, TF: TokenFactory<'input>>(DefaultErrorStrategy<'input, TF>);
+pub struct BailErrorStrategy<'input, Ctx: ParserNodeType<'input>>(DefaultErrorStrategy<'input, Ctx>);
 
-impl<'input, TF: TokenFactory<'input>> BailErrorStrategy<'input, TF> {
+impl<'input, Ctx: ParserNodeType<'input>> BailErrorStrategy<'input, Ctx> {
     pub fn new() -> Self { Self(DefaultErrorStrategy::new()) }
 
-    fn process_error<T: Parser<'input, TF=TF>>(&self, recognizer: &mut T, e: &ANTLRError) -> ANTLRError {
+    fn process_error<T: Parser<'input, Node=Ctx, TF=Ctx::TF>>(&self, recognizer: &mut T, e: &ANTLRError) -> ANTLRError {
         let mut ctx = recognizer.get_parser_rule_context().clone();
         let _: Option<()> = try {
             loop {
@@ -401,7 +403,7 @@ impl Display for ParseCancelledError {
     }
 }
 
-impl<'a, T: Parser<'a>> ErrorStrategy<'a, T> for BailErrorStrategy<'a, T::TF> {
+impl<'a, T: Parser<'a>> ErrorStrategy<'a, T> for BailErrorStrategy<'a, T::Node> {
     fn reset(&mut self, recognizer: &mut T) { self.0.reset(recognizer) }
 
     fn recover_inline(&mut self, recognizer: &mut T) -> Result<<T::TF as TokenFactory<'a>>::Tok, ANTLRError> {

src/errors.rs

@@ -8,8 +8,8 @@ use std::rc::Rc;
 
 use crate::atn_simulator::IATNSimulator;
 use crate::interval_set::IntervalSet;
-use crate::parser::Parser;
-use crate::parser_rule_context::{ParserRuleContext, ParserRuleContextType};
+use crate::parser::{Parser, ParserNodeType};
+use crate::parser_rule_context::ParserRuleContext;
 use crate::token::{OwningToken, Token};
 use crate::transition::PredicateTransition;
 use crate::transition::TransitionType::TRANSITION_PREDICATE;
@@ -19,7 +19,7 @@ use crate::transition::TransitionType::TRANSITION_PREDICATE;
 pub enum ANTLRError {
     /// Returned from Lexer when it fails to find matching token type for current input
     ///
-    /// Usually Lexers contain rule that captures all invalid tokens like:
+    /// Usually Lexers contain last rule that captures all invalid tokens like:
     /// ```text
     /// ERROR_TOKEN: . ;
     /// ```
@@ -119,7 +119,7 @@ pub struct BaseRecognitionError {
 impl BaseRecognitionError {
     pub fn get_expected_tokens<'a, T: Parser<'a>>(&self, recognizer: &T) -> IntervalSet {
         recognizer.get_interpreter().atn()
-            .get_expected_tokens(self.offending_state, recognizer.get_parser_rule_context())
+            .get_expected_tokens::<'a, T::Node>(self.offending_state, recognizer.get_parser_rule_context())
     }
 
     fn new<'a, T: Parser<'a>>(recog: &mut T) -> BaseRecognitionError {
@@ -187,7 +187,7 @@ impl InputMisMatchError {
         }
     }
 
-    pub fn with_state<'a, T: Parser<'a>>(recognizer: &mut T, offending_state: isize, ctx: ParserRuleContextType<'a, T::TF>) -> InputMisMatchError {
+    pub fn with_state<'a, T: Parser<'a>>(recognizer: &mut T, offending_state: isize, ctx: Rc<<T::Node as ParserNodeType<'a>>::Type>) -> InputMisMatchError {
         let mut a = Self::new(recognizer);
         // a.base.ctx = ctx;
         a.base.offending_state = offending_state;

src/interval_set.rs

@@ -11,6 +11,8 @@ pub struct Interval {
     pub b: isize,
 }
 
+pub const INVALID: Interval = Interval { a: -1, b: -2 };
+
 impl Interval {
     /* stop is not included! */
     fn new(a: isize, b: isize) -> Interval {
@@ -21,7 +23,6 @@ impl Interval {
         unimplemented!()
     }
 
-
     fn length(&self) -> isize {
         self.b - self.a
     }