Merge pull request #66 from 1c-syntax/feature/query_v3

Парсер и лексер запросов, вариант 3

.gitignore

@@ -69,3 +69,4 @@ fabric.properties
 \.idea/sonarlint-state\.xml
 
 \.idea/sonarlint\.xml
+**/antlr/.antlr/**

src/main/java/com/github/_1c_syntax/bsl/parser/BSLParserRuleContext.java

@@ -23,9 +23,7 @@
 
 import com.github._1c_syntax.utils.Lazy;
 import org.antlr.v4.runtime.ParserRuleContext;
-import org.antlr.v4.runtime.RuleContext;
 import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.tree.ErrorNode;
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.TerminalNode;
 
@@ -56,7 +54,7 @@ public List<Token> getTokens() {
   }
 
   private List<Token> computeTokens() {
-    if ( children == null ) {
+    if (children == null) {
       return Collections.emptyList();
     }
 

src/main/java/com/github/_1c_syntax/bsl/parser/BSLTokenizer.java

@@ -0,0 +1,36 @@
+/*
+ * This file is a part of BSL Parser.
+ *
+ * Copyright © 2018-2020
+ * Alexey Sosnoviy <[email protected]>, Nikita Gryzlov <[email protected]>, Sergey Batanov <[email protected]>
+ *
+ * SPDX-License-Identifier: LGPL-3.0-or-later
+ *
+ * BSL Parser is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3.0 of the License, or (at your option) any later version.
+ *
+ * BSL Parser is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with BSL Parser.
+ */
+package com.github._1c_syntax.bsl.parser;
+
+import org.antlr.v4.runtime.CharStreams;
+
+public class BSLTokenizer extends Tokenizer<BSLParser.FileContext, BSLParser> {
+  public BSLTokenizer(String content) {
+    super(content, new BSLLexer(CharStreams.fromString(""), true), BSLParser.class);
+  }
+
+  @Override
+  protected BSLParser.FileContext rootAST() {
+    return parser.file();
+  }
+
+}

src/main/java/com/github/_1c_syntax/bsl/parser/SDBLTokenizer.java

@@ -0,0 +1,35 @@
+/*
+ * This file is a part of BSL Parser.
+ *
+ * Copyright © 2018-2020
+ * Alexey Sosnoviy <[email protected]>, Nikita Gryzlov <[email protected]>, Sergey Batanov <[email protected]>
+ *
+ * SPDX-License-Identifier: LGPL-3.0-or-later
+ *
+ * BSL Parser is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3.0 of the License, or (at your option) any later version.
+ *
+ * BSL Parser is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with BSL Parser.
+ */
+package com.github._1c_syntax.bsl.parser;
+
+import org.antlr.v4.runtime.CharStreams;
+
+public class SDBLTokenizer extends Tokenizer<SDBLParser.QueryPackageContext, SDBLParser> {
+  public SDBLTokenizer(String content) {
+    super(content, new SDBLLexer(CharStreams.fromString(""), true), SDBLParser.class);
+  }
+
+  @Override
+  protected SDBLParser.QueryPackageContext rootAST() {
+    return parser.queryPackage();
+  }
+}

src/main/java/com/github/_1c_syntax/bsl/parser/Tokenizer.java

@@ -28,46 +28,51 @@
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.ConsoleErrorListener;
 import org.antlr.v4.runtime.Lexer;
+import org.antlr.v4.runtime.Parser;
 import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenStream;
 import org.antlr.v4.runtime.atn.PredictionMode;
 import org.apache.commons.io.IOUtils;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.lang.reflect.InvocationTargetException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
+import static java.util.Objects.requireNonNull;
 import static org.antlr.v4.runtime.Token.EOF;
 
-public class Tokenizer {
+abstract public class Tokenizer<T extends BSLParserRuleContext, P extends Parser> {
 
   private final InputStream content;
-  private Lexer lexer;
+  private final Lexer lexer;
   private final Lazy<CommonTokenStream> tokenStream = new Lazy<>(this::computeTokenStream);
   private final Lazy<List<Token>> tokens = new Lazy<>(this::computeTokens);
-  private final Lazy<BSLParser.FileContext> ast = new Lazy<>(this::computeAST);
+  private final Lazy<T> ast = new Lazy<>(this::computeAST);
+  private final Class<P> parserClass;
+  protected P parser;
 
-  public Tokenizer(String content) {
-    this(content, null);
+  protected Tokenizer(String content, Lexer lexer, Class<P> parserClass) {
+    this(IOUtils.toInputStream(content, StandardCharsets.UTF_8), lexer, parserClass);
   }
 
-  protected Tokenizer(String content, Lexer lexer) {
-    this(IOUtils.toInputStream(content, StandardCharsets.UTF_8), lexer);
-  }
-
-  protected Tokenizer(InputStream content, Lexer lexer) {
+  protected Tokenizer(InputStream content, Lexer lexer, Class<P> parserClass) {
+    requireNonNull(content);
+    requireNonNull(lexer);
     this.content = content;
     this.lexer = lexer;
+    this.parserClass = parserClass;
   }
 
   public List<Token> getTokens() {
     return tokens.getOrCompute();
   }
 
-  public BSLParser.FileContext getAst() {
+  public T getAst() {
     return ast.getOrCompute();
   }
 
@@ -76,55 +81,61 @@ private List<Token> computeTokens() {
 
     Token lastToken = tokensTemp.get(tokensTemp.size() - 1);
     if (lastToken.getType() == EOF && lastToken instanceof CommonToken) {
-      ((CommonToken)lastToken).setChannel(Lexer.HIDDEN);
+      ((CommonToken) lastToken).setChannel(Lexer.HIDDEN);
     }
 
     return tokensTemp;
   }
 
-  private BSLParser.FileContext computeAST() {
-    BSLParser parser = new BSLParser(getTokenStream());
+  private T computeAST() {
+    parser = createParser(getTokenStream());
     parser.removeErrorListener(ConsoleErrorListener.INSTANCE);
     try {
       parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
-      return parser.file();
+      return rootAST();
     } catch (Exception ex) {
       parser.reset(); // rewind input stream
       parser.getInterpreter().setPredictionMode(PredictionMode.LL);
     }
-    return parser.file();
+    return rootAST();
   }
 
+  abstract protected T rootAST();
+
   private CommonTokenStream computeTokenStream() {
 
     CharStream input;
 
     try (
       UnicodeBOMInputStream ubis = new UnicodeBOMInputStream(content);
-      Reader inputStreamReader = new InputStreamReader(ubis, StandardCharsets.UTF_8);
+      Reader inputStreamReader = new InputStreamReader(ubis, StandardCharsets.UTF_8)
     ) {
       ubis.skipBOM();
       input = CharStreams.fromReader(inputStreamReader);
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
 
-    if (lexer == null) {
-      lexer = new BSLLexer(input, true);
-    } else {
-      lexer.setInputStream(input);
-    }
+    lexer.setInputStream(input);
     lexer.removeErrorListener(ConsoleErrorListener.INSTANCE);
 
     CommonTokenStream tempTokenStream = new CommonTokenStream(lexer);
     tempTokenStream.fill();
     return tempTokenStream;
   }
 
-  private CommonTokenStream getTokenStream() {
+  protected CommonTokenStream getTokenStream() {
     final CommonTokenStream tokenStreamUnboxed = tokenStream.getOrCompute();
     tokenStreamUnboxed.seek(0);
     return tokenStreamUnboxed;
   }
 
+  private P createParser(CommonTokenStream tokenStream) {
+    try {
+      return parserClass.getDeclaredConstructor(TokenStream.class)
+        .newInstance(tokenStream);
+    } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
+      throw new RuntimeException(e);
+    }
+  }
 }