From bffe0bfd16a626b33626a6ad7b8f9fb9a0a7799e Mon Sep 17 00:00:00 2001 From: Zach Levis Date: Sun, 21 Jun 2020 01:18:41 -0700 Subject: [PATCH] stub out some of the json5 loader --- .../configurate/build/ConfigurateDevPlugin.kt | 2 + etc/checkstyle/suppressions.xml | 3 + format/json5/build.gradle.kts | 17 +++ .../spongepowered/configurate/json5/Json5.g4 | 128 ++++++++++++++++++ .../configurate/json5/Helpers.java | 34 +++++ .../json5/Json5ConfigurationLoader.java | 120 ++++++++++++++++ .../configurate/json5/NumberLayout.java | 34 +++++ .../configurate/json5/QuotingStyle.java | 28 ++++ .../configurate/json5/ToNodeListener.java | 97 +++++++++++++ .../json5/Json5ConfigurationLoaderTest.java | 47 +++++++ .../configurate/json5/site-example.json5 | 12 ++ settings.gradle.kts | 2 +- 12 files changed, 523 insertions(+), 1 deletion(-) create mode 100644 format/json5/build.gradle.kts create mode 100644 format/json5/src/main/antlr/org/spongepowered/configurate/json5/Json5.g4 create mode 100644 format/json5/src/main/java/org/spongepowered/configurate/json5/Helpers.java create mode 100644 format/json5/src/main/java/org/spongepowered/configurate/json5/Json5ConfigurationLoader.java create mode 100644 format/json5/src/main/java/org/spongepowered/configurate/json5/NumberLayout.java create mode 100644 format/json5/src/main/java/org/spongepowered/configurate/json5/QuotingStyle.java create mode 100644 format/json5/src/main/java/org/spongepowered/configurate/json5/ToNodeListener.java create mode 100644 format/json5/src/test/java/org/spongepowered/configurate/json5/Json5ConfigurationLoaderTest.java create mode 100644 format/json5/src/test/resources/org/spongepowered/configurate/json5/site-example.json5 diff --git a/buildSrc/src/main/kotlin/org/spongepowered/configurate/build/ConfigurateDevPlugin.kt b/buildSrc/src/main/kotlin/org/spongepowered/configurate/build/ConfigurateDevPlugin.kt index 1dda3926d..90d4a1b81 100644 --- a/buildSrc/src/main/kotlin/org/spongepowered/configurate/build/ConfigurateDevPlugin.kt +++ b/buildSrc/src/main/kotlin/org/spongepowered/configurate/build/ConfigurateDevPlugin.kt @@ -62,6 +62,8 @@ class ConfigurateDevPlugin : Plugin { header = rootProject.file("LICENSE_HEADER") include("**/*.java") include("**/*.kt") + // TODO: this is probably not very efficient + exclude { it.file.absolutePath.contains("generated") } newLine = false } } diff --git a/etc/checkstyle/suppressions.xml b/etc/checkstyle/suppressions.xml index 15b21d729..e4fbf0049 100644 --- a/etc/checkstyle/suppressions.xml +++ b/etc/checkstyle/suppressions.xml @@ -9,4 +9,7 @@ + + + diff --git a/format/json5/build.gradle.kts b/format/json5/build.gradle.kts new file mode 100644 index 000000000..87e0e77b7 --- /dev/null +++ b/format/json5/build.gradle.kts @@ -0,0 +1,17 @@ +import org.spongepowered.configurate.build.core + +plugins { + id("org.spongepowered.configurate-component") + antlr +} + +configurations.compile { + exclude("org.antlr", "antlr4") +} + +dependencies { + antlr("org.antlr:antlr4:4.8-1") + implementation("org.antlr:antlr4-runtime:4.8-1") + + api(core()) +} diff --git a/format/json5/src/main/antlr/org/spongepowered/configurate/json5/Json5.g4 b/format/json5/src/main/antlr/org/spongepowered/configurate/json5/Json5.g4 new file mode 100644 index 000000000..44761be28 --- /dev/null +++ b/format/json5/src/main/antlr/org/spongepowered/configurate/json5/Json5.g4 @@ -0,0 +1,128 @@ +grammar Json5; +/** + * A grammar for the JSON5 document format. + * + * Based on the example grammar for JSON from The Definitive ANTLR 4 Reference ch6, + * and the JSON5 and ECMAScript 5.1 specifications. + */ + +@header { +package org.spongepowered.configurate.json5; +} + +// Value types // + +document: object EOF; + +value: object#Compound + | array#Compound + | Json5NumericLiteral#Literal + | Json5String#Literal + | BooleanLiteral#Literal + | NullLiteral#Literal + ; + +object: '{' member (',' member)* ','? '}' + | '{' '}' + ; +member : memberName ':' value; +memberName: IdentifierName + | Json5String + ; + +array: '[' value (',' value)* ','? ']' + | '[' ']' + ; + +NullLiteral: 'null'; +BooleanLiteral: 'true' + | 'false'; + +// ECMA Identifiers and Keywords // + +IdentifierName: IdentifierStart IdentifierPart*; + +Keyword: 'break' | 'do' | 'instanceof' | 'typeof' + | 'case' | 'else' | 'new' | 'var' + | 'catch' | 'finally' | 'return' | 'void' + | 'continue' | 'for' | 'switch' | 'while' + | 'debugger' | 'function' | 'this' | 'with' + | 'default' | 'if' | 'throw' + | 'delete' | 'in' | 'try' + ; + +FutureReservedWord: 'class' | 'enum' | 'extends' | 'super' + | 'const' | 'export' | 'import' + ; + + +fragment IdentifierStart: UnicodeLetter | '$' | '_' | ('\\' UnicodeEscapeSequence); +fragment IdentifierPart: IdentifierStart + | UnicodeCombiningMark + | UnicodeDigit + | UnicodeConnectorPunctuation + ; + +fragment UnicodeLetter: [\p{L}\p{Nl}]; // Unicode Letter or letterlike numeric character +fragment UnicodeCombiningMark: [\p{Mn}\p{Mc}]; // Non-spacing mark or Combining spacing mark +fragment UnicodeDigit: [\p{Nd}]; // Decimal number +fragment UnicodeConnectorPunctuation: [\p{Pc}]; + + +// Strings // + +WS: [ \t\n\r\u000B\u000C\u2028\u2029\ufeff]+ -> skip; + +Json5String: '"' Json5DoubleStringCharacter* '"' + | '\'' Json5SingleStringCharacter* '\''; + +fragment Json5DoubleStringCharacter: ~('\\' | '"' | '\r' | '\n') + | '\\' EscapeSequence + | LineContinuation + | '\u2028' + | '\u2029'; + +fragment Json5SingleStringCharacter: ~('\\' | '\'' | '\r' | '\n') + | '\\' EscapeSequence + | LineContinuation + | '\u2028' + | '\u2029'; + +fragment EscapeSequence: '0' DecimalDigit + | HexEscapeSequence + | UnicodeEscapeSequence + | ~[\r\n]; + +fragment LineContinuation: '\\' NL; + + +fragment HexEscapeSequence: 'x' HexDigit HexDigit; +fragment UnicodeEscapeSequence: 'u' HexDigit HexDigit HexDigit HexDigit; + +// Numbers // + +Json5NumericLiteral: [+-]? (NumericLiteral + | 'Inifinity' + | 'NaN'); + +fragment NumericLiteral: DecimalLiteral + | HexIntegerLiteral; + +DecimalLiteral: DecimalIntegerLiteral '.' DecimalDigit* ExponentPart? + | '.' DecimalDigit+ ExponentPart? + | DecimalIntegerLiteral ExponentPart?; + +HexIntegerLiteral: '0x' HexDigit+; +fragment DecimalIntegerLiteral: '0' + | NonZeroDigit DecimalDigit*; +fragment HexDigit: [0-9a-fA-F]; +fragment DecimalDigit: '0' | NonZeroDigit; +fragment NonZeroDigit: [1-9]; +fragment ExponentPart: [eE] [+-]? DecimalDigit+; + +// Comments and newlines // + +fragment NL : '\r'? '\n'; + +LINE_COMMENT: '//' .*? NL -> channel(HIDDEN); +BLOCK_COMMENT: '/*' .*? '*/' -> channel(HIDDEN); diff --git a/format/json5/src/main/java/org/spongepowered/configurate/json5/Helpers.java b/format/json5/src/main/java/org/spongepowered/configurate/json5/Helpers.java new file mode 100644 index 000000000..017b27b97 --- /dev/null +++ b/format/json5/src/main/java/org/spongepowered/configurate/json5/Helpers.java @@ -0,0 +1,34 @@ +/* + * Configurate + * Copyright (C) zml and Configurate contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.spongepowered.configurate.json5; + +/** + * Helpers for interacting with parsed values. + */ +final class Helpers { + + private Helpers() {} + + public static String unescape(final String input) { + return input; // TODO: implement + } + + public static String unquote(final String input) { + return input.substring(1, input.length() - 1); + } + +} diff --git a/format/json5/src/main/java/org/spongepowered/configurate/json5/Json5ConfigurationLoader.java b/format/json5/src/main/java/org/spongepowered/configurate/json5/Json5ConfigurationLoader.java new file mode 100644 index 000000000..092e304b3 --- /dev/null +++ b/format/json5/src/main/java/org/spongepowered/configurate/json5/Json5ConfigurationLoader.java @@ -0,0 +1,120 @@ +/* + * Configurate + * Copyright (C) zml and Configurate contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.spongepowered.configurate.json5; + +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.ConsoleErrorListener; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.atn.PredictionMode; +import org.antlr.v4.runtime.misc.ParseCancellationException; +import org.antlr.v4.runtime.tree.ParseTreeWalker; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.spongepowered.configurate.CommentedConfigurationNode; +import org.spongepowered.configurate.ConfigurationNode; +import org.spongepowered.configurate.ConfigurationOptions; +import org.spongepowered.configurate.RepresentationHint; +import org.spongepowered.configurate.loader.AbstractConfigurationLoader; +import org.spongepowered.configurate.loader.CommentHandler; +import org.spongepowered.configurate.loader.CommentHandlers; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Writer; + +// TODO: Package-private generated files -- we may have to provide our own templates (sigh) +// TODO: see https://github.com/antlr/antlr4/blob/master/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg + +/** + * A loader for the JSON5 language. + * + *

We use our own implementation, and aim to be fully complaint with the + * spec. Within those bounds, any features will be considered.

+ */ +public final class Json5ConfigurationLoader extends AbstractConfigurationLoader { + + public static final RepresentationHint NUMBER_FORMAT = RepresentationHint.of("number_format", NumberLayout.class, + NumberLayout.DECIMAL); + + public static Json5ConfigurationLoader.Builder builder() { + return new Builder(); + } + + Json5ConfigurationLoader(final @NonNull Builder builder) { + super(builder, new CommentHandler[]{CommentHandlers.SLASH_BLOCK, CommentHandlers.DOUBLE_SLASH}); + } + + @Override + protected void loadInternal(final CommentedConfigurationNode node, final BufferedReader reader) throws IOException { + final CharStream stream = CharStreams.fromReader(reader); + final Json5Lexer lexer = new Json5Lexer(stream); + final CommonTokenStream tokens = new CommonTokenStream(lexer); + final Json5Parser parser = new Json5Parser(tokens); + final ToNodeListener listener = new ToNodeListener(tokens, node); + + // ANTLR 4 Reference 13.7, improving speed + // TODO: Use actions instead to avoid generating parse tree. + parser.getInterpreter().setPredictionMode(PredictionMode.SLL); + parser.removeErrorListeners(); + parser.setErrorHandler(new BailErrorStrategy()); + try { + ParseTreeWalker.DEFAULT.walk(listener, parser.document()); + // success with the simpler method + } catch (final ParseCancellationException ex) { + // Reset node + node.setValue(null); + node.setComment(null); + // Reset state + tokens.seek(0); + parser.reset(); + + parser.addErrorListener(ConsoleErrorListener.INSTANCE); // TODO: Capture + throw all errors + parser.setErrorHandler(new DefaultErrorStrategy()); + + // with full prediction + parser.getInterpreter().setPredictionMode(PredictionMode.LL); + ParseTreeWalker.DEFAULT.walk(listener, parser.document()); + } + for (Token token : tokens.getTokens()) { + System.out.println(token); + } + } + + @Override + protected void saveInternal(final ConfigurationNode node, final Writer writer) throws IOException { + // TODO: writing + } + + @Override + public CommentedConfigurationNode createNode(final ConfigurationOptions options) { + return CommentedConfigurationNode.root(options); + } + + public static final class Builder extends AbstractConfigurationLoader.Builder { + + // TODO: provide options to customize output + + @Override + public @NonNull Json5ConfigurationLoader build() { + return new Json5ConfigurationLoader(this); + } + } + +} diff --git a/format/json5/src/main/java/org/spongepowered/configurate/json5/NumberLayout.java b/format/json5/src/main/java/org/spongepowered/configurate/json5/NumberLayout.java new file mode 100644 index 000000000..a1a3f2615 --- /dev/null +++ b/format/json5/src/main/java/org/spongepowered/configurate/json5/NumberLayout.java @@ -0,0 +1,34 @@ +/* + * Configurate + * Copyright (C) zml and Configurate contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.spongepowered.configurate.json5; + +/** + * A method of presenting numbers. + */ +public enum NumberLayout { + + /** + * Base-10 number representation. + */ + DECIMAL, + + /** + * Base-16 number representation. + */ + HEX; + +} diff --git a/format/json5/src/main/java/org/spongepowered/configurate/json5/QuotingStyle.java b/format/json5/src/main/java/org/spongepowered/configurate/json5/QuotingStyle.java new file mode 100644 index 000000000..48a9a7b65 --- /dev/null +++ b/format/json5/src/main/java/org/spongepowered/configurate/json5/QuotingStyle.java @@ -0,0 +1,28 @@ +/* + * Configurate + * Copyright (C) zml and Configurate contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.spongepowered.configurate.json5; + +/** + * The preferred style for quoted strings. + */ +public enum QuotingStyle { + + PREFER_SINGLE, + PREFER_DOUBLE, + PREFER_UNQUOTED; + +} diff --git a/format/json5/src/main/java/org/spongepowered/configurate/json5/ToNodeListener.java b/format/json5/src/main/java/org/spongepowered/configurate/json5/ToNodeListener.java new file mode 100644 index 000000000..56e488007 --- /dev/null +++ b/format/json5/src/main/java/org/spongepowered/configurate/json5/ToNodeListener.java @@ -0,0 +1,97 @@ +/* + * Configurate + * Copyright (C) zml and Configurate contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.spongepowered.configurate.json5; + +import org.antlr.v4.runtime.BufferedTokenStream; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.tree.TerminalNode; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.spongepowered.configurate.CommentedConfigurationNode; + +import java.util.List; + +/** + * TODO: Use actions and avoid generating a parse tree. + */ +class ToNodeListener extends Json5BaseListener { + + private final BufferedTokenStream tokens; + private CommentedConfigurationNode position; + + ToNodeListener(final BufferedTokenStream tokens, final CommentedConfigurationNode root) { + this.tokens = tokens; + this.position = root; + } + + private void applyCommentsToPosition(final ParserRuleContext ctx) { + final List commentChannel = this.tokens.getHiddenTokensToLeft(ctx.getStart().getTokenIndex(), Json5Lexer.HIDDEN); + if (commentChannel != null) { + final Token comment = commentChannel.get(0); + if (comment != null) { + final String rawText = comment.getText(); + switch (comment.getType()) { + case Json5Lexer.BLOCK_COMMENT: + this.position.setComment(rawText.substring(2, rawText.length() - 2)); + break; + case Json5Lexer.LINE_COMMENT: + this.position.setComment(rawText.substring(2)); + break; + default: // no-op + } + } + } + } + + @Override + public void enterCompound(final Json5Parser.CompoundContext ctx) { + // only advance if we're an array, otherwise this is handled in memberName + if (ctx.parent instanceof Json5Parser.ArrayContext) { // we're in an array + this.position = this.position.appendListNode(); + applyCommentsToPosition(ctx); + } + } + + @Override + public void exitCompound(final Json5Parser.CompoundContext ctx) { + this.position = this.position.getParent(); + super.exitCompound(ctx); + } + + @Override + public void enterLiteral(final Json5Parser.LiteralContext ctx) { + // only advance if we're an array, otherwise this is handled in memberName + if (ctx.parent instanceof Json5Parser.ArrayContext) { // we're in an array + this.position = this.position.appendListNode(); + applyCommentsToPosition(ctx); + } + this.position.setValue(ctx.getText()); // TODO: actually handle values + this.position = this.position.getParent(); + } + + @Override + public void enterMemberName(final Json5Parser.MemberNameContext ctx) { // a key in an object + final @Nullable TerminalNode identifier = ctx.IdentifierName(); + if (identifier != null) { // plain name + this.position = this.position.getNode(identifier.getText()); + } else { // quoted name + this.position = this.position.getNode(Helpers.unquote(ctx.getText())); + } + applyCommentsToPosition(ctx); + } + +} diff --git a/format/json5/src/test/java/org/spongepowered/configurate/json5/Json5ConfigurationLoaderTest.java b/format/json5/src/test/java/org/spongepowered/configurate/json5/Json5ConfigurationLoaderTest.java new file mode 100644 index 000000000..d73148056 --- /dev/null +++ b/format/json5/src/test/java/org/spongepowered/configurate/json5/Json5ConfigurationLoaderTest.java @@ -0,0 +1,47 @@ +/* + * Configurate + * Copyright (C) zml and Configurate contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.spongepowered.configurate.json5; + +import org.junit.jupiter.api.Test; +import org.spongepowered.configurate.CommentedConfigurationNode; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; + +public class Json5ConfigurationLoaderTest { + + private CommentedConfigurationNode loadFromString(final String content) throws IOException { + final Json5ConfigurationLoader loader = Json5ConfigurationLoader.builder() + .setSource(() -> new BufferedReader(new StringReader(content))) + .build(); + + return loader.load(); + } + + @Test + public void testWebsiteExample() throws IOException { + final Json5ConfigurationLoader loader = Json5ConfigurationLoader.builder() + .setUrl(getClass().getResource("site-example.json5")) + .build(); + + final CommentedConfigurationNode node = loader.load(); + + System.out.println(node); + } + +} diff --git a/format/json5/src/test/resources/org/spongepowered/configurate/json5/site-example.json5 b/format/json5/src/test/resources/org/spongepowered/configurate/json5/site-example.json5 new file mode 100644 index 000000000..476fe2d03 --- /dev/null +++ b/format/json5/src/test/resources/org/spongepowered/configurate/json5/site-example.json5 @@ -0,0 +1,12 @@ +{ + // comments + unquoted: 'and you can quote me on that', + singleQuotes: 'I can use "double quotes" here', + lineBreaks: "Look, Mom! \ +No \\n's!", + hexadecimal: 0xdecaf, + leadingDecimalPoint: .8675309, andTrailing: 8675309., + positiveSign: +1, + trailingComma: 'in objects', andIn: ['arrays',], + "backwardsCompatible": "with JSON", +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 1ae482159..1ecd9d7d7 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -9,7 +9,7 @@ listOf("core", "tool", "ext-kotlin", "bom", "examples").forEach { } // formats -listOf("gson", "hocon", "jackson", "xml", "yaml").forEach { +listOf("gson", "hocon", "jackson", "json5", "xml", "yaml").forEach { include(":format:$it") // findProject(":format:$it")?.name = "$prefix-$it" }