From 3ba4964a69b74bdce725af95fd5257dc135bf390 Mon Sep 17 00:00:00 2001 From: Jamie Willis Date: Wed, 10 Jan 2024 13:18:38 +0000 Subject: [PATCH] restructured naming for comment descriptions, closing #168 --- docs/tutorial/interlude-1-haskell.md | 8 +-- .../machine/instructions/TokenInstrs.scala | 12 ++--- .../token/descriptions/SpaceDesc.scala | 51 +++++++++---------- .../test/scala/parsley/token/SpaceTests.scala | 32 ++++++------ .../scala/parsley/token/TokeniserTests.scala | 12 ++--- 5 files changed, 57 insertions(+), 58 deletions(-) diff --git a/docs/tutorial/interlude-1-haskell.md b/docs/tutorial/interlude-1-haskell.md index e00965513..1f94db1f7 100644 --- a/docs/tutorial/interlude-1-haskell.md +++ b/docs/tutorial/interlude-1-haskell.md @@ -105,10 +105,10 @@ object lexer { escapeSequences = text.EscapeDesc.haskell, ), SpaceDesc.plain.copy( - commentStart = "{-", - commentEnd = "-}", - commentLine = "--", - nestedComments = true, + lineCommentStart = "--", + multiLineCommentStart = "{-", + multiLineCommentEnd = "-}", + multiLineNestedComments = true, space = Basic(c => c == ' ' || c == '\t'), ) ) diff --git a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/TokenInstrs.scala b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/TokenInstrs.scala index 9c613ddc0..f659d2486 100644 --- a/parsley/shared/src/main/scala/parsley/internal/machine/instructions/TokenInstrs.scala +++ b/parsley/shared/src/main/scala/parsley/internal/machine/instructions/TokenInstrs.scala @@ -75,8 +75,8 @@ private [internal] final class TokenComment private ( endOfSingleComment: Iterable[ExpectDesc], ) extends CommentLexer { def this(desc: SpaceDesc, errConfig: ErrorConfig) = { - this(desc.commentStart, desc.commentEnd, desc.commentLine, desc.nestedComments, desc.commentLineAllowsEOF, - errConfig.labelSpaceEndOfMultiComment.asExpectItems(desc.commentEnd), + this(desc.multiLineCommentStart, desc.multiLineCommentEnd, desc.lineCommentStart, desc.multiLineNestedComments, desc.lineCommentAllowsEOF, + errConfig.labelSpaceEndOfMultiComment.asExpectItems(desc.multiLineCommentEnd), errConfig.labelSpaceEndOfLineComment.asExpectDescs("end of line")) } private [this] final val openingSize = Math.max(start.codePointCount(0, start.length), line.codePointCount(0, line.length)) @@ -173,8 +173,8 @@ private [internal] final class TokenWhiteSpace private ( protected [this] val endOfMultiComment: Iterable[ExpectItem], protected [this] val endOfSingleComment: Iterable[ExpectDesc]) extends WhiteSpaceLike { def this(ws: Char => Boolean, desc: SpaceDesc, errConfig: ErrorConfig) = { - this(ws, desc.commentStart, desc.commentEnd, desc.commentLine, desc.nestedComments, desc.commentLineAllowsEOF, - errConfig.labelSpaceEndOfMultiComment.asExpectItems(desc.commentEnd), + this(ws, desc.multiLineCommentStart, desc.multiLineCommentEnd, desc.lineCommentStart, desc.multiLineNestedComments, desc.lineCommentAllowsEOF, + errConfig.labelSpaceEndOfMultiComment.asExpectItems(desc.multiLineCommentEnd), errConfig.labelSpaceEndOfLineComment.asExpectDescs("end of line")) } override def spaces(ctx: Context): Unit = { @@ -196,8 +196,8 @@ private [internal] final class TokenSkipComments private ( protected [this] val endOfMultiComment: Iterable[ExpectItem], protected [this] val endOfSingleComment: Iterable[ExpectDesc]) extends WhiteSpaceLike { def this(desc: SpaceDesc, errConfig: ErrorConfig) = { - this(desc.commentStart, desc.commentEnd, desc.commentLine, desc.nestedComments, desc.commentLineAllowsEOF, - errConfig.labelSpaceEndOfMultiComment.asExpectItems(desc.commentEnd), + this(desc.multiLineCommentStart, desc.multiLineCommentEnd, desc.lineCommentStart, desc.multiLineNestedComments, desc.lineCommentAllowsEOF, + errConfig.labelSpaceEndOfMultiComment.asExpectItems(desc.multiLineCommentEnd), errConfig.labelSpaceEndOfLineComment.asExpectDescs("end of line")) } override def spaces(ctx: Context): Unit = () diff --git a/parsley/shared/src/main/scala/parsley/token/descriptions/SpaceDesc.scala b/parsley/shared/src/main/scala/parsley/token/descriptions/SpaceDesc.scala index 62a3e3f4d..cd74fc726 100644 --- a/parsley/shared/src/main/scala/parsley/token/descriptions/SpaceDesc.scala +++ b/parsley/shared/src/main/scala/parsley/token/descriptions/SpaceDesc.scala @@ -9,30 +9,29 @@ import parsley.token.predicate.{CharPredicate, Unicode} /** This class describes how whitespace should be handled lexically. * - * @param commentStart how do multi-line comments start? (empty for no multi-line comments) - * @param commentEnd how do multi-line comments end? (empty for no multi-line comments) - * @param commentLine how do single-line comments start? (empty for no single-line comments) - * @param commentLineAllowsEOF can a single-line comment be terminated by the end-of-file, or '''must''' it ends with a newline - * @param nestedComments can multi-line comments be nested within each other? + * @param lineCommentStart how do single-line comments start? (empty for no single-line comments) + * @param lineCommentAllowsEOF can a single-line comment be terminated by the end-of-file, or '''must''' it ends with a newline + * @param multiLineCommentStart how do multi-line comments start? (empty for no multi-line comments) + * @param multiLineCommentEnd how do multi-line comments end? (empty for no multi-line comments) + * @param multiLineNestedComments can multi-line comments be nested within each other? * @param space what characters serve as whitespace within the language? * @param whitespaceIsContextDependent can the definition of whitespace change depending on context? (in Python, say, newlines are valid whitespace * within parentheses, but are significant outside of them) * @since 4.0.0 */ -final case class SpaceDesc (commentStart: String, - commentEnd: String, - commentLine: String, - commentLineAllowsEOF: Boolean, - nestedComments: Boolean, +final case class SpaceDesc (lineCommentStart: String, + lineCommentAllowsEOF: Boolean, + multiLineCommentStart: String, + multiLineCommentEnd: String, + multiLineNestedComments: Boolean, space: CharPredicate, whitespaceIsContextDependent: Boolean) { - private [token] lazy val supportsComments = { - require(commentStart.nonEmpty == commentEnd.nonEmpty, "multi-line comments must describe both start and end") - val on = commentStart.nonEmpty || commentLine.nonEmpty - require(commentStart.isEmpty || commentLine.isEmpty || !commentLine.startsWith(commentStart), - "multi-line comments which are a valid prefix of a single-line comment are not allowed as this causes ambiguity in the parser") - on - } + require(multiLineCommentStart.nonEmpty == multiLineCommentEnd.nonEmpty, "multi-line comments must describe both start and end") + require(multiLineCommentStart.isEmpty || lineCommentStart.isEmpty || !lineCommentStart.startsWith(multiLineCommentStart), + "multi-line comments which are a valid prefix of a single-line comment are not allowed as this causes ambiguity in the parser") + require(!multiLineCommentStart.contains('\n') && !multiLineCommentEnd.contains('\n') && !lineCommentStart.contains('\n'), + "comment descriptions cannot include newlines") + private [token] lazy val supportsComments = multiLineCommentStart.nonEmpty || lineCommentStart.nonEmpty } /** This object contains any default configurations describing whitespace. @@ -42,11 +41,11 @@ object SpaceDesc { /** The plain definition of space, with no comments, no nested comments, and any unicode space character. * * {{{ - * commentStart = "" + * multiLineCommentStart = "" * commentEnd = "" - * commentLine = "" - * commentLineAllowsEOF = true - * nestedComments = false + * lineComment = "" + * lineCommentAllowsEOF = true + * multiLineNestedComments = false * space = Unicode(Character.isWhitespace) * whitespaceIsContextDependent = false * }}} @@ -54,11 +53,11 @@ object SpaceDesc { * @since 4.0.0 */ val plain = SpaceDesc( - commentStart = "", - commentEnd = "", - commentLine = "", - commentLineAllowsEOF = true, - nestedComments = false, + multiLineCommentStart = "", + multiLineCommentEnd = "", + lineCommentStart = "", + lineCommentAllowsEOF = true, + multiLineNestedComments = false, space = Unicode(Character.isWhitespace), whitespaceIsContextDependent = false ) diff --git a/parsley/shared/src/test/scala/parsley/token/SpaceTests.scala b/parsley/shared/src/test/scala/parsley/token/SpaceTests.scala index 84fb622f9..7ca2e6ee0 100644 --- a/parsley/shared/src/test/scala/parsley/token/SpaceTests.scala +++ b/parsley/shared/src/test/scala/parsley/token/SpaceTests.scala @@ -39,8 +39,8 @@ class SpaceTests extends ParsleyTest { "/**/ a" -> None, ) - val basicLine = basicNoComments.copy(commentLine = "--") - val unicodeLine = unicodeNoComments.copy(commentLine = "--") + val basicLine = basicNoComments.copy(lineCommentStart = "--") + val unicodeLine = unicodeNoComments.copy(lineCommentStart = "--") it should "parse spaces and line comments when defined" in { cases(makeSpace(basicLine).whiteSpace *> string("a")) ( @@ -59,8 +59,8 @@ class SpaceTests extends ParsleyTest { ) } - val basicMulti = basicNoComments.copy(commentStart = "/*", commentEnd = "*/") - val unicodeMulti = unicodeNoComments.copy(commentStart = "/*", commentEnd = "*/") + val basicMulti = basicNoComments.copy(multiLineCommentStart = "/*", multiLineCommentEnd = "*/") + val unicodeMulti = unicodeNoComments.copy(multiLineCommentStart = "/*", multiLineCommentEnd = "*/") it should "parse spaces and multi-line comment when defined" in { cases(makeSpace(basicMulti).whiteSpace *> string("a")) ( @@ -79,8 +79,8 @@ class SpaceTests extends ParsleyTest { ) } - val basicMixed = basicNoComments.copy(commentLine = "#", commentStart = "##", commentEnd = "##") - val unicodeMixed = unicodeNoComments.copy(commentLine = "#", commentStart = "##", commentEnd = "##") + val basicMixed = basicNoComments.copy(lineCommentStart = "#", multiLineCommentStart = "##", multiLineCommentEnd = "##") + val unicodeMixed = unicodeNoComments.copy(lineCommentStart = "#", multiLineCommentStart = "##", multiLineCommentEnd = "##") it should "parse spaces and mixed comments when defined" in { cases(makeSpace(basicMixed).whiteSpace *> string("a")) ( @@ -111,10 +111,10 @@ class SpaceTests extends ParsleyTest { unicode.whiteSpace shouldBe unicode.skipComments } - val basicLineEOF = basicLine.copy(commentLineAllowsEOF = true) - val basicLineNoEOF = basicLine.copy(commentLineAllowsEOF = false) - val basicMixedEOF = basicMixed.copy(commentLineAllowsEOF = true) - val basicMixedNoEOF = basicMixed.copy(commentLineAllowsEOF = false) + val basicLineEOF = basicLine.copy(lineCommentAllowsEOF = true) + val basicLineNoEOF = basicLine.copy(lineCommentAllowsEOF = false) + val basicMixedEOF = basicMixed.copy(lineCommentAllowsEOF = true) + val basicMixedNoEOF = basicMixed.copy(lineCommentAllowsEOF = false) it should "allow for line comments to end in EOF" in { cases(makeSpace(basicLineEOF).whiteSpace)("--hello world" -> Some(())) @@ -126,9 +126,9 @@ class SpaceTests extends ParsleyTest { cases(makeSpace(basicMixedNoEOF).whiteSpace)("--hello world" -> None) } - val basicMultiNested = basicMulti.copy(nestedComments = true) + val basicMultiNested = basicMulti.copy(multiLineNestedComments = true) // having the same start and end makes this... weird - val basicMixedNested = basicMixed.copy(nestedComments = true, commentStart = "#-", commentEnd = "-#") + val basicMixedNested = basicMixed.copy(multiLineNestedComments = true, multiLineCommentStart = "#-", multiLineCommentEnd = "-#") it should "parse nested comments when applicable" in { cases(makeSpace(basicMultiNested).whiteSpace) ( @@ -143,8 +143,8 @@ class SpaceTests extends ParsleyTest { ) } - val basicMultiNonNested = basicMulti.copy(nestedComments = false) - val basicMixedNonNested = basicMulti.copy(nestedComments = false) + val basicMultiNonNested = basicMulti.copy(multiLineNestedComments = false) + val basicMixedNonNested = basicMulti.copy(multiLineNestedComments = false) it should "not parse nested comments when applicable" in { cases(makeSpace(basicMultiNonNested).whiteSpace) ( @@ -220,8 +220,8 @@ class SpaceTests extends ParsleyTest { } it should "not aggressively eat everything" in { - val lexer1 = makeSpace(basicCommentsOnly.copy(commentStart = "", commentEnd = "")) - val lexer2 = makeSpace(basicCommentsOnly.copy(commentLine = "")) + val lexer1 = makeSpace(basicCommentsOnly.copy(multiLineCommentStart = "", multiLineCommentEnd = "")) + val lexer2 = makeSpace(basicCommentsOnly.copy(lineCommentStart = "")) val lexer3 = makeSpace(unicodeCommentsOnly) (lexer1.skipComments *> char('a')).parse("a") shouldBe a [Success[_]] (lexer2.skipComments *> char('a')).parse("a") shouldBe a [Success[_]] diff --git a/parsley/shared/src/test/scala/parsley/token/TokeniserTests.scala b/parsley/shared/src/test/scala/parsley/token/TokeniserTests.scala index 68b54f721..46f7dd558 100644 --- a/parsley/shared/src/test/scala/parsley/token/TokeniserTests.scala +++ b/parsley/shared/src/test/scala/parsley/token/TokeniserTests.scala @@ -27,16 +27,16 @@ class TokeniserTests extends ParsleyTest { caseSensitive = true), desc.numeric.NumericDesc.plain, desc.text.TextDesc.plain, - desc.SpaceDesc(commentStart = "/*", - commentEnd = "*/", - commentLine = "//", - commentLineAllowsEOF = true, - nestedComments = true, + desc.SpaceDesc(multiLineCommentStart = "/*", + multiLineCommentEnd = "*/", + lineCommentStart = "//", + lineCommentAllowsEOF = true, + multiLineNestedComments = true, space = token.predicate.Basic(_.isWhitespace), whitespaceIsContextDependent = false)) val scala_ = scala.copy( - spaceDesc = scala.spaceDesc.copy(nestedComments = false) + spaceDesc = scala.spaceDesc.copy(multiLineNestedComments = false) ) val tokeniser = new token.Lexer(scala) val tokeniser_ = new token.Lexer(scala_)