Skip to content

Commit

Permalink
Removed a bunch of redundant AST Nodes and instructions
Browse files Browse the repository at this point in the history
  • Loading branch information
@j-mie6
j-mie6 committed Jan 5, 2021
1 parent 08807aa commit 6d79f01
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 85 deletions.
46 changes: 22 additions & 24 deletions src/main/scala/parsley/Token.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,30 +114,33 @@ object BitGen
*/
final class TokenParser(lang: LanguageDef)
{
private def keyOrOp(startImpl: Impl, letterImpl: Impl, parser: Parsley[String], predicate: String => Boolean, name: String,
builder: (TokenSet, TokenSet) => deepembedding.Parsley[String]) = (startImpl, letterImpl) match
{
case (BitSetImpl(start), BitSetImpl(letter)) => lexeme(new Parsley(builder(start, letter)))
case (BitSetImpl(start), Predicate(letter)) => lexeme(new Parsley(builder(start, letter)))
case (Predicate(start), BitSetImpl(letter)) => lexeme(new Parsley(builder(start, letter)))
case (Predicate(start), Predicate(letter)) => lexeme(new Parsley(builder(start, letter)))
case _ => lexeme(attempt(parser.guard(predicate, s"unexpected $name " + _)))
private def keyOrOp(startImpl: Impl, letterImpl: Impl, parser: Parsley[String], predicate: String => Boolean,
combinatorName: String, name: String, illegalName: String) = {
val builder = (start: TokenSet, letter: TokenSet) =>
new Parsley(new deepembedding.NonSpecific(combinatorName, name, illegalName, start, letter, !predicate(_)))
lexeme((startImpl, letterImpl) match
{
case (BitSetImpl(start), BitSetImpl(letter)) => builder(start, letter)
case (BitSetImpl(start), Predicate(letter)) => builder(start, letter)
case (Predicate(start), BitSetImpl(letter)) => builder(start, letter)
case (Predicate(start), Predicate(letter)) => builder(start, letter)
case _ => attempt((parser ? name).guard(predicate, s"unexpected $illegalName " + _))
})
}

// Identifiers & Reserved words
/**This lexeme parser parses a legal identifier. Returns the identifier string. This parser will
* fail on identifiers that are reserved words (i.e. keywords). Legal identifier characters and
* keywords are defined in the `LanguageDef` provided to the token parser. An identifier is treated
* as a single token using `attempt`.*/
lazy val identifier: Parsley[String] = keyOrOp(lang.identStart, lang.identLetter, ident, !isReservedName(_), "keyword",
(start, letter) => new deepembedding.Identifier(start, letter, theReservedNames))
lazy val identifier: Parsley[String] = keyOrOp(lang.identStart, lang.identLetter, ident, !isReservedName(_), "identifier", "identifier", "keyword")

/**The lexeme parser `keyword(name)` parses the symbol `name`, but it also checks that the `name`
* is not a prefix of a valid identifier. A `keyword` is treated as a single token using `attempt`.*/
def keyword(name: String): Parsley[Unit] = lang.identLetter match
{
case BitSetImpl(letter) => lexeme(new Parsley(new deepembedding.Keyword(name, letter, lang.caseSensitive)))
case Predicate(letter) => lexeme(new Parsley(new deepembedding.Keyword(name, letter, lang.caseSensitive)))
case BitSetImpl(letter) => lexeme(new Parsley(new deepembedding.Specific("keyword", name, letter, lang.caseSensitive)))
case Predicate(letter) => lexeme(new Parsley(new deepembedding.Specific("keyword", name, letter, lang.caseSensitive)))
case _ => lexeme(attempt(caseString(name) *> notFollowedBy(identLetter) ? ("end of " + name)))
}

Expand All @@ -151,21 +154,19 @@ final class TokenParser(lang: LanguageDef)
private val theReservedNames = if (lang.caseSensitive) lang.keywords else lang.keywords.map(_.toLowerCase)
private lazy val identStart = toParser(lang.identStart)
private lazy val identLetter = toParser(lang.identLetter)
private lazy val ident = lift2((c: Char, cs: List[Char]) => (c::cs).mkString, identStart, many(identLetter)) ? "identifier"
private lazy val ident = lift2((c: Char, cs: List[Char]) => (c::cs).mkString, identStart, many(identLetter))

// Operators & Reserved ops
/**This lexeme parser parses a legal operator. Returns the name of the operator. This parser
* will fail on any operators that are reserved operators. Legal operator characters and
* reserved operators are defined in the `LanguageDef` provided to the token parser. A
* `userOp` is treated as a single token using `attempt`.*/
lazy val userOp: Parsley[String] = keyOrOp(lang.opStart, lang.opLetter, oper, !isReservedOp(_), "reserved operator",
(start, letter) => new deepembedding.UserOp(start, letter, lang.operators))
lazy val userOp: Parsley[String] = keyOrOp(lang.opStart, lang.opLetter, oper, !isReservedOp(_), "userOp", "operator", "reserved operator")

/**This non-lexeme parser parses a reserved operator. Returns the name of the operator.
* Legal operator characters and reserved operators are defined in the `LanguageDef`
* provided to the token parser. A `reservedOp_` is treated as a single token using `attempt`.*/
lazy val reservedOp_ : Parsley[String] = keyOrOp(lang.opStart, lang.opLetter, oper, isReservedOp(_), "non-reserved operator",
(start, letter) => new deepembedding.ReservedOp(start, letter, lang.operators))
lazy val reservedOp_ : Parsley[String] = keyOrOp(lang.opStart, lang.opLetter, oper, isReservedOp(_), "reservedOp", "operator", "non-reserved operator")

/**This lexeme parser parses a reserved operator. Returns the name of the operator. Legal
* operator characters and reserved operators are defined in the `LanguageDef` provided
Expand All @@ -182,8 +183,8 @@ final class TokenParser(lang: LanguageDef)
* `attempt`.*/
def operator_(name: String): Parsley[Unit] = lang.opLetter match
{
case BitSetImpl(letter) => new Parsley(new deepembedding.Operator(name, letter))
case Predicate(letter) => new Parsley(new deepembedding.Operator(name, letter))
case BitSetImpl(letter) => new Parsley(new deepembedding.Specific("operator", name, letter, true))
case Predicate(letter) => new Parsley(new deepembedding.Specific("operator", name, letter, true))
case _ => attempt(name *> notFollowedBy(opLetter) ? ("end of " + name))
}

Expand All @@ -200,7 +201,7 @@ final class TokenParser(lang: LanguageDef)
private def isReservedOp(op: String): Boolean = lang.operators.contains(op)
private lazy val opStart = toParser(lang.opStart)
private lazy val opLetter = toParser(lang.opLetter)
private lazy val oper = lift2((c: Char, cs: List[Char]) => (c::cs).mkString, opStart, many(opLetter)) ? "operator"
private lazy val oper = lift2((c: Char, cs: List[Char]) => (c::cs).mkString, opStart, many(opLetter))

// Chars & Strings
/**This lexeme parser parses a single literal character. Returns the literal character value.
Expand Down Expand Up @@ -312,10 +313,7 @@ final class TokenParser(lang: LanguageDef)
* or "0O". Returns the value of the number.*/
lazy val octal: Parsley[Int] = lexeme('0' *> octal_)

private def number(base: Int, baseDigit: Parsley[Char]): Parsley[Int] =
{
for (digits <- some(baseDigit)) yield digits.foldLeft(0)((x, d) => base*x + d.asDigit)
}
private def number(base: Int, baseDigit: Parsley[Char]): Parsley[Int] = baseDigit.foldLeft(0)((x, d) => base*x + d.asDigit)

// White space & symbols
/**Lexeme parser `symbol(s)` parses `string(s)` and skips trailing white space.*/
Expand Down
5 changes: 2 additions & 3 deletions src/main/scala/parsley/internal/deepembedding/AlternativeEmbedding.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,7 @@ private [parsley] final class <|>[A, B](_p: =>Parsley[A], _q: =>Parsley[B]) exte
val (c, expected) = lead match {
case ct@CharTok(d) => (d, ct.expected)
case st@StringTok(s) => (s.head, if (st.expected == null) "\"" + s + "\"" else st.expected)
case kw@Keyword(k) => (k.head, if (kw.expected == null) k else kw.expected)
case op@Operator(o) => (o.head, if (op.expected == null) o else op.expected)
case st@Specific(s) => (s.head, if (st.expected == null) s else st.expected)
case op@MaxOp(o) => (o.head, if (op.expected == null) o else op.expected)
case sl: StringLiteral => ('"', if (sl.expected == null) "string" else sl.expected)
case rs: RawStringLiteral => ('"', if (rs.expected == null) "string" else rs.expected)
Expand All @@ -155,7 +154,7 @@ private [parsley] final class <|>[A, B](_p: =>Parsley[A], _q: =>Parsley[B]) exte
}
@tailrec private def tablable(p: Parsley[_]): Option[Parsley[_]] = p match {
// CODO: Numeric parsers by leading digit (This one would require changing the foldTablified function a bit)
case t@(_: CharTok | _: StringTok | _: Keyword | _: StringLiteral | _: RawStringLiteral | _: Operator | _: MaxOp) => Some(t)
case t@(_: CharTok | _: StringTok | _: Specific | _: StringLiteral | _: RawStringLiteral | _: MaxOp) => Some(t)
case Attempt(t) => tablable(t)
case (_: Pure[_]) <*> t => tablable(t)
case Lift2(_, t, _) => tablable(t)
Expand Down
50 changes: 20 additions & 30 deletions src/main/scala/parsley/internal/deepembedding/TokenEmbedding.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,51 +4,44 @@ import parsley.TokenParser.TokenSet
import Sign.SignType
import parsley.internal.{instructions, UnsafeOption}

private [parsley] class WhiteSpace(ws: TokenSet, start: String, end: String, line: String, nested: Boolean)
private [parsley] final class WhiteSpace(ws: TokenSet, start: String, end: String, line: String, nested: Boolean)
extends Singleton[Unit]("whiteSpace", new instructions.TokenWhiteSpace(ws, start, end, line, nested))

private [parsley] class SkipComments(start: String, end: String, line: String, nested: Boolean)
private [parsley] final class SkipComments(start: String, end: String, line: String, nested: Boolean)
extends Singleton[Unit]("skipComments", new instructions.TokenSkipComments(start, end, line, nested))

private [parsley] class Comment(start: String, end: String, line: String, nested: Boolean)
private [parsley] final class Comment(start: String, end: String, line: String, nested: Boolean)
extends Singleton[Unit]("comment", new instructions.TokenComment(start, end, line, nested))

private [parsley] class Sign[A](ty: SignType, val expected: UnsafeOption[String] = null)
private [parsley] final class Sign[A](ty: SignType, val expected: UnsafeOption[String] = null)
extends SingletonExpect[A => A]("sign", new Sign(ty, _), new instructions.TokenSign(ty, expected))

private [parsley] class Natural(val expected: UnsafeOption[String] = null)
private [parsley] final class Natural(val expected: UnsafeOption[String] = null)
extends SingletonExpect[Int]("natural", new Natural(_), new instructions.TokenNatural(expected))

private [parsley] class Float(val expected: UnsafeOption[String] = null)
private [parsley] final class Float(val expected: UnsafeOption[String] = null)
extends SingletonExpect[Double]("float", new Float(_), new instructions.TokenFloat(expected))

private [parsley] class Escape(val expected: UnsafeOption[String] = null)
private [parsley] final class Escape(val expected: UnsafeOption[String] = null)
extends SingletonExpect[Char]("escape", new Escape(_), new instructions.TokenEscape(expected))

private [parsley] class StringLiteral(ws: TokenSet, val expected: UnsafeOption[String] = null)
private [parsley] final class StringLiteral(ws: TokenSet, val expected: UnsafeOption[String] = null)
extends SingletonExpect[String]("stringLiteral", new StringLiteral(ws, _), new instructions.TokenString(ws, expected))

private [parsley] class RawStringLiteral(val expected: UnsafeOption[String] = null)
private [parsley] final class RawStringLiteral(val expected: UnsafeOption[String] = null)
extends SingletonExpect[String]("rawStringLiteral", new RawStringLiteral(_), new instructions.TokenRawString(expected))

private [parsley] class Identifier(start: TokenSet, letter: TokenSet, keywords: Set[String], val expected: UnsafeOption[String] = null)
extends SingletonExpect[String]("identifier", new Identifier(start, letter, keywords, _),
new instructions.TokenIdentifier(start, letter, keywords, expected))
private [parsley] class NonSpecific(combinatorName: String, name: String, illegalName: String, start: TokenSet,
letter: TokenSet, illegal: String => Boolean, val expected: UnsafeOption[String] = null)
extends SingletonExpect[String](combinatorName, new NonSpecific(combinatorName, name, illegalName, start, letter, illegal, _),
new instructions.TokenNonSpecific(name, illegalName)(start, letter, illegal, expected))

private [parsley] class UserOp(start: TokenSet, letter: TokenSet, ops: Set[String], val expected: UnsafeOption[String] = null)
extends SingletonExpect[String]("userOp", new UserOp(start, letter, ops, _), new instructions.TokenUserOperator(start, letter, ops, expected))
private [parsley] final class Specific(name: String, private [Specific] val specific: String,
letter: TokenSet, caseSensitive: Boolean, val expected: UnsafeOption[String] = null)
extends SingletonExpect[Unit](s"$name($specific)", new Specific(name, specific, letter, caseSensitive, _),
new instructions.TokenSpecific(specific, letter, caseSensitive, expected))

private [parsley] class ReservedOp(start: TokenSet, letter: TokenSet, ops: Set[String], val expected: UnsafeOption[String] = null)
extends SingletonExpect[String]("reservedOp", new ReservedOp(start, letter, ops, _), new instructions.TokenOperator(start, letter, ops, expected))

private [parsley] class Keyword(private [Keyword] val keyword: String, letter: TokenSet, caseSensitive: Boolean, val expected: UnsafeOption[String] = null)
extends SingletonExpect[Unit](s"keyword($keyword)", new Keyword(keyword, letter, caseSensitive, _),
new instructions.TokenKeyword(keyword, letter, caseSensitive, expected))

private [parsley] class Operator(private [Operator] val operator: String, letter: TokenSet, val expected: UnsafeOption[String] = null)
extends SingletonExpect[Unit](s"operator($operator)", new Operator(operator, letter, _), new instructions.TokenOperator_(operator, letter, expected))

private [parsley] class MaxOp(private [MaxOp] val operator: String, ops: Set[String], val expected: UnsafeOption[String] = null)
private [parsley] final class MaxOp(private [MaxOp] val operator: String, ops: Set[String], val expected: UnsafeOption[String] = null)
extends SingletonExpect[Unit](s"maxOp($operator)", new MaxOp(operator, ops, _), new instructions.TokenMaxOp(operator, ops, expected))

private [parsley] object Sign {
Expand All @@ -64,11 +57,8 @@ private [parsley] object Sign {
}

// $COVERAGE-OFF$
private [deepembedding] object Keyword {
def unapply(self: Keyword): Option[String] = Some(self.keyword)
}
private [deepembedding] object Operator {
def unapply(self: Operator): Option[String] = Some(self.operator)
private [deepembedding] object Specific {
def unapply(self: Specific): Option[String] = Some(self.specific)
}
private [deepembedding] object MaxOp {
def unapply(self: MaxOp): Option[String] = Some(self.operator)
Expand Down
41 changes: 13 additions & 28 deletions src/main/scala/parsley/internal/instructions/TokenInstrs.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,11 +121,11 @@ private [internal] final class TokenSkipComments(start: String, end: String, lin
// $COVERAGE-ON$
}

private [instructions] abstract class TokenLexi(name: String, illegalName: String)
private [internal] final class TokenNonSpecific(name: String, illegalName: String)
(start: TokenSet, letter: TokenSet, illegal: String => Boolean, _expected: UnsafeOption[String]) extends Instr {
private val expected = if (_expected == null) name else _expected

final override def apply(ctx: Context): Unit = {
override def apply(ctx: Context): Unit = {
if (ctx.moreInput && start(ctx.nextChar)) {
val name = new StringBuilder()
name += ctx.nextChar
Expand All @@ -146,7 +146,7 @@ private [instructions] abstract class TokenLexi(name: String, illegalName: Strin
}
}

@tailrec private final def restOfToken(ctx: Context, tok: StringBuilder): Unit = {
@tailrec private def restOfToken(ctx: Context, tok: StringBuilder): Unit = {
if (ctx.moreInput && letter(ctx.nextChar)) {
tok += ctx.nextChar
ctx.offset += 1
Expand All @@ -156,20 +156,11 @@ private [instructions] abstract class TokenLexi(name: String, illegalName: Strin
}

// $COVERAGE-OFF$
final override def toString: String = s"TokenLexi($name)"
override def toString: String = s"TokenNonSpecific($name)"
// $COVERAGE-ON$
}

private [internal] final class TokenIdentifier(start: TokenSet, letter: TokenSet, keywords: Set[String], _expected: UnsafeOption[String])
extends TokenLexi("identifier", "keyword")(start, letter, keywords, _expected)

private [internal] final class TokenUserOperator(start: TokenSet, letter: TokenSet, reservedOps: Set[String], _expected: UnsafeOption[String])
extends TokenLexi("operator", "reserved operator")(start, letter, reservedOps, _expected)

private [internal] final class TokenOperator(start: TokenSet, letter: TokenSet, reservedOps: Set[String], _expected: UnsafeOption[String])
extends TokenLexi("operator", "non-reserved operator")(start, letter, reservedOps.andThen(!_), _expected)

private [instructions] abstract class TokenSpecific(_specific: String, caseSensitive: Boolean, _expected: UnsafeOption[String]) extends Instr {
private [instructions] abstract class TokenSpecificAllowTrailing(_specific: String, caseSensitive: Boolean, _expected: UnsafeOption[String]) extends Instr {
private final val expected = if (_expected == null) _specific else _expected
protected final val expectedEnd = if (_expected == null) "end of " + _specific else _expected
private final val specific = (if (caseSensitive) _specific else _specific.toLowerCase).toCharArray
Expand All @@ -195,15 +186,11 @@ private [instructions] abstract class TokenSpecific(_specific: String, caseSensi
if (ctx.inputsz >= ctx.offset + strsz) readSpecific(ctx, ctx.offset, 0)
else ctx.fail(expected)
}

// $COVERAGE-OFF$
override def toString: String = s"TokenSpecific(${_specific})"
// $COVERAGE-ON$
}

private [internal] abstract class TokenSpecificNoTrailLetter(keyword: String, letter: TokenSet, caseSensitive: Boolean, expected: UnsafeOption[String])
extends TokenSpecific(keyword, caseSensitive, expected) {
final override def postprocess(ctx: Context, i: Int): Unit = {
private [internal] final class TokenSpecific(_specific: String, letter: TokenSet, caseSensitive: Boolean, expected: UnsafeOption[String])
extends TokenSpecificAllowTrailing(_specific, caseSensitive, expected) {
override def postprocess(ctx: Context, i: Int): Unit = {
if (i < ctx.inputsz && letter(ctx.input(i))) {
ctx.fail(expectedEnd)
ctx.restoreState()
Expand All @@ -213,16 +200,14 @@ private [internal] abstract class TokenSpecificNoTrailLetter(keyword: String, le
ctx.pushAndContinue(())
}
}
}

private [internal] final class TokenKeyword(keyword: String, letter: TokenSet, caseSensitive: Boolean, expected: UnsafeOption[String])
extends TokenSpecificNoTrailLetter(keyword, letter, caseSensitive, expected)

private [internal] final class TokenOperator_(operator: String, letter: TokenSet, expected: UnsafeOption[String])
extends TokenSpecificNoTrailLetter(operator, letter, true, expected)
// $COVERAGE-OFF$
override def toString: String = s"TokenSpecific(${_specific})"
// $COVERAGE-ON$
}

private [internal] final class TokenMaxOp(operator: String, _ops: Set[String], expected: UnsafeOption[String])
extends TokenSpecific(operator, true, expected) {
extends TokenSpecificAllowTrailing(operator, true, expected) {
private val ops = Radix(_ops.collect {
case op if op.length > operator.length && op.startsWith(operator) => op.substring(operator.length)
})
Expand Down

0 comments on commit 6d79f01

Please sign in to comment.