Skip to content

Commit

Permalink
feat(token)!: removed implicits for predicate
Browse files Browse the repository at this point in the history
  • Loading branch information
j-mie6 committed Dec 27, 2024
1 parent ddfef94 commit eed5882
Show file tree
Hide file tree
Showing 8 changed files with 58 additions and 84 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ object SpaceDesc {
lineCommentStart = "",
lineCommentAllowsEOF = true,
multiLineNestedComments = false,
space = Unicode(Character.isWhitespace),
space = Unicode(Character.isWhitespace(_)),
whitespaceIsContextDependent = false
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import parsley.character.satisfy
import parsley.exceptions.ParsleyException
import parsley.unicode.{satisfy => satisfyUtf16}

import scala.collection.immutable.NumericRange

/** Base class for character predicates.
* @since 4.0.0
*/
Expand All @@ -35,13 +37,43 @@ sealed abstract class CharPredicate {
* @since 4.0.0
*/
final case class Unicode(predicate: Int => Boolean) extends CharPredicate {
def this(c: Int) = this(_ == c)
def this(cs: NumericRange[Int]) = this(cs.contains)
def this(cs: Range) = this(cs.contains)
private [token] override def toBmp = satisfy(c => predicate(c.toInt))
private [token] override def toUnicode = satisfyUtf16(predicate)
private [token] override def toNative = toUnicode.void
private [token] def startsWith(s: String) = s.nonEmpty && predicate(s.codePointAt(0))
private [token] def endsWith(s: String) = s.nonEmpty && predicate(s.codePointBefore(s.length))
private [parsley] def asInternalPredicate = new parsley.internal.machine.instructions.token.Unicode(predicate)
}
object Unicode {
/** Lifts a regular full-width character predicate.
* @since 5.0.0
*/
def apply(c: Int): Unicode = new Unicode(_ == c)
/** Constructs a predicate for anything in a range of specific unicode codepoints.
* @since 5.0.0
*/
def apply(cs: NumericRange[Int]): Unicode = new Unicode(cs.contains)
/** Constructs a predicate for anything in a range of specific unicode codepoints.
* @since 5.0.0
*/
def apply(cs: Range): Unicode = new Unicode(cs.contains)

/** Lifts a regular character predicate.
* @since 5.0.0
*/
def char(pred: Char => Boolean): Unicode = new Unicode(c => c.isValidChar && pred(c.toChar))
/** Constructs a predicate for the specific given character.
* @since 5.0.0
*/
def char(c: Char): Unicode = new Unicode(c.toInt)
/** Constructs a predicate for anything in a range of specific characters.
* @since 5.0.0
*/
def char(cs: NumericRange[Char]): Unicode = char(cs.contains)
}

/** Basic character predicate, which reads regular Scala 16-bit characters.
*
Expand All @@ -51,6 +83,8 @@ final case class Unicode(predicate: Int => Boolean) extends CharPredicate {
* @since 4.0.0
*/
final case class Basic(predicate: Char => Boolean) extends CharPredicate {
def this(c: Char) = this(_ == c)
def this(cs: NumericRange[Char]) = this(cs.contains)
private [token] override def toBmp = satisfy(predicate)
// $COVERAGE-OFF$
private [token] override def toUnicode =
Expand All @@ -61,6 +95,16 @@ final case class Basic(predicate: Char => Boolean) extends CharPredicate {
private [token] def endsWith(s: String) = s.lastOption.exists(predicate)
private [parsley] def asInternalPredicate = new parsley.internal.machine.instructions.token.Basic(predicate)
}
object Basic {
/** Constructs a predicate for the specific given character.
* @since 5.0.0
*/
def apply(c: Char): Basic = new Basic(c)
/** Constructs a predicate for anything in a range of specific characters.
* @since 5.0.0
*/
def apply(cs: NumericRange[Char]): Basic = new Basic(cs)
}

/** Character predicate that never succeeds.
*
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class SpaceTests extends ParsleyTest {
private def makeSpace(space: SpaceDesc) = makeLexer(space).space

val basicNoComments = SpaceDesc.plain.copy(space = predicate.Basic(Character.isWhitespace))
val unicodeNoComments = basicNoComments.copy(space = predicate.Unicode(Character.isWhitespace))
val unicodeNoComments = basicNoComments.copy(space = predicate.Unicode(Character.isWhitespace(_)))

"whiteSpace" should "parse spaces when no comments are defined" in cases(makeSpace(basicNoComments).whiteSpace *> string("a")) (
"a" -> Some("a"),
Expand Down
10 changes: 5 additions & 5 deletions parsley/shared/src/test/scala/parsley/token/TokeniserTests.scala
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,15 @@ import parsley.Parsley.eof
import parsley.character.string

import token.{descriptions => desc}
import token.predicate.implicits.Basic._
import token.predicate.Basic

class TokeniserTests extends ParsleyTest {
val scala =
desc.LexicalDesc(
desc.NameDesc(identifierStart = ('a' to 'z').toSet ++ ('A' to 'Z').toSet + '_',
identifierLetter = ('a' to 'z').toSet ++ ('A' to 'Z').toSet ++ ('0' to '9').toSet + '_',
operatorStart = Set('+', '-', ':', '/', '*', '='),
operatorLetter = Set('+', '-', '/', '*')),
desc.NameDesc(identifierStart = Basic(('a' to 'z').toSet ++ ('A' to 'Z').toSet + '_'),
identifierLetter = Basic(('a' to 'z').toSet ++ ('A' to 'Z').toSet ++ ('0' to '9').toSet + '_'),
operatorStart = Basic(Set('+', '-', ':', '/', '*', '=')),
operatorLetter = Basic(Set('+', '-', '/', '*'))),
desc.SymbolDesc(hardKeywords = Set("if", "else", "for", "yield", "while", "def", "class",
"trait", "abstract", "override", "val", "var", "lazy"),
hardOperators = Set(":", "=", "::", ":="),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
package parsley.token.descriptions

import parsley.token.predicate._
import parsley.token.predicate.implicits.Basic.charToBasic

import org.scalacheck.Gen
import org.scalacheck.Arbitrary
Expand All @@ -19,7 +18,7 @@ object DescGen {
NotRequired,
Unicode(Character.isLetter(_)),
Unicode(Character.isLetterOrDigit(_)),
'$'
Basic('$'),
)

private val opCharGen = Gen.nonEmptyContainerOf[Set, Char](Gen.oneOf('+', '*', '/', 'a'))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,13 @@ class NamesTests extends ParsleyTest {
"hi" -> Some("hi"),
"x7" -> Some("x7"),
)
identCases(Unicode(Character.isAlphabetic), Unicode(Character.isLetterOrDigit))(
identCases(Unicode(Character.isAlphabetic), Unicode(Character.isLetterOrDigit(_)))(
"hello1" -> Some("hello1"),
"7f" -> None,
"hi" -> Some("hi"),
"x7" -> Some("x7"),
)
identCases(Basic(_.isLetter), Unicode(Character.isDigit))(
identCases(Basic(_.isLetter), Unicode(Character.isDigit(_)))(
"hello1" -> None,
"7f" -> None,
"7" -> None,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import parsley.token.LexemeImpl._

import parsley.token.descriptions._
import parsley.token.errors.ErrorConfig
import parsley.token.predicate._, implicits.Basic.charToBasic, implicits.Unicode.funToUnicode
import parsley.token.predicate._
import parsley.character.{spaces, string}
import org.scalactic.source.Position

Expand All @@ -24,13 +24,13 @@ class SymbolTests extends ParsleyTest {
}
def makeSymbol(nameDesc: NameDesc, symDesc: SymbolDesc): Symbol = new LexemeSymbol(new ConcreteSymbol(nameDesc, symDesc, errConfig), spaces)

val plainName = NameDesc.plain.copy(identifierLetter = Basic(_.isLetter), operatorLetter = ':')
val plainName = NameDesc.plain.copy(identifierLetter = Basic(_.isLetter), operatorLetter = Basic(':'))
val plainSym = SymbolDesc.plain.copy(hardKeywords = Set("keyword", "hard"), hardOperators = Set("+", "<", "<="))

val plainSymbol = makeSymbol(plainName, plainSym)
val unicodeSymbol = makeSymbol(plainName.copy(identifierLetter = Character.isAlphabetic(_)), plainSym)
val unicodeSymbol = makeSymbol(plainName.copy(identifierLetter = Unicode(Character.isAlphabetic(_))), plainSym)
val caseInsensitive = makeSymbol(plainName, plainSym.copy(caseSensitive = false))
val caseInsensitiveUni = makeSymbol(plainName.copy(identifierLetter = Character.isAlphabetic(_)), plainSym.copy(caseSensitive = false))
val caseInsensitiveUni = makeSymbol(plainName.copy(identifierLetter = Unicode(Character.isAlphabetic(_))), plainSym.copy(caseSensitive = false))

def boolCases(p: Parsley[Unit])(tests: (String, Boolean, Position)*): Unit = cases(p, noEof = true)(tests.map { case (i, r, pos) => (i, if (r) Some(()) else None, pos) }: _*)
def namedCases(sym: String => Parsley[Unit])(ktests: (String, Seq[(String, Boolean, Position)])*): Unit = {
Expand Down

0 comments on commit eed5882

Please sign in to comment.