Skip to content

Commit 3d70c39

Browse files
committed
A proposed new interface for skipping characters automatically - implemented for Scanners but not yet for RegexParsers
1 parent feaa71e commit 3d70c39

File tree

4 files changed

+66
-35
lines changed

4 files changed

+66
-35
lines changed

shared/src/main/scala/scala/util/parsing/combinator/PackratParsers.scala

+2-2
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ trait PackratParsers extends Parsers {
104104
override def phrase[T](p: Parser[T]): PackratParser[T] = {
105105
val q = super.phrase(p)
106106
new PackratParser[T] {
107-
def apply(in: Input) = in match {
107+
def parse(in: Input) = in match {
108108
case in: PackratReader[_] => q(in)
109109
case in => q(new PackratReader(in))
110110
}
@@ -231,7 +231,7 @@ to update each parser involved in the recursion.
231231
*/
232232
def memo[T](p: super.Parser[T]): PackratParser[T] = {
233233
new PackratParser[T] {
234-
def apply(in: Input) = {
234+
def parse(in: Input) = {
235235
/*
236236
* transformed reader
237237
*/

shared/src/main/scala/scala/util/parsing/combinator/Parsers.scala

+53-16
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
package scala
1414
package util.parsing.combinator
1515

16+
import scala.util.DynamicVariable
1617
import scala.util.parsing.input._
1718
import scala.collection.mutable.ListBuffer
1819
import scala.annotation.tailrec
@@ -221,7 +222,7 @@ trait Parsers {
221222
}
222223

223224
def Parser[T](f: Input => ParseResult[T]): Parser[T]
224-
= new Parser[T]{ def apply(in: Input) = f(in) }
225+
= new Parser[T]{ def parse(in: Input) = f(in) }
225226

226227
private[combinator] def Success[U](res: U, next: Input, failure: Option[Failure]): ParseResult[U] =
227228
new Success(res, next) { override val lastFailure: Option[Failure] = failure }
@@ -236,8 +237,27 @@ trait Parsers {
236237
case _ => None
237238
}
238239

240+
val skipParser: DynamicVariable[Option[Parser[Any]]] = new DynamicVariable(None);
241+
242+
final def skip(in: Input): Input = {
243+
skipParser.value match {
244+
case None => in
245+
case Some(parser) => {
246+
skipParser.withValue(None) {
247+
parser(in) match {
248+
case Success(_,next) => {
249+
next
250+
}
251+
// A parser whose purpose is to skip shouldn't fail; it should just not skip stuff
252+
case _ => in
253+
}
254+
}
255+
}
256+
}
257+
}
258+
239259
def OnceParser[T](f: Input => ParseResult[T]): Parser[T] with OnceParser[T]
240-
= new Parser[T] with OnceParser[T] { def apply(in: Input) = f(in) }
260+
= new Parser[T] with OnceParser[T] { def parse(in: Input) = f(in) }
241261

242262
/** The root class of parsers.
243263
* Parsers are functions from the Input type to ParseResult.
@@ -248,7 +268,10 @@ trait Parsers {
248268
override def toString = s"Parser ($name)"
249269

250270
/** An unspecified method that defines the behaviour of this parser. */
251-
def apply(in: Input): ParseResult[T]
271+
def parse(in: Input): ParseResult[T]
272+
def apply(in: Input): ParseResult[T] = {
273+
parse(skip(in))
274+
}
252275

253276
def flatMap[U](f: T => Parser[U]): Parser[U]
254277
= Parser{ in => this(in) flatMapWithNext(f)}
@@ -268,6 +291,20 @@ trait Parsers {
268291
Parser{ in => this(in) append p(in)}
269292
}
270293

294+
/** A parser combinator that changes skipping behavior
295+
*/
296+
def << (toSkip: => Option[Parser[Any]]): Parser[T] = {
297+
val originalParse: Input => ParseResult[T] = parse
298+
new Parser[T] {
299+
override def apply(in: Input): ParseResult[T] = {
300+
skipParser.withValue(toSkip) {
301+
parse(skip(in))
302+
}
303+
}
304+
def parse(in: Input): ParseResult[T] = originalParse(in)
305+
}.named(name)
306+
}
307+
271308
// the operator formerly known as +++, ++, &, but now, behold the venerable ~
272309
// it's short, light (looks like whitespace), has few overloaded meaning (thanks to the recent change from ~ to unary_~)
273310
// and we love it! (or do we like `,` better?)
@@ -324,7 +361,7 @@ trait Parsers {
324361

325362
/* not really useful: V cannot be inferred because Parser is covariant in first type parameter (V is always trivially Nothing)
326363
def ~~ [U, V](q: => Parser[U])(implicit combine: (T, U) => V): Parser[V] = new Parser[V] {
327-
def apply(in: Input) = seq(Parser.this, q)((x, y) => combine(x,y))(in)
364+
def parse(in: Input) = seq(Parser.this, q)((x, y) => combine(x,y))(in)
328365
} */
329366

330367
/** A parser combinator for non-back-tracking sequential composition.
@@ -391,7 +428,7 @@ trait Parsers {
391428
*/
392429
def ||| [U >: T](q0: => Parser[U]): Parser[U] = new Parser[U] {
393430
lazy val q = q0 // lazy argument
394-
def apply(in: Input) = {
431+
def parse(in: Input) = {
395432
val res1 = Parser.this(in)
396433
val res2 = q(in)
397434

@@ -427,7 +464,7 @@ trait Parsers {
427464
*/
428465
def ^^^ [U](v: => U): Parser[U] = new Parser[U] {
429466
lazy val v0 = v // lazy argument
430-
def apply(in: Input) = Parser.this(in) map (x => v0)
467+
def parse(in: Input) = Parser.this(in) map (x => v0)
431468
}.named(toString+"^^^")
432469

433470
/** A parser combinator for partial function application.
@@ -769,18 +806,18 @@ trait Parsers {
769806

770807
def continue(in: Input, failure: Option[Failure]): ParseResult[List[T]] = {
771808
val p0 = p // avoid repeatedly re-evaluating by-name parser
772-
@tailrec def applyp(in0: Input, failure: Option[Failure]): ParseResult[List[T]] = p0(in0) match {
809+
@tailrec def parsep(in0: Input, failure: Option[Failure]): ParseResult[List[T]] = p0(in0) match {
773810
case s @ Success(x, rest) =>
774811
val selectedFailure = selectLastFailure(s.lastFailure, failure)
775812
elems += x
776-
applyp(rest, selectedFailure)
813+
parsep(rest, selectedFailure)
777814
case e @ Error(_, _) => e // still have to propagate error
778815
case f: Failure =>
779816
val selectedFailure = selectLastFailure(failure, Some(f))
780817
Success(elems.toList, in0, selectedFailure)
781818
}
782819

783-
applyp(in, failure)
820+
parsep(in, failure)
784821
}
785822

786823
first(in) match {
@@ -804,14 +841,14 @@ trait Parsers {
804841
val elems = new ListBuffer[T]
805842
val p0 = p // avoid repeatedly re-evaluating by-name parser
806843

807-
@tailrec def applyp(in0: Input, failure: Option[Failure]): ParseResult[List[T]] =
844+
@tailrec def parsep(in0: Input, failure: Option[Failure]): ParseResult[List[T]] =
808845
if (elems.length == num) Success(elems.toList, in0, failure)
809846
else p0(in0) match {
810-
case s @ Success(x, rest) => elems += x ; applyp(rest, s.lastFailure)
847+
case s @ Success(x, rest) => elems += x ; parsep(rest, s.lastFailure)
811848
case ns: NoSuccess => ns
812849
}
813850

814-
applyp(in, None)
851+
parsep(in, None)
815852
}
816853

817854
/** A parser generator for a specified range of repetitions interleaved by a
@@ -835,13 +872,13 @@ trait Parsers {
835872

836873
def continue(in: Input): ParseResult[List[T]] = {
837874
val p0 = sep ~> p // avoid repeatedly re-evaluating by-name parser
838-
@tailrec def applyp(in0: Input): ParseResult[List[T]] = p0(in0) match {
839-
case Success(x, rest) => elems += x; if (elems.length == m) Success(elems.toList, rest, None) else applyp(rest)
875+
@tailrec def parsep(in0: Input): ParseResult[List[T]] = p0(in0) match {
876+
case Success(x, rest) => elems += x; if (elems.length == m) Success(elems.toList, rest, None) else parsep(rest)
840877
case e @ Error(_, _) => e // still have to propagate error
841878
case _ => Success(elems.toList, in0, None)
842879
}
843880

844-
applyp(in)
881+
parsep(in)
845882
}
846883

847884
mandatory(in) match {
@@ -973,7 +1010,7 @@ trait Parsers {
9731010
* if `p` consumed all the input.
9741011
*/
9751012
def phrase[T](p: Parser[T]) = new Parser[T] {
976-
def apply(in: Input) = p(in) match {
1013+
def parse(in: Input) = p(in) match {
9771014
case s @ Success(out, in1) =>
9781015
if (in1.atEnd) s
9791016
else s.lastFailure match {

shared/src/main/scala/scala/util/parsing/combinator/RegexParsers.scala

+5-5
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ import scala.language.implicitConversions
4848
* }
4949
* }
5050
*
51-
* def apply(input: String): Double = parseAll(expr, input) match {
51+
* def parse(input: String): Double = parseAll(expr, input) match {
5252
* case Success(result, _) => result
5353
* case failure : NoSuccess => scala.sys.error(failure.msg)
5454
* }
@@ -83,7 +83,7 @@ trait RegexParsers extends Parsers {
8383

8484
/** A parser that matches a literal string */
8585
implicit def literal(s: String): Parser[String] = new Parser[String] {
86-
def apply(in: Input) = {
86+
def parse(in: Input) = {
8787
val source = in.source
8888
val offset = in.offset
8989
val start = handleWhiteSpace(source, offset)
@@ -104,7 +104,7 @@ trait RegexParsers extends Parsers {
104104

105105
/** A parser that matches a regex string */
106106
implicit def regex(r: Regex): Parser[String] = new Parser[String] {
107-
def apply(in: Input) = {
107+
def parse(in: Input) = {
108108
val source = in.source
109109
val offset = in.offset
110110
val start = handleWhiteSpace(source, offset)
@@ -131,7 +131,7 @@ trait RegexParsers extends Parsers {
131131
override def positioned[T <: Positional](p: => Parser[T]): Parser[T] = {
132132
val pp = super.positioned(p)
133133
new Parser[T] {
134-
def apply(in: Input) = {
134+
def parse(in: Input) = {
135135
val offset = in.offset
136136
val start = handleWhiteSpace(in.source, offset)
137137
pp(in.drop (start - offset))
@@ -141,7 +141,7 @@ trait RegexParsers extends Parsers {
141141

142142
// we might want to make it public/protected in a future version
143143
private def ws[T](p: Parser[T]): Parser[T] = new Parser[T] {
144-
def apply(in: Input) = {
144+
def parse(in: Input) = {
145145
val offset = in.offset
146146
val start = handleWhiteSpace(in.source, offset)
147147
p(in.drop (start - offset))

shared/src/main/scala/scala/util/parsing/combinator/lexical/Scanners.scala

+6-12
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,6 @@ trait Scanners extends Parsers {
3232
/** A parser that produces a token (from a stream of characters). */
3333
def token: Parser[Token]
3434

35-
/** A parser for white-space -- its result will be discarded. */
36-
def whitespace: Parser[Any]
37-
3835
/** `Scanner` is essentially¹ a parser that produces `Token`s
3936
* from a stream of characters. The tokens it produces are typically
4037
* passed to parsers in `TokenParsers`.
@@ -44,21 +41,18 @@ trait Scanners extends Parsers {
4441
class Scanner(in: Reader[Char]) extends Reader[Token] {
4542
/** Convenience constructor (makes a character reader out of the given string) */
4643
def this(in: String) = this(new CharArrayReader(in.toCharArray))
47-
private val (tok, rest1, rest2) = whitespace(in) match {
48-
case Success(_, in1) =>
49-
token(in1) match {
50-
case Success(tok, in2) => (tok, in1, in2)
51-
case ns: NoSuccess => (errorToken(ns.msg), ns.next, skip(ns.next))
52-
}
53-
case ns: NoSuccess => (errorToken(ns.msg), ns.next, skip(ns.next))
44+
private val in1 = skip(in)
45+
private val (tok, rest1, rest2) = token(in1) match {
46+
case Success(tok, in2) => (tok, in1, in2)
47+
case ns: NoSuccess => (errorToken(ns.msg), ns.next, skipChar(ns.next))
5448
}
55-
private def skip(in: Reader[Char]) = if (in.atEnd) in else in.rest
49+
private def skipChar(in: Reader[Char]) = if (in.atEnd) in else in.rest
5650

5751
override def source: java.lang.CharSequence = in.source
5852
override def offset: Int = in.offset
5953
def first = tok
6054
def rest = new Scanner(rest2)
6155
def pos = rest1.pos
62-
def atEnd = in.atEnd || (whitespace(in) match { case Success(_, in1) => in1.atEnd case _ => false })
56+
def atEnd = in.atEnd || skip(in).atEnd
6357
}
6458
}

0 commit comments

Comments
 (0)