Fast-ANF implementation

core/src/main/scala/squid/lang/InspectableBase.scala

@@ -95,7 +95,7 @@ trait InspectableBase extends IntermediateBase with quasi.QuasiBase with TraceDe
 
   }
   protected implicit class ProtectedInspectableRepOps(private val self: Rep) {
-    def extract (that: Rep) = baseSelf.extract(self, that)
+    def extract (that: Rep) = baseSelf.extractRep(self, that)
   }
   implicit class InspectableRepOps(private val self: Rep) {
     /** Note: this is only a to-level call to `base.extractRep`; not supposed to be called in implementations of `extract` itself */

core/src/main/scala/squid/lang/IntermediateBase.scala

@@ -21,6 +21,7 @@ trait IntermediateBase extends Base { ibase: IntermediateBase =>
 
   def reinterpret(r: Rep, newBase: Base)(extrudedHandle: (BoundVal => newBase.Rep) = DefaultExtrudedHandler): newBase.Rep
 
+  // TODO rename to `defaultValue`
   def nullValue[T: IRType]: IR[T,{}]
 
 

core/src/main/scala/squid/lang/Optimizer.scala

@@ -7,7 +7,7 @@ trait Optimizer {
   //final def optimizeRep(pgrm: Rep): Rep = pipeline(pgrm)
   final def optimizeRep(pgrm: Rep): Rep = { // TODO do this Transformer's `pipeline`......?
     val r = pipeline(pgrm)
-    if (!(r eq pgrm)) substitute(r) else r  // only calls substitute if a transformation actually happened
+    if (!(r eq pgrm)) insertAfterTransformation(r) else r  // only calls insertAfterTransformation if a transformation actually happened
   }
   final def optimize[T,C](pgrm: IR[T,C]): IR[T,C] = `internal IR`[T,C](optimizeRep(pgrm.rep))
 

core/src/main/scala/squid/quasi/MetaBases.scala

@@ -170,6 +170,8 @@ trait MetaBases {
     def hole(name: String, typ: TypeRep): Rep = q"$Base.hole($name, $typ)"
     def hopHole(name: String, typ: TypeRep, yes: List[List[BoundVal]], no: List[BoundVal]): Rep =
       q"$Base.hopHole($name, $typ, ${yes map (_ map (_._2))}, ${no map (_._2)})"
+    override def hopHole2(name: String, typ: TypeRep, argss: List[List[Rep]], visible: List[BoundVal]): Rep =
+      q"$Base.hopHole2($name, $typ, ${argss}, ${visible map (_._2)})"
     def splicedHole(name: String, typ: TypeRep): Rep = q"$Base.splicedHole($name, $typ)"
 
     def substitute(r: => Rep, defs: Map[String, Rep]): Rep =

core/src/main/scala/squid/quasi/QuasiBase.scala

@@ -33,6 +33,8 @@ self: Base =>
   def substitute(r: => Rep, defs: Map[String, Rep]): Rep
   def substituteLazy(r: => Rep, defs: Map[String, () => Rep]): Rep = substitute(r, defs map (kv => kv._1 -> kv._2()))
 
+  def insertAfterTransformation(r: => Rep, defs: Map[String, Rep]): Rep = substitute(r, defs)
+
   /** Not yet used: should eventually be defined by all IRs as something different than hole */
   def freeVar(name: String, typ: TypeRep) = hole(name, typ)
 
@@ -46,6 +48,14 @@ self: Base =>
     * this should check that the extracted term does not contain any reference to a bound value contained in the `no` 
     * parameter, and it should extract a function term with the arity of the `yes` parameter. */
   def hopHole(name: String, typ: TypeRep, yes: List[List[BoundVal]], no: List[BoundVal]): Rep
+  def hopHole2(name: String, typ: TypeRep, args: List[List[Rep]], visible: List[BoundVal]): Rep = {
+    // TODO remove `hopHole` and implement this correctly everywhere
+    //val vars = args map (_ map (_.asInstanceOf[BoundVal]))\
+    // ^ Note: this will succeed even if there are some non-BoundVal, because BoundVal is eliminated by erasure
+    //   it's actually wrong (in AST, readVal(_:BoundVal) adds a Rep wrapper!
+    //hopHole(name, typ, vars, visible.toSet -- vars.flatten toList)
+    throw new UnsupportedOperationException("Higher-order patterns")
+  }
 
   /** Pattern hole in type position */
   def typeHole(name: String): TypeRep
@@ -62,6 +72,8 @@ self: Base =>
   /* if (defs isEmpty) r else */  // <- This "optimization" is not welcome, as some IRs (ANF) may relie on `substitute` being called for all insertions
     substitute(r, defs.toMap)
 
+  final def insertAfterTransformation(r: => Rep, defs: (String, Rep)*): Rep = insertAfterTransformation(r, defs.toMap)
+
   protected def mkIR[T,C](r: Rep): IR[T,C] = new IR[T,C] {
     val rep = r
     override def equals(that: Any): Boolean = that match {
@@ -291,6 +303,8 @@ self: Base =>
     res
   }
 
+  protected def mergeAll(as: Extract*): Option[Extract] = mergeAll(as.map(Some(_)))
+
   def mergeableReps(a: Rep, b: Rep): Boolean = a =~= b
 
   def mergeTypes(a: TypeRep, b: TypeRep): Option[TypeRep] =
@@ -320,11 +334,14 @@ self: Base =>
   def $Code[T](q: Code[T]): T = ??? // TODO B/E  -- also, rename to 'unquote'?
   def $Code[A,B](q: Code[A] => Code[B]): A => B = ???
 
-  /* To support hole syntax `xs?` (old syntax `$$xs`) (in ction) or `$xs` (in xtion)  */
+  /* To support hole syntax `?xs` (old syntax `$$xs`) (in ction) or `$xs` (in xtion)  */
   def $$[T](name: Symbol): T = ???
   /* To support hole syntax `$$xs: _*` (in ction) or `$xs: _*` (in xtion)  */
   def $$_*[T](name: Symbol): Seq[T] = ???
 
+  /** Higher-order pattern */
+  def $$_hop[T](name: Symbol)(args: Any*): T = ???
+
 
   implicit def liftFun[A:IRType,B,C](qf: IR[A,C] => IR[B,C]): IR[A => B,C] = {
     val bv = bindVal("lifted", typeRepOf[A], Nil) // add TODO annotation recording the lifting?

core/src/main/scala/squid/quasi/QuasiEmbedder.scala

@@ -42,13 +42,11 @@ class QuasiEmbedder[C <: whitebox.Context](val c: C) {
 
   /** holes: Seq(either term or type); splicedHoles: which holes are spliced term holes (eg: List($xs*))
     * holes in ction mode are interpreted as free variables (and are never spliced)
-    * unquotedTypes contains the types unquoted in (in ction mode with $ and in xtion mode with $$)
-    * TODO: actually use `unquotedTypes`!
-    * TODO maybe this should go in QuasiMacros... */
-  def applyQQ(Base: Tree, tree: c.Tree, holes: Seq[Either[TermName,TypeName]], 
-            splicedHoles: collection.Set[TermName], hopvHoles: collection.Map[TermName,List[List[TermName]]],
-            typeBounds: Map[TypeName,EitherOrBoth[Tree,Tree]],
-            unquotedTypes: Seq[(TypeName, Type, Tree)], unapply: Option[c.Tree], config: QuasiConfig): c.Tree = {
+    * unquotedTypes contains the types that are inserted (in ction mode with $ and in xtion mode with $$) */
+  def applyQQ(Base: Tree, tree: c.Tree, holes: Seq[Either[TermName,TypeName]],
+              splicedHoles: collection.Set[TermName], hopHoles: collection.Set[TermName],
+              typeBounds: Map[TypeName,EitherOrBoth[Tree,Tree]],
+              unquotedTypes: Seq[(TypeName, Type, Tree)], unapply: Option[c.Tree], config: QuasiConfig): c.Tree = {
 
     //debug("HOLES:",holes)
 
@@ -226,7 +224,7 @@ class QuasiEmbedder[C <: whitebox.Context](val c: C) {
 
 
     apply(Base, finalTree, termScope, config, unapply, 
-      typeSymbols, holeSymbols, holes, splicedHoles, hopvHoles, termHoles, typeHoles, typedTree, typedTreeType, stmts, convNames, unquotedTypes)
+      typeSymbols, holeSymbols, holes, splicedHoles, hopHoles, termHoles, typeHoles, typedTree, typedTreeType, stmts, convNames, unquotedTypes)
 
   }
 
@@ -242,7 +240,7 @@ class QuasiEmbedder[C <: whitebox.Context](val c: C) {
   def apply(
     baseTree: Tree, rawTree: c.Tree, termScopeParam: List[Type], config: QuasiConfig, unapply: Option[c.Tree],
     typeSymbols: Map[TypeName, Symbol], holeSymbols: Set[Symbol], 
-    holes: Seq[Either[TermName,TypeName]], splicedHoles: collection.Set[TermName], hopvHoles: collection.Map[TermName,List[List[TermName]]],
+    holes: Seq[Either[TermName,TypeName]], splicedHoles: collection.Set[TermName], hopHoles: collection.Set[TermName],
     termHoles: Set[TermName], typeHoles: Set[TypeName], typedTree: Tree, typedTreeType: Type, stmts: List[Tree], 
     convNames: Set[TermName], unquotedTypes: Seq[(TypeName, Type, Tree)]
   ): c.Tree = {
@@ -322,6 +320,23 @@ class QuasiEmbedder[C <: whitebox.Context](val c: C) {
 
           override def liftTerm(x: Tree, parent: Tree, expectedType: Option[Type], inVarargsPos: Boolean)(implicit ctx: Map[TermSymbol, b.BoundVal]): b.Rep = {
           object HoleName { def unapply(tr: Tree) = Some(holeName(tr,x)) }
+          def mkHoleType(name: String, tpt: Tree) = expectedType match {
+            case None if tpt.tpe <:< Nothing => throw EmbeddingException(s"No type info for hole '$name'" + (
+              if (debug.debugOptionEnabled) s", in: $parent" else "" )) // TODO: check only after we have explored all repetitions of the hole? (not sure if possible)
+            case Some(tp) =>
+              assert(!SCALA_REPEATED(tp.typeSymbol.fullName.toString))
+
+              if (tp <:< Nothing) parent match {
+                case q"$_: $_" => // this hole is ascribed explicitly; the Nothing type must be desired
+                case _ => macroContext.warning(macroContext.enclosingPosition,
+                  s"Type inferred for hole '$name' was Nothing. Ascribe the hole explicitly to remove this warning.") // TODO show real hole in its original form
+              }
+
+              val mergedTp = if (tp <:< tpt.tpe || tpt.tpe <:< Nothing) tp else tpt.tpe
+              debug(s"[Hole '$name'] Expected",tp," required",tpt.tpe," merged",mergedTp)
+
+              mergedTp
+          }
           x match {
 
 
@@ -465,74 +480,57 @@ class QuasiEmbedder[C <: whitebox.Context](val c: C) {
               throw QuasiException("Unknown use of free variable syntax operator `?`.")
 
 
+            /** Handling of higher-order patterns: */
+            case q"$baseTree.$$$$_hop[$tpt](${nameTree @ HoleName(name)})(..$args)" => // TODO check baseTree
+              // TODO remove _extracted_ binders (as in `val $x = ...`) from 'visible' bindings?; add them to context requirements...
+              debug("HOP",tpt,nameTree,args)
+
+              val holeType = mkHoleType(name, tpt)
+
+              // TODO handle contexts and FVs...
+              // TODO make sure HOPV holes with the same name are not repeated? or at least have the same param types
+              // Q: handle HOPV holes in spliced position?
+
+              val termName = TermName(name)
+
+              val hopvType = FunctionType(args map (_.tpe) : _*)(holeType)
+              termHoleInfo(termName) = Map() -> hopvType
+
+              val largs = args map (arg => b.byName(liftTerm(arg,x,None,false)))
+
+              b.hopHole2(name, liftType(holeType), (largs)::Nil, ctx.values.toList)
+
 
             /** Replaces calls to $$(name) with actual holes */
             case q"$baseTree.$$$$[$tpt]($nameTree)" => // TODO check baseTree
 
               val name = holeName(nameTree, x)
 
-              val holeType = expectedType match {
-                case None if tpt.tpe <:< Nothing => throw EmbeddingException(s"No type info for hole '$name'" + (
-                  if (debug.debugOptionEnabled) s", in: $parent" else "" )) // TODO: check only after we have explored all repetitions of the hole? (not sure if possible)
-                case Some(tp) =>
-                  assert(!SCALA_REPEATED(tp.typeSymbol.fullName.toString))
-
-                  if (tp <:< Nothing) parent match {
-                    case q"$_: $_" => // this hole is ascribed explicitly; the Nothing type must be desired
-                    case _ => macroContext.warning(macroContext.enclosingPosition,
-                      s"Type inferred for hole '$name' was Nothing. Ascribe the hole explicitly to remove this warning.") // TODO show real hole in its original form
-                  }
-
-                  val mergedTp = if (tp <:< tpt.tpe || tpt.tpe <:< Nothing) tp else tpt.tpe
-                  debug(s"[Hole '$name'] Expected",tp," required",tpt.tpe," merged",mergedTp)
-
-                  mergedTp
-              }
-
+              val holeType = mkHoleType(name, tpt)
 
               //if (splicedHoles(TermName(name))) throw EmbeddingException(s"Misplaced spliced hole: '$name'") // used to be: q"$Base.splicedHole[${_expectedType}](${name.toString})"
 
               val termName = TermName(name)
 
-              hopvHoles get TermName(name) match {
-                case Some(idents) =>
-
-                  // TODO make sure HOPV holes withb the same name are not repeated? or at least have the same param types
-                  // TODO handle HOPV holes in spliced position?
-
-                  val scp = ctx map { case k -> v => k.name -> (k -> v) }
-                  val identVals = idents map (_ map scp)
-                  val yes = identVals map (_ map (_._2))
-                  val keys = idents.flatten.toSet
-                  val no = scp.filterKeys(!keys(_)).values.map(_._2).toList
-                  debug(s"HOPV: yes=$yes; no=$no")
-                  val List(identVals2) = identVals // FIXME generalize
-                  val hopvType = FunctionType(identVals2 map (_._1.typeSignature) : _*)(holeType)
-                  termHoleInfo(termName) = Map() -> hopvType
-                  b.hopHole(name, liftType(holeType), yes, no)
-
-                case _ =>
-                  if (unapply isEmpty) {
-                    debug(s"Free variable: $name: $tpt")
-                    freeVariableInstances ::= name -> holeType
-                    // TODO maybe aggregate glb type beforehand so we can pass the precise type here... could even pass the _same_ hole!
-                  } else {
-                    //val termName = TermName(name)
-                    val scp = ctx.keys map (k => k.name -> k.typeSignature) toMap;
-                    termHoleInfo get termName map { case (scp0, holeType0) =>
-                      val newScp = scp0 ++ scp.map { case (n,t) => lub(t :: scp0.getOrElse(n, Any) :: Nil) }
-                      newScp -> lub(holeType :: holeType0 :: Nil)
-                    } getOrElse {
-                      termHoleInfo(termName) = scp -> holeType
-                    }
-                  }
-
-                  if (splicedHoles(TermName(name)))
-                       b.splicedHole(name, liftType(holeType))
-                  else b.hole(name, liftType(holeType))
-
+              if (unapply isEmpty) {
+                debug(s"Free variable: $name: $tpt")
+                freeVariableInstances ::= name -> holeType
+                // TODO maybe aggregate glb type beforehand so we can pass the precise type here... could even pass the _same_ hole!
+              } else {
+                //val termName = TermName(name)
+                val scp = ctx.keys map (k => k.name -> k.typeSignature) toMap;
+                termHoleInfo get termName map { case (scp0, holeType0) =>
+                  val newScp = scp0 ++ scp.map { case (n,t) => lub(t :: scp0.getOrElse(n, Any) :: Nil) }
+                  newScp -> lub(holeType :: holeType0 :: Nil)
+                } getOrElse {
+                  termHoleInfo(termName) = scp -> holeType
+                }
               }
 
+              if (splicedHoles(TermName(name)))
+                   b.splicedHole(name, liftType(holeType))
+              else b.hole(name, liftType(holeType))
+
             /** Removes implicit conversions that were generated in order to apply the subtyping knowledge extracted from pattern matching */
             case q"${Ident(name:TermName)}.apply($x)" if convNames(name) => 
               liftTerm(x, parent, typeIfNotNothing(x.tpe))
@@ -642,7 +640,7 @@ class QuasiEmbedder[C <: whitebox.Context](val c: C) {
         val typeTypesToExtract = typeSymbols mapValues { sym => tq"$baseTree.IRType[$sym]" }
 
         val extrTyps = holes.map {
-          case Left(vname) => termTypesToExtract(vname)
+          case Left(vname) => termTypesToExtract(vname) // TODO B/E
           case Right(tname) => typeTypesToExtract(tname) // TODO B/E
         }
         debug("Extracted Types: "+extrTyps.map(showCode(_)).mkString(", "))

core/src/main/scala/squid/quasi/QuasiMacros.scala

@@ -106,7 +106,7 @@ class QuasiMacros(val c: whitebox.Context) {
       holeSymbols = Set(),
       holes = Seq(),
       splicedHoles = Set(),
-      hopvHoles = Map(),
+      hopHoles = Set(),
       termHoles = Set(),
       typeHoles = Set(),
       typedTree = code,
@@ -200,7 +200,7 @@ class QuasiMacros(val c: whitebox.Context) {
 
     var holes: List[(Either[TermName,TypeName], Tree)] = Nil // (Left(value-hole) | Right(type-hole), original-hole-tree)
     val splicedHoles = mutable.Set[TermName]()
-    val hopvHoles = mutable.Map[TermName,List[List[TermName]]]()
+    val hopvHoles = mutable.Set[TermName]()
     var typeBounds: List[(TypeName, EitherOrBoth[Tree,Tree])] = Nil
 
     // Keeps track of which holes still have not been found in the source code
@@ -302,18 +302,17 @@ class QuasiMacros(val c: whitebox.Context) {
       case Ident(name: TermName) if builder.holes.contains(name) =>
         mkTermHole(name, false)
 
-      // Identify and treat Higher-Order Pattern Variables (HOPV)
-      case q"${Ident(name: TermName)}(...$argss)" if isUnapply && builder.holes.contains(name) =>
-        val idents = argss map (_ map {
-          case Ident(name:TermName) => name
-          case e => throw EmbeddingException(s"Unexpected expression in higher-order pattern variable argument: ${showCode(e)}")
-        })
-        val hole = builder.holes(name)
-        val n = hole.name filter (_.toString != "_") getOrElse (
+      // Identify and treat Higher-Order Patterns (HOP)
+      case q"${Ident(nme: TermName)}(...$argss)" if isUnapply && builder.holes.contains(nme) =>
+        val List(args) = argss // TODO handle ...$argss
+        val hole = builder.holes(nme)
+        val name = hole.name filter (_.toString != "_") getOrElse (
           throw QuasiException("All higher-order holes should be named.") // Q: necessary restriction?
         ) toTermName;
-        hopvHoles += n -> idents
-        mkTermHole(name, false)
+        hopvHoles += name
+        remainingHoles -= nme
+        holes ::= Left(name) -> hole.tree
+        q"$base.$$$$_hop(${Symbol(name toString)})(..$args)"
 
       // Interprets bounds on extracted types, like in: `case List[$t where (Null <:< t <:< AnyRef)]`:
       case tq"${Ident(name: TypeName)} where $bounds" if isUnapply && builder.holes.contains(name.toTermName) =>

src/main/scala/squid/ir/AST.scala

@@ -298,6 +298,12 @@ trait AST extends InspectableBase with ScalaTyping with ASTReinterpreter with Ru
   override def hopHole(name: String, typ: TypeRep, yes: List[List[Val]], no: List[Val]) = rep(new HOPHole(name, typ, yes, no))
   class HOPHole(name: String, typ: TypeRep, val yes: List[List[Val]], val no: List[Val]) extends HoleClass(name, typ)()
 
+  override def hopHole2(name: String, typ: TypeRep, args: List[List[Rep]], visible: List[BoundVal]): Rep = {
+    // TODO replace `hopHole`
+    val vars = args map (_ map (r => dfn(r).asInstanceOf[BoundVal]))
+    hopHole(name, typ, vars, visible.toSet -- vars.flatten toList)
+  }
+
   case class Constant(value: Any) extends Def {
     lazy val typ = value match {
       case () => TypeRep(ruh.Unit)