Move IsXLTuple to Boilerplate.scala, split tests, wip LabelledGeneric…

…Listable

Conflicts:
	build.sbt

project/Boilerplate.scala

@@ -18,7 +18,8 @@ object Boilerplate {
 
  val templates: Seq[Template] = List(
  GenXLTuples,
- GenXLTuplerInstances
+ GenXLTuplerInstances,
+ GenIsXLTupleBoilerplate
  )
 
  def gen(dir: File) =
@@ -109,5 +110,35 @@ object Boilerplate {
  """
  } 
  }
+
+ object GenIsXLTupleBoilerplate extends Template(1, 64) {
+ def filename(root: File) = root / "IsXLTupleBoilerplate.scala"
+ def content(tv: TemplateVals) = {
+ import tv._
+ block"""
+ |package typedframe
+ |
+ |import scala.reflect.macros.whitebox
+ |
+ |class IsXLTupleMacro(val c: whitebox.Context) {
+ | import c.universe._
+ | import internal.constantType
+ | 
+ | def mk[T: WeakTypeTag]: Tree = {
+ | val tTpe = weakTypeOf[T]
+ | if(!isTuple(tTpe))
+ | c.abort(c.enclosingPosition, s"Unable to materialize IsXLTuple for non-tuple type $$tTpe")
+ | 
+ | q"new IsXLTuple[$$tTpe] {}"
+ | }
+ | 
+ | def isTuple(tpe: Type): Boolean = {
+ - tpe <:< typeOf[Tuple${arity}[${`_.._`}]] ||
+ | tpe <:< typeOf[Unit]
+ | }
+ |}
+ """
+ }
+ }
 
 }

src/main/scala/LabelledGenericListable.scala

@@ -0,0 +1,23 @@
+package typedframe
+
+import shapeless._
+import shapeless.ops.hlist.ToList
+import shapeless.ops.record.Keys
+
+trait LabelledGenericListable[S <: Product] {
+ def list: Seq[String]
+}
+
+object LabelledGenericListable {
+ def apply[S <: Product](implicit t: LabelledGenericListable[S]): LabelledGenericListable[S] = t
+
+ implicit def labelledGenericListable[S <: Product, B <: HList, Y <: HList]
+ (implicit
+ b: LabelledGeneric.Aux[S, B],
+ y: Keys.Aux[B, Y],
+ o: ToList[Y, Symbol]
+ ): LabelledGenericListable[S] =
+ new LabelledGenericListable[S] {
+ val list = y().toList.map(_.name)
+ }
+}

src/main/scala/ShapelessExtra.scala

@@ -1,26 +1,27 @@
 package typedframe
 
 import shapeless._
-import shapeless.ops.hlist._
 import shapeless.ops.record.Remover
 import scala.collection.GenTraversable
-import scala.language.higherKinds
+import scala.language.experimental.macros
+import scala.language.higherKinds // TODO: Needed?
+import scala.reflect.macros.whitebox
 
 /** Type class supporting multiple record field removal. */
 @annotation.implicitNotFound(msg = "No fields ${K} in record ${L}")
 trait AllRemover[L <: HList, K <: HList] extends DepFn1[L] with Serializable { type Out <: HList }
 
 object AllRemover {
  def apply[L <: HList, K <: HList](implicit rf: AllRemover[L, K]): Aux[L, K, rf.Out] = rf
-
+ 
  type Aux[L <: HList, K <: HList, Out0 <: HList] = AllRemover[L, K] { type Out = Out0 }
-
+ 
  implicit def hnilAllRemover[L <: HList]: Aux[L, HNil, L] =
  new AllRemover[L, HNil] {
  type Out = L
  def apply(l: L): Out = l
  }
-
+ 
  implicit def hconsAllRemover[L <: HList, H, T <: HList, V, R <: HList]
  (implicit
  r: Remover.Aux[L, H, (V, R)],
@@ -32,35 +33,36 @@ object AllRemover {
  }
 }
 
-/** Typeclass witnessing that all the elements of an HList have instances of the given typeclass.
+/** Type class witnessing that all the elements of an `HList` have instances of the given typeclass.
  * (To be removed when https://github.com/milessabin/shapeless/pull/503 is published) */
 sealed trait LiftAll[F[_], In <: HList] {
  type Out <: HList
  def instances: Out
 }
 
 object LiftAll {
- type Aux[F[_], In0 <: HList, Out0 <: HList] = LiftAll[F, In0] {type Out = Out0}
- class Curried[F[_]] {def apply[In <: HList](in: In)(implicit ev: LiftAll[F, In]) = ev}
+ type Aux[F[_], In0 <: HList, Out0 <: HList] = LiftAll[F, In0] { type Out = Out0 }
+
+ class Curried[F[_]] {
+ def apply[In <: HList](in: In)(implicit ev: LiftAll[F, In]) = ev
+ }
  def apply[F[_]] = new Curried[F]
  def apply[F[_], In <: HList](implicit ev: LiftAll[F, In]) = ev
+
  implicit def hnil[F[_]]: LiftAll.Aux[F, HNil, HNil] = new LiftAll[F, HNil] {
  type Out = HNil
  def instances = HNil
  }
+
  implicit def hcons[F[_], H, T <: HList]
- (implicit headInstance: F[H], tailInstances: LiftAll[F, T]): Aux[F, H :: T, F[H] :: tailInstances.Out] =
+ (implicit
+ headInstance: F[H],
+ tailInstances: LiftAll[F, T]
+ ): Aux[F, H :: T, F[H] :: tailInstances.Out] =
  new LiftAll[F, H :: T] {
  type Out = F[H] :: tailInstances.Out
  def instances = headInstance :: tailInstances.instances
- }
-}
-
-/** Type class supporting conversion of this `HList` to a tuple, up to Tuple64. */
-trait XLTupler[L <: HList] extends DepFn1[L] with Serializable
-
-object XLTupler extends XLTuplerInstances {
- def apply[L <: HList](implicit tupler: XLTupler[L]): Aux[L, tupler.Out] = tupler
+ }
 }
 
 /** Type class supporting type safe cast.
@@ -70,7 +72,7 @@ trait NullTypeable[T] extends Serializable {
 }
 
 object NullTypeable {
- // implicit val nullTypeableany: NullTypeable[Any] = nullTypeableFromTypeable(Typeable.anyTypeable)
+ def apply[T](implicit t: NullTypeable[T]): NullTypeable[T] = t
 
  implicit def nullTypeableFromTypeable[T](implicit typeable: Typeable[T]): NullTypeable[T] =
  new NullTypeable[T] {
@@ -105,53 +107,16 @@ object FromTraversableNullable {
  }
 }
 
-import scala.language.experimental.macros
-import scala.reflect.macros.whitebox
+/** Type class supporting conversion of this `HList` to a tuple, up to Tuple64. */
+trait XLTupler[L <: HList] extends DepFn1[L] with Serializable
+
+object XLTupler extends XLTuplerInstances {
+ def apply[L <: HList](implicit tupler: XLTupler[L]): Aux[L, tupler.Out] = tupler
+}
 
+/** Type class witnessing that a type a tuple, up to Tuple64. */
 trait IsXLTuple[T]
 
 object IsXLTuple {
  implicit def apply[T]: IsXLTuple[T] = macro IsXLTupleMacro.mk[T]
 }
-
-class IsXLTupleMacro(val c: whitebox.Context) {
- import c.universe._
- import internal.constantType
- import Flag._
-
- def abort(msg: String) =
- c.abort(c.enclosingPosition, msg)
-
- def mk[T: WeakTypeTag]: Tree = {
- val tTpe = weakTypeOf[T]
- if(!isTuple(tTpe))
- abort(s"Unable to materialize IsXLTuple for non-tuple type $tTpe")
-
- q"""new IsXLTuple[$tTpe] {}"""
- }
-
- def isTuple(tpe: Type): Boolean =
- tpe <:< typeOf[Unit] ||
- tpe <:< typeOf[Tuple1[_]] ||
- tpe <:< typeOf[(_, _)] ||
- tpe <:< typeOf[(_, _, _)] ||
- tpe <:< typeOf[(_, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _)] ||
- tpe <:< typeOf[(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _)]
-}

src/main/scala/TypeableRow.scala

@@ -5,7 +5,7 @@ import shapeless.ops.hlist.{Length, Fill}
 import scala.reflect.runtime.universe.TypeTag
 import org.apache.spark.sql.Row
 
-trait TypeableRow[S <: Product] {
+trait TypeableRow[S <: Product] extends Serializable {
  def apply(row: Row): S
 }
 
@@ -25,6 +25,8 @@ trait LowPriorityTypeableRow {
 }
 
 object TypeableRow extends LowPriorityTypeableRow {
+ def apply[S <: Product](implicit t: TypeableRow[S]): TypeableRow[S] = t
+
  implicit def typeableRowTuple[S <: Product, G <: HList, N <: Nat, F <: HList, T <: Product]
  (implicit
  c: TypeTag[G],

src/main/scala/TypedFrame.scala

@@ -31,6 +31,9 @@ object TypedFrame {
  o: ToList[Y, Symbol]
  ): TypedFrame[S] =
  new TypedFrame[S](dataFrame.toDF(y().toList.map(_.name): _*))
+
+ def mk[S <: Product](dataFrame: DataFrame)(implicit l: LabelledGenericListable[S]): TypedFrame[S] =
+ new TypedFrame[S](dataFrame.toDF(l.list: _*))
 }
 
 final class TypedFrame[Schema <: Product] private[typedframe] (val df: DataFrame) extends Serializable {

src/main/scala/package.scala

@@ -0,0 +1,5 @@
+import shapeless._
+
+package object typedframe {
+ implicit val TODO: Typeable[Any] = Typeable.anyTypeable
+}

src/test/resources/log4j.properties

@@ -1,3 +1,9 @@
-log4j.logger.org.apache.spark=ERROR
 log4j.logger.Remoting=ERROR
+log4j.logger.akka.event.slf4j=ERROR
+log4j.logger.org.spark-project.jetty.server=ERROR
+log4j.logger.org.apache.spark=ERROR
+log4j.logger.com.anjuke.dm=ERROR
 log4j.logger.org.eclipse.jetty=ERROR
+log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR
+log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=ERROR
+log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=ERROR

src/test/scala/SpecWithContext.scala

@@ -3,8 +3,7 @@ package org.apache.spark.sql
 import org.apache.spark.{SparkConf, SparkContext}
 import org.scalatest.{FunSuite, BeforeAndAfterAll}
 import scala.reflect.runtime.universe.TypeTag
-import typedframe.{TypedFrame, FromTraversableNullable}
-import typedframe.TypeableRow
+import typedframe.{TypedFrame, FromTraversableNullable, TypeableRow}
 
 trait SpecWithContext extends FunSuite with BeforeAndAfterAll {
  implicit var implicitContext: SQLContext = null