Merge pull request typelevel#170 from Voltir/improve-sparktest-2

Improve SparkTest
KamalKang · Sep 7, 2017 · b5453de · b5453de
2 parents 919cdd5 + 47f401c
commit b5453de
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 8 deletions.
diff --git a/build.sbt b/build.sbt
@@ -57,7 +57,8 @@ lazy val framelessSettings = Seq(
  "com.chuusai" %% "shapeless" % shapeless,
  "org.scalatest" %% "scalatest" % scalatest % "test",
  "org.scalacheck" %% "scalacheck" % scalacheck % "test"),
- fork in Test := false,
+ javaOptions in Test ++= Seq("-Xmx1G"),
+ fork in Test := true,
  parallelExecution in Test := false
 )
 

diff --git a/dataset/src/test/scala/frameless/JobTests.scala b/dataset/src/test/scala/frameless/JobTests.scala
@@ -1,11 +1,11 @@
 package frameless
 
 import org.scalacheck.Arbitrary
-import org.scalatest.{FreeSpec, Matchers}
+import org.scalatest.{BeforeAndAfterAll, FreeSpec, Matchers}
 import org.scalatest.prop.GeneratorDrivenPropertyChecks
 
 
-class JobTests extends FreeSpec with SparkTesting with GeneratorDrivenPropertyChecks with Matchers {
+class JobTests extends FreeSpec with BeforeAndAfterAll with SparkTesting with GeneratorDrivenPropertyChecks with Matchers {
 
  "map" - {
  "identity" in {

diff --git a/dataset/src/test/scala/frameless/TypedDatasetSuite.scala b/dataset/src/test/scala/frameless/TypedDatasetSuite.scala
@@ -3,10 +3,11 @@ package frameless
 import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.sql.{SQLContext, SparkSession}
 import org.scalactic.anyvals.PosZInt
-import org.scalatest.FunSuite
+import org.scalatest.{BeforeAndAfterAll, FunSuite}
 import org.scalatest.prop.Checkers
 
-trait SparkTesting {
+trait SparkTesting { self: BeforeAndAfterAll =>
+
  val appID: String = new java.util.Date().toString + math.floor(math.random * 10E4).toLong.toString
 
  val conf: SparkConf = new SparkConf()
@@ -15,12 +16,27 @@ trait SparkTesting {
  .set("spark.ui.enabled", "false")
  .set("spark.app.id", appID)
 
- implicit def session: SparkSession = SparkSession.builder().config(conf).getOrCreate()
+ private var s: SparkSession = _
+
+ implicit def session: SparkSession = s
  implicit def sc: SparkContext = session.sparkContext
  implicit def sqlContext: SQLContext = session.sqlContext
+
+ override def beforeAll(): Unit = {
+ assert(s == null)
+ s = SparkSession.builder().config(conf).getOrCreate()
+ }
+
+ override def afterAll(): Unit = {
+ if (s != null) {
+ s.stop()
+ s = null
+ }
+ }
 }
 
-class TypedDatasetSuite extends FunSuite with Checkers with SparkTesting {
+
+class TypedDatasetSuite extends FunSuite with Checkers with BeforeAndAfterAll with SparkTesting {
  // Limit size of generated collections and number of checks because Travis
  implicit override val generatorDrivenConfig =
  PropertyCheckConfiguration(sizeRange = PosZInt(10), minSize = PosZInt(10))

diff --git a/dataset/src/test/scala/frameless/ops/SmartProjectTest.scala b/dataset/src/test/scala/frameless/ops/SmartProjectTest.scala
@@ -12,7 +12,8 @@ case class InvalidFooProjectionType(i: Int, x: Boolean)
 case class InvalidFooProjectionName(i: Int, xerr: String)
 
 class SmartProjectTest extends TypedDatasetSuite {
- val dataset = TypedDataset.create(Foo(1, 2, "hi") :: Foo(2, 3, "there") :: Nil)
+ // Lazy needed to prevent initialization anterior to the `beforeAll` hook
+ lazy val dataset = TypedDataset.create(Foo(1, 2, "hi") :: Foo(2, 3, "there") :: Nil)
 
  test("project Foo to Bar") {
  assert(dataset.project[Bar].count().run() === 2)