From eb061754fcfbf6ce82585aa58201064c4a2fd775 Mon Sep 17 00:00:00 2001 From: Nick Childers Date: Fri, 1 Sep 2017 13:52:59 -0700 Subject: [PATCH 1/3] hopefully less metaspace errors --- build.sbt | 2 +- .../src/test/scala/frameless/JobTests.scala | 4 ++-- .../scala/frameless/TypedDatasetSuite.scala | 24 +++++++++++++++---- .../frameless/ops/SmartProjectTest.scala | 2 +- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/build.sbt b/build.sbt index d701adb3..bf88cd7c 100644 --- a/build.sbt +++ b/build.sbt @@ -57,7 +57,7 @@ lazy val framelessSettings = Seq( "com.chuusai" %% "shapeless" % shapeless, "org.scalatest" %% "scalatest" % scalatest % "test", "org.scalacheck" %% "scalacheck" % scalacheck % "test"), - fork in Test := false, + fork in Test := true, parallelExecution in Test := false ) diff --git a/dataset/src/test/scala/frameless/JobTests.scala b/dataset/src/test/scala/frameless/JobTests.scala index 172322c0..d3a8ae92 100644 --- a/dataset/src/test/scala/frameless/JobTests.scala +++ b/dataset/src/test/scala/frameless/JobTests.scala @@ -1,11 +1,11 @@ package frameless import org.scalacheck.Arbitrary -import org.scalatest.{FreeSpec, Matchers} +import org.scalatest.{BeforeAndAfterAll, FreeSpec, Matchers} import org.scalatest.prop.GeneratorDrivenPropertyChecks -class JobTests extends FreeSpec with SparkTesting with GeneratorDrivenPropertyChecks with Matchers { +class JobTests extends FreeSpec with BeforeAndAfterAll with SparkTesting with GeneratorDrivenPropertyChecks with Matchers { "map" - { "identity" in { diff --git a/dataset/src/test/scala/frameless/TypedDatasetSuite.scala b/dataset/src/test/scala/frameless/TypedDatasetSuite.scala index 9ebf29d2..a2f5a618 100644 --- a/dataset/src/test/scala/frameless/TypedDatasetSuite.scala +++ b/dataset/src/test/scala/frameless/TypedDatasetSuite.scala @@ -3,10 +3,11 @@ package frameless import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.{SQLContext, SparkSession} import org.scalactic.anyvals.PosZInt -import org.scalatest.FunSuite +import org.scalatest.{BeforeAndAfterAll, FunSuite} import org.scalatest.prop.Checkers -trait SparkTesting { +trait SparkTesting { self: BeforeAndAfterAll => + val appID: String = new java.util.Date().toString + math.floor(math.random * 10E4).toLong.toString val conf: SparkConf = new SparkConf() @@ -15,12 +16,27 @@ trait SparkTesting { .set("spark.ui.enabled", "false") .set("spark.app.id", appID) - implicit def session: SparkSession = SparkSession.builder().config(conf).getOrCreate() + private var s: SparkSession = _ + + implicit def session: SparkSession = s implicit def sc: SparkContext = session.sparkContext implicit def sqlContext: SQLContext = session.sqlContext + + override def beforeAll(): Unit = { + assert(s == null) + s = SparkSession.builder().config(conf).getOrCreate() + } + + override def afterAll(): Unit = { + if (s != null) { + s.stop() + s = null + } + } } -class TypedDatasetSuite extends FunSuite with Checkers with SparkTesting { + +class TypedDatasetSuite extends FunSuite with Checkers with BeforeAndAfterAll with SparkTesting { // Limit size of generated collections and number of checks because Travis implicit override val generatorDrivenConfig = PropertyCheckConfiguration(sizeRange = PosZInt(10), minSize = PosZInt(10)) diff --git a/dataset/src/test/scala/frameless/ops/SmartProjectTest.scala b/dataset/src/test/scala/frameless/ops/SmartProjectTest.scala index a1a182d6..ec58911e 100644 --- a/dataset/src/test/scala/frameless/ops/SmartProjectTest.scala +++ b/dataset/src/test/scala/frameless/ops/SmartProjectTest.scala @@ -12,7 +12,7 @@ case class InvalidFooProjectionType(i: Int, x: Boolean) case class InvalidFooProjectionName(i: Int, xerr: String) class SmartProjectTest extends TypedDatasetSuite { - val dataset = TypedDataset.create(Foo(1, 2, "hi") :: Foo(2, 3, "there") :: Nil) + lazy val dataset = TypedDataset.create(Foo(1, 2, "hi") :: Foo(2, 3, "there") :: Nil) test("project Foo to Bar") { assert(dataset.project[Bar].count().run() === 2) From 773a5ecb7d741a4fd8f239d050ecf0dd56eff474 Mon Sep 17 00:00:00 2001 From: Nick Childers Date: Fri, 1 Sep 2017 14:33:21 -0700 Subject: [PATCH 2/3] Maybe this folklore from the internet works --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index bf88cd7c..9abc5d38 100644 --- a/build.sbt +++ b/build.sbt @@ -57,6 +57,7 @@ lazy val framelessSettings = Seq( "com.chuusai" %% "shapeless" % shapeless, "org.scalatest" %% "scalatest" % scalatest % "test", "org.scalacheck" %% "scalacheck" % scalacheck % "test"), + javaOptions in Test ++= Seq("-Xmx1G"), fork in Test := true, parallelExecution in Test := false ) From 47f401ce5ca73584ca5b346a200354ca0485179d Mon Sep 17 00:00:00 2001 From: Nick Childers Date: Tue, 5 Sep 2017 09:44:56 -0700 Subject: [PATCH 3/3] added comment regarding the use of lazy in SmartProjectTest --- dataset/src/test/scala/frameless/ops/SmartProjectTest.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/dataset/src/test/scala/frameless/ops/SmartProjectTest.scala b/dataset/src/test/scala/frameless/ops/SmartProjectTest.scala index ec58911e..b1836a3f 100644 --- a/dataset/src/test/scala/frameless/ops/SmartProjectTest.scala +++ b/dataset/src/test/scala/frameless/ops/SmartProjectTest.scala @@ -12,6 +12,7 @@ case class InvalidFooProjectionType(i: Int, x: Boolean) case class InvalidFooProjectionName(i: Int, xerr: String) class SmartProjectTest extends TypedDatasetSuite { + // Lazy needed to prevent initialization anterior to the `beforeAll` hook lazy val dataset = TypedDataset.create(Foo(1, 2, "hi") :: Foo(2, 3, "there") :: Nil) test("project Foo to Bar") {