Upgrade to latest versions

Address #217 and #213

build.sbt

@@ -3,11 +3,14 @@ val v = IO.readLines(new File("VERSION")).head
 val sparkVersions: List[String] = IO.readLines(new File("sparkVersions")).map(_.trim)
 
 val scala11 = "2.11.12"
-val scala12 = "2.12.16"
+val scala12 = "2.12.17"
+val scala13 = "2.13.10"
 
-val Spark233 = "2.3.3"
-val Spark245 = "2.4.5"
-val Spark312 = "3.1.2"
+val Spark23 = "2.3.4"
+val Spark24 = "2.4.8"
+val Spark31 = "3.1.3"
+val Spark32 = "3.2.3"
+val Spark33 = "3.3.1"
 
 val sparkVersionSystem = System.getProperty("sparkVersion", sparkVersions.head)
 val sparkVersion = settingKey[String]("Spark version")
@@ -29,13 +32,17 @@ lazy val rootSettings =
  sparkVersion := sparkVersionSystem,
  crossScalaVersions := {
  sparkVersion.value match {
- case Spark233 => Seq(scala11)
- case Spark245 => Seq(scala11, scala12)
- case Spark312 => Seq(scala12)
+ case Spark23 => Seq(scala11)
+ case Spark24 => Seq(scala11, scala12)
+ case Spark31 => Seq(scala12)
+ case Spark32 => Seq(scala12, scala13)
+ case Spark33 => Seq(scala12, scala13)
+ case s =>
+ throw new Exception(s"crossScalaVersions: Do not know what to do with spark version $s")
  }
  },
  scalaVersion := crossScalaVersions.value.head,
- resolvers += Resolver.sonatypeRepo("releases"),
+ resolvers ++= Resolver.sonatypeOssRepos("releases"),
  soteriaAddSemantic := false,
  version ~= (v => s"${sparkVersionSystem}_$v"),
  dynver ~= (v => s"${sparkVersionSystem}_$v")
@@ -56,6 +63,15 @@ lazy val commonSettings =
  "org.apache.spark" %% "spark-sql" % sparkVersion.value % Provided,
  "org.scalatest" %% "scalatest" % "3.2.14" % Test
  ),
+ libraryDependencies ++= {
+ sparkVersion.value match {
+ case Spark32 => Seq(
+ "io.netty" % "netty-all" % "4.1.68.Final",
+ "io.netty" % "netty-buffer" % "4.1.68.Final"
+ )
+ case _ => Seq.empty
+ }
+ },
  updateOptions := updateOptions.value.withGigahorse(false),
  Test / publishArtifact := false,
  pomIncludeRepository := (_ => false)
@@ -76,8 +92,12 @@ lazy val library = (project in file("Library")).settings(
 lazy val sparkTestingBaseVersion =
  sparkVersionSystem match {
  // https://mvnrepository.com/artifact/com.holdenkarau/spark-testing-base
- case Spark312 => "3.1.2_1.1.0"
- case _ => s"${sparkVersionSystem}_0.14.0"
+ case Spark23 => "2.3.3_0.14.0"
+ case Spark24 => "2.4.8_1.3.0"
+ case Spark31 => "3.1.2_1.3.0"
+ case Spark32 => "3.2.2_1.3.0"
+ case Spark33 => "3.3.1_1.3.0"
+ case s => throw new Exception(s"sparkTestingBaseVersion: Unknown mapping for spark version $s")
  }
 
 lazy val testHelper = (project in file("testModules/TestHelper"))
@@ -142,3 +162,7 @@ lazy val example2Small = (project in file("examples/Example2_small"))
  libraryConfigsScallop,
  testHelper % Test
  )
+
+// https://github.com/sbt/sbt/issues/6997#issuecomment-1310637232
+ThisBuild / libraryDependencySchemes +=
+ "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always

configLibs/Scallop/src/test/scala/com/leobenkel/zparkio/config/scallop/CommandLineArgumentScallopTest.scala

@@ -114,7 +114,7 @@ class CommandLineArgumentScallopTest extends AnyFreeSpec {
  .run(
  arg.checkValidity().tapError { case h: HelpHandlerException => h.printHelpMessage }
  ) match {
- case Success(a) => assert(true)
+ case Success(_) => assert(true)
  case Failure(ex) => assertThrows[HelpHandlerException](throw ex.squash)
  }
  }

examples/Example1_mini/src/test/scala/com/leobenkel/example1/ApplicationTest.scala

@@ -2,6 +2,7 @@ package com.leobenkel.example1
 
 import com.leobenkel.zparkio.ZparkioApp.ZIOEnv
 import com.leobenkel.zparkiotest.TestWithSpark
+import org.scalatest.Assertions
 import org.scalatest.freespec.AnyFreeSpec
 import zio.{Runtime, Unsafe, ZIO}
 import zio.Exit.{Failure, Success}
@@ -19,7 +20,7 @@ class ApplicationTest extends AnyFreeSpec with TestWithSpark {
  case Success(value) =>
  println(s"Read: $value")
  assertResult(0)(value)
- case Failure(cause) => fail(cause.prettyPrint)
+ case Failure(cause) => Assertions.fail(cause.prettyPrint)
  }
  }
 
@@ -34,7 +35,7 @@ class ApplicationTest extends AnyFreeSpec with TestWithSpark {
  case Success(value) =>
  println(s"Read: $value")
  assertResult(1)(value)
- case Failure(cause) => fail(cause.prettyPrint)
+ case Failure(cause) => Assertions.fail(cause.prettyPrint)
  }
  }
 
@@ -44,7 +45,7 @@ class ApplicationTest extends AnyFreeSpec with TestWithSpark {
  case Success(value) =>
  println(s"Read: $value")
  assertResult(0)(value)
- case Failure(cause) => fail(cause.prettyPrint)
+ case Failure(cause) => Assertions.fail(cause.prettyPrint)
  }
  }
  }
@@ -55,3 +56,5 @@ object TestApp extends Application {
 
  lazy final override val makeRuntime: Runtime[ZIOEnv] = super.makeRuntime
 }
+
+

project/build.properties

@@ -1 +1 @@
-sbt.version = 1.7.1
+sbt.version = 1.8.0

project/plugins.sbt

@@ -1,2 +1,6 @@
 // To publish
 addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.11")
+
+// https://github.com/sbt/sbt/issues/6997#issuecomment-1310637232
+ThisBuild / libraryDependencySchemes +=
+ "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always

project/soteria.sbt

@@ -1,2 +1,2 @@
 // https://github.com/leobenkel/safety_plugin
-addSbtPlugin("com.leobenkel" % "soteria" % "0.4.9")
+addSbtPlugin("com.leobenkel" % "soteria" % "0.5.1")

soteria.json

@@ -56,24 +56,6 @@
  "version": "2.5"
  }
  },
- "io.netty": {
- "netty": {
- "version": "3.10.6.Final"
- },
- "netty-all": {
- "overrideIsEnough": false,
- "version": "4.1.48.Final"
- },
- "netty-codec": {
- "version": "4.1.48.Final"
- },
- "netty-codec-http": {
- "version": "4.1.48.Final"
- },
- "netty-handler": {
- "version": "4.1.48.Final"
- }
- },
  "jline": {
  "jline": {
  "overrideIsEnough": false,
@@ -106,7 +88,6 @@
  },
  "hadoop-hdfs": {
  "dependenciesToRemove": [
- "io.netty | netty-all",
  "org.codehaus.jackson | jackson-mapper-asl"
  ],
  "version": "2.7.5"
@@ -151,7 +132,6 @@
  "commons-beanutils | commons-beanutils-core",
  "commons-collections | commons-collections",
  "commons-io | commons-io",
- "io.netty | netty-all",
  "org.apache.xbean | xbean-asm6-shaded",
  "org.apache.commons | commons-compress",
  "org.apache.hadoop | hadoop-hdfs",
@@ -179,7 +159,7 @@
  }
  }
  },
- "sbtVersion": "1.7.1",
+ "sbtVersion": "1.8.0",
  "scalaCFlags": [
  "-feature",
  "-Yrangepos",
@@ -194,7 +174,7 @@
  ],
  "scalaVersions": [
  "2.11.12",
- "2.12.16",
- "2.13.3"
+ "2.12.17",
+ "2.13.10"
  ]
 }

sparkVersions

@@ -1,3 +1,5 @@
-2.3.3
-2.4.5
-3.1.2
+2.3.4
+2.4.8
+3.1.3
+3.2.3
+3.3.1