WIP server

build.sbt

@@ -18,7 +18,8 @@ javacOptions in ThisBuild ++= Seq("-source", "1.7", "-target", "1.7",
 
 lazy val root = project in file(".") aggregate(
  core,
- engines//,
+ engines,
+ tools
  //experiment
  )
 
@@ -28,3 +29,5 @@ lazy val core = (project in file("core"))
 
 lazy val engines = (project in file("engines")).
  dependsOn(core)
+
+lazy val tools = project in file("tools") dependsOn(core)

core/src/main/scala/Controller.scala

@@ -29,7 +29,7 @@ trait Algorithm[
  def predict(model: M, feature: F): P
 }
 
-trait Server[-F <: BaseFeature, P <: BasePrediction, SP <: BaseServerParams]
+trait Server[F <: BaseFeature, P <: BasePrediction, SP <: BaseServerParams]
  extends BaseServer[F, P, SP] {
  def init(serverParams: SP): Unit
 

core/src/main/scala/core/BaseController.scala

@@ -9,8 +9,8 @@ import scala.reflect.ClassTag
 import org.apache.spark.SparkContext
 import org.apache.spark.SparkContext._
 import org.apache.spark.SparkConf
- 
-abstract 
+
+abstract
 class BaseCleanser[
  TD <: BaseTrainingData,
  CD <: BaseCleansedData,
@@ -21,7 +21,7 @@ class BaseCleanser[
 }
 
 
-abstract 
+abstract
 class LocalCleanser[
  TD <: BaseTrainingData,
  CD <: BaseCleansedData : Manifest,
@@ -62,15 +62,15 @@ class SparkCleanser[
 
 abstract class BaseAlgorithm[
  CD <: BaseCleansedData,
- F <: BaseFeature,
+ F <: BaseFeature : Manifest,
  P <: BasePrediction,
  M <: BaseModel,
  AP <: BaseAlgoParams: Manifest]
  extends AbstractParameterizedDoer[AP] {
 
  def trainBase(
- sc: SparkContext, 
- cleansedData: BaseCleansedData): RDD[BaseModel] 
+ sc: SparkContext,
+ cleansedData: BaseCleansedData): RDD[BaseModel]
 
  def predictBase(baseModel: BaseModel, baseFeature: BaseFeature)
  : BasePrediction = {
@@ -80,11 +80,14 @@ abstract class BaseAlgorithm[
  }
 
  def predict(model: M, feature: F): P
+
+ def featureClass() = manifest[F]
+
 }
 
 abstract class LocalAlgorithm[
  CD <: BaseCleansedData,
- F <: BaseFeature,
+ F <: BaseFeature : Manifest,
  P <: BasePrediction,
  M <: BaseModel : Manifest,
  AP <: BaseAlgoParams: Manifest]
@@ -102,20 +105,20 @@ abstract class LocalAlgorithm[
  }
 
  def train(cleansedData: CD): M
- 
+
  def predict(model: M, feature: F): P
 }
 
 abstract class SparkAlgorithm[
  CD <: BaseCleansedData,
- F <: BaseFeature,
+ F <: BaseFeature : Manifest,
  P <: BasePrediction,
  M <: BaseModel : Manifest,
  AP <: BaseAlgoParams: Manifest]
  extends BaseAlgorithm[CD, F, P, M, AP] {
  // train returns a local object M, and we parallelize it.
  def trainBase(
- sc: SparkContext, 
+ sc: SparkContext,
  cleansedData: BaseCleansedData): RDD[BaseModel] = {
  val m: BaseModel = train(cleansedData.asInstanceOf[CD])
  sc.parallelize(Array(m))
@@ -132,7 +135,7 @@ abstract class SparkAlgorithm[
 /* Server */
 
 abstract class BaseServer[
- -F <: BaseFeature,
+ F <: BaseFeature,
  P <: BasePrediction,
  SP <: BaseServerParams: Manifest]
  extends AbstractParameterizedDoer[SP] {
@@ -179,7 +182,8 @@ class BaseEngine[
  : Map[String,
  Class[_ <:
  BaseAlgorithm[CD, F, P, _ <: BaseModel, _ <: BaseAlgoParams]]],
- val serverClass: Class[_ <: BaseServer[F, P, _ <: BaseServerParams]]) {}
+ val serverClass: Class[_ <: BaseServer[F, P, _ <: BaseServerParams]]) {
+ }
 
 class LocalEngine[
  TD <: BaseTrainingData,
@@ -194,7 +198,7 @@ class LocalEngine[
  LocalAlgorithm[CD, F, P, _ <: BaseModel, _ <: BaseAlgoParams]]],
  serverClass: Class[_ <: BaseServer[F, P, _ <: BaseServerParams]])
  extends BaseEngine(cleanserClass, algorithmClassMap, serverClass)
- 
+
 class SparkEngine[
  TD <: BaseTrainingData,
  CD <: BaseCleansedData,
@@ -208,7 +212,3 @@ class SparkEngine[
  SparkAlgorithm[CD, F, P, _ <: BaseModel, _ <: BaseAlgoParams]]],
  serverClass: Class[_ <: BaseServer[F, P, _ <: BaseServerParams]])
  extends BaseEngine(cleanserClass, algorithmClassMap, serverClass)
-
-
-
-

engines/src/main/scala/itemrank/examples/manifest.json

@@ -4,7 +4,7 @@
  "name": "Kenneth Chan Smart Item Ranking Engine",
  "jars": [
  "../engines/target/scala-2.10/engines_2.10-0.8.0-SNAPSHOT.jar",
- "target/scala-2.10/core_2.10-0.8.0-SNAPSHOT.jar",
+ "../core/target/scala-2.10/core_2.10-0.8.0-SNAPSHOT.jar",
  "../engines/target/scala-2.10/engines-assembly-0.8.0-SNAPSHOT-deps.jar"
  ],
  "engineFactory": "io.prediction.engines.itemrank.ItemRankEngine",

engines/src/main/scala/stock/examples/manifest.json

@@ -4,7 +4,7 @@
  "name": "Justin Yip Always Win Stock Engine",
  "jars": [
  "../engines/target/scala-2.10/engines_2.10-0.8.0-SNAPSHOT.jar",
- "target/scala-2.10/core_2.10-0.8.0-SNAPSHOT.jar",
+ "../core/target/scala-2.10/core_2.10-0.8.0-SNAPSHOT.jar",
  "../engines/target/scala-2.10/engines-assembly-0.8.0-SNAPSHOT-deps.jar"
  ],
  "engineFactory": "io.prediction.engines.stock.StockEngine",

project/plugins.sbt

@@ -1 +1,3 @@
 addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.7.4")
+
+addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.5.1")

tools/build.sbt

@@ -0,0 +1,25 @@
+name := "tools"
+
+libraryDependencies ++= Seq(
+ "ch.qos.logback" % "logback-classic" % "1.1.2",
+ "com.github.scopt" %% "scopt" % "3.2.0",
+ "com.twitter" %% "chill" % "0.3.6" exclude("com.esotericsoftware.minlog", "minlog"),
+ "com.twitter" %% "chill-bijection" % "0.3.6",
+ "com.typesafe" % "config" % "1.2.1",
+ "com.typesafe.akka" %% "akka-actor" % "2.3.3",
+ "com.typesafe.akka" %% "akka-cluster" % "2.3.3",
+ "com.typesafe.akka" %% "akka-contrib" % "2.3.3",
+ "com.typesafe.akka" %% "akka-testkit" % "2.3.3",
+ "io.spray" % "spray-can" % "1.3.1",
+ "io.spray" % "spray-routing" % "1.3.1",
+ "org.apache.spark" %% "spark-core" % "1.0.0" % "provided",
+ "org.clapper" %% "grizzled-slf4j" % "1.0.2",
+ "org.mongodb" %% "casbah" % "2.7.2",
+ "org.json4s" %% "json4s-native" % "3.2.10",
+ "org.json4s" %% "json4s-ext" % "3.2.10")
+
+Revolver.settings
+
+packSettings
+
+packMain := Map("runserver" -> "io.prediction.tools.RunServer")