Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow convertion from Date and Timestamp Spark types to Date and DateTime TransmogrifAI types #188

Merged
merged 4 commits into from
Dec 1, 2018
Merged

Allow convertion from Date and Timestamp Spark types to Date and DateTime TransmogrifAI types #188

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
80acbdf
Allow convertion from Date and Timestamp Spark types to Date and Date…
tovbinm Dec 1, 2018
8c49475
ident
tovbinm Dec 1, 2018
a35218d
blank
tovbinm Dec 1, 2018
54f6e96
fix test
tovbinm Dec 1, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Allow convertion from Date and Timestamp Spark types to Date and Date… 
…Time TransmogrifAI types
  • Loading branch information
@tovbinm
tovbinm committed Dec 1, 2018
commit 80acbdf4ebb1f406587b5528ba5513318190902e
8 changes: 7 additions & 1 deletion features/src/main/scala/com/salesforce/op/features/types/FeatureTypeSparkConverter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,12 +147,18 @@ case object FeatureTypeSparkConverter {

// Date & Time
case wt if wt <:< weakTypeOf[t.DateTime] => (value: Any) =>
if (value == null) FeatureTypeDefaults.DateTime.value else Some(value.asInstanceOf[Long])
value match {
case null => FeatureTypeDefaults.DateTime.value
case v: Long => Some(v)
case v: java.util.Date => Some(v.getTime)
case v => throw new IllegalArgumentException(s"DateTime type mapping is not defined for ${v.getClass}")
}
case wt if wt <:< weakTypeOf[t.Date] => (value: Any) =>
value match {
case null => FeatureTypeDefaults.Date.value
case v: Int => Some(v.toLong)
case v: Long => Some(v)
case v: java.util.Date => Some(v.getTime)
case v => throw new IllegalArgumentException(s"Date type mapping is not defined for ${v.getClass}")
}

Expand Down
16 changes: 15 additions & 1 deletion features/src/test/scala/com/salesforce/op/features/FeatureBuilderTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,14 +84,28 @@ class FeatureBuilderTest extends FlatSpec with TestSparkContext {
assertFeature[Row, Real](feature)(name = name, in = Row(1.0, "2"), out = 1.toReal, isResponse = true)
}

it should "build features from a dataframe" in {
it should "build text & numeric features from a dataframe" in {
val row = data.head()
val (label, Array(fs, fl)) = FeatureBuilder.fromDataFrame[RealNN](data, response = "d")
assertFeature(label)(name = "d", in = row, out = 2.0.toRealNN, isResponse = true)
assertFeature(fs.asInstanceOf[Feature[Text]])(name = "s", in = row, out = "blah1".toText, isResponse = false)
assertFeature(fl.asInstanceOf[Feature[Integral]])(name = "l", in = row, out = 10.toIntegral, isResponse = false)
}

it should "build time & date features from a dataframe " in {
val ts = java.sql.Timestamp.valueOf("2018-12-02 11:05:33.523")
val dt = java.sql.Date.valueOf("2018-12-1")
val df = spark.createDataFrame(Seq((1.0, ts, dt)))
val Array(lblName, tsName, dtName) = df.schema.fieldNames
val row = df.head()
val (label, Array(time, date)) = FeatureBuilder.fromDataFrame[RealNN](df, response = lblName)
assertFeature(label)(name = lblName, in = row, out = 1.0.toRealNN, isResponse = true)
assertFeature(time.asInstanceOf[Feature[DateTime]])(
name = tsName, in = row, out = ts.getTime.toDateTime, isResponse = false)
assertFeature(date.asInstanceOf[Feature[Date]])(
name = dtName, in = row, out = dt.getTime.toDate, isResponse = false)
}

it should "error on invalid response" in {
intercept[RuntimeException](FeatureBuilder.fromDataFrame[RealNN](data, response = "non_existent"))
.getMessage shouldBe "Response feature 'non_existent' was not found in dataframe schema"
Expand Down
4 changes: 2 additions & 2 deletions features/src/test/scala/com/salesforce/op/features/FeatureSparkTypeTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
package com.salesforce.op.features

import com.salesforce.op.features.types.FeatureType
import com.salesforce.op.test.TestSparkContext
import com.salesforce.op.test.{TestCommon, TestSparkContext}
import org.apache.spark.ml.linalg.SQLDataTypes.VectorType
import org.apache.spark.sql.types._
import org.junit.runner.RunWith
Expand All @@ -41,7 +41,7 @@ import org.scalatest.junit.JUnitRunner
import scala.reflect.runtime.universe._

@RunWith(classOf[JUnitRunner])
class FeatureSparkTypeTest extends FlatSpec with TestSparkContext {
class FeatureSparkTypeTest extends FlatSpec with TestCommon {
val primitiveTypes = Seq(
(DoubleType, weakTypeTag[types.Real], DoubleType),
(FloatType, weakTypeTag[types.Real], DoubleType),
Expand Down