Improved datetime unit transformer shortcuts - Part 2

core/src/main/scala/com/salesforce/op/dsl/RichListFeature.scala

@@ -199,6 +199,14 @@ trait RichListFeature {
  */
  implicit class RichDateListFeature(val f: FeatureLike[DateList]) {
 
+ /**
+ * Convert to specified time period
+ * @param period type of [[TimePeriod]] to convert date feature to
+ * @return OPVector feature of time period values
+ */
+ def toTimePeriod(period: TimePeriod): FeatureLike[OPVector] =
+ new TimePeriodListTransformer[DateList](period).setInput(f).getOutput()
+
  /**
  * Apply DateList vectorizer: Converts a sequence of DateLists features into a vector feature.
  *
@@ -240,6 +248,14 @@ trait RichListFeature {
  */
  implicit class RichDateTimeListFeature(val f: FeatureLike[DateTimeList]) {
 
+ /**
+ * Convert to specified time period
+ * @param period type of [[TimePeriod]] to convert date feature to
+ * @return OPVector feature of time period values
+ */
+ def toTimePeriod(period: TimePeriod): FeatureLike[OPVector] =
+ new TimePeriodListTransformer[DateTimeList](period).setInput(f).getOutput()
+
  /**
  * Apply DateList vectorizer: Converts a sequence of DateTimeLists features into a vector feature.
  *

core/src/main/scala/com/salesforce/op/dsl/RichMapFeature.scala

@@ -702,6 +702,14 @@ trait RichMapFeature {
  */
  implicit class RichDateMapFeature(val f: FeatureLike[DateMap]) {
 
+ /**
+ * Convert to specified time period
+ * @param period type of [[TimePeriod]] to convert date feature to
+ * @return Integer map feature of time period values
+ */
+ def toTimePeriod(period: TimePeriod): FeatureLike[IntegralMap] =
+ new TimePeriodMapTransformer[DateMap](period).setInput(f).getOutput()
+
  /**
  * transforms a DateMap field into a series of cartesian coordinate representation
  * of an extracted time period on the unit circle
@@ -781,6 +789,13 @@ trait RichMapFeature {
  */
  implicit class RichDateTimeMapFeature(val f: FeatureLike[DateTimeMap]) {
 
+ /**
+ * Convert to specified time period
+ * @param period type of [[TimePeriod]] to convert date feature to
+ * @return Integer map feature of time period values
+ */
+ def toTimePeriod(period: TimePeriod): FeatureLike[IntegralMap] =
+ new TimePeriodMapTransformer[DateTimeMap](period).setInput(f).getOutput()
 
  /**
  * transforms a DateTimeMap field into a series of cartesian coordinate representation

core/src/main/scala/com/salesforce/op/stages/impl/feature/TimePeriodListTransformer.scala

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2017, Salesforce.com, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.salesforce.op.stages.impl.feature
+
+import com.salesforce.op.UID
+import com.salesforce.op.features.types._
+import com.salesforce.op.stages.base.unary.UnaryTransformer
+
+import scala.reflect.runtime.universe.TypeTag
+
+/**
+ * TimePeriodMapTransformer extracts one of a set of time periods from a date/datetime list
+ *
+ * @param period time period to extract from date
+ * @param uid uid for instance
+ * @param tti type tag for input
+ * @tparam I input feature type
+ */
+class TimePeriodListTransformer[I <: DateList]
+(
+ val period: TimePeriod,
+ uid: String = UID[TimePeriodListTransformer[_]]
+)(
+ implicit override val tti: TypeTag[I]
+) extends UnaryTransformer[I, OPVector](operationName = "dateListToTimePeriod", uid = uid) {
+
+ override def transformFn: I => OPVector =
+ (i: I) => i.value.map(t => period.extractFromTime(t).toDouble).toOPVector
+}

core/src/main/scala/com/salesforce/op/stages/impl/feature/TimePeriodMapTransformer.scala

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2017, Salesforce.com, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.salesforce.op.stages.impl.feature
+
+import com.salesforce.op.UID
+import com.salesforce.op.features.types._
+import com.salesforce.op.stages.base.unary.UnaryTransformer
+
+import scala.reflect.runtime.universe.TypeTag
+
+/**
+ * TimePeriodMapTransformer extracts one of a set of time periods from a date/datetime map
+ *
+ * @param period time period to extract from date
+ * @param uid uid for instance
+ * @param tti type tag for input
+ * @tparam I input feature type
+ */
+class TimePeriodMapTransformer[I <: DateMap]
+(
+ val period: TimePeriod,
+ uid: String = UID[TimePeriodMapTransformer[_]]
+)(
+ implicit override val tti: TypeTag[I]
+) extends UnaryTransformer[I, IntegralMap](operationName = "dateMapToTimePeriod", uid = uid) {
+
+ override def transformFn: I => IntegralMap =
+ (i: I) => i.value.mapValues(t => period.extractFromTime(t).toLong).toIntegralMap
+}

core/src/test/scala/com/salesforce/op/stages/impl/feature/TimePeriodListTransformerTest.scala

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2017, Salesforce.com, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.salesforce.op.stages.impl.feature
+
+import com.salesforce.op.features.FeatureLike
+import com.salesforce.op.features.types._
+import com.salesforce.op.test.{OpTransformerSpec, TestFeatureBuilder}
+import com.salesforce.op.utils.date.DateTimeUtils
+import com.salesforce.op.utils.spark.RichDataset._
+import org.apache.spark.ml.Transformer
+import org.joda.time.{DateTime => JDateTime}
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+
+@RunWith(classOf[JUnitRunner])
+class TimePeriodListTransformerTest extends OpTransformerSpec[OPVector, TimePeriodListTransformer[DateList]] {
+
+ val dateList: DateList = Seq[Long](
+ new JDateTime(1879, 3, 14, 0, 0, DateTimeUtils.DefaultTimeZone).getMillis,
+ new JDateTime(1955, 11, 12, 10, 4, DateTimeUtils.DefaultTimeZone).getMillis,
+ new JDateTime(1999, 3, 8, 12, 0, DateTimeUtils.DefaultTimeZone).getMillis,
+ new JDateTime(2019, 4, 30, 13, 0, DateTimeUtils.DefaultTimeZone).getMillis
+ ).toDateList
+
+ val (inputData, f1) = TestFeatureBuilder(Seq(dateList))
+
+ override val transformer: TimePeriodListTransformer[DateList] =
+ new TimePeriodListTransformer(TimePeriod.DayOfMonth).setInput(f1)
+
+ override val expectedResult: Seq[OPVector] = Seq(Seq(14, 12, 8, 30).map(_.toDouble).toVector.toOPVector)
+
+ it should "transform with rich shortcuts" in {
+ val dlist = List(new JDateTime(1879, 3, 14, 0, 0, DateTimeUtils.DefaultTimeZone).getMillis)
+ val (inputData2, d1, d2) = TestFeatureBuilder(
+ Seq[(DateList, DateTimeList)]((dlist.toDateList, dlist.toDateTimeList))
+ )
+
+ def assertFeature(feature: FeatureLike[OPVector], expected: Seq[OPVector]): Unit = {
+ val transformed = feature.originStage.asInstanceOf[Transformer].transform(inputData2)
+ val actual = transformed.collect(feature)
+ actual shouldBe expected
+ }
+
+ assertFeature(d1.toTimePeriod(TimePeriod.DayOfMonth), Seq(Vector(14.0).toOPVector))
+ assertFeature(d2.toTimePeriod(TimePeriod.DayOfMonth), Seq(Vector(14.0).toOPVector))
+ }
+}

core/src/test/scala/com/salesforce/op/stages/impl/feature/TimePeriodMapTransformerTest.scala

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2017, Salesforce.com, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.salesforce.op.stages.impl.feature
+
+import com.salesforce.op.features.FeatureLike
+import com.salesforce.op.features.types._
+import com.salesforce.op.test.{OpTransformerSpec, TestFeatureBuilder}
+import com.salesforce.op.utils.date.DateTimeUtils
+import com.salesforce.op.utils.spark.RichDataset._
+import org.apache.spark.ml.Transformer
+import org.joda.time.{DateTime => JDateTime}
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+
+@RunWith(classOf[JUnitRunner])
+class TimePeriodMapTransformerTest extends OpTransformerSpec[IntegralMap, TimePeriodMapTransformer[DateMap]] {
+
+ val names: Seq[String] = Seq("n1", "n2", "n3", "n4")
+
+ val dates: Seq[Long] = Seq(
+ new JDateTime(1879, 3, 14, 0, 0, DateTimeUtils.DefaultTimeZone).getMillis,
+ new JDateTime(1955, 11, 12, 10, 4, DateTimeUtils.DefaultTimeZone).getMillis,
+ new JDateTime(1999, 3, 8, 12, 0, DateTimeUtils.DefaultTimeZone).getMillis,
+ new JDateTime(2019, 4, 30, 13, 0, DateTimeUtils.DefaultTimeZone).getMillis
+ )
+
+ val dateMap: DateMap = names.zip(dates).toMap.toDateMap
+
+ val (inputData, f1) = TestFeatureBuilder(Seq[DateMap](dateMap))
+
+ override val transformer: TimePeriodMapTransformer[DateMap] =
+ new TimePeriodMapTransformer(TimePeriod.DayOfMonth).setInput(f1)
+
+ override val expectedResult: Seq[IntegralMap] = Seq(
+ names.zip(Seq(14L, 12L, 8L, 30L)).toMap.toIntegralMap
+ )
+
+ it should "transform with rich shortcuts" in {
+ val n = "n1"
+ val dmap = Map(n -> new JDateTime(1879, 3, 14, 0, 0, DateTimeUtils.DefaultTimeZone).getMillis)
+ val (inputData2, d1, d2) = TestFeatureBuilder(
+ Seq[(DateMap, DateTimeMap)]((dmap.toDateMap, dmap.toDateTimeMap))
+ )
+
+ def assertFeature(feature: FeatureLike[IntegralMap], expected: Seq[IntegralMap]): Unit = {
+ val transformed = feature.originStage.asInstanceOf[Transformer].transform(inputData2)
+ val actual = transformed.collect(feature)
+ actual shouldBe expected
+ }
+
+ assertFeature(d1.toTimePeriod(TimePeriod.DayOfMonth), Seq(IntegralMap(Map(n -> 14))))
+ assertFeature(d2.toTimePeriod(TimePeriod.DayOfMonth), Seq(IntegralMap(Map(n -> 14))))
+ }
+}