WIP

core/src/main/scala/com/salesforce/op/stages/impl/insights/RecordInsightsLOCO.scala

@@ -30,25 +30,23 @@
 
 package com.salesforce.op.stages.impl.insights
 
-import com.salesforce.op.{FeatureInsights, UID}
+import com.salesforce.op.UID
 import com.salesforce.op.features.types._
 import com.salesforce.op.stages.base.unary.UnaryTransformer
-import com.salesforce.op.stages.impl.feature.{DateToUnitCircle, TimePeriod}
+import com.salesforce.op.stages.impl.feature.TimePeriod
 import com.salesforce.op.stages.impl.selector.SelectedModel
 import com.salesforce.op.stages.sparkwrappers.specific.OpPredictorWrapperModel
 import com.salesforce.op.stages.sparkwrappers.specific.SparkModelConverter._
 import com.salesforce.op.utils.spark.{OpVectorColumnHistory, OpVectorMetadata}
 import enumeratum.{Enum, EnumEntry}
 import org.apache.spark.annotation.Experimental
 import org.apache.spark.ml.Model
-import org.apache.spark.ml.linalg.Vectors
+import org.apache.spark.ml.linalg.{DenseVector, Vectors}
 import org.apache.spark.ml.param.{IntParam, Param, Params}
 
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
 
-import scala.reflect.runtime.universe._
-
 
 trait RecordInsightsLOCOParams extends Params {
 
@@ -132,17 +130,15 @@ class RecordInsightsLOCO[T <: Model[T]]
 
  private def computeDiffs
  (
- i: Int,
  oldInd: Int,
  oldVal: Double,
- featureArray: Array[(Int, Double)],
- featureSize: Int,
+ featureVec: DenseVector,
  baseScore: Array[Double]
  ): Array[Double] = {
- featureArray.update(i, (oldInd, 0.0))
- val score = modelApply(labelDummy, Vectors.sparse(featureSize, featureArray).toOPVector).score
+ featureVec.values.update(oldInd, 0.0)
+ val score = modelApply(labelDummy, featureVec.toOPVector).score
  val diffs = baseScore.zip(score).map { case (b, s) => b - s }
- featureArray.update(i, (oldInd, oldVal))
+ featureVec.values.update(oldInd, oldVal)
  diffs
  }
 
@@ -165,38 +161,38 @@ class RecordInsightsLOCO[T <: Model[T]]
 
  private def returnTopPosNeg
  (
- featureArray: Array[(Int, Double)],
- featureSize: Int,
+ featureVec: DenseVector,
  baseScore: Array[Double],
  k: Int,
  indexToExamine: Int
  ): Seq[LOCOValue] = {
 
  val minMaxHeap = new MinMaxHeap(k)
  val aggregationMap = mutable.Map.empty[String, (Array[Int], Array[Double])]
- for {i <- featureArray.indices} {
- val (oldInd, oldVal) = featureArray(i)
- val diffToExamine = computeDiffs(i, oldInd, oldVal, featureArray, featureSize, baseScore)
- val history = histories(oldInd)
-
- history match {
- // If indicator value and descriptor value of a derived text feature are empty, then it is likely
- // to be a hashing tf output. We aggregate such features for each (rawFeatureName).
- case h if h.indicatorValue.isEmpty && h.descriptorValue.isEmpty && textFeatureIndices.contains(oldInd) =>
- for {name <- getRawFeatureName(h)} {
- val (indices, array) = aggregationMap.getOrElse(name, (Array.empty[Int], Array.empty[Double]))
- aggregationMap.update(name, (indices :+ i, sumArrays(array, diffToExamine)))
- }
- // If the descriptor value of a derived date feature exists, then it is likely to be
- // from unit circle transformer. We aggregate such features for each (rawFeatureName, timePeriod).
- case h if h.descriptorValue.isDefined && dateFeatureIndices.contains(oldInd) =>
- for {name <- getRawFeatureName(h)} {
- val key = name + h.descriptorValue.flatMap(convertToTimePeriod).map(p => "_" + p.entryName).getOrElse("")
- val (indices, array) = aggregationMap.getOrElse(key, (Array.empty[Int], Array.empty[Double]))
- aggregationMap.update(key, (indices :+ i, sumArrays(array, diffToExamine)))
- }
- case _ => minMaxHeap enqueue LOCOValue(i, diffToExamine(indexToExamine), diffToExamine)
- }
+ featureVec.foreachActive {
+ case (oldInd, oldVal) if oldVal != 0.0 =>
+ val diffToExamine = computeDiffs(oldInd, oldVal, featureVec, baseScore)
+ val history = histories(oldInd)
+
+ history match {
+ // If indicator value and descriptor value of a derived text feature are empty, then it is likely
+ // to be a hashing tf output. We aggregate such features for each (rawFeatureName).
+ case h if h.indicatorValue.isEmpty && h.descriptorValue.isEmpty && textFeatureIndices.contains(oldInd) =>
+ for {name <- getRawFeatureName(h)} {
+ val (indices, array) = aggregationMap.getOrElse(name, (Array.empty[Int], Array.empty[Double]))
+ aggregationMap.update(name, (indices :+ oldInd, sumArrays(array, diffToExamine)))
+ }
+ // If the descriptor value of a derived date feature exists, then it is likely to be
+ // from unit circle transformer. We aggregate such features for each (rawFeatureName, timePeriod).
+ case h if h.descriptorValue.isDefined && dateFeatureIndices.contains(oldInd) =>
+ for {name <- getRawFeatureName(h)} {
+ val key = name + h.descriptorValue.flatMap(convertToTimePeriod).map(p => "_" + p.entryName).getOrElse("")
+ val (indices, array) = aggregationMap.getOrElse(key, (Array.empty[Int], Array.empty[Double]))
+ aggregationMap.update(key, (indices :+ oldInd, sumArrays(array, diffToExamine)))
+ }
+ case _ => minMaxHeap enqueue LOCOValue(oldInd, diffToExamine(indexToExamine), diffToExamine)
+ }
+ case _ => ()
  }
 
  // Adding LOCO results from aggregation map into heaps
@@ -222,6 +218,7 @@ class RecordInsightsLOCO[T <: Model[T]]
  (textFeatureIndices ++ dateFeatureIndices).foreach(i => if (!featuresSparse.indices.contains(i)) res += i -> 0.0)
  val featureArray = res.toArray
  val featureSize = featuresSparse.size
+ val featureDense = Vectors.sparse(featureSize, featureArray).toDense
 
  val k = $(topK)
  // Index where to examine the difference in the prediction vector
@@ -232,7 +229,7 @@ class RecordInsightsLOCO[T <: Model[T]]
  // For MultiClassification, the value is from the predicted class(i.e. the class having the highest probability)
  case n if n > 2 => baseResult.prediction.toInt
  }
- val topPosNeg = returnTopPosNeg(featureArray, featureSize, baseScore, k, indexToExamine)
+ val topPosNeg = returnTopPosNeg(featureDense, baseScore, k, indexToExamine)
  val top = getTopKStrategy match {
  case TopKStrategy.Abs => topPosNeg.sortBy { case LOCOValue(_, v, _) => -math.abs(v) }.take(k)
  // Take top K positive and top K negative LOCOs, hence 2 * K

gradle.properties

@@ -1,3 +1,3 @@
-version=0.6.1-SNAPSHOT
+version=0.6.1-SNAPSHOT-gera-1
 group=com.salesforce.transmogrifai
 org.gradle.caching=true