Revert "Turn off logging for cardEstimate & track token length instea…

…d of token value (#416)" (#419)
salesforce · Oct 11, 2019 · 6589656 · 6589656
1 parent e0ea2ed
commit 6589656
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/core/src/main/scala/com/salesforce/op/filters/FeatureDistribution.scala b/core/src/main/scala/com/salesforce/op/filters/FeatureDistribution.scala
@@ -167,6 +167,7 @@ case class FeatureDistribution
       "nulls" -> nulls.toString,
       "distribution" -> distribution.mkString("[", ",", "]"),
       "summaryInfo" -> summaryInfo.mkString("[", ",", "]"),
+      "cardinality" -> cardEstimate.map(_.toString).getOrElse(""),
       "moments" -> moments.map(_.toString).getOrElse("")
     ).map { case (n, v) => s"$n = $v" }.mkString(", ")
 
@@ -278,7 +279,7 @@ object FeatureDistribution {
    */
   private def cardinalityValues(values: ProcessedSeq): TextStats = {
     val population = values match {
-      case Left(seq) => seq.map(_.size.toString)
+      case Left(seq) => seq
       case Right(seq) => seq.map(_.toString)
     }
     TextStats(population.groupBy(identity).map{case (key, value) => (key, value.size)})

diff --git a/core/src/test/scala/com/salesforce/op/filters/FeatureDistributionTest.scala b/core/src/test/scala/com/salesforce/op/filters/FeatureDistributionTest.scala
@@ -75,7 +75,7 @@ class FeatureDistributionTest extends FlatSpec with PassengerSparkFixtureTest wi
     distribs(3).distribution.sum shouldBe 0
     distribs(4).distribution.sum shouldBe 3
     distribs(4).summaryInfo.length shouldBe bins
-    distribs(2).cardEstimate.get shouldBe TextStats(Map("4" -> 1, "6" -> 1))
+    distribs(2).cardEstimate.get shouldBe TextStats(Map("male" -> 1, "female" -> 1))
     distribs(2).moments.get shouldBe Moments(2, 5.0, 2.0, 0.0, 2.0)
     distribs(4).cardEstimate.get shouldBe TextStats(Map("5.0" -> 1, "1.0" -> 1, "3.0" -> 1))
     distribs(4).moments.get shouldBe Moments(3, 3.0, 8.0, 0.0, 32.0)
@@ -196,7 +196,7 @@ class FeatureDistributionTest extends FlatSpec with PassengerSparkFixtureTest wi
   it should "have toString" in {
     FeatureDistribution("A", None, 10, 1, Array(1, 4, 0, 0, 6), Array.empty).toString() shouldBe
       "FeatureDistribution(type = Training, name = A, key = None, count = 10, nulls = 1, " +
-        "distribution = [1.0,4.0,0.0,0.0,6.0], summaryInfo = [], moments = )"
+        "distribution = [1.0,4.0,0.0,0.0,6.0], summaryInfo = [], cardinality = , moments = )"
   }
 
   it should "marshall to/from json" in {