Add 'levels' parameter to as_discrete()

commons/src/commonMain/kotlin/org/jetbrains/letsPlot/commons/intern/CollectionsEx.kt

@@ -12,6 +12,13 @@ fun <K, V> Map<K?, V>.filterNotNullKeys(): Map<K, V> {
  .toMap()
 }
 
+fun <K, V> Map<K, V?>.filterNotNullValues(): Map<K, V> {
+ return entries
+ .asSequence()
+ .mapNotNull { (k, v) -> v?.let { k to v } }
+ .toMap()
+}
+
 fun <T> Collection<T>.splitBy(comp: Comparator<T>): List<List<T>> {
  val result = mutableListOf<List<T>>()
  var chunk = mutableListOf<T>()

demo/plot-common/src/commonMain/kotlin/demo/plot/common/model/plotConfig/AsDiscrete.kt

@@ -16,7 +16,8 @@ class AsDiscrete {
  fillFactorWithScaleColor(),
  layerData_DiscreteGroup(),
  smoothStatAsDiscrete(),
- smoothStatWithGroup()
+ smoothStatWithGroup(),
+ factorLevels()
  )
  }
 
@@ -226,4 +227,34 @@ class AsDiscrete {
  return parsePlotSpec(spec)
  }
 
+ private fun factorLevels(): MutableMap<String, Any> {
+ val spec = """{
+ 'data': { 
+ 'name' : ['c', 'c', 'a', 'a', 'd', 'b', 'b', 'a'],
+ 'value': [1, 2, 3, 2, 2, 1, 4, 1] 
+ },
+ 'kind': 'plot',
+ 'layers': [
+ { 
+ 'geom': 'bar',
+ 'stat': 'identity',
+ 'mapping': {'x': 'name', 'y': 'value', 'fill': 'value'},
+ 'data_meta': {
+ 'series_annotations': [
+ { 
+ 'column': 'name',
+ 'factor_levels': ['a','c','b']
+ },
+ { 
+ 'column': 'value',
+ 'factor_levels': [1,2,3],
+ 'order': -1
+ } 
+ ]
+ } 
+ }
+ ]
+ }""".trimIndent()
+ return parsePlotSpec(spec)
+ }
 }

future_changes.md

@@ -18,6 +18,11 @@
  See: [example notebook](https://nbviewer.jupyter.org/github/JetBrains/lets-plot/blob/master/docs/f-23f/scale_lablim.ipynb).
 
 
+- The `levels` parameter in `as_discrete` function [[#931](https://github.com/JetBrains/lets-plot/issues/931)].
+
+ See: [example notebook](https://nbviewer.jupyter.org/github/JetBrains/lets-plot/blob/master/docs/f-23f/factor_levels.ipynb).
+
+
 ### Changed
 
 - The `plot_margin` parameter in `theme()` and the `margin` parameter in `element_text()` accept a number or a list of numbers:

plot-stem/src/commonMain/kotlin/org/jetbrains/letsPlot/core/spec/Option.kt

@@ -65,6 +65,7 @@ object Option {
  const val COLUMN = "column" // a.k.a. variable name
  const val TYPE = "type"
  const val FACTOR_LEVELS = "factor_levels" // annotation for discrete variables
+ const val ORDER = "order"
 
  // Values of the "TYPE" property
  object DateTime {

plot-stem/src/commonMain/kotlin/org/jetbrains/letsPlot/core/spec/back/PlotConfigBackend.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.letsPlot.core.spec.back
 
+import org.jetbrains.letsPlot.commons.intern.filterNotNullKeys
 import org.jetbrains.letsPlot.core.plot.base.*
 import org.jetbrains.letsPlot.core.plot.base.DataFrame.Variable
 import org.jetbrains.letsPlot.core.plot.base.data.DataFrameUtil
@@ -49,6 +50,15 @@ open class PlotConfigBackend(
  }
  }
 
+ // match the specified 'factor_levels' to the actual contents of the data set (on combined df before stat)
+ val specifiedFactorLevelsByLayers = layerConfigs.map { layerConfig ->
+ prepareLayerFactorLevelsByVariable(
+ layerConfig.combinedData,
+ plotDataMeta = getMap(DATA_META),
+ layerDataMeta = layerConfig.getMap(DATA_META)
+ )
+ }
+
  // replace layer data with data after stat
  layerConfigs.withIndex().forEach { (layerIndex, layerConfig) ->
  // optimization: only replace layer' data if 'combined' data was changed (because of stat or sampling occurred)
@@ -61,24 +71,28 @@ open class PlotConfigBackend(
  // Clean-up data before sending it to the front-end.
  dropUnusedDataBeforeEncoding(layerConfigs)
 
- // Re-create the "natural order" existed before faceting.
- if (facets.isDefined) {
- // When faceting, each layer' data was split to panels, then re-combined with loss of 'natural order'.
- layerConfigs.forEach { layerConfig ->
- val layerData = layerConfig.ownData
- if (facets.isFacettable(layerData)) {
+ // Re-create the "natural order" existed before faceting
+ // or apply the specified order
+ if (facets.isDefined || specifiedFactorLevelsByLayers.any { it.isNotEmpty() }) {
+ layerConfigs.zip(specifiedFactorLevelsByLayers)
+ .filter { (layerConfig, factorLevels) ->
+ // When faceting, each layer' data was split to panels, then re-combined with loss of 'natural order'.
+ facets.isFacettable(layerConfig.ownData)
+ || factorLevels.isNotEmpty()
+ }
+ .forEach { (layerConfig, factorLevels) ->
  val layerDataMetaUpdated = addFactorLevelsDataMeta(
- layerData = layerData,
+ layerData = layerConfig.ownData,
  layerDataMeta = layerConfig.getMap(DATA_META),
  stat = layerConfig.stat,
  varBindings = layerConfig.varBindings,
  transformByAes = transformByAes,
  orderOptions = layerConfig.orderOptions,
- yOrientation = layerConfig.isYOrientation
+ yOrientation = layerConfig.isYOrientation,
+ specifiedLayerFactorLevers = factorLevels
  )
  layerConfig.update(DATA_META, layerDataMetaUpdated)
  }
- }
  }
  }
 
@@ -273,6 +287,7 @@ open class PlotConfigBackend(
  transformByAes: Map<Aes<*>, Transform>,
  orderOptions: List<OrderOption>,
  yOrientation: Boolean,
+ specifiedLayerFactorLevers: Map<String, List<Any>>
  ): Map<String, Any> {
 
  // Use "discrete transforms" to re-create the "natural order" existed before faceting.
@@ -304,7 +319,34 @@ open class PlotConfigBackend(
  levelsByVariable[variable.name] = orderedDistinctValues
  }
 
+ // apply specified factors
+ levelsByVariable += specifiedLayerFactorLevers
+
  return DataMetaUtil.updateFactorLevelsByVariable(layerDataMeta, levelsByVariable)
  }
+
+ private fun prepareLayerFactorLevelsByVariable(
+ data: DataFrame,
+ plotDataMeta: Map<*, *>,
+ layerDataMeta: Map<*, *>
+ ): Map<String, List<Any>> {
+ val plotFactorLevelsByVar = DataMetaUtil.getFactorLevelsByVariable(plotDataMeta)
+ val layerFactorLevelsByVar = DataMetaUtil.getFactorLevelsByVariable(layerDataMeta)
+ val factorLevelsByVar = (plotFactorLevelsByVar + layerFactorLevelsByVar)
+ .mapKeys { (varName, _) -> data.variables().find { it.name == varName } }
+ .filterNotNullKeys()
+
+ val orderDirectionsByVar = DataMetaUtil.getFactorLevelsOrderByVariable(plotDataMeta) +
+ DataMetaUtil.getFactorLevelsOrderByVariable(layerDataMeta)
+
+ return factorLevelsByVar.map { (variable, levels) ->
+ // append missed values to the tail of specified levels
+ val distinctValues = data.distinctValues(variable)
+ val tail = distinctValues - levels.toSet()
+ val order = orderDirectionsByVar.getOrElse(variable.name) { 0 }
+ val factors = (levels + tail).let { if (order >= 0) it else it.reversed() }
+ variable.name to factors
+ }.toMap()
+ }
  }
 }

plot-stem/src/commonMain/kotlin/org/jetbrains/letsPlot/core/spec/config/DataConfigUtil.kt

@@ -155,6 +155,11 @@ internal object DataConfigUtil {
  val asDiscreteVariables = varBindings.filter { it.aes.name in asDiscreteAesSet }.map { it.variable.name }
  val variablesToMarkAsDiscrete = rawCombinedData.variables().filter { it.name in asDiscreteVariables }
 
+ // categorical variables
+ val categoricalVariables = DataMetaUtil.getCategoricalVariables(ownDataMeta).let { categoricalVarNames ->
+ rawCombinedData.variables().filter { it.name in categoricalVarNames }
+ }
+
  fun DataFrame.Builder.addVariables(
  variables: List<DataFrame.Variable>,
  put: (DataFrame.Builder, DataFrame.Variable, List<*>) -> DataFrame.Builder
@@ -169,6 +174,7 @@ internal object DataConfigUtil {
 
  addVariables(variablesToMarkAsDateTime, DataFrame.Builder::putDateTime)
  addVariables(variablesToMarkAsDiscrete, DataFrame.Builder::putDiscrete)
+ addVariables(categoricalVariables, DataFrame.Builder::putDiscrete)
 
  if (clientSide) {
  val variables = rawCombinedData.variables()

plot-stem/src/commonMain/kotlin/org/jetbrains/letsPlot/core/spec/config/DataMetaUtil.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.letsPlot.core.spec.config
 
+import org.jetbrains.letsPlot.commons.intern.filterNotNullValues
 import org.jetbrains.letsPlot.core.plot.builder.data.OrderOptionUtil
 import org.jetbrains.letsPlot.core.spec.*
 import org.jetbrains.letsPlot.core.spec.Option.Meta.MappingAnnotation
@@ -96,12 +97,25 @@ object DataMetaUtil {
  ?: emptySet()
  }
 
+ fun getCategoricalVariables(dataMeta: Map<*, *>): Set<String> {
+ return getFactorLevelsByVariable(dataMeta).keys
+ }
+
  fun getFactorLevelsByVariable(dataMeta: Map<*, *>): Map<String, List<Any>> {
  return (dataMeta
  .getMaps(SeriesAnnotation.TAG)
  ?.associate { it.getString(COLUMN)!! to it.getList(FACTOR_LEVELS) }
- ?.filterValues { list -> list?.isNotEmpty() ?: false }
- ?.mapValues { (_, list) -> list!!.map { v -> v as Any } }
+ ?.filterNotNullValues()
+ ?.mapValues { (_, factorLevels) -> factorLevels.map { v -> v as Any } }
+ ?: emptyMap())
+ }
+
+ fun getFactorLevelsOrderByVariable(dataMeta: Map<*, *>): Map<String, Int> {
+ return (dataMeta
+ .getMaps(SeriesAnnotation.TAG)
+ ?.associate { it.getString(COLUMN)!! to it.getNumber(SeriesAnnotation.ORDER) }
+ ?.filterNotNullValues()
+ ?.mapValues { (_, order) -> order.toInt() }
  ?: emptyMap())
  }
 

plot-stem/src/jvmTest/kotlin/org/jetbrains/letsPlot/core/spec/back/DataMetaStoreFactorLevelsTest.kt

@@ -94,6 +94,34 @@ class DataMetaStoreFactorLevelsTest {
  .hasDataMetaFacetLevels("chrom", listOf("chr1", "chr2", "chr4", "chr5"))
  }
 
+ @Test
+ fun `specified 'factor_levels' for variable`() {
+ val plotSpecs = plotSpecs_With_FactorLevels(
+ varListWithLevels = mapOf(
+ "chrom" to listOf("chr5", "chr4", "chr2", "chr1"),
+ "arm" to listOf("q", "p")
+ )
+ )
+ val layerConfigs = BackendTestUtil.createLayerConfigs(plotSpecs)
+ SingleLayerAssert.assertThat(layerConfigs)
+ .hasDataMetaFacetLevels("chrom", listOf("chr5", "chr4", "chr2", "chr1"))
+ .hasDataMetaFacetLevels("arm", listOf("q", "p"))
+ }
+
+ @Test
+ fun `specified 'factor_levels' will be consistent with the actual dataset contents`() {
+ val plotSpecs = plotSpecs_With_FactorLevels(
+ varListWithLevels = mapOf(
+ "chrom" to listOf("chr4", "chr5"), // should append "chr1", "chr2"
+ "arm" to listOf("q") // should append "p"
+ )
+ )
+ val layerConfigs = BackendTestUtil.createLayerConfigs(plotSpecs)
+ SingleLayerAssert.assertThat(layerConfigs)
+ .hasDataMetaFacetLevels("chrom", listOf("chr4", "chr5", "chr1", "chr2"))
+ .hasDataMetaFacetLevels("arm", listOf("q", "p"))
+ }
+
 
  private companion object {
  // See LP issue #746
@@ -165,21 +193,33 @@ class DataMetaStoreFactorLevelsTest {
  )
 
  private fun plotSpecs_No_Ordering_No_DateTime(): MutableMap<String, Any> {
- val plotSpec = plotSpec(null, null)
+ val plotSpec = plotSpec(
+ asDiscreteAnnotationsSpec = null,
+ dateTimeAnnotationsSpec = null,
+ factorLevelsAnnotationsSpec = null
+ )
  plotSpec["data"] = DATA_20
  return plotSpec
  }
 
- private fun plotSpecs_With_asDiscrete(aesList: List<String>, order: Boolean): MutableMap<String, Any> {
+ fun plotSpecs_With_asDiscrete(aesList: List<String>, order: Boolean): MutableMap<String, Any> {
  val asDiscreteAnnotations = asDiscreteAnnotationsSpec(aesList, order)
- val plotSpec = plotSpec(asDiscreteAnnotations, null)
+ val plotSpec = plotSpec(
+ asDiscreteAnnotations,
+ dateTimeAnnotationsSpec = null,
+ factorLevelsAnnotationsSpec = null
+ )
  plotSpec["data"] = DATA_20
  return plotSpec
  }
 
  private fun plotSpecs_With_DateTime(varList: List<String>): MutableMap<String, Any> {
  val dateTimeAnnotations = dateTimeAnnotationsSpec(varList)
- val plotSpec = plotSpec(null, dateTimeAnnotations)
+ val plotSpec = plotSpec(
+ asDiscreteAnnotationsSpec = null,
+ dateTimeAnnotationsSpec = dateTimeAnnotations,
+ factorLevelsAnnotationsSpec = null
+ )
  plotSpec["data"] = DATA_20
  return plotSpec
  }
@@ -191,21 +231,40 @@ class DataMetaStoreFactorLevelsTest {
  ): MutableMap<String, Any> {
  val dateTimeAnnotations = dateTimeAnnotationsSpec(varList)
  val asDiscreteAnnotations = asDiscreteAnnotationsSpec(aesList, order)
- val plotSpec = plotSpec(asDiscreteAnnotations, dateTimeAnnotations)
+ val plotSpec = plotSpec(
+ asDiscreteAnnotations,
+ dateTimeAnnotations,
+ factorLevelsAnnotationsSpec = null
+ )
+ plotSpec["data"] = DATA_20
+ return plotSpec
+ }
+
+ fun plotSpecs_With_FactorLevels(
+ varListWithLevels: Map<String, List<Any>>
+ ): MutableMap<String, Any> {
+ val factorLevelsAnnotationsSpec = factorLevelsAnnotationsSpec(varListWithLevels)
+ val plotSpec = plotSpec(
+ asDiscreteAnnotationsSpec = null,
+ dateTimeAnnotationsSpec = null,
+ factorLevelsAnnotationsSpec = factorLevelsAnnotationsSpec
+ )
  plotSpec["data"] = DATA_20
  return plotSpec
  }
 
  private fun plotSpec(
  asDiscreteAnnotationsSpec: String?,
- dateTimeAnnotationsSpec: String?
+ dateTimeAnnotationsSpec: String?,
+ factorLevelsAnnotationsSpec: String?
  ): MutableMap<String, Any> {
  val annotationSpecs = ArrayList<String>().apply {
  if (asDiscreteAnnotationsSpec != null) {
  add("'mapping_annotations': [$asDiscreteAnnotationsSpec]")
  }
- if (dateTimeAnnotationsSpec != null) {
- add("'series_annotations': [$dateTimeAnnotationsSpec]")
+ val seriesAnnotations = listOfNotNull(dateTimeAnnotationsSpec, factorLevelsAnnotationsSpec).joinToString()
+ if (seriesAnnotations.isNotEmpty()) {
+ add("'series_annotations': [$seriesAnnotations]")
  }
  }
 
@@ -247,6 +306,17 @@ class DataMetaStoreFactorLevelsTest {
  }
  }
 
+ private fun factorLevelsAnnotationsSpec(varListWithLevels: Map<String, List<Any>>): String {
+ return varListWithLevels.toList().joinToString { (variable, factorLevels) ->
+ """
+ {
+ 'column': '$variable',
+ 'factor_levels': [ ${factorLevels.joinToString { "\'$it\'" }} ]
+ } 
+ """.trimIndent()
+ }
+ }
+
  private fun dateTimeAnnotationsSpec(varList: List<String>): String {
  return varList.joinToString { variable ->
  """