Fix #936: Way to calculate proportion of points with same coordinate

future_changes.md

@@ -2,6 +2,11 @@
 
 ### Added
 
+- New variables computed by `'count'` and `'count2d'` statistics: `'..sumprop..'`, `'..sumpct..'`.
+
+ See: [example notebook](https://nbviewer.jupyter.org/github/JetBrains/lets-plot/blob/master/docs/f-23f/new_stat_count_vars.ipynb).
+
+
 ### Changed
 
 ### Fixed
@@ -11,3 +16,4 @@
 - geom_livemap: freeze at zoom 10 [[#892](https://github.com/JetBrains/lets-plot/issues/892)].
 - Enormous CPU / Time/ Memory consumption on some data [[#932](https://github.com/JetBrains/lets-plot/issues/932)].
 - scale_x_log2(), scale_y_log2() as a shortcut for trans='log2' [[#922](https://github.com/JetBrains/lets-plot/issues/922)].
+- How to calculate proportion of points with same coordinate [[#936](https://github.com/JetBrains/lets-plot/issues/936)].

plot-base/src/commonMain/kotlin/org/jetbrains/letsPlot/core/plot/base/stat/AbstractCountStat.kt

@@ -80,14 +80,19 @@ abstract class AbstractCountStat(
  // compute total location weights on the whole data
  val summary = groupAndSum(locations, weights)
  val totalWeights = locations.map { summary[it]!! }
+ val totalWeightsSum = summary.values.sum()
 
  val prop = weights.zip(totalWeights).map { (groupWeight, totalWeight) -> groupWeight / totalWeight }
  val propPercent = prop.map { it * 100 }
+ val sumProp = totalWeights.map { it / totalWeightsSum }
+ val sumPropPercent = sumProp.map { it * 100 }
 
  val statDf = dataAfterStat.builder()
  statDf.putNumeric(Stats.SUM, totalWeights)
  statDf.putNumeric(Stats.PROP, prop)
  statDf.putNumeric(Stats.PROPPCT, propPercent)
+ statDf.putNumeric(Stats.SUMPROP, sumProp)
+ statDf.putNumeric(Stats.SUMPCT, sumPropPercent)
  return statDf.build()
  }
 

plot-base/src/commonMain/kotlin/org/jetbrains/letsPlot/core/plot/base/stat/Stats.kt

@@ -35,6 +35,8 @@ object Stats {
  val SUM = DataFrame.Variable("..sum..", STAT, "sum")
  val PROP = DataFrame.Variable("..prop..", STAT, "prop")
  val PROPPCT = DataFrame.Variable("..proppct..", STAT, "proppct")
+ val SUMPROP = DataFrame.Variable("..sumprop..", STAT, "sumprop")
+ val SUMPCT = DataFrame.Variable("..sumpct..", STAT, "sumpct")
 
  val SCALED = DataFrame.Variable("..scaled..", STAT, "scaled")
 
@@ -66,6 +68,8 @@ object Stats {
  SUM,
  PROP,
  PROPPCT,
+ SUMPROP,
+ SUMPCT,
  SCALED,
  GROUP,
  )

plot-builder/src/commonMain/kotlin/org/jetbrains/letsPlot/core/plot/builder/tooltip/TooltipFormatting.kt

@@ -31,8 +31,10 @@ internal object TooltipFormatting {
 
  fun createFormatter(variable: DataFrame.Variable): (Any) -> String {
  return when (variable) {
- Stats.PROP -> StringFormat.forOneArg(".2f", formatFor = variable.name)::format
- Stats.PROPPCT -> StringFormat.forOneArg("{.1f} %", formatFor = variable.name)::format
+ Stats.PROP,
+ Stats.SUMPROP -> StringFormat.forOneArg(".2f", formatFor = variable.name)::format
+ Stats.PROPPCT,
+ Stats.SUMPCT -> StringFormat.forOneArg("{.1f} %", formatFor = variable.name)::format
  else -> { value -> value.toString() }
  }
  }

plot-builder/src/jvmTest/kotlin/org/jetbrains/letsPlot/core/plot/builder/data/Count2dStatTest.kt

@@ -20,7 +20,7 @@ class Count2dStatTest {
  val statDf = dataProcessor.applyStat(Stats.count2d())
 
  assertThat(statDf.variables())
- .containsExactlyInAnyOrder(x, y, Stats.X, Stats.Y, Stats.SUM, Stats.COUNT, Stats.PROP, Stats.PROPPCT)
+ .containsExactlyInAnyOrder(x, y, Stats.X, Stats.Y, Stats.SUM, Stats.COUNT, Stats.PROP, Stats.PROPPCT, Stats.SUMPROP, Stats.SUMPCT)
 
  assertThat(statDf.rowCount()).isZero()
  }
@@ -34,14 +34,16 @@ class Count2dStatTest {
  val statDf = dataProcessor.applyStat(Stats.count2d())
 
  assertThat(statDf.variables())
- .containsExactlyInAnyOrder(x, y, Stats.X, Stats.Y, Stats.SUM, Stats.COUNT, Stats.PROP, Stats.PROPPCT)
+ .containsExactlyInAnyOrder(x, y, Stats.X, Stats.Y, Stats.SUM, Stats.COUNT, Stats.PROP, Stats.PROPPCT, Stats.SUMPROP, Stats.SUMPCT)
 
  assertThat(statDf[Stats.X]).containsExactly("0")
  assertThat(statDf[Stats.Y]).containsExactly("0")
  assertThat(statDf[Stats.SUM]).containsExactly(1.0)
  assertThat(statDf[Stats.COUNT]).containsExactly(1.0)
  assertThat(statDf[Stats.PROP]).containsExactly(1.0)
  assertThat(statDf[Stats.PROPPCT]).containsExactly(100.0)
+ assertThat(statDf[Stats.SUMPROP]).containsExactly(1.0)
+ assertThat(statDf[Stats.SUMPCT]).containsExactly(100.0)
  }
 
  @Test
@@ -55,6 +57,7 @@ class Count2dStatTest {
  assertThat(statDf[Stats.SUM]).containsExactly(1.0, 1.0)
  assertThat(statDf[Stats.COUNT]).containsExactly(1.0, 1.0)
  assertThat(statDf[Stats.PROP]).containsExactly(1.0, 1.0)
+ assertThat(statDf[Stats.SUMPROP]).containsExactly(0.5, 0.5)
  }
 
  @Test
@@ -68,6 +71,7 @@ class Count2dStatTest {
  assertThat(statDf[Stats.SUM]).containsExactly(2.0)
  assertThat(statDf[Stats.COUNT]).containsExactly(2.0)
  assertThat(statDf[Stats.PROP]).containsExactly(1.0)
+ assertThat(statDf[Stats.SUMPROP]).containsExactly(1.0)
  }
 
  @Test
@@ -82,6 +86,7 @@ class Count2dStatTest {
  assertThat(statDf[Stats.SUM]).containsExactly(4.0)
  assertThat(statDf[Stats.COUNT]).containsExactly(4.0)
  assertThat(statDf[Stats.PROP]).containsExactly(1.0)
+ assertThat(statDf[Stats.SUMPROP]).containsExactly(1.0)
  }
 
  @Test
@@ -98,6 +103,7 @@ class Count2dStatTest {
  assertThat(statDf[Stats.SUM]).containsExactly(4.0, 4.0)
  assertThat(statDf[Stats.COUNT]).containsExactly(1.0, 3.0)
  assertThat(statDf[Stats.PROP]).containsExactly(0.25, 0.75)
+ assertThat(statDf[Stats.SUMPROP]).containsExactly(1.0, 1.0)
  }
 
  @Test
@@ -113,13 +119,15 @@ class Count2dStatTest {
  assertThat(statDf[Stats.SUM]).containsExactly(4.0)
  assertThat(statDf[Stats.COUNT]).containsExactly(4.0)
  assertThat(statDf[Stats.PROP]).containsExactly(1.0)
+ assertThat(statDf[Stats.SUMPROP]).containsExactly(1.0)
  }
 
  dataProcessor.groupingVarName = "g"
  dataProcessor.applyStat(Stats.count2d()).let { statDf ->
  assertThat(statDf[Stats.SUM]).containsExactly(4.0, 4.0)
  assertThat(statDf[Stats.COUNT]).containsExactly(1.0, 3.0)
  assertThat(statDf[Stats.PROP]).containsExactly(0.25, 0.75)
+ assertThat(statDf[Stats.SUMPROP]).containsExactly(1.0, 1.0)
  }
  }
 }

python-package/lets_plot/plot/geom.py

@@ -731,6 +731,8 @@ def geom_bar(mapping=None, *, data=None, stat=None, position=None, show_legend=N
  - ..sum.. : total number of points with same x-axis coordinate.
  - ..prop.. : groupwise proportion.
  - ..proppct.. : groupwise proportion in percent.
+ - ..sumprop.. : total proportion, i.e. ratio of total number of points with same x-axis coordinate to the total number of points in the data.
+ - ..sumpct.. : total proportion in percent.
 
  `geom_bar()` understands the following aesthetics mappings:
 
@@ -6172,6 +6174,8 @@ def geom_pie(mapping=None, *, data=None, stat=None, position=None, show_legend=N
  - ..sum.. : total number of points with same (x,y) coordinate.
  - ..prop.. : groupwise proportion.
  - ..proppct.. : groupwise proportion in percent.
+ - ..sumprop.. : total proportion, i.e. ratio of total number of points with same (x,y) coordinate to the total number of points in the data.
+ - ..sumpct.. : total proportion in percent.
 
  `geom_pie()` understands the following aesthetics mappings: