Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Summary stat #803

Merged
merged 26 commits into from
Jun 30, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
7886cbc
First implementation of the summary stat.
ASmirnov-HORIS Jun 16, 2023
d95a062
Update API for the stat_summary() function.
ASmirnov-HORIS Jun 20, 2023
b28bf7f
Small refactor in SummaryStat.
ASmirnov-HORIS Jun 21, 2023
6ce963b
Remove extra enum class from the SummaryStatUtil.
ASmirnov-HORIS Jun 21, 2023
b7a8f33
Remove SummaryStatUtil.
ASmirnov-HORIS Jun 21, 2023
05345ee
Use references instead of lambdas for the SummaryCalculator.
ASmirnov-HORIS Jun 21, 2023
756b626
Replace SummaryCalculator by the SummaryStatUtil.
ASmirnov-HORIS Jun 21, 2023
3b2bca7
Refactor functions in SummaryStatUtil.
ASmirnov-HORIS Jun 22, 2023
1f03da4
Refactor summary stat options in StatProto.
ASmirnov-HORIS Jun 22, 2023
baccb49
Fix statData emptiness case in the SummaryStat.
ASmirnov-HORIS Jun 22, 2023
c9e154c
Further code refactoring.
ASmirnov-HORIS Jun 22, 2023
0082f74
Small fixes.
ASmirnov-HORIS Jun 22, 2023
cbabbe3
Add new stat variables and use them in the SummaryStat.
ASmirnov-HORIS Jun 23, 2023
7fea03b
Add prefix to min/max stats in stat_summary().
ASmirnov-HORIS Jun 23, 2023
0ecc6ec
Change API of the summary_stat() - add 'quantiles' parameter.
ASmirnov-HORIS Jun 26, 2023
722b43a
Tiny refactor in SummaryStat and AggregateFunctions.
ASmirnov-HORIS Jun 27, 2023
e30b93b
Use AggregateFunctions in the FiveNumberSummary.
ASmirnov-HORIS Jun 27, 2023
f076938
Add tests for AggregateFunctions.
ASmirnov-HORIS Jun 27, 2023
2313872
Replace parameter fun_map by usual aesthetics list for the stat_summa…
ASmirnov-HORIS Jun 29, 2023
4bba413
Small fixes in code for summary stat.
ASmirnov-HORIS Jun 29, 2023
9cfeea5
Merge branch 'master' into summary-stats
ASmirnov-HORIS Jun 29, 2023
4586451
Add getMapping() method to the Flipped stat context.
ASmirnov-HORIS Jun 29, 2023
9dcdd79
Add docstrings to the stat_summary() function.
ASmirnov-HORIS Jun 29, 2023
37526eb
Add demo notebook for stat_summary().
ASmirnov-HORIS Jun 29, 2023
0a1d636
Mention stat_summary() in the future_changes.
ASmirnov-HORIS Jun 29, 2023
a392ba1
Refactor StatContext.
ASmirnov-HORIS Jun 30, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Replace parameter fun_map by usual aesthetics list for the stat_summa…
…ry().
  • Loading branch information
ASmirnov-HORIS committed Jun 29, 2023
commit 231387297093c98f5a02dad9297f4965dd474e62
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ interface StatContext {

fun overallYRange(): DoubleSpan?

fun getMapping(): Map<Aes<*>, DataFrame.Variable>

fun getFlipped(): StatContext {
return Flipped(this)
}
Expand All @@ -25,6 +27,10 @@ interface StatContext {
return orig.overallXRange()
}

override fun getMapping(): Map<Aes<*>, DataFrame.Variable> {
return orig.getMapping()
}

override fun getFlipped(): StatContext {
return orig
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package jetbrains.datalore.plot.base.stat

import jetbrains.datalore.base.interval.DoubleSpan
import jetbrains.datalore.plot.base.Aes
import jetbrains.datalore.plot.base.DataFrame
import jetbrains.datalore.plot.base.StatContext
import jetbrains.datalore.plot.base.data.TransformVar
Expand All @@ -20,4 +21,8 @@ class SimpleStatContext(private val myDataFrame: DataFrame) :
override fun overallYRange(): DoubleSpan? {
return myDataFrame.range(TransformVar.Y)
}

override fun getMapping(): Map<Aes<*>, DataFrame.Variable> {
return emptyMap()
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@ import jetbrains.datalore.plot.base.data.TransformVar
import jetbrains.datalore.plot.common.data.SeriesUtil

class SummaryStat(
private val aggFunctionsMap: Map<DataFrame.Variable, (List<Double>) -> Double>
private val yAggFunction: (List<Double>) -> Double,
private val yMinAggFunction: (List<Double>) -> Double,
private val yMaxAggFunction: (List<Double>) -> Double,
private val sortedQuantiles: List<Double>
) : BaseStat(DEF_MAPPING) {

override fun consumes(): List<Aes<*>> {
Expand All @@ -32,7 +35,7 @@ class SummaryStat(
List(ys.size) { 0.0 }
}

val statData = buildStat(xs, ys)
val statData = buildStat(xs, ys, statCtx)
if (statData.isEmpty()) {
return withEmptyStatValues()
}
Expand All @@ -46,7 +49,8 @@ class SummaryStat(

private fun buildStat(
xs: List<Double?>,
ys: List<Double?>
ys: List<Double?>,
statCtx: StatContext
): Map<DataFrame.Variable, List<Double>> {
val binnedData = SeriesUtil.filterFinite(xs, ys)
.let { (xs, ys) -> xs zip ys }
Expand All @@ -57,17 +61,46 @@ class SummaryStat(
}

val statX = ArrayList<Double>()
val statAggValues: Map<DataFrame.Variable, MutableList<Double>> = aggFunctionsMap.keys.associateWith { mutableListOf() }
val statY = ArrayList<Double>()
val statYMin = ArrayList<Double>()
val statYMax = ArrayList<Double>()
val statAggValues: Map<DataFrame.Variable, MutableList<Double>> = statCtx.getMapping().values.filter { it.isStat }.associateWith { mutableListOf() }
for ((x, bin) in binnedData) {
statX.add(x)
val sortedBin = Ordering.natural<Double>().sortedCopy(bin)
statX.add(x)
statY.add(yAggFunction(sortedBin))
statYMin.add(yMinAggFunction(sortedBin))
statYMax.add(yMaxAggFunction(sortedBin))
for ((statVar, aggValues) in statAggValues) {
val aggFunction = aggFunctionsMap[statVar] ?: { Double.NaN }
val aggFunction = aggFunctionByStat(statVar)
aggValues.add(aggFunction(sortedBin))
}
}

return mapOf(Stats.X to statX) + statAggValues
return mapOf(
Stats.X to statX,
Stats.Y to statY,
Stats.Y_MIN to statYMin,
Stats.Y_MAX to statYMax,
) + statAggValues
}

private fun aggFunctionByStat(statVar: DataFrame.Variable): (List<Double>) -> Double {
return when (statVar) {
Stats.COUNT -> AggregateFunctions::count
Stats.SUM -> AggregateFunctions::sum
Stats.MEAN -> AggregateFunctions::mean
Stats.MEDIAN -> AggregateFunctions::median
Stats.Y_MIN -> AggregateFunctions::min
Stats.Y_MAX -> AggregateFunctions::max
Stats.LOWER_QUANTILE -> { values -> AggregateFunctions.quantile(values, sortedQuantiles[0]) }
Stats.MIDDLE_QUANTILE -> { values -> AggregateFunctions.quantile(values, sortedQuantiles[1]) }
Stats.UPPER_QUANTILE -> { values -> AggregateFunctions.quantile(values, sortedQuantiles[2]) }
else -> throw IllegalStateException(
"Unsupported stat variable: '${statVar.name}'\n" +
"Use one of: ..count.., ..sum.., ..mean.., ..median.., ..ymin.., ..ymax.., ..lq.., ..mq.., ..uq.."
)
}
}

companion object {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -416,51 +416,30 @@ object StatProto {
SummaryStat.DEF_QUANTILES
}

val defaultAggFunctions = mapOf(
Stats.Y to (options.getString(Summary.FUN)?.lowercase()?.let { getAggFunction(it, sortedQuantiles) } ?: AggregateFunctions::mean),
Stats.Y_MIN to (options.getString(Summary.FUN_MIN)?.lowercase()?.let { getAggFunction(it, sortedQuantiles) } ?: AggregateFunctions::min),
Stats.Y_MAX to (options.getString(Summary.FUN_MAX)?.lowercase()?.let { getAggFunction(it, sortedQuantiles) } ?: AggregateFunctions::max)
)

val additionalAggFunctions = configureAggFunMap(options.getMap(Summary.FUN_MAP), sortedQuantiles)

return SummaryStat(defaultAggFunctions + additionalAggFunctions)
}

private fun configureAggFunMap(
aggFunMap: Map<String, Any>,
sortedQuantiles: List<Double>
): Map<DataFrame.Variable, (List<Double>) -> Double> {
val aggFunOptions = OptionsAccessor(aggFunMap)
return aggFunMap.keys.mapNotNull { option ->
aggFunOptions.getString(option)?.lowercase()?.let { aggFunName ->
val statVar = when (aggFunName) {
"min", "max" -> "..y$aggFunName.."
else -> "..$aggFunName.."
}.let { Stats.statVar(it) }
Pair(statVar, getAggFunction(aggFunName, sortedQuantiles))
fun getAggFunction(option: String): ((List<Double>) -> Double)? {
return options.getString(option)?.let {
when (it.lowercase()) {
"count" -> AggregateFunctions::count
"sum" -> AggregateFunctions::sum
"mean" -> AggregateFunctions::mean
"median" -> AggregateFunctions::median
"min" -> AggregateFunctions::min
"max" -> AggregateFunctions::max
"lq" -> { values -> AggregateFunctions.quantile(values, sortedQuantiles[0]) }
"mq" -> { values -> AggregateFunctions.quantile(values, sortedQuantiles[1]) }
"uq" -> { values -> AggregateFunctions.quantile(values, sortedQuantiles[2]) }
else -> throw IllegalArgumentException(
"Unsupported function name: '$it'\n" +
"Use one of: count, sum, mean, median, min, max, lq, mq, uq."
)
}
}
}.toMap()
}

private fun getAggFunction(
aggFunName: String,
sortedQuantiles: List<Double>
): ((List<Double>) -> Double) {
return when (aggFunName) {
"count" -> AggregateFunctions::count
"sum" -> AggregateFunctions::sum
"mean" -> AggregateFunctions::mean
"median" -> AggregateFunctions::median
"min" -> AggregateFunctions::min
"max" -> AggregateFunctions::max
"lq" -> { values -> AggregateFunctions.quantile(values, sortedQuantiles[0]) }
"mq" -> { values -> AggregateFunctions.quantile(values, sortedQuantiles[1]) }
"uq" -> { values -> AggregateFunctions.quantile(values, sortedQuantiles[2]) }
else -> throw IllegalArgumentException(
"Unsupported function name: '$aggFunName'\n" +
"Use one of: count, sum, mean, median, min, max, lq, mq, uq."
)
}

val yAggFunction = getAggFunction(Summary.FUN) ?: AggregateFunctions::mean
val yMinAggFunction = getAggFunction(Summary.FUN_MIN) ?: AggregateFunctions::min
val yMaxAggFunction = getAggFunction(Summary.FUN_MAX) ?: AggregateFunctions::max

return SummaryStat(yAggFunction, yMinAggFunction, yMaxAggFunction, sortedQuantiles)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@ import jetbrains.datalore.base.interval.DoubleSpan
import jetbrains.datalore.plot.base.*
import jetbrains.datalore.plot.base.data.DataFrameUtil
import jetbrains.datalore.plot.base.scale.ScaleUtil
import jetbrains.datalore.plot.builder.VarBinding
import jetbrains.datalore.plot.common.data.SeriesUtil

internal class ConfiguredStatContext(
private val dataFrames: List<DataFrame>,
private val transformByAes: Map<Aes<*>, Transform>
private val transformByAes: Map<Aes<*>, Transform>,
private val varBindings: List<VarBinding>
) : StatContext {

private fun overallRange(variable: DataFrame.Variable, dataFrames: List<DataFrame>): DoubleSpan? {
Expand All @@ -34,6 +36,10 @@ internal class ConfiguredStatContext(
return overallRange(Aes.Y)
}

override fun getMapping(): Map<Aes<*>, DataFrame.Variable> {
return varBindings.associate { it.aes to it.variable }
}

private fun overallRange(aes: Aes<*>): DoubleSpan? {
val transformVar = DataFrameUtil.transformVarFor(aes)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import jetbrains.datalore.plot.config.*
import jetbrains.datalore.plot.config.Option.Meta.DATA_META
import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame.GDF
import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame.GEOMETRY
import jetbrains.datalore.plot.server.config.transform.bistro.util.layer

open class PlotConfigServerSide(
opts: Map<String, Any>
Expand Down Expand Up @@ -118,13 +119,12 @@ open class PlotConfigServerSide(
val dataByLayer: List<DataFrame> = layerConfigs.map { layer ->
DataProcessing.transformOriginals(layer.combinedData, layer.varBindings, transformByAes)
}
val statCtx = ConfiguredStatContext(dataByLayer, transformByAes)

return layerConfigs.mapIndexed { layerIndex, layerConfig ->
applyLayerStatistic(
layerConfig,
layerData = dataByLayer[layerIndex],
statCtx,
ConfiguredStatContext(dataByLayer, transformByAes, layerConfig.varBindings),
) { message ->
layerMessageHandler(layerIndex, message)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ class Summary {
'mapping': {'middle': '..median..'},
'stat': 'summary',
'fun_min': 'lq',
'fun_map': {'middle': 'median'},
'quantiles': [0.45, 0.5, 0.55]
}
]
Expand Down
16 changes: 3 additions & 13 deletions python-package/lets_plot/plot/stat.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# Copyright (c) 2023. JetBrains s.r.o.
# Use of this source code is governed by the MIT license that can be found in the LICENSE file.
from .core import aes
from .geom import _geom

#
Expand All @@ -12,29 +11,20 @@
def stat_summary(mapping=None, *, data=None, geom='pointrange',
position=None, show_legend=None, sampling=None, tooltips=None,
orientation=None,
fun=None, fun_min=None, fun_max=None, fun_map=None,
fun=None, fun_min=None, fun_max=None,
quantiles=None,
color_by=None, fill_by=None,
**other_args):
def fun_to_stat(fun_name):
prefix = "y" if fun_name in ["min", "max"] else ""
return "..{0}{1}..".format(prefix, fun_name)

mapping_dict = mapping.as_dict() if mapping is not None else {}
fun_mapping_dict = {aes_name: fun_to_stat(fun_name) for aes_name, fun_name in (fun_map or {}).items()}
inner_mapping_dict = {**fun_mapping_dict, **mapping_dict}
inner_mapping = aes(**inner_mapping_dict) if len(inner_mapping_dict.keys()) > 0 else None

return _geom(geom,
mapping=inner_mapping,
mapping=mapping,
data=data,
stat='summary',
position=position,
show_legend=show_legend,
sampling=sampling,
tooltips=tooltips,
orientation=orientation,
fun=fun, fun_min=fun_min, fun_max=fun_max, fun_map=fun_map,
fun=fun, fun_min=fun_min, fun_max=fun_max,
quantiles=quantiles,
color_by=color_by, fill_by=fill_by,
**other_args)