From b4b6935e38349b2ca60ebb405299437c8691d337 Mon Sep 17 00:00:00 2001 From: Ivan Kupriyanov Date: Fri, 29 May 2020 13:21:46 +0300 Subject: [PATCH 01/11] Remove map_id --- .../jetbrains/datalore/plot/base/Aes.kt | 13 +- .../datalore/plot/base/DataPointAesthetics.kt | 6 +- .../jetbrains/datalore/plot/base/GeomMeta.kt | 21 +- .../datalore/plot/base/aes/AesInitValue.kt | 6 +- .../datalore/plot/base/aes/AesVisitor.kt | 20 +- .../plot/base/aes/AestheticsBuilder.kt | 27 +- .../datalore/plot/base/data/TransformVar.kt | 16 +- .../geom/util/DataPointAestheticsDelegate.kt | 12 +- .../builder/scale/DefaultMapperProvider.kt | 39 +-- .../plot/builder/scale/DefaultNaValue.kt | 6 +- .../TooltipSpecFactorySkippedAesTest.kt | 11 - .../scale/DefaultMapperProviderTest.kt | 17 -- .../builder/scale/ScaleProviderHelperTest.kt | 3 +- .../plot/config/GeoPositionsDataUtil.kt | 219 +++++++--------- .../datalore/plot/config/LayerConfig.kt | 9 +- .../jetbrains/datalore/plot/config/Option.kt | 9 +- .../plot/config/PlotConfigClientSideUtil.kt | 5 +- .../config/aes/TypedOptionConverterMap.kt | 7 +- .../geo/GeometryFromGeoDataFrameChange.kt | 6 +- .../server/config/PlotConfigServerSide.kt | 17 +- .../transform/GeoDataFrameMappingChange.kt | 9 +- .../transform/GeoPositionMappingChange.kt | 11 +- .../LonLatSpecInMappingSpecChange.kt | 152 ----------- .../server/config/transform/MapJoinChange.kt | 4 +- .../PlotConfigServerSideTransforms.kt | 4 - .../GeomInteractionBuilderCreationTest.kt | 10 +- .../kotlin/plot/config/ConfigUtilTest.kt | 10 +- .../plot/server/config/DropUnusedDataTest.kt | 53 ++-- .../config/GeoDataFrameMappingChangeTest.kt | 9 +- .../plot/server/config/SingleLayerAssert.kt | 4 +- .../transform/LivemapLonLatOptionsTest.kt | 152 ----------- .../LonLatSpecInMappingSpecChangeTest.kt | 236 ------------------ .../LiveMapDataPointAestheticsProcessor.kt | 29 +-- .../datalore/plot/livemap/LiveMapUtil.kt | 16 +- .../plot/livemap/MultiDataPointHelper.kt | 10 +- .../MapIdAndGeoPointDataProcessingTest.kt | 6 +- .../plot/livemap/MultiDataPointHelperTest.kt | 6 +- .../plot/livemap/PointConverterTest.kt | 3 +- .../plot/livemap/PolygonConverterTest.kt | 4 +- 39 files changed, 299 insertions(+), 898 deletions(-) delete mode 100644 plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/LonLatSpecInMappingSpecChange.kt delete mode 100644 plot-config/src/jvmTest/kotlin/plot/server/config/transform/LivemapLonLatOptionsTest.kt delete mode 100644 plot-config/src/jvmTest/kotlin/plot/server/config/transform/LonLatSpecInMappingSpecChangeTest.kt diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/Aes.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/Aes.kt index 3dd0c1e12b5..6a2ae3017e0 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/Aes.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/Aes.kt @@ -34,10 +34,8 @@ class Aes private constructor(val name: String, val isNumeric: Boolean = true val COLOR: Aes = Aes("color", false) val FILL: Aes = Aes("fill", false) val ALPHA: Aes = Aes("alpha") - val SHAPE: Aes = - Aes("shape", false) - val LINETYPE: Aes = - Aes("linetype", false) + val SHAPE: Aes = Aes("shape", false) + val LINETYPE: Aes = Aes("linetype", false) val SIZE: Aes = Aes("size") val WIDTH: Aes = Aes("width") @@ -58,7 +56,6 @@ class Aes private constructor(val name: String, val isNumeric: Boolean = true val XEND: Aes = Aes("xend") val YEND: Aes = Aes("yend") - val MAP_ID: Aes = Aes("map_id", false) val FRAME: Aes = Aes("frame", false) val SPEED: Aes = Aes("speed") @@ -76,6 +73,9 @@ class Aes private constructor(val name: String, val isNumeric: Boolean = true val ANGLE: Aes = Aes("angle") + val SYM_X: Aes = Aes("sym_x") + val SYM_Y: Aes = Aes("sym_y") + fun numeric(unfiltered: Iterable>): Iterable> { // safe to cast all 'numeric' aesthetics are 'Double' @@ -135,8 +135,7 @@ class Aes private constructor(val name: String, val isNumeric: Boolean = true } fun noGuideNeeded(aes: Aes<*>): Boolean { - return aes == MAP_ID || - aes == FRAME || + return aes == FRAME || aes == SPEED || aes == FLOW || aes == LABEL || diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/DataPointAesthetics.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/DataPointAesthetics.kt index 67f45478078..af601d75bf5 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/DataPointAesthetics.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/DataPointAesthetics.kt @@ -54,8 +54,6 @@ interface DataPointAesthetics { fun upper(): Double? - fun mapId(): Any - fun frame(): String fun speed(): Double? @@ -82,6 +80,10 @@ interface DataPointAesthetics { fun angle(): Double? + fun symX(): Double? + + fun symY(): Double? + fun group(): Int? fun numeric(aes: Aes): Double? diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/GeomMeta.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/GeomMeta.kt index a4bb321d881..a6bed4564ef 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/GeomMeta.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/GeomMeta.kt @@ -29,8 +29,7 @@ object GeomMeta { Aes.COLOR, Aes.FILL, Aes.ALPHA, - Aes.SHAPE, - Aes.MAP_ID + Aes.SHAPE // strokeWidth ) @@ -40,7 +39,6 @@ object GeomMeta { Aes.LINETYPE, Aes.COLOR, Aes.ALPHA, - Aes.MAP_ID, Aes.SPEED, Aes.FLOW ) @@ -51,8 +49,7 @@ object GeomMeta { Aes.LINETYPE, Aes.COLOR, Aes.FILL, - Aes.ALPHA, - Aes.MAP_ID + Aes.ALPHA ) private val AREA = listOf( @@ -167,8 +164,7 @@ object GeomMeta { Aes.LINETYPE, Aes.COLOR, Aes.FILL, - Aes.ALPHA, - Aes.MAP_ID + Aes.ALPHA ) GeomKind.AB_LINE -> listOf( @@ -237,8 +233,7 @@ object GeomMeta { Aes.LINETYPE, Aes.COLOR, Aes.FILL, - Aes.ALPHA, - Aes.MAP_ID + Aes.ALPHA ) GeomKind.SEGMENT -> listOf( @@ -262,12 +257,10 @@ object GeomMeta { Aes.FONTFACE, Aes.HJUST, Aes.VJUST, - Aes.ANGLE, - Aes.MAP_ID + Aes.ANGLE ) GeomKind.LIVE_MAP -> listOf( // ToDo: not static - depends on 'display mode' - Aes.MAP_ID, Aes.ALPHA, Aes.COLOR, Aes.FILL, @@ -275,7 +268,9 @@ object GeomMeta { Aes.SHAPE, Aes.FRAME, Aes.X, - Aes.Y + Aes.Y, + Aes.SYM_X, + Aes.SYM_Y ) GeomKind.RASTER -> listOf( diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AesInitValue.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AesInitValue.kt index 5e0bf60c989..49dae5e8cf8 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AesInitValue.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AesInitValue.kt @@ -23,12 +23,13 @@ import jetbrains.datalore.plot.base.Aes.Companion.INTERCEPT import jetbrains.datalore.plot.base.Aes.Companion.LABEL import jetbrains.datalore.plot.base.Aes.Companion.LINETYPE import jetbrains.datalore.plot.base.Aes.Companion.LOWER -import jetbrains.datalore.plot.base.Aes.Companion.MAP_ID import jetbrains.datalore.plot.base.Aes.Companion.MIDDLE import jetbrains.datalore.plot.base.Aes.Companion.SHAPE import jetbrains.datalore.plot.base.Aes.Companion.SIZE import jetbrains.datalore.plot.base.Aes.Companion.SLOPE import jetbrains.datalore.plot.base.Aes.Companion.SPEED +import jetbrains.datalore.plot.base.Aes.Companion.SYM_X +import jetbrains.datalore.plot.base.Aes.Companion.SYM_Y import jetbrains.datalore.plot.base.Aes.Companion.UPPER import jetbrains.datalore.plot.base.Aes.Companion.VJUST import jetbrains.datalore.plot.base.Aes.Companion.WEIGHT @@ -73,7 +74,6 @@ object AesInitValue { VALUE_MAP[LOWER] = Double.NaN VALUE_MAP[MIDDLE] = Double.NaN VALUE_MAP[UPPER] = Double.NaN - VALUE_MAP[MAP_ID] = "empty map_id" VALUE_MAP[FRAME] = "empty frame" VALUE_MAP[SPEED] = 10.0 VALUE_MAP[FLOW] = 0.1 @@ -87,6 +87,8 @@ object AesInitValue { VALUE_MAP[HJUST] = 0.5 // 'middle' VALUE_MAP[VJUST] = 0.5 // 'middle' VALUE_MAP[ANGLE] = 0.0 + VALUE_MAP[SYM_X] = 0.0 + VALUE_MAP[SYM_Y] = 0.0 } /** diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AesVisitor.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AesVisitor.kt index 3cc7b74c7ea..ae3f007d981 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AesVisitor.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AesVisitor.kt @@ -20,12 +20,13 @@ import jetbrains.datalore.plot.base.Aes.Companion.INTERCEPT import jetbrains.datalore.plot.base.Aes.Companion.LABEL import jetbrains.datalore.plot.base.Aes.Companion.LINETYPE import jetbrains.datalore.plot.base.Aes.Companion.LOWER -import jetbrains.datalore.plot.base.Aes.Companion.MAP_ID import jetbrains.datalore.plot.base.Aes.Companion.MIDDLE import jetbrains.datalore.plot.base.Aes.Companion.SHAPE import jetbrains.datalore.plot.base.Aes.Companion.SIZE import jetbrains.datalore.plot.base.Aes.Companion.SLOPE import jetbrains.datalore.plot.base.Aes.Companion.SPEED +import jetbrains.datalore.plot.base.Aes.Companion.SYM_X +import jetbrains.datalore.plot.base.Aes.Companion.SYM_Y import jetbrains.datalore.plot.base.Aes.Companion.UPPER import jetbrains.datalore.plot.base.Aes.Companion.VJUST import jetbrains.datalore.plot.base.Aes.Companion.WEIGHT @@ -125,9 +126,6 @@ abstract class AesVisitor { if (aes == UPPER) { return upper() } - if (aes == MAP_ID) { - return mapId() - } if (aes == FRAME) { return frame() } @@ -168,6 +166,14 @@ abstract class AesVisitor { return angle() } + if (aes == SYM_X) { + return symX() + } + + if (aes == SYM_Y) { + return symY() + } + throw IllegalArgumentException("Unexpected aes: $aes") } @@ -213,8 +219,6 @@ abstract class AesVisitor { protected abstract fun upper(): T - protected abstract fun mapId(): T - protected abstract fun frame(): T protected abstract fun speed(): T @@ -240,4 +244,8 @@ abstract class AesVisitor { protected abstract fun vjust(): T protected abstract fun angle(): T + + protected abstract fun symX(): T + + protected abstract fun symY(): T } diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AestheticsBuilder.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AestheticsBuilder.kt index 2744bc8e0d8..b5d5e510867 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AestheticsBuilder.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AestheticsBuilder.kt @@ -26,12 +26,13 @@ import jetbrains.datalore.plot.base.Aes.Companion.INTERCEPT import jetbrains.datalore.plot.base.Aes.Companion.LABEL import jetbrains.datalore.plot.base.Aes.Companion.LINETYPE import jetbrains.datalore.plot.base.Aes.Companion.LOWER -import jetbrains.datalore.plot.base.Aes.Companion.MAP_ID import jetbrains.datalore.plot.base.Aes.Companion.MIDDLE import jetbrains.datalore.plot.base.Aes.Companion.SHAPE import jetbrains.datalore.plot.base.Aes.Companion.SIZE import jetbrains.datalore.plot.base.Aes.Companion.SLOPE import jetbrains.datalore.plot.base.Aes.Companion.SPEED +import jetbrains.datalore.plot.base.Aes.Companion.SYM_X +import jetbrains.datalore.plot.base.Aes.Companion.SYM_Y import jetbrains.datalore.plot.base.Aes.Companion.UPPER import jetbrains.datalore.plot.base.Aes.Companion.VJUST import jetbrains.datalore.plot.base.Aes.Companion.WEIGHT @@ -123,10 +124,6 @@ class AestheticsBuilder @JvmOverloads constructor(private var myDataPointCount: return aes(WEIGHT, v) } - fun mapId(v: (Int) -> Any?): AestheticsBuilder { - return aes(MAP_ID, v) - } - fun frame(v: (Int) -> String?): AestheticsBuilder { return aes(FRAME, v) } @@ -184,6 +181,14 @@ class AestheticsBuilder @JvmOverloads constructor(private var myDataPointCount: return aes(YMAX, v) } + fun symX(v: (Int) -> Double?): AestheticsBuilder { + return aes(SYM_X, v) + } + + fun symY(v: (Int) -> Double?): AestheticsBuilder { + return aes(SYM_Y, v) + } + fun constantAes(aes: Aes, v: T): AestheticsBuilder { myConstantAes.add(aes) myIndexFunctionMap[aes] = constant(v) @@ -438,10 +443,6 @@ class AestheticsBuilder @JvmOverloads constructor(private var myDataPointCount: return get(UPPER) } - override fun mapId(): Any { - return get(MAP_ID) - } - override fun frame(): String { return get(FRAME) } @@ -494,6 +495,14 @@ class AestheticsBuilder @JvmOverloads constructor(private var myDataPointCount: return get(ANGLE) } + override fun symX(): Double? { + return get(SYM_X) + } + + override fun symY(): Double? { + return get(SYM_Y) + } + override fun group(): Int? { return myAesthetics.group(myIndex!!) } diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/data/TransformVar.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/data/TransformVar.kt index 7f1926bb14a..aac66464658 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/data/TransformVar.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/data/TransformVar.kt @@ -33,7 +33,6 @@ object TransformVar { val LOWER = DataFrame.Variable("transform.LOWER", TRANSFORM) val MIDDLE = DataFrame.Variable("transform.MIDDLE", TRANSFORM) val UPPER = DataFrame.Variable("transform.UPPER", TRANSFORM) - val MAP_ID = DataFrame.Variable("transform.MAP_ID", TRANSFORM) val FRAME = DataFrame.Variable("transform.FRAME", TRANSFORM) val SPEED = DataFrame.Variable("transform.SPEED", TRANSFORM) val FLOW = DataFrame.Variable("transform.FLOW", TRANSFORM) @@ -47,6 +46,8 @@ object TransformVar { val HJUST = DataFrame.Variable("transform.HJUST", TRANSFORM) val VJUST = DataFrame.Variable("transform.VJUST", TRANSFORM) val ANGLE = DataFrame.Variable("transform.ANGLE", TRANSFORM) + val SYM_X = DataFrame.Variable("transform.SYM_X", TRANSFORM) + val SYM_Y = DataFrame.Variable("transform.SYM_Y", TRANSFORM) private val VAR_BY_AES = TransformVarByAes() private val VARS: Map @@ -160,11 +161,6 @@ object TransformVar { return UPPER } - - override fun mapId(): DataFrame.Variable { - return MAP_ID - } - override fun frame(): DataFrame.Variable { return FRAME } @@ -216,5 +212,13 @@ object TransformVar { override fun angle(): DataFrame.Variable { return ANGLE } + + override fun symX(): DataFrame.Variable { + return SYM_X + } + + override fun symY(): DataFrame.Variable { + return SYM_Y + } } } diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/util/DataPointAestheticsDelegate.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/util/DataPointAestheticsDelegate.kt index 6e59201b96b..e59a07d8dd3 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/util/DataPointAestheticsDelegate.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/util/DataPointAestheticsDelegate.kt @@ -102,10 +102,6 @@ open class DataPointAestheticsDelegate(private val p: DataPointAesthetics) : return p.upper() } - override fun mapId(): Any { - return p.mapId() - } - override fun frame(): String { return p.frame() } @@ -158,6 +154,14 @@ open class DataPointAestheticsDelegate(private val p: DataPointAesthetics) : return p.angle() } + override fun symX(): Double? { + return p.symX() + } + + override fun symY(): Double? { + return p.symY() + } + override fun group(): Int? { return p.group() } diff --git a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/scale/DefaultMapperProvider.kt b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/scale/DefaultMapperProvider.kt index e509e7bc51b..0db96aa7431 100644 --- a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/scale/DefaultMapperProvider.kt +++ b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/scale/DefaultMapperProvider.kt @@ -20,12 +20,13 @@ import jetbrains.datalore.plot.base.Aes.Companion.INTERCEPT import jetbrains.datalore.plot.base.Aes.Companion.LABEL import jetbrains.datalore.plot.base.Aes.Companion.LINETYPE import jetbrains.datalore.plot.base.Aes.Companion.LOWER -import jetbrains.datalore.plot.base.Aes.Companion.MAP_ID import jetbrains.datalore.plot.base.Aes.Companion.MIDDLE import jetbrains.datalore.plot.base.Aes.Companion.SHAPE import jetbrains.datalore.plot.base.Aes.Companion.SIZE import jetbrains.datalore.plot.base.Aes.Companion.SLOPE import jetbrains.datalore.plot.base.Aes.Companion.SPEED +import jetbrains.datalore.plot.base.Aes.Companion.SYM_X +import jetbrains.datalore.plot.base.Aes.Companion.SYM_Y import jetbrains.datalore.plot.base.Aes.Companion.UPPER import jetbrains.datalore.plot.base.Aes.Companion.VJUST import jetbrains.datalore.plot.base.Aes.Companion.WEIGHT @@ -74,17 +75,13 @@ object DefaultMapperProvider { init { for (aes in Aes.allPositional()) { - put(aes, - NUMERIC_UNDEFINED - ) + put(aes, NUMERIC_UNDEFINED) } this.put(X, NUMERIC_IDENTITY) this.put(Y, NUMERIC_IDENTITY) - this.put(Z, - NUMERIC_IDENTITY - ) + this.put(Z, NUMERIC_IDENTITY) this.put(YMIN, NUMERIC_IDENTITY) this.put(YMAX, NUMERIC_IDENTITY) this.put(COLOR, createColor()) @@ -94,15 +91,9 @@ object DefaultMapperProvider { this.put(LINETYPE, createWithDiscreteOutput(LineTypeMapper.allLineTypes(), LineTypeMapper.NA_VALUE)) this.put(SIZE, SizeMapperProvider.DEFAULT) - this.put(WIDTH, - NUMERIC_IDENTITY - ) - this.put(HEIGHT, - NUMERIC_IDENTITY - ) - this.put(WEIGHT, - NUMERIC_IDENTITY - ) + this.put(WIDTH, NUMERIC_IDENTITY) + this.put(HEIGHT, NUMERIC_IDENTITY) + this.put(WEIGHT, NUMERIC_IDENTITY) this.put(INTERCEPT, NUMERIC_IDENTITY) this.put(SLOPE, NUMERIC_IDENTITY) this.put(XINTERCEPT, NUMERIC_IDENTITY) @@ -111,15 +102,10 @@ object DefaultMapperProvider { this.put(MIDDLE, NUMERIC_IDENTITY) this.put(UPPER, NUMERIC_IDENTITY) - this.put(MAP_ID, createObjectIdentityDiscrete(MAP_ID)) this.put(FRAME, createStringIdentity(FRAME)) - this.put(SPEED, - NUMERIC_IDENTITY - ) - this.put(FLOW, - NUMERIC_IDENTITY - ) + this.put(SPEED, NUMERIC_IDENTITY) + this.put(FLOW, NUMERIC_IDENTITY) this.put(XMIN, NUMERIC_IDENTITY) this.put(XMAX, NUMERIC_IDENTITY) @@ -135,9 +121,10 @@ object DefaultMapperProvider { // text vertical justification (numbers [0..1] or predefined strings, not positional) this.put(VJUST, createObjectIdentityDiscrete(VJUST)) - this.put(ANGLE, - NUMERIC_IDENTITY - ) + this.put(ANGLE, NUMERIC_IDENTITY) + + this.put(SYM_X, NUMERIC_IDENTITY) + this.put(SYM_Y, NUMERIC_IDENTITY) } internal operator fun get(aes: Aes): MapperProvider { diff --git a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/scale/DefaultNaValue.kt b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/scale/DefaultNaValue.kt index 1e93843759e..e1fe60ae2a1 100644 --- a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/scale/DefaultNaValue.kt +++ b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/scale/DefaultNaValue.kt @@ -23,12 +23,13 @@ import jetbrains.datalore.plot.base.Aes.Companion.INTERCEPT import jetbrains.datalore.plot.base.Aes.Companion.LABEL import jetbrains.datalore.plot.base.Aes.Companion.LINETYPE import jetbrains.datalore.plot.base.Aes.Companion.LOWER -import jetbrains.datalore.plot.base.Aes.Companion.MAP_ID import jetbrains.datalore.plot.base.Aes.Companion.MIDDLE import jetbrains.datalore.plot.base.Aes.Companion.SHAPE import jetbrains.datalore.plot.base.Aes.Companion.SIZE import jetbrains.datalore.plot.base.Aes.Companion.SLOPE import jetbrains.datalore.plot.base.Aes.Companion.SPEED +import jetbrains.datalore.plot.base.Aes.Companion.SYM_X +import jetbrains.datalore.plot.base.Aes.Companion.SYM_Y import jetbrains.datalore.plot.base.Aes.Companion.UPPER import jetbrains.datalore.plot.base.Aes.Companion.VJUST import jetbrains.datalore.plot.base.Aes.Companion.WEIGHT @@ -73,7 +74,6 @@ object DefaultNaValue { VALUE_MAP.put(LOWER, 0.0) VALUE_MAP.put(MIDDLE, 0.0) VALUE_MAP.put(UPPER, 0.0) - VALUE_MAP.put(MAP_ID, "empty map_id") VALUE_MAP.put(FRAME, "empty frame") VALUE_MAP.put(SPEED, 10.0) VALUE_MAP.put(FLOW, 0.1) @@ -87,6 +87,8 @@ object DefaultNaValue { VALUE_MAP.put(HJUST, 0.5) // 'middle' VALUE_MAP.put(VJUST, 0.5) // 'middle' VALUE_MAP.put(ANGLE, 0.0) + VALUE_MAP.put(SYM_X, 0.0) + VALUE_MAP.put(SYM_Y, 0.0) } /** diff --git a/plot-builder-portable/src/jvmTest/kotlin/plot/builder/interact/TooltipSpecFactorySkippedAesTest.kt b/plot-builder-portable/src/jvmTest/kotlin/plot/builder/interact/TooltipSpecFactorySkippedAesTest.kt index 0ccdee463d7..c345460d534 100644 --- a/plot-builder-portable/src/jvmTest/kotlin/plot/builder/interact/TooltipSpecFactorySkippedAesTest.kt +++ b/plot-builder-portable/src/jvmTest/kotlin/plot/builder/interact/TooltipSpecFactorySkippedAesTest.kt @@ -34,15 +34,4 @@ class TooltipSpecFactorySkippedAesTest : jetbrains.datalore.plot.builder.interac assertTooltipsCount(1) assertLines(0, widthMapping.longTooltipText()) } - - @Test - fun shouldNotSkipMapIdMappingVar_whenOtherMappingWithSameVarExists() { - val commonLabel = "foo" - addMappedData(variable().name(commonLabel).mapping(Aes.MAP_ID)) - addMappedData(variable().name(commonLabel).mapping(Aes.FILL)) - - buildTooltipSpecs() - - assertTooltipsCount(1) - } } diff --git a/plot-builder-portable/src/jvmTest/kotlin/plot/builder/scale/DefaultMapperProviderTest.kt b/plot-builder-portable/src/jvmTest/kotlin/plot/builder/scale/DefaultMapperProviderTest.kt index d9c1755d29a..7477fdc33b8 100644 --- a/plot-builder-portable/src/jvmTest/kotlin/plot/builder/scale/DefaultMapperProviderTest.kt +++ b/plot-builder-portable/src/jvmTest/kotlin/plot/builder/scale/DefaultMapperProviderTest.kt @@ -6,27 +6,10 @@ package jetbrains.datalore.plot.builder.scale import jetbrains.datalore.plot.base.Aes -import jetbrains.datalore.plot.base.data.DataFrameUtil import kotlin.test.Test -import kotlin.test.assertEquals import kotlin.test.assertTrue class DefaultMapperProviderTest { - @Test - fun mapId() { - val serie = listOf("a", "a", "b", "c") - val d = mapOf("var" to serie) - val df = DataFrameUtil.fromMap(d) - - val variable = df.variables().iterator().next() - - val mapperProvider = DefaultMapperProvider[Aes.MAP_ID] - val mapper = mapperProvider.createDiscreteMapper(df, variable) - - val mapped = arrayOf(0.0, 0.0, 1.0, 2.0).map { v -> mapper.apply(v) as String } - assertEquals(serie, mapped) - } - @Test fun everyAesHasMapperProvider() { for (aes in Aes.values()) { diff --git a/plot-builder-portable/src/jvmTest/kotlin/plot/builder/scale/ScaleProviderHelperTest.kt b/plot-builder-portable/src/jvmTest/kotlin/plot/builder/scale/ScaleProviderHelperTest.kt index 9d02ba89c6e..5677d9a153b 100644 --- a/plot-builder-portable/src/jvmTest/kotlin/plot/builder/scale/ScaleProviderHelperTest.kt +++ b/plot-builder-portable/src/jvmTest/kotlin/plot/builder/scale/ScaleProviderHelperTest.kt @@ -7,7 +7,6 @@ package jetbrains.datalore.plot.builder.scale import jetbrains.datalore.plot.base.Aes import jetbrains.datalore.plot.base.DataFrame -import org.junit.Ignore import kotlin.test.Test class ScaleProviderHelperTest { @@ -19,6 +18,6 @@ class ScaleProviderHelperTest { .put(region, listOf("Europe", "Asia", null, "Australia")) .build() - ScaleProviderHelper.createDefault(Aes.MAP_ID).createScale(df, region) + ScaleProviderHelper.createDefault(Aes.HJUST).createScale(df, region) } } diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoPositionsDataUtil.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoPositionsDataUtil.kt index 577b67fe067..2d9b6e977fa 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoPositionsDataUtil.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoPositionsDataUtil.kt @@ -5,7 +5,6 @@ package jetbrains.datalore.plot.config -import jetbrains.datalore.base.gcommon.base.Preconditions.checkState import jetbrains.datalore.base.values.Pair import jetbrains.datalore.plot.base.Aes import jetbrains.datalore.plot.base.DataFrame @@ -13,7 +12,8 @@ import jetbrains.datalore.plot.base.DataFrame.Variable import jetbrains.datalore.plot.base.GeomKind import jetbrains.datalore.plot.base.data.DataFrameUtil import jetbrains.datalore.plot.builder.map.GeoPositionField -import jetbrains.datalore.plot.config.Option.Geom.Choropleth +import jetbrains.datalore.plot.config.GeoPositionsDataUtil.GeoDataKind.* +import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS import jetbrains.datalore.plot.config.Option.Meta.MapJoin object GeoPositionsDataUtil { @@ -22,69 +22,54 @@ object GeoPositionsDataUtil { const val MAP_OSM_ID_COLUMN = "__geoid__" - private val GEOMS_SUPPORT = mapOf( - GeomKind.MAP to GeoDataSupport.boundary(), - GeomKind.POLYGON to GeoDataSupport.boundary(), - GeomKind.POINT to GeoDataSupport.point(), - GeomKind.RECT to GeoDataSupport.bbox(), - GeomKind.PATH to GeoDataSupport.path(), - GeomKind.TEXT to GeoDataSupport.point() + val GEOMS_SUPPORT = mapOf( + GeomKind.MAP to GeoDataSupport(BOUNDARY, ::createPointMapping), + GeomKind.POLYGON to GeoDataSupport(BOUNDARY, ::createPointMapping), + GeomKind.POINT to GeoDataSupport(POINT, ::createPointMapping), + GeomKind.RECT to GeoDataSupport(BBOX, ::createRectMapping), + GeomKind.PATH to GeoDataSupport(PATH, ::createPointMapping), + GeomKind.TEXT to GeoDataSupport(POINT, ::createPointMapping) ) fun isGeomSupported(geomKind: GeomKind): Boolean { - return GEOMS_SUPPORT.containsKey(geomKind) + return geomKind in GEOMS_SUPPORT } fun getGeoDataKind(geomKind: GeomKind): GeoDataKind { return GEOMS_SUPPORT[geomKind]!!.geoDataKind } - internal fun hasGeoPositionsData(layerConfig: LayerConfig): Boolean { - return layerConfig.has(Choropleth.GEO_POSITIONS) - } - internal fun getGeoPositionsData(layerConfig: LayerConfig): DataFrame { - val mapOptions = layerConfig.getMap(Choropleth.GEO_POSITIONS) - return ConfigUtil.createDataFrame(mapOptions) + return ConfigUtil.createDataFrame(layerConfig.getMap(GEO_POSITIONS)) } internal fun initDataAndMappingForGeoPositions( - geomKind: GeomKind, layerData: DataFrame, mapOptions: DataFrame, mappingOptions: Map<*, *>): Pair, Variable>> { + geomKind: GeomKind, + layerData: DataFrame, + mapOptions: DataFrame, + mappingOptions: Map<*, *>, + leftMapId: String, + rightMapId: String + ): Pair, Variable>> { @Suppress("NAME_SHADOWING") var layerData = layerData - val leftMapId = mappingOptions[Option.Mapping.MAP_ID] - checkState(leftMapId != null || mappingOptions.isEmpty(), "'map_join' parameter is required if 'map' parameter is used") - - return when { - leftMapId != null -> { - val rightMapId = getGeoPositionsIdVar(mapOptions).name - layerData = ConfigUtil.rightJoin( - layerData, - leftMapId.toString(), - mapOptions, - rightMapId - ) + //val rightMapId = getGeoPositionsIdVar(mapOptions).name + layerData = ConfigUtil.rightJoin(layerData, leftMapId, mapOptions, rightMapId) - val aesMapping = HashMap( - ConfigUtil.createAesMapping( - layerData, - mappingOptions - ) - ) - aesMapping.putAll( - generateMappings( - geomKind, - layerData - ) - ) - Pair(layerData, aesMapping) - } - else -> // just show a blank map - Pair(mapOptions, - generateMappings(geomKind, mapOptions) - ) - } + val aesMapping = HashMap( + ConfigUtil.createAesMapping( + layerData, + mappingOptions + ) + ) + aesMapping.putAll( + generateMappings( + geomKind, + layerData + ) + ) + return Pair(layerData, aesMapping) } private fun generateMappings(geomKind: GeomKind, layerData: DataFrame): Map, Variable> { @@ -96,7 +81,7 @@ object GeoPositionsDataUtil { } private fun getGeoPositionsIdVar(mapOptions: DataFrame): Variable { - val variable = findFirstVariable(mapOptions, listOf(MapJoin.ID, MAP_REGION_COLUMN)) + val variable = findFirstVariable(mapOptions, listOf(MapJoin.MAP_ID, MAP_REGION_COLUMN)) if (variable != null) { return variable } @@ -104,7 +89,7 @@ object GeoPositionsDataUtil { throw IllegalArgumentException( geoPositionsColumnNotFoundError( "region id", - listOf(MapJoin.ID, MAP_REGION_COLUMN) + listOf(MapJoin.MAP_ID, MAP_REGION_COLUMN) ) ) } @@ -142,85 +127,69 @@ object GeoPositionsDataUtil { BOUNDARY } - internal class GeoDataSupport(val geoDataKind: GeoDataKind, private val mappingsGenerator: (DataFrame) -> Map, Variable>) { - - fun generateMapping(df: DataFrame): Map, Variable> { - return mappingsGenerator(df) - } - - companion object { - fun boundary(): GeoDataSupport { - return GeoDataSupport(GeoDataKind.BOUNDARY, ::createPointMapping) - } - - fun point(): GeoDataSupport { - return GeoDataSupport(GeoDataKind.POINT, ::createPointMapping) - } + private fun createRectMapping(dataFrame: DataFrame): Map, Variable> { + val mapping = HashMap, Variable>() + mapping.putAll( + findMapping( + Aes.XMIN, + listOf(GeoPositionField.RECT_XMIN), + dataFrame + ) + ) + mapping.putAll( + findMapping( + Aes.XMAX, + listOf(GeoPositionField.RECT_XMAX), + dataFrame + ) + ) + mapping.putAll( + findMapping( + Aes.YMIN, + listOf(GeoPositionField.RECT_YMIN), + dataFrame + ) + ) + mapping.putAll( + findMapping( + Aes.YMAX, + listOf(GeoPositionField.RECT_YMAX), + dataFrame + ) + ) + return mapping + } - fun path(): GeoDataSupport { - return GeoDataSupport(GeoDataKind.PATH, ::createPointMapping) - } + private fun createPointMapping(dataFrame: DataFrame): Map, Variable> { + val mapping = HashMap, Variable>() + mapping.putAll( + findMapping( + Aes.X, + listOf( + GeoPositionField.POINT_X, + "x", + GeoPositionField.POINT_X2 + ), + dataFrame + ) + ) + mapping.putAll( + findMapping( + Aes.Y, + listOf(GeoPositionField.POINT_Y, "y"), + dataFrame + ) + ) - fun bbox(): GeoDataSupport { - return GeoDataSupport(GeoDataKind.BBOX, ::createRectMapping) - } + return mapping + } - private fun createRectMapping(dataFrame: DataFrame): Map, Variable> { - val mapping = HashMap, Variable>() - mapping.putAll( - findMapping( - Aes.XMIN, - listOf(GeoPositionField.RECT_XMIN), - dataFrame - ) - ) - mapping.putAll( - findMapping( - Aes.XMAX, - listOf(GeoPositionField.RECT_XMAX), - dataFrame - ) - ) - mapping.putAll( - findMapping( - Aes.YMIN, - listOf(GeoPositionField.RECT_YMIN), - dataFrame - ) - ) - mapping.putAll( - findMapping( - Aes.YMAX, - listOf(GeoPositionField.RECT_YMAX), - dataFrame - ) - ) - return mapping - } + class GeoDataSupport( + val geoDataKind: GeoDataKind, + private val mappingsGenerator: (DataFrame) -> Map, Variable> + ) { - private fun createPointMapping(dataFrame: DataFrame): Map, Variable> { - val mapping = HashMap, Variable>() - mapping.putAll( - findMapping( - Aes.X, - listOf( - GeoPositionField.POINT_X, - "x", - GeoPositionField.POINT_X2 - ), - dataFrame - ) - ) - mapping.putAll( - findMapping( - Aes.Y, - listOf(GeoPositionField.POINT_Y, "y"), - dataFrame - ) - ) + fun generateMapping(df: DataFrame): Map, Variable> = mappingsGenerator(df) - return mapping - } - } } } diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/LayerConfig.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/LayerConfig.kt index e7ec3fa5e16..f2149376e1e 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/LayerConfig.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/LayerConfig.kt @@ -19,6 +19,7 @@ import jetbrains.datalore.plot.builder.assemble.TypedScaleProviderMap import jetbrains.datalore.plot.builder.sampling.Sampling import jetbrains.datalore.plot.builder.tooltip.TooltipLineSpecification import jetbrains.datalore.plot.config.DataMetaUtil.createDataFrame +import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS import jetbrains.datalore.plot.config.Option.Layer.GEOM import jetbrains.datalore.plot.config.Option.Layer.SHOW_LEGEND import jetbrains.datalore.plot.config.Option.Layer.STAT @@ -96,13 +97,15 @@ class LayerConfig( var aesMappings: Map, DataFrame.Variable>? - if (GeoPositionsDataUtil.hasGeoPositionsData(this) && myClientSide) { + if (has(GEO_POSITIONS) && myClientSide) { // join dataset and geo-positions data val dataAndMapping = GeoPositionsDataUtil.initDataAndMappingForGeoPositions( geomProto.geomKind, combinedData, GeoPositionsDataUtil.getGeoPositionsData(this), - combinedMappings + combinedMappings, + getString(MapJoin.DATA_JOIN_COLUMN)!!, + getString(MapJoin.MAP_JOIN_COLUMN)!! ) combinedData = dataAndMapping.first aesMappings = dataAndMapping.second @@ -166,7 +169,7 @@ class LayerConfig( else null - if (fieldName == null && GeoPositionsDataUtil.hasGeoPositionsData(this)) { + if (fieldName == null && has(GEO_POSITIONS)) { // 'default' group is important for 'geom_map' val groupVar = variables(data)["group"] if (groupVar != null) { diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/Option.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/Option.kt index dccf746b106..ac23128f106 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/Option.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/Option.kt @@ -48,9 +48,13 @@ object Option { } object MapJoin { - // Column with keys used for join - const val ID = "__id__" const val MAP_JOIN_COLUMN = "__map_join_column__" + const val DATA_JOIN_COLUMN = "__data_join_column__" + + // column in map used for join with data + const val MAP_ID = "__map_id__" + // generated data column for joined geometries from map + const val DATA_ID = "__data_id__" } object MappingAnnotation { @@ -236,7 +240,6 @@ object Option { object Mapping { const val GROUP = "group" - val MAP_ID = toOption(Aes.MAP_ID) // map_id is 'aes' but also used as option in geom_map() private val AES_BY_OPTION = HashMap>() val REAL_AES_OPTION_NAMES: Iterable = AES_BY_OPTION.keys diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSideUtil.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSideUtil.kt index 8332f84f2ec..77f87a679e9 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSideUtil.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSideUtil.kt @@ -251,10 +251,7 @@ object PlotConfigClientSideUtil { } // remove auto generated mappings - aesListForTooltip.removeAll { layerConfig.getScaleForAes(it)?.name == MapJoin.ID } - - // remove map_id mapping - aesListForTooltip.removeAll { it === Aes.MAP_ID } + aesListForTooltip.removeAll { layerConfig.getScaleForAes(it)?.name == MapJoin.MAP_ID } return aesListForTooltip } diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/aes/TypedOptionConverterMap.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/aes/TypedOptionConverterMap.kt index 329360bd8f2..1a0c7635204 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/aes/TypedOptionConverterMap.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/aes/TypedOptionConverterMap.kt @@ -20,12 +20,13 @@ import jetbrains.datalore.plot.base.Aes.Companion.INTERCEPT import jetbrains.datalore.plot.base.Aes.Companion.LABEL import jetbrains.datalore.plot.base.Aes.Companion.LINETYPE import jetbrains.datalore.plot.base.Aes.Companion.LOWER -import jetbrains.datalore.plot.base.Aes.Companion.MAP_ID import jetbrains.datalore.plot.base.Aes.Companion.MIDDLE import jetbrains.datalore.plot.base.Aes.Companion.SHAPE import jetbrains.datalore.plot.base.Aes.Companion.SIZE import jetbrains.datalore.plot.base.Aes.Companion.SLOPE import jetbrains.datalore.plot.base.Aes.Companion.SPEED +import jetbrains.datalore.plot.base.Aes.Companion.SYM_X +import jetbrains.datalore.plot.base.Aes.Companion.SYM_Y import jetbrains.datalore.plot.base.Aes.Companion.UPPER import jetbrains.datalore.plot.base.Aes.Companion.VJUST import jetbrains.datalore.plot.base.Aes.Companion.WEIGHT @@ -71,7 +72,6 @@ internal class TypedOptionConverterMap { this.put(MIDDLE, DOUBLE_CVT) this.put(UPPER, DOUBLE_CVT) - this.put(MAP_ID, IDENTITY_O_CVT) this.put(FRAME, IDENTITY_S_CVT) this.put(SPEED, DOUBLE_CVT) @@ -88,6 +88,9 @@ internal class TypedOptionConverterMap { this.put(HJUST, IDENTITY_O_CVT) // text horizontal justification (numbers [0..1] or predefined strings, DOUBLE_CVT; not positional) this.put(VJUST, IDENTITY_O_CVT) // text vertical justification (numbers [0..1] or predefined strings, not positional) this.put(ANGLE, DOUBLE_CVT) + + this.put(SYM_X, DOUBLE_CVT) + this.put(SYM_Y, DOUBLE_CVT) } private fun put(aes: Aes, value: (Any?) -> T?) { diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeometryFromGeoDataFrameChange.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeometryFromGeoDataFrameChange.kt index 71fdf64f76d..e24f0fd2787 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeometryFromGeoDataFrameChange.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeometryFromGeoDataFrameChange.kt @@ -40,12 +40,12 @@ class GeometryFromGeoDataFrameChange : GeometryFromGeoPositionsChange() { ) } - val dataTable = mapSpec.getList(MapJoin.ID) + val dataTable = mapSpec.getList(MapJoin.MAP_ID) ?.zip(geometryTables) - ?.fold(mutableMapOf>(), { dataFrame, (id, geometryTable) -> + ?.fold(mutableMapOf>(), { dataFrame, (mapId, geometryTable) -> dataFrame .concat(geometryTable) - .concat(MapJoin.ID, MutableList(geometryTable.rowCount) { id!! }) + .concat(MapJoin.DATA_ID, MutableList(geometryTable.rowCount) { mapId!! }) }) ?: geometryTables.fold(mutableMapOf>(), { dataFrame, geometryTable -> dataFrame.concat(geometryTable) diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/PlotConfigServerSide.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/PlotConfigServerSide.kt index f0c429b2f26..485a51ccec0 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/PlotConfigServerSide.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/PlotConfigServerSide.kt @@ -16,6 +16,7 @@ import jetbrains.datalore.plot.builder.data.GroupingContext import jetbrains.datalore.plot.builder.tooltip.TooltipLineSpecification import jetbrains.datalore.plot.builder.tooltip.VariableValue import jetbrains.datalore.plot.config.* +import jetbrains.datalore.plot.config.Option.Layer.MAP_JOIN import jetbrains.datalore.plot.server.config.transform.PlotConfigServerSideTransforms.entryTransform import jetbrains.datalore.plot.server.config.transform.PlotConfigServerSideTransforms.migrationTransform @@ -185,17 +186,11 @@ open class PlotConfigServerSide(opts: Map) : PlotConfig(opts) { varsToKeep.removeAll(notRenderedVars) varsToKeep.addAll(renderedVars) - val varNamesToKeep = HashSet() - for (`var` in varsToKeep) { - varNamesToKeep.add(`var`.name) - } - varNamesToKeep.add(Stats.GROUP.name) - facets.xVar?.let(varNamesToKeep::add) - facets.yVar?.let(varNamesToKeep::add) - - if (layerConfig.hasExplicitGrouping()) { - varNamesToKeep.add(layerConfig.explicitGroupingVarName!!) - } + val varNamesToKeep = HashSet() + + varsToKeep.map(Variable::name) + + Stats.GROUP.name + + listOfNotNull(layerConfig.mergedOptions.getList(MAP_JOIN)?.get(0) as? String) + + listOfNotNull(facets.xVar, facets.yVar, layerConfig.explicitGroupingVarName) layerData = DataFrameUtil.removeAllExcept(layerData!!, varNamesToKeep) layerConfig.replaceOwnData(layerData) diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoDataFrameMappingChange.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoDataFrameMappingChange.kt index 41e2709e5d9..2f37b07510a 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoDataFrameMappingChange.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoDataFrameMappingChange.kt @@ -7,14 +7,12 @@ package jetbrains.datalore.plot.server.config.transform import jetbrains.datalore.plot.config.* import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS -import jetbrains.datalore.plot.config.Option.Mapping.MAP_ID import jetbrains.datalore.plot.config.Option.Meta.DATA_META import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame.GEOMETRY_COLUMN_NAME import jetbrains.datalore.plot.config.Option.Meta.MapJoin import jetbrains.datalore.plot.config.Option.Plot import jetbrains.datalore.plot.config.Option.PlotBase.DATA -import jetbrains.datalore.plot.config.Option.PlotBase.MAPPING import jetbrains.datalore.plot.config.transform.SpecChange import jetbrains.datalore.plot.config.transform.SpecChangeContext import jetbrains.datalore.plot.config.transform.SpecSelector @@ -27,10 +25,11 @@ class GeoDataFrameMappingChange : SpecChange { val ids = geometries.indices.map(Int::toString) spec.remove(DATA, geometryColumnName) - spec.write(DATA, MapJoin.ID) { ids } - spec.write(GEO_POSITIONS, MapJoin.ID) { ids } + spec.write(DATA, MapJoin.MAP_ID) { ids } + spec.write(GEO_POSITIONS, MapJoin.MAP_ID) { ids } spec.write(GEO_POSITIONS, GeoDataFrame.GEOMETRIES) { geometries} - spec.write(MAPPING, MAP_ID) { MapJoin.ID } + // TODO: Fix MAP_ID + //spec.write(MAPPING, MAP_ID) { MapJoin.ID } } override fun isApplicable(spec: Map): Boolean { diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoPositionMappingChange.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoPositionMappingChange.kt index 76d0ff0f98e..02847f51b2b 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoPositionMappingChange.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoPositionMappingChange.kt @@ -5,7 +5,6 @@ package jetbrains.datalore.plot.server.config.transform -import jetbrains.datalore.plot.config.* import jetbrains.datalore.plot.config.GeoPositionsDataUtil.MAP_OSM_ID_COLUMN import jetbrains.datalore.plot.config.GeoPositionsDataUtil.MAP_REGION_COLUMN import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS @@ -16,9 +15,13 @@ import jetbrains.datalore.plot.config.Option.Meta.GeoReference import jetbrains.datalore.plot.config.Option.Meta.MAP_DATA_META import jetbrains.datalore.plot.config.Option.Meta.MapJoin import jetbrains.datalore.plot.config.Option.Plot +import jetbrains.datalore.plot.config.getMap +import jetbrains.datalore.plot.config.has +import jetbrains.datalore.plot.config.read import jetbrains.datalore.plot.config.transform.SpecChange import jetbrains.datalore.plot.config.transform.SpecChangeContext import jetbrains.datalore.plot.config.transform.SpecSelector +import jetbrains.datalore.plot.config.write class GeoPositionMappingChange : SpecChange { @@ -32,7 +35,7 @@ class GeoPositionMappingChange : SpecChange { mapJoinIds, // user defined column via parameter `map_join` mapSpec.read(GeoReference.REQUEST), // Regions object from our geocoding mapSpec.read(MAP_REGION_COLUMN) // ??? - ).firstOrNull()?.let { mapSpec.write(MapJoin.ID) { it } } + ).firstOrNull()?.let { mapSpec.write(MapJoin.MAP_ID) { it } } spec.read(MAP_DATA_META, GeoDataFrame.TAG, GEOMETRY_COLUMN_NAME) ?.let { geometryColumnName -> mapSpec.read(geometryColumnName as String)!! } @@ -40,12 +43,12 @@ class GeoPositionMappingChange : SpecChange { } if (spec.has(MAP_DATA_META, GeoReference.TAG)) { - mapSpec.write(MapJoin.ID) { mapSpec.read(GeoReference.REQUEST)!! } + mapSpec.write(MapJoin.MAP_ID) { mapSpec.read(GeoReference.REQUEST)!! } mapSpec.write(MAP_OSM_ID_COLUMN) { mapSpec.read(GeoReference.OSM_ID)!! } } if (spec.has(MAP_DATA_META, GeoDict.TAG)) { - mapJoinIds?.let { mapSpec.write(MapJoin.ID) { it } } + mapJoinIds?.let { mapSpec.write(MapJoin.MAP_ID) { it } } } } diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/LonLatSpecInMappingSpecChange.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/LonLatSpecInMappingSpecChange.kt deleted file mode 100644 index 4737952c4bb..00000000000 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/LonLatSpecInMappingSpecChange.kt +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (c) 2019. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.server.config.transform - -import jetbrains.datalore.base.gcommon.base.Preconditions.checkArgument -import jetbrains.datalore.base.spatial.MercatorUtils.checkLat -import jetbrains.datalore.base.spatial.MercatorUtils.checkLon -import jetbrains.datalore.plot.config.Option.GeomName -import jetbrains.datalore.plot.config.Option.Layer.GEOM -import jetbrains.datalore.plot.config.Option.Mapping -import jetbrains.datalore.plot.config.Option.Plot -import jetbrains.datalore.plot.config.Option.PlotBase.DATA -import jetbrains.datalore.plot.config.Option.PlotBase.MAPPING -import jetbrains.datalore.plot.config.transform.SpecChange -import jetbrains.datalore.plot.config.transform.SpecChangeContext -import jetbrains.datalore.plot.config.transform.SpecFinder -import jetbrains.datalore.plot.config.transform.SpecSelector - -class LonLatSpecInMappingSpecChange : SpecChange { - - override fun isApplicable(spec: Map): Boolean { - return GeomName.LIVE_MAP == spec[GEOM] && - spec.containsKey(MAPPING) && - spec[MAPPING] is Map<*, *> - } - - override fun apply(spec: MutableMap, ctx: SpecChangeContext) { - //System.out.println("Finding mapping....."); - // find: aes(... , map_id = lon_lat(lon = 'x', lat = 'y') ) - val finder = SpecFinder(MAPPING, Mapping.MAP_ID) - val specs = finder.findSpecs(spec) - if (specs.isEmpty()) { - //System.out.println("Not found"); - return - } - - val mapIdMapping = specs[0] - //System.out.println("Found " + mapIdMapping); - - // check if the value is: lon_lat(lon = 'x', lat = 'y') - if (mapIdMapping.containsKey(LONLAT_SPEC_KEY) - && LONLAT_SPEC_VALUE == mapIdMapping[LONLAT_SPEC_KEY] - && mapIdMapping.containsKey(LON_KEY) - && mapIdMapping.containsKey(LAT_KEY) - ) { - - val lonDataKey = mapIdMapping[LON_KEY].toString() - val latDataKey = mapIdMapping[LAT_KEY].toString() - val keys = listOf(lonDataKey, latDataKey) - - // Find data spec containing both: lon and lat data keys - var dataSpec: MutableMap? = null - if (spec[DATA] is Map<*, *>) { - val layerDataSpec = spec[DATA] as Map<*, *> - if (layerDataSpec.keys.containsAll(keys)) { - @Suppress("UNCHECKED_CAST") - dataSpec = layerDataSpec as MutableMap - } - } - - if (dataSpec == null) { - val list = ctx.getSpecsAbsolute(DATA) - if (list.isNotEmpty() && list[0].keys.containsAll(keys)) { - dataSpec = list[0] as MutableMap - } - } - - //System.out.println("data found: " + dataSpec); - if (dataSpec == null) { - throw IllegalArgumentException("Can not find data containing keys: $lonDataKey and $latDataKey") - } - - val coords = - concatColumns( - dataSpec[lonDataKey] as List<*>, - dataSpec[latDataKey] as List<*> - ) - - // add 'lon/lat' data vector - dataSpec[GENERATED_LONLAT_COLUMN_NAME] = coords - // update map_id mapping - @Suppress("UNCHECKED_CAST") - val aesSpec = spec[MAPPING] as MutableMap - aesSpec[Mapping.MAP_ID] = - GENERATED_LONLAT_COLUMN_NAME - } - } - - companion object { - const val LON_KEY = "lon" - const val LAT_KEY = "lat" - const val LONLAT_SPEC_KEY = "name" - const val LONLAT_SPEC_VALUE = "lon_lat" - const val GENERATED_LONLAT_COLUMN_NAME = "generated_lonlat" - - fun specSelector(): SpecSelector { - return SpecSelector.of(Plot.LAYERS) - } - - private fun concatColumns(lon: List<*>, lat: List<*>): List { - checkArgument(lon.isNotEmpty(), "Empty longitude data.") - checkArgument(lat.isNotEmpty(), "Empty latitude data.") - checkArgument(lon.size == lat.size, "Longitude and latitude have different size") - - val coordinates = ArrayList() - var i = 0 - val n = lon.size - while (i < n) { - coordinates.add( - concatCoordinates( - lon[i], - lat[i] - ) - ) - ++i - } - - return coordinates - } - - private fun concatCoordinates(lonObj: Any?, latObj: Any?): String { - val lon = - toDouble(lonObj) - val lat = - toDouble(latObj) - - checkArgument(checkLon(lon), "Invalid longitude: $lon") - checkArgument(checkLat(lat), "Invalid latitude: $lat") - - return "$lon, $lat" - } - - private fun toDouble(v: Any?): Double { - val errorMessage = "Invalid coordinate data format" - checkArgument(v is String || v is Number, errorMessage) - - if (v is String) { - try { - return v.toDouble() - } catch (e: NumberFormatException) { - throw IllegalArgumentException(errorMessage, e) - } - - } - - return (v as Number).toDouble() - } - } -} diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/MapJoinChange.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/MapJoinChange.kt index 0decd1dcd2b..fa1d2b9efe0 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/MapJoinChange.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/MapJoinChange.kt @@ -7,10 +7,8 @@ package jetbrains.datalore.plot.server.config.transform import jetbrains.datalore.plot.config.Option import jetbrains.datalore.plot.config.Option.Layer.MAP_JOIN -import jetbrains.datalore.plot.config.Option.Mapping.MAP_ID import jetbrains.datalore.plot.config.Option.Meta.MAP_DATA_META import jetbrains.datalore.plot.config.Option.Meta.MapJoin -import jetbrains.datalore.plot.config.Option.PlotBase.MAPPING import jetbrains.datalore.plot.config.getList import jetbrains.datalore.plot.config.transform.SpecChange import jetbrains.datalore.plot.config.transform.SpecChangeContext @@ -20,7 +18,7 @@ import jetbrains.datalore.plot.config.write class MapJoinChange: SpecChange { override fun apply(spec: MutableMap, ctx: SpecChangeContext) { val (dataJoinColumn, mapJoinColumn) = spec.getList(MAP_JOIN)!! - dataJoinColumn?.let { spec.write(MAPPING, MAP_ID) { it } } + dataJoinColumn?.let { spec.write(MapJoin.DATA_JOIN_COLUMN) { it } } mapJoinColumn?.let { spec.write(MapJoin.MAP_JOIN_COLUMN) { it } } } diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/PlotConfigServerSideTransforms.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/PlotConfigServerSideTransforms.kt index 1ba4ae31cd8..1b42271f4d7 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/PlotConfigServerSideTransforms.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/PlotConfigServerSideTransforms.kt @@ -45,10 +45,6 @@ object PlotConfigServerSideTransforms { MapJoinChange.specSelector(), MapJoinChange() ) - .change( - LonLatSpecInMappingSpecChange.specSelector(), - LonLatSpecInMappingSpecChange() - ) .change( GeoDataFrameMappingChange.specSelector(), GeoDataFrameMappingChange() diff --git a/plot-config-portable/src/commonTest/kotlin/plot/config/GeomInteractionBuilderCreationTest.kt b/plot-config-portable/src/commonTest/kotlin/plot/config/GeomInteractionBuilderCreationTest.kt index 48faf1d4a96..ef2e4269b98 100644 --- a/plot-config-portable/src/commonTest/kotlin/plot/config/GeomInteractionBuilderCreationTest.kt +++ b/plot-config-portable/src/commonTest/kotlin/plot/config/GeomInteractionBuilderCreationTest.kt @@ -8,15 +8,12 @@ package plot.config import jetbrains.datalore.plot.base.Aes import jetbrains.datalore.plot.config.LayerConfig import jetbrains.datalore.plot.config.Option -import jetbrains.datalore.plot.config.Option.Meta.MapJoin -import jetbrains.datalore.plot.config.Option.PlotBase.DATA import jetbrains.datalore.plot.config.Option.PlotBase.MAPPING import jetbrains.datalore.plot.config.PlotConfigClientSideUtil import jetbrains.datalore.plot.server.config.PlotConfigServerSide import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFalse -import kotlin.test.assertNotNull class GeomInteractionBuilderCreationTest { @@ -53,7 +50,7 @@ class GeomInteractionBuilderCreationTest { assertAesListCount(expectedAesListCount, builder.aesListForTooltip) } - + /* TODO: Fix MAP_ID @Test fun shouldSkipMapIdMapping() { val mappedData = data + mapOf( @@ -117,7 +114,7 @@ class GeomInteractionBuilderCreationTest { assertAesListCount(expectedAxisCount, builder.axisAesListForTooltip) assertAesListCount(expectedAesListCount, builder.aesListForTooltip) - } + }*/ @Test fun shouldNotDuplicateVarToAxisAndGenericTooltip() { @@ -152,6 +149,7 @@ class GeomInteractionBuilderCreationTest { assertAesListCount(expectedAesListCount, builder.aesListForTooltip) } + /* TODO: Fix MAP_ID @Test fun shouldSkipAutoGeneratedMappings() { @@ -193,7 +191,7 @@ class GeomInteractionBuilderCreationTest { ) assertFalse { builder.aesListForTooltip.contains(binding.aes) } - } + }*/ private fun createLayerConfig(plotOpts: MutableMap): LayerConfig { val plotSpec = PlotConfigServerSide.processTransform(plotOpts) diff --git a/plot-config/src/jvmTest/kotlin/plot/config/ConfigUtilTest.kt b/plot-config/src/jvmTest/kotlin/plot/config/ConfigUtilTest.kt index de6077830c3..6166620944e 100644 --- a/plot-config/src/jvmTest/kotlin/plot/config/ConfigUtilTest.kt +++ b/plot-config/src/jvmTest/kotlin/plot/config/ConfigUtilTest.kt @@ -7,7 +7,7 @@ package jetbrains.datalore.plot.config import jetbrains.datalore.plot.base.DataFrame import jetbrains.datalore.plot.base.DataFrame.Variable -import jetbrains.datalore.plot.config.Option.Meta.MapJoin.ID +import jetbrains.datalore.plot.config.Option.Meta.MapJoin.MAP_ID import org.assertj.core.api.Assertions.assertThat import kotlin.test.Test import kotlin.test.assertEquals @@ -21,20 +21,20 @@ class ConfigUtilTest { val dataValues = listOf("a", "b", "c", "d") val data = DataFrame.Builder() - .put(Variable(ID), idList) + .put(Variable(MAP_ID), idList) .put(Variable("foo"), dataValues) .build() val map = DataFrame.Builder() - .put(Variable(ID), idList) + .put(Variable(MAP_ID), idList) .put(Variable("lon"), listOf(13.0, 24.0, -65.0, 117.0)) .put(Variable("lat"), listOf(42.0, 21.0, -12.0, 77.0)) .build() - val joinedDf = ConfigUtil.rightJoin(data, ID, map, ID) + val joinedDf = ConfigUtil.rightJoin(data, MAP_ID, map, MAP_ID) assertThat(joinedDf.variables().map { it.toString() }) - .containsExactlyInAnyOrder(ID, "foo", "lon", "lat") + .containsExactlyInAnyOrder(MAP_ID, "foo", "lon", "lat") var dataVar: Variable? = null for (variable in joinedDf.variables()) { diff --git a/plot-config/src/jvmTest/kotlin/plot/server/config/DropUnusedDataTest.kt b/plot-config/src/jvmTest/kotlin/plot/server/config/DropUnusedDataTest.kt index a107907bcb0..989e23dbb38 100644 --- a/plot-config/src/jvmTest/kotlin/plot/server/config/DropUnusedDataTest.kt +++ b/plot-config/src/jvmTest/kotlin/plot/server/config/DropUnusedDataTest.kt @@ -7,6 +7,7 @@ package jetbrains.datalore.plot.server.config import jetbrains.datalore.plot.base.data.TransformVar import jetbrains.datalore.plot.config.TestUtil +import jetbrains.datalore.plot.parsePlotSpec import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFalse @@ -554,28 +555,38 @@ class DropUnusedDataTest { } @Test - fun shouldNotDropMapIdMappingData() { - val spec = "{" + - " 'layers': [" + - " {" + - " 'data': {" + - " 'name': ['New York']" + - " }," + - " 'geom': {" + - " 'name': 'polygon'," + - " 'mapping': {" + - " 'map_id': 'name'" + - " }" + - " }" + - " }" + - " ]" + - "}" - + fun shouldNotDropMapJoin() { + val spec = """ +{ + "kind": "plot", + "layers": [ + { + "geom": "polygon", + "data": { + "name": ["A", "B", "C"], + "value": [42, 23, 87] + }, + "mapping": { "fill": "value" }, + "map_data_meta": { + "geodataframe": { + "geometry": "coord" + } + }, + "map": { + "id": ["A", "B", "C"], + "coord": [ + "{\"type\": \"Point\", \"coordinates\": [-5.0, 17.0]}", + "{\"type\": \"Polygon\", \"coordinates\": [[[1.0, 1.0], [1.0, 9.0], [9.0, 9.0], [9.0, 1.0], [1.0, 1.0]], [[2.0, 2.0], [3.0, 2.0], [3.0, 3.0], [2.0, 3.0], [2.0, 2.0]], [[4.0, 4.0], [6.0, 4.0], [6.0, 6.0], [4.0, 6.0], [4.0, 4.0]]]}", + "{\"type\": \"MultiPolygon\", \"coordinates\": [[[[11.0, 12.0], [13.0, 14.0], [15.0, 13.0], [11.0, 12.0]]]]}" + ] + }, + "map_join": ["name", "id"] + } + ] +} +}""" - val opts = ServerSideTestUtil.parseOptionsServerSide(spec, - mapOf( - "name" to listOf("New York")) - ) + val opts = parsePlotSpec(spec) TestUtil.checkOptionsClientSide(opts, 1) diff --git a/plot-config/src/jvmTest/kotlin/plot/server/config/GeoDataFrameMappingChangeTest.kt b/plot-config/src/jvmTest/kotlin/plot/server/config/GeoDataFrameMappingChangeTest.kt index 2ea3d323616..290159574cc 100644 --- a/plot-config/src/jvmTest/kotlin/plot/server/config/GeoDataFrameMappingChangeTest.kt +++ b/plot-config/src/jvmTest/kotlin/plot/server/config/GeoDataFrameMappingChangeTest.kt @@ -42,10 +42,11 @@ class GeoDataFrameMappingChangeTest { val expectedIdList = listOf("0", "1", "2", "3", "4") assertThat(cfg) - .haveBinding(Aes.MAP_ID, MapJoin.ID) + //.haveBinding(Aes.MAP_ID, MapJoin.ID) // TODO: Fix MAP_ID .haveDataVectors(mapOf( - MapJoin.ID to expectedIdList, - NAME_COLUMN to MAP_DATA_WITH_IDS[NAME_COLUMN] as List<*>)) + // MapJoin.ID to expectedIdList, // TODO: Fix MAP_ID + NAME_COLUMN to MAP_DATA_WITH_IDS[NAME_COLUMN] as List<*> + )) .haveMapIds(expectedIdList) .haveMapGeometries(MAP_DATA_WITH_IDS[GEOMETRY_COLUMN] as List<*>) } @@ -98,7 +99,7 @@ class GeoDataFrameMappingChangeTest { DATA_FRAME + mapOf( GeoReference.REQUEST to MAP_DATA_REQUESTS, MAP_REGION_COLUMN to MAP_DATA_REGIONS, - MapJoin.ID to MAP_DATA_JOIN_KEYS, + MapJoin.MAP_ID to MAP_DATA_JOIN_KEYS, GEOMETRY_COLUMN to GEOMETRIES ) diff --git a/plot-config/src/jvmTest/kotlin/plot/server/config/SingleLayerAssert.kt b/plot-config/src/jvmTest/kotlin/plot/server/config/SingleLayerAssert.kt index 09c49bae3d2..8313c76210f 100644 --- a/plot-config/src/jvmTest/kotlin/plot/server/config/SingleLayerAssert.kt +++ b/plot-config/src/jvmTest/kotlin/plot/server/config/SingleLayerAssert.kt @@ -13,7 +13,7 @@ import jetbrains.datalore.plot.config.GeoPositionsDataUtil.MAP_OSM_ID_COLUMN import jetbrains.datalore.plot.config.LayerConfig import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame.GEOMETRIES -import jetbrains.datalore.plot.config.Option.Meta.MapJoin.ID +import jetbrains.datalore.plot.config.Option.Meta.MapJoin.MAP_ID import org.assertj.core.api.AbstractAssert import org.assertj.core.api.Assertions import kotlin.test.assertEquals @@ -78,7 +78,7 @@ class SingleLayerAssert private constructor(layers: List) : } internal fun haveMapIds(expectedIds: List<*>): SingleLayerAssert { - return haveMapValues(ID, expectedIds) + return haveMapValues(MAP_ID, expectedIds) } internal fun haveMapGeometries(expectedGeometries: List<*>): SingleLayerAssert { diff --git a/plot-config/src/jvmTest/kotlin/plot/server/config/transform/LivemapLonLatOptionsTest.kt b/plot-config/src/jvmTest/kotlin/plot/server/config/transform/LivemapLonLatOptionsTest.kt deleted file mode 100644 index deecff954ff..00000000000 --- a/plot-config/src/jvmTest/kotlin/plot/server/config/transform/LivemapLonLatOptionsTest.kt +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (c) 2019. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.server.config.transform - -import jetbrains.datalore.plot.base.Aes -import jetbrains.datalore.plot.config.Option.GeomName -import jetbrains.datalore.plot.config.Option.Layer.GEOM -import jetbrains.datalore.plot.config.Option.Plot.LAYERS -import jetbrains.datalore.plot.config.Option.PlotBase.DATA -import jetbrains.datalore.plot.config.Option.PlotBase.MAPPING -import jetbrains.datalore.plot.server.config.ServerSideTestUtil -import jetbrains.datalore.plot.server.config.SingleLayerAssert.Companion.assertThat -import jetbrains.datalore.plot.server.config.transform.LivemapLonLatOptionsTest.PlotWithLonLatData.Companion.LAT_DATA_KEY -import jetbrains.datalore.plot.server.config.transform.LivemapLonLatOptionsTest.PlotWithLonLatData.Companion.LON_DATA_KEY -import jetbrains.datalore.plot.server.config.transform.LonLatSpecInMappingSpecChange.Companion.GENERATED_LONLAT_COLUMN_NAME -import jetbrains.datalore.plot.server.config.transform.LonLatSpecInMappingSpecChange.Companion.LAT_KEY -import jetbrains.datalore.plot.server.config.transform.LonLatSpecInMappingSpecChange.Companion.LONLAT_SPEC_KEY -import jetbrains.datalore.plot.server.config.transform.LonLatSpecInMappingSpecChange.Companion.LONLAT_SPEC_VALUE -import jetbrains.datalore.plot.server.config.transform.LonLatSpecInMappingSpecChange.Companion.LON_KEY -import kotlin.test.Test -import kotlin.test.assertEquals - - -class LivemapLonLatOptionsTest { - @Test - fun livemapGeoCoordProcessorIntegrationTest() { - livemapGeoCoordProcessorIntegrationTest(PlotWithLonLatData()) - livemapGeoCoordProcessorIntegrationTest(PlotWithLonLatData.withDataInPlot()) - } - - private fun livemapGeoCoordProcessorIntegrationTest(plotWithLonLatData: PlotWithLonLatData) { - val layerConfigs = ServerSideTestUtil.createLayerConfigsWithoutEncoding(plotWithLonLatData.plotOpts) - - assertThat(layerConfigs) - .haveBinding(Aes.MAP_ID, GENERATED_LONLAT_COLUMN_NAME) - .haveDataVector( - GENERATED_LONLAT_COLUMN_NAME, plotWithLonLatData.formattedLonLatData - ) - } - - @Test - fun shouldDropLonLatDataWithoutMapping() { - val plotWithLonLatData = PlotWithLonLatData.withoutMapping() - val layerConfigs = ServerSideTestUtil.createLayerConfigsWithoutEncoding(plotWithLonLatData.plotOpts) - - assertEquals(1, layerConfigs[0].getMap(DATA).size.toLong()) - assertThat(layerConfigs) - .haveBinding(Aes.MAP_ID, GENERATED_LONLAT_COLUMN_NAME) - .haveDataVector( - GENERATED_LONLAT_COLUMN_NAME, plotWithLonLatData.formattedLonLatData - ) - } - - - @Test - fun shouldNotDropLonLatDataWithMapping() { - val plotWithLonLatData = PlotWithLonLatData.withMapping(true, true) - val layerConfigs = ServerSideTestUtil.createLayerConfigsWithoutEncoding(plotWithLonLatData.plotOpts) - - assertEquals(3, layerConfigs[0].getMap(DATA).size.toLong()) - assertThat(layerConfigs) - .haveBindings(mapOf( - Aes.MAP_ID to GENERATED_LONLAT_COLUMN_NAME, - Aes.FILL to LON_DATA_KEY, - Aes.COLOR to LAT_DATA_KEY - )) - .haveDataVectors(mapOf( - GENERATED_LONLAT_COLUMN_NAME to plotWithLonLatData.formattedLonLatData, - LON_DATA_KEY to plotWithLonLatData.lonDataVector, - LAT_DATA_KEY to plotWithLonLatData.latDataVector)) - } - - internal class PlotWithLonLatData @JvmOverloads constructor(dataInLayer: Boolean = true) { - - internal val lonDataVector = listOf(10.0, 20.0, 30.0) - internal val latDataVector = listOf(40.0, 50.0, 60.0) - internal val plotOpts: MutableMap - private val mapping: MutableMap - internal val formattedLonLatData = listOf("10.0, 40.0", "20.0, 50.0", "30.0, 60.0") - - init { - val data = object : HashMap() { - init { - put(LON_DATA_KEY, lonDataVector) - put(LAT_DATA_KEY, latDataVector) - } - } - - val lonLatSpecData = mapOf( - LONLAT_SPEC_KEY to LONLAT_SPEC_VALUE, - LON_KEY to LON_DATA_KEY, - LAT_KEY to LAT_DATA_KEY) - - mapping = object : HashMap() { - init { - put(Aes.MAP_ID.name, lonLatSpecData) - } - } - - plotOpts = object : HashMap() { - init { - if (!dataInLayer) { - put(DATA, data) - } - put(LAYERS, ArrayList(listOf( - object : HashMap() { - init { - put(GEOM, GeomName.LIVE_MAP) - put(MAPPING, mapping) - if (dataInLayer) { - put(DATA, data) - } - } - } - )) - ) - } - } - } - - companion object { - - const val LON_DATA_KEY = "LON_DATA_KEY" - const val LAT_DATA_KEY = "LAT_DATA_KEY" - - fun withoutMapping(): PlotWithLonLatData { - return PlotWithLonLatData() - } - - fun withDataInPlot(): PlotWithLonLatData { - return PlotWithLonLatData(false) - } - - fun withMapping(lonMapping: Boolean, latMapping: Boolean): PlotWithLonLatData { - val data = PlotWithLonLatData() - - if (lonMapping) { - data.mapping[Aes.FILL.name] = LON_DATA_KEY - } - - if (latMapping) { - data.mapping[Aes.COLOR.name] = LAT_DATA_KEY - } - - return data - } - } - } -} \ No newline at end of file diff --git a/plot-config/src/jvmTest/kotlin/plot/server/config/transform/LonLatSpecInMappingSpecChangeTest.kt b/plot-config/src/jvmTest/kotlin/plot/server/config/transform/LonLatSpecInMappingSpecChangeTest.kt deleted file mode 100644 index aaee8aeb0fe..00000000000 --- a/plot-config/src/jvmTest/kotlin/plot/server/config/transform/LonLatSpecInMappingSpecChangeTest.kt +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright (c) 2019. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.server.config.transform - -import jetbrains.datalore.plot.base.Aes -import jetbrains.datalore.plot.config.Option.GeomName -import jetbrains.datalore.plot.config.Option.Layer.GEOM -import jetbrains.datalore.plot.config.Option.Plot.LAYERS -import jetbrains.datalore.plot.config.Option.PlotBase.DATA -import jetbrains.datalore.plot.config.Option.PlotBase.MAPPING -import jetbrains.datalore.plot.config.transform.PlotSpecTransform -import jetbrains.datalore.plot.config.transform.SpecFinder -import jetbrains.datalore.plot.server.config.transform.LonLatSpecInMappingSpecChange.Companion.GENERATED_LONLAT_COLUMN_NAME -import jetbrains.datalore.plot.server.config.transform.LonLatSpecInMappingSpecChange.Companion.LAT_KEY -import jetbrains.datalore.plot.server.config.transform.LonLatSpecInMappingSpecChange.Companion.LONLAT_SPEC_KEY -import jetbrains.datalore.plot.server.config.transform.LonLatSpecInMappingSpecChange.Companion.LONLAT_SPEC_VALUE -import jetbrains.datalore.plot.server.config.transform.LonLatSpecInMappingSpecChange.Companion.LON_KEY -import kotlin.test.BeforeTest -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertFalse - -class LonLatSpecInMappingSpecChangeTest { - - private lateinit var livemapLayer: MutableMap - private lateinit var opt: MutableMap - private lateinit var layers: MutableList - private lateinit var livemapMapping: MutableMap - private lateinit var livemapData: MutableMap - private lateinit var plotData: Map - - @BeforeTest - fun setUp() { - livemapMapping = HashMap() - livemapData = HashMap() - livemapLayer = HashMap() - plotData = HashMap() - - livemapLayer[GEOM] = GeomName.LIVE_MAP - livemapLayer[MAPPING] = livemapMapping - livemapLayer[DATA] = livemapData - - layers = ArrayList() - layers.add(livemapLayer) - - opt = HashMap() - opt[LAYERS] = layers - opt[DATA] = plotData - } - - @Test - fun whenGeoCoordsInLivemapData_AreGood_AndTypeDouble_ShouldReplaceMappingAndData() { - val lon = listOf(10.0, 11.0, 12.0) - val lat = listOf(13.0, 14.0, 15.0) - - addLonLatSpec(livemapData, lon, lat) - - process() - - assertFormattedGeoData( - listOf( - "10.0, 13.0", - "11.0, 14.0", - "12.0, 15.0" - ) - ) - } - - @Test - fun whenGeoCoordsAreGood_AndTypeString_ShouldReplaceMappingAndData() { - val lon = listOf("10.0", "11.0", "12.0") - val lat = listOf("13.0", "14.0", "15.0") - addLonLatSpec(livemapData, lon, lat) - - process() - - assertFormattedGeoData( - listOf( - "10.0, 13.0", - "11.0, 14.0", - "12.0, 15.0") - ) - } - - @Test - fun whenGeoCoordsAreGood_AndTypeInteger_ShouldReplaceMappingAndData() { - val lon = listOf(1, 2) - val lat = listOf(3, 4) - addLonLatSpec(livemapData, lon, lat) - - process() - - assertFormattedGeoData( - listOf( - "1.0, 3.0", - "2.0, 4.0" - ) - ) - } - - @Test(expected = IllegalArgumentException::class) - fun whenColumnNameIsWrong_ShouldDoNothing() { - val lon = listOf("10.", "11.", "12.") - val lat = listOf("13.", "14.", "15.") - - // columns 'foo' and 'bar' are not present - addLonLatSpec(livemapData, lon, lat, "foo", "bar") - - process() - } - - @Test(expected = IllegalArgumentException::class) - fun whenGeoCoordsAreBad_AndTypeString_ShouldDoNothing() { - val lon = listOf("10.", "a", "b") - val lat = listOf("13.", "14.", "c") - addLonLatSpec(livemapData, lon, lat) - - process() - } - - @Test(expected = IllegalArgumentException::class) - fun whenGeoCoordsAreBad_AndTypeNotSupported_ShouldDoNothing() { - val lon = listOf(emptyList(), "a", "b") - val lat = listOf("13.", emptyList(), "c") - addLonLatSpec(livemapData, lon, lat) - - process() - } - - @Test(expected = IllegalArgumentException::class) - fun whenDataColumnsAreEmpty_ShouldDoNothing() { - val lon = emptyList() - val lat = emptyList() - addLonLatSpec(livemapData, lon, lat) - - process() - } - - @Test(expected = IllegalArgumentException::class) - fun whenDataColumnsHaveDifferentSize_ShouldDoNothing() { - val lon = listOf(1.0, 2.0, 3.0) - val lat = listOf(4.0, 5.0) - addLonLatSpec(livemapData, lon, lat) - - process() - } - - @Test - fun whenNoMappedColumns_ShouldDoNothing() { - - process() - - assertEquals(0, livemapData.size.toLong()) - } - - @Test - fun whenNotLivemap_ShouldDoNothing() { - layers.remove(livemapLayer) - - process() - } - - @Test - fun onGetMappedColumns_whenMapIdIsString_ShouldReturnNull() { - addGeoNameMapping() - - assertFalse(containsLoanLatSpec(livemapLayer)) - } - - @Test - fun onGetMappedColumns_whenMapIdIsStringList_ShouldReturnNull() { - addGeoNameListMapping() - - assertFalse(containsLoanLatSpec(livemapLayer)) - } - - private fun process() { - opt = PlotSpecTransform.builderForRawSpec() - .change(LonLatSpecInMappingSpecChange.specSelector(), LonLatSpecInMappingSpecChange()) - .build() - .apply(opt) - } - - - private fun assertFormattedGeoData(expected: List) { - val mapData = SpecFinder(LAYERS, DATA).findSpecs(opt)[0] - assertEquals(expected, mapData[GENERATED_LONLAT_COLUMN_NAME]) - } - - private fun addLonLatSpec(data: MutableMap, lon: List<*>, lat: List<*>, lonColumnName: String = LON_COLUMN, latColumnName: String = LAT_COLUMN) { - data[LON_COLUMN] = lon - data[LAT_COLUMN] = lat - - val lonLatSpec = HashMap() - lonLatSpec[LONLAT_SPEC_KEY] = LONLAT_SPEC_VALUE - lonLatSpec[LON_KEY] = lonColumnName - lonLatSpec[LAT_KEY] = latColumnName - - livemapMapping[AES_MAP_ID_NAME] = lonLatSpec - } - - private fun addGeoNameMapping() { - - livemapMapping[AES_MAP_ID_NAME] = GEO_NAME - } - - private fun addGeoNameListMapping() { - - livemapMapping[AES_MAP_ID_NAME] = listOf(GEO_NAME) - } - - companion object { - internal const val GEO_COORD_FORMAT = "%s, %s" - private const val GEO_NAME = "Texas" - private const val LON_COLUMN = "col_with_lon" - private const val LAT_COLUMN = "col_with_lat" - private val AES_MAP_ID_NAME = Aes.MAP_ID.name - - private fun containsLoanLatSpec(livemapOptions: Map<*, *>?): Boolean { - val specs = SpecFinder(MAPPING, AES_MAP_ID_NAME).findSpecs(livemapOptions!!) - if (specs.isEmpty()) { - return false - } - - val lonLatSpec = specs[0] - - return (lonLatSpec.containsKey(LONLAT_SPEC_KEY) - && LONLAT_SPEC_VALUE == lonLatSpec[LONLAT_SPEC_KEY] - && lonLatSpec.containsKey(LON_KEY) - && lonLatSpec.containsKey(LAT_KEY)) - } - } -} diff --git a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapDataPointAestheticsProcessor.kt b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapDataPointAestheticsProcessor.kt index 3e858167204..e57a17277bb 100644 --- a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapDataPointAestheticsProcessor.kt +++ b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapDataPointAestheticsProcessor.kt @@ -5,8 +5,6 @@ package jetbrains.datalore.plot.livemap -import jetbrains.datalore.base.spatial.LonLat -import jetbrains.datalore.base.typedGeometry.Vec import jetbrains.datalore.plot.base.Aes import jetbrains.datalore.plot.base.Aesthetics import jetbrains.datalore.plot.base.DataPointAesthetics @@ -14,9 +12,9 @@ import jetbrains.datalore.plot.base.aes.AesInitValue import jetbrains.datalore.plot.base.livemap.LiveMapOptions import jetbrains.datalore.plot.base.livemap.LivemapConstants import jetbrains.datalore.plot.livemap.LiveMapUtil.createLayersConfigurator +import jetbrains.datalore.plot.livemap.MapLayerKind.POINT import jetbrains.datalore.plot.livemap.MultiDataPointHelper.SortingMode import jetbrains.livemap.api.LayersBuilder -import jetbrains.datalore.plot.livemap.MapLayerKind.POINT internal class LiveMapDataPointAestheticsProcessor( private val myAesthetics: Aesthetics, @@ -25,7 +23,6 @@ internal class LiveMapDataPointAestheticsProcessor( private val myLayerKind: MapLayerKind = getLayerKind(liveMapOptions.displayMode) private val myGeodesic: Boolean = liveMapOptions.geodesic private val myFrameSpecified: Boolean = allAesMatch(myAesthetics, ::isFrameSet) - private val myLonLatInsideMapIdSpecified: Boolean = allAesMatch(myAesthetics, ::isLiveMapWithLonLat) val mapEntityBuilders: List get() = (if (useMultiDataPoint()) processMultiDataPoints() else processDataPoints()).onEach { it.layerIndex = 0 } @@ -34,10 +31,6 @@ internal class LiveMapDataPointAestheticsProcessor( return p.frame() != AesInitValue[Aes.FRAME] } - private fun isLiveMapWithLonLat(p: DataPointAesthetics): Boolean { - return LonLatParser.parse(p.mapId().toString()) != null - } - private fun getLayerKind(displayMode: LivemapConstants.DisplayMode): MapLayerKind { return when (displayMode) { LivemapConstants.DisplayMode.POLYGON -> MapLayerKind.POLYGON @@ -62,37 +55,23 @@ internal class LiveMapDataPointAestheticsProcessor( private fun processDataPoints(): List { return myAesthetics.dataPoints() - .map { MapEntityBuilder(it, myLayerKind).apply { setIfNeeded(it) } } + .map { MapEntityBuilder(it, myLayerKind).apply { setIfNeeded() } } } private fun processMultiDataPoints(): List { return MultiDataPointHelper .getPoints(myAesthetics, getSortingMode(myLayerKind)) - .map { MapEntityBuilder(it, myLayerKind).apply { setIfNeeded(it.aes) } } + .map { MapEntityBuilder(it, myLayerKind).apply { setIfNeeded() } } } private fun useMultiDataPoint(): Boolean { return myLayerKind === MapLayerKind.PIE || myLayerKind === MapLayerKind.BAR } - private fun MapEntityBuilder.setIfNeeded( - p: DataPointAesthetics - ) { - setGeometryPointIfNeeded(p, this) + private fun MapEntityBuilder.setIfNeeded() { setGeodesicIfNeeded(this) } - private fun setGeometryPointIfNeeded(p: DataPointAesthetics, mapEntityBuilder: MapEntityBuilder) { - var lonlat: Vec? = null - if (myLonLatInsideMapIdSpecified) { - lonlat = LonLatParser.parse(p.mapId().toString()) - } - - if (lonlat != null) { - mapEntityBuilder.setGeometryPoint(lonlat) - } - } - private fun setGeodesicIfNeeded(mapEntityBuilder: MapEntityBuilder) { if (myLayerKind == MapLayerKind.PATH) { mapEntityBuilder.geodesic = myGeodesic diff --git a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapUtil.kt b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapUtil.kt index 3c3e81dc0e1..2dd14f86a47 100644 --- a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapUtil.kt +++ b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapUtil.kt @@ -84,10 +84,7 @@ object LiveMapUtil { } private fun getHiddenAes(geomKind: GeomKind): List> { - val hiddenAes = ArrayList>() - hiddenAes.add(Aes.MAP_ID) - - when (geomKind) { + return when (geomKind) { POINT, POLYGON, CONTOUR, @@ -98,17 +95,14 @@ object LiveMapUtil { TILE, BIN_2D, V_LINE, - H_LINE -> hiddenAes.addAll(listOf(Aes.X, Aes.Y)) + H_LINE -> listOf(Aes.X, Aes.Y) - RECT -> hiddenAes.addAll(listOf(Aes.YMIN, Aes.YMAX, Aes.XMIN, Aes.XMAX)) + RECT -> listOf(Aes.YMIN, Aes.YMAX, Aes.XMIN, Aes.XMAX) - SEGMENT -> hiddenAes.addAll(listOf(Aes.X, Aes.Y, Aes.XEND, Aes.YEND)) + SEGMENT -> listOf(Aes.X, Aes.Y, Aes.XEND, Aes.YEND) - else -> { - } + else -> emptyList() } - - return hiddenAes } fun createContextualMapping(geomKind: GeomKind, dataAccess: MappedDataAccess): ContextualMapping { diff --git a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelper.kt b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelper.kt index 057ecc0fd6f..78e81d4f276 100644 --- a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelper.kt +++ b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelper.kt @@ -17,7 +17,7 @@ internal class MultiDataPointHelper private constructor( val builders = HashMap() fun fetchBuilder(p: DataPointAesthetics): MultiDataPointBuilder = - builders.getOrPut(p.mapId(), { MultiDataPointBuilder(p, sortingMode) }) + builders.getOrPut(p.group()!!, { MultiDataPointBuilder(p, sortingMode) }) aesthetics.dataPoints().forEach { fetchBuilder(it).add(it) } return builders.values.map { it.build() } @@ -37,7 +37,7 @@ internal class MultiDataPointHelper private constructor( private var myUsesOrder: Boolean = false internal fun add(p: DataPointAesthetics) { - if (p.x() != 0.0) { + if (p.symX() != 0.0) { myUsesOrder = true } @@ -54,7 +54,7 @@ internal class MultiDataPointHelper private constructor( return MultiDataPoint( aes = myAes, indices = myPoints.map { it.index() }, - values = myPoints.map { it.y()!! }, + values = myPoints.map { it.symY()!! }, colors = myPoints.map { it.fill()!! } ) } @@ -72,8 +72,8 @@ internal class MultiDataPointHelper private constructor( } companion object { - private val BY_ORDER: (DataPointAesthetics) -> Double = { it.x()!! } - private val BY_VALUE: (DataPointAesthetics) -> Double = { it.y()!! } + private val BY_ORDER: (DataPointAesthetics) -> Double = { it.symX()!! } + private val BY_VALUE: (DataPointAesthetics) -> Double = { it.symY()!! } } } diff --git a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MapIdAndGeoPointDataProcessingTest.kt b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MapIdAndGeoPointDataProcessingTest.kt index 361c5941587..72368d6f271 100644 --- a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MapIdAndGeoPointDataProcessingTest.kt +++ b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MapIdAndGeoPointDataProcessingTest.kt @@ -20,11 +20,13 @@ import jetbrains.datalore.plot.config.Option.Geom.LiveMap.DISPLAY_MODE import jetbrains.datalore.plot.config.OptionsAccessor import jetbrains.datalore.plot.livemap.ConverterDataHelper.createDefaultMatcher import jetbrains.datalore.plot.livemap.MapObjectMatcher.Companion.eq +import kotlin.test.Ignore import kotlin.test.Test class MapIdAndGeoPointDataProcessingTest { + @Ignore // TODO: Fix MAP_ID @Test fun whenSingleGeoName() { Expectations( @@ -34,6 +36,7 @@ class MapIdAndGeoPointDataProcessingTest { ).doAssert() } + @Ignore // TODO: Fix MAP_ID @Test fun whenSingleLonLat() { Expectations( @@ -61,6 +64,7 @@ class MapIdAndGeoPointDataProcessingTest { ).doAssert() } + @Ignore // TODO: Fix MAP_ID @Test fun multiDataLonLat() { Expectations( @@ -131,7 +135,7 @@ class MapIdAndGeoPointDataProcessingTest { return AestheticsBuilder(1) .x(constant(1.0)) .y(constant(1.0)) - .mapId(constant(mapId)) + //.mapId(constant(mapId)) // TODO: Fix MAP_ID .build() } diff --git a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelperTest.kt b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelperTest.kt index 622112945be..af5e64961a2 100644 --- a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelperTest.kt +++ b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelperTest.kt @@ -13,6 +13,7 @@ import jetbrains.datalore.plot.livemap.MultiDataPointHelper.SortingMode import jetbrains.datalore.plot.livemap.MultiDataPointHelper.SortingMode.BAR import jetbrains.datalore.plot.livemap.MultiDataPointHelper.SortingMode.PIE_CHART import jetbrains.datalore.plot.livemap.MultiDataPointHelperTest.MultiDataBuilder.DataPointBuilder +import kotlin.test.Ignore import kotlin.test.Test import kotlin.test.assertEquals @@ -20,6 +21,7 @@ class MultiDataPointHelperTest { private var myMultiDataBuilder = MultiDataBuilder() + @Ignore // TODO: Fix MAP_ID @Test fun whenSortingModePieChart_AndPointOrderSet_ShouldSortByOrder() { val dataPointBuilders = listOf( @@ -37,6 +39,7 @@ class MultiDataPointHelperTest { ) } + @Ignore // TODO: Fix MAP_ID @Test fun whenSortingModePieChart_AndPointOrderNotSet_ShouldUseSpecialSortingByValue() { assertPointsOrder( @@ -61,6 +64,7 @@ class MultiDataPointHelperTest { ) } + @Ignore // TODO: Fix MAP_ID @Test fun whenDataNotSorted_AndSortingModeBar_ShouldSortByOrder() { assertPointsOrder( @@ -126,7 +130,7 @@ class MultiDataPointHelperTest { } myBuilder - .mapId(collection(mapId)) + //.mapId(collection(mapId)) // TODO: Fix MAP_ID .x(collection(order)) .y(collection(values)) .dataPointCount(order.size) diff --git a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PointConverterTest.kt b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PointConverterTest.kt index d59ff6c5e18..485969a1f3a 100644 --- a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PointConverterTest.kt +++ b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PointConverterTest.kt @@ -96,7 +96,8 @@ class PointConverterTest { @Test fun whenMapIdSet_ShouldUseGeometry() { - aesData.builder().mapId(constant("New York City")) + aesData.builder() + //.mapId(constant("New York City")) // TODO: Fix MAP_ID assertPointGeometryDataArray() } diff --git a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PolygonConverterTest.kt b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PolygonConverterTest.kt index 53ae7baed98..6c2accb73aa 100644 --- a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PolygonConverterTest.kt +++ b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PolygonConverterTest.kt @@ -5,7 +5,6 @@ package jetbrains.datalore.plot.livemap -import jetbrains.datalore.plot.base.aes.AestheticsBuilder.Companion.constant import jetbrains.datalore.plot.livemap.ConverterDataHelper.AestheticsDataHelper import jetbrains.datalore.plot.livemap.ConverterDataHelper.MULTIPOLYGON import jetbrains.datalore.plot.livemap.ConverterDataHelper.createDefaultMatcher @@ -36,7 +35,8 @@ class PolygonConverterTest { @Test fun withMapIdAndGeometry_ShouldUseGeometry() { - aesData.builder().mapId(constant("New York City")) + aesData.builder() + // .mapId(constant("New York City")) // TODO: Fix MAP_ID matcher.geometry(geometryEq(Boundary.create(MULTIPOLYGON))) From 48278c3e1e1072df6bfc7934c6806b985420bfa3 Mon Sep 17 00:00:00 2001 From: Ivan Kupriyanov Date: Fri, 5 Jun 2020 15:47:53 +0300 Subject: [PATCH 02/11] Remove map_id from implementation, move data and map join implementation from SpecChange to LayerConfig --- .../geodataframe_and_geoms.ipynb | 177 +++- .../geopandas_GeoDataFrame.ipynb | 807 ++++++++++++++++-- .../datalore/plot/base/data/DataFrameUtil.kt | 2 +- .../plot/builder/assemble/GeomLayerBuilder.kt | 9 +- .../plot/builder/data/DataProcessing.kt | 4 +- .../plot/builder/data/GroupingContext.kt | 16 +- .../plot/builder/map/GeoPositionField.kt | 1 - .../datalore/plot/config/GeoConfig.kt | 282 ++++++ .../plot/config/GeoPositionsDataUtil.kt | 195 ----- .../datalore/plot/config/LayerConfig.kt | 29 +- .../jetbrains/datalore/plot/config/Option.kt | 16 +- .../plot/config/PlotConfigClientSide.kt | 2 - .../plot/config/PlotConfigClientSideUtil.kt | 6 +- .../geo/GeometryFromGeoDataFrameChange.kt | 145 ---- .../geo/GeometryFromGeoPositionsChange.kt | 48 -- .../server/config/PlotConfigServerSide.kt | 19 +- .../transform/GeoDataFrameMappingChange.kt | 43 - .../transform/GeoPositionMappingChange.kt | 64 -- .../server/config/transform/MapJoinChange.kt | 35 - .../PlotConfigServerSideTransforms.kt | 12 - .../plot/config/geo/GeoPositionResolver.kt | 64 -- .../kotlin/plot/config/AsDiscreteTest.kt | 114 +-- .../jvmTest/kotlin/plot/config/Assertions.kt | 97 +++ .../kotlin/plot/config/GeoConfigTest.kt | 263 ++++++ .../plot/config/geo/DataFrameBuilderTest.kt | 222 ----- .../plot/server/config/DropUnusedDataTest.kt | 95 ++- .../config/GeoDataFrameMappingChangeTest.kt | 117 --- .../config/GeoReferenceMappingChangeTest.kt | 40 - .../plot/server/config/SingleLayerAssert.kt | 5 - .../plotDemo/model/plotConfig/GeoData.kt | 128 ++- 30 files changed, 1758 insertions(+), 1299 deletions(-) create mode 100644 plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt delete mode 100644 plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoPositionsDataUtil.kt delete mode 100644 plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeometryFromGeoDataFrameChange.kt delete mode 100644 plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeometryFromGeoPositionsChange.kt delete mode 100644 plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoDataFrameMappingChange.kt delete mode 100644 plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoPositionMappingChange.kt delete mode 100644 plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/MapJoinChange.kt delete mode 100644 plot-config/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeoPositionResolver.kt create mode 100644 plot-config/src/jvmTest/kotlin/plot/config/Assertions.kt create mode 100644 plot-config/src/jvmTest/kotlin/plot/config/GeoConfigTest.kt delete mode 100644 plot-config/src/jvmTest/kotlin/plot/config/geo/DataFrameBuilderTest.kt delete mode 100644 plot-config/src/jvmTest/kotlin/plot/server/config/GeoDataFrameMappingChangeTest.kt delete mode 100644 plot-config/src/jvmTest/kotlin/plot/server/config/GeoReferenceMappingChangeTest.kt diff --git a/docs/examples/jupyter-notebooks-dev/geodataframe_and_geoms.ipynb b/docs/examples/jupyter-notebooks-dev/geodataframe_and_geoms.ipynb index ab60bc411bc..235ec0bf114 100644 --- a/docs/examples/jupyter-notebooks-dev/geodataframe_and_geoms.ipynb +++ b/docs/examples/jupyter-notebooks-dev/geodataframe_and_geoms.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -18,16 +18,36 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "LetsPlot.setup_html()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -37,9 +57,72 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
keycoord
0CaliforniaPOINT (-119.99411 37.27734)
1NevadaPOINT (-116.66696 38.50308)
2UtahPOINT (-111.54916 39.49887)
3ArizonaPOINT (-111.66859 34.16854)
\n", + "
" + ], + "text/plain": [ + " key coord\n", + "0 California POINT (-119.99411 37.27734)\n", + "1 Nevada POINT (-116.66696 38.50308)\n", + "2 Utah POINT (-111.54916 39.49887)\n", + "3 Arizona POINT (-111.66859 34.16854)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "centroids = gpd.GeoDataFrame(\n", " data={'key': ['California', 'Nevada', 'Utah', 'Arizona'],\n", @@ -54,7 +137,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -69,9 +152,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/ikupriyanov/miniconda3/lib/python3.7/site-packages/pyproj/crs/crs.py:53: FutureWarning: '+init=:' syntax is deprecated. ':' is the preferred initialization method. When making the change, be mindful of axis order changes: https://pyproj4.github.io/pyproj/stable/gotchas.html#axis-order-changes-in-proj-6\n", + " return _prepare_from_string(\" \".join(pjargs))\n" + ] + } + ], "source": [ "nv = download_geometry(165473, 'Nevada')\n", "ca = download_geometry(165475, 'California')\n", @@ -81,9 +173,72 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
geometrykey
0POLYGON ((-114.81358 32.49408, -111.07483 31.3...Arizona
1MULTIPOLYGON (((-124.32884 41.99833, -119.9994...California
2POLYGON ((-114.05283 37.57353, -114.05005 37.0...Utah
3POLYGON ((-120.00556 39.25849, -120.00101 38.9...Nevada
\n", + "
" + ], + "text/plain": [ + " geometry key\n", + "0 POLYGON ((-114.81358 32.49408, -111.07483 31.3... Arizona\n", + "1 MULTIPOLYGON (((-124.32884 41.99833, -119.9994... California\n", + "2 POLYGON ((-114.05283 37.57353, -114.05005 37.0... Utah\n", + "3 POLYGON ((-120.00556 39.25849, -120.00101 38.9... Nevada" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "boundaries = ar\n", "boundaries = pd.concat([boundaries, ca], ignore_index=True)\n", diff --git a/docs/examples/jupyter-notebooks-dev/geopandas_GeoDataFrame.ipynb b/docs/examples/jupyter-notebooks-dev/geopandas_GeoDataFrame.ipynb index 2c6d90b2f6f..504ddb02f95 100644 --- a/docs/examples/jupyter-notebooks-dev/geopandas_GeoDataFrame.ipynb +++ b/docs/examples/jupyter-notebooks-dev/geopandas_GeoDataFrame.ipynb @@ -2,23 +2,43 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", - "from lets_plot import *" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "LetsPlot.setup_html()" + "from lets_plot import *\n", + "LetsPlot.setup_html()\n", + "\n", + "def dump_plot(plot):\n", + " import json\n", + " from lets_plot._type_utils import standardize_dict\n", + " \n", + " plot_dict = standardize_dict(plot.as_dict())\n", + " plot_json = json.dumps(plot_dict, indent=2)\n", + " print(plot_json)" ] }, { @@ -30,22 +50,107 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "from geopandas import GeoDataFrame\n", - "from shapely.geometry import MultiPolygon, Polygon, LinearRing, Point, mapping" + "from shapely.geometry import MultiPolygon, Polygon, LinearRing, Point, MultiPoint, LineString, MultiLineString, mapping" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
kindcoord
0PointPOINT (-5.00000 17.00000)
1MPointMULTIPOINT (3.00000 15.00000, 6.00000 13.00000)
2LineLINESTRING (0.00000 0.00000, 5.00000 5.00000)
3MLineMULTILINESTRING ((10.00000 0.00000, 15.00000 5...
4PolygonPOLYGON ((1.00000 1.00000, 1.00000 9.00000, 9....
5MPolygonMULTIPOLYGON (((11.00000 12.00000, 13.00000 14...
\n", + "
" + ], + "text/plain": [ + " kind coord\n", + "0 Point POINT (-5.00000 17.00000)\n", + "1 MPoint MULTIPOINT (3.00000 15.00000, 6.00000 13.00000)\n", + "2 Line LINESTRING (0.00000 0.00000, 5.00000 5.00000)\n", + "3 MLine MULTILINESTRING ((10.00000 0.00000, 15.00000 5...\n", + "4 Polygon POLYGON ((1.00000 1.00000, 1.00000 9.00000, 9....\n", + "5 MPolygon MULTIPOLYGON (((11.00000 12.00000, 13.00000 14..." + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "POINT = Point(-5, 17)\n", "\n", + "MULTI_POINT = MultiPoint([Point(3, 15), Point(6, 13)])\n", + "\n", + "LINE = LineString([(0, 0), (5, 5)])\n", + "\n", + "MULTI_LINE = MultiLineString([\n", + " LineString([(10, 0), (15, 5)]),\n", + " LineString([(10, 5), (15, 0)])\n", + "])\n", + "\n", + "\n", "POLYGON = Polygon(\n", " LinearRing([(1, 1), (1, 9), (9, 9), (9, 1)]),\n", " [LinearRing([(2, 2), (3, 2), (3, 3), (2, 3)]),\n", @@ -54,157 +159,715 @@ "\n", "MULTIPOLYGON = MultiPolygon([\n", " Polygon(LinearRing([(11, 12), (13, 14), (15, 13)]))\n", - " ])" + " ])\n", + "\n", + "gdf = GeoDataFrame(\n", + " data={\n", + " 'kind': ['Point', 'MPoint', 'Line', 'MLine', 'Polygon', 'MPolygon'],\n", + " 'coord': [POINT, MULTI_POINT, LINE, MULTI_LINE, POLYGON, MULTIPOLYGON]\n", + " },\n", + " geometry='coord'\n", + ")\n", + "gdf" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "gdf = GeoDataFrame(\n", - " data={'id': ['A', 'B', 'C'],\n", - " 'coord': [POINT, POLYGON, MULTIPOLYGON]},\n", - " geometry='coord')" + "ggplot() + geom_polygon(aes(fill = 'kind'), gdf)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "ggplot() + geom_polygon(aes(fill = 'id'), gdf)" + "ggplot() + geom_polygon(map = gdf)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 6, "metadata": {}, + "outputs": [], "source": [ - "Mapping doesn't work for DataFrame in map" + "\n", + "df = pd.DataFrame(\n", + " {\n", + " 'value': [42, 23, 66],\n", + " 'fig': ['Polygon', 'MPolygon', 'C'] \n", + " }\n", + ")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "ggplot() + geom_polygon(aes(fill = 'id'), map = gdf)" + "ggplot() + geom_polygon(aes(fill = 'value'), df, map = gdf, map_join=['fig', 'kind'])" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "ggplot() + geom_polygon(map = gdf)" + "#### Geom kinds that support GeoDataFrame" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "df = pd.DataFrame(\n", - " {'name': ['A', 'B', 'C'],\n", - " 'value': [42, 23, 87]})" + "ggplot() + geom_polygon(aes(fill='kind'), data = gdf)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "ggplot() + geom_polygon(aes(fill = 'value'), df, map = gdf, map_join=('name', 'id'))" + "ggplot() + geom_point(aes(color='kind'), gdf, size=5)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 17, "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "#### Geom kinds that support GeoDataFrame" + "ggplot() + geom_rect(aes(fill='kind'), gdf)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "ggplot() + geom_polygon(data = gdf)" + "p = ggplot() + geom_path(data = gdf)\n", + "def dump_plot(plot):\n", + " import json\n", + " from lets_plot._type_utils import standardize_dict\n", + " \n", + " plot_dict = standardize_dict(plot.as_dict())\n", + " plot_json = json.dumps(plot_dict, indent=2)\n", + " print(plot_json)\n", + " \n", + "p" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "ggplot() + geom_point(aes(color='id'), gdf, size=5)" + "ggplot() + geom_map(map = gdf)" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "ggplot() + geom_rect(aes(fill='id'), gdf)" + "#### Problems" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "ggplot() + geom_path(data = gdf)" + "ggplot() + geom_map(data = gdf)" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "ggplot() + geom_map(map = gdf)" + "Error message contains autogenerated column \"\\_\\_key\\_\\_\". It's ok." ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 14, "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "#### Problems" + "ggplot() + geom_polygon(aes(fill = 'kind'), gdf)" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 15, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "ggplot() + geom_map(data = gdf)" + "tiny_gdf = GeoDataFrame(\n", + " #data={'lll': ['A'], 'coord': [MULTIPOLYGON]},\n", + " data={'lll': ['A', 'B', 'C'], 'coord': [POINT, POLYGON, MULTIPOLYGON]},\n", + " geometry='coord')\n", + "ggplot() + geom_polygon(map = tiny_gdf)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Error message contains autogenerated column \"\\_\\_key\\_\\_\". It's ok." + "Matching doesn't work for DataFrame in map" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "ggplot() + geom_polygon(aes(fill = 'id'), gdf)" + "ggplot() + geom_polygon(aes(fill = 'kind'), map = gdf)" ] } ], diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/data/DataFrameUtil.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/data/DataFrameUtil.kt index d79d5b236a0..6aac6a2631a 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/data/DataFrameUtil.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/data/DataFrameUtil.kt @@ -98,7 +98,7 @@ object DataFrameUtil { } fun variables(df: DataFrame): Map { - return df.variables().associateBy { it.name } + return df.variables().associateBy(DataFrame.Variable::name) } fun appendReplace(df0: DataFrame, df1: DataFrame): DataFrame { diff --git a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/assemble/GeomLayerBuilder.kt b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/assemble/GeomLayerBuilder.kt index dec25e0ff9e..b19466e3992 100644 --- a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/assemble/GeomLayerBuilder.kt +++ b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/assemble/GeomLayerBuilder.kt @@ -36,6 +36,7 @@ class GeomLayerBuilder { private lateinit var myPosProvider: PosProvider private lateinit var myGeomProvider: GeomProvider private var myGroupingVarName: String? = null + private var myPathIdVarName: String? = null private val myScaleProviderByAes = HashMap, ScaleProvider<*>>() private var myDataPreprocessor: ((DataFrame) -> DataFrame)? = null @@ -74,6 +75,11 @@ class GeomLayerBuilder { return this } + fun pathIdVarName(v: String): GeomLayerBuilder { + myPathIdVarName = v + return this + } + fun addConstantAes(aes: Aes, v: T): GeomLayerBuilder { myConstantByAes.put(aes, v) return this @@ -148,7 +154,7 @@ class GeomLayerBuilder { myPosProvider, // handledAes(), myGeomProvider.renders(), - GroupingContext(data, myBindings, myGroupingVarName, handlesGroups()).groupMapper, + GroupingContext(data, myBindings, myGroupingVarName, myPathIdVarName, handlesGroups()).groupMapper, replacementBindings.values, myConstantByAes, dataAccess, @@ -276,6 +282,7 @@ class GeomLayerBuilder { transformedData, builder.myBindings, builder.myGroupingVarName, + builder.myPathIdVarName, true ) val dataAndGroupingContext = DataProcessing.buildStatData( diff --git a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/data/DataProcessing.kt b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/data/DataProcessing.kt index 32fa9e8265d..238667674db 100644 --- a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/data/DataProcessing.kt +++ b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/data/DataProcessing.kt @@ -304,8 +304,8 @@ object DataProcessing { return inverseTransformedStatSeries } - internal fun computeGroups(data: DataFrame, bindings: List, groupingVar: Variable?): (Int) -> Int { - val groupingVariables = getGroupingVariables(data, bindings, groupingVar) + internal fun computeGroups(data: DataFrame, bindings: List, groupingVar: Variable?, pathIdVar: Variable?): (Int) -> Int { + val groupingVariables = getGroupingVariables(data, bindings, groupingVar) + listOfNotNull(pathIdVar) var currentGroups: List? = null if (groupingVar != null) { diff --git a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/data/GroupingContext.kt b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/data/GroupingContext.kt index ecea85ae4f3..344e5303128 100644 --- a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/data/GroupingContext.kt +++ b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/data/GroupingContext.kt @@ -15,11 +15,13 @@ class GroupingContext( private val myData: DataFrame, bindings: List, groupingVarName: String?, + pathIdVarName: String?, private val myExpectMultiple: Boolean ) { - private val myBindings: List - internal val optionalGroupingVar: Variable? + private val myBindings: List = ArrayList(bindings) + internal val optionalGroupingVar: Variable? = findOptionalVariable(myData, groupingVarName) + private val pathIdVar: Variable? = findOptionalVariable(myData, pathIdVarName) private var myGroupSizeList: List? = null private var myGroupMapper: ((Int) -> Int)? = null @@ -33,11 +35,6 @@ class GroupingContext( myGroupMapper!!(index) } - init { - myBindings = ArrayList(bindings) - optionalGroupingVar = findOptionalVariable(myData, groupingVarName) - } - private fun computeGroups(): (Int) -> Int { if (myData.has(Stats.GROUP)) { val list = myData.getNumeric(Stats.GROUP) @@ -54,7 +51,8 @@ class GroupingContext( return DataProcessing.computeGroups( myData, myBindings, - optionalGroupingVar + optionalGroupingVar, + pathIdVar ) } return GroupUtil.SINGLE_GROUP @@ -62,7 +60,7 @@ class GroupingContext( companion object { internal fun withOrderedGroups(data: DataFrame, groupSizeList: List): GroupingContext { - val groupingContext = GroupingContext(data, emptyList(), null, false) + val groupingContext = GroupingContext(data, emptyList(), null, null, false) groupingContext.myGroupSizeList = ArrayList(groupSizeList) return groupingContext } diff --git a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/map/GeoPositionField.kt b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/map/GeoPositionField.kt index ca1c5349351..c7e85b62196 100644 --- a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/map/GeoPositionField.kt +++ b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/map/GeoPositionField.kt @@ -12,7 +12,6 @@ object GeoPositionField { // fixed columns in 'boundaries' of 'centroids' data frames const val POINT_X = "lon" const val POINT_X1 = "longitude" - const val POINT_X2 = "long" const val POINT_Y = "lat" const val POINT_Y1 = "latitude" diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt new file mode 100644 index 00000000000..7cf4edebc3e --- /dev/null +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2020. JetBrains s.r.o. + * Use of this source code is governed by the MIT license that can be found in the LICENSE file. + */ + +package jetbrains.datalore.plot.config + +import jetbrains.datalore.base.spatial.* +import jetbrains.datalore.base.typedGeometry.* +import jetbrains.datalore.plot.base.Aes +import jetbrains.datalore.plot.base.DataFrame +import jetbrains.datalore.plot.base.GeomKind +import jetbrains.datalore.plot.base.GeomKind.* +import jetbrains.datalore.plot.base.data.DataFrameUtil.variables +import jetbrains.datalore.plot.config.ConfigUtil.createAesMapping +import jetbrains.datalore.plot.config.ConfigUtil.createDataFrame +import jetbrains.datalore.plot.config.ConfigUtil.rightJoin +import jetbrains.datalore.plot.config.CoordinatesBuilder.Companion.createCoordinateBuilder +import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS +import jetbrains.datalore.plot.config.Option.Layer.MAP_JOIN +import jetbrains.datalore.plot.config.Option.Meta.DATA_META +import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame.GDF +import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame.GEOMETRY +import jetbrains.datalore.plot.config.Option.Meta.MAP_DATA_META +import jetbrains.datalore.plot.config.Option.PlotBase.DATA + +class GeoConfig( + geomKind: GeomKind, + data: DataFrame, + layerOptions: Map<*, *>, + mappingOptions: Map<*, *> +) { + val dataAndCoordinates: DataFrame + val mappings: Map, DataFrame.Variable> + + init { + fun getGeoJson(gdfLocation: String): List { + val geoColumn: String + val geoDataFrame: Map + when(gdfLocation) { + GEO_POSITIONS -> { + geoDataFrame = layerOptions.getMap(GEO_POSITIONS) ?: error("require 'map' parameter") + geoColumn = layerOptions.getString(MAP_DATA_META, GDF, GEOMETRY) ?: error("Geometry column not set") + } + DATA -> { + geoDataFrame = layerOptions.getMap(DATA) ?: error("require 'data' parameter") + geoColumn = layerOptions.getString(DATA_META, GDF, GEOMETRY) ?: error("Geometry column not set") + } + else -> error("Unknown gdf location: $gdfLocation") + } + return geoDataFrame.getList(geoColumn)?.map { it as String } ?: error("$geoColumn not found in $gdfLocation") + } + + val joinIds: List + val dataJoinColumn: String + val mapJoinColumn: String + val geoJson: List + val dataFrame: DataFrame + val autoId = "__gdf_id__" + + when { + // (aes(color='cyl'), data=data, map=gdf) - how to join without `map_join`? + with(layerOptions) { has(GEO_POSITIONS) && !has(MAP_JOIN) && !data.isEmpty && mappingOptions.isNotEmpty() } -> { + error(MAP_JOIN_REQUIRED_MESSAGE) + } + + // (map=gdf) - simple geometry + with(layerOptions) { has(GEO_POSITIONS) && !has(MAP_JOIN) && has(MAP_DATA_META, GDF, GEOMETRY) } -> { + geoJson = getGeoJson(GEO_POSITIONS) + + dataJoinColumn = autoId + mapJoinColumn = autoId + joinIds = geoJson.indices.map(Int::toString) + dataFrame = DataFrame.Builder(data).put(DataFrame.Variable(dataJoinColumn), joinIds).build() + } + + // (data=data, map=gdf, map_join=('id', 'city')) + with(layerOptions) { has(GEO_POSITIONS) && has(MAP_DATA_META, GDF, GEOMETRY) && has(MAP_JOIN) } -> { + geoJson = getGeoJson(GEO_POSITIONS) + + val mapJoin = layerOptions.getList(MAP_JOIN) ?: error("require map_join parameter") + dataJoinColumn = mapJoin[0] as String + mapJoinColumn = mapJoin[1] as String + joinIds = layerOptions.getMap(GEO_POSITIONS)?.getList(mapJoinColumn)?.requireNoNulls() ?: error("MapJoinColumn '$mapJoinColumn' is not found") + dataFrame = data + } + + // (data=gdf) + with(layerOptions) { !has(GEO_POSITIONS) && has(DATA_META, GDF, GEOMETRY) } -> { + geoJson = getGeoJson(DATA) + + dataJoinColumn = autoId + mapJoinColumn = autoId + joinIds = geoJson.indices.map(Int::toString) + dataFrame = DataFrame.Builder(data).put(DataFrame.Variable(dataJoinColumn), joinIds).build() + } + else -> error("GeoDataFrame not found in data or map") + } + + val coordinatesBuilder = createCoordinateBuilder(geomKind) + .append(geoJson) + .setIdColumn(columnName = mapJoinColumn, values = joinIds) + + dataAndCoordinates = rightJoin( + left = dataFrame, + leftKey = dataJoinColumn, + right = createDataFrame(coordinatesBuilder.build()), + rightKey = mapJoinColumn + ) + + val coordinatesAutoMapping = coordinatesBuilder.columns + .filterKeys { coordName -> coordName in variables(dataAndCoordinates) } + .map { (coordName, aes) -> aes to variables(dataAndCoordinates).getValue(coordName) } + .toMap() + mappings = createAesMapping(dataAndCoordinates, mappingOptions) + coordinatesAutoMapping + } + + companion object { + const val MAP_JOIN_REQUIRED_MESSAGE = "map_join is required when both data and map parameters used" + + fun isApplicable(layerOptions: Map<*, *>): Boolean { + return layerOptions.has(MAP_DATA_META, GDF, GEOMETRY) || + layerOptions.has(DATA_META, GDF, GEOMETRY) + } + } +} + +const val POINT_X = "__gdf_x__" +const val POINT_Y = "__gdf_y__" +const val RECT_XMIN = "__gdf_xmin__" +const val RECT_YMIN = "__gdf_ymin__" +const val RECT_XMAX = "__gdf_xmax__" +const val RECT_YMAX = "__gdf_ymax__" + +internal abstract class CoordinatesBuilder( + val columns: Map> +) { + companion object { + + fun createCoordinateBuilder(geomKind: GeomKind): CoordinatesBuilder { + return when(geomKind) { + MAP, POLYGON -> BoundaryCoordinatesBuilder() + POINT, TEXT -> PointCoordinatesBuilder() + RECT -> BboxCoordinatesBuilder() + PATH -> PathCoordinatesBuilder() + else -> error("Unsupported geom: $geomKind") + } + } + + val POINT_COLUMNS = mapOf( + POINT_X to Aes.X, + POINT_Y to Aes.Y + ) + + val RECT_COLUMNS = mapOf( + RECT_XMIN to Aes.XMIN, + RECT_YMIN to Aes.YMIN, + RECT_XMAX to Aes.XMAX, + RECT_YMAX to Aes.YMAX + ) + + internal fun Map>.append(p: Vec) { + append(POINT_X, p.x) + append(POINT_Y, p.y) + } + + internal fun Map>.append(rect: Rect) { + append(RECT_XMIN, rect.left) + append(RECT_XMAX, rect.right) + append(RECT_YMIN, rect.top) + append(RECT_YMAX, rect.bottom) + } + + private fun Map>.append(key: String, value: Double) { + get(key)?.add(value) ?: error("$key is not found") + } + } + + private var idColumnName: String? = null + private var ids: List? = null + private val groupLengths = mutableListOf() + protected val coordinates: Map> = columns.keys.associateBy({ it }) { mutableListOf() } + protected abstract val geoJsonConsumer: SimpleFeature.Consumer + protected abstract val supportedFeatures: List + + fun append(geoJsons: List): CoordinatesBuilder { + geoJsons.forEach { + val oldRowCount = coordinates.rowCount + GeoJson.parse(it, geoJsonConsumer) + groupLengths += coordinates.rowCount - oldRowCount + } + return this + } + + fun setIdColumn(columnName: String, values: List): CoordinatesBuilder { + idColumnName = columnName + ids = values + return this + } + + fun build(): Map> { + if (coordinates.rowCount == 0) { + error("Geometries are empty or no matching types. Expected: " + supportedFeatures) + } + + if (idColumnName == null && ids == null) { + return coordinates + } + + if (idColumnName != null && ids != null) { + require(groupLengths.size == ids!!.size) { "Groups and ids should have same size" } + + // (['a', 'b'], [2, 3]) => ['a', 'a', 'b', 'b', 'b'] + fun copies(values: Collection, count: Collection) = + values.asSequence().zip(count.asSequence()) + .fold(mutableListOf()) { acc, (value, count) -> repeat(count) { acc += value }; acc } + + return coordinates + (idColumnName!! to copies(ids!!, groupLengths)) + } + + error("idColumnName and idValues should be both null or not null") + } + + internal fun defaultConsumer(config: SimpleFeature.Consumer.() -> Unit) = + SimpleFeature.Consumer( + onPoint = {}, + onMultiPoint = {}, + onLineString = {}, + onMultiLineString = {}, + onPolygon = {}, + onMultiPolygon = {} + ).apply(config) + + private val > Map.rowCount get() = values.firstOrNull()?.size ?: 0 + + class PointCoordinatesBuilder : CoordinatesBuilder(POINT_COLUMNS) { + override val supportedFeatures = listOf("Point, MultiPoint") + override val geoJsonConsumer: SimpleFeature.Consumer = defaultConsumer { + onPoint = { p -> coordinates.append(p) } + onMultiPoint = { it.forEach { p -> coordinates.append(p) } } + } + } + + class PathCoordinatesBuilder : CoordinatesBuilder(POINT_COLUMNS) { + override val supportedFeatures = listOf("LineString, MultiLineString") + override val geoJsonConsumer: SimpleFeature.Consumer = defaultConsumer { + onLineString = { it.forEach { p -> coordinates.append(p) } } + onMultiLineString = { it.asSequence().flatten().forEach { p -> coordinates.append(p) } } + } + } + + class BoundaryCoordinatesBuilder : CoordinatesBuilder(POINT_COLUMNS) { + override val supportedFeatures = listOf("Polygon, MultiPolygon") + override val geoJsonConsumer: SimpleFeature.Consumer = defaultConsumer { + onPolygon = { it.asSequence().flatten().forEach { p -> coordinates.append(p) } } + onMultiPolygon = { it.asSequence().flatten().flatten().forEach { p -> coordinates.append(p) } } + } + } + + class BboxCoordinatesBuilder : CoordinatesBuilder(RECT_COLUMNS) { + override val supportedFeatures = listOf("MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon") + override val geoJsonConsumer: SimpleFeature.Consumer = defaultConsumer { + fun insert(bboxes: List>) = + bboxes + .run(BBOX_CALCULATOR::union) + .run(::convertToGeoRectangle) + .run(GeoRectangle::splitByAntiMeridian) + .forEach{ r -> coordinates.append(r) } + + fun insert(bbox: Rect) = insert(listOf(bbox)) + + onMultiPoint = { insert(it.boundingBox()) } + onLineString = { insert(it.boundingBox()) } + onMultiLineString = { insert(it.flatten().boundingBox()) } + onPolygon = { insert(it.limit()) } + onMultiPolygon = { insert(it.limit()) } + } + } +} + + +fun Map<*, *>.dataJoinVariable() = getList(MAP_JOIN)?.get(0) as? String \ No newline at end of file diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoPositionsDataUtil.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoPositionsDataUtil.kt deleted file mode 100644 index 2d9b6e977fa..00000000000 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoPositionsDataUtil.kt +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright (c) 2019. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.config - -import jetbrains.datalore.base.values.Pair -import jetbrains.datalore.plot.base.Aes -import jetbrains.datalore.plot.base.DataFrame -import jetbrains.datalore.plot.base.DataFrame.Variable -import jetbrains.datalore.plot.base.GeomKind -import jetbrains.datalore.plot.base.data.DataFrameUtil -import jetbrains.datalore.plot.builder.map.GeoPositionField -import jetbrains.datalore.plot.config.GeoPositionsDataUtil.GeoDataKind.* -import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS -import jetbrains.datalore.plot.config.Option.Meta.MapJoin - -object GeoPositionsDataUtil { - // Provided by regions object - const val MAP_REGION_COLUMN = "region" - - const val MAP_OSM_ID_COLUMN = "__geoid__" - - val GEOMS_SUPPORT = mapOf( - GeomKind.MAP to GeoDataSupport(BOUNDARY, ::createPointMapping), - GeomKind.POLYGON to GeoDataSupport(BOUNDARY, ::createPointMapping), - GeomKind.POINT to GeoDataSupport(POINT, ::createPointMapping), - GeomKind.RECT to GeoDataSupport(BBOX, ::createRectMapping), - GeomKind.PATH to GeoDataSupport(PATH, ::createPointMapping), - GeomKind.TEXT to GeoDataSupport(POINT, ::createPointMapping) - ) - - fun isGeomSupported(geomKind: GeomKind): Boolean { - return geomKind in GEOMS_SUPPORT - } - - fun getGeoDataKind(geomKind: GeomKind): GeoDataKind { - return GEOMS_SUPPORT[geomKind]!!.geoDataKind - } - - internal fun getGeoPositionsData(layerConfig: LayerConfig): DataFrame { - return ConfigUtil.createDataFrame(layerConfig.getMap(GEO_POSITIONS)) - } - - internal fun initDataAndMappingForGeoPositions( - geomKind: GeomKind, - layerData: DataFrame, - mapOptions: DataFrame, - mappingOptions: Map<*, *>, - leftMapId: String, - rightMapId: String - ): Pair, Variable>> { - @Suppress("NAME_SHADOWING") - var layerData = layerData - - //val rightMapId = getGeoPositionsIdVar(mapOptions).name - layerData = ConfigUtil.rightJoin(layerData, leftMapId, mapOptions, rightMapId) - - val aesMapping = HashMap( - ConfigUtil.createAesMapping( - layerData, - mappingOptions - ) - ) - aesMapping.putAll( - generateMappings( - geomKind, - layerData - ) - ) - return Pair(layerData, aesMapping) - } - - private fun generateMappings(geomKind: GeomKind, layerData: DataFrame): Map, Variable> { - return if (isGeomSupported(geomKind)) { - GEOMS_SUPPORT[geomKind]!!.generateMapping(layerData) - } else { - emptyMap() - } - } - - private fun getGeoPositionsIdVar(mapOptions: DataFrame): Variable { - val variable = findFirstVariable(mapOptions, listOf(MapJoin.MAP_ID, MAP_REGION_COLUMN)) - if (variable != null) { - return variable - } - - throw IllegalArgumentException( - geoPositionsColumnNotFoundError( - "region id", - listOf(MapJoin.MAP_ID, MAP_REGION_COLUMN) - ) - ) - } - - private fun findMapping(aes: Aes<*>, names: List, dataFrame: DataFrame): Map, Variable> { - val variable = findFirstVariable(dataFrame, names) - ?: throw IllegalArgumentException( - geoPositionsColumnNotFoundError( - aes.name + "-column", - names - ) - ) - return mapOf(aes to variable) - } - - private fun findFirstVariable(data: DataFrame, names: Iterable): Variable? { - val variableMap = DataFrameUtil.variables(data) - for (name in names) { - if (variableMap.containsKey(name)) { - return variableMap[name] - } - } - return null - } - - private fun geoPositionsColumnNotFoundError(what: String, names: List): String { - return "Can't draw map: " + what + " not found. Geo position data must contain column " + - names.joinToString(" or ") { s -> "'$s'" } - } - - enum class GeoDataKind { - POINT, - PATH, - BBOX, - BOUNDARY - } - - private fun createRectMapping(dataFrame: DataFrame): Map, Variable> { - val mapping = HashMap, Variable>() - mapping.putAll( - findMapping( - Aes.XMIN, - listOf(GeoPositionField.RECT_XMIN), - dataFrame - ) - ) - mapping.putAll( - findMapping( - Aes.XMAX, - listOf(GeoPositionField.RECT_XMAX), - dataFrame - ) - ) - mapping.putAll( - findMapping( - Aes.YMIN, - listOf(GeoPositionField.RECT_YMIN), - dataFrame - ) - ) - mapping.putAll( - findMapping( - Aes.YMAX, - listOf(GeoPositionField.RECT_YMAX), - dataFrame - ) - ) - return mapping - } - - private fun createPointMapping(dataFrame: DataFrame): Map, Variable> { - val mapping = HashMap, Variable>() - mapping.putAll( - findMapping( - Aes.X, - listOf( - GeoPositionField.POINT_X, - "x", - GeoPositionField.POINT_X2 - ), - dataFrame - ) - ) - mapping.putAll( - findMapping( - Aes.Y, - listOf(GeoPositionField.POINT_Y, "y"), - dataFrame - ) - ) - - return mapping - } - - class GeoDataSupport( - val geoDataKind: GeoDataKind, - private val mappingsGenerator: (DataFrame) -> Map, Variable> - ) { - - fun generateMapping(df: DataFrame): Map, Variable> = mappingsGenerator(df) - - } -} diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/LayerConfig.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/LayerConfig.kt index f2149376e1e..13f5f53ccff 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/LayerConfig.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/LayerConfig.kt @@ -18,6 +18,7 @@ import jetbrains.datalore.plot.builder.assemble.PosProvider import jetbrains.datalore.plot.builder.assemble.TypedScaleProviderMap import jetbrains.datalore.plot.builder.sampling.Sampling import jetbrains.datalore.plot.builder.tooltip.TooltipLineSpecification +import jetbrains.datalore.plot.config.ConfigUtil.createAesMapping import jetbrains.datalore.plot.config.DataMetaUtil.createDataFrame import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS import jetbrains.datalore.plot.config.Option.Layer.GEOM @@ -97,30 +98,23 @@ class LayerConfig( var aesMappings: Map, DataFrame.Variable>? - if (has(GEO_POSITIONS) && myClientSide) { - // join dataset and geo-positions data - val dataAndMapping = GeoPositionsDataUtil.initDataAndMappingForGeoPositions( + if (myClientSide && GeoConfig.isApplicable(layerOptions)) { + val geoConfig = GeoConfig( geomProto.geomKind, combinedData, - GeoPositionsDataUtil.getGeoPositionsData(this), - combinedMappings, - getString(MapJoin.DATA_JOIN_COLUMN)!!, - getString(MapJoin.MAP_JOIN_COLUMN)!! + layerOptions, + combinedMappings ) - combinedData = dataAndMapping.first - aesMappings = dataAndMapping.second + combinedData = geoConfig.dataAndCoordinates + aesMappings = geoConfig.mappings + } else { - aesMappings = ConfigUtil.createAesMapping(combinedData, combinedMappings) + aesMappings = createAesMapping(combinedData, combinedMappings) } // exclude constant aes from mapping - val constants = LayerConfigUtil.initConstants(this) - if (constants.isNotEmpty()) { - aesMappings = HashMap(aesMappings) - for (aes in constants.keys) { - aesMappings.remove(aes) - } - } + constantsMap = LayerConfigUtil.initConstants(this) + aesMappings = aesMappings - constantsMap.keys // grouping explicitGroupingVarName = initGroupingVarName(combinedData, combinedMappings) @@ -131,7 +125,6 @@ class LayerConfig( this, geomProto.preferredPositionAdjustments(this) ) - constantsMap = constants val consumedAesSet = HashSet(geomProto.renders()) if (!myClientSide) { diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/Option.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/Option.kt index ac23128f106..f5f98ee6e55 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/Option.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/Option.kt @@ -29,32 +29,20 @@ object Option { } object GeoDataFrame { - const val TAG = "geodataframe" - const val GEOMETRY_COLUMN_NAME = "geometry" + const val GDF = "geodataframe" + const val GEOMETRY = "geometry" // Column with geometries extracted from GeoDataFrame. Can be used either in DATA and MAP const val GEOMETRIES = "__geometry__" } - object GeoReference { - const val TAG = "georeference" - const val REQUEST = "request" - const val MAP_REGION_COLUMN = "region" - const val OSM_ID = "id" - } - object GeoDict { const val TAG = "geodict" } object MapJoin { - const val MAP_JOIN_COLUMN = "__map_join_column__" - const val DATA_JOIN_COLUMN = "__data_join_column__" - // column in map used for join with data const val MAP_ID = "__map_id__" - // generated data column for joined geometries from map - const val DATA_ID = "__data_id__" } object MappingAnnotation { diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSide.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSide.kt index 138f5a70f09..9cdda006d38 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSide.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSide.kt @@ -14,7 +14,6 @@ import jetbrains.datalore.plot.builder.theme.Theme import jetbrains.datalore.plot.config.Option.Plot.COORD import jetbrains.datalore.plot.config.Option.Plot.THEME import jetbrains.datalore.plot.config.PlotConfigClientSideUtil.createGuideOptionsMap -import jetbrains.datalore.plot.config.geo.GeometryFromGeoDataFrameChange import jetbrains.datalore.plot.config.theme.ThemeConfig import jetbrains.datalore.plot.config.transform.PlotSpecTransform import jetbrains.datalore.plot.config.transform.migration.MoveGeomPropertiesToLayerMigration @@ -73,7 +72,6 @@ class PlotConfigClientSide private constructor(opts: Map) : PlotCon val isGGBunch = isGGBunchSpec(plotSpec) plotSpec = PlotSpecTransform.builderForRawSpec() - .change(GeometryFromGeoDataFrameChange.specSelector(isGGBunch), GeometryFromGeoDataFrameChange()) .build() .apply(plotSpec) diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSideUtil.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSideUtil.kt index 77f87a679e9..2810dc31c79 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSideUtil.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSideUtil.kt @@ -21,8 +21,8 @@ import jetbrains.datalore.plot.builder.interact.GeomInteractionBuilder.Companion import jetbrains.datalore.plot.builder.interact.GeomInteractionBuilder.Companion.NON_AREA_GEOM import jetbrains.datalore.plot.builder.theme.Theme import jetbrains.datalore.plot.builder.tooltip.CompositeValue -import jetbrains.datalore.plot.config.Option.Meta.MapJoin import jetbrains.datalore.plot.builder.tooltip.TooltipLineSpecification +import jetbrains.datalore.plot.config.Option.Meta.MapJoin object PlotConfigClientSideUtil { internal fun createGuideOptionsMap(scaleConfigs: List>): Map, GuideOptions> { @@ -133,6 +133,10 @@ object PlotConfigClientSideUtil { layerBuilder.groupingVarName(layerConfig.explicitGroupingVarName!!) } + layerConfig.mergedOptions.dataJoinVariable()?.let { + layerBuilder.pathIdVarName(it) + } + // variable bindings val bindings = layerConfig.varBindings for (binding in bindings) { diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeometryFromGeoDataFrameChange.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeometryFromGeoDataFrameChange.kt deleted file mode 100644 index e24f0fd2787..00000000000 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeometryFromGeoDataFrameChange.kt +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) 2020. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.config.geo - -import jetbrains.datalore.base.spatial.* -import jetbrains.datalore.base.typedGeometry.* -import jetbrains.datalore.plot.builder.map.GeoPositionField.POINT_X -import jetbrains.datalore.plot.builder.map.GeoPositionField.POINT_Y -import jetbrains.datalore.plot.builder.map.GeoPositionField.RECT_XMAX -import jetbrains.datalore.plot.builder.map.GeoPositionField.RECT_XMIN -import jetbrains.datalore.plot.builder.map.GeoPositionField.RECT_YMAX -import jetbrains.datalore.plot.builder.map.GeoPositionField.RECT_YMIN -import jetbrains.datalore.plot.config.GeoPositionsDataUtil.GeoDataKind -import jetbrains.datalore.plot.config.Option -import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame -import jetbrains.datalore.plot.config.Option.Meta.MapJoin -import jetbrains.datalore.plot.config.getList -import jetbrains.datalore.plot.config.transform.SpecSelector - - -class GeometryFromGeoDataFrameChange : GeometryFromGeoPositionsChange() { - override val geoPositionsKeys: Set - get() = GEO_DATA_FRAME_KEYS - - override fun changeGeoPositions(mapSpec: MutableMap, geoDataKind: GeoDataKind) { - val geometryTables = mapSpec.getList(GeoDataFrame.GEOMETRIES)!!.map { parseGeometry(it as String, geoDataKind) } - - if (geometryTables.sumBy { it.rowCount } == 0) { - error( - "Geometries are empty or no matching types. Expected: " + - when (geoDataKind) { - GeoDataKind.POINT -> "Point, MultiPoint" - GeoDataKind.PATH -> "LineString, MultiLineString" - GeoDataKind.BOUNDARY -> "Polygon, MultiPolygon" - GeoDataKind.BBOX -> "MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon" - } - ) - } - - val dataTable = mapSpec.getList(MapJoin.MAP_ID) - ?.zip(geometryTables) - ?.fold(mutableMapOf>(), { dataFrame, (mapId, geometryTable) -> - dataFrame - .concat(geometryTable) - .concat(MapJoin.DATA_ID, MutableList(geometryTable.rowCount) { mapId!! }) - }) - ?: geometryTables.fold(mutableMapOf>(), { dataFrame, geometryTable -> - dataFrame.concat(geometryTable) - }) - - mapSpec.clear() - mapSpec.putAll(dataTable) - } - - private fun parseGeometry(geoJson: String, geoDataKind: GeoDataKind): MutableMap> { - val geometryTable = mutableMapOf>() - - when (geoDataKind) { - GeoDataKind.POINT -> defaultConsumer { - onPoint = geometryTable::append - onMultiPoint = { it.forEach(geometryTable::append) } - } - GeoDataKind.PATH -> defaultConsumer { - onLineString = { it.forEach(geometryTable::append) } - onMultiLineString = { it.flatten().forEach(geometryTable::append) } - } - GeoDataKind.BOUNDARY -> defaultConsumer { - onPolygon = { it.flatten().forEach(geometryTable::append) } - onMultiPolygon = { it.flatten().flatten().forEach(geometryTable::append) } - } - GeoDataKind.BBOX -> { - fun insert(bboxes: List>) = - bboxes - .run(BBOX_CALCULATOR::union) - .run(::convertToGeoRectangle) - .run(GeoRectangle::splitByAntiMeridian) - .forEach(geometryTable::append) - - fun insert(bbox: Rect) = insert(listOf(bbox)) - - defaultConsumer { - onMultiPoint = { insert(it.boundingBox()) } - onLineString = { insert(it.boundingBox()) } - onMultiLineString = { insert(it.flatten().boundingBox()) } - onPolygon = { insert(it.limit()) } - onMultiPolygon = { insert(it.limit()) } - } - - } - }.let { GeoJson.parse(geoJson, it) } - - return geometryTable - } - - companion object { - fun specSelector(isGGBunch: Boolean) = SpecSelector.from( - if (isGGBunch) { - listOf(Option.GGBunch.ITEMS, Option.GGBunch.Item.FEATURE_SPEC, Option.Plot.LAYERS); - } else { - listOf(Option.Plot.LAYERS); - } - ) - - private val GEO_DATA_FRAME_KEYS: Set = setOf(GeoDataFrame.GEOMETRIES) - - internal fun defaultConsumer(config: SimpleFeature.Consumer.() -> Unit) = - SimpleFeature.Consumer( - onPoint = {}, - onMultiPoint = {}, - onLineString = {}, - onMultiLineString = {}, - onPolygon = {}, - onMultiPolygon = {} - ).apply(config) - } -} - -private val > Map.rowCount get() = values.firstOrNull()?.size ?: 0 - -private fun MutableMap>.concat(other: Map>) = apply { - other.forEach { (key, value) -> getOrPut(key, { ArrayList() }).addAll(value) } -} - -private fun MutableMap>.concat(column: String, values: List) = apply { - concat(mutableMapOf(column to values)) -} - -private fun MutableMap>.append(key: String, value: Double) { - getOrPut(key, { mutableListOf() }).add(value) -} - -private fun MutableMap>.append(p: Vec) { - append(POINT_X, p.x) - append(POINT_Y, p.y) -} - -private fun MutableMap>.append(rect: Rect) { - append(RECT_XMIN, rect.left) - append(RECT_XMAX, rect.right) - append(RECT_YMIN, rect.top) - append(RECT_YMAX, rect.bottom) -} diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeometryFromGeoPositionsChange.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeometryFromGeoPositionsChange.kt deleted file mode 100644 index 536b57d0a90..00000000000 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeometryFromGeoPositionsChange.kt +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2020. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.config.geo - -import jetbrains.datalore.plot.base.GeomKind -import jetbrains.datalore.plot.config.GeoPositionsDataUtil -import jetbrains.datalore.plot.config.GeoPositionsDataUtil.getGeoDataKind -import jetbrains.datalore.plot.config.GeoPositionsDataUtil.isGeomSupported -import jetbrains.datalore.plot.config.Option -import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS -import jetbrains.datalore.plot.config.Option.Layer.GEOM -import jetbrains.datalore.plot.config.asMutable -import jetbrains.datalore.plot.config.getMap -import jetbrains.datalore.plot.config.transform.SpecChange -import jetbrains.datalore.plot.config.transform.SpecChangeContext - - -abstract class GeometryFromGeoPositionsChange : SpecChange { - override fun isApplicable(spec: Map): Boolean { - if (spec[GEO_POSITIONS] !is MutableMap<*, *>) { - return false - } - if (!(spec[GEO_POSITIONS] as MutableMap<*, *>?)!!.keys.containsAll(geoPositionsKeys)) { - return false - } - return isGeomSupported(getGeomKind(spec)) - } - - override fun apply(spec: MutableMap, ctx: SpecChangeContext) { - changeGeoPositions( - mapSpec = spec.getMap(GEO_POSITIONS)!!.asMutable(), - geoDataKind = getGeoDataKind(getGeomKind(spec)) - ) - } - - abstract val geoPositionsKeys: Set - abstract fun changeGeoPositions(mapSpec: MutableMap, geoDataKind: GeoPositionsDataUtil.GeoDataKind) - - companion object { - private fun getGeomKind(layerSpec: Map): GeomKind { - val name = layerSpec[GEOM] as String - return Option.GeomName.toGeomKind(name) - } - } -} \ No newline at end of file diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/PlotConfigServerSide.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/PlotConfigServerSide.kt index 485a51ccec0..e4dae3f6870 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/PlotConfigServerSide.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/PlotConfigServerSide.kt @@ -17,6 +17,9 @@ import jetbrains.datalore.plot.builder.tooltip.TooltipLineSpecification import jetbrains.datalore.plot.builder.tooltip.VariableValue import jetbrains.datalore.plot.config.* import jetbrains.datalore.plot.config.Option.Layer.MAP_JOIN +import jetbrains.datalore.plot.config.Option.Meta.DATA_META +import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame.GDF +import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame.GEOMETRY import jetbrains.datalore.plot.server.config.transform.PlotConfigServerSideTransforms.entryTransform import jetbrains.datalore.plot.server.config.transform.PlotConfigServerSideTransforms.migrationTransform @@ -116,12 +119,10 @@ open class PlotConfigServerSide(opts: Map) : PlotConfig(opts) { if (!DataFrameUtil.variables(layerData!!).containsKey(varName)) { dropPlotVar = when { - // don't drop if used in mapping - layerConfig.hasVarBinding(varName) -> false + layerConfig.hasVarBinding(varName) -> false // don't drop if used in mapping layerConfig.isExplicitGrouping(varName) -> false - // don't drop if used for facets - varName == facets.xVar -> false - varName == facets.yVar -> false + varName == facets.xVar -> false // don't drop if used for facets + varName == facets.yVar -> false // don't drop if used for facets else -> true } if (!dropPlotVar) { @@ -189,6 +190,7 @@ open class PlotConfigServerSide(opts: Map) : PlotConfig(opts) { val varNamesToKeep = HashSet() + varsToKeep.map(Variable::name) + Stats.GROUP.name + + listOfNotNull(layerConfig.mergedOptions.getString(DATA_META, GDF, GEOMETRY)) + listOfNotNull(layerConfig.mergedOptions.getList(MAP_JOIN)?.get(0) as? String) + listOfNotNull(facets.xVar, facets.yVar, layerConfig.explicitGroupingVarName) @@ -226,8 +228,11 @@ open class PlotConfigServerSide(opts: Map) : PlotConfig(opts) { val tileLayerInputData = inputDataByTileByLayer[tileIndex][layerIndex] val varBindings = layerConfig.varBindings val groupingContext = GroupingContext( - tileLayerInputData, - varBindings, layerConfig.explicitGroupingVarName, true + myData = tileLayerInputData, + bindings = varBindings, + groupingVarName = layerConfig.explicitGroupingVarName, + pathIdVarName = null, // only on client side + myExpectMultiple = true ) val groupingContextAfterStat: GroupingContext diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoDataFrameMappingChange.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoDataFrameMappingChange.kt deleted file mode 100644 index 2f37b07510a..00000000000 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoDataFrameMappingChange.kt +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.server.config.transform - -import jetbrains.datalore.plot.config.* -import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS -import jetbrains.datalore.plot.config.Option.Meta.DATA_META -import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame -import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame.GEOMETRY_COLUMN_NAME -import jetbrains.datalore.plot.config.Option.Meta.MapJoin -import jetbrains.datalore.plot.config.Option.Plot -import jetbrains.datalore.plot.config.Option.PlotBase.DATA -import jetbrains.datalore.plot.config.transform.SpecChange -import jetbrains.datalore.plot.config.transform.SpecChangeContext -import jetbrains.datalore.plot.config.transform.SpecSelector - -class GeoDataFrameMappingChange : SpecChange { - - override fun apply(spec: MutableMap, ctx: SpecChangeContext) { - val geometryColumnName = spec.read(DATA_META, GeoDataFrame.TAG, GEOMETRY_COLUMN_NAME) as String - val geometries = spec.getList(DATA, geometryColumnName)!! - val ids = geometries.indices.map(Int::toString) - - spec.remove(DATA, geometryColumnName) - spec.write(DATA, MapJoin.MAP_ID) { ids } - spec.write(GEO_POSITIONS, MapJoin.MAP_ID) { ids } - spec.write(GEO_POSITIONS, GeoDataFrame.GEOMETRIES) { geometries} - // TODO: Fix MAP_ID - //spec.write(MAPPING, MAP_ID) { MapJoin.ID } - } - - override fun isApplicable(spec: Map): Boolean { - return (!spec.has(GEO_POSITIONS) - && spec.has(DATA_META, GeoDataFrame.TAG)) - } - - companion object { - internal fun specSelector() = SpecSelector.of(Plot.LAYERS) - } -} diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoPositionMappingChange.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoPositionMappingChange.kt deleted file mode 100644 index 02847f51b2b..00000000000 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/GeoPositionMappingChange.kt +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2019. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.server.config.transform - -import jetbrains.datalore.plot.config.GeoPositionsDataUtil.MAP_OSM_ID_COLUMN -import jetbrains.datalore.plot.config.GeoPositionsDataUtil.MAP_REGION_COLUMN -import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS -import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame -import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame.GEOMETRY_COLUMN_NAME -import jetbrains.datalore.plot.config.Option.Meta.GeoDict -import jetbrains.datalore.plot.config.Option.Meta.GeoReference -import jetbrains.datalore.plot.config.Option.Meta.MAP_DATA_META -import jetbrains.datalore.plot.config.Option.Meta.MapJoin -import jetbrains.datalore.plot.config.Option.Plot -import jetbrains.datalore.plot.config.getMap -import jetbrains.datalore.plot.config.has -import jetbrains.datalore.plot.config.read -import jetbrains.datalore.plot.config.transform.SpecChange -import jetbrains.datalore.plot.config.transform.SpecChangeContext -import jetbrains.datalore.plot.config.transform.SpecSelector -import jetbrains.datalore.plot.config.write - -class GeoPositionMappingChange : SpecChange { - - override fun apply(spec: MutableMap, ctx: SpecChangeContext) { - val mapSpec = spec.getMap(GEO_POSITIONS)!! - val mapJoinIds = spec.read(MapJoin.MAP_JOIN_COLUMN)?.let { mapSpec.read(it as String) } - - if (spec.has(MAP_DATA_META, GeoDataFrame.TAG)) { - // Select column applicable for join - listOfNotNull( - mapJoinIds, // user defined column via parameter `map_join` - mapSpec.read(GeoReference.REQUEST), // Regions object from our geocoding - mapSpec.read(MAP_REGION_COLUMN) // ??? - ).firstOrNull()?.let { mapSpec.write(MapJoin.MAP_ID) { it } } - - spec.read(MAP_DATA_META, GeoDataFrame.TAG, GEOMETRY_COLUMN_NAME) - ?.let { geometryColumnName -> mapSpec.read(geometryColumnName as String)!! } - ?.let { geometries -> mapSpec.write(GeoDataFrame.GEOMETRIES) { geometries } } - } - - if (spec.has(MAP_DATA_META, GeoReference.TAG)) { - mapSpec.write(MapJoin.MAP_ID) { mapSpec.read(GeoReference.REQUEST)!! } - mapSpec.write(MAP_OSM_ID_COLUMN) { mapSpec.read(GeoReference.OSM_ID)!! } - } - - if (spec.has(MAP_DATA_META, GeoDict.TAG)) { - mapJoinIds?.let { mapSpec.write(MapJoin.MAP_ID) { it } } - } - } - - override fun isApplicable(spec: Map): Boolean { - return spec.has(MAP_DATA_META, GeoDataFrame.TAG) || - spec.has(MAP_DATA_META, GeoReference.TAG) || - spec.has(MAP_DATA_META, GeoDict.TAG) - } - - companion object { - internal fun specSelector() = SpecSelector.of(Plot.LAYERS) - } -} diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/MapJoinChange.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/MapJoinChange.kt deleted file mode 100644 index fa1d2b9efe0..00000000000 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/MapJoinChange.kt +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2020. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.server.config.transform - -import jetbrains.datalore.plot.config.Option -import jetbrains.datalore.plot.config.Option.Layer.MAP_JOIN -import jetbrains.datalore.plot.config.Option.Meta.MAP_DATA_META -import jetbrains.datalore.plot.config.Option.Meta.MapJoin -import jetbrains.datalore.plot.config.getList -import jetbrains.datalore.plot.config.transform.SpecChange -import jetbrains.datalore.plot.config.transform.SpecChangeContext -import jetbrains.datalore.plot.config.transform.SpecSelector -import jetbrains.datalore.plot.config.write - -class MapJoinChange: SpecChange { - override fun apply(spec: MutableMap, ctx: SpecChangeContext) { - val (dataJoinColumn, mapJoinColumn) = spec.getList(MAP_JOIN)!! - dataJoinColumn?.let { spec.write(MapJoin.DATA_JOIN_COLUMN) { it } } - mapJoinColumn?.let { spec.write(MapJoin.MAP_JOIN_COLUMN) { it } } - } - - override fun isApplicable(spec: Map): Boolean { - return spec.contains(MAP_JOIN) - && spec.contains(MAP_DATA_META) - } - - companion object { - internal fun specSelector(): SpecSelector { - return SpecSelector.of(Option.Plot.LAYERS) - } - } -} diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/PlotConfigServerSideTransforms.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/PlotConfigServerSideTransforms.kt index 1b42271f4d7..e5debc46a56 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/PlotConfigServerSideTransforms.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/server/config/transform/PlotConfigServerSideTransforms.kt @@ -41,18 +41,6 @@ object PlotConfigServerSideTransforms { ReplaceDataVectorsInAesMappingChange.specSelector(), ReplaceDataVectorsInAesMappingChange() ) - .change( - MapJoinChange.specSelector(), - MapJoinChange() - ) - .change( - GeoDataFrameMappingChange.specSelector(), - GeoDataFrameMappingChange() - ) - .change( - GeoPositionMappingChange.specSelector(), - GeoPositionMappingChange() - ) .build() } } diff --git a/plot-config/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeoPositionResolver.kt b/plot-config/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeoPositionResolver.kt deleted file mode 100644 index 9d2be246423..00000000000 --- a/plot-config/src/commonMain/kotlin/jetbrains/datalore/plot/config/geo/GeoPositionResolver.kt +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2019. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.config.geo - -import jetbrains.datalore.base.async.Async -import jetbrains.datalore.base.async.Asyncs -import jetbrains.datalore.plot.config.Option.GGBunch.ITEMS -import jetbrains.datalore.plot.config.Option.GGBunch.Item.FEATURE_SPEC -import jetbrains.datalore.plot.config.Option.GeomName.LIVE_MAP -import jetbrains.datalore.plot.config.Option.Layer.GEOM -import jetbrains.datalore.plot.config.Option.Plot.LAYERS -import jetbrains.datalore.plot.config.PlotConfig -import jetbrains.datalore.plot.config.transform.PlotSpecTransform -import jetbrains.datalore.plot.config.transform.SpecFinder -import jetbrains.datalore.plot.config.transform.SpecSelector.Companion.of - -object GeoPositionsResolver { - - public fun fetchAndReplace( - plotSpec: Map - ) : Async> { - val specTransformBuilder = PlotSpecTransform.builderForRawSpec(); - - // Process geodataframe - specTransformBuilder.change( - of(*layersPath(plotSpec).toTypedArray()), - GeometryFromGeoDataFrameChange() - ); - - - // run - val resultPlotSpec: Map = specTransformBuilder.build().apply(plotSpec.toMutableMap()); - - return Asyncs.constant(resultPlotSpec) - } - - private fun containsLivemap(plotSpec: Map): Boolean { - var containsLivemap = false; - for (it in SpecFinder(layersPath(plotSpec)).findSpecs(plotSpec as Map<*, *>)) { - if (it.get(GEOM)!! == LIVE_MAP) { - containsLivemap = true - break - } - } - return containsLivemap; - } - - internal fun layersPath(plotSpec: Map): List { - if (PlotConfig.isPlotSpec(plotSpec)) { - return listOf(LAYERS); - } - - if (PlotConfig.isGGBunchSpec(plotSpec)) { - return listOf(ITEMS, FEATURE_SPEC, LAYERS); - } - - throw IllegalArgumentException("Plot spec kind is not set"); - } -} - - diff --git a/plot-config/src/jvmTest/kotlin/plot/config/AsDiscreteTest.kt b/plot-config/src/jvmTest/kotlin/plot/config/AsDiscreteTest.kt index 1bd8eec42eb..fd6f1b9f28e 100644 --- a/plot-config/src/jvmTest/kotlin/plot/config/AsDiscreteTest.kt +++ b/plot-config/src/jvmTest/kotlin/plot/config/AsDiscreteTest.kt @@ -6,23 +6,14 @@ package jetbrains.datalore.plot.config import jetbrains.datalore.plot.base.Aes -import jetbrains.datalore.plot.base.DataFrame -import jetbrains.datalore.plot.base.data.DataFrameUtil -import jetbrains.datalore.plot.base.data.DataFrameUtil.variables import jetbrains.datalore.plot.base.stat.Stats import jetbrains.datalore.plot.config.AsDiscreteTest.Storage.LAYER import jetbrains.datalore.plot.config.AsDiscreteTest.Storage.PLOT import jetbrains.datalore.plot.config.DataMetaUtil.toDiscrete -import jetbrains.datalore.plot.config.PlotConfig.Companion.getErrorMessage -import jetbrains.datalore.plot.config.PlotConfig.Companion.isFailure -import jetbrains.datalore.plot.parsePlotSpec -import jetbrains.datalore.plot.server.config.ServerSideTestUtil import org.junit.Ignore import org.junit.Test import kotlin.math.pow import kotlin.test.assertEquals -import kotlin.test.assertTrue -import kotlin.test.fail class AsDiscreteTest { @@ -91,7 +82,7 @@ class AsDiscreteTest { mappingStorage = LAYER ) - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable(toDiscrete("g"), isDiscrete = true) } @@ -104,7 +95,7 @@ class AsDiscreteTest { mappingStorage = PLOT ) - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable(toDiscrete("g"), isDiscrete = true) } @@ -117,7 +108,7 @@ class AsDiscreteTest { mappingStorage = LAYER ) - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable(toDiscrete("g"), isDiscrete = true) } @@ -130,7 +121,7 @@ class AsDiscreteTest { mappingStorage = PLOT ) - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable(toDiscrete("g"), isDiscrete = true) } @@ -143,7 +134,7 @@ class AsDiscreteTest { mappingStorage = LAYER ) - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable(toDiscrete("g"), isDiscrete = true) } @@ -156,7 +147,7 @@ class AsDiscreteTest { mappingStorage = PLOT ) - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable(toDiscrete("g"), isDiscrete = true) } @@ -169,7 +160,7 @@ class AsDiscreteTest { mappingStorage = LAYER ) - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable(toDiscrete("g"), isDiscrete = true) } @@ -182,7 +173,7 @@ class AsDiscreteTest { mappingStorage = PLOT ) - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable(toDiscrete("g"), isDiscrete = true) } @@ -216,7 +207,7 @@ class AsDiscreteTest { | ] |}""".trimMargin() - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable(toDiscrete("g"), isDiscrete = true) } @@ -251,7 +242,7 @@ class AsDiscreteTest { | ] |}""".trimMargin() - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true) .hasVariable(Stats.GROUP) .assertVariable(varName = "g", isDiscrete = false) @@ -337,7 +328,7 @@ class AsDiscreteTest { barData = case.barData, barMapping = case.barMapping ) - .let(::toClientPlotConfig) + .let(::transformToClientPlotConfig) .assertScale(Aes.COLOR, isDiscrete = true) { case.toString() } .assertScale(Aes.FILL, isDiscrete = true) { case.toString() } .assertVariable(toDiscrete("foo"), isDiscrete = true) { case.toString() } @@ -365,7 +356,7 @@ class AsDiscreteTest { | ] |}""".trimMargin() - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable("d", isDiscrete = true) } @@ -396,7 +387,7 @@ class AsDiscreteTest { | } | ] |}""".trimMargin() - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable(toDiscrete("g"), isDiscrete = true) } @@ -430,7 +421,7 @@ class AsDiscreteTest { | ] |}""".trimMargin() - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.FILL, isDiscrete = true) .assertScale(Aes.COLOR, isDiscrete = false) .assertVariable(toDiscrete("cyl"), isDiscrete = true) @@ -466,7 +457,7 @@ class AsDiscreteTest { | ] |}""".trimMargin() - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable(toDiscrete("cyl"), isDiscrete = true) } @@ -484,7 +475,7 @@ class AsDiscreteTest { layerData = null, layerMapping = false, layerAnnotation = false - ).let(::toClientPlotConfig) + ).let(::transformToClientPlotConfig) .assertScale(Aes.COLOR, isDiscrete = true) // as_discrete in plot .assertVariable(toDiscrete("cyl"), isDiscrete = true) // no overriding in layer } @@ -499,7 +490,7 @@ class AsDiscreteTest { layerData = null, layerMapping = true, layerAnnotation = true - ).let(::toClientPlotConfig) + ).let(::transformToClientPlotConfig) .assertScale(Aes.COLOR, isDiscrete = true) // as_discrete in plot .assertVariable(toDiscrete("cyl"), isDiscrete = true) // as_discrete in layer } @@ -514,7 +505,7 @@ class AsDiscreteTest { layerData = null, layerMapping = true, layerAnnotation = true - ).let(::toClientPlotConfig) + ).let(::transformToClientPlotConfig) .assertScale(Aes.COLOR, isDiscrete = true) // as_discrete in layer .assertVariable(toDiscrete("cyl"), isDiscrete = true) // as_discrete in layer } @@ -529,7 +520,7 @@ class AsDiscreteTest { layerData = null, layerMapping = true, layerAnnotation = false - ).let(::toClientPlotConfig) + ).let(::transformToClientPlotConfig) .assertScale(Aes.COLOR, isDiscrete = true) // as_discrete in plot .assertVariable("cyl", isDiscrete = false) // as is (numeric, overrided by layer) } @@ -545,7 +536,7 @@ class AsDiscreteTest { layerMapping = true, layerAnnotation = false ) - .let(::toClientPlotConfig) + .let(::transformToClientPlotConfig) .assertScale(Aes.COLOR, isDiscrete = false) .assertVariable("cyl", isDiscrete = false) .assertValue("cyl", listOf(4.0, 5.0, 6.0)) @@ -562,7 +553,7 @@ class AsDiscreteTest { layerMapping = false, layerAnnotation = false ) - .let(::toClientPlotConfig) + .let(::transformToClientPlotConfig) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable(toDiscrete("cyl"), isDiscrete = true) .assertValue(toDiscrete("cyl"), listOf(4.0, 5.0, 6.0)) @@ -579,7 +570,7 @@ class AsDiscreteTest { layerMapping = true, layerAnnotation = true ) - .let(::toClientPlotConfig) + .let(::transformToClientPlotConfig) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable(toDiscrete("cyl"), isDiscrete = true) .assertValue(toDiscrete("cyl"), listOf(4.0, 5.0, 6.0)) @@ -596,7 +587,7 @@ class AsDiscreteTest { layerMapping = true, layerAnnotation = false ) - .let(::toClientPlotConfig) + .let(::transformToClientPlotConfig) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable("cyl", isDiscrete = false) .assertValue("cyl", listOf(4.0, 5.0, 6.0)) @@ -613,7 +604,7 @@ class AsDiscreteTest { layerMapping = true, layerAnnotation = false ) - .let(::toClientPlotConfig) + .let(::transformToClientPlotConfig) .assertScale(Aes.COLOR, isDiscrete = true) .assertVariable("cyl", isDiscrete = false) .assertValue("cyl", listOf(1.0, 2.0, 3.0)) @@ -630,7 +621,7 @@ class AsDiscreteTest { layerMapping = true, layerAnnotation = false ) - .let(::toClientPlotConfig) + .let(::transformToClientPlotConfig) .assertScale(Aes.COLOR, isDiscrete = false) .assertVariable("cyl", isDiscrete = false) .assertValue("cyl", listOf(1.0, 2.0, 3.0)) @@ -668,7 +659,7 @@ class AsDiscreteTest { | ] |}""".trimMargin() - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true, name = "clndr") .assertVariable(toDiscrete("cyl"), isDiscrete = true) } @@ -716,7 +707,7 @@ class AsDiscreteTest { | ] |}""".trimMargin() - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true, name = "clndr") .assertVariable(toDiscrete("cyl"), isDiscrete = true) } @@ -762,61 +753,12 @@ class AsDiscreteTest { | ] |}""".trimMargin() - toClientPlotConfig(spec) + transformToClientPlotConfig(spec) .assertScale(Aes.COLOR, isDiscrete = true, name = "ndr") .assertVariable(toDiscrete("cyl"), isDiscrete = true) } } - - -private fun PlotConfigClientSide.assertValue(variable: String, values: List<*>): PlotConfigClientSide { - val data = layerConfigs.single().combinedData - assertEquals(values, data.get(variables(data)[variable]!!)) - return this -} - -private fun PlotConfigClientSide.assertVariable( - varName: String, - isDiscrete: Boolean, - msg: () -> String = { "" } -): PlotConfigClientSide { - val layer = layerConfigs.single() - if (!DataFrameUtil.hasVariable(layer.combinedData, varName)) { - fail("Variable $varName is not found in ${layer.combinedData.variables().map(DataFrame.Variable::name)}") - } - val dfVar = DataFrameUtil.findVariableOrFail(layer.combinedData, varName) - assertEquals(!isDiscrete, layer.combinedData.isNumeric(dfVar), msg()) - return this -} - -private fun PlotConfigClientSide.assertScale( - aes: Aes<*>, - isDiscrete: Boolean, - name: String? = null, - msg: () -> String = { "" } -): PlotConfigClientSide { - val layer = layerConfigs.single() - val binding = layer.varBindings.firstOrNull { it.aes == aes } ?: fail("$aes not found. ${msg()}") - assertEquals(!isDiscrete, binding.scale!!.isContinuous) - name?.let { assertEquals(it, binding.scale!!.name)} - return this -} - -private fun PlotConfigClientSide.hasVariable(variable: DataFrame.Variable): PlotConfigClientSide { - val layer = layerConfigs.single() - assertTrue(DataFrameUtil.hasVariable(layer.combinedData, variable.name)) - return this -} - -private fun toClientPlotConfig(spec: String): PlotConfigClientSide { - return parsePlotSpec(spec) - .let(ServerSideTestUtil::serverTransformWithoutEncoding) - .also { require(!isFailure(it)) { getErrorMessage(it) } } - .let(TestUtil::assertClientWontFail) -} - - @Suppress("ComplexRedundantLet") private fun buildSpecWithOverriding( geom: String, diff --git a/plot-config/src/jvmTest/kotlin/plot/config/Assertions.kt b/plot-config/src/jvmTest/kotlin/plot/config/Assertions.kt new file mode 100644 index 00000000000..51c5bea1869 --- /dev/null +++ b/plot-config/src/jvmTest/kotlin/plot/config/Assertions.kt @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2020. JetBrains s.r.o. + * Use of this source code is governed by the MIT license that can be found in the LICENSE file. + */ + +package jetbrains.datalore.plot.config + +import jetbrains.datalore.plot.base.Aes +import jetbrains.datalore.plot.base.DataFrame +import jetbrains.datalore.plot.base.data.DataFrameUtil +import jetbrains.datalore.plot.parsePlotSpec +import jetbrains.datalore.plot.server.config.ServerSideTestUtil +import kotlin.test.assertEquals +import kotlin.test.assertTrue +import kotlin.test.fail + +fun transformToClientPlotConfig(spec: String): PlotConfigClientSide { + return parsePlotSpec(spec) + .let(ServerSideTestUtil::serverTransformWithoutEncoding) + .also { require(!PlotConfig.isFailure(it)) { PlotConfig.getErrorMessage(it) } } + .let(TestUtil::assertClientWontFail) +} + +fun failedTransformToClientPlotConfig(spec: String): String { + return parsePlotSpec(spec) + .let(ServerSideTestUtil::serverTransformWithoutEncoding) + .let { + try { + PlotConfigClientSide.create(it) + } catch (e: Throwable) { + return@let e.localizedMessage as String + } + fail("Error expected") + } +} + +fun PlotConfigClientSide.assertValue(variable: String, values: List<*>): PlotConfigClientSide { + val data = layerConfigs.single().combinedData + assertEquals(values, data.get(DataFrameUtil.variables(data)[variable]!!)) + return this +} + +fun PlotConfigClientSide.assertVariable( + varName: String, + isDiscrete: Boolean, + msg: () -> String = { "" } +): PlotConfigClientSide { + val layer = layerConfigs.single() + if (!DataFrameUtil.hasVariable(layer.combinedData, varName)) { + fail("Variable $varName is not found in ${layer.combinedData.variables().map(DataFrame.Variable::name)}") + } + val dfVar = DataFrameUtil.findVariableOrFail(layer.combinedData, varName) + assertEquals(!isDiscrete, layer.combinedData.isNumeric(dfVar), msg()) + return this +} + +fun PlotConfigClientSide.assertScale( + aes: Aes<*>, + isDiscrete: Boolean, + name: String? = null, + msg: () -> String = { "" } +): PlotConfigClientSide { + val layer = layerConfigs.single() + val binding = layer.varBindings.firstOrNull { it.aes == aes } ?: fail("$aes not found. ${msg()}") + assertEquals(!isDiscrete, binding.scale!!.isContinuous) + name?.let { assertEquals(it, binding.scale!!.name) } + return this +} + +fun PlotConfigClientSide.hasVariable(variable: DataFrame.Variable): PlotConfigClientSide { + val layer = layerConfigs.single() + assertTrue(DataFrameUtil.hasVariable(layer.combinedData, variable.name)) + return this +} + +fun PlotConfigClientSide.assertBinding( + aes: Aes<*>, + varName: String +): PlotConfigClientSide { + val layer = layerConfigs.single() + + val aesBindings = layer.varBindings.associateBy { it.aes } + aesBindings[aes] + ?.let { binding -> assertEquals(varName, binding.variable.name) } + ?: fail("Binding for aes $aes was not found") + return this +} + +fun PlotConfigClientSide.assertNoBinding( + aes: Aes<*> +): PlotConfigClientSide { + val layer = layerConfigs.single() + + assert(layer.varBindings.none { it.aes == aes }) { "Binding for aes $aes was found" } + return this +} + diff --git a/plot-config/src/jvmTest/kotlin/plot/config/GeoConfigTest.kt b/plot-config/src/jvmTest/kotlin/plot/config/GeoConfigTest.kt new file mode 100644 index 00000000000..97f06b8d68c --- /dev/null +++ b/plot-config/src/jvmTest/kotlin/plot/config/GeoConfigTest.kt @@ -0,0 +1,263 @@ +/* + * Copyright (c) 2020. JetBrains s.r.o. + * Use of this source code is governed by the MIT license that can be found in the LICENSE file. + */ + +package jetbrains.datalore.plot.config + +import jetbrains.datalore.plot.base.Aes +import jetbrains.datalore.plot.base.DataPointAesthetics +import jetbrains.datalore.plot.builder.LayerRendererUtil.createLayerRendererData +import jetbrains.datalore.plot.config.GeoConfig.Companion.MAP_JOIN_REQUIRED_MESSAGE +import jetbrains.datalore.plot.config.PlotConfigClientSideUtil.createPlotAssembler +import jetbrains.datalore.plot.parsePlotSpec +import kotlin.test.Test +import kotlin.test.assertEquals + +class GeoConfigTest { + private val point = """{\"type\": \"Point\", \"coordinates\": [1.0, 2.0]}""" + private val multiPoint = """{\"type\": \"MultiPoint\", \"coordinates\": [[3.0, 4.0], [5.0, 6.0]]}""" + private val lineString = """{\"type\": \"LineString\", \"coordinates\": [[7.0, 8.0], [9.0, 10.0]]}""" + private val multiLineString = """{\"type\": \"MultiLineString\", \"coordinates\": [[[11.0, 12.0], [13.0, 14.0]], [[15.0, 16.0], [17.0, 18.0]]]}""" + private val polygon = """ +|{ +| \"type\": \"Polygon\", +| \"coordinates\":[ +| [ +| [21.0, 21.0], +| [21.0, 29.0], +| [29.0, 29.0], +| [29.0, 21.0], +| [21.0, 21.0] +| ], +| [ +| [22.0, 22.0], +| [23.0, 22.0], +| [23.0, 23.0], +| [22.0, 23.0], +| [22.0, 22.0] +| ], +| [ +| [24.0, 24.0], +| [26.0, 24.0], +| [26.0, 26.0], +| [24.0, 26.0], +| [24.0, 24.0] +| ] +| ] +|}""".trimMargin() + + private val multipolygon = """ +|{ +| \"type\": \"MultiPolygon\", +| \"coordinates\": [ +| [ +| [ +| [11.0, 12.0], +| [13.0, 14.0], +| [15.0, 13.0], +| [11.0, 12.0] +| ] +| ] +| ] +|}""".trimMargin() + + private val gdf = """ + |{ + | "kind": ["Point", "MPoint", "Line", "MLine", "Polygon", "MPolygon"], + | "coord": ["$point", "$multiPoint", "$lineString", "$multiLineString", "$polygon", "$multipolygon"] + |} + """.trimMargin() + + private val df = """ + |{ + | "fig": ["Point", "MPoint", "Line", "MLine", "Polygon", "MPolygon"], + | "value": [1, 2, 3, 4, 5, 6] + |} + """.trimMargin() + private fun `aes(color='kind'), data=gdf`(geom: String): PlotConfigClientSide { + return transformToClientPlotConfig(""" + |{ + | "kind": "plot", + | "layers": [{ + | "geom": "$geom", + | "mapping": {"color": "kind"}, + | "data": $gdf, + | "data_meta": {"geodataframe": {"geometry": "coord"}} + | }] + |} + """.trimMargin() + ) + } + + @Test + fun `geom_point(aes(color='kind'), gdf)`() { + `aes(color='kind'), data=gdf`(geom="point") + .assertBinding(Aes.X, POINT_X) + .assertBinding(Aes.Y, POINT_Y) + .assertBinding(Aes.COLOR, "kind") + } + + @Test + fun `geom_rect(aes(color='kind'), gdf)`() { + `aes(color='kind'), data=gdf`(geom="rect") + .assertBinding(Aes.XMIN, RECT_XMIN) + .assertBinding(Aes.XMAX, RECT_XMAX) + .assertBinding(Aes.YMIN, RECT_YMIN) + .assertBinding(Aes.YMAX, RECT_YMAX) + .assertBinding(Aes.COLOR, "kind") + } + + @Test + fun `geom_polygon(aes(color = 'kind'), gdf)`() { + `aes(color='kind'), data=gdf`(geom="polygon") + .assertBinding(Aes.X, POINT_X) + .assertBinding(Aes.Y, POINT_Y) + .assertBinding(Aes.COLOR, "kind") + } + + @Test + fun `geom_path(aes(color = 'kind'), gdf)`() { + `aes(color='kind'), data=gdf`(geom="path") + .assertBinding(Aes.X, POINT_X) + .assertBinding(Aes.Y, POINT_Y) + .assertBinding(Aes.COLOR, "kind") + } + + private fun `aes(color='value'), data=df, map=gdf, map_join=('fig', 'kind')`(geom: String): PlotConfigClientSide { + return transformToClientPlotConfig(""" + |{ + | "kind": "plot", + | "layers": [{ + | "geom": "$geom", + | "data": $df, + | "mapping": {"color": "value"}, + | "map": $gdf, + | "map_data_meta": {"geodataframe": {"geometry": "coord"}}, + | "map_join": ["fig", "kind"] + | }] + |} + """.trimMargin()) + } + + @Test + fun `geom_point(aes(color = 'value'), df, map = gdf, map_join=('fig', 'kind'))`() { + `aes(color='value'), data=df, map=gdf, map_join=('fig', 'kind')`(geom="point") + .assertBinding(Aes.X, POINT_X) + .assertBinding(Aes.Y, POINT_Y) + .assertBinding(Aes.COLOR, "value") + } + + @Test + fun `geom_polygon(aes(color = 'value'), df, map = gdf, map_join=('fig', 'kind'))`() { + `aes(color='value'), data=df, map=gdf, map_join=('fig', 'kind')`(geom="polygon") + .assertBinding(Aes.X, POINT_X) + .assertBinding(Aes.Y, POINT_Y) + .assertBinding(Aes.COLOR, "value") + } + + @Test + fun `geom_path(aes(color = 'value'), df, map = gdf, map_join=('fig', 'kind'))`() { + `aes(color='value'), data=df, map=gdf, map_join=('fig', 'kind')`(geom="path") + .assertBinding(Aes.X, POINT_X) + .assertBinding(Aes.Y, POINT_Y) + .assertBinding(Aes.COLOR, "value") + } + + @Test + fun `geom_rect(aes(color = 'value'), df, map = gdf, map_join=('fig', 'kind'))`() { + `aes(color='value'), data=df, map=gdf, map_join=('fig', 'kind')`(geom="rect") + .assertBinding(Aes.XMIN, RECT_XMIN) + .assertBinding(Aes.XMAX, RECT_XMAX) + .assertBinding(Aes.YMIN, RECT_YMIN) + .assertBinding(Aes.YMAX, RECT_YMAX) + .assertBinding(Aes.COLOR, "value") + } + + private fun `map=gdf`(geom: String): PlotConfigClientSide { + val spec = """ + |{ + | "kind": "plot", + | "layers": [{ + | "geom": "$geom", + | "map": $gdf, + | "map_data_meta": {"geodataframe": {"geometry": "coord"}} + | }] + |} + """.trimMargin() + + return transformToClientPlotConfig(spec) + } + + @Test + fun `geom_point(map=gdf)`() { + `map=gdf`(geom="point") + .assertBinding(Aes.X, POINT_X) + .assertBinding(Aes.Y, POINT_Y) + } + + @Test + fun `geom_polygon(map=gdf)`() { + `map=gdf`(geom="polygon") + .assertBinding(Aes.X, POINT_X) + .assertBinding(Aes.Y, POINT_Y) + } + + @Test + fun `geom_path(map=gdf)`() { + `map=gdf`(geom="path") + .assertBinding(Aes.X, POINT_X) + .assertBinding(Aes.Y, POINT_Y) + } + + @Test + fun `geom_rect(map=gdf)`() { + `map=gdf`(geom="rect") + .assertBinding(Aes.XMIN, RECT_XMIN) + .assertBinding(Aes.XMAX, RECT_XMAX) + .assertBinding(Aes.YMIN, RECT_YMIN) + .assertBinding(Aes.YMAX, RECT_YMAX) + } + + @Test + fun `with map_join=('fig', 'kind') should group by 'fig'`() { + val spec = """ + |{ + | "kind": "plot", + | "layers": [{ + | "geom": "polygon", + | "data": { + | "fig": ["Polygon", "MPolygon", "C"], + | "value": [42, 23, 66] + | }, + | "mapping": {"fill": "value"}, + | "map": $gdf, + | "map_data_meta": {"geodataframe": {"geometry": "coord"}}, + | "map_join": ["fig", "kind"] + | }] + |} + """.trimMargin() + + val plotAssembler = createPlotAssembler(parsePlotSpec(spec)) + val aesthetics = createLayerRendererData(plotAssembler.layersByTile.single().single(), emptyMap(), emptyMap()).aesthetics + assertEquals((0..14).map { 0 } + (0..3).map { 1 } , aesthetics.dataPoints().map(DataPointAesthetics::group)) + } + + @Test + fun `aes(color'fig'), data=df, map=gdf without map_join should fail`() { + // don't know how to join data with map without id columns. + assertEquals(MAP_JOIN_REQUIRED_MESSAGE, failedTransformToClientPlotConfig(""" + |{ + | "kind": "plot", + | "layers": [{ + | "geom": "polygon", + | "data": { "fig": ["a", "b", "C"] }, + | "mapping": { "color": "fig" }, + | "map": $gdf, + | "map_data_meta": {"geodataframe": {"geometry": "coord"}} + | }] + |}""".trimMargin() + ) + ) + } +} diff --git a/plot-config/src/jvmTest/kotlin/plot/config/geo/DataFrameBuilderTest.kt b/plot-config/src/jvmTest/kotlin/plot/config/geo/DataFrameBuilderTest.kt deleted file mode 100644 index d553f42bd80..00000000000 --- a/plot-config/src/jvmTest/kotlin/plot/config/geo/DataFrameBuilderTest.kt +++ /dev/null @@ -1,222 +0,0 @@ -//package jetbrains.datalore.plot.config.geo -// -//import jetbrains.datalore.base.spatial.GeoRectangle -//import jetbrains.datalore.base.spatial.LonLat -//import jetbrains.datalore.base.typedGeometry.* -//import jetbrains.datalore.plot.builder.map.GeoPositionField.POINT_X -//import jetbrains.datalore.plot.builder.map.GeoPositionField.POINT_Y -//import jetbrains.datalore.plot.builder.map.GeoPositionField.RECT_XMAX -//import jetbrains.datalore.plot.builder.map.GeoPositionField.RECT_XMIN -//import jetbrains.datalore.plot.builder.map.GeoPositionField.RECT_YMAX -//import jetbrains.datalore.plot.builder.map.GeoPositionField.RECT_YMIN -//import jetbrains.datalore.plot.config.GeoPositionsDataUtil.GeoDataKind.* -//import jetbrains.datalore.plot.config.GeoPositionsDataUtil.MAP_JOIN_ID_COLUMN -//import kotlin.test.Test -//import kotlin.test.assertEquals -// -// -//class DataFrameBuilderTest { -// -// @Test -// fun shouldNotAddPointForPolygonMode() { -// val pointCombiner = createBoundaryCombiner() -// pointCombiner.addPoint(OBJECT_ID, POINT) -// -// assertEquals( -// ExpectedDataBuilder().buildPoints(), -// pointCombiner.data -// ) -// } -// -// @Test -// fun shouldAddPointForPathMode() { -// val pointCombiner = createCentroidCombiner() -// pointCombiner.addPoint(OBJECT_ID, POINT) -// -// assertEquals( -// ExpectedDataBuilder() -// .addPoint(OBJECT_ID, POINT) -// .buildPoints(), -// pointCombiner.data -// ) -// } -// -// @Test -// fun shouldClosePathForPolygonMode() { -// val polygonCombiner = createBoundaryCombiner() -// polygonCombiner.addBoundary(OBJECT_ID, MultiPolygon(listOf(Polygon(listOf(OPEN_PATH))))) -// -// assertEquals( -// ExpectedDataBuilder() -// .addPoints(OBJECT_ID, OPEN_PATH) -// .addPoint(OBJECT_ID, OPEN_PATH.get(0)) -// .buildPoints(), -// polygonCombiner.data -// ) -// } -// -// @Test -// fun shouldNotClosePathForNotPolygonMode() { -// val pointCombiner = createCentroidCombiner() -// pointCombiner.addBoundary(OBJECT_ID, MultiPolygon(listOf(Polygon(listOf(OPEN_PATH))))) -// -// assertEquals( -// ExpectedDataBuilder() -// .addPoints(OBJECT_ID, OPEN_PATH) -// .buildPoints(), -// pointCombiner.data -// ) -// } -// -// @Test -// fun whenGeoRectangleCrossAntiMeridian_itShouldBeCut() { -// val rectCombiner = createLimitCombiner() -// rectCombiner.addGeoRectangle(OBJECT_ID, RECT_CROSSED_ANTI_MERIDIAN) -// -// assertEquals( -// ExpectedDataBuilder() -// .addRect(OBJECT_ID, POSITIVE_LON, 180.0, NEGATIVE_LAT, POSITIVE_LAT) -// .addRect(OBJECT_ID, -180.0, NEGATIVE_LON, NEGATIVE_LAT, POSITIVE_LAT) -// .buildLimits(), -// rectCombiner.data -// ) -// } -// -// @Test -// fun whenGeoRectangleNotCrossAntiMeridian_itShouldNotBeCut() { -// val rectCombiner = createLimitCombiner() -// rectCombiner.addGeoRectangle(OBJECT_ID, RECT_NOT_CROSSED_ANTI_MERIDIAN) -// -// assertEquals( -// ExpectedDataBuilder() -// .addRect(OBJECT_ID, NEGATIVE_LON, POSITIVE_LON, NEGATIVE_LAT, POSITIVE_LAT) -// .buildLimits(), -// rectCombiner.data -// ) -// } -// -// @Test -// fun shouldCalculateBBoxForBoundary() { -// val rectCombiner = createLimitCombiner() -// rectCombiner.addBoundary( -// OBJECT_ID, MultiPolygon(listOf( -// Polygon(listOf( -// Ring(listOf( -// point(NEGATIVE_LON, ZERO), -// point(POSITIVE_LON, ZERO), -// point(ZERO, NEGATIVE_LAT), -// point(ZERO, POSITIVE_LAT) -// )) -// )) -// )) -// ) -// -// assertEquals( -// ExpectedDataBuilder() -// .addRect(OBJECT_ID, NEGATIVE_LON, POSITIVE_LON, NEGATIVE_LAT, POSITIVE_LAT) -// .buildLimits(), -// rectCombiner.data -// ) -// } -// -// @Test -// fun shouldNotAddPoint() { -// val rectCombiner = createLimitCombiner() -// rectCombiner.append(OBJECT_ID, point(POSITIVE_LON, POSITIVE_LAT)) -// -// assertEquals( -// ExpectedDataBuilder().buildLimits(), -// rectCombiner.data -// ) -// } -// -// private class ExpectedDataBuilder { -// private val myIdList = ArrayList() -// private val myXList = ArrayList() -// private val myYList = ArrayList() -// private val myXMinList = ArrayList() -// private val myXMaxList = ArrayList() -// private val myYMinList = ArrayList() -// private val myYMaxList = ArrayList() -// -// internal fun addRect( -// id: String, -// xmin: Double, -// xmax: Double, -// ymin: Double, -// ymax: Double -// ): ExpectedDataBuilder { -// myIdList.add(id) -// myXMinList.add(xmin) -// myXMaxList.add(xmax) -// myYMinList.add(ymin) -// myYMaxList.add(ymax) -// return this -// } -// -// internal fun addPoints(id: String, coords: List>): ExpectedDataBuilder { -// coords.forEach { coord -> addPoint(id, coord) } -// return this -// } -// -// internal fun addPoint(id: String, coord: Vec<*>): ExpectedDataBuilder { -// myIdList.add(id) -// myXList.add(coord.x) -// myYList.add(coord.y) -// return this -// } -// -// internal fun buildPoints(): Map { -// return mapOf( -// MAP_JOIN_ID_COLUMN to myIdList, -// POINT_X to myXList, -// POINT_Y to myYList -// ) -// } -// -// internal fun buildLimits(): Map { -// return mapOf( -// MAP_JOIN_ID_COLUMN to myIdList, -// RECT_XMIN to myXMinList, -// RECT_XMAX to myXMaxList, -// RECT_YMIN to myYMinList, -// RECT_YMAX to myYMaxList -// ) -// } -// } -// -// companion object { -// private val OBJECT_ID: String? = "42" -// private val POINT = point(15.0, 27.0) -// private val OPEN_PATH = Ring(listOf( -// point(0.0, 0.0), -// point(0.0, 1.0), -// point(1.0, 1.0) -// ) -// ) -// private val NEGATIVE_LON = -90.0 -// private val POSITIVE_LON = 150.0 -// private val NEGATIVE_LAT = -10.0 -// private val POSITIVE_LAT = 30.0 -// private val ZERO = 0.0 -// private val RECT_CROSSED_ANTI_MERIDIAN = GeoRectangle(POSITIVE_LON, NEGATIVE_LAT, NEGATIVE_LON, POSITIVE_LAT) -// private val RECT_NOT_CROSSED_ANTI_MERIDIAN = -// GeoRectangle(NEGATIVE_LON, NEGATIVE_LAT, POSITIVE_LON, POSITIVE_LAT) -// -// private fun point(x: Double, y: Double): Vec { -// return explicitVec(x, y) -// } -// -// private fun createBoundaryCombiner(): DataFrameBuilder { -// return DataFrameBuilder(BOUNDARY) -// } -// -// private fun createCentroidCombiner(): DataFrameBuilder { -// return DataFrameBuilder(CENTROID) -// } -// -// private fun createLimitCombiner(): DataFrameBuilder { -// return DataFrameBuilder(LIMIT) -// } -// } -//} \ No newline at end of file diff --git a/plot-config/src/jvmTest/kotlin/plot/server/config/DropUnusedDataTest.kt b/plot-config/src/jvmTest/kotlin/plot/server/config/DropUnusedDataTest.kt index 989e23dbb38..4b25684ec2a 100644 --- a/plot-config/src/jvmTest/kotlin/plot/server/config/DropUnusedDataTest.kt +++ b/plot-config/src/jvmTest/kotlin/plot/server/config/DropUnusedDataTest.kt @@ -5,8 +5,9 @@ package jetbrains.datalore.plot.server.config +import jetbrains.datalore.plot.base.Aes import jetbrains.datalore.plot.base.data.TransformVar -import jetbrains.datalore.plot.config.TestUtil +import jetbrains.datalore.plot.config.* import jetbrains.datalore.plot.parsePlotSpec import kotlin.test.Test import kotlin.test.assertEquals @@ -74,7 +75,8 @@ class DropUnusedDataTest { } } - init { + @Test + fun specialVariables() { val x = listOf(0.0, 0.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 3.0, 3.0) val group = listOf("0", "0", "1", "0", "0", "1", "1", "0", "1", "1") val facetX = listOf("a", "a", "b", "a", "a", "b", "b", "a", "b", "b") @@ -555,7 +557,7 @@ class DropUnusedDataTest { } @Test - fun shouldNotDropMapJoin() { + fun `map_join with GeoDataFrame should not drop data variable`() { val spec = """ { "kind": "plot", @@ -586,17 +588,84 @@ class DropUnusedDataTest { } }""" - val opts = parsePlotSpec(spec) - TestUtil.checkOptionsClientSide(opts, 1) + parsePlotSpec(spec) + .let(ServerSideTestUtil::serverTransformWithoutEncoding) + .also { require(!PlotConfig.isFailure(it)) { PlotConfig.getErrorMessage(it) } } + .let(TestUtil::assertClientWontFail) - val unknownSize = -1 - val droppedVars = emptyList() - checkSingleLayerData(opts, 1, - mapOf( - "name" to unknownSize - ), - droppedVars - ) + } + + @Test + fun `should not drop geometry column when GeoDataFrame in data`() { + val spec = """ +{ + "kind": "plot", + "layers": [ + { + "geom": "polygon", + "data": { + "id": ["A", "B", "C"], + "coord": [ + "{\"type\": \"Point\", \"coordinates\": [-5.0, 17.0]}", + "{\"type\": \"Polygon\", \"coordinates\": [[[1.0, 1.0], [1.0, 9.0], [9.0, 9.0], [9.0, 1.0], [1.0, 1.0]], [[2.0, 2.0], [3.0, 2.0], [3.0, 3.0], [2.0, 3.0], [2.0, 2.0]], [[4.0, 4.0], [6.0, 4.0], [6.0, 6.0], [4.0, 6.0], [4.0, 4.0]]]}", + "{\"type\": \"MultiPolygon\", \"coordinates\": [[[[11.0, 12.0], [13.0, 14.0], [15.0, 13.0], [11.0, 12.0]]]]}" + ] + }, + "mapping": { "fill": "id" }, + "data_meta": { + "geodataframe": { + "geometry": "coord" + } + } + } + ] +} +}""" + + + parsePlotSpec(spec) + .let(ServerSideTestUtil::serverTransformWithoutEncoding) + .also { require(!PlotConfig.isFailure(it)) { PlotConfig.getErrorMessage(it) } } + .let(TestUtil::assertClientWontFail) + .assertBinding(Aes.X, POINT_X) + .assertBinding(Aes.Y, POINT_Y) + + } + + @Test + fun `map_join with GeoDict should not drop data variable`() { + val spec = """ +{ + "ggsize": { + "width": 500, + "height": 300 + }, + "kind": "plot", + "layers": [ + { + "geom": "polygon", + "data": { + "Country": ["UK", "Germany", "France"], + "Population": [66650000.0, 83020000.0, 66990000.0] + }, + "mapping": {"fill": "Population"}, + "map": { + "lon": [-2.598046, -2.37832, -2.46621, -3.169335, -1.938867, -0.576562, -0.31289, 0.873632, 0.082617, -2.598046, 7.685156, 9.926367, 13.661718, 14.101171, 11.464453, 12.870703, 8.564062, 8.651953, 6.80625, 6.938085, 7.685156, -2.246484, 2.367773, 7.245703, 5.268164, 6.586523, 2.895117, 2.279882, -0.532617, -0.356835, -2.246484], + "lat": [ 51.030349, 51.797754, 53.94575, 54.561879, 55.193929, 53.816229, 52.924809, 52.525588, 51.113188, 51.030349, 53.294124, 54.049078, 53.60816, 51.305902, 50.221916, 48.679365, 48.007575, 49.485266, 50.024691, 51.552493, 53.294124, 48.095702, 50.586036, 48.795295, 46.365136, 44.169607, 43.663114, 43.088157, 43.631315, 46.51655, 48.095702], + "country": [ "UK", "UK", "UK", "UK", "UK", "UK", "UK", "UK", "UK", "UK", "Germany", "Germany", "Germany", "Germany", "Germany", "Germany", "Germany", "Germany", "Germany", "Germany", "Germany", "France", "France", "France", "France", "France", "France", "France", "France", "France", "France"] + }, + "map_join": ["Country", "country"], + "map_data_meta": {"geodict": {}}, + "alpha": 0.3 + } + ] +} + """.trimIndent() + + parsePlotSpec(spec) + .let(ServerSideTestUtil::serverTransformWithoutEncoding) + .also { require(!PlotConfig.isFailure(it)) { PlotConfig.getErrorMessage(it) } } + .let(TestUtil::assertClientWontFail) } } diff --git a/plot-config/src/jvmTest/kotlin/plot/server/config/GeoDataFrameMappingChangeTest.kt b/plot-config/src/jvmTest/kotlin/plot/server/config/GeoDataFrameMappingChangeTest.kt deleted file mode 100644 index 290159574cc..00000000000 --- a/plot-config/src/jvmTest/kotlin/plot/server/config/GeoDataFrameMappingChangeTest.kt +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2019. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.server.config - -import jetbrains.datalore.plot.base.Aes -import jetbrains.datalore.plot.config.GeoPositionsDataUtil.MAP_REGION_COLUMN -import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS -import jetbrains.datalore.plot.config.Option.GeomName -import jetbrains.datalore.plot.config.Option.Layer.GEOM -import jetbrains.datalore.plot.config.Option.Layer.POS -import jetbrains.datalore.plot.config.Option.Layer.STAT -import jetbrains.datalore.plot.config.Option.Meta.DATA_META -import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame -import jetbrains.datalore.plot.config.Option.Meta.GeoReference -import jetbrains.datalore.plot.config.Option.Meta.MAP_DATA_META -import jetbrains.datalore.plot.config.Option.Meta.MapJoin -import jetbrains.datalore.plot.config.Option.PlotBase.DATA -import jetbrains.datalore.plot.config.Option.PlotBase.MAPPING -import jetbrains.datalore.plot.server.config.ServerSideTestUtil.createLayerConfigsByLayerSpec -import jetbrains.datalore.plot.server.config.ServerSideTestUtil.geomPolygonSpec -import jetbrains.datalore.plot.server.config.SingleLayerAssert.Companion.assertThat -import kotlin.test.Test - -class GeoDataFrameMappingChangeTest { - - @Test - fun whenDataContainsGeoDataFrame_shouldMoveGeometryColumnToMap() { - val cfg = createLayerConfigsByLayerSpec(mapOf( - GEOM to GeomName.POLYGON, - DATA to MAP_DATA_WITH_IDS, - DATA_META to GEO_DATA_FRAME_META, - GEO_POSITIONS to null, - MAP_DATA_META to null, - MAPPING to mapOf(Aes.FILL.name to NAME_COLUMN), - STAT to null, - POS to null) - ) - - val expectedIdList = listOf("0", "1", "2", "3", "4") - - assertThat(cfg) - //.haveBinding(Aes.MAP_ID, MapJoin.ID) // TODO: Fix MAP_ID - .haveDataVectors(mapOf( - // MapJoin.ID to expectedIdList, // TODO: Fix MAP_ID - NAME_COLUMN to MAP_DATA_WITH_IDS[NAME_COLUMN] as List<*> - )) - .haveMapIds(expectedIdList) - .haveMapGeometries(MAP_DATA_WITH_IDS[GEOMETRY_COLUMN] as List<*>) - } - - @Test - fun whenMapContainsGeoDataFrameWithID_shouldKeepIdAndGeometryColumns() { - val cfg = createLayerConfigsByLayerSpec( - geomPolygonSpec(MAP_DATA_WITH_IDS, GEO_DATA_FRAME_META) - ) - - assertThat(cfg) - .haveMapIds(MAP_DATA_REQUESTS) - .haveMapGeometries(GEOMETRIES) - } - - @Test - fun whenMapContainsGeoDataFrameWithoutID_shouldKeepOnlyGeometryColumns() { - val cfg = createLayerConfigsByLayerSpec( - geomPolygonSpec(GEO_DATA_FRAME_WITHOUT_IDS, GEO_DATA_FRAME_META) - ) - - assertThat(cfg) - .haveMapGeometries(GEOMETRIES) - } - - companion object { - private const val NAME_COLUMN = "name" - private const val VALUE_COLUMN = "value" - private const val GEOMETRY_COLUMN = "coord" - private val NAMES = listOf("foo", "bar", "qwe", "tmp", "xyz") - private val VALUES = listOf("254", "313", "142", "89", "3") - - private val DATA_FRAME = mapOf( - NAME_COLUMN to NAMES, - VALUE_COLUMN to VALUES - ) - - private val MAP_DATA_REQUESTS = listOf("rq_foo", "rq_bar", "rq_qwe", "rq_tmp", "rq_xyz") - private val MAP_DATA_REGIONS = listOf("rg_foo", "rg_bar", "rg_qwe", "rg_tmp", "rg_xyz") - private val MAP_DATA_JOIN_KEYS = listOf("id_foo", "id_bar", "id_qwe", "id_tmp", "id_xyz") - private val GEOMETRIES = listOf( - "{\"type: \"Point\", \"coordinates\":[-58, -34]}", - "{\"type: \"Point\", \"coordinates\":[-47, -15]}", - "{\"type: \"Point\", \"coordinates\":[-70, -33]}", - "{\"type: \"Point\", \"coordinates\":[-74, -11]}", - "{\"type: \"Point\", \"coordinates\":[-66, -17]}" - ) - - private val MAP_DATA_WITH_IDS = - DATA_FRAME + mapOf( - GeoReference.REQUEST to MAP_DATA_REQUESTS, - MAP_REGION_COLUMN to MAP_DATA_REGIONS, - MapJoin.MAP_ID to MAP_DATA_JOIN_KEYS, - GEOMETRY_COLUMN to GEOMETRIES - ) - - private val GEO_DATA_FRAME_WITHOUT_IDS = - DATA_FRAME + mapOf( - GEOMETRY_COLUMN to GEOMETRIES - ) - - private val GEO_DATA_FRAME_META = mapOf( - GeoDataFrame.TAG to mapOf( - GeoDataFrame.GEOMETRY_COLUMN_NAME to GEOMETRY_COLUMN - ) - ) - } -} \ No newline at end of file diff --git a/plot-config/src/jvmTest/kotlin/plot/server/config/GeoReferenceMappingChangeTest.kt b/plot-config/src/jvmTest/kotlin/plot/server/config/GeoReferenceMappingChangeTest.kt deleted file mode 100644 index 63df81413ec..00000000000 --- a/plot-config/src/jvmTest/kotlin/plot/server/config/GeoReferenceMappingChangeTest.kt +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.server.config - -import jetbrains.datalore.plot.config.Option.Meta.GeoReference -import jetbrains.datalore.plot.server.config.ServerSideTestUtil.createLayerConfigsByLayerSpec -import jetbrains.datalore.plot.server.config.ServerSideTestUtil.geomPolygonSpec -import jetbrains.datalore.plot.server.config.SingleLayerAssert.Companion.assertThat -import kotlin.test.Test - -class GeoReferenceMappingChangeTest { - - @Test - fun whenMapContainsGeoReference_shouldKeepIdAndGeocodeColumns() { - val cfg = createLayerConfigsByLayerSpec(geomPolygonSpec(GEO_REFERENCE, GEO_REFERENCE_META)) - - assertThat(cfg) - .haveMapIds(GEO_REFERENCE[GeoReference.REQUEST] as List<*>) - .haveMapGeocode(GEO_REFERENCE[GeoReference.OSM_ID] as List<*>) - } - - companion object { - private val GEO_REFERENCE = mapOf( - GeoReference.REQUEST to listOf("foo", "bar", "xyz"), - GeoReference.OSM_ID to listOf("123", "42", "27"), - "found name" to listOf("Foo", "Bar", "xyz"), - "highlights" to listOf( - listOf("foo baz"), - listOf("baz bar"), - listOf("baz xyz baz")) - ) - - private val GEO_REFERENCE_META = mapOf( - GeoReference.TAG to mapOf() - ) - } -} \ No newline at end of file diff --git a/plot-config/src/jvmTest/kotlin/plot/server/config/SingleLayerAssert.kt b/plot-config/src/jvmTest/kotlin/plot/server/config/SingleLayerAssert.kt index 8313c76210f..e8e5c14021a 100644 --- a/plot-config/src/jvmTest/kotlin/plot/server/config/SingleLayerAssert.kt +++ b/plot-config/src/jvmTest/kotlin/plot/server/config/SingleLayerAssert.kt @@ -9,7 +9,6 @@ import jetbrains.datalore.plot.base.Aes import jetbrains.datalore.plot.base.data.DataFrameUtil import jetbrains.datalore.plot.builder.tooltip.MappedAes import jetbrains.datalore.plot.builder.tooltip.VariableValue -import jetbrains.datalore.plot.config.GeoPositionsDataUtil.MAP_OSM_ID_COLUMN import jetbrains.datalore.plot.config.LayerConfig import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame.GEOMETRIES @@ -85,10 +84,6 @@ class SingleLayerAssert private constructor(layers: List) : return haveMapValues(GEOMETRIES, expectedGeometries) } - internal fun haveMapGeocode(expectedGeocode: List<*>): SingleLayerAssert { - return haveMapValues(MAP_OSM_ID_COLUMN, expectedGeocode) - } - private fun haveMapValues(key: String, expectedMapValues: List<*>): SingleLayerAssert { val geoPositions = myLayer[GEO_POSITIONS] as Map<*, *>? assertTrue(geoPositions!!.containsKey(key)) diff --git a/plot-demo/src/commonMain/kotlin/jetbrains/datalore/plotDemo/model/plotConfig/GeoData.kt b/plot-demo/src/commonMain/kotlin/jetbrains/datalore/plotDemo/model/plotConfig/GeoData.kt index 42dc64ccbc6..1d3529dad5b 100644 --- a/plot-demo/src/commonMain/kotlin/jetbrains/datalore/plotDemo/model/plotConfig/GeoData.kt +++ b/plot-demo/src/commonMain/kotlin/jetbrains/datalore/plotDemo/model/plotConfig/GeoData.kt @@ -12,30 +12,64 @@ class GeoData : PlotConfigDemoBase() { fun plotSpecList(): List> { return listOf( - mapJoinDict(), - mapJoinPair(), - mapIdAndMapJoinNoneString(), - emptyDataGdf(), - emptyMapGdf(), - geomText(), - mixedShapesGeom("polygon"), - mixedShapesGeom("point"), - mixedShapesGeom("path"), - mapRegionId() + ttt() +// mapGeoDataFrame_MapJoin(), +// mapGeoDataFrame_NoMapJoin_MixedShapes("polygon"), +// mapGeoDataFrame_NoMapJoin_MixedShapes("point"), +// mapGeoDataFrame_NoMapJoin_MixedShapes("path"), +// mapGeoDataFrame_Empty(), +// mapGeoDict_MapJoin(), +// dataGeoDataFrame_Empty(), +// dataGeoDataFrame_NoMapJoin_GeomText() ) } companion object { - private const val pointA = """{\"type\": \"Point\", \"coordinates\": [12.0, 22.0]}""" - private const val pointB = """{\"type\": \"Point\", \"coordinates\": [25.0, 11.0]}""" + private const val pointA = """{\"type\": \"Point\", \"coordinates\": [0.0, 0.0]}""" + private const val pointB = """{\"type\": \"Point\", \"coordinates\": [10.0, 10.0]}""" private const val lineA = """{\"type\": \"LineString\", \"coordinates\": [[15.0, 21.0], [29, 14], [33, 19]]}""" private const val lineB = """{\"type\": \"LineString\", \"coordinates\": [[3.0, 3.0], [7, 7], [10, 10]]}""" private const val multipolygon = """{\"type\": \"MultiPolygon\", \"coordinates\": [[[[11.0, 12.0], [13.0, 14.0], [15.0, 13.0], [11.0, 12.0]]]]}""" - private fun mapJoinDict(): Map { + + fun ttt(): MutableMap { + val point = """{\"type\": \"Point\", \"coordinates\": [-5.0, 17.0]}""" + val multiPoint = """{\"type\": \"MultiPoint\", \"coordinates\": [[3.0, 15.0], [6.0, 13.0]]}""" + val lineString = """{\"type\": \"LineString\", \"coordinates\": [[0.0, 0.0], [5.0, 5.0]]}""" + val multiLineString = """{\"type\": \"MultiLineString\", \"coordinates\": [[[10.0, 0.0], [15.0, 5.0]], [[10.0, 5.0], [15.0, 0.0]]]}""" + val polygon = """{\"type\": \"Polygon\", \"coordinates\": [[[1.0, 1.0], [1.0, 9.0], [9.0, 9.0], [9.0, 1.0], [1.0, 1.0]], [[2.0, 2.0], [3.0, 2.0], [3.0, 3.0], [2.0, 3.0], [2.0, 2.0]], [[4.0, 4.0], [6.0, 4.0], [6.0, 6.0], [4.0, 6.0], [4.0, 4.0]]]}""" + val multipolygon = """{\"type\": \"MultiPolygon\", \"coordinates\": [[[[11.0, 12.0], [13.0, 14.0], [15.0, 13.0], [11.0, 12.0]]]]}""" + val gdf = """ +{ + "kind": ["Point", "MPoint", "Line", "MLine", "Polygon", "MPolygon"], + "coord": ["$point", "$multiPoint", "$lineString", "$multiLineString", "$polygon", "$multipolygon"] +} +""" + val spec = """ + |{ + | "kind": "plot", + | "layers": [{ + | "geom": "polygon", + | "data": { + | "fig": ["Polygon", "MPolygon", "C"], + | "value": [42, 23, 66] + | }, + | "mapping": {"fill": "value"}, + | "map": $gdf, + | "map_data_meta": {"geodataframe": {"geometry": "coord"}}, + | "map_join": ["fig", "kind"] + | }] + |} + """.trimMargin() + return parsePlotSpec(spec) + + } + + private fun mapGeoDict_MapJoin(): Map { val spec = """ { + "ggtitle": {"text": "mapJoinDict"}, "ggsize": { "width": 500, "height": 300 @@ -64,9 +98,10 @@ class GeoData : PlotConfigDemoBase() { return parsePlotSpec(spec) } - private fun mapJoinPair(): Map { + private fun mapGeoDataFrame_MapJoin(): Map { val spec = """ |{ + | "ggtitle": {"text": "mapGeoDataFrame_MapJoin"}, | "kind": "plot", | "layers": [{ | "geom": "point", @@ -85,33 +120,10 @@ class GeoData : PlotConfigDemoBase() { return parsePlotSpec(spec) } - private fun mapIdAndMapJoinNoneString(): Map { - val spec = """ - |{ - | "kind": "plot", - | "layers": [{ - | "geom": "point", - | "data": {"labels": ["A", "B"], "values": [12, 3]}, - | "mapping": { - | "color": "values", - | "map_id": "labels" - | }, - | "map_data_meta": {"geodataframe": {"geometry": "coord"}}, - | "map_join": [null, "map_names"], - | "map": { - | "map_names": ["A", "B"], - | "coord": ["$pointA", "$pointB"] - | } - | }] - |} - """.trimMargin() - - return parsePlotSpec(spec) - } - - fun emptyDataGdf(): MutableMap { + fun dataGeoDataFrame_Empty(): MutableMap { val spec = """ |{ + | "ggtitle": {"text": "emptyDataGdf"}, | "kind": "plot", | "layers": [{ | "geom": "polygon", @@ -127,9 +139,10 @@ class GeoData : PlotConfigDemoBase() { return parsePlotSpec(spec) } - fun emptyMapGdf(): MutableMap { + fun mapGeoDataFrame_Empty(): MutableMap { val spec = """ |{ + | "ggtitle": {"text": "mapGeoDataFrame_Empty"}, | "kind": "plot", | "layers": [{ | "geom": "polygon", @@ -145,9 +158,10 @@ class GeoData : PlotConfigDemoBase() { return parsePlotSpec(spec) } - fun geomText(): MutableMap { + fun dataGeoDataFrame_NoMapJoin_GeomText(): MutableMap { val spec = """ |{ + | "ggtitle": {"text": "dataGeoDataFrame_GeomText"}, | "kind": "plot", | "layers": [{ | "geom": "text", @@ -163,38 +177,10 @@ class GeoData : PlotConfigDemoBase() { return parsePlotSpec(spec) } - fun mapRegionId(): MutableMap { - val spec = """ - |{ - | "kind": "plot", - | "layers": [{ - | "geom": "point", - | "map_data_meta": {"geodataframe": {"geometry": "coord"}}, - | "map_join": ["labels", "map_names"], - | "mapping": { - | "color": "values", - | "map_id": "labels" - | }, - | "data": { - | "labels": ["A", "B"], - | "values": [12, 3] - | }, - | "map": { - | "map_names": ["A", "B"], - | "coord": [ - | "$pointA", - | "$pointB" - | ] - | } - | }] - |} - """.trimMargin() - return parsePlotSpec(spec) - } - - fun mixedShapesGeom(geomName: String): MutableMap { + fun mapGeoDataFrame_NoMapJoin_MixedShapes(geomName: String): MutableMap { val plotSpec = """ |{ + | "ggtitle": {"text": "mapGeoDataFrame_MixedShapes geom_$geomName"}, | "kind": "plot", | "layers": [{ | "geom": "$geomName", From 0345d83cf69490f6353ac35c63aaa348dba2bfcd Mon Sep 17 00:00:00 2001 From: Ivan Kupriyanov Date: Tue, 9 Jun 2020 20:39:15 +0300 Subject: [PATCH 03/11] sym_x and sym_y in geom_livemap for pie/bar, x and y for positions --- .../LiveMapDataPointAestheticsProcessor.kt | 15 +++++--- .../datalore/plot/livemap/LonLatParser.kt | 30 ---------------- .../datalore/plot/livemap/MapEntityBuilder.kt | 4 +-- .../plot/livemap/MultiDataPointHelper.kt | 11 ++++-- .../datalore/plot/livemap/LonLatParserTest.kt | 35 ------------------- .../plot/livemap/MultiDataPointHelperTest.kt | 21 ++++++----- 6 files changed, 32 insertions(+), 84 deletions(-) delete mode 100644 plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LonLatParser.kt delete mode 100644 plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/LonLatParserTest.kt diff --git a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapDataPointAestheticsProcessor.kt b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapDataPointAestheticsProcessor.kt index e57a17277bb..592a12f1585 100644 --- a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapDataPointAestheticsProcessor.kt +++ b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapDataPointAestheticsProcessor.kt @@ -5,6 +5,7 @@ package jetbrains.datalore.plot.livemap +import jetbrains.datalore.base.typedGeometry.explicitVec import jetbrains.datalore.plot.base.Aes import jetbrains.datalore.plot.base.Aesthetics import jetbrains.datalore.plot.base.DataPointAesthetics @@ -12,7 +13,6 @@ import jetbrains.datalore.plot.base.aes.AesInitValue import jetbrains.datalore.plot.base.livemap.LiveMapOptions import jetbrains.datalore.plot.base.livemap.LivemapConstants import jetbrains.datalore.plot.livemap.LiveMapUtil.createLayersConfigurator -import jetbrains.datalore.plot.livemap.MapLayerKind.POINT import jetbrains.datalore.plot.livemap.MultiDataPointHelper.SortingMode import jetbrains.livemap.api.LayersBuilder @@ -34,7 +34,7 @@ internal class LiveMapDataPointAestheticsProcessor( private fun getLayerKind(displayMode: LivemapConstants.DisplayMode): MapLayerKind { return when (displayMode) { LivemapConstants.DisplayMode.POLYGON -> MapLayerKind.POLYGON - LivemapConstants.DisplayMode.POINT -> POINT + LivemapConstants.DisplayMode.POINT -> MapLayerKind.POINT LivemapConstants.DisplayMode.PIE -> MapLayerKind.PIE LivemapConstants.DisplayMode.HEATMAP -> MapLayerKind.HEATMAP LivemapConstants.DisplayMode.BAR -> MapLayerKind.BAR @@ -55,23 +55,28 @@ internal class LiveMapDataPointAestheticsProcessor( private fun processDataPoints(): List { return myAesthetics.dataPoints() - .map { MapEntityBuilder(it, myLayerKind).apply { setIfNeeded() } } + .map { MapEntityBuilder(it, myLayerKind).apply { setIfNeeded(it) } } } private fun processMultiDataPoints(): List { return MultiDataPointHelper .getPoints(myAesthetics, getSortingMode(myLayerKind)) - .map { MapEntityBuilder(it, myLayerKind).apply { setIfNeeded() } } + .map { MapEntityBuilder(it, myLayerKind).apply { setIfNeeded(it.aes) } } } private fun useMultiDataPoint(): Boolean { return myLayerKind === MapLayerKind.PIE || myLayerKind === MapLayerKind.BAR } - private fun MapEntityBuilder.setIfNeeded() { + private fun MapEntityBuilder.setIfNeeded(p: DataPointAesthetics) { + setGeometryPointIfNeeded(p, this) setGeodesicIfNeeded(this) } + private fun setGeometryPointIfNeeded(p: DataPointAesthetics, mapEntityBuilder: MapEntityBuilder) { + mapEntityBuilder.setGeometryPoint(explicitVec(p.x()!!, p.y()!!)) + } + private fun setGeodesicIfNeeded(mapEntityBuilder: MapEntityBuilder) { if (myLayerKind == MapLayerKind.PATH) { mapEntityBuilder.geodesic = myGeodesic diff --git a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LonLatParser.kt b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LonLatParser.kt deleted file mode 100644 index ad22f1cccf6..00000000000 --- a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LonLatParser.kt +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2019. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.livemap - -import jetbrains.datalore.base.spatial.LonLat -import jetbrains.datalore.base.spatial.MercatorUtils.checkLat -import jetbrains.datalore.base.spatial.MercatorUtils.checkLon -import jetbrains.datalore.base.typedGeometry.Vec -import jetbrains.datalore.base.typedGeometry.explicitVec - -internal object LonLatParser { - private val LON_LAT = Regex("^(-?\\d+\\.?\\d*), ?(-?\\d+\\.?\\d*)$") - private const val LON = 1 - private const val LAT = 2 - - fun parse(lonlat: String): Vec? { - val matcher = LON_LAT.matchEntire(lonlat)?.groups ?: return null - val lon = matcher[LON]?.value?.toDouble() ?: return null - val lat = matcher[LAT]?.value?.toDouble() ?: return null - - return if (checkLon(lon) && checkLat(lat)) { - explicitVec(lon, lat) - } else { - null - } - } -} \ No newline at end of file diff --git a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/MapEntityBuilder.kt b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/MapEntityBuilder.kt index df88c5f853b..ea4c666b39c 100644 --- a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/MapEntityBuilder.kt +++ b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/MapEntityBuilder.kt @@ -20,8 +20,8 @@ import jetbrains.datalore.plot.base.geom.util.GeomHelper import jetbrains.datalore.plot.base.render.svg.TextLabel.HorizontalAnchor.* import jetbrains.datalore.plot.base.render.svg.TextLabel.VerticalAnchor.* import jetbrains.datalore.plot.builder.scale.DefaultNaValue -import jetbrains.livemap.api.* import jetbrains.datalore.plot.livemap.MapLayerKind.* +import jetbrains.livemap.api.* import kotlin.math.ceil internal class MapEntityBuilder { @@ -109,7 +109,7 @@ internal class MapEntityBuilder { } private fun allZeroes(values: List): Boolean { - return values.all { value -> value == 0.0 } + return values.all(0.0::equals) } private fun createNaColorList(size: Int): List { diff --git a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelper.kt b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelper.kt index 78e81d4f276..d8e8bdc9aa6 100644 --- a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelper.kt +++ b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelper.kt @@ -5,6 +5,9 @@ package jetbrains.datalore.plot.livemap +import jetbrains.datalore.base.spatial.LonLat +import jetbrains.datalore.base.typedGeometry.Vec +import jetbrains.datalore.base.typedGeometry.explicitVec import jetbrains.datalore.base.values.Color import jetbrains.datalore.plot.base.Aesthetics import jetbrains.datalore.plot.base.DataPointAesthetics @@ -14,10 +17,12 @@ internal class MultiDataPointHelper private constructor( companion object { fun getPoints(aesthetics: Aesthetics, sortingMode: SortingMode): List { - val builders = HashMap() + val builders = HashMap, MultiDataPointBuilder>() - fun fetchBuilder(p: DataPointAesthetics): MultiDataPointBuilder = - builders.getOrPut(p.group()!!, { MultiDataPointBuilder(p, sortingMode) }) + fun fetchBuilder(p: DataPointAesthetics): MultiDataPointBuilder { + val coord = explicitVec(p.x()!!, p.y()!!) + return builders.getOrPut(coord, { MultiDataPointBuilder(p, sortingMode) }) + } aesthetics.dataPoints().forEach { fetchBuilder(it).add(it) } return builders.values.map { it.build() } diff --git a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/LonLatParserTest.kt b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/LonLatParserTest.kt deleted file mode 100644 index 12de8d9fc5b..00000000000 --- a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/LonLatParserTest.kt +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.livemap - -import jetbrains.datalore.base.typedGeometry.explicitVec -import kotlin.test.Test - -import kotlin.test.assertEquals -import kotlin.test.assertTrue - -class LonLatParserTest { - - @Test - fun withString_ShouldNotParse() { - assertTrue { null == LonLatParser.parse("Double, Double") } - } - - @Test - fun withString_AndWrongStructure_ShouldNotParse() { - assertTrue { LonLatParser.parse("Texas") == null } - } - - @Test - fun withoutSpaces_ShouldParse() { - assertEquals(explicitVec(-1.0, 9.0), LonLatParser.parse("-1.0,9")) - } - - @Test - fun withSpaces_ShouldParse() { - assertEquals(explicitVec(-1.0, 9.0), LonLatParser.parse("-1.0, 9")) - } -} \ No newline at end of file diff --git a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelperTest.kt b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelperTest.kt index af5e64961a2..c342c5145a2 100644 --- a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelperTest.kt +++ b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelperTest.kt @@ -5,6 +5,9 @@ package jetbrains.datalore.plot.livemap +import jetbrains.datalore.base.spatial.LonLat +import jetbrains.datalore.base.typedGeometry.Vec +import jetbrains.datalore.base.typedGeometry.explicitVec import jetbrains.datalore.plot.base.Aesthetics import jetbrains.datalore.plot.base.aes.AestheticsBuilder import jetbrains.datalore.plot.base.aes.AestheticsBuilder.Companion.collection @@ -102,7 +105,7 @@ class MultiDataPointHelperTest { ) { val points = myMultiDataBuilder .sortingMode(sortingMode) - .multiData("TX", dataPointBuilders) + .multiData(explicitVec(0.0, 0.0), dataPointBuilders) .points assertPointsOrder( @@ -126,13 +129,13 @@ class MultiDataPointHelperTest { for (pointBuilder in myPoints) { values.add(pointBuilder.myValue) order.add(pointBuilder.myOrder) - mapId.add(pointBuilder.myMapId) + mapId.add(pointBuilder.myCoord) } myBuilder //.mapId(collection(mapId)) // TODO: Fix MAP_ID - .x(collection(order)) - .y(collection(values)) + .symX(collection(order)) + .symY(collection(values)) .dataPointCount(order.size) return myBuilder.build() @@ -143,9 +146,9 @@ class MultiDataPointHelperTest { return this } - fun multiData(mapId: Any, v: List): MultiDataBuilder { + fun multiData(coord: Vec, v: List): MultiDataBuilder { for (point in v) { - point.mapId(mapId) + point.coord(coord) myPoints.add(point) } return this @@ -154,7 +157,7 @@ class MultiDataPointHelperTest { internal class DataPointBuilder { var myValue: Double = 0.0 var myOrder: Double = 0.0 - lateinit var myMapId: Any + lateinit var myCoord: Vec fun value(v: Double): DataPointBuilder { myValue = v @@ -166,8 +169,8 @@ class MultiDataPointHelperTest { return this } - fun mapId(v: Any): DataPointBuilder { - myMapId = v + fun coord(v: Vec): DataPointBuilder { + myCoord = v return this } } From e79089b218351cd7c8c0b8df1559ff7b68fa8212 Mon Sep 17 00:00:00 2001 From: Ivan Kupriyanov Date: Tue, 9 Jun 2020 23:44:17 +0300 Subject: [PATCH 04/11] cleanup --- .../jetbrains/livemap/plotDemo/LiveMap.kt | 3 +- .../jetbrains/datalore/plot/base/Aes.kt | 2 + .../datalore/plot/base/geom/LiveMapGeom.kt | 3 - .../datalore/plot/base/geom/MapGeom.kt | 2 - .../datalore/plot/base/geom/RectGeom.kt | 1 - .../jetbrains/datalore/plot/config/Option.kt | 8 - .../plot/config/PlotConfigClientSideUtil.kt | 4 +- .../GeomInteractionBuilderCreationTest.kt | 121 +++--------- .../kotlin/plot/config/ConfigUtilTest.kt | 9 +- .../plot/server/config/SingleLayerAssert.kt | 10 - .../MapIdAndGeoPointDataProcessingTest.kt | 187 ------------------ .../plot/livemap/MultiDataPointHelperTest.kt | 11 +- .../plot/livemap/PointConverterTest.kt | 8 - .../plot/livemap/PolygonConverterTest.kt | 10 - python-package/lets_plot/plot/geom.py | 11 +- 15 files changed, 47 insertions(+), 343 deletions(-) delete mode 100644 plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MapIdAndGeoPointDataProcessingTest.kt diff --git a/livemap-demo/src/commonMain/kotlin/jetbrains/livemap/plotDemo/LiveMap.kt b/livemap-demo/src/commonMain/kotlin/jetbrains/livemap/plotDemo/LiveMap.kt index 47c51aba9ab..5bb6cd9f644 100644 --- a/livemap-demo/src/commonMain/kotlin/jetbrains/livemap/plotDemo/LiveMap.kt +++ b/livemap-demo/src/commonMain/kotlin/jetbrains/livemap/plotDemo/LiveMap.kt @@ -249,7 +249,8 @@ class LiveMap : PlotConfigDemoBase() { { "geom":"livemap", "mapping":{ - "map_id":"lonlat", + "x":"x", + "y":"y", "color":"label" }, "display_mode":"point", diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/Aes.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/Aes.kt index 6a2ae3017e0..fd257953113 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/Aes.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/Aes.kt @@ -147,6 +147,8 @@ class Aes private constructor(val name: String, val isNumeric: Boolean = true aes == ANGLE || aes == FAMILY || aes == FONTFACE || + aes == SYM_X || + aes == SYM_Y || isPositional(aes) } diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/LiveMapGeom.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/LiveMapGeom.kt index 205f9d71e87..2cb7da21c24 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/LiveMapGeom.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/LiveMapGeom.kt @@ -5,8 +5,6 @@ package jetbrains.datalore.plot.base.geom -import jetbrains.datalore.base.geometry.DoubleVector -import jetbrains.datalore.base.values.SomeFig import jetbrains.datalore.base.geometry.DoubleRectangle import jetbrains.datalore.plot.base.* import jetbrains.datalore.plot.base.geom.legend.GenericLegendKeyElementFactory @@ -49,7 +47,6 @@ class LiveMapGeom(private val myDisplayMode: DisplayMode) : Geom { // ToDo: not static, depends on 'display mode' // val RENDERS = listOf( -// Aes.MAP_ID, // Aes.ALPHA, // Aes.COLOR, // Aes.FILL, diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/MapGeom.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/MapGeom.kt index 99f05304679..733bb8d9dc2 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/MapGeom.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/MapGeom.kt @@ -18,8 +18,6 @@ class MapGeom : PolygonGeom() { // Aes.COLOR, // Aes.FILL, // Aes.ALPHA, -// -// Aes.MAP_ID // ) const val HANDLES_GROUPS = true diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/RectGeom.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/RectGeom.kt index 808e5830600..1831c699a49 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/RectGeom.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/geom/RectGeom.kt @@ -40,7 +40,6 @@ class RectGeom : GeomBase() { // Aes.COLOR, // Aes.FILL, // Aes.ALPHA, -// Aes.MAP_ID // ) //rectangle groups are used in geom_livemap const val HANDLES_GROUPS = true diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/Option.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/Option.kt index f5f98ee6e55..20d281b9cfc 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/Option.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/Option.kt @@ -31,20 +31,12 @@ object Option { object GeoDataFrame { const val GDF = "geodataframe" const val GEOMETRY = "geometry" - - // Column with geometries extracted from GeoDataFrame. Can be used either in DATA and MAP - const val GEOMETRIES = "__geometry__" } object GeoDict { const val TAG = "geodict" } - object MapJoin { - // column in map used for join with data - const val MAP_ID = "__map_id__" - } - object MappingAnnotation { const val TAG = "mapping_annotations" const val AES = "aes" diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSideUtil.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSideUtil.kt index 2810dc31c79..49651c2a362 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSideUtil.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/PlotConfigClientSideUtil.kt @@ -22,7 +22,6 @@ import jetbrains.datalore.plot.builder.interact.GeomInteractionBuilder.Companion import jetbrains.datalore.plot.builder.theme.Theme import jetbrains.datalore.plot.builder.tooltip.CompositeValue import jetbrains.datalore.plot.builder.tooltip.TooltipLineSpecification -import jetbrains.datalore.plot.config.Option.Meta.MapJoin object PlotConfigClientSideUtil { internal fun createGuideOptionsMap(scaleConfigs: List>): Map, GuideOptions> { @@ -255,7 +254,8 @@ object PlotConfigClientSideUtil { } // remove auto generated mappings - aesListForTooltip.removeAll { layerConfig.getScaleForAes(it)?.name == MapJoin.MAP_ID } + val autoGenerated = listOf() + aesListForTooltip.removeAll { layerConfig.getScaleForAes(it)?.name in autoGenerated } return aesListForTooltip } diff --git a/plot-config-portable/src/commonTest/kotlin/plot/config/GeomInteractionBuilderCreationTest.kt b/plot-config-portable/src/commonTest/kotlin/plot/config/GeomInteractionBuilderCreationTest.kt index ef2e4269b98..58c5958151e 100644 --- a/plot-config-portable/src/commonTest/kotlin/plot/config/GeomInteractionBuilderCreationTest.kt +++ b/plot-config-portable/src/commonTest/kotlin/plot/config/GeomInteractionBuilderCreationTest.kt @@ -6,10 +6,16 @@ package plot.config import jetbrains.datalore.plot.base.Aes -import jetbrains.datalore.plot.config.LayerConfig -import jetbrains.datalore.plot.config.Option +import jetbrains.datalore.plot.config.* +import jetbrains.datalore.plot.config.Option.GeomName.POINT +import jetbrains.datalore.plot.config.Option.Layer.GEOM +import jetbrains.datalore.plot.config.Option.Meta.DATA_META +import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame +import jetbrains.datalore.plot.config.Option.Meta.KIND +import jetbrains.datalore.plot.config.Option.Meta.Kind.PLOT +import jetbrains.datalore.plot.config.Option.Plot.LAYERS +import jetbrains.datalore.plot.config.Option.PlotBase.DATA import jetbrains.datalore.plot.config.Option.PlotBase.MAPPING -import jetbrains.datalore.plot.config.PlotConfigClientSideUtil import jetbrains.datalore.plot.server.config.PlotConfigServerSide import kotlin.test.Test import kotlin.test.assertEquals @@ -29,9 +35,9 @@ class GeomInteractionBuilderCreationTest { ) val plotOpts = mutableMapOf( MAPPING to mappedData, - Option.Plot.LAYERS to listOf( + LAYERS to listOf( mapOf( - Option.Layer.GEOM to Option.GeomName.HISTOGRAM + GEOM to Option.GeomName.HISTOGRAM ) ) ) @@ -50,72 +56,6 @@ class GeomInteractionBuilderCreationTest { assertAesListCount(expectedAesListCount, builder.aesListForTooltip) } - /* TODO: Fix MAP_ID - @Test - fun shouldSkipMapIdMapping() { - val mappedData = data + mapOf( - Aes.MAP_ID.name to listOf('a') - ) - val plotOpts = mutableMapOf( - MAPPING to mappedData, - Option.Plot.LAYERS to listOf( - mapOf( - Option.Layer.GEOM to Option.GeomName.POINT - ) - ) - ) - val layerConfig = createLayerConfig(plotOpts) - - val builder = PlotConfigClientSideUtil.createGeomInteractionBuilder( - layerConfig, - emptyList(), - false - ) - - assertFalse { builder.aesListForTooltip.contains(Aes.MAP_ID) } - - val expectedAxisList = listOf(Aes.X, Aes.Y) - // without Aes.MAP_ID: - val expectedAesListCount = (layerConfig.geomProto.renders() - expectedAxisList).size - 1 - - assertAesListCount(expectedAxisList.size, builder.axisAesListForTooltip) - assertAesListCount(expectedAesListCount, builder.aesListForTooltip) - } - - @Test - fun shouldSkipMapIdMappingAndAxisVisibilityIsFalse() { - val mappedData = data + mapOf( - Aes.MAP_ID.name to listOf('a') - ) - - val plotOpts = mutableMapOf( - MAPPING to mappedData, - Option.Plot.LAYERS to listOf( - mapOf( - Option.Layer.GEOM to Option.GeomName.POLYGON - ) - ) - ) - val layerConfig = createLayerConfig(plotOpts) - - val builder = PlotConfigClientSideUtil.createGeomInteractionBuilder( - layerConfig, - emptyList(), - false - ) - - assertFalse { builder.aesListForTooltip.contains(Aes.MAP_ID) } - - // builder's axis tooltip visibility is false: - val expectedAxisCount = 0 - // without Aes.MAP_ID: - val expectedAesListCount = (layerConfig.geomProto.renders() - listOf(Aes.X, Aes.Y)).size - 1 - - assertAesListCount(expectedAxisCount, builder.axisAesListForTooltip) - assertAesListCount(expectedAesListCount, builder.aesListForTooltip) - - }*/ - @Test fun shouldNotDuplicateVarToAxisAndGenericTooltip() { val mappedData = mapOf( @@ -125,9 +65,9 @@ class GeomInteractionBuilderCreationTest { val plotOpts = mutableMapOf( MAPPING to mappedData, - Option.Plot.LAYERS to listOf( + LAYERS to listOf( mapOf( - Option.Layer.GEOM to Option.GeomName.HISTOGRAM + GEOM to Option.GeomName.HISTOGRAM ) ) ) @@ -149,14 +89,13 @@ class GeomInteractionBuilderCreationTest { assertAesListCount(expectedAesListCount, builder.aesListForTooltip) } - /* TODO: Fix MAP_ID @Test fun shouldSkipAutoGeneratedMappings() { val GEOMETRIES = listOf( - "{\"type: \"Point\", \"coordinates\":[-10, -20]}", - "{\"type: \"Point\", \"coordinates\":[-30, -40]}", - "{\"type: \"Point\", \"coordinates\":[-50, -60]}" + "{\"type\": \"Point\", \"coordinates\":[-10, -20]}", + "{\"type\": \"Point\", \"coordinates\":[-30, -40]}", + "{\"type\": \"Point\", \"coordinates\":[-50, -60]}" ) val geomData = mapOf( "name" to listOf("a", "b", "c"), @@ -164,25 +103,28 @@ class GeomInteractionBuilderCreationTest { "coord" to GEOMETRIES ) val GEO_DATA_FRAME_META = mapOf( - Option.Meta.GeoDataFrame.TAG to mapOf( - Option.Meta.GeoDataFrame.GEOMETRY_COLUMN_NAME to "coord" + GeoDataFrame.GDF to mapOf( + GeoDataFrame.GEOMETRY to "coord" ) ) val plotOpts = mutableMapOf( - Option.Meta.KIND to Option.Meta.Kind.PLOT, - Option.Plot.LAYERS to listOf( + KIND to PLOT, + LAYERS to listOf( mutableMapOf( - Option.Layer.GEOM to Option.GeomName.POLYGON, + GEOM to POINT, DATA to geomData, - Option.Meta.DATA_META to GEO_DATA_FRAME_META, + DATA_META to GEO_DATA_FRAME_META, MAPPING to mapOf(Aes.FILL.name to "name") ) ) ) - val layerConfig = createLayerConfig(plotOpts) - val binding = layerConfig.varBindings.find { it.variable.name == MapJoin.ID } - assertNotNull(binding) + + val opts = PlotConfigServerSide.processTransform(plotOpts) + val layerConfig = PlotConfigClientSide.create(opts).layerConfigs.first() + + val pointX = layerConfig.varBindings.find { it.variable.name == POINT_X } + val pointY = layerConfig.varBindings.find { it.variable.name == POINT_Y } val builder = PlotConfigClientSideUtil.createGeomInteractionBuilder( layerConfig, @@ -190,15 +132,16 @@ class GeomInteractionBuilderCreationTest { false ) - assertFalse { builder.aesListForTooltip.contains(binding.aes) } - }*/ + assertFalse { builder.aesListForTooltip.contains(pointX!!.aes) } + assertFalse { builder.aesListForTooltip.contains(pointY!!.aes) } + } private fun createLayerConfig(plotOpts: MutableMap): LayerConfig { val plotSpec = PlotConfigServerSide.processTransform(plotOpts) return PlotConfigServerSide(plotSpec).layerConfigs.first() } - internal fun assertAesListCount(expectedCount: Int, aesList: List>) { + private fun assertAesListCount(expectedCount: Int, aesList: List>) { assertEquals(expectedCount, aesList.size) } } \ No newline at end of file diff --git a/plot-config/src/jvmTest/kotlin/plot/config/ConfigUtilTest.kt b/plot-config/src/jvmTest/kotlin/plot/config/ConfigUtilTest.kt index 6166620944e..cae60a97638 100644 --- a/plot-config/src/jvmTest/kotlin/plot/config/ConfigUtilTest.kt +++ b/plot-config/src/jvmTest/kotlin/plot/config/ConfigUtilTest.kt @@ -7,7 +7,6 @@ package jetbrains.datalore.plot.config import jetbrains.datalore.plot.base.DataFrame import jetbrains.datalore.plot.base.DataFrame.Variable -import jetbrains.datalore.plot.config.Option.Meta.MapJoin.MAP_ID import org.assertj.core.api.Assertions.assertThat import kotlin.test.Test import kotlin.test.assertEquals @@ -21,20 +20,20 @@ class ConfigUtilTest { val dataValues = listOf("a", "b", "c", "d") val data = DataFrame.Builder() - .put(Variable(MAP_ID), idList) + .put(Variable("id"), idList) .put(Variable("foo"), dataValues) .build() val map = DataFrame.Builder() - .put(Variable(MAP_ID), idList) + .put(Variable("id"), idList) .put(Variable("lon"), listOf(13.0, 24.0, -65.0, 117.0)) .put(Variable("lat"), listOf(42.0, 21.0, -12.0, 77.0)) .build() - val joinedDf = ConfigUtil.rightJoin(data, MAP_ID, map, MAP_ID) + val joinedDf = ConfigUtil.rightJoin(data, "id", map, "id") assertThat(joinedDf.variables().map { it.toString() }) - .containsExactlyInAnyOrder(MAP_ID, "foo", "lon", "lat") + .containsExactlyInAnyOrder("id", "foo", "lon", "lat") var dataVar: Variable? = null for (variable in joinedDf.variables()) { diff --git a/plot-config/src/jvmTest/kotlin/plot/server/config/SingleLayerAssert.kt b/plot-config/src/jvmTest/kotlin/plot/server/config/SingleLayerAssert.kt index e8e5c14021a..1486a4b8ac3 100644 --- a/plot-config/src/jvmTest/kotlin/plot/server/config/SingleLayerAssert.kt +++ b/plot-config/src/jvmTest/kotlin/plot/server/config/SingleLayerAssert.kt @@ -11,8 +11,6 @@ import jetbrains.datalore.plot.builder.tooltip.MappedAes import jetbrains.datalore.plot.builder.tooltip.VariableValue import jetbrains.datalore.plot.config.LayerConfig import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS -import jetbrains.datalore.plot.config.Option.Meta.GeoDataFrame.GEOMETRIES -import jetbrains.datalore.plot.config.Option.Meta.MapJoin.MAP_ID import org.assertj.core.api.AbstractAssert import org.assertj.core.api.Assertions import kotlin.test.assertEquals @@ -76,14 +74,6 @@ class SingleLayerAssert private constructor(layers: List) : return this } - internal fun haveMapIds(expectedIds: List<*>): SingleLayerAssert { - return haveMapValues(MAP_ID, expectedIds) - } - - internal fun haveMapGeometries(expectedGeometries: List<*>): SingleLayerAssert { - return haveMapValues(GEOMETRIES, expectedGeometries) - } - private fun haveMapValues(key: String, expectedMapValues: List<*>): SingleLayerAssert { val geoPositions = myLayer[GEO_POSITIONS] as Map<*, *>? assertTrue(geoPositions!!.containsKey(key)) diff --git a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MapIdAndGeoPointDataProcessingTest.kt b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MapIdAndGeoPointDataProcessingTest.kt deleted file mode 100644 index 72368d6f271..00000000000 --- a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MapIdAndGeoPointDataProcessingTest.kt +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright (c) 2019. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.livemap - - -import jetbrains.datalore.base.spatial.LonLat -import jetbrains.datalore.base.typedGeometry.Vec -import jetbrains.datalore.base.typedGeometry.explicitVec -import jetbrains.datalore.plot.base.Aesthetics -import jetbrains.datalore.plot.base.aes.AestheticsBuilder -import jetbrains.datalore.plot.base.aes.AestheticsBuilder.Companion.constant -import jetbrains.datalore.plot.base.livemap.LivemapConstants.DisplayMode -import jetbrains.datalore.plot.base.livemap.LivemapConstants.DisplayMode.PIE -import jetbrains.datalore.plot.base.livemap.LivemapConstants.DisplayMode.POINT -import jetbrains.datalore.plot.config.LiveMapOptionsParser.Companion.parseFromLayerOptions -import jetbrains.datalore.plot.config.Option.Geom.LiveMap.DISPLAY_MODE -import jetbrains.datalore.plot.config.OptionsAccessor -import jetbrains.datalore.plot.livemap.ConverterDataHelper.createDefaultMatcher -import jetbrains.datalore.plot.livemap.MapObjectMatcher.Companion.eq -import kotlin.test.Ignore -import kotlin.test.Test - - -class MapIdAndGeoPointDataProcessingTest { - - @Ignore // TODO: Fix MAP_ID - @Test - fun whenSingleGeoName() { - Expectations( - DataPointKind.SINGLE, - MapIdDataKind.NAME, - null - ).doAssert() - } - - @Ignore // TODO: Fix MAP_ID - @Test - fun whenSingleLonLat() { - Expectations( - DataPointKind.SINGLE, - MapIdDataKind.LONLAT, - LONLAT_MERCATOR_GEO_COORD - ).doAssert() - } - - @Test - fun whenSingleOsmId() { - Expectations( - DataPointKind.SINGLE, - MapIdDataKind.OSM_ID, - null - ).doAssert() - } - - @Test - fun multiDataGeoName() { - Expectations( - DataPointKind.MULTI, - MapIdDataKind.NAME, - null - ).doAssert() - } - - @Ignore // TODO: Fix MAP_ID - @Test - fun multiDataLonLat() { - Expectations( - DataPointKind.MULTI, - MapIdDataKind.LONLAT, - LONLAT_MERCATOR_GEO_COORD - ).doAssert() - } - - @Test - fun multiDataOsmId() { - Expectations( - DataPointKind.MULTI, - MapIdDataKind.OSM_ID, - null - ).doAssert() - } - - internal enum class MapIdDataKind { - NAME, - LONLAT, - OSM_ID - } - - internal enum class DataPointKind { - SINGLE, - MULTI - } - - private class Expectations constructor( - private val myDataPointKind: DataPointKind, - private val myMapIdDataKind: MapIdDataKind, - private val myExpectedPoint: Vec? - ) { - private val myMapObjectMatcher = createDefaultMatcher() - .point(eq(myExpectedPoint)) - - internal fun doAssert() { - createMapObject( - myDataPointKind, - myMapIdDataKind - )?.let { - myMapObjectMatcher.match(it) - } ?: if (myExpectedPoint != null) throw AssertionError("Expect:<$myExpectedPoint> but mapObject not created") - } - } - - internal class LiveMapDataPointAestheticsProcessorBuilder( - mapIdDataKind: MapIdDataKind, - dataPointKind: DataPointKind - ) { - private val myOptions = HashMap() - private val myAes: Aesthetics - - init { - myAes = initAes(mapIdDataKind) - processOptions(dataPointKind) - } - - private fun initAes(mapIdDataKind: MapIdDataKind): Aesthetics { - - val mapId: Any = when (mapIdDataKind) { - MapIdDataKind.NAME -> TEXAS_GEO_STRING - MapIdDataKind.LONLAT -> LONLAT_GEO_STRING - MapIdDataKind.OSM_ID -> OSM_ID_STRING - } - - return AestheticsBuilder(1) - .x(constant(1.0)) - .y(constant(1.0)) - //.mapId(constant(mapId)) // TODO: Fix MAP_ID - .build() - } - - private fun processOptions(dataPointKind: DataPointKind) { - myOptions[DISPLAY_MODE] = displayModeToDataPointKind(dataPointKind) - } - - private fun displayModeToDataPointKind(dataPointKind: DataPointKind): DisplayMode { - return when (dataPointKind) { - DataPointKind.SINGLE -> POINT - DataPointKind.MULTI -> PIE - } - } - - fun build(): LiveMapDataPointAestheticsProcessor { - return LiveMapDataPointAestheticsProcessor( - myAes, - parseFromLayerOptions(OptionsAccessor(myOptions)) - ) - } - } - - companion object { - private const val TEXAS_GEO_STRING = "TEXAS" - private const val LONLAT_GEO_STRING = "0,0" - private const val OSM_ID_STRING = "123456" - private val LONLAT_MERCATOR_GEO_COORD = explicitVec(0.0, 0.0) - - private fun createMapObject(dataPointKind: DataPointKind, mapIdDataKind: MapIdDataKind): MapEntityBuilder? { - val mapObjects = createProcessorBuilder( - dataPointKind, - mapIdDataKind - ) - .build() - .mapEntityBuilders - - return if (mapObjects.isNotEmpty()) mapObjects[0] else null - } - - private fun createProcessorBuilder( - dataPointKind: DataPointKind, mapIdDataKind: MapIdDataKind - ): LiveMapDataPointAestheticsProcessorBuilder { - return LiveMapDataPointAestheticsProcessorBuilder( - mapIdDataKind, - dataPointKind - ) - } - } -} \ No newline at end of file diff --git a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelperTest.kt b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelperTest.kt index c342c5145a2..c84463a4707 100644 --- a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelperTest.kt +++ b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/MultiDataPointHelperTest.kt @@ -16,7 +16,6 @@ import jetbrains.datalore.plot.livemap.MultiDataPointHelper.SortingMode import jetbrains.datalore.plot.livemap.MultiDataPointHelper.SortingMode.BAR import jetbrains.datalore.plot.livemap.MultiDataPointHelper.SortingMode.PIE_CHART import jetbrains.datalore.plot.livemap.MultiDataPointHelperTest.MultiDataBuilder.DataPointBuilder -import kotlin.test.Ignore import kotlin.test.Test import kotlin.test.assertEquals @@ -24,7 +23,6 @@ class MultiDataPointHelperTest { private var myMultiDataBuilder = MultiDataBuilder() - @Ignore // TODO: Fix MAP_ID @Test fun whenSortingModePieChart_AndPointOrderSet_ShouldSortByOrder() { val dataPointBuilders = listOf( @@ -42,7 +40,6 @@ class MultiDataPointHelperTest { ) } - @Ignore // TODO: Fix MAP_ID @Test fun whenSortingModePieChart_AndPointOrderNotSet_ShouldUseSpecialSortingByValue() { assertPointsOrder( @@ -67,7 +64,6 @@ class MultiDataPointHelperTest { ) } - @Ignore // TODO: Fix MAP_ID @Test fun whenDataNotSorted_AndSortingModeBar_ShouldSortByOrder() { assertPointsOrder( @@ -125,15 +121,16 @@ class MultiDataPointHelperTest { private fun build(): Aesthetics { val values = ArrayList() val order = ArrayList() - val mapId = ArrayList() + val coord = ArrayList>() for (pointBuilder in myPoints) { values.add(pointBuilder.myValue) order.add(pointBuilder.myOrder) - mapId.add(pointBuilder.myCoord) + coord.add(pointBuilder.myCoord) } myBuilder - //.mapId(collection(mapId)) // TODO: Fix MAP_ID + .x(collection(coord.map(Vec::x))) + .y(collection(coord.map(Vec::y))) .symX(collection(order)) .symY(collection(values)) .dataPointCount(order.size) diff --git a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PointConverterTest.kt b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PointConverterTest.kt index 485969a1f3a..3395b534433 100644 --- a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PointConverterTest.kt +++ b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PointConverterTest.kt @@ -94,14 +94,6 @@ class PointConverterTest { assertMapObject(1) } - @Test - fun whenMapIdSet_ShouldUseGeometry() { - aesData.builder() - //.mapId(constant("New York City")) // TODO: Fix MAP_ID - - assertPointGeometryDataArray() - } - private fun assertMapObject(index: Int) { val mapObjectList = aesData.buildConverter().toPoint(PointGeom()) assertEquals(2, mapObjectList.size.toLong()) diff --git a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PolygonConverterTest.kt b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PolygonConverterTest.kt index 6c2accb73aa..b4417ad8b4b 100644 --- a/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PolygonConverterTest.kt +++ b/plot-livemap/src/commonTest/kotlin/jetbrains/datalore/plot/livemap/PolygonConverterTest.kt @@ -33,16 +33,6 @@ class PolygonConverterTest { assertMapObject() } - @Test - fun withMapIdAndGeometry_ShouldUseGeometry() { - aesData.builder() - // .mapId(constant("New York City")) // TODO: Fix MAP_ID - - matcher.geometry(geometryEq(Boundary.create(MULTIPOLYGON))) - - assertMapObject() - } - @Test fun twoRings_ShouldContainOneBoundingBox() { matcher.locationBoundingBoxes(sizeEq(1)) diff --git a/python-package/lets_plot/plot/geom.py b/python-package/lets_plot/plot/geom.py index b0b1f69e627..a0f67a261a7 100644 --- a/python-package/lets_plot/plot/geom.py +++ b/python-package/lets_plot/plot/geom.py @@ -48,7 +48,6 @@ def geom_point(mapping=None, data=None, stat=None, position=None, show_legend=No Codes and names: 0 = "none" (default), 1 = "ripple". map : dictionary, pandas DataFrame or GeoDataFrame (supported shapes Point and MultiPoint) Data containing coordinates of points. - Can be used with aesthetic parameter 'map_id' for joining data and map coordinates. Dictionary and DataFrame object must contain keys/columns: 1. 'x' or 'lon' or 'long' 2. 'y' or 'lat' @@ -70,7 +69,6 @@ def geom_point(mapping=None, data=None, stat=None, position=None, show_legend=No or two categorical variables. geom_point understands the following aesthetics mappings: - - map_id : name used to join data with map coordinates - x : x-axis value - y : y-axis value - alpha : transparency level of the point @@ -127,7 +125,6 @@ def geom_path(mapping=None, data=None, stat=None, position=None, show_legend=Non animation : type of the animation, optional Codes and names: 0 = "none" (default), 1 = "dash", 2 = "plane", 3 = "circle". map : dictionary, pandas DataFrame or GeoDataFrame (supported shapes LineString and MultiLineString) - Can be used with aesthetic parameter 'map_id' for joining data and map coordinates. Dictionary and DataFrame object must contain keys/columns: 1. 'x' or 'lon' or 'long' @@ -149,7 +146,6 @@ def geom_path(mapping=None, data=None, stat=None, position=None, show_legend=Non geom_path lets you explore how two variables are related over time. geom_path understands the following aesthetics mappings: - - map_id : name used to join data with map coordinates - x : x-axis value - y : y-axis value - alpha : transparency level of a point @@ -1110,7 +1106,6 @@ def geom_polygon(mapping=None, data=None, stat=None, position=None, show_legend= position adjustment function. map : dictionary, pandas DataFrame or GeoDataFrame (supported shapes Polygon and MultiPolygon) Data (Dictionary, DataFrame or GeoDataFrame object) contains coordinates of polygon vertices on map. - Can be used with aesthetic parameter 'map_id' for joining data and map coordinates. Dictionary and DataFrame object must contain keys/columns: 1. 'x' or 'lon' or 'long' 2. 'y' or 'lat' @@ -1130,7 +1125,6 @@ def geom_polygon(mapping=None, data=None, stat=None, position=None, show_legend= geom_polygon draws polygons, which are filled paths. Each vertex of the polygon requires a separate row in the data. geom_polygon understands the following aesthetics mappings: - - map_id : name used to join data with map coordinates - x : x-axis coordinates of the vertices of the polygon. - y : y-axis coordinates of the vertices of the polygon. - alpha : transparency level of a layer @@ -1202,10 +1196,9 @@ def geom_map(mapping=None, data=None, stat=None, show_legend=None, sampling=None ----- geom_map draws polygons which boundaries are specified by 'map' parameter. Aesthetics of ploygons (fill etc.) are computed basing on input data and mapping - (see 'data' and 'mapping' arguments, 'map_id' aesthetic). + (see 'data' and 'mapping' arguments). geom_map understands the following aesthetics: - - map_id : name used to join data with map coordinates (region boundaries). - alpha : transparency level of a layer Understands numbers between 0 and 1. - color (colour) : color of a geometry lines @@ -2058,7 +2051,6 @@ def geom_rect(mapping=None, data=None, stat=None, position=None, show_legend=Non geom_rect draws rectangles geom_rect understands the following aesthetics mappings: - - map_id : name used to join data with map coordinates - xmin : x-axis value - xmax : x-axis value - ymin : y-axis value @@ -2190,7 +2182,6 @@ def geom_text(mapping=None, data=None, stat=None, position=None, show_legend=Non Adds text directly to the plot. geom_text understands the following aesthetics mappings: - - map_id : name used to join data with map coordinates - x : x-axis value - y : y-axis value - label : text to add to plot From a66866904fccbff4f1af772e1b8b2f89605b8583 Mon Sep 17 00:00:00 2001 From: Ivan Kupriyanov Date: Wed, 10 Jun 2020 12:36:48 +0300 Subject: [PATCH 05/11] Revert nb changes --- .../geodataframe_and_geoms.ipynb | 177 +--- .../geopandas_GeoDataFrame.ipynb | 807 ++---------------- 2 files changed, 83 insertions(+), 901 deletions(-) diff --git a/docs/examples/jupyter-notebooks-dev/geodataframe_and_geoms.ipynb b/docs/examples/jupyter-notebooks-dev/geodataframe_and_geoms.ipynb index 235ec0bf114..ab60bc411bc 100644 --- a/docs/examples/jupyter-notebooks-dev/geodataframe_and_geoms.ipynb +++ b/docs/examples/jupyter-notebooks-dev/geodataframe_and_geoms.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -18,36 +18,16 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "LetsPlot.setup_html()" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -57,72 +37,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
keycoord
0CaliforniaPOINT (-119.99411 37.27734)
1NevadaPOINT (-116.66696 38.50308)
2UtahPOINT (-111.54916 39.49887)
3ArizonaPOINT (-111.66859 34.16854)
\n", - "
" - ], - "text/plain": [ - " key coord\n", - "0 California POINT (-119.99411 37.27734)\n", - "1 Nevada POINT (-116.66696 38.50308)\n", - "2 Utah POINT (-111.54916 39.49887)\n", - "3 Arizona POINT (-111.66859 34.16854)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "centroids = gpd.GeoDataFrame(\n", " data={'key': ['California', 'Nevada', 'Utah', 'Arizona'],\n", @@ -137,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -152,18 +69,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/ikupriyanov/miniconda3/lib/python3.7/site-packages/pyproj/crs/crs.py:53: FutureWarning: '+init=:' syntax is deprecated. ':' is the preferred initialization method. When making the change, be mindful of axis order changes: https://pyproj4.github.io/pyproj/stable/gotchas.html#axis-order-changes-in-proj-6\n", - " return _prepare_from_string(\" \".join(pjargs))\n" - ] - } - ], + "outputs": [], "source": [ "nv = download_geometry(165473, 'Nevada')\n", "ca = download_geometry(165475, 'California')\n", @@ -173,72 +81,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
geometrykey
0POLYGON ((-114.81358 32.49408, -111.07483 31.3...Arizona
1MULTIPOLYGON (((-124.32884 41.99833, -119.9994...California
2POLYGON ((-114.05283 37.57353, -114.05005 37.0...Utah
3POLYGON ((-120.00556 39.25849, -120.00101 38.9...Nevada
\n", - "
" - ], - "text/plain": [ - " geometry key\n", - "0 POLYGON ((-114.81358 32.49408, -111.07483 31.3... Arizona\n", - "1 MULTIPOLYGON (((-124.32884 41.99833, -119.9994... California\n", - "2 POLYGON ((-114.05283 37.57353, -114.05005 37.0... Utah\n", - "3 POLYGON ((-120.00556 39.25849, -120.00101 38.9... Nevada" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "boundaries = ar\n", "boundaries = pd.concat([boundaries, ca], ignore_index=True)\n", diff --git a/docs/examples/jupyter-notebooks-dev/geopandas_GeoDataFrame.ipynb b/docs/examples/jupyter-notebooks-dev/geopandas_GeoDataFrame.ipynb index 504ddb02f95..2c6d90b2f6f 100644 --- a/docs/examples/jupyter-notebooks-dev/geopandas_GeoDataFrame.ipynb +++ b/docs/examples/jupyter-notebooks-dev/geopandas_GeoDataFrame.ipynb @@ -2,43 +2,23 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", - "from lets_plot import *\n", - "LetsPlot.setup_html()\n", - "\n", - "def dump_plot(plot):\n", - " import json\n", - " from lets_plot._type_utils import standardize_dict\n", - " \n", - " plot_dict = standardize_dict(plot.as_dict())\n", - " plot_json = json.dumps(plot_dict, indent=2)\n", - " print(plot_json)" + "from lets_plot import *" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "LetsPlot.setup_html()" ] }, { @@ -50,107 +30,22 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from geopandas import GeoDataFrame\n", - "from shapely.geometry import MultiPolygon, Polygon, LinearRing, Point, MultiPoint, LineString, MultiLineString, mapping" + "from shapely.geometry import MultiPolygon, Polygon, LinearRing, Point, mapping" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
kindcoord
0PointPOINT (-5.00000 17.00000)
1MPointMULTIPOINT (3.00000 15.00000, 6.00000 13.00000)
2LineLINESTRING (0.00000 0.00000, 5.00000 5.00000)
3MLineMULTILINESTRING ((10.00000 0.00000, 15.00000 5...
4PolygonPOLYGON ((1.00000 1.00000, 1.00000 9.00000, 9....
5MPolygonMULTIPOLYGON (((11.00000 12.00000, 13.00000 14...
\n", - "
" - ], - "text/plain": [ - " kind coord\n", - "0 Point POINT (-5.00000 17.00000)\n", - "1 MPoint MULTIPOINT (3.00000 15.00000, 6.00000 13.00000)\n", - "2 Line LINESTRING (0.00000 0.00000, 5.00000 5.00000)\n", - "3 MLine MULTILINESTRING ((10.00000 0.00000, 15.00000 5...\n", - "4 Polygon POLYGON ((1.00000 1.00000, 1.00000 9.00000, 9....\n", - "5 MPolygon MULTIPOLYGON (((11.00000 12.00000, 13.00000 14..." - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "POINT = Point(-5, 17)\n", "\n", - "MULTI_POINT = MultiPoint([Point(3, 15), Point(6, 13)])\n", - "\n", - "LINE = LineString([(0, 0), (5, 5)])\n", - "\n", - "MULTI_LINE = MultiLineString([\n", - " LineString([(10, 0), (15, 5)]),\n", - " LineString([(10, 5), (15, 0)])\n", - "])\n", - "\n", - "\n", "POLYGON = Polygon(\n", " LinearRing([(1, 1), (1, 9), (9, 9), (9, 1)]),\n", " [LinearRing([(2, 2), (3, 2), (3, 3), (2, 3)]),\n", @@ -159,715 +54,157 @@ "\n", "MULTIPOLYGON = MultiPolygon([\n", " Polygon(LinearRing([(11, 12), (13, 14), (15, 13)]))\n", - " ])\n", - "\n", - "gdf = GeoDataFrame(\n", - " data={\n", - " 'kind': ['Point', 'MPoint', 'Line', 'MLine', 'Polygon', 'MPolygon'],\n", - " 'coord': [POINT, MULTI_POINT, LINE, MULTI_LINE, POLYGON, MULTIPOLYGON]\n", - " },\n", - " geometry='coord'\n", - ")\n", - "gdf" + " ])" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "ggplot() + geom_polygon(aes(fill = 'kind'), gdf)" + "gdf = GeoDataFrame(\n", + " data={'id': ['A', 'B', 'C'],\n", + " 'coord': [POINT, POLYGON, MULTIPOLYGON]},\n", + " geometry='coord')" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "ggplot() + geom_polygon(map = gdf)" + "ggplot() + geom_polygon(aes(fill = 'id'), gdf)" ] }, { - "cell_type": "code", - "execution_count": 6, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "\n", - "df = pd.DataFrame(\n", - " {\n", - " 'value': [42, 23, 66],\n", - " 'fig': ['Polygon', 'MPolygon', 'C'] \n", - " }\n", - ")" + "Mapping doesn't work for DataFrame in map" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "ggplot() + geom_polygon(aes(fill = 'value'), df, map = gdf, map_join=['fig', 'kind'])" + "ggplot() + geom_polygon(aes(fill = 'id'), map = gdf)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "#### Geom kinds that support GeoDataFrame" + "ggplot() + geom_polygon(map = gdf)" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "ggplot() + geom_polygon(aes(fill='kind'), data = gdf)" + "df = pd.DataFrame(\n", + " {'name': ['A', 'B', 'C'],\n", + " 'value': [42, 23, 87]})" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "ggplot() + geom_point(aes(color='kind'), gdf, size=5)" + "ggplot() + geom_polygon(aes(fill = 'value'), df, map = gdf, map_join=('name', 'id'))" ] }, { - "cell_type": "code", - "execution_count": 17, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "ggplot() + geom_rect(aes(fill='kind'), gdf)" + "#### Geom kinds that support GeoDataFrame" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "p = ggplot() + geom_path(data = gdf)\n", - "def dump_plot(plot):\n", - " import json\n", - " from lets_plot._type_utils import standardize_dict\n", - " \n", - " plot_dict = standardize_dict(plot.as_dict())\n", - " plot_json = json.dumps(plot_dict, indent=2)\n", - " print(plot_json)\n", - " \n", - "p" + "ggplot() + geom_polygon(data = gdf)" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "ggplot() + geom_map(map = gdf)" + "ggplot() + geom_point(aes(color='id'), gdf, size=5)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "#### Problems" + "ggplot() + geom_rect(aes(fill='id'), gdf)" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "ggplot() + geom_map(data = gdf)" + "ggplot() + geom_path(data = gdf)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "Error message contains autogenerated column \"\\_\\_key\\_\\_\". It's ok." + "ggplot() + geom_map(map = gdf)" ] }, { - "cell_type": "code", - "execution_count": 14, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "ggplot() + geom_polygon(aes(fill = 'kind'), gdf)" + "#### Problems" ] }, { "cell_type": "code", - "execution_count": 15, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "tiny_gdf = GeoDataFrame(\n", - " #data={'lll': ['A'], 'coord': [MULTIPOLYGON]},\n", - " data={'lll': ['A', 'B', 'C'], 'coord': [POINT, POLYGON, MULTIPOLYGON]},\n", - " geometry='coord')\n", - "ggplot() + geom_polygon(map = tiny_gdf)" + "ggplot() + geom_map(data = gdf)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Matching doesn't work for DataFrame in map" + "Error message contains autogenerated column \"\\_\\_key\\_\\_\". It's ok." ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "ggplot() + geom_polygon(aes(fill = 'kind'), map = gdf)" + "ggplot() + geom_polygon(aes(fill = 'id'), gdf)" ] } ], From 1ea6fda8927433b2aee60aa5a380c91bbe7cd303 Mon Sep 17 00:00:00 2001 From: Ivan Kupriyanov Date: Wed, 10 Jun 2020 13:01:59 +0300 Subject: [PATCH 06/11] Update pie/bar demo notebook --- .../livemap/pie_bar_on_livemap.ipynb | 229 ++---------------- 1 file changed, 16 insertions(+), 213 deletions(-) diff --git a/docs/examples/jupyter-notebooks-dev/livemap/pie_bar_on_livemap.ipynb b/docs/examples/jupyter-notebooks-dev/livemap/pie_bar_on_livemap.ipynb index 9ec2decd6bf..c1bb86113f6 100644 --- a/docs/examples/jupyter-notebooks-dev/livemap/pie_bar_on_livemap.ipynb +++ b/docs/examples/jupyter-notebooks-dev/livemap/pie_bar_on_livemap.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -11,45 +11,26 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "LetsPlot.setup_html()" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "LetsPlot.set(maptiles_zxy(url='https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png'))" + "from lets_plot.settings_utils import maptiles_lets_plot\n", + "LetsPlot.set(maptiles_lets_plot(url='ws://10.0.0.127:3933'))" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -62,212 +43,34 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "scrolled": false }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "ggplot(data) + geom_livemap(aes(map_id='lnglat', x = 'order', y='val', fill = 'order'), symbol='bar', size = 30)" + "ggplot(data) + geom_livemap(aes(x='lng', y='lat', sym_y='val', fill = 'val'), symbol='bar', size = 30)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "ggplot(data) \\\n", - " + geom_livemap(aes(map_id='lnglat', x='order', y= 'val', fill='order'), symbol='pie', size=30) \\\n", + " + geom_livemap(aes(x='lng', y='lat', sym_y='val', fill='val'), symbol='pie', size=60) \\\n", " + scale_fill_hue(h = [0.,120.], name = 'order')" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "ggplot(data) \\\n", - " + geom_livemap(aes(map_id='lnglat', x='order', y= 'val', fill='order'), symbol='bar', size=50) \\\n", + " + geom_livemap(aes(x='lng', y='lat', sym_x='order', sym_y= 'val', fill='val'), symbol='bar', size=50) \\\n", " + scale_fill_hue(h = [0.,120.], name = 'order')" ] } From 3fb161e6ee28351dc6f97d6b2270640900135eee Mon Sep 17 00:00:00 2001 From: Ivan Kupriyanov Date: Wed, 10 Jun 2020 16:43:09 +0300 Subject: [PATCH 07/11] Cleanup --- .../plot/base/aes/AestheticsDefaults.kt | 4 +--- .../plot/base/livemap/LiveMapConstants.kt | 2 -- .../builder/assemble/geom/GeomProvider.kt | 2 +- .../plot/builder/map/GeoPositionField.kt | 23 ------------------- .../plot/builder/tooltip/MappedAes.kt | 18 +-------------- .../interact/TooltipSpecAxisTooltipTest.kt | 15 ------------ .../plot/config/LiveMapOptionsParser.kt | 2 +- .../LiveMapDataPointAestheticsProcessor.kt | 2 -- .../plot/livemap/LiveMapSpecBuilder.kt | 18 +++++++++------ 9 files changed, 15 insertions(+), 71 deletions(-) delete mode 100644 plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/map/GeoPositionField.kt diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AestheticsDefaults.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AestheticsDefaults.kt index e4d47f362de..948ac025168 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AestheticsDefaults.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/aes/AestheticsDefaults.kt @@ -152,9 +152,8 @@ open class AestheticsDefaults { return crossBar() } - fun livemap(displayMode: LivemapConstants.DisplayMode, scaled: Boolean): AestheticsDefaults { + fun livemap(displayMode: LivemapConstants.DisplayMode): AestheticsDefaults { return when (displayMode) { - LivemapConstants.DisplayMode.POLYGON -> polygon() LivemapConstants.DisplayMode.POINT -> point() .updateInLegend(Aes.SIZE, 5.0) LivemapConstants.DisplayMode.BAR -> base() @@ -164,7 +163,6 @@ open class AestheticsDefaults { .update(Aes.SIZE, 20.0) .update(Aes.COLOR, Color.TRANSPARENT) .updateInLegend(Aes.SIZE, 5.0) - LivemapConstants.DisplayMode.HEATMAP -> base().update(Aes.SIZE, if (scaled) 0.01 else 10.0) } } diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/livemap/LiveMapConstants.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/livemap/LiveMapConstants.kt index 88eb1cefe96..18c35becea6 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/livemap/LiveMapConstants.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/livemap/LiveMapConstants.kt @@ -7,10 +7,8 @@ package jetbrains.datalore.plot.base.livemap interface LivemapConstants { enum class DisplayMode { - POLYGON, POINT, PIE, - HEATMAP, BAR } diff --git a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/assemble/geom/GeomProvider.kt b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/assemble/geom/GeomProvider.kt index 20d3f16b5a5..d16e1799997 100644 --- a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/assemble/geom/GeomProvider.kt +++ b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/assemble/geom/GeomProvider.kt @@ -233,7 +233,7 @@ abstract class GeomProvider private constructor(val geomKind: GeomKind) { ): GeomProvider { return GeomProviderBuilder( GeomKind.LIVE_MAP, - AestheticsDefaults.livemap(options.displayMode, options.scaled), + AestheticsDefaults.livemap(options.displayMode), LiveMapGeom.HANDLES_GROUPS, myGeomSupplier = { LiveMapGeom(options.displayMode) } ).build() diff --git a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/map/GeoPositionField.kt b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/map/GeoPositionField.kt deleted file mode 100644 index c7e85b62196..00000000000 --- a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/map/GeoPositionField.kt +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2019. JetBrains s.r.o. - * Use of this source code is governed by the MIT license that can be found in the LICENSE file. - */ - -package jetbrains.datalore.plot.builder.map - - -// column names in data-frames provided by geocoding -object GeoPositionField { - - // fixed columns in 'boundaries' of 'centroids' data frames - const val POINT_X = "lon" - const val POINT_X1 = "longitude" - const val POINT_Y = "lat" - const val POINT_Y1 = "latitude" - - // fixed columns in 'limits' - const val RECT_XMIN = "lonmin" - const val RECT_XMAX = "lonmax" - const val RECT_YMIN = "latmin" - const val RECT_YMAX = "latmax" -} diff --git a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/tooltip/MappedAes.kt b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/tooltip/MappedAes.kt index ad476f55dcc..902a8fb337c 100644 --- a/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/tooltip/MappedAes.kt +++ b/plot-builder-portable/src/commonMain/kotlin/jetbrains/datalore/plot/builder/tooltip/MappedAes.kt @@ -10,7 +10,6 @@ import jetbrains.datalore.plot.base.interact.DataContext import jetbrains.datalore.plot.base.interact.MappedDataAccess import jetbrains.datalore.plot.base.interact.ValueSource import jetbrains.datalore.plot.base.interact.ValueSource.DataPoint -import jetbrains.datalore.plot.builder.map.GeoPositionField open class MappedAes( protected val aes: Aes<*>, @@ -28,7 +27,7 @@ open class MappedAes( val mappedDataPoint = getMappedDataPoint(index) ?: return null val label2value: Pair = when { - isAxis && !isAxisTooltipAllowed(mappedDataPoint) -> null + isAxis && !mappedDataPoint.isContinuous -> null isAxis -> "" to mappedDataPoint.value else -> createLabeledValue( index = index, @@ -62,13 +61,6 @@ open class MappedAes( ) } - private fun isAxisTooltipAllowed(sourceDataPoint: DataPoint): Boolean { - return when { - MAP_COORDINATE_NAMES.contains(sourceDataPoint.label) -> false - else -> sourceDataPoint.isContinuous - } - } - private fun createLabeledValue( index: Int, value: String, @@ -85,7 +77,6 @@ open class MappedAes( } fun shortText() = "" to value - fun fullText() = label to value return when { @@ -101,13 +92,6 @@ open class MappedAes( } companion object { - private val MAP_COORDINATE_NAMES = setOf( - GeoPositionField.POINT_X, - GeoPositionField.POINT_X1, - GeoPositionField.POINT_Y, - GeoPositionField.POINT_Y1 - ) - fun createMappedAxis(aes: Aes<*>, dataContext: DataContext): ValueSource = MappedAes(aes, isOutlier = true, isAxis = true).also { it.setDataPointProvider(dataContext) } diff --git a/plot-builder-portable/src/jvmTest/kotlin/plot/builder/interact/TooltipSpecAxisTooltipTest.kt b/plot-builder-portable/src/jvmTest/kotlin/plot/builder/interact/TooltipSpecAxisTooltipTest.kt index 46cf28d83dd..d3499f13056 100644 --- a/plot-builder-portable/src/jvmTest/kotlin/plot/builder/interact/TooltipSpecAxisTooltipTest.kt +++ b/plot-builder-portable/src/jvmTest/kotlin/plot/builder/interact/TooltipSpecAxisTooltipTest.kt @@ -65,19 +65,4 @@ class TooltipSpecAxisTooltipTest : jetbrains.datalore.plot.builder.interact.Tool assertLines(0, fillMapping.shortTooltipText()) assertLines(1, yMapping.shortTooltipText()) } - - @Test - fun mapVarsShouldNotBeAddedToAxisTooltip() { - val namesToIgnore = listOf("lon", "longitude", "lat", "latitude") - - for (name in namesToIgnore) { - val var1 = variable().name(name).value("0").isContinuous(true) - - addMappedData(var1.mapping(Aes.X)) - - buildTooltipSpecs() - - assertTooltipsCount(0) - } - } } diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/LiveMapOptionsParser.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/LiveMapOptionsParser.kt index 9e0a7f45588..576dcc488c3 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/LiveMapOptionsParser.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/LiveMapOptionsParser.kt @@ -34,7 +34,7 @@ class LiveMapOptionsParser { location = opts.get(LiveMap.LOCATION), stroke = opts.getDouble(LiveMap.STROKE), interactive = opts.getBoolean(LiveMap.INTERACTIVE, true), - displayMode = opts.getString(LiveMap.DISPLAY_MODE)?.let(::parseDisplayMode) ?: DisplayMode.POLYGON, + displayMode = opts.getString(LiveMap.DISPLAY_MODE)?.let(::parseDisplayMode) ?: DisplayMode.POINT, scaled = opts.getBoolean(LiveMap.SCALED, false), clustering = opts.getBoolean(LiveMap.CLUSTERING, false), labels = opts.getBoolean(LiveMap.LABELS, true), diff --git a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapDataPointAestheticsProcessor.kt b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapDataPointAestheticsProcessor.kt index 592a12f1585..4a97a91c9ad 100644 --- a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapDataPointAestheticsProcessor.kt +++ b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapDataPointAestheticsProcessor.kt @@ -33,10 +33,8 @@ internal class LiveMapDataPointAestheticsProcessor( private fun getLayerKind(displayMode: LivemapConstants.DisplayMode): MapLayerKind { return when (displayMode) { - LivemapConstants.DisplayMode.POLYGON -> MapLayerKind.POLYGON LivemapConstants.DisplayMode.POINT -> MapLayerKind.POINT LivemapConstants.DisplayMode.PIE -> MapLayerKind.PIE - LivemapConstants.DisplayMode.HEATMAP -> MapLayerKind.HEATMAP LivemapConstants.DisplayMode.BAR -> MapLayerKind.BAR } } diff --git a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapSpecBuilder.kt b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapSpecBuilder.kt index eb7f37ec2b5..72f484d7082 100644 --- a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapSpecBuilder.kt +++ b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapSpecBuilder.kt @@ -14,13 +14,6 @@ import jetbrains.datalore.plot.base.Aesthetics import jetbrains.datalore.plot.base.interact.MappedDataAccess import jetbrains.datalore.plot.base.livemap.LiveMapOptions import jetbrains.datalore.plot.base.livemap.LivemapConstants -import jetbrains.datalore.plot.builder.map.GeoPositionField.POINT_X -import jetbrains.datalore.plot.builder.map.GeoPositionField.POINT_Y -import jetbrains.datalore.plot.builder.map.GeoPositionField.RECT_XMAX -import jetbrains.datalore.plot.builder.map.GeoPositionField.RECT_XMIN -import jetbrains.datalore.plot.builder.map.GeoPositionField.RECT_YMAX -import jetbrains.datalore.plot.builder.map.GeoPositionField.RECT_YMIN -import jetbrains.gis.geoprotocol.FeatureLevel import jetbrains.gis.geoprotocol.GeocodingService import jetbrains.gis.geoprotocol.MapRegion import jetbrains.gis.tileprotocol.TileService @@ -129,6 +122,16 @@ internal class LiveMapSpecBuilder { private const val REGION_TYPE_COORDINATES = "coordinates" private const val REGION_TYPE_DATAFRAME = "data_frame" + // fixed columns in 'boundaries' of 'centroids' data frames + private const val POINT_X = "lon" + private const val POINT_Y = "lat" + + // fixed columns in 'limits' + private const val RECT_XMIN = "lonmin" + private const val RECT_XMAX = "lonmax" + private const val RECT_YMIN = "latmin" + private const val RECT_YMAX = "latmax" + object Tile { const val KIND = "kind" const val URL = "url" @@ -342,4 +345,5 @@ internal class LiveMapSpecBuilder { ) } } + } \ No newline at end of file From 8e42b8a98be16b4885abd9dfb323ca72929e27f7 Mon Sep 17 00:00:00 2001 From: Ivan Kupriyanov Date: Wed, 10 Jun 2020 23:26:12 +0300 Subject: [PATCH 08/11] Minor fixes --- .../datalore/plot/config/GeoConfig.kt | 176 +++++++++--------- .../plot/livemap/LiveMapSpecBuilder.kt | 2 - 2 files changed, 86 insertions(+), 92 deletions(-) diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt index 7cf4edebc3e..3b5562c429f 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt @@ -15,7 +15,7 @@ import jetbrains.datalore.plot.base.data.DataFrameUtil.variables import jetbrains.datalore.plot.config.ConfigUtil.createAesMapping import jetbrains.datalore.plot.config.ConfigUtil.createDataFrame import jetbrains.datalore.plot.config.ConfigUtil.rightJoin -import jetbrains.datalore.plot.config.CoordinatesBuilder.Companion.createCoordinateBuilder +import jetbrains.datalore.plot.config.CoordinatesCollector.* import jetbrains.datalore.plot.config.Option.Geom.Choropleth.GEO_POSITIONS import jetbrains.datalore.plot.config.Option.Layer.MAP_JOIN import jetbrains.datalore.plot.config.Option.Meta.DATA_META @@ -60,22 +60,14 @@ class GeoConfig( when { // (aes(color='cyl'), data=data, map=gdf) - how to join without `map_join`? - with(layerOptions) { has(GEO_POSITIONS) && !has(MAP_JOIN) && !data.isEmpty && mappingOptions.isNotEmpty() } -> { + with(layerOptions) { has(MAP_DATA_META, GDF, GEOMETRY) && !has(MAP_JOIN) && !data.isEmpty && mappingOptions.isNotEmpty() } -> { + require(layerOptions.has(GEO_POSITIONS)) { "'map' parameter is mandatory with MAP_DATA_META" } error(MAP_JOIN_REQUIRED_MESSAGE) } - - // (map=gdf) - simple geometry - with(layerOptions) { has(GEO_POSITIONS) && !has(MAP_JOIN) && has(MAP_DATA_META, GDF, GEOMETRY) } -> { - geoJson = getGeoJson(GEO_POSITIONS) - - dataJoinColumn = autoId - mapJoinColumn = autoId - joinIds = geoJson.indices.map(Int::toString) - dataFrame = DataFrame.Builder(data).put(DataFrame.Variable(dataJoinColumn), joinIds).build() - } // (data=data, map=gdf, map_join=('id', 'city')) - with(layerOptions) { has(GEO_POSITIONS) && has(MAP_DATA_META, GDF, GEOMETRY) && has(MAP_JOIN) } -> { + with(layerOptions) { has(MAP_DATA_META, GDF, GEOMETRY) && has(MAP_JOIN) } -> { + require(layerOptions.has(GEO_POSITIONS)) { "'map' parameter is mandatory with MAP_DATA_META" } geoJson = getGeoJson(GEO_POSITIONS) val mapJoin = layerOptions.getList(MAP_JOIN) ?: error("require map_join parameter") @@ -85,8 +77,20 @@ class GeoConfig( dataFrame = data } + // (map=gdf) - simple geometry + with(layerOptions) { has(MAP_DATA_META, GDF, GEOMETRY) && !has(MAP_JOIN) } -> { + require(layerOptions.has(GEO_POSITIONS)) { "'map' parameter is mandatory with MAP_DATA_META" } + geoJson = getGeoJson(GEO_POSITIONS) + + dataJoinColumn = autoId + mapJoinColumn = autoId + joinIds = geoJson.indices.map(Int::toString) + dataFrame = DataFrame.Builder(data).put(DataFrame.Variable(dataJoinColumn), joinIds).build() + } + // (data=gdf) - with(layerOptions) { !has(GEO_POSITIONS) && has(DATA_META, GDF, GEOMETRY) } -> { + with(layerOptions) { has(DATA_META, GDF, GEOMETRY) && !has(GEO_POSITIONS) && !has(MAP_JOIN) } -> { + require(layerOptions.has(DATA)) { "'data' parameter is mandatory with DATA_META" } geoJson = getGeoJson(DATA) dataJoinColumn = autoId @@ -94,23 +98,32 @@ class GeoConfig( joinIds = geoJson.indices.map(Int::toString) dataFrame = DataFrame.Builder(data).put(DataFrame.Variable(dataJoinColumn), joinIds).build() } + else -> error("GeoDataFrame not found in data or map") } - val coordinatesBuilder = createCoordinateBuilder(geomKind) + val coordinatesCollector = when(geomKind) { + MAP, POLYGON -> BoundaryCoordinatesCollector() + POINT, TEXT -> PointCoordinatesCollector() + RECT -> BboxCoordinatesCollector() + PATH -> PathCoordinatesCollector() + else -> error("Unsupported geom: $geomKind") + } + + coordinatesCollector .append(geoJson) .setIdColumn(columnName = mapJoinColumn, values = joinIds) dataAndCoordinates = rightJoin( left = dataFrame, leftKey = dataJoinColumn, - right = createDataFrame(coordinatesBuilder.build()), + right = createDataFrame(coordinatesCollector.buildCoordinatesMap()), rightKey = mapJoinColumn ) - val coordinatesAutoMapping = coordinatesBuilder.columns - .filterKeys { coordName -> coordName in variables(dataAndCoordinates) } - .map { (coordName, aes) -> aes to variables(dataAndCoordinates).getValue(coordName) } + val coordinatesAutoMapping = coordinatesCollector.mappings + .filterValues { coordName -> coordName in variables(dataAndCoordinates) } + .map { (aes, coordName) -> aes to variables(dataAndCoordinates).getValue(coordName) } .toMap() mappings = createAesMapping(dataAndCoordinates, mappingOptions) + coordinatesAutoMapping } @@ -132,93 +145,45 @@ const val RECT_YMIN = "__gdf_ymin__" const val RECT_XMAX = "__gdf_xmax__" const val RECT_YMAX = "__gdf_ymax__" -internal abstract class CoordinatesBuilder( - val columns: Map> +internal abstract class CoordinatesCollector( + val mappings: Map, String> ) { - companion object { - - fun createCoordinateBuilder(geomKind: GeomKind): CoordinatesBuilder { - return when(geomKind) { - MAP, POLYGON -> BoundaryCoordinatesBuilder() - POINT, TEXT -> PointCoordinatesBuilder() - RECT -> BboxCoordinatesBuilder() - PATH -> PathCoordinatesBuilder() - else -> error("Unsupported geom: $geomKind") - } - } - - val POINT_COLUMNS = mapOf( - POINT_X to Aes.X, - POINT_Y to Aes.Y - ) - - val RECT_COLUMNS = mapOf( - RECT_XMIN to Aes.XMIN, - RECT_YMIN to Aes.YMIN, - RECT_XMAX to Aes.XMAX, - RECT_YMAX to Aes.YMAX - ) - - internal fun Map>.append(p: Vec) { - append(POINT_X, p.x) - append(POINT_Y, p.y) - } - - internal fun Map>.append(rect: Rect) { - append(RECT_XMIN, rect.left) - append(RECT_XMAX, rect.right) - append(RECT_YMIN, rect.top) - append(RECT_YMAX, rect.bottom) - } - - private fun Map>.append(key: String, value: Double) { - get(key)?.add(value) ?: error("$key is not found") - } - } - - private var idColumnName: String? = null - private var ids: List? = null + private lateinit var idColumnName: String + private lateinit var ids: List private val groupLengths = mutableListOf() - protected val coordinates: Map> = columns.keys.associateBy({ it }) { mutableListOf() } + protected val coordinates: Map> = mappings.values.associateBy({ it }) { mutableListOf() } protected abstract val geoJsonConsumer: SimpleFeature.Consumer protected abstract val supportedFeatures: List - fun append(geoJsons: List): CoordinatesBuilder { + fun append(geoJsons: List): CoordinatesCollector { geoJsons.forEach { val oldRowCount = coordinates.rowCount GeoJson.parse(it, geoJsonConsumer) groupLengths += coordinates.rowCount - oldRowCount } + + if (coordinates.rowCount == 0) { + error("Geometries are empty or no matching types. Expected: " + supportedFeatures) + } + return this } - fun setIdColumn(columnName: String, values: List): CoordinatesBuilder { + fun setIdColumn(columnName: String, values: List): CoordinatesCollector { idColumnName = columnName ids = values return this } - fun build(): Map> { - if (coordinates.rowCount == 0) { - error("Geometries are empty or no matching types. Expected: " + supportedFeatures) - } - - if (idColumnName == null && ids == null) { - return coordinates - } - - if (idColumnName != null && ids != null) { - require(groupLengths.size == ids!!.size) { "Groups and ids should have same size" } + fun buildCoordinatesMap(): Map> { + require(groupLengths.size == ids.size) { "Groups and ids should have same size" } - // (['a', 'b'], [2, 3]) => ['a', 'a', 'b', 'b', 'b'] - fun copies(values: Collection, count: Collection) = - values.asSequence().zip(count.asSequence()) - .fold(mutableListOf()) { acc, (value, count) -> repeat(count) { acc += value }; acc } + // (['a', 'b'], [2, 3]) => ['a', 'a', 'b', 'b', 'b'] + fun copies(values: Collection, count: Collection) = + values.asSequence().zip(count.asSequence()) + .fold(mutableListOf()) { acc, (value, count) -> repeat(count) { acc += value }; acc } - return coordinates + (idColumnName!! to copies(ids!!, groupLengths)) - } - - error("idColumnName and idValues should be both null or not null") + return coordinates + (idColumnName to copies(ids, groupLengths)) } internal fun defaultConsumer(config: SimpleFeature.Consumer.() -> Unit) = @@ -233,7 +198,7 @@ internal abstract class CoordinatesBuilder( private val > Map.rowCount get() = values.firstOrNull()?.size ?: 0 - class PointCoordinatesBuilder : CoordinatesBuilder(POINT_COLUMNS) { + class PointCoordinatesCollector : CoordinatesCollector(POINT_COLUMNS) { override val supportedFeatures = listOf("Point, MultiPoint") override val geoJsonConsumer: SimpleFeature.Consumer = defaultConsumer { onPoint = { p -> coordinates.append(p) } @@ -241,7 +206,7 @@ internal abstract class CoordinatesBuilder( } } - class PathCoordinatesBuilder : CoordinatesBuilder(POINT_COLUMNS) { + class PathCoordinatesCollector : CoordinatesCollector(POINT_COLUMNS) { override val supportedFeatures = listOf("LineString, MultiLineString") override val geoJsonConsumer: SimpleFeature.Consumer = defaultConsumer { onLineString = { it.forEach { p -> coordinates.append(p) } } @@ -249,7 +214,7 @@ internal abstract class CoordinatesBuilder( } } - class BoundaryCoordinatesBuilder : CoordinatesBuilder(POINT_COLUMNS) { + class BoundaryCoordinatesCollector : CoordinatesCollector(POINT_COLUMNS) { override val supportedFeatures = listOf("Polygon, MultiPolygon") override val geoJsonConsumer: SimpleFeature.Consumer = defaultConsumer { onPolygon = { it.asSequence().flatten().forEach { p -> coordinates.append(p) } } @@ -257,7 +222,7 @@ internal abstract class CoordinatesBuilder( } } - class BboxCoordinatesBuilder : CoordinatesBuilder(RECT_COLUMNS) { + class BboxCoordinatesCollector : CoordinatesCollector(RECT_MAPPINGS) { override val supportedFeatures = listOf("MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon") override val geoJsonConsumer: SimpleFeature.Consumer = defaultConsumer { fun insert(bboxes: List>) = @@ -276,6 +241,37 @@ internal abstract class CoordinatesBuilder( onMultiPolygon = { insert(it.limit()) } } } + + companion object { + + val POINT_COLUMNS = mapOf, String>( + Aes.X to POINT_X, + Aes.Y to POINT_Y + ) + + val RECT_MAPPINGS = mapOf, String>( + Aes.XMIN to RECT_XMIN, + Aes.YMIN to RECT_YMIN, + Aes.XMAX to RECT_XMAX, + Aes.YMAX to RECT_YMAX + ) + + internal fun Map>.append(p: Vec) { + append(POINT_X, p.x) + append(POINT_Y, p.y) + } + + internal fun Map>.append(rect: Rect) { + append(RECT_XMIN, rect.left) + append(RECT_XMAX, rect.right) + append(RECT_YMIN, rect.top) + append(RECT_YMAX, rect.bottom) + } + + private fun Map>.append(key: String, value: Double) { + get(key)?.add(value) ?: error("$key is not found") + } + } } diff --git a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapSpecBuilder.kt b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapSpecBuilder.kt index 72f484d7082..6ed4aab9a76 100644 --- a/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapSpecBuilder.kt +++ b/plot-livemap/src/commonMain/kotlin/jetbrains/datalore/plot/livemap/LiveMapSpecBuilder.kt @@ -122,11 +122,9 @@ internal class LiveMapSpecBuilder { private const val REGION_TYPE_COORDINATES = "coordinates" private const val REGION_TYPE_DATAFRAME = "data_frame" - // fixed columns in 'boundaries' of 'centroids' data frames private const val POINT_X = "lon" private const val POINT_Y = "lat" - // fixed columns in 'limits' private const val RECT_XMIN = "lonmin" private const val RECT_XMAX = "lonmax" private const val RECT_YMIN = "latmin" From 50644ec970b7f1ccbd1dae071bc39d02e815a735 Mon Sep 17 00:00:00 2001 From: Ivan Kupriyanov Date: Mon, 15 Jun 2020 14:00:49 +0300 Subject: [PATCH 09/11] Better error message for missing data key, more tests --- .../datalore/plot/base/data/DataFrameUtil.kt | 2 + .../datalore/plot/config/GeoConfig.kt | 55 +++++++++++-------- .../kotlin/plot/config/GeoConfigTest.kt | 33 ++++++++++- .../plotDemo/model/plotConfig/GeoData.kt | 51 +++-------------- 4 files changed, 72 insertions(+), 69 deletions(-) diff --git a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/data/DataFrameUtil.kt b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/data/DataFrameUtil.kt index 6aac6a2631a..934d949de55 100644 --- a/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/data/DataFrameUtil.kt +++ b/plot-base-portable/src/commonMain/kotlin/jetbrains/datalore/plot/base/data/DataFrameUtil.kt @@ -169,3 +169,5 @@ object DataFrameUtil { return b.build() } } + + diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt index 3b5562c429f..510f57e734a 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt @@ -11,6 +11,7 @@ import jetbrains.datalore.plot.base.Aes import jetbrains.datalore.plot.base.DataFrame import jetbrains.datalore.plot.base.GeomKind import jetbrains.datalore.plot.base.GeomKind.* +import jetbrains.datalore.plot.base.data.DataFrameUtil.findVariableOrFail import jetbrains.datalore.plot.base.data.DataFrameUtil.variables import jetbrains.datalore.plot.config.ConfigUtil.createAesMapping import jetbrains.datalore.plot.config.ConfigUtil.createDataFrame @@ -51,12 +52,12 @@ class GeoConfig( return geoDataFrame.getList(geoColumn)?.map { it as String } ?: error("$geoColumn not found in $gdfLocation") } - val joinIds: List - val dataJoinColumn: String - val mapJoinColumn: String + val mapKeys: List + val dataKeyColumn: String + val mapKeyColumn: String val geoJson: List val dataFrame: DataFrame - val autoId = "__gdf_id__" + val autoId = "__id__" when { // (aes(color='cyl'), data=data, map=gdf) - how to join without `map_join`? @@ -71,9 +72,15 @@ class GeoConfig( geoJson = getGeoJson(GEO_POSITIONS) val mapJoin = layerOptions.getList(MAP_JOIN) ?: error("require map_join parameter") - dataJoinColumn = mapJoin[0] as String - mapJoinColumn = mapJoin[1] as String - joinIds = layerOptions.getMap(GEO_POSITIONS)?.getList(mapJoinColumn)?.requireNoNulls() ?: error("MapJoinColumn '$mapJoinColumn' is not found") + dataKeyColumn = mapJoin[0] as String + mapKeyColumn = mapJoin[1] as String + mapKeys = layerOptions.getMap(GEO_POSITIONS)?.getList(mapKeyColumn)?.requireNoNulls() ?: error("'$mapKeyColumn' is not found in map") + + // All data keys should be present in map + data.get(findVariableOrFail(data, dataKeyColumn)) + .firstOrNull { dataKey -> dataKey !in mapKeys } + ?.let { error("'$it' not found in map") } + dataFrame = data } @@ -82,10 +89,10 @@ class GeoConfig( require(layerOptions.has(GEO_POSITIONS)) { "'map' parameter is mandatory with MAP_DATA_META" } geoJson = getGeoJson(GEO_POSITIONS) - dataJoinColumn = autoId - mapJoinColumn = autoId - joinIds = geoJson.indices.map(Int::toString) - dataFrame = DataFrame.Builder(data).put(DataFrame.Variable(dataJoinColumn), joinIds).build() + dataKeyColumn = autoId + mapKeyColumn = autoId + mapKeys = geoJson.indices.map(Int::toString) + dataFrame = DataFrame.Builder(data).put(DataFrame.Variable(dataKeyColumn), mapKeys).build() } // (data=gdf) @@ -93,10 +100,10 @@ class GeoConfig( require(layerOptions.has(DATA)) { "'data' parameter is mandatory with DATA_META" } geoJson = getGeoJson(DATA) - dataJoinColumn = autoId - mapJoinColumn = autoId - joinIds = geoJson.indices.map(Int::toString) - dataFrame = DataFrame.Builder(data).put(DataFrame.Variable(dataJoinColumn), joinIds).build() + dataKeyColumn = autoId + mapKeyColumn = autoId + mapKeys = geoJson.indices.map(Int::toString) + dataFrame = DataFrame.Builder(data).put(DataFrame.Variable(dataKeyColumn), mapKeys).build() } else -> error("GeoDataFrame not found in data or map") @@ -112,13 +119,13 @@ class GeoConfig( coordinatesCollector .append(geoJson) - .setIdColumn(columnName = mapJoinColumn, values = joinIds) + .setIdColumn(columnName = mapKeyColumn, values = mapKeys) dataAndCoordinates = rightJoin( left = dataFrame, - leftKey = dataJoinColumn, + leftKey = dataKeyColumn, right = createDataFrame(coordinatesCollector.buildCoordinatesMap()), - rightKey = mapJoinColumn + rightKey = mapKeyColumn ) val coordinatesAutoMapping = coordinatesCollector.mappings @@ -138,12 +145,12 @@ class GeoConfig( } } -const val POINT_X = "__gdf_x__" -const val POINT_Y = "__gdf_y__" -const val RECT_XMIN = "__gdf_xmin__" -const val RECT_YMIN = "__gdf_ymin__" -const val RECT_XMAX = "__gdf_xmax__" -const val RECT_YMAX = "__gdf_ymax__" +const val POINT_X = "__x__" +const val POINT_Y = "__y__" +const val RECT_XMIN = "__xmin__" +const val RECT_YMIN = "__ymin__" +const val RECT_XMAX = "__xmax__" +const val RECT_YMAX = "__ymax__" internal abstract class CoordinatesCollector( val mappings: Map, String> diff --git a/plot-config/src/jvmTest/kotlin/plot/config/GeoConfigTest.kt b/plot-config/src/jvmTest/kotlin/plot/config/GeoConfigTest.kt index 97f06b8d68c..4311ab9665c 100644 --- a/plot-config/src/jvmTest/kotlin/plot/config/GeoConfigTest.kt +++ b/plot-config/src/jvmTest/kotlin/plot/config/GeoConfigTest.kt @@ -13,6 +13,7 @@ import jetbrains.datalore.plot.config.PlotConfigClientSideUtil.createPlotAssembl import jetbrains.datalore.plot.parsePlotSpec import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.fail class GeoConfigTest { private val point = """{\"type\": \"Point\", \"coordinates\": [1.0, 2.0]}""" @@ -227,8 +228,8 @@ class GeoConfigTest { | "layers": [{ | "geom": "polygon", | "data": { - | "fig": ["Polygon", "MPolygon", "C"], - | "value": [42, 23, 66] + | "fig": ["Polygon", "MPolygon"], + | "value": [42, 23] | }, | "mapping": {"fill": "value"}, | "map": $gdf, @@ -260,4 +261,32 @@ class GeoConfigTest { ) ) } + + + @Test + fun `should show data key missing in map`() { + val spec = """ + |{ + | "kind": "plot", + | "layers": [{ + | "geom": "polygon", + | "data": { + | "fig": ["Polygon", "MPolygon", "Missing_Key"], + | "value": [42, 23, 66] + | }, + | "mapping": {"fill": "value"}, + | "map": $gdf, + | "map_data_meta": {"geodataframe": {"geometry": "coord"}}, + | "map_join": ["fig", "kind"] + | }] + |} + """.trimMargin() + + try { + createPlotAssembler(parsePlotSpec(spec)) + fail("'Missing_Key' is missing in map - should throw exception") + } catch (e: Exception) { + assertEquals("'Missing_Key' not found in map", e.localizedMessage) + } + } } diff --git a/plot-demo/src/commonMain/kotlin/jetbrains/datalore/plotDemo/model/plotConfig/GeoData.kt b/plot-demo/src/commonMain/kotlin/jetbrains/datalore/plotDemo/model/plotConfig/GeoData.kt index 1d3529dad5b..544a2c3628e 100644 --- a/plot-demo/src/commonMain/kotlin/jetbrains/datalore/plotDemo/model/plotConfig/GeoData.kt +++ b/plot-demo/src/commonMain/kotlin/jetbrains/datalore/plotDemo/model/plotConfig/GeoData.kt @@ -12,15 +12,14 @@ class GeoData : PlotConfigDemoBase() { fun plotSpecList(): List> { return listOf( - ttt() -// mapGeoDataFrame_MapJoin(), -// mapGeoDataFrame_NoMapJoin_MixedShapes("polygon"), -// mapGeoDataFrame_NoMapJoin_MixedShapes("point"), -// mapGeoDataFrame_NoMapJoin_MixedShapes("path"), -// mapGeoDataFrame_Empty(), -// mapGeoDict_MapJoin(), -// dataGeoDataFrame_Empty(), -// dataGeoDataFrame_NoMapJoin_GeomText() + mapGeoDataFrame_MapJoin(), + mapGeoDataFrame_NoMapJoin_MixedShapes("polygon"), + mapGeoDataFrame_NoMapJoin_MixedShapes("point"), + mapGeoDataFrame_NoMapJoin_MixedShapes("path"), + mapGeoDataFrame_Empty(), + mapGeoDict_MapJoin(), + dataGeoDataFrame_Empty(), + dataGeoDataFrame_NoMapJoin_GeomText() ) } @@ -32,40 +31,6 @@ class GeoData : PlotConfigDemoBase() { private const val multipolygon = """{\"type\": \"MultiPolygon\", \"coordinates\": [[[[11.0, 12.0], [13.0, 14.0], [15.0, 13.0], [11.0, 12.0]]]]}""" - - fun ttt(): MutableMap { - val point = """{\"type\": \"Point\", \"coordinates\": [-5.0, 17.0]}""" - val multiPoint = """{\"type\": \"MultiPoint\", \"coordinates\": [[3.0, 15.0], [6.0, 13.0]]}""" - val lineString = """{\"type\": \"LineString\", \"coordinates\": [[0.0, 0.0], [5.0, 5.0]]}""" - val multiLineString = """{\"type\": \"MultiLineString\", \"coordinates\": [[[10.0, 0.0], [15.0, 5.0]], [[10.0, 5.0], [15.0, 0.0]]]}""" - val polygon = """{\"type\": \"Polygon\", \"coordinates\": [[[1.0, 1.0], [1.0, 9.0], [9.0, 9.0], [9.0, 1.0], [1.0, 1.0]], [[2.0, 2.0], [3.0, 2.0], [3.0, 3.0], [2.0, 3.0], [2.0, 2.0]], [[4.0, 4.0], [6.0, 4.0], [6.0, 6.0], [4.0, 6.0], [4.0, 4.0]]]}""" - val multipolygon = """{\"type\": \"MultiPolygon\", \"coordinates\": [[[[11.0, 12.0], [13.0, 14.0], [15.0, 13.0], [11.0, 12.0]]]]}""" - val gdf = """ -{ - "kind": ["Point", "MPoint", "Line", "MLine", "Polygon", "MPolygon"], - "coord": ["$point", "$multiPoint", "$lineString", "$multiLineString", "$polygon", "$multipolygon"] -} -""" - val spec = """ - |{ - | "kind": "plot", - | "layers": [{ - | "geom": "polygon", - | "data": { - | "fig": ["Polygon", "MPolygon", "C"], - | "value": [42, 23, 66] - | }, - | "mapping": {"fill": "value"}, - | "map": $gdf, - | "map_data_meta": {"geodataframe": {"geometry": "coord"}}, - | "map_join": ["fig", "kind"] - | }] - |} - """.trimMargin() - return parsePlotSpec(spec) - - } - private fun mapGeoDict_MapJoin(): Map { val spec = """ { From c0245d018320ed4742c58927109aa09a8c07f4fa Mon Sep 17 00:00:00 2001 From: Ivan Kupriyanov Date: Tue, 16 Jun 2020 00:01:57 +0300 Subject: [PATCH 10/11] Fix NPE when map has extra entries, not matching data --- .../datalore/plot/config/GeoConfig.kt | 95 +++++++++++-------- .../kotlin/plot/config/GeoConfigTest.kt | 24 +++++ 2 files changed, 79 insertions(+), 40 deletions(-) diff --git a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt index 510f57e734a..0dfde4caef9 100644 --- a/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt +++ b/plot-config-portable/src/commonMain/kotlin/jetbrains/datalore/plot/config/GeoConfig.kt @@ -9,6 +9,7 @@ import jetbrains.datalore.base.spatial.* import jetbrains.datalore.base.typedGeometry.* import jetbrains.datalore.plot.base.Aes import jetbrains.datalore.plot.base.DataFrame +import jetbrains.datalore.plot.base.DataFrame.Variable import jetbrains.datalore.plot.base.GeomKind import jetbrains.datalore.plot.base.GeomKind.* import jetbrains.datalore.plot.base.data.DataFrameUtil.findVariableOrFail @@ -32,10 +33,10 @@ class GeoConfig( mappingOptions: Map<*, *> ) { val dataAndCoordinates: DataFrame - val mappings: Map, DataFrame.Variable> + val mappings: Map, Variable> init { - fun getGeoJson(gdfLocation: String): List { + fun getGeoJson(gdfLocation: String, keys: Collection): Map { val geoColumn: String val geoDataFrame: Map when(gdfLocation) { @@ -49,15 +50,14 @@ class GeoConfig( } else -> error("Unknown gdf location: $gdfLocation") } - return geoDataFrame.getList(geoColumn)?.map { it as String } ?: error("$geoColumn not found in $gdfLocation") + val geoJsons = geoDataFrame.getList(geoColumn)?.map { it as String } ?: error("$geoColumn not found in $gdfLocation") + return keys.zip(geoJsons).toMap() } - val mapKeys: List val dataKeyColumn: String - val mapKeyColumn: String - val geoJson: List + val geoKeyColumn: String + val geometries: Map val dataFrame: DataFrame - val autoId = "__id__" when { // (aes(color='cyl'), data=data, map=gdf) - how to join without `map_join`? @@ -69,41 +69,51 @@ class GeoConfig( // (data=data, map=gdf, map_join=('id', 'city')) with(layerOptions) { has(MAP_DATA_META, GDF, GEOMETRY) && has(MAP_JOIN) } -> { require(layerOptions.has(GEO_POSITIONS)) { "'map' parameter is mandatory with MAP_DATA_META" } - geoJson = getGeoJson(GEO_POSITIONS) + dataFrame = data val mapJoin = layerOptions.getList(MAP_JOIN) ?: error("require map_join parameter") + geoKeyColumn = mapJoin[1] as String dataKeyColumn = mapJoin[0] as String - mapKeyColumn = mapJoin[1] as String - mapKeys = layerOptions.getMap(GEO_POSITIONS)?.getList(mapKeyColumn)?.requireNoNulls() ?: error("'$mapKeyColumn' is not found in map") + val mapKeys = layerOptions + .getMap(GEO_POSITIONS) + ?.getList(geoKeyColumn) + ?.requireNoNulls() + ?.toSet() + ?: error("'$geoKeyColumn' is not found in map") + + val dataKeys = dataFrame.getOrFail(dataKeyColumn).requireNoNulls().toSet() // All data keys should be present in map - data.get(findVariableOrFail(data, dataKeyColumn)) - .firstOrNull { dataKey -> dataKey !in mapKeys } - ?.let { error("'$it' not found in map") } + (dataKeys - mapKeys).firstOrNull() ?.let { error("'$it' not found in map") } - dataFrame = data + geometries = getGeoJson(gdfLocation = GEO_POSITIONS, keys = mapKeys) + .filterKeys { it in dataKeys } // if not in data rightJoin adds null values and cause NPE when compute groups } // (map=gdf) - simple geometry with(layerOptions) { has(MAP_DATA_META, GDF, GEOMETRY) && !has(MAP_JOIN) } -> { require(layerOptions.has(GEO_POSITIONS)) { "'map' parameter is mandatory with MAP_DATA_META" } - geoJson = getGeoJson(GEO_POSITIONS) + dataKeyColumn = AUTO_ID + geoKeyColumn = AUTO_ID + geometries = run { + val indicies = layerOptions.getMap(GEO_POSITIONS)?.indicies?.map(Int::toString) ?: emptyList() + getGeoJson(gdfLocation = GEO_POSITIONS, keys = indicies) + } - dataKeyColumn = autoId - mapKeyColumn = autoId - mapKeys = geoJson.indices.map(Int::toString) - dataFrame = DataFrame.Builder(data).put(DataFrame.Variable(dataKeyColumn), mapKeys).build() + dataFrame = DataFrame.Builder(data).put(Variable(dataKeyColumn), geometries.keys.toList()).build() } // (data=gdf) with(layerOptions) { has(DATA_META, GDF, GEOMETRY) && !has(GEO_POSITIONS) && !has(MAP_JOIN) } -> { require(layerOptions.has(DATA)) { "'data' parameter is mandatory with DATA_META" } - geoJson = getGeoJson(DATA) + dataKeyColumn = AUTO_ID + geoKeyColumn = "__geo_id__" + geometries = run { + val indicies = layerOptions.getMap(DATA)?.indicies?.map(Int::toString) ?: emptyList() + getGeoJson(gdfLocation = DATA, keys = indicies) + } - dataKeyColumn = autoId - mapKeyColumn = autoId - mapKeys = geoJson.indices.map(Int::toString) - dataFrame = DataFrame.Builder(data).put(DataFrame.Variable(dataKeyColumn), mapKeys).build() + dataFrame = DataFrame.Builder(data).put(Variable(dataKeyColumn), geometries.keys.toList()).build() } else -> error("GeoDataFrame not found in data or map") @@ -117,15 +127,17 @@ class GeoConfig( else -> error("Unsupported geom: $geomKind") } - coordinatesCollector - .append(geoJson) - .setIdColumn(columnName = mapKeyColumn, values = mapKeys) + val geoFrame = coordinatesCollector + .append(geometries) + .setKeyColumn(geoKeyColumn) + .buildCoordinatesMap() + .let(::createDataFrame) dataAndCoordinates = rightJoin( left = dataFrame, leftKey = dataKeyColumn, - right = createDataFrame(coordinatesCollector.buildCoordinatesMap()), - rightKey = mapKeyColumn + right = geoFrame, + rightKey = geoKeyColumn ) val coordinatesAutoMapping = coordinatesCollector.mappings @@ -145,6 +157,7 @@ class GeoConfig( } } +const val AUTO_ID = "__id__" const val POINT_X = "__x__" const val POINT_Y = "__y__" const val RECT_XMIN = "__xmin__" @@ -155,18 +168,19 @@ const val RECT_YMAX = "__ymax__" internal abstract class CoordinatesCollector( val mappings: Map, String> ) { - private lateinit var idColumnName: String - private lateinit var ids: List + private lateinit var keyColumnName: String + private val groupKeys = mutableListOf() private val groupLengths = mutableListOf() protected val coordinates: Map> = mappings.values.associateBy({ it }) { mutableListOf() } protected abstract val geoJsonConsumer: SimpleFeature.Consumer protected abstract val supportedFeatures: List - fun append(geoJsons: List): CoordinatesCollector { - geoJsons.forEach { + fun append(rows: Map): CoordinatesCollector { + rows.forEach { (key, geoJson) -> val oldRowCount = coordinates.rowCount - GeoJson.parse(it, geoJsonConsumer) + GeoJson.parse(geoJson, geoJsonConsumer) groupLengths += coordinates.rowCount - oldRowCount + groupKeys += key } if (coordinates.rowCount == 0) { @@ -176,21 +190,20 @@ internal abstract class CoordinatesCollector( return this } - fun setIdColumn(columnName: String, values: List): CoordinatesCollector { - idColumnName = columnName - ids = values + fun setKeyColumn(columnName: String): CoordinatesCollector { + keyColumnName = columnName return this } fun buildCoordinatesMap(): Map> { - require(groupLengths.size == ids.size) { "Groups and ids should have same size" } + require(groupLengths.size == groupKeys.size) { "Groups and ids should have same size" } // (['a', 'b'], [2, 3]) => ['a', 'a', 'b', 'b', 'b'] fun copies(values: Collection, count: Collection) = values.asSequence().zip(count.asSequence()) .fold(mutableListOf()) { acc, (value, count) -> repeat(count) { acc += value }; acc } - return coordinates + (idColumnName to copies(ids, groupLengths)) + return coordinates + (keyColumnName to copies(groupKeys, groupLengths)) } internal fun defaultConsumer(config: SimpleFeature.Consumer.() -> Unit) = @@ -282,4 +295,6 @@ internal abstract class CoordinatesCollector( } -fun Map<*, *>.dataJoinVariable() = getList(MAP_JOIN)?.get(0) as? String \ No newline at end of file +fun Map<*, *>.dataJoinVariable() = getList(MAP_JOIN)?.get(0) as? String +private fun DataFrame.getOrFail(varName: String) = this.get(findVariableOrFail(this, varName)) +private val Map.indicies get() = (values.firstOrNull() as? List<*>)?.indices diff --git a/plot-config/src/jvmTest/kotlin/plot/config/GeoConfigTest.kt b/plot-config/src/jvmTest/kotlin/plot/config/GeoConfigTest.kt index 4311ab9665c..31db1774d27 100644 --- a/plot-config/src/jvmTest/kotlin/plot/config/GeoConfigTest.kt +++ b/plot-config/src/jvmTest/kotlin/plot/config/GeoConfigTest.kt @@ -289,4 +289,28 @@ class GeoConfigTest { assertEquals("'Missing_Key' not found in map", e.localizedMessage) } } + + @Test + fun `should not fail if map has extra entries`() { + val spec = """ + |{ + | "kind": "plot", + | "layers": [{ + | "geom": "polygon", + | "data": { + | "fig": ["Polygon"], + | "value": [42] + | }, + | "mapping": {"fill": "value"}, + | "map": $gdf, + | "map_data_meta": {"geodataframe": {"geometry": "coord"}}, + | "map_join": ["fig", "kind"] + | }] + |} + """.trimMargin() + + val plotAssembler = createPlotAssembler(parsePlotSpec(spec)) + val aesthetics = createLayerRendererData(plotAssembler.layersByTile.single().single(), emptyMap(), emptyMap()).aesthetics + assertEquals((0..14).map { 0 }, aesthetics.dataPoints().map(DataPointAesthetics::group)) + } } From 25bac91f608e7185d7dc7d2dea72f65d4ac8aab7 Mon Sep 17 00:00:00 2001 From: Ivan Kupriyanov Date: Tue, 16 Jun 2020 00:17:05 +0300 Subject: [PATCH 11/11] Update pie/bar demo --- .../livemap/pie_bar_on_livemap.ipynb | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/docs/examples/jupyter-notebooks-dev/livemap/pie_bar_on_livemap.ipynb b/docs/examples/jupyter-notebooks-dev/livemap/pie_bar_on_livemap.ipynb index c1bb86113f6..a68428ae423 100644 --- a/docs/examples/jupyter-notebooks-dev/livemap/pie_bar_on_livemap.ipynb +++ b/docs/examples/jupyter-notebooks-dev/livemap/pie_bar_on_livemap.ipynb @@ -6,7 +6,8 @@ "metadata": {}, "outputs": [], "source": [ - "from lets_plot import *" + "from lets_plot import *\n", + "import lets_plot.mapping as pm" ] }, { @@ -34,11 +35,12 @@ "metadata": {}, "outputs": [], "source": [ - "data = {'val': [3000, 0, -2000, 1500, 1000, 2500], \\\n", - " 'order': [1, 2, 3, 1, 2, 3], \\\n", - " 'lat': [31, 31, 31, 42, 42, 42],\\\n", - " 'lng': [-100, -100, -100, -93, -93, -93],\\\n", - " 'lnglat': [\"-100,31\",\"-100,31\",\"-100,31\",\"-93,42\",\"-93,42\",\"-93,42\"]}" + "data = {\n", + " 'val': [1500, 1000, 1000, 500, 600, 700, 500, 600, 700, 700],\n", + " 'order': [1, 1, 2, 2, 3, 3, 4, 4 , 5, 5],\n", + " 'lon': [-82, -100, -82, -100, -82, -100, -82, -100, -82, -100],\n", + " 'lat': [28, 31, 28, 31, 28, 31, 28, 31, 28, 31]\n", + "}" ] }, { @@ -49,7 +51,7 @@ }, "outputs": [], "source": [ - "ggplot(data) + geom_livemap(aes(x='lng', y='lat', sym_y='val', fill = 'val'), symbol='bar', size = 30)" + "ggplot(data) + geom_livemap(aes(x='lon', y='lat', sym_y='val', fill='val'), symbol='bar', size = 30)" ] }, { @@ -59,8 +61,8 @@ "outputs": [], "source": [ "ggplot(data) \\\n", - " + geom_livemap(aes(x='lng', y='lat', sym_y='val', fill='val'), symbol='pie', size=60) \\\n", - " + scale_fill_hue(h = [0.,120.], name = 'order')" + " + geom_livemap(aes(x='lon', y='lat', sym_x='order', sym_y= 'val', fill='val'), symbol='bar', size=50) \\\n", + " + scale_fill_gradient(low='yellow', high='red')" ] }, { @@ -70,9 +72,27 @@ "outputs": [], "source": [ "ggplot(data) \\\n", - " + geom_livemap(aes(x='lng', y='lat', sym_x='order', sym_y= 'val', fill='val'), symbol='bar', size=50) \\\n", - " + scale_fill_hue(h = [0.,120.], name = 'order')" + " + geom_livemap(aes(x='lon', y='lat', sym_x='order', sym_y='val', fill='val'), symbol='pie', size=32, color='orange') \\\n", + " + scale_fill_gradient(low='yellow', high='red') + ggsize(600,200)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ggplot(data) \\\n", + " + geom_livemap(aes(x='lon', y='lat', sym_y='val', fill='val'), symbol='pie', size=32, color='orange') \\\n", + " + scale_fill_gradient(low='yellow', high='red') + ggsize(600,200)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {