WIP: answers in protocol

python-package/lets_plot/export/simple.py

@@ -5,7 +5,6 @@
 from os.path import abspath
 from typing import Union
 
-from .. import _kbridge as kbr
 from .._global_settings import is_production
 from .._version import __version__
 from ..plot.core import PlotSpec
@@ -32,6 +31,8 @@ def export_svg(plot: Union[PlotSpec, GGBunch], filename: str) -> str:
  if not (isinstance(plot, PlotSpec) or isinstance(plot, GGBunch)):
  raise ValueError("PlotSpec or GGBunch expected but was: {}".format(type(plot)))
 
+ from .. import _kbridge as kbr
+
  svg = kbr._generate_svg(plot.as_dict())
  with io.open(filename, mode="w", encoding="utf-8") as f:
  f.write(svg)
@@ -62,6 +63,9 @@ def export_html(plot: Union[PlotSpec, GGBunch], filename: str, iframe: bool = Fa
  raise ValueError("PlotSpec or GGBunch expected but was: {}".format(type(plot)))
 
  version = __version__ if is_production() else "latest"
+
+ from .. import _kbridge as kbr
+
  html_page = kbr._generate_static_html_page(plot.as_dict(), version, iframe)
  with io.open(filename, mode="w", encoding="utf-8") as f:
  f.write(html_page)

python-package/lets_plot/geo_data/gis/json_response.py

@@ -7,13 +7,14 @@
 from .geometry import Ring
 from .response import Multipolygon, GeoPoint, GeoRect, Boundary, Polygon
 from .response import Response, ResponseBuilder, SuccessResponse, AmbiguousResponse
-from .response import Status, LevelKind, GeocodedFeature, AmbiguousFeature, Namesake, NamesakeParent, FeatureBuilder
+from .response import Status, LevelKind, Answer, GeocodedFeature, AmbiguousFeature, Namesake, NamesakeParent, FeatureBuilder
 
 
 class ResponseField(Enum):
  status = 'status'
  message = 'message'
  data = 'data'
+ answers = 'answers'
  features = 'features'
  geocoded_data = 'good_features'
  incorrect_data = 'bad_features'
@@ -66,7 +67,7 @@ def parse(response_json: Dict) -> Response:
  .visit_enum_existing(ResponseField.level, LevelKind, response.set_level)
 
  if response.status == Status.success:
- data_dict.visit(ResponseField.features, partial(ResponseParser._parse_geocoded_features, response=response))
+ data_dict.visit(ResponseField.answers, partial(ResponseParser._parse_answers, response=response))
  elif response.status == Status.ambiguous:
  data_dict.visit(ResponseField.features, partial(ResponseParser._parse_ambiguous_features, response=response))
  else:
@@ -75,23 +76,28 @@ def parse(response_json: Dict) -> Response:
  return response.build()
 
  @staticmethod
- def _parse_geocoded_features(features_json: List[Dict], response: ResponseBuilder):
- geocoded_features: List[GeocodedFeature] = []
- for feature_json in features_json:
- feature = FeatureBuilder()
- FluentDict(feature_json) \
- .visit_str(ResponseField.query, feature.set_query) \
- .visit_str(ResponseField.geo_object_id, feature.set_id) \
- .visit_str(ResponseField.name, feature.set_name) \
- .visit_str_list_optional(ResponseField.highlights, feature.set_highlights) \
- .visit_str_existing(ResponseField.boundary, lambda json: feature.set_boundary(GeoJson().parse_geometry(json))) \
- .visit_object_optional(ResponseField.centroid, lambda json: feature.set_centroid(ResponseParser._parse_point(json))) \
- .visit_object_optional(ResponseField.limit, lambda json: feature.set_limit(ResponseParser._parse_rect(json))) \
- .visit_object_optional(ResponseField.position, lambda json: feature.set_position(ResponseParser._parse_rect(json)))
-
- geocoded_features.append(feature.build_geocoded())
-
- response.set_geocoded_features(geocoded_features)
+ def _parse_answers(answers_json: List[Dict], response: ResponseBuilder):
+ answers: List[Answer] = []
+ for answer_json in answers_json:
+ query = answer_json.get(ResponseField.query.value)
+ features_json = answer_json.get(ResponseField.features.value, [])
+ geocoded_features: List[GeocodedFeature] = []
+ for feature_json in features_json:
+ feature = FeatureBuilder().set_query(query)
+
+ FluentDict(feature_json) \
+ .visit_str(ResponseField.geo_object_id, feature.set_id) \
+ .visit_str(ResponseField.name, feature.set_name) \
+ .visit_str_list_optional(ResponseField.highlights, feature.set_highlights) \
+ .visit_str_existing(ResponseField.boundary, lambda json: feature.set_boundary(GeoJson().parse_geometry(json))) \
+ .visit_object_optional(ResponseField.centroid, lambda json: feature.set_centroid(ResponseParser._parse_point(json))) \
+ .visit_object_optional(ResponseField.limit, lambda json: feature.set_limit(ResponseParser._parse_rect(json))) \
+ .visit_object_optional(ResponseField.position, lambda json: feature.set_position(ResponseParser._parse_rect(json)))
+
+ geocoded_features.append(feature.build_geocoded())
+ answers.append(Answer(query, geocoded_features))
+
+ response.set_answers(answers)
 
  @staticmethod
  def _parse_ambiguous_features(features_json: List[Dict], response: ResponseBuilder):
@@ -146,7 +152,7 @@ def format(response: Response) -> Dict:
  .put(ResponseField.message, response.message) \
  .put(ResponseField.data, FluentDict()
  .put(ResponseField.level, response.level.value)
- .put(ResponseField.features, list(map(ResponseFormatter._format_geocoded_feature, response.features)))) \
+ .put(ResponseField.answers, list(map(ResponseFormatter._format_answer, response.answers)))) \
  .to_dict()
  elif isinstance(response, AmbiguousResponse):
  return FluentDict() \
@@ -158,17 +164,27 @@ def format(response: Response) -> Dict:
  .to_dict()
 
  @staticmethod
- def _format_geocoded_feature(feature: GeocodedFeature) -> Dict:
+ def _format_answer(answer: Answer) -> Dict:
+ features = []
+ for feature in answer.features:
+ features.append(
+ FluentDict() \
+ .put(ResponseField.query, feature.query) \
+ .put(ResponseField.geo_object_id, feature.id) \
+ .put(ResponseField.name, feature.name) \
+ .put(ResponseField.boundary, ResponseFormatter._format_boundary(feature.boundary)) \
+ .put(ResponseField.centroid, ResponseFormatter._format_centroid(feature.centroid)) \
+ .put(ResponseField.limit, ResponseFormatter._format_rect(feature.limit)) \
+ .put(ResponseField.position, ResponseFormatter._format_rect(feature.position)) \
+ .to_dict()
+ )
+
  return FluentDict() \
- .put(ResponseField.query, feature.query) \
- .put(ResponseField.geo_object_id, feature.id) \
- .put(ResponseField.name, feature.name) \
- .put(ResponseField.boundary, ResponseFormatter._format_boundary(feature.boundary)) \
- .put(ResponseField.centroid, ResponseFormatter._format_centroid(feature.centroid)) \
- .put(ResponseField.limit, ResponseFormatter._format_rect(feature.limit)) \
- .put(ResponseField.position, ResponseFormatter._format_rect(feature.position)) \
+ .put(ResponseField.query, answer.query) \
+ .put(ResponseField.features, features) \
  .to_dict()
 
+
  @staticmethod
  def _format_centroid(point: Optional[GeoPoint]) -> Optional[Dict]:
  if point is None:

python-package/lets_plot/geo_data/gis/response.py

@@ -77,16 +77,30 @@ def __init__(self, message: str):
  assert_type(message, str)
  self.message: str = message
 
+class Answer:
+ def __init__(self, query: str, features: List[GeocodedFeature]):
+ assert_type(query, str)
+ assert_list_type(features, GeocodedFeature)
+
+ self.query: str = query
+ self.features: List[GeocodedFeature] = features
+
 
 class SuccessResponse(Response):
- def __init__(self, message: str, level: LevelKind, features: List[GeocodedFeature]):
+ def __init__(self, message: str, level: LevelKind, answers: List[Answer]):
  super().__init__(message)
 
  assert_type(message, str)
  assert_optional_type(level, LevelKind)
- assert_list_type(features, GeocodedFeature)
+ assert_list_type(answers, Answer)
 
  self.level: LevelKind = level
+ self.answers: List[Answer] = answers
+
+ features = []
+ for answer in answers:
+ features.extend(answer.features)
+
  self.features: List[GeocodedFeature] = features
 
 
@@ -187,7 +201,7 @@ def __init__(self):
  self.status: Status = None
  self.level: LevelKind = None
  self.message: str = None
- self.geocoded_features: List[GeocodedFeature] = None
+ self.answers: List[Answer] = None
  self.ambiguous_features: List[AmbiguousFeature] = None
  self.data: Dict = None
 
@@ -211,16 +225,25 @@ def set_ambiguous_features(self, v: List[AmbiguousFeature]) -> 'ResponseBuilder'
  self.ambiguous_features = v
  return self
 
- def set_geocoded_features(self, v: List[GeocodedFeature]) -> 'ResponseBuilder':
+ def set_answers(self, v: List[Answer]) -> 'ResponseBuilder':
+ assert_list_type(v, Answer)
+ self.answers = v
+ return self
+
+ def set_geocoded_features(self, v: List[GeocodedFeature]):
+ '''
+ Exactly matching non-exploding features, i.e. one feature per answer
+ '''
  assert_list_type(v, GeocodedFeature)
- self.geocoded_features = v
+ self.answers = [Answer(f.query, [f]) for f in v]
  return self
 
+
  def build(self) -> Response:
  if self.status == Status.error:
  return ErrorResponse(self.message)
  elif self.status == Status.success:
- return SuccessResponse(self.message, self.level, self.geocoded_features)
+ return SuccessResponse(self.message, self.level, self.answers)
  elif self.status == Status.ambiguous:
  return AmbiguousResponse(self.message, self.level, self.ambiguous_features)
  else:

python-package/lets_plot/geo_data/regions.py

@@ -49,6 +49,21 @@ def select_not_empty_name(feature: GeocodedFeature) -> str:
  return feature.name if feature.query is None or feature.query == '' else feature.query
 
 
+def arrange_queries(features: List[GeocodedFeature], queries: List[RegionQuery]) -> List[Optional[RegionQuery]]:
+ res = []
+ for feature in features:
+ request: Optional[RegionQuery] = None
+
+ for query in queries:
+ if query.request == feature.query:
+ request = query
+ break
+
+ res.append(request)
+
+ return res
+
+
 class PlacesDataFrameBuilder:
  def __init__(self):
  self._request: List[str] = []
@@ -57,16 +72,16 @@ def __init__(self):
  self._state: List[Optional[str]] = []
  self._country: List[Optional[str]] = []
 
- def append_row(self, request: str, found_name: str, queries: Optional[List[RegionQuery]], parent_row: int):
+ def append_row(self, request: str, found_name: str, queries: List[Optional[RegionQuery]], parent_row: int):
  self._request.append(request)
  self._found_name.append(found_name)
 
- if queries is None or len(queries) == 0:
- self._county.append(None)
- self._state.append(None)
- self._country.append(None)
+ query: Optional[RegionQuery] = queries[parent_row]
+ if query is None:
+ self._county.append(MapRegion.name_or_none(None))
+ self._state.append(MapRegion.name_or_none(None))
+ self._country.append(MapRegion.name_or_none(None))
  else:
- query: RegionQuery = queries[parent_row]
  self._county.append(MapRegion.name_or_none(query.county))
  self._state.append(MapRegion.name_or_none(query.state))
  self._country.append(MapRegion.name_or_none(query.country))
@@ -252,7 +267,7 @@ def to_data_frame(self) -> DataFrame:
  data[DF_ID] = [feature.id for feature in self._geocoded_features]
 
  # for us-48 queries doesnt' count
- queries = self._queries if len(self._queries) == len(self._geocoded_features) else None
+ queries: List[Optional[RegionQuery]] = arrange_queries(self._geocoded_features, self._queries)
 
  for i in range(len(self._geocoded_features)):
  feature = self._geocoded_features[i]
@@ -272,7 +287,12 @@ def _execute(self, request_builder: RequestBuilder, df_converter):
  if not isinstance(response, SuccessResponse):
  _raise_exception(response)
 
- self._join_payload(response.features)
+ features = []
+
+ for a in response.answers:
+ features.extend(a.features)
+
+ self._join_payload(features)
 
  return df_converter.to_data_frame(self._geocoded_features, self._queries)
 

python-package/lets_plot/geo_data/to_geo_data_frame.py

@@ -5,7 +5,7 @@
 from pandas import DataFrame
 from shapely.geometry import box
 
-from lets_plot.geo_data import PlacesDataFrameBuilder, select_not_empty_name, DF_REQUEST, DF_FOUND_NAME, abstractmethod
+from lets_plot.geo_data import PlacesDataFrameBuilder, arrange_queries, select_not_empty_name, DF_REQUEST, DF_FOUND_NAME, abstractmethod
 from lets_plot.geo_data.gis.response import GeocodedFeature, GeoRect, Boundary, Multipolygon, Polygon, GeoPoint
 from lets_plot.geo_data.gis.request import RegionQuery
 
@@ -42,7 +42,7 @@ def __init__(self):
 
  def to_data_frame(self, features: List[GeocodedFeature], queries: List[RegionQuery] = []) -> DataFrame:
  places = PlacesDataFrameBuilder()
- queries = queries if len(queries) == len(features) else None
+ queries: List[Optional[RegionQuery]] = arrange_queries(features, queries)
 
  for i in range(len(features)):
  feature = features[i]
@@ -81,7 +81,7 @@ def __init__(self):
 
  def to_data_frame(self, features: List[GeocodedFeature], queries: List[RegionQuery] = []) -> DataFrame:
  places = PlacesDataFrameBuilder()
- queries = queries if len(queries) == len(features) else None
+ queries: List[Optional[RegionQuery]] = arrange_queries(features, queries)
 
  for i in range(len(features)):
  feature = features[i]
@@ -99,7 +99,7 @@ def __init__(self):
 
  def to_data_frame(self, features: List[GeocodedFeature], queries: List[RegionQuery] = []) -> DataFrame:
  places = PlacesDataFrameBuilder()
- queries = queries if len(queries) == len(features) else None
+ queries: List[Optional[RegionQuery]] = arrange_queries(features, queries)
 
  geometry = []
  for i in range(len(features)):

python-package/test/geo_data/geo_data.py

@@ -74,7 +74,7 @@ def assert_str(column, expected):
 
  if isinstance(expected, list):
  actual = df[column][index:index + len(expected)].tolist()
- assert actual == expected
+ assert actual == expected, str(expected) + '!=' + str(actual)
  return
 
  raise ValueError('Not support type of expected: {}'.format(str(type(expected))))

python-package/test/geo_data/gis/test_response_parser.py

@@ -52,9 +52,9 @@ def test_valid_success_response():
 
  assert isinstance(response, SuccessResponse)
  assert 'OK' == response.message
- assert 2 == len(response.features)
- assert_geocoded(foo, response.features[0])
- assert_geocoded(foofoo, response.features[1])
+ assert 2 == len(response.answers)
+ assert_geocoded(foo, response.answers[0].features[0])
+ assert_geocoded(foofoo, response.answers[1].features[0])
 
 
 def test_ambiguous_response():