Fix more tests

docs/examples/jupyter-notebooks-dev/map_US_household_income_new.ipynb

@@ -1942,7 +1942,7 @@
  "name": "python",
  "nbconvert_exporter": "python",
  "pygments_lexer": "ipython3",
- "version": "3.7.8"
+ "version": "3.7.6"
  }
  },
  "nbformat": 4,

python-package/lets_plot/_global_settings.py

@@ -43,7 +43,7 @@ def _get_env_val(actual_name: str) -> Any:
  'dev_offline': _init_value('dev_offline', True), # default: embed js into the notebook
  'dev_js_base_url': "https://0.0.0.0:8080",
  'dev_js_name': '', # default: lets-plot-<version>.js
- 'dev_geocoding_url': _init_value('dev_geocoding_url', 'https:https://geo.datalore.jetbrains.com')
+ 'dev_geocoding_url': _init_value('dev_geocoding_url', 'http:https://localhost:3012')
 }
 
 

python-package/lets_plot/geo_data/gis/geocoding_service.py

@@ -97,5 +97,5 @@ def _execute(self, request: Request) -> Response:
  except Exception as e:
  return ResponseBuilder() \
  .set_status(Status.error) \
- .set_message('Geocoding service exception: {}'.format(str(e))) \
+ .set_message('Internal service exception: {}'.format(str(e))) \
  .build()

python-package/lets_plot/geo_data/gis/json_request.py

@@ -3,7 +3,7 @@
 
 from .fluent_dict import FluentDict, FluentList
 from .geometry import GeoPoint
-from .request import RegionQuery, MapRegion, PayloadKind, RegionQueryBuilder, IgnoringStrategyKind, MapRegionBuilder
+from .request import RegionQuery, MapRegion, MapRegionKind, PayloadKind, RegionQueryBuilder, IgnoringStrategyKind, MapRegionBuilder
 from .request import Request, GeocodingRequest, ExplicitRequest, RequestBuilder, RequestKind, ReverseGeocodingRequest
 from .response import LevelKind, GeoRect
 
@@ -118,6 +118,14 @@ def _format_map_region(parent: Optional[MapRegion]) -> Optional[Dict]:
  if parent is None:
  return None
 
+ # special case - place is just a geocoded object with id and extra information, used by client
+ # server doesn't need this extra information
+ if parent.kind.value == 'place':
+ return FluentDict() \
+ .put(Field.map_region_kind, MapRegionKind.id.value) \
+ .put(Field.map_region_values, parent.values) \
+ .to_dict()
+
  return FluentDict() \
  .put(Field.map_region_kind, parent.kind.value) \
  .put(Field.map_region_values, parent.values) \

python-package/lets_plot/geo_data/gis/request.py

@@ -57,12 +57,17 @@ class MapRegion:
  with_name - single name, not yet geocoded.
  '''
  @staticmethod
- def request_or_none(place: Optional['MapRegion']):
+ def name_or_none(place: Optional['MapRegion']):
  if place is None:
  return None
 
- assert place.kind == MapRegionKind.place, 'Only palce MapRegion contains request'
- return place._request
+ if place.kind == MapRegionKind.place:
+ return place.request()
+
+ if place.kind == MapRegionKind.name:
+ return place.name()
+
+ raise ValueError('MapRegion with kind \'{}\' doesn\'t have a name'.format(place.kind))
 
 
  @staticmethod
@@ -94,10 +99,16 @@ def __init__(self, kind: MapRegionKind, values: List[str], request: Optional[str
  self._level_kind: Optional[LevelKind] = level_kind
  self._hash = hash((self.values, self.kind))
 
- def request(self) -> Optional[str]:
- assert self.kind == MapRegionKind.place, 'Invalid MapRegion kind: only place contains request'
+ def request(self) -> str:
+ assert self.kind == MapRegionKind.place, 'Invalid MapRegion kind. Expected \'place\', but was ' + str(self.kind)
+ assert_type(self._request, str)
  return self._request
 
+ def name(self) -> str:
+ assert self.kind == MapRegionKind.name, 'Invalid MapRegion kind. Expected \'name\', but was ' + str(self.kind)
+ assert_type(self.values[0], str)
+ return self.values[0]
+
  def level_kind(self) -> Optional[LevelKind]:
  assert self.kind == MapRegionKind.place, 'Invalid MapRegion kind: only place contains level_kind'
  return self._level_kind

python-package/lets_plot/geo_data/regions.py

@@ -67,9 +67,9 @@ def append_row(self, request: str, found_name: str, queries: Optional[List[Regio
  self._country.append(None)
  else:
  query: RegionQuery = queries[parent_row]
- self._county.append(MapRegion.request_or_none(query.county))
- self._state.append(MapRegion.request_or_none(query.state))
- self._country.append(MapRegion.request_or_none(query.country))
+ self._county.append(MapRegion.name_or_none(query.county))
+ self._state.append(MapRegion.name_or_none(query.state))
+ self._country.append(MapRegion.name_or_none(query.country))
 
 
  def build_dict(self):

python-package/lets_plot/geo_data/regions_builder.py

@@ -75,9 +75,23 @@ def _create_queries(request: request_types, scope: scope_types, ambiguity_resovl
 
  return [RegionQuery(r, scopes, ambiguity_resovler) for r in requests]
  else:
- countries = _ensure_is_list(countries)
- states = _ensure_is_list(states)
- counties = _ensure_is_list(counties)
+ def ensure_is_parent_list(obj):
+ if obj is None:
+ return None
+
+ if isinstance(obj, str):
+ return [obj]
+ if isinstance(obj, Regions):
+ return obj.as_list()
+
+ if isinstance(obj, list):
+ return obj
+
+ return [obj]
+
+ countries = ensure_is_parent_list(countries)
+ states = ensure_is_parent_list(states)
+ counties = ensure_is_parent_list(counties)
 
  assert countries is None or len(countries) == len(requests)
  assert states is None or len(states) == len(requests)
@@ -232,7 +246,7 @@ def build(self) -> Regions:
  .build()
 
  # Too many queries - can fail with timeout. Chunk queries.
- if len(self._get_queries()) > 100 and self._chunk_size is None:
+ if len(self._get_queries()) > 100_000 and self._chunk_size is None:
  self.chunk_request(self._on_progress, 40)
 
  response: Response = GeocodingService().do_request(request, self._chunk_size, self._on_progress)

python-package/test/geo_data/geo_data.py

@@ -4,14 +4,15 @@
 import json
 from typing import List, Union
 
-from lets_plot.geo_data import DF_REQUEST, DF_FOUND_NAME
+from lets_plot.geo_data import DF_ID, DF_REQUEST, DF_FOUND_NAME, DF_PARENT_COUNTY, DF_PARENT_STATE, DF_PARENT_COUNTRY
 from lets_plot.geo_data.gis.geometry import Ring, Polygon, Multipolygon
 from lets_plot.geo_data.gis.json_response import ResponseField, GeometryKind
 from lets_plot.geo_data.gis.response import GeocodedFeature, FeatureBuilder, LevelKind, Status, GeoRect, GeoPoint, \
  Response, SuccessResponse, AmbiguousResponse, ErrorResponse, ResponseBuilder
 from lets_plot.geo_data.regions import Regions
 
 from pandas import DataFrame
+from shapely.geometry import Point
 
 GEOJSON_TYPE = ResponseField.boundary_type.value
 GEOJSON_COORDINATES = ResponseField.boundary_coordinates.value
@@ -44,6 +45,57 @@
 GEO_RECT_MAX_LON: float = 9
 GEO_RECT_MAX_LAT: float = 7
 
+def run_intergration_tests() -> bool:
+ import os
+ if 'RUN_GEOCODING_INTEGRATION_TEST' in os.environ.keys():
+ return os.environ.get('RUN_GEOCODING_INTEGRATION_TEST').lower() == 'true'
+ return False
+
+
+def use_local_server():
+ old = os.environ.copy()
+ os.environ.update({'GEOSERVER_URL': 'https://localhost:3012', **old})
+
+NO_COLUMN = '<no column>'
+IGNORED = '__value_ignored__'
+
+def assert_row(df, index: int = 0, request: Union[str, List] = IGNORED, found_name: Union[str, List] = IGNORED, id: Union[str, List] = IGNORED, county: Union[str, List] = IGNORED, state: Union[str, List] = IGNORED, country: Union[str, List] = IGNORED, lon=None, lat=None) -> dict:
+ def assert_str(column, expected):
+ if expected == IGNORED:
+ return
+
+ if expected == NO_COLUMN:
+ assert column not in df.columns.tolist()
+ return
+
+ if isinstance(expected, str):
+ assert expected == df[column][index]
+ return
+
+ if isinstance(expected, list):
+ actual = df[column][index:index + len(expected)].tolist()
+ assert actual == expected
+ return
+
+ raise ValueError('Not support type of expected: {}'.format(str(type(expected))))
+
+
+ assert_str(DF_ID, id)
+ assert_str(DF_REQUEST, request)
+ assert_str(DF_FOUND_NAME, found_name)
+ assert_str(DF_PARENT_COUNTY, county)
+ assert_str(DF_PARENT_STATE, state)
+ assert_str(DF_PARENT_COUNTRY, country)
+ if lon is not None:
+ assert Point(df.geometry[index]).x == lon
+
+ if lat is not None:
+ assert Point(df.geometry[index]).y == lat
+
+
+def assert_found_names(df: DataFrame, names: List[str]):
+ assert names == df[DF_FOUND_NAME].tolist()
+
 
 def make_geocode_region(request: str, name: str, geo_object_id: str, highlights: List[str], level_kind: LevelKind = LevelKind.city) -> Regions:
  return Regions(level_kind, [make_region(request, name, geo_object_id, highlights)])

python-package/test/geo_data/test_integration_new_api.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2020. JetBrains s.r.o.
+# Use of this source code is governed by the MIT license that can be found in the LICENSE file.
+
+import lets_plot.geo_data as geodata
+from lets_plot.geo_data import DF_FOUND_NAME, DF_ID, DF_REQUEST, DF_PARENT_COUNTRY, DF_PARENT_STATE, DF_PARENT_COUNTY
+from .geo_data import assert_row, NO_COLUMN
+
+def test_all_columns_order():
+ boston = geodata.regions_builder2(level='city', names='boston', counties='suffolk', states='massachusetts', countries='usa').build()
+ assert boston.to_data_frame().columns.tolist() == [DF_ID, DF_REQUEST, DF_FOUND_NAME, DF_PARENT_COUNTY, DF_PARENT_STATE, DF_PARENT_COUNTRY]
+ assert boston.limits().columns.tolist() == [DF_REQUEST, DF_FOUND_NAME, DF_PARENT_COUNTY, DF_PARENT_STATE, DF_PARENT_COUNTRY, 'geometry']
+ assert boston.centroids().columns.tolist() == [DF_REQUEST, DF_FOUND_NAME, DF_PARENT_COUNTY, DF_PARENT_STATE, DF_PARENT_COUNTRY, 'geometry']
+ assert boston.boundaries().columns.tolist() == [DF_REQUEST, DF_FOUND_NAME, DF_PARENT_COUNTY, DF_PARENT_STATE, DF_PARENT_COUNTRY, 'geometry']
+
+def test_do_not_add_unsued_parents_columns():
+ moscow = geodata.regions_builder2(level='city', names='moscow', countries='rusria').build()
+ assert moscow.to_data_frame().columns.tolist() == [DF_ID, DF_REQUEST, DF_FOUND_NAME, DF_PARENT_COUNTRY]
+ assert moscow.limits().columns.tolist() == [DF_REQUEST, DF_FOUND_NAME, DF_PARENT_COUNTRY, 'geometry']
+ assert moscow.centroids().columns.tolist() == [DF_REQUEST, DF_FOUND_NAME, DF_PARENT_COUNTRY, 'geometry']
+ assert moscow.boundaries().columns.tolist() == [DF_REQUEST, DF_FOUND_NAME, DF_PARENT_COUNTRY, 'geometry']
+
+
+#@pytest.mark.skipif(TURN_OFF_INTERACTION_TEST, reason='Need proper server ip')
+def test_parents_in_regions_object_and_geo_data_frame():
+ boston = geodata.regions_builder2(level='city', names='boston', counties='suffolk', states='massachusetts', countries='usa').build()
+
+ assert_row(boston.to_data_frame(), request='boston', county='suffolk', state='massachusetts', country='usa')
+ assert_row(boston.limits(), request='boston', county='suffolk', state='massachusetts', country='usa')
+ assert_row(boston.centroids(), request='boston', county='suffolk', state='massachusetts', country='usa')
+ assert_row(boston.boundaries(), request='boston', county='suffolk', state='massachusetts', country='usa')
+
+ # antimeridian
+ ru = geodata.regions_builder2(level='country', names='russia').build()
+ assert_row(ru.to_data_frame(), request='russia', county=NO_COLUMN, state=NO_COLUMN, country=NO_COLUMN)
+ assert_row(ru.limits(), request=['russia', 'russia'], county=NO_COLUMN, state=NO_COLUMN, country=NO_COLUMN)
+
+
+#@pytest.mark.skipif(TURN_OFF_INTERACTION_TEST, reason='Need proper server ip')
+def test_regions_parents_in_regions_object_and_geo_data_frame():
+ # parent request from regions object should be propagated to resulting GeoDataFrame
+ massachusetts = geodata.regions_builder2(level='state', names='massachusetts').build()
+ boston = geodata.regions_builder2(level='city', names='boston', states=massachusetts).build()
+
+ assert_row(boston.to_data_frame(), request='boston', state='massachusetts', county=NO_COLUMN, country=NO_COLUMN)
+ assert_row(boston.centroids(), request='boston', state='massachusetts', county=NO_COLUMN, country=NO_COLUMN)
+
+def test_list_of_regions_parents_in_regions_object_and_geo_data_frame():
+ # parent request from regions object should be propagated to resulting GeoDataFrame
+ states = geodata.regions_builder2(level='state', names=['massachusetts', 'texas']).build()
+ cities = geodata.regions_builder2(level='city', names=['boston', 'austin'], states=states).build()
+
+ assert_row(cities.to_data_frame(),
+ request=['boston', 'austin'],
+ state=['massachusetts', 'texas'],
+ county=NO_COLUMN,
+ country=NO_COLUMN
+ )
+
+ assert_row(cities.centroids(),
+ request=['boston', 'austin'],
+ state=['massachusetts', 'texas'],
+ county=NO_COLUMN,
+ country=NO_COLUMN
+ )
+
+def test_parents_lists():
+ states = geodata.regions_builder2(level='state', names=['texas', 'nevada'], countries=['usa', 'usa']).build()
+ assert_row(states.to_data_frame(), request=['texas', 'nevada'], found_name=['Texas', 'Nevada'], country=['usa', 'usa'])
+
+def test_with_drop_not_found():
+ states = geodata.regions_builder2(
+ level='state',
+ names=['texas', 'trololo', 'nevada'],
+ countries=['usa', 'usa', 'usa']
+ )\
+ .drop_not_found() \
+ .build()
+
+ assert_row(states.to_data_frame(), request=['texas', 'nevada'], found_name=['Texas', 'Nevada'], country=['usa', 'usa'])
+