Unit tests for new geocoding API

python-package/lets_plot/geo_data/new_api.py

@@ -1,33 +1,190 @@
 # Copyright (c) 2020. JetBrains s.r.o.
 # Use of this source code is governed by the MIT license that can be found in the LICENSE file.
-from typing import Any, Union, List, Optional
+from typing import Any, Union, List, Optional, Dict
+from collections import namedtuple
 
 import numpy as np
 from pandas import Series
 
 from .gis.geocoding_service import GeocodingService
 from .gis.geometry import GeoPoint
-from .gis.request import RequestBuilder, RequestKind, MapRegion, MapRegionKind
+from .gis.request import RequestBuilder, Request, GeocodingRequest, RequestKind, MapRegion, MapRegionKind, AmbiguityResolver, RegionQuery, \
+ LevelKind, IgnoringStrategyKind
 from .gis.response import Response, SuccessResponse
-from .regions import Regions, _raise_exception, _to_level_kind, _to_scope
-from .regions_builder import RegionsBuilder
+from .regions import Regions, _raise_exception, _to_level_kind, _to_scope, _make_parent_region
+from .regions_builder import RegionsBuilder, NAMESAKE_MAX_COUNT
 from .type_assertion import assert_list_type
+from .regions import _to_level_kind, request_types, parent_types, scope_types, Regions, _raise_exception, \
+ _ensure_is_list
 
 __all__ = [
  'regions_builder2'
 ]
 
-def _prepare_new_scope(scope) -> List[MapRegion]:
+QuerySpec = namedtuple('QuerySpec', 'name, county, state, country')
+WithinSpec = namedtuple('WithinSpec', 'scope, polygon')
+
+def _get_or_none(list, index):
+ if index >= len(list):
+ return None
+ return list[index]
+
+
+def _ensure_is_parent_list(obj):
+ if obj is None:
+ return None
+
+ if isinstance(obj, str):
+ return [obj]
+ if isinstance(obj, Regions):
+ return obj.as_list()
+
+ if isinstance(obj, list):
+ return obj
+
+ return [obj]
+
+
+
+
+class RegionsBuilder2:
+ def __init__(self,
+ level: Optional[Union[str, LevelKind]] = None,
+ request: request_types = None,
+ scope: scope_types = None,
+ highlights: bool = False,
+ allow_ambiguous=False
+ ):
+
+ self._level: Optional[LevelKind] = _to_level_kind(level)
+ self._default_ambiguity_resolver: AmbiguityResolver = AmbiguityResolver.empty() # TODO rename to geohint
+ self._highlights: bool = highlights
+ self._allow_ambiguous = allow_ambiguous
+ self._countries: List[Optional[MapRegion]] = []
+ self._states: List[Optional[MapRegion]] = []
+ self._counties: List[Optional[MapRegion]] = []
+ self._overridings: Dict[QuerySpec, WhereSpec] = {} # query to scope
+
+ requests: Optional[List[str]] = _ensure_is_list(request)
+
+ self._names: List[Optional[str]] = list(map(lambda name: name if requests is not None else None, requests))
+ self._scope: List[Optional[MapRegion]] = _prepare_new_scope(scope)
+
+ def countries(self, countries) -> 'RegionsBuilder2':
+ self._countries = self._make_parents(countries, 'countries')
+ return self
+
+ def states(self, states) -> 'RegionsBuilder2':
+ self._states = self._make_parents(states, 'states')
+ return self
+
+ def counties(self, counties) -> 'RegionsBuilder2':
+ self._counties = self._make_parents(counties, 'counties')
+ return self
+
+ def drop_not_found(self) -> 'RegionsBuilder2':
+ self._default_ambiguity_resolver = AmbiguityResolver(IgnoringStrategyKind.skip_missing)
+ return self
+
+ def drop_not_matched(self) -> 'RegionsBuilder2':
+ self._default_ambiguity_resolver = AmbiguityResolver(IgnoringStrategyKind.skip_all)
+ return self
+
+ def allow_ambiguous(self) -> 'RegionsBuilder2':
+ self._default_ambiguity_resolver = AmbiguityResolver(IgnoringStrategyKind.take_namesakes)
+ self._allow_ambiguous = True
+ return self
+
+ def where(self, name: str,
+ county: Optional[parent_types]=None,
+ state: Optional[parent_types]=None,
+ country: Optional[parent_types]=None,
+ scope: scope_types=None):
+ new_scope: List[MapRegion] = _prepare_new_scope(scope)
+ if len(new_scope) != 1:
+ raise ValueError('Scope for where should have length of 1, but was {}'.format(len(new_scope)))
+
+ county_region = _make_parent_region(county)
+ state_region = _make_parent_region(state)
+ country_region = _make_parent_region(country)
+
+ self._overridings[QuerySpec(name, county_region, state_region, country_region)] = WithinSpec(new_scope[0], AmbiguityResolver.empty())
+ return self
+
+ def _build_request(self) -> GeocodingRequest:
+ queries = []
+ for i in range(len(self._names)):
+ name = self._names[i]
+ country = _get_or_none(self._countries, i)
+ state = _get_or_none(self._states, i)
+ county = _get_or_none(self._counties, i)
+
+ scope, ambiguity_resolver = self._overridings.get(QuerySpec(name, county, state, country),
+ WithinSpec(None, self._default_ambiguity_resolver))
+
+ query = RegionQuery(
+ request=name,
+ country=country,
+ state=state,
+ county=county,
+ scope=scope,
+ ambiguity_resolver=ambiguity_resolver
+ )
+
+ queries.append(query)
+
+ request = RequestBuilder() \
+ .set_request_kind(RequestKind.geocoding) \
+ .set_requested_payload([PayloadKind.highlights] if self._highlights else []) \
+ .set_queries(queries) \
+ .set_scope(self._scope) \
+ .set_level(self._level) \
+ .set_namesake_limit(NAMESAKE_MAX_COUNT) \
+ .set_allow_ambiguous(self._allow_ambiguous) \
+ .build()
+
+ return request
+
+ def build(self) -> Regions:
+ request: GeocodingRequest = self._build_request()
+
+ response: Response = GeocodingService().do_request(request)
+
+ if not isinstance(response, SuccessResponse):
+ _raise_exception(response)
+
+ return Regions(response.level, response.answers, request.region_queries, self._highlights)
+
+ def _make_parents(self, values, kind) -> List[Optional[MapRegion]]:
+ values = _ensure_is_parent_list(values)
+
+ if values is None:
+ return []
+
+ if values is not None and len(values) != len(self._names):
+ raise ValueError('{} count({}) != names count({})'.format(kind, len(values), len(self._names)))
+
+ return list(map(lambda v: _make_parent_region(v) if values is not None else None, values))
+
+ def __eq__(self, o):
+ return isinstance(o, RegionsBuilder2) \
+ and self._overridings == o._overridings
+
+ def __ne__(self, o):
+ return not self == o
+
+
+def _prepare_new_scope(scope: Optional[Union[str, Regions, MapRegion, List]]) -> List[MapRegion]:
  def obj_to_map_region(obj) -> List[MapRegion]:
  if isinstance(obj, str):
  return [MapRegion.with_name(obj)]
  if isinstance(obj, Regions):
  return obj.to_map_regions()
  if isinstance(obj, MapRegion):
  if scope.kind == MapRegionKind.id:
- return [MapRegion.scope([id]) for id in obj.values] # flat_map ids
+ return [MapRegion.scope([id]) for id in obj.values]  # flat_map ids
  else:
- assert len(obj.values) == 1 # only id have more than one value
+ assert len(obj.values) == 1  # only id have more than one value
  return [obj]
 
  if scope is None:
@@ -43,7 +200,8 @@ def obj_to_map_region(obj) -> List[MapRegion]:
  return flatten_scope
 
 
-def regions_builder2(level=None, names=None, countries=None, states=None, counties=None, scope=None, highlights=False) -> RegionsBuilder:
+def regions_builder2(level=None, names=None, countries=None, states=None, counties=None, scope=None,
+ highlights=False) -> RegionsBuilder2:
  """
  Create a RegionBuilder class by level and request. Allows to refine ambiguous request with
  where method. build() method creates Regions object or shows details for ambiguous result.
@@ -85,6 +243,11 @@ def regions_builder2(level=None, names=None, countries=None, states=None, counti
 
  new_scope = _prepare_new_scope(scope)
 
+ return RegionsBuilder2(level, names, scope, highlights, allow_ambiguous=False)\
+ .countries(countries) \
+ .states(states) \
+ .counties(counties)
+
  return RegionsBuilder(level, names, scope, highlights,
  allow_ambiguous=False,
  countries=countries,
@@ -93,6 +256,7 @@ def regions_builder2(level=None, names=None, countries=None, states=None, counti
  new_api=True,
  new_scope=new_scope)
 
+
 def regions2(level=None, names=None, countries=None, states=None, counties=None, scope=None) -> Regions:
  """
  Returns regions object.
@@ -116,4 +280,4 @@ def regions2(level=None, names=None, countries=None, states=None, counties=None,
  If all parents are set (including countries) then the scope parameter is ignored.
  If scope is an array then geocoding will try to search objects in all scopes.
  """
- return regions_builder2(level, names, countries, states, counties, scope).build()
+ return regions_builder2(level, names, countries, states, counties, scope).build()

python-package/lets_plot/geo_data/regions_builder.py

@@ -169,13 +169,14 @@ def __init__(self,
  new_scope: List[MapRegion]=[]
  ):
 
+ self._new_api: bool = new_api
  self._level: Optional[LevelKind] = _to_level_kind(level)
  self._default_ambiguity_resolver: AmbiguityResolver = AmbiguityResolver.empty() # TODO rename to geohint
  self._highlights: bool = highlights
  self._allow_ambiguous = allow_ambiguous
  self._overridings: List[RegionQuery] = []
 
- if new_api:
+ if self._new_api:
  self._scope: List[MapRegion] = new_scope
  self._queries: List[RegionQuery] = _create_new_queries(request, self._default_ambiguity_resolver, countries, states, counties)
  else:
@@ -199,7 +200,11 @@ def allow_ambiguous(self) -> 'RegionsBuilder':
  def where(self,
  request: request_types = None,
  within: Optional[Union[str, List[str], Regions, List[Regions], ShapelyPolygonType]] = None,
- near: Optional[Union[Regions, ShapelyPointType]] = None) -> 'RegionsBuilder':
+ near: Optional[Union[Regions, ShapelyPointType]] = None,
+ country: Optional[str] = None,
+ state: Optional[str] = None,
+ county: Optional[str] = None,
+ ) -> 'RegionsBuilder':
  """
  If request is not exist - append it to a list with specified scope.
  If request is already exist in the list - specify scope exactly for that request.
@@ -236,19 +241,19 @@ def where(self,
  """
 
  scope, box = _split(within)
+ if self._new_api:
+ if scope is not None:
+ raise ValueError('Parameter scope is not available in new version of where function')
+ else:
+ ambiguity_resolver = AmbiguityResolver(None, _to_near_coord(near), box)
 
- ambiguity_resolver = AmbiguityResolver(
- None,
- _to_near_coord(near),
- box
- )
+ new_overridings = _create_queries(request, scope, ambiguity_resolver)
+ for overriding in self._overridings:
+ if overriding.request in set([overriding.request for overriding in new_overridings]):
+ self._overridings.remove(overriding)
 
- new_overridings = _create_queries(request, scope, ambiguity_resolver)
- for overriding in self._overridings:
- if overriding.request in set([overriding.request for overriding in new_overridings]):
- self._overridings.remove(overriding)
+ self._overridings.extend(new_overridings)
 
- self._overridings.extend(new_overridings)
  return self
 
  def build(self) -> Regions:

python-package/test/geo_data/request_assertion.py

@@ -0,0 +1,88 @@
+# Copyright (c) 2020. JetBrains s.r.o.
+# Use of this source code is governed by the MIT license that can be found in the LICENSE file.
+from typing import TypeVar, Generic, Optional
+
+from lets_plot.geo_data.gis.request import Request, GeocodingRequest, RegionQuery, MapRegion, AmbiguityResolver
+
+T = TypeVar('T')
+
+class ValueMatcher(Generic[T]):
+ def check(self, value):
+ raise ValueError('abstract')
+
+class any(ValueMatcher[T]):
+ def check(self, value):
+ return
+
+class eq(ValueMatcher[T]):
+ def __init__(self, v):
+ self.expected = v
+
+ def check(self, value):
+ assert self.expected == value, '{} != {}'.format(self.expected, value)
+
+class eq_map_region(eq):
+ def __init__(self, v):
+ self.expected = MapRegion.with_name(v)
+
+
+class empty(ValueMatcher[T]):
+ def check(self, value):
+ assert value is None, '{} is not None'.format(value)
+
+class RegionQueryMatcher:
+ def __init__(self,
+ request: ValueMatcher[Optional[str]] = any(),
+ scope: ValueMatcher[Optional[MapRegion]] = any(),
+ ambiguity_resolver: ValueMatcher[AmbiguityResolver] = any(),
+ country: ValueMatcher[Optional[MapRegion]] = any(),
+ state: ValueMatcher[Optional[MapRegion]] = any(),
+ county: ValueMatcher[Optional[MapRegion]] = any()
+ ):
+ self._request: ValueMatcher[Optional[str]] = request
+ self._scope: ValueMatcher[Optional[MapRegion]] = scope
+ self._ambiguity_resolver: ValueMatcher[AmbiguityResolver] = ambiguity_resolver
+ self._country: ValueMatcher[Optional[MapRegion]] = country
+ self._state: ValueMatcher[Optional[MapRegion]] = state
+ self._county: ValueMatcher[Optional[MapRegion]] = county
+
+ def request(self, request: ValueMatcher[Optional[str]]) -> 'RegionQueryMatcher':
+ self._request = request
+ return self
+
+ def scope(self, scope: ValueMatcher[Optional[MapRegion]]) -> 'RegionQueryMatcher':
+ self._scope = scope
+ return self
+
+ def ambiguity_resolver(self, ambiguity_resolver: ValueMatcher[AmbiguityResolver]) -> 'RegionQueryMatcher':
+ self._ambiguity_resolver = ambiguity_resolver
+ return self
+
+ def country(self, country: ValueMatcher[Optional[MapRegion]]) -> 'RegionQueryMatcher':
+ self._country: ValueMatcher[Optional[MapRegion]]
+ return self
+
+ def state(self, state: ValueMatcher[Optional[MapRegion]]) -> 'RegionQueryMatcher':
+ self._state = state
+ return self
+
+ def county(self, county: ValueMatcher[Optional[MapRegion]]) -> 'RegionQueryMatcher':
+ self._county = county
+ return self
+
+ def check(self, q: RegionQuery):
+ self._request.check(q.request)
+ self._scope.check(q.scope)
+ self._ambiguity_resolver.check(q.ambiguity_resolver)
+ self._country.check(q.country)
+ self._state.check(q.state)
+ self._county.check(q.county)
+
+class GeocodingRequestAssertion:
+ def __init__(self, request: Request):
+ assert isinstance(request, GeocodingRequest)
+ self._request: GeocodingRequest = request
+
+ def has_query(self, i: int, query_matcher: RegionQueryMatcher) -> 'GeocodingRequestAssertion':
+ query_matcher.check(self._request.region_queries[i])
+ return self

python-package/test/geo_data/test_integration_new_api.py

@@ -126,3 +126,8 @@ def test_simple_scope():
  florida_with_scope = geodata.regions_builder2('state', names=['florida'], scope='Uruguay').build().to_data_frame()
 
  assert florida_with_country[DF_ID][0] == florida_with_scope[DF_ID][0]
+
+
+def test_where():
+ geodata.regions_builder2('city', names='warwick')\
+ .where('warwick', scope='oklahoma').build()