Remove chunked requests, better docs for new API

python-package/lets_plot/geo_data/gis/geocoding_service.py

@@ -12,64 +12,8 @@
 
 
 class GeocodingService:
- def do_request(self, request: Request, chunk_size=None, progress_callback=None) -> Response:
- # level autodetection can work only with whole request
- if chunk_size is not None and isinstance(request, GeocodingRequest) and request.level is not None:
- return self._execute_chunked(request, chunk_size, progress_callback)
- else:
- return self._execute(request)
-
- def _execute_chunked(self, request: GeocodingRequest, chunk_size, progress_callback) -> Response:
- success_chunks = []
- ambiguous_chunks = []
-
- def chunked(items):
- for i in range(0, len(items), chunk_size):
- yield items[i:i + chunk_size]
-
- total_count = len(request.region_queries)
- i = 0
- if progress_callback:
- progress_callback(i, total_count)
-
- for q in chunked(request.region_queries):
- chunked_request = GeocodingRequest(
- requested_payload=request.requested_payload,
- resolution=request.resolution,
- region_queries=q,
- level=request.level,
- namesake_example_limit=request.namesake_example_limit,
- allow_ambiguous=request.allow_ambiguous
- )
-
- response = self._execute(chunked_request)
- if progress_callback:
- i = i + len(q)
- progress_callback(i, total_count)
-
- if isinstance(response, ErrorResponse):
- return response
- elif isinstance(response, SuccessResponse):
- success_chunks.append(response)
- elif isinstance(response, AmbiguousResponse):
- ambiguous_chunks.append(response)
- else:
- raise ValueError('Unknown response type: ' + type(response).__name__)
-
- # combine ambiguous features from all chunks
- if ambiguous_chunks:
- ambiguous_features = []
- for response in ambiguous_chunks:
- ambiguous_features.extend(response.features)
-
- return AmbiguousResponse(ambiguous_chunks[0].message, request.level, ambiguous_features)
-
- # no errors or ambiguous responses - combine success features from all chunks
- success_features = []
- for response in success_chunks:
- success_features.extend(response.features)
-
- return SuccessResponse(success_chunks[0].message, request.level, success_features)
+ def do_request(self, request: Request) -> Response:
+ return self._execute(request)
 
  def _execute(self, request: Request) -> Response:
  if not has_global_value(GEOCODING_PROVIDER_URL):

python-package/lets_plot/geo_data/new_api.py

@@ -29,20 +29,19 @@ def regions_builder2(level=None, names=None, scope=None, countries=None, states=
  level : ['country' | 'state' | 'county' | 'city' | None]
  The level of administrative division. Default is a 'state'.
  names : [array | string | None]
- Data can be filtered by full names at any level (only exact matching).
+ Names of objects to be geocoded.
  For 'state' level:
  -'US-48' returns continental part of United States (48 states) in a compact form.
- countries : [array | string | None]
- Parent countries. Should have same size as names.
- states : [array | string | None]
- Parent states. Should have same size as names.
- counties : [array | string | None]
- Parent counties. Should have same size as names.
+ countries : [array | None]
+ Parent countries. Should have same size as names. Can contain strings or Regions objects.
+ states : [array | None]
+ Parent states. Should have same size as names. Can contain strings or Regions objects.
+ counties : [array | None]
+ Parent counties. Should have same size as names. Can contain strings or Regions objects.
  scope : [array | string | Regions | None]
- Data can be filtered by within name.
- If within is array then request and within will be merged positionally (size should be equal).
- If within is Regions then request will be searched in any of these regions.
- 'US-48' includes continental part of United States (48 states).
+ Limits area of geocoding. Applyed to a highest admin level of parents that are set or to names, if no parents given.
+ If all parents are set (including countries) then the scope parameter is ignored.
+ If scope is an array then geocoding will try to search objects in all scopes.
 
  Returns
  -------
@@ -58,8 +57,6 @@ def regions_builder2(level=None, names=None, scope=None, countries=None, states=
  >>> r = regions_builder(level='city', request=['moscow', 'york']).where('york', regions_state('New York')).build()
  """
  return RegionsBuilder(level, names, scope, highlights,
- progress_callback=None,
- chunk_size=None,
  allow_ambiguous=False,
  countries=countries,
  states=states,

python-package/lets_plot/geo_data/regions.py

@@ -45,9 +45,11 @@ class Resolution(enum.Enum):
 def contains_values(column):
  return any(v is not None for v in column)
 
+
 def select_request(query: RegionQuery, answer: Answer, feature: GeocodedFeature) -> str:
  return query.request if len(answer.features) <= 1 else feature.name
 
+
 def zip_answers(queries: List, answers: List):
  if len(queries) > 0:
  return zip(queries, answers)
@@ -76,7 +78,6 @@ def append_row(self, request: str, found_name: str, query: Optional[RegionQuery]
  self._state.append(MapRegion.name_or_none(query.state))
  self._country.append(MapRegion.name_or_none(query.country))
 
-
  def build_dict(self):
  data = {}
  data[DF_REQUEST] = self._request
@@ -93,14 +94,14 @@ def build_dict(self):
 
  return data
 
-
  @abstractmethod
  def to_data_frame(self, answers: List[Answer], queries: List[RegionQuery] = []) -> DataFrame:
  raise ValueError('Not implemented')
 
 
 class Regions(CanToDataFrame):
- def __init__(self, level_kind: LevelKind, answers: List[Answer], queries: List[RegionQuery], highlights: bool = False):
+ def __init__(self, level_kind: LevelKind, answers: List[Answer], queries: List[RegionQuery],
+ highlights: bool = False):
  assert_list_type(answers, Answer)
  assert_list_type(queries, RegionQuery)
  assert len(answers) == len(queries)
@@ -136,11 +137,12 @@ def to_map_regions(self):
 
  def as_list(self) -> List['Regions']:
  if len(self._queries) == 0:
- return [Regions(self._level_kind, [answer], [RegionQuery(request=None)], self._highlights) for answer in self._answers]
+ return [Regions(self._level_kind, [answer], [RegionQuery(request=None)], self._highlights) for answer in
+ self._answers]
 
  assert len(self._queries) == len(self._answers)
- return [Regions(self._level_kind, [answer], [query], self._highlights) for query, answer in zip(self._queries, self._answers)]
-
+ return [Regions(self._level_kind, [answer], [query], self._highlights) for query, answer in
+ zip(self._queries, self._answers)]
 
  def unique_ids(self) -> List[str]:
  seen = set()
@@ -287,7 +289,6 @@ def to_data_frame(self) -> DataFrame:
 
  return DataFrame(data)
 
-
  def _execute(self, request_builder: RequestBuilder, df_converter):
  response = GeocodingService().do_request(request_builder.build())
 

python-package/lets_plot/geo_data/regions_builder.py

@@ -36,21 +36,23 @@ def _to_near_coord(near: Optional[Union[Regions, ShapelyPointType]]) -> Optional
  else:
  raise ValueError("Unexpected geocoding response for id " + str(near_id[0]))
 
- if ShapelyWrapper.is_point(near):
+ if LazyShapely.is_point(near):
  return GeoPoint(lon=near.x, lat=near.y)
 
  raise ValueError('Not supported type')
 
 
-def _split(box: Optional[Union[str, List[str], Regions, List[Regions], ShapelyPolygonType]]) -> Tuple[scope_types, Optional[GeoRect]]:
- if not ShapelyWrapper.is_polygon(box):
+def _split(box: Optional[Union[str, List[str], Regions, List[Regions], ShapelyPolygonType]]) -> Tuple[
+ scope_types, Optional[GeoRect]]:
+ if not LazyShapely.is_polygon(box):
  return box, None
 
  return None, GeoRect(min_lon=box.bounds[0], min_lat=box.bounds[1], max_lon=box.bounds[2], max_lat=box.bounds[3])
 
 
 def _create_queries(request: request_types, scope: scope_types, ambiguity_resovler: AmbiguityResolver,
- countries: parent_types = None, states: parent_types = None, counties: parent_types = None) -> List[RegionQuery]:
+ countries: parent_types = None, states: parent_types = None, counties: parent_types = None) -> List[
+ RegionQuery]:
  requests: Optional[List[str]] = _ensure_is_list(request)
 
  if (countries is None and states is None and counties is None) or requests is None:
@@ -93,7 +95,6 @@ def ensure_is_parent_list(obj):
  states = ensure_is_parent_list(states)
  counties = ensure_is_parent_list(counties)
 
-
  if countries is not None and len(countries) != len(requests):
  raise ValueError('Countries count({}) != names count({})'.format(len(countries), len(requests)))
 
@@ -118,25 +119,25 @@ def ensure_is_parent_list(obj):
  return queries
 
 
-class ShapelyWrapper:
+class LazyShapely:
  @staticmethod
  def is_point(p) -> bool:
- if not ShapelyWrapper.is_shapely_available():
+ if not LazyShapely._is_shapely_available():
  return False
 
  from shapely.geometry import Point
  return isinstance(p, Point)
 
  @staticmethod
  def is_polygon(p):
- if not ShapelyWrapper.is_shapely_available():
+ if not LazyShapely._is_shapely_available():
  return False
 
  from shapely.geometry import Polygon
  return isinstance(p, Polygon)
 
  @staticmethod
- def is_shapely_available():
+ def _is_shapely_available():
  try:
  import shapely
  return True
@@ -150,21 +151,18 @@ def __init__(self,
  request: request_types = None,
  scope: scope_types = None,
  highlights: bool = False,
- progress_callback = None,
- chunk_size = None,
- allow_ambiguous = False,
+ allow_ambiguous=False,
  countries=None,
  states=None,
  counties=None
  ):
 
  self._level: Optional[LevelKind] = _to_level_kind(level)
  self._overridings: List[RegionQuery] = []
- self._default_ambiguity_resolver: AmbiguityResolver = AmbiguityResolver.empty() # TODO rename to geohint
- self._queries: List[RegionQuery] = _create_queries(request, scope, self._default_ambiguity_resolver, countries, states, counties)
+ self._default_ambiguity_resolver: AmbiguityResolver = AmbiguityResolver.empty() # TODO rename to geohint
+ self._queries: List[RegionQuery] = _create_queries(request, scope, self._default_ambiguity_resolver, countries,
+ states, counties)
  self._highlights: bool = highlights
- self._on_progress = progress_callback
- self._chunk_size = chunk_size
  self._allow_ambiguous = allow_ambiguous
 
  def drop_not_found(self) -> 'RegionsBuilder':
@@ -180,11 +178,6 @@ def allow_ambiguous(self) -> 'RegionsBuilder':
  self._allow_ambiguous = True
  return self
 
- def chunk_request(self, on_progress=None, chunk_size=40):
- self._chunk_size = chunk_size
- self._on_progress = on_progress
- return self
-
  def where(self,
  request: request_types = None,
  within: Optional[Union[str, List[str], Regions, List[Regions], ShapelyPolygonType]] = None,
@@ -251,11 +244,7 @@ def build(self) -> Regions:
  .set_allow_ambiguous(self._allow_ambiguous) \
  .build()
 
- # Too many queries - can fail with timeout. Chunk queries.
- if len(self._get_queries()) > 100_000 and self._chunk_size is None:
- self.chunk_request(self._on_progress, 40)
-
- response: Response = GeocodingService().do_request(request, self._chunk_size, self._on_progress)
+ response: Response = GeocodingService().do_request(request)
 
  if not isinstance(response, SuccessResponse):
  _raise_exception(response)

python-package/test/geo_data/test_core.py

@@ -70,9 +70,7 @@ def test_regions(mock_geocoding):
  level=LEVEL_KIND,
  namesake_example_limit=NAMESAKES_EXAMPLE_LIMIT,
  allow_ambiguous=False
- ),
- None, # chunk_size
- None # progress_callback
+ )
  )
 
 
@@ -90,9 +88,7 @@ def test_regions_with_highlights(mock_geocoding):
  level=LEVEL_KIND,
  namesake_example_limit=NAMESAKES_EXAMPLE_LIMIT,
  allow_ambiguous=False
- ),
- None, # chunk_size
- None # progress_callback
+ )
  )
 
 FOO_FEATURE = FeatureBuilder().set_name('fooname').set_id('fooid').build_geocoded()