[BEAM-7424] Retry HTTP 429 errors from GCS

match Python SDK retrying threshold to Java SDK threshold.
mransley · Jun 26, 2019 · b6e45f4 · b6e45f4
1 parent 221e0a5
commit b6e45f4
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py
@@ -522,7 +522,8 @@ def __init__(self, client, path, buffer_size):
  # Initialize read buffer state.
  self._download_stream = io.BytesIO()
  self._downloader = transfer.Download(
- self._download_stream, auto_transfer=False, chunksize=self._buffer_size)
+ self._download_stream, auto_transfer=False, chunksize=self._buffer_size,
+ num_retries=20)
  self._client.objects.Get(self._get_request, download=self._downloader)
 
  @retry.with_exponential_backoff(

diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py
@@ -52,7 +52,7 @@ def __init__(self, url='', credentials=None,
  super(StorageV1, self).__init__(
  url, credentials=credentials,
  get_credentials=get_credentials, http=http, model=model,
- log_request=log_request, log_response=log_response,
+ log_request=log_request, log_response=log_response, num_retries=20,
  credentials_args=credentials_args,
  default_global_params=default_global_params,
  additional_http_headers=additional_http_headers,