From b6e45f48905755e9a27c897f7d7eb8377f12b5c4 Mon Sep 17 00:00:00 2001 From: Heejong Lee Date: Mon, 24 Jun 2019 11:55:52 -0700 Subject: [PATCH] [BEAM-7424] Retry HTTP 429 errors from GCS match Python SDK retrying threshold to Java SDK threshold. --- sdks/python/apache_beam/io/gcp/gcsio.py | 3 ++- .../io/gcp/internal/clients/storage/storage_v1_client.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py index 5586eb48fe8e1..6a27c273f8e43 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio.py +++ b/sdks/python/apache_beam/io/gcp/gcsio.py @@ -522,7 +522,8 @@ def __init__(self, client, path, buffer_size): # Initialize read buffer state. self._download_stream = io.BytesIO() self._downloader = transfer.Download( - self._download_stream, auto_transfer=False, chunksize=self._buffer_size) + self._download_stream, auto_transfer=False, chunksize=self._buffer_size, + num_retries=20) self._client.objects.Get(self._get_request, download=self._downloader) @retry.with_exponential_backoff( diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py index cb5a0c9365be8..735b1597a28e5 100644 --- a/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py +++ b/sdks/python/apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py @@ -52,7 +52,7 @@ def __init__(self, url='', credentials=None, super(StorageV1, self).__init__( url, credentials=credentials, get_credentials=get_credentials, http=http, model=model, - log_request=log_request, log_response=log_response, + log_request=log_request, log_response=log_response, num_retries=20, credentials_args=credentials_args, default_global_params=default_global_params, additional_http_headers=additional_http_headers,