From b180ded392c1cb257ffdada6b18680bb6c87baa9 Mon Sep 17 00:00:00 2001 From: "dsansome@chromium.org" Date: Tue, 29 Mar 2016 03:27:41 +0000 Subject: [PATCH] Retry errors setting executable headers. Fixes this flake: https://build.chromium.org/p/chromium.infra/builders/infra-continuous-mac-10.9-64/builds/1729/steps/upload%20go%20bin/logs/stdio BUG= Review URL: https://codereview.chromium.org/1824223002 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@299525 0039d316-1c4b-4281-b951-d872f2087c98 --- download_from_google_storage.py | 17 +++++++++++++++++ upload_to_google_storage.py | 10 +++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/download_from_google_storage.py b/download_from_google_storage.py index b186c63ad..d71de9868 100755 --- a/download_from_google_storage.py +++ b/download_from_google_storage.py @@ -55,6 +55,11 @@ def GetNormalizedPlatform(): class Gsutil(object): """Call gsutil with some predefined settings. This is a convenience object, and is also immutable.""" + + MAX_TRIES = 5 + RETRY_BASE_DELAY = 5.0 + RETRY_DELAY_MULTIPLE = 1.3 + def __init__(self, path, boto_path=None, timeout=None, version='4.15'): if not os.path.exists(path): raise FileNotFoundError('GSUtil not found in %s' % path) @@ -100,6 +105,18 @@ class Gsutil(object): return (404, out, err) return (code, out, err) + def check_call_with_retries(self, *args): + delay = self.RETRY_BASE_DELAY + for i in xrange(self.MAX_TRIES): + code, out, err = self.check_call(*args) + if not code or i == self.MAX_TRIES - 1: + break + + time.sleep(delay) + delay *= self.RETRY_DELAY_MULTIPLE + + return code, out, err + def check_platform(target): """Checks if any parent directory of target matches (win|mac|linux).""" diff --git a/upload_to_google_storage.py b/upload_to_google_storage.py index 4487457bd..83f2f0e16 100755 --- a/upload_to_google_storage.py +++ b/upload_to_google_storage.py @@ -76,7 +76,7 @@ def _upload_worker( file_url = '%s/%s' % (base_url, sha1_sum) if gsutil.check_call('ls', file_url)[0] == 0 and not force: # File exists, check MD5 hash. - _, out, _ = gsutil.check_call('ls', '-L', file_url) + _, out, _ = gsutil.check_call_with_retries('ls', '-L', file_url) etag_match = re.search('ETag:\s+([a-z0-9]{32})', out) if etag_match: remote_md5 = etag_match.group(1) @@ -97,7 +97,7 @@ def _upload_worker( if gzip: gsutil_args.extend(['-z', gzip]) gsutil_args.extend([filename, file_url]) - code, _, err = gsutil.check_call(*gsutil_args) + code, _, err = gsutil.check_call_with_retries(*gsutil_args) if code != 0: ret_codes.put( (code, @@ -109,9 +109,9 @@ def _upload_worker( # the download script will check for to preserve the executable bit. if not sys.platform.startswith('win'): if os.stat(filename).st_mode & stat.S_IEXEC: - code, _, err = gsutil.check_call('setmeta', '-h', - 'x-goog-meta-executable:1', file_url) - if code: + code, _, err = gsutil.check_call_with_retries( + 'setmeta', '-h', 'x-goog-meta-executable:1', file_url) + if not code: ret_codes.put( (code, 'Encountered error on setting metadata on %s\n%s' %