Retry errors setting executable headers.

Fixes this flake: https://build.chromium.org/p/chromium.infra/builders/infra-continuous-mac-10.9-64/builds/1729/steps/upload%20go%20bin/logs/stdio

BUG=

Review URL: https://codereview.chromium.org/1824223002

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@299525 0039d316-1c4b-4281-b951-d872f2087c98

download_from_google_storage.py

@@ -55,6 +55,11 @@ def GetNormalizedPlatform():
 class Gsutil(object):
   """Call gsutil with some predefined settings.  This is a convenience object,
   and is also immutable."""
+
+  MAX_TRIES = 5
+  RETRY_BASE_DELAY = 5.0
+  RETRY_DELAY_MULTIPLE = 1.3
+
   def __init__(self, path, boto_path=None, timeout=None, version='4.15'):
     if not os.path.exists(path):
       raise FileNotFoundError('GSUtil not found in %s' % path)
@@ -100,6 +105,18 @@ class Gsutil(object):
       return (404, out, err)
     return (code, out, err)
 
+  def check_call_with_retries(self, *args):
+    delay = self.RETRY_BASE_DELAY
+    for i in xrange(self.MAX_TRIES):
+      code, out, err = self.check_call(*args)
+      if not code or i == self.MAX_TRIES - 1:
+        break
+
+      time.sleep(delay)
+      delay *= self.RETRY_DELAY_MULTIPLE
+
+    return code, out, err
+
 
 def check_platform(target):
   """Checks if any parent directory of target matches (win|mac|linux)."""

upload_to_google_storage.py

@@ -76,7 +76,7 @@ def _upload_worker(
     file_url = '%s/%s' % (base_url, sha1_sum)
     if gsutil.check_call('ls', file_url)[0] == 0 and not force:
       # File exists, check MD5 hash.
-      _, out, _ = gsutil.check_call('ls', '-L', file_url)
+      _, out, _ = gsutil.check_call_with_retries('ls', '-L', file_url)
       etag_match = re.search('ETag:\s+([a-z0-9]{32})', out)
       if etag_match:
         remote_md5 = etag_match.group(1)
@@ -97,7 +97,7 @@ def _upload_worker(
     if gzip:
       gsutil_args.extend(['-z', gzip])
     gsutil_args.extend([filename, file_url])
-    code, _, err = gsutil.check_call(*gsutil_args)
+    code, _, err = gsutil.check_call_with_retries(*gsutil_args)
     if code != 0:
       ret_codes.put(
           (code,
@@ -109,9 +109,9 @@ def _upload_worker(
     # the download script will check for to preserve the executable bit.
     if not sys.platform.startswith('win'):
       if os.stat(filename).st_mode & stat.S_IEXEC:
-        code, _, err = gsutil.check_call('setmeta', '-h',
+        code, _, err = gsutil.check_call_with_retries(
-                                         'x-goog-meta-executable:1', file_url)
+            'setmeta', '-h', 'x-goog-meta-executable:1', file_url)
-        if code:
+        if not code:
           ret_codes.put(
               (code,
                'Encountered error on setting metadata on %s\n%s' %