From af121aeec9a2e4436c7609a39fcb61131a815b83 Mon Sep 17 00:00:00 2001 From: Dirk Pranke Date: Wed, 19 May 2021 18:32:07 +0000 Subject: [PATCH] Revert "Update gsutil to use gsutil version 4.61, python3." This reverts commit f059ec936899adfd83f63fff46fdb61ae3a39537. Reason for revert: Reverting because we probably need to be using vpython and a compiled crcmod instead. See, e.g.,. b/188591640. Original change's description: > Update gsutil to use gsutil version 4.61, python3. > > This CL updates the gsutil.py wrapper to download and use > v4.61 of GCP's gsutil, which is Python3-compatible. > > v4.61 appears to be fully self-contained and have all of the > packages it needs vendored into it. So, there's no reason to > use vpython anymore, and this CL removes that. > > Also, this CL removes the 'fallback' option to gsutil and > the ability to force a version switch, as this should no > longer be necessary (it was added for a migration back in 2014 > but apparently this code was never removed afterwards). > > This CL also updates download_from_google_storage.py and > upload_to_google_storage.py to similarly not have the version flags > and to just use regular python3, not vpython3. > > Bug: 1184108 > Change-Id: I0d1a8351dba2d3ad1f927afa333fb10959f19443 > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2898439 > Reviewed-by: Mike Frysinger > Reviewed-by: Josip Sokcevic > Reviewed-by: Robbie Iannucci > Commit-Queue: Dirk Pranke Bug: 1184108 Change-Id: I8e21a9a40d81e4e185642f866855b6838f80f1c2 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2905904 Bot-Commit: Rubber Stamper Commit-Queue: Dirk Pranke --- download_from_google_storage.py | 9 ++- gsutil.py | 32 ++++----- gsutil.vpython | 115 ++++++++++++++++++++++++++++++++ tests/gsutil_test.py | 6 +- upload_to_google_storage.py | 2 +- 5 files changed, 142 insertions(+), 22 deletions(-) create mode 100644 gsutil.vpython diff --git a/download_from_google_storage.py b/download_from_google_storage.py index d416d1444e..84bc8e30db 100755 --- a/download_from_google_storage.py +++ b/download_from_google_storage.py @@ -76,12 +76,15 @@ class Gsutil(object): MAX_TRIES = 5 RETRY_BASE_DELAY = 5.0 RETRY_DELAY_MULTIPLE = 1.3 + VPYTHON3 = ('vpython3.bat' + if GetNormalizedPlatform() == 'win32' else 'vpython3') - def __init__(self, path, boto_path=None): + def __init__(self, path, boto_path=None, version='4.28'): if not os.path.exists(path): raise FileNotFoundError('GSUtil not found in %s' % path) self.path = path self.boto_path = boto_path + self.version = version def get_sub_env(self): env = os.environ.copy() @@ -98,12 +101,12 @@ class Gsutil(object): return env def call(self, *args): - cmd = [sys.executable, self.path] + cmd = [self.VPYTHON3, self.path, '--force-version', self.version] cmd.extend(args) return subprocess2.call(cmd, env=self.get_sub_env()) def check_call(self, *args): - cmd = [sys.executable, self.path] + cmd = [self.VPYTHON3, self.path, '--force-version', self.version] cmd.extend(args) ((out, err), code) = subprocess2.communicate( cmd, diff --git a/gsutil.py b/gsutil.py index 6ed3773852..d5423ad7ef 100755 --- a/gsutil.py +++ b/gsutil.py @@ -29,14 +29,13 @@ import zipfile GSUTIL_URL = 'https://storage.googleapis.com/pub/' API_URL = 'https://www.googleapis.com/storage/v1/b/pub/o/' - THIS_DIR = os.path.dirname(os.path.abspath(__file__)) DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil') +DEFAULT_FALLBACK_GSUTIL = os.path.join( + THIS_DIR, 'third_party', 'gsutil', 'gsutil') IS_WINDOWS = os.name == 'nt' -VERSION = '4.61' - class InvalidGsutilError(Exception): pass @@ -130,8 +129,11 @@ def ensure_gsutil(version, target, clean): return gsutil_bin -def run_gsutil(target, args, clean=False): - gsutil_bin = ensure_gsutil(VERSION, target, clean) +def run_gsutil(force_version, fallback, target, args, clean=False): + if force_version: + gsutil_bin = ensure_gsutil(force_version, target, clean) + else: + gsutil_bin = fallback disable_update = ['-o', 'GSUtil:software_update_check_period=0'] if sys.platform == 'cygwin': @@ -145,8 +147,13 @@ def run_gsutil(target, args, clean=False): sys.exit(subprocess.call(cmd)) assert sys.platform != 'cygwin' + # Run "gsutil" through "vpython". We need to do this because on GCE instances, + # expectations are made about Python having access to "google-compute-engine" + # and "boto" packages that are not met with non-system Python (e.g., bundles). cmd = [ - sys.executable, + 'vpython', + '-vpython-spec', os.path.join(THIS_DIR, 'gsutil.vpython'), + '--', gsutil_bin ] + disable_update + args return subprocess.call(cmd, shell=IS_WINDOWS) @@ -156,19 +163,13 @@ def parse_args(): bin_dir = os.environ.get('DEPOT_TOOLS_GSUTIL_BIN_DIR', DEFAULT_BIN_DIR) parser = argparse.ArgumentParser() - + parser.add_argument('--force-version', default='4.30') parser.add_argument('--clean', action='store_true', help='Clear any existing gsutil package, forcing a new download.') + parser.add_argument('--fallback', default=DEFAULT_FALLBACK_GSUTIL) parser.add_argument('--target', default=bin_dir, help='The target directory to download/store a gsutil version in. ' '(default is %(default)s).') - - # These two args exist for backwards-compatibility but are no-ops. - parser.add_argument('--force-version', default=VERSION, - help='(deprecated, this flag has no effect)') - parser.add_argument('--fallback', - help='(deprecated, this flag has no effect)') - parser.add_argument('args', nargs=argparse.REMAINDER) args, extras = parser.parse_known_args() @@ -181,7 +182,8 @@ def parse_args(): def main(): args = parse_args() - return run_gsutil(args.target, args.args, clean=args.clean) + return run_gsutil(args.force_version, args.fallback, args.target, args.args, + clean=args.clean) if __name__ == '__main__': diff --git a/gsutil.vpython b/gsutil.vpython new file mode 100644 index 0000000000..aa47e5918f --- /dev/null +++ b/gsutil.vpython @@ -0,0 +1,115 @@ +# vpython VirtualEnv protobuf for "gsutil". +# +# See: +# https://chromium.googlesource.com/infra/luci/luci-go/+/HEAD/vpython/api/vpython/spec.proto +# +# This is a "vpython" VirtualEnv specification applied to invocations of "gsutil" +# by the bootstrap wrapper, "gsutil.py". It ensures that any Python distribution +# has the expected Python packages installed. +# +# This is specifically relevant on Google Compute Engine invocations of +# "gsutil", where a stock system-deployed file in "/etc/" explicitly specifies +# that the "google_compute_engine" and "boto" Python packages are available. + +python_version: "2.7" + +wheel < + name: "infra/python/wheels/google_compute_engine-py2_py3" + version: "version:2.6.2" +> +wheel < + name: "infra/python/wheels/boto-py2_py3" + version: "version:2.48.0" +> + +# "gsutil" on non-GCE can require PyOpenSSL, which, in turn, requires +# "cryptography". + +wheel: < + name: "infra/python/wheels/pyopenssl-py2_py3" + version: "version:17.2.0" +> + +## +# BEGIN "cryptography" dependencies. +## + +wheel: < + name: "infra/python/wheels/cryptography/${vpython_platform}" + version: "version:2.9.2" +> + +wheel: < + name: "infra/python/wheels/appdirs-py2_py3" + version: "version:1.4.3" +> + +wheel: < + name: "infra/python/wheels/enum34-py2" + version: "version:1.1.6" +> + +wheel: < + name: "infra/python/wheels/cffi/${vpython_platform}" + version: "version:1.14.5" +> + +wheel: < + name: "infra/python/wheels/idna-py2_py3" + version: "version:2.5" +> + +wheel: < + name: "infra/python/wheels/ipaddress-py2" + version: "version:1.0.18" +> + +wheel: < + name: "infra/python/wheels/packaging-py2_py3" + version: "version:16.8" +> + +wheel: < + name: "infra/python/wheels/pyasn1-py2_py3" + version: "version:0.2.3" +> + +wheel: < + name: "infra/python/wheels/pycparser-py2_py3" + version: "version:2.17" +> + +wheel: < + name: "infra/python/wheels/pyparsing-py2_py3" + version: "version:2.2.0" +> + +wheel: < + name: "infra/python/wheels/setuptools-py2_py3" + version: "version:34.3.2" +> + +wheel: < + name: "infra/python/wheels/six-py2_py3" + version: "version:1.10.0" +> + +## +# END "cryptography" dependencies. +## + +wheel: < + name: "infra/python/wheels/crcmod/${vpython_platform}" + version: "version:1.7" + match_tag: < + abi: "cp27mu" + platform: "manylinux1_i686" + > + match_tag: < + abi: "cp27mu" + platform: "manylinux1_x86_64" + > + match_tag: < + platform: "macosx_10_6_intel" + > +> diff --git a/tests/gsutil_test.py b/tests/gsutil_test.py index d9b7bbbd5c..cd81291046 100755 --- a/tests/gsutil_test.py +++ b/tests/gsutil_test.py @@ -72,7 +72,7 @@ class GsutilUnitTests(unittest.TestCase): setattr(subprocess, 'call', self.old_call) def test_download_gsutil(self): - version = gsutil.VERSION + version = '4.2' filename = 'gsutil_%s.zip' % version full_filename = os.path.join(self.tempdir, filename) fake_file = b'This is gsutil.zip' @@ -113,7 +113,7 @@ class GsutilUnitTests(unittest.TestCase): self.assertEqual(self.fake.expectations, []) def test_ensure_gsutil_full(self): - version = gsutil.VERSION + version = '4.2' gsutil_dir = os.path.join(self.tempdir, 'gsutil_%s' % version, 'gsutil') gsutil_bin = os.path.join(gsutil_dir, 'gsutil') gsutil_flag = os.path.join(gsutil_dir, 'install.flag') @@ -137,7 +137,7 @@ class GsutilUnitTests(unittest.TestCase): self.assertEqual(self.fake.expectations, []) def test_ensure_gsutil_short(self): - version = gsutil.VERSION + version = '4.2' gsutil_dir = os.path.join(self.tempdir, 'gsutil_%s' % version, 'gsutil') gsutil_bin = os.path.join(gsutil_dir, 'gsutil') gsutil_flag = os.path.join(gsutil_dir, 'install.flag') diff --git a/upload_to_google_storage.py b/upload_to_google_storage.py index 2ecdf4e337..332bb9a47b 100755 --- a/upload_to_google_storage.py +++ b/upload_to_google_storage.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file.