From 83aafc979fa6536d4d38b18b51e972bdeccdf1d3 Mon Sep 17 00:00:00 2001 From: Edward Lemur Date: Mon, 25 Nov 2019 23:25:05 +0000 Subject: [PATCH] depot_tools: Make gsutil compatible with python3. Bug: 1026371 Change-Id: Iddef64ac2d3a9f97fdb0540a048dfb9dc5679b5c Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1925407 Auto-Submit: Edward Lesmes Reviewed-by: Anthony Polito Commit-Queue: Edward Lesmes --- gsutil.py | 2 +- tests/gsutil_test.py | 66 ++++++++++++++++++++------------------------ 2 files changed, 31 insertions(+), 37 deletions(-) diff --git a/gsutil.py b/gsutil.py index 8d5de5df3..46682c572 100755 --- a/gsutil.py +++ b/gsutil.py @@ -59,7 +59,7 @@ def download_gsutil(version, target_dir): metadata_url = '%s%s' % (API_URL, filename) metadata = json.load(urllib.urlopen(metadata_url)) - remote_md5 = base64.b64decode(metadata['md5Hash']) + remote_md5 = base64.b64decode(metadata['md5Hash']).decode('utf-8') if local_md5 == remote_md5: return target_filename diff --git a/tests/gsutil_test.py b/tests/gsutil_test.py index 888d0dbd0..cd8129104 100755 --- a/tests/gsutil_test.py +++ b/tests/gsutil_test.py @@ -1,14 +1,16 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython3 # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Test gsutil.py.""" +from __future__ import unicode_literals + -import __builtin__ import base64 import hashlib +import io import json import os import shutil @@ -16,9 +18,12 @@ import subprocess import sys import tempfile import unittest -import urllib2 import zipfile +try: + import urllib2 as urllib +except ImportError: # For Py3 compatibility + import urllib.request as urllib # Add depot_tools to path THIS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -32,21 +37,6 @@ class TestError(Exception): pass -class Buffer(object): - def __init__(self, data=None): - self.data = data or '' - - def write(self, buf): - self.data += buf - - def read(self, amount=None): - if not amount: - amount = len(self.data) - result = self.data[:amount] - self.data = self.data[amount:] - return result - - class FakeCall(object): def __init__(self): self.expectations = [] @@ -70,51 +60,55 @@ class GsutilUnitTests(unittest.TestCase): def setUp(self): self.fake = FakeCall() self.tempdir = tempfile.mkdtemp() - self.old_urlopen = getattr(urllib2, 'urlopen') + self.old_urlopen = getattr(urllib, 'urlopen') self.old_call = getattr(subprocess, 'call') - setattr(urllib2, 'urlopen', self.fake) + setattr(urllib, 'urlopen', self.fake) setattr(subprocess, 'call', self.fake) def tearDown(self): self.assertEqual(self.fake.expectations, []) shutil.rmtree(self.tempdir) - setattr(urllib2, 'urlopen', self.old_urlopen) + setattr(urllib, 'urlopen', self.old_urlopen) setattr(subprocess, 'call', self.old_call) def test_download_gsutil(self): version = '4.2' filename = 'gsutil_%s.zip' % version full_filename = os.path.join(self.tempdir, filename) - fake_file = 'This is gsutil.zip' - fake_file2 = 'This is other gsutil.zip' + fake_file = b'This is gsutil.zip' + fake_file2 = b'This is other gsutil.zip' url = '%s%s' % (gsutil.GSUTIL_URL, filename) - self.fake.add_expectation(url, _returns=Buffer(fake_file)) + self.fake.add_expectation(url, _returns=io.BytesIO(fake_file)) self.assertEqual( gsutil.download_gsutil(version, self.tempdir), full_filename) - with open(full_filename, 'r') as f: + with open(full_filename, 'rb') as f: self.assertEqual(fake_file, f.read()) metadata_url = gsutil.API_URL + filename md5_calc = hashlib.md5() md5_calc.update(fake_file) - b64_md5 = base64.b64encode(md5_calc.hexdigest()) - self.fake.add_expectation(metadata_url, _returns=Buffer(json.dumps({ - 'md5Hash': b64_md5 - }))) + b64_md5 = base64.b64encode(md5_calc.hexdigest().encode('utf-8')) + self.fake.add_expectation( + metadata_url, + _returns=io.BytesIO( + json.dumps({'md5Hash': b64_md5.decode('utf-8')}).encode('utf-8'))) self.assertEqual( gsutil.download_gsutil(version, self.tempdir), full_filename) - with open(full_filename, 'r') as f: + with open(full_filename, 'rb') as f: self.assertEqual(fake_file, f.read()) self.assertEqual(self.fake.expectations, []) - self.fake.add_expectation(metadata_url, _returns=Buffer(json.dumps({ - 'md5Hash': base64.b64encode('aaaaaaa') # Bad MD5 - }))) - self.fake.add_expectation(url, _returns=Buffer(fake_file2)) + self.fake.add_expectation( + metadata_url, + _returns=io.BytesIO( + json.dumps({ + 'md5Hash': base64.b64encode(b'aaaaaaa').decode('utf-8') # Bad MD5 + }).encode('utf-8'))) + self.fake.add_expectation(url, _returns=io.BytesIO(fake_file2)) self.assertEqual( gsutil.download_gsutil(version, self.tempdir), full_filename) - with open(full_filename, 'r') as f: + with open(full_filename, 'rb') as f: self.assertEqual(fake_file2, f.read()) self.assertEqual(self.fake.expectations, []) @@ -132,7 +126,7 @@ class GsutilUnitTests(unittest.TestCase): with zipfile.ZipFile(tempzip, 'w') as zf: zf.writestr('gsutil/gsutil', fake_gsutil) with open(tempzip, 'rb') as f: - self.fake.add_expectation(url, _returns=Buffer(f.read())) + self.fake.add_expectation(url, _returns=io.BytesIO(f.read())) # This should write the gsutil_bin with 'Fake gsutil' gsutil.ensure_gsutil(version, self.tempdir, False)