Drop py2 support in gclient and related files

python3 is the only supported version of python in depot_tools.

Bug: 1475402
Change-Id: I17174d7252b5cbf698700333d3cd561c6591d0a1
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4809190
Reviewed-by: Aravind Vasudevan <aravindvasudev@google.com>
Commit-Queue: Gavin Mak <gavinmak@google.com>
Reviewed-by: Josip Sokcevic <sokcevic@chromium.org>
changes/90/4809190/7
Gavin Mak 2 years ago committed by LUCI CQ
parent 5819c303f0
commit 65c49b18b9

@ -7,8 +7,6 @@
# gclient-new-workdir.py [options] <repository> <new_workdir>
#
from __future__ import print_function
import argparse
import os
import shutil

@ -80,11 +80,8 @@
# To specify a target CPU, the variables target_cpu and target_cpu_only
# are available and are analogous to target_os and target_os_only.
from __future__ import print_function
__version__ = '0.7'
import collections
import copy
import json
import logging
@ -96,32 +93,21 @@ import pprint
import re
import sys
import time
try:
import urlparse
except ImportError: # For Py3 compatibility
import urllib.parse as urlparse
import urllib.parse
import detect_host_arch
import fix_encoding
import gclient_eval
import gclient_scm
import gclient_paths
import gclient_scm
import gclient_utils
import git_cache
import metrics
import metrics_utils
from third_party.repo.progress import Progress
import setup_color
import subcommand
import subprocess2
import setup_color
from third_party import six
# TODO(crbug.com/953884): Remove this when python3 migration is done.
if six.PY3:
# pylint: disable=redefined-builtin
basestring = str
from third_party.repo.progress import Progress
DEPOT_TOOLS_DIR = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
@ -148,16 +134,13 @@ def ToGNString(value, allow_dicts = True):
allow_dicts indicates if this function will allow converting dictionaries
to GN scopes. This is only possible at the top level, you can't nest a
GN scope in a list, so this should be set to False for recursive calls."""
if isinstance(value, basestring):
if isinstance(value, str):
if value.find('\n') >= 0:
raise GNException("Trying to print a string with a newline in it.")
return '"' + \
value.replace('\\', '\\\\').replace('"', '\\"').replace('$', '\\$') + \
'"'
if sys.version_info.major == 2 and isinstance(value, unicode):
return ToGNString(value.encode('utf-8'))
if isinstance(value, bool):
if value:
return "true"
@ -176,11 +159,11 @@ class Hook(object):
"""Constructor.
Arguments:
action (list of basestring): argv of the command to run
pattern (basestring regex): noop with git; deprecated
name (basestring): optional name; no effect on operation
cwd (basestring): working directory to use
condition (basestring): condition when to run the hook
action (list of str): argv of the command to run
pattern (str regex): noop with git; deprecated
name (str): optional name; no effect on operation
cwd (str): working directory to use
condition (str): condition when to run the hook
variables (dict): variables for evaluating the condition
"""
self._action = gclient_utils.freeze(action)
@ -314,7 +297,7 @@ class DependencySettings(object):
self._custom_hooks = custom_hooks or []
# Post process the url to remove trailing slashes.
if isinstance(self.url, basestring):
if isinstance(self.url, str):
# urls are sometime incorrectly written as proto://host/path/@rev. Replace
# it to proto://host/path@rev.
self.set_url(self.url.replace('/@', '@'))
@ -479,7 +462,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
self._OverrideUrl()
# This is inherited from WorkItem. We want the URL to be a resource.
if self.url and isinstance(self.url, basestring):
if self.url and isinstance(self.url, str):
# The url is usually given to gclient either as https://blah@123
# or just https://blah. The @123 portion is irrelevant.
self.resources.append(self.url.split('@')[0])
@ -509,7 +492,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
logging.info('Dependency(%s)._OverrideUrl(None) -> None', self._name)
return
if not isinstance(self.url, basestring):
if not isinstance(self.url, str):
raise gclient_utils.Error('Unknown url type')
# self.url is a local path
@ -518,7 +501,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
return
# self.url is a URL
parsed_url = urlparse.urlparse(self.url)
parsed_url = urllib.parse.urlparse(self.url)
if parsed_url[0] or re.match(r'^\w+\@[\w\.-]+\:[\w\/]+', parsed_url[2]):
return
@ -856,7 +839,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
if 'recursedeps' in local_scope:
for ent in local_scope['recursedeps']:
if isinstance(ent, basestring):
if isinstance(ent, str):
self.recursedeps[ent] = self.deps_file
else: # (depname, depsfilename)
self.recursedeps[ent[0]] = ent[1]
@ -1035,8 +1018,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
# Don't enforce this for custom_deps.
if dep.name in self._custom_deps:
continue
if isinstance(dep.url, basestring):
parsed_url = urlparse.urlparse(dep.url)
if isinstance(dep.url, str):
parsed_url = urllib.parse.urlparse(dep.url)
if parsed_url.netloc and parsed_url.netloc not in self._allowed_hosts:
bad_deps.append(dep)
return bad_deps
@ -1258,7 +1241,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
value = variables[arg]
if isinstance(value, gclient_eval.ConstantString):
value = value.value
elif isinstance(value, basestring):
elif isinstance(value, str):
value = gclient_eval.EvaluateCondition(value, variables)
lines.append('%s = %s' % (arg, ToGNString(value)))
@ -2324,8 +2307,8 @@ class CipdDependency(Dependency):
custom_vars, should_process, relative, condition):
package = dep_value['package']
version = dep_value['version']
url = urlparse.urljoin(
cipd_root.service_url, '%s@%s' % (package, version))
url = urllib.parse.urljoin(cipd_root.service_url,
'%s@%s' % (package, version))
super(CipdDependency, self).__init__(
parent=parent,
name=name + ':' + package,

@ -4,24 +4,13 @@
import ast
import collections
from io import StringIO
import logging
import sys
import tokenize
import gclient_utils
from third_party import schema
from third_party import six
if six.PY2:
# We use cStringIO.StringIO because it is equivalent to Py3's io.StringIO.
from cStringIO import StringIO
import collections as collections_abc
else:
from collections import abc as collections_abc
from io import StringIO
# pylint: disable=redefined-builtin
basestring = str
# git_dependencies migration states. Used within the DEPS file to indicate
@ -52,7 +41,7 @@ class ConstantString(object):
return self.value.__hash__()
class _NodeDict(collections_abc.MutableMapping):
class _NodeDict(collections.abc.MutableMapping):
"""Dict-like type that also stores information on AST nodes and tokens."""
def __init__(self, data=None, tokens=None):
self.data = collections.OrderedDict(data or [])
@ -111,53 +100,60 @@ def _NodeDictSchema(dict_schema):
# See https://github.com/keleshev/schema for docs how to configure schema.
_GCLIENT_DEPS_SCHEMA = _NodeDictSchema({
schema.Optional(basestring):
schema.Or(
None,
basestring,
_NodeDictSchema({
# Repo and revision to check out under the path
# (same as if no dict was used).
'url': schema.Or(None, basestring),
# Optional condition string. The dep will only be processed
# if the condition evaluates to True.
schema.Optional('condition'): basestring,
schema.Optional('dep_type', default='git'): basestring,
}),
# CIPD package.
_NodeDictSchema({
'packages': [
_NodeDictSchema({
'package': basestring,
'version': basestring,
})
],
schema.Optional('condition'): basestring,
schema.Optional('dep_type', default='cipd'): basestring,
}),
),
schema.Optional(str):
schema.Or(
None,
str,
_NodeDictSchema({
# Repo and revision to check out under the path
# (same as if no dict was used).
'url':
schema.Or(None, str),
# Optional condition string. The dep will only be processed
# if the condition evaluates to True.
schema.Optional('condition'):
str,
schema.Optional('dep_type', default='git'):
str,
}),
# CIPD package.
_NodeDictSchema({
'packages': [_NodeDictSchema({
'package': str,
'version': str,
})],
schema.Optional('condition'):
str,
schema.Optional('dep_type', default='cipd'):
str,
}),
),
})
_GCLIENT_HOOKS_SCHEMA = [
_NodeDictSchema({
# Hook action: list of command-line arguments to invoke.
'action': [schema.Or(basestring)],
'action': [schema.Or(str)],
# Name of the hook. Doesn't affect operation.
schema.Optional('name'): basestring,
schema.Optional('name'):
str,
# Hook pattern (regex). Originally intended to limit some hooks to run
# only when files matching the pattern have changed. In practice, with
# git, gclient runs all the hooks regardless of this field.
schema.Optional('pattern'): basestring,
schema.Optional('pattern'):
str,
# Working directory where to execute the hook.
schema.Optional('cwd'): basestring,
schema.Optional('cwd'):
str,
# Optional condition string. The hook will only be run
# if the condition evaluates to True.
schema.Optional('condition'): basestring,
schema.Optional('condition'):
str,
})
]
@ -171,7 +167,7 @@ _GCLIENT_SCHEMA = schema.Schema(
# List of host names from which dependencies are allowed (allowlist).
# NOTE: when not present, all hosts are allowed.
# NOTE: scoped to current DEPS file, not recursive.
schema.Optional('allowed_hosts'): [schema.Optional(basestring)],
schema.Optional('allowed_hosts'): [schema.Optional(str)],
# Mapping from paths to repo and revision to check out under that path.
# Applying this mapping to the on-disk checkout is the main purpose
@ -188,21 +184,21 @@ _GCLIENT_SCHEMA = schema.Schema(
# Also see 'target_os'.
schema.Optional('deps_os'):
_NodeDictSchema({
schema.Optional(basestring): _GCLIENT_DEPS_SCHEMA,
schema.Optional(str): _GCLIENT_DEPS_SCHEMA,
}),
# Dependency to get gclient_gn_args* settings from. This allows these
# values to be set in a recursedeps file, rather than requiring that
# they exist in the top-level solution.
schema.Optional('gclient_gn_args_from'):
basestring,
str,
# Path to GN args file to write selected variables.
schema.Optional('gclient_gn_args_file'):
basestring,
str,
# Subset of variables to write to the GN args file (see above).
schema.Optional('gclient_gn_args'): [schema.Optional(basestring)],
schema.Optional('gclient_gn_args'): [schema.Optional(str)],
# Hooks executed after gclient sync (unless suppressed), or explicitly
# on gclient hooks. See _GCLIENT_HOOKS_SCHEMA for details.
@ -212,11 +208,11 @@ _GCLIENT_SCHEMA = schema.Schema(
# Similar to 'hooks', also keyed by OS.
schema.Optional('hooks_os'):
_NodeDictSchema({schema.Optional(basestring): _GCLIENT_HOOKS_SCHEMA}),
_NodeDictSchema({schema.Optional(str): _GCLIENT_HOOKS_SCHEMA}),
# Rules which #includes are allowed in the directory.
# Also see 'skip_child_includes' and 'specific_include_rules'.
schema.Optional('include_rules'): [schema.Optional(basestring)],
schema.Optional('include_rules'): [schema.Optional(str)],
# Optionally discards rules from parent directories, similar to
# "noparent" in OWNERS files. For example, if
@ -237,22 +233,20 @@ _GCLIENT_SCHEMA = schema.Schema(
# Allowlists deps for which recursion should be enabled.
schema.Optional('recursedeps'): [
schema.Optional(
schema.Or(basestring, (basestring, basestring),
[basestring, basestring])),
schema.Optional(schema.Or(str, (str, str), [str, str])),
],
# Blocklists directories for checking 'include_rules'.
schema.Optional('skip_child_includes'): [schema.Optional(basestring)],
schema.Optional('skip_child_includes'): [schema.Optional(str)],
# Mapping from paths to include rules specific for that path.
# See 'include_rules' for more details.
schema.Optional('specific_include_rules'):
_NodeDictSchema({schema.Optional(basestring): [basestring]}),
_NodeDictSchema({schema.Optional(str): [str]}),
# List of additional OS names to consider when selecting dependencies
# from deps_os.
schema.Optional('target_os'): [schema.Optional(basestring)],
schema.Optional('target_os'): [schema.Optional(str)],
# For recursed-upon sub-dependencies, check out their own dependencies
# relative to the parent's path, rather than relative to the .gclient
@ -268,8 +262,8 @@ _GCLIENT_SCHEMA = schema.Schema(
# Variables that can be referenced using Var() - see 'deps'.
schema.Optional('vars'):
_NodeDictSchema({
schema.Optional(basestring):
schema.Or(ConstantString, basestring, bool),
schema.Optional(str):
schema.Or(ConstantString, str, bool),
}),
}))
@ -279,7 +273,7 @@ def _gclient_eval(node_or_string, filename='<unknown>', vars_dict=None):
_allowed_names = {'None': None, 'True': True, 'False': False}
if isinstance(node_or_string, ConstantString):
return node_or_string.value
if isinstance(node_or_string, basestring):
if isinstance(node_or_string, str):
node_or_string = ast.parse(node_or_string, filename=filename, mode='eval')
if isinstance(node_or_string, ast.Expression):
node_or_string = node_or_string.body
@ -338,7 +332,7 @@ def _gclient_eval(node_or_string, filename='<unknown>', vars_dict=None):
filename, getattr(node, 'lineno', '<unknown>')))
arg = _convert(node.args[0])
if not isinstance(arg, basestring):
if not isinstance(arg, str):
raise ValueError(
'Var\'s argument must be a variable name (file %r, line %s)' % (
filename, getattr(node, 'lineno', '<unknown>')))
@ -458,7 +452,7 @@ def _StandardizeDeps(deps_dict, vars_dict):
new_deps_dict = {}
for dep_name, dep_info in deps_dict.items():
dep_name = dep_name.format(**vars_dict)
if not isinstance(dep_info, collections_abc.Mapping):
if not isinstance(dep_info, collections.abc.Mapping):
dep_info = {'url': dep_info}
dep_info.setdefault('dep_type', 'git')
new_deps_dict[dep_name] = dep_info
@ -585,7 +579,7 @@ def EvaluateCondition(condition, variables, referenced_variables=None):
# Allow using "native" types, without wrapping everything in strings.
# Note that schema constraints still apply to variables.
if not isinstance(value, basestring):
if not isinstance(value, str):
return value
# Recursively evaluate the variable reference.
@ -927,11 +921,11 @@ def GetRevision(gclient_dict, dep_name):
if dep is None:
return None
if isinstance(dep, basestring):
if isinstance(dep, str):
_, _, revision = dep.partition('@')
return revision or None
if isinstance(dep, collections_abc.Mapping) and 'url' in dep:
if isinstance(dep, collections.abc.Mapping) and 'url' in dep:
_, _, revision = dep['url'].partition('@')
return revision or None

@ -8,14 +8,13 @@
# particularly for modules that are not builtin (see sys.builtin_modules_names,
# os isn't built in, but it's essential to this file).
from __future__ import print_function
import gclient_utils
import logging
import os
import subprocess2
import sys
import gclient_utils
import subprocess2
def FindGclientRoot(from_dir, filename='.gclient'):
"""Tries to find the gclient root."""
@ -78,8 +77,7 @@ def GetPrimarySolutionPath():
try:
top_dir = subprocess2.check_output(['git', 'rev-parse', '--show-toplevel'],
stderr=subprocess2.DEVNULL)
if sys.version_info.major == 3:
top_dir = top_dir.decode('utf-8', 'replace')
top_dir = top_dir.decode('utf-8', 'replace')
top_dir = os.path.normpath(top_dir.strip())
except subprocess2.CalledProcessError:
pass

@ -4,8 +4,6 @@
"""Gclient-specific SCM-specific operations."""
from __future__ import print_function
import collections
import contextlib
import errno
@ -15,21 +13,16 @@ import os
import platform
import posixpath
import re
import shutil
import sys
import tempfile
import threading
import traceback
try:
import urlparse
except ImportError: # For Py3 compatibility
import urllib.parse as urlparse
import gclient_utils
import gerrit_util
import git_cache
import scm
import shutil
import subprocess2

@ -4,8 +4,6 @@
"""Generic utils."""
from __future__ import print_function
import codecs
import collections
import contextlib
@ -18,6 +16,7 @@ import operator
import os
import pipes
import platform
import queue
import re
import stat
import subprocess
@ -25,18 +24,9 @@ import sys
import tempfile
import threading
import time
import subprocess2
import urllib.parse
if sys.version_info.major == 2:
from cStringIO import StringIO
import collections as collections_abc
import Queue as queue
import urlparse
else:
from collections import abc as collections_abc
from io import StringIO
import queue
import urllib.parse as urlparse
import subprocess2
# Git wrapper retries on a transient error, and some callees do retries too,
@ -57,17 +47,13 @@ THREADED_INDEX_PACK_BLOCKLIST = [
'https://chromium.googlesource.com/chromium/reference_builds/chrome_win.git'
]
"""To support rethrowing exceptions with tracebacks on both Py2 and 3."""
if sys.version_info.major == 2:
# We have to use exec to avoid a SyntaxError in Python 3.
exec("def reraise(typ, value, tb=None):\n raise typ, value, tb\n")
else:
def reraise(typ, value, tb=None):
if value is None:
value = typ()
if value.__traceback__ is not tb:
raise value.with_traceback(tb)
raise value
def reraise(typ, value, tb=None):
"""To support rethrowing exceptions with tracebacks."""
if value is None:
value = typ()
if value.__traceback__ is not tb:
raise value.with_traceback(tb)
raise value
class Error(Exception):
@ -202,10 +188,8 @@ def AskForData(message):
# Windows. Fall back to simple input handling.
pass
# Use this so that it can be mocked in tests on Python 2 and 3.
# Use this so that it can be mocked in tests.
try:
if sys.version_info.major == 2:
return raw_input(message)
return input(message)
except KeyboardInterrupt:
# Hide the exception.
@ -840,7 +824,7 @@ class WorkItem(object):
def __init__(self, name):
# A unique string representing this work item.
self._name = name
self.outbuf = StringIO()
self.outbuf = io.StringIO()
self.start = self.finish = None
self.resources = [] # List of resources this work item requires.
@ -1201,11 +1185,11 @@ def UpgradeToHttps(url):
# relative url and will use http:///foo. Note that it defaults to http://
# for compatibility with naked url like "localhost:8080".
url = 'http://%s' % url
parsed = list(urlparse.urlparse(url))
parsed = list(urllib.parse.urlparse(url))
# Do not automatically upgrade http to https if a port number is provided.
if parsed[0] == 'http' and not re.match(r'^.+?\:\d+$', parsed[1]):
parsed[0] = 'https'
return urlparse.urlunparse(parsed)
return urllib.parse.urlunparse(parsed)
def ParseCodereviewSettingsContent(content):
@ -1310,7 +1294,7 @@ def freeze(obj):
Will raise TypeError if you pass an object which is not hashable.
"""
if isinstance(obj, collections_abc.Mapping):
if isinstance(obj, collections.abc.Mapping):
return FrozenDict((freeze(k), freeze(v)) for k, v in obj.items())
if isinstance(obj, (list, tuple)):
@ -1323,7 +1307,7 @@ def freeze(obj):
return obj
class FrozenDict(collections_abc.Mapping):
class FrozenDict(collections.abc.Mapping):
"""An immutable OrderedDict.
Modified From: http://stackoverflow.com/a/2704866
@ -1337,7 +1321,7 @@ class FrozenDict(collections_abc.Mapping):
operator.xor, (hash(i) for i in enumerate(self._d.items())), 0)
def __eq__(self, other):
if not isinstance(other, collections_abc.Mapping):
if not isinstance(other, collections.abc.Mapping):
return NotImplemented
if self is other:
return True

@ -8,19 +8,16 @@ import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from io import StringIO
from unittest import mock
import gclient_paths
import gclient_utils
import subprocess2
if sys.version_info.major == 2:
from StringIO import StringIO
import mock
else:
from io import StringIO
from unittest import mock
EXCEPTION = subprocess2.CalledProcessError(
128, ['cmd'], 'cwd', 'stdout', 'stderr')

@ -9,33 +9,26 @@
from __future__ import unicode_literals
from subprocess import Popen, PIPE, STDOUT
from io import StringIO
import json
import logging
import os
import re
from subprocess import Popen, PIPE, STDOUT
import sys
import tempfile
import unittest
from unittest import mock
if sys.version_info.major == 2:
from cStringIO import StringIO
import mock
else:
from io import StringIO
from unittest import mock
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from testing_support import fake_repos
from testing_support import test_case_utils
import gclient_scm
import gclient_utils
import git_cache
import subprocess2
from testing_support import fake_repos
from testing_support import test_case_utils
GIT = 'git' if sys.platform != 'win32' else 'git.bat'

@ -15,6 +15,7 @@ import unittest
import gclient_smoketest_base
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT_DIR)
@ -49,8 +50,7 @@ class GClientSmoke(gclient_smoketest_base.GClientSmokeBase):
os.remove(p)
results = self.gclient(cmd)
self.check(('', '', 0), results)
mode = 'r' if sys.version_info.major == 3 else 'rU'
with open(p, mode) as f:
with open(p, 'r') as f:
actual = {}
exec(f.read(), {}, actual)
self.assertEqual(expected, actual)

@ -8,6 +8,7 @@ import os
import re
import sys
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
GCLIENT_PATH = os.path.join(ROOT_DIR, 'gclient')
sys.path.insert(0, ROOT_DIR)
@ -25,7 +26,6 @@ class GClientSmokeBase(fake_repos.FakeReposTestBase):
self.env['DEPOT_TOOLS_METRICS'] = '0'
# Suppress Python 3 warnings and other test undesirables.
self.env['GCLIENT_TEST'] = '1'
self.env['GCLIENT_PY3'] = '0' if sys.version_info.major == 2 else '1'
self.maxDiff = None
def gclient(self, cmd, cwd=None, error_ok=False):

@ -8,34 +8,33 @@
See gclient_smoketest.py for integration tests.
"""
import copy
import json
import logging
import ntpath
import os
import queue
import sys
import six
import unittest
from unittest import mock
import six
if sys.version_info.major == 2:
import mock
import Queue
else:
from unittest import mock
import queue as Queue
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import metrics_utils
# We have to disable monitoring before importing gclient.
metrics_utils.COLLECT_METRICS = False
import gclient
import gclient_eval
import gclient_utils
import gclient_scm
import gclient_utils
from testing_support import trial_dir
def write(filename, content):
"""Writes the content of a file and create the directories as needed."""
filename = os.path.abspath(filename)
@ -84,7 +83,7 @@ class SCMMock(object):
class GclientTest(trial_dir.TestCase):
def setUp(self):
super(GclientTest, self).setUp()
self.processed = Queue.Queue()
self.processed = queue.Queue()
self.previous_dir = os.getcwd()
os.chdir(self.root_dir)
# Manual mocks.
@ -216,7 +215,7 @@ class GclientTest(trial_dir.TestCase):
try:
while True:
items.append(self.processed.get_nowait())
except Queue.Empty:
except queue.Empty:
pass
return items

@ -4,29 +4,18 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from __future__ import print_function
from __future__ import unicode_literals
import io
import os
import sys
import tempfile
import time
import unittest
from unittest import mock
if sys.version_info.major == 2:
from StringIO import StringIO
import mock
else:
from io import StringIO
from unittest import mock
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from testing_support import trial_dir
import gclient_utils
import subprocess2
from testing_support import trial_dir
class CheckCallAndFilterTestCase(unittest.TestCase):

Loading…
Cancel
Save