repo: pull in v1.13.4 changes

This is mostly for Python 3 improvements.

Bug: gerrit:10418
Change-Id: Ice96b8dc5f700015ae6ec53c0e5b86700bcf6532
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1734587
Auto-Submit: Mike Frysinger <vapier@chromium.org>
Reviewed-by: Alex Klein <saklein@chromium.org>
Commit-Queue: Mike Frysinger <vapier@chromium.org>

repo

@@ -1,4 +1,14 @@
 #!/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""Repo launcher.
+
+This is a standalone tool that people may copy to anywhere in their system.
+It is used to get an initial repo client checkout, and after that it runs the
+copy of repo in the checkout.
+"""
+
+from __future__ import print_function
 
 # repo default configuration
 #
@@ -245,6 +255,7 @@ GITC_CONFIG_FILE = '/gitc/.config'
 GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
 
 
+import collections
 import errno
 import optparse
 import platform
@@ -265,22 +276,15 @@ else:
   urllib.error = urllib2
 
 
-def _print(*objects, **kwargs):
-  sep = kwargs.get('sep', ' ')
-  end = kwargs.get('end', '\n')
-  out = kwargs.get('file', sys.stdout)
-  out.write(sep.join(objects) + end)
-
-
 # Python version check
 ver = sys.version_info
 if (ver[0], ver[1]) < MIN_PYTHON_VERSION:
-  _print('error: Python version {} unsupported.\n'
-         'Please use Python {}.{} instead.'.format(
-             sys.version.split(' ')[0],
-             MIN_PYTHON_VERSION[0],
-             MIN_PYTHON_VERSION[1],
-         ), file=sys.stderr)
+  print('error: Python version {} unsupported.\n'
+        'Please use Python {}.{} instead.'.format(
+            sys.version.split(' ')[0],
+            MIN_PYTHON_VERSION[0],
+            MIN_PYTHON_VERSION[1],
+        ), file=sys.stderr)
   sys.exit(1)
 
 home_dot_repo = os.path.expanduser('~/.repoconfig')
@@ -316,9 +320,19 @@ group.add_option('--mirror',
 group.add_option('--reference',
                  dest='reference',
                  help='location of mirror directory', metavar='DIR')
+group.add_option('--dissociate',
+                 dest='dissociate', action='store_true',
+                 help='dissociate from reference mirrors after clone')
 group.add_option('--depth', type='int', default=None,
                  dest='depth',
                  help='create a shallow clone with given depth; see git clone')
+group.add_option('--partial-clone', action='store_true',
+                 dest='partial_clone',
+                 help='perform partial clone (https://git-scm.com/'
+                 'docs/gitrepository-layout#_code_partialclone_code)')
+group.add_option('--clone-filter', action='store', default='blob:none',
+                 dest='clone_filter',
+                 help='filter for use with --partial-clone [default: %default]')
 group.add_option('--archive',
                  dest='archive', action='store_true',
                  help='checkout an archive instead of a git repository for '
@@ -443,21 +457,21 @@ def _Init(args, gitc_init=False):
   if branch.startswith('refs/heads/'):
     branch = branch[len('refs/heads/'):]
   if branch.startswith('refs/'):
-    _print("fatal: invalid branch name '%s'" % branch, file=sys.stderr)
+    print("fatal: invalid branch name '%s'" % branch, file=sys.stderr)
     raise CloneFailure()
 
   try:
     if gitc_init:
       gitc_manifest_dir = get_gitc_manifest_dir()
       if not gitc_manifest_dir:
-        _print('fatal: GITC filesystem is not available. Exiting...',
-               file=sys.stderr)
+        print('fatal: GITC filesystem is not available. Exiting...',
+              file=sys.stderr)
         sys.exit(1)
       gitc_client = opt.gitc_client
       if not gitc_client:
         gitc_client = gitc_parse_clientdir(os.getcwd())
       if not gitc_client:
-        _print('fatal: GITC client (-c) is required.', file=sys.stderr)
+        print('fatal: GITC client (-c) is required.', file=sys.stderr)
         sys.exit(1)
       client_dir = os.path.join(gitc_manifest_dir, gitc_client)
       if not os.path.exists(client_dir):
@@ -470,8 +484,8 @@ def _Init(args, gitc_init=False):
     os.mkdir(repodir)
   except OSError as e:
     if e.errno != errno.EEXIST:
-      _print('fatal: cannot make %s directory: %s'
-             % (repodir, e.strerror), file=sys.stderr)
+      print('fatal: cannot make %s directory: %s'
+            % (repodir, e.strerror), file=sys.stderr)
       # Don't raise CloneFailure; that would delete the
       # name. Instead exit immediately.
       #
@@ -487,62 +501,81 @@ def _Init(args, gitc_init=False):
     dst = os.path.abspath(os.path.join(repodir, S_repo))
     _Clone(url, dst, opt.quiet, not opt.no_clone_bundle)
 
-    if not os.path.isfile('%s/repo' % dst):
-      _print("warning: '%s' does not look like a git-repo repository, is "
-             "REPO_URL set correctly?" % url, file=sys.stderr)
-
     if can_verify and not opt.no_repo_verify:
       rev = _Verify(dst, branch, opt.quiet)
     else:
       rev = 'refs/remotes/origin/%s^0' % branch
 
     _Checkout(dst, branch, rev, opt.quiet)
+
+    if not os.path.isfile(os.path.join(dst, 'repo')):
+      print("warning: '%s' does not look like a git-repo repository, is "
+            "REPO_URL set correctly?" % url, file=sys.stderr)
+
   except CloneFailure:
     if opt.quiet:
-      _print('fatal: repo init failed; run without --quiet to see why',
-             file=sys.stderr)
+      print('fatal: repo init failed; run without --quiet to see why',
+            file=sys.stderr)
     raise
 
 
-def ParseGitVersion(ver_str):
+# The git version info broken down into components for easy analysis.
+# Similar to Python's sys.version_info.
+GitVersion = collections.namedtuple(
+    'GitVersion', ('major', 'minor', 'micro', 'full'))
+
+def ParseGitVersion(ver_str=None):
+  if ver_str is None:
+    # Load the version ourselves.
+    ver_str = _GetGitVersion()
+
   if not ver_str.startswith('git version '):
     return None
 
-  num_ver_str = ver_str[len('git version '):].strip().split('-')[0]
+  full_version = ver_str[len('git version '):].strip()
+  num_ver_str = full_version.split('-')[0]
   to_tuple = []
   for num_str in num_ver_str.split('.')[:3]:
     if num_str.isdigit():
       to_tuple.append(int(num_str))
     else:
       to_tuple.append(0)
-  return tuple(to_tuple)
+  to_tuple.append(full_version)
+  return GitVersion(*to_tuple)
 
 
-def _CheckGitVersion():
+def _GetGitVersion():
   cmd = [GIT, '--version']
   try:
     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
   except OSError as e:
-    _print(file=sys.stderr)
-    _print("fatal: '%s' is not available" % GIT, file=sys.stderr)
-    _print('fatal: %s' % e, file=sys.stderr)
-    _print(file=sys.stderr)
-    _print('Please make sure %s is installed and in your path.' % GIT,
-           file=sys.stderr)
-    raise CloneFailure()
+    print(file=sys.stderr)
+    print("fatal: '%s' is not available" % GIT, file=sys.stderr)
+    print('fatal: %s' % e, file=sys.stderr)
+    print(file=sys.stderr)
+    print('Please make sure %s is installed and in your path.' % GIT,
+          file=sys.stderr)
+    raise
 
   ver_str = proc.stdout.read().strip()
   proc.stdout.close()
   proc.wait()
+  return ver_str.decode('utf-8')
+
+
+def _CheckGitVersion():
+  try:
+    ver_act = ParseGitVersion()
+  except OSError:
+    raise CloneFailure()
 
-  ver_act = ParseGitVersion(ver_str)
   if ver_act is None:
-    _print('error: "%s" unsupported' % ver_str, file=sys.stderr)
+    print('error: "%s" unsupported' % ver_str, file=sys.stderr)
     raise CloneFailure()
 
   if ver_act < MIN_GIT_VERSION:
     need = '.'.join(map(str, MIN_GIT_VERSION))
-    _print('fatal: git %s or later required' % need, file=sys.stderr)
+    print('fatal: git %s or later required' % need, file=sys.stderr)
     raise CloneFailure()
 
 
@@ -569,16 +602,16 @@ def SetupGnuPG(quiet):
     os.mkdir(home_dot_repo)
   except OSError as e:
     if e.errno != errno.EEXIST:
-      _print('fatal: cannot make %s directory: %s'
-             % (home_dot_repo, e.strerror), file=sys.stderr)
+      print('fatal: cannot make %s directory: %s'
+            % (home_dot_repo, e.strerror), file=sys.stderr)
       sys.exit(1)
 
   try:
     os.mkdir(gpg_dir, stat.S_IRWXU)
   except OSError as e:
     if e.errno != errno.EEXIST:
-      _print('fatal: cannot make %s directory: %s' % (gpg_dir, e.strerror),
-             file=sys.stderr)
+      print('fatal: cannot make %s directory: %s' % (gpg_dir, e.strerror),
+            file=sys.stderr)
       sys.exit(1)
 
   env = os.environ.copy()
@@ -594,18 +627,18 @@ def SetupGnuPG(quiet):
                             stdin=subprocess.PIPE)
   except OSError as e:
     if not quiet:
-      _print('warning: gpg (GnuPG) is not available.', file=sys.stderr)
-      _print('warning: Installing it is strongly encouraged.', file=sys.stderr)
-      _print(file=sys.stderr)
+      print('warning: gpg (GnuPG) is not available.', file=sys.stderr)
+      print('warning: Installing it is strongly encouraged.', file=sys.stderr)
+      print(file=sys.stderr)
     return False
 
   proc.stdin.write(MAINTAINER_KEYS)
   proc.stdin.close()
 
   if proc.wait() != 0:
-    _print('fatal: registering repo maintainer keys failed', file=sys.stderr)
+    print('fatal: registering repo maintainer keys failed', file=sys.stderr)
     sys.exit(1)
-  _print()
+  print()
 
   fd = open(os.path.join(home_dot_repo, 'keyring-version'), 'w')
   fd.write('.'.join(map(str, KEYRING_VERSION)) + '\n')
@@ -632,7 +665,7 @@ def _InitHttp():
       p = n.hosts[host]
       mgr.add_password(p[1], 'http://%s/' % host, p[0], p[2])
       mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
-  except:  # pylint: disable=bare-except
+  except:
     pass
   handlers.append(urllib.request.HTTPBasicAuthHandler(mgr))
   handlers.append(urllib.request.HTTPDigestAuthHandler(mgr))
@@ -648,7 +681,7 @@ def _InitHttp():
 
 def _Fetch(url, local, src, quiet):
   if not quiet:
-    _print('Get %s' % url, file=sys.stderr)
+    print('Get %s' % url, file=sys.stderr)
 
   cmd = [GIT, 'fetch']
   if quiet:
@@ -658,7 +691,7 @@ def _Fetch(url, local, src, quiet):
     err = None
   cmd.append(src)
   cmd.append('+refs/heads/*:refs/remotes/origin/*')
-  cmd.append('refs/tags/*:refs/tags/*')
+  cmd.append('+refs/tags/*:refs/tags/*')
 
   proc = subprocess.Popen(cmd, cwd=local, stderr=err)
   if err:
@@ -698,19 +731,19 @@ def _DownloadBundle(url, local, quiet):
     except urllib.error.HTTPError as e:
       if e.code in [401, 403, 404, 501]:
         return False
-      _print('fatal: Cannot get %s' % url, file=sys.stderr)
-      _print('fatal: HTTP error %s' % e.code, file=sys.stderr)
+      print('fatal: Cannot get %s' % url, file=sys.stderr)
+      print('fatal: HTTP error %s' % e.code, file=sys.stderr)
       raise CloneFailure()
     except urllib.error.URLError as e:
-      _print('fatal: Cannot get %s' % url, file=sys.stderr)
-      _print('fatal: error %s' % e.reason, file=sys.stderr)
+      print('fatal: Cannot get %s' % url, file=sys.stderr)
+      print('fatal: error %s' % e.reason, file=sys.stderr)
       raise CloneFailure()
     try:
       if not quiet:
-        _print('Get %s' % url, file=sys.stderr)
+        print('Get %s' % url, file=sys.stderr)
       while True:
         buf = r.read(8192)
-        if buf == '':
+        if not buf:
           return True
         dest.write(buf)
     finally:
@@ -733,23 +766,23 @@ def _Clone(url, local, quiet, clone_bundle):
   try:
     os.mkdir(local)
   except OSError as e:
-    _print('fatal: cannot make %s directory: %s' % (local, e.strerror),
-           file=sys.stderr)
+    print('fatal: cannot make %s directory: %s' % (local, e.strerror),
+          file=sys.stderr)
     raise CloneFailure()
 
   cmd = [GIT, 'init', '--quiet']
   try:
     proc = subprocess.Popen(cmd, cwd=local)
   except OSError as e:
-    _print(file=sys.stderr)
-    _print("fatal: '%s' is not available" % GIT, file=sys.stderr)
-    _print('fatal: %s' % e, file=sys.stderr)
-    _print(file=sys.stderr)
-    _print('Please make sure %s is installed and in your path.' % GIT,
-           file=sys.stderr)
+    print(file=sys.stderr)
+    print("fatal: '%s' is not available" % GIT, file=sys.stderr)
+    print('fatal: %s' % e, file=sys.stderr)
+    print(file=sys.stderr)
+    print('Please make sure %s is installed and in your path.' % GIT,
+          file=sys.stderr)
     raise CloneFailure()
   if proc.wait() != 0:
-    _print('fatal: could not create %s' % local, file=sys.stderr)
+    print('fatal: could not create %s' % local, file=sys.stderr)
     raise CloneFailure()
 
   _InitHttp()
@@ -777,18 +810,18 @@ def _Verify(cwd, branch, quiet):
   proc.stderr.close()
 
   if proc.wait() != 0 or not cur:
-    _print(file=sys.stderr)
-    _print("fatal: branch '%s' has not been signed" % branch, file=sys.stderr)
+    print(file=sys.stderr)
+    print("fatal: branch '%s' has not been signed" % branch, file=sys.stderr)
     raise CloneFailure()
 
   m = re.compile(r'^(.*)-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur)
   if m:
     cur = m.group(1)
     if not quiet:
-      _print(file=sys.stderr)
-      _print("info: Ignoring branch '%s'; using tagged release '%s'"
-             % (branch, cur), file=sys.stderr)
-      _print(file=sys.stderr)
+      print(file=sys.stderr)
+      print("info: Ignoring branch '%s'; using tagged release '%s'"
+            % (branch, cur), file=sys.stderr)
+      print(file=sys.stderr)
 
   env = os.environ.copy()
   try:
@@ -809,10 +842,10 @@ def _Verify(cwd, branch, quiet):
   proc.stderr.close()
 
   if proc.wait() != 0:
-    _print(file=sys.stderr)
-    _print(out, file=sys.stderr)
-    _print(err, file=sys.stderr)
-    _print(file=sys.stderr)
+    print(file=sys.stderr)
+    print(out, file=sys.stderr)
+    print(err, file=sys.stderr)
+    print(file=sys.stderr)
     raise CloneFailure()
   return '%s^0' % cur
 
@@ -883,7 +916,7 @@ def _Usage():
   if get_gitc_manifest_dir():
     gitc_usage = "  gitc-init Initialize a GITC Client.\n"
 
-  _print(
+  print(
       """usage: repo COMMAND [ARGS]
 
 repo is not yet installed.  Use "repo init" to install it here.
@@ -895,8 +928,8 @@ The most commonly used repo commands are:
       """  help      Display detailed help on a command
 
 For access to the full online help, install repo ("repo init").
-""", file=sys.stderr)
-  sys.exit(1)
+""")
+  sys.exit(0)
 
 
 def _Help(args):
@@ -909,23 +942,23 @@ def _Help(args):
       init_optparse.print_help()
       sys.exit(0)
     else:
-      _print("error: '%s' is not a bootstrap command.\n"
-             '        For access to online help, install repo ("repo init").'
-             % args[0], file=sys.stderr)
+      print("error: '%s' is not a bootstrap command.\n"
+            '        For access to online help, install repo ("repo init").'
+            % args[0], file=sys.stderr)
   else:
     _Usage()
   sys.exit(1)
 
 
 def _NotInstalled():
-  _print('error: repo is not installed.  Use "repo init" to install it here.',
-         file=sys.stderr)
+  print('error: repo is not installed.  Use "repo init" to install it here.',
+        file=sys.stderr)
   sys.exit(1)
 
 
 def _NoCommands(cmd):
-  _print("""error: command '%s' requires repo to be installed first.
-         Use "repo init" to install it here.""" % cmd, file=sys.stderr)
+  print("""error: command '%s' requires repo to be installed first.
+        Use "repo init" to install it here.""" % cmd, file=sys.stderr)
   sys.exit(1)
 
 
@@ -962,7 +995,7 @@ def _SetDefaultsTo(gitdir):
   proc.stderr.close()
 
   if proc.wait() != 0:
-    _print('fatal: %s has no current branch' % gitdir, file=sys.stderr)
+    print('fatal: %s has no current branch' % gitdir, file=sys.stderr)
     sys.exit(1)
 
 
@@ -979,10 +1012,10 @@ def main(orig_args):
 
   cwd = os.getcwd()
   if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
-    _print('error: repo cannot be used in the GITC local manifest directory.'
-           '\nIf you want to work on this GITC client please rerun this '
-           'command from the corresponding client under /gitc/',
-           file=sys.stderr)
+    print('error: repo cannot be used in the GITC local manifest directory.'
+          '\nIf you want to work on this GITC client please rerun this '
+          'command from the corresponding client under /gitc/',
+          file=sys.stderr)
     sys.exit(1)
   if not repo_main:
     if opt.help:
@@ -998,8 +1031,8 @@ def main(orig_args):
         _Init(args, gitc_init=(cmd == 'gitc-init'))
       except CloneFailure:
         path = os.path.join(repodir, S_repo)
-        _print("fatal: cloning the git-repo repository failed, will remove "
-               "'%s' " % path, file=sys.stderr)
+        print("fatal: cloning the git-repo repository failed, will remove "
+              "'%s' " % path, file=sys.stderr)
         shutil.rmtree(path, ignore_errors=True)
         sys.exit(1)
       repo_main, rel_repo_dir = _FindRepo()
@@ -1023,14 +1056,10 @@ def main(orig_args):
     else:
       os.execv(sys.executable, me)
   except OSError as e:
-    _print("fatal: unable to start %s" % repo_main, file=sys.stderr)
-    _print("fatal: %s" % e, file=sys.stderr)
+    print("fatal: unable to start %s" % repo_main, file=sys.stderr)
+    print("fatal: %s" % e, file=sys.stderr)
     sys.exit(148)
 
 
 if __name__ == '__main__':
-  if ver[0] == 3:
-    _print('warning: Python 3 support is currently experimental. YMMV.\n'
-           'Please use Python 2.7 instead.',
-           file=sys.stderr)
   main(sys.argv[1:])