From fe0d1902b3cb59799c0290e62ece72720ab6935b Mon Sep 17 00:00:00 2001 From: "szager@chromium.org" Date: Tue, 8 Apr 2014 20:50:44 +0000 Subject: [PATCH] Revamped terminal output for update. Features: - Non-verbose output is now limited to a one-line progress indicator. - Verbose output is now collated per subprocess. As soon as a subprocess finishes, its full output is dumped to terminal. - Verbose output is prefixed with timestamps representing elapsed time since the beginning of the gclient invocation. - git progress indicators ("Receiving objects", etc.) are limited to one line every 10 seconds. - In both verbose and non-verbose mode, if a failure occurs, the full output of the failed update operation is dumped to terminal just before exit. - In the event that updates are progressing, but slowly, "Still working" messages will be printed periodically, to pacify users and buildbots. BUG= R=hinoka@google.com Review URL: https://codereview.chromium.org/227163002 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@262500 0039d316-1c4b-4281-b951-d872f2087c98 --- gclient.py | 29 +++-- gclient_scm.py | 215 +++++++++++++++++++---------------- gclient_utils.py | 122 ++++++++++++++++---- tests/gclient_scm_test.py | 39 +++++-- tests/gclient_smoketest.py | 71 +++++------- tests/gclient_test.py | 2 +- third_party/repo/progress.py | 14 ++- 7 files changed, 303 insertions(+), 189 deletions(-) diff --git a/gclient.py b/gclient.py index 1b5d8f4a39..5fa3565405 100755 --- a/gclient.py +++ b/gclient.py @@ -444,7 +444,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): parent_url = self.parent.parsed_url if isinstance(parent_url, self.FileImpl): parent_url = parent_url.file_location - scm = gclient_scm.CreateSCM(parent_url, self.root.root_dir, None) + scm = gclient_scm.CreateSCM( + parent_url, self.root.root_dir, None, self.outbuf) parsed_url = scm.FullUrlForRelativeUrl(url) else: parsed_url = url @@ -657,7 +658,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): # pylint: disable=E1103 options.revision = parsed_url.GetRevision() self._used_scm = gclient_scm.SVNWrapper( - parsed_url.GetPath(), self.root.root_dir, self.name) + parsed_url.GetPath(), self.root.root_dir, self.name, + out_cb=work_queue.out_cb) self._used_scm.RunCommand('updatesingle', options, args + [parsed_url.GetFilename()], file_list) else: @@ -667,7 +669,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): self.maybeGetParentRevision( command, options, parsed_url, self.parent.name, revision_overrides) self._used_scm = gclient_scm.CreateSCM( - parsed_url, self.root.root_dir, self.name) + parsed_url, self.root.root_dir, self.name, self.outbuf, + out_cb=work_queue.out_cb) self._got_revision = self._used_scm.RunCommand(command, options, args, file_list) if file_list: @@ -724,7 +727,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): match = re.match('^Binary file ([^\0]+) matches$', line) if match: - print 'Binary file %s matches' % mod_path(match.group(1)) + print 'Binary file %s matches\n' % mod_path(match.group(1)) return items = line.split('\0') @@ -1050,7 +1053,8 @@ solutions = [ solutions.""" for dep in self.dependencies: if dep.managed and dep.url: - scm = gclient_scm.CreateSCM(dep.url, self.root_dir, dep.name) + scm = gclient_scm.CreateSCM( + dep.url, self.root_dir, dep.name, self.outbuf) actual_url = scm.GetActualRemoteURL(self._options) if actual_url and not scm.DoesRemoteURLMatch(self._options): raise gclient_utils.Error(''' @@ -1234,7 +1238,8 @@ want to set 'managed': False in .gclient. 'It appears your safesync_url (%s) is not working properly\n' '(as it returned an empty response). Check your config.' % dep.safesync_url) - scm = gclient_scm.CreateSCM(dep.url, dep.root.root_dir, dep.name) + scm = gclient_scm.CreateSCM( + dep.url, dep.root.root_dir, dep.name, self.outbuf) safe_rev = scm.GetUsableRev(rev, self._options) if self._options.verbose: print('Using safesync_url revision: %s.\n' % safe_rev) @@ -1265,7 +1270,8 @@ want to set 'managed': False in .gclient. elif command == 'recurse': pm = Progress(' '.join(args), 1) work_queue = gclient_utils.ExecutionQueue( - self._options.jobs, pm, ignore_requirements=ignore_requirements) + self._options.jobs, pm, ignore_requirements=ignore_requirements, + verbose=self._options.verbose) for s in self.dependencies: work_queue.enqueue(s) work_queue.flush(revision_overrides, command, args, options=self._options) @@ -1301,7 +1307,8 @@ want to set 'managed': False in .gclient. if (entry not in entries and (not any(path.startswith(entry + '/') for path in entries)) and os.path.exists(e_dir)): - scm = gclient_scm.CreateSCM(prev_url, self.root_dir, entry_fixed) + scm = gclient_scm.CreateSCM( + prev_url, self.root_dir, entry_fixed, self.outbuf) # Check to see if this directory is now part of a higher-up checkout. if scm.GetCheckoutRoot() in full_entries: @@ -1332,7 +1339,8 @@ want to set 'managed': False in .gclient. if not self.dependencies: raise gclient_utils.Error('No solution specified') # Load all the settings. - work_queue = gclient_utils.ExecutionQueue(self._options.jobs, None, False) + work_queue = gclient_utils.ExecutionQueue( + self._options.jobs, None, False, verbose=self._options.verbose) for s in self.dependencies: work_queue.enqueue(s) work_queue.flush({}, None, [], options=self._options) @@ -1346,7 +1354,8 @@ want to set 'managed': False in .gclient. else: original_url = dep.parsed_url url, _ = gclient_utils.SplitUrlRevision(original_url) - scm = gclient_scm.CreateSCM(original_url, self.root_dir, dep.name) + scm = gclient_scm.CreateSCM( + original_url, self.root_dir, dep.name, self.outbuf) if not os.path.isdir(scm.checkout_path): return None return '%s@%s' % (url, scm.revinfo(self._options, [], None)) diff --git a/gclient_scm.py b/gclient_scm.py index 91f03b0277..29684f458b 100644 --- a/gclient_scm.py +++ b/gclient_scm.py @@ -4,6 +4,8 @@ """Gclient-specific SCM-specific operations.""" +from __future__ import print_function + import logging import os import posixpath @@ -35,11 +37,12 @@ class DiffFiltererWrapper(object): original_prefix = "--- " working_prefix = "+++ " - def __init__(self, relpath): + def __init__(self, relpath, print_func): # Note that we always use '/' as the path separator to be # consistent with svn's cygwin-style output on Windows self._relpath = relpath.replace("\\", "/") self._current_file = None + self._print_func = print_func def SetCurrentFile(self, current_file): self._current_file = current_file @@ -59,7 +62,7 @@ class DiffFiltererWrapper(object): if (line.startswith(self.original_prefix) or line.startswith(self.working_prefix)): line = self._Replace(line) - print(line) + self._print_func(line) class SvnDiffFilterer(DiffFiltererWrapper): @@ -94,7 +97,7 @@ def GetScmName(url): return None -def CreateSCM(url, root_dir=None, relpath=None): +def CreateSCM(url, root_dir=None, relpath=None, out_fh=None, out_cb=None): SCM_MAP = { 'svn' : SVNWrapper, 'git' : GitWrapper, @@ -106,7 +109,7 @@ def CreateSCM(url, root_dir=None, relpath=None): scm_class = SCM_MAP[scm_name] if not scm_class.BinaryExists(): raise gclient_utils.Error('%s command not found' % scm_name) - return scm_class(url, root_dir, relpath) + return scm_class(url, root_dir, relpath, out_fh, out_cb) # SCMWrapper base class @@ -116,7 +119,9 @@ class SCMWrapper(object): This is the abstraction layer to bind to different SCM. """ - def __init__(self, url=None, root_dir=None, relpath=None): + + def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None, + out_cb=None): self.url = url self._root_dir = root_dir if self._root_dir: @@ -126,6 +131,16 @@ class SCMWrapper(object): self.relpath = self.relpath.replace('/', os.sep) if self.relpath and self._root_dir: self.checkout_path = os.path.join(self._root_dir, self.relpath) + if out_fh is None: + out_fh = sys.stdout + self.out_fh = out_fh + self.out_cb = out_cb + + def Print(self, *args, **kwargs): + kwargs.setdefault('file', self.out_fh) + if kwargs.pop('timestamp', True): + self.out_fh.write('[%s] ' % gclient_utils.Elapsed()) + print(*args, **kwargs) def RunCommand(self, command, options, args, file_list=None): commands = ['cleanup', 'update', 'updatesingle', 'revert', @@ -191,11 +206,12 @@ class GitWrapper(SCMWrapper): cache_dir = None - def __init__(self, url=None, root_dir=None, relpath=None): + def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None, + out_cb=None): """Removes 'git+' fake prefix from git URL.""" if url.startswith('git+http://') or url.startswith('git+https://'): url = url[4:] - SCMWrapper.__init__(self, url, root_dir, relpath) + SCMWrapper.__init__(self, url, root_dir, relpath, out_fh, out_cb) @staticmethod def BinaryExists(): @@ -241,7 +257,7 @@ class GitWrapper(SCMWrapper): gclient_utils.CheckCallAndFilter( ['git', 'diff', merge_base], cwd=self.checkout_path, - filter_fn=GitDiffFilterer(self.relpath).Filter) + filter_fn=GitDiffFilterer(self.relpath).Filter, print_func=self.Print) def UpdateSubmoduleConfig(self): submod_cmd = ['git', 'config', '-f', '$toplevel/.git/config', @@ -316,7 +332,7 @@ class GitWrapper(SCMWrapper): # expired yet. Use rev-list to get the corresponding revision. # git rev-list -n 1 --before='time-stamp' branchname if options.transitive: - print('Warning: --transitive only works for SVN repositories.') + self.Print('Warning: --transitive only works for SVN repositories.') revision = default_rev rev_str = ' at %s' % revision @@ -325,7 +341,7 @@ class GitWrapper(SCMWrapper): printed_path = False verbose = [] if options.verbose: - print('\n_____ %s%s' % (self.relpath, rev_str)) + self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False) verbose = ['--verbose'] printed_path = True @@ -352,13 +368,13 @@ class GitWrapper(SCMWrapper): if not verbose: # Make the output a little prettier. It's nice to have some whitespace # between projects when cloning. - print('') + self.Print('') return self._Capture(['rev-parse', '--verify', 'HEAD']) if not managed: self._UpdateBranchHeads(options, fetch=False) self.UpdateSubmoduleConfig() - print ('________ unmanaged solution; skipping %s' % self.relpath) + self.Print('________ unmanaged solution; skipping %s' % self.relpath) return self._Capture(['rev-parse', '--verify', 'HEAD']) if not os.path.exists(os.path.join(self.checkout_path, '.git')): @@ -383,7 +399,7 @@ class GitWrapper(SCMWrapper): subprocess2.capture( ['git', 'config', 'remote.%s.gclient-auto-fix-url' % self.remote], cwd=self.checkout_path).strip() != 'False'): - print('_____ switching %s to a new upstream' % self.relpath) + self.Print('_____ switching %s to a new upstream' % self.relpath) # Make sure it's clean self._CheckClean(rev_str) # Switch over to the new upstream @@ -436,7 +452,7 @@ class GitWrapper(SCMWrapper): remote_output = scm.GIT.Capture(['remote'] + verbose + ['update'], cwd=self.checkout_path) if verbose: - print(remote_output) + self.Print(remote_output) self._UpdateBranchHeads(options, fetch=True) @@ -453,7 +469,7 @@ class GitWrapper(SCMWrapper): self._CheckDetachedHead(rev_str, options) self._Capture(['checkout', '--quiet', '%s' % revision]) if not printed_path: - print('\n_____ %s%s' % (self.relpath, rev_str)) + self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False) elif current_type == 'hash': # case 1 if scm.GIT.IsGitSvn(self.checkout_path) and upstream_branch is not None: @@ -482,7 +498,7 @@ class GitWrapper(SCMWrapper): # case 4 new_base = revision.replace('heads', 'remotes/' + self.remote) if not printed_path: - print('\n_____ %s%s' % (self.relpath, rev_str)) + self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False) switch_error = ("Switching upstream branch from %s to %s\n" % (upstream_branch, new_base) + "Please merge or rebase manually:\n" + @@ -494,7 +510,7 @@ class GitWrapper(SCMWrapper): if files is not None: files = self._Capture(['diff', upstream_branch, '--name-only']).split() if verbose: - print('Trying fast-forward merge to branch : %s' % upstream_branch) + self.Print('Trying fast-forward merge to branch : %s' % upstream_branch) try: merge_args = ['merge'] if options.merge: @@ -502,12 +518,12 @@ class GitWrapper(SCMWrapper): else: merge_args.append('--ff-only') merge_args.append(upstream_branch) - merge_output = scm.GIT.Capture(merge_args, cwd=self.checkout_path) + merge_output = self._Capture(merge_args) except subprocess2.CalledProcessError as e: if re.match('fatal: Not possible to fast-forward, aborting.', e.stderr): files = [] if not printed_path: - print('\n_____ %s%s' % (self.relpath, rev_str)) + self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False) printed_path = True while True: try: @@ -529,34 +545,34 @@ class GitWrapper(SCMWrapper): "cd %s && git " % self.checkout_path + "rebase %s" % upstream_branch) elif re.match(r'skip|s', action, re.I): - print('Skipping %s' % self.relpath) + self.Print('Skipping %s' % self.relpath) return else: - print('Input not recognized') + self.Print('Input not recognized') elif re.match("error: Your local changes to '.*' would be " "overwritten by merge. Aborting.\nPlease, commit your " "changes or stash them before you can merge.\n", e.stderr): if not printed_path: - print('\n_____ %s%s' % (self.relpath, rev_str)) + self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False) printed_path = True raise gclient_utils.Error(e.stderr) else: # Some other problem happened with the merge logging.error("Error during fast-forward merge in %s!" % self.relpath) - print(e.stderr) + self.Print(e.stderr) raise else: # Fast-forward merge was successful if not re.match('Already up-to-date.', merge_output) or verbose: if not printed_path: - print('\n_____ %s%s' % (self.relpath, rev_str)) + self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False) printed_path = True - print(merge_output.strip()) + self.Print(merge_output.strip()) if not verbose: # Make the output a little prettier. It's nice to have some # whitespace between projects when syncing. - print('') + self.Print('') self.UpdateSubmoduleConfig() if file_list is not None: @@ -571,7 +587,8 @@ class GitWrapper(SCMWrapper): % (self.relpath, rev_str)) if verbose: - print('Checked out revision %s' % self.revinfo(options, (), None)) + self.Print('Checked out revision %s' % self.revinfo(options, (), None), + timestamp=False) # If --reset and --delete_unversioned_trees are specified, remove any # untracked directories. @@ -585,7 +602,7 @@ class GitWrapper(SCMWrapper): for path in (p for p in paths.splitlines() if p.endswith('/')): full_path = os.path.join(self.checkout_path, path) if not os.path.islink(full_path): - print('\n_____ removing unversioned directory %s' % path) + self.Print('_____ removing unversioned directory %s' % path) gclient_utils.rmtree(full_path) return self._Capture(['rev-parse', '--verify', 'HEAD']) @@ -599,7 +616,7 @@ class GitWrapper(SCMWrapper): if not os.path.isdir(self.checkout_path): # revert won't work if the directory doesn't exist. It needs to # checkout instead. - print('\n_____ %s is missing, synching instead' % self.relpath) + self.Print('_____ %s is missing, synching instead' % self.relpath) # Don't reuse the args. return self.update(options, [], file_list) @@ -634,11 +651,12 @@ class GitWrapper(SCMWrapper): def status(self, options, _args, file_list): """Display status information.""" if not os.path.isdir(self.checkout_path): - print(('\n________ couldn\'t run status in %s:\n' - 'The directory does not exist.') % self.checkout_path) + self.Print('________ couldn\'t run status in %s:\n' + 'The directory does not exist.' % self.checkout_path) else: merge_base = self._Capture(['merge-base', 'HEAD', self.remote]) - self._Run(['diff', '--name-status', merge_base], options) + self._Run(['diff', '--name-status', merge_base], options, + stdout=self.out_fh) if file_list is not None: files = self._Capture(['diff', '--name-only', merge_base]).split() file_list.extend([os.path.join(self.checkout_path, f) for f in files]) @@ -672,14 +690,14 @@ class GitWrapper(SCMWrapper): logging.debug('git config --get svn-remote.svn.fetch failed, ' 'ignoring possible optimization.') if options.verbose: - print('Running git svn fetch. This might take a while.\n') + self.Print('Running git svn fetch. This might take a while.\n') scm.GIT.Capture(['svn', 'fetch'], cwd=self.checkout_path) try: sha1 = scm.GIT.GetBlessedSha1ForSvnRev( cwd=self.checkout_path, rev=rev) except gclient_utils.Error, e: sha1 = e.message - print('\nWarning: Could not find a git revision with accurate\n' + self.Print('Warning: Could not find a git revision with accurate\n' '.DEPS.git that maps to SVN revision %s. Sync-ing to\n' 'the closest sane git revision, which is:\n' ' %s\n' % (rev, e.message)) @@ -741,7 +759,7 @@ class GitWrapper(SCMWrapper): if not options.verbose: # git clone doesn't seem to insert a newline properly before printing # to stdout - print('') + self.Print('') template_path = os.path.join( os.path.dirname(THIS_FILE_PATH), 'git-templates') cfg = gclient_utils.DefaultIndexPackConfig(self.url) @@ -766,11 +784,11 @@ class GitWrapper(SCMWrapper): gclient_utils.safe_rename(os.path.join(tmp_dir, '.git'), os.path.join(self.checkout_path, '.git')) except: - traceback.print_exc(file=sys.stderr) + traceback.print_exc(file=self.out_fh) raise finally: if os.listdir(tmp_dir): - print('\n_____ removing non-empty tmp dir %s' % tmp_dir) + self.Print('_____ removing non-empty tmp dir %s' % tmp_dir) gclient_utils.rmtree(tmp_dir) if revision.startswith('refs/heads/'): self._Run( @@ -778,7 +796,7 @@ class GitWrapper(SCMWrapper): else: # Squelch git's very verbose detached HEAD warning and use our own self._Run(['checkout', '--quiet', revision], options) - print( + self.Print( ('Checked out %s to a detached HEAD. Before making any commits\n' 'in this repo, you should use \'git checkout \' to switch to\n' 'an existing branch or use \'git checkout %s -b \' to\n' @@ -807,16 +825,16 @@ class GitWrapper(SCMWrapper): revision = newbase action = 'merge' if merge else 'rebase' if not printed_path: - print('\n_____ %s : Attempting %s onto %s...' % ( + self.Print('_____ %s : Attempting %s onto %s...' % ( self.relpath, action, revision)) printed_path = True else: - print('Attempting %s onto %s...' % (action, revision)) + self.Print('Attempting %s onto %s...' % (action, revision)) if merge: merge_output = self._Capture(['merge', revision]) if options.verbose: - print(merge_output) + self.Print(merge_output) return # Build the rebase command here using the args @@ -852,7 +870,7 @@ class GitWrapper(SCMWrapper): "cd %s && git " % self.checkout_path + "%s" % ' '.join(rebase_cmd)) elif re.match(r'show|s', rebase_action, re.I): - print('\n%s' % e.stderr.strip()) + self.Print('%s' % e.stderr.strip()) continue else: gclient_utils.Error("Input not recognized") @@ -862,18 +880,18 @@ class GitWrapper(SCMWrapper): "Fix the conflict and run gclient again.\n" "See 'man git-rebase' for details.\n") else: - print(e.stdout.strip()) - print('Rebase produced error output:\n%s' % e.stderr.strip()) + self.Print(e.stdout.strip()) + self.Print('Rebase produced error output:\n%s' % e.stderr.strip()) raise gclient_utils.Error("Unrecognized error, please merge or rebase " "manually.\ncd %s && git " % self.checkout_path + "%s" % ' '.join(rebase_cmd)) - print(rebase_output.strip()) + self.Print(rebase_output.strip()) if not options.verbose: # Make the output a little prettier. It's nice to have some # whitespace between projects when syncing. - print('') + self.Print('') @staticmethod def _CheckMinVersion(min_version): @@ -932,7 +950,7 @@ class GitWrapper(SCMWrapper): name = ('saved-by-gclient-' + self._Capture(['rev-parse', '--short', 'HEAD'])) self._Capture(['branch', '-f', name]) - print('\n_____ found an unreferenced commit and saved it as \'%s\'' % + self.Print('_____ found an unreferenced commit and saved it as \'%s\'' % name) def _GetCurrentBranch(self): @@ -942,11 +960,10 @@ class GitWrapper(SCMWrapper): return None return branch - def _Capture(self, args, cwd=None): - return subprocess2.check_output( - ['git'] + args, - stderr=subprocess2.VOID, - cwd=cwd or self.checkout_path).strip() + def _Capture(self, args, cwd=None, **kwargs): + kwargs.setdefault('cwd', self.checkout_path) + kwargs.setdefault('stderr', subprocess2.PIPE) + return subprocess2.check_output(['git'] + args, **kwargs).strip() def _UpdateBranchHeads(self, options, fetch=False): """Adds, and optionally fetches, "branch-heads" refspecs if requested.""" @@ -963,26 +980,29 @@ class GitWrapper(SCMWrapper): self._Run(fetch_cmd, options, retry=True) def _Run(self, args, options, **kwargs): - kwargs.setdefault('cwd', self.checkout_path) - git_filter = not options.verbose - if git_filter: - kwargs['filter_fn'] = gclient_utils.GitFilter(kwargs.get('filter_fn')) - kwargs.setdefault('print_stdout', False) - # Don't prompt for passwords; just fail quickly and noisily. - # By default, git will use an interactive terminal prompt when a username/ - # password is needed. That shouldn't happen in the chromium workflow, - # and if it does, then gclient may hide the prompt in the midst of a flood - # of terminal spew. The only indication that something has gone wrong - # will be when gclient hangs unresponsively. Instead, we disable the - # password prompt and simply allow git to fail noisily. The error - # message produced by git will be copied to gclient's output. - env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy()) - env.setdefault('GIT_ASKPASS', 'true') - env.setdefault('SSH_ASKPASS', 'true') - else: - kwargs.setdefault('print_stdout', True) + cwd = kwargs.setdefault('cwd', self.checkout_path) + kwargs.setdefault('stdout', self.out_fh) + filter_kwargs = { 'time_throttle': 10, 'out_fh': self.out_fh } + if self.out_cb: + filter_kwargs['predicate'] = self.out_cb + kwargs['filter_fn'] = git_filter = gclient_utils.GitFilter(**filter_kwargs) + kwargs.setdefault('print_stdout', False) + # Don't prompt for passwords; just fail quickly and noisily. + # By default, git will use an interactive terminal prompt when a username/ + # password is needed. That shouldn't happen in the chromium workflow, + # and if it does, then gclient may hide the prompt in the midst of a flood + # of terminal spew. The only indication that something has gone wrong + # will be when gclient hangs unresponsively. Instead, we disable the + # password prompt and simply allow git to fail noisily. The error + # message produced by git will be copied to gclient's output. + env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy()) + env.setdefault('GIT_ASKPASS', 'true') + env.setdefault('SSH_ASKPASS', 'true') + cmd = ['git'] + args - return gclient_utils.CheckCallAndFilterAndHeader(cmd, **kwargs) + header = "running '%s' in '%s'" % (' '.join(cmd), cwd) + git_filter(header) + return gclient_utils.CheckCallAndFilter(cmd, **kwargs) class SVNWrapper(SCMWrapper): @@ -1032,7 +1052,7 @@ class SVNWrapper(SCMWrapper): ['svn', 'diff', '-x', '--ignore-eol-style'] + args, cwd=self.checkout_path, print_stdout=False, - filter_fn=SvnDiffFilterer(self.relpath).Filter) + filter_fn=SvnDiffFilterer(self.relpath).Filter, print_func=self.Print) def update(self, options, args, file_list): """Runs svn to update or transparently checkout the working copy. @@ -1045,12 +1065,12 @@ class SVNWrapper(SCMWrapper): # Only update if git or hg is not controlling the directory. git_path = os.path.join(self.checkout_path, '.git') if os.path.exists(git_path): - print('________ found .git directory; skipping %s' % self.relpath) + self.Print('________ found .git directory; skipping %s' % self.relpath) return hg_path = os.path.join(self.checkout_path, '.hg') if os.path.exists(hg_path): - print('________ found .hg directory; skipping %s' % self.relpath) + self.Print('________ found .hg directory; skipping %s' % self.relpath) return if args: @@ -1086,7 +1106,7 @@ class SVNWrapper(SCMWrapper): [], os.path.join(self.checkout_path, '.')) except (gclient_utils.Error, subprocess2.CalledProcessError): if options.reset and options.delete_unversioned_trees: - print 'Removing troublesome path %s' % self.checkout_path + self.Print('Removing troublesome path %s' % self.checkout_path) gclient_utils.rmtree(self.checkout_path) exists = False else: @@ -1127,14 +1147,14 @@ class SVNWrapper(SCMWrapper): latest_checkout = sorted_items[-1] tempdir = tempfile.mkdtemp() - print 'Downloading %s...' % latest_checkout + self.Print('Downloading %s...' % latest_checkout) code, out, err = gsutil.check_call('cp', latest_checkout, tempdir) if code: - print '%s\n%s' % (out, err) + self.Print('%s\n%s' % (out, err)) raise Exception() filename = latest_checkout.split('/')[-1] tarball = os.path.join(tempdir, filename) - print 'Unpacking into %s...' % self.checkout_path + self.Print('Unpacking into %s...' % self.checkout_path) gclient_utils.safe_makedirs(self.checkout_path) # TODO(hinoka): Use 7z for windows. cmd = ['tar', '--extract', '--ungzip', @@ -1143,7 +1163,7 @@ class SVNWrapper(SCMWrapper): gclient_utils.CheckCallAndFilter( cmd, stdout=sys.stdout, print_stdout=True) - print 'Deleting temp file' + self.Print('Deleting temp file') gclient_utils.rmtree(tempdir) # Rewrite the repository root to match. @@ -1154,14 +1174,14 @@ class SVNWrapper(SCMWrapper): tarball_parsed.netloc) if tarball_root != local_root: - print 'Switching repository root to %s' % local_root + self.Print('Switching repository root to %s' % local_root) self._Run(['switch', '--relocate', tarball_root, local_root, self.checkout_path], options) except Exception as e: - print 'We tried to get a source tarball but failed.' - print 'Resuming normal operations.' - print str(e) + self.Print('We tried to get a source tarball but failed.') + self.Print('Resuming normal operations.') + self.Print(str(e)) gclient_utils.safe_makedirs(os.path.dirname(self.checkout_path)) # We need to checkout. @@ -1171,7 +1191,7 @@ class SVNWrapper(SCMWrapper): return self.Svnversion() if not managed: - print ('________ unmanaged solution; skipping %s' % self.relpath) + self.Print(('________ unmanaged solution; skipping %s' % self.relpath)) return self.Svnversion() if 'URL' not in from_info: @@ -1201,12 +1221,13 @@ class SVNWrapper(SCMWrapper): assert not os.path.isabs(d[1]) path_to_remove = os.path.normpath( os.path.join(self.checkout_path, d[1])) - print 'Removing troublesome path %s' % path_to_remove + self.Print('Removing troublesome path %s' % path_to_remove) gclient_utils.rmtree(path_to_remove) else: - print 'Not removing troublesome path %s automatically.' % d[1] + self.Print( + 'Not removing troublesome path %s automatically.' % d[1]) if d[0][0] == '!': - print 'You can pass --force to enable automatic removal.' + self.Print('You can pass --force to enable automatic removal.') raise e # Retrieve the current HEAD version because svn is slow at null updates. @@ -1226,7 +1247,7 @@ class SVNWrapper(SCMWrapper): can_switch = ((from_info['Repository Root'] != to_info['Repository Root']) and (from_info['UUID'] == to_info['UUID'])) if can_switch: - print('\n_____ relocating %s to a new checkout' % self.relpath) + self.Print('_____ relocating %s to a new checkout' % self.relpath) # We have different roots, so check if we can switch --relocate. # Subversion only permits this if the repository UUIDs match. # Perform the switch --relocate, then rewrite the from_url @@ -1254,7 +1275,7 @@ class SVNWrapper(SCMWrapper): 'there is local changes in %s. Delete the directory and ' 'try again.') % (url, self.checkout_path)) # Ok delete it. - print('\n_____ switching %s to a new checkout' % self.relpath) + self.Print('_____ switching %s to a new checkout' % self.relpath) gclient_utils.rmtree(self.checkout_path) # We need to checkout. command = ['checkout', url, self.checkout_path] @@ -1266,7 +1287,7 @@ class SVNWrapper(SCMWrapper): # number of the existing directory, then we don't need to bother updating. if not options.force and str(from_info['Revision']) == revision: if options.verbose or not forced_revision: - print('\n_____ %s%s' % (self.relpath, rev_str)) + self.Print('_____ %s%s' % (self.relpath, rev_str), timestamp=False) else: command = ['update', self.checkout_path] command = self._AddAdditionalUpdateFlags(command, options, revision) @@ -1280,7 +1301,7 @@ class SVNWrapper(SCMWrapper): if (status[0][0] == '?' and os.path.isdir(full_path) and not os.path.islink(full_path)): - print('\n_____ removing unversioned directory %s' % status[1]) + self.Print('_____ removing unversioned directory %s' % status[1]) gclient_utils.rmtree(full_path) return self.Svnversion() @@ -1323,20 +1344,20 @@ class SVNWrapper(SCMWrapper): gclient_utils.rmtree(self.checkout_path) # svn revert won't work if the directory doesn't exist. It needs to # checkout instead. - print('\n_____ %s is missing, synching instead' % self.relpath) + self.Print('_____ %s is missing, synching instead' % self.relpath) # Don't reuse the args. return self.update(options, [], file_list) if not os.path.isdir(os.path.join(self.checkout_path, '.svn')): if os.path.isdir(os.path.join(self.checkout_path, '.git')): - print('________ found .git directory; skipping %s' % self.relpath) + self.Print('________ found .git directory; skipping %s' % self.relpath) return if os.path.isdir(os.path.join(self.checkout_path, '.hg')): - print('________ found .hg directory; skipping %s' % self.relpath) + self.Print('________ found .hg directory; skipping %s' % self.relpath) return if not options.force: raise gclient_utils.Error('Invalid checkout path, aborting') - print( + self.Print( '\n_____ %s is not a valid svn checkout, synching instead' % self.relpath) gclient_utils.rmtree(self.checkout_path) @@ -1349,7 +1370,7 @@ class SVNWrapper(SCMWrapper): if logging.getLogger().isEnabledFor(logging.INFO): logging.info('%s%s' % (file_status[0], file_status[1])) else: - print(os.path.join(self.checkout_path, file_status[1])) + self.Print(os.path.join(self.checkout_path, file_status[1])) scm.SVN.Revert(self.checkout_path, callback=printcb) # Revert() may delete the directory altogether. @@ -1382,7 +1403,7 @@ class SVNWrapper(SCMWrapper): command = ['status'] + args if not os.path.isdir(self.checkout_path): # svn status won't work if the directory doesn't exist. - print(('\n________ couldn\'t run \'%s\' in \'%s\':\n' + self.Print(('\n________ couldn\'t run \'%s\' in \'%s\':\n' 'The directory does not exist.') % (' '.join(command), self.checkout_path)) # There's no file list to retrieve. diff --git a/gclient_utils.py b/gclient_utils.py index 3517946435..f89b601f3a 100644 --- a/gclient_utils.py +++ b/gclient_utils.py @@ -6,6 +6,7 @@ import codecs import cStringIO +import datetime import logging import os import pipes @@ -25,6 +26,7 @@ import subprocess2 RETRY_MAX = 3 RETRY_INITIAL_SLEEP = 0.5 +START = datetime.datetime.now() _WARNINGS = [] @@ -47,6 +49,12 @@ class Error(Exception): super(Error, self).__init__(msg, *args, **kwargs) +def Elapsed(until=None): + if until is None: + until = datetime.datetime.now() + return str(until - START).partition('.')[0] + + def PrintWarnings(): """Prints any accumulated warnings.""" if _WARNINGS: @@ -483,12 +491,8 @@ def CheckCallAndFilter(args, stdout=None, filter_fn=None, output.write(in_byte) if print_stdout: stdout.write(in_byte) - if in_byte != '\r': - if in_byte != '\n': - in_line += in_byte - else: - filter_fn(in_line) - in_line = '' + if in_byte not in ['\r', '\n']: + in_line += in_byte else: filter_fn(in_line) in_line = '' @@ -525,9 +529,9 @@ class GitFilter(object): Allows a custom function to skip certain lines (predicate), and will throttle the output of percentage completed lines to only output every X seconds. """ - PERCENT_RE = re.compile('.* ([0-9]{1,2})% .*') + PERCENT_RE = re.compile('(.*) ([0-9]{1,3})% .*') - def __init__(self, time_throttle=0, predicate=None): + def __init__(self, time_throttle=0, predicate=None, out_fh=None): """ Args: time_throttle (int): GitFilter will throttle 'noisy' output (such as the @@ -535,10 +539,13 @@ class GitFilter(object): seconds apart. predicate (f(line)): An optional function which is invoked for every line. The line will be skipped if predicate(line) returns False. + out_fh: File handle to write output to. """ self.last_time = 0 self.time_throttle = time_throttle self.predicate = predicate + self.out_fh = out_fh or sys.stdout + self.progress_prefix = None def __call__(self, line): # git uses an escape sequence to clear the line; elide it. @@ -549,11 +556,14 @@ class GitFilter(object): return now = time.time() match = self.PERCENT_RE.match(line) - if not match: - self.last_time = 0 - if (now - self.last_time) >= self.time_throttle: - self.last_time = now - print line + if match: + if match.group(1) != self.progress_prefix: + self.progress_prefix = match.group(1) + elif now - self.last_time < self.time_throttle: + return + self.last_time = now + self.out_fh.write('[%s] ' % Elapsed()) + print >> self.out_fh, line def FindGclientRoot(from_dir, filename='.gclient'): @@ -683,6 +693,8 @@ class WorkItem(object): def __init__(self, name): # A unique string representing this work item. self._name = name + self.outbuf = cStringIO.StringIO() + self.start = self.finish = None def run(self, work_queue): """work_queue is passed as keyword argument so it should be @@ -704,7 +716,7 @@ class ExecutionQueue(object): Methods of this class are thread safe. """ - def __init__(self, jobs, progress, ignore_requirements): + def __init__(self, jobs, progress, ignore_requirements, verbose=False): """jobs specifies the number of concurrent tasks to allow. progress is a Progress instance.""" # Set when a thread is done or a new item is enqueued. @@ -725,6 +737,9 @@ class ExecutionQueue(object): self.progress.update(0) self.ignore_requirements = ignore_requirements + self.verbose = verbose + self.last_join = None + self.last_subproc_output = None def enqueue(self, d): """Enqueue one Dependency to be executed later once its requirements are @@ -743,9 +758,30 @@ class ExecutionQueue(object): finally: self.ready_cond.release() + def out_cb(self, _): + self.last_subproc_output = datetime.datetime.now() + return True + + @staticmethod + def format_task_output(task, comment=''): + if comment: + comment = ' (%s)' % comment + if task.start and task.finish: + elapsed = ' (Elapsed: %s)' % ( + str(task.finish - task.start).partition('.')[0]) + else: + elapsed = '' + return """ +%s%s%s +---------------------------------------- +%s +----------------------------------------""" % ( + task.name, comment, task.outbuf.getvalue().strip(), elapsed) + def flush(self, *args, **kwargs): """Runs all enqueued items until all are executed.""" kwargs['work_queue'] = self + self.last_subproc_output = self.last_join = datetime.datetime.now() self.ready_cond.acquire() try: while True: @@ -778,6 +814,17 @@ class ExecutionQueue(object): # We need to poll here otherwise Ctrl-C isn't processed. try: self.ready_cond.wait(10) + # If we haven't printed to terminal for a while, but we have received + # spew from a suprocess, let the user know we're still progressing. + now = datetime.datetime.now() + if (now - self.last_join > datetime.timedelta(seconds=60) and + self.last_subproc_output > self.last_join): + if self.progress: + print >> sys.stdout, '' + elapsed = Elapsed() + print >> sys.stdout, '[%s] Still working on:' % elapsed + for task in self.running: + print >> sys.stdout, '[%s] %s' % (elapsed, task.item.name) except KeyboardInterrupt: # Help debugging by printing some information: print >> sys.stderr, ( @@ -788,7 +835,10 @@ class ExecutionQueue(object): ', '.join(self.ran), len(self.running))) for i in self.queued: - print >> sys.stderr, '%s: %s' % (i.name, ', '.join(i.requirements)) + print >> sys.stderr, '%s (not started): %s' % ( + i.name, ', '.join(i.requirements)) + for i in self.running: + print >> sys.stderr, self.format_task_output(i.item, 'interrupted') raise # Something happened: self.enqueue() or a thread terminated. Loop again. finally: @@ -796,11 +846,14 @@ class ExecutionQueue(object): assert not self.running, 'Now guaranteed to be single-threaded' if not self.exceptions.empty(): + if self.progress: + print >> sys.stdout, '' # To get back the stack location correctly, the raise a, b, c form must be # used, passing a tuple as the first argument doesn't work. - e = self.exceptions.get() + e, task = self.exceptions.get() + print >> sys.stderr, self.format_task_output(task.item, 'ERROR') raise e[0], e[1], e[2] - if self.progress: + elif self.progress: self.progress.end() def _flush_terminated_threads(self): @@ -812,7 +865,10 @@ class ExecutionQueue(object): self.running.append(t) else: t.join() + self.last_join = datetime.datetime.now() sys.stdout.flush() + if self.verbose: + print >> sys.stdout, self.format_task_output(t.item) if self.progress: self.progress.update(1, t.item.name) if t.item.name in self.ran: @@ -832,10 +888,26 @@ class ExecutionQueue(object): else: # Run the 'thread' inside the main thread. Don't try to catch any # exception. - task_item.run(*args, **kwargs) - self.ran.append(task_item.name) - if self.progress: - self.progress.update(1, ', '.join(t.item.name for t in self.running)) + try: + task_item.start = datetime.datetime.now() + print >> task_item.outbuf, '[%s] Started.' % Elapsed(task_item.start) + task_item.run(*args, **kwargs) + task_item.finish = datetime.datetime.now() + print >> task_item.outbuf, '[%s] Finished.' % Elapsed(task_item.finish) + self.ran.append(task_item.name) + if self.verbose: + if self.progress: + print >> sys.stdout, '' + print >> sys.stdout, self.format_task_output(task_item) + if self.progress: + self.progress.update(1, ', '.join(t.item.name for t in self.running)) + except KeyboardInterrupt: + print >> sys.stderr, self.format_task_output(task_item, 'interrupted') + raise + except Exception: + print >> sys.stderr, self.format_task_output(task_item, 'ERROR') + raise + class _Worker(threading.Thread): """One thread to execute one WorkItem.""" @@ -853,17 +925,21 @@ class ExecutionQueue(object): logging.debug('_Worker.run(%s)' % self.item.name) work_queue = self.kwargs['work_queue'] try: + self.item.start = datetime.datetime.now() + print >> self.item.outbuf, '[%s] Started.' % Elapsed(self.item.start) self.item.run(*self.args, **self.kwargs) + self.item.finish = datetime.datetime.now() + print >> self.item.outbuf, '[%s] Finished.' % Elapsed(self.item.finish) except KeyboardInterrupt: logging.info('Caught KeyboardInterrupt in thread %s', self.item.name) logging.info(str(sys.exc_info())) - work_queue.exceptions.put(sys.exc_info()) + work_queue.exceptions.put((sys.exc_info(), self)) raise except Exception: # Catch exception location. logging.info('Caught exception in thread %s', self.item.name) logging.info(str(sys.exc_info())) - work_queue.exceptions.put(sys.exc_info()) + work_queue.exceptions.put((sys.exc_info(), self)) finally: logging.info('_Worker.run(%s) done', self.item.name) work_queue.ready_cond.acquire() diff --git a/tests/gclient_scm_test.py b/tests/gclient_scm_test.py index a0a08872ef..6b667a00ab 100755 --- a/tests/gclient_scm_test.py +++ b/tests/gclient_scm_test.py @@ -13,6 +13,7 @@ from subprocess import Popen, PIPE, STDOUT import logging import os +import re import sys import tempfile import unittest @@ -28,6 +29,14 @@ import subprocess2 # Shortcut since this function is used often join = gclient_scm.os.path.join +TIMESTAMP_RE = re.compile('\[[0-9]{1,2}:[0-9]{2}:[0-9]{2}\] (.*)', re.DOTALL) +def strip_timestamps(value): + lines = value.splitlines(True) + for i in xrange(len(lines)): + m = TIMESTAMP_RE.match(lines[i]) + if m: + lines[i] = m.group(1) + return ''.join(lines) # Access to a protected member XXX of a client class # pylint: disable=W0212 @@ -89,6 +98,12 @@ class SVNWrapperTestCase(BaseTestCase): self.jobs = 1 self.delete_unversioned_trees = False + def checkstdout(self, expected): + value = sys.stdout.getvalue() + sys.stdout.close() + # pylint: disable=E1101 + self.assertEquals(expected, strip_timestamps(value)) + def Options(self, *args, **kwargs): return self.OptionsObject(*args, **kwargs) @@ -179,7 +194,7 @@ class SVNWrapperTestCase(BaseTestCase): relpath=self.relpath) scm.revert(options, self.args, files_list) self.checkstdout( - ('\n_____ %s is missing, synching instead\n' % self.relpath)) + ('_____ %s is missing, synching instead\n' % self.relpath)) def testRevertNoDotSvn(self): options = self.Options(verbose=True, force=True) @@ -416,7 +431,7 @@ class SVNWrapperTestCase(BaseTestCase): scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, relpath=self.relpath) scm.update(options, (), files_list) - self.checkstdout('\n_____ %s at 42\n' % self.relpath) + self.checkstdout('_____ %s at 42\n' % self.relpath) def testUpdateResetDeleteUnversionedTrees(self): options = self.Options(verbose=True) @@ -461,8 +476,8 @@ class SVNWrapperTestCase(BaseTestCase): files_list = [] scm.update(options, (), files_list) self.checkstdout( - ('\n_____ %s at 42\n' - '\n_____ removing unversioned directory dir\n') % self.relpath) + ('_____ %s at 42\n' + '_____ removing unversioned directory dir\n') % self.relpath) def testUpdateSingleCheckout(self): options = self.Options(verbose=True) @@ -509,7 +524,7 @@ class SVNWrapperTestCase(BaseTestCase): scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, relpath=self.relpath) scm.updatesingle(options, ['DEPS'], files_list) - self.checkstdout('\n_____ %s at 42\n' % self.relpath) + self.checkstdout('_____ %s at 42\n' % self.relpath) def testUpdateSingleCheckoutSVN14(self): options = self.Options(verbose=True) @@ -581,7 +596,7 @@ class SVNWrapperTestCase(BaseTestCase): relpath=self.relpath) scm.updatesingle(options, ['DEPS'], files_list) self.checkstdout( - ('\n_____ %s at 42\n' % self.relpath)) + ('_____ %s at 42\n' % self.relpath)) def testUpdateSingleUpdate(self): options = self.Options(verbose=True) @@ -616,7 +631,7 @@ class SVNWrapperTestCase(BaseTestCase): scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, relpath=self.relpath) scm.updatesingle(options, ['DEPS'], files_list) - self.checkstdout('\n_____ %s at 42\n' % self.relpath) + self.checkstdout('_____ %s at 42\n' % self.relpath) def testUpdateGit(self): options = self.Options(verbose=True) @@ -745,6 +760,12 @@ from :3 def Options(self, *args, **kwargs): return self.OptionsObject(*args, **kwargs) + def checkstdout(self, expected): + value = sys.stdout.getvalue() + sys.stdout.close() + # pylint: disable=E1101 + self.assertEquals(expected, strip_timestamps(value)) + @staticmethod def CreateGitRepo(git_import, path): """Do it for real.""" @@ -895,7 +916,7 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): scm.status(options, self.args, file_list) self.assertEquals(file_list, [file_path]) self.checkstdout( - ('\n________ running \'git diff --name-status ' + ('running \'git diff --name-status ' '069c602044c5388d2d15c3f875b057c852003458\' in \'%s\'\nM\ta\n') % join(self.root_dir, '.')) @@ -915,7 +936,7 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): expected_file_list = [join(self.base_path, x) for x in ['a', 'b']] self.assertEquals(sorted(file_list), expected_file_list) self.checkstdout( - ('\n________ running \'git diff --name-status ' + ('running \'git diff --name-status ' '069c602044c5388d2d15c3f875b057c852003458\' in \'%s\'\nM\ta\nM\tb\n') % join(self.root_dir, '.')) diff --git a/tests/gclient_smoketest.py b/tests/gclient_smoketest.py index 68264fada8..5a58ea44d3 100755 --- a/tests/gclient_smoketest.py +++ b/tests/gclient_smoketest.py @@ -544,7 +544,7 @@ class GClientSmokeSVN(GClientSmokeBase): out = self.splitBlock(results[0]) # src, src/other is missing, src/other, src/third_party/foo is missing, # src/third_party/foo, 2 svn hooks, 3 related to File(). - self.assertEquals(10, len(out)) + self.assertEquals( 8, len(out)) self.checkString('', results[1]) self.assertEquals(0, results[2]) tree = self.mangle_svn_tree( @@ -597,14 +597,14 @@ class GClientSmokeSVN(GClientSmokeBase): ['running', join(self.root_dir, 'src', 'third_party', 'fpp')], ['running', join(self.root_dir, 'src', 'third_party', 'prout')]]) out = self.svnBlockCleanup(out) - self.checkString('other', out[0][1]) - self.checkString(join('third_party', 'fpp'), out[0][2]) - self.checkString(join('third_party', 'prout'), out[0][3]) - self.checkString('hi', out[1][1]) - self.assertEquals(4, len(out[0])) - self.assertEquals(2, len(out[1])) - self.assertEquals(1, len(out[2])) - self.assertEquals(1, len(out[3])) + self.checkString('other', out[0][5]) + self.checkString(join('third_party', 'fpp'), out[0][7]) + self.checkString(join('third_party', 'prout'), out[0][8]) + self.checkString('hi', out[1][5]) + self.assertEquals(9, len(out[0])) + self.assertEquals(7, len(out[1])) + self.assertEquals(6, len(out[2])) + self.assertEquals(6, len(out[3])) self.assertEquals(4, len(out)) # Revert implies --force implies running hooks without looking at pattern @@ -613,7 +613,7 @@ class GClientSmokeSVN(GClientSmokeBase): # the file list after some ___ running 'svn status' results = self.gclient(['revert', '--deps', 'mac', '--jobs', '1']) out = self.splitBlock(results[0]) - self.assertEquals(7, len(out)) + self.assertEquals(4, len(out)) self.checkString('', results[1]) self.assertEquals(0, results[2]) tree = self.mangle_svn_tree( @@ -848,7 +848,7 @@ class GClientSmokeGIT(GClientSmokeBase): # Test unversioned checkout. self.parseGclient( ['sync', '--deps', 'mac', '--jobs', '1'], - ['running', 'running', 'running', 'running', 'running']) + ['running', 'running']) # TODO(maruel): http://crosbug.com/3582 hooks run even if not matching, must # add sync parsing to get the list of updated files. tree = self.mangle_git_tree(('repo_1@2', 'src'), @@ -867,7 +867,7 @@ class GClientSmokeGIT(GClientSmokeBase): ['sync', '--jobs', '1', '--revision', 'src@' + self.githash('repo_1', 1), '--deps', 'mac', '--delete_unversioned_trees'], - ['running', 'running', 'deleting']) + ['deleting']) tree = self.mangle_git_tree(('repo_1@1', 'src'), ('repo_2@2', 'src/repo2'), ('repo_3@1', 'src/repo2/repo3'), @@ -877,7 +877,7 @@ class GClientSmokeGIT(GClientSmokeBase): # Test incremental sync: delete-unversioned_trees isn't there. self.parseGclient( ['sync', '--deps', 'mac', '--jobs', '1'], - ['running', 'running', 'running']) + ['running', 'running']) tree = self.mangle_git_tree(('repo_1@2', 'src'), ('repo_2@1', 'src/repo2'), ('repo_3@1', 'src/repo2/repo3'), @@ -895,7 +895,7 @@ class GClientSmokeGIT(GClientSmokeBase): self.parseGclient( ['sync', '--deps', 'mac', '--jobs', '1', '--revision', 'invalid@' + self.githash('repo_1', 1)], - ['running', 'running', 'running', 'running', 'running'], + ['running', 'running'], 'Please fix your script, having invalid --revision flags ' 'will soon considered an error.\n') tree = self.mangle_git_tree(('repo_1@2', 'src'), @@ -913,7 +913,7 @@ class GClientSmokeGIT(GClientSmokeBase): self.parseGclient( ['sync', '--deps', 'mac', '--jobs', '1', '--revision', self.githash('repo_1', 1)], - ['running', 'running', 'running', 'running']) + []) tree = self.mangle_git_tree(('repo_1@1', 'src'), ('repo_2@2', 'src/repo2'), ('repo_3@1', 'src/repo2/repo3'), @@ -928,7 +928,7 @@ class GClientSmokeGIT(GClientSmokeBase): # Test unversioned checkout. self.parseGclient( ['sync', '--deps', 'mac', '--jobs', '8'], - ['running', 'running', 'running', 'running', 'running'], + ['running', 'running'], untangle=True) # TODO(maruel): http://crosbug.com/3582 hooks run even if not matching, must # add sync parsing to get the list of updated files. @@ -948,7 +948,7 @@ class GClientSmokeGIT(GClientSmokeBase): self.parseGclient( ['sync', '--revision', 'src@' + self.githash('repo_1', 1), '--deps', 'mac', '--delete_unversioned_trees', '--jobs', '1'], - [ 'running', 'running', 'deleting'], + ['deleting'], untangle=True) tree = self.mangle_git_tree(('repo_1@1', 'src'), ('repo_2@2', 'src/repo2'), @@ -959,7 +959,7 @@ class GClientSmokeGIT(GClientSmokeBase): # Test incremental sync: delete-unversioned_trees isn't there. self.parseGclient( ['sync', '--deps', 'mac', '--jobs', '8'], - ['running', 'running', 'running'], + ['running', 'running'], untangle=True) tree = self.mangle_git_tree(('repo_1@2', 'src'), ('repo_2@1', 'src/repo2'), @@ -1001,16 +1001,13 @@ class GClientSmokeGIT(GClientSmokeBase): return self.gclient(['config', self.git_base + 'repo_5', '--name', 'src']) expectation = [ - ('running', self.root_dir), # git clone repo_5 ('running', self.root_dir), # pre-deps hook - ('running', self.root_dir), # git clone repo_1 - ('running', self.root_dir), # git clone repo_1 ] out = self.parseGclient(['sync', '--deps', 'mac', '--jobs=1', '--revision', 'src@' + self.githash('repo_5', 2)], expectation) - self.assertEquals(2, len(out[1])) - self.assertEquals('pre-deps hook', out[1][1]) + self.assertEquals(2, len(out[0])) + self.assertEquals('pre-deps hook', out[0][1]) tree = self.mangle_git_tree(('repo_5@2', 'src'), ('repo_1@2', 'src/repo1'), ('repo_2@1', 'src/repo2') @@ -1054,7 +1051,6 @@ class GClientSmokeGIT(GClientSmokeBase): return self.gclient(['config', self.git_base + 'repo_5', '--name', 'src']) expectated_stdout = [ - ('running', self.root_dir), # git clone repo_5 ('running', self.root_dir), # pre-deps hook ('running', self.root_dir), # pre-deps hook (fails) ] @@ -1134,9 +1130,6 @@ class GClientSmokeGITMutates(GClientSmokeBase): self.gclient(['sync', '--deps', 'mac']) write(join(self.root_dir, 'src', 'repo2', 'hi'), 'Hey!') - expected1 = ('running', os.path.join(self.root_dir, 'src')) - expected2 = ('running', os.path.join(expected1[1], 'repo2')) - expected3 = ('running', os.path.join(expected2[1], 'repo_renamed')) out = self.parseGclient(['status', '--deps', 'mac', '--jobs', '1'], []) # TODO(maruel): http://crosbug.com/3584 It should output the unversioned # files. @@ -1147,13 +1140,9 @@ class GClientSmokeGITMutates(GClientSmokeBase): # there should be two results for each. The last two results should reflect # writing git_hooked1 and git_hooked2. There's only one result for the third # because it is clean and has no output for 'git clean'. - expected4 = ('running', self.root_dir) out = self.parseGclient(['revert', '--deps', 'mac', '--jobs', '1'], - [expected1, expected1, - expected2, expected2, - expected3, - expected4, expected4]) - self.assertEquals(7, len(out)) + ['running', 'running']) + self.assertEquals(2, len(out)) tree = self.mangle_git_tree(('repo_1@3', 'src'), ('repo_2@1', 'src/repo2'), ('repo_3@2', 'src/repo2/repo_renamed')) @@ -1171,11 +1160,8 @@ class GClientSmokeGITMutates(GClientSmokeBase): 'custom_vars': {'r2hash': self.FAKE_REPOS.git_hashes['repo_2'][-1][0] } }]) out = self.parseGclient(['revert', '--deps', 'mac', '--jobs', '1'], - [expected1, expected1, - expected2, expected2, - expected3, - expected4, expected4]) - self.assertEquals(7, len(out)) + ['running', 'running']) + self.assertEquals(2, len(out)) tree = self.mangle_git_tree(('repo_1@3', 'src'), ('repo_2@3', 'src/repo2'), ('repo_3@2', 'src/repo2/repo_renamed')) @@ -1204,12 +1190,12 @@ class GClientSmokeBoth(GClientSmokeBase): '{"name": "src-git",' '"url": "' + self.git_base + 'repo_1"}]']) self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'], - ['running', 'running', 'running', + ['running', 'running', # This is due to the way svn update is called for a single # file when File() is used in a DEPS file. ('running', self.root_dir + '/src/file/other'), 'running', 'running', 'running', 'running', - 'running', 'running', 'running', 'running']) + 'running', 'running']) tree = self.mangle_git_tree(('repo_1@2', 'src-git'), ('repo_2@1', 'src/repo2'), ('repo_3@2', 'src/repo2/repo_renamed')) @@ -1240,7 +1226,7 @@ class GClientSmokeBoth(GClientSmokeBase): self.checkString('', stderr) self.assertEquals(0, returncode) results = self.splitBlock(stdout) - self.assertEquals(12, len(results)) + self.assertEquals(9, len(results)) tree = self.mangle_git_tree(('repo_1@2', 'src-git'), ('repo_2@1', 'src/repo2'), ('repo_3@2', 'src/repo2/repo_renamed')) @@ -1267,8 +1253,7 @@ class GClientSmokeBoth(GClientSmokeBase): self.parseGclient( ['sync', '--deps', 'mac', '--jobs', '1', '--revision', '1', '-r', 'src-git@' + self.githash('repo_1', 1)], - ['running', 'running', 'running', 'running', 'running', - 'running', 'running', 'running'], + ['running', 'running', 'running', 'running'], expected_stderr= 'You must specify the full solution name like --revision src@1\n' 'when you have multiple solutions setup in your .gclient file.\n' diff --git a/tests/gclient_test.py b/tests/gclient_test.py index 86cb69300f..ea61b0a1d5 100755 --- a/tests/gclient_test.py +++ b/tests/gclient_test.py @@ -72,7 +72,7 @@ class GclientTest(trial_dir.TestCase): os.chdir(self.previous_dir) super(GclientTest, self).tearDown() - def _createscm(self, parsed_url, root_dir, name): + def _createscm(self, parsed_url, root_dir, name, out_fh=None, out_cb=None): self.assertTrue(parsed_url.startswith('svn://example.com/'), parsed_url) self.assertTrue(root_dir.startswith(self.root_dir), root_dir) return SCMMock(self, parsed_url) diff --git a/third_party/repo/progress.py b/third_party/repo/progress.py index 3d302814c2..b21e1bc6f0 100644 --- a/third_party/repo/progress.py +++ b/third_party/repo/progress.py @@ -59,15 +59,17 @@ class Progress(object): return if self._total <= 0: - sys.stdout.write('%s: %d, done.\n' % ( + text = '%s: %d, done.' % ( self._title, - self._done)) - sys.stdout.flush() + self._done) else: p = (100 * self._done) / self._total - sys.stdout.write('%s: %3d%% (%d/%d), done.\n' % ( + text = '%s: %3d%% (%d/%d), done.' % ( self._title, p, self._done, - self._total)) - sys.stdout.flush() + self._total) + + spaces = max(self._width - len(text), 0) + sys.stdout.write('%s%*s\n' % (text, spaces, '')) + sys.stdout.flush()