From f2ed3fb1f04a5da4a0a90a62bb12a575dda1fc1f Mon Sep 17 00:00:00 2001 From: "ilevy@chromium.org" Date: Fri, 9 Nov 2012 23:39:49 +0000 Subject: [PATCH] Add gclient grep for git repos - Adds a gclient grep command to search through git repos. BUG=157950 Review URL: https://chromiumcodereview.appspot.com/11312116 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@167007 0039d316-1c4b-4281-b951-d872f2087c98 --- gclient.py | 101 +++++++++++++++++++++++++++++++++--------- gclient_utils.py | 31 ++++++------- tests/gclient_test.py | 2 +- 3 files changed, 98 insertions(+), 36 deletions(-) diff --git a/gclient.py b/gclient.py index fd4075610..48ac3a7f0 100644 --- a/gclient.py +++ b/gclient.py @@ -618,7 +618,18 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): # Strip any leading path separators. while file_list[i].startswith(('\\', '/')): file_list[i] = file_list[i][1:] - elif command == 'recurse': + + # Always parse the DEPS file. + self.ParseDepsFile() + + self._run_is_done(file_list, parsed_url) + + if self.recursion_limit: + # Parse the dependencies of this dependency. + for s in self.dependencies: + work_queue.enqueue(s) + + if command == 'recurse': if not isinstance(parsed_url, self.FileImpl): # Skip file only checkout. scm = gclient_scm.GetScmName(parsed_url) @@ -630,25 +641,39 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): env['GCLIENT_SCM'] = scm if parsed_url: env['GCLIENT_URL'] = parsed_url + if options.prepend_dir: + print_stdout = False + def filter_fn(line): + items = line.split('\0') + if len(items) == 1: + match = re.match('Binary file (.*) matches$', line) + if match: + print 'Binary file %s matches' % os.path.join( + self.name, match.group(1)) + else: + print line + elif len(items) == 2 and items[1]: + print '%s : %s' % (os.path.join(self.name, items[0]), items[1]) + else: + # Multiple null bytes or a single trailing null byte indicate + # git is likely displaying filenames only (such as with -l) + print '\n'.join(os.path.join(self.name, f) for f in items if f) + else: + print_stdout = True + filter_fn = None + if os.path.isdir(cwd): try: gclient_utils.CheckCallAndFilter( - args, cwd=cwd, env=env, print_stdout=True) + args, cwd=cwd, env=env, print_stdout=print_stdout, + filter_fn=filter_fn, + ) except subprocess2.CalledProcessError: if not options.ignore: raise else: print >> sys.stderr, 'Skipped missing %s' % cwd - # Always parse the DEPS file. - self.ParseDepsFile() - - self._run_is_done(file_list, parsed_url) - - if self.recursion_limit: - # Parse the dependencies of this dependency. - for s in self.dependencies: - work_queue.enqueue(s) @gclient_utils.lockedmethod def _run_is_done(self, file_list, parsed_url): @@ -1050,7 +1075,7 @@ solutions = [ print('Using safesync_url revision: %s.\n' % safe_rev) self._options.revisions.append('%s@%s' % (dep.name, safe_rev)) - def RunOnDeps(self, command, args): + def RunOnDeps(self, command, args, ignore_requirements=False, progress=True): """Runs a command on each dependency in a client and its dependencies. Args: @@ -1066,12 +1091,13 @@ solutions = [ revision_overrides = self._EnforceRevisions() pm = None # Disable progress for non-tty stdout. - if (sys.stdout.isatty() and not self._options.verbose): + if (sys.stdout.isatty() and not self._options.verbose and progress): if command in ('update', 'revert'): pm = Progress('Syncing projects', 1) elif command == 'recurse': pm = Progress(' '.join(args), 1) - work_queue = gclient_utils.ExecutionQueue(self._options.jobs, pm) + work_queue = gclient_utils.ExecutionQueue( + self._options.jobs, pm, ignore_requirements=ignore_requirements) for s in self.dependencies: work_queue.enqueue(s) work_queue.flush(revision_overrides, command, args, options=self._options) @@ -1119,7 +1145,7 @@ solutions = [ if not self.dependencies: raise gclient_utils.Error('No solution specified') # Load all the settings. - work_queue = gclient_utils.ExecutionQueue(self._options.jobs, None) + work_queue = gclient_utils.ExecutionQueue(self._options.jobs, None, False) for s in self.dependencies: work_queue.enqueue(s) work_queue.flush({}, None, [], options=self._options) @@ -1234,9 +1260,13 @@ def CMDrecurse(parser, args): # Stop parsing at the first non-arg so that these go through to the command parser.disable_interspersed_args() parser.add_option('-s', '--scm', action='append', default=[], - help='choose scm types to operate upon') + help='Choose scm types to operate upon.') parser.add_option('-i', '--ignore', action='store_true', - help='continue processing in case of non zero return code') + help='Ignore non-zero return codes from subcommands.') + parser.add_option('--prepend-dir', action='store_true', + help='Prepend relative dir for use with git --null.') + parser.add_option('--no-progress', action='store_true', + help='Disable progress bar that shows sub-command updates') options, args = parser.parse_args(args) if not args: print >> sys.stderr, 'Need to supply a command!' @@ -1256,7 +1286,8 @@ def CMDrecurse(parser, args): options.nohooks = True client = GClient.LoadCurrentConfig(options) - return client.RunOnDeps('recurse', args) + return client.RunOnDeps('recurse', args, ignore_requirements=True, + progress=not options.no_progress) @attr('usage', '[args ...]') @@ -1266,8 +1297,38 @@ def CMDfetch(parser, args): Completely git-specific. Simply runs 'git fetch [args ...]' for each module. """ (options, args) = parser.parse_args(args) - args = ['-j%d' % options.jobs, '-s', 'git', 'git', 'fetch'] + args - return CMDrecurse(parser, args) + return CMDrecurse(Parser(), [ + '--jobs=%d' % options.jobs, '--scm=git', 'git', 'fetch'] + args) + + +def CMDgrep(parser, args): + """Greps through git repos managed by gclient. + +Runs 'git grep [args...]' for each module. +""" + + # We can't use optparse because it will try to parse arguments sent + # to git grep and throw an error. :-( + if not args or re.match('(-h|--help)$', args[0]): + print >> sys.stderr, ( + 'Usage: gclient grep [-j ] git-grep-args...\n\n' + 'Example: "gclient grep -j10 -A2 RefCountedBase" runs\n"git grep ' + '-A2 RefCountedBase" on each of gclient\'s git\nrepos with up to ' + '10 jobs.\n\nBonus: page output by appending "|& less -FRSX" to the' + ' end of your query.' + ) + return 1 + + jobs_arg = ['--jobs=1'] + if re.match(r'(-j|--jobs=)\d+$', args[0]): + jobs_arg, args = args[:1], args[1:] + elif re.match(r'(-j|--jobs)$', args[0]): + jobs_arg, args = args[:2], args[2:] + + return CMDrecurse( + parser, + jobs_arg + ['--ignore', '--prepend-dir', '--no-progress', '--scm=git', + 'git', 'grep', '--null', '--color=Always'] + args) @attr('usage', '[url] [safesync url]') diff --git a/gclient_utils.py b/gclient_utils.py index 78fa7222a..0eeb185dc 100644 --- a/gclient_utils.py +++ b/gclient_utils.py @@ -192,29 +192,31 @@ def safe_makedirs(tree): raise -def CheckCallAndFilterAndHeader(args, always=False, **kwargs): +def CheckCallAndFilterAndHeader(args, always=False, header=None, **kwargs): """Adds 'header' support to CheckCallAndFilter. If |always| is True, a message indicating what is being done is printed to stdout all the time even if not output is generated. Otherwise the message header is printed only if the call generated any ouput. """ - stdout = kwargs.get('stdout', None) or sys.stdout + stdout = kwargs.setdefault('stdout', sys.stdout) + if header is None: + header = "\n________ running '%s' in '%s'\n" % ( + ' '.join(args), kwargs.get('cwd', '.')) + if always: - stdout.write('\n________ running \'%s\' in \'%s\'\n' - % (' '.join(args), kwargs.get('cwd', '.'))) + stdout.write(header) else: - filter_fn = kwargs.get('filter_fn', None) + filter_fn = kwargs.get('filter_fn') def filter_msg(line): if line is None: - stdout.write('\n________ running \'%s\' in \'%s\'\n' - % (' '.join(args), kwargs.get('cwd', '.'))) + stdout.write(header) elif filter_fn: filter_fn(line) kwargs['filter_fn'] = filter_msg kwargs['call_filter_on_first_line'] = True # Obviously. - kwargs['print_stdout'] = True + kwargs.setdefault('print_stdout', True) return CheckCallAndFilter(args, **kwargs) @@ -450,7 +452,7 @@ def PathDifference(root, subpath): def FindFileUpwards(filename, path=None): """Search upwards from the a directory (default: current) to find a file. - + Returns nearest upper-level directory with the passed in file. """ if not path: @@ -526,7 +528,7 @@ class ExecutionQueue(object): Methods of this class are thread safe. """ - def __init__(self, jobs, progress): + def __init__(self, jobs, progress, ignore_requirements): """jobs specifies the number of concurrent tasks to allow. progress is a Progress instance.""" # Set when a thread is done or a new item is enqueued. @@ -546,6 +548,8 @@ class ExecutionQueue(object): if self.progress: self.progress.update(0) + self.ignore_requirements = ignore_requirements + def enqueue(self, d): """Enqueue one Dependency to be executed later once its requirements are satisfied. @@ -583,11 +587,8 @@ class ExecutionQueue(object): # Check for new tasks to start. for i in xrange(len(self.queued)): # Verify its requirements. - for r in self.queued[i].requirements: - if not r in self.ran: - # Requirement not met. - break - else: + if (self.ignore_requirements or + not (set(self.queued[i].requirements) - set(self.ran))): # Start one work item: all its requirements are satisfied. self._run_one_task(self.queued.pop(i), args, kwargs) break diff --git a/tests/gclient_test.py b/tests/gclient_test.py index 35f2be29c..e3bf9fd6c 100755 --- a/tests/gclient_test.py +++ b/tests/gclient_test.py @@ -269,7 +269,7 @@ class GclientTest(trial_dir.TestCase): options, _ = parser.parse_args([]) options.force = True client = gclient.GClient.LoadCurrentConfig(options) - work_queue = gclient_utils.ExecutionQueue(options.jobs, None) + work_queue = gclient_utils.ExecutionQueue(options.jobs, None, False) for s in client.dependencies: work_queue.enqueue(s) work_queue.flush({}, None, [], options=options)