From e79161aa2d7ce63b9c53481f0bbfe1e08a3084bf Mon Sep 17 00:00:00 2001 From: "petermayo@chromium.org" Date: Tue, 9 Jul 2013 14:40:37 +0000 Subject: [PATCH] Add custom hooks. Sometimes we wish to pull in a complicated dependency but want to suppress or replace one or more of the hooks rules. Say for example we want to use a different way of generating the projects, or have a different set of landmine expectations. Here we add a custom_hooks section mirroring custom_deps to allow us to override sections we have identified in the DEPS file. To do so, we add an optional name to the elements of the hooks list, and overwrite those whose name matches. Conventions between included DEPS and the .gclient as to the meanings of the name are equivalent to the meaning of the customized deps, and so do not benefit from further structure or definition. BUG=None TEST=local unit test Review URL: https://chromiumcodereview.appspot.com/17742004 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@210573 0039d316-1c4b-4281-b951-d872f2087c98 --- gclient.py | 45 ++++++++++++++++++++++++------- tests/gclient_test.py | 62 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 91 insertions(+), 16 deletions(-) diff --git a/gclient.py b/gclient.py index 9cb89d286..d61540233 100755 --- a/gclient.py +++ b/gclient.py @@ -10,8 +10,9 @@ Files Format is a Python script defining 'solutions', a list whose entries each are maps binding the strings "name" and "url" to strings specifying the name and location of the client - module, as well as "custom_deps" to a map similar to the DEPS - file below. + module, as well as "custom_deps" to a map similar to the deps + section of the DEPS file below, as well as "custom_hooks" to + a list similar to the hooks sections of the DEPS file below. .gclient_entries : A cache constructed by 'update' command. Format is a Python script defining 'entries', a list of the names of all modules in the client @@ -24,8 +25,8 @@ Hooks working copy as a result of a "sync"/"update" or "revert" operation. This can be prevented by using --nohooks (hooks run by default). Hooks can also be forced to run with the "runhooks" operation. If "sync" is run with - --force, all known hooks will run regardless of the state of the working - copy. + --force, all known but not suppressed hooks will run regardless of the state + of the working copy. Each item in a "hooks" list is a dict, containing these two keys: "pattern" The associated value is a string containing a regular @@ -41,11 +42,16 @@ Hooks to run the command. If the list contains string "$matching_files" it will be removed from the list and the list will be extended by the list of matching files. + "name" An optional string specifying the group to which a hook belongs + for overriding and organizing. Example: hooks = [ { "pattern": "\\.(gif|jpe?g|pr0n|png)$", "action": ["python", "image_indexer.py", "--all"]}, + { "pattern": ".", + "name": "gyp", + "action": ["python", "src/build/gyp_chromium"]}, ] Specifying a target OS @@ -159,7 +165,7 @@ class DependencySettings(GClientKeywords): """Immutable configuration settings.""" def __init__( self, parent, url, safesync_url, managed, custom_deps, custom_vars, - deps_file, should_process): + custom_hooks, deps_file, should_process): GClientKeywords.__init__(self) # These are not mutable: @@ -186,6 +192,7 @@ class DependencySettings(GClientKeywords): # These are only set in .gclient and not in DEPS files. self._custom_vars = custom_vars or {} self._custom_deps = custom_deps or {} + self._custom_hooks = custom_hooks or [] # TODO(iannucci): Remove this when all masters are correctly substituting # the new blink url. @@ -246,6 +253,10 @@ class DependencySettings(GClientKeywords): def custom_deps(self): return self._custom_deps.copy() + @property + def custom_hooks(self): + return self._custom_hooks[:] + @property def url(self): return self._url @@ -276,11 +287,11 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): """Object that represents a dependency checkout.""" def __init__(self, parent, name, url, safesync_url, managed, custom_deps, - custom_vars, deps_file, should_process): + custom_vars, custom_hooks, deps_file, should_process): gclient_utils.WorkItem.__init__(self, name) DependencySettings.__init__( self, parent, url, safesync_url, managed, custom_deps, custom_vars, - deps_file, should_process) + custom_hooks, deps_file, should_process) # This is in both .gclient and DEPS files: self._deps_hooks = [] @@ -528,10 +539,23 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): for name, url in deps.iteritems(): should_process = self.recursion_limit and self.should_process deps_to_add.append(Dependency( - self, name, url, None, None, None, None, + self, name, url, None, None, None, None, None, self.deps_file, should_process)) deps_to_add.sort(key=lambda x: x.name) - self.add_dependencies_and_close(deps_to_add, local_scope.get('hooks', [])) + + # override named sets of hooks by the custom hooks + hooks_to_run = [] + hook_names_to_suppress = [c.get('name', '') for c in self.custom_hooks] + for hook in local_scope.get('hooks', []): + if hook.get('name', '') not in hook_names_to_suppress: + hooks_to_run.append(hook) + + # add the replacements and any additions + for hook in self.custom_hooks: + if 'action' in hook: + hooks_to_run.append(hook) + + self.add_dependencies_and_close(deps_to_add, hooks_to_run) logging.info('ParseDepsFile(%s) done' % self.name) def add_dependencies_and_close(self, deps_to_add, hooks): @@ -923,7 +947,7 @@ solutions = [ # Do not change previous behavior. Only solution level and immediate DEPS # are processed. self._recursion_limit = 2 - Dependency.__init__(self, None, None, None, None, True, None, None, + Dependency.__init__(self, None, None, None, None, True, None, None, None, 'unused', True) self._options = options if options.deps_os: @@ -967,6 +991,7 @@ solutions = [ s.get('managed', True), s.get('custom_deps', {}), s.get('custom_vars', {}), + s.get('custom_hooks', []), s.get('deps_file', 'DEPS'), True)) except KeyError: diff --git a/tests/gclient_test.py b/tests/gclient_test.py index e3bf9fd6c..822a35c79 100755 --- a/tests/gclient_test.py +++ b/tests/gclient_test.py @@ -208,7 +208,7 @@ class GclientTest(trial_dir.TestCase): # Invalid urls causes pain when specifying requirements. Make sure it's # auto-fixed. d = gclient.Dependency( - None, 'name', 'proto://host/path/@revision', None, None, None, + None, 'name', 'proto://host/path/@revision', None, None, None, None, None, '', True) self.assertEquals('proto://host/path@revision', d.url) @@ -219,24 +219,24 @@ class GclientTest(trial_dir.TestCase): obj.add_dependencies_and_close( [ gclient.Dependency( - obj, 'foo', 'url', None, None, None, None, 'DEPS', True), + obj, 'foo', 'url', None, None, None, None, None, 'DEPS', True), gclient.Dependency( - obj, 'bar', 'url', None, None, None, None, 'DEPS', True), + obj, 'bar', 'url', None, None, None, None, None, 'DEPS', True), ], []) obj.dependencies[0].add_dependencies_and_close( [ gclient.Dependency( obj.dependencies[0], 'foo/dir1', 'url', None, None, None, None, - 'DEPS', True), + None, 'DEPS', True), gclient.Dependency( obj.dependencies[0], 'foo/dir2', gclient.GClientKeywords.FromImpl('bar'), None, None, None, None, - 'DEPS', True), + None, 'DEPS', True), gclient.Dependency( obj.dependencies[0], 'foo/dir3', gclient.GClientKeywords.FileImpl('url'), None, None, None, None, - 'DEPS', True), + None, 'DEPS', True), ], []) # Make sure __str__() works fine. @@ -275,6 +275,56 @@ class GclientTest(trial_dir.TestCase): work_queue.flush({}, None, [], options=options) self.assertEqual(client.GetHooks(options), [x['action'] for x in hooks]) + def testCustomHooks(self): + topdir = self.root_dir + gclient_fn = os.path.join(topdir, '.gclient') + fh = open(gclient_fn, 'w') + extra_hooks = [{'name': 'append', 'pattern':'.', 'action':['supercmd']}] + print >> fh, ('solutions = [{"name":"top","url":"svn://svn.top.com/top",' + '"custom_hooks": %s},' ) % repr(extra_hooks + [{'name': 'skip'}]) + print >> fh, '{"name":"bottom","url":"svn://svn.top.com/bottom"}]' + fh.close() + subdir_fn = os.path.join(topdir, 'top') + os.mkdir(subdir_fn) + deps_fn = os.path.join(subdir_fn, 'DEPS') + fh = open(deps_fn, 'w') + hooks = [{'pattern':'.', 'action':['cmd1', 'arg1', 'arg2']}] + hooks.append({'pattern':'.', 'action':['cmd2', 'arg1', 'arg2']}) + skip_hooks = [ + {'name': 'skip', 'pattern':'.', 'action':['cmd3', 'arg1', 'arg2']}] + skip_hooks.append( + {'name': 'skip', 'pattern':'.', 'action':['cmd4', 'arg1', 'arg2']}) + print >> fh, 'hooks = %s' % repr(hooks + skip_hooks) + fh.close() + + # Make sure the custom hooks for that project don't affect the next one. + subdir_fn = os.path.join(topdir, 'bottom') + os.mkdir(subdir_fn) + deps_fn = os.path.join(subdir_fn, 'DEPS') + fh = open(deps_fn, 'w') + sub_hooks = [{'pattern':'.', 'action':['response1', 'yes1', 'yes2']}] + sub_hooks.append( + {'name': 'skip', 'pattern':'.', 'action':['response2', 'yes', 'sir']}) + print >> fh, 'hooks = %s' % repr(sub_hooks) + fh.close() + + fh = open(os.path.join(subdir_fn, 'fake.txt'), 'w') + print >> fh, 'bogus content' + fh.close() + + os.chdir(topdir) + + parser = gclient.Parser() + options, _ = parser.parse_args([]) + options.force = True + client = gclient.GClient.LoadCurrentConfig(options) + work_queue = gclient_utils.ExecutionQueue(options.jobs, None, False) + for s in client.dependencies: + work_queue.enqueue(s) + work_queue.flush({}, None, [], options=options) + self.assertEqual(client.GetHooks(options), + [x['action'] for x in hooks + extra_hooks + sub_hooks]) + def testTargetOS(self): """Verifies that specifying a target_os pulls in all relevant dependencies.