From 6f64a0522bb8b6d20686028768024ebce5cfbba9 Mon Sep 17 00:00:00 2001 From: Edward Lesmes Date: Tue, 20 Mar 2018 17:35:49 -0400 Subject: [PATCH] Reland "gclient: Add commands to edit dependencies and variables in DEPS" This is a reland of 7f4c905fc53e7cbcc3277074c6a339ade8bc0f66 Original change's description: > gclient: Add commands to edit dependencies and variables in DEPS > > Adds 'gclient setvar' and 'gclient setdep' commands to edit variables > and dependencies in a DEPS file. > > Bug: 760633 > Change-Id: I6c0712cc079dbbbaee6541b7eda71f4b4813b77b > Reviewed-on: https://chromium-review.googlesource.com/950405 > Commit-Queue: Edward Lesmes > Reviewed-by: Aaron Gable Bug: 760633 Change-Id: Ia46c74d02e5cc3b67517dfa248f597cb3d98ef3d Reviewed-on: https://chromium-review.googlesource.com/969457 Commit-Queue: Edward Lesmes Reviewed-by: Aaron Gable --- gclient.py | 56 ++++++++++ gclient_eval.py | 197 ++++++++++++++++++++++++++++----- gclient_utils.py | 2 +- tests/gclient_eval_unittest.py | 103 ++++++++++++++++- tests/gclient_test.py | 40 +++++++ 5 files changed, 366 insertions(+), 32 deletions(-) diff --git a/gclient.py b/gclient.py index b35d98ee7..3169791d9 100755 --- a/gclient.py +++ b/gclient.py @@ -2859,6 +2859,62 @@ def CMDrevinfo(parser, args): return 0 +def CMDsetdep(parser, args): + parser.add_option('--var', action='append', + dest='vars', metavar='VAR=VAL', default=[], + help='Sets a variable to the given value with the format ' + 'name=value.') + parser.add_option('-r', '--revision', action='append', + dest='revisions', metavar='DEP@REV', default=[], + help='Sets the revision/version for the dependency with ' + 'the format dep@rev. If it is a git dependency, dep ' + 'must be a path and rev must be a git hash or ' + 'reference (e.g. src/dep@deadbeef). If it is a CIPD ' + 'dependency, dep must be of the form path:package and ' + 'rev must be the package version ' + '(e.g. src/pkg:chromium/pkg@2.1-cr0).') + parser.add_option('--deps-file', default='DEPS', + # TODO(ehmaldonado): Try to find the DEPS file pointed by + # .gclient first. + help='The DEPS file to be edited. Defaults to the DEPS ' + 'file in the current directory.') + (options, args) = parser.parse_args(args) + + global_scope = {'Var': lambda var: '{%s}' % var} + + if not os.path.isfile(options.deps_file): + raise gclient_utils.Error( + 'DEPS file %s does not exist.' % options.deps_file) + with open(options.deps_file) as f: + contents = f.read() + local_scope = gclient_eval.Exec(contents, global_scope, {}) + + for var in options.vars: + name, _, value = var.partition('=') + if not name or not value: + raise gclient_utils.Error( + 'Wrong var format: %s should be of the form name=value.' % var) + gclient_eval.SetVar(local_scope, name, value) + + for revision in options.revisions: + name, _, value = revision.partition('@') + if not name or not value: + raise gclient_utils.Error( + 'Wrong dep format: %s should be of the form dep@rev.' % revision) + if ':' in name: + name, _, package = name.partition(':') + if not name or not package: + raise gclient_utils.Error( + 'Wrong CIPD format: %s:%s should be of the form path:pkg@version.' + % (name, package)) + gclient_eval.SetCIPD(local_scope, name, package, value) + else: + gclient_eval.SetRevision(local_scope, global_scope, name, value) + + with open(options.deps_file, 'w') as f: + f.write(gclient_eval.RenderDEPSFile(local_scope)) + + def CMDverify(parser, args): """Verifies the DEPS file deps are only from allowed_hosts.""" (options, args) = parser.parse_args(args) diff --git a/gclient_eval.py b/gclient_eval.py index 38d37e672..2cdf9e7a0 100644 --- a/gclient_eval.py +++ b/gclient_eval.py @@ -3,17 +3,58 @@ # found in the LICENSE file. import ast +import cStringIO import collections +import tokenize from third_party import schema +class _NodeDict(collections.MutableMapping): + """Dict-like type that also stores information on AST nodes and tokens.""" + def __init__(self, data, tokens=None): + self.data = collections.OrderedDict(data) + self.tokens = tokens + + def __str__(self): + return str({k: v[0] for k, v in self.data.iteritems()}) + + def __getitem__(self, key): + return self.data[key][0] + + def __setitem__(self, key, value): + self.data[key] = (value, None) + + def __delitem__(self, key): + del self.data[key] + + def __iter__(self): + return iter(self.data) + + def __len__(self): + return len(self.data) + + def GetNode(self, key): + return self.data[key][1] + + def _SetNode(self, key, value, node): + self.data[key] = (value, node) + + +def _NodeDictSchema(dict_schema): + """Validate dict_schema after converting _NodeDict to a regular dict.""" + def validate(d): + schema.Schema(dict_schema).validate(dict(d)) + return True + return validate + + # See https://github.com/keleshev/schema for docs how to configure schema. -_GCLIENT_DEPS_SCHEMA = { +_GCLIENT_DEPS_SCHEMA = _NodeDictSchema({ schema.Optional(basestring): schema.Or( None, basestring, - { + _NodeDictSchema({ # Repo and revision to check out under the path # (same as if no dict was used). 'url': basestring, @@ -23,25 +64,25 @@ _GCLIENT_DEPS_SCHEMA = { schema.Optional('condition'): basestring, schema.Optional('dep_type', default='git'): basestring, - }, + }), # CIPD package. - { + _NodeDictSchema({ 'packages': [ - { + _NodeDictSchema({ 'package': basestring, 'version': basestring, - } + }) ], schema.Optional('condition'): basestring, schema.Optional('dep_type', default='cipd'): basestring, - }, + }), ), -} +}) -_GCLIENT_HOOKS_SCHEMA = [{ +_GCLIENT_HOOKS_SCHEMA = [_NodeDictSchema({ # Hook action: list of command-line arguments to invoke. 'action': [basestring], @@ -59,9 +100,9 @@ _GCLIENT_HOOKS_SCHEMA = [{ # Optional condition string. The hook will only be run # if the condition evaluates to True. schema.Optional('condition'): basestring, -}] +})] -_GCLIENT_SCHEMA = schema.Schema({ +_GCLIENT_SCHEMA = schema.Schema(_NodeDictSchema({ # List of host names from which dependencies are allowed (whitelist). # NOTE: when not present, all hosts are allowed. # NOTE: scoped to current DEPS file, not recursive. @@ -79,9 +120,9 @@ _GCLIENT_SCHEMA = schema.Schema({ # Similar to 'deps' (see above) - also keyed by OS (e.g. 'linux'). # Also see 'target_os'. - schema.Optional('deps_os'): { + schema.Optional('deps_os'): _NodeDictSchema({ schema.Optional(basestring): _GCLIENT_DEPS_SCHEMA, - }, + }), # Path to GN args file to write selected variables. schema.Optional('gclient_gn_args_file'): basestring, @@ -95,9 +136,9 @@ _GCLIENT_SCHEMA = schema.Schema({ schema.Optional('hooks'): _GCLIENT_HOOKS_SCHEMA, # Similar to 'hooks', also keyed by OS. - schema.Optional('hooks_os'): { + schema.Optional('hooks_os'): _NodeDictSchema({ schema.Optional(basestring): _GCLIENT_HOOKS_SCHEMA - }, + }), # Rules which #includes are allowed in the directory. # Also see 'skip_child_includes' and 'specific_include_rules'. @@ -123,9 +164,9 @@ _GCLIENT_SCHEMA = schema.Schema({ # Mapping from paths to include rules specific for that path. # See 'include_rules' for more details. - schema.Optional('specific_include_rules'): { + schema.Optional('specific_include_rules'): _NodeDictSchema({ schema.Optional(basestring): [basestring] - }, + }), # List of additional OS names to consider when selecting dependencies # from deps_os. @@ -136,10 +177,10 @@ _GCLIENT_SCHEMA = schema.Schema({ schema.Optional('use_relative_paths'): bool, # Variables that can be referenced using Var() - see 'deps'. - schema.Optional('vars'): { + schema.Optional('vars'): _NodeDictSchema({ schema.Optional(basestring): schema.Or(basestring, bool), - }, -}) + }), +})) def _gclient_eval(node_or_string, global_scope, filename=''): @@ -159,8 +200,7 @@ def _gclient_eval(node_or_string, global_scope, filename=''): elif isinstance(node, ast.List): return list(map(_convert, node.elts)) elif isinstance(node, ast.Dict): - return collections.OrderedDict( - (_convert(k), _convert(v)) + return _NodeDict((_convert(k), (_convert(v), v)) for k, v in zip(node.keys, node.values)) elif isinstance(node, ast.Name): if node.id not in _allowed_names: @@ -197,6 +237,7 @@ def Exec(content, global_scope, local_scope, filename=''): if isinstance(node_or_string, ast.Expression): node_or_string = node_or_string.body + defined_variables = set() def _visit_in_module(node): if isinstance(node, ast.Assign): if len(node.targets) != 1: @@ -210,12 +251,13 @@ def Exec(content, global_scope, local_scope, filename=''): filename, getattr(node, 'lineno', ''))) value = _gclient_eval(node.value, global_scope, filename=filename) - if target.id in local_scope: + if target.id in defined_variables: raise ValueError( 'invalid assignment: overrides var %r (file %r, line %s)' % ( target.id, filename, getattr(node, 'lineno', ''))) - local_scope[target.id] = value + defined_variables.add(target.id) + return target.id, (value, node.value) else: raise ValueError( 'unexpected AST node: %s %s (file %r, line %s)' % ( @@ -223,8 +265,15 @@ def Exec(content, global_scope, local_scope, filename=''): getattr(node, 'lineno', ''))) if isinstance(node_or_string, ast.Module): + data = [] for stmt in node_or_string.body: - _visit_in_module(stmt) + data.append(_visit_in_module(stmt)) + tokens = { + token[2]: list(token) + for token in tokenize.generate_tokens( + cStringIO.StringIO(content).readline) + } + local_scope = _NodeDict(data, tokens) else: raise ValueError( 'unexpected AST node: %s %s (file %r, line %s)' % ( @@ -333,3 +382,101 @@ def EvaluateCondition(condition, variables, referenced_variables=None): 'unexpected AST node: %s %s (inside %r)' % ( node, ast.dump(node), condition)) return _convert(main_node) + + +def RenderDEPSFile(gclient_dict): + contents = sorted(gclient_dict.tokens.values(), key=lambda token: token[2]) + return tokenize.untokenize(contents) + + +def _UpdateAstString(tokens, node, value): + position = node.lineno, node.col_offset + tokens[position][1] = repr(value) + node.s = value + + +def SetVar(gclient_dict, var_name, value): + if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None: + raise ValueError( + "Can't use SetVar for the given gclient dict. It contains no " + "formatting information.") + tokens = gclient_dict.tokens + + if 'vars' not in gclient_dict or var_name not in gclient_dict['vars']: + raise ValueError( + "Could not find any variable called %s." % var_name) + + node = gclient_dict['vars'].GetNode(var_name) + if node is None: + raise ValueError( + "The vars entry for %s has no formatting information." % var_name) + + _UpdateAstString(tokens, node, value) + gclient_dict['vars']._SetNode(var_name, value, node) + + +def SetCIPD(gclient_dict, dep_name, package_name, new_version): + if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None: + raise ValueError( + "Can't use SetCIPD for the given gclient dict. It contains no " + "formatting information.") + tokens = gclient_dict.tokens + + if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']: + raise ValueError( + "Could not find any dependency called %s." % dep_name) + + # Find the package with the given name + packages = [ + package + for package in gclient_dict['deps'][dep_name]['packages'] + if package['package'] == package_name + ] + if len(packages) != 1: + raise ValueError( + "There must be exactly one package with the given name (%s), " + "%s were found." % (package_name, len(packages))) + + # TODO(ehmaldonado): Support Var in package's version. + node = packages[0].GetNode('version') + if node is None: + raise ValueError( + "The deps entry for %s:%s has no formatting information." % + (dep_name, package_name)) + + new_version = 'version:' + new_version + _UpdateAstString(tokens, node, new_version) + packages[0]._SetNode('version', new_version, node) + + +def SetRevision(gclient_dict, global_scope, dep_name, new_revision): + if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None: + raise ValueError( + "Can't use SetRevision for the given gclient dict. It contains no " + "formatting information.") + tokens = gclient_dict.tokens + + if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']: + raise ValueError( + "Could not find any dependency called %s." % dep_name) + + def _UpdateRevision(dep_dict, dep_key): + dep_node = dep_dict.GetNode(dep_key) + if dep_node is None: + raise ValueError( + "The deps entry for %s has no formatting information." % dep_name) + + node = dep_node + if isinstance(node, ast.BinOp): + node = node.right + if isinstance(node, ast.Call): + SetVar(gclient_dict, node.args[0].s, new_revision) + else: + _UpdateAstString(tokens, node, new_revision) + value = _gclient_eval(dep_node, global_scope) + dep_dict._SetNode(dep_key, value, dep_node) + + if isinstance(gclient_dict['deps'][dep_name], _NodeDict): + _UpdateRevision(gclient_dict['deps'][dep_name], 'url') + else: + _UpdateRevision(gclient_dict['deps'], dep_name) diff --git a/gclient_utils.py b/gclient_utils.py index b67916935..81bed14b6 100644 --- a/gclient_utils.py +++ b/gclient_utils.py @@ -1278,7 +1278,7 @@ def freeze(obj): Will raise TypeError if you pass an object which is not hashable. """ - if isinstance(obj, dict): + if isinstance(obj, collections.Mapping): return FrozenDict((freeze(k), freeze(v)) for k, v in obj.iteritems()) elif isinstance(obj, (list, tuple)): return tuple(freeze(i) for i in obj) diff --git a/tests/gclient_eval_unittest.py b/tests/gclient_eval_unittest.py index bc0b5923d..28380add1 100755 --- a/tests/gclient_eval_unittest.py +++ b/tests/gclient_eval_unittest.py @@ -8,6 +8,7 @@ import itertools import logging import os import sys +import textwrap import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -18,6 +19,45 @@ import gclient import gclient_eval +_SAMPLE_DEPS_FILE = textwrap.dedent("""\ +deps = { + 'src/dep': Var('git_repo') + '/dep' + '@' + 'deadbeef', + # Some comment + 'src/android/dep_2': { + 'url': Var('git_repo') + '/dep_2' + '@' + Var('dep_2_rev'), + 'condition': 'checkout_android', + }, + + 'src/dep_3': Var('git_repo') + '/dep_3@' + Var('dep_3_rev'), + + 'src/cipd/package': { + 'packages': [ + { + 'package': 'some/cipd/package', + 'version': 'version:1234', + }, + { + 'package': 'another/cipd/package', + 'version': 'version:5678', + }, + ], + 'condition': 'checkout_android', + 'dep_type': 'cipd', + }, +} + +vars = { + 'git_repo': 'https://example.com/repo.git', + # Some comment with bad indentation + 'dep_2_rev': '1ced', + # Some more comments + # 1 + # 2 + # 3 + 'dep_3_rev': '5p1e5', +}""") + + class GClientEvalTest(unittest.TestCase): def test_str(self): self.assertEqual('foo', gclient_eval._gclient_eval('"foo"', {})) @@ -82,17 +122,13 @@ class ExecTest(unittest.TestCase): self.assertIn( 'invalid assignment: overrides var \'a\'', str(cm.exception)) - def test_schema_unknown_key(self): - with self.assertRaises(schema.SchemaWrongKeyError): - gclient_eval.Exec('foo = "bar"', {}, {}, '') - def test_schema_wrong_type(self): with self.assertRaises(schema.SchemaError): gclient_eval.Exec('include_rules = {}', {}, {}, '') def test_recursedeps_list(self): local_scope = {} - gclient_eval.Exec( + local_scope = gclient_eval.Exec( 'recursedeps = [["src/third_party/angle", "DEPS.chromium"]]', {}, local_scope, '') @@ -105,7 +141,7 @@ class ExecTest(unittest.TestCase): global_scope = { 'Var': lambda var_name: '{%s}' % var_name, } - gclient_eval.Exec('\n'.join([ + local_scope = gclient_eval.Exec('\n'.join([ 'vars = {', ' "foo": "bar",', '}', @@ -118,6 +154,10 @@ class ExecTest(unittest.TestCase): 'deps': collections.OrderedDict([('a_dep', 'a{foo}b')]), }, local_scope) + def test_empty_deps(self): + local_scope = gclient_eval.Exec('deps = {}', {}, {}, '') + self.assertEqual({'deps': {}}, local_scope) + class EvaluateConditionTest(unittest.TestCase): def test_true(self): @@ -169,6 +209,57 @@ class EvaluateConditionTest(unittest.TestCase): str(cm.exception)) +class SetVarTest(unittest.TestCase): + def testSetVar(self): + local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, {'Var': str}, {}) + + gclient_eval.SetVar(local_scope, 'dep_2_rev', 'c0ffee') + result = gclient_eval.RenderDEPSFile(local_scope) + + self.assertEqual( + result, + _SAMPLE_DEPS_FILE.replace('1ced', 'c0ffee')) + + +class SetCipdTest(unittest.TestCase): + def testSetCIPD(self): + local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, {'Var': str}, {}) + + gclient_eval.SetCIPD( + local_scope, 'src/cipd/package', 'another/cipd/package', '6.789') + result = gclient_eval.RenderDEPSFile(local_scope) + + self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('5678', '6.789')) + + +class SetRevisionTest(unittest.TestCase): + def setUp(self): + self.global_scope = {'Var': str} + self.local_scope = gclient_eval.Exec( + _SAMPLE_DEPS_FILE, self.global_scope, {}) + + def testSetRevision(self): + gclient_eval.SetRevision( + self.local_scope, self.global_scope, 'src/dep', 'deadfeed') + result = gclient_eval.RenderDEPSFile(self.local_scope) + + self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('deadbeef', 'deadfeed')) + + def testSetRevisionInUrl(self): + gclient_eval.SetRevision( + self.local_scope, self.global_scope, 'src/dep_3', '0ff1ce') + result = gclient_eval.RenderDEPSFile(self.local_scope) + + self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('5p1e5', '0ff1ce')) + + def testSetRevisionInVars(self): + gclient_eval.SetRevision( + self.local_scope, self.global_scope, 'src/android/dep_2', 'c0ffee') + result = gclient_eval.RenderDEPSFile(self.local_scope) + + self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('1ced', 'c0ffee')) + + if __name__ == '__main__': level = logging.DEBUG if '-v' in sys.argv else logging.FATAL logging.basicConfig( diff --git a/tests/gclient_test.py b/tests/gclient_test.py index 98f24391e..d53fbb25a 100755 --- a/tests/gclient_test.py +++ b/tests/gclient_test.py @@ -1155,6 +1155,46 @@ class GclientTest(trial_dir.TestCase): finally: self._get_processed() + def testCreatesCipdDependencies(self): + """Verifies something.""" + write( + '.gclient', + 'solutions = [\n' + ' { "name": "foo", "url": "svn://example.com/foo",\n' + ' "deps_file" : ".DEPS.git",\n' + ' },\n' + ']') + write( + os.path.join('foo', 'DEPS'), + 'vars = {\n' + ' "lemur_version": "version:1234",\n' + '}\n' + 'deps = {\n' + ' "bar": {\n' + ' "packages": [{\n' + ' "package": "lemur",\n' + ' "version": Var("lemur_version"),\n' + ' }],\n' + ' "dep_type": "cipd",\n' + ' }\n' + '}') + options, _ = gclient.OptionParser().parse_args([]) + options.validate_syntax = True + obj = gclient.GClient.LoadCurrentConfig(options) + + self.assertEquals(1, len(obj.dependencies)) + sol = obj.dependencies[0] + sol._condition = 'some_condition' + + sol.ParseDepsFile() + self.assertEquals(1, len(sol.dependencies)) + dep = sol.dependencies[0] + + self.assertIsInstance(dep, gclient.CipdDependency) + self.assertEquals( + 'https://chrome-infra-packages.appspot.com/lemur@version:1234', + dep.url) + def testSameDirAllowMultipleCipdDeps(self): """Verifies gclient allow multiple cipd deps under same directory.""" parser = gclient.OptionParser()