diff --git a/gclient.py b/gclient.py index c4bf01176..230ce7878 100755 --- a/gclient.py +++ b/gclient.py @@ -736,7 +736,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): deps_to_add.sort(key=lambda x: x.name) return deps_to_add - def ParseDepsFile(self): + def ParseDepsFile(self, expand_vars=True): """Parses the DEPS file for this dependency.""" assert not self.deps_parsed assert not self.dependencies @@ -765,15 +765,14 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): local_scope = {} if deps_content: - # Eval the content. try: - if self._get_option('validate_syntax', False): - local_scope = gclient_eval.Exec(deps_content, filepath) - else: - global_scope = { - 'Var': lambda var_name: '{%s}' % var_name, - } - exec(deps_content, global_scope, local_scope) + vars_override = self.get_vars() + if self.parent: + vars_override.update(self.parent.get_vars()) + local_scope = gclient_eval.Parse( + deps_content, expand_vars, + self._get_option('validate_syntax', False), + filepath, vars_override) except SyntaxError as e: gclient_utils.SyntaxErrorToError(filepath, e) @@ -988,7 +987,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): file_list[i] = file_list[i][1:] # Always parse the DEPS file. - self.ParseDepsFile() + self.ParseDepsFile(expand_vars=(command != 'flatten')) self._run_is_done(file_list or [], parsed_url) if command in ('update', 'revert') and not options.noprehooks: self.RunPreDepsHooks() @@ -1864,7 +1863,7 @@ it or fix the checkout. print('%s: %s' % (x, entries[x])) logging.info(str(self)) - def ParseDepsFile(self): + def ParseDepsFile(self, expand_vars=None): """No DEPS to parse for a .gclient file.""" raise gclient_utils.Error('Internal error') @@ -1973,7 +1972,7 @@ class CipdDependency(Dependency): self._cipd_package = self._cipd_root.add_package( self._cipd_subdir, self._package_name, self._package_version) - def ParseDepsFile(self): + def ParseDepsFile(self, expand_vars=None): """CIPD dependencies are not currently allowed to have nested deps.""" self.add_dependencies_and_close([], []) @@ -2930,7 +2929,9 @@ def CMDsetdep(parser, args): 'DEPS file %s does not exist.' % options.deps_file) with open(options.deps_file) as f: contents = f.read() - local_scope = gclient_eval.Exec(contents) + local_scope = gclient_eval.Parse( + contents, expand_vars=True, validate_syntax=True, + filename=options.deps_file) for var in options.vars: name, _, value = var.partition('=') diff --git a/gclient_eval.py b/gclient_eval.py index 081c96fdb..0e4d5457f 100644 --- a/gclient_eval.py +++ b/gclient_eval.py @@ -37,7 +37,7 @@ class _NodeDict(collections.MutableMapping): def GetNode(self, key): return self.data[key][1] - def _SetNode(self, key, value, node): + def SetNode(self, key, value, node): self.data[key] = (value, node) @@ -183,8 +183,9 @@ _GCLIENT_SCHEMA = schema.Schema(_NodeDictSchema({ })) -def _gclient_eval(node_or_string, filename=''): +def _gclient_eval(node_or_string, vars_dict, expand_vars, filename): """Safely evaluates a single expression. Returns the result.""" + vars_dict = vars_dict or {} _allowed_names = {'None': None, 'True': True, 'False': False} if isinstance(node_or_string, basestring): node_or_string = ast.parse(node_or_string, filename=filename, mode='eval') @@ -192,7 +193,15 @@ def _gclient_eval(node_or_string, filename=''): node_or_string = node_or_string.body def _convert(node): if isinstance(node, ast.Str): - return node.s + if not expand_vars: + return node.s + try: + return node.s.format(**vars_dict) + except KeyError as e: + raise ValueError( + '%s was used as a variable, but was not declared in the vars dict ' + '(file %r, line %s)' % ( + e.message, filename, getattr(node, 'lineno', ''))) elif isinstance(node, ast.Num): return node.n elif isinstance(node, ast.Tuple): @@ -222,7 +231,18 @@ def _gclient_eval(node_or_string, filename=''): raise ValueError( 'Var\'s argument must be a variable name (file %r, line %s)' % ( filename, getattr(node, 'lineno', ''))) - return '{%s}' % arg + if not expand_vars: + return '{%s}' % arg + if vars_dict is None: + raise ValueError( + 'vars must be declared before Var can be used (file %r, line %s)' + % (filename, getattr(node, 'lineno', ''))) + if arg not in vars_dict: + raise ValueError( + '%s was used as a variable, but was not declared in the vars dict ' + '(file %r, line %s)' % ( + arg, filename, getattr(node, 'lineno', ''))) + return vars_dict[arg] elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): return _convert(node.left) + _convert(node.right) elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Mod): @@ -235,13 +255,18 @@ def _gclient_eval(node_or_string, filename=''): return _convert(node_or_string) -def Exec(content, filename=''): - """Safely execs a set of assignments. Mutates |local_scope|.""" +def Exec(content, expand_vars=True, filename=''): + """Safely execs a set of assignments.""" node_or_string = ast.parse(content, filename=filename, mode='exec') if isinstance(node_or_string, ast.Expression): node_or_string = node_or_string.body - defined_variables = set() + tokens = { + token[2]: list(token) + for token in tokenize.generate_tokens( + cStringIO.StringIO(content).readline) + } + local_scope = _NodeDict({}, tokens) def _visit_in_module(node): if isinstance(node, ast.Assign): if len(node.targets) != 1: @@ -253,15 +278,15 @@ def Exec(content, filename=''): raise ValueError( 'invalid assignment: target should be a name (file %r, line %s)' % ( filename, getattr(node, 'lineno', ''))) - value = _gclient_eval(node.value, filename=filename) + value = _gclient_eval(node.value, local_scope.get('vars', None), + expand_vars, filename) - if target.id in defined_variables: + if target.id in local_scope: raise ValueError( 'invalid assignment: overrides var %r (file %r, line %s)' % ( target.id, filename, getattr(node, 'lineno', ''))) - defined_variables.add(target.id) - return target.id, (value, node.value) + local_scope.SetNode(target.id, value, node.value) else: raise ValueError( 'unexpected AST node: %s %s (file %r, line %s)' % ( @@ -269,15 +294,8 @@ def Exec(content, filename=''): getattr(node, 'lineno', ''))) if isinstance(node_or_string, ast.Module): - data = [] for stmt in node_or_string.body: - data.append(_visit_in_module(stmt)) - tokens = { - token[2]: list(token) - for token in tokenize.generate_tokens( - cStringIO.StringIO(content).readline) - } - local_scope = _NodeDict(data, tokens) + _visit_in_module(stmt) else: raise ValueError( 'unexpected AST node: %s %s (file %r, line %s)' % ( @@ -289,6 +307,69 @@ def Exec(content, filename=''): return _GCLIENT_SCHEMA.validate(local_scope) +def Parse(content, expand_vars, validate_syntax, filename, vars_override=None): + """Parses DEPS strings. + + Executes the Python-like string stored in content, resulting in a Python + dictionary specifyied by the schema above. Supports syntax validation and + variable expansion. + + Args: + content: str. DEPS file stored as a string. + expand_vars: bool. Whether variables should be expanded to their values. + validate_syntax: bool. Whether syntax should be validated using the schema + defined above. + filename: str. The name of the DEPS file, or a string describing the source + of the content, e.g. '', ''. + vars_override: dict, optional. A dictionary with overrides for the variables + defined by the DEPS file. + + Returns: + A Python dict with the parsed contents of the DEPS file, as specified by the + schema above. + """ + # TODO(ehmaldonado): Make validate_syntax = True the only case + if validate_syntax: + return Exec(content, expand_vars, filename) + + vars_dict = {} + def _DeepFormat(node): + if isinstance(node, basestring): + return node.format(**vars_dict) + elif isinstance(node, dict): + return { + k.format(**vars_dict): _DeepFormat(v) + for k, v in node.iteritems() + } + elif isinstance(node, list): + return [ + _DeepFormat(elem) + for elem in node + ] + elif isinstance(node, tuple): + return tuple( + _DeepFormat(elem) + for elem in node + ) + else: + return node + + local_scope = {} + global_scope = {'Var': lambda var_name: '{%s}' % var_name} + exec(content, global_scope, local_scope) + + if 'vars' not in local_scope or not expand_vars: + return local_scope + + vars_dict.update(local_scope['vars']) + vars_override = vars_override or {} + for var, value in vars_override.iteritems(): + if var in vars_dict: + vars_dict[var] = value + + return _DeepFormat(local_scope) + + def EvaluateCondition(condition, variables, referenced_variables=None): """Safely evaluates a boolean condition. Returns the result.""" if not referenced_variables: @@ -416,7 +497,7 @@ def SetVar(gclient_dict, var_name, value): "The vars entry for %s has no formatting information." % var_name) _UpdateAstString(tokens, node, value) - gclient_dict['vars']._SetNode(var_name, value, node) + gclient_dict['vars'].SetNode(var_name, value, node) def SetCIPD(gclient_dict, dep_name, package_name, new_version): @@ -450,7 +531,7 @@ def SetCIPD(gclient_dict, dep_name, package_name, new_version): new_version = 'version:' + new_version _UpdateAstString(tokens, node, new_version) - packages[0]._SetNode('version', new_version, node) + packages[0].SetNode('version', new_version, node) def SetRevision(gclient_dict, dep_name, new_revision): @@ -477,8 +558,9 @@ def SetRevision(gclient_dict, dep_name, new_revision): SetVar(gclient_dict, node.args[0].s, new_revision) else: _UpdateAstString(tokens, node, new_revision) - value = _gclient_eval(dep_node) - dep_dict._SetNode(dep_key, value, dep_node) + value = _gclient_eval(dep_node, gclient_dict.get('vars', None), + expand_vars=True, filename='') + dep_dict.SetNode(dep_key, value, dep_node) if isinstance(gclient_dict['deps'][dep_name], _NodeDict): _UpdateRevision(gclient_dict['deps'][dep_name], 'url') diff --git a/tests/gclient_eval_unittest.py b/tests/gclient_eval_unittest.py index 7f841d692..f50e5f58e 100755 --- a/tests/gclient_eval_unittest.py +++ b/tests/gclient_eval_unittest.py @@ -20,6 +20,17 @@ import gclient_eval _SAMPLE_DEPS_FILE = textwrap.dedent("""\ +vars = { + 'git_repo': 'https://example.com/repo.git', + # Some comment with bad indentation + 'dep_2_rev': '1ced', + # Some more comments + # 1 + # 2 + # 3 + 'dep_3_rev': '5p1e5', +} + deps = { 'src/dep': Var('git_repo') + '/dep' + '@' + 'deadbeef', # Some comment @@ -45,59 +56,82 @@ deps = { 'dep_type': 'cipd', }, } - -vars = { - 'git_repo': 'https://example.com/repo.git', - # Some comment with bad indentation - 'dep_2_rev': '1ced', - # Some more comments - # 1 - # 2 - # 3 - 'dep_3_rev': '5p1e5', -}""") +""") class GClientEvalTest(unittest.TestCase): def test_str(self): - self.assertEqual('foo', gclient_eval._gclient_eval('"foo"')) + self.assertEqual( + 'foo', + gclient_eval._gclient_eval('"foo"', vars_dict=None, expand_vars=False, + filename='')) def test_tuple(self): - self.assertEqual(('a', 'b'), gclient_eval._gclient_eval('("a", "b")')) + self.assertEqual( + ('a', 'b'), + gclient_eval._gclient_eval('("a", "b")', vars_dict=None, + expand_vars=False, filename='')) def test_list(self): - self.assertEqual(['a', 'b'], gclient_eval._gclient_eval('["a", "b"]')) + self.assertEqual( + ['a', 'b'], + gclient_eval._gclient_eval('["a", "b"]', vars_dict=None, + expand_vars=False, filename='')) def test_dict(self): - self.assertEqual({'a': 'b'}, gclient_eval._gclient_eval('{"a": "b"}')) + self.assertEqual( + {'a': 'b'}, + gclient_eval._gclient_eval('{"a": "b"}', vars_dict=None, + expand_vars=False, filename='')) def test_name_safe(self): - self.assertEqual(True, gclient_eval._gclient_eval('True')) + self.assertEqual( + True, + gclient_eval._gclient_eval('True', vars_dict=None, + expand_vars=False, filename='')) def test_name_unsafe(self): with self.assertRaises(ValueError) as cm: - gclient_eval._gclient_eval('UnsafeName') + gclient_eval._gclient_eval('UnsafeName', vars_dict=None, + expand_vars=False, filename='') self.assertIn('invalid name \'UnsafeName\'', str(cm.exception)) + def test_invalid_call(self): + with self.assertRaises(ValueError) as cm: + gclient_eval._gclient_eval('Foo("bar")', vars_dict=None, + expand_vars=False, filename='') + self.assertIn('Var is the only allowed function', str(cm.exception)) + def test_call(self): self.assertEqual( '{bar}', - gclient_eval._gclient_eval('Var("bar")')) + gclient_eval._gclient_eval('Var("bar")', vars_dict=None, + expand_vars=False, filename='')) def test_plus(self): - self.assertEqual('foo', gclient_eval._gclient_eval('"f" + "o" + "o"')) + self.assertEqual( + 'foo', + gclient_eval._gclient_eval('"f" + "o" + "o"', vars_dict=None, + expand_vars=False, filename='')) def test_format(self): - self.assertEqual('foo', gclient_eval._gclient_eval('"%s" % "foo"')) + self.assertEqual( + 'foo', + gclient_eval._gclient_eval('"%s" % "foo"', vars_dict=None, + expand_vars=False, filename='')) def test_not_expression(self): with self.assertRaises(SyntaxError) as cm: - gclient_eval._gclient_eval('def foo():\n pass') + gclient_eval._gclient_eval( + 'def foo():\n pass', vars_dict=None, expand_vars=False, + filename='') self.assertIn('invalid syntax', str(cm.exception)) def test_not_whitelisted(self): with self.assertRaises(ValueError) as cm: - gclient_eval._gclient_eval('[x for x in [1, 2, 3]]') + gclient_eval._gclient_eval( + '[x for x in [1, 2, 3]]', vars_dict=None, expand_vars=False, + filename='') self.assertIn( 'unexpected AST node: <_ast.ListComp object', str(cm.exception)) @@ -105,7 +139,9 @@ class GClientEvalTest(unittest.TestCase): for test_case in itertools.permutations(range(4)): input_data = ['{'] + ['"%s": "%s",' % (n, n) for n in test_case] + ['}'] expected = [(str(n), str(n)) for n in test_case] - result = gclient_eval._gclient_eval(''.join(input_data)) + result = gclient_eval._gclient_eval( + ''.join(input_data), vars_dict=None, expand_vars=False, + filename='') self.assertEqual(expected, result.items()) @@ -124,12 +160,11 @@ class ExecTest(unittest.TestCase): def test_schema_wrong_type(self): with self.assertRaises(schema.SchemaError): - gclient_eval.Exec('include_rules = {}', '') + gclient_eval.Exec('include_rules = {}') def test_recursedeps_list(self): local_scope = gclient_eval.Exec( - 'recursedeps = [["src/third_party/angle", "DEPS.chromium"]]', - '') + 'recursedeps = [["src/third_party/angle", "DEPS.chromium"]]') self.assertEqual( {'recursedeps': [['src/third_party/angle', 'DEPS.chromium']]}, local_scope) @@ -142,7 +177,35 @@ class ExecTest(unittest.TestCase): 'deps = {', ' "a_dep": "a" + Var("foo") + "b",', '}', - ]), '') + ])) + self.assertEqual({ + 'vars': collections.OrderedDict([('foo', 'bar')]), + 'deps': collections.OrderedDict([('a_dep', 'abarb')]), + }, local_scope) + + def test_braces_var(self): + local_scope = gclient_eval.Exec('\n'.join([ + 'vars = {', + ' "foo": "bar",', + '}', + 'deps = {', + ' "a_dep": "a{foo}b",', + '}', + ])) + self.assertEqual({ + 'vars': collections.OrderedDict([('foo', 'bar')]), + 'deps': collections.OrderedDict([('a_dep', 'abarb')]), + }, local_scope) + + def test_var_unexpanded(self): + local_scope = gclient_eval.Exec('\n'.join([ + 'vars = {', + ' "foo": "bar",', + '}', + 'deps = {', + ' "a_dep": "a" + Var("foo") + "b",', + '}', + ]), expand_vars=False) self.assertEqual({ 'vars': collections.OrderedDict([('foo', 'bar')]), 'deps': collections.OrderedDict([('a_dep', 'a{foo}b')]),