diff --git a/gclient.py b/gclient.py index c4bf01176..239ff28bf 100755 --- a/gclient.py +++ b/gclient.py @@ -736,7 +736,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): deps_to_add.sort(key=lambda x: x.name) return deps_to_add - def ParseDepsFile(self): + def ParseDepsFile(self, expand_vars=True): """Parses the DEPS file for this dependency.""" assert not self.deps_parsed assert not self.dependencies @@ -765,15 +765,15 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): local_scope = {} if deps_content: - # Eval the content. try: - if self._get_option('validate_syntax', False): - local_scope = gclient_eval.Exec(deps_content, filepath) - else: - global_scope = { - 'Var': lambda var_name: '{%s}' % var_name, - } - exec(deps_content, global_scope, local_scope) + vars_override = {} + if self.parent: + vars_override = self.parent.get_vars() + vars_override.update(self.get_vars()) + local_scope = gclient_eval.Parse( + deps_content, expand_vars, + self._get_option('validate_syntax', False), + filepath, vars_override) except SyntaxError as e: gclient_utils.SyntaxErrorToError(filepath, e) @@ -988,7 +988,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): file_list[i] = file_list[i][1:] # Always parse the DEPS file. - self.ParseDepsFile() + self.ParseDepsFile(expand_vars=(command != 'flatten')) self._run_is_done(file_list or [], parsed_url) if command in ('update', 'revert') and not options.noprehooks: self.RunPreDepsHooks() @@ -1864,7 +1864,7 @@ it or fix the checkout. print('%s: %s' % (x, entries[x])) logging.info(str(self)) - def ParseDepsFile(self): + def ParseDepsFile(self, expand_vars=None): """No DEPS to parse for a .gclient file.""" raise gclient_utils.Error('Internal error') @@ -1973,7 +1973,7 @@ class CipdDependency(Dependency): self._cipd_package = self._cipd_root.add_package( self._cipd_subdir, self._package_name, self._package_version) - def ParseDepsFile(self): + def ParseDepsFile(self, expand_vars=None): """CIPD dependencies are not currently allowed to have nested deps.""" self.add_dependencies_and_close([], []) @@ -2930,7 +2930,9 @@ def CMDsetdep(parser, args): 'DEPS file %s does not exist.' % options.deps_file) with open(options.deps_file) as f: contents = f.read() - local_scope = gclient_eval.Exec(contents) + local_scope = gclient_eval.Parse( + contents, expand_vars=True, validate_syntax=True, + filename=options.deps_file) for var in options.vars: name, _, value = var.partition('=') diff --git a/gclient_eval.py b/gclient_eval.py index 081c96fdb..0ebb1c528 100644 --- a/gclient_eval.py +++ b/gclient_eval.py @@ -37,7 +37,7 @@ class _NodeDict(collections.MutableMapping): def GetNode(self, key): return self.data[key][1] - def _SetNode(self, key, value, node): + def SetNode(self, key, value, node): self.data[key] = (value, node) @@ -183,7 +183,7 @@ _GCLIENT_SCHEMA = schema.Schema(_NodeDictSchema({ })) -def _gclient_eval(node_or_string, filename=''): +def _gclient_eval(node_or_string, vars_dict, expand_vars, filename): """Safely evaluates a single expression. Returns the result.""" _allowed_names = {'None': None, 'True': True, 'False': False} if isinstance(node_or_string, basestring): @@ -192,7 +192,15 @@ def _gclient_eval(node_or_string, filename=''): node_or_string = node_or_string.body def _convert(node): if isinstance(node, ast.Str): - return node.s + if not expand_vars: + return node.s + try: + return node.s.format(**vars_dict) + except KeyError as e: + raise ValueError( + '%s was used as a variable, but was not declared in the vars dict ' + '(file %r, line %s)' % ( + e.message, filename, getattr(node, 'lineno', ''))) elif isinstance(node, ast.Num): return node.n elif isinstance(node, ast.Tuple): @@ -222,7 +230,18 @@ def _gclient_eval(node_or_string, filename=''): raise ValueError( 'Var\'s argument must be a variable name (file %r, line %s)' % ( filename, getattr(node, 'lineno', ''))) - return '{%s}' % arg + if not expand_vars: + return '{%s}' % arg + if vars_dict is None: + raise ValueError( + 'vars must be declared before Var can be used (file %r, line %s)' + % (filename, getattr(node, 'lineno', ''))) + if arg not in vars_dict: + raise ValueError( + '%s was used as a variable, but was not declared in the vars dict ' + '(file %r, line %s)' % ( + arg, filename, getattr(node, 'lineno', ''))) + return vars_dict[arg] elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): return _convert(node.left) + _convert(node.right) elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Mod): @@ -235,13 +254,19 @@ def _gclient_eval(node_or_string, filename=''): return _convert(node_or_string) -def Exec(content, filename=''): - """Safely execs a set of assignments. Mutates |local_scope|.""" +def Exec(content, expand_vars, filename='', vars_override=None): + """Safely execs a set of assignments.""" node_or_string = ast.parse(content, filename=filename, mode='exec') if isinstance(node_or_string, ast.Expression): node_or_string = node_or_string.body - defined_variables = set() + tokens = { + token[2]: list(token) + for token in tokenize.generate_tokens( + cStringIO.StringIO(content).readline) + } + local_scope = _NodeDict({}, tokens) + vars_dict = {} def _visit_in_module(node): if isinstance(node, ast.Assign): if len(node.targets) != 1: @@ -253,15 +278,22 @@ def Exec(content, filename=''): raise ValueError( 'invalid assignment: target should be a name (file %r, line %s)' % ( filename, getattr(node, 'lineno', ''))) - value = _gclient_eval(node.value, filename=filename) + value = _gclient_eval(node.value, vars_dict, expand_vars, filename) - if target.id in defined_variables: + if target.id in local_scope: raise ValueError( 'invalid assignment: overrides var %r (file %r, line %s)' % ( target.id, filename, getattr(node, 'lineno', ''))) - defined_variables.add(target.id) - return target.id, (value, node.value) + if target.id == 'vars': + vars_dict.update(value) + if vars_override: + vars_dict.update({ + k: v + for k, v in vars_override.iteritems() + if k in vars_dict}) + + local_scope.SetNode(target.id, value, node.value) else: raise ValueError( 'unexpected AST node: %s %s (file %r, line %s)' % ( @@ -269,15 +301,8 @@ def Exec(content, filename=''): getattr(node, 'lineno', ''))) if isinstance(node_or_string, ast.Module): - data = [] for stmt in node_or_string.body: - data.append(_visit_in_module(stmt)) - tokens = { - token[2]: list(token) - for token in tokenize.generate_tokens( - cStringIO.StringIO(content).readline) - } - local_scope = _NodeDict(data, tokens) + _visit_in_module(stmt) else: raise ValueError( 'unexpected AST node: %s %s (file %r, line %s)' % ( @@ -289,6 +314,70 @@ def Exec(content, filename=''): return _GCLIENT_SCHEMA.validate(local_scope) +def Parse(content, expand_vars, validate_syntax, filename, vars_override=None): + """Parses DEPS strings. + + Executes the Python-like string stored in content, resulting in a Python + dictionary specifyied by the schema above. Supports syntax validation and + variable expansion. + + Args: + content: str. DEPS file stored as a string. + expand_vars: bool. Whether variables should be expanded to their values. + validate_syntax: bool. Whether syntax should be validated using the schema + defined above. + filename: str. The name of the DEPS file, or a string describing the source + of the content, e.g. '', ''. + vars_override: dict, optional. A dictionary with overrides for the variables + defined by the DEPS file. + + Returns: + A Python dict with the parsed contents of the DEPS file, as specified by the + schema above. + """ + # TODO(ehmaldonado): Make validate_syntax = True the only case + if validate_syntax: + return Exec(content, expand_vars, filename, vars_override) + + local_scope = {} + global_scope = {'Var': lambda var_name: '{%s}' % var_name} + + # If we use 'exec' directly, it complains that 'Parse' contains a nested + # function with free variables. + # This is because on versions of Python < 2.7.9, "exec(a, b, c)" not the same + # as "exec a in b, c" (See https://bugs.python.org/issue21591). + eval(compile(content, filename, 'exec'), global_scope, local_scope) + + if 'vars' not in local_scope or not expand_vars: + return local_scope + + vars_dict = {} + vars_dict.update(local_scope['vars']) + if vars_override: + vars_dict.update({ + k: v + for k, v in vars_override.iteritems() + if k in vars_dict + }) + + def _DeepFormat(node): + if isinstance(node, basestring): + return node.format(**vars_dict) + elif isinstance(node, dict): + return { + k.format(**vars_dict): _DeepFormat(v) + for k, v in node.iteritems() + } + elif isinstance(node, list): + return [_DeepFormat(elem) for elem in node] + elif isinstance(node, tuple): + return tuple(_DeepFormat(elem) for elem in node) + else: + return node + + return _DeepFormat(local_scope) + + def EvaluateCondition(condition, variables, referenced_variables=None): """Safely evaluates a boolean condition. Returns the result.""" if not referenced_variables: @@ -416,7 +505,7 @@ def SetVar(gclient_dict, var_name, value): "The vars entry for %s has no formatting information." % var_name) _UpdateAstString(tokens, node, value) - gclient_dict['vars']._SetNode(var_name, value, node) + gclient_dict['vars'].SetNode(var_name, value, node) def SetCIPD(gclient_dict, dep_name, package_name, new_version): @@ -450,7 +539,7 @@ def SetCIPD(gclient_dict, dep_name, package_name, new_version): new_version = 'version:' + new_version _UpdateAstString(tokens, node, new_version) - packages[0]._SetNode('version', new_version, node) + packages[0].SetNode('version', new_version, node) def SetRevision(gclient_dict, dep_name, new_revision): @@ -477,8 +566,9 @@ def SetRevision(gclient_dict, dep_name, new_revision): SetVar(gclient_dict, node.args[0].s, new_revision) else: _UpdateAstString(tokens, node, new_revision) - value = _gclient_eval(dep_node) - dep_dict._SetNode(dep_key, value, dep_node) + value = _gclient_eval(dep_node, gclient_dict.get('vars', None), + expand_vars=True, filename='') + dep_dict.SetNode(dep_key, value, dep_node) if isinstance(gclient_dict['deps'][dep_name], _NodeDict): _UpdateRevision(gclient_dict['deps'][dep_name], 'url') diff --git a/tests/gclient_eval_unittest.py b/tests/gclient_eval_unittest.py index 7f841d692..ff4f5a873 100755 --- a/tests/gclient_eval_unittest.py +++ b/tests/gclient_eval_unittest.py @@ -20,6 +20,15 @@ import gclient_eval _SAMPLE_DEPS_FILE = textwrap.dedent("""\ +vars = { + 'git_repo': 'https://example.com/repo.git', + # Some comment with bad indentation + 'dep_2_rev': '1ced', + # Some more comments with bad indentation + # and trailing whitespaces + 'dep_3_rev': '5p1e5', +} + deps = { 'src/dep': Var('git_repo') + '/dep' + '@' + 'deadbeef', # Some comment @@ -45,59 +54,101 @@ deps = { 'dep_type': 'cipd', }, } - -vars = { - 'git_repo': 'https://example.com/repo.git', - # Some comment with bad indentation - 'dep_2_rev': '1ced', - # Some more comments - # 1 - # 2 - # 3 - 'dep_3_rev': '5p1e5', -}""") +""") class GClientEvalTest(unittest.TestCase): def test_str(self): - self.assertEqual('foo', gclient_eval._gclient_eval('"foo"')) + self.assertEqual( + 'foo', + gclient_eval._gclient_eval('"foo"', vars_dict=None, expand_vars=False, + filename='')) def test_tuple(self): - self.assertEqual(('a', 'b'), gclient_eval._gclient_eval('("a", "b")')) + self.assertEqual( + ('a', 'b'), + gclient_eval._gclient_eval('("a", "b")', vars_dict=None, + expand_vars=False, filename='')) def test_list(self): - self.assertEqual(['a', 'b'], gclient_eval._gclient_eval('["a", "b"]')) + self.assertEqual( + ['a', 'b'], + gclient_eval._gclient_eval('["a", "b"]', vars_dict=None, + expand_vars=False, filename='')) def test_dict(self): - self.assertEqual({'a': 'b'}, gclient_eval._gclient_eval('{"a": "b"}')) + self.assertEqual( + {'a': 'b'}, + gclient_eval._gclient_eval('{"a": "b"}', vars_dict=None, + expand_vars=False, filename='')) def test_name_safe(self): - self.assertEqual(True, gclient_eval._gclient_eval('True')) + self.assertEqual( + True, + gclient_eval._gclient_eval('True', vars_dict=None, + expand_vars=False, filename='')) def test_name_unsafe(self): with self.assertRaises(ValueError) as cm: - gclient_eval._gclient_eval('UnsafeName') + gclient_eval._gclient_eval('UnsafeName', vars_dict=None, + expand_vars=False, filename='') self.assertIn('invalid name \'UnsafeName\'', str(cm.exception)) + def test_invalid_call(self): + with self.assertRaises(ValueError) as cm: + gclient_eval._gclient_eval('Foo("bar")', vars_dict=None, + expand_vars=False, filename='') + self.assertIn('Var is the only allowed function', str(cm.exception)) + def test_call(self): self.assertEqual( '{bar}', - gclient_eval._gclient_eval('Var("bar")')) + gclient_eval._gclient_eval('Var("bar")', vars_dict=None, + expand_vars=False, filename='')) + + def test_expands_vars(self): + self.assertEqual( + 'foo', + gclient_eval._gclient_eval('Var("bar")', vars_dict={'bar': 'foo'}, + expand_vars=True, filename='')) + + def test_expands_vars_with_braces(self): + self.assertEqual( + 'foo', + gclient_eval._gclient_eval('"{bar}"', vars_dict={'bar': 'foo'}, + expand_vars=True, filename='')) + + def test_invalid_var(self): + with self.assertRaises(ValueError) as cm: + gclient_eval._gclient_eval('"{bar}"', vars_dict={}, expand_vars=True, + filename='') + self.assertIn('bar was used as a variable, but was not declared', + str(cm.exception)) def test_plus(self): - self.assertEqual('foo', gclient_eval._gclient_eval('"f" + "o" + "o"')) + self.assertEqual( + 'foo', + gclient_eval._gclient_eval('"f" + "o" + "o"', vars_dict=None, + expand_vars=False, filename='')) def test_format(self): - self.assertEqual('foo', gclient_eval._gclient_eval('"%s" % "foo"')) + self.assertEqual( + 'foo', + gclient_eval._gclient_eval('"%s" % "foo"', vars_dict=None, + expand_vars=False, filename='')) def test_not_expression(self): with self.assertRaises(SyntaxError) as cm: - gclient_eval._gclient_eval('def foo():\n pass') + gclient_eval._gclient_eval( + 'def foo():\n pass', vars_dict=None, expand_vars=False, + filename='') self.assertIn('invalid syntax', str(cm.exception)) def test_not_whitelisted(self): with self.assertRaises(ValueError) as cm: - gclient_eval._gclient_eval('[x for x in [1, 2, 3]]') + gclient_eval._gclient_eval( + '[x for x in [1, 2, 3]]', vars_dict=None, expand_vars=False, + filename='') self.assertIn( 'unexpected AST node: <_ast.ListComp object', str(cm.exception)) @@ -105,31 +156,33 @@ class GClientEvalTest(unittest.TestCase): for test_case in itertools.permutations(range(4)): input_data = ['{'] + ['"%s": "%s",' % (n, n) for n in test_case] + ['}'] expected = [(str(n), str(n)) for n in test_case] - result = gclient_eval._gclient_eval(''.join(input_data)) + result = gclient_eval._gclient_eval( + ''.join(input_data), vars_dict=None, expand_vars=False, + filename='') self.assertEqual(expected, result.items()) class ExecTest(unittest.TestCase): def test_multiple_assignment(self): with self.assertRaises(ValueError) as cm: - gclient_eval.Exec('a, b, c = "a", "b", "c"') + gclient_eval.Exec('a, b, c = "a", "b", "c"', expand_vars=True) self.assertIn( 'invalid assignment: target should be a name', str(cm.exception)) def test_override(self): with self.assertRaises(ValueError) as cm: - gclient_eval.Exec('a = "a"\na = "x"') + gclient_eval.Exec('a = "a"\na = "x"', expand_vars=True) self.assertIn( 'invalid assignment: overrides var \'a\'', str(cm.exception)) def test_schema_wrong_type(self): with self.assertRaises(schema.SchemaError): - gclient_eval.Exec('include_rules = {}', '') + gclient_eval.Exec('include_rules = {}', expand_vars=True) def test_recursedeps_list(self): local_scope = gclient_eval.Exec( 'recursedeps = [["src/third_party/angle", "DEPS.chromium"]]', - '') + expand_vars=True) self.assertEqual( {'recursedeps': [['src/third_party/angle', 'DEPS.chromium']]}, local_scope) @@ -142,16 +195,71 @@ class ExecTest(unittest.TestCase): 'deps = {', ' "a_dep": "a" + Var("foo") + "b",', '}', - ]), '') + ]), expand_vars=True) + self.assertEqual({ + 'vars': collections.OrderedDict([('foo', 'bar')]), + 'deps': collections.OrderedDict([('a_dep', 'abarb')]), + }, local_scope) + + def test_braces_var(self): + local_scope = gclient_eval.Exec('\n'.join([ + 'vars = {', + ' "foo": "bar",', + '}', + 'deps = {', + ' "a_dep": "a{foo}b",', + '}', + ]), expand_vars=True) + self.assertEqual({ + 'vars': collections.OrderedDict([('foo', 'bar')]), + 'deps': collections.OrderedDict([('a_dep', 'abarb')]), + }, local_scope) + + def test_var_unexpanded(self): + local_scope = gclient_eval.Exec('\n'.join([ + 'vars = {', + ' "foo": "bar",', + '}', + 'deps = {', + ' "a_dep": "a" + Var("foo") + "b",', + '}', + ]), expand_vars=False) self.assertEqual({ 'vars': collections.OrderedDict([('foo', 'bar')]), 'deps': collections.OrderedDict([('a_dep', 'a{foo}b')]), }, local_scope) def test_empty_deps(self): - local_scope = gclient_eval.Exec('deps = {}', '') + local_scope = gclient_eval.Exec('deps = {}', expand_vars=True) self.assertEqual({'deps': {}}, local_scope) + def test_overrides_vars(self): + local_scope = gclient_eval.Exec('\n'.join([ + 'vars = {', + ' "foo": "bar",', + '}', + 'deps = {', + ' "a_dep": "a{foo}b",', + '}', + ]), expand_vars=True, vars_override={'foo': 'baz'}) + self.assertEqual({ + 'vars': collections.OrderedDict([('foo', 'bar')]), + 'deps': collections.OrderedDict([('a_dep', 'abazb')]), + }, local_scope) + + def test_doesnt_override_undeclared_vars(self): + with self.assertRaises(ValueError) as cm: + gclient_eval.Exec('\n'.join([ + 'vars = {', + ' "foo": "bar",', + '}', + 'deps = {', + ' "a_dep": "a{baz}b",', + '}', + ]), expand_vars=True, vars_override={'baz': 'lalala'}) + self.assertIn('baz was used as a variable, but was not declared', + str(cm.exception)) + class EvaluateConditionTest(unittest.TestCase): def test_true(self): @@ -204,8 +312,8 @@ class EvaluateConditionTest(unittest.TestCase): class SetVarTest(unittest.TestCase): - def testSetVar(self): - local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE) + def test_sets_var(self): + local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, expand_vars=True) gclient_eval.SetVar(local_scope, 'dep_2_rev', 'c0ffee') result = gclient_eval.RenderDEPSFile(local_scope) @@ -216,8 +324,8 @@ class SetVarTest(unittest.TestCase): class SetCipdTest(unittest.TestCase): - def testSetCIPD(self): - local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE) + def test_sets_cipd(self): + local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, expand_vars=True) gclient_eval.SetCIPD( local_scope, 'src/cipd/package', 'another/cipd/package', '6.789') @@ -228,23 +336,23 @@ class SetCipdTest(unittest.TestCase): class SetRevisionTest(unittest.TestCase): def setUp(self): - self.local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE) + self.local_scope = gclient_eval.Exec(_SAMPLE_DEPS_FILE, expand_vars=True) - def testSetRevision(self): + def test_sets_revision(self): gclient_eval.SetRevision( self.local_scope, 'src/dep', 'deadfeed') result = gclient_eval.RenderDEPSFile(self.local_scope) self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('deadbeef', 'deadfeed')) - def testSetRevisionInUrl(self): + def test_sets_revision_inside_dict(self): gclient_eval.SetRevision( self.local_scope, 'src/dep_3', '0ff1ce') result = gclient_eval.RenderDEPSFile(self.local_scope) self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('5p1e5', '0ff1ce')) - def testSetRevisionInVars(self): + def test_sets_revision_in_vars(self): gclient_eval.SetRevision( self.local_scope, 'src/android/dep_2', 'c0ffee') result = gclient_eval.RenderDEPSFile(self.local_scope) @@ -252,6 +360,68 @@ class SetRevisionTest(unittest.TestCase): self.assertEqual(result, _SAMPLE_DEPS_FILE.replace('1ced', 'c0ffee')) +class ParseTest(unittest.TestCase): + def callParse(self, expand_vars=True, validate_syntax=True, + vars_override=None): + return gclient_eval.Parse('\n'.join([ + 'vars = {', + ' "foo": "bar",', + '}', + 'deps = {', + ' "a_dep": "a{foo}b",', + '}', + ]), expand_vars, validate_syntax, '', vars_override) + + def test_expands_vars(self): + for validate_syntax in True, False: + local_scope = self.callParse(validate_syntax=validate_syntax) + self.assertEqual({ + 'vars': collections.OrderedDict([('foo', 'bar')]), + 'deps': collections.OrderedDict([('a_dep', 'abarb')]), + }, local_scope) + + def test_no_expands_vars(self): + for validate_syntax in True, False: + local_scope = self.callParse(expand_vars=False, + validate_syntax=validate_syntax) + self.assertEqual({ + 'vars': collections.OrderedDict([('foo', 'bar')]), + 'deps': collections.OrderedDict([('a_dep', 'a{foo}b')]), + }, local_scope) + + def test_overrides_vars(self): + for validate_syntax in True, False: + local_scope = self.callParse(validate_syntax=validate_syntax, + vars_override={'foo': 'baz'}) + self.assertEqual({ + 'vars': collections.OrderedDict([('foo', 'bar')]), + 'deps': collections.OrderedDict([('a_dep', 'abazb')]), + }, local_scope) + + def test_no_extra_vars(self): + deps_file = '\n'.join([ + 'vars = {', + ' "foo": "bar",', + '}', + 'deps = {', + ' "a_dep": "a{baz}b",', + '}', + ]) + + with self.assertRaises(ValueError) as cm: + gclient_eval.Parse( + deps_file, expand_vars=True, validate_syntax=True, + filename='', vars_override={'baz': 'lalala'}) + self.assertIn('baz was used as a variable, but was not declared', + str(cm.exception)) + + with self.assertRaises(KeyError) as cm: + gclient_eval.Parse( + deps_file, expand_vars=True, validate_syntax=False, + filename='', vars_override={'baz': 'lalala'}) + self.assertIn('baz', str(cm.exception)) + + if __name__ == '__main__': level = logging.DEBUG if '-v' in sys.argv else logging.FATAL logging.basicConfig(