From 0f7b2007a53d5158e42e97b938fd03f57cdf9786 Mon Sep 17 00:00:00 2001 From: John Budorick Date: Fri, 19 Jan 2018 15:46:17 -0800 Subject: [PATCH] Add cipd support to gclient. Bug: 789809 Change-Id: I9942eaa613d51edd77ee5195603932a103f5e3cd Reviewed-on: https://chromium-review.googlesource.com/829953 Commit-Queue: John Budorick Reviewed-by: Aaron Gable --- gclient.py | 219 ++++++++++++++++++++++++++++----- gclient_eval.py | 18 ++- gclient_scm.py | 253 +++++++++++++++++++++++++++++++++----- tests/gclient_scm_test.py | 236 ++++++++++++++++++++++++++--------- tests/gclient_test.py | 6 +- 5 files changed, 608 insertions(+), 124 deletions(-) diff --git a/gclient.py b/gclient.py index cc072e85c7..fd46eceee2 100755 --- a/gclient.py +++ b/gclient.py @@ -277,8 +277,9 @@ class DependencySettings(object): ('dependency url must be either string or None, ' 'instead of %s') % self._url.__class__.__name__) # Make any deps_file path platform-appropriate. - for sep in ['/', '\\']: - self._deps_file = self._deps_file.replace(sep, os.sep) + if self._deps_file: + for sep in ['/', '\\']: + self._deps_file = self._deps_file.replace(sep, os.sep) @property def deps_file(self): @@ -422,6 +423,22 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): if not self.name and self.parent: raise gclient_utils.Error('Dependency without name') + def ToLines(self): + s = [] + condition_part = ([' "condition": %r,' % self.condition] + if self.condition else []) + s.extend([ + ' # %s' % self.hierarchy(include_url=False), + ' "%s": {' % (self.name,), + ' "url": "%s",' % (self.raw_url,), + ] + condition_part + [ + ' },', + '', + ]) + return s + + + @property def requirements(self): """Calculate the list of requirements.""" @@ -534,7 +551,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): 'relative DEPS entry \'%s\' must begin with a slash' % url) # Create a scm just to query the full url. parent_url = self.parent.parsed_url - scm = gclient_scm.CreateSCM( + scm = self.CreateSCM( parent_url, self.root.root_dir, None, self.outbuf) parsed_url = scm.FullUrlForRelativeUrl(url) else: @@ -623,6 +640,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): def _deps_to_objects(self, deps, use_relative_paths): """Convert a deps dict to a dict of Dependency objects.""" deps_to_add = [] + cipd_root = None for name, dep_value in deps.iteritems(): should_process = self.recursion_limit and self.should_process deps_file = self.deps_file @@ -632,30 +650,58 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): deps_file = ent['deps_file'] if dep_value is None: continue + condition = None condition_value = True if isinstance(dep_value, basestring): raw_url = dep_value + dep_type = None else: # This should be guaranteed by schema checking in gclient_eval. assert isinstance(dep_value, collections.Mapping) - raw_url = dep_value['url'] + raw_url = dep_value.get('url') # Take into account should_process metadata set by MergeWithOsDeps. should_process = (should_process and dep_value.get('should_process', True)) condition = dep_value.get('condition') - - url = raw_url.format(**self.get_vars()) + dep_type = dep_value.get('dep_type') if condition: condition_value = gclient_eval.EvaluateCondition( condition, self.get_vars()) if not self._get_option('process_all_deps', False): should_process = should_process and condition_value - deps_to_add.append(Dependency( - self, name, raw_url, url, None, None, self.custom_vars, None, - deps_file, should_process, use_relative_paths, condition, - condition_value)) + + if dep_type == 'cipd': + if not cipd_root: + cipd_root = gclient_scm.CipdRoot( + os.path.join(self.root.root_dir, self.name), + # TODO(jbudorick): Support other service URLs as necessary. + # Service URLs should be constant over the scope of a cipd + # root, so a var per DEPS file specifying the service URL + # should suffice. + 'https://chrome-infra-packages.appspot.com') + for package in dep_value.get('packages', []): + deps_to_add.append( + CipdDependency( + self, name, package, cipd_root, + self.custom_vars, should_process, use_relative_paths, + condition, condition_value)) + elif dep_type == 'git': + url = raw_url.format(**self.get_vars()) + deps_to_add.append( + GitDependency( + self, name, raw_url, url, None, None, self.custom_vars, None, + deps_file, should_process, use_relative_paths, condition, + condition_value)) + else: + url = raw_url.format(**self.get_vars()) + deps_to_add.append( + Dependency( + self, name, raw_url, url, None, None, self.custom_vars, None, + deps_file, should_process, use_relative_paths, condition, + condition_value)) + deps_to_add.sort(key=lambda x: x.name) return deps_to_add @@ -695,7 +741,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): # Eval the content. try: if self._get_option('validate_syntax', False): - gclient_eval.Exec(deps_content, global_scope, local_scope, filepath) + local_scope = gclient_eval.Exec( + deps_content, global_scope, local_scope, filepath) else: exec(deps_content, global_scope, local_scope) except SyntaxError as e: @@ -878,7 +925,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): options = copy.copy(options) options.revision = revision_override self._used_revision = options.revision - self._used_scm = gclient_scm.CreateSCM( + self._used_scm = self.CreateSCM( parsed_url, self.root.root_dir, self.name, self.outbuf, out_cb=work_queue.out_cb) self._got_revision = self._used_scm.RunCommand(command, options, args, @@ -913,7 +960,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): if command == 'recurse': # Skip file only checkout. - scm = gclient_scm.GetScmName(parsed_url) + scm = self.GetScmName(parsed_url) if not options.scm or scm in options.scm: cwd = os.path.normpath(os.path.join(self.root.root_dir, self.name)) # Pass in the SCM type as an env variable. Make sure we don't put @@ -967,6 +1014,40 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): else: print('Skipped missing %s' % cwd, file=sys.stderr) + def GetScmName(self, url): + """Get the name of the SCM for the given URL. + + While we currently support both git and cipd as SCM implementations, + this currently cannot return 'cipd', regardless of the URL, as CIPD + has no canonical URL format. If you want to use CIPD as an SCM, you + must currently do so by explicitly using a CipdDependency. + """ + if not url: + return None + url, _ = gclient_utils.SplitUrlRevision(url) + if url.endswith('.git'): + return 'git' + protocol = url.split('://')[0] + if protocol in ( + 'file', 'git', 'git+http', 'git+https', 'http', 'https', 'ssh', 'sso'): + return 'git' + return None + + def CreateSCM(self, url, root_dir=None, relpath=None, out_fh=None, + out_cb=None): + SCM_MAP = { + 'cipd': gclient_scm.CipdWrapper, + 'git': gclient_scm.GitWrapper, + } + + scm_name = self.GetScmName(url) + if not scm_name in SCM_MAP: + raise gclient_utils.Error('No SCM found for url %s' % url) + scm_class = SCM_MAP[scm_name] + if not scm_class.BinaryExists(): + raise gclient_utils.Error('%s command not found' % scm_name) + return scm_class(url, root_dir, relpath, out_fh, out_cb) + def HasGNArgsFile(self): return self._gn_args_file is not None @@ -1004,7 +1085,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): # what files have changed so we always run all hooks. It'd be nice to fix # that. if (options.force or - gclient_scm.GetScmName(self.parsed_url) in ('git', None) or + self.GetScmName(self.parsed_url) in ('git', None) or os.path.isdir(os.path.join(self.root.root_dir, self.name, '.git'))): result.extend(self.deps_hooks) else: @@ -1280,7 +1361,7 @@ solutions = [ solutions.""" for dep in self.dependencies: if dep.managed and dep.url: - scm = gclient_scm.CreateSCM( + scm = self.CreateSCM( dep.url, self.root_dir, dep.name, self.outbuf) actual_url = scm.GetActualRemoteURL(self._options) if actual_url and not scm.DoesRemoteURLMatch(self._options): @@ -1305,10 +1386,10 @@ You should ensure that the URL listed in .gclient is correct and either change it or fix the checkout. ''' % {'checkout_path': os.path.join(self.root_dir, dep.name), 'expected_url': dep.url, - 'expected_scm': gclient_scm.GetScmName(dep.url), + 'expected_scm': self.GetScmName(dep.url), 'mirror_string' : mirror_string, 'actual_url': actual_url, - 'actual_scm': gclient_scm.GetScmName(actual_url)}) + 'actual_scm': self.GetScmName(actual_url)}) def SetConfig(self, content): assert not self.dependencies @@ -1529,7 +1610,7 @@ it or fix the checkout. (not any(path.startswith(entry + '/') for path in entries)) and os.path.exists(e_dir)): # The entry has been removed from DEPS. - scm = gclient_scm.CreateSCM( + scm = self.CreateSCM( prev_url, self.root_dir, entry_fixed, self.outbuf) # Check to see if this directory is now part of a higher-up checkout. @@ -1619,7 +1700,7 @@ it or fix the checkout. if dep.parsed_url is None: return None url, _ = gclient_utils.SplitUrlRevision(dep.parsed_url) - scm = gclient_scm.CreateSCM( + scm = dep.CreateSCM( dep.parsed_url, self.root_dir, dep.name, self.outbuf) if not os.path.isdir(scm.checkout_path): return None @@ -1699,6 +1780,91 @@ it or fix the checkout. return self._enforced_os +class GitDependency(Dependency): + """A Dependency object that represents a single git checkout.""" + + #override + def GetScmName(self, url): + """Always 'git'.""" + del url + return 'git' + + #override + def CreateSCM(self, url, root_dir=None, relpath=None, out_fh=None, + out_cb=None): + """Create a Wrapper instance suitable for handling this git dependency.""" + return gclient_scm.GitWrapper(url, root_dir, relpath, out_fh, out_cb) + + +class CipdDependency(Dependency): + """A Dependency object that represents a single CIPD package.""" + + def __init__( + self, parent, name, dep_value, cipd_root, + custom_vars, should_process, relative, condition, condition_value): + package = dep_value['package'] + version = dep_value['version'] + url = urlparse.urljoin( + cipd_root.service_url, '%s@%s' % (package, version)) + super(CipdDependency, self).__init__( + parent, name, url, url, None, None, custom_vars, + None, None, should_process, relative, condition, condition_value) + if relative: + # TODO(jbudorick): Implement relative if necessary. + raise gclient_utils.Error( + 'Relative CIPD dependencies are not currently supported.') + self._cipd_root = cipd_root + + self._cipd_subdir = os.path.relpath( + os.path.join(self.root.root_dir, self.name), cipd_root.root_dir) + self._cipd_package = self._cipd_root.add_package( + self._cipd_subdir, package, version) + + def ParseDepsFile(self): + """CIPD dependencies are not currently allowed to have nested deps.""" + self.add_dependencies_and_close([], []) + + #override + def GetScmName(self, url): + """Always 'cipd'.""" + del url + return 'cipd' + + #override + def CreateSCM(self, url, root_dir=None, relpath=None, out_fh=None, + out_cb=None): + """Create a Wrapper instance suitable for handling this CIPD dependency.""" + return gclient_scm.CipdWrapper( + url, root_dir, relpath, out_fh, out_cb, + root=self._cipd_root, + package=self._cipd_package) + + def ToLines(self): + """Return a list of lines representing this in a DEPS file.""" + s = [] + if self._cipd_package.authority_for_subdir: + condition_part = ([' "condition": %r,' % self.condition] + if self.condition else []) + s.extend([ + ' # %s' % self.hierarchy(include_url=False), + ' "%s": {' % (self.name,), + ' "packages": [', + ]) + for p in self._cipd_root.packages(self._cipd_subdir): + s.extend([ + ' "package": "%s",' % p.name, + ' "version": "%s",' % p.version, + ]) + s.extend([ + ' ],', + ' "dep_type": "cipd",', + ] + condition_part + [ + ' },', + '', + ]) + return s + + #### gclient commands. @@ -1809,7 +1975,7 @@ class Flattener(object): if revision and gclient_utils.IsFullGitSha(revision): return - scm = gclient_scm.CreateSCM( + scm = dep.CreateSCM( dep.parsed_url, self._client.root_dir, dep.name, dep.outbuf) revinfo = scm.revinfo(self._client._options, [], None) @@ -2028,17 +2194,8 @@ def _DepsToLines(deps): if not deps: return [] s = ['deps = {'] - for name, dep in sorted(deps.iteritems()): - condition_part = ([' "condition": %r,' % dep.condition] - if dep.condition else []) - s.extend([ - ' # %s' % dep.hierarchy(include_url=False), - ' "%s": {' % (name,), - ' "url": "%s",' % (dep.raw_url,), - ] + condition_part + [ - ' },', - '', - ]) + for _, dep in sorted(deps.iteritems()): + s.extend(dep.ToLines()) s.extend(['}', '']) return s diff --git a/gclient_eval.py b/gclient_eval.py index 34ac4d65f1..38d37e672c 100644 --- a/gclient_eval.py +++ b/gclient_eval.py @@ -21,6 +21,22 @@ _GCLIENT_DEPS_SCHEMA = { # Optional condition string. The dep will only be processed # if the condition evaluates to True. schema.Optional('condition'): basestring, + + schema.Optional('dep_type', default='git'): basestring, + }, + # CIPD package. + { + 'packages': [ + { + 'package': basestring, + + 'version': basestring, + } + ], + + schema.Optional('condition'): basestring, + + schema.Optional('dep_type', default='cipd'): basestring, }, ), } @@ -217,7 +233,7 @@ def Exec(content, global_scope, local_scope, filename=''): filename, getattr(node_or_string, 'lineno', ''))) - _GCLIENT_SCHEMA.validate(local_scope) + return _GCLIENT_SCHEMA.validate(local_scope) def EvaluateCondition(condition, variables, referenced_variables=None): diff --git a/gclient_scm.py b/gclient_scm.py index d4c9c41ef3..50cfd79776 100644 --- a/gclient_scm.py +++ b/gclient_scm.py @@ -6,13 +6,17 @@ from __future__ import print_function +import collections +import contextlib import errno +import json import logging import os import posixpath import re import sys import tempfile +import threading import traceback import urlparse @@ -81,37 +85,6 @@ class GitDiffFilterer(DiffFiltererWrapper): return re.sub("[a|b]/" + self._current_file, self._replacement_file, line) -### SCM abstraction layer - -# Factory Method for SCM wrapper creation - -def GetScmName(url): - if not url: - return None - url, _ = gclient_utils.SplitUrlRevision(url) - if url.endswith('.git'): - return 'git' - protocol = url.split('://')[0] - if protocol in ( - 'file', 'git', 'git+http', 'git+https', 'http', 'https', 'ssh', 'sso'): - return 'git' - return None - - -def CreateSCM(url, root_dir=None, relpath=None, out_fh=None, out_cb=None): - SCM_MAP = { - 'git' : GitWrapper, - } - - scm_name = GetScmName(url) - if not scm_name in SCM_MAP: - raise gclient_utils.Error('No SCM found for url %s' % url) - scm_class = SCM_MAP[scm_name] - if not scm_class.BinaryExists(): - raise gclient_utils.Error('%s command not found' % scm_name) - return scm_class(url, root_dir, relpath, out_fh, out_cb) - - # SCMWrapper base class class SCMWrapper(object): @@ -238,11 +211,11 @@ class GitWrapper(SCMWrapper): cache_dir = None - def __init__(self, url=None, *args): + def __init__(self, url=None, *args, **kwargs): """Removes 'git+' fake prefix from git URL.""" if url.startswith('git+http://') or url.startswith('git+https://'): url = url[4:] - SCMWrapper.__init__(self, url, *args) + SCMWrapper.__init__(self, url, *args, **kwargs) filter_kwargs = { 'time_throttle': 1, 'out_fh': self.out_fh } if self.out_cb: filter_kwargs['predicate'] = self.out_cb @@ -1230,3 +1203,217 @@ class GitWrapper(SCMWrapper): gclient_utils.CheckCallAndFilterAndHeader(cmd, env=env, **kwargs) else: gclient_utils.CheckCallAndFilter(cmd, env=env, **kwargs) + + +class CipdPackage(object): + """A representation of a single CIPD package.""" + + def __init__(self, name, version, authority_for_root, authority_for_subdir): + self._authority_for_root = authority_for_root + self._authority_for_subdir = authority_for_subdir + self._name = name + self._version = version + + @property + def authority_for_root(self): + """Whether this package has authority to act on behalf of its root. + + Some operations should only be performed once per cipd root. A package + that has authority for its cipd root is the only package that should + perform such operations. + + Returns: + bool; whether this package has root authority. + """ + return self._authority_for_root + + @property + def authority_for_subdir(self): + """Whether this package has authority to act on behalf of its subdir. + + Some operations should only be performed once per subdirectory. A package + that has authority for its subdirectory is the only package that should + perform such operations. + + Returns: + bool; whether this package has subdir authority. + """ + return self._authority_for_subdir + + @property + def name(self): + return self._name + + @property + def version(self): + return self._version + + +class CipdRoot(object): + """A representation of a single CIPD root.""" + def __init__(self, root_dir, service_url): + self._all_packages = set() + self._mutator_lock = threading.Lock() + self._packages_by_subdir = collections.defaultdict(list) + self._root_dir = root_dir + self._service_url = service_url + + def add_package(self, subdir, package, version): + """Adds a package to this CIPD root. + + As far as clients are concerned, this grants both root and subdir authority + to packages arbitrarily. (The implementation grants root authority to the + first package added and subdir authority to the first package added for that + subdir, but clients should not depend on or expect that behavior.) + + Args: + subdir: str; relative path to where the package should be installed from + the cipd root directory. + package: str; the cipd package name. + version: str; the cipd package version. + Returns: + CipdPackage; the package that was created and added to this root. + """ + with self._mutator_lock: + cipd_package = CipdPackage( + package, version, + not self._packages_by_subdir, + not self._packages_by_subdir[subdir]) + self._all_packages.add(cipd_package) + self._packages_by_subdir[subdir].append(cipd_package) + return cipd_package + + def packages(self, subdir): + """Get the list of configured packages for the given subdir.""" + return list(self._packages_by_subdir[subdir]) + + def clobber(self): + """Remove the .cipd directory. + + This is useful for forcing ensure to redownload and reinitialize all + packages. + """ + with self._mutator_lock: + cipd_cache_dir = os.path.join(self._cipd_root, '.cipd') + try: + gclient_utils.rmtree(os.path.join(cipd_cache_dir)) + except OSError: + if os.path.exists(cipd_cache_dir): + raise + + @contextlib.contextmanager + def _create_ensure_file(self): + try: + ensure_file = None + with tempfile.NamedTemporaryFile( + suffix='.ensure', delete=False) as ensure_file: + for subdir, packages in sorted(self._packages_by_subdir.iteritems()): + ensure_file.write('@Subdir %s\n' % subdir) + for package in packages: + ensure_file.write('%s %s\n' % (package.name, package.version)) + ensure_file.write('\n') + yield ensure_file.name + finally: + if ensure_file is not None and os.path.exists(ensure_file.name): + os.remove(ensure_file.name) + + def ensure(self): + """Run `cipd ensure`.""" + with self._mutator_lock: + with self._create_ensure_file() as ensure_file: + cmd = [ + 'cipd', 'ensure', + '-log-level', 'error', + '-root', self.root_dir, + '-ensure-file', ensure_file, + ] + gclient_utils.CheckCallAndFilterAndHeader(cmd) + + def created_package(self, package): + """Checks whether this root created the given package. + + Args: + package: CipdPackage; the package to check. + Returns: + bool; whether this root created the given package. + """ + return package in self._all_packages + + @property + def root_dir(self): + return self._root_dir + + @property + def service_url(self): + return self._service_url + + +class CipdWrapper(SCMWrapper): + """Wrapper for CIPD. + + Currently only supports chrome-infra-packages.appspot.com. + """ + + def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None, + out_cb=None, root=None, package=None): + super(CipdWrapper, self).__init__( + url=url, root_dir=root_dir, relpath=relpath, out_fh=out_fh, + out_cb=out_cb) + assert root.created_package(package) + self._package = package + self._root = root + + #override + def GetCacheMirror(self): + return None + + #override + def GetActualRemoteURL(self, options): + return self._root.service_url + + #override + def DoesRemoteURLMatch(self, options): + del options + return True + + def revert(self, options, args, file_list): + """Deletes .cipd and reruns ensure.""" + if self._package.authority_for_root: + self._root.clobber() + self._root.ensure() + + def diff(self, options, args, file_list): + """CIPD has no notion of diffing.""" + pass + + def pack(self, options, args, file_list): + """CIPD has no notion of diffing.""" + pass + + def revinfo(self, options, args, file_list): + """Grab the instance ID.""" + try: + tmpdir = tempfile.mkdtemp() + describe_json_path = os.path.join(tmpdir, 'describe.json') + cmd = [ + 'cipd', 'describe', + self._package.name, + '-log-level', 'error', + '-version', self._package.version, + '-json-output', describe_json_path + ] + gclient_utils.CheckCallAndFilter( + cmd, filter_fn=lambda _line: None, print_stdout=False) + with open(describe_json_path) as f: + describe_json = json.load(f) + return describe_json.get('result', {}).get('pin', {}).get('instance_id') + finally: + gclient_utils.rmtree(tmpdir) + + def status(self, options, args, file_list): + pass + + def update(self, options, args, file_list): + """Runs ensure.""" + if self._package.authority_for_root: + self._root.ensure() diff --git a/tests/gclient_scm_test.py b/tests/gclient_scm_test.py index 71b5f54a43..7a829a1ef1 100755 --- a/tests/gclient_scm_test.py +++ b/tests/gclient_scm_test.py @@ -11,6 +11,7 @@ from shutil import rmtree from subprocess import Popen, PIPE, STDOUT +import json import logging import os import re @@ -65,7 +66,7 @@ class BaseTestCase(GCBaseTestCase, SuperMoxTestBase): self.mox.StubOutWithMock(gclient_scm.gclient_utils, 'rmtree') self.mox.StubOutWithMock(subprocess2, 'communicate') self.mox.StubOutWithMock(subprocess2, 'Popen') - self._scm_wrapper = gclient_scm.CreateSCM + self._scm_wrapper = gclient_scm.GitWrapper self._original_GitBinaryExists = gclient_scm.GitWrapper.BinaryExists gclient_scm.GitWrapper.BinaryExists = staticmethod(lambda : True) # Absolute path of the fake checkout directory. @@ -256,8 +257,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): return options = self.Options() file_path = join(self.base_path, 'a') - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_list = [] scm.update(options, None, file_list) gclient_scm.os.remove(file_path) @@ -273,8 +274,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): if not self.enabled: return options = self.Options() - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_list = [] scm.update(options, None, file_list) file_list = [] @@ -288,8 +289,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): if not self.enabled: return options = self.Options() - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_list = [] scm.update(options, None, file_list) file_path = join(self.base_path, 'a') @@ -308,8 +309,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): if not self.enabled: return options = self.Options() - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_list = [] scm.update(options, None, file_list) file_path = join(self.base_path, 'c') @@ -334,8 +335,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): options = self.Options() file_path = join(self.base_path, 'a') open(file_path, 'a').writelines('touched\n') - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_list = [] scm.status(options, self.args, file_list) self.assertEquals(file_list, [file_path]) @@ -353,8 +354,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): file_path = join(self.base_path, f) open(file_path, 'a').writelines('touched\n') expected_file_list.extend([file_path]) - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_list = [] scm.status(options, self.args, file_list) expected_file_list = [join(self.base_path, x) for x in ['a', 'b']] @@ -369,8 +370,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): return options = self.Options() expected_file_list = [join(self.base_path, x) for x in ['a', 'b']] - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_list = [] scm.update(options, (), file_list) self.assertEquals(file_list, expected_file_list) @@ -383,8 +384,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): return options = self.Options() options.merge = True - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) scm._Run(['checkout', '-q', 'feature'], options) rev = scm.revinfo(options, (), None) file_list = [] @@ -404,8 +405,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): if not self.enabled: return options = self.Options() - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) scm._Run(['checkout', '-q', 'feature'], options) file_list = [] # Fake a 'y' key press. @@ -436,8 +437,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): file_path = join(self.base_path, 'file') open(file_path, 'w').writelines('new\n') - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_list = [] scm.update(options, (), file_list) self.assert_(gclient_scm.os.path.isdir(dir_path)) @@ -458,8 +459,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): file_path = join(self.base_path, 'file') open(file_path, 'w').writelines('new\n') - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_list = [] scm.update(options, (), file_list) self.assert_(not gclient_scm.os.path.isdir(dir_path)) @@ -470,8 +471,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): if not self.enabled: return options = self.Options() - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_path = join(self.base_path, 'b') open(file_path, 'w').writelines('conflict\n') try: @@ -491,8 +492,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): if not self.enabled: return options = self.Options() - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_path = join(self.base_path, '.git', 'index.lock') with open(file_path, 'w'): pass @@ -505,8 +506,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): return options = self.Options() options.break_repo_locks = True - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_path = join(self.base_path, '.git', 'index.lock') with open(file_path, 'w'): pass @@ -520,8 +521,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): if not self.enabled: return options = self.Options() - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_path = join(self.base_path, 'b') open(file_path, 'w').writelines('conflict\n') scm._Run(['commit', '-am', 'test'], options) @@ -542,8 +543,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): if not self.enabled: return options = self.Options() - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) rev_info = scm.revinfo(options, (), None) self.assertEquals(rev_info, '069c602044c5388d2d15c3f875b057c852003458') @@ -605,8 +606,8 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase): self.mox.ReplayAll() - git_scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + git_scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) # A [fake] git sha1 with a git repo should work (this is in the case that # the LKGR gets flipped to git sha1's some day). self.assertEquals(git_scm.GetUsableRev(self.fake_hash_1, options), @@ -641,8 +642,8 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase): ).AndReturn('') self.mox.ReplayAll() - scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = self._scm_wrapper(self.url, self.root_dir, + self.relpath) scm.update(options, None, []) self.checkstdout('\n') @@ -678,8 +679,8 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase): ).AndReturn('') self.mox.ReplayAll() - scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = self._scm_wrapper(self.url, self.root_dir, + self.relpath) scm.update(options, None, []) self.checkstdout('\n') @@ -715,9 +716,9 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase): self.relpath = '.' self.base_path = join(self.root_dir, self.relpath) - scm = gclient_scm.CreateSCM(url=origin_root_dir, - root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(origin_root_dir, + self.root_dir, + self.relpath) expected_file_list = [join(self.base_path, "a"), join(self.base_path, "b")] @@ -747,9 +748,9 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase): url_with_commit_ref = origin_root_dir +\ '@a7142dc9f0009350b96a11f372b6ea658592aa95' - scm = gclient_scm.CreateSCM(url=url_with_commit_ref, - root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(url_with_commit_ref, + self.root_dir, + self.relpath) expected_file_list = [join(self.base_path, "a"), join(self.base_path, "b")] @@ -778,9 +779,9 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase): self.base_path = join(self.root_dir, self.relpath) url_with_branch_ref = origin_root_dir + '@feature' - scm = gclient_scm.CreateSCM(url=url_with_branch_ref, - root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(url_with_branch_ref, + self.root_dir, + self.relpath) expected_file_list = [join(self.base_path, "a"), join(self.base_path, "b"), @@ -811,9 +812,9 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase): self.base_path = join(self.root_dir, self.relpath) url_with_branch_ref = origin_root_dir + '@refs/remotes/origin/feature' - scm = gclient_scm.CreateSCM(url=url_with_branch_ref, - root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(url_with_branch_ref, + self.root_dir, + self.relpath) expected_file_list = [join(self.base_path, "a"), join(self.base_path, "b"), @@ -843,9 +844,9 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase): self.base_path = join(self.root_dir, self.relpath) url_with_branch_ref = origin_root_dir + '@refs/heads/feature' - scm = gclient_scm.CreateSCM(url=url_with_branch_ref, - root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(url_with_branch_ref, + self.root_dir, + self.relpath) expected_file_list = [join(self.base_path, "a"), join(self.base_path, "b"), @@ -876,8 +877,8 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase): return options = self.Options() expected_file_list = [] - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) + scm = gclient_scm.GitWrapper(self.url, self.root_dir, + self.relpath) file_list = [] options.revision = 'unmanaged' scm.update(options, (), file_list) @@ -887,6 +888,129 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase): self.checkstdout('________ unmanaged solution; skipping .\n') +class CipdWrapperTestCase(BaseTestCase): + + def setUp(self): + # Create this before setting up mocks. + self._cipd_root_dir = tempfile.mkdtemp() + self._workdir = tempfile.mkdtemp() + BaseTestCase.setUp(self) + + self._cipd_instance_url = 'https://chrome-infra-packages.appspot.com' + self._cipd_root = gclient_scm.CipdRoot( + self._cipd_root_dir, + self._cipd_instance_url) + self._cipd_packages = [ + self._cipd_root.add_package('f', 'foo_package', 'foo_version'), + self._cipd_root.add_package('b', 'bar_package', 'bar_version'), + self._cipd_root.add_package('b', 'baz_package', 'baz_version'), + ] + self.mox.StubOutWithMock(gclient_scm.CipdRoot, 'add_package') + self.mox.StubOutWithMock(gclient_scm.CipdRoot, 'clobber') + self.mox.StubOutWithMock(gclient_scm.CipdRoot, 'ensure') + + def tearDown(self): + BaseTestCase.tearDown(self) + rmtree(self._cipd_root_dir) + rmtree(self._workdir) + + def createScmWithPackageThatSatisfies(self, condition): + return gclient_scm.CipdWrapper( + url=self._cipd_instance_url, + root_dir=self._cipd_root_dir, + relpath='fake_relpath', + root=self._cipd_root, + package=self.getPackageThatSatisfies(condition)) + + def getPackageThatSatisfies(self, condition): + for p in self._cipd_packages: + if condition(p): + return p + + self.fail('Unable to find a satisfactory package.') + + def testSingleRootAuthority(self): + """Checks that exactly one package has root authority.""" + self.assertEquals(1, len([p for p in self._cipd_packages + if p.authority_for_root])) + + def testRevert(self): + """Checks that revert w/ root authority clobbers and reruns ensure.""" + scm = self.createScmWithPackageThatSatisfies( + lambda p: p.authority_for_root) + + gclient_scm.CipdRoot.clobber() + gclient_scm.CipdRoot.ensure() + + self.mox.ReplayAll() + + scm.revert(None, (), []) + + def testRevertWithoutAuthority(self): + """Checks that revert w/o root authority does nothing.""" + scm = self.createScmWithPackageThatSatisfies( + lambda p: not p.authority_for_root) + + self.mox.ReplayAll() + + scm.revert(None, (), []) + + def testRevinfo(self): + """Checks that revinfo uses the JSON from cipd describe.""" + scm = self.createScmWithPackageThatSatisfies(lambda _: True) + + expected_revinfo = '0123456789abcdef0123456789abcdef01234567' + json_contents = { + 'result': { + 'pin': { + 'instance_id': expected_revinfo, + } + } + } + describe_json_path = join(self._workdir, 'describe.json') + with open(describe_json_path, 'w') as describe_json: + json.dump(json_contents, describe_json) + + cmd = [ + 'cipd', 'describe', 'foo_package', + '-log-level', 'error', + '-version', 'foo_version', + '-json-output', describe_json_path, + ] + + self.mox.StubOutWithMock(tempfile, 'mkdtemp') + + tempfile.mkdtemp().AndReturn(self._workdir) + gclient_scm.gclient_utils.CheckCallAndFilter( + cmd, filter_fn=mox.IgnoreArg(), print_stdout=False) + gclient_scm.gclient_utils.rmtree(self._workdir) + + self.mox.ReplayAll() + + revinfo = scm.revinfo(None, (), []) + self.assertEquals(revinfo, expected_revinfo) + + def testUpdate(self): + """Checks that update w/ root authority runs ensure.""" + scm = self.createScmWithPackageThatSatisfies( + lambda p: p.authority_for_root) + + gclient_scm.CipdRoot.ensure() + + self.mox.ReplayAll() + + scm.update(None, (), []) + + def testUpdateWithoutAuthority(self): + """Checks that update w/o root authority does nothing.""" + scm = self.createScmWithPackageThatSatisfies( + lambda p: not p.authority_for_root) + + self.mox.ReplayAll() + + scm.update(None, (), []) + + if __name__ == '__main__': level = logging.DEBUG if '-v' in sys.argv else logging.FATAL logging.basicConfig( diff --git a/tests/gclient_test.py b/tests/gclient_test.py index 529b166f69..99d0b0e369 100755 --- a/tests/gclient_test.py +++ b/tests/gclient_test.py @@ -60,15 +60,15 @@ class GclientTest(trial_dir.TestCase): self.previous_dir = os.getcwd() os.chdir(self.root_dir) # Manual mocks. - self._old_createscm = gclient.gclient_scm.CreateSCM - gclient.gclient_scm.CreateSCM = self._createscm + self._old_createscm = gclient.Dependency.CreateSCM + gclient.Dependency.CreateSCM = self._createscm self._old_sys_stdout = sys.stdout sys.stdout = gclient.gclient_utils.MakeFileAutoFlush(sys.stdout) sys.stdout = gclient.gclient_utils.MakeFileAnnotated(sys.stdout) def tearDown(self): self.assertEquals([], self._get_processed()) - gclient.gclient_scm.CreateSCM = self._old_createscm + gclient.Dependency.CreateSCM = self._old_createscm sys.stdout = self._old_sys_stdout os.chdir(self.previous_dir) super(GclientTest, self).tearDown()