diff --git a/git_cache.py b/git_cache.py index e80923cdb..9f835b5ff 100755 --- a/git_cache.py +++ b/git_cache.py @@ -145,9 +145,24 @@ class Mirror(object): os.path.dirname(os.path.abspath(__file__)), 'gsutil.py') cachepath_lock = threading.Lock() + @staticmethod + def parse_fetch_spec(spec): + """Parses and canonicalizes a fetch spec. + + Returns (fetchspec, value_regex), where value_regex can be used + with 'git config --replace-all'. + """ + parts = spec.split(':', 1) + src = parts[0].lstrip('+').rstrip('/') + if not src.startswith('refs/'): + src = 'refs/heads/%s' % src + dest = parts[1].rstrip('/') if len(parts) > 1 else src + regex = r'\+%s:.*' % src.replace('*', r'\*') + return ('+%s:%s' % (src, dest), regex) + def __init__(self, url, refs=None, print_func=None): self.url = url - self.refs = refs or [] + self.fetch_specs = set([self.parse_fetch_spec(ref) for ref in (refs or [])]) self.basedir = self.UrlToCacheDir(url) self.mirror_path = os.path.join(self.GetCachePath(), self.basedir) if print_func: @@ -236,16 +251,9 @@ class Mirror(object): self.RunGit(['config', 'remote.origin.url', self.url], cwd=cwd) self.RunGit(['config', '--replace-all', 'remote.origin.fetch', '+refs/heads/*:refs/heads/*', r'\+refs/heads/\*:.*'], cwd=cwd) - for ref in self.refs: - ref = ref.lstrip('+').rstrip('/') - if ref.startswith('refs/'): - refspec = '+%s:%s' % (ref, ref) - regex = r'\+%s:.*' % ref.replace('*', r'\*') - else: - refspec = '+refs/%s/*:refs/%s/*' % (ref, ref) - regex = r'\+refs/heads/%s:.*' % ref.replace('*', r'\*') + for spec, value_regex in self.fetch_specs: self.RunGit( - ['config', '--replace-all', 'remote.origin.fetch', refspec, regex], + ['config', '--replace-all', 'remote.origin.fetch', spec, value_regex], cwd=cwd) def bootstrap_repo(self, directory): @@ -314,9 +322,27 @@ class Mirror(object): def exists(self): return os.path.isfile(os.path.join(self.mirror_path, 'config')) + def _preserve_fetchspec(self): + """Read and preserve remote.origin.fetch from an existing mirror. + + This modifies self.fetch_specs. + """ + if not self.exists(): + return + try: + config_fetchspecs = subprocess.check_output( + [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], + cwd=self.mirror_path) + for fetchspec in config_fetchspecs.splitlines(): + self.fetch_specs.add(self.parse_fetch_spec(fetchspec)) + except subprocess.CalledProcessError: + logging.warn('Tried and failed to preserve remote.origin.fetch from the ' + 'existing cache directory. You may need to manually edit ' + '%s and "git cache fetch" again.' + % os.path.join(self.mirror_path, 'config')) + def _ensure_bootstrapped(self, depth, bootstrap, force=False): tempdir = None - config_file = os.path.join(self.mirror_path, 'config') pack_dir = os.path.join(self.mirror_path, 'objects', 'pack') pack_files = [] @@ -324,16 +350,19 @@ class Mirror(object): pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')] should_bootstrap = (force or - not os.path.exists(config_file) or + not self.exists() or len(pack_files) > GC_AUTOPACKLIMIT) if should_bootstrap: + if self.exists(): + # Re-bootstrapping an existing mirror; preserve existing fetch spec. + self._preserve_fetchspec() tempdir = tempfile.mkdtemp( prefix='_cache_tmp', suffix=self.basedir, dir=self.GetCachePath()) bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir) if bootstrapped: # Bootstrap succeeded; delete previous cache, if any. gclient_utils.rmtree(self.mirror_path) - elif not os.path.exists(config_file): + elif not self.exists(): # Bootstrap failed, no previous cache; start with a bare git dir. self.RunGit(['init', '--bare'], cwd=tempdir) else: @@ -563,6 +592,9 @@ def CMDpopulate(parser, args): def CMDfetch(parser, args): """Update mirror, and fetch in cwd.""" parser.add_option('--all', action='store_true', help='Fetch all remotes') + parser.add_option('--no_bootstrap', '--no-bootstrap', + action='store_true', + help='Don\'t (re)bootstrap from Google Storage') options, args = parser.parse_args(args) # Figure out which remotes to fetch. This mimics the behavior of regular @@ -593,7 +625,7 @@ def CMDfetch(parser, args): git_dir = os.path.abspath(git_dir) if git_dir.startswith(cachepath): mirror = Mirror.FromPath(git_dir) - mirror.populate() + mirror.populate(bootstrap=not options.no_bootstrap) return 0 for remote in remotes: remote_url = subprocess.check_output( @@ -602,7 +634,7 @@ def CMDfetch(parser, args): mirror = Mirror.FromPath(remote_url) mirror.print = lambda *args: None print('Updating git cache...') - mirror.populate() + mirror.populate(bootstrap=not options.no_bootstrap) subprocess.check_call([Mirror.git_exe, 'fetch', remote]) return 0 diff --git a/tests/git_cache_test.py b/tests/git_cache_test.py new file mode 100755 index 000000000..55ea1269d --- /dev/null +++ b/tests/git_cache_test.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unit tests for git_cache.py""" + +import os +import shutil +import sys +import tempfile +import unittest + +DEPOT_TOOLS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, DEPOT_TOOLS_ROOT) + +from testing_support import coverage_utils +import git_cache + +class GitCacheTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.cache_dir = tempfile.mkdtemp(prefix='git_cache_test_') + git_cache.Mirror.SetCachePath(cls.cache_dir) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.cache_dir, ignore_errors=True) + + def testParseFetchSpec(self): + testData = [ + ([], []), + (['master'], [('+refs/heads/master:refs/heads/master', + r'\+refs/heads/master:.*')]), + (['master/'], [('+refs/heads/master:refs/heads/master', + r'\+refs/heads/master:.*')]), + (['+master'], [('+refs/heads/master:refs/heads/master', + r'\+refs/heads/master:.*')]), + (['refs/heads/*'], [('+refs/heads/*:refs/heads/*', + r'\+refs/heads/\*:.*')]), + (['foo/bar/*', 'baz'], [('+refs/heads/foo/bar/*:refs/heads/foo/bar/*', + r'\+refs/heads/foo/bar/\*:.*'), + ('+refs/heads/baz:refs/heads/baz', + r'\+refs/heads/baz:.*')]), + (['refs/foo/*:refs/bar/*'], [('+refs/foo/*:refs/bar/*', + r'\+refs/foo/\*:.*')]) + ] + + mirror = git_cache.Mirror('test://phony.example.biz') + for fetch_specs, expected in testData: + mirror = git_cache.Mirror('test://phony.example.biz', refs=fetch_specs) + self.assertItemsEqual(mirror.fetch_specs, expected) + +if __name__ == '__main__': + sys.exit(coverage_utils.covered_main(( + os.path.join(DEPOT_TOOLS_ROOT, 'git_cache.py') + ), required_percentage=0))