From 3534aa56f48111e9906b3b2d0924ca8998165c32 Mon Sep 17 00:00:00 2001 From: "ilevy@chromium.org" Date: Sat, 20 Jul 2013 01:58:08 +0000 Subject: [PATCH] Allow gclient clone in non-empty directories Add an option in DEPS files to clone a project into a temp dir and then copy into expected final dir. This allows checking out a git repo into a folder which is non-empty. It is useful for projects that are embedded in src/ but want to specify the revision of src/ in the embedded project (such as android private). BUG=165280 Review URL: https://chromiumcodereview.appspot.com/19359002 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@212720 0039d316-1c4b-4281-b951-d872f2087c98 --- gclient_scm.py | 103 ++++++++++++------------------------- tests/gclient_scm_test.py | 16 ------ tests/gclient_smoketest.py | 86 +++++++++++++++++++++---------- 3 files changed, 91 insertions(+), 114 deletions(-) diff --git a/gclient_scm.py b/gclient_scm.py index e7147ce405..af74b754d1 100644 --- a/gclient_scm.py +++ b/gclient_scm.py @@ -10,6 +10,7 @@ import os import posixpath import re import sys +import tempfile import threading import time @@ -350,10 +351,9 @@ class GitWrapper(SCMWrapper): # hash is also a tag, only make a distinction at checkout rev_type = "hash" - if not os.path.exists(self.checkout_path) or ( - os.path.isdir(self.checkout_path) and - not os.listdir(self.checkout_path)): - gclient_utils.safe_makedirs(os.path.dirname(self.checkout_path)) + if (not os.path.exists(self.checkout_path) or + (os.path.isdir(self.checkout_path) and + not os.path.exists(os.path.join(self.checkout_path, '.git')))): self._Clone(revision, url, options) self.UpdateSubmoduleConfig() if file_list is not None: @@ -408,19 +408,6 @@ class GitWrapper(SCMWrapper): if return_early: return - if not self._IsValidGitRepo(): - # .git directory is hosed for some reason, set it back up. - print('_____ %s/.git is corrupted, rebuilding' % self.relpath) - self._Run(['init'], options) - self._Run(['remote', 'set-url', 'origin', url], options) - - if not self._HasHead(): - # Previous checkout was aborted before branches could be created in repo, - # so we need to reconstruct them here. - self._Run(['-c', 'core.deltaBaseCacheLimit=2g', 'pull', 'origin', - 'master'], options) - self._FetchAndReset(revision, file_list, options) - cur_branch = self._GetCurrentBranch() # Cases: @@ -859,47 +846,45 @@ class GitWrapper(SCMWrapper): print('') template_path = os.path.join( os.path.dirname(THIS_FILE_PATH), 'git-templates') - clone_cmd = ['-c', 'core.deltaBaseCacheLimit=2g', 'clone', '--progress', - '--template=%s' % template_path] + clone_cmd = ['-c', 'core.deltaBaseCacheLimit=2g', 'clone', '--no-checkout', + '--progress', '--template=%s' % template_path] if self.cache_dir: clone_cmd.append('--shared') - if revision.startswith('refs/heads/'): - clone_cmd.extend(['-b', revision.replace('refs/heads/', '')]) - detach_head = False - else: - detach_head = True if options.verbose: clone_cmd.append('--verbose') - clone_cmd.extend([url, self.checkout_path]) - + clone_cmd.append(url) # If the parent directory does not exist, Git clone on Windows will not # create it, so we need to do it manually. parent_dir = os.path.dirname(self.checkout_path) - if not os.path.exists(parent_dir): - gclient_utils.safe_makedirs(parent_dir) - - for _ in range(3): - try: - self._Run(clone_cmd, options, cwd=self._root_dir, git_filter=True) - break - except subprocess2.CalledProcessError, e: - # Too bad we don't have access to the actual output yet. - # We should check for "transfer closed with NNN bytes remaining to - # read". In the meantime, just make sure .git exists. - if (e.returncode == 128 and - os.path.exists(os.path.join(self.checkout_path, '.git'))): + gclient_utils.safe_makedirs(parent_dir) + tmp_dir = tempfile.mkdtemp( + prefix='_gclient_%s_' % os.path.basename(self.checkout_path), + dir=parent_dir) + try: + clone_cmd.append(tmp_dir) + for i in xrange(3): + try: + self._Run(clone_cmd, options, cwd=self._root_dir, git_filter=True) + break + except subprocess2.CalledProcessError as e: + gclient_utils.rmtree(os.path.join(tmp_dir, '.git')) + if e.returncode != 128 or i == 2: + raise print(str(e)) print('Retrying...') - continue - raise e - - # Update the "branch-heads" remote-tracking branches, since we might need it - # to checkout a specific revision below. - self._UpdateBranchHeads(options, fetch=True) - - if detach_head: + gclient_utils.safe_makedirs(self.checkout_path) + os.rename(os.path.join(tmp_dir, '.git'), + os.path.join(self.checkout_path, '.git')) + finally: + if os.listdir(tmp_dir): + print('\n_____ removing non-empty tmp dir %s' % tmp_dir) + gclient_utils.rmtree(tmp_dir) + if revision.startswith('refs/heads/'): + self._Run( + ['checkout', '--quiet', revision.replace('refs/heads/', '')], options) + else: # Squelch git's very verbose detached HEAD warning and use our own - self._Capture(['checkout', '--quiet', '%s' % revision]) + self._Run(['checkout', '--quiet', revision], options) print( ('Checked out %s to a detached HEAD. Before making any commits\n' 'in this repo, you should use \'git checkout \' to switch to\n' @@ -977,28 +962,6 @@ class GitWrapper(SCMWrapper): # whitespace between projects when syncing. print('') - def _IsValidGitRepo(self): - """Returns if the directory is a valid git repository. - - Checks if git status works. - """ - try: - self._Capture(['status']) - return True - except subprocess2.CalledProcessError: - return False - - def _HasHead(self): - """Returns True if any commit is checked out. - - This is done by checking if rev-parse HEAD works in the current repository. - """ - try: - self._GetCurrentBranch() - return True - except subprocess2.CalledProcessError: - return False - @staticmethod def _CheckMinVersion(min_version): (ok, current_version) = scm.GIT.AssertVersion(min_version) diff --git a/tests/gclient_scm_test.py b/tests/gclient_scm_test.py index e91636854f..465d172fd9 100755 --- a/tests/gclient_scm_test.py +++ b/tests/gclient_scm_test.py @@ -8,7 +8,6 @@ # pylint: disable=E1103 # Import before super_mox to keep valid references. -from os import rename from shutil import rmtree from subprocess import Popen, PIPE, STDOUT @@ -1026,21 +1025,6 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase): self.assertRaisesError(exception, scm.update, options, (), []) sys.stdout.close() - def testUpdateNotGit(self): - if not self.enabled: - return - options = self.Options() - scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir, - relpath=self.relpath) - git_path = join(self.base_path, '.git') - rename(git_path, git_path + 'foo') - exception = ('\n____ . at refs/heads/master\n' - '\tPath is not a git repo. No .git dir.\n' - '\tTo resolve:\n' - '\t\trm -rf .\n' - '\tAnd run gclient sync again\n') - self.assertRaisesError(exception, scm.update, options, (), []) - def testRevinfo(self): if not self.enabled: return diff --git a/tests/gclient_smoketest.py b/tests/gclient_smoketest.py index 8c9a414d5d..4f0e30601d 100755 --- a/tests/gclient_smoketest.py +++ b/tests/gclient_smoketest.py @@ -839,8 +839,12 @@ class GClientSmokeGIT(GClientSmokeBase): # TODO(maruel): safesync. self.gclient(['config', self.git_base + 'repo_1', '--name', 'src']) # Test unversioned checkout. - self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'], - ['running', 'running', 'running', 'running', 'running']) + self.parseGclient( + ['sync', '--deps', 'mac', '--jobs', '1'], + ['running', ('running', self.root_dir + '/src'), + 'running', ('running', self.root_dir + '/src/repo2'), + 'running', ('running', self.root_dir + '/src/repo2/repo_renamed'), + 'running', 'running']) # TODO(maruel): http://crosbug.com/3582 hooks run even if not matching, must # add sync parsing to get the list of updated files. tree = self.mangle_git_tree(('repo_1@2', 'src'), @@ -856,10 +860,13 @@ class GClientSmokeGIT(GClientSmokeBase): # Test incremental versioned sync: sync backward. diffdir = os.path.join(self.root_dir, 'src', 'repo2', 'repo_renamed') - self.parseGclient(['sync', '--jobs', '1', '--revision', - 'src@' + self.githash('repo_1', 1), - '--deps', 'mac', '--delete_unversioned_trees'], - ['running', 'running', ('running', diffdir), 'deleting']) + self.parseGclient( + ['sync', '--jobs', '1', '--revision', + 'src@' + self.githash('repo_1', 1), + '--deps', 'mac', '--delete_unversioned_trees'], + ['running', ('running', self.root_dir + '/src/repo2/repo3'), + 'running', ('running', self.root_dir + '/src/repo4'), + ('running', diffdir), 'deleting']) tree = self.mangle_git_tree(('repo_1@1', 'src'), ('repo_2@2', 'src/repo2'), ('repo_3@1', 'src/repo2/repo3'), @@ -869,8 +876,10 @@ class GClientSmokeGIT(GClientSmokeBase): # Test incremental sync: delete-unversioned_trees isn't there. expect3 = ('running', os.path.join(self.root_dir, 'src', 'repo2', 'repo3')) expect4 = ('running', os.path.join(self.root_dir, 'src', 'repo4')) - self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'], - ['running', 'running', 'running', expect3, expect4]) + self.parseGclient( + ['sync', '--deps', 'mac', '--jobs', '1'], + ['running', ('running', self.root_dir + '/src/repo2/repo_renamed'), + 'running', 'running', expect3, expect4]) tree = self.mangle_git_tree(('repo_1@2', 'src'), ('repo_2@1', 'src/repo2'), ('repo_3@1', 'src/repo2/repo3'), @@ -888,9 +897,12 @@ class GClientSmokeGIT(GClientSmokeBase): self.parseGclient( ['sync', '--deps', 'mac', '--jobs', '1', '--revision', 'invalid@' + self.githash('repo_1', 1)], - ['running', 'running', 'running', 'running', 'running'], + ['running', ('running', self.root_dir + '/src'), + 'running', ('running', self.root_dir + '/src/repo2'), + 'running', ('running', self.root_dir + '/src/repo2/repo_renamed'), + 'running', 'running'], 'Please fix your script, having invalid --revision flags ' - 'will soon considered an error.\n') + 'will soon considered an error.\n') tree = self.mangle_git_tree(('repo_1@2', 'src'), ('repo_2@1', 'src/repo2'), ('repo_3@2', 'src/repo2/repo_renamed')) @@ -903,9 +915,13 @@ class GClientSmokeGIT(GClientSmokeBase): return # When no solution name is provided, gclient uses the first solution listed. self.gclient(['config', self.git_base + 'repo_1', '--name', 'src']) - self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1', - '--revision', self.githash('repo_1', 1)], - ['running', 'running', 'running', 'running']) + self.parseGclient( + ['sync', '--deps', 'mac', '--jobs', '1', + '--revision', self.githash('repo_1', 1)], + ['running', ('running', self.root_dir + '/src'), + 'running', ('running', self.root_dir + '/src/repo2'), + 'running', ('running', self.root_dir + '/src/repo2/repo3'), + 'running', ('running', self.root_dir + '/src/repo4')]) tree = self.mangle_git_tree(('repo_1@1', 'src'), ('repo_2@2', 'src/repo2'), ('repo_3@1', 'src/repo2/repo3'), @@ -918,9 +934,13 @@ class GClientSmokeGIT(GClientSmokeBase): # TODO(maruel): safesync. self.gclient(['config', self.git_base + 'repo_1', '--name', 'src']) # Test unversioned checkout. - self.parseGclient(['sync', '--deps', 'mac', '--jobs', '8'], - ['running', 'running', 'running', 'running', 'running'], - untangle=True) + self.parseGclient( + ['sync', '--deps', 'mac', '--jobs', '8'], + ['running', ('running', self.root_dir + '/src'), + 'running', ('running', self.root_dir + '/src/repo2'), + 'running', ('running', self.root_dir + '/src/repo2/repo_renamed'), + 'running', 'running'], + untangle=True) # TODO(maruel): http://crosbug.com/3582 hooks run even if not matching, must # add sync parsing to get the list of updated files. tree = self.mangle_git_tree(('repo_1@2', 'src'), @@ -940,7 +960,9 @@ class GClientSmokeGIT(GClientSmokeBase): self.parseGclient( ['sync', '--revision', 'src@' + self.githash('repo_1', 1), '--deps', 'mac', '--delete_unversioned_trees', '--jobs', '8'], - ['running', 'running', expect3, 'deleting'], + ['running', ('running', self.root_dir + '/src/repo4'), + 'running', ('running', self.root_dir + '/src/repo2/repo3'), + expect3, 'deleting'], untangle=True) tree = self.mangle_git_tree(('repo_1@1', 'src'), ('repo_2@2', 'src/repo2'), @@ -951,11 +973,11 @@ class GClientSmokeGIT(GClientSmokeBase): # Test incremental sync: delete-unversioned_trees isn't there. expect4 = os.path.join(self.root_dir, 'src', 'repo2', 'repo3') expect5 = os.path.join(self.root_dir, 'src', 'repo4') - self.parseGclient(['sync', '--deps', 'mac', '--jobs', '8'], - ['running', 'running', 'running', - ('running', expect4), - ('running', expect5)], - untangle=True) + self.parseGclient( + ['sync', '--deps', 'mac', '--jobs', '8'], + ['running', ('running', self.root_dir + '/src/repo2/repo_renamed'), + 'running', 'running', ('running', expect4), ('running', expect5)], + untangle=True) tree = self.mangle_git_tree(('repo_1@2', 'src'), ('repo_2@1', 'src/repo2'), ('repo_3@1', 'src/repo2/repo3'), @@ -1074,12 +1096,16 @@ class GClientSmokeBoth(GClientSmokeBase): '{"name": "src-git",' '"url": "' + self.git_base + 'repo_1"}]']) self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'], - ['running', 'running', 'running', + ['running', + 'running', ('running', self.root_dir + '/src-git'), + 'running', # This is due to the way svn update is called for a single # file when File() is used in a DEPS file. ('running', self.root_dir + '/src/file/other'), - 'running', 'running', 'running', 'running', 'running', 'running', - 'running', 'running']) + 'running', 'running', 'running', + 'running', ('running', self.root_dir + '/src/repo2'), + 'running', ('running', self.root_dir + '/src/repo2/repo_renamed'), + 'running', 'running', 'running']) tree = self.mangle_git_tree(('repo_1@2', 'src-git'), ('repo_2@1', 'src/repo2'), ('repo_3@2', 'src/repo2/repo_renamed')) @@ -1110,7 +1136,7 @@ class GClientSmokeBoth(GClientSmokeBase): self.checkString('', stderr) self.assertEquals(0, returncode) results = self.splitBlock(stdout) - self.assertEquals(12, len(results)) + self.assertEquals(15, len(results)) tree = self.mangle_git_tree(('repo_1@2', 'src-git'), ('repo_2@1', 'src/repo2'), ('repo_3@2', 'src/repo2/repo_renamed')) @@ -1137,8 +1163,12 @@ class GClientSmokeBoth(GClientSmokeBase): self.parseGclient( ['sync', '--deps', 'mac', '--jobs', '1', '--revision', '1', '-r', 'src-git@' + self.githash('repo_1', 1)], - ['running', 'running', 'running', 'running', - 'running', 'running', 'running', 'running'], + ['running', + 'running', ('running', self.root_dir + '/src-git'), + 'running', 'running', 'running', + 'running', ('running', self.root_dir + '/src/repo2'), + 'running', ('running', self.root_dir + '/src/repo2/repo3'), + 'running', ('running', self.root_dir + '/src/repo4')], expected_stderr= 'You must specify the full solution name like --revision src@1\n' 'when you have multiple solutions setup in your .gclient file.\n'