diff --git a/gclient_utils.py b/gclient_utils.py index d8bbf023e..4c8edb6bf 100644 --- a/gclient_utils.py +++ b/gclient_utils.py @@ -595,9 +595,9 @@ class ExecutionQueue(object): t.kwargs['options'].stdout.full_flush() if self.progress: self.progress.update(1) - assert not t.name in self.ran - if not t.name in self.ran: - self.ran.append(t.name) + assert not t.item.name in self.ran + if not t.item.name in self.ran: + self.ran.append(t.item.name) def _run_one_task(self, task_item, args, kwargs): if self.jobs > 1: diff --git a/tests/gclient_smoketest.py b/tests/gclient_smoketest.py index 29913e27f..ad166a93b 100755 --- a/tests/gclient_smoketest.py +++ b/tests/gclient_smoketest.py @@ -49,9 +49,25 @@ class GClientSmokeBase(FakeReposTestBase): return (stdout.replace('\r\n', '\n'), stderr.replace('\r\n', '\n'), process.returncode) - def parseGclient(self, cmd, items, expected_stderr=''): - """Parse gclient's output to make it easier to test.""" + def parseGclient(self, cmd, items, expected_stderr='', untangle=False): + """Parse gclient's output to make it easier to test. + If untangle is True, tries to sort out the output from parallel checkout.""" (stdout, stderr, returncode) = self.gclient(cmd) + if untangle: + tasks = {} + remaining = [] + for line in stdout.splitlines(False): + m = re.match(r'^(\d)+>(.*)$', line) + if not m: + remaining.append(line) + else: + self.assertEquals([], remaining) + tasks.setdefault(int(m.group(1)), []).append(m.group(2)) + out = [] + for key in sorted(tasks.iterkeys()): + out.extend(tasks[key]) + out.extend(remaining) + stdout = '\n'.join(out) self.checkString(expected_stderr, stderr) self.assertEquals(0, returncode) return self.checkBlock(stdout, items) @@ -78,9 +94,14 @@ class GClientSmokeBase(FakeReposTestBase): # Blah, it's when a dependency is deleted, we should probably not # output this message. results.append([line]) - else: - print line - raise Exception('fail', line) + elif ( + not re.match( + r'_____ [^ ]+ : Attempting rebase onto [0-9a-f]+...', + line) and + not re.match(r'_____ [^ ]+ at [^ ]+', line)): + # The two regexp above are a bit too broad, they are necessary only + # for git checkouts. + self.fail(line) else: results.append([[match.group(1), match.group(2), match.group(3)]]) else: @@ -224,7 +245,7 @@ class GClientSmoke(GClientSmokeBase): 'deps = { "src": "%strunk/src" }' % (self.svn_base)) src = join(self.root_dir, 'src') os.mkdir(src) - res = self.gclient(['status'], src) + res = self.gclient(['status', '--jobs', '1'], src) self.checkBlock(res[0], [('running', deps), ('running', src)]) @@ -237,7 +258,7 @@ class GClientSmokeSVN(GClientSmokeBase): # TODO(maruel): safesync. self.gclient(['config', self.svn_base + 'trunk/src/']) # Test unversioned checkout. - self.parseGclient(['sync', '--deps', 'mac'], + self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'], ['running', 'running', # This is due to the way svn update is called for a # single file when File() is used in a DEPS file. @@ -257,9 +278,10 @@ class GClientSmokeSVN(GClientSmokeBase): os.remove(join(self.root_dir, 'src', 'svn_hooked1')) # Test incremental versioned sync: sync backward. - self.parseGclient(['sync', '--revision', 'src@1', '--deps', 'mac', - '--delete_unversioned_trees'], - ['running', 'running', 'running', 'running', 'deleting']) + self.parseGclient( + ['sync', '--revision', 'src@1', '--deps', 'mac', + '--delete_unversioned_trees', '--jobs', '1'], + ['running', 'running', 'running', 'running', 'deleting']) tree = self.mangle_svn_tree( ('trunk/src@1', 'src'), ('trunk/third_party/foo@2', 'src/third_party/fpp'), @@ -269,7 +291,7 @@ class GClientSmokeSVN(GClientSmokeBase): self.FAKE_REPOS.svn_revs[2]['trunk/other/DEPS']) self.assertTree(tree) # Test incremental sync: delete-unversioned_trees isn't there. - self.parseGclient(['sync', '--deps', 'mac'], + self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'], ['running', 'running', 'running', 'running', 'running']) tree = self.mangle_svn_tree( ('trunk/src@2', 'src'), @@ -285,7 +307,8 @@ class GClientSmokeSVN(GClientSmokeBase): def testSyncIgnoredSolutionName(self): """TODO(maruel): This will become an error soon.""" self.gclient(['config', self.svn_base + 'trunk/src/']) - results = self.gclient(['sync', '--deps', 'mac', '-r', 'invalid@1']) + results = self.gclient( + ['sync', '--deps', 'mac', '-r', 'invalid@1', '--jobs', '1']) self.checkBlock(results[0], [ 'running', 'running', # This is due to the way svn update is called for a single file when @@ -307,7 +330,7 @@ class GClientSmokeSVN(GClientSmokeBase): def testSyncNoSolutionName(self): # When no solution name is provided, gclient uses the first solution listed. self.gclient(['config', self.svn_base + 'trunk/src/']) - self.parseGclient(['sync', '--deps', 'mac', '-r', '1'], + self.parseGclient(['sync', '--deps', 'mac', '-r', '1', '--jobs', '1'], ['running', 'running', 'running', 'running']) tree = self.mangle_svn_tree( ('trunk/src@1', 'src'), @@ -316,13 +339,67 @@ class GClientSmokeSVN(GClientSmokeBase): ('trunk/third_party/foo@2', 'src/third_party/prout')) self.assertTree(tree) + def testSyncJobs(self): + # TODO(maruel): safesync. + self.gclient(['config', self.svn_base + 'trunk/src/']) + # Test unversioned checkout. + self.parseGclient( + ['sync', '--deps', 'mac', '--jobs', '8'], + ['running', 'running', + # This is due to the way svn update is called for a + # single file when File() is used in a DEPS file. + ('running', self.root_dir + '/src/file/other'), + 'running', 'running', 'running', 'running'], + untangle=True) + tree = self.mangle_svn_tree( + ('trunk/src@2', 'src'), + ('trunk/third_party/foo@1', 'src/third_party/foo'), + ('trunk/other@2', 'src/other')) + tree['src/file/other/DEPS'] = ( + self.FAKE_REPOS.svn_revs[2]['trunk/other/DEPS']) + tree['src/svn_hooked1'] = 'svn_hooked1' + self.assertTree(tree) + + # Manually remove svn_hooked1 before synching to make sure it's not + # recreated. + os.remove(join(self.root_dir, 'src', 'svn_hooked1')) + + # Test incremental versioned sync: sync backward. + self.parseGclient( + ['sync', '--revision', 'src@1', '--deps', 'mac', + '--delete_unversioned_trees', '--jobs', '8'], + ['running', 'running', 'running', 'running', 'deleting'], + untangle=True) + tree = self.mangle_svn_tree( + ('trunk/src@1', 'src'), + ('trunk/third_party/foo@2', 'src/third_party/fpp'), + ('trunk/other@1', 'src/other'), + ('trunk/third_party/foo@2', 'src/third_party/prout')) + tree['src/file/other/DEPS'] = ( + self.FAKE_REPOS.svn_revs[2]['trunk/other/DEPS']) + self.assertTree(tree) + # Test incremental sync: delete-unversioned_trees isn't there. + self.parseGclient(['sync', '--deps', 'mac', '--jobs', '8'], + ['running', 'running', 'running', 'running', 'running'], + untangle=True) + tree = self.mangle_svn_tree( + ('trunk/src@2', 'src'), + ('trunk/third_party/foo@2', 'src/third_party/fpp'), + ('trunk/third_party/foo@1', 'src/third_party/foo'), + ('trunk/other@2', 'src/other'), + ('trunk/third_party/foo@2', 'src/third_party/prout')) + tree['src/file/other/DEPS'] = ( + self.FAKE_REPOS.svn_revs[2]['trunk/other/DEPS']) + tree['src/svn_hooked1'] = 'svn_hooked1' + self.assertTree(tree) + def testRevertAndStatus(self): self.gclient(['config', self.svn_base + 'trunk/src/']) # Tested in testSync. self.gclient(['sync', '--deps', 'mac']) write(join(self.root_dir, 'src', 'other', 'hi'), 'Hey!') - out = self.parseGclient(['status', '--deps', 'mac'], + out = self.parseGclient(['status', '--deps', 'mac', '--jobs', '1'], [['running', join(self.root_dir, 'src')], ['running', join(self.root_dir, 'src', 'other')]]) out = self.svnBlockCleanup(out) @@ -336,7 +413,7 @@ class GClientSmokeSVN(GClientSmokeBase): # Revert implies --force implies running hooks without looking at pattern # matching. - results = self.gclient(['revert', '--deps', 'mac']) + results = self.gclient(['revert', '--deps', 'mac', '--jobs', '1']) out = self.splitBlock(results[0]) # src, src/other is missing, src/other, src/third_party/foo is missing, # src/third_party/foo, 2 svn hooks, 3 related to File(). @@ -353,7 +430,7 @@ class GClientSmokeSVN(GClientSmokeBase): tree['src/svn_hooked2'] = 'svn_hooked2' self.assertTree(tree) - out = self.parseGclient(['status', '--deps', 'mac'], + out = self.parseGclient(['status', '--deps', 'mac', '--jobs', '1'], [['running', join(self.root_dir, 'src')]]) out = self.svnBlockCleanup(out) self.checkString('file', out[0][1]) @@ -372,7 +449,7 @@ class GClientSmokeSVN(GClientSmokeBase): # Without --verbose, gclient won't output the directories without # modification. - out = self.parseGclient(['status', '--deps', 'mac'], + out = self.parseGclient(['status', '--deps', 'mac', '--jobs', '1'], [['running', join(self.root_dir, 'src')], ['running', join(self.root_dir, 'src', 'other')]]) out = self.svnBlockCleanup(out) @@ -384,7 +461,8 @@ class GClientSmokeSVN(GClientSmokeBase): self.assertEquals(2, len(out[1])) # So verify it works with --verbose. - out = self.parseGclient(['status', '--deps', 'mac', '--verbose'], + out = self.parseGclient( + ['status', '--deps', 'mac', '--verbose', '--jobs', '1'], [['running', join(self.root_dir, 'src')], ['running', join(self.root_dir, 'src', 'third_party', 'fpp')], ['running', join(self.root_dir, 'src', 'other')], @@ -404,7 +482,7 @@ class GClientSmokeSVN(GClientSmokeBase): # matching. # TODO(maruel): In general, gclient revert output is wrong. It should output # the file list after some ___ running 'svn status' - results = self.gclient(['revert', '--deps', 'mac']) + results = self.gclient(['revert', '--deps', 'mac', '--jobs', '1']) out = self.splitBlock(results[0]) self.assertEquals(7, len(out)) self.checkString('', results[1]) @@ -416,7 +494,7 @@ class GClientSmokeSVN(GClientSmokeBase): ('trunk/third_party/prout@2', 'src/third_party/prout')) self.assertTree(tree) - out = self.parseGclient(['status', '--deps', 'mac'], + out = self.parseGclient(['status', '--deps', 'mac', '--jobs', '1'], [['running', join(self.root_dir, 'src')]]) out = self.svnBlockCleanup(out) self.checkString('other', out[0][1]) @@ -490,14 +568,14 @@ class GClientSmokeSVN(GClientSmokeBase): self.gclient(['config', self.svn_base + 'trunk/src/']) self.gclient(['sync']) src = join(self.root_dir, 'src') - res = self.gclient(['status'], src) + res = self.gclient(['status', '--jobs', '1'], src) self.checkBlock(res[0], [('running', src)]) def testInitialCheckoutNotYetDone(self): # Check that gclient can be executed when the initial checkout hasn't been # done yet. self.gclient(['config', self.svn_base + 'trunk/src/']) - self.parseGclient(['sync'], + self.parseGclient(['sync', '--jobs', '1'], ['running', 'running', # This is due to the way svn update is called for a # single file when File() is used in a DEPS file. @@ -512,7 +590,7 @@ class GClientSmokeSVN(GClientSmokeBase): # Cripple the checkout. os.remove(join(self.root_dir, '.gclient_entries')) src = join(self.root_dir, 'src') - res = self.gclient(['sync'], src) + res = self.gclient(['sync', '--jobs', '1'], src) self.checkBlock(res[0], ['running', 'running', 'running']) @@ -528,7 +606,7 @@ class GClientSmokeGIT(GClientSmokeBase): # TODO(maruel): safesync. self.gclient(['config', self.git_base + 'repo_1', '--name', 'src']) # Test unversioned checkout. - self.parseGclient(['sync', '--deps', 'mac'], + self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'], ['running', 'running', 'running', 'running', 'running']) # TODO(maruel): http://crosbug.com/3582 hooks run even if not matching, must # add sync parsing to get the list of updated files. @@ -544,15 +622,10 @@ class GClientSmokeGIT(GClientSmokeBase): os.remove(join(self.root_dir, 'src', 'git_hooked1')) # Test incremental versioned sync: sync backward. - results = self.gclient(['sync', '--revision', - 'src@' + self.githash('repo_1', 1), - '--deps', 'mac', '--delete_unversioned_trees']) - # gclient's git output is unparsable and all messed up. Don't look at it, it - # hurts the smoke test's eyes. Add "print out" here if you want to dare to - # parse it. So just assert it's not empty and look at the result on the file - # system instead. - self.assertEquals('', results[1]) - self.assertEquals(0, results[2]) + self.parseGclient(['sync', '--jobs', '1', '--revision', + 'src@' + self.githash('repo_1', 1), + '--deps', 'mac', '--delete_unversioned_trees'], + ['running', 'running', 'deleting']) tree = self.mangle_git_tree(('repo_1@1', 'src'), ('repo_2@2', 'src/repo2'), ('repo_3@1', 'src/repo2/repo3'), @@ -560,10 +633,8 @@ class GClientSmokeGIT(GClientSmokeBase): tree['src/git_hooked2'] = 'git_hooked2' self.assertTree(tree) # Test incremental sync: delete-unversioned_trees isn't there. - results = self.gclient(['sync', '--deps', 'mac']) - # See comment above about parsing gclient's git output. - self.assertEquals('', results[1]) - self.assertEquals(0, results[2]) + self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'], + ['running', 'running', 'running']) tree = self.mangle_git_tree(('repo_1@2', 'src'), ('repo_2@1', 'src/repo2'), ('repo_3@1', 'src/repo2/repo3'), @@ -579,7 +650,7 @@ class GClientSmokeGIT(GClientSmokeBase): return self.gclient(['config', self.git_base + 'repo_1', '--name', 'src']) self.parseGclient( - ['sync', '--deps', 'mac', + ['sync', '--deps', 'mac', '--jobs', '1', '--revision', 'invalid@' + self.githash('repo_1', 1)], ['running', 'running', 'running', 'running', 'running'], 'Please fix your script, having invalid --revision flags ' @@ -596,7 +667,7 @@ class GClientSmokeGIT(GClientSmokeBase): return # When no solution name is provided, gclient uses the first solution listed. self.gclient(['config', self.git_base + 'repo_1', '--name', 'src']) - self.parseGclient(['sync', '--deps', 'mac', + self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1', '--revision', self.githash('repo_1', 1)], ['running', 'running', 'running', 'running']) tree = self.mangle_git_tree(('repo_1@1', 'src'), @@ -605,6 +676,52 @@ class GClientSmokeGIT(GClientSmokeBase): ('repo_4@2', 'src/repo4')) self.assertTree(tree) + def testSyncJobs(self): + if not self.enabled: + return + # TODO(maruel): safesync. + self.gclient(['config', self.git_base + 'repo_1', '--name', 'src']) + # Test unversioned checkout. + self.parseGclient(['sync', '--deps', 'mac', '--jobs', '8'], + ['running', 'running', 'running', 'running', 'running'], + untangle=True) + # TODO(maruel): http://crosbug.com/3582 hooks run even if not matching, must + # add sync parsing to get the list of updated files. + tree = self.mangle_git_tree(('repo_1@2', 'src'), + ('repo_2@1', 'src/repo2'), + ('repo_3@2', 'src/repo2/repo_renamed')) + tree['src/git_hooked1'] = 'git_hooked1' + tree['src/git_hooked2'] = 'git_hooked2' + self.assertTree(tree) + + # Manually remove git_hooked1 before synching to make sure it's not + # recreated. + os.remove(join(self.root_dir, 'src', 'git_hooked1')) + + # Test incremental versioned sync: sync backward. + self.parseGclient( + ['sync', '--revision', 'src@' + self.githash('repo_1', 1), + '--deps', 'mac', '--delete_unversioned_trees', '--jobs', '8'], + ['running', 'running', 'deleting'], + untangle=True) + tree = self.mangle_git_tree(('repo_1@1', 'src'), + ('repo_2@2', 'src/repo2'), + ('repo_3@1', 'src/repo2/repo3'), + ('repo_4@2', 'src/repo4')) + tree['src/git_hooked2'] = 'git_hooked2' + self.assertTree(tree) + # Test incremental sync: delete-unversioned_trees isn't there. + self.parseGclient(['sync', '--deps', 'mac', '--jobs', '8'], + ['running', 'running', 'running'], untangle=True) + tree = self.mangle_git_tree(('repo_1@2', 'src'), + ('repo_2@1', 'src/repo2'), + ('repo_3@1', 'src/repo2/repo3'), + ('repo_3@2', 'src/repo2/repo_renamed'), + ('repo_4@2', 'src/repo4')) + tree['src/git_hooked1'] = 'git_hooked1' + tree['src/git_hooked2'] = 'git_hooked2' + self.assertTree(tree) + def testRevertAndStatus(self): """TODO(maruel): Remove this line once this test is fixed.""" if not self.enabled: @@ -712,7 +829,7 @@ class GClientSmokeBoth(GClientSmokeBase): ' "url": "' + self.svn_base + 'trunk/src/"},' '{"name": "src-git",' '"url": "' + self.git_base + 'repo_1"}]']) - self.parseGclient(['sync', '--deps', 'mac'], + self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'], ['running', 'running', 'running', # This is due to the way svn update is called for a single # file when File() is used in a DEPS file. @@ -733,6 +850,37 @@ class GClientSmokeBoth(GClientSmokeBase): tree['src/svn_hooked1'] = 'svn_hooked1' self.assertTree(tree) + def testMultiSolutionsJobs(self): + if not self.enabled: + return + self.gclient(['config', '--spec', + 'solutions=[' + '{"name": "src",' + ' "url": "' + self.svn_base + 'trunk/src/"},' + '{"name": "src-git",' + '"url": "' + self.git_base + 'repo_1"}]']) + self.parseGclient(['sync', '--deps', 'mac', '--jobs', '8'], + ['running', 'running', 'running', + # This is due to the way svn update is called for a single + # file when File() is used in a DEPS file. + ('running', self.root_dir + '/src/file/other'), + 'running', 'running', 'running', 'running', 'running', 'running', + 'running', 'running'], + untangle=True) + tree = self.mangle_git_tree(('repo_1@2', 'src-git'), + ('repo_2@1', 'src/repo2'), + ('repo_3@2', 'src/repo2/repo_renamed')) + tree.update(self.mangle_svn_tree( + ('trunk/src@2', 'src'), + ('trunk/third_party/foo@1', 'src/third_party/foo'), + ('trunk/other@2', 'src/other'))) + tree['src/file/other/DEPS'] = ( + self.FAKE_REPOS.svn_revs[2]['trunk/other/DEPS']) + tree['src/git_hooked1'] = 'git_hooked1' + tree['src/git_hooked2'] = 'git_hooked2' + tree['src/svn_hooked1'] = 'svn_hooked1' + self.assertTree(tree) + def testMultiSolutionsMultiRev(self): if not self.enabled: return @@ -743,7 +891,7 @@ class GClientSmokeBoth(GClientSmokeBase): '{"name": "src-git",' '"url": "' + self.git_base + 'repo_1"}]']) self.parseGclient( - ['sync', '--deps', 'mac', '--revision', '1', + ['sync', '--deps', 'mac', '--jobs', '1', '--revision', '1', '-r', 'src-git@' + self.githash('repo_1', 1)], ['running', 'running', 'running', 'running', 'running', 'running', 'running', 'running']) @@ -843,7 +991,8 @@ class GClientSmokeFromCheckout(GClientSmokeBase): '--username', 'user1', '--password', 'foo']) def testSync(self): - self.parseGclient(['sync', '--deps', 'mac'], ['running', 'running']) + self.parseGclient(['sync', '--deps', 'mac', '--jobs', '1'], + ['running', 'running']) tree = self.mangle_svn_tree( ('trunk/webkit@2', ''), ('trunk/third_party/foo@1', 'foo/bar')) @@ -853,11 +1002,11 @@ class GClientSmokeFromCheckout(GClientSmokeBase): self.gclient(['sync']) # TODO(maruel): This is incorrect. - out = self.parseGclient(['status', '--deps', 'mac'], []) + out = self.parseGclient(['status', '--deps', 'mac', '--jobs', '1'], []) # Revert implies --force implies running hooks without looking at pattern # matching. - results = self.gclient(['revert', '--deps', 'mac']) + results = self.gclient(['revert', '--deps', 'mac', '--jobs', '1']) out = self.splitBlock(results[0]) self.assertEquals(2, len(out)) self.checkString(2, len(out[0])) @@ -903,10 +1052,12 @@ class GClientSmokeFromCheckout(GClientSmokeBase): def testRest(self): self.gclient(['sync']) # TODO(maruel): This is incorrect, it should run on ./ too. - out = self.parseGclient(['cleanup', '--deps', 'mac', '--verbose'], - [('running', join(self.root_dir, 'foo', 'bar'))]) - out = self.parseGclient(['diff', '--deps', 'mac', '--verbose'], - [('running', join(self.root_dir, 'foo', 'bar'))]) + out = self.parseGclient( + ['cleanup', '--deps', 'mac', '--verbose', '--jobs', '1'], + [('running', join(self.root_dir, 'foo', 'bar'))]) + out = self.parseGclient( + ['diff', '--deps', 'mac', '--verbose', '--jobs', '1'], + [('running', join(self.root_dir, 'foo', 'bar'))]) if __name__ == '__main__':