From fb86a018357903a702ca065dea6e1ced67c4dd2b Mon Sep 17 00:00:00 2001 From: Andrii Shyshkalov Date: Fri, 26 Apr 2019 01:03:46 +0000 Subject: [PATCH] Revert "Stop using compression for git cache update bootstrap." This reverts commit 749139969f34788a65dc1e5e27484325035129ce. Reason for revert: check for existence is wrong: Failed to check GS: CommandException: One or more URLs matched no objects. See more https://logs.chromium.org/logs/infra-internal/buildbucket/cr-buildbucket.appspot.com/8915157478918019280/+/steps/Updating_https:__chromium.googlesource.com_android_apks/0/stdout Original change's description: > Stop using compression for git cache update bootstrap. > > Change git cache to stop zipping the git directory and instead upload directly to GS. > > Bug: 943696 > Change-Id: I310fb48ff3d7fd6781914e14e0e17530d5d9b328 > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1585029 > Commit-Queue: Karen Qian > Reviewed-by: Andrii Shyshkalov TBR=tandrii@chromium.org,karenqian@google.com Change-Id: I4c57cd6b2defdb2cf842eb0acb9c1b1cd7a96298 No-Presubmit: true No-Tree-Checks: true No-Try: true Bug: 943696 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1585445 Reviewed-by: Andrii Shyshkalov Commit-Queue: Andrii Shyshkalov --- git_cache.py | 53 +++++++++++++++++----------------------------------- 1 file changed, 17 insertions(+), 36 deletions(-) diff --git a/git_cache.py b/git_cache.py index 7a6c907d47..4a2a237411 100755 --- a/git_cache.py +++ b/git_cache.py @@ -258,10 +258,6 @@ class Mirror(object): # Not recognized. return None - @property - def gs_folder(self): - return 'gs://%s/v2/%s' % (self.bootstrap_bucket, self.basedir) - @classmethod def FromPath(cls, path): return cls(cls.CacheDirToUrl(path)) @@ -604,43 +600,28 @@ class Mirror(object): lockfile.unlock() def update_bootstrap(self, prune=False): - # The folder is + # The files are named .zip gen_number = subprocess.check_output( [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip() # Run Garbage Collect to compress packfile. self.RunGit(['gc', '--prune=all']) - + # Creating a temp file and then deleting it ensures we can use this name. + _, tmp_zipfile = tempfile.mkstemp(suffix='.zip') + os.remove(tmp_zipfile) + subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path) gsutil = Gsutil(path=self.gsutil_exe, boto_path=None) - - src_name = self.mirror_path - dest_name = '%s/%s' % (self.gs_folder, gen_number) - - # check to see if folder already exists in gs - _, ls_out, ls_err = gsutil.check_call('ls', dest_name) - _, ls_out_ready, ls_err_ready = ( - gsutil.check_call('ls', dest_name + '.ready')) - - if ls_err: - print('Failed to check GS:\n%s' % (ls_err)) - return - - if ls_err_ready: - print('Failed to check GS:\n%s' % (ls_err_ready)) - return - - if not (ls_out == '' and ls_out_ready == ''): - return - - gsutil.call('-m', 'cp', '-r', src_name, dest_name) - - #TODO(karenqian): prune old caches - - # create .ready file and upload - _, ready_file_name = tempfile.mkstemp(suffix='.ready') - try: - gsutil.call('cp', ready_file_name, '%s.ready' % (dest_name)) - finally: - os.remove(ready_file_name) + gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir) + dest_name = '%s/%s.zip' % (gs_folder, gen_number) + gsutil.call('cp', tmp_zipfile, dest_name) + os.remove(tmp_zipfile) + + # Remove all other files in the same directory. + if prune: + _, ls_out, _ = gsutil.check_call('ls', gs_folder) + for filename in ls_out.splitlines(): + if filename == dest_name: + continue + gsutil.call('rm', filename) @staticmethod def DeleteTmpPackFiles(path):