From 704be878162e5e0f776d77b182f4969103aa886b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wei-Yin=20Chen=20=28=E9=99=B3=E5=A8=81=E5=B0=B9=29?= Date: Thu, 11 May 2017 00:58:26 -0700 Subject: [PATCH] Add reflink support for gclient-new-workdir.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On filesystems supporting copy-on-write, like Btrfs and ZFS, use "cp --reflink" to achieve faster speed and lower disk space usage. Option --reflink and --no-reflink can override the automatic detection. When reflink is supported, toolchains, support libraries, and artifacts would also be preserved. This means you can do incremental builds right after creating a new workdir. For Linux Chromium checkout, one additional workdir only takes 0.8~1 GB disk space in Btrfs, including data and metadata, depending on how many artifacts are there inside the out*/ directories. Change-Id: I8b913efba9e5afaeea2adc1c10a561f63cb50282 Bug: 721585 Reviewed-on: https://chromium-review.googlesource.com/499567 Commit-Queue: Wei-Yin Chen (陳威尹) Reviewed-by: Robbie Iannucci --- gclient-new-workdir.py | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/gclient-new-workdir.py b/gclient-new-workdir.py index 5e663b363d..96fccf700d 100755 --- a/gclient-new-workdir.py +++ b/gclient-new-workdir.py @@ -27,6 +27,12 @@ def parse_options(): parser.add_argument('repository', type=os.path.abspath, help='should contain a .gclient file') parser.add_argument('new_workdir', help='must not exist') + parser.add_argument('--reflink', action='store_true', default=None, + help='''force to use "cp --reflink" for speed and disk + space. need supported FS like btrfs or ZFS.''') + parser.add_argument('--no-reflink', action='store_false', dest='reflink', + help='''force not to use "cp --reflink" even on supported + FS like btrfs or ZFS.''') args = parser.parse_args() if not os.path.exists(args.repository): @@ -42,21 +48,52 @@ def parse_options(): return args +def support_cow(src, dest): + try: + subprocess.check_output(['cp', '-a', '--reflink', src, dest], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + return False + finally: + os.remove(dest) + return True + + def main(): args = parse_options() gclient = os.path.join(args.repository, '.gclient') + new_gclient = os.path.join(args.new_workdir, '.gclient') os.makedirs(args.new_workdir) - os.symlink(gclient, os.path.join(args.new_workdir, '.gclient')) + if args.reflink is None: + args.reflink = support_cow(gclient, new_gclient) + if args.reflink: + print('Copy-on-write is supported. Using reflink to copy the repo.') + os.symlink(gclient, new_gclient) for root, dirs, _ in os.walk(args.repository): if '.git' in dirs: workdir = root.replace(args.repository, args.new_workdir, 1) print('Creating: %s' % workdir) + + if args.reflink: + if not os.path.exists(workdir): + print('Copying: %s' % workdir) + subprocess.check_call(['cp', '-a', '--reflink', root, workdir]) + shutil.rmtree(os.path.join(workdir, '.git')) + git_common.make_workdir(os.path.join(root, '.git'), os.path.join(workdir, '.git')) - subprocess.check_call(['git', 'checkout', '-f'], cwd=workdir) + if args.reflink: + subprocess.check_call(['cp', '-a', '--reflink', + os.path.join(root, '.git', 'index'), + os.path.join(workdir, '.git', 'index')]) + else: + subprocess.check_call(['git', 'checkout', '-f'], cwd=workdir) + + if args.reflink: + subprocess.check_call(['git', 'clean', '-df'], cwd=workdir) if __name__ == '__main__':