From 7a0fe8bf3cf3f51531827e5c02f7b7aff27e3937 Mon Sep 17 00:00:00 2001 From: Ben Segall Date: Mon, 24 Apr 2023 20:36:55 +0000 Subject: [PATCH] Move reclient cache dir to gclient root to avoid wipeout on gn clean This speeds up small builds as we will still persist the credential cache and it will speed up all builds as we will keep the deps cache and compiler info cache. Bug: b/278498980 Change-Id: I2fe26395a7493c461860b34ef3f109b2147abb5e Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4454889 Reviewed-by: Junji Watanabe Reviewed-by: Gavin Mak Commit-Queue: Ben Segall --- reclient_helper.py | 28 +++++++++++++++++++++++++++- tests/OWNERS | 1 + tests/ninja_reclient_test.py | 12 ++++++++++-- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/reclient_helper.py b/reclient_helper.py index 54f258d207..1570d49538 100644 --- a/reclient_helper.py +++ b/reclient_helper.py @@ -71,10 +71,36 @@ def find_ninja_out_dir(args): return '.' +def find_cache_dir(tmp_dir): + """Helper to find the correct cache directory for a build. + + tmp_dir should be a build specific temp directory within the out directory. + + If this is called from within a gclient checkout, the cache dir will be: + /.reproxy_cache/md5(tmp_dir)/ + If this is not called from within a gclient checkout, the cache dir will be: + tmp_dir/cache + """ + gclient_root = gclient_paths.FindGclientRoot(os.getcwd()) + if gclient_root: + return os.path.join(gclient_root, '.reproxy_cache', + hashlib.md5(tmp_dir.encode()).hexdigest()) + return os.path.join(tmp_dir, 'cache') + + def set_reproxy_path_flags(out_dir, make_dirs=True): """Helper to setup the logs and cache directories for reclient. Creates the following directory structure if make_dirs is true: + If in a gclient checkout + out_dir/ + .reproxy_tmp/ + logs/ + + .reproxy_cache/ + md5(out_dir/.reproxy_tmp)/ + + If not in a gclient checkout out_dir/ .reproxy_tmp/ logs/ @@ -92,7 +118,7 @@ def set_reproxy_path_flags(out_dir, make_dirs=True): """ tmp_dir = os.path.abspath(os.path.join(out_dir, '.reproxy_tmp')) log_dir = os.path.join(tmp_dir, 'logs') - cache_dir = os.path.join(tmp_dir, 'cache') + cache_dir = find_cache_dir(tmp_dir) if make_dirs: os.makedirs(tmp_dir, exist_ok=True) os.makedirs(log_dir, exist_ok=True) diff --git a/tests/OWNERS b/tests/OWNERS index 317807ef25..2e7ea0e0a6 100644 --- a/tests/OWNERS +++ b/tests/OWNERS @@ -2,3 +2,4 @@ per-file autoninja_test.py=brucedawson@chromium.org per-file autoninja_test.py=tikuta@chromium.org per-file ninjalog_uploader_test.py=tikuta@chromium.org per-file ninja_reclient_test.py=file://BUILD_OWNERS +per-file ninja_reclient_test.py=file://RECLIENT_OWNERS diff --git a/tests/ninja_reclient_test.py b/tests/ninja_reclient_test.py index 9adaec0ace..1aa6494fcc 100755 --- a/tests/ninja_reclient_test.py +++ b/tests/ninja_reclient_test.py @@ -56,7 +56,11 @@ class NinjaReclientTest(trial_dir.TestCase): os.path.isdir(os.path.join(self.root_dir, "out", "a", ".reproxy_tmp"))) self.assertTrue( os.path.isdir( - os.path.join(self.root_dir, "out", "a", ".reproxy_tmp", "cache"))) + os.path.join( + self.root_dir, ".reproxy_cache", + hashlib.md5( + os.path.join(self.root_dir, "out", "a", + ".reproxy_tmp").encode()).hexdigest()))) self.assertTrue( os.path.isdir( os.path.join(self.root_dir, "out", "a", ".reproxy_tmp", "logs"))) @@ -68,7 +72,11 @@ class NinjaReclientTest(trial_dir.TestCase): os.path.join(self.root_dir, "out", "a", ".reproxy_tmp", "logs")) self.assertEqual( os.environ.get('RBE_cache_dir'), - os.path.join(self.root_dir, "out", "a", ".reproxy_tmp", "cache")) + os.path.join( + self.root_dir, ".reproxy_cache", + hashlib.md5( + os.path.join(self.root_dir, "out", "a", + ".reproxy_tmp").encode()).hexdigest())) if sys.platform.startswith('win'): self.assertEqual( os.environ.get('RBE_server_address'),