From 20eda953269e14f055afd58be97ad5915c6c33dd Mon Sep 17 00:00:00 2001
From: Michael Savigny <msavigny@google.com>
Date: Wed, 20 Jan 2021 01:16:27 +0000
Subject: [PATCH] Add reproxy setup/teardown to autoninja.

Adds reproxy setup and teardown to autoninja.  Since reproxy is intended
to run for a single build (unlike the goma proxy), having setup and
teardown happen as part of autoninja makes management of the proxy
execution easier.

To use this as it is currently implemented, set the RBE_BIN_DIR and
RBE_CFG_DIR environment variables to point to the reclient binaries and
reclient configuration files.  Note the reproxy.cfg file is NOT included
in this change, and at the time of this CL you need to provide one
yourself.

Bug: 1149386
Change-Id: I23601cc9b13193ac617ffc5963b9d443f6840d33
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2597837
Reviewed-by: Dirk Pranke <dpranke@google.com>
Reviewed-by: Ye Kuang <yekuang@google.com>
Commit-Queue: Michael Savigny <msavigny@google.com>
Auto-Submit: Michael Savigny <msavigny@google.com>
---
 autoninja.py | 38 ++++++++++++++++++++++++++++++++------
 1 file changed, 32 insertions(+), 6 deletions(-)

diff --git a/autoninja.py b/autoninja.py
index aef7f0f61..0322ba273 100755
--- a/autoninja.py
+++ b/autoninja.py
@@ -67,7 +67,16 @@ for index, arg in enumerate(input_args[1:]):
 # Strip -o/--offline so ninja doesn't see them.
 input_args = [ arg for arg in input_args if arg != '-o' and arg != '--offline']
 
-use_remote_build = False
+use_goma = False
+use_rbe = False
+
+# Currently get reclient binary and config dirs relative to output_dir.  If
+# they exist and using rbe, then automatically call bootstrap to start
+# reproxy.  This works under the current assumption that the output
+# directory is two levels up from chromium/src.
+reclient_bin_dir = os.path.join(output_dir, "../../buildtools/reclient")
+reclient_cfg = os.path.join(
+  output_dir, "../../buildtools/reclient_cfgs/reproxy.cfg")
 
 # Attempt to auto-detect remote build acceleration. We support gn-based
 # builds, where we look for args.gn in the build tree, and cmake-based builds
@@ -83,9 +92,13 @@ if os.path.exists(os.path.join(output_dir, 'args.gn')):
       #
       # Anything after a comment is not consider a valid argument.
       line_without_comment = line.split('#')[0]
-      if re.search(r'(^|\s)(use_goma|use_rbe)\s*=\s*true($|\s)',
+      if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)',
+                   line_without_comment):
+        use_goma = True
+        continue
+      if re.search(r'(^|\s)(use_rbe)\s*=\s*true($|\s)',
                    line_without_comment):
-        use_remote_build = True
+        use_rbe = True
         continue
 else:
   for relative_path in [
@@ -97,7 +110,7 @@ else:
       with open(path) as file_handle:
         for line in file_handle:
           if re.match(r'^\s*command\s*=\s*\S+gomacc', line):
-            use_remote_build = True
+            use_goma = True
             break
 
 # If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1" (case-insensitive)
@@ -109,7 +122,7 @@ else:
 # autoninja uses an appropriate -j value in this situation.
 goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower()
 if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']:
-  use_remote_build = False
+  use_goma = False
 
 # Specify ninja.exe on Windows so that ninja.bat can call autoninja and not
 # be called back.
@@ -132,7 +145,7 @@ args = prefix_args + [ninja_exe_path] + input_args[1:]
 
 num_cores = psutil.cpu_count()
 if not j_specified and not t_specified:
-  if use_remote_build:
+  if use_goma or use_rbe:
     args.append('-j')
     core_multiplier = int(os.environ.get('NINJA_CORE_MULTIPLIER', '40'))
     j_value = num_cores * core_multiplier
@@ -166,6 +179,19 @@ for i in range(len(args)):
 if os.environ.get('NINJA_SUMMARIZE_BUILD', '0') == '1':
   args += ['-d', 'stats']
 
+# If using rbe and the necessary environment variables are set, also start
+# reproxy (via bootstrap) before running ninja.
+if (not offline and use_rbe and os.path.exists(reclient_bin_dir) 
+    and os.path.exists(reclient_cfg)):
+  setup_args = [
+    'RBE_cfg=' + reclient_cfg,
+    reclient_bin_dir + '/bootstrap',
+    '--re_proxy=' + reclient_bin_dir + '/reproxy']
+
+  teardown_args = [reclient_bin_dir + '/bootstrap', '--shutdown']
+
+  args = setup_args + ['&&'] + args + ['&&'] + teardown_args
+
 if offline and not sys.platform.startswith('win'):
   # Tell goma or reclient to do local compiles. On Windows these environment
   # variables are set by the wrapper batch file.