You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
219 lines
8.8 KiB
Python
219 lines
8.8 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright (c) 2017 The Chromium Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""
|
|
This script (intended to be invoked by autoninja or autoninja.bat) detects
|
|
whether a build is accelerated using a service like goma. If so, it runs with a
|
|
large -j value, and otherwise it chooses a small one. This auto-adjustment
|
|
makes using remote build acceleration simpler and safer, and avoids errors that
|
|
can cause slow goma builds or swap-storms on unaccelerated builds.
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
|
|
import multiprocessing
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
# The -t tools are incompatible with -j
|
|
t_specified = False
|
|
j_specified = False
|
|
offline = False
|
|
output_dir = '.'
|
|
input_args = sys.argv
|
|
# On Windows the autoninja.bat script passes along the arguments enclosed in
|
|
# double quotes. This prevents multiple levels of parsing of the special '^'
|
|
# characters needed when compiling a single file but means that this script gets
|
|
# called with a single argument containing all of the actual arguments,
|
|
# separated by spaces. When this case is detected we need to do argument
|
|
# splitting ourselves. This means that arguments containing actual spaces are
|
|
# not supported by autoninja, but that is not a real limitation.
|
|
if (sys.platform.startswith('win') and len(sys.argv) == 2 and
|
|
input_args[1].count(' ') > 0):
|
|
input_args = sys.argv[:1] + sys.argv[1].split()
|
|
|
|
# Ninja uses getopt_long, which allow to intermix non-option arguments.
|
|
# To leave non supported parameters untouched, we do not use getopt.
|
|
for index, arg in enumerate(input_args[1:]):
|
|
if arg.startswith('-j'):
|
|
j_specified = True
|
|
if arg.startswith('-t'):
|
|
t_specified = True
|
|
if arg == '-C':
|
|
# + 1 to get the next argument and +1 because we trimmed off input_args[0]
|
|
output_dir = input_args[index + 2]
|
|
elif arg.startswith('-C'):
|
|
# Support -Cout/Default
|
|
output_dir = arg[2:]
|
|
elif arg in ('-o', '--offline'):
|
|
offline = True
|
|
elif arg == '-h':
|
|
print('autoninja: Use -o/--offline to temporary disable goma.',
|
|
file=sys.stderr)
|
|
print(file=sys.stderr)
|
|
|
|
# Strip -o/--offline so ninja doesn't see them.
|
|
input_args = [ arg for arg in input_args if arg not in ('-o', '--offline')]
|
|
|
|
use_goma = False
|
|
use_remoteexec = False
|
|
|
|
# Currently get reclient binary and config dirs relative to output_dir. If
|
|
# they exist and using remoteexec, then automatically call bootstrap to start
|
|
# reproxy. This works under the current assumption that the output
|
|
# directory is two levels up from chromium/src.
|
|
reclient_bin_dir = os.path.join(
|
|
output_dir, '..', '..', 'buildtools', 'reclient')
|
|
reclient_cfg = os.path.join(
|
|
output_dir, '..', '..', 'buildtools', 'reclient_cfgs', 'reproxy.cfg')
|
|
|
|
# Attempt to auto-detect remote build acceleration. We support gn-based
|
|
# builds, where we look for args.gn in the build tree, and cmake-based builds
|
|
# where we look for rules.ninja.
|
|
if os.path.exists(os.path.join(output_dir, 'args.gn')):
|
|
with open(os.path.join(output_dir, 'args.gn')) as file_handle:
|
|
for line in file_handle:
|
|
# Either use_goma, use_remoteexec or use_rbe (in deprecation)
|
|
# activate build acceleration.
|
|
#
|
|
# This test can match multi-argument lines. Examples of this are:
|
|
# is_debug=false use_goma=true is_official_build=false
|
|
# use_goma=false# use_goma=true This comment is ignored
|
|
#
|
|
# Anything after a comment is not consider a valid argument.
|
|
line_without_comment = line.split('#')[0]
|
|
if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)',
|
|
line_without_comment):
|
|
use_goma = True
|
|
continue
|
|
if re.search(r'(^|\s)(use_rbe|use_remoteexec)\s*=\s*true($|\s)',
|
|
line_without_comment):
|
|
use_remoteexec = True
|
|
continue
|
|
else:
|
|
for relative_path in [
|
|
'', # GN keeps them in the root of output_dir
|
|
'CMakeFiles'
|
|
]:
|
|
path = os.path.join(output_dir, relative_path, 'rules.ninja')
|
|
if os.path.exists(path):
|
|
with open(path) as file_handle:
|
|
for line in file_handle:
|
|
if re.match(r'^\s*command\s*=\s*\S+gomacc', line):
|
|
use_goma = True
|
|
break
|
|
|
|
# If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1" (case-insensitive)
|
|
# then gomacc will use the local compiler instead of doing a goma compile. This
|
|
# is convenient if you want to briefly disable goma. It avoids having to rebuild
|
|
# the world when transitioning between goma/non-goma builds. However, it is not
|
|
# as fast as doing a "normal" non-goma build because an extra process is created
|
|
# for each compile step. Checking this environment variable ensures that
|
|
# autoninja uses an appropriate -j value in this situation.
|
|
goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower()
|
|
if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']:
|
|
use_goma = False
|
|
|
|
if use_goma:
|
|
gomacc_file = 'gomacc.exe' if sys.platform.startswith('win') else 'gomacc'
|
|
goma_dir = os.environ.get('GOMA_DIR', os.path.join(SCRIPT_DIR, '.cipd_bin'))
|
|
gomacc_path = os.path.join(goma_dir, gomacc_file)
|
|
# Don't invoke gomacc if it doesn't exist.
|
|
if os.path.exists(gomacc_path):
|
|
# Check to make sure that goma is running. If not, don't start the build.
|
|
status = subprocess.call([gomacc_path, 'port'], stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, shell=False)
|
|
if status == 1:
|
|
print('Goma is not running. Use "goma_ctl ensure_start" to start it.',
|
|
file=sys.stderr)
|
|
if sys.platform.startswith('win'):
|
|
# Set an exit code of 1 in the batch file.
|
|
print('cmd "/c exit 1"')
|
|
else:
|
|
# Set an exit code of 1 by executing 'false' in the bash script.
|
|
print('false')
|
|
sys.exit(1)
|
|
|
|
# Specify ninja.exe on Windows so that ninja.bat can call autoninja and not
|
|
# be called back.
|
|
ninja_exe = 'ninja.exe' if sys.platform.startswith('win') else 'ninja'
|
|
ninja_exe_path = os.path.join(SCRIPT_DIR, ninja_exe)
|
|
|
|
# A large build (with or without goma) tends to hog all system resources.
|
|
# Launching the ninja process with 'nice' priorities improves this situation.
|
|
prefix_args = []
|
|
if (sys.platform.startswith('linux')
|
|
and os.environ.get('NINJA_BUILD_IN_BACKGROUND', '0') == '1'):
|
|
# nice -10 is process priority 10 lower than default 0
|
|
# ionice -c 3 is IO priority IDLE
|
|
prefix_args = ['nice'] + ['-10']
|
|
|
|
|
|
# Use absolute path for ninja path,
|
|
# or fail to execute ninja if depot_tools is not in PATH.
|
|
args = prefix_args + [ninja_exe_path] + input_args[1:]
|
|
|
|
num_cores = multiprocessing.cpu_count()
|
|
if not j_specified and not t_specified:
|
|
if use_goma or use_remoteexec:
|
|
args.append('-j')
|
|
core_multiplier = int(os.environ.get('NINJA_CORE_MULTIPLIER', '40'))
|
|
j_value = num_cores * core_multiplier
|
|
|
|
if sys.platform.startswith('win'):
|
|
# On windows, j value higher than 1000 does not improve build performance.
|
|
j_value = min(j_value, 1000)
|
|
elif sys.platform == 'darwin':
|
|
# On Mac, j value higher than 500 causes 'Too many open files' error
|
|
# (crbug.com/936864).
|
|
j_value = min(j_value, 500)
|
|
|
|
args.append('%d' % j_value)
|
|
else:
|
|
j_value = num_cores
|
|
# Ninja defaults to |num_cores + 2|
|
|
j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2'))
|
|
args.append('-j')
|
|
args.append('%d' % j_value)
|
|
|
|
# On Windows, fully quote the path so that the command processor doesn't think
|
|
# the whole output is the command.
|
|
# On Linux and Mac, if people put depot_tools in directories with ' ',
|
|
# shell would misunderstand ' ' as a path separation.
|
|
# TODO(yyanagisawa): provide proper quoting for Windows.
|
|
# see https://cs.chromium.org/chromium/src/tools/mb/mb.py
|
|
for i in range(len(args)):
|
|
if (i == 0 and sys.platform.startswith('win')) or ' ' in args[i]:
|
|
args[i] = '"%s"' % args[i].replace('"', '\\"')
|
|
|
|
if os.environ.get('NINJA_SUMMARIZE_BUILD', '0') == '1':
|
|
args += ['-d', 'stats']
|
|
|
|
# If using remoteexec and the necessary environment variables are set,
|
|
# also start reproxy (via bootstrap) before running ninja.
|
|
if (not offline and use_remoteexec and os.path.exists(reclient_bin_dir)
|
|
and os.path.exists(reclient_cfg)):
|
|
bootstrap = os.path.join(reclient_bin_dir, 'bootstrap')
|
|
setup_args = [
|
|
bootstrap,
|
|
'--cfg=' + reclient_cfg,
|
|
'--re_proxy=' + os.path.join(reclient_bin_dir, 'reproxy')]
|
|
|
|
teardown_args = [bootstrap, '--cfg=' + reclient_cfg, '--shutdown']
|
|
|
|
cmd_sep = '\n' if sys.platform.startswith('win') else '&&'
|
|
args = setup_args + [cmd_sep] + args + [cmd_sep] + teardown_args
|
|
|
|
if offline and not sys.platform.startswith('win'):
|
|
# Tell goma or reclient to do local compiles. On Windows these environment
|
|
# variables are set by the wrapper batch file.
|
|
print('RBE_remote_disabled=1 GOMA_DISABLED=1 ' + ' '.join(args))
|
|
else:
|
|
print(' '.join(args))
|