#!/usr/bin/env python3 # Copyright (c) 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """ This script (intended to be invoked by autoninja or autoninja.bat) detects whether a build is accelerated using a service like goma. If so, it runs with a large -j value, and otherwise it chooses a small one. This auto-adjustment makes using remote build acceleration simpler and safer, and avoids errors that can cause slow goma builds or swap-storms on unaccelerated builds. """ import multiprocessing import os import platform import re import subprocess import sys if sys.platform == 'darwin': import resource SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) def main(args): # The -t tools are incompatible with -j t_specified = False j_specified = False offline = False output_dir = '.' input_args = args # On Windows the autoninja.bat script passes along the arguments enclosed in # double quotes. This prevents multiple levels of parsing of the special '^' # characters needed when compiling a single file but means that this script # gets called with a single argument containing all of the actual arguments, # separated by spaces. When this case is detected we need to do argument # splitting ourselves. This means that arguments containing actual spaces # are not supported by autoninja, but that is not a real limitation. if (sys.platform.startswith('win') and len(args) == 2 and input_args[1].count(' ') > 0): input_args = args[:1] + args[1].split() # Ninja uses getopt_long, which allow to intermix non-option arguments. # To leave non supported parameters untouched, we do not use getopt. for index, arg in enumerate(input_args[1:]): if arg.startswith('-j'): j_specified = True if arg.startswith('-t'): t_specified = True if arg == '-C': # + 1 to get the next argument and +1 because we trimmed off # input_args[0] output_dir = input_args[index + 2] elif arg.startswith('-C'): # Support -Cout/Default output_dir = arg[2:] elif arg in ('-o', '--offline'): offline = True elif arg == '-h': print('autoninja: Use -o/--offline to temporary disable goma.', file=sys.stderr) print(file=sys.stderr) use_goma = False use_remoteexec = False use_rbe = False use_siso = False # Attempt to auto-detect remote build acceleration. We support gn-based # builds, where we look for args.gn in the build tree, and cmake-based # builds where we look for rules.ninja. if os.path.exists(os.path.join(output_dir, 'args.gn')): with open(os.path.join(output_dir, 'args.gn')) as file_handle: for line in file_handle: # use_goma, use_remoteexec, or use_rbe will activate build # acceleration. # # This test can match multi-argument lines. Examples of this # are: is_debug=false use_goma=true is_official_build=false # use_goma=false# use_goma=true This comment is ignored # # Anything after a comment is not consider a valid argument. line_without_comment = line.split('#')[0] if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)', line_without_comment): use_goma = True continue if re.search(r'(^|\s)(use_remoteexec)\s*=\s*true($|\s)', line_without_comment): use_remoteexec = True continue if re.search(r'(^|\s)(use_rbe)\s*=\s*true($|\s)', line_without_comment): use_rbe = True continue if re.search(r'(^|\s)(use_siso)\s*=\s*true($|\s)', line_without_comment): use_siso = True continue siso_marker = os.path.join(output_dir, '.siso_deps') if use_siso: ninja_marker = os.path.join(output_dir, '.ninja_log') # autosiso generates a .ninja_log file so the mere existence of a # .ninja_log file doesn't imply that a ninja build was done. However # if there is a .ninja_log but no .siso_deps then that implies a # ninja build. if os.path.exists(ninja_marker) and not os.path.exists(siso_marker): return ( 'echo Run gn clean before switching from ninja to siso in ' '%s' % output_dir) siso = ['autosiso'] if use_remoteexec else ['siso', 'ninja'] if sys.platform.startswith('win'): # An explicit 'call' is needed to make sure the invocation of # autosiso returns to autoninja.bat, and the command prompt # title gets reset. siso = ['call'] + siso return ' '.join(siso + input_args[1:]) if os.path.exists(siso_marker): return ( 'echo Run gn clean before switching from siso to ninja in %s' % output_dir) else: for relative_path in [ '', # GN keeps them in the root of output_dir 'CMakeFiles' ]: path = os.path.join(output_dir, relative_path, 'rules.ninja') if os.path.exists(path): with open(path) as file_handle: for line in file_handle: if re.match(r'^\s*command\s*=\s*\S+gomacc', line): use_goma = True break # Strip -o/--offline so ninja doesn't see them. input_args = [arg for arg in input_args if arg not in ('-o', '--offline')] # If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1" # (case-insensitive) then gomacc will use the local compiler instead of # doing a goma compile. This is convenient if you want to briefly disable # goma. It avoids having to rebuild the world when transitioning between # goma/non-goma builds. However, it is not as fast as doing a "normal" # non-goma build because an extra process is created for each compile step. # Checking this environment variable ensures that autoninja uses an # appropriate -j value in this situation. goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower() if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']: use_goma = False if use_goma: gomacc_file = 'gomacc.exe' if sys.platform.startswith( 'win') else 'gomacc' goma_dir = os.environ.get('GOMA_DIR', os.path.join(SCRIPT_DIR, '.cipd_bin')) gomacc_path = os.path.join(goma_dir, gomacc_file) # Don't invoke gomacc if it doesn't exist. if os.path.exists(gomacc_path): # Check to make sure that goma is running. If not, don't start the # build. status = subprocess.call([gomacc_path, 'port'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) if status == 1: print( 'Goma is not running. Use "goma_ctl ensure_start" to start ' 'it.', file=sys.stderr) if sys.platform.startswith('win'): # Set an exit code of 1 in the batch file. print('cmd "/c exit 1"') else: # Set an exit code of 1 by executing 'false' in the bash # script. print('false') sys.exit(1) # A large build (with or without goma) tends to hog all system resources. # Launching the ninja process with 'nice' priorities improves this # situation. prefix_args = [] if (sys.platform.startswith('linux') and os.environ.get('NINJA_BUILD_IN_BACKGROUND', '0') == '1'): # nice -10 is process priority 10 lower than default 0 # ionice -c 3 is IO priority IDLE prefix_args = ['nice'] + ['-10'] # Tell goma or reclient to do local compiles. On Windows these environment # variables are set by the wrapper batch file. offline_env = ['RBE_remote_disabled=1', 'GOMA_DISABLED=1' ] if offline and not sys.platform.startswith('win') else [] # On macOS, the default limit of open file descriptors is too low (256). # This causes a large j value to result in 'Too many open files' errors. # Check whether the limit can be raised to a large enough value. If yes, # use `ulimit -n .... &&` as a prefix to increase the limit when running # ninja. if sys.platform == 'darwin': wanted_limit = 200000 # Large enough to avoid any risk of exhaustion. fileno_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) if fileno_limit <= wanted_limit: try: resource.setrlimit(resource.RLIMIT_NOFILE, (wanted_limit, hard_limit)) except Exception as _: pass fileno_limit, hard_limit = resource.getrlimit( resource.RLIMIT_NOFILE) if fileno_limit >= wanted_limit: prefix_args = ['ulimit', '-n', f'{wanted_limit}', '&&' ] + offline_env offline_env = [] # Call ninja.py so that it can find ninja binary installed by DEPS or one in # PATH. ninja_path = os.path.join(SCRIPT_DIR, 'ninja.py') # If using remoteexec, use ninja_reclient.py which wraps ninja.py with # starting and stopping reproxy. if use_remoteexec: ninja_path = os.path.join(SCRIPT_DIR, 'ninja_reclient.py') args = offline_env + prefix_args + [sys.executable, ninja_path ] + input_args[1:] num_cores = multiprocessing.cpu_count() if not j_specified and not t_specified: if not offline and (use_goma or use_remoteexec or use_rbe): args.append('-j') default_core_multiplier = 80 if platform.machine() in ('x86_64', 'AMD64'): # Assume simultaneous multithreading and therefore half as many # cores as logical processors. num_cores //= 2 core_multiplier = int( os.environ.get('NINJA_CORE_MULTIPLIER', default_core_multiplier)) j_value = num_cores * core_multiplier core_limit = int(os.environ.get('NINJA_CORE_LIMIT', j_value)) j_value = min(j_value, core_limit) if sys.platform.startswith('win'): # On windows, j value higher than 1000 does not improve build # performance. j_value = min(j_value, 1000) elif sys.platform == 'darwin': # If the number of open file descriptors is large enough (or it # can be raised to a large enough value), then set j value to # 1000. This limit comes from ninja which is limited to at most # FD_SETSIZE (1024) open file descriptors (using 1000 leave a # bit of head room). # # If the number of open file descriptors cannot be raised, then # use a j value of 200 which is the maximum value that reliably # work with the default limit of 256. if fileno_limit >= wanted_limit: j_value = min(j_value, 1000) else: j_value = min(j_value, 200) args.append('%d' % j_value) else: j_value = num_cores # Ninja defaults to |num_cores + 2| j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2')) args.append('-j') args.append('%d' % j_value) # On Windows, fully quote the path so that the command processor doesn't # think the whole output is the command. On Linux and Mac, if people put # depot_tools in directories with ' ', shell would misunderstand ' ' as a # path separation. TODO(yyanagisawa): provide proper quoting for Windows. # see https://cs.chromium.org/chromium/src/tools/mb/mb.py for i in range(len(args)): if (i == 0 and sys.platform.startswith('win')) or ' ' in args[i]: args[i] = '"%s"' % args[i].replace('"', '\\"') if os.environ.get('NINJA_SUMMARIZE_BUILD', '0') == '1': args += ['-d', 'stats'] return ' '.join(args) if __name__ == '__main__': print(main(sys.argv))