From 381db68adc980da8e158f53405d25495c65aa8b2 Mon Sep 17 00:00:00 2001 From: Takuto Ikuta Date: Wed, 27 Apr 2022 23:54:02 +0000 Subject: [PATCH] autoninja: add simple test This is to prevent revert like https://crrev.com/c/3607513 Bug: 1317620 Change-Id: I6ab7aba5f92719bd573d22d90358f58e48aeb10c Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3607514 Reviewed-by: Bruce Dawson Reviewed-by: Gavin Mak Commit-Queue: Takuto Ikuta --- PRESUBMIT.py | 5 +- autoninja.py | 393 ++++++++++++++++++++-------------------- tests/OWNERS | 2 + tests/autoninja_test.py | 40 ++++ 4 files changed, 247 insertions(+), 193 deletions(-) create mode 100755 tests/autoninja_test.py diff --git a/PRESUBMIT.py b/PRESUBMIT.py index 6571a4ee7..3379d9ab9 100644 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -124,7 +124,10 @@ def CheckUnitTestsOnCommit(input_api, output_api): 'recipes_test.py', ] - py3_only_tests = ['ninjalog_uploader_test.py'] + py3_only_tests = [ + 'autoninja_test.py', + 'ninjalog_uploader_test.py', + ] tests = input_api.canned_checks.GetUnitTestsInDirectory( input_api, diff --git a/autoninja.py b/autoninja.py index 28f9ff39a..2ba7c864a 100755 --- a/autoninja.py +++ b/autoninja.py @@ -21,198 +21,207 @@ import sys SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -# The -t tools are incompatible with -j -t_specified = False -j_specified = False -offline = False -output_dir = '.' -input_args = sys.argv -# On Windows the autoninja.bat script passes along the arguments enclosed in -# double quotes. This prevents multiple levels of parsing of the special '^' -# characters needed when compiling a single file but means that this script gets -# called with a single argument containing all of the actual arguments, -# separated by spaces. When this case is detected we need to do argument -# splitting ourselves. This means that arguments containing actual spaces are -# not supported by autoninja, but that is not a real limitation. -if (sys.platform.startswith('win') and len(sys.argv) == 2 and - input_args[1].count(' ') > 0): - input_args = sys.argv[:1] + sys.argv[1].split() - -# Ninja uses getopt_long, which allow to intermix non-option arguments. -# To leave non supported parameters untouched, we do not use getopt. -for index, arg in enumerate(input_args[1:]): - if arg.startswith('-j'): - j_specified = True - if arg.startswith('-t'): - t_specified = True - if arg == '-C': - # + 1 to get the next argument and +1 because we trimmed off input_args[0] - output_dir = input_args[index + 2] - elif arg.startswith('-C'): - # Support -Cout/Default - output_dir = arg[2:] - elif arg in ('-o', '--offline'): - offline = True - elif arg == '-h': - print('autoninja: Use -o/--offline to temporary disable goma.', - file=sys.stderr) - print(file=sys.stderr) - -# Strip -o/--offline so ninja doesn't see them. -input_args = [ arg for arg in input_args if arg not in ('-o', '--offline')] - -use_goma = False -use_remoteexec = False - -# Currently get reclient binary and config dirs relative to output_dir. If -# they exist and using remoteexec, then automatically call bootstrap to start -# reproxy. This works under the current assumption that the output -# directory is two levels up from chromium/src. -reclient_bin_dir = os.path.join( - output_dir, '..', '..', 'buildtools', 'reclient') -reclient_cfg = os.path.join( - output_dir, '..', '..', 'buildtools', 'reclient_cfgs', 'reproxy.cfg') - -# Attempt to auto-detect remote build acceleration. We support gn-based -# builds, where we look for args.gn in the build tree, and cmake-based builds -# where we look for rules.ninja. -if os.path.exists(os.path.join(output_dir, 'args.gn')): - with open(os.path.join(output_dir, 'args.gn')) as file_handle: - for line in file_handle: - # Either use_goma, use_remoteexec or use_rbe (in deprecation) - # activate build acceleration. - # - # This test can match multi-argument lines. Examples of this are: - # is_debug=false use_goma=true is_official_build=false - # use_goma=false# use_goma=true This comment is ignored - # - # Anything after a comment is not consider a valid argument. - line_without_comment = line.split('#')[0] - if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)', - line_without_comment): - use_goma = True - continue - if re.search(r'(^|\s)(use_rbe|use_remoteexec)\s*=\s*true($|\s)', - line_without_comment): - use_remoteexec = True - continue -else: - for relative_path in [ - '', # GN keeps them in the root of output_dir - 'CMakeFiles' - ]: - path = os.path.join(output_dir, relative_path, 'rules.ninja') - if os.path.exists(path): - with open(path) as file_handle: - for line in file_handle: - if re.match(r'^\s*command\s*=\s*\S+gomacc', line): - use_goma = True - break - -# If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1" (case-insensitive) -# then gomacc will use the local compiler instead of doing a goma compile. This -# is convenient if you want to briefly disable goma. It avoids having to rebuild -# the world when transitioning between goma/non-goma builds. However, it is not -# as fast as doing a "normal" non-goma build because an extra process is created -# for each compile step. Checking this environment variable ensures that -# autoninja uses an appropriate -j value in this situation. -goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower() -if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']: - use_goma = False -if use_goma: - gomacc_file = 'gomacc.exe' if sys.platform.startswith('win') else 'gomacc' - goma_dir = os.environ.get('GOMA_DIR', os.path.join(SCRIPT_DIR, '.cipd_bin')) - gomacc_path = os.path.join(goma_dir, gomacc_file) - # Don't invoke gomacc if it doesn't exist. - if os.path.exists(gomacc_path): - # Check to make sure that goma is running. If not, don't start the build. - status = subprocess.call([gomacc_path, 'port'], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=False) - if status == 1: - print('Goma is not running. Use "goma_ctl ensure_start" to start it.', +def main(args): + # The -t tools are incompatible with -j + t_specified = False + j_specified = False + offline = False + output_dir = '.' + input_args = args + # On Windows the autoninja.bat script passes along the arguments enclosed in + # double quotes. This prevents multiple levels of parsing of the special '^' + # characters needed when compiling a single file but means that this script + # gets called with a single argument containing all of the actual arguments, + # separated by spaces. When this case is detected we need to do argument + # splitting ourselves. This means that arguments containing actual spaces are + # not supported by autoninja, but that is not a real limitation. + if (sys.platform.startswith('win') and len(args) == 2 + and input_args[1].count(' ') > 0): + input_args = args[:1] + args[1].split() + + # Ninja uses getopt_long, which allow to intermix non-option arguments. + # To leave non supported parameters untouched, we do not use getopt. + for index, arg in enumerate(input_args[1:]): + if arg.startswith('-j'): + j_specified = True + if arg.startswith('-t'): + t_specified = True + if arg == '-C': + # + 1 to get the next argument and +1 because we trimmed off input_args[0] + output_dir = input_args[index + 2] + elif arg.startswith('-C'): + # Support -Cout/Default + output_dir = arg[2:] + elif arg in ('-o', '--offline'): + offline = True + elif arg == '-h': + print('autoninja: Use -o/--offline to temporary disable goma.', file=sys.stderr) - if sys.platform.startswith('win'): - # Set an exit code of 1 in the batch file. - print('cmd "/c exit 1"') - else: - # Set an exit code of 1 by executing 'false' in the bash script. - print('false') - sys.exit(1) - -# Specify ninja.exe on Windows so that ninja.bat can call autoninja and not -# be called back. -ninja_exe = 'ninja.exe' if sys.platform.startswith('win') else 'ninja' -ninja_exe_path = os.path.join(SCRIPT_DIR, ninja_exe) - -# A large build (with or without goma) tends to hog all system resources. -# Launching the ninja process with 'nice' priorities improves this situation. -prefix_args = [] -if (sys.platform.startswith('linux') - and os.environ.get('NINJA_BUILD_IN_BACKGROUND', '0') == '1'): - # nice -10 is process priority 10 lower than default 0 - # ionice -c 3 is IO priority IDLE - prefix_args = ['nice'] + ['-10'] - - -# Use absolute path for ninja path, -# or fail to execute ninja if depot_tools is not in PATH. -args = prefix_args + [ninja_exe_path] + input_args[1:] - -num_cores = multiprocessing.cpu_count() -if not j_specified and not t_specified: - if use_goma or use_remoteexec: - args.append('-j') - core_multiplier = int(os.environ.get('NINJA_CORE_MULTIPLIER', '40')) - j_value = num_cores * core_multiplier - - if sys.platform.startswith('win'): - # On windows, j value higher than 1000 does not improve build performance. - j_value = min(j_value, 1000) - elif sys.platform == 'darwin': - # On Mac, j value higher than 500 causes 'Too many open files' error - # (crbug.com/936864). - j_value = min(j_value, 500) - - args.append('%d' % j_value) + print(file=sys.stderr) + + # Strip -o/--offline so ninja doesn't see them. + input_args = [arg for arg in input_args if arg not in ('-o', '--offline')] + + use_goma = False + use_remoteexec = False + + # Currently get reclient binary and config dirs relative to output_dir. If + # they exist and using remoteexec, then automatically call bootstrap to start + # reproxy. This works under the current assumption that the output + # directory is two levels up from chromium/src. + reclient_bin_dir = os.path.join(output_dir, '..', '..', 'buildtools', + 'reclient') + reclient_cfg = os.path.join(output_dir, '..', '..', 'buildtools', + 'reclient_cfgs', 'reproxy.cfg') + + # Attempt to auto-detect remote build acceleration. We support gn-based + # builds, where we look for args.gn in the build tree, and cmake-based builds + # where we look for rules.ninja. + if os.path.exists(os.path.join(output_dir, 'args.gn')): + with open(os.path.join(output_dir, 'args.gn')) as file_handle: + for line in file_handle: + # Either use_goma, use_remoteexec or use_rbe (in deprecation) + # activate build acceleration. + # + # This test can match multi-argument lines. Examples of this are: + # is_debug=false use_goma=true is_official_build=false + # use_goma=false# use_goma=true This comment is ignored + # + # Anything after a comment is not consider a valid argument. + line_without_comment = line.split('#')[0] + if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)', + line_without_comment): + use_goma = True + continue + if re.search(r'(^|\s)(use_rbe|use_remoteexec)\s*=\s*true($|\s)', + line_without_comment): + use_remoteexec = True + continue else: - j_value = num_cores - # Ninja defaults to |num_cores + 2| - j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2')) - args.append('-j') - args.append('%d' % j_value) - -# On Windows, fully quote the path so that the command processor doesn't think -# the whole output is the command. -# On Linux and Mac, if people put depot_tools in directories with ' ', -# shell would misunderstand ' ' as a path separation. -# TODO(yyanagisawa): provide proper quoting for Windows. -# see https://cs.chromium.org/chromium/src/tools/mb/mb.py -for i in range(len(args)): - if (i == 0 and sys.platform.startswith('win')) or ' ' in args[i]: - args[i] = '"%s"' % args[i].replace('"', '\\"') - -if os.environ.get('NINJA_SUMMARIZE_BUILD', '0') == '1': - args += ['-d', 'stats'] - -# If using remoteexec and the necessary environment variables are set, -# also start reproxy (via bootstrap) before running ninja. -if (not offline and use_remoteexec and os.path.exists(reclient_bin_dir) - and os.path.exists(reclient_cfg)): - bootstrap = os.path.join(reclient_bin_dir, 'bootstrap') - setup_args = [ - bootstrap, - '--cfg=' + reclient_cfg, - '--re_proxy=' + os.path.join(reclient_bin_dir, 'reproxy')] - - teardown_args = [bootstrap, '--cfg=' + reclient_cfg, '--shutdown'] - - cmd_sep = '\n' if sys.platform.startswith('win') else '&&' - args = setup_args + [cmd_sep] + args + [cmd_sep] + teardown_args - -if offline and not sys.platform.startswith('win'): - # Tell goma or reclient to do local compiles. On Windows these environment - # variables are set by the wrapper batch file. - print('RBE_remote_disabled=1 GOMA_DISABLED=1 ' + ' '.join(args)) -else: - print(' '.join(args)) + for relative_path in [ + '', # GN keeps them in the root of output_dir + 'CMakeFiles' + ]: + path = os.path.join(output_dir, relative_path, 'rules.ninja') + if os.path.exists(path): + with open(path) as file_handle: + for line in file_handle: + if re.match(r'^\s*command\s*=\s*\S+gomacc', line): + use_goma = True + break + + # If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1" + # (case-insensitive) then gomacc will use the local compiler instead of doing + # a goma compile. This is convenient if you want to briefly disable goma. It + # avoids having to rebuild the world when transitioning between goma/non-goma + # builds. However, it is not as fast as doing a "normal" non-goma build + # because an extra process is created for each compile step. Checking this + # environment variable ensures that autoninja uses an appropriate -j value in + # this situation. + goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower() + if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']: + use_goma = False + + if use_goma: + gomacc_file = 'gomacc.exe' if sys.platform.startswith('win') else 'gomacc' + goma_dir = os.environ.get('GOMA_DIR', os.path.join(SCRIPT_DIR, '.cipd_bin')) + gomacc_path = os.path.join(goma_dir, gomacc_file) + # Don't invoke gomacc if it doesn't exist. + if os.path.exists(gomacc_path): + # Check to make sure that goma is running. If not, don't start the build. + status = subprocess.call([gomacc_path, 'port'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False) + if status == 1: + print('Goma is not running. Use "goma_ctl ensure_start" to start it.', + file=sys.stderr) + if sys.platform.startswith('win'): + # Set an exit code of 1 in the batch file. + print('cmd "/c exit 1"') + else: + # Set an exit code of 1 by executing 'false' in the bash script. + print('false') + sys.exit(1) + + # Specify ninja.exe on Windows so that ninja.bat can call autoninja and not + # be called back. + ninja_exe = 'ninja.exe' if sys.platform.startswith('win') else 'ninja' + ninja_exe_path = os.path.join(SCRIPT_DIR, ninja_exe) + + # A large build (with or without goma) tends to hog all system resources. + # Launching the ninja process with 'nice' priorities improves this situation. + prefix_args = [] + if (sys.platform.startswith('linux') + and os.environ.get('NINJA_BUILD_IN_BACKGROUND', '0') == '1'): + # nice -10 is process priority 10 lower than default 0 + # ionice -c 3 is IO priority IDLE + prefix_args = ['nice'] + ['-10'] + + # Use absolute path for ninja path, + # or fail to execute ninja if depot_tools is not in PATH. + args = prefix_args + [ninja_exe_path] + input_args[1:] + + num_cores = multiprocessing.cpu_count() + if not j_specified and not t_specified: + if use_goma or use_remoteexec: + args.append('-j') + core_multiplier = int(os.environ.get('NINJA_CORE_MULTIPLIER', '40')) + j_value = num_cores * core_multiplier + + if sys.platform.startswith('win'): + # On windows, j value higher than 1000 does not improve build + # performance. + j_value = min(j_value, 1000) + elif sys.platform == 'darwin': + # On Mac, j value higher than 500 causes 'Too many open files' error + # (crbug.com/936864). + j_value = min(j_value, 500) + + args.append('%d' % j_value) + else: + j_value = num_cores + # Ninja defaults to |num_cores + 2| + j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2')) + args.append('-j') + args.append('%d' % j_value) + + # On Windows, fully quote the path so that the command processor doesn't think + # the whole output is the command. + # On Linux and Mac, if people put depot_tools in directories with ' ', + # shell would misunderstand ' ' as a path separation. + # TODO(yyanagisawa): provide proper quoting for Windows. + # see https://cs.chromium.org/chromium/src/tools/mb/mb.py + for i in range(len(args)): + if (i == 0 and sys.platform.startswith('win')) or ' ' in args[i]: + args[i] = '"%s"' % args[i].replace('"', '\\"') + + if os.environ.get('NINJA_SUMMARIZE_BUILD', '0') == '1': + args += ['-d', 'stats'] + + # If using remoteexec and the necessary environment variables are set, + # also start reproxy (via bootstrap) before running ninja. + if (not offline and use_remoteexec and os.path.exists(reclient_bin_dir) + and os.path.exists(reclient_cfg)): + bootstrap = os.path.join(reclient_bin_dir, 'bootstrap') + setup_args = [ + bootstrap, '--cfg=' + reclient_cfg, + '--re_proxy=' + os.path.join(reclient_bin_dir, 'reproxy') + ] + + teardown_args = [bootstrap, '--cfg=' + reclient_cfg, '--shutdown'] + + cmd_sep = '\n' if sys.platform.startswith('win') else '&&' + args = setup_args + [cmd_sep] + args + [cmd_sep] + teardown_args + + if offline and not sys.platform.startswith('win'): + # Tell goma or reclient to do local compiles. On Windows these environment + # variables are set by the wrapper batch file. + return 'RBE_remote_disabled=1 GOMA_DISABLED=1 ' + ' '.join(args) + + return ' '.join(args) + + +if __name__ == '__main__': + print(main(sys.argv)) diff --git a/tests/OWNERS b/tests/OWNERS index 8ea5226f1..2dc80dcda 100644 --- a/tests/OWNERS +++ b/tests/OWNERS @@ -1 +1,3 @@ +per-file autoninja_test.py=brucedawson@chromium.org +per-file autoninja_test.py=tikuta@chromium.org per-file ninjalog_uploader_test.py=tikuta@chromium.org diff --git a/tests/autoninja_test.py b/tests/autoninja_test.py new file mode 100755 index 000000000..55e6b5b17 --- /dev/null +++ b/tests/autoninja_test.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import multiprocessing +import os +import os.path +import sys +import unittest +import unittest.mock + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT_DIR) + +import autoninja + + +class AutoninjaTest(unittest.TestCase): + def test_autoninja(self): + autoninja.main([]) + + def test_autoninja_goma(self): + with unittest.mock.patch( + 'os.path.exists', + return_value=True) as mock_exists, unittest.mock.patch( + 'autoninja.open', unittest.mock.mock_open( + read_data='use_goma=true')) as mock_open, unittest.mock.patch( + 'subprocess.call', return_value=0): + args = autoninja.main([]).split() + mock_exists.assert_called() + mock_open.assert_called_once() + + self.assertIn('-j', args) + parallel_j = int(args[args.index('-j') + 1]) + self.assertGreater(parallel_j, multiprocessing.cpu_count()) + + +if __name__ == '__main__': + unittest.main()