aiden
/
depot_tools
mirror of https://chromium.googlesource.com/chromium/tools/depot_tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

apply black and `git cl format` for ninja related python files

Browse Source 
This is made by
$ black --line-length 79 *ninja*.py
$ git cl format

Change-Id: Ic446898a5461ae536542f6312cae2ce126dfe82a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5035265
Auto-Submit: Takuto Ikuta <tikuta@chromium.org>
Reviewed-by: Junji Watanabe <jwata@google.com>
Commit-Queue: Takuto Ikuta <tikuta@chromium.org>
Reviewed-by: Philipp Wollermann <philwo@chromium.org>
Reviewed-by: Fumitoshi Ukai <ukai@google.com>
changes/65/5035265/9
Takuto Ikuta 2 years ago committed by LUCI CQ
parent 0b98e7c063
commit df3e577855
6 changed files with 299 additions and 248 deletions
Show Stats Download Patch File Download Diff File

161
autoninja.py
Unescape Escape View File

@ -27,7 +27,7 @@ import ninja
import ninja_reclient import ninja_reclient
import siso import siso
if sys.platform in ['darwin', 'linux']: if sys.platform in ["darwin", "linux"]:
import resource import resource
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
@ -39,13 +39,13 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
# pylint: disable=line-too-long # pylint: disable=line-too-long
# [1] https://learn.microsoft.com/en-us/archive/blogs/twistylittlepassagesallalike/everyone-quotes-command-line-arguments-the-wrong-way # noqa # [1] https://learn.microsoft.com/en-us/archive/blogs/twistylittlepassagesallalike/everyone-quotes-command-line-arguments-the-wrong-way # noqa
# [2] https://web.archive.org/web/20150815000000*/https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/set.mspx # noqa # [2] https://web.archive.org/web/20150815000000*/https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/set.mspx # noqa
_UNSAFE_FOR_CMD = set('^<>&|()%') _UNSAFE_FOR_CMD = set("^<>&|()%")
_ALL_META_CHARS = _UNSAFE_FOR_CMD.union(set('"')) _ALL_META_CHARS = _UNSAFE_FOR_CMD.union(set('"'))
def _quote_for_cmd(arg): def _quote_for_cmd(arg):
# First, escape the arg so that CommandLineToArgvW will parse it properly. # First, escape the arg so that CommandLineToArgvW will parse it properly.
if arg == '' or ' ' in arg or '"' in arg: if arg == "" or " " in arg or '"' in arg:
quote_re = re.compile(r'(\\*)"') quote_re = re.compile(r'(\\*)"')
arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg)) arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg))
@ -53,13 +53,13 @@ def _quote_for_cmd(arg):
# double quotes; if it does, quote everything (including the double # double quotes; if it does, quote everything (including the double
# quotes) for safety. # quotes) for safety.
if any(a in _UNSAFE_FOR_CMD for a in arg): if any(a in _UNSAFE_FOR_CMD for a in arg):
arg = ''.join('^' + a if a in _ALL_META_CHARS else a for a in arg) arg = "".join("^" + a if a in _ALL_META_CHARS else a for a in arg)
return arg return arg
def _print_cmd(cmd): def _print_cmd(cmd):
shell_quoter = shlex.quote shell_quoter = shlex.quote
if sys.platform.startswith('win'): if sys.platform.startswith("win"):
shell_quoter = _quote_for_cmd shell_quoter = _quote_for_cmd
print(*[shell_quoter(arg) for arg in cmd], file=sys.stderr) print(*[shell_quoter(arg) for arg in cmd], file=sys.stderr)
@ -70,14 +70,14 @@ def _gn_lines(output_dir, path):
import directives as needed. import directives as needed.
""" """
import_re = re.compile(r'\s*import\("(.*)"\)') import_re = re.compile(r'\s*import\("(.*)"\)')
with open(path, encoding='utf-8') as f: with open(path, encoding="utf-8") as f:
for line in f: for line in f:
match = import_re.match(line) match = import_re.match(line)
if match: if match:
raw_import_path = match.groups()[0] raw_import_path = match.groups()[0]
if raw_import_path[:2] == "//": if raw_import_path[:2] == "//":
import_path = os.path.normpath( import_path = os.path.normpath(
os.path.join(output_dir, '..', '..', os.path.join(output_dir, "..", "..",
raw_import_path[2:])) raw_import_path[2:]))
else: else:
import_path = os.path.normpath( import_path = os.path.normpath(
@ -93,9 +93,9 @@ def main(args):
t_specified = False t_specified = False
j_specified = False j_specified = False
offline = False offline = False
output_dir = '.' output_dir = "."
input_args = args input_args = args
summarize_build = os.environ.get('NINJA_SUMMARIZE_BUILD') == '1' summarize_build = os.environ.get("NINJA_SUMMARIZE_BUILD") == "1"
# On Windows the autoninja.bat script passes along the arguments enclosed in # On Windows the autoninja.bat script passes along the arguments enclosed in
# double quotes. This prevents multiple levels of parsing of the special '^' # double quotes. This prevents multiple levels of parsing of the special '^'
# characters needed when compiling a single file but means that this script # characters needed when compiling a single file but means that this script
@ -103,29 +103,31 @@ def main(args):
# separated by spaces. When this case is detected we need to do argument # separated by spaces. When this case is detected we need to do argument
# splitting ourselves. This means that arguments containing actual spaces # splitting ourselves. This means that arguments containing actual spaces
# are not supported by autoninja, but that is not a real limitation. # are not supported by autoninja, but that is not a real limitation.
if (sys.platform.startswith('win') and len(args) == 2 if (sys.platform.startswith("win") and len(args) == 2
and input_args[1].count(' ') > 0): and input_args[1].count(" ") > 0):
input_args = args[:1] + args[1].split() input_args = args[:1] + args[1].split()
# Ninja uses getopt_long, which allow to intermix non-option arguments. # Ninja uses getopt_long, which allow to intermix non-option arguments.
# To leave non supported parameters untouched, we do not use getopt. # To leave non supported parameters untouched, we do not use getopt.
for index, arg in enumerate(input_args[1:]): for index, arg in enumerate(input_args[1:]):
if arg.startswith('-j'): if arg.startswith("-j"):
j_specified = True j_specified = True
if arg.startswith('-t'): if arg.startswith("-t"):
t_specified = True t_specified = True
if arg == '-C': if arg == "-C":
# + 1 to get the next argument and +1 because we trimmed off # + 1 to get the next argument and +1 because we trimmed off
# input_args[0] # input_args[0]
output_dir = input_args[index + 2] output_dir = input_args[index + 2]
elif arg.startswith('-C'): elif arg.startswith("-C"):
# Support -Cout/Default # Support -Cout/Default
output_dir = arg[2:] output_dir = arg[2:]
elif arg in ('-o', '--offline'): elif arg in ("-o", "--offline"):
offline = True offline = True
elif arg == '-h': elif arg == "-h":
print('autoninja: Use -o/--offline to temporary disable goma.', print(
file=sys.stderr) "autoninja: Use -o/--offline to temporary disable goma.",
file=sys.stderr,
)
print(file=sys.stderr) print(file=sys.stderr)
use_goma = False use_goma = False
@ -135,8 +137,8 @@ def main(args):
# Attempt to auto-detect remote build acceleration. We support gn-based # Attempt to auto-detect remote build acceleration. We support gn-based
# builds, where we look for args.gn in the build tree, and cmake-based # builds, where we look for args.gn in the build tree, and cmake-based
# builds where we look for rules.ninja. # builds where we look for rules.ninja.
if os.path.exists(os.path.join(output_dir, 'args.gn')): if os.path.exists(os.path.join(output_dir, "args.gn")):
for line in _gn_lines(output_dir, os.path.join(output_dir, 'args.gn')): for line in _gn_lines(output_dir, os.path.join(output_dir, "args.gn")):
# use_goma, or use_remoteexec will activate build # use_goma, or use_remoteexec will activate build
# acceleration. # acceleration.
# #
@ -145,62 +147,70 @@ def main(args):
# use_goma=false# use_goma=true This comment is ignored # use_goma=false# use_goma=true This comment is ignored
# #
# Anything after a comment is not consider a valid argument. # Anything after a comment is not consider a valid argument.
line_without_comment = line.split('#')[0] line_without_comment = line.split("#")[0]
if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)', if re.search(r"(^|\s)(use_goma)\s*=\s*true($|\s)",
line_without_comment): line_without_comment):
use_goma = True use_goma = True
continue continue
if re.search(r'(^|\s)(use_remoteexec)\s*=\s*true($|\s)', if re.search(
line_without_comment): r"(^|\s)(use_remoteexec)\s*=\s*true($|\s)",
line_without_comment,
):
use_remoteexec = True use_remoteexec = True
continue continue
if re.search(r'(^|\s)(use_siso)\s*=\s*true($|\s)', if re.search(r"(^|\s)(use_siso)\s*=\s*true($|\s)",
line_without_comment): line_without_comment):
use_siso = True use_siso = True
continue continue
siso_marker = os.path.join(output_dir, '.siso_deps') siso_marker = os.path.join(output_dir, ".siso_deps")
if use_siso: if use_siso:
# autosiso generates a .ninja_log file so the mere existence of a # autosiso generates a .ninja_log file so the mere existence of a
# .ninja_log file doesn't imply that a ninja build was done. However # .ninja_log file doesn't imply that a ninja build was done. However
# if there is a .ninja_log but no .siso_deps then that implies a # if there is a .ninja_log but no .siso_deps then that implies a
# ninja build. # ninja build.
ninja_marker = os.path.join(output_dir, '.ninja_log') ninja_marker = os.path.join(output_dir, ".ninja_log")
if os.path.exists(ninja_marker) and not os.path.exists(siso_marker): if os.path.exists(ninja_marker) and not os.path.exists(siso_marker):
print('Run gn clean before switching from ninja to siso in %s' % print(
"Run gn clean before switching from ninja to siso in %s" %
output_dir, output_dir,
file=sys.stderr) file=sys.stderr,
)
return 1 return 1
if use_goma: if use_goma:
print('Siso does not support Goma.', file=sys.stderr) print("Siso does not support Goma.", file=sys.stderr)
print('Do not use use_siso=true and use_goma=true', print(
file=sys.stderr) "Do not use use_siso=true and use_goma=true",
file=sys.stderr,
)
return 1 return 1
if use_remoteexec: if use_remoteexec:
return autosiso.main(['autosiso'] + input_args[1:]) return autosiso.main(["autosiso"] + input_args[1:])
return siso.main(['siso', 'ninja', '--offline'] + input_args[1:]) return siso.main(["siso", "ninja", "--offline"] + input_args[1:])
if os.path.exists(siso_marker): if os.path.exists(siso_marker):
print('Run gn clean before switching from siso to ninja in %s' % print(
"Run gn clean before switching from siso to ninja in %s" %
output_dir, output_dir,
file=sys.stderr) file=sys.stderr,
)
return 1 return 1
else: else:
for relative_path in [ for relative_path in [
'', # GN keeps them in the root of output_dir "", # GN keeps them in the root of output_dir
'CMakeFiles' "CMakeFiles",
]: ]:
path = os.path.join(output_dir, relative_path, 'rules.ninja') path = os.path.join(output_dir, relative_path, "rules.ninja")
if os.path.exists(path): if os.path.exists(path):
with open(path, encoding='utf-8') as file_handle: with open(path, encoding="utf-8") as file_handle:
for line in file_handle: for line in file_handle:
if re.match(r'^\s*command\s*=\s*\S+gomacc', line): if re.match(r"^\s*command\s*=\s*\S+gomacc", line):
use_goma = True use_goma = True
break break
# Strip -o/--offline so ninja doesn't see them. # Strip -o/--offline so ninja doesn't see them.
input_args = [arg for arg in input_args if arg not in ('-o', '--offline')] input_args = [arg for arg in input_args if arg not in ("-o", "--offline")]
# If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1" # If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1"
# (case-insensitive) then gomacc will use the local compiler instead of # (case-insensitive) then gomacc will use the local compiler instead of
@ -210,43 +220,46 @@ def main(args):
# non-goma build because an extra process is created for each compile step. # non-goma build because an extra process is created for each compile step.
# Checking this environment variable ensures that autoninja uses an # Checking this environment variable ensures that autoninja uses an
# appropriate -j value in this situation. # appropriate -j value in this situation.
goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower() goma_disabled_env = os.environ.get("GOMA_DISABLED", "0").lower()
if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']: if offline or goma_disabled_env in ["true", "t", "yes", "y", "1"]:
use_goma = False use_goma = False
if use_goma: if use_goma:
gomacc_file = 'gomacc.exe' if sys.platform.startswith( gomacc_file = ("gomacc.exe"
'win') else 'gomacc' if sys.platform.startswith("win") else "gomacc")
goma_dir = os.environ.get('GOMA_DIR', goma_dir = os.environ.get("GOMA_DIR",
os.path.join(SCRIPT_DIR, '.cipd_bin')) os.path.join(SCRIPT_DIR, ".cipd_bin"))
gomacc_path = os.path.join(goma_dir, gomacc_file) gomacc_path = os.path.join(goma_dir, gomacc_file)
# Don't invoke gomacc if it doesn't exist. # Don't invoke gomacc if it doesn't exist.
if os.path.exists(gomacc_path): if os.path.exists(gomacc_path):
# Check to make sure that goma is running. If not, don't start the # Check to make sure that goma is running. If not, don't start the
# build. # build.
status = subprocess.call([gomacc_path, 'port'], status = subprocess.call(
[gomacc_path, "port"],
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
shell=False) shell=False,
)
if status == 1: if status == 1:
print( print(
'Goma is not running. Use "goma_ctl ensure_start" to start ' 'Goma is not running. Use "goma_ctl ensure_start" to start '
'it.', "it.",
file=sys.stderr) file=sys.stderr,
if sys.platform.startswith('win'): )
if sys.platform.startswith("win"):
# Set an exit code of 1 in the batch file. # Set an exit code of 1 in the batch file.
print('cmd "/c exit 1"') print('cmd "/c exit 1"')
else: else:
# Set an exit code of 1 by executing 'false' in the bash # Set an exit code of 1 by executing 'false' in the bash
# script. # script.
print('false') print("false")
sys.exit(1) sys.exit(1)
# A large build (with or without goma) tends to hog all system resources. # A large build (with or without goma) tends to hog all system resources.
# Depending on the operating system, we might have mechanisms available # Depending on the operating system, we might have mechanisms available
# to run at a lower priority, which improves this situation. # to run at a lower priority, which improves this situation.
if os.environ.get('NINJA_BUILD_IN_BACKGROUND') == '1': if os.environ.get("NINJA_BUILD_IN_BACKGROUND") == "1":
if sys.platform in ['darwin', 'linux']: if sys.platform in ["darwin", "linux"]:
# nice-level 10 is usually considered a good default for background # nice-level 10 is usually considered a good default for background
# tasks. The niceness is inherited by child processes, so we can # tasks. The niceness is inherited by child processes, so we can
# just set it here for us and it'll apply to the build tool we # just set it here for us and it'll apply to the build tool we
@ -255,8 +268,8 @@ def main(args):
# Tell goma or reclient to do local compiles. # Tell goma or reclient to do local compiles.
if offline: if offline:
os.environ['RBE_remote_disabled'] = '1' os.environ["RBE_remote_disabled"] = "1"
os.environ['GOMA_DISABLED'] = '1' os.environ["GOMA_DISABLED"] = "1"
# On macOS and most Linux distributions, the default limit of open file # On macOS and most Linux distributions, the default limit of open file
# descriptors is too low (256 and 1024, respectively). # descriptors is too low (256 and 1024, respectively).
@ -264,7 +277,7 @@ def main(args):
# Check whether the limit can be raised to a large enough value. If yes, # Check whether the limit can be raised to a large enough value. If yes,
# use `ulimit -n .... &&` as a prefix to increase the limit when running # use `ulimit -n .... &&` as a prefix to increase the limit when running
# ninja. # ninja.
if sys.platform in ['darwin', 'linux']: if sys.platform in ["darwin", "linux"]:
# Increase the number of allowed open file descriptors to the maximum. # Increase the number of allowed open file descriptors to the maximum.
fileno_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) fileno_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
if fileno_limit < hard_limit: if fileno_limit < hard_limit:
@ -278,54 +291,54 @@ def main(args):
# Call ninja.py so that it can find ninja binary installed by DEPS or one in # Call ninja.py so that it can find ninja binary installed by DEPS or one in
# PATH. # PATH.
ninja_path = os.path.join(SCRIPT_DIR, 'ninja.py') ninja_path = os.path.join(SCRIPT_DIR, "ninja.py")
# If using remoteexec, use ninja_reclient.py which wraps ninja.py with # If using remoteexec, use ninja_reclient.py which wraps ninja.py with
# starting and stopping reproxy. # starting and stopping reproxy.
if use_remoteexec: if use_remoteexec:
ninja_path = os.path.join(SCRIPT_DIR, 'ninja_reclient.py') ninja_path = os.path.join(SCRIPT_DIR, "ninja_reclient.py")
args = [sys.executable, ninja_path] + input_args[1:] args = [sys.executable, ninja_path] + input_args[1:]
num_cores = multiprocessing.cpu_count() num_cores = multiprocessing.cpu_count()
if not j_specified and not t_specified: if not j_specified and not t_specified:
if not offline and (use_goma or use_remoteexec): if not offline and (use_goma or use_remoteexec):
args.append('-j') args.append("-j")
default_core_multiplier = 80 default_core_multiplier = 80
if platform.machine() in ('x86_64', 'AMD64'): if platform.machine() in ("x86_64", "AMD64"):
# Assume simultaneous multithreading and therefore half as many # Assume simultaneous multithreading and therefore half as many
# cores as logical processors. # cores as logical processors.
num_cores //= 2 num_cores //= 2
core_multiplier = int( core_multiplier = int(
os.environ.get('NINJA_CORE_MULTIPLIER', os.environ.get("NINJA_CORE_MULTIPLIER",
default_core_multiplier)) default_core_multiplier))
j_value = num_cores * core_multiplier j_value = num_cores * core_multiplier
core_limit = int(os.environ.get('NINJA_CORE_LIMIT', j_value)) core_limit = int(os.environ.get("NINJA_CORE_LIMIT", j_value))
j_value = min(j_value, core_limit) j_value = min(j_value, core_limit)
# On Windows, a -j higher than 1000 doesn't improve build times. # On Windows, a -j higher than 1000 doesn't improve build times.
# On macOS, ninja is limited to at most FD_SETSIZE (1024) open file # On macOS, ninja is limited to at most FD_SETSIZE (1024) open file
# descriptors. # descriptors.
if sys.platform in ['darwin', 'win32']: if sys.platform in ["darwin", "win32"]:
j_value = min(j_value, 1000) j_value = min(j_value, 1000)
# Use a j value that reliably works with the open file descriptors # Use a j value that reliably works with the open file descriptors
# limit. # limit.
if sys.platform in ['darwin', 'linux']: if sys.platform in ["darwin", "linux"]:
j_value = min(j_value, int(fileno_limit * 0.8)) j_value = min(j_value, int(fileno_limit * 0.8))
args.append('%d' % j_value) args.append("%d" % j_value)
else: else:
j_value = num_cores j_value = num_cores
# Ninja defaults to |num_cores + 2| # Ninja defaults to |num_cores + 2|
j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2')) j_value += int(os.environ.get("NINJA_CORE_ADDITION", "2"))
args.append('-j') args.append("-j")
args.append('%d' % j_value) args.append("%d" % j_value)
if summarize_build: if summarize_build:
# Enable statistics collection in Ninja. # Enable statistics collection in Ninja.
args += ['-d', 'stats'] args += ["-d", "stats"]
# Print the command-line to reassure the user that the right settings # Print the command-line to reassure the user that the right settings
# are being used. # are being used.
_print_cmd(args) _print_cmd(args)
@ -335,7 +348,7 @@ def main(args):
return ninja.main(args[1:]) return ninja.main(args[1:])
if __name__ == '__main__': if __name__ == "__main__":
try: try:
sys.exit(main(sys.argv)) sys.exit(main(sys.argv))
except KeyboardInterrupt: except KeyboardInterrupt:

39
ninja.py
Unescape Escape View File

@ -15,14 +15,14 @@ import gclient_paths
def findNinjaInPath(): def findNinjaInPath():
env_path = os.getenv('PATH') env_path = os.getenv("PATH")
if not env_path: if not env_path:
return return
exe = 'ninja' exe = "ninja"
if sys.platform in ('win32', 'cygwin'): if sys.platform in ("win32", "cygwin"):
exe += '.exe' exe += ".exe"
for bin_dir in env_path.split(os.pathsep): for bin_dir in env_path.split(os.pathsep):
if bin_dir.rstrip(os.sep).endswith('depot_tools'): if bin_dir.rstrip(os.sep).endswith("depot_tools"):
# skip depot_tools to avoid calling ninja.py infinitely. # skip depot_tools to avoid calling ninja.py infinitely.
continue continue
ninja_path = os.path.join(bin_dir, exe) ninja_path = os.path.join(bin_dir, exe)
@ -37,12 +37,13 @@ def fallback(ninja_args):
return subprocess.call([ninja_path] + ninja_args) return subprocess.call([ninja_path] + ninja_args)
print( print(
'depot_tools/ninja.py: Could not find Ninja in the third_party of ' "depot_tools/ninja.py: Could not find Ninja in the third_party of "
'the current project, nor in your PATH.\n' "the current project, nor in your PATH.\n"
'Please take one of the following actions to install Ninja:\n' "Please take one of the following actions to install Ninja:\n"
'- If your project has DEPS, add a CIPD Ninja dependency to DEPS.\n' "- If your project has DEPS, add a CIPD Ninja dependency to DEPS.\n"
'- Otherwise, add Ninja to your PATH *after* depot_tools.', "- Otherwise, add Ninja to your PATH *after* depot_tools.",
file=sys.stderr) file=sys.stderr,
)
return 1 return 1
@ -53,14 +54,14 @@ def main(args):
# detected, we need to split the argument. This means that arguments # detected, we need to split the argument. This means that arguments
# containing actual spaces are not supported by ninja.bat, but that is not a # containing actual spaces are not supported by ninja.bat, but that is not a
# real limitation. # real limitation.
if (sys.platform.startswith('win') and len(args) == 2): if sys.platform.startswith("win") and len(args) == 2:
args = args[:1] + args[1].split() args = args[:1] + args[1].split()
# macOS's python sets CPATH, LIBRARY_PATH, SDKROOT implicitly. # macOS's python sets CPATH, LIBRARY_PATH, SDKROOT implicitly.
# https://openradar.appspot.com/radar?id=5608755232243712 # https://openradar.appspot.com/radar?id=5608755232243712
# #
# Removing those environment variables to avoid affecting clang's behaviors. # Removing those environment variables to avoid affecting clang's behaviors.
if sys.platform == 'darwin': if sys.platform == "darwin":
os.environ.pop("CPATH", None) os.environ.pop("CPATH", None)
os.environ.pop("LIBRARY_PATH", None) os.environ.pop("LIBRARY_PATH", None)
os.environ.pop("SDKROOT", None) os.environ.pop("SDKROOT", None)
@ -70,21 +71,25 @@ def main(args):
gclient_root_path = gclient_paths.FindGclientRoot(os.getcwd()) gclient_root_path = gclient_paths.FindGclientRoot(os.getcwd())
gclient_src_root_path = None gclient_src_root_path = None
if gclient_root_path: if gclient_root_path:
gclient_src_root_path = os.path.join(gclient_root_path, 'src') gclient_src_root_path = os.path.join(gclient_root_path, "src")
for base_path in set( for base_path in set(
[primary_solution_path, gclient_root_path, gclient_src_root_path]): [primary_solution_path, gclient_root_path, gclient_src_root_path]):
if not base_path: if not base_path:
continue continue
ninja_path = os.path.join(base_path, 'third_party', 'ninja', ninja_path = os.path.join(
'ninja' + gclient_paths.GetExeSuffix()) base_path,
"third_party",
"ninja",
"ninja" + gclient_paths.GetExeSuffix(),
)
if os.path.isfile(ninja_path): if os.path.isfile(ninja_path):
return subprocess.call([ninja_path] + args[1:]) return subprocess.call([ninja_path] + args[1:])
return fallback(args[1:]) return fallback(args[1:])
if __name__ == '__main__': if __name__ == "__main__":
try: try:
sys.exit(main(sys.argv)) sys.exit(main(sys.argv))
except KeyboardInterrupt: except KeyboardInterrupt:

4
ninja_reclient.py
Unescape Escape View File

@ -14,7 +14,7 @@ import reclient_helper
def main(argv): def main(argv):
with reclient_helper.build_context(argv, 'ninja_reclient') as ret_code: with reclient_helper.build_context(argv, "ninja_reclient") as ret_code:
if ret_code: if ret_code:
return ret_code return ret_code
try: try:
@ -24,5 +24,5 @@ def main(argv):
return 1 return 1
if __name__ == '__main__': if __name__ == "__main__":
sys.exit(main(sys.argv)) sys.exit(main(sys.argv))

119
ninjalog_uploader.py
Unescape Escape View File

@ -28,22 +28,35 @@ import time
import urllib.request import urllib.request
# These build configs affect build performance. # These build configs affect build performance.
ALLOWLISTED_CONFIGS = ('symbol_level', 'use_goma', 'is_debug', ALLOWLISTED_CONFIGS = (
'is_component_build', 'enable_nacl', 'host_os', "symbol_level",
'host_cpu', 'target_os', 'target_cpu', "use_goma",
'blink_symbol_level', 'is_java_debug', "is_debug",
'treat_warnings_as_errors', 'disable_android_lint', "is_component_build",
'use_errorprone_java_compiler', 'incremental_install', "enable_nacl",
'android_static_analysis') "host_os",
"host_cpu",
"target_os",
"target_cpu",
"blink_symbol_level",
"is_java_debug",
"treat_warnings_as_errors",
"disable_android_lint",
"use_errorprone_java_compiler",
"incremental_install",
"android_static_analysis",
)
def IsGoogler(): def IsGoogler():
"""Check whether this user is Googler or not.""" """Check whether this user is Googler or not."""
p = subprocess.run('goma_auth info', p = subprocess.run(
"goma_auth info",
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True, universal_newlines=True,
shell=True) shell=True,
)
if p.returncode != 0: if p.returncode != 0:
return False return False
lines = p.stdout.splitlines() lines = p.stdout.splitlines()
@ -51,7 +64,7 @@ def IsGoogler():
return False return False
l = lines[0] l = lines[0]
# |l| will be like 'Login as <user>@google.com' for googler using goma. # |l| will be like 'Login as <user>@google.com' for googler using goma.
return l.startswith('Login as ') and l.endswith('@google.com') return l.startswith("Login as ") and l.endswith("@google.com")
def ParseGNArgs(gn_args): def ParseGNArgs(gn_args):
@ -63,10 +76,10 @@ def ParseGNArgs(gn_args):
key = config["name"] key = config["name"]
if key not in ALLOWLISTED_CONFIGS: if key not in ALLOWLISTED_CONFIGS:
continue continue
if 'current' in config: if "current" in config:
build_configs[key] = config['current']['value'] build_configs[key] = config["current"]["value"]
else: else:
build_configs[key] = config['default']['value'] build_configs[key] = config["default"]["value"]
return build_configs return build_configs
@ -79,8 +92,8 @@ def GetBuildTargetFromCommandLine(cmdline):
# Skipping all args that involve these flags, and taking all remaining args # Skipping all args that involve these flags, and taking all remaining args
# as targets. # as targets.
onearg_flags = ('-C', '-d', '-f', '-j', '-k', '-l', '-p', '-t', '-w') onearg_flags = ("-C", "-d", "-f", "-j", "-k", "-l", "-p", "-t", "-w")
zeroarg_flags = ('--version', '-n', '-v') zeroarg_flags = ("--version", "-n", "-v")
targets = [] targets = []
@ -90,12 +103,12 @@ def GetBuildTargetFromCommandLine(cmdline):
idx += 2 idx += 2
continue continue
if (arg[:2] in onearg_flags or arg in zeroarg_flags): if arg[:2] in onearg_flags or arg in zeroarg_flags:
idx += 1 idx += 1
continue continue
# A target doesn't start with '-'. # A target doesn't start with '-'.
if arg.startswith('-'): if arg.startswith("-"):
idx += 1 idx += 1
continue continue
@ -114,12 +127,12 @@ def GetJflag(cmdline):
"""Parse cmdline to get flag value for -j""" """Parse cmdline to get flag value for -j"""
for i in range(len(cmdline)): for i in range(len(cmdline)):
if (cmdline[i] == '-j' and i + 1 < len(cmdline) if (cmdline[i] == "-j" and i + 1 < len(cmdline)
and cmdline[i + 1].isdigit()): and cmdline[i + 1].isdigit()):
return int(cmdline[i + 1]) return int(cmdline[i + 1])
if (cmdline[i].startswith('-j') and cmdline[i][len('-j'):].isdigit()): if cmdline[i].startswith("-j") and cmdline[i][len("-j"):].isdigit():
return int(cmdline[i][len('-j'):]) return int(cmdline[i][len("-j"):])
def GetMetadata(cmdline, ninjalog): def GetMetadata(cmdline, ninjalog):
@ -136,10 +149,10 @@ def GetMetadata(cmdline, ninjalog):
build_configs = {} build_configs = {}
try: try:
args = ['gn', 'args', build_dir, '--list', '--short', '--json'] args = ["gn", "args", build_dir, "--list", "--short", "--json"]
if sys.platform == 'win32': if sys.platform == "win32":
# gn in PATH is bat file in windows environment (except cygwin). # gn in PATH is bat file in windows environment (except cygwin).
args = ['cmd', '/c'] + args args = ["cmd", "/c"] + args
gn_args = subprocess.check_output(args) gn_args = subprocess.check_output(args)
build_configs = ParseGNArgs(gn_args) build_configs = ParseGNArgs(gn_args)
@ -152,15 +165,15 @@ def GetMetadata(cmdline, ninjalog):
build_configs[k] = str(build_configs[k]) build_configs[k] = str(build_configs[k])
metadata = { metadata = {
'platform': platform.system(), "platform": platform.system(),
'cpu_core': multiprocessing.cpu_count(), "cpu_core": multiprocessing.cpu_count(),
'build_configs': build_configs, "build_configs": build_configs,
'targets': GetBuildTargetFromCommandLine(cmdline), "targets": GetBuildTargetFromCommandLine(cmdline),
} }
jflag = GetJflag(cmdline) jflag = GetJflag(cmdline)
if jflag is not None: if jflag is not None:
metadata['jobs'] = jflag metadata["jobs"] = jflag
return metadata return metadata
@ -168,36 +181,40 @@ def GetMetadata(cmdline, ninjalog):
def GetNinjalog(cmdline): def GetNinjalog(cmdline):
"""GetNinjalog returns the path to ninjalog from cmdline.""" """GetNinjalog returns the path to ninjalog from cmdline."""
# ninjalog is in current working directory by default. # ninjalog is in current working directory by default.
ninjalog_dir = '.' ninjalog_dir = "."
i = 0 i = 0
while i < len(cmdline): while i < len(cmdline):
cmd = cmdline[i] cmd = cmdline[i]
i += 1 i += 1
if cmd == '-C' and i < len(cmdline): if cmd == "-C" and i < len(cmdline):
ninjalog_dir = cmdline[i] ninjalog_dir = cmdline[i]
i += 1 i += 1
continue continue
if cmd.startswith('-C') and len(cmd) > len('-C'): if cmd.startswith("-C") and len(cmd) > len("-C"):
ninjalog_dir = cmd[len('-C'):] ninjalog_dir = cmd[len("-C"):]
return os.path.join(ninjalog_dir, '.ninja_log') return os.path.join(ninjalog_dir, ".ninja_log")
def main(): def main():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('--server', parser.add_argument(
default='chromium-build-stats.appspot.com', "--server",
help='server to upload ninjalog file.') default="chromium-build-stats.appspot.com",
parser.add_argument('--ninjalog', help='ninjalog file to upload.') help="server to upload ninjalog file.",
parser.add_argument('--verbose', )
action='store_true', parser.add_argument("--ninjalog", help="ninjalog file to upload.")
help='Enable verbose logging.') parser.add_argument("--verbose",
parser.add_argument('--cmdline', action="store_true",
help="Enable verbose logging.")
parser.add_argument(
"--cmdline",
required=True, required=True,
nargs=argparse.REMAINDER, nargs=argparse.REMAINDER,
help='command line args passed to ninja.') help="command line args passed to ninja.",
)
args = parser.parse_args() args = parser.parse_args()
@ -224,27 +241,29 @@ def main():
output = io.BytesIO() output = io.BytesIO()
with open(ninjalog) as f: with open(ninjalog) as f:
with gzip.GzipFile(fileobj=output, mode='wb') as g: with gzip.GzipFile(fileobj=output, mode="wb") as g:
g.write(f.read().encode()) g.write(f.read().encode())
g.write(b'# end of ninja log\n') g.write(b"# end of ninja log\n")
metadata = GetMetadata(args.cmdline, ninjalog) metadata = GetMetadata(args.cmdline, ninjalog)
logging.info('send metadata: %s', json.dumps(metadata)) logging.info("send metadata: %s", json.dumps(metadata))
g.write(json.dumps(metadata).encode()) g.write(json.dumps(metadata).encode())
resp = urllib.request.urlopen( resp = urllib.request.urlopen(
urllib.request.Request('https://' + args.server + '/upload_ninja_log/', urllib.request.Request(
"https://" + args.server + "/upload_ninja_log/",
data=output.getvalue(), data=output.getvalue(),
headers={'Content-Encoding': 'gzip'})) headers={"Content-Encoding": "gzip"},
))
if resp.status != 200: if resp.status != 200:
logging.warning("unexpected status code for response: %s", resp.status) logging.warning("unexpected status code for response: %s", resp.status)
return 1 return 1
logging.info('response header: %s', resp.headers) logging.info("response header: %s", resp.headers)
logging.info('response content: %s', resp.read()) logging.info("response content: %s", resp.read())
return 0 return 0
if __name__ == '__main__': if __name__ == "__main__":
sys.exit(main()) sys.exit(main())

63
ninjalog_uploader_wrapper.py
Unescape Escape View File

@ -13,43 +13,43 @@ import ninjalog_uploader
import subprocess2 import subprocess2
THIS_DIR = os.path.dirname(__file__) THIS_DIR = os.path.dirname(__file__)
UPLOADER = os.path.join(THIS_DIR, 'ninjalog_uploader.py') UPLOADER = os.path.join(THIS_DIR, "ninjalog_uploader.py")
CONFIG = os.path.join(THIS_DIR, 'ninjalog.cfg') CONFIG = os.path.join(THIS_DIR, "ninjalog.cfg")
VERSION = 3 VERSION = 3
def LoadConfig(): def LoadConfig():
if os.path.isfile(CONFIG): if os.path.isfile(CONFIG):
with open(CONFIG, 'r') as f: with open(CONFIG, "r") as f:
try: try:
config = json.load(f) config = json.load(f)
except Exception: except Exception:
# Set default value when failed to load config. # Set default value when failed to load config.
config = { config = {
'is-googler': ninjalog_uploader.IsGoogler(), "is-googler": ninjalog_uploader.IsGoogler(),
'countdown': 10, "countdown": 10,
'version': VERSION, "version": VERSION,
} }
if config['version'] == VERSION: if config["version"] == VERSION:
config['countdown'] = max(0, config['countdown'] - 1) config["countdown"] = max(0, config["countdown"] - 1)
return config return config
return { return {
'is-googler': ninjalog_uploader.IsGoogler(), "is-googler": ninjalog_uploader.IsGoogler(),
'countdown': 10, "countdown": 10,
'version': VERSION, "version": VERSION,
} }
def SaveConfig(config): def SaveConfig(config):
with open(CONFIG, 'w') as f: with open(CONFIG, "w") as f:
json.dump(config, f) json.dump(config, f)
def ShowMessage(countdown): def ShowMessage(countdown):
allowlisted = '\n'.join( allowlisted = "\n".join(
[' * %s' % config for config in ninjalog_uploader.ALLOWLISTED_CONFIGS]) [" * %s" % config for config in ninjalog_uploader.ALLOWLISTED_CONFIGS])
print(""" print("""
Your ninjalog will be uploaded to build stats server. The uploaded log will be Your ninjalog will be uploaded to build stats server. The uploaded log will be
used to analyze user side build performance. used to analyze user side build performance.
@ -78,27 +78,32 @@ You can find a more detailed explanation in
or or
https://chromium.googlesource.com/chromium/tools/depot_tools/+/main/ninjalog.README.md https://chromium.googlesource.com/chromium/tools/depot_tools/+/main/ninjalog.README.md
""" % (allowlisted, countdown, __file__, __file__, """ % (
os.path.abspath(os.path.join(THIS_DIR, "ninjalog.README.md")))) allowlisted,
countdown,
__file__,
__file__,
os.path.abspath(os.path.join(THIS_DIR, "ninjalog.README.md")),
))
def main(): def main():
config = LoadConfig() config = LoadConfig()
if len(sys.argv) == 2 and sys.argv[1] == 'opt-in': if len(sys.argv) == 2 and sys.argv[1] == "opt-in":
config['opt-in'] = True config["opt-in"] = True
config['countdown'] = 0 config["countdown"] = 0
SaveConfig(config) SaveConfig(config)
print('ninjalog upload is opted in.') print("ninjalog upload is opted in.")
return 0 return 0
if len(sys.argv) == 2 and sys.argv[1] == 'opt-out': if len(sys.argv) == 2 and sys.argv[1] == "opt-out":
config['opt-in'] = False config["opt-in"] = False
SaveConfig(config) SaveConfig(config)
print('ninjalog upload is opted out.') print("ninjalog upload is opted out.")
return 0 return 0
if 'opt-in' in config and not config['opt-in']: if "opt-in" in config and not config["opt-in"]:
# Upload is opted out. # Upload is opted out.
return 0 return 0
@ -121,13 +126,15 @@ def main():
# Run upload script without wait. # Run upload script without wait.
devnull = open(os.devnull, "w") devnull = open(os.devnull, "w")
creationflags = 0 creationflags = 0
if platform.system() == 'Windows': if platform.system() == "Windows":
creationflags = subprocess.CREATE_NEW_PROCESS_GROUP creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
subprocess2.Popen([sys.executable, UPLOADER] + sys.argv[1:], subprocess2.Popen(
[sys.executable, UPLOADER] + sys.argv[1:],
stdout=devnull, stdout=devnull,
stderr=devnull, stderr=devnull,
creationflags=creationflags) creationflags=creationflags,
)
if __name__ == '__main__': if __name__ == "__main__":
sys.exit(main()) sys.exit(main())

137
post_build_ninja_summary.py
Unescape Escape View File

@ -94,9 +94,9 @@ class Target:
""" """
# Allow for modest floating-point errors # Allow for modest floating-point errors
epsilon = 0.000002 epsilon = 0.000002
if (self.weighted_duration > self.Duration() + epsilon): if self.weighted_duration > self.Duration() + epsilon:
print('%s > %s?' % (self.weighted_duration, self.Duration())) print("%s > %s?" % (self.weighted_duration, self.Duration()))
assert (self.weighted_duration <= self.Duration() + epsilon) assert self.weighted_duration <= self.Duration() + epsilon
return self.weighted_duration return self.weighted_duration
def DescribeTargets(self): def DescribeTargets(self):
@ -104,10 +104,10 @@ class Target:
# Some build steps generate dozens of outputs - handle them sanely. # Some build steps generate dozens of outputs - handle them sanely.
# The max_length was chosen so that it can fit most of the long # The max_length was chosen so that it can fit most of the long
# single-target names, while minimizing word wrapping. # single-target names, while minimizing word wrapping.
result = ', '.join(self.targets) result = ", ".join(self.targets)
max_length = 65 max_length = 65
if len(result) > max_length: if len(result) > max_length:
result = result[:max_length] + '...' result = result[:max_length] + "..."
return result return result
@ -121,12 +121,12 @@ def ReadTargets(log, show_all):
# targets. # targets.
if not header: if not header:
return [] return []
assert header == '# ninja log v5\n', ('unrecognized ninja log version %r' % assert header == "# ninja log v5\n", ("unrecognized ninja log version %r" %
header) header)
targets_dict = {} targets_dict = {}
last_end_seen = 0.0 last_end_seen = 0.0
for line in log: for line in log:
parts = line.strip().split('\t') parts = line.strip().split("\t")
if len(parts) != 5: if len(parts) != 5:
# If ninja.exe is rudely halted then the .ninja_log file may be # If ninja.exe is rudely halted then the .ninja_log file may be
# corrupt. Silently continue. # corrupt. Silently continue.
@ -170,12 +170,12 @@ def GetExtension(target, extra_patterns):
steps by type.""" steps by type."""
for output in target.targets: for output in target.targets:
if extra_patterns: if extra_patterns:
for fn_pattern in extra_patterns.split(';'): for fn_pattern in extra_patterns.split(";"):
if fnmatch.fnmatch(output, '*' + fn_pattern + '*'): if fnmatch.fnmatch(output, "*" + fn_pattern + "*"):
return fn_pattern return fn_pattern
# Not a true extension, but a good grouping. # Not a true extension, but a good grouping.
if output.endswith('type_mappings'): if output.endswith("type_mappings"):
extension = 'type_mappings' extension = "type_mappings"
break break
# Capture two extensions if present. For example: file.javac.jar should # Capture two extensions if present. For example: file.javac.jar should
@ -185,26 +185,26 @@ def GetExtension(target, extra_patterns):
extension = ext2 + ext1 # Preserve the order in the file name. extension = ext2 + ext1 # Preserve the order in the file name.
if len(extension) == 0: if len(extension) == 0:
extension = '(no extension found)' extension = "(no extension found)"
if ext1 in ['.pdb', '.dll', '.exe']: if ext1 in [".pdb", ".dll", ".exe"]:
extension = 'PEFile (linking)' extension = "PEFile (linking)"
# Make sure that .dll and .exe are grouped together and that the # Make sure that .dll and .exe are grouped together and that the
# .dll.lib files don't cause these to be listed as libraries # .dll.lib files don't cause these to be listed as libraries
break break
if ext1 in ['.so', '.TOC']: if ext1 in [".so", ".TOC"]:
extension = '.so (linking)' extension = ".so (linking)"
# Attempt to identify linking, avoid identifying as '.TOC' # Attempt to identify linking, avoid identifying as '.TOC'
break break
# Make sure .obj files don't get categorized as mojo files # Make sure .obj files don't get categorized as mojo files
if ext1 in ['.obj', '.o']: if ext1 in [".obj", ".o"]:
break break
# Jars are the canonical output of java targets. # Jars are the canonical output of java targets.
if ext1 == '.jar': if ext1 == ".jar":
break break
# Normalize all mojo related outputs to 'mojo'. # Normalize all mojo related outputs to 'mojo'.
if output.count('.mojom') > 0: if output.count(".mojom") > 0:
extension = 'mojo' extension = "mojo"
break break
return extension return extension
@ -229,8 +229,8 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting):
if target.end > latest: if target.end > latest:
latest = target.end latest = target.end
total_cpu_time += target.Duration() total_cpu_time += target.Duration()
task_start_stop_times.append((target.start, 'start', target)) task_start_stop_times.append((target.start, "start", target))
task_start_stop_times.append((target.end, 'stop', target)) task_start_stop_times.append((target.end, "stop", target))
length = latest - earliest length = latest - earliest
weighted_total = 0.0 weighted_total = 0.0
@ -256,10 +256,10 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting):
if num_running > 0: if num_running > 0:
# Update the total weighted time up to this moment. # Update the total weighted time up to this moment.
last_weighted_time += (time - last_time) / float(num_running) last_weighted_time += (time - last_time) / float(num_running)
if action_name == 'start': if action_name == "start":
# Record the total weighted task time when this task starts. # Record the total weighted task time when this task starts.
running_tasks[target] = last_weighted_time running_tasks[target] = last_weighted_time
if action_name == 'stop': if action_name == "stop":
# Record the change in the total weighted task time while this task # Record the change in the total weighted task time while this task
# ran. # ran.
weighted_duration = last_weighted_time - running_tasks[target] weighted_duration = last_weighted_time - running_tasks[target]
@ -267,24 +267,26 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting):
weighted_total += weighted_duration weighted_total += weighted_duration
del running_tasks[target] del running_tasks[target]
last_time = time last_time = time
assert (len(running_tasks) == 0) assert len(running_tasks) == 0
# Warn if the sum of weighted times is off by more than half a second. # Warn if the sum of weighted times is off by more than half a second.
if abs(length - weighted_total) > 500: if abs(length - weighted_total) > 500:
print('Warning: Possible corrupt ninja log, results may be ' print("Warning: Possible corrupt ninja log, results may be "
'untrustworthy. Length = %.3f, weighted total = %.3f' % "untrustworthy. Length = %.3f, weighted total = %.3f" %
(length, weighted_total)) (length, weighted_total))
# Print the slowest build steps: # Print the slowest build steps:
print(' Longest build steps:') print(" Longest build steps:")
if elapsed_time_sorting: if elapsed_time_sorting:
entries.sort(key=lambda x: x.Duration()) entries.sort(key=lambda x: x.Duration())
else: else:
entries.sort(key=lambda x: x.WeightedDuration()) entries.sort(key=lambda x: x.WeightedDuration())
for target in entries[-long_count:]: for target in entries[-long_count:]:
print(' %8.1f weighted s to build %s (%.1f s elapsed time)' % print(" %8.1f weighted s to build %s (%.1f s elapsed time)" % (
(target.WeightedDuration(), target.DescribeTargets(), target.WeightedDuration(),
target.Duration())) target.DescribeTargets(),
target.Duration(),
))
# Sum up the time by file extension/type of the output file # Sum up the time by file extension/type of the output file
count_by_ext = {} count_by_ext = {}
@ -293,13 +295,13 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting):
# Scan through all of the targets to build up per-extension statistics. # Scan through all of the targets to build up per-extension statistics.
for target in entries: for target in entries:
extension = GetExtension(target, extra_step_types) extension = GetExtension(target, extra_step_types)
time_by_ext[extension] = time_by_ext.get(extension, time_by_ext[extension] = (time_by_ext.get(extension, 0) +
0) + target.Duration() target.Duration())
weighted_time_by_ext[extension] = weighted_time_by_ext.get( weighted_time_by_ext[extension] = (
extension, 0) + target.WeightedDuration() weighted_time_by_ext.get(extension, 0) + target.WeightedDuration())
count_by_ext[extension] = count_by_ext.get(extension, 0) + 1 count_by_ext[extension] = count_by_ext.get(extension, 0) + 1
print(' Time by build-step type:') print(" Time by build-step type:")
# Copy to a list with extension name and total time swapped, to (time, ext) # Copy to a list with extension name and total time swapped, to (time, ext)
if elapsed_time_sorting: if elapsed_time_sorting:
weighted_time_by_ext_sorted = sorted( weighted_time_by_ext_sorted = sorted(
@ -309,34 +311,39 @@ def SummarizeEntries(entries, extra_step_types, elapsed_time_sorting):
(y, x) for (x, y) in weighted_time_by_ext.items()) (y, x) for (x, y) in weighted_time_by_ext.items())
# Print the slowest build target types: # Print the slowest build target types:
for time, extension in weighted_time_by_ext_sorted[-long_ext_count:]: for time, extension in weighted_time_by_ext_sorted[-long_ext_count:]:
print( print(" %8.1f s weighted time to generate %d %s files "
' %8.1f s weighted time to generate %d %s files ' "(%1.1f s elapsed time sum)" % (
'(%1.1f s elapsed time sum)' % time,
(time, count_by_ext[extension], extension, time_by_ext[extension])) count_by_ext[extension],
extension,
print(' %.1f s weighted time (%.1f s elapsed time sum, %1.1fx ' time_by_ext[extension],
'parallelism)' % ))
print(" %.1f s weighted time (%.1f s elapsed time sum, %1.1fx "
"parallelism)" %
(length, total_cpu_time, total_cpu_time * 1.0 / length)) (length, total_cpu_time, total_cpu_time * 1.0 / length))
print(' %d build steps completed, average of %1.2f/s' % print(" %d build steps completed, average of %1.2f/s" %
(len(entries), len(entries) / (length))) (len(entries), len(entries) / (length)))
def main(): def main():
log_file = '.ninja_log' log_file = ".ninja_log"
metrics_file = 'siso_metrics.json' metrics_file = "siso_metrics.json"
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('-C', dest='build_directory', help='Build directory.') parser.add_argument("-C", dest="build_directory", help="Build directory.")
parser.add_argument( parser.add_argument(
'-s', "-s",
'--step-types', "--step-types",
help='semicolon separated fnmatch patterns for build-step grouping') help="semicolon separated fnmatch patterns for build-step grouping",
)
parser.add_argument( parser.add_argument(
'-e', "-e",
'--elapsed_time_sorting', "--elapsed_time_sorting",
default=False, default=False,
action='store_true', action="store_true",
help='Sort output by elapsed time instead of weighted time') help="Sort output by elapsed time instead of weighted time",
parser.add_argument('--log-file', )
parser.add_argument("--log-file",
help="specific ninja log file to analyze.") help="specific ninja log file to analyze.")
args, _extra_args = parser.parse_known_args() args, _extra_args = parser.parse_known_args()
if args.build_directory: if args.build_directory:
@ -348,34 +355,34 @@ def main():
# Offer a convenient way to add extra step types automatically, # Offer a convenient way to add extra step types automatically,
# including when this script is run by autoninja. get() returns None if # including when this script is run by autoninja. get() returns None if
# the variable isn't set. # the variable isn't set.
args.step_types = os.environ.get('chromium_step_types') args.step_types = os.environ.get("chromium_step_types")
if args.step_types: if args.step_types:
# Make room for the extra build types. # Make room for the extra build types.
global long_ext_count global long_ext_count
long_ext_count += len(args.step_types.split(';')) long_ext_count += len(args.step_types.split(";"))
if os.path.exists(metrics_file): if os.path.exists(metrics_file):
# Automatically handle summarizing siso builds. # Automatically handle summarizing siso builds.
cmd = ['siso.bat' if 'win32' in sys.platform else 'siso'] cmd = ["siso.bat" if "win32" in sys.platform else "siso"]
cmd.extend(['metrics', 'summary']) cmd.extend(["metrics", "summary"])
if args.build_directory: if args.build_directory:
cmd.extend(['-C', args.build_directory]) cmd.extend(["-C", args.build_directory])
if args.step_types: if args.step_types:
cmd.extend(['--step_types', args.step_types]) cmd.extend(["--step_types", args.step_types])
if args.elapsed_time_sorting: if args.elapsed_time_sorting:
cmd.append('--elapsed_time_sorting') cmd.append("--elapsed_time_sorting")
subprocess.run(cmd) subprocess.run(cmd)
else: else:
try: try:
with open(log_file, 'r') as log: with open(log_file, "r") as log:
entries = ReadTargets(log, False) entries = ReadTargets(log, False)
if entries: if entries:
SummarizeEntries(entries, args.step_types, SummarizeEntries(entries, args.step_types,
args.elapsed_time_sorting) args.elapsed_time_sorting)
except IOError: except IOError:
print('Log file %r not found, no build summary created.' % log_file) print("Log file %r not found, no build summary created." % log_file)
return errno.ENOENT return errno.ENOENT
if __name__ == '__main__': if __name__ == "__main__":
sys.exit(main()) sys.exit(main())

Write Preview
Loading…
Cancel
Save