diff --git a/autoninja b/autoninja index 7943e55732..156bd29b4b 100755 --- a/autoninja +++ b/autoninja @@ -24,12 +24,12 @@ if eval "$command"; then fi # Collect ninjalog from googler. - python "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command + python3 "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command exit fi # Collect ninjalog from googler. -python "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command +python3 "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command # Return an error code of 1 so that if a developer types: # "autoninja chrome && chrome" then chrome won't run if the build fails. diff --git a/autoninja.bat b/autoninja.bat index 056560f1c5..b5bac57881 100755 --- a/autoninja.bat +++ b/autoninja.bat @@ -36,14 +36,14 @@ REM Also print it to reassure that the right settings are being used. FOR /f "usebackq tokens=*" %%a in (`vpython %scriptdir%autoninja.py "%*"`) do echo %%a & %%a @if errorlevel 1 goto buildfailure -REM Use call to invoke vpython script here, because we use vpython via vpython.bat. +REM Use call to invoke python script here, because we may use python3 via python3.bat. @if "%NINJA_SUMMARIZE_BUILD%" == "1" call python3 %scriptdir%post_build_ninja_summary.py %* -@call python.bat %scriptdir%ninjalog_uploader_wrapper.py --cmdline %* +@call python3 %scriptdir%ninjalog_uploader_wrapper.py --cmdline %* exit /b :buildfailure -@call python.bat %scriptdir%ninjalog_uploader_wrapper.py --cmdline %* +@call python3 %scriptdir%ninjalog_uploader_wrapper.py --cmdline %* REM Return an error code of 1 so that if a developer types: REM "autoninja chrome && chrome" then chrome won't run if the build fails. diff --git a/ninjalog_uploader.py b/ninjalog_uploader.py index 308815ea24..3de06648e6 100755 --- a/ninjalog_uploader.py +++ b/ninjalog_uploader.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2018 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -15,8 +15,8 @@ The log will be used to analyze user side build performance. """ import argparse -import cStringIO import gzip +import io import json import logging import multiprocessing @@ -26,7 +26,8 @@ import subprocess import sys import time -from third_party import httplib2 +from third_party.six.moves.urllib import error +from third_party.six.moves.urllib import request # These build configs affect build performance a lot. # TODO(https://crbug.com/900161): Add 'blink_symbol_level' and @@ -39,10 +40,9 @@ WHITELISTED_CONFIGS = ('symbol_level', 'use_goma', 'is_debug', def IsGoogler(server): """Check whether this script run inside corp network.""" try: - h = httplib2.Http() - _, content = h.request('https://' + server + '/should-upload', 'GET') - return content == 'Success' - except httplib2.HttpLib2Error: + resp = request.urlopen('https://' + server + '/should-upload') + return resp.read() == b'Success' + except error.URLError: return False @@ -202,31 +202,28 @@ def main(): logging.info("ninjalog is not updated recently %s", ninjalog) return 0 - output = cStringIO.StringIO() + output = io.BytesIO() with open(ninjalog) as f: with gzip.GzipFile(fileobj=output, mode='wb') as g: - g.write(f.read()) - g.write('# end of ninja log\n') + g.write(f.read().encode()) + g.write(b'# end of ninja log\n') metadata = GetMetadata(args.cmdline, ninjalog) logging.info('send metadata: %s', json.dumps(metadata)) - g.write(json.dumps(metadata)) - - h = httplib2.Http() - resp_headers, content = h.request('https://' + args.server + - '/upload_ninja_log/', - 'POST', - body=output.getvalue(), - headers={'Content-Encoding': 'gzip'}) - - if resp_headers.status != 200: - logging.warning("unexpected status code for response: %s", - resp_headers.status) + g.write(json.dumps(metadata).encode()) + + resp = request.urlopen( + request.Request('https://' + args.server + '/upload_ninja_log/', + data=output.getvalue(), + headers={'Content-Encoding': 'gzip'})) + + if resp.status != 200: + logging.warning("unexpected status code for response: %s", resp.status) return 1 - logging.info('response header: %s', resp_headers) - logging.info('response content: %s', content) + logging.info('response header: %s', resp.headers) + logging.info('response content: %s', resp.read()) return 0 diff --git a/ninjalog_uploader_wrapper.py b/ninjalog_uploader_wrapper.py index edb03be994..83ba9a44fb 100755 --- a/ninjalog_uploader_wrapper.py +++ b/ninjalog_uploader_wrapper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2018 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -115,7 +115,7 @@ def main(): creationnflags = 0 if platform.system() == 'Windows': creationnflags = subprocess.CREATE_NEW_PROCESS_GROUP - subprocess2.Popen(['vpython', UPLOADER] + sys.argv[1:], + subprocess2.Popen([sys.executable, UPLOADER] + sys.argv[1:], stdout=devnull, stderr=devnull, creationflags=creationnflags) diff --git a/tests/ninjalog_uploader_test.py b/tests/ninjalog_uploader_test.py index 7716794d88..5b22f26886 100755 --- a/tests/ninjalog_uploader_test.py +++ b/tests/ninjalog_uploader_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2018 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -15,105 +15,105 @@ import ninjalog_uploader class NinjalogUploaderTest(unittest.TestCase): - def test_parse_gn_args(self): - self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([])), {}) - - # Extract current configs from GN's output json. - self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([ - { - 'current': {'value': 'true'}, - 'default': {'value': 'false'}, - 'name': 'is_component_build' - }, - { - 'default': {'value': '"x64"'}, - 'name': 'host_cpu' - }, - ])), { - 'is_component_build': 'true', - 'host_cpu': '"x64"', - }) - - self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([ - { - 'current': {'value': 'true'}, - 'default': {'value': 'false'}, - 'name': 'is_component_build' - }, - { - 'current': {'value': 'false'}, - 'default': {'value': 'false'}, - 'name': 'use_goma' - }, - ])), {'is_component_build': 'true', - 'use_goma': 'false'}) - - def test_get_ninjalog(self): - # No args => default to cwd. - self.assertEqual(ninjalog_uploader.GetNinjalog(['ninja']), - './.ninja_log') - - # Specified by -C case. - self.assertEqual( - ninjalog_uploader.GetNinjalog(['ninja', '-C', 'out/Release']), - 'out/Release/.ninja_log') - self.assertEqual( - ninjalog_uploader.GetNinjalog(['ninja', '-Cout/Release']), - 'out/Release/.ninja_log') - - # Invalid -C flag case. - self.assertEqual(ninjalog_uploader.GetNinjalog(['ninja', '-C']), - './.ninja_log') - - # Multiple target directories => use the last directory. - self.assertEqual(ninjalog_uploader.GetNinjalog( - ['ninja', '-C', 'out/Release', '-C', 'out/Debug']), - 'out/Debug/.ninja_log') - - def test_get_build_target_from_command_line(self): - self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( - ['ninja', 'chrome']), ['chrome']) - - self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( - ['ninja']), []) - - self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( - ['ninja', '-j', '1000', 'chrome']), ['chrome']) - - self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( - ['ninja', 'chrome', '-j', '1000']), ['chrome']) - - self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( - ['ninja', '-C', 'chrome']), []) - - self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( - ['ninja', '-Cout/Release', 'chrome']), ['chrome']) - - self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( - ['ninja', '-C', 'out/Release', 'chrome', 'all']), ['chrome', 'all']) - - def test_get_j_flag(self): - self.assertEqual(ninjalog_uploader.GetJflag( - ['ninja']), None) - - self.assertEqual(ninjalog_uploader.GetJflag( - ['ninja','-j', '1000']), 1000) - - self.assertEqual(ninjalog_uploader.GetJflag( - ['ninja','-j', '1000a']), None) - - self.assertEqual(ninjalog_uploader.GetJflag( - ['ninja','-j', 'a']), None) - - self.assertEqual(ninjalog_uploader.GetJflag( - ['ninja','-j1000']), 1000) - - self.assertEqual(ninjalog_uploader.GetJflag( - ['ninja','-ja']), None) - - self.assertEqual(ninjalog_uploader.GetJflag( - ['ninja','-j']), None) + def test_parse_gn_args(self): + self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([])), {}) + + # Extract current configs from GN's output json. + self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([ + { + 'current': {'value': 'true'}, + 'default': {'value': 'false'}, + 'name': 'is_component_build' + }, + { + 'default': {'value': '"x64"'}, + 'name': 'host_cpu' + }, + ])), { + 'is_component_build': 'true', + 'host_cpu': '"x64"', + }) + + self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([ + { + 'current': {'value': 'true'}, + 'default': {'value': 'false'}, + 'name': 'is_component_build' + }, + { + 'current': {'value': 'false'}, + 'default': {'value': 'false'}, + 'name': 'use_goma' + }, + ])), {'is_component_build': 'true', + 'use_goma': 'false'}) + + def test_get_ninjalog(self): + # No args => default to cwd. + self.assertEqual(ninjalog_uploader.GetNinjalog(['ninja']), + './.ninja_log') + + # Specified by -C case. + self.assertEqual( + ninjalog_uploader.GetNinjalog(['ninja', '-C', 'out/Release']), + 'out/Release/.ninja_log') + self.assertEqual( + ninjalog_uploader.GetNinjalog(['ninja', '-Cout/Release']), + 'out/Release/.ninja_log') + + # Invalid -C flag case. + self.assertEqual(ninjalog_uploader.GetNinjalog(['ninja', '-C']), + './.ninja_log') + + # Multiple target directories => use the last directory. + self.assertEqual(ninjalog_uploader.GetNinjalog( + ['ninja', '-C', 'out/Release', '-C', 'out/Debug']), + 'out/Debug/.ninja_log') + + def test_get_build_target_from_command_line(self): + self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( + ['ninja', 'chrome']), ['chrome']) + + self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( + ['ninja']), []) + + self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( + ['ninja', '-j', '1000', 'chrome']), ['chrome']) + + self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( + ['ninja', 'chrome', '-j', '1000']), ['chrome']) + + self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( + ['ninja', '-C', 'chrome']), []) + + self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( + ['ninja', '-Cout/Release', 'chrome']), ['chrome']) + + self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine( + ['ninja', '-C', 'out/Release', 'chrome', 'all']), ['chrome', 'all']) + + def test_get_j_flag(self): + self.assertEqual(ninjalog_uploader.GetJflag( + ['ninja']), None) + + self.assertEqual(ninjalog_uploader.GetJflag( + ['ninja','-j', '1000']), 1000) + + self.assertEqual(ninjalog_uploader.GetJflag( + ['ninja','-j', '1000a']), None) + + self.assertEqual(ninjalog_uploader.GetJflag( + ['ninja','-j', 'a']), None) + + self.assertEqual(ninjalog_uploader.GetJflag( + ['ninja','-j1000']), 1000) + + self.assertEqual(ninjalog_uploader.GetJflag( + ['ninja','-ja']), None) + + self.assertEqual(ninjalog_uploader.GetJflag( + ['ninja','-j']), None) if __name__ == '__main__': - unittest.main() + unittest.main()