From 3f722d1f6358963ad61824e78b97335fb63b1868 Mon Sep 17 00:00:00 2001 From: "scottmg@chromium.org" Date: Tue, 2 Jun 2015 02:54:51 +0000 Subject: [PATCH] Rework/update toolchain script for vs2015 This is the other side of https://codereview.chromium.org/1163723003/ The changes here are to remove the use of 'vs2013_files' and 'win8sdk' (as those will be different numbers soon enough) but still maintain behaviour for the old "style" while in transition. Secondarily, to remove the dependence of these two scripts on 'toolchain2013.py' as most of the script is now unused. R=dpranke@chromium.org BUG=440500, 492774 Review URL: https://codereview.chromium.org/1165563003 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@295485 0039d316-1c4b-4281-b951-d872f2087c98 --- win_toolchain/get_toolchain_if_necessary.py | 82 ++++++++++-- win_toolchain/package_from_installed.py | 133 ++++++++++++++++---- 2 files changed, 184 insertions(+), 31 deletions(-) diff --git a/win_toolchain/get_toolchain_if_necessary.py b/win_toolchain/get_toolchain_if_necessary.py index 7bb3eba5a..cb4b5da2c 100755 --- a/win_toolchain/get_toolchain_if_necessary.py +++ b/win_toolchain/get_toolchain_if_necessary.py @@ -33,13 +33,20 @@ import os import shutil import subprocess import sys +import tempfile import time +import zipfile BASEDIR = os.path.dirname(os.path.abspath(__file__)) DEPOT_TOOLS_PATH = os.path.join(BASEDIR, '..') sys.path.append(DEPOT_TOOLS_PATH) -import download_from_google_storage +try: + import download_from_google_storage +except ImportError: + # Allow use of utility functions in this script from package_from_installed + # on bare VM that doesn't have a full depot_tools. + pass if sys.platform != 'cygwin': import ctypes.wintypes @@ -186,6 +193,32 @@ def DelayBeforeRemoving(target_dir): print +def DownloadUsingGsutil(filename): + """Downloads the given file from Google Storage chrome-wintoolchain bucket.""" + temp_dir = tempfile.mkdtemp() + assert os.path.basename(filename) == filename + target_path = os.path.join(temp_dir, filename) + gsutil = download_from_google_storage.Gsutil( + download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None) + code = gsutil.call('cp', 'gs://chrome-wintoolchain/' + filename, target_path) + if code != 0: + sys.exit('gsutil failed') + return temp_dir, target_path + + +def DoTreeMirror(target_dir, tree_sha1): + """In order to save temporary space on bots that do not have enough space to + download ISOs, unpack them, and copy to the target location, the whole tree + is uploaded as a zip to internal storage, and then mirrored here.""" + temp_dir, local_zip = DownloadUsingGsutil(tree_sha1 + '.zip') + sys.stdout.write('Extracting %s...\n' % local_zip) + sys.stdout.flush() + with zipfile.ZipFile(local_zip, 'r', zipfile.ZIP_DEFLATED, True) as zf: + zf.extractall(target_dir) + if temp_dir: + subprocess.check_call('rmdir /s/q "%s"' % temp_dir, shell=True) + + def main(): if not sys.platform.startswith(('cygwin', 'win32')): return 0 @@ -215,7 +248,13 @@ def main(): # the downloader script is. os.chdir(os.path.normpath(os.path.join(BASEDIR))) toolchain_dir = '.' - target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files')) + if os.environ.get('GYP_MSVS_VERSION') == '2015': + target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs_files')) + else: + target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files')) + abs_target_dir = os.path.abspath(target_dir) + + got_new_toolchain = False # If the current hash doesn't match what we want in the file, nuke and pave. # Typically this script is only run when the .sha1 one file is updated, but @@ -224,8 +263,8 @@ def main(): current_hash = CalculateHash(target_dir) if current_hash not in desired_hashes: should_use_gs = False - if (HaveSrcInternalAccess() or - LooksLikeGoogler() or + if (HaveSrcInternalAccess() or + LooksLikeGoogler() or CanAccessToolchainBucket()): should_use_gs = True if not CanAccessToolchainBucket(): @@ -246,12 +285,35 @@ def main(): stdin=nul, stdout=nul, stderr=nul) if os.path.isdir(target_dir): subprocess.check_call('rmdir /s/q "%s"' % target_dir, shell=True) - args = [sys.executable, - 'toolchain2013.py', - '--targetdir', target_dir, - '--sha1', desired_hashes[0], - '--use-gs'] - subprocess.check_call(args) + + DoTreeMirror(target_dir, desired_hashes[0]) + + got_new_toolchain = True + + win_sdk = os.path.join(abs_target_dir, 'win_sdk') + try: + with open(os.path.join(target_dir, 'VS_VERSION'), 'rb') as f: + vs_version = f.read().strip() + except IOError: + # Older toolchains didn't have the VS_VERSION file, and used 'win8sdk' + # instead of just 'win_sdk'. + vs_version = '2013' + win_sdk = os.path.join(abs_target_dir, 'win8sdk') + + data = { + 'path': abs_target_dir, + 'version': vs_version, + 'win_sdk': win_sdk, + 'wdk': os.path.join(abs_target_dir, 'wdk'), + 'runtime_dirs': [ + os.path.join(abs_target_dir, 'sys64'), + os.path.join(abs_target_dir, 'sys32'), + ], + } + with open(os.path.join(target_dir, '..', 'data.json'), 'w') as f: + json.dump(data, f) + + if got_new_toolchain: current_hash = CalculateHash(target_dir) if current_hash not in desired_hashes: print >> sys.stderr, ( diff --git a/win_toolchain/package_from_installed.py b/win_toolchain/package_from_installed.py index 22f52acbd..6fe606f18 100644 --- a/win_toolchain/package_from_installed.py +++ b/win_toolchain/package_from_installed.py @@ -28,15 +28,16 @@ import tempfile import zipfile import get_toolchain_if_necessary -import toolchain2013 # pylint: disable=F0401 + + +VS_VERSION = None def BuildFileList(): result = [] # Subset of VS corresponding roughly to VC. - vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 12.0' - for path in [ + paths = [ 'DIA SDK/bin', 'DIA SDK/idl', 'DIA SDK/include', @@ -47,15 +48,39 @@ def BuildFileList(): 'VC/include', 'VC/lib', 'VC/redist', - ('VC/redist/x86/Microsoft.VC120.CRT', 'sys32'), - ('VC/redist/x86/Microsoft.VC120.MFC', 'sys32'), - ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugCRT', 'sys32'), - ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugMFC', 'sys32'), - ('VC/redist/x64/Microsoft.VC120.CRT', 'sys64'), - ('VC/redist/x64/Microsoft.VC120.MFC', 'sys64'), - ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugCRT', 'sys64'), - ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugMFC', 'sys64'), - ]: + ] + + if VS_VERSION == '2013': + paths += [ + ('VC/redist/x86/Microsoft.VC120.CRT', 'sys32'), + ('VC/redist/x86/Microsoft.VC120.MFC', 'sys32'), + ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugCRT', 'sys32'), + ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugMFC', 'sys32'), + ('VC/redist/x64/Microsoft.VC120.CRT', 'sys64'), + ('VC/redist/x64/Microsoft.VC120.MFC', 'sys64'), + ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugCRT', 'sys64'), + ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugMFC', 'sys64'), + ] + elif VS_VERSION == '2015': + paths += [ + ('VC/redist/x86/Microsoft.VC140.CRT', 'sys32'), + ('VC/redist/x86/Microsoft.VC140.MFC', 'sys32'), + ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugCRT', 'sys32'), + ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugMFC', 'sys32'), + ('VC/redist/x64/Microsoft.VC140.CRT', 'sys64'), + ('VC/redist/x64/Microsoft.VC140.MFC', 'sys64'), + ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugCRT', 'sys64'), + ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugMFC', 'sys64'), + ] + else: + raise ValueError('VS_VERSION %s' % VS_VERSION) + + if VS_VERSION == '2013': + vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 12.0' + else: + vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 14.0' + + for path in paths: src = path[0] if isinstance(path, tuple) else path combined = os.path.join(vs_path, src) assert os.path.exists(combined) and os.path.isdir(combined) @@ -68,8 +93,8 @@ def BuildFileList(): else: assert final_from.startswith(vs_path) dest = final_from[len(vs_path) + 1:] - if dest.lower().endswith('\\xtree'): - # Patch for C4702 in xtree. http://crbug.com/346399. + if VS_VERSION == '2013' and dest.lower().endswith('\\xtree'): + # Patch for C4702 in xtree on VS2013. http://crbug.com/346399. (handle, patched) = tempfile.mkstemp() with open(final_from, 'rb') as unpatched_f: unpatched_contents = unpatched_f.read() @@ -85,21 +110,80 @@ def BuildFileList(): for root, _, files in os.walk(sdk_path): for f in files: combined = os.path.normpath(os.path.join(root, f)) - to = os.path.join('win8sdk', combined[len(sdk_path) + 1:]) + to = os.path.join('win_sdk', combined[len(sdk_path) + 1:]) result.append((combined, to)) + if VS_VERSION == '2015': + for ucrt_path in ( + (r'C:\Program Files (x86)\Windows Kits\10\Include', 'Include'), + (r'C:\Program Files (x86)\Windows Kits\10\Lib', 'Lib'), + (r'C:\Program Files (x86)\Windows Kits\10\Source', 'Source')): + src, target = ucrt_path + for root, _, files in os.walk(src): + for f in files: + combined = os.path.normpath(os.path.join(root, f)) + to = os.path.join('ucrt', target, combined[len(src) + 1:]) + result.append((combined, to)) + # Generically drop all arm stuff that we don't need. - return [(f, t) for f, t in result if 'arm\\' not in f.lower()] + return [(f, t) for f, t in result if 'arm\\' not in f.lower() and + 'arm64\\' not in f.lower()] + + +def GenerateSetEnvCmd(target_dir): + """Generate a batch file that gyp expects to exist to set up the compiler + environment. + + This is normally generated by a full install of the SDK, but we + do it here manually since we do not do a full install.""" + with open(os.path.join( + target_dir, r'win_sdk\bin\SetEnv.cmd'), 'w') as f: + f.write('@echo off\n' + ':: Generated by win_toolchain\\package_from_installed.py.\n' + # Common to x86 and x64 + 'set PATH=%~dp0..\\..\\Common7\\IDE;%PATH%\n' + 'set INCLUDE=%~dp0..\\..\\win_sdk\\Include\\um;' + '%~dp0..\\..\\win_sdk\\Include\\shared;' + '%~dp0..\\..\\win_sdk\\Include\\winrt;' + '%~dp0..\\..\\ucrt\\Include\\10.0.10056.0\\ucrt;' + '%~dp0..\\..\\VC\\include;' + '%~dp0..\\..\\VC\\atlmfc\\include\n' + 'if "%1"=="/x64" goto x64\n') + + # x86. Always use amd64_x86 cross, not x86 on x86. + f.write('set PATH=%~dp0..\\..\\win_sdk\\bin\\x86;' + '%~dp0..\\..\\VC\\bin\\amd64_x86;' + '%~dp0..\\..\\VC\\bin\\amd64;' # Needed for mspdb1x0.dll. + '%PATH%\n') + f.write('set LIB=%~dp0..\\..\\VC\\lib;' + '%~dp0..\\..\\win_sdk\\Lib\\winv6.3\\um\\x86;' + '%~dp0..\\..\\ucrt\\Lib\\10.0.10056.0\\ucrt\\x86;' + '%~dp0..\\..\\VC\\atlmfc\\lib\n' + 'goto :EOF\n') + + # x64. + f.write(':x64\n' + 'set PATH=%~dp0..\\..\\win_sdk\\bin\\x64;' + '%~dp0..\\..\\VC\\bin\\amd64;' + '%PATH%\n') + f.write('set LIB=%~dp0..\\..\\VC\\lib\\amd64;' + '%~dp0..\\..\\win_sdk\\Lib\\winv6.3\\um\\x64;' + '%~dp0..\\..\\ucrt\\Lib\\10.0.10056.0\\ucrt\\x64;' + '%~dp0..\\..\\VC\\atlmfc\\lib\\amd64\n') def AddEnvSetup(files): """We need to generate this file in the same way that the "from pieces" script does, so pull that in here.""" tempdir = tempfile.mkdtemp() - os.makedirs(os.path.join(tempdir, 'win8sdk', 'bin')) - toolchain2013.GenerateSetEnvCmd(tempdir, True) - files.append((os.path.join(tempdir, 'win8sdk', 'bin', 'SetEnv.cmd'), - 'win8sdk\\bin\\SetEnv.cmd')) + os.makedirs(os.path.join(tempdir, 'win_sdk', 'bin')) + GenerateSetEnvCmd(tempdir) + files.append((os.path.join(tempdir, 'win_sdk', 'bin', 'SetEnv.cmd'), + 'win_sdk\\bin\\SetEnv.cmd')) + vs_version_file = os.path.join(tempdir, 'VS_VERSION') + with open(vs_version_file, 'wb') as version: + print >>version, VS_VERSION + files.append((vs_version_file, 'VS_VERSION')) def RenameToSha1(output): @@ -109,7 +193,7 @@ def RenameToSha1(output): tempdir = tempfile.mkdtemp() old_dir = os.getcwd() os.chdir(tempdir) - rel_dir = 'vs2013_files' + rel_dir = 'vs_files' with zipfile.ZipFile( os.path.join(old_dir, output), 'r', zipfile.ZIP_DEFLATED, True) as zf: zf.extractall(rel_dir) @@ -123,6 +207,13 @@ def RenameToSha1(output): def main(): + if len(sys.argv) != 2 or sys.argv[1] not in ('2013', '2015'): + print 'Usage: package_from_installed.py 2013|2015' + return 1 + + global VS_VERSION + VS_VERSION = sys.argv[1] + print 'Building file list...' files = BuildFileList()