diff --git a/recipes/README.recipes.md b/recipes/README.recipes.md index 0e5069ece..ec7a384f8 100644 --- a/recipes/README.recipes.md +++ b/recipes/README.recipes.md @@ -13,6 +13,7 @@ * [gitiles](#recipe_modules-gitiles) * [gsutil](#recipe_modules-gsutil) * [infra_paths](#recipe_modules-infra_paths) + * [osx_sdk](#recipe_modules-osx_sdk) — The `osx_sdk` module provides safe functions to access a semi-hermetic XCode installation. * [presubmit](#recipe_modules-presubmit) * [tryserver](#recipe_modules-tryserver) * [windows_sdk](#recipe_modules-windows_sdk) — The `windows_sdk` module provides safe functions to access a hermetic Microsoft Visual Studio installation. @@ -32,6 +33,7 @@ * [gitiles:examples/full](#recipes-gitiles_examples_full) * [gsutil:examples/full](#recipes-gsutil_examples_full) * [infra_paths:examples/full](#recipes-infra_paths_examples_full) + * [osx_sdk:examples/full](#recipes-osx_sdk_examples_full) * [presubmit:examples/full](#recipes-presubmit_examples_full) * [tryserver:examples/full](#recipes-tryserver_examples_full) * [windows_sdk:examples/full](#recipes-windows_sdk_examples_full) @@ -651,6 +653,66 @@ It returns git_cache path if it is defined (Buildbot world), otherwise uses the more generic [CACHE]/git path (LUCI world). — **def [initialize](/recipes/recipe_modules/infra_paths/api.py#11)(self):** +### *recipe_modules* / [osx\_sdk](/recipes/recipe_modules/osx_sdk) + +[DEPS](/recipes/recipe_modules/osx_sdk/__init__.py#5): [recipe\_engine/cipd][recipe_engine/recipe_modules/cipd], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/step][recipe_engine/recipe_modules/step] + +The `osx_sdk` module provides safe functions to access a semi-hermetic +XCode installation. + +Available only to Google-run bots. + +#### **class [OSXSDKApi](/recipes/recipe_modules/osx_sdk/api.py#15)([RecipeApi][recipe_engine/wkt/RecipeApi]):** + +API for using OS X SDK distributed via CIPD. + +  **@contextmanager**
— **def [\_\_call\_\_](/recipes/recipe_modules/osx_sdk/api.py#25)(self, kind):** + +Sets up the XCode SDK environment. + +Is a no-op on non-mac platforms. + +This will deploy the helper tool and the XCode.app bundle at +`[START_DIR]/cache/osx_sdk`. + +To avoid machines rebuilding these on every run, set up a named cache in +your cr-buildbucket.cfg file like: + + caches: { + # Cache for mac_toolchain tool and XCode.app + name: "osx_sdk" + path: "osx_sdk" + } + +If you have builders which e.g. use a non-current SDK, you can give them +a uniqely named cache: + + caches: { + # Cache for N-1 version mac_toolchain tool and XCode.app + name: "osx_sdk_old" + path: "osx_sdk" + } + +Similarly, if you have mac and iOS builders you may want to distinguish the +cache name by adding '_ios' to it. However, if you're sharing the same bots +for both mac and iOS, consider having a single cache and just always +fetching the iOS version. This will lead to lower overall disk utilization +and should help to reduce cache thrashing. + +Usage: + with api.osx_sdk('mac'): + # sdk with mac build bits + + with api.osx_sdk('ios'): + # sdk with mac+iOS build bits + +Args: + kind ('mac'|'ios'): How the SDK should be configured. iOS includes the + base XCode distribution, as well as the iOS simulators (which can be + quite large). + +Raises: + StepFailure or InfraFailure. ### *recipe_modules* / [presubmit](/recipes/recipe_modules/presubmit) [DEPS](/recipes/recipe_modules/presubmit/__init__.py#1): [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/step][recipe_engine/recipe_modules/step] @@ -845,6 +907,11 @@ Move things around in a loop! [DEPS](/recipes/recipe_modules/infra_paths/examples/full.py#7): [infra\_paths](#recipe_modules-infra_paths), [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step] — **def [RunSteps](/recipes/recipe_modules/infra_paths/examples/full.py#16)(api):** +### *recipes* / [osx\_sdk:examples/full](/recipes/recipe_modules/osx_sdk/examples/full.py) + +[DEPS](/recipes/recipe_modules/osx_sdk/examples/full.py#5): [osx\_sdk](#recipe_modules-osx_sdk), [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step] + +— **def [RunSteps](/recipes/recipe_modules/osx_sdk/examples/full.py#13)(api):** ### *recipes* / [presubmit:examples/full](/recipes/recipe_modules/presubmit/examples/full.py) [DEPS](/recipes/recipe_modules/presubmit/examples/full.py#5): [presubmit](#recipe_modules-presubmit) diff --git a/recipes/recipe_modules/osx_sdk/__init__.py b/recipes/recipe_modules/osx_sdk/__init__.py new file mode 100644 index 000000000..ed790c34e --- /dev/null +++ b/recipes/recipe_modules/osx_sdk/__init__.py @@ -0,0 +1,40 @@ +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'recipe_engine/cipd', + 'recipe_engine/context', + 'recipe_engine/json', + 'recipe_engine/path', + 'recipe_engine/platform', + 'recipe_engine/step', +] + +from recipe_engine.recipe_api import Property +from recipe_engine.config import ConfigGroup, Single + +PROPERTIES = { + '$depot_tools/osx_sdk': Property( + help='Properties specifically for the infra osx_sdk module.', + param_name='sdk_properties', + kind=ConfigGroup( # pylint: disable=line-too-long + # XCode build version number. Internally maps to an XCode build id like + # '9c40b'. See + # + # https://chrome-infra-packages.appspot.com/p/infra_internal/ios/xcode/mac/+/ + # + # For an up to date list of the latest SDK builds. + sdk_version=Single(str), + + # The CIPD toolchain tool package and version + toolchain_pkg=Single(str), + toolchain_ver=Single(str), + ), default={ + 'sdk_version': '9c40b', + + 'toolchain_pkg': 'infra/tools/mac_toolchain/${platform}', + 'toolchain_ver': 'git_revision:796d2b92cff93fc2059623ce0a66284373ceea0a', + }, + ) +} diff --git a/recipes/recipe_modules/osx_sdk/api.py b/recipes/recipe_modules/osx_sdk/api.py new file mode 100644 index 000000000..4fc4c2968 --- /dev/null +++ b/recipes/recipe_modules/osx_sdk/api.py @@ -0,0 +1,104 @@ +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""The `osx_sdk` module provides safe functions to access a semi-hermetic +XCode installation. + +Available only to Google-run bots.""" + +from contextlib import contextmanager + +from recipe_engine import recipe_api + + +class OSXSDKApi(recipe_api.RecipeApi): + """API for using OS X SDK distributed via CIPD.""" + + def __init__(self, sdk_properties, *args, **kwargs): + super(OSXSDKApi, self).__init__(*args, **kwargs) + + self._sdk_version = sdk_properties['sdk_version'].lower() + self._tool_pkg = sdk_properties['toolchain_pkg'] + self._tool_ver = sdk_properties['toolchain_ver'] + + @contextmanager + def __call__(self, kind): + """Sets up the XCode SDK environment. + + Is a no-op on non-mac platforms. + + This will deploy the helper tool and the XCode.app bundle at + `[START_DIR]/cache/osx_sdk`. + + To avoid machines rebuilding these on every run, set up a named cache in + your cr-buildbucket.cfg file like: + + caches: { + # Cache for mac_toolchain tool and XCode.app + name: "osx_sdk" + path: "osx_sdk" + } + + If you have builders which e.g. use a non-current SDK, you can give them + a uniqely named cache: + + caches: { + # Cache for N-1 version mac_toolchain tool and XCode.app + name: "osx_sdk_old" + path: "osx_sdk" + } + + Similarly, if you have mac and iOS builders you may want to distinguish the + cache name by adding '_ios' to it. However, if you're sharing the same bots + for both mac and iOS, consider having a single cache and just always + fetching the iOS version. This will lead to lower overall disk utilization + and should help to reduce cache thrashing. + + Usage: + with api.osx_sdk('mac'): + # sdk with mac build bits + + with api.osx_sdk('ios'): + # sdk with mac+iOS build bits + + Args: + kind ('mac'|'ios'): How the SDK should be configured. iOS includes the + base XCode distribution, as well as the iOS simulators (which can be + quite large). + + Raises: + StepFailure or InfraFailure. + """ + assert kind in ('mac', 'ios'), 'Invalid kind %r' % (kind,) + if not self.m.platform.is_mac: + yield + return + + try: + with self.m.context(infra_steps=True): + app = self._ensure_sdk(kind) + self.m.step('select XCode', ['sudo', 'xcode-select', '--switch', app]) + yield + finally: + with self.m.context(infra_steps=True): + self.m.step('reset XCode', ['sudo', 'xcode-select', '--reset']) + + def _ensure_sdk(self, kind): + """Ensures the mac_toolchain tool and OS X SDK packages are installed. + + Returns Path to the installed sdk app bundle.""" + cache_dir = self.m.path['cache'].join('osx_sdk') + + ef = self.m.cipd.EnsureFile() + ef.add_package(self._tool_pkg, self._tool_ver) + self.m.cipd.ensure(cache_dir, ef) + + sdk_app = cache_dir.join('XCode.app') + self.m.step('install xcode', [ + cache_dir.join('mac_toolchain'), 'install', + '-kind', kind, + '-xcode-version', self._sdk_version, + '-output-dir', sdk_app, + ]) + return sdk_app diff --git a/recipes/recipe_modules/osx_sdk/examples/full.expected/linux.json b/recipes/recipe_modules/osx_sdk/examples/full.expected/linux.json new file mode 100644 index 000000000..51d94309f --- /dev/null +++ b/recipes/recipe_modules/osx_sdk/examples/full.expected/linux.json @@ -0,0 +1,23 @@ +[ + { + "cmd": [ + "gn", + "gen", + "out/Release" + ], + "name": "gn" + }, + { + "cmd": [ + "ninja", + "-C", + "out/Release" + ], + "name": "ninja" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/osx_sdk/examples/full.expected/mac.json b/recipes/recipe_modules/osx_sdk/examples/full.expected/mac.json new file mode 100644 index 000000000..1defa9b94 --- /dev/null +++ b/recipes/recipe_modules/osx_sdk/examples/full.expected/mac.json @@ -0,0 +1,83 @@ +[ + { + "cmd": [ + "cipd", + "ensure", + "-root", + "[CACHE]/osx_sdk", + "-ensure-file", + "infra/tools/mac_toolchain/${platform} git_revision:796d2b92cff93fc2059623ce0a66284373ceea0a", + "-json-output", + "/path/to/tmp/json" + ], + "infra_step": true, + "name": "ensure_installed", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:796\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/mac_toolchain/${platform}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[CACHE]/osx_sdk/mac_toolchain", + "install", + "-kind", + "mac", + "-xcode-version", + "9c40b", + "-output-dir", + "[CACHE]/osx_sdk/XCode.app" + ], + "infra_step": true, + "name": "install xcode" + }, + { + "cmd": [ + "sudo", + "xcode-select", + "--switch", + "[CACHE]/osx_sdk/XCode.app" + ], + "infra_step": true, + "name": "select XCode" + }, + { + "cmd": [ + "gn", + "gen", + "out/Release" + ], + "name": "gn" + }, + { + "cmd": [ + "ninja", + "-C", + "out/Release" + ], + "name": "ninja" + }, + { + "cmd": [ + "sudo", + "xcode-select", + "--reset" + ], + "infra_step": true, + "name": "reset XCode" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/osx_sdk/examples/full.expected/win.json b/recipes/recipe_modules/osx_sdk/examples/full.expected/win.json new file mode 100644 index 000000000..51d94309f --- /dev/null +++ b/recipes/recipe_modules/osx_sdk/examples/full.expected/win.json @@ -0,0 +1,23 @@ +[ + { + "cmd": [ + "gn", + "gen", + "out/Release" + ], + "name": "gn" + }, + { + "cmd": [ + "ninja", + "-C", + "out/Release" + ], + "name": "ninja" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/osx_sdk/examples/full.py b/recipes/recipe_modules/osx_sdk/examples/full.py new file mode 100644 index 000000000..f28a5e428 --- /dev/null +++ b/recipes/recipe_modules/osx_sdk/examples/full.py @@ -0,0 +1,23 @@ +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'osx_sdk', + 'recipe_engine/platform', + 'recipe_engine/properties', + 'recipe_engine/step', +] + + +def RunSteps(api): + with api.osx_sdk('mac'): + api.step('gn', ['gn', 'gen', 'out/Release']) + api.step('ninja', ['ninja', '-C', 'out/Release']) + + +def GenTests(api): + for platform in ('linux', 'mac', 'win'): + yield (api.test(platform) + + api.platform.name(platform) + + api.properties.generic(buildername='test_builder'))