From dd1bc7c0b0ff9c36f53b3adcac5ebf5bfff0bcf8 Mon Sep 17 00:00:00 2001 From: Dan Jacques Date: Sun, 16 Jul 2017 22:16:18 -0700 Subject: [PATCH] [cipd] Generate other platform suffixes. Update the CIPD recipe module to generate platform suffixes for arbitrary OS, architecture, and bitness. BUG=None TEST=expectations Change-Id: Ic713adc403dc135895a1cfaaa6ba1546742125d9 Reviewed-on: https://chromium-review.googlesource.com/573412 Commit-Queue: Daniel Jacques Reviewed-by: Robbie Iannucci --- recipes/README.recipes.md | 38 +++++++----- recipes/recipe_modules/cipd/api.py | 42 ++++++++++--- .../platform_suffix.expected/junk arch.json | 7 +++ .../platform_suffix.expected/junk bits.json | 7 +++ .../linux_arm_32.json | 14 +++++ .../linux_arm_64.json | 14 +++++ .../linux_intel_32.json | 14 +++++ .../linux_intel_64.json | 14 +++++ .../linux_mips_64.json | 14 +++++ .../mac_intel_64.json | 14 +++++ .../win_intel_32.json | 14 +++++ .../win_intel_64.json | 14 +++++ .../cipd/examples/platform_suffix.py | 59 +++++++++++++++++++ 13 files changed, 244 insertions(+), 21 deletions(-) create mode 100644 recipes/recipe_modules/cipd/examples/platform_suffix.expected/junk arch.json create mode 100644 recipes/recipe_modules/cipd/examples/platform_suffix.expected/junk bits.json create mode 100644 recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_arm_32.json create mode 100644 recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_arm_64.json create mode 100644 recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_intel_32.json create mode 100644 recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_intel_64.json create mode 100644 recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_mips_64.json create mode 100644 recipes/recipe_modules/cipd/examples/platform_suffix.expected/mac_intel_64.json create mode 100644 recipes/recipe_modules/cipd/examples/platform_suffix.expected/win_intel_32.json create mode 100644 recipes/recipe_modules/cipd/examples/platform_suffix.expected/win_intel_64.json create mode 100644 recipes/recipe_modules/cipd/examples/platform_suffix.py diff --git a/recipes/README.recipes.md b/recipes/README.recipes.md index 784476ef2..853cbbb56 100644 --- a/recipes/README.recipes.md +++ b/recipes/README.recipes.md @@ -20,6 +20,7 @@ **[Recipes](#Recipes)** * [bot_update:examples/full](#recipes-bot_update_examples_full) * [cipd:examples/full](#recipes-cipd_examples_full) + * [cipd:examples/platform_suffix](#recipes-cipd_examples_platform_suffix) * [depot_tools:examples/full](#recipes-depot_tools_examples_full) * [gclient:examples/full](#recipes-gclient_examples_full) * [gclient:tests/patch_project](#recipes-gclient_tests_patch_project) @@ -102,7 +103,7 @@ installed somewhere in $PATH. This will be true if you use depot_tools, or if your recipe is running inside of chrome-infrastructure's systems (buildbot, swarming). -— **def [build](/recipes/recipe_modules/cipd/api.py#193)(self, input_dir, output_package, package_name, install_mode=None):** +— **def [build](/recipes/recipe_modules/cipd/api.py#221)(self, input_dir, output_package, package_name, install_mode=None):** Builds, but does not upload, a cipd package from a directory. @@ -115,7 +116,7 @@ Args: should use when installing this package. If None, defaults to the platform default ('copy' on windows, 'symlink' on everything else). -— **def [create\_from\_pkg](/recipes/recipe_modules/cipd/api.py#289)(self, pkg_def, refs=None, tags=None):** +— **def [create\_from\_pkg](/recipes/recipe_modules/cipd/api.py#317)(self, pkg_def, refs=None, tags=None):** Builds and uploads a package based on a PackageDefinition object. @@ -133,7 +134,7 @@ Returns the JSON 'result' section, e.g.: { "instance_id": "433bfdf86c0bb82d1eee2d1a0473d3709c25d2c4" } -— **def [create\_from\_yaml](/recipes/recipe_modules/cipd/api.py#269)(self, pkg_def, refs=None, tags=None):** +— **def [create\_from\_yaml](/recipes/recipe_modules/cipd/api.py#297)(self, pkg_def, refs=None, tags=None):** Builds and uploads a package based on on-disk YAML package definition file. @@ -151,11 +152,11 @@ Returns the JSON 'result' section, e.g.: { "instance_id": "433bfdf86c0bb82d1eee2d1a0473d3709c25d2c4" } -  **@property**
— **def [default\_bot\_service\_account\_credentials](/recipes/recipe_modules/cipd/api.py#169)(self):** +  **@property**
— **def [default\_bot\_service\_account\_credentials](/recipes/recipe_modules/cipd/api.py#185)(self):** -— **def [describe](/recipes/recipe_modules/cipd/api.py#396)(self, package_name, version, test_data_refs=None, test_data_tags=None):** +— **def [describe](/recipes/recipe_modules/cipd/api.py#424)(self, package_name, version, test_data_refs=None, test_data_tags=None):** -— **def [ensure](/recipes/recipe_modules/cipd/api.py#311)(self, root, packages):** +— **def [ensure](/recipes/recipe_modules/cipd/api.py#339)(self, root, packages):** Ensures that packages are installed in a given root dir. @@ -166,11 +167,11 @@ packages must be a mapping from package name to its version, where If installing a package requires credentials, call ``set_service_account_credentials`` before calling this function. -  **@property**
— **def [executable](/recipes/recipe_modules/cipd/api.py#165)(self):** +  **@property**
— **def [executable](/recipes/recipe_modules/cipd/api.py#181)(self):** -— **def [initialize](/recipes/recipe_modules/cipd/api.py#159)(self):** +— **def [initialize](/recipes/recipe_modules/cipd/api.py#175)(self):** -— **def [platform\_suffix](/recipes/recipe_modules/cipd/api.py#178)(self):** +— **def [platform\_suffix](/recipes/recipe_modules/cipd/api.py#194)(self, name=None, arch=None, bits=None):** Use to get full package name that is platform indepdent. @@ -178,15 +179,19 @@ Example: >>> 'my/package/%s' % api.cipd.platform_suffix() 'my/package/linux-amd64' -— **def [register](/recipes/recipe_modules/cipd/api.py#221)(self, package_name, package_path, refs=None, tags=None):** +Optional platform bits and architecture may be supplied to generate CIPD +suffixes for other platforms. If any are omitted, the current platform +parameters will be used. -— **def [search](/recipes/recipe_modules/cipd/api.py#378)(self, package_name, tag):** +— **def [register](/recipes/recipe_modules/cipd/api.py#249)(self, package_name, package_path, refs=None, tags=None):** -— **def [set\_ref](/recipes/recipe_modules/cipd/api.py#358)(self, package_name, version, refs):** +— **def [search](/recipes/recipe_modules/cipd/api.py#406)(self, package_name, tag):** -— **def [set\_service\_account\_credentials](/recipes/recipe_modules/cipd/api.py#162)(self, path):** +— **def [set\_ref](/recipes/recipe_modules/cipd/api.py#386)(self, package_name, version, refs):** -— **def [set\_tag](/recipes/recipe_modules/cipd/api.py#338)(self, package_name, version, tags):** +— **def [set\_service\_account\_credentials](/recipes/recipe_modules/cipd/api.py#178)(self, path):** + +— **def [set\_tag](/recipes/recipe_modules/cipd/api.py#366)(self, package_name, version, tags):** ### *recipe_modules* / [depot\_tools](/recipes/recipe_modules/depot_tools) [DEPS](/recipes/recipe_modules/depot_tools/__init__.py#1): [recipe\_engine/platform][recipe_engine/recipe_modules/platform] @@ -759,6 +764,11 @@ like checkout or compile), and some of these tests have failed. [DEPS](/recipes/recipe_modules/cipd/examples/full.py#8): [cipd](#recipe_modules-cipd), [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step] — **def [RunSteps](/recipes/recipe_modules/cipd/examples/full.py#27)(api, use_pkg, pkg_files, pkg_dirs, ver_files, install_mode):** +### *recipes* / [cipd:examples/platform\_suffix](/recipes/recipe_modules/cipd/examples/platform_suffix.py) + +[DEPS](/recipes/recipe_modules/cipd/examples/platform_suffix.py#8): [cipd](#recipe_modules-cipd), [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step] + +— **def [RunSteps](/recipes/recipe_modules/cipd/examples/platform_suffix.py#22)(api, arch_override, bits_override, expect_error):** ### *recipes* / [depot\_tools:examples/full](/recipes/recipe_modules/depot_tools/examples/full.py) [DEPS](/recipes/recipe_modules/depot_tools/examples/full.py#5): [depot\_tools](#recipe_modules-depot_tools), [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/step][recipe_engine/recipe_modules/step] diff --git a/recipes/recipe_modules/cipd/api.py b/recipes/recipe_modules/cipd/api.py index 4aefd0d7e..681e1846e 100644 --- a/recipes/recipe_modules/cipd/api.py +++ b/recipes/recipe_modules/cipd/api.py @@ -155,6 +155,22 @@ class CIPDApi(recipe_api.RecipeApi): """ PackageDefinition = PackageDefinition + # Map for architecture mapping. First key is platform module arch, second is + # platform module bits. + _SUFFIX_ARCH_MAP = { + 'intel': { + 32: '386', + 64: 'amd64', + }, + 'mips': { + 64: 'mips64', + }, + 'arm': { + 32: 'armv6', + 64: 'arm64', + }, + } + # pylint: disable=attribute-defined-outside-init def initialize(self): self._cipd_credentials = None @@ -175,20 +191,32 @@ class CIPDApi(recipe_api.RecipeApi): else: return '/creds/service_accounts/service-account-cipd-builder.json' - def platform_suffix(self): + def platform_suffix(self, name=None, arch=None, bits=None): """Use to get full package name that is platform indepdent. Example: >>> 'my/package/%s' % api.cipd.platform_suffix() 'my/package/linux-amd64' + + Optional platform bits and architecture may be supplied to generate CIPD + suffixes for other platforms. If any are omitted, the current platform + parameters will be used. """ + name = name or self.m.platform.name + arch = arch or self.m.platform.arch + bits = bits or self.m.platform.bits + + arch_map = self._SUFFIX_ARCH_MAP.get(arch) + if not arch_map: + raise KeyError('No architecture mapped for %r.' % (arch,)) + arch_str = arch_map.get(bits) + if not arch_str: + raise KeyError('No architecture mapped for %r with %r bits.' % ( + arch, bits)) + return '%s-%s' % ( - self.m.platform.name.replace('win', 'windows'), - { - 32: '386', - 64: 'amd64', - }[self.m.platform.bits], - ) + name.replace('win', 'windows'), + arch_str) def build(self, input_dir, output_package, package_name, install_mode=None): """Builds, but does not upload, a cipd package from a directory. diff --git a/recipes/recipe_modules/cipd/examples/platform_suffix.expected/junk arch.json b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/junk arch.json new file mode 100644 index 000000000..7726ecb24 --- /dev/null +++ b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/junk arch.json @@ -0,0 +1,7 @@ +[ + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/cipd/examples/platform_suffix.expected/junk bits.json b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/junk bits.json new file mode 100644 index 000000000..7726ecb24 --- /dev/null +++ b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/junk bits.json @@ -0,0 +1,7 @@ +[ + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_arm_32.json b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_arm_32.json new file mode 100644 index 000000000..8d830cf55 --- /dev/null +++ b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_arm_32.json @@ -0,0 +1,14 @@ +[ + { + "cmd": [ + "echo", + "linux-armv6" + ], + "name": "platform_suffix" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_arm_64.json b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_arm_64.json new file mode 100644 index 000000000..71d364f10 --- /dev/null +++ b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_arm_64.json @@ -0,0 +1,14 @@ +[ + { + "cmd": [ + "echo", + "linux-arm64" + ], + "name": "platform_suffix" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_intel_32.json b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_intel_32.json new file mode 100644 index 000000000..f369bf797 --- /dev/null +++ b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_intel_32.json @@ -0,0 +1,14 @@ +[ + { + "cmd": [ + "echo", + "linux-386" + ], + "name": "platform_suffix" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_intel_64.json b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_intel_64.json new file mode 100644 index 000000000..d02a4d5b4 --- /dev/null +++ b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_intel_64.json @@ -0,0 +1,14 @@ +[ + { + "cmd": [ + "echo", + "linux-amd64" + ], + "name": "platform_suffix" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_mips_64.json b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_mips_64.json new file mode 100644 index 000000000..6b11ba11e --- /dev/null +++ b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/linux_mips_64.json @@ -0,0 +1,14 @@ +[ + { + "cmd": [ + "echo", + "linux-mips64" + ], + "name": "platform_suffix" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/cipd/examples/platform_suffix.expected/mac_intel_64.json b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/mac_intel_64.json new file mode 100644 index 000000000..1c61977d2 --- /dev/null +++ b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/mac_intel_64.json @@ -0,0 +1,14 @@ +[ + { + "cmd": [ + "echo", + "mac-amd64" + ], + "name": "platform_suffix" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/cipd/examples/platform_suffix.expected/win_intel_32.json b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/win_intel_32.json new file mode 100644 index 000000000..0722c0612 --- /dev/null +++ b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/win_intel_32.json @@ -0,0 +1,14 @@ +[ + { + "cmd": [ + "echo", + "windows-386" + ], + "name": "platform_suffix" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/cipd/examples/platform_suffix.expected/win_intel_64.json b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/win_intel_64.json new file mode 100644 index 000000000..6d3480d2b --- /dev/null +++ b/recipes/recipe_modules/cipd/examples/platform_suffix.expected/win_intel_64.json @@ -0,0 +1,14 @@ +[ + { + "cmd": [ + "echo", + "windows-amd64" + ], + "name": "platform_suffix" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/cipd/examples/platform_suffix.py b/recipes/recipe_modules/cipd/examples/platform_suffix.py new file mode 100644 index 000000000..d45d07c99 --- /dev/null +++ b/recipes/recipe_modules/cipd/examples/platform_suffix.py @@ -0,0 +1,59 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from recipe_engine.config import List, Single, ConfigList, ConfigGroup +from recipe_engine.recipe_api import Property + +DEPS = [ + 'recipe_engine/path', + 'recipe_engine/platform', + 'recipe_engine/properties', + 'recipe_engine/step', + 'cipd', +] + +PROPERTIES = { + 'arch_override': Property(kind=str, default=None), + 'bits_override': Property(kind=int, default=None), + 'expect_error': Property(kind=bool, default=False), +} + +def RunSteps(api, arch_override, bits_override, expect_error): + was_error = False + try: + api.step('platform_suffix', ['echo', api.cipd.platform_suffix( + arch=arch_override, + bits=bits_override, + )]) + except KeyError: + was_error = True + + assert was_error == expect_error + + +def GenTests(api): + for name, arch, bits in ( + ('linux', 'intel', 32), + ('linux', 'intel', 64), + ('linux', 'mips', 64), + ('linux', 'arm', 32), + ('linux', 'arm', 64), + ('mac', 'intel', 64), + ('win', 'intel', 32), + ('win', 'intel', 64)): + test = ( + api.test('%s_%s_%d' % (name, arch, bits)) + + api.platform(name, bits) + ) + if arch != 'intel': + test += api.properties(arch_override=arch) + yield test + + yield ( + api.test('junk arch') + + api.properties(arch_override='pants', expect_error=True)) + + yield ( + api.test('junk bits') + + api.properties(bits_override=42, expect_error=True))