From 5484b866dc936fd1e148385fa1575b0369eec820 Mon Sep 17 00:00:00 2001 From: Andrii Shyshkalov Date: Tue, 3 Jul 2018 03:33:55 +0000 Subject: [PATCH] bot_update: use buildbucket's gitiles_commit as repo/revision input. Today, whenever LUCI scheduler sets gitiles_commit, a revision property is also always set, thus this CL will have no effect on today's production. The goal of CL is to pave the way to avoid this duplication. For starters, one could start real writing tests with + api.buildbucket.ci_build(..., revision='sha1') without the need to duplicate the same revision as a property. All recipes which WAI with such input are thus ready to work w/o 'revision' property. After that, we'll be able to stop the duplication in LUCI scheduler. R=hinoka Recipe-Nontrivial-Roll: infra Recipe-Nontrivial-Roll: build Change-Id: I1773b1c3d2a98a9dbd1cfdd9ac2b4fb99745ee99 Reviewed-on: https://chromium-review.googlesource.com/1123713 Commit-Queue: Andrii Shyshkalov Reviewed-by: Ryan Tseng --- recipes/README.recipes.md | 49 ++++++++++------- recipes/recipe_modules/bot_update/__init__.py | 1 + recipes/recipe_modules/bot_update/api.py | 6 ++ .../examples/buildbucket.expected/ci.json | 55 +++++++++++++++++++ .../bot_update/examples/buildbucket.py | 24 ++++++++ 5 files changed, 115 insertions(+), 20 deletions(-) create mode 100644 recipes/recipe_modules/bot_update/examples/buildbucket.expected/ci.json create mode 100644 recipes/recipe_modules/bot_update/examples/buildbucket.py diff --git a/recipes/README.recipes.md b/recipes/README.recipes.md index 00135116b..70b7f08d8 100644 --- a/recipes/README.recipes.md +++ b/recipes/README.recipes.md @@ -17,6 +17,7 @@ * [tryserver](#recipe_modules-tryserver) **[Recipes](#Recipes)** + * [bot_update:examples/buildbucket](#recipes-bot_update_examples_buildbucket) * [bot_update:examples/full](#recipes-bot_update_examples_full) * [cipd:examples/full](#recipes-cipd_examples_full) * [cipd:examples/platform_suffix](#recipes-cipd_examples_platform_suffix) @@ -36,24 +37,24 @@ ### *recipe_modules* / [bot\_update](/recipes/recipe_modules/bot_update) -[DEPS](/recipes/recipe_modules/bot_update/__init__.py#1): [depot\_tools](#recipe_modules-depot_tools), [gclient](#recipe_modules-gclient), [gerrit](#recipe_modules-gerrit), [tryserver](#recipe_modules-tryserver), [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/runtime][recipe_engine/recipe_modules/runtime], [recipe\_engine/source\_manifest][recipe_engine/recipe_modules/source_manifest], [recipe\_engine/step][recipe_engine/recipe_modules/step] +[DEPS](/recipes/recipe_modules/bot_update/__init__.py#1): [depot\_tools](#recipe_modules-depot_tools), [gclient](#recipe_modules-gclient), [gerrit](#recipe_modules-gerrit), [tryserver](#recipe_modules-tryserver), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/runtime][recipe_engine/recipe_modules/runtime], [recipe\_engine/source\_manifest][recipe_engine/recipe_modules/source_manifest], [recipe\_engine/step][recipe_engine/recipe_modules/step] Recipe module to ensure a checkout is consistent on a bot. #### **class [BotUpdateApi](/recipes/recipe_modules/bot_update/api.py#10)([RecipeApi][recipe_engine/wkt/RecipeApi]):** -— **def [\_\_call\_\_](/recipes/recipe_modules/bot_update/api.py#31)(self, name, cmd, \*\*kwargs):** +— **def [\_\_call\_\_](/recipes/recipe_modules/bot_update/api.py#37)(self, name, cmd, \*\*kwargs):** Wrapper for easy calling of bot_update. -— **def [apply\_gerrit\_ref](/recipes/recipe_modules/bot_update/api.py#46)(self, root, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, gerrit_repo=None, gerrit_ref=None, step_name='apply_gerrit', \*\*kwargs):** +— **def [apply\_gerrit\_ref](/recipes/recipe_modules/bot_update/api.py#52)(self, root, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, gerrit_repo=None, gerrit_ref=None, step_name='apply_gerrit', \*\*kwargs):** -— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#408)(self, bot_update_step):** +— **def [deapply\_patch](/recipes/recipe_modules/bot_update/api.py#414)(self, bot_update_step):** Deapplies a patch, taking care of DEPS and solution revisions properly. -— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#68)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, manifest_name=None, \*\*kwargs):** +— **def [ensure\_checkout](/recipes/recipe_modules/bot_update/api.py#74)(self, gclient_config=None, suffix=None, patch=True, update_presentation=True, patch_root=None, with_branch_heads=False, with_tags=False, refs=None, patch_oauth2=None, oauth2_json=None, use_site_config_creds=None, clobber=False, root_solution_revision=None, rietveld=None, issue=None, patchset=None, gerrit_no_reset=False, gerrit_no_rebase_patch_ref=False, disable_syntax_validation=False, manifest_name=None, \*\*kwargs):** Args: gclient_config: The gclient configuration to use when running bot_update. @@ -64,7 +65,7 @@ Args: manifest_name: The name of the manifest to upload to LogDog. This must be unique for the whole build. -— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#385)(self, project_name, gclient_config=None):** +— **def [get\_project\_revision\_properties](/recipes/recipe_modules/bot_update/api.py#391)(self, project_name, gclient_config=None):** Returns all property names used for storing the checked-out revision of a given project. @@ -78,7 +79,9 @@ Args: Returns (list of str): All properties that'll hold the checked-out revision of the given project. An empty list if no such properties exist. -  **@property**
— **def [last\_returned\_properties](/recipes/recipe_modules/bot_update/api.py#40)(self):** +— **def [initialize](/recipes/recipe_modules/bot_update/api.py#31)(self):** + +  **@property**
— **def [last\_returned\_properties](/recipes/recipe_modules/bot_update/api.py#46)(self):** ### *recipe_modules* / [cipd](/recipes/recipe_modules/cipd) [DEPS](/recipes/recipe_modules/cipd/__init__.py#1): [infra\_paths](#recipe_modules-infra_paths), [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/step][recipe_engine/recipe_modules/step] @@ -742,6 +745,11 @@ This means we started running actual tests (not prerequisite steps like checkout or compile), and some of these tests have failed. ## Recipes +### *recipes* / [bot\_update:examples/buildbucket](/recipes/recipe_modules/bot_update/examples/buildbucket.py) + +[DEPS](/recipes/recipe_modules/bot_update/examples/buildbucket.py#5): [bot\_update](#recipe_modules-bot_update), [gclient](#recipe_modules-gclient), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket] + +— **def [RunSteps](/recipes/recipe_modules/bot_update/examples/buildbucket.py#11)(api):** ### *recipes* / [bot\_update:examples/full](/recipes/recipe_modules/bot_update/examples/full.py) [DEPS](/recipes/recipe_modules/bot_update/examples/full.py#5): [bot\_update](#recipe_modules-bot_update), [gclient](#recipe_modules-gclient), [gerrit](#recipe_modules-gerrit), [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/runtime][recipe_engine/recipe_modules/runtime] @@ -820,16 +828,17 @@ Move things around in a loop! — **def [RunSteps](/recipes/recipe_modules/tryserver/examples/full.py#17)(api):** -[recipe_engine/recipe_modules/context]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/782ed0377a0a23f393a3a873194954825972b961/README.recipes.md#recipe_modules-context -[recipe_engine/recipe_modules/file]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/782ed0377a0a23f393a3a873194954825972b961/README.recipes.md#recipe_modules-file -[recipe_engine/recipe_modules/json]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/782ed0377a0a23f393a3a873194954825972b961/README.recipes.md#recipe_modules-json -[recipe_engine/recipe_modules/path]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/782ed0377a0a23f393a3a873194954825972b961/README.recipes.md#recipe_modules-path -[recipe_engine/recipe_modules/platform]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/782ed0377a0a23f393a3a873194954825972b961/README.recipes.md#recipe_modules-platform -[recipe_engine/recipe_modules/properties]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/782ed0377a0a23f393a3a873194954825972b961/README.recipes.md#recipe_modules-properties -[recipe_engine/recipe_modules/python]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/782ed0377a0a23f393a3a873194954825972b961/README.recipes.md#recipe_modules-python -[recipe_engine/recipe_modules/raw_io]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/782ed0377a0a23f393a3a873194954825972b961/README.recipes.md#recipe_modules-raw_io -[recipe_engine/recipe_modules/runtime]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/782ed0377a0a23f393a3a873194954825972b961/README.recipes.md#recipe_modules-runtime -[recipe_engine/recipe_modules/source_manifest]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/782ed0377a0a23f393a3a873194954825972b961/README.recipes.md#recipe_modules-source_manifest -[recipe_engine/recipe_modules/step]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/782ed0377a0a23f393a3a873194954825972b961/README.recipes.md#recipe_modules-step -[recipe_engine/recipe_modules/url]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/782ed0377a0a23f393a3a873194954825972b961/README.recipes.md#recipe_modules-url -[recipe_engine/wkt/RecipeApi]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/782ed0377a0a23f393a3a873194954825972b961/recipe_engine/recipe_api.py#1006 +[recipe_engine/recipe_modules/buildbucket]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/README.recipes.md#recipe_modules-buildbucket +[recipe_engine/recipe_modules/context]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/README.recipes.md#recipe_modules-context +[recipe_engine/recipe_modules/file]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/README.recipes.md#recipe_modules-file +[recipe_engine/recipe_modules/json]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/README.recipes.md#recipe_modules-json +[recipe_engine/recipe_modules/path]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/README.recipes.md#recipe_modules-path +[recipe_engine/recipe_modules/platform]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/README.recipes.md#recipe_modules-platform +[recipe_engine/recipe_modules/properties]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/README.recipes.md#recipe_modules-properties +[recipe_engine/recipe_modules/python]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/README.recipes.md#recipe_modules-python +[recipe_engine/recipe_modules/raw_io]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/README.recipes.md#recipe_modules-raw_io +[recipe_engine/recipe_modules/runtime]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/README.recipes.md#recipe_modules-runtime +[recipe_engine/recipe_modules/source_manifest]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/README.recipes.md#recipe_modules-source_manifest +[recipe_engine/recipe_modules/step]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/README.recipes.md#recipe_modules-step +[recipe_engine/recipe_modules/url]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/README.recipes.md#recipe_modules-url +[recipe_engine/wkt/RecipeApi]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/57600791c9ce0381bffff5b8d5ffef235c661a09/recipe_engine/recipe_api.py#1006 diff --git a/recipes/recipe_modules/bot_update/__init__.py b/recipes/recipe_modules/bot_update/__init__.py index 802047e55..805b6e5c6 100644 --- a/recipes/recipe_modules/bot_update/__init__.py +++ b/recipes/recipe_modules/bot_update/__init__.py @@ -2,6 +2,7 @@ DEPS = [ 'depot_tools', 'gclient', 'gerrit', + 'recipe_engine/buildbucket', 'recipe_engine/context', 'recipe_engine/json', 'recipe_engine/path', diff --git a/recipes/recipe_modules/bot_update/api.py b/recipes/recipe_modules/bot_update/api.py index 9bd2a8166..7038eb660 100644 --- a/recipes/recipe_modules/bot_update/api.py +++ b/recipes/recipe_modules/bot_update/api.py @@ -28,6 +28,12 @@ class BotUpdateApi(recipe_api.RecipeApi): self._last_returned_properties = {} super(BotUpdateApi, self).__init__(*args, **kwargs) + def initialize(self): + gm = self.m.buildbucket.build_input.gitiles_commit + if self._revision is None and self._repository is None and gm: + self._revision = gm.id + self._repository = gm.host + '/' + gm.project + def __call__(self, name, cmd, **kwargs): """Wrapper for easy calling of bot_update.""" assert isinstance(cmd, (list, tuple)) diff --git a/recipes/recipe_modules/bot_update/examples/buildbucket.expected/ci.json b/recipes/recipe_modules/bot_update/examples/buildbucket.expected/ci.json new file mode 100644 index 000000000..840f40586 --- /dev/null +++ b/recipes/recipe_modules/bot_update/examples/buildbucket.expected/ci.json @@ -0,0 +1,55 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec-path", + "cache_dir = '[GIT_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'v8', 'url': 'https://chromium.googlesource.com/v8/v8'}]", + "--patch_root", + "v8", + "--revision_mapping_file", + "{}", + "--git-cache-dir", + "[GIT_CACHE]", + "--cleanup-dir", + "[CLEANUP]/bot_update", + "--output_json", + "/path/to/tmp/json", + "--revision", + "v8@2d72510e447ab60a9728aeea2362d8be2cbd7789" + ], + "env_prefixes": { + "PATH": [ + "RECIPE_PACKAGE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"v8\": \"2d72510e447ab60a9728aeea2362d8be2cbd7789\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {}, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"v8\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {}, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"v8\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"directories\": {}, @@@", + "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipes/recipe_modules/bot_update/examples/buildbucket.py b/recipes/recipe_modules/bot_update/examples/buildbucket.py new file mode 100644 index 000000000..0c3184ea8 --- /dev/null +++ b/recipes/recipe_modules/bot_update/examples/buildbucket.py @@ -0,0 +1,24 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'bot_update', + 'gclient', + 'recipe_engine/buildbucket', +] + +def RunSteps(api): + cfg = api.gclient.make_config(CACHE_DIR='[GIT_CACHE]') + soln = cfg.solutions.add() + soln.name = 'v8' + soln.url = 'https://chromium.googlesource.com/v8/v8' + api.gclient.c = cfg + api.bot_update.ensure_checkout() + + +def GenTests(api): + yield api.test('ci') + api.buildbucket.ci_build( + 'v8', 'ci', 'builder', + git_repo='https://chromium.googlesource.com/v8/v8', + revision='2d72510e447ab60a9728aeea2362d8be2cbd7789')