From f7bb4cf0477b41a54eead49faded11167cc57fb6 Mon Sep 17 00:00:00 2001 From: "luqui@chromium.org" Date: Fri, 4 Dec 2015 23:30:38 +0000 Subject: [PATCH] Set up depot_tools as a recipe package, and add depot_tools recipe module. This is a good prototypical example of how to do it :-). TBR for OWNERS change for recipes.cfg. BUG=564920 R=iannucci@chromium.org,martiniss@chromium.org TBR=phajdan.jr@chromium.org Review URL: https://codereview.chromium.org/1494103004 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@297851 0039d316-1c4b-4281-b951-d872f2087c98 --- .gitignore | 3 + PRESUBMIT.py | 2 + infra/config/OWNERS | 2 + infra/config/recipes.cfg | 9 ++ recipe_modules/depot_tools/__init__.py | 1 + recipe_modules/depot_tools/api.py | 12 ++ .../depot_tools/example.expected/basic.json | 17 +++ recipe_modules/depot_tools/example.py | 14 ++ recipes.py | 136 ++++++++++++++++++ tests/recipes_test.py | 18 +++ 10 files changed, 214 insertions(+) create mode 100644 infra/config/recipes.cfg create mode 100644 recipe_modules/depot_tools/__init__.py create mode 100644 recipe_modules/depot_tools/api.py create mode 100644 recipe_modules/depot_tools/example.expected/basic.json create mode 100644 recipe_modules/depot_tools/example.py create mode 100755 recipes.py create mode 100755 tests/recipes_test.py diff --git a/.gitignore b/.gitignore index 3e149d743..30411ae96 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,6 @@ # Ignore intermediate isolate files *.isolated *.isolated.state + +# Ignore recipe working directory. +/.recipe_deps diff --git a/PRESUBMIT.py b/PRESUBMIT.py index 2877144f3..c1cc5de20 100644 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -16,10 +16,12 @@ def CommonChecks(input_api, output_api, tests_to_black_list): results = [] results.extend(input_api.canned_checks.CheckOwners(input_api, output_api)) black_list = list(input_api.DEFAULT_BLACK_LIST) + [ + r'^\.recipe_deps[\/\\].*', r'^cpplint\.py$', r'^cpplint_chromium\.py$', r'^external_bin[\/\\].+', r'^python[0-9]*_bin[\/\\].+', + r'^recipes\.py$', r'^site-packages-py[0-9]\.[0-9][\/\\].+', r'^svn_bin[\/\\].+', r'^testing_support[\/\\]_rietveld[\/\\].+'] diff --git a/infra/config/OWNERS b/infra/config/OWNERS index 2aa95ea9e..6ed730495 100644 --- a/infra/config/OWNERS +++ b/infra/config/OWNERS @@ -3,3 +3,5 @@ akuegel@chromium.org phajdan.jr@chromium.org sergiyb@chromium.org tandrii@chromium.org + +per-file recipes.cfg=luqui@chromium.org diff --git a/infra/config/recipes.cfg b/infra/config/recipes.cfg new file mode 100644 index 000000000..2a2dcf68f --- /dev/null +++ b/infra/config/recipes.cfg @@ -0,0 +1,9 @@ +api_version: 1 +project_id: "depot_tools" +recipes_path: "" +deps { + project_id: "recipe_engine" + url: "https://chromium.googlesource.com/external/github.com/luci/recipes-py.git" + branch: "master" + revision: "cbbcfad66fa7239561436e495963bf9a32cecdf2" +} diff --git a/recipe_modules/depot_tools/__init__.py b/recipe_modules/depot_tools/__init__.py new file mode 100644 index 000000000..c92b78dd2 --- /dev/null +++ b/recipe_modules/depot_tools/__init__.py @@ -0,0 +1 @@ +DEPS = [] diff --git a/recipe_modules/depot_tools/api.py b/recipe_modules/depot_tools/api.py new file mode 100644 index 000000000..3685330e9 --- /dev/null +++ b/recipe_modules/depot_tools/api.py @@ -0,0 +1,12 @@ +# Copyright (c) 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Entry point for interacting with depot_tools from recipes.""" + +from recipe_engine import recipe_api + +class DepotToolsApi(recipe_api.RecipeApi): + @property + def gclient_py(self): + return self.package_resource('gclient.py') diff --git a/recipe_modules/depot_tools/example.expected/basic.json b/recipe_modules/depot_tools/example.expected/basic.json new file mode 100644 index 000000000..6d2435cc6 --- /dev/null +++ b/recipe_modules/depot_tools/example.expected/basic.json @@ -0,0 +1,17 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE[depot_tools]/gclient.py", + "--help" + ], + "cwd": "[SLAVE_BUILD]", + "name": "gclient help" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/recipe_modules/depot_tools/example.py b/recipe_modules/depot_tools/example.py new file mode 100644 index 000000000..4afa8e8ee --- /dev/null +++ b/recipe_modules/depot_tools/example.py @@ -0,0 +1,14 @@ +# Copyright (c) 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'recipe_engine/python', + 'depot_tools', +] + +def RunSteps(api): + api.python('gclient help', api.depot_tools.gclient_py, ['--help']) + +def GenTests(api): + yield api.test('basic') diff --git a/recipes.py b/recipes.py new file mode 100755 index 000000000..e03508472 --- /dev/null +++ b/recipes.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python + +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Bootstrap script to clone and forward to the recipe engine tool.""" + +import ast +import logging +import os +import random +import re +import subprocess +import sys +import time +import traceback + +BOOTSTRAP_VERSION = 1 +# The root of the repository relative to the directory of this file. +REPO_ROOT = '' +# The path of the recipes.cfg file relative to the root of the repository. +RECIPES_CFG = os.path.join('infra', 'config', 'recipes.cfg') + + +def parse_protobuf(fh): + """Parse the protobuf text format just well enough to understand recipes.cfg. + + We don't use the protobuf library because we want to be as self-contained + as possible in this bootstrap, so it can be simply vendored into a client + repo. + + We assume all fields are repeated since we don't have a proto spec to work + with. + + Args: + fh: a filehandle containing the text format protobuf. + Returns: + A recursive dictionary of lists. + """ + def parse_atom(text): + if text == 'true': return True + if text == 'false': return False + return ast.literal_eval(text) + + ret = {} + for line in fh: + line = line.strip() + m = re.match(r'(\w+)\s*:\s*(.*)', line) + if m: + ret.setdefault(m.group(1), []).append(parse_atom(m.group(2))) + continue + + m = re.match(r'(\w+)\s*{', line) + if m: + subparse = parse_protobuf(fh) + ret.setdefault(m.group(1), []).append(subparse) + continue + + if line == '}': return ret + if line == '': continue + + raise Exception('Could not understand line: <%s>' % line) + + return ret + + +def get_unique(things): + if len(things) == 1: + return things[0] + elif len(things) == 0: + raise ValueError("Expected to get one thing, but dinna get none.") + else: + logging.warn('Expected to get one thing, but got a bunch: %s\n%s' % + (things, traceback.format_stack())) + return things[0] + + +def main(): + if sys.platform.startswith(('win', 'cygwin')): + git = 'git.bat' + else: + git = 'git' + + # Find the repository and config file to operate on. + repo_root = os.path.abspath( + os.path.join(os.path.dirname(__file__), REPO_ROOT)) + recipes_cfg_path = os.path.join(repo_root, RECIPES_CFG) + + with open(recipes_cfg_path, 'rU') as fh: + protobuf = parse_protobuf(fh) + + engine_buf = get_unique([ + b for b in protobuf['deps'] if b.get('project_id') == ['recipe_engine'] ]) + engine_url = get_unique(engine_buf['url']) + engine_revision = get_unique(engine_buf['revision']) + engine_subpath = (get_unique(engine_buf.get('path_override', [''])) + .replace('/', os.path.sep)) + + recipes_path = os.path.join(repo_root, + get_unique(protobuf['recipes_path']).replace('/', os.path.sep)) + deps_path = os.path.join(recipes_path, '.recipe_deps') + engine_path = os.path.join(deps_path, 'recipe_engine') + + # Ensure that we have the recipe engine cloned. + def ensure_engine(): + if not os.path.exists(deps_path): + os.makedirs(deps_path) + if not os.path.exists(engine_path): + subprocess.check_call([git, 'clone', engine_url, engine_path]) + + needs_fetch = subprocess.call( + [git, 'rev-parse', '--verify', '%s^{commit}' % engine_revision], + cwd=engine_path, stdout=open(os.devnull, 'w')) + if needs_fetch: + subprocess.check_call([git, 'fetch'], cwd=engine_path) + subprocess.check_call( + [git, 'checkout', '--quiet', engine_revision], cwd=engine_path) + + try: + ensure_engine() + except subprocess.CalledProcessError as e: + if e.returncode == 128: # Thrown when git gets a lock error. + time.sleep(random.uniform(2,5)) + ensure_engine() + else: + raise + + args = ['--package', recipes_cfg_path, + '--bootstrap-script', __file__] + sys.argv[1:] + return subprocess.call([ + sys.executable, '-u', + os.path.join(engine_path, engine_subpath, 'recipes.py')] + args) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/recipes_test.py b/tests/recipes_test.py new file mode 100755 index 000000000..99b149d46 --- /dev/null +++ b/tests/recipes_test.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +# Copyright (c) 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Runs simulation tests and lint on the recipes.""" + +import os +import subprocess + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +def recipes_py(*args): + subprocess.check_call([os.path.join(ROOT_DIR, 'recipes.py')] + list(args)) + +recipes_py('simulation_test', '--threshold=92') +recipes_py('lint')