From b469623afd79cae89597120b8e03692c744ee44b Mon Sep 17 00:00:00 2001 From: "szager@chromium.org" Date: Wed, 16 Oct 2013 19:45:35 +0000 Subject: [PATCH] Add git/gerrit-on-borg utilities. gob_util.py is a general-purpose library for communicating with the gerrit-on-borg service. testing_support/gerrit_test_case.py is a unittest framework for testing code that interacts with gerrit. R=vadimsh@chromium.org, cmp@chromium.org BUG= Review URL: https://codereview.chromium.org/26399002 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@228965 0039d316-1c4b-4281-b951-d872f2087c98 --- gerrit_util.py | 394 +++++++++++++++++++++++ testing_support/gerrit-init.sh | 2 +- testing_support/gerrit_test_case.py | 482 ++++++++++++++++++++++++++++ 3 files changed, 877 insertions(+), 1 deletion(-) create mode 100755 gerrit_util.py create mode 100644 testing_support/gerrit_test_case.py diff --git a/gerrit_util.py b/gerrit_util.py new file mode 100755 index 0000000000..7c44f03107 --- /dev/null +++ b/gerrit_util.py @@ -0,0 +1,394 @@ +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Utilities for requesting information for a gerrit server via https. + +https://gerrit-review.googlesource.com/Documentation/rest-api.html +""" + +import base64 +import httplib +import json +import logging +import netrc +import os +import time +import urllib +from cStringIO import StringIO + +try: + NETRC = netrc.netrc() +except (IOError, netrc.NetrcParseError): + NETRC = netrc.netrc(os.devnull) +LOGGER = logging.getLogger() +TRY_LIMIT = 5 + +# Controls the transport protocol used to communicate with gerrit. +# This is parameterized primarily to enable GerritTestCase. +GERRIT_PROTOCOL = 'https' + + +class GerritError(Exception): + """Exception class for errors commuicating with the gerrit-on-borg service.""" + def __init__(self, http_status, *args, **kwargs): + super(GerritError, self).__init__(*args, **kwargs) + self.http_status = http_status + self.message = '(%d) %s' % (self.http_status, self.message) + + +def _QueryString(param_dict, first_param=None): + """Encodes query parameters in the key:val[+key:val...] format specified here: + + https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes + """ + q = [urllib.quote(first_param)] if first_param else [] + q.extend(['%s:%s' % (key, val) for key, val in param_dict.iteritems()]) + return '+'.join(q) + + +def GetConnectionClass(protocol=None): + if protocol is None: + protocol = GERRIT_PROTOCOL + if protocol == 'https': + return httplib.HTTPSConnection + elif protocol == 'http': + return httplib.HTTPConnection + else: + raise RuntimeError( + "Don't know how to work with protocol '%s'" % protocol) + + +def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None): + """Opens an https connection to a gerrit service, and sends a request.""" + headers = headers or {} + bare_host = host.partition(':')[0] + auth = NETRC.authenticators(bare_host) + if auth: + headers.setdefault('Authorization', 'Basic %s' % ( + base64.b64encode('%s:%s' % (auth[0], auth[2])))) + else: + LOGGER.debug('No authorization found') + if body: + body = json.JSONEncoder().encode(body) + headers.setdefault('Content-Type', 'application/json') + if LOGGER.isEnabledFor(logging.DEBUG): + LOGGER.debug('%s %s://%s/a/%s' % (reqtype, GERRIT_PROTOCOL, host, path)) + for key, val in headers.iteritems(): + if key == 'Authorization': + val = 'HIDDEN' + LOGGER.debug('%s: %s' % (key, val)) + if body: + LOGGER.debug(body) + conn = GetConnectionClass()(host) + conn.req_host = host + conn.req_params = { + 'url': '/a/%s' % path, + 'method': reqtype, + 'headers': headers, + 'body': body, + } + conn.request(**conn.req_params) + return conn + + +def ReadHttpResponse(conn, expect_status=200, ignore_404=True): + """Reads an http response from a connection into a string buffer. + + Args: + conn: An HTTPSConnection or HTTPConnection created by CreateHttpConn, above. + expect_status: Success is indicated by this status in the response. + ignore_404: For many requests, gerrit-on-borg will return 404 if the request + doesn't match the database contents. In most such cases, we + want the API to return None rather than raise an Exception. + Returns: A string buffer containing the connection's reply. + """ + + sleep_time = 0.5 + for idx in range(TRY_LIMIT): + response = conn.getresponse() + # If response.status < 500 then the result is final; break retry loop. + if response.status < 500: + break + # A status >=500 is assumed to be a possible transient error; retry. + http_version = 'HTTP/%s' % ('1.1' if response.version == 11 else '1.0') + msg = ( + 'A transient error occured while querying %s:\n' + '%s %s %s\n' + '%s %d %s' % ( + conn.host, conn.req_params['method'], conn.req_params['url'], + http_version, http_version, response.status, response.reason)) + if TRY_LIMIT - idx > 1: + msg += '\n... will retry %d more times.' % (TRY_LIMIT - idx - 1) + time.sleep(sleep_time) + sleep_time = sleep_time * 2 + req_host = conn.req_host + req_params = conn.req_params + conn = GetConnectionClass()(req_host) + conn.req_host = req_host + conn.req_params = req_params + conn.request(**req_params) + LOGGER.warn(msg) + if ignore_404 and response.status == 404: + return StringIO() + if response.status != expect_status: + raise GerritError(response.status, response.reason) + return StringIO(response.read()) + + +def ReadHttpJsonResponse(conn, expect_status=200, ignore_404=True): + """Parses an https response as json.""" + fh = ReadHttpResponse( + conn, expect_status=expect_status, ignore_404=ignore_404) + # The first line of the response should always be: )]}' + s = fh.readline() + if s and s.rstrip() != ")]}'": + raise GerritError(200, 'Unexpected json output: %s' % s) + s = fh.read() + if not s: + return None + return json.loads(s) + + +def QueryChanges(host, param_dict, first_param=None, limit=None, o_params=None, + sortkey=None): + """ + Queries a gerrit-on-borg server for changes matching query terms. + + Args: + param_dict: A dictionary of search parameters, as documented here: + http://gerrit-documentation.googlecode.com/svn/Documentation/2.6/user-search.html + first_param: A change identifier + limit: Maximum number of results to return. + o_params: A list of additional output specifiers, as documented here: + https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes + Returns: + A list of json-decoded query results. + """ + # Note that no attempt is made to escape special characters; YMMV. + if not param_dict and not first_param: + raise RuntimeError('QueryChanges requires search parameters') + path = 'changes/?q=%s' % _QueryString(param_dict, first_param) + if sortkey: + path = '%s&N=%s' % (path, sortkey) + if limit: + path = '%s&n=%d' % (path, limit) + if o_params: + path = '%s&%s' % (path, '&'.join(['o=%s' % p for p in o_params])) + # Don't ignore 404; a query should always return a list, even if it's empty. + return ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=False) + + +def MultiQueryChanges(host, param_dict, change_list, limit=None, o_params=None, + sortkey=None): + """Initiate a query composed of multiple sets of query parameters.""" + if not change_list: + raise RuntimeError( + "MultiQueryChanges requires a list of change numbers/id's") + q = ['q=%s' % '+OR+'.join([urllib.quote(str(x)) for x in change_list])] + if param_dict: + q.append(_QueryString(param_dict)) + if limit: + q.append('n=%d' % limit) + if sortkey: + q.append('N=%s' % sortkey) + if o_params: + q.extend(['o=%s' % p for p in o_params]) + path = 'changes/?%s' % '&'.join(q) + try: + result = ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=False) + except GerritError as e: + msg = '%s:\n%s' % (e.message, path) + raise GerritError(e.http_status, msg) + return result + + +def GetGerritFetchUrl(host): + """Given a gerrit host name returns URL of a gerrit instance to fetch from.""" + return '%s://%s/' % (GERRIT_PROTOCOL, host) + + +def GetChangePageUrl(host, change_number): + """Given a gerrit host name and change number, return change page url.""" + return '%s://%s/#/c/%d/' % (GERRIT_PROTOCOL, host, change_number) + + +def GetChangeUrl(host, change): + """Given a gerrit host name and change id, return an url for the change.""" + return '%s://%s/a/changes/%s' % (GERRIT_PROTOCOL, host, change) + + +def GetChange(host, change): + """Query a gerrit server for information about a single change.""" + path = 'changes/%s' % change + return ReadHttpJsonResponse(CreateHttpConn(host, path)) + + +def GetChangeDetail(host, change, o_params=None): + """Query a gerrit server for extended information about a single change.""" + path = 'changes/%s/detail' % change + if o_params: + path += '?%s' % '&'.join(['o=%s' % p for p in o_params]) + return ReadHttpJsonResponse(CreateHttpConn(host, path)) + + +def GetChangeCurrentRevision(host, change): + """Get information about the latest revision for a given change.""" + return QueryChanges(host, {}, change, o_params=('CURRENT_REVISION',)) + + +def GetChangeRevisions(host, change): + """Get information about all revisions associated with a change.""" + return QueryChanges(host, {}, change, o_params=('ALL_REVISIONS',)) + + +def GetChangeReview(host, change, revision=None): + """Get the current review information for a change.""" + if not revision: + jmsg = GetChangeRevisions(host, change) + if not jmsg: + return None + elif len(jmsg) > 1: + raise GerritError(200, 'Multiple changes found for ChangeId %s.' % change) + revision = jmsg[0]['current_revision'] + path = 'changes/%s/revisions/%s/review' + return ReadHttpJsonResponse(CreateHttpConn(host, path)) + + +def AbandonChange(host, change, msg=''): + """Abandon a gerrit change.""" + path = 'changes/%s/abandon' % change + body = {'message': msg} if msg else None + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + return ReadHttpJsonResponse(conn, ignore_404=False) + + +def RestoreChange(host, change, msg=''): + """Restore a previously abandoned change.""" + path = 'changes/%s/restore' % change + body = {'message': msg} if msg else None + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + return ReadHttpJsonResponse(conn, ignore_404=False) + + +def SubmitChange(host, change, wait_for_merge=True): + """Submits a gerrit change via Gerrit.""" + path = 'changes/%s/submit' % change + body = {'wait_for_merge': wait_for_merge} + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + return ReadHttpJsonResponse(conn, ignore_404=False) + + +def GetReviewers(host, change): + """Get information about all reviewers attached to a change.""" + path = 'changes/%s/reviewers' % change + return ReadHttpJsonResponse(CreateHttpConn(host, path)) + + +def GetReview(host, change, revision): + """Get review information about a specific revision of a change.""" + path = 'changes/%s/revisions/%s/review' % (change, revision) + return ReadHttpJsonResponse(CreateHttpConn(host, path)) + + +def AddReviewers(host, change, add=None): + """Add reviewers to a change.""" + if not add: + return + if isinstance(add, basestring): + add = (add,) + path = 'changes/%s/reviewers' % change + for r in add: + body = {'reviewer': r} + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + jmsg = ReadHttpJsonResponse(conn, ignore_404=False) + return jmsg + + +def RemoveReviewers(host, change, remove=None): + """Remove reveiewers from a change.""" + if not remove: + return + if isinstance(remove, basestring): + remove = (remove,) + for r in remove: + path = 'changes/%s/reviewers/%s' % (change, r) + conn = CreateHttpConn(host, path, reqtype='DELETE') + try: + ReadHttpResponse(conn, ignore_404=False) + except GerritError as e: + # On success, gerrit returns status 204; anything else is an error. + if e.http_status != 204: + raise + else: + raise GerritError( + 'Unexpectedly received a 200 http status while deleting reviewer "%s"' + ' from change %s' % (r, change)) + + +def SetReview(host, change, msg=None, labels=None, notify=None): + """Set labels and/or add a message to a code review.""" + if not msg and not labels: + return + path = 'changes/%s/revisions/current/review' % change + body = {} + if msg: + body['message'] = msg + if labels: + body['labels'] = labels + if notify: + body['notify'] = notify + conn = CreateHttpConn(host, path, reqtype='POST', body=body) + response = ReadHttpJsonResponse(conn) + if labels: + for key, val in labels.iteritems(): + if ('labels' not in response or key not in response['labels'] or + int(response['labels'][key] != int(val))): + raise GerritError(200, 'Unable to set "%s" label on change %s.' % ( + key, change)) + + +def ResetReviewLabels(host, change, label, value='0', message=None, + notify=None): + """Reset the value of a given label for all reviewers on a change.""" + # This is tricky, because we want to work on the "current revision", but + # there's always the risk that "current revision" will change in between + # API calls. So, we check "current revision" at the beginning and end; if + # it has changed, raise an exception. + jmsg = GetChangeCurrentRevision(host, change) + if not jmsg: + raise GerritError( + 200, 'Could not get review information for change "%s"' % change) + value = str(value) + revision = jmsg[0]['current_revision'] + path = 'changes/%s/revisions/%s/review' % (change, revision) + message = message or ( + '%s label set to %s programmatically.' % (label, value)) + jmsg = GetReview(host, change, revision) + if not jmsg: + raise GerritError(200, 'Could not get review information for revison %s ' + 'of change %s' % (revision, change)) + for review in jmsg.get('labels', {}).get(label, {}).get('all', []): + if str(review.get('value', value)) != value: + body = { + 'message': message, + 'labels': {label: value}, + 'on_behalf_of': review['_account_id'], + } + if notify: + body['notify'] = notify + conn = CreateHttpConn( + host, path, reqtype='POST', body=body) + response = ReadHttpJsonResponse(conn) + if str(response['labels'][label]) != value: + username = review.get('email', jmsg.get('name', '')) + raise GerritError(200, 'Unable to set %s label for user "%s"' + ' on change %s.' % (label, username, change)) + jmsg = GetChangeCurrentRevision(host, change) + if not jmsg: + raise GerritError( + 200, 'Could not get review information for change "%s"' % change) + elif jmsg[0]['current_revision'] != revision: + raise GerritError(200, 'While resetting labels on change "%s", ' + 'a new patchset was uploaded.' % change) diff --git a/testing_support/gerrit-init.sh b/testing_support/gerrit-init.sh index 3169ed45bd..5d10045497 100755 --- a/testing_support/gerrit-init.sh +++ b/testing_support/gerrit-init.sh @@ -147,7 +147,7 @@ fi mkdir -p "${rundir}/etc" cat < "${rundir}/etc/gerrit.config" [auth] - type = http + type = DEVELOPMENT_BECOME_ANY_ACCOUNT gitBasicAuth = true [gerrit] canonicalWebUrl = http://$(hostname):${http_port}/ diff --git a/testing_support/gerrit_test_case.py b/testing_support/gerrit_test_case.py new file mode 100644 index 0000000000..12b2943840 --- /dev/null +++ b/testing_support/gerrit_test_case.py @@ -0,0 +1,482 @@ +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Test framework for code that interacts with gerrit. + +class GerritTestCase +-------------------------------------------------------------------------------- +This class initializes and runs an a gerrit instance on localhost. To use the +framework, define a class that extends GerritTestCase, and then do standard +python unittest development as described here: + +http://docs.python.org/2.7/library/unittest.html#basic-example + +When your test code runs, the framework will: + +- Download the latest stable(-ish) binary release of the gerrit code. +- Start up a live gerrit instance running in a temp directory on the localhost. +- Set up a single gerrit user account with admin priveleges. +- Supply credential helpers for interacting with the gerrit instance via http + or ssh. + +Refer to depot_tools/testing_support/gerrit-init.sh for details about how the +gerrit instance is set up, and refer to helper methods defined below +(createProject, cloneProject, uploadChange, etc.) for ways to interact with the +gerrit instance from your test methods. + + +class RepoTestCase +-------------------------------------------------------------------------------- +This class extends GerritTestCase, and creates a set of project repositories +and a manifest repository that can be used in conjunction with the 'repo' tool. + +Each test method will initialize and sync a brand-new repo working directory. +The 'repo' command may be invoked in a subprocess as part of your tests. + +One gotcha: 'repo upload' will always attempt to use the ssh interface to talk +to gerrit. +""" + +import collections +from cStringIO import StringIO +import errno +import httplib +import imp +import json +import netrc +import os +import re +import shutil +import signal +import socket +import stat +import subprocess +import sys +import tempfile +import unittest +import urllib + +import gerrit_util + + +DEPOT_TOOLS_DIR = os.path.normpath(os.path.join( + os.path.realpath(__file__), '..', '..')) + + +# When debugging test code, it's sometimes helpful to leave the test gerrit +# instance intact and running after the test code exits. Setting TEARDOWN +# to False will do that. +TEARDOWN = True + +class GerritTestCase(unittest.TestCase): + """Test class for tests that interact with a gerrit server. + + The class setup creates and launches a stand-alone gerrit instance running on + localhost, for test methods to interact with. Class teardown stops and + deletes the gerrit instance. + + Note that there is a single gerrit instance for ALL test methods in a + GerritTestCase sub-class. + """ + + COMMIT_RE = re.compile(r'^commit ([0-9a-fA-F]{40})$') + CHANGEID_RE = re.compile('^\s+Change-Id:\s*(\S+)$') + DEVNULL = open(os.devnull, 'w') + TEST_USERNAME = 'test-username' + TEST_EMAIL = 'test-username@test.org' + + GerritInstance = collections.namedtuple('GerritInstance', [ + 'credential_file', + 'gerrit_dir', + 'gerrit_exe', + 'gerrit_host', + 'gerrit_pid', + 'gerrit_url', + 'git_dir', + 'git_host', + 'git_url', + 'http_port', + 'netrc_file', + 'ssh_ident', + 'ssh_port', + ]) + + @classmethod + def check_call(cls, *args, **kwargs): + kwargs.setdefault('stdout', cls.DEVNULL) + kwargs.setdefault('stderr', cls.DEVNULL) + subprocess.check_call(*args, **kwargs) + + @classmethod + def check_output(cls, *args, **kwargs): + kwargs.setdefault('stderr', cls.DEVNULL) + return subprocess.check_output(*args, **kwargs) + + @classmethod + def _create_gerrit_instance(cls, gerrit_dir): + gerrit_init_script = os.path.join( + DEPOT_TOOLS_DIR, 'testing_support', 'gerrit-init.sh') + http_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + http_sock.bind(('', 0)) + http_port = str(http_sock.getsockname()[1]) + ssh_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ssh_sock.bind(('', 0)) + ssh_port = str(ssh_sock.getsockname()[1]) + + # NOTE: this is not completely safe. These port numbers could be + # re-assigned by the OS between the calls to socket.close() and gerrit + # starting up. The only safe way to do this would be to pass file + # descriptors down to the gerrit process, which is not even remotely + # supported. Alas. + http_sock.close() + ssh_sock.close() + cls.check_call(['bash', gerrit_init_script, '--http-port', http_port, + '--ssh-port', ssh_port, gerrit_dir]) + + gerrit_exe = os.path.join(gerrit_dir, 'bin', 'gerrit.sh') + cls.check_call(['bash', gerrit_exe, 'start']) + with open(os.path.join(gerrit_dir, 'logs', 'gerrit.pid')) as fh: + gerrit_pid = int(fh.read().rstrip()) + + return cls.GerritInstance( + credential_file=os.path.join(gerrit_dir, 'tmp', '.git-credentials'), + gerrit_dir=gerrit_dir, + gerrit_exe=gerrit_exe, + gerrit_host='localhost:%s' % http_port, + gerrit_pid=gerrit_pid, + gerrit_url='http://localhost:%s' % http_port, + git_dir=os.path.join(gerrit_dir, 'git'), + git_host='%s/git' % gerrit_dir, + git_url='file://%s/git' % gerrit_dir, + http_port=http_port, + netrc_file=os.path.join(gerrit_dir, 'tmp', '.netrc'), + ssh_ident=os.path.join(gerrit_dir, 'tmp', 'id_rsa'), + ssh_port=ssh_port,) + + @classmethod + def setUpClass(cls): + """Sets up the gerrit instances in a class-specific temp dir.""" + # Create gerrit instance. + gerrit_dir = tempfile.mkdtemp() + os.chmod(gerrit_dir, 0700) + gi = cls.gerrit_instance = cls._create_gerrit_instance(gerrit_dir) + + # Set netrc file for http authentication. + cls.gerrit_util_netrc_orig = gerrit_util.NETRC + gerrit_util.NETRC = netrc.netrc(gi.netrc_file) + + # gerrit_util.py defaults to using https, but for testing, it's much + # simpler to use http connections. + cls.gerrit_util_protocol_orig = gerrit_util.GERRIT_PROTOCOL + gerrit_util.GERRIT_PROTOCOL = 'http' + + # Because we communicate with the test server via http, rather than https, + # libcurl won't add authentication headers to raw git requests unless the + # gerrit server returns 401. That works for pushes, but for read operations + # (like git-ls-remote), gerrit will simply omit any ref that requires + # authentication. By default gerrit doesn't permit anonymous read access to + # refs/meta/config. Override that behavior so tests can access + # refs/meta/config if necessary. + clone_path = os.path.join(gi.gerrit_dir, 'tmp', 'All-Projects') + cls._CloneProject('All-Projects', clone_path) + project_config = os.path.join(clone_path, 'project.config') + cls.check_call(['git', 'config', '--file', project_config, '--add', + 'access.refs/meta/config.read', 'group Anonymous Users']) + cls.check_call(['git', 'add', project_config], cwd=clone_path) + cls.check_call( + ['git', 'commit', '-m', 'Anonyous read for refs/meta/config'], + cwd=clone_path) + cls.check_call(['git', 'push', 'origin', 'HEAD:refs/meta/config'], + cwd=clone_path) + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + os.chmod(self.tempdir, 0700) + + def tearDown(self): + if TEARDOWN: + shutil.rmtree(self.tempdir) + + @classmethod + def createProject(cls, name, description='Test project', owners=None, + submit_type='CHERRY_PICK'): + """Create a project on the test gerrit server.""" + if owners is None: + owners = ['Administrators'] + body = { + 'description': description, + 'submit_type': submit_type, + 'owners': owners, + } + path = 'projects/%s' % urllib.quote(name, '') + conn = gerrit_util.CreateHttpConn( + cls.gerrit_instance.gerrit_host, path, reqtype='PUT', body=body) + jmsg = gerrit_util.ReadHttpJsonResponse(conn, expect_status=201) + assert jmsg['name'] == name + + @classmethod + def _post_clone_bookkeeping(cls, clone_path): + config_path = os.path.join(clone_path, '.git', 'config') + cls.check_call( + ['git', 'config', '--file', config_path, 'user.email', cls.TEST_EMAIL]) + cls.check_call( + ['git', 'config', '--file', config_path, 'credential.helper', + 'store --file=%s' % cls.gerrit_instance.credential_file]) + + @classmethod + def _CloneProject(cls, name, path): + """Clone a project from the test gerrit server.""" + gi = cls.gerrit_instance + parent_dir = os.path.dirname(path) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) + url = '/'.join((gi.gerrit_url, name)) + cls.check_call(['git', 'clone', url, path]) + cls._post_clone_bookkeeping(path) + # Install commit-msg hook to add Change-Id lines. + hook_path = os.path.join(path, '.git', 'hooks', 'commit-msg') + cls.check_call(['curl', '-o', hook_path, + '/'.join((gi.gerrit_url, 'tools/hooks/commit-msg'))]) + os.chmod(hook_path, stat.S_IRWXU) + return path + + def cloneProject(self, name, path=None): + """Clone a project from the test gerrit server.""" + if path is None: + path = os.path.basename(name) + if path.endswith('.git'): + path = path[:-4] + path = os.path.join(self.tempdir, path) + return self._CloneProject(name, path) + + @classmethod + def _CreateCommit(cls, clone_path, fn=None, msg=None, text=None): + """Create a commit in the given git checkout.""" + if not fn: + fn = 'test-file.txt' + if not msg: + msg = 'Test Message' + if not text: + text = 'Another day, another dollar.' + fpath = os.path.join(clone_path, fn) + with open(fpath, 'a') as fh: + fh.write('%s\n' % text) + cls.check_call(['git', 'add', fn], cwd=clone_path) + cls.check_call(['git', 'commit', '-m', msg], cwd=clone_path) + return cls._GetCommit(clone_path) + + def createCommit(self, clone_path, fn=None, msg=None, text=None): + """Create a commit in the given git checkout.""" + clone_path = os.path.join(self.tempdir, clone_path) + return self._CreateCommit(clone_path, fn, msg, text) + + @classmethod + def _GetCommit(cls, clone_path, ref='HEAD'): + """Get the sha1 and change-id for a ref in the git checkout.""" + log_proc = cls.check_output(['git', 'log', '-n', '1', ref], cwd=clone_path) + sha1 = None + change_id = None + for line in log_proc.splitlines(): + match = cls.COMMIT_RE.match(line) + if match: + sha1 = match.group(1) + continue + match = cls.CHANGEID_RE.match(line) + if match: + change_id = match.group(1) + continue + assert sha1 + assert change_id + return (sha1, change_id) + + def getCommit(self, clone_path, ref='HEAD'): + """Get the sha1 and change-id for a ref in the git checkout.""" + clone_path = os.path.join(self.tempdir, clone_path) + return self._GetCommit(clone_path, ref) + + @classmethod + def _UploadChange(cls, clone_path, branch='master', remote='origin'): + """Create a gerrit CL from the HEAD of a git checkout.""" + cls.check_call( + ['git', 'push', remote, 'HEAD:refs/for/%s' % branch], cwd=clone_path) + + def uploadChange(self, clone_path, branch='master', remote='origin'): + """Create a gerrit CL from the HEAD of a git checkout.""" + clone_path = os.path.join(self.tempdir, clone_path) + self._UploadChange(clone_path, branch, remote) + + @classmethod + def _PushBranch(cls, clone_path, branch='master'): + """Push a branch directly to gerrit, bypassing code review.""" + cls.check_call( + ['git', 'push', 'origin', 'HEAD:refs/heads/%s' % branch], + cwd=clone_path) + + def pushBranch(self, clone_path, branch='master'): + """Push a branch directly to gerrit, bypassing code review.""" + clone_path = os.path.join(self.tempdir, clone_path) + self._PushBranch(clone_path, branch) + + @classmethod + def createAccount(cls, name='Test User', email='test-user@test.org', + password=None, groups=None): + """Create a new user account on gerrit.""" + username = email.partition('@')[0] + gerrit_cmd = 'gerrit create-account %s --full-name "%s" --email %s' % ( + username, name, email) + if password: + gerrit_cmd += ' --http-password "%s"' % password + if groups: + gerrit_cmd += ' '.join(['--group %s' % x for x in groups]) + ssh_cmd = ['ssh', '-p', cls.gerrit_instance.ssh_port, + '-i', cls.gerrit_instance.ssh_ident, + '-o', 'NoHostAuthenticationForLocalhost=yes', + '-o', 'StrictHostKeyChecking=no', + '%s@localhost' % cls.TEST_USERNAME, gerrit_cmd] + cls.check_call(ssh_cmd) + + @classmethod + def _stop_gerrit(cls, gerrit_instance): + """Stops the running gerrit instance and deletes it.""" + try: + # This should terminate the gerrit process. + cls.check_call(['bash', gerrit_instance.gerrit_exe, 'stop']) + finally: + try: + # cls.gerrit_pid should have already terminated. If it did, then + # os.waitpid will raise OSError. + os.waitpid(gerrit_instance.gerrit_pid, os.WNOHANG) + except OSError as e: + if e.errno == errno.ECHILD: + # If gerrit shut down cleanly, os.waitpid will land here. + # pylint: disable=W0150 + return + + # If we get here, the gerrit process is still alive. Send the process + # SIGKILL for good measure. + try: + os.kill(gerrit_instance.gerrit_pid, signal.SIGKILL) + except OSError: + if e.errno == errno.ESRCH: + # os.kill raised an error because the process doesn't exist. Maybe + # gerrit shut down cleanly after all. + # pylint: disable=W0150 + return + + # Announce that gerrit didn't shut down cleanly. + msg = 'Test gerrit server (pid=%d) did not shut down cleanly.' % ( + gerrit_instance.gerrit_pid) + print >> sys.stderr, msg + + @classmethod + def tearDownClass(cls): + gerrit_util.NETRC = cls.gerrit_util_netrc_orig + gerrit_util.GERRIT_PROTOCOL = cls.gerrit_util_protocol_orig + if TEARDOWN: + cls._stop_gerrit(cls.gerrit_instance) + shutil.rmtree(cls.gerrit_instance.gerrit_dir) + + +class RepoTestCase(GerritTestCase): + """Test class which runs in a repo checkout.""" + + REPO_URL = 'https://chromium.googlesource.com/external/repo' + MANIFEST_PROJECT = 'remotepath/manifest' + MANIFEST_TEMPLATE = """ + + + + + + + + + +""" + + @classmethod + def setUpClass(cls): + GerritTestCase.setUpClass() + gi = cls.gerrit_instance + + # Create local mirror of repo tool repository. + repo_mirror_path = os.path.join(gi.git_dir, 'repo.git') + cls.check_call( + ['git', 'clone', '--mirror', cls.REPO_URL, repo_mirror_path]) + + # Check out the top-level repo script; it will be used for invocation. + repo_clone_path = os.path.join(gi.gerrit_dir, 'tmp', 'repo') + cls.check_call(['git', 'clone', '-n', repo_mirror_path, repo_clone_path]) + cls.check_call( + ['git', 'checkout', 'origin/stable', 'repo'], cwd=repo_clone_path) + shutil.rmtree(os.path.join(repo_clone_path, '.git')) + cls.repo_exe = os.path.join(repo_clone_path, 'repo') + + # Create manifest repository. + cls.createProject(cls.MANIFEST_PROJECT) + clone_path = os.path.join(gi.gerrit_dir, 'tmp', 'manifest') + cls._CloneProject(cls.MANIFEST_PROJECT, clone_path) + manifest_path = os.path.join(clone_path, 'default.xml') + with open(manifest_path, 'w') as fh: + fh.write(cls.MANIFEST_TEMPLATE % gi.__dict__) + cls.check_call(['git', 'add', 'default.xml'], cwd=clone_path) + cls.check_call(['git', 'commit', '-m', 'Test manifest.'], cwd=clone_path) + cls._PushBranch(clone_path) + + # Create project repositories. + for i in xrange(1, 5): + proj = 'testproj%d' % i + cls.createProject('remotepath/%s' % proj) + clone_path = os.path.join(gi.gerrit_dir, 'tmp', proj) + cls._CloneProject('remotepath/%s' % proj, clone_path) + cls._CreateCommit(clone_path) + cls._PushBranch(clone_path, 'master') + + def setUp(self): + super(RepoTestCase, self).setUp() + manifest_url = '/'.join((self.gerrit_instance.gerrit_url, + self.MANIFEST_PROJECT)) + repo_url = '/'.join((self.gerrit_instance.gerrit_url, 'repo')) + self.check_call( + [self.repo_exe, 'init', '-u', manifest_url, '--repo-url', + repo_url, '--no-repo-verify'], cwd=self.tempdir) + self.check_call([self.repo_exe, 'sync'], cwd=self.tempdir) + for i in xrange(1, 5): + clone_path = os.path.join(self.tempdir, 'localpath', 'testproj%d' % i) + self._post_clone_bookkeeping(clone_path) + # Tell 'repo upload' to upload this project without prompting. + config_path = os.path.join(clone_path, '.git', 'config') + self.check_call( + ['git', 'config', '--file', config_path, 'review.%s.upload' % + self.gerrit_instance.gerrit_host, 'true']) + + @classmethod + def runRepo(cls, *args, **kwargs): + # Unfortunately, munging $HOME appears to be the only way to control the + # netrc file used by repo. + munged_home = os.path.join(cls.gerrit_instance.gerrit_dir, 'tmp') + if 'env' not in kwargs: + env = kwargs['env'] = os.environ.copy() + env['HOME'] = munged_home + else: + env.setdefault('HOME', munged_home) + args[0].insert(0, cls.repo_exe) + cls.check_call(*args, **kwargs) + + def uploadChange(self, clone_path, branch='master', remote='origin'): + review_host = self.check_output( + ['git', 'config', 'remote.%s.review' % remote], + cwd=clone_path).strip() + assert(review_host) + projectname = self.check_output( + ['git', 'config', 'remote.%s.projectname' % remote], + cwd=clone_path).strip() + assert(projectname) + GerritTestCase._UploadChange( + clone_path, branch=branch, remote='%s://%s/%s' % ( + gerrit_util.GERRIT_PROTOCOL, review_host, projectname))