Add git/gerrit-on-borg utilities.
gob_util.py is a general-purpose library for communicating with the gerrit-on-borg service. testing_support/gerrit_test_case.py is a unittest framework for testing code that interacts with gerrit. R=vadimsh@chromium.org, cmp@chromium.org BUG= Review URL: https://codereview.chromium.org/26399002 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@228965 0039d316-1c4b-4281-b951-d872f2087c98experimental/szager/collated-output
szager@chromium.org
12 years ago
parent
71b1357747
commit
b469623afd
@ -0,0 +1,394 @@
|
||||
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""
|
||||
Utilities for requesting information for a gerrit server via https.
|
||||
|
||||
https://gerrit-review.googlesource.com/Documentation/rest-api.html
|
||||
"""
|
||||
|
||||
import base64
|
||||
import httplib
|
||||
import json
|
||||
import logging
|
||||
import netrc
|
||||
import os
|
||||
import time
|
||||
import urllib
|
||||
from cStringIO import StringIO
|
||||
|
||||
try:
|
||||
NETRC = netrc.netrc()
|
||||
except (IOError, netrc.NetrcParseError):
|
||||
NETRC = netrc.netrc(os.devnull)
|
||||
LOGGER = logging.getLogger()
|
||||
TRY_LIMIT = 5
|
||||
|
||||
# Controls the transport protocol used to communicate with gerrit.
|
||||
# This is parameterized primarily to enable GerritTestCase.
|
||||
GERRIT_PROTOCOL = 'https'
|
||||
|
||||
|
||||
class GerritError(Exception):
|
||||
"""Exception class for errors commuicating with the gerrit-on-borg service."""
|
||||
def __init__(self, http_status, *args, **kwargs):
|
||||
super(GerritError, self).__init__(*args, **kwargs)
|
||||
self.http_status = http_status
|
||||
self.message = '(%d) %s' % (self.http_status, self.message)
|
||||
|
||||
|
||||
def _QueryString(param_dict, first_param=None):
|
||||
"""Encodes query parameters in the key:val[+key:val...] format specified here:
|
||||
|
||||
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
|
||||
"""
|
||||
q = [urllib.quote(first_param)] if first_param else []
|
||||
q.extend(['%s:%s' % (key, val) for key, val in param_dict.iteritems()])
|
||||
return '+'.join(q)
|
||||
|
||||
|
||||
def GetConnectionClass(protocol=None):
|
||||
if protocol is None:
|
||||
protocol = GERRIT_PROTOCOL
|
||||
if protocol == 'https':
|
||||
return httplib.HTTPSConnection
|
||||
elif protocol == 'http':
|
||||
return httplib.HTTPConnection
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"Don't know how to work with protocol '%s'" % protocol)
|
||||
|
||||
|
||||
def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None):
|
||||
"""Opens an https connection to a gerrit service, and sends a request."""
|
||||
headers = headers or {}
|
||||
bare_host = host.partition(':')[0]
|
||||
auth = NETRC.authenticators(bare_host)
|
||||
if auth:
|
||||
headers.setdefault('Authorization', 'Basic %s' % (
|
||||
base64.b64encode('%s:%s' % (auth[0], auth[2]))))
|
||||
else:
|
||||
LOGGER.debug('No authorization found')
|
||||
if body:
|
||||
body = json.JSONEncoder().encode(body)
|
||||
headers.setdefault('Content-Type', 'application/json')
|
||||
if LOGGER.isEnabledFor(logging.DEBUG):
|
||||
LOGGER.debug('%s %s://%s/a/%s' % (reqtype, GERRIT_PROTOCOL, host, path))
|
||||
for key, val in headers.iteritems():
|
||||
if key == 'Authorization':
|
||||
val = 'HIDDEN'
|
||||
LOGGER.debug('%s: %s' % (key, val))
|
||||
if body:
|
||||
LOGGER.debug(body)
|
||||
conn = GetConnectionClass()(host)
|
||||
conn.req_host = host
|
||||
conn.req_params = {
|
||||
'url': '/a/%s' % path,
|
||||
'method': reqtype,
|
||||
'headers': headers,
|
||||
'body': body,
|
||||
}
|
||||
conn.request(**conn.req_params)
|
||||
return conn
|
||||
|
||||
|
||||
def ReadHttpResponse(conn, expect_status=200, ignore_404=True):
|
||||
"""Reads an http response from a connection into a string buffer.
|
||||
|
||||
Args:
|
||||
conn: An HTTPSConnection or HTTPConnection created by CreateHttpConn, above.
|
||||
expect_status: Success is indicated by this status in the response.
|
||||
ignore_404: For many requests, gerrit-on-borg will return 404 if the request
|
||||
doesn't match the database contents. In most such cases, we
|
||||
want the API to return None rather than raise an Exception.
|
||||
Returns: A string buffer containing the connection's reply.
|
||||
"""
|
||||
|
||||
sleep_time = 0.5
|
||||
for idx in range(TRY_LIMIT):
|
||||
response = conn.getresponse()
|
||||
# If response.status < 500 then the result is final; break retry loop.
|
||||
if response.status < 500:
|
||||
break
|
||||
# A status >=500 is assumed to be a possible transient error; retry.
|
||||
http_version = 'HTTP/%s' % ('1.1' if response.version == 11 else '1.0')
|
||||
msg = (
|
||||
'A transient error occured while querying %s:\n'
|
||||
'%s %s %s\n'
|
||||
'%s %d %s' % (
|
||||
conn.host, conn.req_params['method'], conn.req_params['url'],
|
||||
http_version, http_version, response.status, response.reason))
|
||||
if TRY_LIMIT - idx > 1:
|
||||
msg += '\n... will retry %d more times.' % (TRY_LIMIT - idx - 1)
|
||||
time.sleep(sleep_time)
|
||||
sleep_time = sleep_time * 2
|
||||
req_host = conn.req_host
|
||||
req_params = conn.req_params
|
||||
conn = GetConnectionClass()(req_host)
|
||||
conn.req_host = req_host
|
||||
conn.req_params = req_params
|
||||
conn.request(**req_params)
|
||||
LOGGER.warn(msg)
|
||||
if ignore_404 and response.status == 404:
|
||||
return StringIO()
|
||||
if response.status != expect_status:
|
||||
raise GerritError(response.status, response.reason)
|
||||
return StringIO(response.read())
|
||||
|
||||
|
||||
def ReadHttpJsonResponse(conn, expect_status=200, ignore_404=True):
|
||||
"""Parses an https response as json."""
|
||||
fh = ReadHttpResponse(
|
||||
conn, expect_status=expect_status, ignore_404=ignore_404)
|
||||
# The first line of the response should always be: )]}'
|
||||
s = fh.readline()
|
||||
if s and s.rstrip() != ")]}'":
|
||||
raise GerritError(200, 'Unexpected json output: %s' % s)
|
||||
s = fh.read()
|
||||
if not s:
|
||||
return None
|
||||
return json.loads(s)
|
||||
|
||||
|
||||
def QueryChanges(host, param_dict, first_param=None, limit=None, o_params=None,
|
||||
sortkey=None):
|
||||
"""
|
||||
Queries a gerrit-on-borg server for changes matching query terms.
|
||||
|
||||
Args:
|
||||
param_dict: A dictionary of search parameters, as documented here:
|
||||
http://gerrit-documentation.googlecode.com/svn/Documentation/2.6/user-search.html
|
||||
first_param: A change identifier
|
||||
limit: Maximum number of results to return.
|
||||
o_params: A list of additional output specifiers, as documented here:
|
||||
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
|
||||
Returns:
|
||||
A list of json-decoded query results.
|
||||
"""
|
||||
# Note that no attempt is made to escape special characters; YMMV.
|
||||
if not param_dict and not first_param:
|
||||
raise RuntimeError('QueryChanges requires search parameters')
|
||||
path = 'changes/?q=%s' % _QueryString(param_dict, first_param)
|
||||
if sortkey:
|
||||
path = '%s&N=%s' % (path, sortkey)
|
||||
if limit:
|
||||
path = '%s&n=%d' % (path, limit)
|
||||
if o_params:
|
||||
path = '%s&%s' % (path, '&'.join(['o=%s' % p for p in o_params]))
|
||||
# Don't ignore 404; a query should always return a list, even if it's empty.
|
||||
return ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=False)
|
||||
|
||||
|
||||
def MultiQueryChanges(host, param_dict, change_list, limit=None, o_params=None,
|
||||
sortkey=None):
|
||||
"""Initiate a query composed of multiple sets of query parameters."""
|
||||
if not change_list:
|
||||
raise RuntimeError(
|
||||
"MultiQueryChanges requires a list of change numbers/id's")
|
||||
q = ['q=%s' % '+OR+'.join([urllib.quote(str(x)) for x in change_list])]
|
||||
if param_dict:
|
||||
q.append(_QueryString(param_dict))
|
||||
if limit:
|
||||
q.append('n=%d' % limit)
|
||||
if sortkey:
|
||||
q.append('N=%s' % sortkey)
|
||||
if o_params:
|
||||
q.extend(['o=%s' % p for p in o_params])
|
||||
path = 'changes/?%s' % '&'.join(q)
|
||||
try:
|
||||
result = ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=False)
|
||||
except GerritError as e:
|
||||
msg = '%s:\n%s' % (e.message, path)
|
||||
raise GerritError(e.http_status, msg)
|
||||
return result
|
||||
|
||||
|
||||
def GetGerritFetchUrl(host):
|
||||
"""Given a gerrit host name returns URL of a gerrit instance to fetch from."""
|
||||
return '%s://%s/' % (GERRIT_PROTOCOL, host)
|
||||
|
||||
|
||||
def GetChangePageUrl(host, change_number):
|
||||
"""Given a gerrit host name and change number, return change page url."""
|
||||
return '%s://%s/#/c/%d/' % (GERRIT_PROTOCOL, host, change_number)
|
||||
|
||||
|
||||
def GetChangeUrl(host, change):
|
||||
"""Given a gerrit host name and change id, return an url for the change."""
|
||||
return '%s://%s/a/changes/%s' % (GERRIT_PROTOCOL, host, change)
|
||||
|
||||
|
||||
def GetChange(host, change):
|
||||
"""Query a gerrit server for information about a single change."""
|
||||
path = 'changes/%s' % change
|
||||
return ReadHttpJsonResponse(CreateHttpConn(host, path))
|
||||
|
||||
|
||||
def GetChangeDetail(host, change, o_params=None):
|
||||
"""Query a gerrit server for extended information about a single change."""
|
||||
path = 'changes/%s/detail' % change
|
||||
if o_params:
|
||||
path += '?%s' % '&'.join(['o=%s' % p for p in o_params])
|
||||
return ReadHttpJsonResponse(CreateHttpConn(host, path))
|
||||
|
||||
|
||||
def GetChangeCurrentRevision(host, change):
|
||||
"""Get information about the latest revision for a given change."""
|
||||
return QueryChanges(host, {}, change, o_params=('CURRENT_REVISION',))
|
||||
|
||||
|
||||
def GetChangeRevisions(host, change):
|
||||
"""Get information about all revisions associated with a change."""
|
||||
return QueryChanges(host, {}, change, o_params=('ALL_REVISIONS',))
|
||||
|
||||
|
||||
def GetChangeReview(host, change, revision=None):
|
||||
"""Get the current review information for a change."""
|
||||
if not revision:
|
||||
jmsg = GetChangeRevisions(host, change)
|
||||
if not jmsg:
|
||||
return None
|
||||
elif len(jmsg) > 1:
|
||||
raise GerritError(200, 'Multiple changes found for ChangeId %s.' % change)
|
||||
revision = jmsg[0]['current_revision']
|
||||
path = 'changes/%s/revisions/%s/review'
|
||||
return ReadHttpJsonResponse(CreateHttpConn(host, path))
|
||||
|
||||
|
||||
def AbandonChange(host, change, msg=''):
|
||||
"""Abandon a gerrit change."""
|
||||
path = 'changes/%s/abandon' % change
|
||||
body = {'message': msg} if msg else None
|
||||
conn = CreateHttpConn(host, path, reqtype='POST', body=body)
|
||||
return ReadHttpJsonResponse(conn, ignore_404=False)
|
||||
|
||||
|
||||
def RestoreChange(host, change, msg=''):
|
||||
"""Restore a previously abandoned change."""
|
||||
path = 'changes/%s/restore' % change
|
||||
body = {'message': msg} if msg else None
|
||||
conn = CreateHttpConn(host, path, reqtype='POST', body=body)
|
||||
return ReadHttpJsonResponse(conn, ignore_404=False)
|
||||
|
||||
|
||||
def SubmitChange(host, change, wait_for_merge=True):
|
||||
"""Submits a gerrit change via Gerrit."""
|
||||
path = 'changes/%s/submit' % change
|
||||
body = {'wait_for_merge': wait_for_merge}
|
||||
conn = CreateHttpConn(host, path, reqtype='POST', body=body)
|
||||
return ReadHttpJsonResponse(conn, ignore_404=False)
|
||||
|
||||
|
||||
def GetReviewers(host, change):
|
||||
"""Get information about all reviewers attached to a change."""
|
||||
path = 'changes/%s/reviewers' % change
|
||||
return ReadHttpJsonResponse(CreateHttpConn(host, path))
|
||||
|
||||
|
||||
def GetReview(host, change, revision):
|
||||
"""Get review information about a specific revision of a change."""
|
||||
path = 'changes/%s/revisions/%s/review' % (change, revision)
|
||||
return ReadHttpJsonResponse(CreateHttpConn(host, path))
|
||||
|
||||
|
||||
def AddReviewers(host, change, add=None):
|
||||
"""Add reviewers to a change."""
|
||||
if not add:
|
||||
return
|
||||
if isinstance(add, basestring):
|
||||
add = (add,)
|
||||
path = 'changes/%s/reviewers' % change
|
||||
for r in add:
|
||||
body = {'reviewer': r}
|
||||
conn = CreateHttpConn(host, path, reqtype='POST', body=body)
|
||||
jmsg = ReadHttpJsonResponse(conn, ignore_404=False)
|
||||
return jmsg
|
||||
|
||||
|
||||
def RemoveReviewers(host, change, remove=None):
|
||||
"""Remove reveiewers from a change."""
|
||||
if not remove:
|
||||
return
|
||||
if isinstance(remove, basestring):
|
||||
remove = (remove,)
|
||||
for r in remove:
|
||||
path = 'changes/%s/reviewers/%s' % (change, r)
|
||||
conn = CreateHttpConn(host, path, reqtype='DELETE')
|
||||
try:
|
||||
ReadHttpResponse(conn, ignore_404=False)
|
||||
except GerritError as e:
|
||||
# On success, gerrit returns status 204; anything else is an error.
|
||||
if e.http_status != 204:
|
||||
raise
|
||||
else:
|
||||
raise GerritError(
|
||||
'Unexpectedly received a 200 http status while deleting reviewer "%s"'
|
||||
' from change %s' % (r, change))
|
||||
|
||||
|
||||
def SetReview(host, change, msg=None, labels=None, notify=None):
|
||||
"""Set labels and/or add a message to a code review."""
|
||||
if not msg and not labels:
|
||||
return
|
||||
path = 'changes/%s/revisions/current/review' % change
|
||||
body = {}
|
||||
if msg:
|
||||
body['message'] = msg
|
||||
if labels:
|
||||
body['labels'] = labels
|
||||
if notify:
|
||||
body['notify'] = notify
|
||||
conn = CreateHttpConn(host, path, reqtype='POST', body=body)
|
||||
response = ReadHttpJsonResponse(conn)
|
||||
if labels:
|
||||
for key, val in labels.iteritems():
|
||||
if ('labels' not in response or key not in response['labels'] or
|
||||
int(response['labels'][key] != int(val))):
|
||||
raise GerritError(200, 'Unable to set "%s" label on change %s.' % (
|
||||
key, change))
|
||||
|
||||
|
||||
def ResetReviewLabels(host, change, label, value='0', message=None,
|
||||
notify=None):
|
||||
"""Reset the value of a given label for all reviewers on a change."""
|
||||
# This is tricky, because we want to work on the "current revision", but
|
||||
# there's always the risk that "current revision" will change in between
|
||||
# API calls. So, we check "current revision" at the beginning and end; if
|
||||
# it has changed, raise an exception.
|
||||
jmsg = GetChangeCurrentRevision(host, change)
|
||||
if not jmsg:
|
||||
raise GerritError(
|
||||
200, 'Could not get review information for change "%s"' % change)
|
||||
value = str(value)
|
||||
revision = jmsg[0]['current_revision']
|
||||
path = 'changes/%s/revisions/%s/review' % (change, revision)
|
||||
message = message or (
|
||||
'%s label set to %s programmatically.' % (label, value))
|
||||
jmsg = GetReview(host, change, revision)
|
||||
if not jmsg:
|
||||
raise GerritError(200, 'Could not get review information for revison %s '
|
||||
'of change %s' % (revision, change))
|
||||
for review in jmsg.get('labels', {}).get(label, {}).get('all', []):
|
||||
if str(review.get('value', value)) != value:
|
||||
body = {
|
||||
'message': message,
|
||||
'labels': {label: value},
|
||||
'on_behalf_of': review['_account_id'],
|
||||
}
|
||||
if notify:
|
||||
body['notify'] = notify
|
||||
conn = CreateHttpConn(
|
||||
host, path, reqtype='POST', body=body)
|
||||
response = ReadHttpJsonResponse(conn)
|
||||
if str(response['labels'][label]) != value:
|
||||
username = review.get('email', jmsg.get('name', ''))
|
||||
raise GerritError(200, 'Unable to set %s label for user "%s"'
|
||||
' on change %s.' % (label, username, change))
|
||||
jmsg = GetChangeCurrentRevision(host, change)
|
||||
if not jmsg:
|
||||
raise GerritError(
|
||||
200, 'Could not get review information for change "%s"' % change)
|
||||
elif jmsg[0]['current_revision'] != revision:
|
||||
raise GerritError(200, 'While resetting labels on change "%s", '
|
||||
'a new patchset was uploaded.' % change)
|
||||
@ -0,0 +1,482 @@
|
||||
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""Test framework for code that interacts with gerrit.
|
||||
|
||||
class GerritTestCase
|
||||
--------------------------------------------------------------------------------
|
||||
This class initializes and runs an a gerrit instance on localhost. To use the
|
||||
framework, define a class that extends GerritTestCase, and then do standard
|
||||
python unittest development as described here:
|
||||
|
||||
http://docs.python.org/2.7/library/unittest.html#basic-example
|
||||
|
||||
When your test code runs, the framework will:
|
||||
|
||||
- Download the latest stable(-ish) binary release of the gerrit code.
|
||||
- Start up a live gerrit instance running in a temp directory on the localhost.
|
||||
- Set up a single gerrit user account with admin priveleges.
|
||||
- Supply credential helpers for interacting with the gerrit instance via http
|
||||
or ssh.
|
||||
|
||||
Refer to depot_tools/testing_support/gerrit-init.sh for details about how the
|
||||
gerrit instance is set up, and refer to helper methods defined below
|
||||
(createProject, cloneProject, uploadChange, etc.) for ways to interact with the
|
||||
gerrit instance from your test methods.
|
||||
|
||||
|
||||
class RepoTestCase
|
||||
--------------------------------------------------------------------------------
|
||||
This class extends GerritTestCase, and creates a set of project repositories
|
||||
and a manifest repository that can be used in conjunction with the 'repo' tool.
|
||||
|
||||
Each test method will initialize and sync a brand-new repo working directory.
|
||||
The 'repo' command may be invoked in a subprocess as part of your tests.
|
||||
|
||||
One gotcha: 'repo upload' will always attempt to use the ssh interface to talk
|
||||
to gerrit.
|
||||
"""
|
||||
|
||||
import collections
|
||||
from cStringIO import StringIO
|
||||
import errno
|
||||
import httplib
|
||||
import imp
|
||||
import json
|
||||
import netrc
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import urllib
|
||||
|
||||
import gerrit_util
|
||||
|
||||
|
||||
DEPOT_TOOLS_DIR = os.path.normpath(os.path.join(
|
||||
os.path.realpath(__file__), '..', '..'))
|
||||
|
||||
|
||||
# When debugging test code, it's sometimes helpful to leave the test gerrit
|
||||
# instance intact and running after the test code exits. Setting TEARDOWN
|
||||
# to False will do that.
|
||||
TEARDOWN = True
|
||||
|
||||
class GerritTestCase(unittest.TestCase):
|
||||
"""Test class for tests that interact with a gerrit server.
|
||||
|
||||
The class setup creates and launches a stand-alone gerrit instance running on
|
||||
localhost, for test methods to interact with. Class teardown stops and
|
||||
deletes the gerrit instance.
|
||||
|
||||
Note that there is a single gerrit instance for ALL test methods in a
|
||||
GerritTestCase sub-class.
|
||||
"""
|
||||
|
||||
COMMIT_RE = re.compile(r'^commit ([0-9a-fA-F]{40})$')
|
||||
CHANGEID_RE = re.compile('^\s+Change-Id:\s*(\S+)$')
|
||||
DEVNULL = open(os.devnull, 'w')
|
||||
TEST_USERNAME = 'test-username'
|
||||
TEST_EMAIL = 'test-username@test.org'
|
||||
|
||||
GerritInstance = collections.namedtuple('GerritInstance', [
|
||||
'credential_file',
|
||||
'gerrit_dir',
|
||||
'gerrit_exe',
|
||||
'gerrit_host',
|
||||
'gerrit_pid',
|
||||
'gerrit_url',
|
||||
'git_dir',
|
||||
'git_host',
|
||||
'git_url',
|
||||
'http_port',
|
||||
'netrc_file',
|
||||
'ssh_ident',
|
||||
'ssh_port',
|
||||
])
|
||||
|
||||
@classmethod
|
||||
def check_call(cls, *args, **kwargs):
|
||||
kwargs.setdefault('stdout', cls.DEVNULL)
|
||||
kwargs.setdefault('stderr', cls.DEVNULL)
|
||||
subprocess.check_call(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def check_output(cls, *args, **kwargs):
|
||||
kwargs.setdefault('stderr', cls.DEVNULL)
|
||||
return subprocess.check_output(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def _create_gerrit_instance(cls, gerrit_dir):
|
||||
gerrit_init_script = os.path.join(
|
||||
DEPOT_TOOLS_DIR, 'testing_support', 'gerrit-init.sh')
|
||||
http_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
http_sock.bind(('', 0))
|
||||
http_port = str(http_sock.getsockname()[1])
|
||||
ssh_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
ssh_sock.bind(('', 0))
|
||||
ssh_port = str(ssh_sock.getsockname()[1])
|
||||
|
||||
# NOTE: this is not completely safe. These port numbers could be
|
||||
# re-assigned by the OS between the calls to socket.close() and gerrit
|
||||
# starting up. The only safe way to do this would be to pass file
|
||||
# descriptors down to the gerrit process, which is not even remotely
|
||||
# supported. Alas.
|
||||
http_sock.close()
|
||||
ssh_sock.close()
|
||||
cls.check_call(['bash', gerrit_init_script, '--http-port', http_port,
|
||||
'--ssh-port', ssh_port, gerrit_dir])
|
||||
|
||||
gerrit_exe = os.path.join(gerrit_dir, 'bin', 'gerrit.sh')
|
||||
cls.check_call(['bash', gerrit_exe, 'start'])
|
||||
with open(os.path.join(gerrit_dir, 'logs', 'gerrit.pid')) as fh:
|
||||
gerrit_pid = int(fh.read().rstrip())
|
||||
|
||||
return cls.GerritInstance(
|
||||
credential_file=os.path.join(gerrit_dir, 'tmp', '.git-credentials'),
|
||||
gerrit_dir=gerrit_dir,
|
||||
gerrit_exe=gerrit_exe,
|
||||
gerrit_host='localhost:%s' % http_port,
|
||||
gerrit_pid=gerrit_pid,
|
||||
gerrit_url='http://localhost:%s' % http_port,
|
||||
git_dir=os.path.join(gerrit_dir, 'git'),
|
||||
git_host='%s/git' % gerrit_dir,
|
||||
git_url='file://%s/git' % gerrit_dir,
|
||||
http_port=http_port,
|
||||
netrc_file=os.path.join(gerrit_dir, 'tmp', '.netrc'),
|
||||
ssh_ident=os.path.join(gerrit_dir, 'tmp', 'id_rsa'),
|
||||
ssh_port=ssh_port,)
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Sets up the gerrit instances in a class-specific temp dir."""
|
||||
# Create gerrit instance.
|
||||
gerrit_dir = tempfile.mkdtemp()
|
||||
os.chmod(gerrit_dir, 0700)
|
||||
gi = cls.gerrit_instance = cls._create_gerrit_instance(gerrit_dir)
|
||||
|
||||
# Set netrc file for http authentication.
|
||||
cls.gerrit_util_netrc_orig = gerrit_util.NETRC
|
||||
gerrit_util.NETRC = netrc.netrc(gi.netrc_file)
|
||||
|
||||
# gerrit_util.py defaults to using https, but for testing, it's much
|
||||
# simpler to use http connections.
|
||||
cls.gerrit_util_protocol_orig = gerrit_util.GERRIT_PROTOCOL
|
||||
gerrit_util.GERRIT_PROTOCOL = 'http'
|
||||
|
||||
# Because we communicate with the test server via http, rather than https,
|
||||
# libcurl won't add authentication headers to raw git requests unless the
|
||||
# gerrit server returns 401. That works for pushes, but for read operations
|
||||
# (like git-ls-remote), gerrit will simply omit any ref that requires
|
||||
# authentication. By default gerrit doesn't permit anonymous read access to
|
||||
# refs/meta/config. Override that behavior so tests can access
|
||||
# refs/meta/config if necessary.
|
||||
clone_path = os.path.join(gi.gerrit_dir, 'tmp', 'All-Projects')
|
||||
cls._CloneProject('All-Projects', clone_path)
|
||||
project_config = os.path.join(clone_path, 'project.config')
|
||||
cls.check_call(['git', 'config', '--file', project_config, '--add',
|
||||
'access.refs/meta/config.read', 'group Anonymous Users'])
|
||||
cls.check_call(['git', 'add', project_config], cwd=clone_path)
|
||||
cls.check_call(
|
||||
['git', 'commit', '-m', 'Anonyous read for refs/meta/config'],
|
||||
cwd=clone_path)
|
||||
cls.check_call(['git', 'push', 'origin', 'HEAD:refs/meta/config'],
|
||||
cwd=clone_path)
|
||||
|
||||
def setUp(self):
|
||||
self.tempdir = tempfile.mkdtemp()
|
||||
os.chmod(self.tempdir, 0700)
|
||||
|
||||
def tearDown(self):
|
||||
if TEARDOWN:
|
||||
shutil.rmtree(self.tempdir)
|
||||
|
||||
@classmethod
|
||||
def createProject(cls, name, description='Test project', owners=None,
|
||||
submit_type='CHERRY_PICK'):
|
||||
"""Create a project on the test gerrit server."""
|
||||
if owners is None:
|
||||
owners = ['Administrators']
|
||||
body = {
|
||||
'description': description,
|
||||
'submit_type': submit_type,
|
||||
'owners': owners,
|
||||
}
|
||||
path = 'projects/%s' % urllib.quote(name, '')
|
||||
conn = gerrit_util.CreateHttpConn(
|
||||
cls.gerrit_instance.gerrit_host, path, reqtype='PUT', body=body)
|
||||
jmsg = gerrit_util.ReadHttpJsonResponse(conn, expect_status=201)
|
||||
assert jmsg['name'] == name
|
||||
|
||||
@classmethod
|
||||
def _post_clone_bookkeeping(cls, clone_path):
|
||||
config_path = os.path.join(clone_path, '.git', 'config')
|
||||
cls.check_call(
|
||||
['git', 'config', '--file', config_path, 'user.email', cls.TEST_EMAIL])
|
||||
cls.check_call(
|
||||
['git', 'config', '--file', config_path, 'credential.helper',
|
||||
'store --file=%s' % cls.gerrit_instance.credential_file])
|
||||
|
||||
@classmethod
|
||||
def _CloneProject(cls, name, path):
|
||||
"""Clone a project from the test gerrit server."""
|
||||
gi = cls.gerrit_instance
|
||||
parent_dir = os.path.dirname(path)
|
||||
if not os.path.exists(parent_dir):
|
||||
os.makedirs(parent_dir)
|
||||
url = '/'.join((gi.gerrit_url, name))
|
||||
cls.check_call(['git', 'clone', url, path])
|
||||
cls._post_clone_bookkeeping(path)
|
||||
# Install commit-msg hook to add Change-Id lines.
|
||||
hook_path = os.path.join(path, '.git', 'hooks', 'commit-msg')
|
||||
cls.check_call(['curl', '-o', hook_path,
|
||||
'/'.join((gi.gerrit_url, 'tools/hooks/commit-msg'))])
|
||||
os.chmod(hook_path, stat.S_IRWXU)
|
||||
return path
|
||||
|
||||
def cloneProject(self, name, path=None):
|
||||
"""Clone a project from the test gerrit server."""
|
||||
if path is None:
|
||||
path = os.path.basename(name)
|
||||
if path.endswith('.git'):
|
||||
path = path[:-4]
|
||||
path = os.path.join(self.tempdir, path)
|
||||
return self._CloneProject(name, path)
|
||||
|
||||
@classmethod
|
||||
def _CreateCommit(cls, clone_path, fn=None, msg=None, text=None):
|
||||
"""Create a commit in the given git checkout."""
|
||||
if not fn:
|
||||
fn = 'test-file.txt'
|
||||
if not msg:
|
||||
msg = 'Test Message'
|
||||
if not text:
|
||||
text = 'Another day, another dollar.'
|
||||
fpath = os.path.join(clone_path, fn)
|
||||
with open(fpath, 'a') as fh:
|
||||
fh.write('%s\n' % text)
|
||||
cls.check_call(['git', 'add', fn], cwd=clone_path)
|
||||
cls.check_call(['git', 'commit', '-m', msg], cwd=clone_path)
|
||||
return cls._GetCommit(clone_path)
|
||||
|
||||
def createCommit(self, clone_path, fn=None, msg=None, text=None):
|
||||
"""Create a commit in the given git checkout."""
|
||||
clone_path = os.path.join(self.tempdir, clone_path)
|
||||
return self._CreateCommit(clone_path, fn, msg, text)
|
||||
|
||||
@classmethod
|
||||
def _GetCommit(cls, clone_path, ref='HEAD'):
|
||||
"""Get the sha1 and change-id for a ref in the git checkout."""
|
||||
log_proc = cls.check_output(['git', 'log', '-n', '1', ref], cwd=clone_path)
|
||||
sha1 = None
|
||||
change_id = None
|
||||
for line in log_proc.splitlines():
|
||||
match = cls.COMMIT_RE.match(line)
|
||||
if match:
|
||||
sha1 = match.group(1)
|
||||
continue
|
||||
match = cls.CHANGEID_RE.match(line)
|
||||
if match:
|
||||
change_id = match.group(1)
|
||||
continue
|
||||
assert sha1
|
||||
assert change_id
|
||||
return (sha1, change_id)
|
||||
|
||||
def getCommit(self, clone_path, ref='HEAD'):
|
||||
"""Get the sha1 and change-id for a ref in the git checkout."""
|
||||
clone_path = os.path.join(self.tempdir, clone_path)
|
||||
return self._GetCommit(clone_path, ref)
|
||||
|
||||
@classmethod
|
||||
def _UploadChange(cls, clone_path, branch='master', remote='origin'):
|
||||
"""Create a gerrit CL from the HEAD of a git checkout."""
|
||||
cls.check_call(
|
||||
['git', 'push', remote, 'HEAD:refs/for/%s' % branch], cwd=clone_path)
|
||||
|
||||
def uploadChange(self, clone_path, branch='master', remote='origin'):
|
||||
"""Create a gerrit CL from the HEAD of a git checkout."""
|
||||
clone_path = os.path.join(self.tempdir, clone_path)
|
||||
self._UploadChange(clone_path, branch, remote)
|
||||
|
||||
@classmethod
|
||||
def _PushBranch(cls, clone_path, branch='master'):
|
||||
"""Push a branch directly to gerrit, bypassing code review."""
|
||||
cls.check_call(
|
||||
['git', 'push', 'origin', 'HEAD:refs/heads/%s' % branch],
|
||||
cwd=clone_path)
|
||||
|
||||
def pushBranch(self, clone_path, branch='master'):
|
||||
"""Push a branch directly to gerrit, bypassing code review."""
|
||||
clone_path = os.path.join(self.tempdir, clone_path)
|
||||
self._PushBranch(clone_path, branch)
|
||||
|
||||
@classmethod
|
||||
def createAccount(cls, name='Test User', email='test-user@test.org',
|
||||
password=None, groups=None):
|
||||
"""Create a new user account on gerrit."""
|
||||
username = email.partition('@')[0]
|
||||
gerrit_cmd = 'gerrit create-account %s --full-name "%s" --email %s' % (
|
||||
username, name, email)
|
||||
if password:
|
||||
gerrit_cmd += ' --http-password "%s"' % password
|
||||
if groups:
|
||||
gerrit_cmd += ' '.join(['--group %s' % x for x in groups])
|
||||
ssh_cmd = ['ssh', '-p', cls.gerrit_instance.ssh_port,
|
||||
'-i', cls.gerrit_instance.ssh_ident,
|
||||
'-o', 'NoHostAuthenticationForLocalhost=yes',
|
||||
'-o', 'StrictHostKeyChecking=no',
|
||||
'%s@localhost' % cls.TEST_USERNAME, gerrit_cmd]
|
||||
cls.check_call(ssh_cmd)
|
||||
|
||||
@classmethod
|
||||
def _stop_gerrit(cls, gerrit_instance):
|
||||
"""Stops the running gerrit instance and deletes it."""
|
||||
try:
|
||||
# This should terminate the gerrit process.
|
||||
cls.check_call(['bash', gerrit_instance.gerrit_exe, 'stop'])
|
||||
finally:
|
||||
try:
|
||||
# cls.gerrit_pid should have already terminated. If it did, then
|
||||
# os.waitpid will raise OSError.
|
||||
os.waitpid(gerrit_instance.gerrit_pid, os.WNOHANG)
|
||||
except OSError as e:
|
||||
if e.errno == errno.ECHILD:
|
||||
# If gerrit shut down cleanly, os.waitpid will land here.
|
||||
# pylint: disable=W0150
|
||||
return
|
||||
|
||||
# If we get here, the gerrit process is still alive. Send the process
|
||||
# SIGKILL for good measure.
|
||||
try:
|
||||
os.kill(gerrit_instance.gerrit_pid, signal.SIGKILL)
|
||||
except OSError:
|
||||
if e.errno == errno.ESRCH:
|
||||
# os.kill raised an error because the process doesn't exist. Maybe
|
||||
# gerrit shut down cleanly after all.
|
||||
# pylint: disable=W0150
|
||||
return
|
||||
|
||||
# Announce that gerrit didn't shut down cleanly.
|
||||
msg = 'Test gerrit server (pid=%d) did not shut down cleanly.' % (
|
||||
gerrit_instance.gerrit_pid)
|
||||
print >> sys.stderr, msg
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
gerrit_util.NETRC = cls.gerrit_util_netrc_orig
|
||||
gerrit_util.GERRIT_PROTOCOL = cls.gerrit_util_protocol_orig
|
||||
if TEARDOWN:
|
||||
cls._stop_gerrit(cls.gerrit_instance)
|
||||
shutil.rmtree(cls.gerrit_instance.gerrit_dir)
|
||||
|
||||
|
||||
class RepoTestCase(GerritTestCase):
|
||||
"""Test class which runs in a repo checkout."""
|
||||
|
||||
REPO_URL = 'https://chromium.googlesource.com/external/repo'
|
||||
MANIFEST_PROJECT = 'remotepath/manifest'
|
||||
MANIFEST_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<manifest>
|
||||
<remote name="remote1"
|
||||
fetch="%(gerrit_url)s"
|
||||
review="%(gerrit_host)s" />
|
||||
<remote name="remote2"
|
||||
fetch="%(gerrit_url)s"
|
||||
review="%(gerrit_host)s" />
|
||||
<default revision="refs/heads/master" remote="remote1" sync-j="1" />
|
||||
<project remote="remote1" path="localpath/testproj1" name="remotepath/testproj1" />
|
||||
<project remote="remote1" path="localpath/testproj2" name="remotepath/testproj2" />
|
||||
<project remote="remote2" path="localpath/testproj3" name="remotepath/testproj3" />
|
||||
<project remote="remote2" path="localpath/testproj4" name="remotepath/testproj4" />
|
||||
</manifest>
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
GerritTestCase.setUpClass()
|
||||
gi = cls.gerrit_instance
|
||||
|
||||
# Create local mirror of repo tool repository.
|
||||
repo_mirror_path = os.path.join(gi.git_dir, 'repo.git')
|
||||
cls.check_call(
|
||||
['git', 'clone', '--mirror', cls.REPO_URL, repo_mirror_path])
|
||||
|
||||
# Check out the top-level repo script; it will be used for invocation.
|
||||
repo_clone_path = os.path.join(gi.gerrit_dir, 'tmp', 'repo')
|
||||
cls.check_call(['git', 'clone', '-n', repo_mirror_path, repo_clone_path])
|
||||
cls.check_call(
|
||||
['git', 'checkout', 'origin/stable', 'repo'], cwd=repo_clone_path)
|
||||
shutil.rmtree(os.path.join(repo_clone_path, '.git'))
|
||||
cls.repo_exe = os.path.join(repo_clone_path, 'repo')
|
||||
|
||||
# Create manifest repository.
|
||||
cls.createProject(cls.MANIFEST_PROJECT)
|
||||
clone_path = os.path.join(gi.gerrit_dir, 'tmp', 'manifest')
|
||||
cls._CloneProject(cls.MANIFEST_PROJECT, clone_path)
|
||||
manifest_path = os.path.join(clone_path, 'default.xml')
|
||||
with open(manifest_path, 'w') as fh:
|
||||
fh.write(cls.MANIFEST_TEMPLATE % gi.__dict__)
|
||||
cls.check_call(['git', 'add', 'default.xml'], cwd=clone_path)
|
||||
cls.check_call(['git', 'commit', '-m', 'Test manifest.'], cwd=clone_path)
|
||||
cls._PushBranch(clone_path)
|
||||
|
||||
# Create project repositories.
|
||||
for i in xrange(1, 5):
|
||||
proj = 'testproj%d' % i
|
||||
cls.createProject('remotepath/%s' % proj)
|
||||
clone_path = os.path.join(gi.gerrit_dir, 'tmp', proj)
|
||||
cls._CloneProject('remotepath/%s' % proj, clone_path)
|
||||
cls._CreateCommit(clone_path)
|
||||
cls._PushBranch(clone_path, 'master')
|
||||
|
||||
def setUp(self):
|
||||
super(RepoTestCase, self).setUp()
|
||||
manifest_url = '/'.join((self.gerrit_instance.gerrit_url,
|
||||
self.MANIFEST_PROJECT))
|
||||
repo_url = '/'.join((self.gerrit_instance.gerrit_url, 'repo'))
|
||||
self.check_call(
|
||||
[self.repo_exe, 'init', '-u', manifest_url, '--repo-url',
|
||||
repo_url, '--no-repo-verify'], cwd=self.tempdir)
|
||||
self.check_call([self.repo_exe, 'sync'], cwd=self.tempdir)
|
||||
for i in xrange(1, 5):
|
||||
clone_path = os.path.join(self.tempdir, 'localpath', 'testproj%d' % i)
|
||||
self._post_clone_bookkeeping(clone_path)
|
||||
# Tell 'repo upload' to upload this project without prompting.
|
||||
config_path = os.path.join(clone_path, '.git', 'config')
|
||||
self.check_call(
|
||||
['git', 'config', '--file', config_path, 'review.%s.upload' %
|
||||
self.gerrit_instance.gerrit_host, 'true'])
|
||||
|
||||
@classmethod
|
||||
def runRepo(cls, *args, **kwargs):
|
||||
# Unfortunately, munging $HOME appears to be the only way to control the
|
||||
# netrc file used by repo.
|
||||
munged_home = os.path.join(cls.gerrit_instance.gerrit_dir, 'tmp')
|
||||
if 'env' not in kwargs:
|
||||
env = kwargs['env'] = os.environ.copy()
|
||||
env['HOME'] = munged_home
|
||||
else:
|
||||
env.setdefault('HOME', munged_home)
|
||||
args[0].insert(0, cls.repo_exe)
|
||||
cls.check_call(*args, **kwargs)
|
||||
|
||||
def uploadChange(self, clone_path, branch='master', remote='origin'):
|
||||
review_host = self.check_output(
|
||||
['git', 'config', 'remote.%s.review' % remote],
|
||||
cwd=clone_path).strip()
|
||||
assert(review_host)
|
||||
projectname = self.check_output(
|
||||
['git', 'config', 'remote.%s.projectname' % remote],
|
||||
cwd=clone_path).strip()
|
||||
assert(projectname)
|
||||
GerritTestCase._UploadChange(
|
||||
clone_path, branch=branch, remote='%s://%s/%s' % (
|
||||
gerrit_util.GERRIT_PROTOCOL, review_host, projectname))
|
||||
Loading…
Reference in New Issue