depot_tools/package_management.py

#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Package Management

This module is used to bring in external Python dependencies via eggs from the
PyPi repository.

The approach is to create a site directory in depot_tools root which will
contain those packages that are listed as dependencies in the module variable
PACKAGES. Since we can't guarantee that setuptools is available in all
distributions this module also contains the ability to bootstrap the site
directory by manually downloading and installing setuptools. Once setuptools is
available it uses that to install the other packages in the traditional
manner.

Use is simple:

  import package_management

  # Before any imports from the site directory, call this. This only needs
  # to be called in one place near the beginning of the program.
  package_management.SetupSiteDirectory()

  # If 'SetupSiteDirectory' fails it will complain with an error message but
  # continue happily. Expect ImportErrors when trying to import any third
  # party modules from the site directory.

  import some_third_party_module

  ... etc ...
"""

import cStringIO
import os
import re
import shutil
import site
import subprocess
import sys
import tempfile
import urllib2


# This is the version of setuptools that we will download if the local
# python distribution does not include one.
SETUPTOOLS = ('setuptools', '0.6c11')

# These are the packages that are to be installed in the site directory.
# easy_install makes it so that the most recently installed version of a
# package is the one that takes precedence, even if a newer version exists
# in the site directory. This allows us to blindly install these one on top
# of the other without worrying about whats already installed.
#
# NOTE: If we are often rolling these dependencies then users' site
#     directories will grow monotonically. We could be purging any orphaned
#     packages using the tools provided by pkg_resources.
PACKAGES = (('logilab-common', '0.57.1'),
            ('logilab-astng', '0.23.1'),
            ('pylint', '0.25.1'))


# The Python version suffix used in generating the site directory and in
# requesting packages from PyPi.
VERSION_SUFFIX = "%d.%d" % sys.version_info[0:2]

# This is the root directory of the depot_tools installation.
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))

# This is the path of the site directory we will use. We make this
# python version specific so that this will continue to work even if the
# python version is rolled.
SITE_DIR = os.path.join(ROOT_DIR, 'site-packages-py%s' % VERSION_SUFFIX)

# This status file is created the last time PACKAGES were rolled successfully.
# It is used to determine if packages need to be rolled by comparing against
# the age of __file__.
LAST_ROLLED = os.path.join(SITE_DIR, 'last_rolled.txt')


class Error(Exception):
  """The base class for all module errors."""
  pass


class InstallError(Error):
  """Thrown if an installation is unable to complete."""
  pass


class Package(object):
  """A package represents a release of a project.

  We use this as a lightweight version of pkg_resources, allowing us to
  perform an end-run around setuptools for the purpose of bootstrapping. Its
  functionality is very limited.

  Attributes:
    name: the name of the package.
    version: the version of the package.
    safe_name: the safe name of the package.
    safe_version: the safe version string of the package.
    file_name: the filename-safe name of the package.
    file_version: the filename-safe version string of the package.
  """

  def __init__(self, name, version):
    """Initialize this package.

    Args:
      name: the name of the package.
      version: the version of the package.
    """
    self.name = name
    self.version = version
    self.safe_name = Package._MakeSafeName(self.name)
    self.safe_version = Package._MakeSafeVersion(self.version)
    self.file_name = Package._MakeSafeForFilename(self.safe_name)
    self.file_version = Package._MakeSafeForFilename(self.safe_version)

  @staticmethod
  def _MakeSafeName(name):
    """Makes a safe package name, as per pkg_resources."""
    return re.sub('[^A-Za-z0-9]+', '-', name)

  @staticmethod
  def _MakeSafeVersion(version):
    """Makes a safe package version string, as per pkg_resources."""
    version = re.sub('\s+', '.', version)
    return re.sub('[^A-Za-z0-9\.]+', '-', version)

  @staticmethod
  def _MakeSafeForFilename(safe_name_or_version):
    """Makes a safe name or safe version safe to use in a file name.
    |safe_name_or_version| must be a safe name or version string as returned
    by GetSafeName or GetSafeVersion.
    """
    return re.sub('-', '_', safe_name_or_version)

  def GetAsRequirementString(self):
    """Builds an easy_install requirements string representing this package."""
    return '%s==%s' % (self.name, self.version)

  def GetFilename(self, extension=None):
    """Builds a filename for this package using the setuptools convention.
    If |extensions| is provided it will be appended to the generated filename,
    otherwise only the basename is returned.

    The following url discusses the filename format:

      http://svn.python.org/projects/sandbox/trunk/setuptools/doc/formats.txt
    """
    filename = '%s-%s-py%s' % (self.file_name, self.file_version,
                               VERSION_SUFFIX)

    if extension:
      if not extension.startswith('.'):
        filename += '.'
      filename += extension

    return filename

  def GetPyPiUrl(self, extension):
    """Returns the URL where this package is hosted on PyPi."""
    return 'http://pypi.python.org/packages/%s/%c/%s/%s' % (VERSION_SUFFIX,
        self.file_name[0], self.file_name, self.GetFilename(extension))

  def DownloadEgg(self, dest_dir, overwrite=False):
    """Downloads the EGG for this URL.

    Args:
      dest_dir: The directory where the EGG should be written. If the EGG
          has already been downloaded and cached the returned path may not
          be in this directory.
      overwite: If True the destination path will be overwritten even if
          it already exists. Defaults to False.

    Returns:
      The path to the written EGG.

    Raises:
      Error: if dest_dir doesn't exist, the EGG is unable to be written,
          the URL doesn't exist, or the server returned an error, or the
          transmission was interrupted.
    """
    if not os.path.exists(dest_dir):
      raise Error('Path does not exist: %s' % dest_dir)

    if not os.path.isdir(dest_dir):
      raise Error('Path is not a directory: %s' % dest_dir)

    filename = os.path.abspath(os.path.join(dest_dir, self.GetFilename('egg')))
    if os.path.exists(filename):
      if os.path.isdir(filename):
        raise Error('Path is a directory: %s' % filename)
      if not overwrite:
        return filename

    url = self.GetPyPiUrl('egg')

    try:
      url_stream = urllib2.urlopen(url)
      local_file = open(filename, 'wb')
      local_file.write(url_stream.read())
      local_file.close()
      return filename
    except (IOError, urllib2.HTTPError, urllib2.URLError):
      # Reraise with a new error type, keeping the original message and
      # traceback.
      raise Error, sys.exc_info()[1], sys.exc_info()[2]


def AddToPythonPath(path):
  """Adds the provided path to the head of PYTHONPATH and sys.path."""
  path = os.path.abspath(path)
  if path not in sys.path:
    sys.path.insert(0, path)

  paths = os.environ.get('PYTHONPATH', '').split(os.pathsep)
  if path not in paths:
    paths.insert(0, path)
    os.environ['PYTHONPATH'] = os.pathsep.join(paths)


def AddSiteDirectory(path):
  """Adds the provided path to the runtime as a site directory.

  Any modules that are in the site directory will be available for importing
  after this returns. If modules are added or deleted this must be called
  again for the changes to be reflected in the runtime.

  This calls both AddToPythonPath and site.addsitedir. Both are needed to
  convince easy_install to treat |path| as a site directory.
  """
  AddToPythonPath(path)
  site.addsitedir(path)  # pylint: disable=E1101

def EnsureSiteDirectory(path):
  """Creates and/or adds the provided path to the runtime as a site directory.

  This works like AddSiteDirectory but it will create the directory if it
  does not yet exist.

  Raise:
    Error: if the site directory is unable to be created, or if it exists and
        is not a directory.
  """
  if os.path.exists(path):
    if not os.path.isdir(path):
      raise Error('Path is not a directory: %s' % path)
  else:
    try:
      os.mkdir(path)
    except IOError:
      raise Error('Unable to create directory: %s' % path)

  AddSiteDirectory(path)


def ModuleIsFromPackage(module, package_path):
  """Determines if a module has been imported from a given package.

  Args:
    module: the module to test.
    package_path: the path to the package to test.

  Returns:
    True if |module| has been imported from |package_path|, False otherwise.
  """
  try:
    m = os.path.abspath(module.__file__)
    p = os.path.abspath(package_path)
    if len(m) <= len(p):
      return False
    if m[0:len(p)] != p:
      return False
    return m[len(p)] == os.sep
  except AttributeError:
    return False


def _CaptureStdStreams(function, *args, **kwargs):
  """Captures stdout and stderr while running the provided function.

  This only works if |function| only accesses sys.stdout and sys.stderr. If
  we need more than this we'll have to use subprocess.Popen.

  Args:
    function: the function to be called.
    args: the arguments to pass to |function|.
    kwargs: the keyword arguments to pass to |function|.
  """
  orig_stdout = sys.stdout
  orig_stderr = sys.stderr
  sys.stdout = cStringIO.StringIO()
  sys.stderr = cStringIO.StringIO()
  try:
    return function(*args, **kwargs)
  finally:
    sys.stdout = orig_stdout
    sys.stderr = orig_stderr


def InstallPackage(url_or_req, site_dir):
  """Installs a package to a site directory.

  |site_dir| must exist and already be an active site directory. setuptools
  must in the path. Uses easy_install which may involve a download from
  pypi.python.org, so this also requires network access.

  Args:
    url_or_req: the package to install, expressed as an URL (may be local),
        or a requirement string.
    site_dir: the site directory in which to install it.

  Raises:
    InstallError: if installation fails for any reason.
  """
  args = ['--quiet', '--install-dir', site_dir, '--exclude-scripts',
          '--always-unzip', '--no-deps', url_or_req]

  # The easy_install script only calls SystemExit if something goes wrong.
  # Otherwise, it falls through returning None.
  try:
    import setuptools.command.easy_install
    _CaptureStdStreams(setuptools.command.easy_install.main, args)
  except (ImportError, SystemExit):
    # Re-raise the error, preserving the stack trace and message.
    raise InstallError, sys.exc_info()[1], sys.exc_info()[2]


def _RunInSubprocess(pycode):
  """Launches a python subprocess with the provided code.

  The subprocess will be launched with the same stdout and stderr. The
  subprocess will use the same instance of python as is currently running,
  passing |pycode| as arguments to this script. |pycode| will be interpreted
  as python code in the context of this module.

  Returns:
    True if the subprocess returned 0, False if it returned an error.
  """
  return not subprocess.call([sys.executable, __file__, pycode])


def _LoadSetupToolsFromEggAndInstall(egg_path):
  """Loads setuptools from the provided egg |egg_path|, and installs it to
  SITE_DIR.

  This is intended to be run from a subprocess as it pollutes the running
  instance of Python by importing a module and then forcibly deleting its
  source.

  Returns:
    True on success, False on failure.
  """
  AddToPythonPath(egg_path)

  try:
    # Import setuptools and ensure it comes from the EGG.
    import setuptools
    if not ModuleIsFromPackage(setuptools, egg_path):
      raise ImportError()
  except ImportError:
    print '  Unable to import downloaded package!'
    return False

  try:
    print '  Using setuptools to install itself ...'
    InstallPackage(egg_path, SITE_DIR)
  except InstallError:
    print '  Unable to install setuptools!'
    return False

  return True


def BootstrapSetupTools():
  """Bootstraps the runtime with setuptools.

  Will try to import setuptools directly. If not found it will attempt to
  download it and load it from there. If the download is successful it will
  then use setuptools to install itself in the site directory.

  This is meant to be run from a child process as it modifies the running
  instance of Python by importing modules and then physically deleting them
  from disk.

  Returns:
    Returns True if 'import setuptools' will succeed, False otherwise.
  """
  AddSiteDirectory(SITE_DIR)

  # Check if setuptools is already available. If so, we're done.
  try:
    import setuptools  # pylint: disable=W0612
    return True
  except ImportError:
    pass

  print 'Bootstrapping setuptools ...'

  EnsureSiteDirectory(SITE_DIR)

  # Download the egg to a temp directory.
  dest_dir = tempfile.mkdtemp('depot_tools')
  path = None
  try:
    package = Package(*SETUPTOOLS)
    print '  Downloading %s ...' % package.GetFilename()
    path = package.DownloadEgg(dest_dir)
  except Error:
    print '  Download failed!'
    shutil.rmtree(dest_dir)
    return False

  try:
    # Load the downloaded egg, and install it to the site directory. Do this
    # in a subprocess so as not to pollute this runtime.
    pycode = '_LoadSetupToolsFromEggAndInstall(%s)' % repr(path)
    if not _RunInSubprocess(pycode):
      raise Error()

    # Reload our site directory, which should now contain setuptools.
    AddSiteDirectory(SITE_DIR)

    # Try to import setuptools
    import setuptools
  except ImportError:
    print '  Unable to import setuptools!'
    return False
  except Error:
    # This happens if RunInSubProcess fails, and the appropriate error has
    # already been written to stdout.
    return False
  finally:
    # Delete the temp directory.
    shutil.rmtree(dest_dir)

  return True


def _GetModTime(path):
  """Gets the last modification time associated with |path| in seconds since
  epoch, returning 0 if |path| does not exist.
  """
  try:
    return os.stat(path).st_mtime
  except:  # pylint: disable=W0702
    # This error is different depending on the OS, hence no specified type.
    return 0


def _SiteDirectoryIsUpToDate():
  return _GetModTime(LAST_ROLLED) > _GetModTime(__file__)


def UpdateSiteDirectory():
  """Installs the packages from PACKAGES if they are not already installed.
  At this point we must have setuptools in the site directory.

  This is intended to be run in a subprocess *prior* to the site directory
  having been added to the parent process as it may cause packages to be
  added and/or removed.

  Returns:
    True on success, False otherwise.
  """
  if _SiteDirectoryIsUpToDate():
    return True

  try:
    EnsureSiteDirectory(SITE_DIR)
    import pkg_resources

    # Determine if any packages actually need installing.
    missing_packages = []
    for package in [SETUPTOOLS] + list(PACKAGES):
      pkg = Package(*package)
      req = pkg.GetAsRequirementString()

      # It may be that this package is already available in the site
      # directory. If so, we can skip past it without trying to install it.
      pkg_req = pkg_resources.Requirement.parse(req)
      try:
        dist = pkg_resources.working_set.find(pkg_req)
        if dist:
          continue
      except pkg_resources.VersionConflict:
        # This happens if another version of the package is already
        # installed in another site directory (ie: the system site directory).
        pass

      missing_packages.append(pkg)

    # Install the missing packages.
    if missing_packages:
      print 'Updating python packages ...'
      for pkg in missing_packages:
        print '  Installing %s ...' % pkg.GetFilename()
        InstallPackage(pkg.GetAsRequirementString(), SITE_DIR)

    # Touch the status file so we know that we're up to date next time.
    open(LAST_ROLLED, 'wb')
  except InstallError, e:
    print '  Installation failed: %s' % str(e)
    return False

  return True


def SetupSiteDirectory():
  """Sets up the site directory, bootstrapping setuptools if necessary.

  If this finishes successfully then SITE_DIR will exist and will contain
  the appropriate version of setuptools and all of the packages listed in
  PACKAGES.

  This is the main workhorse of this module. Calling this will do everything
  necessary to ensure that you have the desired packages installed in the
  site directory, and the site directory enabled in this process.

  Returns:
    True on success, False on failure.
  """
  if _SiteDirectoryIsUpToDate():
    AddSiteDirectory(SITE_DIR)
    return True

  if not _RunInSubprocess('BootstrapSetupTools()'):
    return False

  if not _RunInSubprocess('UpdateSiteDirectory()'):
    return False

  # Process the site directory so that the packages within it are available
  # for import.
  AddSiteDirectory(SITE_DIR)

  return True


def CanImportFromSiteDirectory(package_name):
  """Determines if the given package can be imported from the site directory.

  Args:
    package_name: the name of the package to import.

  Returns:
    True if 'import package_name' will succeed and return a module from the
    site directory, False otherwise.
  """
  try:
    return ModuleIsFromPackage(__import__(package_name), SITE_DIR)
  except ImportError:
    return False


def Test():
  """Runs SetupSiteDirectory and then tries to load pylint, ensuring that it
  comes from the site directory just created. This is an end-to-end unittest
  and allows for simple testing from the command-line by running

    ./package_management.py 'Test()'
  """
  print 'Testing package_management.'
  if not SetupSiteDirectory():
    print 'SetupSiteDirectory failed.'
    return False
  if not CanImportFromSiteDirectory('pylint'):
    print 'CanImportFromSiteDirectory failed.'
    return False
  print 'Success!'
  return True


def Main():
  """The main entry for the package management script.

  If no arguments are provided simply runs SetupSiteDirectory. If arguments
  have been passed we execute the first argument as python code in the
  context of this module. This mechanism is used during the bootstrap
  process so that the main instance of Python does not have its runtime
  polluted by various intermediate packages and imports.

  Returns:
    0 on success, 1 otherwise.
  """
  if len(sys.argv) == 2:
    result = False
    exec('result = %s' % sys.argv[1])

    # Translate the success state to a return code.
    return not result
  else:
    return not SetupSiteDirectory()


if __name__ == '__main__':
  sys.exit(Main())