depot_tools/third_party/gsutil/gslib/commands/chacl.py

# Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
This module provides the chacl command to gsutil.

This command allows users to easily specify changes to access control lists.
"""

import random
import re
import time
from xml.dom import minidom
from boto.exception import GSResponseError
from boto.gs import acl
from gslib import name_expansion
from gslib.command import Command
from gslib.command import COMMAND_NAME
from gslib.command import COMMAND_NAME_ALIASES
from gslib.command import CONFIG_REQUIRED
from gslib.command import FILE_URIS_OK
from gslib.command import MAX_ARGS
from gslib.command import MIN_ARGS
from gslib.command import PROVIDER_URIS_OK
from gslib.command import SUPPORTED_SUB_ARGS
from gslib.command import URIS_START_ARG
from gslib.exception import CommandException
from gslib.help_provider import HELP_NAME
from gslib.help_provider import HELP_NAME_ALIASES
from gslib.help_provider import HELP_ONE_LINE_SUMMARY
from gslib.help_provider import HELP_TEXT
from gslib.help_provider import HELP_TYPE
from gslib.help_provider import HelpType
from gslib.util import NO_MAX
from gslib.util import Retry


class ChangeType(object):
  USER = 'User'
  GROUP = 'Group'


class AclChange(object):
  """Represents a logical change to an access control list."""
  public_scopes = ['AllAuthenticatedUsers', 'AllUsers']
  id_scopes = ['UserById', 'GroupById']
  email_scopes = ['UserByEmail', 'GroupByEmail']
  domain_scopes = ['GroupByDomain']
  scope_types = public_scopes + id_scopes + email_scopes + domain_scopes

  permission_shorthand_mapping = {
      'R': 'READ',
      'W': 'WRITE',
      'FC': 'FULL_CONTROL',
      }

  def __init__(self, acl_change_descriptor, scope_type, logger):
    """Creates an AclChange object.

    acl_change_descriptor: An acl change as described in chacl help.
    scope_type: Either ChangeType.USER or ChangeType.GROUP, specifying the
                extent of the scope.
    logger: An instance of ThreadedLogger.
    """
    self.logger = logger
    self.identifier = ''

    self.raw_descriptor = acl_change_descriptor
    self._Parse(acl_change_descriptor, scope_type)
    self._Validate()

  def __str__(self):
    return 'AclChange<{0}|{1}|{2}>'.format(self.scope_type, self.perm,
                                        self.identifier)

  def _Parse(self, change_descriptor, scope_type):
    """Parses an ACL Change descriptor."""

    def _ClassifyScopeIdentifier(text):
      re_map = {
          'AllAuthenticatedUsers': r'^(AllAuthenticatedUsers|AllAuth)$',
          'AllUsers': '^(AllUsers|All)$',
          'Email': r'^.+@.+\..+$',
          'Id': r'^[0-9A-Fa-f]{64}$',
          'Domain': r'^[^@]+\..+$',
          }
      for type_string, regex in re_map.items():
        if re.match(regex, text, re.IGNORECASE):
          return type_string

    if change_descriptor.count(':') != 1:
      raise CommandException('{0} is an invalid change description.'
                             .format(change_descriptor))

    scope_string, perm_token = change_descriptor.split(':')

    perm_token = perm_token.upper()
    if perm_token in self.permission_shorthand_mapping:
      self.perm = self.permission_shorthand_mapping[perm_token]
    else:
      self.perm = perm_token

    scope_class = _ClassifyScopeIdentifier(scope_string)
    if scope_class == 'Domain':
      # This may produce an invalid UserByDomain scope,
      # which is good because then validate can complain.
      self.scope_type = '{0}ByDomain'.format(scope_type)
      self.identifier = scope_string
    elif scope_class in ['Email', 'Id']:
      self.scope_type = '{0}By{1}'.format(scope_type, scope_class)
      self.identifier = scope_string
    elif scope_class == 'AllAuthenticatedUsers':
      self.scope_type = 'AllAuthenticatedUsers'
    elif scope_class == 'AllUsers':
      self.scope_type = 'AllUsers'
    else:
      # This is just a fallback, so we set it to something
      # and the validate step has something to go on.
      self.scope_type = scope_string

  def _Validate(self):
    """Validates a parsed AclChange object."""

    def _ThrowError(msg):
      raise CommandException('{0} is not a valid ACL change\n{1}'
                             .format(self.raw_descriptor, msg))

    if self.scope_type not in self.scope_types:
      _ThrowError('{0} is not a valid scope type'.format(self.scope_type))

    if self.scope_type in self.public_scopes and self.identifier:
      _ThrowError('{0} requires no arguments'.format(self.scope_type))

    if self.scope_type in self.id_scopes and not self.identifier:
      _ThrowError('{0} requires an id'.format(self.scope_type))

    if self.scope_type in self.email_scopes and not self.identifier:
      _ThrowError('{0} requires an email address'.format(self.scope_type))

    if self.scope_type in self.domain_scopes and not self.identifier:
      _ThrowError('{0} requires domain'.format(self.scope_type))

    if self.perm not in self.permission_shorthand_mapping.values():
      perms = ', '.join(self.permission_shorthand_mapping.values())
      _ThrowError('Allowed permissions are {0}'.format(perms))

  def _YieldMatchingEntries(self, current_acl):
    """Generator that yields entries that match the change descriptor.

    current_acl: An instance of bogo.gs.acl.ACL which will be searched
                 for matching entries.
    """
    for entry in current_acl.entries.entry_list:
      if entry.scope.type == self.scope_type:
        if self.scope_type in ['UserById', 'GroupById']:
          if self.identifier == entry.scope.id:
            yield entry
        elif self.scope_type in ['UserByEmail', 'GroupByEmail']:
          if self.identifier == entry.scope.email_address:
            yield entry
        elif self.scope_type == 'GroupByDomain':
          if self.identifier == entry.scope.domain:
            yield entry
        elif self.scope_type in ['AllUsers', 'AllAuthenticatedUsers']:
          yield entry
        else:
          raise CommandException('Found an unrecognized ACL '
                                 'entry type, aborting.')

  def _AddEntry(self, current_acl):
    """Adds an entry to an ACL."""
    if self.scope_type in ['UserById', 'UserById', 'GroupById']:
      entry = acl.Entry(type=self.scope_type, permission=self.perm,
                        id=self.identifier)
    elif self.scope_type in ['UserByEmail', 'GroupByEmail']:
      entry = acl.Entry(type=self.scope_type, permission=self.perm,
                        email_address=self.identifier)
    elif self.scope_type == 'GroupByDomain':
      entry = acl.Entry(type=self.scope_type, permission=self.perm,
                        domain=self.identifier)
    else:
      entry = acl.Entry(type=self.scope_type, permission=self.perm)

    current_acl.entries.entry_list.append(entry)

  def Execute(self, uri, current_acl):
    """Executes the described change on an ACL.

    uri: The URI object to change.
    current_acl: An instance of boto.gs.acl.ACL to permute.
    """
    self.logger.debug('Executing {0} on {1}'
                      .format(self.raw_descriptor, uri))

    if self.perm == 'WRITE' and uri.names_object():
      self.logger.warn(
          'Skipping {0} on {1}, as WRITE does not apply to objects'
          .format(self.raw_descriptor, uri))
      return 0

    matching_entries = list(self._YieldMatchingEntries(current_acl))
    change_count = 0
    if matching_entries:
      for entry in matching_entries:
        if entry.permission != self.perm:
          entry.permission = self.perm
          change_count += 1
    else:
      self._AddEntry(current_acl)
      change_count = 1

    parsed_acl = minidom.parseString(current_acl.to_xml())
    self.logger.debug('New Acl:\n{0}'.format(parsed_acl.toprettyxml()))
    return change_count


class AclDel(AclChange):
  """Represents a logical change from an access control list."""
  scope_regexes = {
      r'All(Users)?': 'AllUsers',
      r'AllAuth(enticatedUsers)?': 'AllAuthenticatedUsers',
      }

  def __init__(self, identifier, logger):
    self.raw_descriptor = '-d {0}'.format(identifier)
    self.logger = logger
    self.identifier = identifier
    for regex, scope in self.scope_regexes.items():
      if re.match(regex, self.identifier, re.IGNORECASE):
        self.identifier = scope
    self.scope_type = 'Any'
    self.perm = 'NONE'

  def _YieldMatchingEntries(self, current_acl):
    for entry in current_acl.entries.entry_list:
      if self.identifier == entry.scope.id:
        yield entry
      elif self.identifier == entry.scope.email_address:
        yield entry
      elif self.identifier == entry.scope.domain:
        yield entry
      elif self.identifier == 'AllUsers' and entry.scope.type == 'AllUsers':
        yield entry
      elif (self.identifier == 'AllAuthenticatedUsers'
            and entry.scope.type == 'AllAuthenticatedUsers'):
        yield entry

  def Execute(self, uri, current_acl):
    self.logger.debug('Executing {0} on {1}'
                      .format(self.raw_descriptor, uri))
    matching_entries = list(self._YieldMatchingEntries(current_acl))
    for entry in matching_entries:
      current_acl.entries.entry_list.remove(entry)
    parsed_acl = minidom.parseString(current_acl.to_xml())
    self.logger.debug('New Acl:\n{0}'.format(parsed_acl.toprettyxml()))
    return len(matching_entries)


_detailed_help_text = ("""
<B>SYNOPSIS</B>
  gsutil chacl [-R] -u|-g|-d <grant>... uri...

  where each <grant> is one of the following forms:
  -u <id|email>:<perm>
  -g <id|email|domain|All|AllAuth>:<perm>
  -d <id|email|domain|All|AllAuth>

<B>DESCRIPTION</B>
  The chacl command updates access control lists, similar in spirit to the Linux
  chmod command. You can specify multiple access grant additions and deletions
  in a single command run; all changes will be made atomically to each object in
  turn. For example, if the command requests deleting one grant and adding a
  different grant, the ACLs being updated will never be left in an intermediate
  state where one grant has been deleted but the second grant not yet added.
  Each change specifies a user or group grant to add or delete, and for grant
  additions, one of R, W, FC (for the permission to be granted). A more formal
  description is provided in a later section; below we provide examples.

  Note: If you want to set a simple "canned" ACL on each object (such as
  project-private or public), or if you prefer to edit the XML representation
  for ACLs, you can do that with the setacl command (see 'gsutil help setacl').


<B>EXAMPLES</B>

  Grant the user john.doe@example.com WRITE access to the bucket
  example-bucket:

    gsutil chacl -u john.doe@example.com:WRITE gs://example-bucket

  Grant the group admins@example.com FULL_CONTROL access to all jpg files in
  the top level of example-bucket:

    gsutil chacl -g admins@example.com:FC gs://example-bucket/*.jpg

  Grant the user with the specified canonical ID READ access to all objects in
  example-bucket that begin with folder/:

    gsutil chacl -R \\
      -u 84fac329bceSAMPLE777d5d22b8SAMPLE77d85ac2SAMPLE2dfcf7c4adf34da46:R \\
      gs://example-bucket/folder/

  Grant all users from my-domain.org READ access to the bucket
  gcs.my-domain.org:

    gsutil chacl -g my-domain.org:R gs://gcs.my-domain.org

  Remove any current access by john.doe@example.com from the bucket
  example-bucket:

    gsutil chacl -d john.doe@example.com gs://example-bucket

  If you have a large number of objects to update, enabling multi-threading with
  the gsutil -m flag can significantly improve performance. The following
  command adds FULL_CONTROL for admin@example.org using multi-threading:

    gsutil -m chacl -R -u admin@example.org:FC gs://example-bucket

  Grant READ access to everyone from my-domain.org and to all authenticated
  users, and grant FULL_CONTROL to admin@mydomain.org, for the buckets
  my-bucket and my-other-bucket, with multi-threading enabled:

    gsutil -m chacl -R -g my-domain.org:R -g AllAuth:R \\
      -u admin@mydomain.org:FC gs://my-bucket/ gs://my-other-bucket


<B>SCOPES</B>
  There are four different scopes: Users, Groups, All Authenticated Users, and
  All Users.

  Users are added with -u and a plain ID or email address, as in
  "-u john-doe@gmail.com:r"

  Groups are like users, but specified with the -g flag, as in
  "-g power-users@example.com:fc". Groups may also be specified as a full
  domain, as in "-g my-company.com:r".

  AllAuthenticatedUsers and AllUsers are specified directly, as
  in "-g AllUsers:R" or "-g AllAuthenticatedUsers:FC". These are case
  insensitive, and may be shortened to "all" and "allauth", respectively.

  Removing permissions is specified with the -d flag and an ID, email
  address, domain, or one of AllUsers or AllAuthenticatedUsers.

  Many scopes can be specified on the same command line, allowing bundled
  changes to be executed in a single run. This will reduce the number of
  requests made to the server.


<B>PERMISSIONS</B>
  You may specify the following permissions with either their shorthand or
  their full name:

  R: READ
  W: WRITE
  FC: FULL_CONTROL


<B>OPTIONS</B>
  -R, -r      Performs chacl request recursively, to all objects under the
              specified URI.

  -u          Add or modify a user permission as specified in the SCOPES
              and PERMISSIONS sections.

  -g          Add or modify a group permission as specified in the SCOPES
              and PERMISSIONS sections.

  -d          Remove all permissions associated with the matching argument, as
              specified in the SCOPES and PERMISSIONS sections.
""")


class ChAclCommand(Command):
  """Implementation of gsutil chacl command."""

  # Command specification (processed by parent class).
  command_spec = {
    # Name of command.
    COMMAND_NAME : 'chacl',
    # List of command name aliases.
    COMMAND_NAME_ALIASES : [],
    # Min number of args required by this command.
    MIN_ARGS : 1,
    # Max number of args required by this command, or NO_MAX.
    MAX_ARGS : NO_MAX,
    # Getopt-style string specifying acceptable sub args.
    SUPPORTED_SUB_ARGS : 'Rrfg:u:d:',
    # True if file URIs acceptable for this command.
    FILE_URIS_OK : False,
    # True if provider-only URIs acceptable for this command.
    PROVIDER_URIS_OK : False,
    # Index in args of first URI arg.
    URIS_START_ARG : 1,
    # True if must configure gsutil before running command.
    CONFIG_REQUIRED : True,
  }
  help_spec = {
    # Name of command or auxiliary help info for which this help applies.
    HELP_NAME : 'chacl',
    # List of help name aliases.
    HELP_NAME_ALIASES : ['chmod'],
    # Type of help:
    HELP_TYPE : HelpType.COMMAND_HELP,
    # One line summary of this help.
    HELP_ONE_LINE_SUMMARY : 'Add / remove entries on bucket and/or object ACLs',
    # The full help text.
    HELP_TEXT : _detailed_help_text,
  }

  # Command entry point.
  def RunCommand(self):
    """This is the point of entry for the chacl command."""
    self.parse_versions = True
    self.changes = []

    if self.sub_opts:
      for o, a in self.sub_opts:
        if o == '-g':
          self.changes.append(AclChange(a, scope_type=ChangeType.GROUP,
                                        logger=self.THREADED_LOGGER))
        if o == '-u':
          self.changes.append(AclChange(a, scope_type=ChangeType.USER,
                                        logger=self.THREADED_LOGGER))
        if o == '-d':
          self.changes.append(AclDel(a, logger=self.THREADED_LOGGER))

    if not self.changes:
      raise CommandException(
          'Please specify at least one access change '
          'with the -g, -u, or -d flags')

    storage_uri = self.UrisAreForSingleProvider(self.args)
    if not (storage_uri and storage_uri.get_provider().name == 'google'):
      raise CommandException('The "{0}" command can only be used with gs:// URIs'
                             .format(self.command_name))

    bulk_uris = set()
    for uri_arg in self.args:
      for result in self.WildcardIterator(uri_arg):
        uri = result.uri
        if uri.names_bucket():
          if self.recursion_requested:
            bulk_uris.add(uri.clone_replace_name('*').uri)
          else:
            # If applying to a bucket directly, the threading machinery will
            # break, so we have to apply now, in the main thread.
            self.ApplyAclChanges(uri)
        else:
          bulk_uris.add(uri_arg)

    try:
      name_expansion_iterator = name_expansion.NameExpansionIterator(
          self.command_name, self.proj_id_handler, self.headers, self.debug,
          self.bucket_storage_uri_class, bulk_uris, self.recursion_requested)
    except CommandException as e:
      # NameExpansionIterator will complain if there are no URIs, but we don't
      # want to throw an error if we handled bucket URIs.
      if e.reason == 'No URIs matched':
        return 0
      else:
        raise e

    self.everything_set_okay = True
    self.Apply(self.ApplyAclChanges,
               name_expansion_iterator,
               self._ApplyExceptionHandler)
    if not self.everything_set_okay:
      raise CommandException('ACLs for some objects could not be set.')

    return 0

  def _ApplyExceptionHandler(self, exception):
    self.THREADED_LOGGER.error('Encountered a problem: {0}'.format(exception))
    self.everything_set_okay = False

  @Retry(GSResponseError, tries=3, delay=1, backoff=2)
  def ApplyAclChanges(self, uri_or_expansion_result):
    """Applies the changes in self.changes to the provided URI."""
    if isinstance(uri_or_expansion_result, name_expansion.NameExpansionResult):
      uri = self.suri_builder.StorageUri(
          uri_or_expansion_result.expanded_uri_str)
    else:
      uri = uri_or_expansion_result

    try:
      current_acl = uri.get_acl()
    except GSResponseError as e:
      self.THREADED_LOGGER.warning('Failed to set acl for {0}: {1}'
                                   .format(uri, e.reason))
      return

    modification_count = 0
    for change in self.changes:
      modification_count += change.Execute(uri, current_acl)
    if modification_count == 0:
      self.THREADED_LOGGER.info('No changes to {0}'.format(uri))
      return

    # TODO: Remove the concept of forcing when boto provides access to
    # bucket generation and meta_generation.
    headers = dict(self.headers)
    force = uri.names_bucket()
    if not force:
      key = uri.get_key()
      headers['x-goog-if-generation-match'] = key.generation
      headers['x-goog-if-metageneration-match'] = key.meta_generation
    
    # If this fails because of a precondition, it will raise a 
    # GSResponseError for @Retry to handle.
    uri.set_acl(current_acl, uri.object_name, False, headers)
    self.THREADED_LOGGER.info('Updated ACL on {0}'.format(uri))