You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
499 lines
20 KiB
Python
499 lines
20 KiB
Python
# Copyright 2010 Google Inc. All Rights Reserved.
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a
|
|
# copy of this software and associated documentation files (the
|
|
# "Software"), to deal in the Software without restriction, including
|
|
# without limitation the rights to use, copy, modify, merge, publish, dis-
|
|
# tribute, sublicense, and/or sell copies of the Software, and to permit
|
|
# persons to whom the Software is furnished to do so, subject to the fol-
|
|
# lowing conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included
|
|
# in all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
|
|
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
|
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
# IN THE SOFTWARE.
|
|
|
|
"""Implementation of wildcarding over StorageUris.
|
|
|
|
StorageUri is an abstraction that Google introduced in the boto library,
|
|
for representing storage provider-independent bucket and object names with
|
|
a shorthand URI-like syntax (see boto/boto/storage_uri.py) The current
|
|
class provides wildcarding support for StorageUri objects (including both
|
|
bucket and file system objects), allowing one to express collections of
|
|
objects with syntax like the following:
|
|
gs://mybucket/images/*.png
|
|
file:///tmp/???abc???
|
|
|
|
We provide wildcarding support as part of gsutil rather than as part
|
|
of boto because wildcarding is really part of shell command-like
|
|
functionality.
|
|
|
|
A comment about wildcard semantics: We support both single path component
|
|
wildcards (e.g., using '*') and recursive wildcards (using '**'), for both
|
|
file and cloud URIs. For example,
|
|
gs://bucket/doc/*/*.html
|
|
would enumerate HTML files one directory down from gs://bucket/doc, while
|
|
gs://bucket/**/*.html
|
|
would enumerate HTML files in all objects contained in the bucket.
|
|
|
|
Note also that if you use file system wildcards it's likely your shell
|
|
interprets the wildcarding before passing the command to gsutil. For example:
|
|
% gsutil cp /opt/eclipse/*/*.html gs://bucket/eclipse
|
|
would likely be expanded by the shell into the following before running gsutil:
|
|
% gsutil cp /opt/eclipse/RUNNING.html gs://bucket/eclipse
|
|
|
|
Note also that most shells don't support '**' wildcarding (I think only
|
|
zsh does). If you want to use '**' wildcarding with such a shell you can
|
|
single quote each wildcarded string, so it gets passed uninterpreted by the
|
|
shell to gsutil (at which point gsutil will perform the wildcarding expansion):
|
|
% gsutil cp '/opt/eclipse/**/*.html' gs://bucket/eclipse
|
|
"""
|
|
|
|
import boto
|
|
import fnmatch
|
|
import glob
|
|
import os
|
|
import re
|
|
import sys
|
|
import urllib
|
|
|
|
from boto.s3.prefix import Prefix
|
|
from boto.storage_uri import BucketStorageUri
|
|
from bucket_listing_ref import BucketListingRef
|
|
|
|
# Regex to determine if a string contains any wildcards.
|
|
WILDCARD_REGEX = re.compile('[*?\[\]]')
|
|
|
|
WILDCARD_OBJECT_ITERATOR = 'wildcard_object_iterator'
|
|
WILDCARD_BUCKET_ITERATOR = 'wildcard_bucket_iterator'
|
|
|
|
|
|
class WildcardIterator(object):
|
|
"""Base class for wildcarding over StorageUris.
|
|
|
|
This class implements support for iterating over StorageUris that
|
|
contain wildcards.
|
|
|
|
The base class is abstract; you should instantiate using the
|
|
wildcard_iterator() static factory method, which chooses the right
|
|
implementation depending on the StorageUri.
|
|
"""
|
|
|
|
def __repr__(self):
|
|
"""Returns string representation of WildcardIterator."""
|
|
return 'WildcardIterator(%s)' % self.wildcard_uri
|
|
|
|
|
|
class CloudWildcardIterator(WildcardIterator):
|
|
"""WildcardIterator subclass for buckets and objects.
|
|
|
|
Iterates over BucketListingRef matching the StorageUri wildcard. It's
|
|
much more efficient to request the Key from the BucketListingRef (via
|
|
GetKey()) than to request the StorageUri and then call uri.get_key()
|
|
to retrieve the key, for cases where you want to get metadata that's
|
|
available in the Bucket (for example to get the name and size of
|
|
each object), because that information is available in the bucket GET
|
|
results. If you were to iterate over URIs for such cases and then get
|
|
the name and size info from each resulting StorageUri, it would cause
|
|
an additional object GET request for each of the result URIs.
|
|
"""
|
|
|
|
def __init__(self, wildcard_uri, proj_id_handler,
|
|
bucket_storage_uri_class=BucketStorageUri, all_versions=False,
|
|
headers=None, debug=0):
|
|
"""
|
|
Instantiates an iterator over BucketListingRef matching given wildcard URI.
|
|
|
|
Args:
|
|
wildcard_uri: StorageUri that contains the wildcard to iterate.
|
|
proj_id_handler: ProjectIdHandler to use for current command.
|
|
bucket_storage_uri_class: BucketStorageUri interface.
|
|
Settable for testing/mocking.
|
|
headers: Dictionary containing optional HTTP headers to pass to boto.
|
|
debug: Debug level to pass in to boto connection (range 0..3).
|
|
"""
|
|
self.wildcard_uri = wildcard_uri
|
|
# Make a copy of the headers so any updates we make during wildcard
|
|
# expansion aren't left in the input params (specifically, so we don't
|
|
# include the x-goog-project-id header needed by a subset of cases, in
|
|
# the data returned to caller, which could then be used in other cases
|
|
# where that header must not be passed).
|
|
if headers is None:
|
|
self.headers = {}
|
|
else:
|
|
self.headers = headers.copy()
|
|
self.proj_id_handler = proj_id_handler
|
|
self.bucket_storage_uri_class = bucket_storage_uri_class
|
|
self.all_versions = all_versions
|
|
self.debug = debug
|
|
|
|
def __iter__(self):
|
|
"""Python iterator that gets called when iterating over cloud wildcard.
|
|
|
|
Yields:
|
|
BucketListingRef, or empty iterator if no matches.
|
|
"""
|
|
# First handle bucket wildcarding, if any.
|
|
if ContainsWildcard(self.wildcard_uri.bucket_name):
|
|
regex = fnmatch.translate(self.wildcard_uri.bucket_name)
|
|
bucket_uris = []
|
|
prog = re.compile(regex)
|
|
self.proj_id_handler.FillInProjectHeaderIfNeeded(WILDCARD_BUCKET_ITERATOR,
|
|
self.wildcard_uri,
|
|
self.headers)
|
|
for b in self.wildcard_uri.get_all_buckets(headers=self.headers):
|
|
if prog.match(b.name):
|
|
# Use str(b.name) because get_all_buckets() returns Unicode
|
|
# string, which when used to construct x-goog-copy-src metadata
|
|
# requests for object-to-object copies causes pathname '/' chars
|
|
# to be entity-encoded (bucket%2Fdir instead of bucket/dir),
|
|
# which causes the request to fail.
|
|
uri_str = '%s://%s' % (self.wildcard_uri.scheme,
|
|
urllib.quote_plus(str(b.name)))
|
|
bucket_uris.append(
|
|
boto.storage_uri(
|
|
uri_str, debug=self.debug,
|
|
bucket_storage_uri_class=self.bucket_storage_uri_class,
|
|
suppress_consec_slashes=False))
|
|
else:
|
|
bucket_uris = [self.wildcard_uri.clone_replace_name('')]
|
|
|
|
# Now iterate over bucket(s), and handle object wildcarding, if any.
|
|
self.proj_id_handler.FillInProjectHeaderIfNeeded(WILDCARD_OBJECT_ITERATOR,
|
|
self.wildcard_uri,
|
|
self.headers)
|
|
for bucket_uri in bucket_uris:
|
|
if self.wildcard_uri.names_bucket():
|
|
# Bucket-only URI.
|
|
yield BucketListingRef(bucket_uri, key=None, prefix=None,
|
|
headers=self.headers)
|
|
else:
|
|
# URI contains an object name. If there's no wildcard just yield
|
|
# the needed URI.
|
|
if not ContainsWildcard(self.wildcard_uri.object_name):
|
|
uri_to_yield = bucket_uri.clone_replace_name(
|
|
self.wildcard_uri.object_name)
|
|
yield BucketListingRef(uri_to_yield, key=None, prefix=None,
|
|
headers=self.headers)
|
|
else:
|
|
# URI contains a wildcard. Expand iteratively by building
|
|
# prefix/delimiter bucket listing request, filtering the results per
|
|
# the current level's wildcard, and continuing with the next component
|
|
# of the wildcard. See _BuildBucketFilterStrings() documentation
|
|
# for details.
|
|
#
|
|
# Initialize the iteration with bucket name from bucket_uri but
|
|
# object name from self.wildcard_uri. This is needed to handle cases
|
|
# where both the bucket and object names contain wildcards.
|
|
uris_needing_expansion = [
|
|
bucket_uri.clone_replace_name(self.wildcard_uri.object_name)]
|
|
while len(uris_needing_expansion) > 0:
|
|
uri = uris_needing_expansion.pop(0)
|
|
(prefix, delimiter, prefix_wildcard, suffix_wildcard) = (
|
|
self._BuildBucketFilterStrings(uri.object_name))
|
|
prog = re.compile(fnmatch.translate(prefix_wildcard))
|
|
# List bucket for objects matching prefix up to delimiter.
|
|
for key in bucket_uri.list_bucket(prefix=prefix,
|
|
delimiter=delimiter,
|
|
headers=self.headers,
|
|
all_versions=self.all_versions):
|
|
# Check that the prefix regex matches rstripped key.name (to
|
|
# correspond with the rstripped prefix_wildcard from
|
|
# _BuildBucketFilterStrings()).
|
|
if prog.match(key.name.rstrip('/')):
|
|
if suffix_wildcard and key.name.rstrip('/') != suffix_wildcard:
|
|
if isinstance(key, Prefix):
|
|
# There's more wildcard left to expand.
|
|
uris_needing_expansion.append(
|
|
uri.clone_replace_name(key.name.rstrip('/') + '/'
|
|
+ suffix_wildcard))
|
|
else:
|
|
# Done expanding.
|
|
expanded_uri = uri.clone_replace_key(key)
|
|
|
|
if isinstance(key, Prefix):
|
|
yield BucketListingRef(expanded_uri, key=None, prefix=key,
|
|
headers=self.headers)
|
|
else:
|
|
if self.all_versions:
|
|
yield BucketListingRef(expanded_uri, key=key, prefix=None,
|
|
headers=self.headers)
|
|
else:
|
|
# Yield BLR wrapping version-less URI.
|
|
yield BucketListingRef(expanded_uri.clone_replace_name(
|
|
expanded_uri.object_name), key=key, prefix=None,
|
|
headers=self.headers)
|
|
|
|
def _BuildBucketFilterStrings(self, wildcard):
|
|
"""
|
|
Builds strings needed for querying a bucket and filtering results to
|
|
implement wildcard object name matching.
|
|
|
|
Args:
|
|
wildcard: The wildcard string to match to objects.
|
|
|
|
Returns:
|
|
(prefix, delimiter, prefix_wildcard, suffix_wildcard)
|
|
where:
|
|
prefix is the prefix to be sent in bucket GET request.
|
|
delimiter is the delimiter to be sent in bucket GET request.
|
|
prefix_wildcard is the wildcard to be used to filter bucket GET results.
|
|
suffix_wildcard is wildcard to be appended to filtered bucket GET
|
|
results for next wildcard expansion iteration.
|
|
For example, given the wildcard gs://bucket/abc/d*e/f*.txt we
|
|
would build prefix= abc/d, delimiter=/, prefix_wildcard=d*e, and
|
|
suffix_wildcard=f*.txt. Using this prefix and delimiter for a bucket
|
|
listing request will then produce a listing result set that can be
|
|
filtered using this prefix_wildcard; and we'd use this suffix_wildcard
|
|
to feed into the next call(s) to _BuildBucketFilterStrings(), for the
|
|
next iteration of listing/filtering.
|
|
|
|
Raises:
|
|
AssertionError if wildcard doesn't contain any wildcard chars.
|
|
"""
|
|
# Generate a request prefix if the object name part of the wildcard starts
|
|
# with a non-wildcard string (e.g., that's true for 'gs://bucket/abc*xyz').
|
|
match = WILDCARD_REGEX.search(wildcard)
|
|
if not match:
|
|
# Input "wildcard" has no wildcard chars, so just return tuple that will
|
|
# cause a bucket listing to match the given input wildcard. Example: if
|
|
# previous iteration yielded gs://bucket/dir/ with suffix_wildcard abc,
|
|
# the next iteration will call _BuildBucketFilterStrings() with
|
|
# gs://bucket/dir/abc, and we will return prefix ='dir/abc',
|
|
# delimiter='/', prefix_wildcard='dir/abc', and suffix_wildcard=''.
|
|
prefix = wildcard
|
|
delimiter = '/'
|
|
prefix_wildcard = wildcard
|
|
suffix_wildcard = ''
|
|
else:
|
|
if match.start() > 0:
|
|
# Wildcard does not occur at beginning of object name, so construct a
|
|
# prefix string to send to server.
|
|
prefix = wildcard[:match.start()]
|
|
wildcard_part = wildcard[match.start():]
|
|
else:
|
|
prefix = None
|
|
wildcard_part = wildcard
|
|
end = wildcard_part.find('/')
|
|
if end != -1:
|
|
wildcard_part = wildcard_part[:end+1]
|
|
# Remove trailing '/' so we will match gs://bucket/abc* as well as
|
|
# gs://bucket/abc*/ with the same wildcard regex.
|
|
prefix_wildcard = ((prefix or '') + wildcard_part).rstrip('/')
|
|
suffix_wildcard = wildcard[match.end():]
|
|
end = suffix_wildcard.find('/')
|
|
if end == -1:
|
|
suffix_wildcard = ''
|
|
else:
|
|
suffix_wildcard = suffix_wildcard[end+1:]
|
|
# To implement recursive (**) wildcarding, if prefix_wildcard
|
|
# suffix_wildcard starts with '**' don't send a delimiter, and combine
|
|
# suffix_wildcard at end of prefix_wildcard.
|
|
if prefix_wildcard.find('**') != -1:
|
|
delimiter = None
|
|
prefix_wildcard = prefix_wildcard + suffix_wildcard
|
|
suffix_wildcard = ''
|
|
else:
|
|
delimiter = '/'
|
|
delim_pos = suffix_wildcard.find(delimiter)
|
|
# The following debug output is useful for tracing how the algorithm
|
|
# walks through a multi-part wildcard like gs://bucket/abc/d*e/f*.txt
|
|
if self.debug > 1:
|
|
sys.stderr.write(
|
|
'DEBUG: wildcard=%s, prefix=%s, delimiter=%s, '
|
|
'prefix_wildcard=%s, suffix_wildcard=%s\n' %
|
|
(wildcard, prefix, delimiter, prefix_wildcard, suffix_wildcard))
|
|
return (prefix, delimiter, prefix_wildcard, suffix_wildcard)
|
|
|
|
def IterKeys(self):
|
|
"""
|
|
Convenience iterator that runs underlying iterator and returns Key for each
|
|
iteration.
|
|
|
|
Yields:
|
|
Subclass of boto.s3.key.Key, or empty iterator if no matches.
|
|
|
|
Raises:
|
|
WildcardException: for bucket-only uri.
|
|
"""
|
|
for bucket_listing_ref in self. __iter__():
|
|
if bucket_listing_ref.HasKey():
|
|
yield bucket_listing_ref.GetKey()
|
|
|
|
def IterUris(self):
|
|
"""
|
|
Convenience iterator that runs underlying iterator and returns StorageUri
|
|
for each iteration.
|
|
|
|
Yields:
|
|
StorageUri, or empty iterator if no matches.
|
|
"""
|
|
for bucket_listing_ref in self. __iter__():
|
|
yield bucket_listing_ref.GetUri()
|
|
|
|
def IterUrisForKeys(self):
|
|
"""
|
|
Convenience iterator that runs underlying iterator and returns the
|
|
StorageUri for each iterated BucketListingRef that has a Key.
|
|
|
|
Yields:
|
|
StorageUri, or empty iterator if no matches.
|
|
"""
|
|
for bucket_listing_ref in self. __iter__():
|
|
if bucket_listing_ref.HasKey():
|
|
yield bucket_listing_ref.GetUri()
|
|
|
|
|
|
class FileWildcardIterator(WildcardIterator):
|
|
"""WildcardIterator subclass for files and directories.
|
|
|
|
If you use recursive wildcards ('**') only a single such wildcard is
|
|
supported. For example you could use the wildcard '**/*.txt' to list all .txt
|
|
files in any subdirectory of the current directory, but you couldn't use a
|
|
wildcard like '**/abc/**/*.txt' (which would, if supported, let you find .txt
|
|
files in any subdirectory named 'abc').
|
|
"""
|
|
|
|
def __init__(self, wildcard_uri, headers=None, debug=0):
|
|
"""
|
|
Instantiate an iterator over BucketListingRefs matching given wildcard URI.
|
|
|
|
Args:
|
|
wildcard_uri: StorageUri that contains the wildcard to iterate.
|
|
headers: Dictionary containing optional HTTP headers to pass to boto.
|
|
debug: Debug level to pass in to boto connection (range 0..3).
|
|
"""
|
|
self.wildcard_uri = wildcard_uri
|
|
self.headers = headers
|
|
self.debug = debug
|
|
|
|
def __iter__(self):
|
|
wildcard = self.wildcard_uri.object_name
|
|
match = re.search('\*\*', wildcard)
|
|
if match:
|
|
# Recursive wildcarding request ('.../**/...').
|
|
# Example input: wildcard = '/tmp/tmp2pQJAX/**/*'
|
|
base_dir = wildcard[:match.start()-1]
|
|
remaining_wildcard = wildcard[match.start()+2:]
|
|
# At this point for the above example base_dir = '/tmp/tmp2pQJAX' and
|
|
# remaining_wildcard = '/*'
|
|
if remaining_wildcard.startswith('*'):
|
|
raise WildcardException('Invalid wildcard with more than 2 consecutive '
|
|
'*s (%s)' % wildcard)
|
|
# If there was no remaining wildcard past the recursive wildcard,
|
|
# treat it as if it were a '*'. For example, file://tmp/** is equivalent
|
|
# to file://tmp/**/*
|
|
if not remaining_wildcard:
|
|
remaining_wildcard = '*'
|
|
# Skip slash(es).
|
|
remaining_wildcard = remaining_wildcard.lstrip(os.sep)
|
|
filepaths = []
|
|
for dirpath, unused_dirnames, filenames in os.walk(base_dir):
|
|
filepaths.extend(
|
|
os.path.join(dirpath, f) for f in fnmatch.filter(filenames,
|
|
remaining_wildcard)
|
|
)
|
|
else:
|
|
# Not a recursive wildcarding request.
|
|
filepaths = glob.glob(wildcard)
|
|
for filepath in filepaths:
|
|
expanded_uri = self.wildcard_uri.clone_replace_name(filepath)
|
|
yield BucketListingRef(expanded_uri)
|
|
|
|
def IterKeys(self):
|
|
"""
|
|
Placeholder to allow polymorphic use of WildcardIterator.
|
|
|
|
Raises:
|
|
WildcardException: in all cases.
|
|
"""
|
|
raise WildcardException(
|
|
'Iterating over Keys not possible for file wildcards')
|
|
|
|
def IterUris(self):
|
|
"""
|
|
Convenience iterator that runs underlying iterator and returns StorageUri
|
|
for each iteration.
|
|
|
|
Yields:
|
|
StorageUri, or empty iterator if no matches.
|
|
"""
|
|
for bucket_listing_ref in self. __iter__():
|
|
yield bucket_listing_ref.GetUri()
|
|
|
|
|
|
class WildcardException(StandardError):
|
|
"""Exception thrown for invalid wildcard URIs."""
|
|
|
|
def __init__(self, reason):
|
|
StandardError.__init__(self)
|
|
self.reason = reason
|
|
|
|
def __repr__(self):
|
|
return 'WildcardException: %s' % self.reason
|
|
|
|
def __str__(self):
|
|
return 'WildcardException: %s' % self.reason
|
|
|
|
|
|
def wildcard_iterator(uri_or_str, proj_id_handler,
|
|
bucket_storage_uri_class=BucketStorageUri,
|
|
all_versions=False,
|
|
headers=None, debug=0):
|
|
"""Instantiate a WildCardIterator for the given StorageUri.
|
|
|
|
Args:
|
|
uri_or_str: StorageUri or URI string naming wildcard objects to iterate.
|
|
proj_id_handler: ProjectIdHandler to use for current command.
|
|
bucket_storage_uri_class: BucketStorageUri interface.
|
|
Settable for testing/mocking.
|
|
headers: Dictionary containing optional HTTP headers to pass to boto.
|
|
debug: Debug level to pass in to boto connection (range 0..3).
|
|
|
|
Returns:
|
|
A WildcardIterator that handles the requested iteration.
|
|
"""
|
|
|
|
if isinstance(uri_or_str, basestring):
|
|
# Disable enforce_bucket_naming, to allow bucket names containing wildcard
|
|
# chars.
|
|
uri = boto.storage_uri(
|
|
uri_or_str, debug=debug, validate=False,
|
|
bucket_storage_uri_class=bucket_storage_uri_class,
|
|
suppress_consec_slashes=False)
|
|
else:
|
|
uri = uri_or_str
|
|
|
|
if uri.is_cloud_uri():
|
|
return CloudWildcardIterator(
|
|
uri, proj_id_handler,
|
|
bucket_storage_uri_class=bucket_storage_uri_class,
|
|
all_versions=all_versions,
|
|
headers=headers,
|
|
debug=debug)
|
|
elif uri.is_file_uri():
|
|
return FileWildcardIterator(uri, headers=headers, debug=debug)
|
|
else:
|
|
raise WildcardException('Unexpected type of StorageUri (%s)' % uri)
|
|
|
|
|
|
def ContainsWildcard(uri_or_str):
|
|
"""Checks whether uri_or_str contains a wildcard.
|
|
|
|
Args:
|
|
uri_or_str: StorageUri or URI string to check.
|
|
|
|
Returns:
|
|
bool indicator.
|
|
"""
|
|
if isinstance(uri_or_str, basestring):
|
|
return bool(WILDCARD_REGEX.search(uri_or_str))
|
|
else:
|
|
return bool(WILDCARD_REGEX.search(uri_or_str.uri))
|