You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
385 lines
15 KiB
Python
385 lines
15 KiB
Python
#!/usr/bin/env python
|
|
# coding=utf8
|
|
# Copyright 2010 Google Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Main module for Google Cloud Storage command line tool."""
|
|
|
|
|
|
import ConfigParser
|
|
import errno
|
|
import getopt
|
|
import logging
|
|
import os
|
|
import re
|
|
import signal
|
|
import socket
|
|
import sys
|
|
import traceback
|
|
|
|
third_party_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
sys.path.insert(0, os.path.dirname(third_party_dir))
|
|
sys.path.insert(0, third_party_dir)
|
|
|
|
|
|
def _OutputAndExit(message):
|
|
global debug
|
|
if debug == 4:
|
|
stack_trace = traceback.format_exc()
|
|
sys.stderr.write('DEBUG: Exception stack trace:\n %s\n' %
|
|
re.sub('\\n', '\n ', stack_trace))
|
|
else:
|
|
sys.stderr.write('%s\n' % message)
|
|
sys.exit(1)
|
|
|
|
|
|
def _OutputUsageAndExit(command_runner):
|
|
command_runner.RunNamedCommand('help')
|
|
sys.exit(1)
|
|
|
|
|
|
debug = 0
|
|
# Before importing boto, find where gsutil is installed and include its
|
|
# boto sub-directory at the start of the PYTHONPATH, to ensure the versions of
|
|
# gsutil and boto stay in sync after software updates. This also allows gsutil
|
|
# to be used without explicitly adding it to the PYTHONPATH.
|
|
# We use realpath() below to unwind symlinks if any were used in the gsutil
|
|
# installation.
|
|
gsutil_bin_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
|
|
if not gsutil_bin_dir:
|
|
_OutputAndExit('Unable to determine where gsutil is installed. Sorry, '
|
|
'cannot run correctly without this.\n')
|
|
boto_lib_dir = os.path.join(gsutil_bin_dir, '..', 'boto')
|
|
if not os.path.isdir(boto_lib_dir):
|
|
_OutputAndExit('There is no boto library under the gsutil install directory '
|
|
'(%s).\nThe gsutil command cannot work properly when installed '
|
|
'this way.\nPlease re-install gsutil per the installation '
|
|
'instructions.' % gsutil_bin_dir)
|
|
# sys.path.insert(0, boto_lib_dir)
|
|
import boto
|
|
from boto.exception import BotoClientError
|
|
from boto.exception import InvalidAclError
|
|
from boto.exception import InvalidUriError
|
|
from boto.exception import ResumableUploadException
|
|
from boto.exception import StorageResponseError
|
|
from gslib.command_runner import CommandRunner
|
|
from gslib.exception import CommandException
|
|
from gslib.exception import ProjectIdException
|
|
from gslib import util
|
|
from gslib.util import ExtractErrorDetail
|
|
from gslib.util import HasConfiguredCredentials
|
|
from gslib.wildcard_iterator import WildcardException
|
|
|
|
# Load the gsutil version number and append it to boto.UserAgent so the value
|
|
# is set before anything instantiates boto. (If parts of boto were instantiated
|
|
# first those parts would have the old value of boto.UserAgent, so we wouldn't
|
|
# be guaranteed that all code paths send the correct user agent.)
|
|
ver_file_path = os.path.join(gsutil_bin_dir, 'VERSION')
|
|
if not os.path.isfile(ver_file_path):
|
|
raise CommandException(
|
|
'%s not found. Please reinstall gsutil from scratch' % ver_file_path)
|
|
ver_file = open(ver_file_path, 'r')
|
|
gsutil_ver = ver_file.read().rstrip()
|
|
ver_file.close()
|
|
boto.UserAgent += ' gsutil/%s (%s)' % (gsutil_ver, sys.platform)
|
|
|
|
# We don't use the oauth2 authentication plugin directly; importing it here
|
|
# ensures that it's loaded and available by default when an operation requiring
|
|
# authentication is performed.
|
|
try:
|
|
from oauth2_plugin import oauth2_plugin
|
|
except ImportError:
|
|
pass
|
|
|
|
|
|
def main():
|
|
global debug
|
|
|
|
if sys.version_info[:3] < (2, 6):
|
|
raise CommandException('gsutil requires Python 2.6 or higher.')
|
|
|
|
bypass_prodaccess = False
|
|
config_file_list = _GetBotoConfigFileList()
|
|
command_runner = CommandRunner(gsutil_bin_dir, boto_lib_dir, config_file_list,
|
|
gsutil_ver)
|
|
headers = {}
|
|
parallel_operations = False
|
|
debug = 0
|
|
|
|
# If user enters no commands just print the usage info.
|
|
if len(sys.argv) == 1:
|
|
sys.argv.append('help')
|
|
|
|
# Change the default of the 'https_validate_certificates' boto option to
|
|
# True (it is currently False in boto).
|
|
if not boto.config.has_option('Boto', 'https_validate_certificates'):
|
|
if not boto.config.has_section('Boto'):
|
|
boto.config.add_section('Boto')
|
|
boto.config.setbool('Boto', 'https_validate_certificates', True)
|
|
|
|
try:
|
|
opts, args = getopt.getopt(sys.argv[1:], 'dDvh:mb',
|
|
['debug', 'detailedDebug', 'version', 'help',
|
|
'header', 'multithreaded',
|
|
'bypass_prodaccess'])
|
|
except getopt.GetoptError, e:
|
|
_HandleCommandException(CommandException(e.msg))
|
|
for o, a in opts:
|
|
if o in ('-d', '--debug'):
|
|
# Passing debug=2 causes boto to include httplib header output.
|
|
debug = 2
|
|
if o in ('-D', '--detailedDebug'):
|
|
# We use debug level 3 to ask gsutil code to output more detailed
|
|
# debug output. This is a bit of a hack since it overloads the same
|
|
# flag that was originally implemented for boto use. And we use -DD
|
|
# to ask for really detailed debugging (i.e., including HTTP payload).
|
|
if debug == 3:
|
|
debug = 4
|
|
else:
|
|
debug = 3
|
|
if o in ('-?', '--help'):
|
|
_OutputUsageAndExit(command_runner)
|
|
if o in ('-h', '--header'):
|
|
(hdr_name, unused_ptn, hdr_val) = a.partition(':')
|
|
if not hdr_name:
|
|
_OutputUsageAndExit(command_runner)
|
|
headers[hdr_name] = hdr_val
|
|
if o in ('-m', '--multithreaded'):
|
|
parallel_operations = True
|
|
if o in ('-b', '--bypass_prodaccess'):
|
|
bypass_prodaccess = True
|
|
if debug > 1:
|
|
sys.stderr.write(
|
|
'***************************** WARNING *****************************\n'
|
|
'*** You are running gsutil with debug output enabled.\n'
|
|
'*** Be aware that debug output includes authentication '
|
|
'credentials.\n'
|
|
'*** Do not share (e.g., post to support forums) debug output\n'
|
|
'*** unless you have sanitized authentication tokens in the\n'
|
|
'*** output, or have revoked your credentials.\n'
|
|
'***************************** WARNING *****************************\n')
|
|
if debug == 2:
|
|
logging.basicConfig(level=logging.INFO)
|
|
elif debug > 2:
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
command_runner.RunNamedCommand('ver')
|
|
config_items = []
|
|
try:
|
|
config_items.extend(boto.config.items('Boto'))
|
|
config_items.extend(boto.config.items('GSUtil'))
|
|
except ConfigParser.NoSectionError:
|
|
pass
|
|
sys.stderr.write('config_file_list: %s\n' % config_file_list)
|
|
sys.stderr.write('config: %s\n' % str(config_items))
|
|
else:
|
|
logging.basicConfig()
|
|
|
|
if not args:
|
|
command_name = 'help'
|
|
else:
|
|
command_name = args[0]
|
|
|
|
if not bypass_prodaccess:
|
|
import plugins.sso_auth
|
|
|
|
return _RunNamedCommandAndHandleExceptions(command_runner, command_name,
|
|
args[1:], headers, debug,
|
|
parallel_operations,
|
|
bypass_prodaccess)
|
|
|
|
|
|
def _GetBotoConfigFileList():
|
|
"""Returns list of boto config files that exist."""
|
|
config_paths = boto.pyami.config.BotoConfigLocations
|
|
if 'AWS_CREDENTIAL_FILE' in os.environ:
|
|
config_paths.append(os.environ['AWS_CREDENTIAL_FILE'])
|
|
config_files = {}
|
|
for config_path in config_paths:
|
|
if os.path.exists(config_path):
|
|
config_files[config_path] = 1
|
|
cf_list = []
|
|
for config_file in config_files:
|
|
cf_list.append(config_file)
|
|
return cf_list
|
|
|
|
|
|
def _HandleUnknownFailure(e):
|
|
global debug
|
|
# Called if we fall through all known/handled exceptions. Allows us to
|
|
# print a stacktrace if -D option used.
|
|
if debug > 2:
|
|
stack_trace = traceback.format_exc()
|
|
sys.stderr.write('DEBUG: Exception stack trace:\n %s\n' %
|
|
re.sub('\\n', '\n ', stack_trace))
|
|
else:
|
|
_OutputAndExit('Failure: %s.' % e)
|
|
|
|
|
|
def _HandleCommandException(e):
|
|
if e.informational:
|
|
_OutputAndExit(e.reason)
|
|
else:
|
|
_OutputAndExit('CommandException: %s' % e.reason)
|
|
|
|
|
|
def _HandleControlC(signal_num, cur_stack_frame):
|
|
"""Called when user hits ^C so we can print a brief message instead of
|
|
the normal Python stack trace (unless -D option is used)."""
|
|
global debug
|
|
if debug > 2:
|
|
stack_trace = ''.join(traceback.format_list(traceback.extract_stack()))
|
|
_OutputAndExit('DEBUG: Caught signal %d - Exception stack trace:\n'
|
|
' %s' % (signal_num, re.sub('\\n', '\n ', stack_trace)))
|
|
else:
|
|
_OutputAndExit('Caught signal %d - exiting' % signal_num)
|
|
|
|
|
|
def _HandleSigQuit(signal_num, cur_stack_frame):
|
|
"""Called when user hits ^\, so we can force breakpoint a running gsutil."""
|
|
import pdb; pdb.set_trace()
|
|
|
|
|
|
def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None,
|
|
headers=None, debug=0,
|
|
parallel_operations=False,
|
|
bypass_prodaccess=False):
|
|
try:
|
|
# Catch ^C so we can print a brief message instead of the normal Python
|
|
# stack trace.
|
|
signal.signal(signal.SIGINT, _HandleControlC)
|
|
# Catch ^\ so we can force a breakpoint in a running gsutil.
|
|
if not util.IS_WINDOWS:
|
|
signal.signal(signal.SIGQUIT, _HandleSigQuit)
|
|
return command_runner.RunNamedCommand(command_name, args, headers, debug,
|
|
parallel_operations,
|
|
bypass_prodaccess=bypass_prodaccess)
|
|
except AttributeError, e:
|
|
if str(e).find('secret_access_key') != -1:
|
|
_OutputAndExit('Missing credentials for the given URI(s). Does your '
|
|
'boto config file contain all needed credentials?')
|
|
else:
|
|
_OutputAndExit(str(e))
|
|
except BotoClientError, e:
|
|
_OutputAndExit('BotoClientError: %s.' % e.reason)
|
|
except CommandException, e:
|
|
_HandleCommandException(e)
|
|
except getopt.GetoptError, e:
|
|
_HandleCommandException(CommandException(e.msg))
|
|
except InvalidAclError, e:
|
|
_OutputAndExit('InvalidAclError: %s.' % str(e))
|
|
except InvalidUriError, e:
|
|
_OutputAndExit('InvalidUriError: %s.' % e.message)
|
|
except ProjectIdException, e:
|
|
_OutputAndExit('ProjectIdException: %s.' % e.reason)
|
|
except boto.auth_handler.NotReadyToAuthenticate:
|
|
_OutputAndExit('NotReadyToAuthenticate')
|
|
except OSError, e:
|
|
_OutputAndExit('OSError: %s.' % e.strerror)
|
|
except WildcardException, e:
|
|
_OutputAndExit(e.reason)
|
|
except StorageResponseError, e:
|
|
# Check for access denied, and provide detail to users who have no boto
|
|
# config file (who might previously have been using gsutil only for
|
|
# accessing publicly readable buckets and objects).
|
|
if e.status == 403:
|
|
if not HasConfiguredCredentials(bypass_prodaccess):
|
|
_OutputAndExit(
|
|
'You are attempting to access protected data with no configured '
|
|
'credentials.\nPlease see '
|
|
'http://code.google.com/apis/storage/docs/signup.html for\ndetails '
|
|
'about activating the Google Cloud Storage service and then run '
|
|
'the\n"gsutil config" command to configure gsutil to use these '
|
|
'credentials.')
|
|
elif (e.error_code == 'AccountProblem'
|
|
and ','.join(args).find('gs://') != -1):
|
|
default_project_id = boto.config.get_value('GSUtil',
|
|
'default_project_id')
|
|
acct_help_part_1 = (
|
|
"""Your request resulted in an AccountProblem (403) error. Usually this happens
|
|
if you attempt to create a bucket or upload an object without having first
|
|
enabled billing for the project you are using. To remedy this problem, please do
|
|
the following:
|
|
|
|
1. Navigate to the Google APIs console (https://code.google.com/apis/console),
|
|
and ensure the drop-down selector beneath "Google APIs" shows the project
|
|
you're attempting to use.
|
|
|
|
""")
|
|
acct_help_part_2 = '\n'
|
|
if default_project_id:
|
|
acct_help_part_2 = (
|
|
"""2. Click "Google Cloud Storage" on the left hand pane, and then check that
|
|
the value listed for "x-goog-project-id" on this page matches the project ID
|
|
(%s) from your boto config file.
|
|
|
|
""" % default_project_id)
|
|
acct_help_part_3 = (
|
|
"""Check whether there's an "!" next to Billing. If so, click Billing and then
|
|
enable billing for this project. Note that it can take up to one hour after
|
|
enabling billing for the project to become activated for creating buckets and
|
|
uploading objects.
|
|
|
|
If the above doesn't resolve your AccountProblem, please send mail to
|
|
gs-team@google.com requesting assistance, noting the exact command you ran, the
|
|
fact that you received a 403 AccountProblem error, and your project ID. Please
|
|
do not post your project ID on the public discussion forum (gs-discussion) or on
|
|
StackOverflow.
|
|
|
|
Note: It's possible to use Google Cloud Storage without enabling billing if
|
|
you're only listing or reading objects for which you're authorized, or if
|
|
you're uploading objects to a bucket billed to a project that has billing
|
|
enabled. But if you're attempting to create buckets or upload objects to a
|
|
bucket owned by your own project, you must first enable billing for that
|
|
project.""")
|
|
if default_project_id:
|
|
_OutputAndExit(acct_help_part_1 + acct_help_part_2 + '3. '
|
|
+ acct_help_part_3)
|
|
else:
|
|
_OutputAndExit(acct_help_part_1 + '2. ' + acct_help_part_3)
|
|
|
|
if not e.body:
|
|
e.body = ''
|
|
exc_name, error_detail = ExtractErrorDetail(e)
|
|
if error_detail:
|
|
_OutputAndExit('%s: status=%d, code=%s, reason=%s, detail=%s.' %
|
|
(exc_name, e.status, e.code, e.reason, error_detail))
|
|
else:
|
|
_OutputAndExit('%s: status=%d, code=%s, reason=%s.' %
|
|
(exc_name, e.status, e.code, e.reason))
|
|
except ResumableUploadException, e:
|
|
_OutputAndExit('ResumableUploadException: %s.' % e.message)
|
|
except socket.error, e:
|
|
if e.args[0] == errno.EPIPE:
|
|
# Retrying with a smaller file (per suggestion below) works because
|
|
# the library code send loop (in boto/s3/key.py) can get through the
|
|
# entire file and then request the HTTP response before the socket
|
|
# gets closed and the response lost.
|
|
message = (
|
|
"""
|
|
Got a "Broken pipe" error. This can happen to clients using Python 2.x,
|
|
when the server sends an error response and then closes the socket (see
|
|
http://bugs.python.org/issue5542). If you are trying to upload a large
|
|
object you might retry with a small (say 200k) object, and see if you get
|
|
a more specific error code.
|
|
""")
|
|
_OutputAndExit(message)
|
|
else:
|
|
_HandleUnknownFailure(e)
|
|
except Exception, e:
|
|
_HandleUnknownFailure(e)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|