809 lines
34 KiB
Python
809 lines
34 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
# Copyright 2013 Google Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Main module for Google Cloud Storage command line tool."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import print_function
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
import datetime
|
|
import errno
|
|
import getopt
|
|
import logging
|
|
import os
|
|
import re
|
|
import signal
|
|
import socket
|
|
import sys
|
|
import textwrap
|
|
import traceback
|
|
|
|
import six
|
|
from six.moves import configparser
|
|
from six.moves import range
|
|
|
|
from google.auth import exceptions as google_auth_exceptions
|
|
import gslib.exception
|
|
from gslib.exception import CommandException
|
|
from gslib.exception import ControlCException
|
|
|
|
from gslib.utils.version_check import check_python_version_support
|
|
from gslib.utils.arg_helper import GetArgumentsAndOptions
|
|
from gslib.utils.user_agent_helper import GetUserAgent
|
|
|
|
# Load the gsutil version number and append it to boto.UserAgent so the value is
|
|
# set before anything instantiates boto. This has to run after THIRD_PARTY_DIR
|
|
# is modified (done in gsutil.py) but before any calls are made that would cause
|
|
# boto.s3.Connection to be loaded - otherwise the Connection class would end up
|
|
# with a static reference to the pre-modified version of the UserAgent field,
|
|
# so boto requests would not include gsutil/version# in the UserAgent string.
|
|
import boto
|
|
import gslib
|
|
from gslib.utils import system_util, text_util
|
|
|
|
# pylint: disable=g-import-not-at-top
|
|
# This module also imports boto, and will override the UserAgent global variable
|
|
# if imported above.
|
|
from gslib import metrics
|
|
|
|
# We parse the options and arguments here so we can pass the results to the user
|
|
# agent helper.
|
|
try:
|
|
opts, args = GetArgumentsAndOptions()
|
|
except CommandException as e:
|
|
reason = e.reason if e.informational else 'CommandException: %s' % e.reason
|
|
err = '%s\n' % reason
|
|
try:
|
|
text_util.print_to_fd(err, end='', file=sys.stderr)
|
|
except UnicodeDecodeError:
|
|
# Can happen when outputting invalid Unicode filenames.
|
|
sys.stderr.write(err)
|
|
if e:
|
|
metrics.LogFatalError(e)
|
|
sys.exit(1)
|
|
|
|
# This calculated user agent can be stored for use in StorageV1.
|
|
gslib.USER_AGENT = GetUserAgent(args, metrics.MetricsCollector.IsDisabled())
|
|
boto.UserAgent += gslib.USER_AGENT
|
|
|
|
# pylint: disable=g-bad-import-order
|
|
import httplib2
|
|
import oauth2client
|
|
from google_reauth import reauth_creds
|
|
from google_reauth import errors as reauth_errors
|
|
from gslib import context_config
|
|
from gslib import wildcard_iterator
|
|
from gslib.cloud_api import AccessDeniedException
|
|
from gslib.cloud_api import ArgumentException
|
|
from gslib.cloud_api import BadRequestException
|
|
from gslib.cloud_api import ProjectIdException
|
|
from gslib.cloud_api import ServiceException
|
|
from gslib.command_runner import CommandRunner
|
|
import apitools.base.py.exceptions as apitools_exceptions
|
|
from gslib.utils import boto_util
|
|
from gslib.utils import constants
|
|
from gslib.utils import system_util
|
|
from gslib.sig_handling import GetCaughtSignals
|
|
from gslib.sig_handling import InitializeSignalHandling
|
|
from gslib.sig_handling import RegisterSignalHandler
|
|
|
|
CONFIG_KEYS_TO_REDACT = ['proxy', 'proxy_port', 'proxy_user', 'proxy_pass']
|
|
|
|
# We don't use the oauth2 authentication plugin directly; importing it here
|
|
# ensures that it's loaded and available by default when an operation requiring
|
|
# authentication is performed.
|
|
try:
|
|
# pylint: disable=unused-import,g-import-not-at-top
|
|
import gcs_oauth2_boto_plugin
|
|
except ImportError:
|
|
pass
|
|
|
|
DEBUG_WARNING = """
|
|
***************************** WARNING *****************************
|
|
*** You are running gsutil with debug output enabled.
|
|
*** Be aware that debug output includes authentication credentials.
|
|
*** Make sure to remove the value of the Authorization header for
|
|
*** each HTTP request printed to the console prior to posting to
|
|
*** a public medium such as a forum post or Stack Overflow.
|
|
***************************** WARNING *****************************
|
|
""".lstrip()
|
|
|
|
TRACE_WARNING = """
|
|
***************************** WARNING *****************************
|
|
*** You are running gsutil with trace output enabled.
|
|
*** Be aware that trace output includes authentication credentials
|
|
*** and may include the contents of any files accessed during the trace.
|
|
***************************** WARNING *****************************
|
|
""".lstrip()
|
|
|
|
HTTP_WARNING = """
|
|
***************************** WARNING *****************************
|
|
*** You are running gsutil with the "https_validate_certificates" config
|
|
*** variable set to False. This option should always be set to True in
|
|
*** production environments to protect against man-in-the-middle attacks,
|
|
*** and leaking of user data.
|
|
***************************** WARNING *****************************
|
|
""".lstrip()
|
|
|
|
debug_level = 0
|
|
test_exception_traces = False
|
|
|
|
|
|
# pylint: disable=unused-argument
|
|
def _CleanupSignalHandler(signal_num, cur_stack_frame):
|
|
"""Cleans up if process is killed with SIGINT, SIGQUIT or SIGTERM.
|
|
|
|
Note that this method is called after main() has been called, so it has
|
|
access to all the modules imported at the start of main().
|
|
|
|
Args:
|
|
signal_num: Unused, but required in the method signature.
|
|
cur_stack_frame: Unused, but required in the method signature.
|
|
"""
|
|
_Cleanup()
|
|
if (gslib.utils.parallelism_framework_util.
|
|
CheckMultiprocessingAvailableAndInit().is_available):
|
|
gslib.command.TeardownMultiprocessingProcesses()
|
|
|
|
|
|
def _Cleanup():
|
|
for fname in boto_util.GetCleanupFiles():
|
|
try:
|
|
os.unlink(fname)
|
|
except: # pylint: disable=bare-except
|
|
pass
|
|
|
|
|
|
def _OutputAndExit(message, exception=None):
|
|
"""Outputs message to stderr and exits gsutil with code 1.
|
|
|
|
This function should only be called in single-process, single-threaded mode.
|
|
|
|
Args:
|
|
message: Message to print to stderr.
|
|
exception: The exception that caused gsutil to fail.
|
|
"""
|
|
if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS or test_exception_traces:
|
|
stack_trace = traceback.format_exc()
|
|
err = ('DEBUG: Exception stack trace:\n %s\n%s\n' %
|
|
(re.sub('\\n', '\n ', stack_trace), message))
|
|
else:
|
|
err = '%s\n' % message
|
|
try:
|
|
text_util.print_to_fd(err, end='', file=sys.stderr)
|
|
except UnicodeDecodeError:
|
|
# Can happen when outputting invalid Unicode filenames.
|
|
sys.stderr.write(err)
|
|
if exception:
|
|
metrics.LogFatalError(exception)
|
|
sys.exit(1)
|
|
|
|
|
|
def _OutputUsageAndExit(command_runner):
|
|
command_runner.RunNamedCommand('help')
|
|
sys.exit(1)
|
|
|
|
|
|
class GsutilFormatter(logging.Formatter):
|
|
"""A logging.Formatter that supports logging microseconds (%f)."""
|
|
|
|
def formatTime(self, record, datefmt=None):
|
|
if datefmt:
|
|
return datetime.datetime.fromtimestamp(record.created).strftime(datefmt)
|
|
|
|
# Use default implementation if datefmt is not specified.
|
|
return super(GsutilFormatter, self).formatTime(record, datefmt=datefmt)
|
|
|
|
|
|
def _ConfigureRootLogger(level=logging.INFO):
|
|
"""Similar to logging.basicConfig() except it always adds a handler."""
|
|
log_format = '%(levelname)s %(asctime)s %(filename)s] %(message)s'
|
|
date_format = '%m%d %H:%M:%S.%f'
|
|
formatter = GsutilFormatter(fmt=log_format, datefmt=date_format)
|
|
handler = logging.StreamHandler()
|
|
handler.setFormatter(formatter)
|
|
root_logger = logging.getLogger()
|
|
root_logger.addHandler(handler)
|
|
root_logger.setLevel(level)
|
|
|
|
|
|
def main():
|
|
InitializeSignalHandling()
|
|
# Any modules used in initializing multiprocessing variables must be
|
|
# imported after importing gslib.__main__.
|
|
# pylint: disable=redefined-outer-name,g-import-not-at-top
|
|
import gslib.boto_translation
|
|
import gslib.command
|
|
import gslib.utils.parallelism_framework_util
|
|
# pylint: disable=unused-variable
|
|
from gcs_oauth2_boto_plugin import oauth2_client
|
|
from apitools.base.py import credentials_lib
|
|
# pylint: enable=unused-variable
|
|
if (gslib.utils.parallelism_framework_util.
|
|
CheckMultiprocessingAvailableAndInit().is_available):
|
|
# These setup methods must be called, and, on Windows, they can only be
|
|
# called from within an "if __name__ == '__main__':" block.
|
|
gslib.command.InitializeMultiprocessingVariables()
|
|
gslib.boto_translation.InitializeMultiprocessingVariables()
|
|
else:
|
|
gslib.command.InitializeThreadingVariables()
|
|
|
|
# This needs to be done after InitializeMultiprocessingVariables(), since
|
|
# otherwise we can't call CreateLock.
|
|
try:
|
|
# pylint: disable=unused-import,g-import-not-at-top
|
|
import gcs_oauth2_boto_plugin
|
|
gsutil_client_id, gsutil_client_secret = (
|
|
system_util.GetGsutilClientIdAndSecret())
|
|
gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret(
|
|
gsutil_client_id, gsutil_client_secret)
|
|
gcs_oauth2_boto_plugin.oauth2_helper.SetLock(
|
|
gslib.utils.parallelism_framework_util.CreateLock())
|
|
credentials_lib.SetCredentialsCacheFileLock(
|
|
gslib.utils.parallelism_framework_util.CreateLock())
|
|
except ImportError:
|
|
pass
|
|
|
|
global debug_level
|
|
global test_exception_traces
|
|
|
|
supported, err = check_python_version_support()
|
|
if not supported:
|
|
raise CommandException(err)
|
|
sys.exit(1)
|
|
|
|
boto_util.MonkeyPatchBoto()
|
|
system_util.MonkeyPatchHttp()
|
|
|
|
# In gsutil 4.0 and beyond, we don't use the boto library for the JSON
|
|
# API. However, we still store gsutil configuration data in the .boto
|
|
# config file for compatibility with previous versions and user convenience.
|
|
# Many users have a .boto configuration file from previous versions, and it
|
|
# is useful to have all of the configuration for gsutil stored in one place.
|
|
command_runner = CommandRunner()
|
|
if not boto_util.BOTO_IS_SECURE:
|
|
raise CommandException('\n'.join(
|
|
textwrap.wrap(
|
|
'Your boto configuration has is_secure = False. Gsutil cannot be '
|
|
'run this way, for security reasons.')))
|
|
|
|
headers = {}
|
|
parallel_operations = False
|
|
quiet = False
|
|
version = False
|
|
debug_level = 0
|
|
trace_token = None
|
|
perf_trace_token = None
|
|
test_exception_traces = False
|
|
user_project = None
|
|
|
|
# If user enters no commands just print the usage info.
|
|
if len(sys.argv) == 1:
|
|
sys.argv.append('help')
|
|
|
|
# Change the default of the 'https_validate_certificates' boto option to
|
|
# True (it is currently False in boto).
|
|
if not boto.config.has_option('Boto', 'https_validate_certificates'):
|
|
if not boto.config.has_section('Boto'):
|
|
boto.config.add_section('Boto')
|
|
boto.config.setbool('Boto', 'https_validate_certificates', True)
|
|
|
|
for signal_num in GetCaughtSignals():
|
|
RegisterSignalHandler(signal_num, _CleanupSignalHandler)
|
|
|
|
try:
|
|
for o, a in opts:
|
|
if o in ('-d', '--debug'):
|
|
# Also causes boto to include httplib header output.
|
|
debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS
|
|
elif o in ('-D', '--detailedDebug'):
|
|
# We use debug level 3 to ask gsutil code to output more detailed
|
|
# debug output. This is a bit of a hack since it overloads the same
|
|
# flag that was originally implemented for boto use. And we use -DD
|
|
# to ask for really detailed debugging (i.e., including HTTP payload).
|
|
if debug_level == constants.DEBUGLEVEL_DUMP_REQUESTS:
|
|
debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADS
|
|
else:
|
|
debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS
|
|
elif o in ('-?', '--help'):
|
|
_OutputUsageAndExit(command_runner)
|
|
elif o in ('-h', '--header'):
|
|
(hdr_name, _, hdr_val) = a.partition(':')
|
|
if not hdr_name:
|
|
_OutputUsageAndExit(command_runner)
|
|
headers[hdr_name.lower()] = hdr_val
|
|
elif o in ('-m', '--multithreaded'):
|
|
parallel_operations = True
|
|
elif o in ('-q', '--quiet'):
|
|
quiet = True
|
|
elif o == '-u':
|
|
user_project = a
|
|
elif o in ('-v', '--version'):
|
|
version = True
|
|
elif o in ('-i', '--impersonate-service-account'):
|
|
constants.IMPERSONATE_SERVICE_ACCOUNT = a
|
|
elif o == '--perf-trace-token':
|
|
perf_trace_token = a
|
|
elif o == '--trace-token':
|
|
trace_token = a
|
|
elif o == '--testexceptiontraces': # Hidden flag for integration tests.
|
|
test_exception_traces = True
|
|
# Avoid printing extra warnings to stderr regarding long retries by
|
|
# setting the threshold very high.
|
|
constants.LONG_RETRY_WARN_SEC = 3600
|
|
elif o in ('-o', '--option'):
|
|
(opt_section_name, _, opt_value) = a.partition('=')
|
|
if not opt_section_name:
|
|
_OutputUsageAndExit(command_runner)
|
|
(opt_section, _, opt_name) = opt_section_name.partition(':')
|
|
if not opt_section or not opt_name:
|
|
_OutputUsageAndExit(command_runner)
|
|
if not boto.config.has_section(opt_section):
|
|
boto.config.add_section(opt_section)
|
|
boto.config.set(opt_section, opt_name, opt_value)
|
|
|
|
# Now that any Boto option overrides (via `-o` args) have been parsed,
|
|
# perform initialization that depends on those options.
|
|
boto_util.configured_certs_file = (boto_util.ConfigureCertsFile())
|
|
|
|
metrics.LogCommandParams(global_opts=opts)
|
|
httplib2.debuglevel = debug_level
|
|
if trace_token:
|
|
sys.stderr.write(TRACE_WARNING)
|
|
if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS:
|
|
sys.stderr.write(DEBUG_WARNING)
|
|
_ConfigureRootLogger(level=logging.DEBUG)
|
|
command_runner.RunNamedCommand('ver', ['-l'])
|
|
|
|
config_items = []
|
|
for config_section in ('Boto', 'GSUtil'):
|
|
try:
|
|
config_items.extend(boto.config.items(config_section))
|
|
except configparser.NoSectionError:
|
|
pass
|
|
for i in range(len(config_items)):
|
|
config_item_key = config_items[i][0]
|
|
if config_item_key in CONFIG_KEYS_TO_REDACT:
|
|
config_items[i] = (config_item_key, 'REDACTED')
|
|
sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv))
|
|
sys.stderr.write('config_file_list: %s\n' %
|
|
boto_util.GetFriendlyConfigFilePaths())
|
|
sys.stderr.write('config: %s\n' % str(config_items))
|
|
else: # Non-debug log level.
|
|
root_logger_level = logging.WARNING if quiet else logging.INFO
|
|
# oauth2client uses INFO and WARNING logging in places that would better
|
|
# correspond to gsutil's debug logging (e.g., when refreshing
|
|
# access tokens), so we bump the threshold one level higher where
|
|
# appropriate. These log levels work for regular- and quiet-level logging.
|
|
oa2c_logger_level = logging.WARNING
|
|
oa2c_multiprocess_file_storage_logger_level = logging.ERROR
|
|
|
|
_ConfigureRootLogger(level=root_logger_level)
|
|
oauth2client.client.logger.setLevel(oa2c_logger_level)
|
|
oauth2client.contrib.multiprocess_file_storage.logger.setLevel(
|
|
oa2c_multiprocess_file_storage_logger_level)
|
|
# pylint: disable=protected-access
|
|
oauth2client.transport._LOGGER.setLevel(oa2c_logger_level)
|
|
reauth_creds._LOGGER.setLevel(oa2c_logger_level)
|
|
# pylint: enable=protected-access
|
|
|
|
# TODO(reauth): Fix once reauth pins to pyu2f version newer than 0.1.3.
|
|
# Fixes pyu2f v0.1.3 bug.
|
|
import six # pylint: disable=g-import-not-at-top
|
|
six.input = six.moves.input
|
|
|
|
if not boto_util.CERTIFICATE_VALIDATION_ENABLED:
|
|
sys.stderr.write(HTTP_WARNING)
|
|
|
|
if version:
|
|
command_name = 'version'
|
|
elif not args:
|
|
command_name = 'help'
|
|
else:
|
|
command_name = args[0]
|
|
if command_name != 'test':
|
|
# Don't initialize mTLS authentication because
|
|
# tests that need it will do this initialization themselves.
|
|
context_config.create_context_config(logging.getLogger())
|
|
|
|
_CheckAndWarnForProxyDifferences()
|
|
|
|
# Both 1 and 2 are valid _ARGCOMPLETE values; this var tells argcomplete at
|
|
# what argv[] index the command to match starts. We want it to start at the
|
|
# value for the path to gsutil, so:
|
|
# $ gsutil <command> # Should be the 1st argument, so '1'
|
|
# $ python gsutil <command> # Should be the 2nd argument, so '2'
|
|
# Both are valid; most users invoke gsutil in the first style, but our
|
|
# integration and prerelease tests invoke it in the second style, as we need
|
|
# to specify the Python interpreter used to run gsutil.
|
|
if os.environ.get('_ARGCOMPLETE', '0') in ('1', '2'):
|
|
return _PerformTabCompletion(command_runner)
|
|
|
|
return _RunNamedCommandAndHandleExceptions(
|
|
command_runner,
|
|
command_name,
|
|
args=args[1:],
|
|
headers=headers,
|
|
debug_level=debug_level,
|
|
trace_token=trace_token,
|
|
parallel_operations=parallel_operations,
|
|
perf_trace_token=perf_trace_token,
|
|
user_project=user_project)
|
|
finally:
|
|
_Cleanup()
|
|
|
|
|
|
def _CheckAndWarnForProxyDifferences():
|
|
# If there are both boto config and environment variable config present for
|
|
# proxies, unset the environment variable and warn if it differs.
|
|
boto_port = boto.config.getint('Boto', 'proxy_port', 0)
|
|
if boto.config.get('Boto', 'proxy', None) or boto_port:
|
|
for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']:
|
|
if proxy_env_var in os.environ and os.environ[proxy_env_var]:
|
|
differing_values = []
|
|
proxy_info = boto_util.ProxyInfoFromEnvironmentVar(proxy_env_var)
|
|
if proxy_info.proxy_host != boto.config.get('Boto', 'proxy', None):
|
|
differing_values.append(
|
|
'Boto proxy host: "%s" differs from %s proxy host: "%s"' %
|
|
(boto.config.get('Boto', 'proxy',
|
|
None), proxy_env_var, proxy_info.proxy_host))
|
|
if (proxy_info.proxy_user != boto.config.get('Boto', 'proxy_user',
|
|
None)):
|
|
differing_values.append(
|
|
'Boto proxy user: "%s" differs from %s proxy user: "%s"' %
|
|
(boto.config.get('Boto', 'proxy_user',
|
|
None), proxy_env_var, proxy_info.proxy_user))
|
|
if (proxy_info.proxy_pass != boto.config.get('Boto', 'proxy_pass',
|
|
None)):
|
|
differing_values.append(
|
|
'Boto proxy password differs from %s proxy password' %
|
|
proxy_env_var)
|
|
# Only compare ports if at least one is present, since the
|
|
# boto logic for selecting default ports has not yet executed.
|
|
if ((proxy_info.proxy_port or boto_port) and
|
|
proxy_info.proxy_port != boto_port):
|
|
differing_values.append(
|
|
'Boto proxy port: "%s" differs from %s proxy port: "%s"' %
|
|
(boto_port, proxy_env_var, proxy_info.proxy_port))
|
|
if differing_values:
|
|
sys.stderr.write('\n'.join(
|
|
textwrap.wrap(
|
|
'WARNING: Proxy configuration is present in both the %s '
|
|
'environment variable and boto configuration, but '
|
|
'configuration differs. boto configuration proxy values will '
|
|
'be used. Differences detected:' % proxy_env_var)))
|
|
sys.stderr.write('\n%s\n' % '\n'.join(differing_values))
|
|
# Regardless of whether the proxy configuration values matched,
|
|
# delete the environment variable so as not to confuse boto.
|
|
del os.environ[proxy_env_var]
|
|
|
|
|
|
def _HandleUnknownFailure(e):
|
|
# Called if we fall through all known/handled exceptions.
|
|
raise
|
|
_OutputAndExit(message='Failure: %s.' % e, exception=e)
|
|
|
|
|
|
def _HandleCommandException(e):
|
|
if e.informational:
|
|
_OutputAndExit(message=e.reason, exception=e)
|
|
else:
|
|
_OutputAndExit(message='CommandException: %s' % e.reason, exception=e)
|
|
|
|
|
|
# pylint: disable=unused-argument
|
|
def _HandleControlC(signal_num, cur_stack_frame):
|
|
"""Called when user hits ^C.
|
|
|
|
This function prints a brief message instead of the normal Python stack trace
|
|
(unless -D option is used).
|
|
|
|
Args:
|
|
signal_num: Signal that was caught.
|
|
cur_stack_frame: Unused.
|
|
"""
|
|
if debug_level >= 2:
|
|
stack_trace = ''.join(traceback.format_list(traceback.extract_stack()))
|
|
_OutputAndExit('DEBUG: Caught CTRL-C (signal %d) - Exception stack trace:\n'
|
|
' %s' %
|
|
(signal_num, re.sub('\\n', '\n ', stack_trace)),
|
|
exception=ControlCException())
|
|
else:
|
|
_OutputAndExit('Caught CTRL-C (signal %d) - exiting' % signal_num,
|
|
exception=ControlCException())
|
|
|
|
|
|
def _HandleSigQuit(signal_num, cur_stack_frame):
|
|
r"""Called when user hits ^\, so we can force breakpoint a running gsutil."""
|
|
import pdb # pylint: disable=g-import-not-at-top
|
|
pdb.set_trace()
|
|
|
|
|
|
def _ConstructAccountProblemHelp(reason):
|
|
"""Constructs a help string for an access control error.
|
|
|
|
Args:
|
|
reason: e.reason string from caught exception.
|
|
|
|
Returns:
|
|
Contructed help text.
|
|
"""
|
|
default_project_id = boto.config.get_value('GSUtil', 'default_project_id')
|
|
# pylint: disable=line-too-long, g-inconsistent-quotes
|
|
acct_help = (
|
|
"Your request resulted in an AccountProblem (403) error. Usually this "
|
|
"happens if you attempt to create a bucket without first having "
|
|
"enabled billing for the project you are using. Please ensure billing is "
|
|
"enabled for your project by following the instructions at "
|
|
"`Google Cloud Platform Console<https://support.google.com/cloud/answer/6158867>`. "
|
|
)
|
|
if default_project_id:
|
|
acct_help += (
|
|
"In the project overview, ensure that the Project Number listed for "
|
|
"your project matches the project ID (%s) from your boto config file. "
|
|
% default_project_id)
|
|
acct_help += (
|
|
"If the above doesn't resolve your AccountProblem, please send mail to "
|
|
"buganizer-system+187143@google.com requesting assistance, noting the "
|
|
"exact command you ran, the fact that you received a 403 AccountProblem "
|
|
"error, and your project ID. Please do not post your project ID on "
|
|
"StackOverflow. "
|
|
"Note: It's possible to use Google Cloud Storage without enabling "
|
|
"billing if you're only listing or reading objects for which you're "
|
|
"authorized, or if you're uploading objects to a bucket billed to a "
|
|
"project that has billing enabled. But if you're attempting to create "
|
|
"buckets or upload objects to a bucket owned by your own project, you "
|
|
"must first enable billing for that project.")
|
|
return acct_help
|
|
|
|
|
|
def _CheckAndHandleCredentialException(e, args):
|
|
# Provide detail to users who have no boto config file (who might previously
|
|
# have been using gsutil only for accessing publicly readable buckets and
|
|
# objects).
|
|
if (not boto_util.HasConfiguredCredentials() and not boto.config.get_value(
|
|
'Tests', 'bypass_anonymous_access_warning', False)):
|
|
# The check above allows tests to assert that we get a particular,
|
|
# expected failure, rather than always encountering this error message
|
|
# when there are no configured credentials. This allows tests to
|
|
# simulate a second user without permissions, without actually requiring
|
|
# two separate configured users.
|
|
if system_util.InvokedViaCloudSdk():
|
|
message = '\n'.join(
|
|
textwrap.wrap(
|
|
'You are attempting to access protected data with no configured '
|
|
'credentials. Please visit '
|
|
'https://cloud.google.com/console#/project and sign up for an '
|
|
'account, and then run the "gcloud auth login" command to '
|
|
'configure gsutil to use these credentials.'))
|
|
else:
|
|
message = '\n'.join(
|
|
textwrap.wrap(
|
|
'You are attempting to access protected data with no configured '
|
|
'credentials. Please visit '
|
|
'https://cloud.google.com/console#/project and sign up for an '
|
|
'account, and then run the "gsutil config" command to configure '
|
|
'gsutil to use these credentials.'))
|
|
_OutputAndExit(message=message, exception=e)
|
|
elif (e.reason and
|
|
(e.reason == 'AccountProblem' or e.reason == 'Account disabled.' or
|
|
'account for the specified project has been disabled' in e.reason) and
|
|
','.join(args).find('gs://') != -1):
|
|
_OutputAndExit('\n'.join(
|
|
textwrap.wrap(_ConstructAccountProblemHelp(e.reason))),
|
|
exception=e)
|
|
|
|
|
|
def _RunNamedCommandAndHandleExceptions(command_runner,
|
|
command_name,
|
|
args=None,
|
|
headers=None,
|
|
debug_level=0,
|
|
trace_token=None,
|
|
parallel_operations=False,
|
|
perf_trace_token=None,
|
|
user_project=None):
|
|
"""Runs the command and handles common exceptions."""
|
|
# Note that this method is run at the end of main() and thus has access to
|
|
# all of the modules imported there.
|
|
# pylint: disable=g-import-not-at-top
|
|
try:
|
|
# Catch ^C so we can print a brief message instead of the normal Python
|
|
# stack trace. Register as a final signal handler because this handler kills
|
|
# the main gsutil process (so it must run last).
|
|
RegisterSignalHandler(signal.SIGINT, _HandleControlC, is_final_handler=True)
|
|
# Catch ^\ so we can force a breakpoint in a running gsutil.
|
|
if not system_util.IS_WINDOWS:
|
|
RegisterSignalHandler(signal.SIGQUIT, _HandleSigQuit)
|
|
|
|
return command_runner.RunNamedCommand(command_name,
|
|
args,
|
|
headers,
|
|
debug_level,
|
|
trace_token,
|
|
parallel_operations,
|
|
perf_trace_token=perf_trace_token,
|
|
collect_analytics=True,
|
|
user_project=user_project)
|
|
except AttributeError as e:
|
|
if str(e).find('secret_access_key') != -1:
|
|
_OutputAndExit(
|
|
'Missing credentials for the given URI(s). Does your '
|
|
'boto config file contain all needed credentials?',
|
|
exception=e)
|
|
else:
|
|
_OutputAndExit(message=str(e), exception=e)
|
|
except CommandException as e:
|
|
_HandleCommandException(e)
|
|
except getopt.GetoptError as e:
|
|
_HandleCommandException(CommandException(e.msg))
|
|
except boto.exception.InvalidUriError as e:
|
|
_OutputAndExit(message='InvalidUriError: %s.' % e.message, exception=e)
|
|
except gslib.exception.InvalidUrlError as e:
|
|
_OutputAndExit(message='InvalidUrlError: %s.' % e.message, exception=e)
|
|
except boto.auth_handler.NotReadyToAuthenticate as e:
|
|
_OutputAndExit(message='NotReadyToAuthenticate', exception=e)
|
|
except gslib.exception.ExternalBinaryError as e:
|
|
_OutputAndExit(message=str(e), exception=e)
|
|
except OSError as e:
|
|
# In Python 3, IOError (next except) is an alias for OSError
|
|
# Sooo... we need the same logic here
|
|
if (e.errno == errno.EPIPE or
|
|
(system_util.IS_WINDOWS and e.errno == errno.EINVAL) and
|
|
not system_util.IsRunningInteractively()):
|
|
# If we get a pipe error, this just means that the pipe to stdout or
|
|
# stderr is broken. This can happen if the user pipes gsutil to a command
|
|
# that doesn't use the entire output stream. Instead of raising an error,
|
|
# just swallow it up and exit cleanly.
|
|
sys.exit(0)
|
|
else:
|
|
_OutputAndExit(message='OSError: %s.' % e.strerror, exception=e)
|
|
except IOError as e:
|
|
if (e.errno == errno.EPIPE or
|
|
(system_util.IS_WINDOWS and e.errno == errno.EINVAL) and
|
|
not system_util.IsRunningInteractively()):
|
|
# If we get a pipe error, this just means that the pipe to stdout or
|
|
# stderr is broken. This can happen if the user pipes gsutil to a command
|
|
# that doesn't use the entire output stream. Instead of raising an error,
|
|
# just swallow it up and exit cleanly.
|
|
sys.exit(0)
|
|
else:
|
|
raise
|
|
except wildcard_iterator.WildcardException as e:
|
|
_OutputAndExit(message=e.reason, exception=e)
|
|
except ProjectIdException as e:
|
|
_OutputAndExit(
|
|
'You are attempting to perform an operation that requires a '
|
|
'project id, with none configured. Please re-run '
|
|
'gsutil config and make sure to follow the instructions for '
|
|
'finding and entering your default project id.',
|
|
exception=e)
|
|
except BadRequestException as e:
|
|
if e.reason == 'MissingSecurityHeader':
|
|
_CheckAndHandleCredentialException(e, args)
|
|
_OutputAndExit(message=e, exception=e)
|
|
except AccessDeniedException as e:
|
|
_CheckAndHandleCredentialException(e, args)
|
|
_OutputAndExit(message=e, exception=e)
|
|
except ArgumentException as e:
|
|
_OutputAndExit(message=e, exception=e)
|
|
except ServiceException as e:
|
|
_OutputAndExit(message=e, exception=e)
|
|
except (oauth2client.client.HttpAccessTokenRefreshError,
|
|
google_auth_exceptions.OAuthError,
|
|
google_auth_exceptions.RefreshError) as e:
|
|
if system_util.InvokedViaCloudSdk():
|
|
_OutputAndExit(
|
|
'Your credentials are invalid. '
|
|
'Please run\n$ gcloud auth login',
|
|
exception=e)
|
|
else:
|
|
_OutputAndExit(
|
|
'Your credentials are invalid. For more help, see '
|
|
'"gsutil help creds", or re-run the gsutil config command (see '
|
|
'"gsutil help config").',
|
|
exception=e)
|
|
except apitools_exceptions.HttpError as e:
|
|
# These should usually be retried by the underlying implementation or
|
|
# wrapped by CloudApi ServiceExceptions, but if we do get them,
|
|
# print something useful.
|
|
_OutputAndExit('HttpError: %s, %s' %
|
|
(getattr(e.response, 'status', ''), e.content or ''),
|
|
exception=e)
|
|
except socket.error as e:
|
|
if e.args[0] == errno.EPIPE:
|
|
# Retrying with a smaller file (per suggestion below) works because
|
|
# the library code send loop (in boto/s3/key.py) can get through the
|
|
# entire file and then request the HTTP response before the socket
|
|
# gets closed and the response lost.
|
|
_OutputAndExit(
|
|
'Got a "Broken pipe" error. This can happen to clients using Python '
|
|
'2.x, when the server sends an error response and then closes the '
|
|
'socket (see http://bugs.python.org/issue5542). If you are trying to '
|
|
'upload a large object you might retry with a small (say 200k) '
|
|
'object, and see if you get a more specific error code.',
|
|
exception=e)
|
|
elif e.args[0] == errno.ECONNRESET and ' '.join(args).contains('s3://'):
|
|
_OutputAndExit('\n'.join(
|
|
textwrap.wrap(
|
|
'Got a "Connection reset by peer" error. One way this can happen is '
|
|
'when copying data to/from an S3 regional bucket. If you are using a '
|
|
'regional S3 bucket you could try re-running this command using the '
|
|
'regional S3 endpoint, for example '
|
|
's3://s3-<region>.amazonaws.com/your-bucket. For details about this '
|
|
'problem see https://github.com/boto/boto/issues/2207')),
|
|
exception=e)
|
|
else:
|
|
_HandleUnknownFailure(e)
|
|
except oauth2client.client.FlowExchangeError as e:
|
|
_OutputAndExit('\n%s\n\n' % '\n'.join(
|
|
textwrap.wrap(
|
|
'Failed to retrieve valid credentials (%s). Make sure you selected and '
|
|
'pasted the ENTIRE authorization code (including any numeric prefix '
|
|
"e.g. '4/')." % e)),
|
|
exception=e)
|
|
except reauth_errors.ReauthSamlLoginRequiredError:
|
|
if system_util.InvokedViaCloudSdk():
|
|
_OutputAndExit('You must re-authenticate with your SAML IdP. '
|
|
'Please run\n$ gcloud auth login')
|
|
else:
|
|
_OutputAndExit('You must re-authenticate with your SAML IdP. '
|
|
'Please run\n$ gsutil config')
|
|
except Exception as e: # pylint: disable=broad-except
|
|
config_paths = ', '.join(boto_util.GetFriendlyConfigFilePaths())
|
|
# Check for two types of errors related to service accounts. These errors
|
|
# appear to be the same except for their messages, but they are caused by
|
|
# different problems and both have unhelpful error messages. Moreover,
|
|
# the error type belongs to PyOpenSSL, which is not necessarily installed.
|
|
if 'mac verify failure' in str(e):
|
|
_OutputAndExit(
|
|
'Encountered an error while refreshing access token. '
|
|
'If you are using a service account,\nplease verify that the '
|
|
'gs_service_key_file_password field in your config file(s),'
|
|
'\n%s, is correct.' % config_paths,
|
|
exception=e)
|
|
elif 'asn1 encoding routines' in str(e):
|
|
_OutputAndExit(
|
|
'Encountered an error while refreshing access token. '
|
|
'If you are using a service account,\nplease verify that the '
|
|
'gs_service_key_file field in your config file(s),\n%s, is correct.' %
|
|
config_paths,
|
|
exception=e)
|
|
_HandleUnknownFailure(e)
|
|
|
|
|
|
def _PerformTabCompletion(command_runner):
|
|
"""Performs gsutil-specific tab completion for the shell."""
|
|
# argparse and argcomplete are bundled with the Google Cloud SDK.
|
|
# When gsutil is invoked from the Google Cloud SDK, both should be available.
|
|
try:
|
|
import argcomplete
|
|
import argparse
|
|
except ImportError as e:
|
|
_OutputAndExit('A library required for performing tab completion was'
|
|
' not found.\nCause: %s' % e,
|
|
exception=e)
|
|
parser = argparse.ArgumentParser(add_help=False)
|
|
command_runner.ConfigureCommandArgumentParsers(parser)
|
|
argcomplete.autocomplete(parser, exit_method=sys.exit)
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|