667 lines
26 KiB
Python
667 lines
26 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright 2018 Google Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Shared utility structures and methods that require importing boto.
|
|
|
|
This module also imports httplib2 (as it is Boto's http transport and closely
|
|
tied to some of Boto's core functionality) and oauth2client.
|
|
"""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import print_function
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
import os
|
|
import pkgutil
|
|
import tempfile
|
|
import textwrap
|
|
|
|
import six
|
|
import boto
|
|
from boto import config
|
|
import boto.auth
|
|
from boto.exception import NoAuthHandlerFound
|
|
from boto.gs.connection import GSConnection
|
|
from boto.provider import Provider
|
|
from boto.pyami.config import BotoConfigLocations
|
|
|
|
import gslib
|
|
from gslib import context_config
|
|
from gslib.exception import CommandException
|
|
from gslib.utils import system_util
|
|
from gslib.utils.constants import DEFAULT_GCS_JSON_API_VERSION
|
|
from gslib.utils.constants import DEFAULT_GSUTIL_STATE_DIR
|
|
from gslib.utils.constants import SSL_TIMEOUT_SEC
|
|
from gslib.utils.constants import UTF8
|
|
from gslib.utils.unit_util import HumanReadableToBytes
|
|
from gslib.utils.unit_util import ONE_MIB
|
|
|
|
import httplib2
|
|
from oauth2client.client import HAS_CRYPTO
|
|
|
|
if six.PY3:
|
|
long = int
|
|
|
|
# Globals in this module are set according to values in the boto config.
|
|
BOTO_IS_SECURE = config.get('Boto', 'is_secure', True)
|
|
CERTIFICATE_VALIDATION_ENABLED = config.get('Boto',
|
|
'https_validate_certificates', True)
|
|
|
|
configured_certs_file = None # Single certs file for use across all processes.
|
|
temp_certs_file = None # Temporary certs file for cleanup upon exit.
|
|
|
|
|
|
def ConfigureCertsFile():
|
|
"""Configures and returns the CA Certificates file.
|
|
|
|
If one is already configured, use it. Otherwise, use the cert roots
|
|
distributed with gsutil.
|
|
|
|
Returns:
|
|
string filename of the certs file to use.
|
|
"""
|
|
certs_file = boto.config.get('Boto', 'ca_certificates_file', None)
|
|
# The 'system' keyword indicates to use the system installed certs. Some
|
|
# Linux distributions patch the stack such that the Python SSL
|
|
# infrastructure picks up the system installed certs by default, thus no
|
|
# action necessary on our part
|
|
if certs_file == 'system':
|
|
return None
|
|
if not certs_file:
|
|
global configured_certs_file, temp_certs_file
|
|
if not configured_certs_file:
|
|
configured_certs_file = os.path.abspath(
|
|
os.path.join(gslib.GSLIB_DIR, 'data', 'cacerts.txt'))
|
|
if not os.path.exists(configured_certs_file):
|
|
# If the file is not present on disk, this means the gslib module
|
|
# doesn't actually exist on disk anywhere. This can happen if it's
|
|
# being imported from a zip file. Unfortunately, we have to copy the
|
|
# certs file to a local temp file on disk because the underlying SSL
|
|
# socket requires it to be a filesystem path.
|
|
certs_data = pkgutil.get_data('gslib', 'data/cacerts.txt')
|
|
if not certs_data:
|
|
raise CommandException('Certificates file not found. Please '
|
|
'reinstall gsutil from scratch')
|
|
certs_data = six.ensure_str(certs_data)
|
|
fd, fname = tempfile.mkstemp(suffix='.txt', prefix='gsutil-cacerts')
|
|
f = os.fdopen(fd, 'w')
|
|
f.write(certs_data)
|
|
f.close()
|
|
temp_certs_file = fname
|
|
configured_certs_file = temp_certs_file
|
|
certs_file = configured_certs_file
|
|
return certs_file
|
|
|
|
|
|
def ConfigureNoOpAuthIfNeeded():
|
|
"""Sets up no-op auth handler if no boto credentials are configured."""
|
|
if not HasConfiguredCredentials():
|
|
if (config.has_option('Credentials', 'gs_service_client_id') and
|
|
not HAS_CRYPTO):
|
|
if system_util.InvokedViaCloudSdk():
|
|
raise CommandException('\n'.join(
|
|
textwrap.wrap(
|
|
'Your gsutil is configured with an OAuth2 service account, but '
|
|
'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. '
|
|
'Service account authentication requires one of these libraries; '
|
|
'please reactivate your service account via the gcloud auth '
|
|
'command and ensure any gcloud packages necessary for '
|
|
'service accounts are present.')))
|
|
else:
|
|
raise CommandException('\n'.join(
|
|
textwrap.wrap(
|
|
'Your gsutil is configured with an OAuth2 service account, but '
|
|
'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. '
|
|
'Service account authentication requires one of these libraries; '
|
|
'please install either of them to proceed, or configure a '
|
|
'different type of credentials with "gsutil config".')))
|
|
else:
|
|
# With no boto config file the user can still access publicly readable
|
|
# buckets and objects.
|
|
from gslib import no_op_auth_plugin # pylint: disable=unused-variable
|
|
|
|
|
|
def GetCertsFile():
|
|
return configured_certs_file
|
|
|
|
|
|
def GetCleanupFiles():
|
|
"""Returns a list of temp files to delete (if possible) when program exits."""
|
|
return [temp_certs_file] if temp_certs_file else []
|
|
|
|
|
|
def GetConfigFilePaths():
|
|
"""Returns a list of the path(s) to the boto config file(s) to be loaded."""
|
|
potential_config_paths = BotoConfigLocations
|
|
|
|
# When Boto's pyami.config.Config class is initialized, it attempts to read
|
|
# this file, transform it into valid Boto config syntax, and load credentials
|
|
# from it, but it does not add this file to BotoConfigLocations.
|
|
if 'AWS_CREDENTIAL_FILE' in os.environ:
|
|
potential_config_paths.append(os.environ['AWS_CREDENTIAL_FILE'])
|
|
# Provider-specific config files, like the one below, are checked for and
|
|
# loaded when Boto's Provider class is initialized. These aren't listed in
|
|
# BotoConfigLocations, but can still affect what credentials are loaded, so
|
|
# we display them in our config list to make auth'n debugging easier.
|
|
aws_cred_file = os.path.join(os.path.expanduser('~'), '.aws', 'credentials')
|
|
if os.path.isfile(aws_cred_file):
|
|
potential_config_paths.append(aws_cred_file)
|
|
|
|
# Only return credential files which we have permission to read (and thus can
|
|
# actually load credentials from).
|
|
readable_config_paths = []
|
|
for path in potential_config_paths:
|
|
try:
|
|
with open(path, 'r'):
|
|
readable_config_paths.append(path)
|
|
except IOError:
|
|
pass
|
|
return readable_config_paths
|
|
|
|
|
|
def GetFriendlyConfigFilePaths():
|
|
"""Like GetConfigFilePaths but returns a not-found message if paths empty."""
|
|
readable_config_paths = GetConfigFilePaths()
|
|
if len(readable_config_paths) == 0:
|
|
readable_config_paths.append('No config found')
|
|
return readable_config_paths
|
|
|
|
|
|
def GetGsutilStateDir():
|
|
"""Returns the location of the directory for gsutil state files.
|
|
|
|
Certain operations, such as cross-process credential sharing and
|
|
resumable transfer tracking, need a known location for state files which
|
|
are created by gsutil as-needed.
|
|
|
|
This location should only be used for storing data that is required to be in
|
|
a static location.
|
|
|
|
Returns:
|
|
Path to directory for gsutil static state files.
|
|
"""
|
|
config_file_dir = config.get('GSUtil', 'state_dir', DEFAULT_GSUTIL_STATE_DIR)
|
|
system_util.CreateDirIfNeeded(config_file_dir)
|
|
return config_file_dir
|
|
|
|
|
|
def GetCredentialStoreFilename():
|
|
# As of gsutil 4.29, this changed from 'credstore' to 'credstore2' because
|
|
# of a change to the underlying credential storage format.
|
|
return os.path.join(GetGsutilStateDir(), 'credstore2')
|
|
|
|
|
|
def GetGceCredentialCacheFilename():
|
|
return os.path.join(GetGsutilStateDir(), 'gcecredcache')
|
|
|
|
|
|
def GetGcsJsonApiVersion():
|
|
return config.get('GSUtil', 'json_api_version', DEFAULT_GCS_JSON_API_VERSION)
|
|
|
|
|
|
# Resumable downloads and uploads make one HTTP call per chunk (and must be
|
|
# in multiples of 256KiB). Overridable for testing.
|
|
def GetJsonResumableChunkSize():
|
|
chunk_size = config.getint('GSUtil', 'json_resumable_chunk_size',
|
|
long(1024 * 1024 * 100))
|
|
if chunk_size == 0:
|
|
chunk_size = long(1024 * 256)
|
|
elif chunk_size % long(1024 * 256) != 0:
|
|
chunk_size += (long(1024 * 256) - (chunk_size % (long(1024 * 256))))
|
|
return chunk_size
|
|
|
|
|
|
def GetLastCheckedForGsutilUpdateTimestampFile():
|
|
return os.path.join(GetGsutilStateDir(), '.last_software_update_check')
|
|
|
|
|
|
def GetMaxConcurrentCompressedUploads():
|
|
"""Gets the max concurrent transport compressed uploads allowed in parallel.
|
|
|
|
Returns:
|
|
The max number of concurrent transport compressed uploads allowed in
|
|
parallel without exceeding the max_upload_compression_buffer_size.
|
|
"""
|
|
upload_chunk_size = GetJsonResumableChunkSize()
|
|
# From apitools compression.py.
|
|
compression_chunk_size = 16 * ONE_MIB
|
|
total_upload_size = (upload_chunk_size + compression_chunk_size + 17 + 5 *
|
|
(((compression_chunk_size - 1) / 16383) + 1))
|
|
max_concurrent_uploads = (GetMaxUploadCompressionBufferSize() /
|
|
total_upload_size)
|
|
if max_concurrent_uploads <= 0:
|
|
max_concurrent_uploads = 1
|
|
return max_concurrent_uploads
|
|
|
|
|
|
def GetMaxRetryDelay():
|
|
return config.getint('Boto', 'max_retry_delay', 32)
|
|
|
|
|
|
def GetMaxUploadCompressionBufferSize():
|
|
"""Get the max amount of memory compressed transport uploads may buffer."""
|
|
return HumanReadableToBytes(
|
|
config.get('GSUtil', 'max_upload_compression_buffer_size', '2GiB'))
|
|
|
|
|
|
def GetNewHttp(http_class=httplib2.Http, **kwargs):
|
|
"""Creates and returns a new httplib2.Http instance.
|
|
|
|
Args:
|
|
http_class: Optional custom Http class to use.
|
|
**kwargs: Arguments to pass to http_class constructor.
|
|
|
|
Returns:
|
|
An initialized httplib2.Http instance.
|
|
"""
|
|
|
|
##Get Proxy configuration from boto file, defaults are None, 0 and False
|
|
proxy_host = config.get('Boto', 'proxy', None) #needed for if clause below
|
|
boto_proxy_config = {
|
|
'proxy_host':
|
|
proxy_host,
|
|
'proxy_type':
|
|
config.get('Boto', 'proxy_type', 'http'),
|
|
'proxy_port':
|
|
config.getint('Boto', 'proxy_port'),
|
|
'proxy_user':
|
|
config.get('Boto', 'proxy_user', None),
|
|
'proxy_pass':
|
|
config.get('Boto', 'proxy_pass', None),
|
|
'proxy_rdns':
|
|
config.get('Boto', 'proxy_rdns', True if proxy_host else None)
|
|
}
|
|
|
|
#Use SetProxyInfo to convert boto config to httplib2.proxyinfo object
|
|
proxy_info = SetProxyInfo(boto_proxy_config)
|
|
|
|
# Some installers don't package a certs file with httplib2, so use the
|
|
# one included with gsutil.
|
|
kwargs['ca_certs'] = GetCertsFile()
|
|
# Use a non-infinite SSL timeout to avoid hangs during network flakiness.
|
|
kwargs['timeout'] = SSL_TIMEOUT_SEC
|
|
http = http_class(proxy_info=proxy_info, **kwargs)
|
|
http.disable_ssl_certificate_validation = (not config.getbool(
|
|
'Boto', 'https_validate_certificates'))
|
|
|
|
global_context_config = context_config.get_context_config()
|
|
if global_context_config and global_context_config.use_client_certificate:
|
|
http.add_certificate(key=global_context_config.client_cert_path,
|
|
cert=global_context_config.client_cert_path,
|
|
domain='',
|
|
password=global_context_config.client_cert_password)
|
|
|
|
return http
|
|
|
|
|
|
# Retry for 10 minutes with exponential backoff, which corresponds to
|
|
# the maximum Downtime Period specified in the GCS SLA
|
|
# (https://cloud.google.com/storage/sla)
|
|
def GetNumRetries():
|
|
return config.getint('Boto', 'num_retries', 23)
|
|
|
|
|
|
def GetTabCompletionLogFilename():
|
|
return os.path.join(GetGsutilStateDir(), 'tab-completion-logs')
|
|
|
|
|
|
def GetTabCompletionCacheFilename():
|
|
tab_completion_dir = os.path.join(GetGsutilStateDir(), 'tab-completion')
|
|
# Limit read permissions on the directory to owner for privacy.
|
|
system_util.CreateDirIfNeeded(tab_completion_dir, mode=0o700)
|
|
return os.path.join(tab_completion_dir, 'cache')
|
|
|
|
|
|
def HasConfiguredCredentials():
|
|
"""Determines if boto credential/config file exists."""
|
|
has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and
|
|
config.has_option('Credentials', 'gs_secret_access_key'))
|
|
has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and
|
|
config.has_option('Credentials', 'aws_secret_access_key'))
|
|
has_oauth_creds = (config.has_option('Credentials',
|
|
'gs_oauth2_refresh_token'))
|
|
has_external_creds = (config.has_option('Credentials',
|
|
'gs_external_account_file'))
|
|
has_external_account_authorized_user_creds = (config.has_option(
|
|
'Credentials', 'gs_external_account_authorized_user_file'))
|
|
has_service_account_creds = (
|
|
HAS_CRYPTO and
|
|
config.has_option('Credentials', 'gs_service_client_id') and
|
|
config.has_option('Credentials', 'gs_service_key_file'))
|
|
|
|
if (has_goog_creds or has_amzn_creds or has_oauth_creds or
|
|
has_service_account_creds or has_external_creds or
|
|
has_external_account_authorized_user_creds):
|
|
return True
|
|
|
|
valid_auth_handler = None
|
|
try:
|
|
valid_auth_handler = boto.auth.get_auth_handler(GSConnection.DefaultHost,
|
|
config,
|
|
Provider('google'),
|
|
requested_capability=['s3'])
|
|
# Exclude the no-op auth handler as indicating credentials are configured.
|
|
# Note we can't use isinstance() here because the no-op module may not be
|
|
# imported so we can't get a reference to the class type.
|
|
if 'NoOpAuth' == getattr(
|
|
getattr(valid_auth_handler, '__class__', None),
|
|
'__name__',
|
|
None): # yapf: disable
|
|
valid_auth_handler = None
|
|
except NoAuthHandlerFound:
|
|
pass
|
|
|
|
return valid_auth_handler
|
|
|
|
|
|
def JsonResumableChunkSizeDefined():
|
|
chunk_size_defined = config.get('GSUtil', 'json_resumable_chunk_size', None)
|
|
return chunk_size_defined is not None
|
|
|
|
|
|
def MonkeyPatchBoto():
|
|
"""Apply gsutil-specific patches to Boto.
|
|
|
|
Here be dragons. Sorry.
|
|
|
|
Note that this method should not be used as a replacement for contributing
|
|
fixes to the upstream Boto library. However, the Boto library has historically
|
|
not been consistent about release cadence, so upstream fixes may not be
|
|
available immediately in a version which we can pin to. Also, some fixes may
|
|
only be applicable to gsutil. In such cases, patches should be applied to the
|
|
Boto library here (and removed if/when they are included in the upstream
|
|
repository and included in an official new release that we pull in). This
|
|
method should be invoked after all other Boto-related initialization has been
|
|
completed.
|
|
"""
|
|
# We have to do all sorts of gross things here (dynamic imports, invalid names
|
|
# to resolve symbols in copy/pasted methods, invalid spacing from copy/pasted
|
|
# methods, etc.), so we just disable pylint warnings for this whole method.
|
|
# yapf: disable
|
|
# pylint: disable=all
|
|
|
|
# This should have already been imported if this method was called in the
|
|
# correct place, but we need to perform the import within this method for
|
|
# this module to recognize it. We don't import this at module level because
|
|
# its import timing is important and is managed elsewhere.
|
|
import gcs_oauth2_boto_plugin
|
|
|
|
# TODO(https://github.com/boto/boto/issues/3831): Remove this if the GitHub
|
|
# issue is ever addressed.
|
|
orig_get_plugin_method = boto.plugin.get_plugin
|
|
|
|
def _PatchedGetPluginMethod(cls, requested_capability=None):
|
|
handler_subclasses = orig_get_plugin_method(
|
|
cls, requested_capability=requested_capability)
|
|
# In Boto's logic, higher precedence handlers should be closer to the end
|
|
# of the list. We always want to prefer OAuth2 credentials over HMAC
|
|
# credentials if both are present, so we shuffle OAuth2 handlers to the end
|
|
# of the handler list.
|
|
xml_oauth2_handlers = (
|
|
gcs_oauth2_boto_plugin.oauth2_plugin.OAuth2ServiceAccountAuth,
|
|
gcs_oauth2_boto_plugin.oauth2_plugin.OAuth2Auth)
|
|
new_result = (
|
|
# We need to sort each of these to avoid inconsistent handler selection
|
|
# when multiple credential types are configured in Python 3.5. See
|
|
# https://issuetracker.google.com/issues/135709541 for specific logs.
|
|
sorted(
|
|
[r for r in handler_subclasses if r not in xml_oauth2_handlers],
|
|
# Types aren't sortable, so we use their names:
|
|
key=(lambda handler_t: handler_t.__name__),
|
|
) + # Now append XML handlers to the end (highest precedence)
|
|
sorted(
|
|
[r for r in handler_subclasses if r in xml_oauth2_handlers],
|
|
key=(lambda handler_t: handler_t.__name__),
|
|
))
|
|
return new_result
|
|
|
|
boto.plugin.get_plugin = _PatchedGetPluginMethod
|
|
|
|
#########################################################################
|
|
|
|
# TODO(boto>2.49.0): Remove this.
|
|
# Fixes SSL issue where SNI was not being used for OpenSSL 1.1.1+
|
|
# https://github.com/boto/boto/pull/3843/files
|
|
|
|
# Import modules and resolve symbols used by our patch method.
|
|
import socket
|
|
import ssl
|
|
InvalidCertificateException = (
|
|
boto.https_connection.InvalidCertificateException)
|
|
ValidateCertificateHostname = (
|
|
boto.https_connection.ValidateCertificateHostname)
|
|
|
|
def _PatchedConnectMethod(self):
|
|
# The lines below were copied directly from the Boto file, so we don't lint
|
|
# or otherwise alter them.
|
|
if hasattr(self, "timeout"):
|
|
sock = socket.create_connection((self.host, self.port), self.timeout)
|
|
else:
|
|
sock = socket.create_connection((self.host, self.port))
|
|
msg = "wrapping ssl socket; "
|
|
if self.ca_certs:
|
|
msg += "CA certificate file=%s" % self.ca_certs
|
|
else:
|
|
msg += "using system provided SSL certs"
|
|
boto.log.debug(msg)
|
|
if hasattr(ssl, 'SSLContext') and getattr(ssl, 'HAS_SNI', False):
|
|
# Use SSLContext so we can specify server_hostname for SNI
|
|
# (Required for connections to storage.googleapis.com)
|
|
context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
|
|
context.verify_mode = ssl.CERT_REQUIRED
|
|
if self.ca_certs:
|
|
context.load_verify_locations(self.ca_certs)
|
|
if self.cert_file:
|
|
context.load_cert_chain(self.cert_file, self.key_file)
|
|
self.sock = context.wrap_socket(sock, server_hostname=self.host)
|
|
# Add attributes only set in SSLSocket constructor without context:
|
|
self.sock.keyfile = self.key_file
|
|
self.sock.certfile = self.cert_file
|
|
self.sock.cert_reqs = context.verify_mode
|
|
self.sock.ssl_version = ssl.PROTOCOL_SSLv23
|
|
self.sock.ca_certs = self.ca_certs
|
|
self.sock.ciphers = None
|
|
else:
|
|
self.sock = ssl.wrap_socket(sock,
|
|
keyfile=self.key_file,
|
|
certfile=self.cert_file,
|
|
cert_reqs=ssl.CERT_REQUIRED,
|
|
ca_certs=self.ca_certs)
|
|
cert = self.sock.getpeercert()
|
|
hostname = self.host.split(':', 0)[0]
|
|
if not ValidateCertificateHostname(cert, hostname):
|
|
raise InvalidCertificateException(
|
|
hostname, cert, 'remote hostname "%s" does not match '
|
|
'certificate' % hostname)
|
|
# End `_PatchedConnectMethod` declaration.
|
|
|
|
boto.https_connection.CertValidatingHTTPSConnection.connect = (
|
|
_PatchedConnectMethod)
|
|
|
|
|
|
def ProxyInfoFromEnvironmentVar(proxy_env_var):
|
|
"""Reads proxy info from the environment and converts to httplib2.ProxyInfo.
|
|
|
|
Args:
|
|
proxy_env_var: Environment variable string to read, such as http_proxy or
|
|
https_proxy.
|
|
|
|
Returns:
|
|
httplib2.ProxyInfo constructed from the environment string.
|
|
"""
|
|
proxy_url = os.environ.get(proxy_env_var)
|
|
if not proxy_url or not proxy_env_var.lower().startswith('http'):
|
|
return httplib2.ProxyInfo(httplib2.socks.PROXY_TYPE_HTTP, None, 0)
|
|
proxy_protocol = proxy_env_var.lower().split('_')[0]
|
|
if not proxy_url.lower().startswith('http'):
|
|
# proxy_info_from_url requires a protocol, which is always http or https.
|
|
proxy_url = proxy_protocol + '://' + proxy_url
|
|
return httplib2.proxy_info_from_url(proxy_url, method=proxy_protocol)
|
|
|
|
|
|
def ResumableThreshold():
|
|
return config.getint('GSUtil', 'resumable_threshold', 8 * ONE_MIB)
|
|
|
|
|
|
def SetProxyInfo(boto_proxy_config):
|
|
"""Sets proxy info from boto and environment and converts to httplib2.ProxyInfo.
|
|
|
|
Args:
|
|
dict: Values read from the .boto file
|
|
|
|
Returns:
|
|
httplib2.ProxyInfo constructed from boto or environment variable string.
|
|
"""
|
|
#Defining proxy_type based on httplib2 library, accounting for None entry too.
|
|
proxy_type_spec = {'socks4': 1, 'socks5': 2, 'http': 3, 'https': 3}
|
|
|
|
#proxy_type defaults to 'http (3)' for backwards compatibility
|
|
proxy_type = proxy_type_spec.get(
|
|
boto_proxy_config.get('proxy_type').lower(), proxy_type_spec['http'])
|
|
proxy_host = boto_proxy_config.get('proxy_host')
|
|
proxy_port = boto_proxy_config.get('proxy_port')
|
|
proxy_user = boto_proxy_config.get('proxy_user')
|
|
proxy_pass = boto_proxy_config.get('proxy_pass')
|
|
proxy_rdns = bool(boto_proxy_config.get('proxy_rdns'))
|
|
|
|
#For proxy_info below, proxy_rdns fails for socks4 and socks5 so restricting use
|
|
#to http only
|
|
proxy_info = httplib2.ProxyInfo(proxy_host=proxy_host,
|
|
proxy_type=proxy_type,
|
|
proxy_port=proxy_port,
|
|
proxy_user=proxy_user,
|
|
proxy_pass=proxy_pass,
|
|
proxy_rdns=proxy_rdns)
|
|
|
|
#Added to force socks proxies not to use rdns
|
|
if not (proxy_info.proxy_type == proxy_type_spec['http']):
|
|
proxy_info.proxy_rdns = False
|
|
|
|
if not (proxy_info.proxy_host and proxy_info.proxy_port):
|
|
# Fall back to using the environment variable. Use only http proxies.
|
|
for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']:
|
|
if proxy_env_var in os.environ and os.environ[proxy_env_var]:
|
|
proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var)
|
|
# Assume proxy_rnds is True if a proxy environment variable exists.
|
|
if boto_proxy_config.get('proxy_rdns') == None:
|
|
proxy_info.proxy_rdns = True
|
|
break
|
|
|
|
return proxy_info
|
|
|
|
|
|
def UsingCrcmodExtension():
|
|
boto_opt = boto.config.get('GSUtil', 'test_assume_fast_crcmod', None)
|
|
if boto_opt is not None:
|
|
return boto_opt
|
|
# Python 3 makes this attribute tough to access due to the way the top-level
|
|
# crcmod package imports and (identically) names its crcmod module. The only
|
|
# way to get it is "from crcmod.crcmod import _usingExtension". This is the
|
|
# alternative form of that statement, but doesn't pollute this module's
|
|
# namespace with a "_usingExtension" attribute. This also works in both Python
|
|
# 2.7 and 3.5+.
|
|
nested_crcmod = __import__(
|
|
'crcmod.crcmod',
|
|
globals(),
|
|
locals(),
|
|
['_usingExtension'],
|
|
0,
|
|
)
|
|
return getattr(nested_crcmod, '_usingExtension', False)
|
|
|
|
|
|
# TODO(boto-2.49.0): Remove when we pull in the next version of Boto.
|
|
def _PatchedShouldRetryMethod(self, response, chunked_transfer=False):
|
|
"""Replaces boto.s3.key's should_retry() to handle KMS-encrypted objects."""
|
|
# We copy/pasted this from boto and slightly modified it; keep the old
|
|
# formatting style.
|
|
# yapf: disable
|
|
provider = self.bucket.connection.provider
|
|
|
|
if not chunked_transfer:
|
|
if response.status in [500, 503]:
|
|
# 500 & 503 can be plain retries.
|
|
return True
|
|
|
|
if response.getheader('location'):
|
|
# If there's a redirect, plain retry.
|
|
return True
|
|
|
|
if 200 <= response.status <= 299:
|
|
self.etag = response.getheader('etag')
|
|
md5 = self.md5
|
|
if isinstance(md5, bytes):
|
|
md5 = md5.decode(UTF8)
|
|
|
|
# If you use customer-provided encryption keys, the ETag value that
|
|
# Amazon S3 returns in the response will not be the MD5 of the
|
|
# object.
|
|
amz_server_side_encryption_customer_algorithm = response.getheader(
|
|
'x-amz-server-side-encryption-customer-algorithm', None)
|
|
# The same is applicable for KMS-encrypted objects in gs buckets.
|
|
goog_customer_managed_encryption = response.getheader(
|
|
'x-goog-encryption-kms-key-name', None)
|
|
if (amz_server_side_encryption_customer_algorithm is None and
|
|
goog_customer_managed_encryption is None):
|
|
if self.etag != '"%s"' % md5:
|
|
raise provider.storage_data_error(
|
|
'ETag from S3 did not match computed MD5. '
|
|
'%s vs. %s' % (self.etag, self.md5))
|
|
|
|
return True
|
|
|
|
if response.status == 400:
|
|
# The 400 must be trapped so the retry handler can check to
|
|
# see if it was a timeout.
|
|
# If ``RequestTimeout`` is present, we'll retry. Otherwise, bomb
|
|
# out.
|
|
body = response.read()
|
|
err = provider.storage_response_error(
|
|
response.status,
|
|
response.reason,
|
|
body
|
|
)
|
|
|
|
if err.error_code in ['RequestTimeout']:
|
|
raise boto.exception.PleaseRetryException(
|
|
"Saw %s, retrying" % err.error_code,
|
|
response=response
|
|
)
|
|
|
|
return False
|
|
|
|
def HasUserSpecifiedGsHost():
|
|
gs_host = boto.config.get('Credentials', 'gs_host', None)
|
|
default_host = six.ensure_str(boto.gs.connection.GSConnection.DefaultHost)
|
|
|
|
if gs_host is not None:
|
|
return default_host == six.ensure_str(gs_host)
|
|
|
|
return False
|
|
|
|
def UsingGsHmac():
|
|
# NOTE: External credentials are omitted intentionally as HMAC takes priority
|
|
# over the external credentials types.
|
|
config = boto.config
|
|
has_refresh_token = config.has_option('Credentials', 'gs_oauth2_refresh_token')
|
|
has_service_account_credentials = (
|
|
config.has_option('Credentials', 'gs_service_client_id')
|
|
and config.has_option('Credentials', 'gs_service_key_file'))
|
|
has_hmac_credentials = (
|
|
config.has_option('Credentials', 'gs_access_key_id')
|
|
and config.has_option('Credentials', 'gs_secret_access_key'))
|
|
return (not has_refresh_token and not has_service_account_credentials
|
|
and has_hmac_credentials)
|