131 lines
4.8 KiB
Python
131 lines
4.8 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright 2018 Google Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Shared, hard-coded constants.
|
|
|
|
A constant should not be placed in this file if:
|
|
- it requires complicated or conditional logic to initialize.
|
|
- it requires importing any modules outside of the Python standard library. This
|
|
helps reduce dependency graph complexity and the chance of cyclic deps.
|
|
- it is only used in one file (in which case it should be defined within that
|
|
module).
|
|
- it semantically belongs somewhere else (e.g. 'BYTES_PER_KIB' would belong in
|
|
unit_util.py).
|
|
"""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import print_function
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
import os
|
|
import sys
|
|
|
|
import six
|
|
|
|
from gslib.utils.unit_util import ONE_GIB
|
|
from gslib.utils.unit_util import ONE_KIB
|
|
from gslib.utils.unit_util import ONE_MIB
|
|
|
|
# Readable descriptions for httplib2 logging levels.
|
|
DEBUGLEVEL_DUMP_REQUESTS = 3
|
|
DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADS = 4
|
|
|
|
DEFAULT_FILE_BUFFER_SIZE = 8 * ONE_KIB
|
|
|
|
DEFAULT_GCS_JSON_API_VERSION = 'v1'
|
|
|
|
DEFAULT_GSUTIL_STATE_DIR = os.path.expanduser(os.path.join('~', '.gsutil'))
|
|
|
|
GSUTIL_PUB_TARBALL = 'gs://pub/gsutil.tar.gz'
|
|
GSUTIL_PUB_TARBALL_PY2 = 'gs://pub/gsutil4.tar.gz'
|
|
|
|
IAM_POLICY_VERSION = 3
|
|
|
|
IMPERSONATE_SERVICE_ACCOUNT = ''
|
|
|
|
# Number of seconds to wait before printing a long retry warning message.
|
|
LONG_RETRY_WARN_SEC = 10
|
|
|
|
# Compressed transport encoded uploads buffer chunks of compressed data. When
|
|
# running many uploads in parallel, compression may consume more memory than
|
|
# available. This restricts the number of compressed transport encoded uploads
|
|
# running in parallel such that they don't consume more memory than set here.
|
|
MAX_UPLOAD_COMPRESSION_BUFFER_SIZE = 2 * ONE_GIB
|
|
|
|
# On Unix-like systems, we will set the maximum number of open files to avoid
|
|
# hitting the limit imposed by the OS. This number was obtained experimentally.
|
|
MIN_ACCEPTABLE_OPEN_FILES_LIMIT = 1000
|
|
|
|
# For files >= this size, output a message indicating that we're running an
|
|
# operation on the file (like hashing or gzipping) so it does not appear to the
|
|
# user that the command is hanging.
|
|
# TODO: This should say the unit in the name.
|
|
MIN_SIZE_COMPUTE_LOGGING = 100 * ONE_MIB
|
|
|
|
# The way NO_MAX is used, what is really needed here is the maximum container
|
|
# size in the Python C code, so using six.MAXSIZE which provides that portably.
|
|
NO_MAX = six.MAXSIZE
|
|
|
|
# Number of objects to request in listing calls.
|
|
NUM_OBJECTS_PER_LIST_PAGE = 1000
|
|
|
|
RELEASE_NOTES_URL = 'https://pub.storage.googleapis.com/gsutil_ReleaseNotes.txt'
|
|
|
|
REQUEST_REASON_ENV_VAR = 'CLOUDSDK_CORE_REQUEST_REASON'
|
|
REQUEST_REASON_HEADER_KEY = 'x-goog-request-reason'
|
|
|
|
RESUMABLE_THRESHOLD_MIB = 8
|
|
RESUMABLE_THRESHOLD_B = RESUMABLE_THRESHOLD_MIB * ONE_MIB
|
|
|
|
# gsutil-specific GUIDs for marking special metadata for S3 compatibility.
|
|
S3_ACL_MARKER_GUID = '3b89a6b5-b55a-4900-8c44-0b0a2f5eab43-s3-AclMarker'
|
|
S3_DELETE_MARKER_GUID = 'eadeeee8-fa8c-49bb-8a7d-0362215932d8-s3-DeleteMarker'
|
|
S3_MARKER_GUIDS = [S3_ACL_MARKER_GUID, S3_DELETE_MARKER_GUID]
|
|
|
|
# By default, the timeout for SSL read errors is infinite. This could
|
|
# cause gsutil to hang on network disconnect, so pick a more reasonable
|
|
# timeout.
|
|
SSL_TIMEOUT_SEC = 60
|
|
|
|
# Start with a progress callback every <X> KiB during uploads/downloads (JSON
|
|
# API). Callback implementation should back off until it hits some maximum size
|
|
# so that callbacks do not create huge amounts of log output.
|
|
START_CALLBACK_PER_BYTES = 256 * ONE_KIB
|
|
|
|
# Upload/download files in 8 KiB chunks over the HTTP connection.
|
|
# TODO: This should say the unit in the name.
|
|
if 'win32' in str(sys.platform).lower():
|
|
TRANSFER_BUFFER_SIZE = 64 * ONE_KIB
|
|
else:
|
|
TRANSFER_BUFFER_SIZE = 8 * ONE_KIB
|
|
|
|
UTF8 = 'utf-8'
|
|
|
|
WINDOWS_1252 = 'cp1252'
|
|
|
|
# Default number of progress callbacks during transfer (XML API).
|
|
XML_PROGRESS_CALLBACKS = 10
|
|
|
|
|
|
class Scopes(object):
|
|
"""Enum class for auth scopes, as unicode."""
|
|
CLOUD_PLATFORM = 'https://www.googleapis.com/auth/cloud-platform'
|
|
CLOUD_PLATFORM_READ_ONLY = (
|
|
'https://www.googleapis.com/auth/cloud-platform.read-only')
|
|
FULL_CONTROL = 'https://www.googleapis.com/auth/devstorage.full_control'
|
|
READ_ONLY = 'https://www.googleapis.com/auth/devstorage.read_only'
|
|
READ_WRITE = 'https://www.googleapis.com/auth/devstorage.read_write'
|
|
REAUTH = 'https://www.googleapis.com/auth/accounts.reauth'
|