638 lines
20 KiB
Python
638 lines
20 KiB
Python
# -*- coding: utf-8 -*- #
|
|
# Copyright 2017 Google LLC. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Common helper methods for Service Management commands."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
import json
|
|
import re
|
|
|
|
from apitools.base.py import encoding
|
|
from apitools.base.py import exceptions as apitools_exceptions
|
|
from apitools.base.py import list_pager
|
|
|
|
from googlecloudsdk.api_lib.endpoints import exceptions
|
|
from googlecloudsdk.api_lib.util import apis
|
|
from googlecloudsdk.calliope import exceptions as calliope_exceptions
|
|
from googlecloudsdk.core import log
|
|
from googlecloudsdk.core import properties
|
|
from googlecloudsdk.core import resources
|
|
from googlecloudsdk.core import yaml
|
|
from googlecloudsdk.core.resource import resource_printer
|
|
from googlecloudsdk.core.util import files
|
|
from googlecloudsdk.core.util import retry
|
|
import six
|
|
|
|
|
|
EMAIL_REGEX = re.compile(r'^.+@([^.@][^@]+)$')
|
|
FINGERPRINT_REGEX = re.compile(
|
|
r'^([a-f0-9][a-f0-9]:){19}[a-f0-9][a-f0-9]$', re.IGNORECASE)
|
|
OP_BASE_CMD = 'gcloud endpoints operations '
|
|
OP_DESCRIBE_CMD = OP_BASE_CMD + 'describe {0}'
|
|
OP_WAIT_CMD = OP_BASE_CMD + 'wait {0}'
|
|
SERVICES_COLLECTION = 'servicemanagement.services'
|
|
CONFIG_COLLECTION = 'servicemanagement.services.configs'
|
|
|
|
ALL_IAM_PERMISSIONS = [
|
|
'servicemanagement.services.get',
|
|
'servicemanagement.services.getProjectSettings',
|
|
'servicemanagement.services.delete',
|
|
'servicemanagement.services.update',
|
|
'servicemanagement.services.bind',
|
|
'servicemanagement.services.updateProjectSettings',
|
|
'servicemanagement.services.check',
|
|
'servicemanagement.services.report',
|
|
'servicemanagement.services.setIamPolicy',
|
|
'servicemanagement.services.getIamPolicy',
|
|
]
|
|
|
|
|
|
def GetMessagesModule():
|
|
return apis.GetMessagesModule('servicemanagement', 'v1')
|
|
|
|
|
|
def GetClientInstance():
|
|
return apis.GetClientInstance('servicemanagement', 'v1')
|
|
|
|
|
|
def GetServiceManagementServiceName():
|
|
return 'servicemanagement.googleapis.com'
|
|
|
|
|
|
def GetValidatedProject(project_id):
|
|
"""Validate the project ID, if supplied, otherwise return the default project.
|
|
|
|
Args:
|
|
project_id: The ID of the project to validate. If None, gcloud's default
|
|
project's ID will be returned.
|
|
|
|
Returns:
|
|
The validated project ID.
|
|
"""
|
|
if project_id:
|
|
properties.VALUES.core.project.Validate(project_id)
|
|
else:
|
|
project_id = properties.VALUES.core.project.Get(required=True)
|
|
return project_id
|
|
|
|
|
|
def GetProjectSettings(service, consumer_project_id, view):
|
|
"""Returns the project settings for a given service, project, and view.
|
|
|
|
Args:
|
|
service: The service for which to return project settings.
|
|
consumer_project_id: The consumer project id for which to return settings.
|
|
view: The view (CONSUMER_VIEW or PRODUCER_VIEW).
|
|
|
|
Returns:
|
|
A ProjectSettings message with the settings populated.
|
|
"""
|
|
# Shorten the request names for better readability
|
|
get_request = (GetMessagesModule()
|
|
.ServicemanagementServicesProjectSettingsGetRequest)
|
|
|
|
# Get the current list of quota settings to see if the quota override
|
|
# exists in the first place.
|
|
request = get_request(
|
|
serviceName=service,
|
|
consumerProjectId=consumer_project_id,
|
|
view=view,
|
|
)
|
|
|
|
return GetClientInstance().services_projectSettings.Get(request)
|
|
|
|
|
|
def GetProducedListRequest(project_id):
|
|
return GetMessagesModule().ServicemanagementServicesListRequest(
|
|
producerProjectId=project_id
|
|
)
|
|
|
|
|
|
def PrettyPrint(resource, print_format='json'):
|
|
"""Prints the given resource.
|
|
|
|
Args:
|
|
resource: The resource to print out.
|
|
print_format: The print_format value to pass along to the resource_printer.
|
|
"""
|
|
resource_printer.Print(
|
|
resources=[resource],
|
|
print_format=print_format,
|
|
out=log.out)
|
|
|
|
|
|
def PushAdvisorChangeTypeToString(change_type):
|
|
"""Convert a ConfigChange.ChangeType enum to a string.
|
|
|
|
Args:
|
|
change_type: The ConfigChange.ChangeType enum to convert.
|
|
|
|
Returns:
|
|
An easily readable string representing the ConfigChange.ChangeType enum.
|
|
"""
|
|
messages = GetMessagesModule()
|
|
enums = messages.ConfigChange.ChangeTypeValueValuesEnum
|
|
if change_type in [enums.ADDED, enums.REMOVED, enums.MODIFIED]:
|
|
return six.text_type(change_type).lower()
|
|
else:
|
|
return '[unknown]'
|
|
|
|
|
|
def PushAdvisorConfigChangeToString(config_change):
|
|
"""Convert a ConfigChange message to a printable string.
|
|
|
|
Args:
|
|
config_change: The ConfigChange message to convert.
|
|
|
|
Returns:
|
|
An easily readable string representing the ConfigChange message.
|
|
"""
|
|
result = ('Element [{element}] (old value = {old_value}, '
|
|
'new value = {new_value}) was {change_type}. Advice:\n').format(
|
|
element=config_change.element,
|
|
old_value=config_change.oldValue,
|
|
new_value=config_change.newValue,
|
|
change_type=PushAdvisorChangeTypeToString(
|
|
config_change.changeType))
|
|
|
|
for advice in config_change.advices:
|
|
result += '\t* {0}'.format(advice.description)
|
|
|
|
return result
|
|
|
|
|
|
def GetActiveRolloutForService(service):
|
|
"""Return the latest Rollout for a service.
|
|
|
|
This function returns the most recent Rollout that has a status of SUCCESS
|
|
or IN_PROGRESS.
|
|
|
|
Args:
|
|
service: The name of the service for which to retrieve the active Rollout.
|
|
|
|
Returns:
|
|
The Rollout message corresponding to the active Rollout for the service.
|
|
"""
|
|
client = GetClientInstance()
|
|
messages = GetMessagesModule()
|
|
statuses = messages.Rollout.StatusValueValuesEnum
|
|
allowed_statuses = [statuses.SUCCESS, statuses.IN_PROGRESS]
|
|
|
|
req = messages.ServicemanagementServicesRolloutsListRequest(
|
|
serviceName=service)
|
|
|
|
result = list(
|
|
list_pager.YieldFromList(
|
|
client.services_rollouts,
|
|
req,
|
|
predicate=lambda r: r.status in allowed_statuses,
|
|
limit=1,
|
|
batch_size_attribute='pageSize',
|
|
field='rollouts',
|
|
)
|
|
)
|
|
|
|
return result[0] if result else None
|
|
|
|
|
|
def GetActiveServiceConfigIdsFromRollout(rollout):
|
|
"""Get the active service config IDs from a Rollout message.
|
|
|
|
Args:
|
|
rollout: The rollout message to inspect.
|
|
|
|
Returns:
|
|
A list of active service config IDs as indicated in the rollout message.
|
|
"""
|
|
if rollout and rollout.trafficPercentStrategy:
|
|
return [p.key for p in rollout.trafficPercentStrategy.percentages
|
|
.additionalProperties]
|
|
else:
|
|
return []
|
|
|
|
|
|
def GetActiveServiceConfigIdsForService(service):
|
|
active_rollout = GetActiveRolloutForService(service)
|
|
return GetActiveServiceConfigIdsFromRollout(active_rollout)
|
|
|
|
|
|
def FilenameMatchesExtension(filename, extensions):
|
|
"""Checks to see if a file name matches one of the given extensions.
|
|
|
|
Args:
|
|
filename: The full path to the file to check
|
|
extensions: A list of candidate extensions.
|
|
|
|
Returns:
|
|
True if the filename matches one of the extensions, otherwise False.
|
|
"""
|
|
f = filename.lower()
|
|
for ext in extensions:
|
|
if f.endswith(ext.lower()):
|
|
return True
|
|
return False
|
|
|
|
|
|
def IsProtoDescriptor(filename):
|
|
return FilenameMatchesExtension(
|
|
filename, ['.pb', '.descriptor', '.proto.bin'])
|
|
|
|
|
|
def IsRawProto(filename):
|
|
return FilenameMatchesExtension(filename, ['.proto'])
|
|
|
|
|
|
def ReadServiceConfigFile(file_path):
|
|
try:
|
|
if IsProtoDescriptor(file_path):
|
|
return files.ReadBinaryFileContents(file_path)
|
|
return files.ReadFileContents(file_path)
|
|
except files.Error as ex:
|
|
raise calliope_exceptions.BadFileException(
|
|
'Could not open service config file [{0}]: {1}'.format(file_path, ex))
|
|
|
|
|
|
def PushNormalizedGoogleServiceConfig(service_name, project, config_dict,
|
|
config_id=None):
|
|
"""Pushes a given normalized Google service configuration.
|
|
|
|
Args:
|
|
service_name: name of the service
|
|
project: the producer project Id
|
|
config_dict: the parsed contents of the Google Service Config file.
|
|
config_id: The id name for the config
|
|
|
|
Returns:
|
|
Result of the ServicesConfigsCreate request (a Service object)
|
|
"""
|
|
messages = GetMessagesModule()
|
|
client = GetClientInstance()
|
|
|
|
# Be aware: DictToMessage takes the value first and message second;
|
|
# JsonToMessage takes the message first and value second
|
|
service_config = encoding.DictToMessage(config_dict, messages.Service)
|
|
service_config.producerProjectId = project
|
|
service_config.id = config_id
|
|
create_request = (
|
|
messages.ServicemanagementServicesConfigsCreateRequest(
|
|
serviceName=service_name,
|
|
service=service_config,
|
|
))
|
|
return client.services_configs.Create(create_request)
|
|
|
|
|
|
def GetServiceConfigIdFromSubmitConfigSourceResponse(response):
|
|
return response.get('serviceConfig', {}).get('id')
|
|
|
|
|
|
def PushMultipleServiceConfigFiles(service_name, config_files, is_async,
|
|
validate_only=False, config_id=None):
|
|
"""Pushes a given set of service configuration files.
|
|
|
|
Args:
|
|
service_name: name of the service.
|
|
config_files: a list of ConfigFile message objects.
|
|
is_async: whether to wait for aync operations or not.
|
|
validate_only: whether to perform a validate-only run of the operation
|
|
or not.
|
|
config_id: an optional name for the config
|
|
|
|
Returns:
|
|
Full response from the SubmitConfigSource request.
|
|
|
|
Raises:
|
|
ServiceDeployErrorException: the SubmitConfigSource API call returned a
|
|
diagnostic with a level of ERROR.
|
|
"""
|
|
messages = GetMessagesModule()
|
|
client = GetClientInstance()
|
|
|
|
config_source = messages.ConfigSource(id=config_id)
|
|
config_source.files.extend(config_files)
|
|
|
|
config_source_request = messages.SubmitConfigSourceRequest(
|
|
configSource=config_source,
|
|
validateOnly=validate_only,
|
|
)
|
|
submit_request = (
|
|
messages.ServicemanagementServicesConfigsSubmitRequest(
|
|
serviceName=service_name,
|
|
submitConfigSourceRequest=config_source_request,
|
|
))
|
|
api_response = client.services_configs.Submit(submit_request)
|
|
operation = ProcessOperationResult(api_response, is_async)
|
|
|
|
response = operation.get('response', {})
|
|
diagnostics = response.get('diagnostics', [])
|
|
|
|
num_errors = 0
|
|
for diagnostic in diagnostics:
|
|
kind = diagnostic.get('kind', '').upper()
|
|
logger = log.error if kind == 'ERROR' else log.warning
|
|
msg = '{l}: {m}\n'.format(
|
|
l=diagnostic.get('location'), m=diagnostic.get('message'))
|
|
logger(msg)
|
|
|
|
if kind == 'ERROR':
|
|
num_errors += 1
|
|
|
|
if num_errors > 0:
|
|
exception_msg = ('{0} diagnostic error{1} found in service configuration '
|
|
'deployment. See log for details.').format(
|
|
num_errors, 's' if num_errors > 1 else '')
|
|
raise exceptions.ServiceDeployErrorException(exception_msg)
|
|
|
|
return response
|
|
|
|
|
|
def PushOpenApiServiceConfig(
|
|
service_name, spec_file_contents, spec_file_path, is_async,
|
|
validate_only=False):
|
|
"""Pushes a given Open API service configuration.
|
|
|
|
Args:
|
|
service_name: name of the service
|
|
spec_file_contents: the contents of the Open API spec file.
|
|
spec_file_path: the path of the Open API spec file.
|
|
is_async: whether to wait for aync operations or not.
|
|
validate_only: whether to perform a validate-only run of the operation
|
|
or not.
|
|
|
|
Returns:
|
|
Full response from the SubmitConfigSource request.
|
|
"""
|
|
messages = GetMessagesModule()
|
|
|
|
config_file = messages.ConfigFile(
|
|
fileContents=spec_file_contents,
|
|
filePath=spec_file_path,
|
|
# Always use YAML because JSON is a subset of YAML.
|
|
fileType=(messages.ConfigFile.
|
|
FileTypeValueValuesEnum.OPEN_API_YAML),
|
|
)
|
|
return PushMultipleServiceConfigFiles(service_name, [config_file], is_async,
|
|
validate_only=validate_only)
|
|
|
|
|
|
def DoesServiceExist(service_name):
|
|
"""Check if a service resource exists.
|
|
|
|
Args:
|
|
service_name: name of the service to check if exists.
|
|
|
|
Returns:
|
|
Whether or not the service exists.
|
|
"""
|
|
messages = GetMessagesModule()
|
|
client = GetClientInstance()
|
|
get_request = messages.ServicemanagementServicesGetRequest(
|
|
serviceName=service_name,
|
|
)
|
|
try:
|
|
client.services.Get(get_request)
|
|
except (apitools_exceptions.HttpForbiddenError,
|
|
apitools_exceptions.HttpNotFoundError):
|
|
# Older versions of service management backend return a 404 when service is
|
|
# new, but more recent versions return a 403. Check for either one for now.
|
|
return False
|
|
else:
|
|
return True
|
|
|
|
|
|
def CreateService(service_name, project, is_async=False):
|
|
"""Creates a Service resource.
|
|
|
|
Args:
|
|
service_name: name of the service to be created.
|
|
project: the project Id
|
|
is_async: If False, the method will block until the operation completes.
|
|
"""
|
|
messages = GetMessagesModule()
|
|
client = GetClientInstance()
|
|
# create service
|
|
create_request = messages.ManagedService(
|
|
serviceName=service_name,
|
|
producerProjectId=project,
|
|
)
|
|
result = client.services.Create(create_request)
|
|
|
|
GetProcessedOperationResult(result, is_async=is_async)
|
|
|
|
|
|
def ValidateFingerprint(fingerprint):
|
|
return re.match(FINGERPRINT_REGEX, fingerprint) is not None
|
|
|
|
|
|
def ValidateEmailString(email):
|
|
"""Returns true if the input is a valid email string.
|
|
|
|
This method uses a somewhat rudimentary regular expression to determine
|
|
input validity, but it should suffice for basic sanity checking.
|
|
|
|
It also verifies that the email string is no longer than 254 characters,
|
|
since that is the specified maximum length.
|
|
|
|
Args:
|
|
email: The email string to validate
|
|
|
|
Returns:
|
|
A bool -- True if the input is valid, False otherwise
|
|
"""
|
|
return EMAIL_REGEX.match(email or '') is not None and len(email) <= 254
|
|
|
|
|
|
def ProcessOperationResult(result, is_async=False):
|
|
"""Validate and process Operation outcome for user display.
|
|
|
|
Args:
|
|
result: The message to process (expected to be of type Operation)'
|
|
is_async: If False, the method will block until the operation completes.
|
|
|
|
Returns:
|
|
The processed Operation message in Python dict form
|
|
"""
|
|
op = GetProcessedOperationResult(result, is_async)
|
|
if is_async:
|
|
cmd = OP_WAIT_CMD.format(op.get('name'))
|
|
log.status.Print('Asynchronous operation is in progress... '
|
|
'Use the following command to wait for its '
|
|
'completion:\n {0}\n'.format(cmd))
|
|
else:
|
|
cmd = OP_DESCRIBE_CMD.format(op.get('name'))
|
|
log.status.Print('Operation finished successfully. '
|
|
'The following command can describe '
|
|
'the Operation details:\n {0}\n'.format(cmd))
|
|
return op
|
|
|
|
|
|
def GetProcessedOperationResult(result, is_async=False):
|
|
"""Validate and process Operation result message for user display.
|
|
|
|
This method checks to make sure the result is of type Operation and
|
|
converts the StartTime field from a UTC timestamp to a local datetime
|
|
string.
|
|
|
|
Args:
|
|
result: The message to process (expected to be of type Operation)'
|
|
is_async: If False, the method will block until the operation completes.
|
|
|
|
Returns:
|
|
The processed message in Python dict form
|
|
"""
|
|
if not result:
|
|
return
|
|
|
|
messages = GetMessagesModule()
|
|
|
|
RaiseIfResultNotTypeOf(result, messages.Operation)
|
|
|
|
result_dict = encoding.MessageToDict(result)
|
|
|
|
if not is_async:
|
|
op_name = result_dict['name']
|
|
op_ref = resources.REGISTRY.Parse(
|
|
op_name,
|
|
collection='servicemanagement.operations')
|
|
log.status.Print(
|
|
'Waiting for async operation {0} to complete...'.format(op_name))
|
|
result_dict = encoding.MessageToDict(WaitForOperation(
|
|
op_ref, GetClientInstance()))
|
|
|
|
return result_dict
|
|
|
|
|
|
def RaiseIfResultNotTypeOf(test_object, expected_type, nonetype_ok=False):
|
|
if nonetype_ok and test_object is None:
|
|
return
|
|
if not isinstance(test_object, expected_type):
|
|
raise TypeError('result must be of type %s' % expected_type)
|
|
|
|
|
|
def WaitForOperation(operation_ref, client):
|
|
"""Waits for an operation to complete.
|
|
|
|
Args:
|
|
operation_ref: A reference to the operation on which to wait.
|
|
client: The client object that contains the GetOperation request object.
|
|
|
|
Raises:
|
|
TimeoutError: if the operation does not complete in time.
|
|
OperationErrorException: if the operation fails.
|
|
|
|
Returns:
|
|
The Operation object, if successful. Raises an exception on failure.
|
|
"""
|
|
WaitForOperation.operation_response = None
|
|
messages = GetMessagesModule()
|
|
operation_id = operation_ref.operationsId
|
|
|
|
def _CheckOperation(operation_id): # pylint: disable=missing-docstring
|
|
request = messages.ServicemanagementOperationsGetRequest(
|
|
operationsId=operation_id,
|
|
)
|
|
|
|
result = client.operations.Get(request)
|
|
|
|
if result.done:
|
|
WaitForOperation.operation_response = result
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
# Wait for no more than 30 minutes while retrying the Operation retrieval
|
|
try:
|
|
retry.Retryer(exponential_sleep_multiplier=1.1, wait_ceiling_ms=10000,
|
|
max_wait_ms=30*60*1000).RetryOnResult(
|
|
_CheckOperation, [operation_id], should_retry_if=False,
|
|
sleep_ms=1500)
|
|
except retry.MaxRetrialsException:
|
|
raise exceptions.TimeoutError('Timed out while waiting for '
|
|
'operation {0}. Note that the operation '
|
|
'is still pending.'.format(operation_id))
|
|
|
|
# Check to see if the operation resulted in an error
|
|
if WaitForOperation.operation_response.error is not None:
|
|
raise exceptions.OperationErrorException(
|
|
'The operation with ID {0} resulted in a failure.'.format(operation_id))
|
|
|
|
# If we've gotten this far, the operation completed successfully,
|
|
# so return the Operation object
|
|
return WaitForOperation.operation_response
|
|
|
|
|
|
def LoadJsonOrYaml(input_string):
|
|
"""Tries to load input string as JSON first, then YAML if that fails.
|
|
|
|
Args:
|
|
input_string: The string to convert to a dictionary
|
|
|
|
Returns:
|
|
A dictionary of the resulting decoding, or None if neither format could be
|
|
detected.
|
|
"""
|
|
def TryJson():
|
|
try:
|
|
return json.loads(input_string)
|
|
except ValueError:
|
|
log.info('No JSON detected in service config. Trying YAML...')
|
|
|
|
def TryYaml():
|
|
try:
|
|
return yaml.load(input_string)
|
|
except yaml.YAMLParseError as e:
|
|
if hasattr(e.inner_error, 'problem_mark'):
|
|
mark = e.inner_error.problem_mark
|
|
log.error('Service config YAML had an error at position (%s:%s)'
|
|
% (mark.line+1, mark.column+1))
|
|
|
|
# First, try to decode JSON. If that fails, try to decode YAML.
|
|
return TryJson() or TryYaml()
|
|
|
|
|
|
def CreateRollout(service_config_id, service_name, is_async=False):
|
|
"""Creates a Rollout for a Service Config within it's service.
|
|
|
|
Args:
|
|
service_config_id: The service config id
|
|
service_name: The name of the service
|
|
is_async: (Optional) Wheter or not operation should be asynchronous
|
|
|
|
Returns:
|
|
The rollout object or long running operation if is_async is true
|
|
"""
|
|
messages = GetMessagesModule()
|
|
client = GetClientInstance()
|
|
|
|
percentages = messages.TrafficPercentStrategy.PercentagesValue()
|
|
percentages.additionalProperties.append(
|
|
(messages.TrafficPercentStrategy.PercentagesValue.AdditionalProperty(
|
|
key=service_config_id, value=100.0)))
|
|
traffic_percent_strategy = messages.TrafficPercentStrategy(
|
|
percentages=percentages)
|
|
rollout = messages.Rollout(
|
|
serviceName=service_name,
|
|
trafficPercentStrategy=traffic_percent_strategy,)
|
|
rollout_create = messages.ServicemanagementServicesRolloutsCreateRequest(
|
|
rollout=rollout,
|
|
serviceName=service_name,
|
|
)
|
|
rollout_operation = client.services_rollouts.Create(rollout_create)
|
|
op = ProcessOperationResult(rollout_operation, is_async)
|
|
|
|
return op.get('response', None)
|