# -*- coding: utf-8 -*- # # Copyright 2017 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Common helper methods for Service Management commands.""" from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals import json import re from apitools.base.py import encoding from apitools.base.py import exceptions as apitools_exceptions from apitools.base.py import list_pager from googlecloudsdk.api_lib.endpoints import exceptions from googlecloudsdk.api_lib.util import apis from googlecloudsdk.calliope import exceptions as calliope_exceptions from googlecloudsdk.core import log from googlecloudsdk.core import properties from googlecloudsdk.core import resources from googlecloudsdk.core import yaml from googlecloudsdk.core.resource import resource_printer from googlecloudsdk.core.util import files from googlecloudsdk.core.util import retry import six EMAIL_REGEX = re.compile(r'^.+@([^.@][^@]+)$') FINGERPRINT_REGEX = re.compile( r'^([a-f0-9][a-f0-9]:){19}[a-f0-9][a-f0-9]$', re.IGNORECASE) OP_BASE_CMD = 'gcloud endpoints operations ' OP_DESCRIBE_CMD = OP_BASE_CMD + 'describe {0}' OP_WAIT_CMD = OP_BASE_CMD + 'wait {0}' SERVICES_COLLECTION = 'servicemanagement.services' CONFIG_COLLECTION = 'servicemanagement.services.configs' ALL_IAM_PERMISSIONS = [ 'servicemanagement.services.get', 'servicemanagement.services.getProjectSettings', 'servicemanagement.services.delete', 'servicemanagement.services.update', 'servicemanagement.services.bind', 'servicemanagement.services.updateProjectSettings', 'servicemanagement.services.check', 'servicemanagement.services.report', 'servicemanagement.services.setIamPolicy', 'servicemanagement.services.getIamPolicy', ] def GetMessagesModule(): return apis.GetMessagesModule('servicemanagement', 'v1') def GetClientInstance(): return apis.GetClientInstance('servicemanagement', 'v1') def GetServiceManagementServiceName(): return 'servicemanagement.googleapis.com' def GetValidatedProject(project_id): """Validate the project ID, if supplied, otherwise return the default project. Args: project_id: The ID of the project to validate. If None, gcloud's default project's ID will be returned. Returns: The validated project ID. """ if project_id: properties.VALUES.core.project.Validate(project_id) else: project_id = properties.VALUES.core.project.Get(required=True) return project_id def GetProjectSettings(service, consumer_project_id, view): """Returns the project settings for a given service, project, and view. Args: service: The service for which to return project settings. consumer_project_id: The consumer project id for which to return settings. view: The view (CONSUMER_VIEW or PRODUCER_VIEW). Returns: A ProjectSettings message with the settings populated. """ # Shorten the request names for better readability get_request = (GetMessagesModule() .ServicemanagementServicesProjectSettingsGetRequest) # Get the current list of quota settings to see if the quota override # exists in the first place. request = get_request( serviceName=service, consumerProjectId=consumer_project_id, view=view, ) return GetClientInstance().services_projectSettings.Get(request) def GetProducedListRequest(project_id): return GetMessagesModule().ServicemanagementServicesListRequest( producerProjectId=project_id ) def PrettyPrint(resource, print_format='json'): """Prints the given resource. Args: resource: The resource to print out. print_format: The print_format value to pass along to the resource_printer. """ resource_printer.Print( resources=[resource], print_format=print_format, out=log.out) def PushAdvisorChangeTypeToString(change_type): """Convert a ConfigChange.ChangeType enum to a string. Args: change_type: The ConfigChange.ChangeType enum to convert. Returns: An easily readable string representing the ConfigChange.ChangeType enum. """ messages = GetMessagesModule() enums = messages.ConfigChange.ChangeTypeValueValuesEnum if change_type in [enums.ADDED, enums.REMOVED, enums.MODIFIED]: return six.text_type(change_type).lower() else: return '[unknown]' def PushAdvisorConfigChangeToString(config_change): """Convert a ConfigChange message to a printable string. Args: config_change: The ConfigChange message to convert. Returns: An easily readable string representing the ConfigChange message. """ result = ('Element [{element}] (old value = {old_value}, ' 'new value = {new_value}) was {change_type}. Advice:\n').format( element=config_change.element, old_value=config_change.oldValue, new_value=config_change.newValue, change_type=PushAdvisorChangeTypeToString( config_change.changeType)) for advice in config_change.advices: result += '\t* {0}'.format(advice.description) return result def GetActiveRolloutForService(service): """Return the latest Rollout for a service. This function returns the most recent Rollout that has a status of SUCCESS or IN_PROGRESS. Args: service: The name of the service for which to retrieve the active Rollout. Returns: The Rollout message corresponding to the active Rollout for the service. """ client = GetClientInstance() messages = GetMessagesModule() statuses = messages.Rollout.StatusValueValuesEnum allowed_statuses = [statuses.SUCCESS, statuses.IN_PROGRESS] req = messages.ServicemanagementServicesRolloutsListRequest( serviceName=service) result = list( list_pager.YieldFromList( client.services_rollouts, req, predicate=lambda r: r.status in allowed_statuses, limit=1, batch_size_attribute='pageSize', field='rollouts', ) ) return result[0] if result else None def GetActiveServiceConfigIdsFromRollout(rollout): """Get the active service config IDs from a Rollout message. Args: rollout: The rollout message to inspect. Returns: A list of active service config IDs as indicated in the rollout message. """ if rollout and rollout.trafficPercentStrategy: return [p.key for p in rollout.trafficPercentStrategy.percentages .additionalProperties] else: return [] def GetActiveServiceConfigIdsForService(service): active_rollout = GetActiveRolloutForService(service) return GetActiveServiceConfigIdsFromRollout(active_rollout) def FilenameMatchesExtension(filename, extensions): """Checks to see if a file name matches one of the given extensions. Args: filename: The full path to the file to check extensions: A list of candidate extensions. Returns: True if the filename matches one of the extensions, otherwise False. """ f = filename.lower() for ext in extensions: if f.endswith(ext.lower()): return True return False def IsProtoDescriptor(filename): return FilenameMatchesExtension( filename, ['.pb', '.descriptor', '.proto.bin']) def IsRawProto(filename): return FilenameMatchesExtension(filename, ['.proto']) def ReadServiceConfigFile(file_path): try: if IsProtoDescriptor(file_path): return files.ReadBinaryFileContents(file_path) return files.ReadFileContents(file_path) except files.Error as ex: raise calliope_exceptions.BadFileException( 'Could not open service config file [{0}]: {1}'.format(file_path, ex)) def PushNormalizedGoogleServiceConfig(service_name, project, config_dict, config_id=None): """Pushes a given normalized Google service configuration. Args: service_name: name of the service project: the producer project Id config_dict: the parsed contents of the Google Service Config file. config_id: The id name for the config Returns: Result of the ServicesConfigsCreate request (a Service object) """ messages = GetMessagesModule() client = GetClientInstance() # Be aware: DictToMessage takes the value first and message second; # JsonToMessage takes the message first and value second service_config = encoding.DictToMessage(config_dict, messages.Service) service_config.producerProjectId = project service_config.id = config_id create_request = ( messages.ServicemanagementServicesConfigsCreateRequest( serviceName=service_name, service=service_config, )) return client.services_configs.Create(create_request) def GetServiceConfigIdFromSubmitConfigSourceResponse(response): return response.get('serviceConfig', {}).get('id') def PushMultipleServiceConfigFiles(service_name, config_files, is_async, validate_only=False, config_id=None): """Pushes a given set of service configuration files. Args: service_name: name of the service. config_files: a list of ConfigFile message objects. is_async: whether to wait for aync operations or not. validate_only: whether to perform a validate-only run of the operation or not. config_id: an optional name for the config Returns: Full response from the SubmitConfigSource request. Raises: ServiceDeployErrorException: the SubmitConfigSource API call returned a diagnostic with a level of ERROR. """ messages = GetMessagesModule() client = GetClientInstance() config_source = messages.ConfigSource(id=config_id) config_source.files.extend(config_files) config_source_request = messages.SubmitConfigSourceRequest( configSource=config_source, validateOnly=validate_only, ) submit_request = ( messages.ServicemanagementServicesConfigsSubmitRequest( serviceName=service_name, submitConfigSourceRequest=config_source_request, )) api_response = client.services_configs.Submit(submit_request) operation = ProcessOperationResult(api_response, is_async) response = operation.get('response', {}) diagnostics = response.get('diagnostics', []) num_errors = 0 for diagnostic in diagnostics: kind = diagnostic.get('kind', '').upper() logger = log.error if kind == 'ERROR' else log.warning msg = '{l}: {m}\n'.format( l=diagnostic.get('location'), m=diagnostic.get('message')) logger(msg) if kind == 'ERROR': num_errors += 1 if num_errors > 0: exception_msg = ('{0} diagnostic error{1} found in service configuration ' 'deployment. See log for details.').format( num_errors, 's' if num_errors > 1 else '') raise exceptions.ServiceDeployErrorException(exception_msg) return response def PushOpenApiServiceConfig( service_name, spec_file_contents, spec_file_path, is_async, validate_only=False): """Pushes a given Open API service configuration. Args: service_name: name of the service spec_file_contents: the contents of the Open API spec file. spec_file_path: the path of the Open API spec file. is_async: whether to wait for aync operations or not. validate_only: whether to perform a validate-only run of the operation or not. Returns: Full response from the SubmitConfigSource request. """ messages = GetMessagesModule() config_file = messages.ConfigFile( fileContents=spec_file_contents, filePath=spec_file_path, # Always use YAML because JSON is a subset of YAML. fileType=(messages.ConfigFile. FileTypeValueValuesEnum.OPEN_API_YAML), ) return PushMultipleServiceConfigFiles(service_name, [config_file], is_async, validate_only=validate_only) def DoesServiceExist(service_name): """Check if a service resource exists. Args: service_name: name of the service to check if exists. Returns: Whether or not the service exists. """ messages = GetMessagesModule() client = GetClientInstance() get_request = messages.ServicemanagementServicesGetRequest( serviceName=service_name, ) try: client.services.Get(get_request) except (apitools_exceptions.HttpForbiddenError, apitools_exceptions.HttpNotFoundError): # Older versions of service management backend return a 404 when service is # new, but more recent versions return a 403. Check for either one for now. return False else: return True def CreateService(service_name, project, is_async=False): """Creates a Service resource. Args: service_name: name of the service to be created. project: the project Id is_async: If False, the method will block until the operation completes. """ messages = GetMessagesModule() client = GetClientInstance() # create service create_request = messages.ManagedService( serviceName=service_name, producerProjectId=project, ) result = client.services.Create(create_request) GetProcessedOperationResult(result, is_async=is_async) def ValidateFingerprint(fingerprint): return re.match(FINGERPRINT_REGEX, fingerprint) is not None def ValidateEmailString(email): """Returns true if the input is a valid email string. This method uses a somewhat rudimentary regular expression to determine input validity, but it should suffice for basic sanity checking. It also verifies that the email string is no longer than 254 characters, since that is the specified maximum length. Args: email: The email string to validate Returns: A bool -- True if the input is valid, False otherwise """ return EMAIL_REGEX.match(email or '') is not None and len(email) <= 254 def ProcessOperationResult(result, is_async=False): """Validate and process Operation outcome for user display. Args: result: The message to process (expected to be of type Operation)' is_async: If False, the method will block until the operation completes. Returns: The processed Operation message in Python dict form """ op = GetProcessedOperationResult(result, is_async) if is_async: cmd = OP_WAIT_CMD.format(op.get('name')) log.status.Print('Asynchronous operation is in progress... ' 'Use the following command to wait for its ' 'completion:\n {0}\n'.format(cmd)) else: cmd = OP_DESCRIBE_CMD.format(op.get('name')) log.status.Print('Operation finished successfully. ' 'The following command can describe ' 'the Operation details:\n {0}\n'.format(cmd)) return op def GetProcessedOperationResult(result, is_async=False): """Validate and process Operation result message for user display. This method checks to make sure the result is of type Operation and converts the StartTime field from a UTC timestamp to a local datetime string. Args: result: The message to process (expected to be of type Operation)' is_async: If False, the method will block until the operation completes. Returns: The processed message in Python dict form """ if not result: return messages = GetMessagesModule() RaiseIfResultNotTypeOf(result, messages.Operation) result_dict = encoding.MessageToDict(result) if not is_async: op_name = result_dict['name'] op_ref = resources.REGISTRY.Parse( op_name, collection='servicemanagement.operations') log.status.Print( 'Waiting for async operation {0} to complete...'.format(op_name)) result_dict = encoding.MessageToDict(WaitForOperation( op_ref, GetClientInstance())) return result_dict def RaiseIfResultNotTypeOf(test_object, expected_type, nonetype_ok=False): if nonetype_ok and test_object is None: return if not isinstance(test_object, expected_type): raise TypeError('result must be of type %s' % expected_type) def WaitForOperation(operation_ref, client): """Waits for an operation to complete. Args: operation_ref: A reference to the operation on which to wait. client: The client object that contains the GetOperation request object. Raises: TimeoutError: if the operation does not complete in time. OperationErrorException: if the operation fails. Returns: The Operation object, if successful. Raises an exception on failure. """ WaitForOperation.operation_response = None messages = GetMessagesModule() operation_id = operation_ref.operationsId def _CheckOperation(operation_id): # pylint: disable=missing-docstring request = messages.ServicemanagementOperationsGetRequest( operationsId=operation_id, ) result = client.operations.Get(request) if result.done: WaitForOperation.operation_response = result return True else: return False # Wait for no more than 30 minutes while retrying the Operation retrieval try: retry.Retryer(exponential_sleep_multiplier=1.1, wait_ceiling_ms=10000, max_wait_ms=30*60*1000).RetryOnResult( _CheckOperation, [operation_id], should_retry_if=False, sleep_ms=1500) except retry.MaxRetrialsException: raise exceptions.TimeoutError('Timed out while waiting for ' 'operation {0}. Note that the operation ' 'is still pending.'.format(operation_id)) # Check to see if the operation resulted in an error if WaitForOperation.operation_response.error is not None: raise exceptions.OperationErrorException( 'The operation with ID {0} resulted in a failure.'.format(operation_id)) # If we've gotten this far, the operation completed successfully, # so return the Operation object return WaitForOperation.operation_response def LoadJsonOrYaml(input_string): """Tries to load input string as JSON first, then YAML if that fails. Args: input_string: The string to convert to a dictionary Returns: A dictionary of the resulting decoding, or None if neither format could be detected. """ def TryJson(): try: return json.loads(input_string) except ValueError: log.info('No JSON detected in service config. Trying YAML...') def TryYaml(): try: return yaml.load(input_string) except yaml.YAMLParseError as e: if hasattr(e.inner_error, 'problem_mark'): mark = e.inner_error.problem_mark log.error('Service config YAML had an error at position (%s:%s)' % (mark.line+1, mark.column+1)) # First, try to decode JSON. If that fails, try to decode YAML. return TryJson() or TryYaml() def CreateRollout(service_config_id, service_name, is_async=False): """Creates a Rollout for a Service Config within it's service. Args: service_config_id: The service config id service_name: The name of the service is_async: (Optional) Wheter or not operation should be asynchronous Returns: The rollout object or long running operation if is_async is true """ messages = GetMessagesModule() client = GetClientInstance() percentages = messages.TrafficPercentStrategy.PercentagesValue() percentages.additionalProperties.append( (messages.TrafficPercentStrategy.PercentagesValue.AdditionalProperty( key=service_config_id, value=100.0))) traffic_percent_strategy = messages.TrafficPercentStrategy( percentages=percentages) rollout = messages.Rollout( serviceName=service_name, trafficPercentStrategy=traffic_percent_strategy,) rollout_create = messages.ServicemanagementServicesRolloutsCreateRequest( rollout=rollout, serviceName=service_name, ) rollout_operation = client.services_rollouts.Create(rollout_create) op = ProcessOperationResult(rollout_operation, is_async) return op.get('response', None)