320 lines
10 KiB
Python
320 lines
10 KiB
Python
# -*- coding: utf-8 -*- #
|
|
# Copyright 2016 Google LLC. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Utilities for working with long running operations go/long-running-operation.
|
|
"""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
import json
|
|
|
|
from apitools.base.py import encoding
|
|
from apitools.base.py import exceptions as apitools_exceptions
|
|
|
|
import enum
|
|
|
|
from googlecloudsdk.api_lib.app import exceptions as app_exceptions
|
|
from googlecloudsdk.api_lib.util import exceptions as api_exceptions
|
|
from googlecloudsdk.api_lib.util import requests
|
|
from googlecloudsdk.api_lib.util import waiter
|
|
from googlecloudsdk.core import exceptions
|
|
from googlecloudsdk.core import log
|
|
from googlecloudsdk.core import resources
|
|
import six
|
|
|
|
# Default is to retry every 5 seconds for 1 hour.
|
|
DEFAULT_OPERATION_RETRY_INTERVAL = 5
|
|
DEFAULT_OPERATION_MAX_TRIES = (60 // DEFAULT_OPERATION_RETRY_INTERVAL) * 60
|
|
|
|
|
|
def CallAndCollectOpErrors(method, *args, **kwargs):
|
|
"""Wrapper for method(...) which re-raises operation-style errors.
|
|
|
|
Args:
|
|
method: Original method to call.
|
|
*args: Positional arguments to method.
|
|
**kwargs: Keyword arguments to method.
|
|
|
|
Raises:
|
|
MiscOperationError: If the method call itself raises one of the exceptions
|
|
listed below. Otherwise, the original exception is raised. Preserves
|
|
stack trace. Re-uses the error string from original error or in the case
|
|
of HttpError, we synthesize human-friendly string from HttpException.
|
|
However, HttpException is neither raised nor part of the stack trace.
|
|
|
|
Returns:
|
|
Result of calling method(*args, **kwargs).
|
|
"""
|
|
try:
|
|
return method(*args, **kwargs)
|
|
except apitools_exceptions.HttpError as http_err:
|
|
# Create HttpException locally only to get its human friendly string
|
|
_ReraiseMiscOperationError(api_exceptions.HttpException(http_err))
|
|
except (OperationError, OperationTimeoutError, app_exceptions.Error) as err:
|
|
_ReraiseMiscOperationError(err)
|
|
|
|
|
|
def _ReraiseMiscOperationError(err):
|
|
"""Transform and re-raise error helper."""
|
|
exceptions.reraise(MiscOperationError(six.text_type(err)))
|
|
|
|
|
|
class MiscOperationError(exceptions.Error):
|
|
"""Wrapper exception for errors treated as operation failures."""
|
|
|
|
|
|
class OperationError(exceptions.Error):
|
|
pass
|
|
|
|
|
|
class OperationTimeoutError(exceptions.Error):
|
|
pass
|
|
|
|
|
|
class Status(enum.Enum):
|
|
PENDING = 1
|
|
COMPLETED = 2
|
|
ERROR = 3
|
|
|
|
|
|
class Operation(object):
|
|
"""Wrapper around Operation response objects for console output.
|
|
|
|
Attributes:
|
|
project: String, name of the project.
|
|
id: String, ID of operation.
|
|
start_time: String, time the operation started.
|
|
status: Status enum, either PENDING, COMPLETED, or Error.
|
|
op_resource: messages.Operation, the original Operation resource.
|
|
"""
|
|
|
|
def __init__(self, op_response):
|
|
"""Creates the operation wrapper object."""
|
|
res = resources.REGISTRY.ParseRelativeName(op_response.name,
|
|
'appengine.apps.operations')
|
|
self.project = res.appsId
|
|
self.id = res.Name()
|
|
self.start_time = _GetInsertTime(op_response)
|
|
self.status = GetStatus(op_response)
|
|
self.op_resource = op_response
|
|
|
|
def __eq__(self, other):
|
|
return (isinstance(other, Operation) and
|
|
self.project == other.project and
|
|
self.id == other.id and
|
|
self.start_time == other.start_time and
|
|
self.status == other.status and
|
|
self.op_resource == other.op_resource)
|
|
|
|
|
|
def GetStatus(operation):
|
|
"""Returns string status for given operation.
|
|
|
|
Args:
|
|
operation: A messages.Operation instance.
|
|
|
|
Returns:
|
|
The status of the operation in string form.
|
|
"""
|
|
if not operation.done:
|
|
return Status.PENDING.name
|
|
elif operation.error:
|
|
return Status.ERROR.name
|
|
else:
|
|
return Status.COMPLETED.name
|
|
|
|
|
|
def _GetInsertTime(operation):
|
|
"""Finds the insertTime property and return its string form.
|
|
|
|
Args:
|
|
operation: A messages.Operation instance.
|
|
|
|
Returns:
|
|
The time the operation started in string form or None if N/A.
|
|
"""
|
|
if not operation.metadata:
|
|
return None
|
|
properties = operation.metadata.additionalProperties
|
|
for prop in properties:
|
|
if prop.key == 'insertTime':
|
|
return prop.value.string_value
|
|
|
|
|
|
class AppEngineOperationPoller(waiter.OperationPoller):
|
|
"""A poller for appengine operations."""
|
|
|
|
def __init__(self, operation_service, operation_metadata_type=None):
|
|
"""Sets up poller for appengine operations.
|
|
|
|
Args:
|
|
operation_service: apitools.base.py.base_api.BaseApiService, api service
|
|
for retrieving information about ongoing operation.
|
|
operation_metadata_type: Message class for the Operation metadata (for
|
|
instance, OperationMetadataV1, or OperationMetadataV1Beta).
|
|
"""
|
|
self.operation_service = operation_service
|
|
self.operation_metadata_type = operation_metadata_type
|
|
self.warnings_seen = set()
|
|
|
|
def IsDone(self, operation):
|
|
"""Overrides."""
|
|
self._LogNewWarnings(operation)
|
|
if operation.done:
|
|
log.debug('Operation [{0}] complete. Result: {1}'.format(
|
|
operation.name,
|
|
json.dumps(encoding.MessageToDict(operation), indent=4)))
|
|
if operation.error:
|
|
raise OperationError(requests.ExtractErrorMessage(
|
|
encoding.MessageToPyValue(operation.error)))
|
|
return True
|
|
log.debug('Operation [{0}] not complete. Waiting to retry.'.format(
|
|
operation.name))
|
|
return False
|
|
|
|
def Poll(self, operation_ref):
|
|
"""Overrides.
|
|
|
|
Args:
|
|
operation_ref: googlecloudsdk.core.resources.Resource.
|
|
|
|
Returns:
|
|
fetched operation message.
|
|
"""
|
|
request_type = self.operation_service.GetRequestType('Get')
|
|
request = request_type(name=operation_ref.RelativeName())
|
|
operation = self.operation_service.Get(request)
|
|
self._LogNewWarnings(operation)
|
|
return operation
|
|
|
|
def _LogNewWarnings(self, operation):
|
|
if self.operation_metadata_type:
|
|
# Log any new warnings to the end user.
|
|
new_warnings = GetWarningsFromOperation(
|
|
operation, self.operation_metadata_type) - self.warnings_seen
|
|
for warning in new_warnings:
|
|
log.warning(warning + '\n')
|
|
self.warnings_seen.add(warning)
|
|
|
|
def GetResult(self, operation):
|
|
"""Simply returns the operation.
|
|
|
|
Args:
|
|
operation: api_name_messages.Operation.
|
|
|
|
Returns:
|
|
the 'response' field of the Operation.
|
|
"""
|
|
return operation
|
|
|
|
|
|
class AppEngineOperationBuildPoller(AppEngineOperationPoller):
|
|
"""Waits for a build to be present, or for the operation to finish."""
|
|
|
|
def __init__(self, operation_service, operation_metadata_type):
|
|
"""Sets up poller for appengine operations.
|
|
|
|
Args:
|
|
operation_service: apitools.base.py.base_api.BaseApiService, api service
|
|
for retrieving information about ongoing operation.
|
|
operation_metadata_type: Message class for the Operation metadata (for
|
|
instance, OperationMetadataV1, or OperationMetadataV1Beta).
|
|
"""
|
|
super(AppEngineOperationBuildPoller, self).__init__(operation_service,
|
|
operation_metadata_type)
|
|
|
|
def IsDone(self, operation):
|
|
if GetBuildFromOperation(operation, self.operation_metadata_type):
|
|
return True
|
|
return super(AppEngineOperationBuildPoller, self).IsDone(operation)
|
|
|
|
|
|
def GetMetadataFromOperation(operation, operation_metadata_type):
|
|
if not operation.metadata:
|
|
return None
|
|
return encoding.JsonToMessage(
|
|
operation_metadata_type,
|
|
encoding.MessageToJson(operation.metadata))
|
|
|
|
|
|
def GetBuildFromOperation(operation, operation_metadata_type):
|
|
metadata = GetMetadataFromOperation(operation, operation_metadata_type)
|
|
if not metadata or not metadata.createVersionMetadata:
|
|
return None
|
|
return metadata.createVersionMetadata.cloudBuildId
|
|
|
|
|
|
def GetWarningsFromOperation(operation, operation_metadata_type):
|
|
metadata = GetMetadataFromOperation(operation, operation_metadata_type)
|
|
if not metadata:
|
|
return set()
|
|
return set(warning for warning in metadata.warning)
|
|
|
|
|
|
def WaitForOperation(operation_service, operation,
|
|
max_retries=None,
|
|
retry_interval=None,
|
|
operation_collection='appengine.apps.operations',
|
|
message=None,
|
|
poller=None):
|
|
"""Wait until the operation is complete or times out.
|
|
|
|
Args:
|
|
operation_service: The apitools service type for operations
|
|
operation: The operation resource to wait on
|
|
max_retries: Maximum number of times to poll the operation
|
|
retry_interval: Frequency of polling in seconds
|
|
operation_collection: The resource collection of the operation.
|
|
message: str, the message to display while progress tracker displays.
|
|
poller: AppEngineOperationPoller to poll with, defaulting to done.
|
|
Returns:
|
|
The operation resource when it has completed
|
|
Raises:
|
|
OperationError: if the operation contains an error.
|
|
OperationTimeoutError: when the operation polling times out
|
|
|
|
"""
|
|
poller = poller or AppEngineOperationPoller(operation_service)
|
|
if poller.IsDone(operation):
|
|
return poller.GetResult(operation)
|
|
operation_ref = resources.REGISTRY.ParseRelativeName(
|
|
operation.name,
|
|
operation_collection)
|
|
if max_retries is None:
|
|
max_retries = DEFAULT_OPERATION_MAX_TRIES - 1
|
|
if retry_interval is None:
|
|
retry_interval = DEFAULT_OPERATION_RETRY_INTERVAL
|
|
if message is None:
|
|
message = 'Waiting for operation [{}] to complete'.format(
|
|
operation_ref.RelativeName())
|
|
# Convert to milliseconds
|
|
retry_interval *= 1000
|
|
try:
|
|
completed_operation = waiter.WaitFor(
|
|
poller,
|
|
operation_ref,
|
|
message,
|
|
pre_start_sleep_ms=1000,
|
|
max_retrials=max_retries,
|
|
exponential_sleep_multiplier=1.0,
|
|
sleep_ms=retry_interval)
|
|
except waiter.TimeoutError:
|
|
raise OperationTimeoutError(('Operation [{0}] timed out. This operation '
|
|
'may still be underway.').format(
|
|
operation.name))
|
|
return completed_operation
|