333 lines
10 KiB
Python
333 lines
10 KiB
Python
# -*- coding: utf-8 -*- #
|
|
# Copyright 2015 Google LLC. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Utilities to support long running operations."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
import abc
|
|
import time
|
|
|
|
from apitools.base.py import encoding
|
|
from googlecloudsdk.core import exceptions
|
|
from googlecloudsdk.core.console import progress_tracker
|
|
from googlecloudsdk.core.util import retry
|
|
import six
|
|
|
|
|
|
_TIMEOUT_MESSAGE = (
|
|
'The operations may still be underway remotely and may still succeed; '
|
|
'use gcloud list and describe commands or '
|
|
'https://console.developers.google.com/ to check resource state.')
|
|
|
|
|
|
class TimeoutError(exceptions.Error):
|
|
pass
|
|
|
|
|
|
class AbortWaitError(exceptions.Error):
|
|
pass
|
|
|
|
|
|
class OperationError(exceptions.Error):
|
|
pass
|
|
|
|
|
|
class OperationPoller(six.with_metaclass(abc.ABCMeta, object)):
|
|
"""Interface for defining operation which can be polled and waited on.
|
|
|
|
This construct manages operation_ref, operation and result abstract objects.
|
|
Operation_ref is an identifier for operation which is a proxy for result
|
|
object. OperationPoller has three responsibilities:
|
|
1. Given operation object determine if it is done.
|
|
2. Given operation_ref fetch operation object
|
|
3. Given operation object fetch result object
|
|
"""
|
|
|
|
@abc.abstractmethod
|
|
def IsDone(self, operation):
|
|
"""Given result of Poll determines if result is done.
|
|
|
|
Args:
|
|
operation: object representing operation returned by Poll method.
|
|
|
|
Returns:
|
|
|
|
"""
|
|
return True
|
|
|
|
@abc.abstractmethod
|
|
def Poll(self, operation_ref):
|
|
"""Retrieves operation given its reference.
|
|
|
|
Args:
|
|
operation_ref: str, some id for operation.
|
|
|
|
Returns:
|
|
object which represents operation.
|
|
"""
|
|
return None
|
|
|
|
@abc.abstractmethod
|
|
def GetResult(self, operation):
|
|
"""Given operation message retrieves result it represents.
|
|
|
|
Args:
|
|
operation: object, representing operation returned by Poll method.
|
|
Returns:
|
|
some object created by given operation.
|
|
"""
|
|
return None
|
|
|
|
|
|
class CloudOperationPoller(OperationPoller):
|
|
"""Manages a longrunning Operations.
|
|
|
|
See https://cloud.google.com/speech/reference/rpc/google.longrunning
|
|
"""
|
|
|
|
def __init__(self, result_service, operation_service):
|
|
"""Sets up poller for cloud operations.
|
|
|
|
Args:
|
|
result_service: apitools.base.py.base_api.BaseApiService, api service for
|
|
retrieving created result of initiated operation.
|
|
operation_service: apitools.base.py.base_api.BaseApiService, api service
|
|
for retrieving information about ongoing operation.
|
|
|
|
Note that result_service and operation_service Get request must have
|
|
single attribute called 'name'.
|
|
"""
|
|
self.result_service = result_service
|
|
self.operation_service = operation_service
|
|
|
|
def IsDone(self, operation):
|
|
"""Overrides."""
|
|
if operation.done:
|
|
if operation.error:
|
|
raise OperationError(operation.error.message)
|
|
return True
|
|
return False
|
|
|
|
def Poll(self, operation_ref):
|
|
"""Overrides.
|
|
|
|
Args:
|
|
operation_ref: googlecloudsdk.core.resources.Resource.
|
|
|
|
Returns:
|
|
fetched operation message.
|
|
"""
|
|
request_type = self.operation_service.GetRequestType('Get')
|
|
return self.operation_service.Get(
|
|
request_type(name=operation_ref.RelativeName()))
|
|
|
|
def GetResult(self, operation):
|
|
"""Overrides.
|
|
|
|
Args:
|
|
operation: api_name_messages.Operation.
|
|
|
|
Returns:
|
|
result of result_service.Get request.
|
|
"""
|
|
request_type = self.result_service.GetRequestType('Get')
|
|
response_dict = encoding.MessageToPyValue(operation.response)
|
|
return self.result_service.Get(request_type(name=response_dict['name']))
|
|
|
|
|
|
class CloudOperationPollerNoResources(OperationPoller):
|
|
"""Manages longrunning Operations for Cloud API that creates no resources.
|
|
|
|
See https://cloud.google.com/speech/reference/rpc/google.longrunning
|
|
"""
|
|
|
|
# TODO(b/62478975): Remove get_name_func when ML API operation names
|
|
# are compatible with gcloud parsing, and use RelativeName instead.
|
|
def __init__(self, operation_service, get_name_func=None):
|
|
"""Sets up poller for cloud operations.
|
|
|
|
Args:
|
|
operation_service: apitools.base.py.base_api.BaseApiService, api service
|
|
for retrieving information about ongoing operation.
|
|
|
|
Note that the operation_service Get request must have a
|
|
single attribute called 'name'.
|
|
get_name_func: the function to use to get the name from the operation_ref.
|
|
This is to allow polling with non-traditional operation resource names.
|
|
If the resource name is compatible with gcloud parsing, use
|
|
`lambda x: x.RelativeName()`.
|
|
"""
|
|
self.operation_service = operation_service
|
|
self.get_name = get_name_func or (lambda x: x.RelativeName())
|
|
|
|
def IsDone(self, operation):
|
|
"""Overrides."""
|
|
if operation.done:
|
|
if operation.error:
|
|
raise OperationError(operation.error.message)
|
|
return True
|
|
return False
|
|
|
|
def Poll(self, operation_ref):
|
|
"""Overrides.
|
|
|
|
Args:
|
|
operation_ref: googlecloudsdk.core.resources.Resource.
|
|
|
|
Returns:
|
|
fetched operation message.
|
|
"""
|
|
request_type = self.operation_service.GetRequestType('Get')
|
|
return self.operation_service.Get(
|
|
request_type(name=self.get_name(operation_ref)))
|
|
|
|
def GetResult(self, operation):
|
|
"""Overrides to get the response from the completed operation.
|
|
|
|
Args:
|
|
operation: api_name_messages.Operation.
|
|
|
|
Returns:
|
|
the 'response' field of the Operation.
|
|
"""
|
|
return operation.response
|
|
|
|
|
|
def WaitFor(poller,
|
|
operation_ref,
|
|
message=None,
|
|
custom_tracker=None,
|
|
tracker_update_func=None,
|
|
pre_start_sleep_ms=1000,
|
|
max_retrials=None,
|
|
max_wait_ms=1800000,
|
|
exponential_sleep_multiplier=1.4,
|
|
jitter_ms=1000,
|
|
wait_ceiling_ms=180000,
|
|
sleep_ms=2000):
|
|
"""Waits for poller.Poll and displays pending operation spinner.
|
|
|
|
Args:
|
|
poller: OperationPoller, poller to use during retrials.
|
|
operation_ref: object, passed to operation poller poll method.
|
|
message: str, string to display for default progress_tracker.
|
|
custom_tracker: ProgressTracker, progress_tracker to use for display.
|
|
tracker_update_func: func(tracker, result, status), tracker update function.
|
|
pre_start_sleep_ms: int, Time to wait before making first poll request.
|
|
max_retrials: int, max number of retrials before raising RetryException.
|
|
max_wait_ms: int, number of ms to wait before raising WaitException.
|
|
exponential_sleep_multiplier: float, factor to use on subsequent retries.
|
|
jitter_ms: int, random (up to the value) additional sleep between retries.
|
|
wait_ceiling_ms: int, Maximum wait between retries.
|
|
sleep_ms: int or iterable: for how long to wait between trials.
|
|
|
|
Returns:
|
|
poller.GetResult(operation).
|
|
|
|
Raises:
|
|
AbortWaitError: if ctrl-c was pressed.
|
|
TimeoutError: if retryer has finished without being done.
|
|
"""
|
|
aborted_message = 'Aborting wait for operation {0}.\n'.format(operation_ref)
|
|
try:
|
|
with progress_tracker.ProgressTracker(
|
|
message, aborted_message=aborted_message
|
|
) if not custom_tracker else custom_tracker as tracker:
|
|
|
|
if pre_start_sleep_ms:
|
|
_SleepMs(pre_start_sleep_ms)
|
|
|
|
def _StatusUpdate(result, status):
|
|
if tracker_update_func:
|
|
tracker_update_func(tracker, result, status)
|
|
else:
|
|
tracker.Tick()
|
|
|
|
operation = PollUntilDone(
|
|
poller, operation_ref, max_retrials, max_wait_ms,
|
|
exponential_sleep_multiplier, jitter_ms, wait_ceiling_ms,
|
|
sleep_ms, _StatusUpdate)
|
|
|
|
except retry.WaitException:
|
|
raise TimeoutError(
|
|
'Operation {0} has not finished in {1} seconds. {2}'
|
|
.format(operation_ref, max_wait_ms // 1000, _TIMEOUT_MESSAGE))
|
|
except retry.MaxRetrialsException as e:
|
|
raise TimeoutError(
|
|
'Operation {0} has not finished in {1} seconds '
|
|
'after max {2} retrials. {3}'
|
|
.format(operation_ref,
|
|
e.state.time_passed_ms // 1000,
|
|
e.state.retrial,
|
|
_TIMEOUT_MESSAGE))
|
|
|
|
return poller.GetResult(operation)
|
|
|
|
|
|
def PollUntilDone(poller, operation_ref,
|
|
max_retrials=None,
|
|
max_wait_ms=1800000,
|
|
exponential_sleep_multiplier=1.4,
|
|
jitter_ms=1000,
|
|
wait_ceiling_ms=180000,
|
|
sleep_ms=2000,
|
|
status_update=None):
|
|
"""Waits for poller.Poll to complete.
|
|
|
|
Note that this *does not* print nice messages to stderr for the user; most
|
|
callers should use WaitFor instead for the best UX unless there's a good
|
|
reason not to print.
|
|
|
|
Args:
|
|
poller: OperationPoller, poller to use during retrials.
|
|
operation_ref: object, passed to operation poller poll method.
|
|
max_retrials: int, max number of retrials before raising RetryException.
|
|
max_wait_ms: int, number of ms to wait before raising WaitException.
|
|
exponential_sleep_multiplier: float, factor to use on subsequent retries.
|
|
jitter_ms: int, random (up to the value) additional sleep between retries.
|
|
wait_ceiling_ms: int, Maximum wait between retries.
|
|
sleep_ms: int or iterable: for how long to wait between trials.
|
|
status_update: func(result, state) called right after each trial.
|
|
|
|
Returns:
|
|
The return value from poller.Poll.
|
|
"""
|
|
|
|
retryer = retry.Retryer(
|
|
max_retrials=max_retrials,
|
|
max_wait_ms=max_wait_ms,
|
|
exponential_sleep_multiplier=exponential_sleep_multiplier,
|
|
jitter_ms=jitter_ms,
|
|
wait_ceiling_ms=wait_ceiling_ms,
|
|
status_update_func=status_update)
|
|
|
|
def _IsNotDone(operation, unused_state):
|
|
return not poller.IsDone(operation)
|
|
|
|
operation = retryer.RetryOnResult(
|
|
func=poller.Poll,
|
|
args=(operation_ref,),
|
|
should_retry_if=_IsNotDone,
|
|
sleep_ms=sleep_ms)
|
|
|
|
return operation
|
|
|
|
|
|
def _SleepMs(miliseconds):
|
|
time.sleep(miliseconds / 1000)
|