413 lines
15 KiB
Python
413 lines
15 KiB
Python
# -*- coding: utf-8 -*- #
|
|
# Copyright 2019 Google LLC. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Module for making API requests."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
import copy
|
|
import json
|
|
|
|
from googlecloudsdk.api_lib.compute import batch_helper
|
|
from googlecloudsdk.api_lib.compute import single_request_helper
|
|
from googlecloudsdk.api_lib.compute import utils
|
|
from googlecloudsdk.api_lib.compute import waiters
|
|
from googlecloudsdk.core import log
|
|
from googlecloudsdk.core import properties
|
|
|
|
import six
|
|
from six.moves import zip # pylint: disable=redefined-builtin
|
|
|
|
|
|
def _RequestsAreListRequests(requests):
|
|
"""Checks if all requests are of list requests."""
|
|
list_requests = [
|
|
method in (
|
|
'List', 'AggregatedList', 'ListInstances', 'ListManagedInstances'
|
|
) for _, method, _ in requests
|
|
]
|
|
if all(list_requests):
|
|
return True
|
|
elif not any(list_requests):
|
|
return False
|
|
else:
|
|
raise ValueError(
|
|
'All requests must be either list requests or non-list requests.')
|
|
|
|
|
|
def _HandleJsonList(response, service, method, errors):
|
|
"""Extracts data from one *List response page as JSON and stores in dicts.
|
|
|
|
Args:
|
|
response: str, The *List response in JSON
|
|
service: The service which responded to *List request
|
|
method: str, Method used to list resources. One of 'List' or
|
|
'AggregatedList'.
|
|
errors: list, Errors from response will be appended to this list.
|
|
|
|
Returns:
|
|
Pair of:
|
|
- List of items returned in response as dicts
|
|
- Next page token (if present, otherwise None).
|
|
"""
|
|
items = []
|
|
|
|
response = json.loads(response)
|
|
|
|
# If the request is a list call, then yield the items directly.
|
|
if method in ('List', 'ListInstances'):
|
|
items = response.get('items', [])
|
|
elif method == 'ListManagedInstances':
|
|
items = response.get('managedInstances', [])
|
|
|
|
# If the request is an aggregatedList call, then do all the
|
|
# magic necessary to get the actual resources because the
|
|
# aggregatedList responses are very complicated data
|
|
# structures...
|
|
elif method == 'AggregatedList':
|
|
items_field_name = service.GetMethodConfig(
|
|
'AggregatedList').relative_path.split('/')[-1]
|
|
for scope_result in six.itervalues(response['items']):
|
|
# If the given scope is unreachable, record the warning
|
|
# message in the errors list.
|
|
warning = scope_result.get('warning', None)
|
|
if warning and warning['code'] == 'UNREACHABLE':
|
|
errors.append((None, warning['message']))
|
|
|
|
items.extend(scope_result.get(items_field_name, []))
|
|
|
|
return items, response.get('nextPageToken', None)
|
|
|
|
|
|
def _HandleMessageList(response, service, method, errors):
|
|
"""Extracts data from one *List response page as Message object."""
|
|
items = []
|
|
|
|
# If the request is a list call, then yield the items directly.
|
|
if method in ('List', 'ListInstances'):
|
|
items = response.items
|
|
elif method == 'ListManagedInstances':
|
|
items = response.managedInstances
|
|
# If the request is an aggregatedList call, then do all the
|
|
# magic necessary to get the actual resources because the
|
|
# aggregatedList responses are very complicated data
|
|
# structures...
|
|
else:
|
|
items_field_name = service.GetMethodConfig(
|
|
'AggregatedList').relative_path.split('/')[-1]
|
|
for scope_result in response.items.additionalProperties:
|
|
# If the given scope is unreachable, record the warning
|
|
# message in the errors list.
|
|
warning = scope_result.value.warning
|
|
if warning and warning.code == warning.CodeValueValuesEnum.UNREACHABLE:
|
|
errors.append((None, warning.message))
|
|
|
|
items.extend(getattr(scope_result.value, items_field_name))
|
|
|
|
return items, response.nextPageToken
|
|
|
|
|
|
def _ListCore(requests, http, batch_url, errors, response_handler):
|
|
"""Makes a series of list and/or aggregatedList batch requests.
|
|
|
|
Args:
|
|
requests: A list of requests to make. Each element must be a 3-element tuple
|
|
where the first element is the service, the second element is the method
|
|
('List' or 'AggregatedList'), and the third element is a protocol buffer
|
|
representing either a list or aggregatedList request.
|
|
http: An httplib2.Http-like object.
|
|
batch_url: The handler for making batch requests.
|
|
errors: A list for capturing errors. If any response contains an error, it
|
|
is added to this list.
|
|
response_handler: The function to extract information responses.
|
|
|
|
Yields:
|
|
Resources encapsulated in format chosen by response_handler as they are
|
|
received from the server.
|
|
"""
|
|
while requests:
|
|
if not _ForceBatchRequest() and len(requests) == 1:
|
|
service, method, request_body = requests[0]
|
|
responses, request_errors = single_request_helper.MakeSingleRequest(
|
|
service, method, request_body
|
|
)
|
|
errors.extend(request_errors)
|
|
else:
|
|
responses, request_errors = batch_helper.MakeRequests(
|
|
requests=requests, http=http, batch_url=batch_url
|
|
)
|
|
errors.extend(request_errors)
|
|
|
|
new_requests = []
|
|
|
|
for i, response in enumerate(responses):
|
|
if not response:
|
|
continue
|
|
|
|
service, method, request_protobuf = requests[i]
|
|
|
|
items, next_page_token = response_handler(response, service, method,
|
|
errors)
|
|
for item in items:
|
|
yield item
|
|
|
|
if next_page_token:
|
|
new_request_protobuf = copy.deepcopy(request_protobuf)
|
|
new_request_protobuf.pageToken = next_page_token
|
|
new_requests.append((service, method, new_request_protobuf))
|
|
|
|
requests = new_requests
|
|
|
|
|
|
def _List(requests, http, batch_url, errors):
|
|
"""Makes a series of list and/or aggregatedList batch requests.
|
|
|
|
Args:
|
|
requests: A list of requests to make. Each element must be a 3-element tuple
|
|
where the first element is the service, the second element is the method
|
|
('List' or 'AggregatedList'), and the third element is a protocol buffer
|
|
representing either a list or aggregatedList request.
|
|
http: An httplib2.Http-like object.
|
|
batch_url: The handler for making batch requests.
|
|
errors: A list for capturing errors. If any response contains an error, it
|
|
is added to this list.
|
|
|
|
Returns:
|
|
Resources encapsulated as protocol buffers as they are received
|
|
from the server.
|
|
"""
|
|
return _ListCore(requests, http, batch_url, errors, _HandleMessageList)
|
|
|
|
|
|
def _IsEmptyOperation(operation, service):
|
|
"""Checks whether operation argument is empty.
|
|
|
|
Args:
|
|
operation: Operation thats checked for emptyness.
|
|
service: Variable used to access service.client.MESSAGES_MODULE.Operation.
|
|
|
|
Returns:
|
|
True if operation is empty, False otherwise.
|
|
"""
|
|
if not isinstance(operation, service.client.MESSAGES_MODULE.Operation):
|
|
raise ValueError('operation must be instance of'
|
|
+ 'service.client.MESSAGES_MODULE.Operation')
|
|
|
|
for field in operation.all_fields():
|
|
if (field.name != 'kind' and field.name != 'warnings' and
|
|
getattr(operation, field.name) is not None):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _ForceBatchRequest():
|
|
"""Check if compute/force_batch_request property is set."""
|
|
return properties.VALUES.compute.force_batch_request.GetBool()
|
|
|
|
|
|
def ListJson(requests, http, batch_url, errors):
|
|
"""Makes a series of list and/or aggregatedList batch requests.
|
|
|
|
This function does all of:
|
|
- Sends batch of List/AggregatedList requests
|
|
- Extracts items from responses
|
|
- Handles pagination
|
|
|
|
All requests must be sent to the same client - Compute.
|
|
|
|
Args:
|
|
requests: A list of requests to make. Each element must be a 3-element tuple
|
|
where the first element is the service, the second element is the method
|
|
('List' or 'AggregatedList'), and the third element is a protocol buffer
|
|
representing either a list or aggregatedList request.
|
|
http: An httplib2.Http-like object.
|
|
batch_url: The handler for making batch requests.
|
|
errors: A list for capturing errors. If any response contains an error, it
|
|
is added to this list.
|
|
|
|
Yields:
|
|
Resources in dicts as they are received from the server.
|
|
"""
|
|
# This is compute-specific helper. It is assumed at this point that all
|
|
# requests are being sent to the same client (for example Compute).
|
|
with requests[0][0].client.JsonResponseModel():
|
|
for item in _ListCore(requests, http, batch_url, errors, _HandleJsonList):
|
|
yield item
|
|
|
|
|
|
def MakeRequests(
|
|
requests,
|
|
http,
|
|
batch_url,
|
|
errors,
|
|
project_override=None,
|
|
progress_tracker=None,
|
|
no_followup=False,
|
|
always_return_operation=False,
|
|
followup_overrides=None,
|
|
log_result=True,
|
|
log_warnings=True,
|
|
timeout=None,
|
|
):
|
|
"""Makes one or more requests to the API.
|
|
|
|
Each request can be either a synchronous API call or an asynchronous
|
|
one. For synchronous calls (e.g., get and list), the result from the
|
|
server is yielded immediately. For asynchronous calls (e.g., calls
|
|
that return operations like insert), this function waits until the
|
|
operation reaches the DONE state and fetches the corresponding
|
|
object and yields that object (nothing is yielded for deletions).
|
|
|
|
Currently, a heterogeneous set of synchronous calls can be made
|
|
(e.g., get request to fetch a disk and instance), however, the
|
|
asynchronous requests must be homogenous (e.g., they must all be the
|
|
same verb on the same collection). In the future, heterogeneous
|
|
asynchronous requests will be supported. For now, it is up to the
|
|
client to ensure that the asynchronous requests are
|
|
homogenous. Synchronous and asynchronous requests can be mixed.
|
|
|
|
Args:
|
|
requests: A list of requests to make. Each element must be a 3-element tuple
|
|
where the first element is the service, the second element is the string
|
|
name of the method on the service, and the last element is a protocol
|
|
buffer representing the request.
|
|
http: An httplib2.Http-like object.
|
|
batch_url: The handler for making batch requests.
|
|
errors: A list for capturing errors. If any response contains an error, it
|
|
is added to this list.
|
|
project_override: The override project for the returned operation to poll
|
|
from.
|
|
progress_tracker: progress tracker to be ticked while waiting for operations
|
|
to finish.
|
|
no_followup: If True, do not followup operation with a GET request.
|
|
always_return_operation: If True, return operation object even if operation
|
|
fails.
|
|
followup_overrides: A list of new resource names to GET once the operation
|
|
finishes. Generally used in renaming calls.
|
|
log_result: Whether the Operation Waiter should print the result in past
|
|
tense of each request.
|
|
log_warnings: Whether warnings for completed operation should be printed.
|
|
timeout: The maximum amount of time, in seconds, to wait for the operations
|
|
to reach the DONE state.
|
|
|
|
Yields:
|
|
A response for each request. For deletion requests, no corresponding
|
|
responses are returned.
|
|
"""
|
|
if _RequestsAreListRequests(requests):
|
|
for item in _List(
|
|
requests=requests, http=http, batch_url=batch_url, errors=errors
|
|
):
|
|
yield item
|
|
return
|
|
|
|
# send single request only if the requests size one and if enable_single_
|
|
# request is set to true
|
|
if not _ForceBatchRequest() and len(requests) == 1:
|
|
service, method, request_body = requests[0]
|
|
responses, new_errors = single_request_helper.MakeSingleRequest(
|
|
service=service, method=method, request_body=request_body
|
|
)
|
|
else:
|
|
responses, new_errors = batch_helper.MakeRequests(
|
|
requests=requests, http=http, batch_url=batch_url)
|
|
errors.extend(new_errors)
|
|
|
|
operation_service = None
|
|
resource_service = None
|
|
|
|
# Collects all operation objects in a list so they can be waited on
|
|
# and yields all non-operation objects since non-operation responses
|
|
# cannot be waited on.
|
|
operations_data = []
|
|
|
|
if not followup_overrides:
|
|
followup_overrides = [None for _ in requests]
|
|
for request, response, followup_override in zip(requests, responses,
|
|
followup_overrides):
|
|
if response is None:
|
|
continue
|
|
|
|
service, _, request_body = request
|
|
if (isinstance(response, service.client.MESSAGES_MODULE.Operation) and
|
|
not _IsEmptyOperation(response, service) and
|
|
service.__class__.__name__ not in (
|
|
'GlobalOperationsService', 'RegionOperationsService',
|
|
'ZoneOperationsService', 'GlobalOrganizationOperationsService',
|
|
'GlobalAccountsOperationsService')):
|
|
|
|
resource_service = service
|
|
project = None
|
|
if hasattr(request_body, 'project'):
|
|
if project_override:
|
|
project = project_override
|
|
else:
|
|
project = request_body.project
|
|
|
|
if response.zone:
|
|
operation_service = service.client.zoneOperations
|
|
elif response.region:
|
|
operation_service = service.client.regionOperations
|
|
else:
|
|
operation_service = service.client.globalOperations
|
|
else:
|
|
operation_service = service.client.globalOrganizationOperations
|
|
# TODO: b/313849714 - Leave only else block once the bug is fixed.
|
|
if hasattr(request_body, 'instanceGroupManagerResizeRequest'):
|
|
operations_data.append(
|
|
waiters.OperationData(
|
|
response,
|
|
operation_service,
|
|
resource_service,
|
|
project=project,
|
|
resize_request_name=request_body.instanceGroupManagerResizeRequest.name,
|
|
no_followup=no_followup,
|
|
followup_override=followup_override,
|
|
always_return_operation=always_return_operation,
|
|
)
|
|
)
|
|
else:
|
|
operations_data.append(
|
|
waiters.OperationData(
|
|
response,
|
|
operation_service,
|
|
resource_service,
|
|
project=project,
|
|
no_followup=no_followup,
|
|
followup_override=followup_override,
|
|
always_return_operation=always_return_operation))
|
|
|
|
else:
|
|
yield response
|
|
|
|
if operations_data:
|
|
warnings = []
|
|
for response in waiters.WaitForOperations(
|
|
operations_data=operations_data,
|
|
http=http,
|
|
batch_url=batch_url,
|
|
warnings=warnings,
|
|
progress_tracker=progress_tracker,
|
|
errors=errors,
|
|
log_result=log_result,
|
|
timeout=timeout,
|
|
):
|
|
yield response
|
|
|
|
if warnings and log_warnings:
|
|
log.warning(
|
|
utils.ConstructList('Some requests generated warnings:', warnings))
|