feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/util/api_enablement.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for API enablement."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import re
+
+from googlecloudsdk.api_lib.services import enable_api
+from googlecloudsdk.core.console import console_io
+
+
+API_ENABLEMENT_REGEX = re.compile(
+    '.*Enable it by visiting https://console.(?:cloud|developers).google.com'
+    '/apis/api/([^/]+)/overview\\?project=(\\S+) then retry. If you '
+    'enabled this API recently, wait a few minutes for the action to propagate'
+    ' to our systems and retry.\\w*')
+
+
+_PROJECTS_NOT_TO_ENABLE = {'google.com:cloudsdktool'}
+
+
+def ShouldAttemptProjectEnable(project):
+  return project not in _PROJECTS_NOT_TO_ENABLE
+
+
+def GetApiEnablementInfo(status_message):
+  """Parses error message for API enablement messages.
+
+  Args:
+    status_message: str, error message to parse.
+
+  Returns:
+    tuple[str]: The project, service token to be used for prompting to enable
+        the API.
+  """
+  match = API_ENABLEMENT_REGEX.match(status_message)
+  if match:
+    (project, service_token) = (match.group(2), match.group(1))
+    if (project is not None and ShouldAttemptProjectEnable(project)
+        and service_token is not None):
+      return (project, service_token)
+  return None
+
+
+def PromptToEnableApi(project, service_token, enable_by_default=False):
+  """Prompts to enable the API.
+
+  Args:
+    project (str): The project that the API is not enabled on.
+    service_token (str): The service token of the API to prompt for.
+    enable_by_default (bool): The default choice for the enablement prompt.
+
+  Returns:
+    bool, whether or not the API was attempted to be enabled
+  """
+  api_enable_attempted = console_io.PromptContinue(
+      default=enable_by_default,
+      prompt_string=('API [{}] not enabled on project [{}]. '
+                     'Would you like to enable and retry (this will take a '
+                     'few minutes)?').format(service_token, project))
+  if api_enable_attempted:
+    enable_api.EnableService(project, service_token)
+  return api_enable_attempted

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/util/apis.py

@@ -0,0 +1,396 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2016 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Library for obtaining API clients and messages."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from apitools.base.py import exceptions as apitools_exceptions
+
+from googlecloudsdk.api_lib.util import api_enablement
+from googlecloudsdk.api_lib.util import apis_internal
+from googlecloudsdk.api_lib.util import apis_util
+from googlecloudsdk.api_lib.util import exceptions as api_exceptions
+from googlecloudsdk.core import exceptions
+from googlecloudsdk.core import properties
+from googlecloudsdk.core.util import regional
+from googlecloudsdk.generated_clients.apis import apis_map
+
+import six
+
+
+class Error(exceptions.Error):
+  """A base class for apis helper errors."""
+  pass
+
+
+class GapicRestUnsupportedError(Error):
+  """An error for the unsupported REST transport on GAPIC Clients."""
+
+  def __init__(self):
+    super(
+        GapicRestUnsupportedError,
+        self).__init__('REST transport is not yet supported for GAPIC Clients')
+
+
+def AddUnreleasedAPIs(unreleased_apis_map):
+  # Add in unreleased APIs so that they can be looked up via main api map.
+  for api_name, api_versions in six.iteritems(unreleased_apis_map.MAP):
+    for api_version, api_def in six.iteritems(api_versions):
+      _AddToApisMap(api_name, api_version, api_def)
+
+
+def _AddToApisMap(api_name, api_version, api_def):
+  """Adds the APIDef specified by the given arguments to the APIs map.
+
+  This method should only be used for runtime patching of the APIs map.
+  Additions to the map should ensure that there is only one and only one default
+  version for each API.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+    api_version: str, The version of the API.
+    api_def: APIDef for the API version.
+  """
+  # pylint:disable=protected-access
+  api_name, _ = apis_internal._GetApiNameAndAlias(api_name)
+
+  # Register API version as default if this API did not exist,
+  # otherwise we'll set the first APIs map
+  api_versions = apis_map.MAP.get(api_name, {})
+  api_def.default_version = not api_versions
+
+  api_versions[api_version] = api_def
+  apis_map.MAP[api_name] = api_versions
+
+
+def GetVersions(api_name):
+  """Return available versions for given api.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+
+  Raises:
+    UnknownAPIError: If api_name does not exist in the APIs map.
+
+  Returns:
+    list, of version names.
+  """
+  # pylint:disable=protected-access
+  return apis_internal._GetVersions(api_name)
+
+
+def ResolveVersion(api_name, api_version=None):
+  """Resolves the version for an API based on the APIs map and API overrides.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+    api_version: str, The API version.
+
+  Raises:
+    apis_internal.UnknownAPIError: If api_name does not exist in the APIs map.
+
+  Returns:
+    str, The resolved version.
+  """
+  # pylint:disable=protected-access
+  api_name, api_name_alias = apis_internal._GetApiNameAndAlias(api_name)
+  if api_name not in apis_map.MAP:
+    raise apis_util.UnknownAPIError(api_name)
+
+  version_overrides = properties.VALUES.api_client_overrides.AllValues()
+
+  # First try to get api specific override, then try full surface override.
+  api_version_override = None
+  if api_version:
+    api_version_override = version_overrides.get(
+        '{}/{}'.format(api_name_alias, api_version), None)
+  if not api_version_override:
+    api_version_override = version_overrides.get(api_name_alias, api_version)
+
+  return (api_version_override or
+          # pylint:disable=protected-access
+          apis_internal._GetDefaultVersion(api_name))
+
+
+API_ENABLEMENT_ERROR_EXPECTED_STATUS_CODE = 403  # retry status code
+RESOURCE_EXHAUSTED_STATUS_CODE = 429
+MAX_RETRY_DELAY_SEC = 60  # Default max retry delay defined by apitools.
+
+
+def GetApiEnablementInfo(exception):
+  """Returns the API Enablement info or None if prompting is not necessary.
+
+  Args:
+    exception (apitools_exceptions.HttpError): Exception if an error occurred.
+
+  Returns:
+    tuple[str]: The project, service token, exception tuple to be used for
+      prompting to enable the API.
+
+  Raises:
+    api_exceptions.HttpException: If gcloud should not prompt to enable the API.
+  """
+  parsed_error = api_exceptions.HttpException(exception)
+  if (parsed_error.payload.status_code !=
+      API_ENABLEMENT_ERROR_EXPECTED_STATUS_CODE):
+    return None
+
+  enablement_info = api_enablement.GetApiEnablementInfo(
+      parsed_error.payload.status_message)
+  if enablement_info:
+    return enablement_info + (parsed_error,)
+  return None
+
+
+def PromptToEnableApi(project, service_token, exception,
+                      is_batch_request=False):
+  """Prompts to enable the API and throws if the answer is no.
+
+  Args:
+    project (str): The project that the API is not enabled on.
+    service_token (str): The service token of the API to prompt for.
+    exception (api_Exceptions.HttpException): Exception to throw if the prompt
+      is denied.
+    is_batch_request: If the request is a batch request. This determines how to
+      get apitools to retry the request.
+
+  Raises:
+    api_exceptions.HttpException: API not enabled error if the user chooses to
+      not enable the API.
+  """
+  api_enable_attempted = api_enablement.PromptToEnableApi(
+      project, service_token)
+  if api_enable_attempted:
+    if not is_batch_request:
+      raise apitools_exceptions.RequestError('Retry')
+  else:
+    raise exception
+
+
+def CheckResponse(skip_activation_prompt=False):
+  """Returns a callback for checking API errors."""
+  state = {'already_prompted_to_enable': False}
+
+  def _CheckForApiEnablementError(response_as_error):
+    # If it was an API enablement error,
+    # prompt the user to enable the API. If yes, we make that call and then
+    # raise a RequestError, which will prompt the caller to retry. If not, we
+    # raise the actual HTTP error.
+    enablement_info = GetApiEnablementInfo(response_as_error)
+    if enablement_info:
+      if state['already_prompted_to_enable'] or skip_activation_prompt:
+        raise apitools_exceptions.RequestError('Retry')
+      state['already_prompted_to_enable'] = True
+      PromptToEnableApi(*enablement_info)
+
+  def _CheckResponse(response):
+    """Checks API error.
+
+    If it's an enablement error, prompt to enable & retry.
+    If it's a resource exhausted error, no retry & return.
+
+    Args:
+      response: response that had an error.
+
+    Raises:
+      apitools_exceptions.RequestError: error which should signal apitools to
+        retry.
+      api_exceptions.HttpException: the parsed error.
+    """
+    # This will throw if there was a specific type of error. If not, then we can
+    # parse and deal with our own class of errors.
+    if response is None:
+      # Caller shouldn't call us if the response is None, but handle anyway.
+      raise apitools_exceptions.RequestError(
+          'Request to url %s did not return a response.' % response.request_url
+      )
+    elif response.status_code == RESOURCE_EXHAUSTED_STATUS_CODE:
+      if response.retry_after and response.retry_after > MAX_RETRY_DELAY_SEC:
+        # If the retry after is greater than the max retry delay, do not retry.
+        return
+      if response.retry_after:
+        raise apitools_exceptions.RetryAfterError.FromResponse(response)
+      else:
+        raise apitools_exceptions.BadStatusCodeError.FromResponse(response)
+
+    elif response.status_code >= 500:
+      raise apitools_exceptions.BadStatusCodeError.FromResponse(response)
+
+    elif response.retry_after:
+      raise apitools_exceptions.RetryAfterError.FromResponse(response)
+
+    response_as_error = apitools_exceptions.HttpError.FromResponse(response)
+
+    if properties.VALUES.core.should_prompt_to_enable_api.GetBool():
+      _CheckForApiEnablementError(response_as_error)
+
+  return _CheckResponse
+
+
+def GetClientClass(api_name, api_version):
+  """Returns the client class for the API specified in the args.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+    api_version: str, The version of the API.
+
+  Returns:
+    base_api.BaseApiClient, Client class for the specified API.
+  """
+  # pylint:disable=protected-access
+  return apis_internal._GetClientClass(api_name, api_version)
+
+
+def GetClientInstance(
+    api_name,
+    api_version,
+    no_http=False,
+    http_timeout_sec=None,
+    skip_activation_prompt=False,
+    location=None,
+):
+  """Returns an instance of the API client specified in the args.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+    api_version: str, The version of the API.
+    no_http: bool, True to not create an http object for this client.
+    http_timeout_sec: int, seconds for http timeout, default if None.
+    skip_activation_prompt: bool, if true, do not prompt for service activation.
+    location: str, Region, multi-region, or zone to use for regionalized
+      endpoints (REP).
+
+  Returns:
+    base_api.BaseApiClient, An instance of the specified API client.
+  """
+  # pylint:disable=protected-access
+  return apis_internal._GetClientInstance(
+      api_name,
+      api_version,
+      no_http,
+      None,
+      CheckResponse(skip_activation_prompt),
+      http_timeout_sec=http_timeout_sec,
+      region=regional.LocationToRegion(location) if location else None,
+  )
+
+
+def GetGapicClientClass(api_name,
+                        api_version,
+                        transport=apis_util.GapicTransport.GRPC):
+  """Returns the GAPIC client class for the API specified in the args.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+    api_version: str, The version of the API.
+    transport: apis_util.GapicTransport, The transport class to obtain.
+
+  Raises:
+    GapicRestUnsupportedError: If transport is REST.
+
+  Returns:
+    The specified GAPIC API Client class.
+  """
+  if transport == apis_util.GapicTransport.REST:
+    raise GapicRestUnsupportedError()
+  # pylint:disable=protected-access
+  return apis_internal._GetGapicClientClass(
+      api_name, api_version, transport_choice=transport)
+
+
+def GetGapicClientInstance(
+    api_name,
+    api_version,
+    address_override_func=None,
+    transport=apis_util.GapicTransport.GRPC,
+    attempt_direct_path=False,
+    redact_request_body_reason=None,
+    custom_interceptors=None,
+    location=None,
+):
+  """Returns an instance of the GAPIC API client specified in the args.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+    api_version: str, The version of the API.
+    address_override_func: function, function to call to override the client
+      host. It takes a single argument which is the original host.
+    transport: apis_util.GapicTransport, The transport to be used by the client.
+    attempt_direct_path: bool, True if we want to attempt direct path gRPC where
+      possible.
+    redact_request_body_reason: str, the reason why the request body must be
+      redacted if --log-http is used. If None, the body is not redacted.
+    custom_interceptors: list[grpc interceptor], a list of custom
+      interceptors to add to the channel.
+    location: str, Region, multi-region, or zone for regionalized endpoints
+      (REP).
+
+  Raises:
+    GapicRestUnsupportedError: If transport is REST.
+
+  Returns:
+    An instance of the specified GAPIC API client.
+  """
+  # pylint: disable=g-import-not-at-top
+  from googlecloudsdk.core import gapic_util
+  # pylint: enable=g-import-not-at-top
+  if transport == apis_util.GapicTransport.REST:
+    raise GapicRestUnsupportedError()
+  credentials = gapic_util.GetGapicCredentials()
+  # pylint:disable=protected-access
+  return apis_internal._GetGapicClientInstance(
+      api_name,
+      api_version,
+      credentials,
+      address_override_func=address_override_func,
+      transport_choice=transport,
+      attempt_direct_path=attempt_direct_path,
+      redact_request_body_reason=redact_request_body_reason,
+      custom_interceptors=custom_interceptors,
+      region=regional.LocationToRegion(location) if location else None,
+  )
+
+
+def GetEffectiveApiEndpoint(api_name, api_version, client_class=None):
+  """Returns effective endpoint for given api."""
+  # pylint:disable=protected-access
+  return apis_internal._GetEffectiveApiEndpoint(api_name,
+                                                api_version,
+                                                client_class)
+
+
+def GetMessagesModule(api_name, api_version):
+  """Returns the messages module for the API specified in the args.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+    api_version: str, The version of the API.
+
+  Returns:
+    Module containing the definitions of messages for the specified API.
+  """
+  # pylint:disable=protected-access
+  api_def = apis_internal.GetApiDef(api_name, api_version)
+  # fromlist below must not be empty, see:
+  # http://stackoverflow.com/questions/2724260/why-does-pythons-import-require-fromlist.
+  return __import__(api_def.apitools.messages_full_modulepath,
+                    fromlist=['something'])
+
+
+def UniversifyAddress(address):
+  return apis_internal.UniversifyAddress(address)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/util/apis_internal.py

@@ -0,0 +1,563 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Library for obtaining API clients and messages.
+
+This should only be called by api_lib.util.apis, core.resources, gcloud meta
+commands, and module tests.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.util import apis_util
+from googlecloudsdk.api_lib.util import resource as resource_util
+from googlecloudsdk.core import exceptions
+from googlecloudsdk.core import log
+from googlecloudsdk.core import properties
+from googlecloudsdk.core import transport
+from googlecloudsdk.generated_clients.apis import apis_map
+import six
+from six.moves.urllib.parse import urljoin
+from six.moves.urllib.parse import urlparse
+from six.moves.urllib.parse import urlunparse
+
+
+def _GetApiNameAndAlias(api_name):
+  # pylint:disable=protected-access
+  return (apis_util._API_NAME_ALIASES.get(api_name, api_name), api_name)
+
+
+def _GetDefaultVersion(api_name):
+  api_name, _ = _GetApiNameAndAlias(api_name)
+  api_vers = apis_map.MAP.get(api_name, {})
+  for ver, api_def in six.iteritems(api_vers):
+    if api_def.default_version:
+      return ver
+  return None
+
+
+def _GetApiNames():
+  """Returns list of avaiblable apis, ignoring the version."""
+  return sorted(apis_map.MAP.keys())
+
+
+def _GetVersions(api_name):
+  """Return available versions for given api.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+
+  Raises:
+    apis_util.UnknownAPIError: If api_name does not exist in the APIs map.
+
+  Returns:
+    list, of version names.
+  """
+  api_name, _ = _GetApiNameAndAlias(api_name)
+  version_map = apis_map.MAP.get(api_name, None)
+  if version_map is None:
+    raise apis_util.UnknownAPIError(api_name)
+  return list(version_map.keys())
+
+
+def GetApiDef(api_name, api_version):
+  """Returns the APIDef for the specified API and version.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+    api_version: str, The version of the API.
+
+  Raises:
+    apis_util.UnknownAPIError: If api_name does not exist in the APIs map.
+    apis_util.UnknownVersionError: If api_version does not exist for given
+      api_name in the APIs map.
+
+  Returns:
+    APIDef, The APIDef for the specified API and version.
+  """
+  api_name, api_name_alias = _GetApiNameAndAlias(api_name)
+  if api_name not in apis_map.MAP:
+    raise apis_util.UnknownAPIError(api_name)
+
+  version_overrides = properties.VALUES.api_client_overrides.AllValues()
+
+  # First attempt to get api specific override, then full surface override.
+  version_override = version_overrides.get('{}/{}'.format(
+      api_name, api_version))
+  if not version_override:
+    version_override = version_overrides.get(api_name_alias, None)
+
+  api_version = version_override or api_version
+
+  api_versions = apis_map.MAP[api_name]
+  if api_version is None or api_version not in api_versions:
+    raise apis_util.UnknownVersionError(api_name, api_version)
+  else:
+    api_def = api_versions[api_version]
+
+  return api_def
+
+
+def _GetClientClass(api_name, api_version):
+  """Returns the client class for the API specified in the args.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+    api_version: str, The version of the API.
+
+  Returns:
+    base_api.BaseApiClient, Client class for the specified API.
+  """
+  api_def = GetApiDef(api_name, api_version)
+  return _GetClientClassFromDef(api_def)
+
+
+def _GetClientClassFromDef(api_def):
+  """Returns the apitools client class for the API definition specified in args.
+
+  Args:
+    api_def: apis_map.APIDef, The definition of the API.
+
+  Returns:
+    base_api.BaseApiClient, Client class for the specified API.
+  """
+  client_full_classpath = api_def.apitools.client_full_classpath
+  module_path, client_class_name = client_full_classpath.rsplit('.', 1)
+  module_obj = __import__(module_path, fromlist=[client_class_name])
+  return getattr(module_obj, client_class_name)
+
+
+def _GetClientInstance(api_name,
+                       api_version,
+                       no_http=False,
+                       http_client=None,
+                       check_response_func=None,
+                       http_timeout_sec=None,
+                       region=None):
+  """Returns an instance of the API client specified in the args.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+    api_version: str, The version of the API.
+    no_http: bool, True to not create an http object for this client.
+    http_client: bring your own http client to use. Incompatible with
+      no_http=True.
+    check_response_func: error handling callback to give to apitools.
+    http_timeout_sec: int, seconds of http timeout to set, defaults if None.
+    region: str, Region (or multi-region) for regionalized endpoints (REP).
+
+  Returns:
+    base_api.BaseApiClient, An instance of the specified API client.
+  """
+
+  # pylint: disable=g-import-not-at-top
+  if no_http:
+    assert http_client is None
+  elif http_client is None:
+    # Normal gcloud authentication
+    # Import http only when needed, as it depends on credential infrastructure
+    # which is not needed in all cases.
+    from googlecloudsdk.core.credentials import transports
+    http_client = transports.GetApitoolsTransport(
+        response_encoding=transport.ENCODING,
+        timeout=http_timeout_sec if http_timeout_sec else 'unset')
+
+  client_class = _GetClientClass(api_name, api_version)
+  client_instance = client_class(
+      url=_GetEffectiveApiEndpoint(api_name, api_version, client_class, region),
+      get_credentials=False,
+      http=http_client)
+  if check_response_func is not None:
+    client_instance.check_response_func = check_response_func
+  api_key = properties.VALUES.core.api_key.Get()
+  if api_key:
+    client_instance.AddGlobalParam('key', api_key)
+    header = 'X-Google-Project-Override'
+    client_instance.additional_http_headers[header] = 'apikey'
+  return client_instance
+
+
+def _GetGapicClientClass(api_name,
+                         api_version,
+                         transport_choice=apis_util.GapicTransport.GRPC):
+  """Returns the GAPIC client class for the API def specified by the args.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+    api_version: str, The version of the API.
+    transport_choice: apis_util.GapicTransport, The transport to be used by the
+      client.
+  """
+  api_def = GetApiDef(api_name, api_version)
+  if transport_choice == apis_util.GapicTransport.GRPC_ASYNCIO:
+    client_full_classpath = api_def.gapic.async_client_full_classpath
+  elif transport_choice == apis_util.GapicTransport.REST:
+    client_full_classpath = api_def.gapic.rest_client_full_classpath
+  else:
+    client_full_classpath = api_def.gapic.client_full_classpath
+
+  module_path, client_class_name = client_full_classpath.rsplit('.', 1)
+  module_obj = __import__(module_path, fromlist=[client_class_name])
+  return getattr(module_obj, client_class_name)
+
+
+def _GetGapicClientInstance(
+    api_name,
+    api_version,
+    credentials,
+    address_override_func=None,
+    transport_choice=apis_util.GapicTransport.GRPC,
+    attempt_direct_path=False,
+    redact_request_body_reason=None,
+    custom_interceptors=None,
+    region=None,
+):
+  """Returns an instance of the GAPIC API client specified in the args.
+
+  For apitools API clients, the API endpoint override is something like
+  http://compute.googleapis.com/v1/. For GAPIC API clients, the DEFAULT_ENDPOINT
+  is something like compute.googleapis.com. To use the same endpoint override
+  property for both, we use the netloc of the API endpoint override.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+    api_version: str, The version of the API.
+    credentials: google.auth.credentials.Credentials, the credentials to use.
+    address_override_func: function, function to call to override the client
+      host. It takes a single argument which is the original host.
+    transport_choice: apis_util.GapicTransport, The transport to be used by the
+      client.
+    attempt_direct_path: bool, True if we want to attempt direct path gRPC where
+      possible.
+    redact_request_body_reason: str, the reason why the request body must be
+      redacted if --log-http is used. If None, the body is not redacted.
+    custom_interceptors: list[grpc interceptor], a list of custom interceptors
+    to add to the channel.
+    region: str, Region (or multi-region) for regionalized endpoints (REP).
+
+  Returns:
+    An instance of the specified GAPIC API client.
+  """
+  def AddressOverride(address):
+    try:
+      endpoint_override = properties.VALUES.api_endpoint_overrides.Property(
+          api_name).Get()
+    except properties.NoSuchPropertyError:
+      endpoint_override = None
+
+    if endpoint_override:
+      address = urlparse(endpoint_override).netloc
+
+    if address_override_func:
+      address = address_override_func(address)
+
+    if endpoint_override is not None:
+      return address
+
+    # Note that regionalized endpoints are incompatible with mTLS / non-Google
+    # universe domains.
+    if _ShouldUseRegionalEndpoints(api_name, api_version, region):
+      return _GetRegionalizedEndpoint(address, region)
+
+    return UniversifyAddress(address)
+
+  client_class = _GetGapicClientClass(
+      api_name, api_version, transport_choice=transport_choice)
+
+  return client_class(
+      credentials,
+      address_override_func=AddressOverride,
+      mtls_enabled=_MtlsEnabled(api_name, api_version),
+      attempt_direct_path=attempt_direct_path,
+      redact_request_body_reason=redact_request_body_reason,
+      custom_interceptors=custom_interceptors,
+  )
+
+
+def UniversifyAddress(address):
+  """Update a URL based on the current universe domain."""
+  universe_domain_property = properties.VALUES.core.universe_domain
+  universe_domain = universe_domain_property.Get()
+  if (address is not None and
+      universe_domain_property.default != universe_domain):
+    address = address.replace(universe_domain_property.default,
+                              universe_domain, 1)
+  return address
+
+
+def _GetRegionalizedEndpoint(domain, region):
+  """Returns regionalized domain.
+
+  Regional endpoints (REPs) have the following format:
+    {service}.{region}.rep.googleapis.com
+
+  Args:
+    domain: Domain for the API in form {service}.googleapis.com.
+    region: REP region (or multi-region).
+
+  Returns:
+    str, Regionalized domain.
+  """
+  if region is None:
+    raise ValueError('Region must be provided.')
+  if domain.count('.') > 2:
+    raise ValueError(
+        'Base URLs should have form {service}.googleapis.com;'
+        f' received: [{domain}].')
+  service, base_domain = domain.split('.', 1)
+  return f'{service}.{region}.rep.{base_domain}'
+
+
+class UnsupportedEndpointModeError(exceptions.Error):
+  """Error when regional/endpoint_mode property is unsupported by a command."""
+
+
+def _ShouldUseRegionalEndpoints(api_name, api_version, region):
+  """Returns whether or not regional endpoint should be used.
+
+  Depending on the user-specified regional/endpoint_mode setting and the
+  developer-specified command annotation, this function calculates whether or
+  not to use regional endpoints. In case of an invalid combination, an error
+  will be raised. The full matrix is as follows:
+
+  +--------------+-------------------------------------------------------------+
+  |              |                    Command compatibility                    |
+  |  User mode   +---------------+---------------+-----------------------------+
+  |              | Only supports | Only supports | Supports both               |
+  |              | global        | regional      |                             |
+  +--------------+---------------+---------------+-----------------------------+
+  | regional     | Error         | Regional      | Regional client             |
+  |              |               | client        |                             |
+  +--------------+---------------+---------------+-----------------------------+
+  | global       | Global client | Error         | Global client               |
+  +--------------+---------------+---------------+-----------------------------+
+  | regional-    | Global client | Regional      | If region available:        |
+  | preferred    |               | client        | Regional client             |
+  |              |               |               | Otherwise: Global client    |
+  +--------------+---------------+---------------+-----------------------------+
+  | (unset)      | Global client | Regional      | Global client               |
+  |              |               | client        |                             |
+  +--------------+---------------+---------------+-----------------------------+
+
+  Args:
+    api_name: str, The API name.
+    api_version: str, The version of the API.
+    region: The region (or multi-region).
+  Returns:
+    bool, Whether to use regional endpoints.
+  Raises:
+    UnsupportedEndpointModeError: If the command does not support the given
+      endpoint mode.
+  """
+  mode = properties.VALUES.regional.endpoint_mode.Get()
+  compatibility = properties.VALUES.regional.endpoint_compatibility.Get()
+  if mode == properties.VALUES.regional.REGIONAL and compatibility is None:
+    raise UnsupportedEndpointModeError(
+        f'The regional/endpoint_mode property is currently set to [{mode}], but'
+        ' this command does not support regional endpoints.')
+  if (
+      mode == properties.VALUES.regional.GLOBAL and
+      compatibility == properties.VALUES.regional.REQUIRED
+  ):
+    raise UnsupportedEndpointModeError(
+        f'The regional/endpoint_mode property is currently set to [{mode}], but'
+        ' this command only supports regional endpoints.')
+
+  # Global.
+  if (
+      compatibility is None
+      or mode == properties.VALUES.regional.GLOBAL
+      or (
+          mode is None
+          and compatibility == properties.VALUES.regional.SUPPORTED
+      )
+  ):
+    return False
+
+  # Regional if available.
+  known_available_regions = GetApiDef(api_name, api_version).regional_endpoints
+  if (
+      mode == properties.VALUES.regional.REGIONAL_PREFERRED
+      and compatibility == properties.VALUES.regional.SUPPORTED
+  ):
+    return region in known_available_regions
+
+  # Command requires or user explicitly requested regional endpoints.
+  if region not in known_available_regions:
+    # If the requested region isn't known to be available, it could be because
+    # this version of gcloud is older and doesn't know about it yet. We let the
+    # request through and fail open in this case rather than error out here.
+    log.info(
+        'API [%s] does not contain a known regional endpoint for region [%s].',
+        api_name, region)
+  return True
+
+
+def _GetMtlsEndpoint(api_name, api_version, client_class=None):
+  """Returns mtls endpoint."""
+  api_def = GetApiDef(api_name, api_version)
+  if api_def.apitools:
+    client_class = client_class or _GetClientClass(api_name, api_version)
+  else:
+    client_class = client_class or _GetGapicClientClass(api_name, api_version)
+  return api_def.mtls_endpoint_override or client_class.MTLS_BASE_URL
+
+
+def _MtlsEnabled(api_name, api_version):
+  """Checks if the API of the given version should use mTLS.
+
+  If context_aware/always_use_mtls_endpoint is True, then mTLS will always be
+  used.
+
+  If context_aware/use_client_certificate is True, then mTLS will be used only
+  if the API version is in the mTLS allowlist.
+
+  gcloud maintains a client-side allowlist for the mTLS feature
+  (go/gcloud-rollout-mtls).
+
+  Args:
+    api_name: str, The API name.
+    api_version: str, The version of the API.
+
+  Returns:
+    True if the given service and version is in the mTLS allowlist.
+  """
+  if properties.VALUES.context_aware.always_use_mtls_endpoint.GetBool():
+    return True
+
+  if not properties.VALUES.context_aware.use_client_certificate.GetBool():
+    return False
+
+  api_def = GetApiDef(api_name, api_version)
+  return api_def.enable_mtls
+
+
+def _BuildEndpointOverride(endpoint_override, base_url):
+  """Constructs a normalized endpoint URI depending on the client base_url."""
+  url_base = urlparse(base_url)
+  url_endpoint_override = urlparse(endpoint_override)
+  if url_base.path == '/' or url_endpoint_override.path != '/':
+    return endpoint_override
+  return urljoin(
+      '{}://{}'.format(url_endpoint_override.scheme,
+                       url_endpoint_override.netloc), url_base.path)
+
+
+def _GetBaseUrlFromApi(api_name, api_version):
+  """Returns base url for given api."""
+  if GetApiDef(api_name, api_version).apitools:
+    client_class = _GetClientClass(api_name, api_version)
+  else:
+    client_class = _GetGapicClientClass(api_name, api_version)
+
+  if hasattr(client_class, 'BASE_URL'):
+    client_base_url = client_class.BASE_URL
+  else:
+    try:
+      client_base_url = _GetResourceModule(api_name, api_version).BASE_URL
+    except AttributeError:
+      client_base_url = 'https://{}.googleapis.com/{}'.format(
+          api_name, api_version
+      )
+  return UniversifyAddress(client_base_url)
+
+
+def _GetEffectiveApiEndpoint(
+    api_name, api_version, client_class=None, region=None):
+  """Returns effective endpoint for given api."""
+  try:
+    endpoint_override = properties.VALUES.api_endpoint_overrides.Property(
+        api_name).Get()
+  except properties.NoSuchPropertyError:
+    endpoint_override = None
+
+  api_def = GetApiDef(api_name, api_version)
+  if api_def.apitools:
+    client_class = client_class or _GetClientClass(api_name, api_version)
+  else:
+    client_class = client_class or _GetGapicClientClass(api_name, api_version)
+  client_base_url = _GetBaseUrlFromApi(api_name, api_version)
+  if endpoint_override:
+    address = _BuildEndpointOverride(endpoint_override, client_base_url)
+  elif _MtlsEnabled(api_name, api_version):
+    address = UniversifyAddress(
+        _GetMtlsEndpoint(api_name, api_version, client_class)
+    )
+  # Note that regionalized endpoints are incompatible with mTLS / non-Google
+  # universe domains.
+  elif _ShouldUseRegionalEndpoints(api_name, api_version, region):
+    url_base = urlparse(client_base_url)
+    regionalized_domain = _GetRegionalizedEndpoint(url_base.netloc, region)
+    address = urlunparse(url_base._replace(netloc=regionalized_domain))
+  else:
+    address = client_base_url
+
+  return address
+
+
+def _GetMessagesModule(api_name, api_version):
+  """Returns the messages module for the API specified in the args.
+
+  Args:
+    api_name: str, The API name (or the command surface name, if different).
+    api_version: str, The version of the API.
+
+  Returns:
+    Module containing the definitions of messages for the specified API.
+  """
+  api_def = GetApiDef(api_name, api_version)
+  # fromlist below must not be empty, see:
+  # http://stackoverflow.com/questions/2724260/why-does-pythons-import-require-fromlist.
+  return __import__(
+      api_def.apitools.messages_full_modulepath, fromlist=['something'])
+
+
+def _GetResourceModule(api_name, api_version):
+  """Imports and returns given api resources module."""
+
+  api_def = GetApiDef(api_name, api_version)
+  # fromlist below must not be empty, see:
+  # http://stackoverflow.com/questions/2724260/why-does-pythons-import-require-fromlist.
+  if api_def.apitools:
+    return __import__(
+        api_def.apitools.class_path + '.' + 'resources', fromlist=['something']
+    )
+  # ApiDef must be gapic-only:
+  return __import__(
+      api_def.gapic.class_path + '.' + 'resources', fromlist=['something']
+  )
+
+
+def _GetApiCollections(api_name, api_version):
+  """Yields all collections for for given api."""
+
+  try:
+    resources_module = _GetResourceModule(api_name, api_version)
+  except ImportError:
+    pass
+  else:
+    for collection in resources_module.Collections:
+      yield resource_util.CollectionInfo(
+          api_name,
+          api_version,
+          resources_module.BASE_URL,
+          resources_module.DOCS_URL,
+          collection.collection_name,
+          collection.path,
+          collection.flat_paths,
+          collection.params,
+          collection.enable_uri_parsing,
+      )

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/util/apis_util.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Some utilities intended for use around apis."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import enum
+
+from googlecloudsdk.core import exceptions
+
+
+class UnknownAPIError(exceptions.Error):
+  """Unable to find API in APIs map."""
+
+  def __init__(self, api_name):
+    super(UnknownAPIError, self).__init__(
+        'API named [{0}] does not exist in the APIs map'.format(api_name))
+
+
+class UnknownVersionError(exceptions.Error):
+  """Unable to find API version in APIs map."""
+
+  def __init__(self, api_name, api_version):
+    super(UnknownVersionError, self).__init__(
+        'The [{0}] API does not have version [{1}] in the APIs map'.format(
+            api_name, api_version))
+
+
+class GapicTransport(enum.Enum):
+  """Enum options for Gapic Clients."""
+  GRPC = 1
+  GRPC_ASYNCIO = 2
+  REST = 3
+
+
+# This is the map of API name aliases to actual API names.
+# Do not add to this map unless the api definition uses different names for api
+# name, endpoint and/or collection names.
+# The apis_map keys are aliases and values are actual API names.
+# The rest of the Cloud SDK, including
+# property sections, and command surfaces should use the API name alias.
+
+# The general rule for this module is: all apis_map lookups should use the real
+# API name, and all property lookups should use the alias. Any api_name argument
+# expects to receive the name alias (if one exists). The _GetApiNameAndAlias
+# helper method can be used to convert it into a (name, alias) tuple.
+_API_NAME_ALIASES = {
+    'sql': 'sqladmin',
+    'transfer': 'storagetransfer',
+}

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/util/common_args.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2020 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Helpers for parsing common arguments."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+
+def ParseSortByArg(sort_by=None):
+  """Parses and creates the sort by object from parsed arguments.
+
+  Args:
+    sort_by: list of strings, passed in from the --sort-by flag.
+
+  Returns:
+    A parsed sort by string ending in asc or desc, conforming to
+    https://aip.dev/132#ordering
+  """
+  if not sort_by:
+    return None
+
+  fields = []
+  for field in sort_by:
+    if field.startswith('~'):
+      field = field.lstrip('~') + ' desc'
+    else:
+      field += ' asc'
+    fields.append(field)
+  return ','.join(fields)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/util/exceptions.py

@@ -0,0 +1,689 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2016 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A module that converts API exceptions to core exceptions."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import collections
+import io
+import json
+import logging
+import string
+import urllib.parse
+
+from apitools.base.py import exceptions as apitools_exceptions
+from googlecloudsdk.api_lib.util import resource as resource_util
+from googlecloudsdk.core import exceptions as core_exceptions
+from googlecloudsdk.core import log
+from googlecloudsdk.core import properties
+from googlecloudsdk.core.resource import resource_lex
+from googlecloudsdk.core.resource import resource_printer
+from googlecloudsdk.core.resource import resource_property
+from googlecloudsdk.core.util import encoding
+import six
+
+
+# Some formatter characters are special inside {...}. The _Escape / _Expand pair
+# escapes special chars inside {...} and ignores them outside.
+_ESCAPE = '~'  # '\x01'
+_ESCAPED_COLON = 'C'
+_ESCAPED_ESCAPE = 'E'
+_ESCAPED_LEFT_CURLY = 'L'
+_ESCAPED_RIGHT_CURLY = 'R'
+
+# ErrorInfo identifier used for extracting domain based handlers.
+ERROR_INFO_SUFFIX = 'google.rpc.ErrorInfo'
+LOCALIZED_MESSAGE_SUFFIX = 'google.rpc.LocalizedMessage'
+HELP_SUFFIX = 'google.rpc.Help'
+
+
+def _Escape(s):
+  """Return s with format special characters escaped."""
+  r = []
+  n = 0
+  for c in s:
+    if c == _ESCAPE:
+      r.append(_ESCAPE + _ESCAPED_ESCAPE + _ESCAPE)
+    elif c == ':':
+      r.append(_ESCAPE + _ESCAPED_COLON + _ESCAPE)
+    elif c == '{':
+      if n > 0:
+        r.append(_ESCAPE + _ESCAPED_LEFT_CURLY + _ESCAPE)
+      else:
+        r.append('{')
+      n += 1
+    elif c == '}':
+      n -= 1
+      if n > 0:
+        r.append(_ESCAPE + _ESCAPED_RIGHT_CURLY + _ESCAPE)
+      else:
+        r.append('}')
+    else:
+      r.append(c)
+  return ''.join(r)
+
+
+def _Expand(s):
+  """Return s with escaped format special characters expanded."""
+  r = []
+  n = 0
+  i = 0
+  while i < len(s):
+    c = s[i]
+    i += 1
+    if c == _ESCAPE and i + 1 < len(s) and s[i + 1] == _ESCAPE:
+      c = s[i]
+      i += 2
+      if c == _ESCAPED_LEFT_CURLY:
+        if n > 0:
+          r.append(_ESCAPE + _ESCAPED_LEFT_CURLY)
+        else:
+          r.append('{')
+        n += 1
+      elif c == _ESCAPED_RIGHT_CURLY:
+        n -= 1
+        if n > 0:
+          r.append(_ESCAPE + _ESCAPED_RIGHT_CURLY)
+        else:
+          r.append('}')
+      elif n > 0:
+        r.append(s[i - 3:i])
+      elif c == _ESCAPED_COLON:
+        r.append(':')
+      elif c == _ESCAPED_ESCAPE:
+        r.append(_ESCAPE)
+    else:
+      r.append(c)
+  return ''.join(r)
+
+
+class _JsonSortedDict(dict):
+  """A dict with a sorted JSON string representation."""
+
+  def __str__(self):
+    return json.dumps(self, sort_keys=True)
+
+
+class FormattableErrorPayload(string.Formatter):
+  """Generic payload for an HTTP error that supports format strings.
+
+  Attributes:
+    content: The dumped JSON content.
+    message: The human readable error message.
+    status_code: The HTTP status code number.
+    status_description: The status_code description.
+    status_message: Context specific status message.
+  """
+
+  def __init__(self, http_error):
+    """Initialize a FormattableErrorPayload instance.
+
+    Args:
+      http_error: An Exception that subclasses can use to populate class
+        attributes, or a string to use as the error message.
+    """
+    super(FormattableErrorPayload, self).__init__()
+    self._value = '{?}'
+    self.content = {}
+    self.status_code = 0
+    self.status_description = ''
+    self.status_message = ''
+    if isinstance(http_error, six.string_types):
+      self.message = http_error
+    else:
+      self.message = self._MakeGenericMessage()
+
+  def get_field(self, field_name, unused_args, unused_kwargs):
+    r"""Returns the value of field_name for string.Formatter.format().
+
+    Args:
+      field_name: The format string field name to get in the form
+        name - the value of name in the payload, '' if undefined
+        name?FORMAT - if name is non-empty then re-formats with FORMAT, where
+          {?} is the value of name. For example, if name=NAME then
+          {name?\nname is "{?}".} expands to '\nname is "NAME".'.
+        .a.b.c - the value of a.b.c in the JSON decoded payload contents.
+          For example, '{.errors.reason?[{?}]}' expands to [REASON] if
+          .errors.reason is defined.
+      unused_args: Ignored.
+      unused_kwargs: Ignored.
+
+    Returns:
+      The value of field_name for string.Formatter.format().
+    """
+    field_name = _Expand(field_name)
+    if field_name == '?':
+      return self._value, field_name
+    parts = field_name.split('?', 1)
+    subparts = parts.pop(0).split(':', 1)
+    name = subparts.pop(0)
+    # Remove the leading 0. from the name if it exists to keep formatter in sync
+    # with resoruce lexers expecations. Needed for python 3.14 compatibility.
+    if (name.startswith('0.') and len(name) > 1) or name == '0.':
+      name = name[1:]
+    printer_format = subparts.pop(0) if subparts else None
+    recursive_format = parts.pop(0) if parts else None
+    name, value = self._GetField(name)
+    if not value and not isinstance(value, (int, float)):
+      return '', name
+    if printer_format or not isinstance(
+        value, (six.text_type, six.binary_type, float) + six.integer_types):
+      buf = io.StringIO()
+      resource_printer.Print(
+          value, printer_format or 'default', out=buf, single=True)
+      value = buf.getvalue().strip()
+    if recursive_format:
+      self._value = value
+      value = self.format(_Expand(recursive_format))
+    return value, name
+
+  def _GetField(self, name):
+    """Gets the value corresponding to name in self.content or class attributes.
+
+    If `name` starts with a period, treat it as a key in self.content and get
+    the corresponding value. Otherwise get the value of the class attribute
+    named `name` first and fall back to checking keys in self.content.
+
+    Args:
+      name (str): The name of the attribute to return the value of.
+
+    Returns:
+      A tuple where the first value is `name` with any leading periods dropped,
+      and the second value is the value of a class attribute or key in
+      self.content.
+    """
+    if '.' in name:
+      if name.startswith('.'):
+        # Only check self.content.
+        check_payload_attributes = False
+        name = name[1:]
+      else:
+        # Check the payload attributes first, then self.content.
+        check_payload_attributes = True
+      key = resource_lex.Lexer(name).Key()
+      content = self.content
+      if check_payload_attributes and key:
+        value = self.__dict__.get(key[0], None)
+        if value:
+          content = {key[0]: value}
+      value = resource_property.Get(content, key, None)
+    elif name:
+      value = self.__dict__.get(name, None)
+    else:
+      value = None
+
+    return name, value
+
+  def _MakeGenericMessage(self):
+    """Makes a generic human readable message from the HttpError."""
+    description = self._MakeDescription()
+    if self.status_message:
+      return '{0}: {1}'.format(description, self.status_message)
+    return description
+
+  def _MakeDescription(self):
+    """Makes description for error by checking which fields are filled in."""
+    description = self.status_description
+    if description:
+      if description.endswith('.'):
+        description = description[:-1]
+      return description
+    # Example: 'HTTPError 403'
+    return 'HTTPError {0}'.format(self.status_code)
+
+
+class HttpErrorPayload(FormattableErrorPayload):
+  r"""Converts apitools HttpError payload to an object.
+
+  Attributes:
+    api_name: The url api name.
+    api_version: The url version.
+    content: The dumped JSON content.
+    details: A list of {'@type': TYPE, 'detail': STRING} typed details.
+    domain_details: ErrorInfo metadata Indexed by domain.
+    violations: map of subject to error message for that subject.
+    field_violations: map of field name to error message for that field.
+    error_info: content['error'].
+    instance_name: The url instance name.
+    message: The human readable error message.
+    resource_item: The resource type.
+    resource_name: The url resource name.
+    resource_version: The resource version.
+    status_code: The HTTP status code number.
+    status_description: The status_code description.
+    status_message: Context specific status message.
+    unparsed_details: The unparsed details.
+    type_details: ErrorDetails Indexed by type.
+    url: The HTTP url. .<a>.<b>...: The <a>.<b>... attribute in the JSON content
+      (synthesized in get_field()).
+
+  Grammar:
+    Format strings inherit from python's string.formatter. where we pass tokens
+    obtained by the resource projection framework format strings.
+
+  Examples:
+    error_format values and resulting output:
+
+    'Error: [{status_code}] {status_message}{url?\n{?}}{.debugInfo?\n{?}}'
+
+      Error: [404] Not found
+      http://dotcom/foo/bar
+      <content.debugInfo in yaml print format>
+
+    'Error: {status_code} {details?\n\ndetails:\n{?}}'
+
+      Error: 404
+
+      details:
+      - foo
+      - bar
+
+     'Error [{status_code}] {status_message}\n'
+     '{.:value(details.detail.list(separator="\n"))}'
+
+       Error [400] Invalid request.
+       foo
+       bar
+  """
+
+  def __init__(self, http_error):
+    super(HttpErrorPayload, self).__init__(http_error)
+    self.api_name = ''
+    self.api_version = ''
+    self.details = []
+    self.violations = {}
+    self.field_violations = {}
+    self.error_info = None
+    self.instance_name = ''
+    self.resource_item = ''
+    self.resource_name = ''
+    self.resource_version = ''
+    self.url = ''
+    self._cred_info = None
+    if not isinstance(http_error, six.string_types):
+      self._ExtractResponseAndJsonContent(http_error)
+      self._ExtractUrlResourceAndInstanceNames(http_error)
+      self.message = self._MakeGenericMessage()
+
+  def _GetField(self, name):
+    if name.startswith('field_violations.'):
+      _, field = name.split('.', 1)
+      value = self.field_violations.get(field)
+    elif name.startswith('violations.'):
+      _, subject = name.split('.', 1)
+      value = self.violations.get(subject)
+    else:
+      name, value = super(HttpErrorPayload, self)._GetField(name)
+    return name, value
+
+  def _GetMTLSEndpointOverride(self, endpoint_override):
+    """Generates mTLS endpoint override for a given endpoint override.
+
+    Args:
+      endpoint_override: The endpoint to generate the mTLS endpoint override
+        for.
+
+    Returns:
+      The mTLS endpoint override, if one exists. Otherwise, None.
+    """
+    if 'sandbox.googleapis.com' in endpoint_override:
+      return endpoint_override.replace(
+          'sandbox.googleapis.com', 'mtls.sandbox.googleapis.com'
+      )
+    elif 'googleapis.com' in endpoint_override:
+      return endpoint_override.replace('googleapis.com', 'mtls.googleapis.com')
+    else:
+      return None
+
+  def _IsMTLSEnabledAndApiEndpointOverridesPresent(self):
+    """Returns whether mTLS is enabled and an endpoint override is present for the current gcloud invocation."""
+    return (
+        properties.VALUES.api_endpoint_overrides.AllValues()
+        and properties.VALUES.context_aware.use_client_certificate.GetBool()
+        and properties.IsInternalUserCheck()
+    )
+
+  def _IsExistingOverrideMTLS(self, endpoint_override):
+    """Returns whether the existing endpoint override is using mTLS already."""
+    urlparsed = urllib.parse.urlparse(endpoint_override)
+    domain_parts = urlparsed.netloc
+    if not domain_parts:
+      return None
+    domain = domain_parts.split('.')[1]
+    return domain.startswith('mtls')
+
+  def _ExtractResponseAndJsonContent(self, http_error):
+    """Extracts the response and JSON content from the HttpError."""
+    response = getattr(http_error, 'response', None)
+    if response:
+      self.status_code = int(response.get('status', 0))
+      self.status_description = encoding.Decode(response.get('reason', ''))
+    content = encoding.Decode(http_error.content)
+    try:
+      # X-GOOG-API-FORMAT-VERSION: 2
+      self.content = _JsonSortedDict(json.loads(content))
+      self.error_info = _JsonSortedDict(self.content['error'])
+      if not self.status_code:  # Could have been set above.
+        self.status_code = int(self.error_info.get('code', 0))
+
+      if self.status_code in [401, 403, 404] and self.error_info.get(
+          'message', ''
+      ):
+        from googlecloudsdk.core.credentials import store as c_store  # pylint: disable=g-import-not-at-top
+
+        # Append the credential info to the error message.
+        self._cred_info = c_store.CredentialInfo.GetCredentialInfo()
+        if self._cred_info:
+          cred_info_message = self._cred_info.GetInfoString()
+          existing_message = self.content['error']['message']
+          # Some surface actually appends an ending dot at the end of the
+          # message, so we need to make sure not adding an ending dot if the
+          # existing message doesn't have one. (Note that cred_info_message
+          # string we created ends with a dot.)
+          if existing_message[-1] != '.':
+            # add dot after existing_message and remove the ending dot from
+            # cred_info_message.
+            self.content['error']['message'] = (
+                existing_message + '. ' + cred_info_message[:-1]
+            )
+          else:
+            self.content['error']['message'] = (
+                existing_message + ' ' + cred_info_message
+            )
+          self.error_info['message'] = self.content['error']['message']
+
+      if not self.status_description:  # Could have been set above.
+        self.status_description = self.error_info.get('status', '')
+      if self._IsMTLSEnabledAndApiEndpointOverridesPresent():
+        endpoint_overrides = (
+            properties.VALUES.api_endpoint_overrides.AllValues()
+        )
+        for api_name, endpoint_override in endpoint_overrides.items():
+          mtls_endpoint = self._GetMTLSEndpointOverride(endpoint_override)
+          if (
+              not self._IsExistingOverrideMTLS(endpoint_override)
+              and http_error.url.startswith(endpoint_override)
+              and mtls_endpoint
+          ):
+            self.error_info['message'] = (
+                'Certificate-based access is enabled, but the endpoint override'
+                ' for the following API is not using mTLS: {}. Please update'
+                ' the endpoint override to use mTLS endpoint: {} '.format(
+                    [api_name, endpoint_override], mtls_endpoint
+                )
+            )
+            self.error_info['details'] = [{
+                '@type': 'type.googleapis.com/google.rpc.ErrorInfo',
+                'reason': 'ACCESS_DENIED',
+                'metadata': {
+                    'endpoint_override': endpoint_override,
+                    'mtls_endpoint_override': mtls_endpoint,
+                },
+            }]
+            break
+      self.status_message = self.error_info.get('message', '')
+      self.details = self.error_info.get('details', [])
+      self.violations = self._ExtractViolations(self.details)
+      self.field_violations = self._ExtractFieldViolations(self.details)
+      self.type_details = self._IndexErrorDetailsByType(self.details)
+      self.domain_details = self._IndexErrorInfoByDomain(self.details)
+      if properties.VALUES.core.parse_error_details.GetBool():
+        self.unparsed_details = self.RedactParsedTypes(self.details)
+    except (KeyError, TypeError, ValueError):
+      self.status_message = content
+    except AttributeError:
+      pass
+
+  def RedactParsedTypes(self, details):
+    """Redacts the parsed types from the details list."""
+    unparsed_details = []
+    for item in details:
+      error_type = item.get('@type', None)
+      error_suffix = error_type.split('/')[-1]
+      if error_suffix not in (LOCALIZED_MESSAGE_SUFFIX, HELP_SUFFIX):
+        unparsed_details.append(item)
+    return unparsed_details
+
+  def _IndexErrorDetailsByType(self, details):
+    """Extracts and indexes error details list by the type attribute."""
+    type_map = collections.defaultdict(list)
+    for item in details:
+      error_type = item.get('@type', None)
+      if error_type:
+        error_type_suffix = error_type.split('.')[-1]
+        type_map[error_type_suffix].append(item)
+    return type_map
+
+  def _IndexErrorInfoByDomain(self, details):
+    """Extracts and indexes error info list by the domain attribute."""
+    domain_map = collections.defaultdict(list)
+    for item in details:
+      error_type = item.get('@type', None)
+      if error_type.endswith(ERROR_INFO_SUFFIX):
+        domain = item.get('domain', None)
+        if domain:
+          domain_map[domain].append(item)
+    return domain_map
+
+  def _ExtractUrlResourceAndInstanceNames(self, http_error):
+    """Extracts the url resource type and instance names from the HttpError."""
+    self.url = http_error.url
+    if not self.url:
+      return
+
+    try:
+      name, version, resource_path = resource_util.SplitEndpointUrl(
+          self.url)
+    except resource_util.InvalidEndpointException:
+      return
+
+    if name:
+      self.api_name = name
+    if version:
+      self.api_version = version
+
+    # We do not attempt to parse this, as generally it doesn't represent a
+    # resource uri.
+    resource_parts = resource_path.split('/')
+    if not 1 < len(resource_parts) < 4:
+      return
+    self.resource_name = resource_parts[0]
+    instance_name = resource_parts[1]
+
+    self.instance_name = instance_name.split('?')[0]
+    self.resource_item = '{} instance'.format(self.resource_name)
+
+  def _MakeDescription(self):
+    """Makes description for error by checking which fields are filled in."""
+    if self.status_code and self.resource_item and self.instance_name:
+      if self.status_code == 403:
+        if self._cred_info:
+          account = (
+              self._cred_info.impersonated_account or self._cred_info.account
+          )
+        else:
+          account = properties.VALUES.core.account.Get()
+        return (
+            '[{0}] does not have permission to access {1} [{2}] (or '
+            'it may not exist)'
+        ).format(account, self.resource_item, self.instance_name)
+      if self.status_code == 404:
+        return '{0} [{1}] not found'.format(
+            self.resource_item.capitalize(), self.instance_name)
+      if self.status_code == 409:
+        if self.resource_name == 'projects':
+          return ('Resource in projects [{0}] '
+                  'is the subject of a conflict').format(self.instance_name)
+        else:
+          return '{0} [{1}] is the subject of a conflict'.format(
+              self.resource_item.capitalize(), self.instance_name)
+
+    return super(HttpErrorPayload, self)._MakeDescription()
+
+  def _ExtractViolations(self, details):
+    """Extracts a map of violations from the given error's details.
+
+    Args:
+      details: JSON-parsed details field from parsed json of error.
+
+    Returns:
+      Map[str, str] sub -> error description. The iterator of it is ordered by
+      the order the subjects first appear in the errror.
+    """
+    results = collections.OrderedDict()
+    for detail in details:
+      if 'violations' not in detail:
+        continue
+      violations = detail['violations']
+      if not isinstance(violations, list):
+        continue
+      sub = detail.get('subject')
+      for violation in violations:
+        try:
+          local_sub = violation.get('subject')
+          subject = sub or local_sub
+          if subject:
+            if subject in results:
+              results[subject] += '\n' + violation['description']
+            else:
+              results[subject] = violation['description']
+        except (KeyError, TypeError):
+          # If violation or description are the wrong type or don't exist.
+          pass
+    return results
+
+  def _ExtractFieldViolations(self, details):
+    """Extracts a map of field violations from the given error's details.
+
+    Args:
+      details: JSON-parsed details field from parsed json of error.
+
+    Returns:
+      Map[str, str] field (in dotted format) -> error description.
+      The iterator of it is ordered by the order the fields first
+      appear in the error.
+    """
+    results = collections.OrderedDict()
+    for deet in details:
+      if 'fieldViolations' not in deet:
+        continue
+      violations = deet['fieldViolations']
+      if not isinstance(violations, list):
+        continue
+      f = deet.get('field')
+      for viol in violations:
+        try:
+          local_f = viol.get('field')
+          field = f or local_f
+          if field:
+            if field in results:
+              results[field] += '\n' + viol['description']
+            else:
+              results[field] = viol['description']
+        except (KeyError, TypeError):
+          # If violation or description are the wrong type or don't exist.
+          pass
+    return results
+
+
+class HttpException(core_exceptions.Error):
+  """Transforms apitools HttpError to api_lib HttpException.
+
+  Attributes:
+    error: The original HttpError.
+    error_format: An HttpErrorPayload format string.
+    payload: The HttpErrorPayload object.
+  """
+
+  def __init__(self, error, error_format=None, payload_class=HttpErrorPayload):
+    super(HttpException, self).__init__('')
+    self.error = error
+    self.error_format = error_format
+    self.payload = payload_class(error)
+
+  def __str__(self):
+    error_format = self.error_format
+    if error_format is None:
+      error_prefix = '{message?}'
+      if properties.VALUES.core.parse_error_details.GetBool():
+        parsed_localized_messages = (
+            '{type_details.LocalizedMessage'
+            ':value(message.list(separator="\n"))?\n{?}}'
+        )
+        parsed_help_messages = (
+            '{type_details.Help'
+            ':value(links.flatten(show="values",separator="\n"))?\n{?}}'
+        )
+        unparsed_details = '{unparsed_details?\n{?}}'
+        error_format = (
+            error_prefix
+            + parsed_localized_messages
+            + parsed_help_messages
+            + unparsed_details
+        )
+      else:
+        error_format = error_prefix + '{details?\n{?}}'
+      if log.GetVerbosity() <= logging.DEBUG:
+        error_format += '{.debugInfo?\n{?}}'
+    return _Expand(self.payload.format(_Escape(error_format)))
+
+  @property
+  def message(self):
+    return six.text_type(self)
+
+  def __hash__(self):
+    return hash(self.message)
+
+  def __eq__(self, other):
+    if isinstance(other, HttpException):
+      return self.message == other.message
+    return False
+
+
+def CatchHTTPErrorRaiseHTTPException(format_str=None):
+  """Decorator that catches an HttpError and returns a custom error message.
+
+  It catches the raw Http Error and runs it through the given format string to
+  get the desired message.
+
+  Args:
+    format_str: An HttpErrorPayload format string. Note that any properties that
+    are accessed here are on the HTTPErrorPayload object, and not the raw
+    object returned from the server.
+
+  Returns:
+    A custom error message.
+
+  Example:
+    @CatchHTTPErrorRaiseHTTPException('Error [{status_code}]')
+    def some_func_that_might_throw_an_error():
+      ...
+  """
+
+  def CatchHTTPErrorRaiseHTTPExceptionDecorator(run_func):
+    # Need to define a secondary wrapper to get an argument to the outer
+    # decorator.
+    def Wrapper(*args, **kwargs):
+      try:
+        return run_func(*args, **kwargs)
+      except apitools_exceptions.HttpError as error:
+        exc = HttpException(error, format_str)
+        core_exceptions.reraise(exc)
+    return Wrapper
+
+  return CatchHTTPErrorRaiseHTTPExceptionDecorator

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/util/http_retry.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Retry logic for HTTP exceptions."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from apitools.base.py import exceptions
+
+from googlecloudsdk.core.util import retry
+
+
+def RetryOnHttpStatus(status):
+  """Decorator factory to automatically retry a function for HTTP errors."""
+
+  def RetryOnHttpStatusAttribute(func):
+    """Decorator to automatically retry a function for HTTP errors."""
+    # pylint:disable=invalid-name
+    def retryIf(exc_type, exc_value, unused_traceback, unused_state):
+      return (exc_type == exceptions.HttpError and
+              exc_value.status_code == status)
+
+    # pylint:disable=invalid-name
+    def wrapper(*args, **kwargs):
+      retryer = retry.Retryer(max_retrials=3, exponential_sleep_multiplier=2,
+                              jitter_ms=100)
+      return retryer.RetryOnException(func, args, kwargs,
+                                      should_retry_if=retryIf, sleep_ms=1000)
+    return wrapper
+  return RetryOnHttpStatusAttribute

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/util/messages.py

@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for interacting with message classes and instances."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from apitools.base.protorpclite import messages as _messages
+from apitools.base.py import encoding as _encoding
+from googlecloudsdk.core import exceptions
+
+import six
+
+
+def UpdateMessage(message, diff):
+  """Updates given message from diff object recursively.
+
+  The function recurses down through the properties of the diff object,
+  checking, for each key in the diff, if the equivalent property exists on the
+  message at the same depth. If the property exists, it is set to value from the
+  diff. If it does not exist, that diff key is silently ignored. All diff keys
+  are assumed to be strings.
+
+  Args:
+    message: An apitools.base.protorpclite.messages.Message instance.
+    diff: A dict of changes to apply to the message
+      e.g. {'settings': {'availabilityType': 'REGIONAL'}}.
+
+  Returns:
+    The modified message instance.
+  """
+  if diff:
+    return _UpdateMessageHelper(message, diff)
+  return message
+
+
+def _UpdateMessageHelper(message, diff):
+  for key, val in six.iteritems(diff):
+    if hasattr(message, key):
+      if isinstance(val, dict):
+        _UpdateMessageHelper(getattr(message, key), diff[key])
+      else:
+        setattr(message, key, val)
+  return message
+
+
+class Error(exceptions.Error):
+  """Indicates an error with an encoded protorpclite message."""
+
+
+class DecodeError(Error):
+  """Indicates an error in decoding a protorpclite message."""
+
+  @classmethod
+  def _FormatProtoPath(cls, edges, field_names):
+    """Returns a string representation of a path to a proto field.
+
+    The return value represents one or more fields in a python dictionary
+    representation of a message (json/yaml) that could not be decoded into the
+    message as a string. The format is a dot separated list of python like sub
+    field references (name, name[index], name[name]). The final element of the
+    returned dot separated path may be a comma separated list of names enclosed
+    in curly braces to represent multiple subfields (see examples)
+
+    Examples:
+      o Reference to a single field that could not be decoded:
+        'a.b[1].c[x].d'
+
+      o Reference to two subfields
+        'a.b[1].c[x].{d,e}'
+
+    Args:
+      edges: List of objects representing python field references
+             (__str__ suitably defined.)
+      field_names: List of names for subfields of the message
+         that could not be decoded.
+
+    Returns:
+      A string representation of a python reference to a filed or
+      fields in a message that could not be decoded as described
+      above.
+    """
+    # Format the edges.
+    path = [six.text_type(edge) for edge in edges]
+
+    if len(field_names) > 1:
+      # Use braces to group the errors when there are multiple errors.
+      # e.g. foo.bar.{x,y,z}
+      path.append('{{{}}}'.format(','.join(sorted(field_names))))
+    elif field_names:
+      # For single items, omit the braces.
+      # e.g. foo.bar.x
+      path.append(field_names[0])
+
+    return '.'.join(path)
+
+  @classmethod
+  def FromErrorPaths(cls, message, errors):
+    """Returns a DecodeError from a list of locations of errors.
+
+    Args:
+      message: The protorpc Message in which a parsing error occurred.
+      errors: List[(edges, field_names)], A list of locations of errors
+          encountered while decoding the message.
+    """
+    type_ = type(message).__name__
+    base_msg = 'Failed to parse value(s) in protobuf [{type_}]:'.format(
+        type_=type_)
+    error_paths = ['  {type_}.{path}'.format(
+        type_=type_, path=cls._FormatProtoPath(edges, field_names))
+                   for edges, field_names in errors]
+    return cls('\n'.join([base_msg] + error_paths))
+
+
+class ScalarTypeMismatchError(DecodeError):
+  """Incicates a scalar property was provided a value of an unexpected type."""
+
+
+def DictToMessageWithErrorCheck(dict_,
+                                message_type,
+                                throw_on_unexpected_fields=True):
+  """Convert "dict_" to a message of type message_type and check for errors.
+
+  A common use case is to define the dictionary by deserializing yaml or json.
+
+  Args:
+    dict_: The dict to parse into a protorpc Message.
+    message_type: The protorpc Message type.
+    throw_on_unexpected_fields: If this flag is set, an error will be raised if
+    the dictionary contains unrecognized fields.
+
+  Returns:
+    A message of type "message_type" parsed from "dict_".
+
+  Raises:
+    DecodeError: One or more unparsable values were found in the parsed message.
+  """
+  try:
+    message = _encoding.DictToMessage(dict_, message_type)
+  except _messages.ValidationError as e:
+    # NOTE: The ValidationError message is passable but does not specify the
+    # full path to the property where the error occurred.
+    raise ScalarTypeMismatchError(
+        'Failed to parse value in protobuf [{type_}]:\n'
+        '  {type_}.??: "{msg}"'.format(
+            type_=message_type.__name__, msg=six.text_type(e)))
+  except AttributeError:
+    # TODO(b/77547931): This is a bug in apitools and must be fixed upstream.
+    # The decode logic attempts an unchecked access to 'iteritems' assuming the
+    # Message field's associated value is a dict.
+    raise
+  else:
+    errors = list(_encoding.UnrecognizedFieldIter(message))
+    if errors and throw_on_unexpected_fields:
+      raise DecodeError.FromErrorPaths(message, errors)
+
+    return message

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/util/requests.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for making requests using a given client and handling errors.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import io
+
+from googlecloudsdk.core.resource import resource_printer
+
+
+def ExtractErrorMessage(error_details):
+  """Extracts error details from an apitools_exceptions.HttpError.
+
+  Args:
+    error_details: a python dictionary returned from decoding an error that
+        was serialized to json.
+
+  Returns:
+    Multiline string containing a detailed error message suitable to show to a
+    user.
+  """
+  error_message = io.StringIO()
+  error_message.write('Error Response: [{code}] {message}'.format(
+      code=error_details.get('code', 'UNKNOWN'),  # error_details.code is an int
+      message=error_details.get('message', '')))
+
+  if 'url' in error_details:
+    error_message.write('\n' + error_details['url'])
+
+  if 'details' in error_details:
+    error_message.write('\n\nDetails: ')
+    resource_printer.Print(
+        resources=[error_details['details']],
+        print_format='json',
+        out=error_message)
+  return error_message.getvalue()
+
+

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/util/resource.py

@@ -0,0 +1,285 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2019 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for cloud resources."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import re
+
+from googlecloudsdk.core import exceptions
+
+
+class CollectionInfo(object):
+  """Holds information about a resource collection.
+
+  Attributes:
+      api_name: str, name of the api of resources parsed by this parser.
+      api_version: str, version id for this api.
+      path: str, Atomic URI template for this resource.
+      flat_paths: {name->path}, Named detailed URI templates for this resource.
+        If there is an entry ''->path it replaces path and corresponding param
+        attributes for resources parsing. path and params are not used in this
+        case. Also note that key in this dictionary is referred as
+        subcollection, as it extends 'name' attribute.
+      params: list(str), description of parameters in the path.
+      name: str, collection name for this resource without leading api_name.
+      base_url: str, URL for service providing these resources.
+      docs_url: str, URL to the API reference docs for this API.
+      enable_uri_parsing: bool, whether to register a parser to build up a
+        search tree to match URLs against URL templates.
+  """
+
+  def __init__(self,
+               api_name,
+               api_version,
+               base_url,
+               docs_url,
+               name,
+               path,
+               flat_paths,
+               params,
+               enable_uri_parsing=True):
+    self.api_name = api_name
+    self.api_version = api_version
+    self.base_url = base_url
+    self.docs_url = docs_url
+    self.name = name
+    self.path = path
+    self.flat_paths = flat_paths
+    self.params = params
+    self.enable_uri_parsing = enable_uri_parsing
+
+  @property
+  def full_name(self):
+    return self.api_name + '.' + self.name
+
+  def GetSubcollection(self, collection_name):
+    name = self.full_name
+    # collection_name could be equal to name in which case subcollection is
+    # empty string or have additional suffix .subcollection.
+    if collection_name.startswith(name):
+      return collection_name[len(name) + 1:]
+    raise KeyError('{0} does not exist in {1}'.format(collection_name, name))
+
+  def GetPathRegEx(self, subcollection):
+    """Returns regex for matching path template."""
+    path = self.GetPath(subcollection)
+    parts = []
+    prev_end = 0
+    for match in re.finditer('({[^}]+}/)|({[^}]+})$', path):
+      parts.append(path[prev_end:match.start()])
+      parts.append('([^/]+)')
+      if match.group(1):
+        parts.append('/')
+      prev_end = match.end()
+    if prev_end == len(path):
+      parts[-1] = '(.*)$'
+    return ''.join(parts)
+
+  def GetParams(self, subcollection):
+    """Returns ordered list of parameters for given subcollection.
+
+    Args:
+      subcollection: str, key name for flat_paths. If self.flat_paths is empty
+        use '' (or any other falsy value) for subcollection to get default path
+        parameters.
+
+    Returns:
+      Paramaters present in specified subcollection path.
+    Raises:
+      KeyError if given subcollection does not exists.
+    """
+    # If default collection requested and we are not using custom paths.
+    if not subcollection and not self.flat_paths:
+      return self.params
+    return GetParamsFromPath(self.flat_paths[subcollection])
+
+  def GetPath(self, subcollection):
+    """Returns uri template path for given subcollection."""
+    # If default collection requested and we are not using custom paths.
+    if not subcollection and not self.flat_paths:
+      return self.path
+    return self.flat_paths[subcollection]
+
+  def __eq__(self, other):
+    return (self.api_name == other.api_name and
+            self.api_version == other.api_version and self.name == other.name)
+
+  def __ne__(self, other):
+    return not self == other
+
+  @classmethod
+  def _CmpHelper(cls, x, y):
+    """Just a helper equivalent to the cmp() function in Python 2."""
+    return (x > y) - (x < y)
+
+  def __lt__(self, other):
+    return self._CmpHelper((self.api_name, self.api_version, self.name),
+                           (other.api_name, other.api_version, other.name)) < 0
+
+  def __gt__(self, other):
+    return self._CmpHelper((self.api_name, self.api_version, self.name),
+                           (other.api_name, other.api_version, other.name)) > 0
+
+  def __le__(self, other):
+    return not self.__gt__(other)
+
+  def __ge__(self, other):
+    return not self.__lt__(other)
+
+  def __str__(self):
+    return self.full_name
+
+
+class InvalidEndpointException(exceptions.Error):
+  """Exception for when an API endpoint is malformed."""
+
+  def __init__(self, url):
+    super(InvalidEndpointException, self).__init__(
+        "URL does not start with 'http://' or 'https://' [{0}]".format(url))
+
+
+def SplitEndpointUrl(url):
+  """Returns api_name, api_version, resource_path tuple for an API URL.
+
+  Supports the following formats:
+  # Google API production/staging endpoints.
+  http(s)://www.googleapis.com/{api}/{version}/{resource-path}
+  http(s)://stagingdomain/{api}/{version}/{resource-path}
+  http(s)://{api}.googleapis.com/{api}/{version}/{resource-path}
+  http(s)://{api}.googleapis.com/apis/{kube-api.name}/{version}/{resource-path}
+  http(s)://{api}.googleapis.com/{version}/{resource-path}
+  http(s)://{api}.googleapis.com/{api}/{resource-path}
+  http(s)://{api}.googleapis.com/{resource-path}
+  http(s)://googledomain/{api}
+  # Non-Google API endpoints.
+  http(s)://someotherdomain/{api}/{version}/{resource-path}
+
+  Args:
+    url: str, The resource url.
+
+  Returns:
+    (str, str, str): The API name, version, resource_path.
+
+  Raises: InvalidEndpointException.
+  """
+  tokens = _StripUrl(url).split('/')
+  version_index = _GetApiVersionIndex(tokens)
+
+  domain = tokens[0]
+  if version_index == -1:
+    api = _ExtractApiNameFromDomain(domain)
+    if len(tokens) > 1 and tokens[1] == api:
+      # domain/{api}/{resource-path}
+      # Some older APIs have this form e.g. compute.googleapis.com/compute/...
+      resource_index = 2
+    else:
+      # domain/{resource-path}
+      resource_index = 1
+    return api, None, '/'.join(tokens[resource_index:])
+
+  version = tokens[version_index]
+  resource_path = '/'.join(tokens[version_index + 1:])
+
+  if version_index == 1:
+    # domain/{version}/{resource-path}
+    return _ExtractApiNameFromDomain(domain), version, resource_path
+
+  if version_index > 1:
+    # If domain is not a kubernetes api name, use
+    # domain/{api}/{version}/{resource-path}
+    api_name = _FindKubernetesApiName(domain) or tokens[version_index - 1]
+    return api_name, version, resource_path
+
+  raise InvalidEndpointException(url)
+
+
+def GetParamsFromPath(path):
+  """Extract parameters from uri template path.
+
+    See https://tools.ietf.org/html/rfc6570. This function makes simplifying
+    assumption that all parameter names are surrounded by /{ and }/.
+
+  Args:
+    path: str, uri template path.
+
+  Returns:
+    list(str), list of parameters in the template path.
+  """
+  path = path.split(':')[0]
+  parts = path.split('/')
+  params = []
+  for part in parts:
+    if part.startswith('{') and part.endswith('}'):
+      part = part[1:-1]
+      if part.startswith('+'):
+        params.append(part[1:])
+      else:
+        params.append(part)
+  return params
+
+
+def _StripUrl(url):
+  """Strip a http: or https: prefix, then strip leading and trailing slashes."""
+  if url.startswith('https://') or url.startswith('http://'):
+    return url[url.index(':') + 1:].strip('/')
+  raise InvalidEndpointException(url)
+
+
+def IsApiVersion(token):
+  """Check if the token parsed from Url is API version."""
+  versions = ('alpha', 'beta', 'v1', 'v2', 'v3', 'v4', 'dogfood', 'head')
+  for api_version in versions:
+    if api_version in token:
+      return True
+  return False
+
+
+def _GetApiVersionIndex(tokens):
+  for idx, token in enumerate(tokens):
+    if IsApiVersion(token):
+      return idx
+  return -1
+
+
+def _ExtractApiNameFromDomain(domain):
+  # Example: sql.googleapis.com -> sql
+  return domain.split('.')[0]
+
+
+def _FindKubernetesApiName(domain):
+  """Find the name of the kubernetes api.
+
+  Determines the kubernetes api name from the domain of the resource uri.
+  The domain may from a global resource or a regionalized resource.
+
+  Args:
+    domain: Domain from the resource uri.
+
+  Returns:
+    Api name. Returns None if the domain is not a kubernetes api domain.
+  """
+  # Examples:
+  # sql.googleapis.com -> sql
+  # us-central1-run.googleapis.com - > run
+  k8s_api_names = ('run',)
+  domain_first_part = domain.split('.')[0]
+  for api_name in k8s_api_names:
+    if (api_name == domain_first_part or
+        domain_first_part.endswith('-' + api_name)):
+      return api_name
+  return None

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/util/resource_search.py

@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The Cloud Resource Search lister."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from apitools.base.py import list_pager
+
+from googlecloudsdk.api_lib.util import apis
+from googlecloudsdk.core import exceptions
+from googlecloudsdk.core import log
+from googlecloudsdk.core.resource import resource_expr_rewrite
+
+
+class Error(exceptions.Error):
+  """Base exception for this module."""
+
+
+class CollectionNotIndexed(Error):
+  """The collection is not indexed."""
+
+
+class QueryOperatorNotSupported(Error):
+  """An operator in a query is not supported."""
+
+
+PAGE_SIZE = 500
+
+# TODO(b/38192518): Move this to the resource parser yaml config.
+# The Cloud Resource Search type dict indexed by Cloud SDK collection.
+# The cloudresourcesearch API does not allow dotted resource type names, so
+# the flat namespace will get crowded and unwieldy as it grows. For example,
+# how many APIs have an "instance" resource? Sorry, "Instance" already snapped
+# up by compute.  Instead we map the well-defined hierarchical Cloud SDK
+# collection names onto the supported resource types.
+RESOURCE_TYPES = {
+    'cloudresourcemanager.projects': 'Project',
+    'compute.disks': 'Disk',
+    'compute.healthChecks': 'HealthCheck',
+    'compute.httpHealthChecks': 'HttpHealthCheck',
+    'compute.httpsHealthChecks': 'HttpsHealthCheck',
+    'compute.images': 'Image',
+    'compute.instanceGroups': 'InstanceTemplate',
+    'compute.instances': 'Instance',
+    'compute.subnetworks': 'Subnetwork',
+}
+
+CLOUD_RESOURCE_SEARCH_COLLECTION = 'resources'
+
+
+class QueryRewriter(resource_expr_rewrite.Backend):
+  """Resource filter expression rewriter."""
+
+  def RewriteGlobal(self, call):
+    """Rewrites global restriction in call.
+
+    Args:
+      call: A list of resource_lex._TransformCall objects. In this case the list
+        has one element that is a global restriction with a global_restriction
+        property that is the restriction substring to match.
+
+    Returns:
+      The global restriction rewrite which is simply the global_restriction
+      string.
+    """
+    return call.global_restriction
+
+  def RewriteTerm(self, key, op, operand, key_type):
+    """Rewrites <key op operand>."""
+
+    del key_type  # unused in RewriteTerm
+    if op in ('~',):
+      raise QueryOperatorNotSupported(
+          'The [{}] operator is not supported in cloud resource search '
+          'queries.'.format(op))
+
+    # The query API does not support name:(value1 ... valueN), but it does
+    # support OR, so we split multiple operands into separate name:valueI
+    # expressions joined by OR.
+    values = operand if isinstance(operand, list) else [operand]
+
+    if key == 'project':
+      # The query API does not have a 'project' attribute, but It does have
+      # 'selfLink'. The rewrite provides 'project' and massages the query to
+      # operate on 'selfLink'. The resource parser would be perfection here,
+      # but a bit heavweight. We'll stick with this shortcut until proven wrong.
+      key = 'selfLink'
+      values = ['/projects/{}/'.format(value) for value in values]
+    elif key == '@type':
+      # Cloud SDK maps @type:collection => @type:resource-type. This isolates
+      # the user from yet another esoteric namespace.
+      collections = values
+      values = []
+      for collection in collections:
+        if collection.startswith(CLOUD_RESOURCE_SEARCH_COLLECTION + '.'):
+          values.append(collection[len(CLOUD_RESOURCE_SEARCH_COLLECTION) + 1:])
+        else:
+          try:
+            values.append(RESOURCE_TYPES[collection])
+          except KeyError:
+            raise CollectionNotIndexed(
+                'Collection [{}] not indexed for search.'.format(collection))
+
+    parts = ['{key}{op}{value}'.format(key=key, op=op, value=self.Quote(value))
+             for value in values]
+    expr = ' OR '.join(parts)
+    if len(parts) > 1:
+      # This eliminates AND/OR precedence ambiguity.
+      expr = '( ' + expr + ' )'
+    return expr
+
+
+def List(limit=None, page_size=None, query=None, sort_by=None, uri=False):
+  """Yields the list of Cloud Resources for collection.
+
+  Not all collections are indexed for search.
+
+  Args:
+    limit: The max number of resources to return. None for unlimited.
+    page_size: The max number of resources per response page. The defsult is
+      PAGE_SIE.
+    query: A resource filter expression. Use @type:collection to filter
+      resources by collection. Use the resources._RESOURCE_TYPE_ collection to
+      specify CloudResourceSearch resource types. By default all indexed
+      resources are in play.
+    sort_by: A list of field names to sort by. Prefix a name with ~ to reverse
+      the sort for that name.
+    uri: Return the resource URI if true.
+
+  Raises:
+    CollectionNotIndexed: If the collection is not indexed for search.
+    QueryOperatorNotSupported: If the query contains an unsupported operator.
+    HttpError: request/response errors.
+
+  Yields:
+    The list of Cloud Resources for collection.
+  """
+  _, remote_query = QueryRewriter().Rewrite(query)
+  log.info('Resource search query="%s" remote_query="%s"', query, remote_query)
+  if page_size is None:
+    page_size = PAGE_SIZE
+  if sort_by:
+    order_by = ','.join([name[1:] + ' desc' if name.startswith('~') else name
+                         for name in sort_by])
+  else:
+    order_by = None
+
+  client = apis.GetClientInstance('cloudresourcesearch', 'v1')
+
+  for result in list_pager.YieldFromList(
+      service=client.ResourcesService(client),
+      method='Search',
+      request=client.MESSAGES_MODULE.CloudresourcesearchResourcesSearchRequest(
+          orderBy=order_by,
+          query=remote_query,
+      ),
+      field='results',
+      limit=limit,
+      batch_size=page_size,
+      batch_size_attribute='pageSize'):
+    yield result.resourceUrl if uri else result.resource

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/util/waiter.py

@@ -0,0 +1,332 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities to support long running operations."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import abc
+import time
+
+from apitools.base.py import encoding
+from googlecloudsdk.core import exceptions
+from googlecloudsdk.core.console import progress_tracker
+from googlecloudsdk.core.util import retry
+import six
+
+
+_TIMEOUT_MESSAGE = (
+    'The operations may still be underway remotely and may still succeed; '
+    'use gcloud list and describe commands or '
+    'https://console.developers.google.com/ to check resource state.')
+
+
+class TimeoutError(exceptions.Error):
+  pass
+
+
+class AbortWaitError(exceptions.Error):
+  pass
+
+
+class OperationError(exceptions.Error):
+  pass
+
+
+class OperationPoller(six.with_metaclass(abc.ABCMeta, object)):
+  """Interface for defining operation which can be polled and waited on.
+
+  This construct manages operation_ref, operation and result abstract objects.
+  Operation_ref is an identifier for operation which is a proxy for result
+  object. OperationPoller has three responsibilities:
+    1. Given operation object determine if it is done.
+    2. Given operation_ref fetch operation object
+    3. Given operation object fetch result object
+  """
+
+  @abc.abstractmethod
+  def IsDone(self, operation):
+    """Given result of Poll determines if result is done.
+
+    Args:
+      operation: object representing operation returned by Poll method.
+
+    Returns:
+
+    """
+    return True
+
+  @abc.abstractmethod
+  def Poll(self, operation_ref):
+    """Retrieves operation given its reference.
+
+    Args:
+      operation_ref: str, some id for operation.
+
+    Returns:
+      object which represents operation.
+    """
+    return None
+
+  @abc.abstractmethod
+  def GetResult(self, operation):
+    """Given operation message retrieves result it represents.
+
+    Args:
+      operation: object, representing operation returned by Poll method.
+    Returns:
+      some object created by given operation.
+    """
+    return None
+
+
+class CloudOperationPoller(OperationPoller):
+  """Manages a longrunning Operations.
+
+  See https://cloud.google.com/speech/reference/rpc/google.longrunning
+  """
+
+  def __init__(self, result_service, operation_service):
+    """Sets up poller for cloud operations.
+
+    Args:
+      result_service: apitools.base.py.base_api.BaseApiService, api service for
+        retrieving created result of initiated operation.
+      operation_service: apitools.base.py.base_api.BaseApiService, api service
+        for retrieving information about ongoing operation.
+
+      Note that result_service and operation_service Get request must have
+      single attribute called 'name'.
+    """
+    self.result_service = result_service
+    self.operation_service = operation_service
+
+  def IsDone(self, operation):
+    """Overrides."""
+    if operation.done:
+      if operation.error:
+        raise OperationError(operation.error.message)
+      return True
+    return False
+
+  def Poll(self, operation_ref):
+    """Overrides.
+
+    Args:
+      operation_ref: googlecloudsdk.core.resources.Resource.
+
+    Returns:
+      fetched operation message.
+    """
+    request_type = self.operation_service.GetRequestType('Get')
+    return self.operation_service.Get(
+        request_type(name=operation_ref.RelativeName()))
+
+  def GetResult(self, operation):
+    """Overrides.
+
+    Args:
+      operation: api_name_messages.Operation.
+
+    Returns:
+      result of result_service.Get request.
+    """
+    request_type = self.result_service.GetRequestType('Get')
+    response_dict = encoding.MessageToPyValue(operation.response)
+    return self.result_service.Get(request_type(name=response_dict['name']))
+
+
+class CloudOperationPollerNoResources(OperationPoller):
+  """Manages longrunning Operations for Cloud API that creates no resources.
+
+  See https://cloud.google.com/speech/reference/rpc/google.longrunning
+  """
+
+  # TODO(b/62478975): Remove get_name_func when ML API operation names
+  # are compatible with gcloud parsing, and use RelativeName instead.
+  def __init__(self, operation_service, get_name_func=None):
+    """Sets up poller for cloud operations.
+
+    Args:
+      operation_service: apitools.base.py.base_api.BaseApiService, api service
+        for retrieving information about ongoing operation.
+
+        Note that the operation_service Get request must have a
+        single attribute called 'name'.
+      get_name_func: the function to use to get the name from the operation_ref.
+        This is to allow polling with non-traditional operation resource names.
+        If the resource name is compatible with gcloud parsing, use
+        `lambda x: x.RelativeName()`.
+    """
+    self.operation_service = operation_service
+    self.get_name = get_name_func or (lambda x: x.RelativeName())
+
+  def IsDone(self, operation):
+    """Overrides."""
+    if operation.done:
+      if operation.error:
+        raise OperationError(operation.error.message)
+      return True
+    return False
+
+  def Poll(self, operation_ref):
+    """Overrides.
+
+    Args:
+      operation_ref: googlecloudsdk.core.resources.Resource.
+
+    Returns:
+      fetched operation message.
+    """
+    request_type = self.operation_service.GetRequestType('Get')
+    return self.operation_service.Get(
+        request_type(name=self.get_name(operation_ref)))
+
+  def GetResult(self, operation):
+    """Overrides to get the response from the completed operation.
+
+    Args:
+      operation: api_name_messages.Operation.
+
+    Returns:
+      the 'response' field of the Operation.
+    """
+    return operation.response
+
+
+def WaitFor(poller,
+            operation_ref,
+            message=None,
+            custom_tracker=None,
+            tracker_update_func=None,
+            pre_start_sleep_ms=1000,
+            max_retrials=None,
+            max_wait_ms=1800000,
+            exponential_sleep_multiplier=1.4,
+            jitter_ms=1000,
+            wait_ceiling_ms=180000,
+            sleep_ms=2000):
+  """Waits for poller.Poll and displays pending operation spinner.
+
+  Args:
+    poller: OperationPoller, poller to use during retrials.
+    operation_ref: object, passed to operation poller poll method.
+    message: str, string to display for default progress_tracker.
+    custom_tracker: ProgressTracker, progress_tracker to use for display.
+    tracker_update_func: func(tracker, result, status), tracker update function.
+    pre_start_sleep_ms: int, Time to wait before making first poll request.
+    max_retrials: int, max number of retrials before raising RetryException.
+    max_wait_ms: int, number of ms to wait before raising WaitException.
+    exponential_sleep_multiplier: float, factor to use on subsequent retries.
+    jitter_ms: int, random (up to the value) additional sleep between retries.
+    wait_ceiling_ms: int, Maximum wait between retries.
+    sleep_ms: int or iterable: for how long to wait between trials.
+
+  Returns:
+    poller.GetResult(operation).
+
+  Raises:
+    AbortWaitError: if ctrl-c was pressed.
+    TimeoutError: if retryer has finished without being done.
+  """
+  aborted_message = 'Aborting wait for operation {0}.\n'.format(operation_ref)
+  try:
+    with progress_tracker.ProgressTracker(
+        message, aborted_message=aborted_message
+    ) if not custom_tracker else custom_tracker as tracker:
+
+      if pre_start_sleep_ms:
+        _SleepMs(pre_start_sleep_ms)
+
+      def _StatusUpdate(result, status):
+        if tracker_update_func:
+          tracker_update_func(tracker, result, status)
+        else:
+          tracker.Tick()
+
+      operation = PollUntilDone(
+          poller, operation_ref, max_retrials, max_wait_ms,
+          exponential_sleep_multiplier, jitter_ms, wait_ceiling_ms,
+          sleep_ms, _StatusUpdate)
+
+  except retry.WaitException:
+    raise TimeoutError(
+        'Operation {0} has not finished in {1} seconds. {2}'
+        .format(operation_ref, max_wait_ms // 1000, _TIMEOUT_MESSAGE))
+  except retry.MaxRetrialsException as e:
+    raise TimeoutError(
+        'Operation {0} has not finished in {1} seconds '
+        'after max {2} retrials. {3}'
+        .format(operation_ref,
+                e.state.time_passed_ms // 1000,
+                e.state.retrial,
+                _TIMEOUT_MESSAGE))
+
+  return poller.GetResult(operation)
+
+
+def PollUntilDone(poller, operation_ref,
+                  max_retrials=None,
+                  max_wait_ms=1800000,
+                  exponential_sleep_multiplier=1.4,
+                  jitter_ms=1000,
+                  wait_ceiling_ms=180000,
+                  sleep_ms=2000,
+                  status_update=None):
+  """Waits for poller.Poll to complete.
+
+  Note that this *does not* print nice messages to stderr for the user; most
+  callers should use WaitFor instead for the best UX unless there's a good
+  reason not to print.
+
+  Args:
+    poller: OperationPoller, poller to use during retrials.
+    operation_ref: object, passed to operation poller poll method.
+    max_retrials: int, max number of retrials before raising RetryException.
+    max_wait_ms: int, number of ms to wait before raising WaitException.
+    exponential_sleep_multiplier: float, factor to use on subsequent retries.
+    jitter_ms: int, random (up to the value) additional sleep between retries.
+    wait_ceiling_ms: int, Maximum wait between retries.
+    sleep_ms: int or iterable: for how long to wait between trials.
+    status_update: func(result, state) called right after each trial.
+
+  Returns:
+    The return value from poller.Poll.
+  """
+
+  retryer = retry.Retryer(
+      max_retrials=max_retrials,
+      max_wait_ms=max_wait_ms,
+      exponential_sleep_multiplier=exponential_sleep_multiplier,
+      jitter_ms=jitter_ms,
+      wait_ceiling_ms=wait_ceiling_ms,
+      status_update_func=status_update)
+
+  def _IsNotDone(operation, unused_state):
+    return not poller.IsDone(operation)
+
+  operation = retryer.RetryOnResult(
+      func=poller.Poll,
+      args=(operation_ref,),
+      should_retry_if=_IsNotDone,
+      sleep_ms=sleep_ms)
+
+  return operation
+
+
+def _SleepMs(miliseconds):
+  time.sleep(miliseconds / 1000)