feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/core/diagnostics/check_base.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2016 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Base classes for checks."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import abc
+import collections
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Checker(object):
+  """Base class for a single check."""
+
+  @abc.abstractproperty
+  def issue(self):
+    """The aspect of the user's machine that is being checked."""
+
+  @abc.abstractmethod
+  def Check(self):
+    """Runs a single check and returns the result and an optional fix.
+
+    Returns:
+      A tuple of two elements. The first element should have the same attributes
+      as a check_base.Result object. The second element should either be a fixer
+      function that can used to fix an error (indicated by the "passed"
+      attribute being False in the first element), or None if the check passed
+      or if it failed with no applicable fix. If there is a fixer function it is
+      assumed that calling it will return True if it makes changes that warrant
+      running a check again.
+    """
+
+
+class Result(
+    collections.namedtuple('Result', ['passed', 'message', 'failures'])):
+  """Holds information about the result of a single check.
+
+  Attributes:
+    passed: Whether the check passed.
+    message: A summary message about the result of the check.
+    failures: A sequence of checkbase.Failure objects; may be empty if there
+        were no failures.
+  """
+
+  def __new__(cls, passed, message='', failures=None):
+    return super(Result, cls).__new__(cls, passed, message, failures or [])
+
+
+class Failure(collections.namedtuple('Failure', ['message', 'exception'])):
+  """Holds information about the failure of a check.
+
+  Attributes:
+    message: A message detailing the failure; to be shown to the user.
+    exception: An Exception object associated with the failure.
+  """
+
+  def __new__(cls, message='', exception=None):
+    return super(Failure, cls).__new__(cls, message, exception)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/core/diagnostics/diagnostic_base.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2016 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Base classes for diagnostics."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.core import log
+from googlecloudsdk.core.console import progress_tracker
+
+
+class Diagnostic(object):
+  """Base class for diagnostics.
+
+  Attributes:
+    intro: A message to introduce the objectives and tasks of the diagnostic.
+    title: The name of the diagnostic.
+    checklist: An iterator of checkbase.Check objects to be run by the
+       diagnostic.
+  """
+
+  _MAX_RETRIES = 5
+
+  def __init__(self, intro, title, checklist):
+    """Initializes Diagnostic with necessary attributes.
+
+    Args:
+      intro: A message to introduce the objectives and tasks of the diagnostic.
+      title: The name of the diagnostic.
+      checklist: An iterable of checkbase.Check objects to be run by the
+         diagnostic.
+    """
+    self.intro = intro
+    self.title = title
+    self.checklist = checklist
+
+  def RunChecks(self):
+    """Runs one or more checks, tries fixes, and outputs results.
+
+    Returns:
+      True if the diagnostic ultimately passed.
+    """
+    self._Print(self.intro)
+
+    num_checks_passed = 0
+    for check in self.checklist:
+      result, fixer = self._RunCheck(check)
+
+      # If the initial check failed, and a fixer is available try to fix issue
+      # and recheck.
+      num_retries = 0
+      while not result.passed and fixer and num_retries < self._MAX_RETRIES:
+        num_retries += 1
+        should_check_again = fixer()
+        if should_check_again:
+          result, fixer = self._RunCheck(check, first_run=False)
+        else:
+          fixer = None
+
+      if not result.passed and fixer and num_retries == self._MAX_RETRIES:
+        log.warning('Unable to fix {0} failure after {1} attempts.'.format(
+            self.title, num_retries))
+      if result.passed:
+        num_checks_passed += 1
+
+    num_checks = len(self.checklist)
+    passed = num_checks_passed == num_checks
+    summary = ('{check} {status} ({num_passed}/{num_checks} checks passed).\n'.
+               format(check=self.title,
+                      num_passed=num_checks_passed,
+                      num_checks=num_checks,
+                      status='passed' if passed else 'failed'))
+    self._Print(summary, as_error=not passed)
+    return passed
+
+  def _RunCheck(self, check, first_run=True):
+    with progress_tracker.ProgressTracker('{0} {1}'.format(
+        'Checking' if first_run else 'Rechecking', check.issue)):
+      result, fixer = check.Check(first_run=first_run)
+    self._PrintResult(result)
+    return result, fixer
+
+  def _Print(self, message, as_error=False):
+    logger = log.status.Print if not as_error else log.error
+    logger(message)
+
+  def _PrintResult(self, result):
+    self._Print(result.message, not result.passed)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/core/diagnostics/http_proxy_setup.py

@@ -0,0 +1,183 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2016 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A module for changing Cloud SDK proxy settings interactively."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.core import http_proxy
+from googlecloudsdk.core import log
+from googlecloudsdk.core import properties
+from googlecloudsdk.core.console import console_io
+from googlecloudsdk.core.util import encoding
+from googlecloudsdk.core.util import http_proxy_types
+
+import httplib2
+
+
+def ChangeGcloudProxySettings():
+  """Displays proxy information and helps user set up gcloud proxy properties.
+
+  Returns:
+    Whether properties were successfully changed.
+  """
+  try:
+    proxy_info, is_existing_proxy = EffectiveProxyInfo()
+  except properties.InvalidValueError:
+    log.status.Print(
+        'Cloud SDK network proxy settings appear to be invalid. Proxy type, '
+        'address, and port must be specified. Run [gcloud info] for more '
+        'details.\n')
+    is_existing_proxy = True
+  else:
+    _DisplayGcloudProxyInfo(proxy_info, is_existing_proxy)
+
+  if properties.VALUES.core.disable_prompts.GetBool():
+    return False
+
+  if is_existing_proxy:
+    options = ['Change Cloud SDK network proxy properties',
+               'Clear all gcloud proxy properties',
+               'Exit']
+    existing_proxy_idx = console_io.PromptChoice(
+        options, message='What would you like to do?')
+
+    if existing_proxy_idx == 0:
+      return _ProxySetupWalkthrough()
+
+    if existing_proxy_idx == 1:
+      SetGcloudProxyProperties()
+      log.status.Print('Cloud SDK proxy properties cleared.\n')
+      return True
+
+    return False
+  else:
+    if console_io.PromptContinue(prompt_string='Do you have a network proxy '
+                                 'you would like to set in gcloud'):
+      return _ProxySetupWalkthrough()
+    return False
+
+
+def _ProxySetupWalkthrough():
+  """Walks user through setting up gcloud proxy properties."""
+  proxy_type_options = sorted(
+      t.upper() for t in http_proxy_types.PROXY_TYPE_MAP)
+  proxy_type_idx = console_io.PromptChoice(
+      proxy_type_options, message='Select the proxy type:')
+  if proxy_type_idx is None:
+    return False
+  proxy_type = proxy_type_options[proxy_type_idx].lower()
+
+  address = console_io.PromptResponse('Enter the proxy host address: ')
+  log.status.Print()
+  if not address:
+    return False
+
+  port = console_io.PromptResponse('Enter the proxy port: ')
+  log.status.Print()
+  # Restrict port number to 0-65535.
+  if not port:
+    return False
+  try:
+    if not 0 <= int(port) <= 65535:
+      log.status.Print('Port number must be between 0 and 65535.')
+      return False
+  except ValueError:
+    log.status.Print('Please enter an integer for the port number.')
+    return False
+
+  username, password = None, None
+  authenticated = console_io.PromptContinue(
+      prompt_string='Is your proxy authenticated', default=False)
+  if authenticated:
+    username = console_io.PromptResponse('Enter the proxy username: ')
+    log.status.Print()
+    if not username:
+      return False
+    password = console_io.PromptResponse('Enter the proxy password: ')
+    log.status.Print()
+    if not password:
+      return False
+
+  SetGcloudProxyProperties(proxy_type=proxy_type, address=address, port=port,
+                           username=username, password=password)
+  log.status.Print('Cloud SDK proxy properties set.\n')
+  return True
+
+
+def EffectiveProxyInfo():
+  """Returns ProxyInfo effective in gcloud and if it is from gloud properties.
+
+  Returns:
+    A tuple of two elements in which the first element is an httplib2.ProxyInfo
+      object and the second is a bool that is True if the proxy info came from
+      previously set Cloud SDK proxy properties.
+
+  Raises:
+    properties.InvalidValueError: If the properties did not include a valid set.
+      "Valid" means all three of these attributes are present: proxy type, host,
+      and port.
+  """
+  proxy_info = http_proxy.GetHttpProxyInfo()  # raises InvalidValueError
+  if not proxy_info:
+    return None, False
+
+  # googlecloudsdk.core.http_proxy.GetHttpProxyInfo() will return a function
+  # if there are no valid proxy settings in gcloud properties. Otherwise, it
+  # will return an instantiated httplib2.ProxyInfo object.
+  from_gcloud_properties = True
+  if not isinstance(proxy_info, httplib2.ProxyInfo):
+    from_gcloud_properties = False
+    # All Google Cloud SDK network calls use https.
+    proxy_info = proxy_info('https')
+
+  return proxy_info, from_gcloud_properties
+
+
+def _DisplayGcloudProxyInfo(proxy_info, from_gcloud):
+  """Displays Cloud SDK proxy information."""
+  if not proxy_info:
+    log.status.Print()
+    return
+
+  log.status.Print('Current effective Cloud SDK network proxy settings:')
+  if not from_gcloud:
+    log.status.Print('(These settings are from your machine\'s environment, '
+                     'not gcloud properties.)')
+  proxy_type_name = http_proxy_types.REVERSE_PROXY_TYPE_MAP.get(
+      proxy_info.proxy_type, 'UNKNOWN PROXY TYPE')
+  log.status.Print('    type = {0}'.format(proxy_type_name))
+  log.status.Print('    host = {0}'.format(proxy_info.proxy_host))
+  log.status.Print('    port = {0}'.format(proxy_info.proxy_port))
+  # In Python 3, httplib2 encodes the proxy username and password when
+  # initializing ProxyInfo, so we want to ensure they're decoded here before
+  # displaying them.
+  log.status.Print('    username = {0}'.format(
+      encoding.Decode(proxy_info.proxy_user)))
+  log.status.Print('    password = {0}'.format(
+      encoding.Decode(proxy_info.proxy_pass)))
+  log.status.Print()
+
+
+def SetGcloudProxyProperties(proxy_type=None, address=None, port=None,
+                             username=None, password=None):
+  """Sets proxy group properties; clears any property not explicitly set."""
+  properties.PersistProperty(properties.VALUES.proxy.proxy_type, proxy_type)
+  properties.PersistProperty(properties.VALUES.proxy.address, address)
+  properties.PersistProperty(properties.VALUES.proxy.port, port)
+  properties.PersistProperty(properties.VALUES.proxy.username, username)
+  properties.PersistProperty(properties.VALUES.proxy.password, password)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/core/diagnostics/network_diagnostics.py

@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2016 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A module for diagnosing common network and proxy problems."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import socket
+import ssl
+
+from googlecloudsdk.core import config
+from googlecloudsdk.core import http
+from googlecloudsdk.core import properties
+from googlecloudsdk.core import requests as core_requests
+from googlecloudsdk.core.diagnostics import check_base
+from googlecloudsdk.core.diagnostics import diagnostic_base
+from googlecloudsdk.core.diagnostics import http_proxy_setup
+
+import httplib2
+import requests
+from six.moves import http_client
+from six.moves import urllib
+import socks
+
+_NETWORK_TIMEOUT = 60  # Timeout in seconds when testing GET requests
+
+
+class NetworkDiagnostic(diagnostic_base.Diagnostic):
+  """Diagnose and fix local network connection issues."""
+
+  def __init__(self):
+    intro = ('Network diagnostic detects and fixes local network connection '
+             'issues.')
+    super(NetworkDiagnostic, self).__init__(
+        intro=intro, title='Network diagnostic',
+        checklist=[ReachabilityChecker()])
+
+  def RunChecks(self):
+    if not properties.IsDefaultUniverse():
+      # Skip network reachability checks on non-default universes.
+      return True
+    return super().RunChecks()
+
+
+def DefaultUrls():
+  """Returns a list of hosts whose reachability is essential for the Cloud SDK.
+
+  Returns:
+    A list of urls (str) to check reachability for.
+  """
+  urls = ['https://accounts.google.com',
+          'https://cloudresourcemanager.googleapis.com/v1beta1/projects',
+          'https://www.googleapis.com/auth/cloud-platform']
+
+  download_urls = (properties.VALUES.component_manager.snapshot_url.Get() or
+                   config.INSTALLATION_CONFIG.snapshot_url)
+  urls.extend(u for u in download_urls.split(',')
+              if urllib.parse.urlparse(u).scheme in ('http', 'https'))
+  return urls
+
+
+class ReachabilityChecker(check_base.Checker):
+  """Checks whether the hosts of given urls are reachable."""
+
+  @property
+  def issue(self):
+    return 'network connection'
+
+  def Check(self, urls=None, first_run=True):
+    """Run reachability check.
+
+    Args:
+      urls: iterable(str), The list of urls to check connection to. Defaults to
+        DefaultUrls() (above) if not supplied.
+      first_run: bool, True if first time this has been run this invocation.
+
+    Returns:
+      A tuple of (check_base.Result, fixer) where fixer is a function that can
+        be used to fix a failed check, or  None if the check passed or failed
+        with no applicable fix.
+    """
+    if urls is None:
+      urls = DefaultUrls()
+
+    failures = []
+    # Check reachability using httplib2
+    for url in urls:
+      fail = CheckURLHttplib2(url)
+      if fail:
+        failures.append(fail)
+
+    # Check reachability using requests
+    for url in urls:
+      fail = CheckURLRequests(url)
+      if fail:
+        failures.append(fail)
+
+    if failures:
+      fail_message = ConstructMessageFromFailures(failures, first_run)
+      result = check_base.Result(passed=False, message=fail_message,
+                                 failures=failures)
+      fixer = http_proxy_setup.ChangeGcloudProxySettings
+      return result, fixer
+
+    pass_message = 'Reachability Check {0}.'.format('passed' if first_run else
+                                                    'now passes')
+    result = check_base.Result(passed=True, message='No URLs to check.'
+                               if not urls else pass_message)
+    return result, None
+
+
+def CheckURLHttplib2(url):
+  try:
+    http.Http(timeout=_NETWORK_TIMEOUT).request(url, method='GET')
+  except (http_client.HTTPException, socket.error, ssl.SSLError,
+          httplib2.HttpLib2Error, socks.HTTPError) as err:
+    msg = 'httplib2 cannot reach {0}:\n{1}\n'.format(
+        url, err)
+    return check_base.Failure(message=msg, exception=err)
+
+
+def CheckURLRequests(url):
+  try:
+    core_requests.GetSession(timeout=_NETWORK_TIMEOUT).request('GET', url)
+  except requests.exceptions.RequestException as err:
+    msg = 'requests cannot reach {0}:\n{1}\n'.format(
+        url, err)
+    return check_base.Failure(message=msg, exception=err)
+
+
+def ConstructMessageFromFailures(failures, first_run):
+  """Constructs error messages along with diagnostic information."""
+  message = 'Reachability Check {0}.\n'.format('failed' if first_run else
+                                               'still does not pass')
+  for failure in failures:
+    message += '    {0}\n'.format(failure.message)
+  if first_run:
+    message += ('Network connection problems may be due to proxy or '
+                'firewall settings.\n')
+
+  return message

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/core/diagnostics/property_diagnostics.py

@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2016 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A module for diagnosing common problems caused by properties."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.core import config
+from googlecloudsdk.core import properties
+from googlecloudsdk.core.configurations import named_configs
+from googlecloudsdk.core.diagnostics import check_base
+from googlecloudsdk.core.diagnostics import diagnostic_base
+import six
+
+
+class PropertyDiagnostic(diagnostic_base.Diagnostic):
+  """Diagnoses issues that may be caused by properties."""
+
+  def __init__(self, ignore_hidden_property_allowlist):
+    intro = ('Property diagnostic detects issues that may be caused by '
+             'properties.')
+    super(PropertyDiagnostic, self).__init__(
+        intro=intro, title='Property diagnostic',
+        checklist=[HiddenPropertiesChecker(ignore_hidden_property_allowlist)])
+
+
+def _AllProperties():
+  for section in properties.VALUES:
+    for prop in section:
+      yield prop
+
+
+class HiddenPropertiesChecker(check_base.Checker):
+  """Checks whether any hidden properties have been set."""
+
+  _ALLOWLIST = (
+      'metrics/environment',
+      'core/universe_domain',
+  )
+
+  def __init__(self, ignore_hidden_property_allowlist):
+    self.ignore_hidden_property_allowlist = ignore_hidden_property_allowlist
+    self.allowlist = set(
+        (properties.VALUES.diagnostics.hidden_property_allowlist.Get() or '')
+        .split(',')
+    )
+    self._properties_file = named_configs.ActivePropertiesFile.Load()
+
+  @property
+  def issue(self):
+    return 'hidden properties'
+
+  def Check(self, first_run=True):
+    """Run hidden property check.
+
+    Args:
+      first_run: bool, True if first time this has been run this invocation.
+
+    Returns:
+      A tuple of (check_base.Result, fixer) where fixer is a function that can
+        be used to fix a failed check, or None if the check passed or failed
+        with no applicable fix.
+    """
+    failures = []
+    for prop in _AllProperties():
+      if prop.is_internal:
+        continue
+      if prop.is_hidden:
+        fail = self._CheckHiddenProperty(prop)
+        if fail:
+          failures.append(fail)
+    if failures:
+      fail_message = self._ConstructMessageFromFailures(failures, first_run)
+      result = check_base.Result(passed=False, message=fail_message,
+                                 failures=failures)
+      return result, None
+
+    pass_message = 'Hidden Property Check {0}.'.format(
+        'passed' if first_run else 'now passes')
+    result = check_base.Result(passed=True, message=pass_message)
+    return result, None
+
+  def _CheckHiddenProperty(self, prop):
+    if six.text_type(prop) in self._ALLOWLIST:
+      return
+    if (not self.ignore_hidden_property_allowlist and
+        six.text_type(prop) in self.allowlist):
+      return
+
+    # pylint:disable=protected-access
+    value = properties._GetPropertyWithoutCallback(prop, self._properties_file)
+    if value is not None:
+      msg = '[{0}]'.format(prop)
+      return check_base.Failure(message=msg)
+
+  def _ConstructMessageFromFailures(self, failures, first_run):
+    message = 'Hidden Property Check {0}.\n'.format('failed' if first_run else
+                                                    'still does not pass')
+    if failures:
+      message += 'The following hidden properties have been set:\n'
+    for failure in failures:
+      message += '    {0}\n'.format(failure.message)
+    if first_run:
+      message += ('Properties files\n'
+                  '    User: {0}\n'
+                  '    Installation: {1}\n'.format(
+                      named_configs.ConfigurationStore.ActiveConfig().file_path,
+                      config.Paths().installation_properties_path)
+                 )
+    return message