feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/compute/instances/os_inventory/__init__.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2019 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Commands for reading OS inventory data and related resources."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA, base.ReleaseTrack.BETA,
+                    base.ReleaseTrack.ALPHA)
+class InstanceOsInventory(base.Group):
+  """Read Compute Engine OS Inventory Data and Related Resources."""
+
+InstanceOsInventory.detailed_help = {
+    'DESCRIPTION': """
+        Read Compute Engine OS inventory data.
+
+        For more information about OS inventory, see the
+        [OS inventory documentation](https://cloud.google.com/compute/docs/instances/os-inventory-management).
+    """,
+}

gcloud auth application-default login/google-cloud-sdk/lib/surface/compute/instances/os_inventory/describe.py

@@ -0,0 +1,187 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2019 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command for describing instance's OS inventory data."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import base64
+import json
+import zlib
+
+from googlecloudsdk.api_lib.compute import base_classes
+from googlecloudsdk.api_lib.compute import utils
+from googlecloudsdk.calliope import base
+from googlecloudsdk.calliope import exceptions as calliope_exceptions
+from googlecloudsdk.command_lib.compute.instances import flags
+from googlecloudsdk.command_lib.compute.instances.os_inventory import exceptions
+from googlecloudsdk.core.resource import resource_projector
+import six
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA, base.ReleaseTrack.BETA,
+                    base.ReleaseTrack.ALPHA)
+class Describe(base.DescribeCommand):
+  """Describe a Compute Engine virtual instance's OS inventory data.
+
+  *{command}* displays all OS inventory data associated with a Compute
+  Engine virtual machine instance.
+
+  ## EXAMPLES
+
+  To see OS inventory of an instance named my-instance, run:
+
+        $ {command} my-instance
+  """
+
+  _GUEST_ATTRIBUTES_PACKAGE_FIELD_KEYS = ('InstalledPackages', 'PackageUpdates')
+
+  @staticmethod
+  def Args(parser):
+    flags.INSTANCE_ARG.AddArgument(parser, operation_type='describe')
+
+    parser.display_info.AddFormat("""
+          multi(
+            InstalledPackages.cos:format=
+              "table[box,title='Installed Packages (COS)']
+                (Name:sort=1,Version)",
+            InstalledPackages.deb:format=
+              "table[box,title='Installed Packages (DEB)']
+                (Name:sort=1,Arch,Version)",
+            InstalledPackages.gem:format=
+              "table[box,title='Installed Packages (Gem)']
+                (Name:sort=1,Arch,Version)",
+            InstalledPackages.googet:format=
+              "table[box,title='Installed Packages (GooGet)']
+                (Name:sort=1,Arch,Version)",
+            InstalledPackages.pip:format=
+              "table[box,title='Installed Packages (Pip)']
+                (Name:sort=1,Arch,Version)",
+            InstalledPackages.rpm:format=
+              "table[box,title='Installed Packages (RPM)']
+                (Name:sort=1,Arch,Version)",
+            InstalledPackages.zypperPatches:format=
+              "table[box,title='Installed Patches (Zypper Patch)'](
+                Name:sort=1,
+                Category,
+                Severity,
+                Summary:wrap=14)",
+            InstalledPackages.wua:format=
+              "table[all-box,title='Installed Packages (Windows Update Agent)'](
+                Title:sort=1:wrap,
+                Categories.list():wrap,
+                KBArticleIDs.list():wrap=14,
+                SupportURL:wrap=11,
+                LastDeploymentChangeTime:wrap=15:label='LAST_DEPLOYMENT')",
+            InstalledPackages.qfe:format=
+              "table[box,title='Installed Packages (Quick Fix Engineering)']
+                (Caption,Description:wrap=15,HotFixID:sort=1,InstalledOn)",
+            PackageUpdates.apt:format=
+              "table[box,title='Package Updates Available (Apt)']
+                (Name:sort=1,Arch,Version)",
+            PackageUpdates.gem:format=
+              "table[box,title='Package Updates Available (Gem)']
+                (Name:sort=1,Arch,Version)",
+            PackageUpdates.googet:format=
+              "table[box,title='Package Updates Available (GooGet)']
+                (Name:sort=1,Arch,Version)",
+            PackageUpdates.pip:format=
+              "table[box,title='Package Updates Available (Pip)']
+                (Name:sort=1,Arch,Version)",
+            PackageUpdates.yum:format=
+              "table[box,title='Package Updates Available (Yum)']
+                (Name:sort=1,Arch,Version)",
+            PackageUpdates.zypperPatches:format=
+              "table[box,title='Patches Available (Zypper Patch)'](
+                Name:sort=1,
+                Category,
+                Severity,
+                Summary:wrap=14)",
+            PackageUpdates.wua:format=
+              "table[all-box,title='Package Updates Available (Windows Update Agent)'](
+                Title:sort=1:wrap,
+                Categories.list():wrap,
+                KBArticleIDs.list():wrap=14,
+                SupportURL:wrap=11,
+                LastDeploymentChangeTime:wrap=15:label='LAST_DEPLOYMENT')",
+            SystemInformation:format="default"
+          )
+        """)
+
+  def _GetInstanceRef(self, holder, args):
+    return flags.INSTANCE_ARG.ResolveAsResource(
+        args,
+        holder.resources,
+        scope_lister=flags.GetInstanceZoneScopeLister(holder.client))
+
+  def _GetGuestInventoryGuestAttributes(self, holder, instance_ref):
+    try:
+      client = holder.client
+      messages = client.messages
+      request = messages.ComputeInstancesGetGuestAttributesRequest(
+          instance=instance_ref.Name(),
+          project=instance_ref.project,
+          queryPath='guestInventory/',
+          zone=instance_ref.zone)
+      response = holder.client.MakeRequests(
+          [(holder.client.apitools_client.instances, 'GetGuestAttributes',
+            request)])[0]
+
+      for item in response.queryValue.items:
+        if item.key in self._GUEST_ATTRIBUTES_PACKAGE_FIELD_KEYS:
+          item.value = zlib.decompress(
+              base64.b64decode(item.value), zlib.MAX_WBITS | 32)
+
+      return response.queryValue.items
+    except calliope_exceptions.ToolException as e:
+      if ('The resource \'guestInventory/\' of type \'Guest Attribute\' was not'
+          ' found.') in six.text_type(e):
+        problems = [
+            (404,
+             'OS inventory data was not found. Make sure the OS Config agent '
+             'is running on this instance.')
+        ]
+        utils.RaiseException(
+            problems,
+            exceptions.OsInventoryNotFoundException,
+            error_message='Could not fetch resource:')
+      raise e
+
+  def _GetFormattedGuestAttributes(self, guest_attributes):
+    guest_attributes_json = resource_projector.MakeSerializable(
+        guest_attributes)
+
+    formatted_guest_attributes = {'SystemInformation': {}}
+    for guest_attribute in guest_attributes_json:
+      guest_attribute_key = guest_attribute['key']
+
+      # Only reformat the guest attribute value
+      # for certain fields that contain JSON data.
+      if guest_attribute_key in self._GUEST_ATTRIBUTES_PACKAGE_FIELD_KEYS:
+        formatted_guest_attributes[guest_attribute_key] = json.loads(
+            guest_attribute['value'])
+      else:
+        formatted_guest_attributes['SystemInformation'][
+            guest_attribute_key] = guest_attribute['value']
+
+    return json.loads(json.dumps(formatted_guest_attributes))
+
+  def Run(self, args):
+    holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
+    instance_ref = self._GetInstanceRef(holder, args)
+    guest_attributes_json = self._GetGuestInventoryGuestAttributes(
+        holder, instance_ref)
+    return self._GetFormattedGuestAttributes(guest_attributes_json)

gcloud auth application-default login/google-cloud-sdk/lib/surface/compute/instances/os_inventory/list_instances.py

@@ -0,0 +1,335 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2019 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command for listing instances with specific OS inventory data values."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import base64
+import json
+import os
+import zlib
+
+from googlecloudsdk.api_lib.compute import base_classes
+from googlecloudsdk.api_lib.compute import lister
+from googlecloudsdk.api_lib.compute import utils
+from googlecloudsdk.calliope import base
+from googlecloudsdk.calliope import exceptions
+from googlecloudsdk.command_lib.compute import completers
+from googlecloudsdk.command_lib.compute.instances import flags
+from googlecloudsdk.core import properties
+from googlecloudsdk.core.resource import resource_filter
+from googlecloudsdk.core.resource import resource_projector
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA, base.ReleaseTrack.BETA)
+class ListInstances(base.ListCommand):
+  r"""List instances with specific OS inventory data values.
+
+  {command} displays all Compute Engine instances in a project matching
+  an inventory filter. Run $ gcloud topic filters to see the supported filter
+  syntax.
+
+  ## EXAMPLES
+
+  To list all instances with OS inventory data in a project in table form, run:
+
+        $ {command}
+
+  To list the URIs of all instances whose OS short name contains rhel, run:
+
+        $ {command} --inventory-filter="ShortName:rhel" --uri
+
+  To list the URIs of all instances whose OS short name is equal to rhel, run:
+
+        $ {command} --os-shortname="rhel" --uri
+
+  To list all instances with package google-cloud-sdk of version 235.0.0-0
+  installed, run:
+
+        $ {command} --package-name="google-cloud-sdk" \
+        --package-version="235.0.0-0"
+
+  To list all instances with package name matching a regex ^google-cloud*
+  available for update through apt, run:
+
+        $ {command} --inventory-filter="\
+        PackageUpdates.apt[].Name~^google-cloud*"
+
+  To list all instances with package update google-cloud-sdk of version greater
+  than or equal to 235.0.0-0 available through apt, run:
+
+        $ {command} --inventory-filter="\
+        PackageUpdates.apt[].['google-cloud-sdk'].Version>=235.0.0-0"
+
+  To list all instances missing the Stackdriver monitoring package
+  stackdriver-agent, run:
+
+        $ {command} --inventory-filter="\
+        NOT(InstalledPackages:stackdriver-agent)"
+
+  To list all Windows instances with an installed qfe hotfix whose ID equals
+  KB4462930, run:
+
+        $ {command} --inventory-filter="\
+        InstalledPackages.qfe[].HotFixID=KB4462930"
+
+  To list all Windows instances with a wua update whose description contains the
+  word Security, run:
+
+        $ {command} --inventory-filter="\
+        InstalledPackages.wua[].Description:Security"
+
+  """
+
+  _GUEST_ATTRIBUTES_PACKAGE_FIELD_KEYS = ('InstalledPackages', 'PackageUpdates')
+
+  _SPECIAL_PACKAGE_MANAGERS = ('wua', 'qfe', 'zypperPatches')
+  _REGULAR_PACKAGE_MANAGERS = ('cos', 'deb', 'googet', 'rpm', 'gem', 'pip')
+
+  @staticmethod
+  def Args(parser):
+    parser.display_info.AddFormat(flags.DEFAULT_LIST_FORMAT)
+    parser.display_info.AddUriFunc(utils.MakeGetUriFunc())
+    parser.display_info.AddCacheUpdater(completers.InstancesCompleter)
+    parser.add_argument(
+        '--inventory-filter',
+        type=str,
+        help="""Filter expression for matching against OS inventory criteria""")
+    filter_group = parser.add_group(
+        help='Exact match values for OS inventory data:')
+    filter_group.add_argument(
+        '--os-shortname',
+        type=str,
+        help="""If specified, only instances with this OS shortname in their
+        inventory data will be displayed.""")
+    filter_group.add_argument(
+        '--os-version',
+        type=str,
+        help="""If specified, only instances with this OS version in their
+        inventory data will be displayed.""")
+    filter_group.add_argument(
+        '--kernel-version',
+        type=str,
+        help="""If specified, only instances with this kernel version in their
+        inventory data will be displayed.""")
+    filter_group.add_argument(
+        '--package-name',
+        type=str,
+        help="""If specified, only instances with an installed package of this
+        name in their inventory data will be displayed.""")
+    filter_group.add_argument(
+        '--package-version',
+        type=str,
+        help="""If specified with a package name, only instances with the
+        installed package of this version in their inventory data will be
+        displayed.""")
+
+  def _GetGuestAttributesRequest(self, messages, instance_name, project, zone):
+    return messages.ComputeInstancesGetGuestAttributesRequest(
+        instance=instance_name,
+        project=project,
+        queryPath='guestInventory/',
+        zone=zone)
+
+  def _GetAllGuestInventoryGuestAttributes(self, holder, instances):
+    client = holder.client
+    messages = client.messages
+    project = properties.VALUES.core.project.GetOrFail()
+
+    requests = [
+        self._GetGuestAttributesRequest(messages, instance['name'], project,
+                                        os.path.basename(instance['zone']))
+        for instance in instances
+    ]
+    responses = holder.client.AsyncRequests(
+        [
+            (client.apitools_client.instances, 'GetGuestAttributes', request)
+            for request in requests
+        ]
+    )
+
+    for response in filter(None, responses):
+      for item in response.queryValue.items:
+        if item.key in self._GUEST_ATTRIBUTES_PACKAGE_FIELD_KEYS:
+          item.value = zlib.decompress(
+              base64.b64decode(item.value), zlib.MAX_WBITS | 32)
+    return responses
+
+  def _GetFormattedGuestAttributes(self, guest_attributes):
+    guest_attributes_json = resource_projector.MakeSerializable(
+        guest_attributes)
+
+    formatted_guest_attributes = {}
+    for guest_attribute in guest_attributes_json:
+      guest_attribute_key = guest_attribute['key']
+
+      # Only reformat the guest attribute value
+      # for certain fields that contain JSON data.
+      if guest_attribute_key in self._GUEST_ATTRIBUTES_PACKAGE_FIELD_KEYS:
+        formatted_packages_info = {}
+        guest_attribute_json = json.loads(guest_attribute['value'])
+        for package_manager, package_list in guest_attribute_json.items():
+          if package_manager in self._SPECIAL_PACKAGE_MANAGERS:
+            # No reformatting for special package managers.
+            formatted_packages_info[package_manager] = package_list
+          else:
+            # Reformat package info published by standard package managers
+            formatted_packages_list = []
+            for package in package_list:
+              name = package['Name']
+              info = {'Arch': package['Arch'], 'Version': package['Version']}
+              formatted_packages_list.append({'Name': name, name: info})
+            formatted_packages_info[package_manager] = formatted_packages_list
+        guest_attribute['value'] = formatted_packages_info
+
+      formatted_guest_attributes[guest_attribute_key] = guest_attribute['value']
+
+    return json.loads(json.dumps(formatted_guest_attributes))
+
+  def _GetInventoryFilteredInstances(self, instances, responses, query):
+    filtered_instances = []
+
+    for instance, response in zip(instances, responses):
+      # No listing instances without inventory data.
+      if instance is not None and response is not None:
+        guest_attributes = response.queryValue.items
+        formatted_guest_attributes_json = self._GetFormattedGuestAttributes(
+            guest_attributes)
+        if query.Evaluate(formatted_guest_attributes_json):
+          filtered_instances.append(instance)
+
+    return filtered_instances
+
+  def _GetInventoryFilterQuery(self, args):
+    query_list = []
+
+    def _AppendQuery(query):
+      query_list.append('({})'.format(query))
+
+    if args.inventory_filter:
+      _AppendQuery(args.inventory_filter)
+    if args.os_shortname:
+      _AppendQuery('ShortName=' + args.os_shortname)
+    if args.os_version:
+      _AppendQuery('Version=' + args.os_version)
+    if args.kernel_version:
+      _AppendQuery('KernelVersion=' + args.kernel_version)
+
+    installed_packages_query_prefixes = [
+        'InstalledPackages.' + package_manager + '[].'
+        for package_manager in self._REGULAR_PACKAGE_MANAGERS
+    ]
+    if args.package_version:
+      if not args.package_name:
+        raise exceptions.InvalidArgumentException(
+            '--package-version',
+            'package version must be specified together with a package name. '
+            'e.g. --package-name google-cloud-sdk --package-version 235.0.0-0')
+      else:
+        package_name = '[\'{}\']'.format(args.package_name)
+        _AppendQuery(' OR '.join([
+            '({})'.format(prefix + package_name + '.Version=' +
+                          args.package_version)
+            for prefix in installed_packages_query_prefixes
+        ]))
+    else:
+      if args.package_name:
+        _AppendQuery(' OR '.join([
+            '({})'.format(prefix + 'Name=' + args.package_name)
+            for prefix in installed_packages_query_prefixes
+        ]))
+
+    return ' AND '.join(query_list)
+
+  def Run(self, args):
+    query = resource_filter.Compile(self._GetInventoryFilterQuery(args))
+
+    holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
+    client = holder.client
+
+    request_data = lister.ParseMultiScopeFlags(args, holder.resources)
+    list_implementation = lister.MultiScopeLister(
+        client,
+        zonal_service=client.apitools_client.instances,
+        aggregation_service=client.apitools_client.instances)
+
+    instances_iterator = lister.Invoke(request_data, list_implementation)
+    instances = list(instances_iterator)
+
+    responses = self._GetAllGuestInventoryGuestAttributes(holder, instances)
+    return self._GetInventoryFilteredInstances(instances, responses, query)
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
+class ListInstancesAlpha(ListInstances):
+  r"""List instances with specific OS inventory data values.
+
+  {command} displays all Google Compute Engine instances in a project matching
+  an inventory filter. Run $ gcloud topic filters to see the supported filter
+  syntax.
+
+  ## EXAMPLES
+
+  To list all instances with OS inventory data in a project in table form, run:
+
+        $ {command}
+
+  To list the URIs of all instances whose OS short name contains rhel, run:
+
+        $ {command} --inventory-filter="ShortName:rhel" --uri
+
+  To list the URIs of all instances whose OS short name is equal to rhel, run:
+
+        $ {command} --os-shortname="rhel" --uri
+
+  To list all instances with package google-cloud-sdk of version 235.0.0-0
+  installed, run:
+
+        $ {command} --package-name="google-cloud-sdk" \
+        --package-version="235.0.0-0"
+
+  To list all instances with package name matching a regex ^google-cloud*
+  available for update through apt, run:
+
+        $ {command} --inventory-filter="\
+        PackageUpdates.apt[].Name~^google-cloud*"
+
+  To list all instances with package update google-cloud-sdk of version greater
+  than or equal to 235.0.0-0 available through apt, run:
+
+        $ {command} --inventory-filter="\
+        PackageUpdates.apt[].['google-cloud-sdk'].Version>=235.0.0-0"
+
+  To list all instances missing the Stackdriver monitoring package
+  stackdriver-agent, run:
+
+        $ {command} --inventory-filter="\
+        NOT(InstalledPackages:stackdriver-agent)"
+
+  To list all Windows instances with an installed qfe hotfix whose ID equals
+  KB4462930, run:
+
+        $ {command} --inventory-filter="\
+        InstalledPackages.qfe[].HotFixID=KB4462930"
+
+  To list all Windows instances with a wua update whose description contains the
+  word Security, run:
+
+        $ {command} --inventory-filter="\
+        InstalledPackages.wua[].Description:Security"
+
+  """