feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/containeranalysis/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/containeranalysis/filter_util.py

@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2020 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility for creating filters with containeranalysis API."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+# The default maximum number of resource URLs by which to filter when showing
+# occurrences. This is required since filtering by too many causes the
+# API request to be too large. Instead, the requests are chunkified.
+_DEFAULT_RESOURCE_URI_CHUNK_SIZE = 6
+
+
+class ContainerAnalysisFilter:
+  """Utility class for creating filters to send to containeranalysis API.
+
+  If passed to a request, only occurrences that have the resource prefix, is of
+  one of the kinds in self._kinds, is for one of the resources in self._resource
+  and satisfies self._custom_filter will be retrieved.
+
+  Properties:
+    resource_prefixes: list, the resource prefixes filter added to this filter.
+    custom_filter: str, the user provided filter added to this filter.
+    kinds: list, metadata kinds added to this filter.
+    resources: list, resource URLs added to this filter.
+  """
+
+  def __init__(self, max_resource_chunk_size=_DEFAULT_RESOURCE_URI_CHUNK_SIZE):
+    self._resource_prefixes = []
+    self._custom_filter = ''
+    self._kinds = []
+    self._resources = []
+    self._max_resource_chunk_size = max_resource_chunk_size
+
+  @property
+  def resource_prefixes(self):
+    return self._resource_prefixes
+
+  @property
+  def custom_filter(self):
+    return self._custom_filter
+
+  @property
+  def kinds(self):
+    return self._kinds
+
+  @property
+  def resources(self):
+    return self._resources
+
+  def WithKinds(self, kinds):
+    """Add metadata kinds to this filter."""
+    self._kinds = list(kinds)
+    return self
+
+  def WithResources(self, resources):
+    """Add resources to this filter."""
+    self._resources = list(resources)
+    return self
+
+  def WithCustomFilter(self, custom_filter):
+    """Add a custom filter to this filter."""
+    self._custom_filter = custom_filter
+    return self
+
+  def WithResourcePrefixes(self, resource_prefixes):
+    """Add resource prefixes to this filter."""
+    self._resource_prefixes = list(resource_prefixes)
+    return self
+
+  def GetFilter(self):
+    """Returns a filter string with filtering attributes set."""
+    kinds = _OrJoinFilters(*[_HasField('kind', k) for k in self._kinds])
+    resources = _OrJoinFilters(
+        *[_HasField('resourceUrl', r) for r in self._resources])
+    return _AndJoinFilters(
+        _HasPrefixes('resourceUrl', self.resource_prefixes), self.custom_filter,
+        kinds, resources)
+
+  def GetChunkifiedFilters(self):
+    r"""Returns a list of filter strings where each filter has an upper limit of resource filters.
+
+    The upper limit of resource filters in a contructed filter string is set
+    by self._max_resource_chunk_size. This is to avoid having too many
+    filters in one API request and getting the request rejected.
+
+
+    For example, consider this ContainerAnalysisFilter object:
+      ContainerAnalysisFilter() \
+        .WithKinds('VULNERABILITY') \
+        .WithResources([
+          'url/to/resources/1', 'url/to/resources/2', 'url/to/resources/3',
+          'url/to/resources/4', 'url/to/resources/5', 'url/to/resources/6'])
+
+    Calling GetChunkifiedFilters will return the following result:
+    [
+      '''(kind="VULNERABILITY") AND (resource_url="'url/to/resources/1)"
+       OR ("resource_url="'url/to/resources/2")
+       OR ("resource_url="'url/to/resources/3")
+       OR ("resource_url="'url/to/resources/4")
+       OR ("resource_url="'url/to/resources/5")''',
+      '(kind="VULNERABILITY") AND (resource_url="url/to/resources/6")'
+    ]
+    """
+    kinds = _OrJoinFilters(*[_HasField('kind', k) for k in self._kinds])
+    resources = [_HasField('resourceUrl', r) for r in self._resources]
+    base_filter = _AndJoinFilters(
+        _HasPrefixes('resourceUrl', self.resource_prefixes), self.custom_filter,
+        kinds)
+
+    if not resources:
+      return [base_filter]
+
+    chunks = [
+        resources[i:i + self._max_resource_chunk_size]
+        for i in range(0, len(resources), self._max_resource_chunk_size)
+    ]
+    return [
+        _AndJoinFilters(base_filter, _OrJoinFilters(*chunk)) for chunk in chunks
+    ]
+
+
+def _AndJoinFilters(*filters):
+  return ' AND '.join(['({})'.format(f) for f in filters if f])
+
+
+def _OrJoinFilters(*filters):
+  return ' OR '.join(['({})'.format(f) for f in filters if f])
+
+
+def _HasPrefixes(field, prefixes):
+  """Returns a filter string where each field is matched with the prefix.
+
+    _HasPrefixes is always an OR join, because multiple ANDs can just
+    resolve to the longest one, so multiple ANDs shouldn't be provided.
+
+    Note that there should never be more than 2 prefixes (one with and one
+    without https), as then there may be an issue with a request that's too
+    long. This can't be solved with chunking, as we need chunking for the
+    resource list itself, and since they're ANDed together, they can't be
+    chunked separately.
+
+  Args:
+    field: The field that must contain one of the given prefixes.
+    prefixes: The list of values of allowed prefixes.
+
+  Returns:
+    A filter string where each field is matched with the prefix.
+
+  Raises:
+    An ArtifactRegistryError if more than 2 prefixes are passed in.
+  """
+
+  if len(prefixes) > 2:
+    raise ValueError('Can only have at most 2 prefix filters.')
+  return ' OR '.join([
+      'has_prefix({}, "{}")'.format(field, prefix) for prefix in prefixes
+  ]) if prefixes else None
+
+
+def _HasField(field, value):
+  return '{} = "{}"'.format(field, value) if value else None

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/containeranalysis/requests.py

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2020 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility for making containeranalysis API calls."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import itertools
+
+from apitools.base.py import list_pager
+from googlecloudsdk.api_lib.util import apis
+from googlecloudsdk.core import resources
+
+
+def GetClient():
+  return apis.GetClientInstance('containeranalysis', 'v1')
+
+
+def GetMessages():
+  return apis.GetMessagesModule('containeranalysis', 'v1')
+
+
+def GetClientV1beta1():
+  return apis.GetClientInstance('containeranalysis', 'v1beta1')
+
+
+def GetMessagesV1beta1():
+  return apis.GetMessagesModule('containeranalysis', 'v1beta1')
+
+
+def ExportSbomV1beta1(project, uri):
+  """Export SBOM for AR image resources."""
+  client = GetClientV1beta1()
+  messages = GetMessagesV1beta1()
+  resource_ref = resources.REGISTRY.Create(
+      'containeranalysis.projects.resources',
+      projectsId=project,
+      resourcesId=uri
+  )
+  name = resource_ref.RelativeName()
+  req = messages.ContaineranalysisProjectsResourcesExportSBOMRequest(name=name)
+  return client.projects_resources.ExportSBOM(req)
+
+
+def ListOccurrencesV1beta1(project, res_filter, page_size=1000):
+  """List occurrences for resources in a project."""
+  client = GetClientV1beta1()
+  messages = GetMessagesV1beta1()
+  project_ref = resources.REGISTRY.Parse(
+      project, collection='cloudresourcemanager.projects'
+  )
+  return list_pager.YieldFromList(
+      client.projects_occurrences,
+      request=messages.ContaineranalysisProjectsOccurrencesListRequest(
+          parent=project_ref.RelativeName(), filter=res_filter
+      ),
+      field='occurrences',
+      batch_size=page_size,
+      batch_size_attribute='pageSize',
+  )
+
+
+def ListOccurrencesWithFiltersV1beta1(project, filters):
+  """List occurrences for resources in a project with multiple filters."""
+  results = [ListOccurrencesV1beta1(project, f) for f in filters]
+  return itertools.chain(*results)
+
+
+def ListOccurrences(project, res_filter, page_size=1000):
+  """List occurrences for resources in a project."""
+  client = GetClient()
+  messages = GetMessages()
+  project_ref = resources.REGISTRY.Parse(
+      project, collection='cloudresourcemanager.projects')
+  return list_pager.YieldFromList(
+      client.projects_occurrences,
+      request=messages.ContaineranalysisProjectsOccurrencesListRequest(
+          parent=project_ref.RelativeName(), filter=res_filter),
+      field='occurrences',
+      batch_size=page_size,
+      batch_size_attribute='pageSize')
+
+
+def ListOccurrencesWithFilters(project, filters):
+  """List occurrences for resources in a project with multiple filters."""
+  results = [ListOccurrences(project, f) for f in filters]
+  return itertools.chain(*results)
+
+
+def GetVulnerabilitySummary(project, res_filter):
+  """Get vulnerability summary for resources in a project."""
+  client = GetClient()
+  messages = GetMessages()
+  project_ref = resources.REGISTRY.Parse(
+      project, collection='cloudresourcemanager.projects')
+  req = (
+      messages
+      .ContaineranalysisProjectsOccurrencesGetVulnerabilitySummaryRequest(
+          parent=project_ref.RelativeName(), filter=res_filter))
+  return client.projects_occurrences.GetVulnerabilitySummary(req)
+
+
+def GetVulnerabilitySummaryWithFilters(project, filters):
+  """Get vulnerability summary for resources in a project with multiple filters."""
+  return [GetVulnerabilitySummary(project, f) for f in filters]

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/containeranalysis/util.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for the container analysis commands."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import itertools
+
+from apitools.base.py import list_pager
+from googlecloudsdk.api_lib.util import apis
+from googlecloudsdk.core import resources
+from six.moves import range
+
+# The maximum number of resource URLs by which to filter when showing
+# occurrences. This is required since filtering by too many causes the
+# API request to be too large. Instead, the requests are chunkified.
+_MAXIMUM_RESOURCE_URL_CHUNK_SIZE = 5
+
+
+def MakeOccurrenceRequest(
+    project_id, resource_filter, occurrence_filter=None, resource_urls=None):
+  """Helper function to make Fetch Occurrence Request."""
+  client = apis.GetClientInstance('containeranalysis', 'v1alpha1')
+  messages = apis.GetMessagesModule('containeranalysis', 'v1alpha1')
+  base_filter = resource_filter
+  if occurrence_filter:
+    base_filter = (
+        '({occurrence_filter}) AND ({base_filter})'.format(
+            occurrence_filter=occurrence_filter,
+            base_filter=base_filter))
+  project_ref = resources.REGISTRY.Parse(
+      project_id, collection='cloudresourcemanager.projects')
+
+  if not resource_urls:
+    # When there are no resource_urls to filter don't need to do chunkifying
+    # logic or add anything to the base filter.
+    return list_pager.YieldFromList(
+        client.projects_occurrences,
+        request=messages.ContaineranalysisProjectsOccurrencesListRequest(
+            parent=project_ref.RelativeName(), filter=base_filter),
+        field='occurrences',
+        batch_size=1000,
+        batch_size_attribute='pageSize')
+
+  # Occurrences are filtered by resource URLs. If there are more than roughly
+  # _MAXIMUM_RESOURCE_URL_CHUNK_SIZE resource urls in the API request, the
+  # request becomes too big and will be rejected. This block chunkifies the
+  # resource URLs list and issues multiple API requests to circumvent this
+  # limit. The resulting generators (from YieldFromList) are chained together in
+  # the final output.
+  occurrence_generators = []
+  for index in range(0, len(resource_urls), _MAXIMUM_RESOURCE_URL_CHUNK_SIZE):
+    chunk = resource_urls[index : (index + _MAXIMUM_RESOURCE_URL_CHUNK_SIZE)]
+    url_filter = '%s AND (%s)' % (
+        base_filter,
+        ' OR '.join(['resource_url="%s"' % url for url in chunk]))
+    occurrence_generators.append(
+        list_pager.YieldFromList(
+            client.projects_occurrences,
+            request=messages.ContaineranalysisProjectsOccurrencesListRequest(
+                parent=project_ref.RelativeName(), filter=url_filter),
+            field='occurrences',
+            batch_size=1000,
+            batch_size_attribute='pageSize'))
+  return itertools.chain(*occurrence_generators)
+
+
+def _GetNoteRef(note_name, default_project):
+  try:
+    return resources.REGISTRY.ParseRelativeName(
+        note_name, 'containeranalysis.providers.notes')
+  except resources.InvalidResourceException:
+    # Not a relative name, try to pase as URL or name + project.
+    return resources.REGISTRY.Parse(
+        note_name,
+        params={'providersId': default_project},
+        collection='containeranalysis.providers.notes')
+
+
+def MakeGetNoteRequest(note_name, default_project):
+  client = apis.GetClientInstance('containeranalysis', 'v1alpha1')
+  messages = apis.GetMessagesModule('containeranalysis', 'v1alpha1')
+  note_ref = _GetNoteRef(note_name, default_project)
+  request = messages.ContaineranalysisProvidersNotesGetRequest(
+      name=note_ref.RelativeName(),
+  )
+  return client.providers_notes.Get(request)