feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/buckets/notifications/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Cloud Storage buckets notifications commands."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.UniverseCompatible
+class Notifications(base.Group):
+  """Manage Cloud Storage bucket notifications."""

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/buckets/notifications/create.py

@@ -0,0 +1,290 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2022 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of create command for notifications."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import functools
+import time
+
+from apitools.base.py import exceptions as apitools_exceptions
+
+from googlecloudsdk.api_lib.storage import api_factory
+from googlecloudsdk.api_lib.storage import cloud_api
+from googlecloudsdk.api_lib.storage import errors as api_errors
+from googlecloudsdk.api_lib.storage.gcs_json import error_util
+from googlecloudsdk.api_lib.util import apis
+from googlecloudsdk.calliope import arg_parsers
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import notification_configuration_iterator
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.core import log
+from googlecloudsdk.core import properties
+
+
+@error_util.catch_http_error_raise_gcs_api_error()
+def _maybe_create_or_modify_topic(topic_name, service_account_email):
+  """Ensures that topic with SA permissions exists, creating it if needed.
+
+  Args:
+    topic_name (str): Name of the Cloud Pub/Sub topic to use or create.
+    service_account_email (str): The project service account for Google Cloud
+      Storage. This SA needs publish permission on the PubSub topic.
+
+  Returns:
+    True if topic was created or had its IAM permissions modified.
+    Otherwise, False.
+  """
+  pubsub_client = apis.GetClientInstance('pubsub', 'v1')
+  pubsub_messages = apis.GetMessagesModule('pubsub', 'v1')
+
+  try:
+    pubsub_client.projects_topics.Get(
+        pubsub_messages.PubsubProjectsTopicsGetRequest(topic=topic_name))
+    log.warning('Topic already exists: ' + topic_name)
+    created_new_topic = False
+  except apitools_exceptions.HttpError as e:
+    if e.status_code != 404:
+      # Expect an Apitools NotFound error. Raise error otherwise.
+      raise
+    new_topic = pubsub_client.projects_topics.Create(
+        pubsub_messages.Topic(name=topic_name))
+    log.info('Created topic:\n{}'.format(new_topic))
+    created_new_topic = True
+
+  # Verify that the service account is in the IAM policy.
+  topic_iam_policy = pubsub_client.projects_topics.GetIamPolicy(
+      pubsub_messages.PubsubProjectsTopicsGetIamPolicyRequest(
+          resource=topic_name))
+  expected_binding = pubsub_messages.Binding(
+      role='roles/pubsub.publisher',
+      members=['serviceAccount:' + service_account_email])
+
+  # Can be improved by checking for roles stronger than "pubsub.publisher".
+  # We could also recurse up the hierarchy, checking project-level permissions.
+  # However, the caller may not have permission to perform this recursion.
+  # The trade-off of complexity for the benefit of not granting a redundant,
+  # permission is not worth it, so we grant "publisher" if a simple check fails.
+  if expected_binding not in topic_iam_policy.bindings:
+    topic_iam_policy.bindings.append(expected_binding)
+    updated_topic_iam_policy = pubsub_client.projects_topics.SetIamPolicy(
+        pubsub_messages.PubsubProjectsTopicsSetIamPolicyRequest(
+            resource=topic_name,
+            setIamPolicyRequest=pubsub_messages.SetIamPolicyRequest(
+                policy=topic_iam_policy)))
+    log.info('Updated topic IAM policy:\n{}'.format(updated_topic_iam_policy))
+    return True
+  else:
+    log.warning(
+        'Project service account {} already has publish permission for topic {}'
+        .format(service_account_email, topic_name))
+  return created_new_topic
+
+
+@base.UniverseCompatible
+class Create(base.Command):
+  """Create a notification configuration on a bucket."""
+
+  detailed_help = {
+      'DESCRIPTION':
+          """
+      *{command}* creates a notification configuration on a bucket,
+      establishing a flow of event notifications from Cloud Storage to a
+      Cloud Pub/Sub topic. As part of creating this flow, it also verifies
+      that the destination Cloud Pub/Sub topic exists, creating it if necessary,
+      and verifies that the Cloud Storage bucket has permission to publish
+      events to that topic, granting the permission if necessary.
+
+      If a destination Cloud Pub/Sub topic is not specified with the `-t` flag,
+      Cloud Storage chooses a topic name in the default project whose ID is
+      the same as the bucket name. For example, if the default project ID
+      specified is `default-project` and the bucket being configured is
+      `gs://example-bucket`, the create command uses the Cloud Pub/Sub topic
+      `projects/default-project/topics/example-bucket`.
+
+      In order to enable notifications, your project's
+      [Cloud Storage service agent](https://cloud.google.com/storage/docs/projects#service-accounts)
+      must have the IAM permission "pubsub.topics.publish".
+      This command checks to see if the destination Cloud Pub/Sub topic grants
+      the service agent this permission. If not, the create command attempts to
+      grant it.
+
+      A bucket can have up to 100 total notification configurations and up to
+      10 notification configurations set to trigger for a specific event.
+      """,
+      'EXAMPLES':
+          """
+      Send notifications of all changes to the bucket
+      `example-bucket` to the Cloud Pub/Sub topic
+      `projects/default-project/topics/example-bucket`:
+
+        $ {command} gs://example-bucket
+
+      The same as the above but sends no notification payload:
+
+        $ {command} --payload-format=none gs://example-bucket
+
+      Include custom metadata in notification payloads:
+
+        $ {command} --custom-attributes=key1:value1,key2:value2 gs://example-bucket
+
+      Create a notification configuration that only sends an event when a new
+      object has been created or an object is deleted:
+
+        $ {command} --event-types=OBJECT_FINALIZE,OBJECT_DELETE gs://example-bucket
+
+      Create a topic and notification configuration that sends events only when
+      they affect objects with the prefix `photos/`:
+
+        $ {command} --object-prefix=photos/ gs://example-bucket
+
+      Specifies the destination topic ID `files-to-process` in the default
+      project:
+
+        $ {command} --topic=files-to-process gs://example-bucket
+
+      The same as above but specifies a Cloud Pub/Sub topic belonging
+      to the specific cloud project `example-project`:
+
+        $ {command} --topic=projects/example-project/topics/files-to-process gs://example-bucket
+
+      Skip creating a topic when creating the notification configuraiton:
+
+        $ {command} --skip-topic-setup gs://example-bucket
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'url',
+        help='URL of the bucket to create the notification configuration'
+        ' on.')
+    parser.add_argument(
+        '-m',
+        '--custom-attributes',
+        metavar='KEY=VALUE',
+        type=arg_parsers.ArgDict(),
+        help='Specifies key:value attributes that are appended to the set of'
+        ' attributes sent to Cloud Pub/Sub for all events associated with'
+        ' this notification configuration.')
+    parser.add_argument(
+        '-e',
+        '--event-types',
+        metavar='NOTIFICATION_EVENT_TYPE',
+        type=arg_parsers.ArgList(
+            choices=sorted(
+                [status.value for status in cloud_api.NotificationEventType])),
+        help=(
+            'Specify event type filters for this notification configuration.'
+            ' Cloud Storage will send notifications of only these types. By'
+            ' default, Cloud Storage sends notifications for all event types.'
+            ' * OBJECT_FINALIZE: An object has been created.'
+            ' * OBJECT_METADATA_UPDATE: The metadata of an object has changed.'
+            ' * OBJECT_DELETE: An object has been permanently deleted.'
+            ' * OBJECT_ARCHIVE: A live version of an object has become a'
+            ' noncurrent version.'))
+    parser.add_argument(
+        '-p',
+        '--object-prefix',
+        help='Specifies a prefix path for this notification configuration.'
+        ' Cloud Storage will send notifications for only objects in the'
+        ' bucket whose names begin with the prefix.')
+    parser.add_argument(
+        '-f',
+        '--payload-format',
+        choices=sorted(
+            [status.value for status in cloud_api.NotificationPayloadFormat]),
+        default=cloud_api.NotificationPayloadFormat.JSON.value,
+        help='Specifies the payload format of notification messages.'
+        ' Notification details are available in the message attributes.'
+        " 'none' sends no payload.")
+    parser.add_argument(
+        '-s',
+        '--skip-topic-setup',
+        action='store_true',
+        help='Skips creation and permission assignment of the Cloud Pub/Sub'
+        ' topic. This is useful if the caller does not have permission to'
+        ' access the topic in question, or if the topic already exists and has'
+        ' the appropriate publish permission assigned.')
+    parser.add_argument(
+        '-t',
+        '--topic',
+        help='Specifies the Cloud Pub/Sub topic to send notifications to.'
+        ' If not specified, this command chooses a topic whose project is'
+        ' your default project and whose ID is the same as the'
+        ' Cloud Storage bucket name.')
+
+  def Run(self, args):
+    project_id = properties.VALUES.core.project.GetOrFail()
+    url = storage_url.storage_url_from_string(args.url)
+    notification_configuration_iterator.raise_error_if_not_gcs_bucket_matching_url(
+        url)
+    if not args.topic:
+      topic_name = 'projects/{}/topics/{}'.format(project_id, url.bucket_name)
+    elif not args.topic.startswith('projects/'):
+      # A topic ID may be present but not a whole path. Use the default project.
+      topic_name = 'projects/{}/topics/{}'.format(
+          project_id,
+          args.topic.rpartition('/')[-1])
+    else:
+      topic_name = args.topic
+
+    # Notifications supported for only GCS.
+    gcs_client = api_factory.get_api(storage_url.ProviderPrefix.GCS)
+    if not args.skip_topic_setup:
+      # Using generated topic name instead of custom one.
+      # Project number is different than project ID.
+      bucket_project_number = gcs_client.get_bucket(
+          url.bucket_name).metadata.projectNumber
+      # Fetch the email of the service account that will need access to
+      # the new pubsub topic.
+      service_account_email = gcs_client.get_service_agent(
+          project_number=bucket_project_number)
+      log.info(
+          'Checking for topic {} with access for project {} service account {}.'
+          .format(topic_name, project_id, service_account_email))
+      created_new_topic_or_set_new_permissions = _maybe_create_or_modify_topic(
+          topic_name, service_account_email)
+    else:
+      created_new_topic_or_set_new_permissions = False
+
+    if args.event_types:
+      event_types = [
+          cloud_api.NotificationEventType(event_type)
+          for event_type in args.event_types
+      ]
+    else:
+      event_types = None
+    create_notification_configuration = functools.partial(
+        gcs_client.create_notification_configuration,
+        url,
+        topic_name,
+        custom_attributes=args.custom_attributes,
+        event_types=event_types,
+        object_name_prefix=args.object_prefix,
+        payload_format=cloud_api.NotificationPayloadFormat(args.payload_format))
+    try:
+      return create_notification_configuration()
+    except api_errors.CloudApiError:
+      if not created_new_topic_or_set_new_permissions:
+        raise
+      log.warning(
+          'Retrying create notification request because topic changes may'
+          ' take up to 10 seconds to process.')
+      time.sleep(10)
+      return create_notification_configuration()

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/buckets/notifications/delete.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2022 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to delete notification configurations."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import notification_configuration_iterator
+from googlecloudsdk.command_lib.storage.tasks import task_executor
+from googlecloudsdk.command_lib.storage.tasks import task_graph_executor
+from googlecloudsdk.command_lib.storage.tasks import task_status
+from googlecloudsdk.command_lib.storage.tasks.buckets.notifications import delete_notification_configuration_task
+
+
+def _delete_notification_configuration_task_iterator(urls):
+  """Creates delete tasks from notification_configuration_iterator."""
+  for notification_configuration_iterator_result in (
+      notification_configuration_iterator
+      .get_notification_configuration_iterator(urls)):
+    yield (delete_notification_configuration_task
+           .DeleteNotificationConfigurationTask(
+               notification_configuration_iterator_result.bucket_url,
+               notification_configuration_iterator_result
+               .notification_configuration.id))
+
+
+@base.UniverseCompatible
+class Delete(base.DeleteCommand):
+  """Delete notification configurations from a bucket."""
+
+  detailed_help = {
+      'DESCRIPTION':
+          """
+      *{command}* deletes notification configurations from a bucket. If a
+      notification configuration name is passed as a parameter, that
+      configuration alone is deleted. If a bucket name is passed, all
+      notification configurations associated with the bucket are deleted.
+
+      Cloud Pub/Sub topics associated with this notification configuration
+      are not deleted by this command. Those must be deleted separately,
+      for example with the command "gcloud pubsub topics delete".
+      """,
+      'EXAMPLES':
+          """
+      Delete a single notification configuration (with ID 3) in the
+      bucket `example-bucket`:
+
+        $ {command} projects/_/buckets/example-bucket/notificationConfigs/3
+
+      Delete all notification configurations in the bucket `example-bucket`:
+
+        $ {command} gs://example-bucket
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'urls',
+        nargs='+',
+        help='Specifies notification configuration names or buckets.')
+
+  def Run(self, args):
+    task_status_queue = task_graph_executor.multiprocessing_context.Queue()
+    task_executor.execute_tasks(
+        _delete_notification_configuration_task_iterator(args.urls),
+        parallelizable=True,
+        task_status_queue=task_status_queue,
+        progress_manager_args=task_status.ProgressManagerArgs(
+            increment_type=task_status.IncrementType.INTEGER,
+            manifest_path=None),
+    )

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/buckets/notifications/describe.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2022 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Command to show metadata of a notification configuration."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import api_factory
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import errors
+from googlecloudsdk.command_lib.storage import notification_configuration_iterator
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.core.resource import resource_projector
+
+
+@base.UniverseCompatible
+class Describe(base.DescribeCommand):
+  """Show metadata for a notification configuration."""
+
+  detailed_help = {
+      'DESCRIPTION':
+          """
+      *{command}* prints populated metadata for a notification configuration.
+      """,
+      'EXAMPLES':
+          """
+      Describe a single notification configuration (with ID 3) in the
+      bucket `example-bucket`:
+
+        $ {command} projects/_/buckets/example-bucket/notificationConfigs/3
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument('url', help='The url of the notification configuration')
+
+  def Run(self, args):
+    bucket_url, notification_id = (
+        notification_configuration_iterator
+        .get_bucket_url_and_notification_id_from_url(args.url))
+    if not (bucket_url and notification_id):
+      raise errors.InvalidUrlError(
+          'Received invalid notification configuration URL: ' + args.url)
+    return resource_projector.MakeSerializable(
+        api_factory.get_api(
+            storage_url.ProviderPrefix.GCS).get_notification_configuration(
+                bucket_url, notification_id))

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/buckets/notifications/list.py

@@ -0,0 +1,142 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2022 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Command to list notification configurations belonging to a bucket."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import notification_configuration_iterator
+from googlecloudsdk.core.resource import resource_printer
+from googlecloudsdk.core.resource import resource_projector
+
+
+_PUBSUB_DOMAIN_PREFIX_LENGTH = len('//pubsub.googleapis.com/')
+
+
+def _get_human_readable_notification(url, config):
+  """Returns pretty notification string."""
+  if config.custom_attributes:
+    custom_attributes_string = '\n\tCustom attributes:'
+    for attribute in config.custom_attributes.additionalProperties:
+      custom_attributes_string += '\n\t\t{}: {}'.format(
+          attribute.key, attribute.value
+      )
+  else:
+    custom_attributes_string = ''
+
+  if config.event_types or config.object_name_prefix:
+    filters_string = '\n\tFilters:'
+
+    if config.event_types:
+      filters_string += '\n\t\tEvent Types: {}'.format(
+          ', '.join(config.event_types)
+      )
+    if config.object_name_prefix:
+      filters_string += "\n\t\tObject name prefix: '{}'".format(
+          config.object_name_prefix
+      )
+  else:
+    filters_string = ''
+
+  return (
+      'projects/_/buckets/{bucket}/notificationConfigs/{notification}\n'
+      '\tCloud Pub/Sub topic: {topic}'
+      '{custom_attributes}{filters}\n\n'.format(
+          bucket=url.bucket_name,
+          notification=config.id,
+          topic=config.topic[_PUBSUB_DOMAIN_PREFIX_LENGTH:],
+          custom_attributes=custom_attributes_string,
+          filters=filters_string,
+      )
+  )
+
+
+@base.UniverseCompatible
+class List(base.ListCommand):
+  """List the notification configurations belonging to a given bucket."""
+
+  detailed_help = {
+      'DESCRIPTION':
+          """
+      *{command}* provides a list of notification configurations belonging to a
+      given bucket. The listed name of each configuration can be used
+      with the delete sub-command to delete that specific notification config.
+      """,
+      'EXAMPLES':
+          """
+      Fetch the list of notification configs for the bucket `example-bucket`:
+
+        $ {command} gs://example-bucket
+
+      Fetch the notification configs in all buckets matching a wildcard:
+
+        $ {command} gs://example-*
+
+      Fetch all of the notification configs for buckets in the default project:
+
+        $ {command}
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'urls',
+        nargs='*',
+        help='Google Cloud Storage bucket paths. The path must begin '
+        'with gs:// and may contain wildcard characters.')
+    parser.add_argument(
+        '--human-readable',
+        action='store_true',
+        # Used by shim. Could be public but don't want maintainence burden.
+        hidden=True,
+        help=(
+            'Prints notification information in a more descriptive,'
+            ' unstructured format.'
+        ),
+    )
+
+  def Display(self, args, resources):
+    if args.human_readable:
+      resource_printer.Print(resources, 'object')
+    else:
+      resource_printer.Print(resources, args.format or 'yaml')
+
+  def Run(self, args):
+    if not args.urls:
+      # Provider URL will fetch all notification configurations in project.
+      urls = ['gs://']
+    else:
+      urls = args.urls
+
+    # Not bucket URLs raise error in iterator.
+    for notification_configuration_iterator_result in (
+        notification_configuration_iterator
+        .get_notification_configuration_iterator(
+            urls, accept_notification_configuration_urls=False)):
+
+      url, config = notification_configuration_iterator_result
+      if args.human_readable:
+        yield _get_human_readable_notification(url, config)
+      else:
+        yield {
+            'Bucket URL': url.url_string,
+            'Notification Configuration': resource_projector.MakeSerializable(
+                config
+            ),
+        }