291 lines
12 KiB
Python
291 lines
12 KiB
Python
# -*- coding: utf-8 -*- #
|
|
# Copyright 2022 Google LLC. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Implementation of create command for notifications."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
import functools
|
|
import time
|
|
|
|
from apitools.base.py import exceptions as apitools_exceptions
|
|
|
|
from googlecloudsdk.api_lib.storage import api_factory
|
|
from googlecloudsdk.api_lib.storage import cloud_api
|
|
from googlecloudsdk.api_lib.storage import errors as api_errors
|
|
from googlecloudsdk.api_lib.storage.gcs_json import error_util
|
|
from googlecloudsdk.api_lib.util import apis
|
|
from googlecloudsdk.calliope import arg_parsers
|
|
from googlecloudsdk.calliope import base
|
|
from googlecloudsdk.command_lib.storage import notification_configuration_iterator
|
|
from googlecloudsdk.command_lib.storage import storage_url
|
|
from googlecloudsdk.core import log
|
|
from googlecloudsdk.core import properties
|
|
|
|
|
|
@error_util.catch_http_error_raise_gcs_api_error()
|
|
def _maybe_create_or_modify_topic(topic_name, service_account_email):
|
|
"""Ensures that topic with SA permissions exists, creating it if needed.
|
|
|
|
Args:
|
|
topic_name (str): Name of the Cloud Pub/Sub topic to use or create.
|
|
service_account_email (str): The project service account for Google Cloud
|
|
Storage. This SA needs publish permission on the PubSub topic.
|
|
|
|
Returns:
|
|
True if topic was created or had its IAM permissions modified.
|
|
Otherwise, False.
|
|
"""
|
|
pubsub_client = apis.GetClientInstance('pubsub', 'v1')
|
|
pubsub_messages = apis.GetMessagesModule('pubsub', 'v1')
|
|
|
|
try:
|
|
pubsub_client.projects_topics.Get(
|
|
pubsub_messages.PubsubProjectsTopicsGetRequest(topic=topic_name))
|
|
log.warning('Topic already exists: ' + topic_name)
|
|
created_new_topic = False
|
|
except apitools_exceptions.HttpError as e:
|
|
if e.status_code != 404:
|
|
# Expect an Apitools NotFound error. Raise error otherwise.
|
|
raise
|
|
new_topic = pubsub_client.projects_topics.Create(
|
|
pubsub_messages.Topic(name=topic_name))
|
|
log.info('Created topic:\n{}'.format(new_topic))
|
|
created_new_topic = True
|
|
|
|
# Verify that the service account is in the IAM policy.
|
|
topic_iam_policy = pubsub_client.projects_topics.GetIamPolicy(
|
|
pubsub_messages.PubsubProjectsTopicsGetIamPolicyRequest(
|
|
resource=topic_name))
|
|
expected_binding = pubsub_messages.Binding(
|
|
role='roles/pubsub.publisher',
|
|
members=['serviceAccount:' + service_account_email])
|
|
|
|
# Can be improved by checking for roles stronger than "pubsub.publisher".
|
|
# We could also recurse up the hierarchy, checking project-level permissions.
|
|
# However, the caller may not have permission to perform this recursion.
|
|
# The trade-off of complexity for the benefit of not granting a redundant,
|
|
# permission is not worth it, so we grant "publisher" if a simple check fails.
|
|
if expected_binding not in topic_iam_policy.bindings:
|
|
topic_iam_policy.bindings.append(expected_binding)
|
|
updated_topic_iam_policy = pubsub_client.projects_topics.SetIamPolicy(
|
|
pubsub_messages.PubsubProjectsTopicsSetIamPolicyRequest(
|
|
resource=topic_name,
|
|
setIamPolicyRequest=pubsub_messages.SetIamPolicyRequest(
|
|
policy=topic_iam_policy)))
|
|
log.info('Updated topic IAM policy:\n{}'.format(updated_topic_iam_policy))
|
|
return True
|
|
else:
|
|
log.warning(
|
|
'Project service account {} already has publish permission for topic {}'
|
|
.format(service_account_email, topic_name))
|
|
return created_new_topic
|
|
|
|
|
|
@base.UniverseCompatible
|
|
class Create(base.Command):
|
|
"""Create a notification configuration on a bucket."""
|
|
|
|
detailed_help = {
|
|
'DESCRIPTION':
|
|
"""
|
|
*{command}* creates a notification configuration on a bucket,
|
|
establishing a flow of event notifications from Cloud Storage to a
|
|
Cloud Pub/Sub topic. As part of creating this flow, it also verifies
|
|
that the destination Cloud Pub/Sub topic exists, creating it if necessary,
|
|
and verifies that the Cloud Storage bucket has permission to publish
|
|
events to that topic, granting the permission if necessary.
|
|
|
|
If a destination Cloud Pub/Sub topic is not specified with the `-t` flag,
|
|
Cloud Storage chooses a topic name in the default project whose ID is
|
|
the same as the bucket name. For example, if the default project ID
|
|
specified is `default-project` and the bucket being configured is
|
|
`gs://example-bucket`, the create command uses the Cloud Pub/Sub topic
|
|
`projects/default-project/topics/example-bucket`.
|
|
|
|
In order to enable notifications, your project's
|
|
[Cloud Storage service agent](https://cloud.google.com/storage/docs/projects#service-accounts)
|
|
must have the IAM permission "pubsub.topics.publish".
|
|
This command checks to see if the destination Cloud Pub/Sub topic grants
|
|
the service agent this permission. If not, the create command attempts to
|
|
grant it.
|
|
|
|
A bucket can have up to 100 total notification configurations and up to
|
|
10 notification configurations set to trigger for a specific event.
|
|
""",
|
|
'EXAMPLES':
|
|
"""
|
|
Send notifications of all changes to the bucket
|
|
`example-bucket` to the Cloud Pub/Sub topic
|
|
`projects/default-project/topics/example-bucket`:
|
|
|
|
$ {command} gs://example-bucket
|
|
|
|
The same as the above but sends no notification payload:
|
|
|
|
$ {command} --payload-format=none gs://example-bucket
|
|
|
|
Include custom metadata in notification payloads:
|
|
|
|
$ {command} --custom-attributes=key1:value1,key2:value2 gs://example-bucket
|
|
|
|
Create a notification configuration that only sends an event when a new
|
|
object has been created or an object is deleted:
|
|
|
|
$ {command} --event-types=OBJECT_FINALIZE,OBJECT_DELETE gs://example-bucket
|
|
|
|
Create a topic and notification configuration that sends events only when
|
|
they affect objects with the prefix `photos/`:
|
|
|
|
$ {command} --object-prefix=photos/ gs://example-bucket
|
|
|
|
Specifies the destination topic ID `files-to-process` in the default
|
|
project:
|
|
|
|
$ {command} --topic=files-to-process gs://example-bucket
|
|
|
|
The same as above but specifies a Cloud Pub/Sub topic belonging
|
|
to the specific cloud project `example-project`:
|
|
|
|
$ {command} --topic=projects/example-project/topics/files-to-process gs://example-bucket
|
|
|
|
Skip creating a topic when creating the notification configuraiton:
|
|
|
|
$ {command} --skip-topic-setup gs://example-bucket
|
|
""",
|
|
}
|
|
|
|
@staticmethod
|
|
def Args(parser):
|
|
parser.add_argument(
|
|
'url',
|
|
help='URL of the bucket to create the notification configuration'
|
|
' on.')
|
|
parser.add_argument(
|
|
'-m',
|
|
'--custom-attributes',
|
|
metavar='KEY=VALUE',
|
|
type=arg_parsers.ArgDict(),
|
|
help='Specifies key:value attributes that are appended to the set of'
|
|
' attributes sent to Cloud Pub/Sub for all events associated with'
|
|
' this notification configuration.')
|
|
parser.add_argument(
|
|
'-e',
|
|
'--event-types',
|
|
metavar='NOTIFICATION_EVENT_TYPE',
|
|
type=arg_parsers.ArgList(
|
|
choices=sorted(
|
|
[status.value for status in cloud_api.NotificationEventType])),
|
|
help=(
|
|
'Specify event type filters for this notification configuration.'
|
|
' Cloud Storage will send notifications of only these types. By'
|
|
' default, Cloud Storage sends notifications for all event types.'
|
|
' * OBJECT_FINALIZE: An object has been created.'
|
|
' * OBJECT_METADATA_UPDATE: The metadata of an object has changed.'
|
|
' * OBJECT_DELETE: An object has been permanently deleted.'
|
|
' * OBJECT_ARCHIVE: A live version of an object has become a'
|
|
' noncurrent version.'))
|
|
parser.add_argument(
|
|
'-p',
|
|
'--object-prefix',
|
|
help='Specifies a prefix path for this notification configuration.'
|
|
' Cloud Storage will send notifications for only objects in the'
|
|
' bucket whose names begin with the prefix.')
|
|
parser.add_argument(
|
|
'-f',
|
|
'--payload-format',
|
|
choices=sorted(
|
|
[status.value for status in cloud_api.NotificationPayloadFormat]),
|
|
default=cloud_api.NotificationPayloadFormat.JSON.value,
|
|
help='Specifies the payload format of notification messages.'
|
|
' Notification details are available in the message attributes.'
|
|
" 'none' sends no payload.")
|
|
parser.add_argument(
|
|
'-s',
|
|
'--skip-topic-setup',
|
|
action='store_true',
|
|
help='Skips creation and permission assignment of the Cloud Pub/Sub'
|
|
' topic. This is useful if the caller does not have permission to'
|
|
' access the topic in question, or if the topic already exists and has'
|
|
' the appropriate publish permission assigned.')
|
|
parser.add_argument(
|
|
'-t',
|
|
'--topic',
|
|
help='Specifies the Cloud Pub/Sub topic to send notifications to.'
|
|
' If not specified, this command chooses a topic whose project is'
|
|
' your default project and whose ID is the same as the'
|
|
' Cloud Storage bucket name.')
|
|
|
|
def Run(self, args):
|
|
project_id = properties.VALUES.core.project.GetOrFail()
|
|
url = storage_url.storage_url_from_string(args.url)
|
|
notification_configuration_iterator.raise_error_if_not_gcs_bucket_matching_url(
|
|
url)
|
|
if not args.topic:
|
|
topic_name = 'projects/{}/topics/{}'.format(project_id, url.bucket_name)
|
|
elif not args.topic.startswith('projects/'):
|
|
# A topic ID may be present but not a whole path. Use the default project.
|
|
topic_name = 'projects/{}/topics/{}'.format(
|
|
project_id,
|
|
args.topic.rpartition('/')[-1])
|
|
else:
|
|
topic_name = args.topic
|
|
|
|
# Notifications supported for only GCS.
|
|
gcs_client = api_factory.get_api(storage_url.ProviderPrefix.GCS)
|
|
if not args.skip_topic_setup:
|
|
# Using generated topic name instead of custom one.
|
|
# Project number is different than project ID.
|
|
bucket_project_number = gcs_client.get_bucket(
|
|
url.bucket_name).metadata.projectNumber
|
|
# Fetch the email of the service account that will need access to
|
|
# the new pubsub topic.
|
|
service_account_email = gcs_client.get_service_agent(
|
|
project_number=bucket_project_number)
|
|
log.info(
|
|
'Checking for topic {} with access for project {} service account {}.'
|
|
.format(topic_name, project_id, service_account_email))
|
|
created_new_topic_or_set_new_permissions = _maybe_create_or_modify_topic(
|
|
topic_name, service_account_email)
|
|
else:
|
|
created_new_topic_or_set_new_permissions = False
|
|
|
|
if args.event_types:
|
|
event_types = [
|
|
cloud_api.NotificationEventType(event_type)
|
|
for event_type in args.event_types
|
|
]
|
|
else:
|
|
event_types = None
|
|
create_notification_configuration = functools.partial(
|
|
gcs_client.create_notification_configuration,
|
|
url,
|
|
topic_name,
|
|
custom_attributes=args.custom_attributes,
|
|
event_types=event_types,
|
|
object_name_prefix=args.object_prefix,
|
|
payload_format=cloud_api.NotificationPayloadFormat(args.payload_format))
|
|
try:
|
|
return create_notification_configuration()
|
|
except api_errors.CloudApiError:
|
|
if not created_new_topic_or_set_new_permissions:
|
|
raise
|
|
log.warning(
|
|
'Retrying create notification request because topic changes may'
|
|
' take up to 10 seconds to process.')
|
|
time.sleep(10)
|
|
return create_notification_configuration()
|