256 lines
8.7 KiB
Python
256 lines
8.7 KiB
Python
# -*- coding: utf-8 -*- #
|
|
# Copyright 2021 Google LLC. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Implementation of objects update command for updating object settings."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
from googlecloudsdk.api_lib.storage import cloud_api
|
|
from googlecloudsdk.calliope import arg_parsers
|
|
from googlecloudsdk.calliope import base
|
|
from googlecloudsdk.command_lib.storage import encryption_util
|
|
from googlecloudsdk.command_lib.storage import errors
|
|
from googlecloudsdk.command_lib.storage import flags
|
|
from googlecloudsdk.command_lib.storage import name_expansion
|
|
from googlecloudsdk.command_lib.storage import stdin_iterator
|
|
from googlecloudsdk.command_lib.storage import user_request_args_factory
|
|
from googlecloudsdk.command_lib.storage.tasks import task_executor
|
|
from googlecloudsdk.command_lib.storage.tasks import task_graph_executor
|
|
from googlecloudsdk.command_lib.storage.tasks import task_status
|
|
from googlecloudsdk.command_lib.storage.tasks.objects import patch_object_task
|
|
from googlecloudsdk.command_lib.storage.tasks.objects import rewrite_object_task
|
|
|
|
_COMMAND_DESCRIPTION = """
|
|
Update Cloud Storage objects.
|
|
"""
|
|
|
|
_GA_EXAMPLES = """
|
|
Update a Google Cloud Storage object's custom-metadata:
|
|
|
|
$ {command} gs://bucket/my-object --custom-metadata=key1=value1,key2=value2
|
|
|
|
You can use [wildcards](https://cloud.google.com/storage/docs/wildcards)
|
|
to update multiple objects in a single command. For instance to update
|
|
all objects to have a custom-metadata key:
|
|
|
|
$ {command} gs://bucket/** --custom-metadata=key1=value1,key2=value2
|
|
|
|
Rewrite all JPEG images to the NEARLINE storage class, including objects across
|
|
nested directories:
|
|
|
|
$ {command} gs://bucket/**/*.jpg --storage-class=NEARLINE
|
|
|
|
You can also provide a precondition on an object's metageneration in
|
|
order to avoid potential race conditions:
|
|
|
|
$ {command} gs://bucket/*.jpg --storage-class=NEARLINE --if-metageneration-match=123456789
|
|
"""
|
|
|
|
_ALPHA_EXAMPLES = """
|
|
The following command overwrites the custom contexts of an object:
|
|
|
|
$ {command} gs://bucket/my-object --custom-contexts=key1=value1,key2=value2
|
|
|
|
The following example simultaneously updates and removes custom contexts, with
|
|
remove being applied first:
|
|
|
|
$ {command} gs://bucket/my-object --update-custom-contexts=key2=value3 \
|
|
--remove-custom-contexts=key2,key3
|
|
"""
|
|
|
|
|
|
def _get_task_iterator(urls, args):
|
|
"""Yields PatchObjectTask's or RewriteObjectTask's."""
|
|
requires_rewrite = (
|
|
args.encryption_key or args.clear_encryption_key or args.storage_class)
|
|
if requires_rewrite:
|
|
task_type = rewrite_object_task.RewriteObjectTask
|
|
else:
|
|
task_type = patch_object_task.PatchObjectTask
|
|
|
|
user_request_args = (
|
|
user_request_args_factory.get_user_request_args_from_command_args(
|
|
args, metadata_type=user_request_args_factory.MetadataType.OBJECT))
|
|
adds_or_removes_acls = user_request_args_factory.adds_or_removes_acls(
|
|
user_request_args
|
|
)
|
|
# TODO(b/292084011): Remove getattr when object lock is GA.
|
|
updates_retention = getattr(args, 'retain_until', None) or getattr(
|
|
args, 'retention_mode', None
|
|
)
|
|
updates_custom_contexts = (
|
|
getattr(args, 'custom_contexts', None)
|
|
or getattr(args, 'custom_contexts_file', None)
|
|
or getattr(args, 'remove_custom_contexts', None)
|
|
or getattr(args, 'update_custom_contexts', None)
|
|
or getattr(args, 'clear_custom_contexts', None)
|
|
)
|
|
if requires_rewrite or adds_or_removes_acls:
|
|
fields_scope = cloud_api.FieldsScope.FULL
|
|
elif updates_retention or updates_custom_contexts:
|
|
fields_scope = cloud_api.FieldsScope.NO_ACL
|
|
else:
|
|
fields_scope = cloud_api.FieldsScope.SHORT
|
|
|
|
if args.all_versions and not (
|
|
args.predefined_acl or args.acl_file or adds_or_removes_acls
|
|
):
|
|
# TODO(b/264282236) Stop raising error once we confirm that this flag
|
|
# works fine with all types of object update operations.
|
|
raise errors.Error(
|
|
'--all_versions flag is only allowed for ACL modifier flags.')
|
|
|
|
if args.recursive:
|
|
recursion_setting = name_expansion.RecursionSetting.YES
|
|
else:
|
|
recursion_setting = name_expansion.RecursionSetting.NO
|
|
for name_expansion_result in name_expansion.NameExpansionIterator(
|
|
urls,
|
|
fields_scope=fields_scope,
|
|
include_buckets=name_expansion.BucketSetting.NO_WITH_ERROR,
|
|
object_state=flags.get_object_state_from_flags(args),
|
|
recursion_requested=recursion_setting,
|
|
):
|
|
yield task_type(
|
|
name_expansion_result.resource, user_request_args=user_request_args
|
|
)
|
|
|
|
|
|
def _add_common_args(parser, release_track=base.ReleaseTrack.GA):
|
|
"""Register flags for this command.
|
|
|
|
Args:
|
|
parser (argparse.ArgumentParser): The parser to add the arguments to.
|
|
release_track (ReleaseTrack): The release track to add flags for.
|
|
|
|
Returns:
|
|
objects update flag group
|
|
"""
|
|
parser.add_argument(
|
|
'url', nargs='*', help='Specifies URLs of objects to update.')
|
|
|
|
parser.add_argument(
|
|
'--all-versions',
|
|
action='store_true',
|
|
help='Perform the operation on all object versions.',
|
|
)
|
|
|
|
acl_flags_group = parser.add_group()
|
|
flags.add_acl_modifier_flags(acl_flags_group)
|
|
flags.add_preserve_acl_flag(acl_flags_group)
|
|
|
|
parser.add_argument(
|
|
'--event-based-hold',
|
|
action=arg_parsers.StoreTrueFalseAction,
|
|
help='Enables or disables an event-based hold on objects.')
|
|
parser.add_argument(
|
|
'-R',
|
|
'-r',
|
|
'--recursive',
|
|
action='store_true',
|
|
help='Recursively update objects under any buckets or directories that'
|
|
' match the URL expression.')
|
|
parser.add_argument(
|
|
'-s',
|
|
'--storage-class',
|
|
help='Specify the storage class of the object. Using this flag triggers'
|
|
' a rewrite of underlying object data.')
|
|
parser.add_argument(
|
|
'--temporary-hold',
|
|
action=arg_parsers.StoreTrueFalseAction,
|
|
help='Enables or disables a temporary hold on objects.')
|
|
|
|
flags.add_additional_headers_flag(parser)
|
|
flags.add_continue_on_error_flag(parser)
|
|
flags.add_encryption_flags(parser, allow_patch=True)
|
|
flags.add_precondition_flags(parser)
|
|
flags.add_object_metadata_flags(
|
|
parser, allow_patch=True, release_track=release_track
|
|
)
|
|
flags.add_per_object_retention_flags(parser, is_update=True)
|
|
flags.add_read_paths_from_stdin_flag(
|
|
parser,
|
|
help_text=(
|
|
'Read the list of objects to update from stdin. No need to enter'
|
|
' a source argument if this flag is present.\nExample:'
|
|
' "storage objects update -I --content-type=new-type"'
|
|
),
|
|
)
|
|
|
|
|
|
def _add_alpha_args(parser):
|
|
"""Register flags for the alpha version of this command.
|
|
|
|
Args:
|
|
parser (argparse.ArgumentParser): The parser to add the arguments to.
|
|
|
|
Returns:
|
|
objects update flag group
|
|
"""
|
|
del parser # Unused.
|
|
|
|
|
|
@base.ReleaseTracks(base.ReleaseTrack.GA)
|
|
@base.UniverseCompatible
|
|
class Update(base.Command):
|
|
"""Update Cloud Storage objects."""
|
|
|
|
detailed_help = {
|
|
'DESCRIPTION': _COMMAND_DESCRIPTION,
|
|
'EXAMPLES': _GA_EXAMPLES,
|
|
}
|
|
|
|
@staticmethod
|
|
def Args(parser):
|
|
_add_common_args(parser)
|
|
|
|
def Run(self, args):
|
|
encryption_util.initialize_key_store(args)
|
|
if not args.predefined_acl and args.preserve_acl is None:
|
|
# Preserve ACLs by default if nothing set by user.
|
|
args.preserve_acl = True
|
|
|
|
urls = stdin_iterator.get_urls_iterable(
|
|
args.url, args.read_paths_from_stdin
|
|
)
|
|
task_iterator = _get_task_iterator(urls, args)
|
|
|
|
task_status_queue = task_graph_executor.multiprocessing_context.Queue()
|
|
self.exit_code = task_executor.execute_tasks(
|
|
task_iterator,
|
|
parallelizable=True,
|
|
task_status_queue=task_status_queue,
|
|
progress_manager_args=task_status.ProgressManagerArgs(
|
|
increment_type=task_status.IncrementType.INTEGER,
|
|
manifest_path=None),
|
|
continue_on_error=args.continue_on_error,
|
|
)
|
|
|
|
|
|
@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
|
|
class UpdateAlpha(Update):
|
|
"""Update Cloud Storage objects."""
|
|
|
|
detailed_help = {
|
|
'DESCRIPTION': _COMMAND_DESCRIPTION,
|
|
'EXAMPLES': _GA_EXAMPLES + _ALPHA_EXAMPLES,
|
|
}
|
|
|
|
@staticmethod
|
|
def Args(parser):
|
|
_add_common_args(parser, base.ReleaseTrack.ALPHA)
|
|
_add_alpha_args(parser)
|