feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/objects/__init__.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Cloud Storage objects commands."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.UniverseCompatible
+class Objects(base.Group):
+  """Manage Cloud Storage objects."""
+

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/objects/add_iam_policy_binding.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of objects add-iam-policy-binding command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import api_factory
+from googlecloudsdk.api_lib.util import apis
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.iam import iam_util
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import iam_command_util
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.command_lib.storage.tasks import set_iam_policy_task
+
+
+@base.Hidden
+@base.UniverseCompatible
+class AddIamPolicyBinding(base.Command):
+  """Grant a principal access to an object."""
+
+  detailed_help = {
+      'DESCRIPTION':
+          """
+      Add an IAM policy binding to an object. For more information, see [Cloud
+      Identity and Access
+      Management](https://cloud.google.com/storage/docs/access-control/iam).
+      """,
+      'EXAMPLES':
+          """
+      To grant full control of OBJECT in BUCKET to the user
+      john.doe@example.com:
+
+        $ {command} gs://BUCKET/OBJECT --member=user:john.doe@example.com --role=roles/storage.legacyObjectOwner
+
+      To make OBJECT publicly readable:
+
+        $ {command} gs://BUCKET/OBJECT --member=AllUsers --role=roles/storage.legacyObjectReader
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'url', help='URL of bucket to add IAM policy binding to.')
+    iam_util.AddArgsForAddIamPolicyBinding(parser, add_condition=True)
+
+  def Run(self, args):
+    url_object = storage_url.storage_url_from_string(args.url)
+    errors_util.raise_error_if_not_cloud_object(args.command_path, url_object)
+    errors_util.raise_error_if_not_gcs(args.command_path, url_object)
+
+    policy = api_factory.get_api(url_object.scheme).get_object_iam_policy(
+        url_object.bucket_name, url_object.resource_name, url_object.generation)
+    return iam_command_util.add_iam_binding_to_resource(
+        args,
+        url_object,
+        apis.GetMessagesModule('storage', 'v1'),
+        policy,
+        set_iam_policy_task.SetObjectIamPolicyTask,
+    )

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/objects/compose.py

@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of objects compose command for Cloud Storage."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+
+import textwrap
+
+from googlecloudsdk.api_lib.storage import cloud_api
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import encryption_util
+from googlecloudsdk.command_lib.storage import errors
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import flags
+from googlecloudsdk.command_lib.storage import name_expansion
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.command_lib.storage import user_request_args_factory
+from googlecloudsdk.command_lib.storage.resources import resource_reference
+from googlecloudsdk.command_lib.storage.tasks import compose_objects_task
+
+
+_COMMAND_DESCRIPTION = """
+{command} creates a new object whose content is the concatenation
+of a given sequence of source objects in the same bucket.
+For more information, please see:
+[composite objects documentation](https://cloud.google.com/storage/docs/composite-objects).
+
+There is a limit (currently 32) to the number of components
+that can be composed in a single operation.
+"""
+_GA_EXAMPLES = """
+The following command creates a new object `target.txt` by concatenating
+`a.txt` and `b.txt`:
+
+  $ {command} gs://bucket/a.txt gs://bucket/b.txt gs://bucket/target.txt
+"""
+_ALPHA_EXAMPLES = """
+Contexts are merged from source objects on the composed object by default.
+However, the following command overwrites custom contexts on composed object:
+
+  $ {command} gs://bucket/a.txt gs://bucket/b.txt gs://bucket/target.txt \
+      --custom-contexts=key1=value1,key2=value2
+"""
+
+
+@base.UniverseCompatible
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+class Compose(base.Command):
+  """Concatenate a sequence of objects into a new composite object."""
+
+  detailed_help = {
+      'DESCRIPTION': _COMMAND_DESCRIPTION,
+      'EXAMPLES': _GA_EXAMPLES,
+  }
+
+  @classmethod
+  def Args(cls, parser):
+    parser.add_argument(
+        'source',
+        nargs='+',
+        help=textwrap.dedent("""\
+            The list of source objects that will be concatenated into a
+            single object."""))
+    parser.add_argument('destination', help='The destination object.')
+
+    flags.add_additional_headers_flag(parser)
+    flags.add_encryption_flags(parser, hidden=True)
+    flags.add_per_object_retention_flags(parser)
+    flags.add_precondition_flags(parser)
+
+    if cls.ReleaseTrack() == base.ReleaseTrack.ALPHA:
+      context_group = flags.get_object_context_group(parser)
+      flags.add_object_context_setter_flags(context_group)
+
+  def Run(self, args):
+    encryption_util.initialize_key_store(args)
+    if args.source:
+      destination_resource = resource_reference.UnknownResource(
+          storage_url.storage_url_from_string(args.destination))
+      for url_string in args.source:
+        source_url = storage_url.storage_url_from_string(url_string)
+        errors_util.raise_error_if_not_cloud_object(args.command_path,
+                                                    source_url)
+        if source_url.scheme is not destination_resource.storage_url.scheme:
+          raise errors.Error('Composing across providers is not supported.')
+    if (args.destination !=
+        destination_resource.storage_url.versionless_url_string):
+      raise errors.Error(
+          'Verison-specific URLs are not valid destinations because'
+          ' composing always results in creating an object with the'
+          ' latest generation.')
+
+    source_expansion_iterator = name_expansion.NameExpansionIterator(
+        args.source,
+        fields_scope=cloud_api.FieldsScope.NO_ACL,
+        recursion_requested=name_expansion.RecursionSetting.NO)
+
+    objects_to_compose = [
+        source.resource for source in source_expansion_iterator
+    ]
+
+    user_request_args = (
+        user_request_args_factory.get_user_request_args_from_command_args(
+            args, metadata_type=user_request_args_factory.MetadataType.OBJECT))
+
+    compose_objects_task.ComposeObjectsTask(
+        objects_to_compose,
+        destination_resource,
+        print_status_message=True,
+        user_request_args=user_request_args,
+    ).execute()
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
+class ComposeAlpha(Compose):
+  """Concatenate a sequence of objects into a new composite object."""
+
+  detailed_help = {
+      'DESCRIPTION': _COMMAND_DESCRIPTION,
+      'EXAMPLES': _GA_EXAMPLES + _ALPHA_EXAMPLES,
+  }

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/objects/describe.py

@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of objects describe command for getting info on objects."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import api_factory
+from googlecloudsdk.api_lib.storage import cloud_api
+from googlecloudsdk.api_lib.storage import request_config_factory
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import encryption_util
+from googlecloudsdk.command_lib.storage import errors
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import flags
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.command_lib.storage import wildcard_iterator
+from googlecloudsdk.command_lib.storage.resources import contexts_only_formatter
+from googlecloudsdk.command_lib.storage.resources import full_resource_formatter
+from googlecloudsdk.command_lib.storage.resources import gsutil_json_printer
+from googlecloudsdk.command_lib.storage.resources import resource_util
+
+_COMMAND_DESCRIPTION = """
+Describe a Cloud Storage object.
+"""
+_GA_EXAMPLES = """
+Describe a Google Cloud Storage object with the url
+"gs://bucket/my-object":
+
+  $ {command} gs://bucket/my-object
+
+Describe object with JSON formatting, only returning the "name" key:
+
+  $ {command} gs://bucket/my-object --format="json(name)"
+"""
+_ALPHA_EXAMPLES = """
+Describe only contexts attached to objects as key value pairs.
+
+  $ {command} gs://my-bucket/object --format=contextsonly
+"""
+
+
+@base.UniverseCompatible
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+class Describe(base.DescribeCommand):
+  """Describe a Cloud Storage object."""
+
+  detailed_help = {
+      'DESCRIPTION': _COMMAND_DESCRIPTION,
+      'EXAMPLES': _GA_EXAMPLES,
+  }
+
+  @classmethod
+  def Args(cls, parser):
+    parser.add_argument('url', help='Specifies URL of object to describe.')
+    flags.add_additional_headers_flag(parser)
+    flags.add_encryption_flags(parser, command_only_reads_data=True)
+    flags.add_fetch_encrypted_object_hashes_flag(parser, is_list=False)
+    flags.add_raw_display_flag(parser)
+    flags.add_soft_deleted_flag(parser)
+    gsutil_json_printer.GsutilJsonPrinter.Register()
+
+    if cls.ReleaseTrack() == base.ReleaseTrack.ALPHA:
+      contexts_only_formatter.ContextsOnlyPrinter.Register()
+
+  def Run(self, args):
+    encryption_util.initialize_key_store(args)
+    if wildcard_iterator.contains_wildcard(args.url):
+      raise errors.InvalidUrlError(
+          'Describe does not accept wildcards because it returns a single'
+          ' resource. Please use the `ls` or `objects list` command for'
+          ' retrieving multiple resources.')
+
+    url = storage_url.storage_url_from_string(args.url)
+    errors_util.raise_error_if_not_cloud_object(args.command_path, url)
+
+    client = api_factory.get_api(url.scheme)
+    resource = client.get_object_metadata(
+        url.bucket_name,
+        url.resource_name,
+        generation=url.generation,
+        fields_scope=cloud_api.FieldsScope.FULL,
+        soft_deleted=args.soft_deleted,
+    )
+
+    if (args.fetch_encrypted_object_hashes and
+        cloud_api.Capability.ENCRYPTION in client.capabilities and
+        not (resource.md5_hash and resource.crc32c_hash) and
+        resource.decryption_key_hash_sha256):
+      request_config = request_config_factory.get_request_config(
+          resource.storage_url,
+          decryption_key_hash_sha256=resource.decryption_key_hash_sha256,
+          error_on_missing_key=True)
+      final_resource = client.get_object_metadata(
+          resource.bucket,
+          resource.name,
+          fields_scope=cloud_api.FieldsScope.FULL,
+          generation=resource.generation,
+          request_config=request_config,
+          soft_deleted=args.soft_deleted,
+      )
+    else:
+      final_resource = resource
+
+    if args.format == contexts_only_formatter.CONTEXT_ONLY_PRINTER_FORMAT:
+      return final_resource
+
+    return resource_util.get_display_dict_for_resource(
+        final_resource,
+        full_resource_formatter.ObjectDisplayTitlesAndDefaults,
+        display_raw_keys=args.raw,
+    )
+
+
+@base.UniverseCompatible
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
+class DescribeAlpha(Describe):
+  """Describe a Cloud Storage object."""
+
+  detailed_help = {
+      'DESCRIPTION': _COMMAND_DESCRIPTION,
+      'EXAMPLES': _GA_EXAMPLES + _ALPHA_EXAMPLES,
+  }

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/objects/get_iam_policy.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of objects get-iam-policy command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import api_factory
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import iam_command_util
+from googlecloudsdk.command_lib.storage import storage_url
+
+
+@base.Hidden
+@base.UniverseCompatible
+class GetIamPolicy(base.Command):
+  """Get the access policy for an object."""
+
+  detailed_help = {
+      'DESCRIPTION':
+          """
+      *{command}* behaves similarly to *{parent_command} get-object-acl*, but
+      uses the IAM policy binding syntax in the output.
+      """,
+      'EXAMPLES':
+          """
+      To get the access policy for OBJECT in BUCKET:
+
+        $ {command} gs://BUCKET/OBJECT
+
+      To output the access policy for OBJECT in BUCKET to a file:
+
+        $ {command} gs://BUCKET/OBJECT > policy.txt
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument('url', help='Request IAM policy for this object.')
+
+  def Run(self, args):
+    url_object = storage_url.storage_url_from_string(args.url)
+    errors_util.raise_error_if_not_cloud_object(args.command_path, url_object)
+    errors_util.raise_error_if_not_gcs(args.command_path, url_object)
+    matching_url = iam_command_util.get_single_matching_url(args.url)
+    return api_factory.get_api(matching_url.scheme).get_object_iam_policy(
+        matching_url.bucket_name, matching_url.resource_name,
+        matching_url.generation)

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/objects/list.py

@@ -0,0 +1,197 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2022 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of objects list command for getting info on objects."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import cloud_api
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import encryption_util
+from googlecloudsdk.command_lib.storage import errors
+from googlecloudsdk.command_lib.storage import flags
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.command_lib.storage import wildcard_iterator
+from googlecloudsdk.command_lib.storage.resources import contexts_only_formatter
+from googlecloudsdk.command_lib.storage.resources import full_resource_formatter
+from googlecloudsdk.command_lib.storage.resources import gsutil_full_resource_formatter
+from googlecloudsdk.command_lib.storage.resources import resource_reference
+from googlecloudsdk.command_lib.storage.resources import resource_util
+from googlecloudsdk.core import log
+from googlecloudsdk.core.resource import resource_printer
+
+_COMMAND_DESCRIPTION = """
+List Cloud Storage objects.
+
+Bucket URLs like `gs://bucket` match all the objects inside a bucket,
+but `gs://b*` fails because it matches a list of buckets.
+"""
+_GA_EXAMPLES = """
+List all objects in bucket ``my-bucket'' within current directory level:
+
+  $ {command} gs://my-bucket
+
+List all objects across nested directories using wildcards (https://cloud.google.com/storage/docs/wildcards):
+
+  $ {command} gs://my-bucket/**
+
+List all objects in bucket beginning with ``o'', including objects across nested
+directories:
+
+  $ {command} gs://my-bucket/**/o*
+
+List all objects within current directory of bucket with JSON formatting, only
+returning the value of the ``name'' metadata field:
+
+  $ {command} gs://my-bucket --format="json(name)"
+"""
+_ALPHA_EXAMPLES = """
+List only contexts attached to objects as key value pairs.
+
+  $ {command} gs://my-bucket --format=contextsonly
+"""
+
+
+def _object_iterator(
+    url,
+    fetch_encrypted_object_hashes,
+    halt_on_empty_response,
+    next_page_token,
+    object_state,
+    list_filter,
+):
+  """Iterates through resources matching URL and filter out non-objects."""
+  for resource in wildcard_iterator.CloudWildcardIterator(
+      url,
+      error_on_missing_key=False,
+      fetch_encrypted_object_hashes=fetch_encrypted_object_hashes,
+      fields_scope=cloud_api.FieldsScope.FULL,
+      halt_on_empty_response=halt_on_empty_response,
+      next_page_token=next_page_token,
+      object_state=object_state,
+      list_filter=list_filter,
+  ):
+    if isinstance(resource, resource_reference.ObjectResource):
+      yield resource
+
+
+@base.UniverseCompatible
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+class List(base.ListCommand):
+  """Lists Cloud Storage objects."""
+
+  detailed_help = {
+      'DESCRIPTION': _COMMAND_DESCRIPTION,
+      'EXAMPLES': _GA_EXAMPLES,
+  }
+
+  @classmethod
+  def Args(cls, parser):
+    parser.add_argument(
+        'urls', nargs='+', help='Specifies URL of objects to list.')
+    parser.add_argument(
+        '--stat',
+        action='store_true',
+        help='Emulates gsutil stat-style behavior. Does not show past object'
+        ' versions and changes output format.')
+    flags.add_additional_headers_flag(parser)
+    flags.add_encryption_flags(parser, command_only_reads_data=True)
+    flags.add_fetch_encrypted_object_hashes_flag(parser, is_list=True)
+    flags.add_raw_display_flag(parser)
+    flags.add_soft_delete_flags(parser)
+    flags.add_uri_support_to_list_commands(parser)
+
+    if cls.ReleaseTrack() == base.ReleaseTrack.ALPHA:
+      flags.add_metadata_filter_flag(parser)
+      contexts_only_formatter.ContextsOnlyPrinter.Register()
+
+  def Display(self, args, resources):
+    if args.stat:
+      resource_printer.Print(resources, 'object[terminator=""]')
+    else:
+      resource_printer.Print(resources, 'yaml')
+
+  def Run(self, args):
+    encryption_util.initialize_key_store(args)
+    metadata_filter = getattr(args, 'metadata_filter', None)
+
+    urls = []
+    for url_string in args.urls:
+      url = storage_url.storage_url_from_string(url_string)
+      if url.is_provider() or (url.is_bucket() and
+                               wildcard_iterator.contains_wildcard(
+                                   url.bucket_name)):
+        raise errors.InvalidUrlError(
+            'URL does not match objects: {}'.format(url_string))
+      if (
+          metadata_filter is not None
+          and url.scheme != storage_url.ProviderPrefix.GCS
+      ):
+        raise errors.InvalidUrlError(
+            'Metadata filter is only supported for GCS URLs.'
+        )
+      if url.is_bucket():
+        # Convert gs://bucket to gs://bucket/* to retrieve objects.
+        urls.append(url.join('*'))
+      else:
+        urls.append(url)
+
+    if not (args.stat or args.soft_deleted):
+      object_state = cloud_api.ObjectState.LIVE_AND_NONCURRENT
+    else:
+      object_state = flags.get_object_state_from_flags(args)
+    stat_formatter = (
+        gsutil_full_resource_formatter.GsutilFullResourceFormatter()
+    )
+    for url in urls:
+      object_iterator = _object_iterator(
+          url,
+          fetch_encrypted_object_hashes=args.fetch_encrypted_object_hashes,
+          halt_on_empty_response=not getattr(args, 'exhaustive', False),
+          next_page_token=getattr(args, 'next_page_token', None),
+          object_state=object_state,
+          list_filter=metadata_filter,
+      )
+      if args.stat:
+        # Replicating gsutil "stat" command behavior.
+        found_match = False
+        for resource in object_iterator:
+          found_match = True
+          yield stat_formatter.format_object(resource, show_acl=False)
+        if not found_match:
+          log.error('No URLs matched: ' + url.url_string)
+          self.exit_code = 1
+      else:
+        for resource in object_iterator:
+          if args.format == contexts_only_formatter.CONTEXT_ONLY_PRINTER_FORMAT:
+            yield resource
+          else:
+            yield resource_util.get_display_dict_for_resource(
+                resource,
+                full_resource_formatter.ObjectDisplayTitlesAndDefaults,
+                display_raw_keys=args.raw,
+            )
+
+
+@base.UniverseCompatible
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
+class ListAlpha(List):
+  """Lists Cloud Storage objects."""
+
+  detailed_help = {
+      'DESCRIPTION': _COMMAND_DESCRIPTION,
+      'EXAMPLES': _GA_EXAMPLES + _ALPHA_EXAMPLES,
+  }

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/objects/remove_iam_policy_binding.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of objects remove-iam-policy-binding command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import api_factory
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.iam import iam_util
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import iam_command_util
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.command_lib.storage.tasks import set_iam_policy_task
+
+
+@base.Hidden
+@base.UniverseCompatible
+class RemoveIamPolicyBinding(base.Command):
+  """Remove an IAM policy binding from an object."""
+
+  detailed_help = {
+      'DESCRIPTION':
+          """
+      *{command}* behaves similarly to
+      *{parent_command} remove-object-acl-grant*, but uses the IAM policy
+      binding syntax.
+      """,
+      'EXAMPLES':
+          """
+      To remove access equivalent to the IAM role of
+      roles/storage.legacyObjectOwner for the user john.doe@example.com on
+      OBJECT in BUCKET:
+
+        $ {command} gs://BUCKET/OBJECT --member=user:john.doe@example.com --role=roles/storage.legacyObjectOwner
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'url', help='URL of object to remove IAM policy binding from.')
+    iam_util.AddArgsForRemoveIamPolicyBinding(parser, add_condition=True)
+
+  def Run(self, args):
+    url_object = storage_url.storage_url_from_string(args.url)
+    errors_util.raise_error_if_not_cloud_object(args.command_path, url_object)
+    errors_util.raise_error_if_not_gcs(args.command_path, url_object)
+
+    client = api_factory.get_api(url_object.scheme)
+    policy = client.get_object_iam_policy(url_object.bucket_name,
+                                          url_object.resource_name,
+                                          url_object.generation)
+
+    return iam_command_util.remove_iam_binding_from_resource(
+        args, url_object, policy, set_iam_policy_task.SetObjectIamPolicyTask
+    )

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/objects/set_iam_policy.py

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of objects set-iam-policy command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import cloud_api
+from googlecloudsdk.api_lib.storage.gcs_json import metadata_field_converters
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.iam import iam_util
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import flags
+from googlecloudsdk.command_lib.storage import iam_command_util
+from googlecloudsdk.command_lib.storage import name_expansion
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.command_lib.storage.tasks import set_iam_policy_task
+
+
+def _set_iam_policy_task_iterator(url_strings, recurse, object_state, policy):
+  """Generates SetIamPolicyTask's for execution."""
+  if recurse:
+    recursion_requested = name_expansion.RecursionSetting.YES
+  else:
+    recursion_requested = name_expansion.RecursionSetting.NO_WITH_WARNING
+
+  for name_expansion_result in name_expansion.NameExpansionIterator(
+      url_strings,
+      fields_scope=cloud_api.FieldsScope.SHORT,
+      object_state=object_state,
+      recursion_requested=recursion_requested,
+  ):
+    yield set_iam_policy_task.SetObjectIamPolicyTask(
+        name_expansion_result.resource.storage_url, policy
+    )
+
+
+@base.Hidden
+@base.UniverseCompatible
+class SetIamPolicy(base.Command):
+  """Set access policy for an object."""
+
+  detailed_help = {
+      'DESCRIPTION':
+          """
+      *{command}* behaves similarly to *{parent_command} set-object-acl*, but
+      uses the IAM policy binding syntax.
+      """,
+      'EXAMPLES':
+          """
+      To set the access policy for OBJECT on BUCKET to the policy defined in
+      POLICY-FILE run:
+
+        $ {command} gs://BUCKET/OBJECT POLICY-FILE
+
+      To set the IAM policy in POLICY-FILE on all objects in all buckets
+      beginning with "b":
+
+        $ {command} -r gs://b* POLICY-FILE
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'urls',
+        nargs='+',
+        help='The URLs for objects whose access policy is being replaced.')
+    iam_util.AddArgForPolicyFile(parser)
+    parser.add_argument(
+        '--all-versions',
+        action='store_true',
+        help='Update the IAM policies of all versions of an object in a'
+        ' versioned bucket.')
+    parser.add_argument(
+        '-e',
+        '--etag',
+        help='Custom etag to set on IAM policy. API will reject etags that do'
+        ' not match this value, making it useful as a precondition during'
+        ' concurrent operations.')
+    parser.add_argument(
+        '-R',
+        '-r',
+        '--recursive',
+        action='store_true',
+        help='Recursively set the IAM policies of the contents of any'
+        ' directories that match the source path expression.')
+    flags.add_continue_on_error_flag(parser)
+
+  def Run(self, args):
+    for url_string in args.urls:
+      url = storage_url.storage_url_from_string(url_string)
+      if not args.recursive:
+        errors_util.raise_error_if_not_cloud_object(args.command_path, url)
+      errors_util.raise_error_if_not_gcs(args.command_path, url)
+
+    policy = metadata_field_converters.process_iam_file(
+        args.policy_file, custom_etag=args.etag)
+    exit_code, output = iam_command_util.execute_set_iam_task_iterator(
+        _set_iam_policy_task_iterator(
+            args.urls,
+            args.recursive,
+            flags.get_object_state_from_flags(args),
+            policy,
+        ),
+        args.continue_on_error,
+    )
+
+    self.exit_code = exit_code
+    return output

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/objects/update.py

@@ -0,0 +1,255 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of objects update command for updating object settings."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import cloud_api
+from googlecloudsdk.calliope import arg_parsers
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import encryption_util
+from googlecloudsdk.command_lib.storage import errors
+from googlecloudsdk.command_lib.storage import flags
+from googlecloudsdk.command_lib.storage import name_expansion
+from googlecloudsdk.command_lib.storage import stdin_iterator
+from googlecloudsdk.command_lib.storage import user_request_args_factory
+from googlecloudsdk.command_lib.storage.tasks import task_executor
+from googlecloudsdk.command_lib.storage.tasks import task_graph_executor
+from googlecloudsdk.command_lib.storage.tasks import task_status
+from googlecloudsdk.command_lib.storage.tasks.objects import patch_object_task
+from googlecloudsdk.command_lib.storage.tasks.objects import rewrite_object_task
+
+_COMMAND_DESCRIPTION = """
+Update Cloud Storage objects.
+"""
+
+_GA_EXAMPLES = """
+Update a Google Cloud Storage object's custom-metadata:
+
+  $ {command} gs://bucket/my-object --custom-metadata=key1=value1,key2=value2
+
+You can use [wildcards](https://cloud.google.com/storage/docs/wildcards)
+to update multiple objects in a single command. For instance to update
+all objects to have a custom-metadata key:
+
+  $ {command} gs://bucket/** --custom-metadata=key1=value1,key2=value2
+
+Rewrite all JPEG images to the NEARLINE storage class, including objects across
+nested directories:
+
+  $ {command} gs://bucket/**/*.jpg --storage-class=NEARLINE
+
+You can also provide a precondition on an object's metageneration in
+order to avoid potential race conditions:
+
+  $ {command} gs://bucket/*.jpg --storage-class=NEARLINE --if-metageneration-match=123456789
+"""
+
+_ALPHA_EXAMPLES = """
+The following command overwrites the custom contexts of an object:
+
+  $ {command} gs://bucket/my-object --custom-contexts=key1=value1,key2=value2
+
+The following example simultaneously updates and removes custom contexts, with
+remove being applied first:
+
+  $ {command} gs://bucket/my-object --update-custom-contexts=key2=value3 \
+      --remove-custom-contexts=key2,key3
+"""
+
+
+def _get_task_iterator(urls, args):
+  """Yields PatchObjectTask's or RewriteObjectTask's."""
+  requires_rewrite = (
+      args.encryption_key or args.clear_encryption_key or args.storage_class)
+  if requires_rewrite:
+    task_type = rewrite_object_task.RewriteObjectTask
+  else:
+    task_type = patch_object_task.PatchObjectTask
+
+  user_request_args = (
+      user_request_args_factory.get_user_request_args_from_command_args(
+          args, metadata_type=user_request_args_factory.MetadataType.OBJECT))
+  adds_or_removes_acls = user_request_args_factory.adds_or_removes_acls(
+      user_request_args
+  )
+  # TODO(b/292084011): Remove getattr when object lock is GA.
+  updates_retention = getattr(args, 'retain_until', None) or getattr(
+      args, 'retention_mode', None
+  )
+  updates_custom_contexts = (
+      getattr(args, 'custom_contexts', None)
+      or getattr(args, 'custom_contexts_file', None)
+      or getattr(args, 'remove_custom_contexts', None)
+      or getattr(args, 'update_custom_contexts', None)
+      or getattr(args, 'clear_custom_contexts', None)
+  )
+  if requires_rewrite or adds_or_removes_acls:
+    fields_scope = cloud_api.FieldsScope.FULL
+  elif updates_retention or updates_custom_contexts:
+    fields_scope = cloud_api.FieldsScope.NO_ACL
+  else:
+    fields_scope = cloud_api.FieldsScope.SHORT
+
+  if args.all_versions and not (
+      args.predefined_acl or args.acl_file or adds_or_removes_acls
+  ):
+    # TODO(b/264282236) Stop raising error once we confirm that this flag
+    # works fine with all types of object update operations.
+    raise errors.Error(
+        '--all_versions flag is only allowed for ACL modifier flags.')
+
+  if args.recursive:
+    recursion_setting = name_expansion.RecursionSetting.YES
+  else:
+    recursion_setting = name_expansion.RecursionSetting.NO
+  for name_expansion_result in name_expansion.NameExpansionIterator(
+      urls,
+      fields_scope=fields_scope,
+      include_buckets=name_expansion.BucketSetting.NO_WITH_ERROR,
+      object_state=flags.get_object_state_from_flags(args),
+      recursion_requested=recursion_setting,
+  ):
+    yield task_type(
+        name_expansion_result.resource, user_request_args=user_request_args
+    )
+
+
+def _add_common_args(parser, release_track=base.ReleaseTrack.GA):
+  """Register flags for this command.
+
+  Args:
+    parser (argparse.ArgumentParser): The parser to add the arguments to.
+    release_track (ReleaseTrack): The release track to add flags for.
+
+  Returns:
+    objects update flag group
+  """
+  parser.add_argument(
+      'url', nargs='*', help='Specifies URLs of objects to update.')
+
+  parser.add_argument(
+      '--all-versions',
+      action='store_true',
+      help='Perform the operation on all object versions.',
+  )
+
+  acl_flags_group = parser.add_group()
+  flags.add_acl_modifier_flags(acl_flags_group)
+  flags.add_preserve_acl_flag(acl_flags_group)
+
+  parser.add_argument(
+      '--event-based-hold',
+      action=arg_parsers.StoreTrueFalseAction,
+      help='Enables or disables an event-based hold on objects.')
+  parser.add_argument(
+      '-R',
+      '-r',
+      '--recursive',
+      action='store_true',
+      help='Recursively update objects under any buckets or directories that'
+      ' match the URL expression.')
+  parser.add_argument(
+      '-s',
+      '--storage-class',
+      help='Specify the storage class of the object. Using this flag triggers'
+      ' a rewrite of underlying object data.')
+  parser.add_argument(
+      '--temporary-hold',
+      action=arg_parsers.StoreTrueFalseAction,
+      help='Enables or disables a temporary hold on objects.')
+
+  flags.add_additional_headers_flag(parser)
+  flags.add_continue_on_error_flag(parser)
+  flags.add_encryption_flags(parser, allow_patch=True)
+  flags.add_precondition_flags(parser)
+  flags.add_object_metadata_flags(
+      parser, allow_patch=True, release_track=release_track
+  )
+  flags.add_per_object_retention_flags(parser, is_update=True)
+  flags.add_read_paths_from_stdin_flag(
+      parser,
+      help_text=(
+          'Read the list of objects to update from stdin. No need to enter'
+          ' a source argument if this flag is present.\nExample:'
+          ' "storage objects update -I --content-type=new-type"'
+      ),
+  )
+
+
+def _add_alpha_args(parser):
+  """Register flags for the alpha version of this command.
+
+  Args:
+    parser (argparse.ArgumentParser): The parser to add the arguments to.
+
+  Returns:
+    objects update flag group
+  """
+  del parser  # Unused.
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+@base.UniverseCompatible
+class Update(base.Command):
+  """Update Cloud Storage objects."""
+
+  detailed_help = {
+      'DESCRIPTION': _COMMAND_DESCRIPTION,
+      'EXAMPLES': _GA_EXAMPLES,
+  }
+
+  @staticmethod
+  def Args(parser):
+    _add_common_args(parser)
+
+  def Run(self, args):
+    encryption_util.initialize_key_store(args)
+    if not args.predefined_acl and args.preserve_acl is None:
+      # Preserve ACLs by default if nothing set by user.
+      args.preserve_acl = True
+
+    urls = stdin_iterator.get_urls_iterable(
+        args.url, args.read_paths_from_stdin
+    )
+    task_iterator = _get_task_iterator(urls, args)
+
+    task_status_queue = task_graph_executor.multiprocessing_context.Queue()
+    self.exit_code = task_executor.execute_tasks(
+        task_iterator,
+        parallelizable=True,
+        task_status_queue=task_status_queue,
+        progress_manager_args=task_status.ProgressManagerArgs(
+            increment_type=task_status.IncrementType.INTEGER,
+            manifest_path=None),
+        continue_on_error=args.continue_on_error,
+    )
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
+class UpdateAlpha(Update):
+  """Update Cloud Storage objects."""
+
+  detailed_help = {
+      'DESCRIPTION': _COMMAND_DESCRIPTION,
+      'EXAMPLES': _GA_EXAMPLES + _ALPHA_EXAMPLES,
+  }
+
+  @staticmethod
+  def Args(parser):
+    _add_common_args(parser, base.ReleaseTrack.ALPHA)
+    _add_alpha_args(parser)