feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/managed_folders/__init__.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Cloud Storage managed folder commands."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.UniverseCompatible
+class ManagedFolders(base.Group):
+  """Manage Cloud Storage managed folders."""
+
+  def Filter(self, context, args):
+    # TODO(b/190541521):  Determine if command group works with project number
+    base.RequireProjectID(args)
+    del context, args

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/managed_folders/add_iam_policy_binding.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of managed-folders add-iam-policy-binding command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import api_factory
+from googlecloudsdk.api_lib.storage import errors as api_errors
+from googlecloudsdk.api_lib.util import apis
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.iam import iam_util
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import iam_command_util
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.command_lib.storage.tasks import set_iam_policy_task
+
+
+@base.UniverseCompatible
+class AddIamPolicyBinding(base.Command):
+  """Add an IAM policy binding to a managed folder."""
+
+  detailed_help = {
+      'DESCRIPTION': """
+      Add an IAM policy binding to a managed folder. For more information, see [Cloud
+      Identity and Access
+      Management](https://cloud.google.com/storage/docs/access-control/iam).
+      """,
+      'EXAMPLES': """
+      To grant a single role to a single principal for a managed folder `managed-folder` in a bucket `bucket`:
+
+        $ {command} gs://bucket/managed-folder --member=user:john.doe@example.com --role=roles/storage.objectCreator
+
+      To make objects in `gs://bucket/managed-folder` publicly readable:
+
+        $ {command} gs://bucket/managed-folder --member=allUsers --role=roles/storage.objectViewer
+
+      To specify a custom role for a principal on `gs://bucket/managed-folder`:
+
+        $ {command} gs://bucket/managed-folder --member=user:john.doe@example.com --role=roles/customRoleName
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'url', help='URL of the managed folder to add IAM policy binding to.'
+    )
+    iam_util.AddArgsForAddIamPolicyBinding(parser, add_condition=True)
+
+  def Run(self, args):
+    url = storage_url.storage_url_from_string(args.url)
+    errors_util.raise_error_if_not_gcs_folder_type(args.command_path, url)
+
+    api_client = api_factory.get_api(url.scheme)
+    messages = apis.GetMessagesModule('storage', 'v1')
+
+    try:
+      policy = api_client.get_managed_folder_iam_policy(
+          url.bucket_name, url.resource_name
+      )
+    except api_errors.NotFoundError:
+      api_client.create_managed_folder(url.bucket_name, url.resource_name)
+      policy = messages.Policy()
+
+    return iam_command_util.add_iam_binding_to_resource(
+        args,
+        url,
+        messages,
+        policy,
+        set_iam_policy_task.SetManagedFolderIamPolicyTask,
+    )

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/managed_folders/create.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of create command for making managed folders."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import api_factory
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import flags
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.core import log
+
+
+@base.UniverseCompatible
+class Create(base.Command):
+  """Create managed folders."""
+
+  detailed_help = {
+      'DESCRIPTION': 'Create managed folders.',
+      'EXAMPLES': """
+      The following command creates a managed folder called `folder/` in a bucket
+      named `my-bucket`:
+
+        $ {command} gs://my-bucket/folder/
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'url', type=str, nargs='+', help='The URLs of the folders to create.'
+    )
+
+    flags.add_additional_headers_flag(parser)
+
+  def Run(self, args):
+    urls = []
+    for url_string in args.url:
+      url = storage_url.storage_url_from_string(url_string)
+      errors_util.raise_error_if_not_gcs_folder_type(args.command_path, url)
+      urls.append(url)
+
+    for url in urls:
+      client = api_factory.get_api(url.scheme)
+      log.status.Print('Creating {}...'.format(url))
+      client.create_managed_folder(url.bucket_name, url.resource_name)

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/managed_folders/delete.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of command for deleting managed folders."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import flags
+from googlecloudsdk.command_lib.storage import folder_util
+from googlecloudsdk.command_lib.storage import name_expansion
+from googlecloudsdk.command_lib.storage import rm_command_util
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.command_lib.storage.tasks import task_graph_executor
+
+
+@base.UniverseCompatible
+class Delete(base.Command):
+  """Delete managed folders."""
+
+  detailed_help = {
+      'DESCRIPTION': """Delete managed folders.""",
+      'EXAMPLES': """
+      The following command deletes a managed folder named `folder` in a bucket
+      called `my-bucket`:
+
+        $ {command} gs://my-bucket/folder/
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'url',
+        type=str,
+        nargs='+',
+        help='The URLs of the managed folders to delete.',
+    )
+    flags.add_additional_headers_flag(parser)
+    flags.add_continue_on_error_flag(parser)
+
+  def Run(self, args):
+    for url_string in args.url:
+      url = storage_url.storage_url_from_string(url_string)
+      errors_util.raise_error_if_not_gcs_folder_type(args.command_path, url)
+
+    managed_folder_expansion_iterator = name_expansion.NameExpansionIterator(
+        args.url,
+        managed_folder_setting=folder_util.ManagedFolderSetting.LIST_WITHOUT_OBJECTS,
+        raise_error_for_unmatched_urls=True,
+    )
+    self.exit_code = rm_command_util.remove_managed_folders(
+        args,
+        managed_folder_expansion_iterator,
+        task_status_queue=task_graph_executor.multiprocessing_context.Queue(),
+        raise_error_for_unmatched_urls=True,
+    )

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/managed_folders/describe.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of command for describing managed folders."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import api_factory
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import flags
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.command_lib.storage.resources import full_resource_formatter
+from googlecloudsdk.command_lib.storage.resources import resource_util
+
+
+@base.UniverseCompatible
+class Describe(base.Command):
+  """Describe managed folders."""
+
+  detailed_help = {
+      'DESCRIPTION': """Describe managed folders.""",
+      'EXAMPLES': """
+      The following command shows information about a managed folder named
+      `folder` in a bucket called `my-bucket`:
+
+        $ {command} gs://my-bucket/folder/
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'url',
+        type=str,
+        help='The URL of the managed folder to describe.',
+    )
+    flags.add_additional_headers_flag(parser)
+    flags.add_raw_display_flag(parser)
+
+  def Run(self, args):
+    url = storage_url.storage_url_from_string(args.url)
+    errors_util.raise_error_if_not_gcs_folder_type(args.command_path, url)
+    client = api_factory.get_api(url.scheme)
+    resource = client.get_managed_folder(
+        url.bucket_name,
+        url.resource_name,
+    )
+    return resource_util.get_display_dict_for_resource(
+        resource,
+        full_resource_formatter.ManagedFolderDisplayTitlesAndDefaults,
+        display_raw_keys=args.raw,
+    )

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/managed_folders/get_iam_policy.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of managed-folders get-iam-policy command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import api_factory
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import storage_url
+
+
+@base.UniverseCompatible
+class GetIamPolicy(base.Command):
+  """Get the IAM policy for a managed folder."""
+
+  detailed_help = {
+      'DESCRIPTION': """
+      Get the IAM policy for a managed folder. For more information, see [Cloud
+      Identity and Access
+      Management](https://cloud.google.com/storage/docs/access-control/iam).
+      """,
+      'EXAMPLES': """
+      To get the IAM policy for a managed folder `managed-folder` in a bucket `bucket`:
+
+        $ {command} gs://bucket/managed-folder/
+
+      To output the IAM policy for `gs://bucket/managed-folder` to a file:
+
+        $ {command} gs://bucket/managed-folder/ > policy.txt
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'url', help='URL of the managed folder to get the IAM policy of.'
+    )
+
+  def Run(self, args):
+    url = storage_url.storage_url_from_string(args.url)
+    errors_util.raise_error_if_not_gcs_folder_type(args.command_path, url)
+    client = api_factory.get_api(url.scheme)
+    return client.get_managed_folder_iam_policy(
+        url.bucket_name, url.resource_name
+    )

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/managed_folders/list.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of command for listing managed folders."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import flags
+from googlecloudsdk.command_lib.storage import folder_util
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.command_lib.storage import wildcard_iterator
+from googlecloudsdk.command_lib.storage.resources import full_resource_formatter
+from googlecloudsdk.command_lib.storage.resources import resource_util
+
+
+@base.UniverseCompatible
+class List(base.ListCommand):
+  """List managed folders."""
+
+  detailed_help = {
+      'DESCRIPTION': """List managed folders.""",
+      'EXAMPLES': """
+      The following command lists all managed folders in a bucket:
+
+        $ {command} gs://my-bucket/
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'url', type=str, nargs='+', help='The URLs of the resources to list.'
+    )
+    flags.add_additional_headers_flag(parser)
+    flags.add_raw_display_flag(parser)
+
+  def Run(self, args):
+    urls = []
+    for url_string in args.url:
+      url = storage_url.storage_url_from_string(url_string)
+      errors_util.raise_error_if_not_gcs(args.command_path, url)
+      urls.append(url)
+
+    for url in urls:
+      for resource in wildcard_iterator.CloudWildcardIterator(
+          url.join('**'),
+          managed_folder_setting=folder_util.ManagedFolderSetting.LIST_WITHOUT_OBJECTS,
+      ):
+        yield resource_util.get_display_dict_for_resource(
+            resource,
+            full_resource_formatter.ManagedFolderDisplayTitlesAndDefaults,
+            display_raw_keys=args.raw,
+        )

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/managed_folders/remove_iam_policy_binding.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of managed-folders remove-iam-policy-binding command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import api_factory
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.iam import iam_util
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import iam_command_util
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.command_lib.storage.tasks import set_iam_policy_task
+
+
+@base.UniverseCompatible
+class RemoveIamPolicyBinding(base.Command):
+  """Remove an IAM policy binding from a managed folder."""
+
+  detailed_help = {
+      'DESCRIPTION': """
+      Remove a policy binding from the IAM policy of a managed folder, given a managed folder
+      URL and the binding. For more information, see [Cloud
+      Identity and Access
+      Management](https://cloud.google.com/storage/docs/access-control/iam).
+      """,
+      'EXAMPLES': """
+      To remove an IAM policy binding from the role of
+      roles/storage.objectCreator for the user john.doe@example.com on a managed folder `managed-folder` in a bucket `bucket`:
+
+        $ {command} gs://bucket/managed-folder --member=user:john.doe@example.com --role=roles/storage.objectCreator
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'url', help='URL of managed folder to remove IAM policy binding from.'
+    )
+    iam_util.AddArgsForRemoveIamPolicyBinding(parser, add_condition=True)
+
+  def Run(self, args):
+    url = storage_url.storage_url_from_string(args.url)
+    errors_util.raise_error_if_not_gcs_folder_type(args.command_path, url)
+    policy = api_factory.get_api(url.scheme).get_managed_folder_iam_policy(
+        url.bucket_name, url.resource_name
+    )
+    return iam_command_util.remove_iam_binding_from_resource(
+        args,
+        url,
+        policy,
+        set_iam_policy_task.SetManagedFolderIamPolicyTask,
+    )

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/managed_folders/set_iam_policy.py

@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of managed-folders set-iam-policy command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import collections
+
+from googlecloudsdk.api_lib.storage import api_factory
+from googlecloudsdk.api_lib.storage.gcs_json import metadata_field_converters
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.iam import iam_util
+from googlecloudsdk.command_lib.storage import errors_util
+from googlecloudsdk.command_lib.storage import flags
+from googlecloudsdk.command_lib.storage import folder_util
+from googlecloudsdk.command_lib.storage import iam_command_util
+from googlecloudsdk.command_lib.storage import name_expansion
+from googlecloudsdk.command_lib.storage import storage_url
+from googlecloudsdk.command_lib.storage import wildcard_iterator
+from googlecloudsdk.command_lib.storage.tasks import set_iam_policy_task
+from googlecloudsdk.core import log
+
+
+def _set_iam_policy_task_iterator(url_strings, policy):
+  """Generates SetIamPolicyTask's for execution."""
+  url_found_match_tracker = collections.OrderedDict()
+  for name_expansion_result in name_expansion.NameExpansionIterator(
+      url_strings,
+      managed_folder_setting=folder_util.ManagedFolderSetting.LIST_WITHOUT_OBJECTS,
+      raise_error_for_unmatched_urls=False,
+      url_found_match_tracker=url_found_match_tracker,
+  ):
+    yield set_iam_policy_task.SetManagedFolderIamPolicyTask(
+        name_expansion_result.resource.storage_url,
+        policy,
+    )
+
+  for url_string, found_match in url_found_match_tracker.items():
+    if found_match:
+      continue
+    if wildcard_iterator.contains_wildcard(url_string):
+      log.warning(
+          'Not creating managed folder for URL containing wildcard that did not'
+          ' match any managed folders: '
+          + url_string
+      )
+      continue
+    url = storage_url.storage_url_from_string(url_string)
+    api_factory.get_api(url.scheme).create_managed_folder(
+        url.bucket_name, url.resource_name
+    )
+    yield set_iam_policy_task.SetManagedFolderIamPolicyTask(url, policy)
+
+
+@base.UniverseCompatible
+class SetIamPolicy(base.Command):
+  """Set the IAM policy for a managed folder."""
+
+  detailed_help = {
+      'DESCRIPTION': """
+      Set the IAM policy for a managed folder. For more information, see [Cloud
+      Identity and Access
+      Management](https://cloud.google.com/storage/docs/access-control/iam).
+      """,
+      'EXAMPLES': """
+      To set the IAM policy in POLICY-FILE on a managed folder `managed-folder` in a bucket `bucket`:
+
+        $ {command} gs://bucket/managed-folder POLICY-FILE
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    parser.add_argument(
+        'urls',
+        nargs='+',
+        help='URLs for managed folders to apply the IAM policy to.',
+    )
+    parser.add_argument(
+        '-e',
+        '--etag',
+        help=(
+            'Custom etag to set on IAM policy. API will reject etags that do'
+            ' not match this value, making it useful as a precondition during'
+            ' concurrent operations.'
+        ),
+    )
+    iam_util.AddArgForPolicyFile(parser)
+    flags.add_continue_on_error_flag(parser)
+
+  def Run(self, args):
+    for url_string in args.urls:
+      url = storage_url.storage_url_from_string(url_string)
+      errors_util.raise_error_if_not_gcs_folder_type(args.command_path, url)
+    policy = metadata_field_converters.process_iam_file(
+        args.policy_file, custom_etag=args.etag
+    )
+    exit_code, output = iam_command_util.execute_set_iam_task_iterator(
+        _set_iam_policy_task_iterator(args.urls, policy), args.continue_on_error
+    )
+
+    self.exit_code = exit_code
+    return output