feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/batch_operations/__init__.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Cloud Storage batch operations commands."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.DefaultUniverseOnly
+class BatchOperations(base.Group):
+  """Manage Cloud Storage batch operations."""
+
+  def Filter(self, context, args):
+    # TODO(b/190541521):  Determine if command group works with project number
+    base.RequireProjectID(args)
+    del context, args

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/batch_operations/jobs/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Cloud Storage batch operations jobs."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.DefaultUniverseOnly
+class Jobs(base.Group):
+  """Manage Cloud Storage batch operations jobs."""

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/batch_operations/jobs/cancel.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of cancel command for batch operations jobs."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import storage_batch_operations_api
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage.batch_operations.jobs import resource_args
+from googlecloudsdk.core import log
+
+
+@base.DefaultUniverseOnly
+class Cancel(base.Command):
+  """Cancel a batch-operations job."""
+
+  detailed_help = {
+      "DESCRIPTION": """
+      Cancel the batch operation job.
+      """,
+      "EXAMPLES": """
+      To cancel a batch job with the name `my-job` in location `us-central1`:
+
+          $ {command} my-job --location=us-central1
+
+      To cancel the same batch job with fully specified name:
+
+          $ {command} projects/my-project/locations/us-central1/jobs/my-job
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    resource_args.add_batch_job_resource_arg(parser, "to cancel")
+
+  def Run(self, args):
+    job_name = args.CONCEPTS.batch_job.Parse().RelativeName()
+    storage_batch_operations_api.StorageBatchOperationsApi().cancel_batch_job(
+        job_name
+    )
+    log.status.Print("Canceled batch job: {}".format(job_name))

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/batch_operations/jobs/create.py

@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of create command for batch actions."""
+
+from googlecloudsdk.api_lib.storage import storage_batch_operations_api
+from googlecloudsdk.calliope import base as calliope_base
+from googlecloudsdk.command_lib.storage import flags
+from googlecloudsdk.command_lib.storage.batch_operations.jobs import resource_args
+from googlecloudsdk.core import log
+from googlecloudsdk.core.console import console_io
+
+
+@calliope_base.ReleaseTracks(calliope_base.ReleaseTrack.GA)
+@calliope_base.DefaultUniverseOnly
+class Create(calliope_base.Command):
+  """Create a new batch operation job."""
+
+  detailed_help = {
+      "DESCRIPTION": """
+      Create a batch operation job, allowing you to perform operations
+      such as deletion, updating metadata, and more on objects in a
+      serverless manner.
+      """,
+      "EXAMPLES": """
+      The following example command creates a batch job, named `my-job`,
+      that performs object deletion on bucket `my-bucket` for objects
+      specified in the manifest file `gs://my-bucket/manifest.csv`:
+
+          $ {command} my-job --bucket=my-bucket --manifest-location=gs://my-bucket/manifest.csv
+          --delete-object
+
+      The following example command creates a batch job, named `my-job`,
+      that updates object metadata `Content-Disposition` to `inline`,
+      `Content-Language` to `en`, and sets object retention mode to `locked`
+      on bucket `my-bucket` for objects with prefixes `prefix1` or `prefix2`:
+
+          $ {command} my-job --bucket=my-bucket --included-object-prefixes=prefix1,prefix2
+          --put-metadata=Content-Disposition=inline,Content-Language=en,Retain-Until=2025-01-01T00:00:00Z,Retention-Mode=locked
+
+      The following example command creates a batch job, named `my-job`,
+      that puts object event based hold on objects in bucket `my-bucket`
+      with logging config enabled for corresponding transform action and
+      succeeded and failed action states:
+
+          $ {command} my-job --bucket=my-bucket --put-object-event-based-hold
+          --put-metadata=Content-Disposition=inline,Content-Language=en
+          --log-actions=transform --log-action-states=succeeded,failed
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    resource_args.add_batch_job_resource_arg(parser, "to create")
+    flags.add_batch_jobs_flags(parser)
+
+  def Run(self, args):
+    # Prompts to confirm deletion if --delete-object is specified.
+    dry_run = getattr(args, "dry_run", False)
+    if args.delete_object and not dry_run:
+      delete_prompt = (
+          "This command will delete objects specified in the batch operation"
+          " job. Please ensure that you have soft delete enabled on the bucket"
+          " if you want to restore the objects within the retention duration."
+      )
+      console_io.PromptContinue(
+          message=delete_prompt,
+          cancel_on_no=True,
+      )
+    job_ref = args.CONCEPTS.batch_job.Parse()
+    storage_batch_operations_api.StorageBatchOperationsApi().create_batch_job(
+        args, job_ref.RelativeName()
+    )
+    log.status.Print("Created batch job: {}".format(job_ref.RelativeName()))
+
+
+@calliope_base.ReleaseTracks(calliope_base.ReleaseTrack.ALPHA)
+class CreateAlpha(Create):
+  """Create a new batch operation job."""
+  detailed_help = {
+      "DESCRIPTION": """
+      Create a batch operation job, allowing you to perform operations
+      such as deletion, updating metadata, and more on objects in a
+      serverless manner.
+      """,
+      "EXAMPLES": """
+      The following example command creates a batch job, named `my-job`,
+      that performs object deletion on bucket `my-bucket` for objects
+      specified in the manifest file `gs://my-bucket/manifest.csv`:
+
+          $ {command} my-job --bucket=my-bucket --manifest-location=gs://my-bucket/manifest.csv
+          --delete-object
+
+      The following example command creates a batch job, named `my-job`,
+      that performs object deletion on buckets `my-bucket-1` and
+      `my-bucket-2` for all objects in them:
+
+          $ {command} my-job \
+          --bucket-list=my-bucket-1,my-bucket-2 \
+          --included-object-prefixes='' --delete-object
+
+      The following example command creates a batch job, named `my-job`,
+      that updates object metadata `Content-Disposition` to `inline`,
+      `Content-Language` to `en`, and sets object retention mode to `locked`
+      on bucket `my-bucket` for objects with prefixes `prefix1` or `prefix2`:
+
+          $ {command} my-job --bucket=my-bucket --included-object-prefixes=prefix1,prefix2
+          --put-metadata=Content-Disposition=inline,Content-Language=en,Retain-Until=2025-01-01T00:00:00Z,Retention-Mode=locked
+
+      The following example command creates a batch job, named `my-job`,
+      that puts object event based hold on objects in bucket `my-bucket`
+      with logging config enabled for corresponding transform action and
+      succeeded and failed action states:
+
+          $ {command} my-job --bucket=my-bucket --put-object-event-based-hold
+          --put-metadata=Content-Disposition=inline,Content-Language=en
+          --log-actions=transform --log-action-states=succeeded,failed
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    resource_args.add_batch_job_resource_arg(parser, "to create")
+    flags.add_batch_jobs_flags(parser, track=calliope_base.ReleaseTrack.ALPHA)
+    flags.add_batch_jobs_dry_run_flag(parser)

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/batch_operations/jobs/delete.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of delete command for batch operations jobs."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import storage_batch_operations_api
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage.batch_operations.jobs import resource_args
+from googlecloudsdk.core import log
+
+
+@base.DefaultUniverseOnly
+class Delete(base.Command):
+  """Delete a batch-operations job."""
+
+  detailed_help = {
+      "DESCRIPTION": """
+      Delete the batch operation job.
+      """,
+      "EXAMPLES": """
+      To delete a batch job with the name `my-job` in location `us-central1`:
+
+          $ {command} my-job --location=us-central1
+
+      To delete the same batch job with fully specified name:
+
+          $ {command} projects/my-project/locations/us-central1/jobs/my-job
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    resource_args.add_batch_job_resource_arg(parser, "to delete")
+
+  def Run(self, args):
+    job_name = args.CONCEPTS.batch_job.Parse().RelativeName()
+    storage_batch_operations_api.StorageBatchOperationsApi().delete_batch_job(
+        job_name
+    )
+    log.status.Print("Deleted batch job: {}".format(job_name))

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/batch_operations/jobs/describe.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of describe command for batch operations jobs."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import storage_batch_operations_api
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.storage.batch_operations.jobs import resource_args
+
+
+@base.DefaultUniverseOnly
+class Describe(base.DescribeCommand):
+  """Describe a batch-operations job."""
+
+  detailed_help = {
+      "DESCRIPTION": """
+      Describe the batch operation job.
+      """,
+      "EXAMPLES": """
+      To describe a batch job with the name `my-job`:
+
+          $ {command} my-job
+
+      To describe the same batch job with fully specified name:
+
+          $ {command} projects/my-project/locations/global/jobs/my-job
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    resource_args.add_batch_job_resource_arg(parser, "to describe")
+
+  def Run(self, args):
+    job_ref = args.CONCEPTS.batch_job.Parse()
+    return (
+        storage_batch_operations_api.StorageBatchOperationsApi().get_batch_job(
+            job_ref.RelativeName()
+        )
+    )

gcloud auth application-default login/google-cloud-sdk/lib/surface/storage/batch_operations/jobs/list.py

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of list command for batch operations jobs."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.storage import storage_batch_operations_api
+from googlecloudsdk.calliope import base
+
+_SBO_CLH_LOCATION_GLOBAL = "global"
+
+
+def _TransformTransformation(job):
+  """Transform for the TRANSFORMATION field in the table output.
+
+  TRANSFORMATION is generated from the oneof field transformation.
+
+  Args:
+    job: job dictionary for transform
+
+  Returns:
+    A dictionary of the transformation type and its values.
+  """
+  transformation = {}
+  transform_types = [
+      "putObjectHold",
+      "deleteObject",
+      "putMetadata",
+      "rewriteObject",
+  ]
+  for transform in transform_types:
+    if transform in job:
+      transformation[transform] = job[transform]
+      return transformation
+
+
+def _TransformDryRun(job):
+  """Transform for the DRY_RUN field in the table output."""
+  return job.get("dry_run", False)
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+@base.DefaultUniverseOnly
+class List(base.ListCommand):
+  """List batch-operations jobs."""
+
+  detailed_help = {
+      "DESCRIPTION": """
+      List batch operation jobs.
+      """,
+      "EXAMPLES": """
+      To list all batch jobs:
+
+          $ {command}
+
+      To list all batch jobs with a page size of `10`:
+
+          $ {command} --page-size=10
+
+      To list a limit of `20` batch jobs:
+
+          $ {command} --limit=20
+
+      To list all batch jobs in `JSON` format:
+
+          $ {command} --format=json
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    base.URI_FLAG.RemoveFromParser(parser)
+    parser.display_info.AddFormat("""
+      table(
+        name.basename():wrap=20:label=BATCH_JOB_ID,
+        bucketList.buckets:wrap=20:label=SOURCE,
+        transformation():wrap=20:label=TRANSFORMATION,
+        createTime:wrap=20:label=CREATE_TIME,
+        counters:wrap=20:label=COUNTERS,
+        errorSummaries:wrap=20:label=ERROR_SUMMARIES,
+        state:wrap=20:label=STATE
+      )
+    """)
+    parser.display_info.AddTransforms({
+        "transformation": _TransformTransformation,
+    })
+
+  def Run(self, args):
+    return storage_batch_operations_api.StorageBatchOperationsApi().list_batch_jobs(
+        _SBO_CLH_LOCATION_GLOBAL, args.limit, args.page_size
+    )
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
+class ListAlpha(List):
+  """List batch-operations jobs."""
+
+  detailed_help = {
+      "DESCRIPTION": """
+      List batch operation jobs.
+      """,
+      "EXAMPLES": """
+      To list all batch jobs:
+
+          $ {command}
+
+      To list all batch jobs that are `not` dry run:
+
+          $ {command} --filter="dry_run=false"
+
+      To list all batch jobs with a page size of `10`:
+
+          $ {command} --page-size=10
+
+      To list a limit of `20` batch jobs:
+
+          $ {command} --limit=20
+
+      To list all batch jobs in `JSON` format:
+
+          $ {command} --format=json
+      """,
+  }
+
+  @staticmethod
+  def Args(parser):
+    base.URI_FLAG.RemoveFromParser(parser)
+    parser.display_info.AddFormat("""
+      table(
+        name.basename():wrap=20:label=BATCH_JOB_ID,
+        dryrun():wrap=20:label=DRY_RUN,
+        bucketList.buckets:wrap=20:label=SOURCE,
+        transformation():wrap=20:label=TRANSFORMATION,
+        createTime:wrap=20:label=CREATE_TIME,
+        counters:wrap=20:label=COUNTERS,
+        errorSummaries:wrap=20:label=ERROR_SUMMARIES,
+        state:wrap=20:label=STATE
+      )
+    """)
+    parser.display_info.AddTransforms({
+        "transformation": _TransformTransformation,
+        "dryrun": _TransformDryRun,
+    })