feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/eventarc/pipelines/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The pipelines command group for Eventarc."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+@base.DefaultUniverseOnly
+class Pipelines(base.Group):
+  """Manage Eventarc pipelines."""
+
+  category = base.SERVERLESS_CATEGORY

gcloud auth application-default login/google-cloud-sdk/lib/surface/eventarc/pipelines/create.py

@@ -0,0 +1,154 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to create a pipeline."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.eventarc import pipelines
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.eventarc import flags
+from googlecloudsdk.command_lib.util.args import labels_util
+from googlecloudsdk.core import log
+
+_DETAILED_HELP = {
+    'DESCRIPTION': '{description}',
+    'EXAMPLES': """ \
+        To create a new pipeline `my-pipeline` in location `us-central1` with its destination targeting HTTP endpoint URI 'https://example-endpoint.com', run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com'
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with an HTTP endpoint URI destination and a message binding template, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com',http_endpoint_message_binding_template='{"headers": {"new-header-key": "new-header-value"}}'
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with a Cloud Workflow destination `my-workflow`, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=workflow=my-workflow
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with a message bus destination `my-message-bus`, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=message_bus=my-message-bus
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with a Cloud Pub/Sub Topic destination `my-topic`, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=pubsub_topic=my-topic
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with a Cloud Workflow in project `example-project` and location `us-east1`, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=workflow=my-workflow,project=example-project,location=us-east1
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with an HTTP endpoint URI destination `https://example-endpoint.com` and a service account `example-service-account@example-project.gserviceaccount.iam.com` for OIDC authentication, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com',google_oidc_authentication_service_account=example-service-account@example-project.gserviceaccount.iam.com
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with an HTTP endpoint URI destination `https://example-endpoint.com` and a service account `example-service-account@example-project.gserviceaccount.iam.com` for OIDC authentication with audience `https://example.com`, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com',google_oidc_authentication_service_account=example-service-account@example-project.gserviceaccount.iam.com,google_oidc_authentication_audience='https://example.com'
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with an HTTP endpoint URI destination `https://example-endpoint.com` and a service account `example-service-account@example-project.gserviceaccount.iam.com` for OAuth token authentication, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com',oauth_token_authentication_service_account=example-service-account@example-project.gserviceaccount.iam.com
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with an HTTP endpoint URI destination `https://example-endpoint.com` and a service account `example-service-account@example-project.gserviceaccount.iam.com` for OAuth token authentication with scope `https://www.googleapis.com/auth/cloud-platform`, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com',oauth_token_authentication_service_account=example-service-account@example-project.gserviceaccount.iam.com,oauth_token_authentication_scope='https://www.googleapis.com/auth/cloud-platform'
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with an HTTP endpoint URI destination `https://example-endpoint.com` and the JSON input and output payload formats, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com',output_payload_format_json= --input-payload-format-json=
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with an HTTP endpoint URI destination `https://example-endpoint.com` and the Avro input and output payload formats, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com',output_payload_format_avro_schema_definition='{"type": "record", "name": "my_record", "fields": [{"name": "my_field", "type": "string"}]}' --input-payload-format-avro-schema-definition='{"type": "record", "name": "my_record", "fields": [{"name": "my_field", "type": "string"}]}'
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with an HTTP endpoint URI destination `https://example-endpoint.com` and the Protobuf input and output payload formats, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com',output_payload_format_protobuf_schema_definition='syntax = "proto3"; message Location { string home_address = 1; }' --input-payload-format-protobuf-schema-definition='syntax = "proto3"; message Location { string home_address = 1; }'
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with an HTTP endpoint URI destination `https://example-endpoint.com` and a transformation mediation, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com'--mediations=transformation_template='message.removeFields(["data.credit_card_number","data.ssn"])'
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with an HTTP endpoint URI destination `https://example-endpoint.com` and a INFO level logging configuration, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com'--logging_config=INFO
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with an HTTP endpoint URI destination `https://example-endpoint.com` and a custom retry policy, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com' --max-retry-attempts=10 --min-retry-delay=2s --max-retry-delay=64s
+
+        To create a new pipeline `my-pipeline` in location `us-central1` with an HTTP endpoint URI destination `https://example-endpoint.com` and a Cloud KMS CryptoKey, run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com'  --crypto-key=projects/PROJECT_ID/locations/KMS_LOCATION/keyRings/KEYRING/cryptoKeys/KEY
+        """,
+}
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+@base.DefaultUniverseOnly
+class Create(base.CreateCommand):
+  """Create an Eventarc pipeline."""
+
+  detailed_help = _DETAILED_HELP
+
+  @classmethod
+  def Args(cls, parser):
+    flags.AddPipelineResourceArg(
+        parser, 'The pipeline to create.', required=True
+    )
+    flags.AddPipelineDestinationsArg(parser, required=True)
+    flags.AddInputPayloadFormatArgs(parser)
+    flags.AddMediationsArg(parser)
+    flags.AddLoggingConfigArg(parser, 'The logging config of the pipeline.')
+    flags.AddPipelineRetryPolicyArgs(parser)
+    flags.AddCryptoKeyArg(parser, with_clear=False, hidden=False)
+    labels_util.AddCreateLabelsFlags(parser)
+    base.ASYNC_FLAG.AddToParser(parser)
+
+  def Run(self, args):
+    """Run the create command."""
+    client = pipelines.PipelineClientV1()
+    pipelines_ref = args.CONCEPTS.pipeline.Parse()
+
+    log.debug(
+        'Creating pipeline {} for project {} in location {}'.format(
+            pipelines_ref.pipelinesId,
+            pipelines_ref.projectsId,
+            pipelines_ref.locationsId,
+        )
+    )
+    operation = client.Create(
+        pipelines_ref,
+        client.BuildPipeline(
+            pipeline_ref=pipelines_ref,
+            destinations=args.destinations,
+            input_payload_format_json=args.input_payload_format_json,
+            input_payload_format_avro_schema_definition=args.input_payload_format_avro_schema_definition,
+            input_payload_format_protobuf_schema_definition=args.input_payload_format_protobuf_schema_definition,
+            mediations=args.mediations,
+            logging_config=args.logging_config,
+            max_retry_attempts=args.max_retry_attempts,
+            min_retry_delay=args.min_retry_delay,
+            max_retry_delay=args.max_retry_delay,
+            crypto_key_name=args.crypto_key,
+            labels=labels_util.ParseCreateArgs(args, client.LabelsValueClass()),
+        ),
+    )
+
+    if args.async_:
+      return operation
+    return client.WaitFor(operation, 'Creating', pipelines_ref)

gcloud auth application-default login/google-cloud-sdk/lib/surface/eventarc/pipelines/delete.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to delete the specified pipeline."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.eventarc import pipelines
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.eventarc import flags
+from googlecloudsdk.core.console import console_io
+
+_DETAILED_HELP = {
+    'DESCRIPTION': '{description}',
+    'EXAMPLES': """ \
+        To delete the pipeline `my-pipeline` in location `us-central1`, run:
+
+          $ {command} my-pipeline --location=us-central1
+        """,
+}
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+@base.DefaultUniverseOnly
+class Delete(base.DeleteCommand):
+  """Delete an Eventarc pipeline."""
+
+  detailed_help = _DETAILED_HELP
+
+  @classmethod
+  def Args(cls, parser):
+    flags.AddPipelineResourceArg(parser, 'Pipeline to delete.', required=True)
+    base.ASYNC_FLAG.AddToParser(parser)
+
+  def Run(self, args):
+    """Run the delete command."""
+    client = pipelines.PipelineClientV1()
+    pipeline_ref = args.CONCEPTS.pipeline.Parse()
+
+    console_io.PromptContinue(
+        message=(
+            'The following pipeline will be deleted.\n'
+            '[{name}] in location [{location}]'.format(
+                name=pipeline_ref.pipelinesId,
+                location=pipeline_ref.locationsId,
+            )
+        ),
+        throw_if_unattended=True,
+        cancel_on_no=True,
+    )
+    operation = client.Delete(pipeline_ref)
+
+    if args.async_:
+      return operation
+    return client.WaitFor(operation, 'Deleting', pipeline_ref)

gcloud auth application-default login/google-cloud-sdk/lib/surface/eventarc/pipelines/describe.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to describe the specified pipeline."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.eventarc import pipelines
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.eventarc import flags
+
+_DETAILED_HELP = {
+    'DESCRIPTION': '{description}',
+    'EXAMPLES': """ \
+        To describe the pipeline `my-pipeline` in location `us-central1`, run:
+
+          $ {command} my-pipeline --location=us-central1
+        """,
+}
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+@base.DefaultUniverseOnly
+class Describe(base.DescribeCommand):
+  """Describe an Eventarc pipeline."""
+
+  detailed_help = _DETAILED_HELP
+
+  @classmethod
+  def Args(cls, parser):
+    flags.AddPipelineResourceArg(parser, 'Pipeline to describe.', required=True)
+
+  def Run(self, args):
+    client = pipelines.PipelineClientV1()
+    pipeline_ref = args.CONCEPTS.pipeline.Parse()
+
+    return client.Get(pipeline_ref)

gcloud auth application-default login/google-cloud-sdk/lib/surface/eventarc/pipelines/list.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to list all pipelines in a project and location."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.eventarc import pipelines
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.eventarc import flags
+
+_DETAILED_HELP = {
+    "DESCRIPTION": "{description}",
+    "EXAMPLES": """\
+        To list all pipelines in location ``us-central1'', run:
+
+          $ {command} --location=us-central1
+
+        To list all pipelines in all locations, run:
+
+          $ {command} --location=-
+
+        or
+
+          $ {command}
+        """,
+}
+
+_FORMAT = """\
+table(
+    name.scope("pipelines"):label=NAME,
+    name.scope("locations").segment(0):label=LOCATION,
+    loggingConfig.logSeverity:label=LOGGING_CONFIG,
+    inputPayloadFormat():label=INPUT_PAYLOAD_FORMAT,
+    retryPolicy.maxAttempts:label=MAX_RETRY_ATTEMPTS,
+    retryPolicy.minRetryDelay:label=MIN_RETRY_DELAY,
+    retryPolicy.maxRetryDelay:label=MAX_RETRY_DELAY
+)
+"""
+
+
+def _InputPayloadFormat(pipeline):
+  """Generate an input payload format string for the pipeline."""
+  input_payload_format = pipeline.get("inputPayloadFormat")
+  if input_payload_format is None:
+    return "None"
+  if input_payload_format.get("json") is not None:
+    return "Json"
+  if input_payload_format.get("avro") is not None:
+    return "Avro"
+  if input_payload_format.get("protobuf") is not None:
+    return "Protobuf"
+  return "Unknown Format"
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+@base.DefaultUniverseOnly
+class List(base.ListCommand):
+  """List Eventarc pipelines."""
+
+  detailed_help = _DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    flags.AddLocationResourceArg(
+        parser,
+        "Location for which to list pipelines. This should be one of the"
+        " supported regions.",
+        required=False,
+        allow_aggregation=True,
+    )
+    flags.AddProjectResourceArg(parser)
+    parser.display_info.AddFormat(_FORMAT)
+    parser.display_info.AddUriFunc(pipelines.GetPipelineURI)
+    parser.display_info.AddTransforms({
+        "inputPayloadFormat": _InputPayloadFormat,
+    })
+
+  def Run(self, args):
+    client = pipelines.PipelineClientV1()
+    location_ref = args.CONCEPTS.location.Parse()
+    return client.List(location_ref, args.limit, args.page_size)

gcloud auth application-default login/google-cloud-sdk/lib/surface/eventarc/pipelines/update.py

@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to update the specified pipeline."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.eventarc import pipelines
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.eventarc import flags
+from googlecloudsdk.command_lib.util.args import labels_util
+from googlecloudsdk.core import log
+
+_DETAILED_HELP = {
+    'DESCRIPTION': '{description}',
+    'EXAMPLES': """ \
+        To update the pipeline `my-pipeline` with its destination targeting HTTP endpoint URI 'https://example-endpoint.com' and network attachment 'my-network-attachment', run:
+
+          $ {command} my-pipeline --location=us-central1 --destinations=http_endpoint_uri='https://example-endpoint.com',network_attachment=my-network-attachment
+        """,
+}
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+@base.DefaultUniverseOnly
+class Update(base.UpdateCommand):
+  """Update an Eventarc pipeline."""
+
+  detailed_help = _DETAILED_HELP
+
+  @classmethod
+  def Args(cls, parser):
+    flags.AddPipelineResourceArg(
+        parser, group_help_text='The pipeline to update.', required=True
+    )
+    flags.AddPipelineDestinationsArg(parser, required=False)
+    flags.AddInputPayloadFormatArgs(parser)
+    flags.AddMediationsArg(parser)
+    flags.AddLoggingConfigArg(parser, 'The logging config of the pipeline.')
+    flags.AddPipelineRetryPolicyArgs(parser)
+    flags.AddCryptoKeyArg(parser, with_clear=True, hidden=False)
+    labels_util.AddUpdateLabelsFlags(parser)
+
+    base.ASYNC_FLAG.AddToParser(parser)
+
+  def Run(self, args):
+    """Run the update command."""
+    client = pipelines.PipelineClientV1()
+    pipeline_ref = args.CONCEPTS.pipeline.Parse()
+
+    log.debug(
+        'Updating pipeline {} for project {} in location {}'.format(
+            pipeline_ref.pipelinesId,
+            pipeline_ref.projectsId,
+            pipeline_ref.locationsId,
+        )
+    )
+
+    original_pipeline = client.Get(pipeline_ref)
+    labels_update_result = labels_util.Diff.FromUpdateArgs(args).Apply(
+        client.LabelsValueClass(), original_pipeline.labels
+    )
+
+    update_mask = client.BuildUpdateMask(
+        destinations=args.IsSpecified('destinations'),
+        input_payload_format_json=args.IsSpecified('input_payload_format_json'),
+        input_payload_format_avro_schema_definition=args.IsSpecified(
+            'input_payload_format_avro_schema_definition'
+        ),
+        input_payload_format_protobuf_schema_definition=args.IsSpecified(
+            'input_payload_format_protobuf_schema_definition'
+        ),
+        mediations=args.IsSpecified('mediations'),
+        logging_config=args.IsSpecified('logging_config'),
+        max_retry_attempts=args.IsSpecified('max_retry_attempts'),
+        min_retry_delay=args.IsSpecified('min_retry_delay'),
+        max_retry_delay=args.IsSpecified('max_retry_delay'),
+        crypto_key=args.IsSpecified('crypto_key'),
+        clear_crypto_key=args.clear_crypto_key,
+        labels=labels_update_result.needs_update,
+    )
+
+    operation = client.Patch(
+        pipeline_ref,
+        client.BuildPipeline(
+            pipeline_ref=pipeline_ref,
+            destinations=args.destinations,
+            input_payload_format_json=args.input_payload_format_json,
+            input_payload_format_avro_schema_definition=args.input_payload_format_avro_schema_definition,
+            input_payload_format_protobuf_schema_definition=args.input_payload_format_protobuf_schema_definition,
+            mediations=args.mediations,
+            logging_config=args.logging_config,
+            max_retry_attempts=args.max_retry_attempts,
+            min_retry_delay=args.min_retry_delay,
+            max_retry_delay=args.max_retry_delay,
+            crypto_key_name=args.crypto_key,
+            labels=labels_update_result.GetOrNone(),
+        ),
+        update_mask,
+    )
+
+    if args.async_:
+      return operation
+    return client.WaitFor(operation, 'Updating', pipeline_ref)