feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/datapipelines/pipeline/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The pipeline command group for the Data Pipelines CLI."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA)
+class Pipeline(base.Group):
+  """Manage Data Pipelines.
+
+  Commands for managing Data Pipelines resources.
+  """

gcloud auth application-default login/google-cloud-sdk/lib/surface/datapipelines/pipeline/create.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to create a Pipeline for the Data Pipelines API."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.datapipelines import util
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.datapipelines import flags
+
+_DETAILED_HELP = {
+    'DESCRIPTION':
+        '{description}',
+    'EXAMPLES':
+        """ \
+        To create a BATCH Data Pipeline ``PIPELINE_NAME'' in project ``example'' in region ``us-central1'', run:
+
+          $ {command} PIPELINE_NAME --project=example --region=us-central1
+              --pipeline-type=BATCH
+              --template-file-gcs-location='gs://path_to_template_file'
+              --parameters=inputFile="gs://path_to_input_file",output="gs://path_to_output_file"
+              --schedule="0 * * * *" --temp-location="gs://path_to_temp_location"
+        """,
+}
+
+
+@base.DefaultUniverseOnly
+@base.ReleaseTracks(base.ReleaseTrack.BETA)
+class Create(base.CreateCommand):
+  """Creates Data Pipelines Pipeline."""
+
+  detailed_help = _DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    flags.AddCreatePipelineFlags(parser)
+    flags.GetDisplayNameArg('Data Pipelines pipeline').AddToParser(parser)
+    flags.GetPipelineTypeArg(required=True).AddToParser(parser)
+    flags.GetTemplateTypeArg(required=False).AddToParser(parser)
+    flags.GetScheduleArg(required=False).AddToParser(parser)
+    flags.GetTimeZoneArg(required=False).AddToParser(parser)
+    flags.GetTemplateFileGcsLocationArg(required=False).AddToParser(parser)
+    flags.GetParametersArg(required=False).AddToParser(parser)
+    flags.GetMaxWorkersArg(required=False).AddToParser(parser)
+    flags.GetNumWorkersArg(required=False).AddToParser(parser)
+    flags.GetNetworkArg(required=False).AddToParser(parser)
+    flags.GetSubnetworkArg(required=False).AddToParser(parser)
+    flags.GetWorkerMachineTypeArg(required=False).AddToParser(parser)
+    flags.GetTempLocationArg(required=False).AddToParser(parser)
+    flags.GetDataflowKmsKeyArg(required=False).AddToParser(parser)
+    flags.GetDisablePublicIpsArg(required=False).AddToParser(parser)
+    flags.GetDataflowServiceAccountEmailArg(required=False).AddToParser(parser)
+    flags.GetSchedulerServiceAccountEmailArg(required=False).AddToParser(parser)
+    flags.GetEnableStreamingEngineArg(required=False).AddToParser(parser)
+    flags.GetAdditionalExperimentsArg(required=False).AddToParser(parser)
+    flags.GetAdditionalUserLabelsArg(required=False).AddToParser(parser)
+    flags.GetWorkerRegionArgs(required=False).AddToParser(parser)
+    flags.GetFlexRsGoalArg(required=False).AddToParser(parser)
+    flags.GetStreamingUpdateArgs(required=False).AddToParser(parser)
+
+  def Run(self, args):
+    """Run the create command."""
+    client = util.PipelinesClient()
+    pipelines_ref = args.CONCEPTS.pipeline.Parse()
+    region_ref = pipelines_ref.Parent()
+    return client.Create(
+        pipeline=pipelines_ref.RelativeName(),
+        parent=region_ref.RelativeName(),
+        args=args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/datapipelines/pipeline/delete.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to delete a Pipeline for the Data Pipelines API."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.datapipelines import util
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.datapipelines import flags
+
+_DETAILED_HELP = {
+    'DESCRIPTION':
+        '{description}',
+    'EXAMPLES':
+        """ \
+        To delete a Data Pipeline ``PIPELINE_NAME'' of project ``example'' in region ``us-central1'', run:
+
+          $ {command} PIPELINE_NAME --project=example --region=us-central1
+        """,
+}
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA)
+class Delete(base.DescribeCommand):
+  """Delete Data Pipelines Pipeline."""
+
+  detailed_help = _DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    flags.AddDescribePipelineFlags(parser)
+
+  def Run(self, args):
+    """Run the delete command."""
+    client = util.PipelinesClient()
+    pipelines_ref = args.CONCEPTS.pipeline.Parse()
+    return client.Delete(pipeline=pipelines_ref.RelativeName())

gcloud auth application-default login/google-cloud-sdk/lib/surface/datapipelines/pipeline/describe.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to describe a Pipeline for the Data Pipelines API."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.datapipelines import util
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.datapipelines import flags
+
+_DETAILED_HELP = {
+    'DESCRIPTION':
+        '{description}',
+    'EXAMPLES':
+        """ \
+        To describe a Data Pipeline ``PIPELINE_NAME'' of project ``example'' in region ``us-central1'', run:
+
+          $ {command} PIPELINE_NAME --project=example --region=us-central1
+        """,
+}
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA)
+class Describe(base.DescribeCommand):
+  """Describe Data Pipelines Pipeline."""
+
+  detailed_help = _DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    flags.AddDescribePipelineFlags(parser)
+
+  def Run(self, args):
+    """Run the describe command."""
+    client = util.PipelinesClient()
+    pipelines_ref = args.CONCEPTS.pipeline.Parse()
+    return client.Describe(pipeline=pipelines_ref.RelativeName())

gcloud auth application-default login/google-cloud-sdk/lib/surface/datapipelines/pipeline/job/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2022 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The job command group for the Data Pipelines CLI."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA)
+class Job(base.Group):
+  """Manage Data Pipeline Jobs.
+
+  Commands for managing Job resources of Data Pipelines.
+  """

gcloud auth application-default login/google-cloud-sdk/lib/surface/datapipelines/pipeline/job/list.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2022 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to list all the Jobs in a given Pipeline."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.datapipelines import util
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.datapipelines import flags
+
+_DETAILED_HELP = {
+    'DESCRIPTION':
+        '{description}',
+    'EXAMPLES':
+        """ \
+        To list all the jobs within a pipeline ``pipeline-1'' in project ``project-1'' and region ``us-central1'', run:
+
+          $ {command} --pipeline=pipeline-1 --project=project-1 --region=us-central1
+        """,
+}
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA)
+class List(base.ListCommand):
+  """List Jobs within a pipeline in a specific project and region."""
+
+  detailed_help = _DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    parser.display_info.AddUriFunc(util.GetJobURI)
+    flags.AddListJobsFlags(parser)
+
+  def Run(self, args):
+    """Run the list command."""
+    client = util.JobsClient()
+    pipeline_ref = args.CONCEPTS.pipeline.Parse()
+    return client.List(
+        limit=args.limit,
+        page_size=args.page_size,
+        pipeline=pipeline_ref.RelativeName())

gcloud auth application-default login/google-cloud-sdk/lib/surface/datapipelines/pipeline/list.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to list all the Pipelines in a given project & location."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.datapipelines import util
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.datapipelines import flags
+
+_DETAILED_HELP = {
+    'DESCRIPTION':
+        '{description}',
+    'EXAMPLES':
+        """ \
+        To list all the Data Pipelines for project ``example'' in region ``us-central1'', run:
+
+          $ {command} --project=example --region=us-central1
+        """,
+}
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA)
+class List(base.ListCommand):
+  """List Pipelines in a project and location."""
+
+  detailed_help = _DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    parser.display_info.AddUriFunc(util.GetPipelineURI)
+    flags.AddRegionResourceArg(parser, 'to list pipelines')
+
+  def Run(self, args):
+    """Run the list command."""
+    client = util.PipelinesClient()
+    region_ref = args.CONCEPTS.region.Parse()
+    return client.List(
+        limit=args.limit,
+        page_size=args.page_size,
+        input_filter=args.filter,
+        region=region_ref.RelativeName())

gcloud auth application-default login/google-cloud-sdk/lib/surface/datapipelines/pipeline/run.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to run a Pipeline for the Data Pipelines API."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.datapipelines import util
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.datapipelines import flags
+
+_DETAILED_HELP = {
+    'DESCRIPTION':
+        '{description}',
+    'EXAMPLES':
+        """ \
+        To run a Data Pipeline ``PIPELINE_NAME'' of project ``example'' in region ``us-central1'', run:
+
+          $ {command} PIPELINE_NAME --project=example --region=us-central1
+        """,
+}
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA)
+class Delete(base.DescribeCommand):
+  """Run Data Pipelines Pipeline."""
+
+  detailed_help = _DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    flags.AddStopPipelineFlags(parser)
+
+  def Run(self, args):
+    """Run the run command."""
+    client = util.PipelinesClient()
+    pipelines_ref = args.CONCEPTS.pipeline.Parse()
+    return client.Run(pipeline=pipelines_ref.RelativeName())

gcloud auth application-default login/google-cloud-sdk/lib/surface/datapipelines/pipeline/stop.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to stop a Pipeline for the Data Pipelines API."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.datapipelines import util
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.datapipelines import flags
+
+_DETAILED_HELP = {
+    'DESCRIPTION':
+        '{description}',
+    'EXAMPLES':
+        """ \
+        To stop a Data Pipeline ``PIPELINE_NAME'' of project ``example'' in region ``us-central1'', run:
+
+          $ {command} PIPELINE_NAME --project=example --region=us-central1
+        """,
+}
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA)
+class Delete(base.DescribeCommand):
+  """Stop Data Pipelines Pipeline."""
+
+  detailed_help = _DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    flags.AddStopPipelineFlags(parser)
+
+  def Run(self, args):
+    """Run the stop command."""
+    client = util.PipelinesClient()
+    pipelines_ref = args.CONCEPTS.pipeline.Parse()
+    return client.Stop(pipeline=pipelines_ref.RelativeName())

gcloud auth application-default login/google-cloud-sdk/lib/surface/datapipelines/pipeline/update.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to update a Pipeline for the Data Pipelines API."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.datapipelines import util
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.datapipelines import flags
+
+_DETAILED_HELP = {
+    'DESCRIPTION':
+        '{description}',
+    'EXAMPLES':
+        """ \
+        To update a Data Pipeline ``PIPELINE_NAME'' of project ``example'' in region ``us-central1'', run:
+
+          $ {command} abc PIPELINE_NAME --project=example --region=us-central1
+              --display-name="new_name"
+        """,
+}
+
+
+@base.DefaultUniverseOnly
+@base.ReleaseTracks(base.ReleaseTrack.BETA)
+class Update(base.UpdateCommand):
+  """Updates a Data Pipelines Pipeline."""
+
+  detailed_help = _DETAILED_HELP
+
+  @staticmethod
+  def Args(parser):
+    flags.AddUpdatePipelineFlags(parser)
+    flags.GetDisplayNameArg('Data Pipelines pipeline').AddToParser(parser)
+    flags.GetTemplateTypeArg(required=False).AddToParser(parser)
+    flags.GetScheduleArg(required=False).AddToParser(parser)
+    flags.GetTimeZoneArg(required=False).AddToParser(parser)
+    flags.GetTemplateFileGcsLocationArg(required=False).AddToParser(parser)
+    flags.GetParametersArg(required=False).AddToParser(parser)
+    flags.GetMaxWorkersArg(required=False).AddToParser(parser)
+    flags.GetNumWorkersArg(required=False).AddToParser(parser)
+    flags.GetNetworkArg(required=False).AddToParser(parser)
+    flags.GetSubnetworkArg(required=False).AddToParser(parser)
+    flags.GetWorkerMachineTypeArg(required=False).AddToParser(parser)
+    flags.GetTempLocationArg(required=False).AddToParser(parser)
+    flags.GetDataflowKmsKeyArg(required=False).AddToParser(parser)
+    flags.GetDisablePublicIpsArg(required=False).AddToParser(parser)
+    flags.GetDataflowServiceAccountEmailArg(required=False).AddToParser(parser)
+    flags.GetSchedulerServiceAccountEmailArg(required=False).AddToParser(parser)
+    flags.GetEnableStreamingEngineArg(required=False).AddToParser(parser)
+    flags.GetAdditionalExperimentsArg(required=False).AddToParser(parser)
+    flags.GetAdditionalUserLabelsArg(required=False).AddToParser(parser)
+    flags.GetWorkerRegionArgs(required=False).AddToParser(parser)
+    flags.GetFlexRsGoalArg(required=False).AddToParser(parser)
+    flags.GetStreamingUpdateArgs(required=False).AddToParser(parser)
+
+  def Run(self, args):
+    """Run the update command."""
+    client = util.PipelinesClient()
+    pipelines_ref = args.CONCEPTS.pipeline.Parse()
+    return client.Patch(pipeline=pipelines_ref.RelativeName(), args=args)