feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/batches/submit/__init__.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The command group for cloud dataproc batches submit."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc import flags
+from googlecloudsdk.command_lib.dataproc.batches import (
+    batches_create_request_factory)
+
+
+@base.DefaultUniverseOnly
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+class Submit(base.Group):
+  """Submit a Dataproc batch job."""
+  detailed_help = {
+      'EXAMPLES':
+          """\
+          To submit a PySpark job, run:
+
+            $ {command} pyspark my-pyspark.py --region='us-central1' --deps-bucket=gs://my-bucket --py-files='path/to/my/python/scripts'
+
+          To submit a Spark job, run:
+
+            $ {command} spark --region='us-central1' --deps-bucket=gs://my-bucket --jar='my_jar.jar' -- ARG1 ARG2
+
+          To submit a Spark-R job, run:
+
+            $ {command} spark-r my-main-r.r --region='us-central1' --deps-bucket=gs://my-bucket -- ARG1 ARG2
+
+          To submit a Spark-Sql job, run:
+
+            $ {command} spark-sql 'my-sql-script.sql' --region='us-central1' --deps-bucket=gs://my-bucket --vars='variable=value'
+        """
+  }
+
+  @staticmethod
+  def Args(parser):
+    flags.AddAsync(parser)
+    batches_create_request_factory.AddArguments(
+        parser, dp.Dataproc().api_version)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/batches/submit/pyspark.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a PySpark batch job."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc.batches import batch_submitter
+from googlecloudsdk.command_lib.dataproc.batches import pyspark_batch_factory
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+class PySpark(base.Command):
+  """Submit a PySpark batch job."""
+  detailed_help = {
+      'EXAMPLES':
+          """\
+          To submit a PySpark batch job called "my-batch" that runs "my-pyspark.py", run:
+
+            $ {command} my-pyspark.py --batch=my-batch --deps-bucket=gs://my-bucket --region=us-central1 --py-files='path/to/my/python/script.py'
+          """
+  }
+
+  @staticmethod
+  def Args(parser):
+    pyspark_batch_factory.AddArguments(parser)
+
+  def Run(self, args):
+    dataproc = dp.Dataproc(base.ReleaseTrack.GA)
+    pyspark_batch = pyspark_batch_factory.PySparkBatchFactory(
+        dataproc).UploadLocalFilesAndGetMessage(args)
+
+    return batch_submitter.Submit(pyspark_batch, dataproc, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/batches/submit/ray.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a Ray batch job."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc import flags
+from googlecloudsdk.command_lib.dataproc.batches import batch_submitter
+from googlecloudsdk.command_lib.dataproc.batches import ray_batch_factory
+
+
+@base.Hidden
+class Ray(base.Command):
+  """Submit a Ray batch job."""
+  detailed_help = {
+      'EXAMPLES':
+          """\
+          To submit a Ray batch job called "my-batch" that runs "my-ray.py", run:
+
+            $ {command} my-ray.py --batch=my-batch --deps-bucket=gs://my-bucket --location=us-central1
+          """
+  }
+
+  @staticmethod
+  def Args(parser):
+    ray_batch_factory.AddArguments(parser)
+    flags.AddLocationFlag(parser)
+
+  def Run(self, args):
+    dataproc = dp.Dataproc(base.ReleaseTrack.BETA)
+    ray_batch = ray_batch_factory.RayBatchFactory(
+        dataproc).UploadLocalFilesAndGetMessage(args)
+
+    return batch_submitter.Submit(ray_batch, dataproc, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/batches/submit/spark.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a Spark batch job."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc.batches import batch_submitter
+from googlecloudsdk.command_lib.dataproc.batches import spark_batch_factory
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+class Spark(base.Command):
+  """Submit a Spark batch job."""
+  detailed_help = {
+      'DESCRIPTION':
+          """\
+          Submit a Spark batch job.
+          """,
+      'EXAMPLES':
+          """\
+          To submit a Spark job, run:
+
+            $ {command} --region=us-central1 --jar=my_jar.jar --deps-bucket=gs://my-bucket -- ARG1 ARG2
+
+          To submit a Spark job that runs a specific class of a jar, run:
+
+            $ {command} --region=us-central1 --class=org.my.main.Class --jars=my_jar1.jar,my_jar2.jar --deps-bucket=gs://my-bucket -- ARG1 ARG2
+
+          To submit a Spark job that runs a jar installed on the cluster, run:
+
+            $ {command} --region=us-central1 --class=org.apache.spark.examples.SparkPi --deps-bucket=gs://my-bucket --jars=file:///usr/lib/spark/examples/jars/spark-examples.jar -- 15
+          """
+  }
+
+  @staticmethod
+  def Args(parser):
+    spark_batch_factory.AddArguments(parser)
+
+  def Run(self, args):
+    dataproc = dp.Dataproc(base.ReleaseTrack.GA)
+    spark_batch = spark_batch_factory.SparkBatchFactory(
+        dataproc).UploadLocalFilesAndGetMessage(args)
+
+    return batch_submitter.Submit(spark_batch, dataproc, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/batches/submit/spark_r.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a SparkR batch job."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc.batches import batch_submitter
+from googlecloudsdk.command_lib.dataproc.batches import sparkr_batch_factory
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+class SparkR(base.Command):
+  """Submit a Spark R batch job."""
+  detailed_help = {
+      'EXAMPLES':
+          """\
+          To submit a Spark R batch job running "my-main-r.r" script and upload it to "gs://my-bucket", run:
+
+            $ {command} my-main-r.r --deps-bucket=gs://my-bucket --region='us-central1' -- ARG1 ARG2
+          """
+  }
+
+  @staticmethod
+  def Args(parser):
+    sparkr_batch_factory.AddArguments(parser)
+
+  def Run(self, args):
+    dataproc = dp.Dataproc(base.ReleaseTrack.GA)
+    sparkr_batch = sparkr_batch_factory.SparkRBatchFactory(
+        dataproc).UploadLocalFilesAndGetMessage(args)
+
+    return batch_submitter.Submit(sparkr_batch, dataproc, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/batches/submit/spark_sql.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a SparkSql batch job."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc.batches import batch_submitter
+from googlecloudsdk.command_lib.dataproc.batches import sparksql_batch_factory
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+class SparkSql(base.Command):
+  """Submit a Spark SQL batch job."""
+  detailed_help = {
+      'EXAMPLES':
+          """\
+          To submit a Spark SQL job running "my-sql-script.sql" and upload it to "gs://my-bucket", run:
+
+            $ {command} my-sql-script.sql --deps-bucket=gs://my-bucket --region=us-central1 --vars="NAME=VALUE,NAME2=VALUE2"
+          """
+  }
+
+  @staticmethod
+  def Args(parser):
+    sparksql_batch_factory.AddArguments(parser)
+
+  def Run(self, args):
+    dataproc = dp.Dataproc(base.ReleaseTrack.GA)
+    sparksql_batch = sparksql_batch_factory.SparkSqlBatchFactory(
+        dataproc).UploadLocalFilesAndGetMessage(args)
+
+    return batch_submitter.Submit(sparksql_batch, dataproc, args)