feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/__init__.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The command group for cloud dataproc jobs."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.UniverseCompatible
+@base.ReleaseTracks(
+    base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA, base.ReleaseTrack.GA
+)
+class Jobs(base.Group):
+  """Submit and manage Dataproc jobs.
+
+  Submit and manage Dataproc jobs.
+
+  ## EXAMPLES
+
+  To learn about the types of jobs that can be submitted, run:
+
+    $ {command} submit
+
+  To view the output of a job as it runs, run:
+
+    $ {command} wait job_id
+
+  To cancel an active job, run:
+
+    $ {command} kill job_id
+
+  To view the details of a job, run:
+
+    $ {command} describe job_id
+
+  To see the list of all jobs, run:
+
+    $ {command} list
+
+  To delete the record of an inactive job, run:
+
+    $ {command} delete job_id
+  """

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/delete.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Delete job command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.api_lib.dataproc import util
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc import flags
+from googlecloudsdk.core import log
+from googlecloudsdk.core.console import console_io
+
+
+class Delete(base.DeleteCommand):
+  """Delete the record of an inactive job.
+
+  Delete the record of an inactive job.
+
+  ## EXAMPLES
+
+  To delete the record of a job, run:
+
+    $ {command} job_id
+  """
+
+  @classmethod
+  def Args(cls, parser):
+    dataproc = dp.Dataproc(cls.ReleaseTrack())
+    flags.AddJobResourceArg(parser, 'delete', dataproc.api_version)
+
+  def Run(self, args):
+    dataproc = dp.Dataproc(self.ReleaseTrack())
+
+    job_ref = args.CONCEPTS.job.Parse()
+    request = dataproc.messages.DataprocProjectsRegionsJobsDeleteRequest(
+        projectId=job_ref.projectId,
+        region=job_ref.region,
+        jobId=job_ref.jobId)
+
+    console_io.PromptContinue(
+        message="The job '{0}' will be deleted.".format(args.job),
+        cancel_on_no=True,
+        cancel_string='Deletion aborted by user.')
+
+    dataproc.client.projects_regions_jobs.Delete(request)
+
+    def _GetJob(job_ref):
+      return dataproc.client.projects_regions_jobs.Get(
+          dataproc.messages.DataprocProjectsRegionsJobsGetRequest(
+              projectId=job_ref.projectId,
+              region=job_ref.region,
+              jobId=job_ref.jobId))
+
+    util.WaitForResourceDeletion(
+        _GetJob, job_ref, message='Waiting for job deletion')
+
+    log.DeletedResource(job_ref)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/describe.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Describe job command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc import flags
+
+
+class Describe(base.DescribeCommand):
+  """View the details of a job.
+
+  View the details of a job.
+
+  ## EXAMPLES
+
+  To view the details of a job, run:
+
+    $ {command} job_id
+  """
+
+  @classmethod
+  def Args(cls, parser):
+    dataproc = dp.Dataproc(cls.ReleaseTrack())
+    flags.AddJobResourceArg(parser, 'describe', dataproc.api_version)
+
+  def Run(self, args):
+    dataproc = dp.Dataproc(self.ReleaseTrack())
+
+    job_ref = args.CONCEPTS.job.Parse()
+
+    return dataproc.client.projects_regions_jobs.Get(
+        dataproc.messages.DataprocProjectsRegionsJobsGetRequest(
+            projectId=job_ref.projectId,
+            region=job_ref.region,
+            jobId=job_ref.jobId))

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/get_iam_policy.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Get IAM job policy command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.api_lib.dataproc import iam_helpers
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc import flags
+
+
+@base.UniverseCompatible
+@base.ReleaseTracks(
+    base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA, base.ReleaseTrack.GA
+)
+class GetIamPolicy(base.ListCommand):
+  """Get IAM policy for a job.
+
+  Gets the IAM policy for a job, given a job ID.
+
+  ## EXAMPLES
+
+  The following command prints the IAM policy for a job with the ID
+  `example-job`:
+
+    $ {command} example-job
+  """
+
+  @classmethod
+  def Args(cls, parser):
+    dataproc = dp.Dataproc(cls.ReleaseTrack())
+    flags.AddJobResourceArg(parser, 'retrieve the policy for',
+                            dataproc.api_version)
+    base.URI_FLAG.RemoveFromParser(parser)
+
+  def Run(self, args):
+    dataproc = dp.Dataproc(self.ReleaseTrack())
+    msgs = dataproc.messages
+
+    job = args.CONCEPTS.job.Parse()
+    request = msgs.DataprocProjectsRegionsJobsGetIamPolicyRequest(
+        resource=job.RelativeName(),
+        getIamPolicyRequest=msgs.GetIamPolicyRequest(
+            options=msgs.GetPolicyOptions(
+                requestedPolicyVersion=iam_helpers.MAX_LIBRARY_IAM_SUPPORTED_VERSION
+            )
+        ),
+    )
+
+    return dataproc.client.projects_regions_jobs.GetIamPolicy(request)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/kill.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Kill job command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.api_lib.dataproc import util
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc import flags
+from googlecloudsdk.core import log
+from googlecloudsdk.core.console import console_io
+
+
+class Kill(base.Command):
+  """Kill an active job.
+
+  Kill an active job.
+
+  ## EXAMPLES
+
+  To cancel a job, run:
+
+    $ {command} job_id
+  """
+
+  @classmethod
+  def Args(cls, parser):
+    dataproc = dp.Dataproc(cls.ReleaseTrack())
+    flags.AddJobResourceArg(parser, 'kill', dataproc.api_version)
+    flags.AddAsync(parser)
+
+  def Run(self, args):
+    dataproc = dp.Dataproc(self.ReleaseTrack())
+
+    job_ref = args.CONCEPTS.job.Parse()
+    request = dataproc.messages.DataprocProjectsRegionsJobsCancelRequest(
+        projectId=job_ref.projectId,
+        region=job_ref.region,
+        jobId=job_ref.jobId,
+        cancelJobRequest=dataproc.messages.CancelJobRequest())
+
+    console_io.PromptContinue(
+        message="The job '{0}' will be killed.".format(args.job),
+        cancel_on_no=True,
+        cancel_string='Cancellation aborted by user.')
+
+    job = dataproc.client.projects_regions_jobs.Cancel(request)
+    log.status.Print(
+        'Job cancellation initiated for [{0}].'.format(job_ref.jobId))
+
+    if args.async_:
+      output_job = job
+    else:
+      output_job = util.WaitForJobTermination(
+          dataproc,
+          job,
+          job_ref,
+          message='Waiting for job cancellation',
+          goal_state=dataproc.messages.JobStatus.StateValueValuesEnum.CANCELLED)
+      log.status.Print('Killed [{0}].'.format(job_ref))
+
+    return output_job

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/list.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""List job command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import constants
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.api_lib.dataproc import display_helper
+from googlecloudsdk.api_lib.dataproc import util
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc import flags
+from googlecloudsdk.core import properties
+
+
+STATE_MATCHER_ENUM_MAP = {'active': 'ACTIVE', 'inactive': 'NON_ACTIVE'}
+
+
+class List(base.ListCommand):
+  """List jobs in a project.
+
+  List jobs in a project. An optional filter can be used to constrain the jobs
+  returned. Filters are case-sensitive and have the following syntax:
+
+    [field = value] AND [field [= value]] ...
+
+  where `field` is `status.state` or `labels.[KEY]`, and `[KEY]` is a label
+  key. `value` can be ```*``` to match all values. `status.state` can be either
+  `ACTIVE` or `INACTIVE`. Only the logical `AND` operator is supported;
+  space-separated items are treated as having an implicit `AND` operator.
+
+  ## EXAMPLES
+
+  To see the list of all jobs in Dataproc's 'us-central1' region, run:
+
+    $ {command} --region=us-central1
+
+  To see a list of all active jobs in cluster 'mycluster' with a label
+  `env=staging` located in 'us-central1', run:
+
+    $ {command} --region=us-central1 --filter='status.state = ACTIVE AND
+        placement.clusterName = 'mycluster' AND labels.env = staging'
+  """
+
+  @staticmethod
+  def Args(parser):
+    base.URI_FLAG.RemoveFromParser(parser)
+    base.PAGE_SIZE_FLAG.SetDefault(parser, constants.DEFAULT_PAGE_SIZE)
+    flags.AddRegionFlag(parser)
+
+    parser.add_argument(
+        '--cluster',
+        help='Restrict to the jobs of this Dataproc cluster.')
+
+    parser.add_argument(
+        '--state-filter',
+        choices=sorted(STATE_MATCHER_ENUM_MAP.keys()),
+        help='Filter by job state.')
+    parser.display_info.AddFormat("""
+          table(
+            reference.jobId,
+            jobType.yesno(no="-"):label=TYPE,
+            status.state:label=STATUS
+          )
+    """)
+
+  def Run(self, args):
+    dataproc = dp.Dataproc(self.ReleaseTrack())
+
+    project = properties.VALUES.core.project.GetOrFail()
+    region = util.ResolveRegion()
+
+    request = self.GetRequest(dataproc.messages, project, region, args)
+
+    if args.cluster:
+      request.clusterName = args.cluster
+
+    if args.state_filter:
+      state = STATE_MATCHER_ENUM_MAP.get(args.state_filter)
+      request.jobStateMatcher = (
+          dataproc.messages.DataprocProjectsRegionsJobsListRequest
+          .JobStateMatcherValueValuesEnum.lookup_by_name(state))
+
+    jobs = util.YieldFromListWithUnreachableList(
+        'The following jobs are unreachable: %s',
+        dataproc.client.projects_regions_jobs,
+        request,
+        limit=args.limit,
+        field='jobs',
+        batch_size=args.page_size,
+        batch_size_attribute='pageSize',
+    )
+    return (display_helper.DisplayHelper(job) for job in jobs)
+
+  @staticmethod
+  def GetRequest(messages, project, region, args):
+    # Explicitly null out args.filter if present because by default args.filter
+    # also acts as a postfilter to the things coming back from the backend
+    backend_filter = None
+    if args.filter:
+      backend_filter = args.filter
+      args.filter = None
+
+    return messages.DataprocProjectsRegionsJobsListRequest(
+        projectId=project, region=region, filter=backend_filter)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/set_iam_policy.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Set IAM job policy command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.api_lib.dataproc import iam_helpers
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc import flags
+from googlecloudsdk.command_lib.iam import iam_util
+
+
+@base.UniverseCompatible
+@base.ReleaseTracks(
+    base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA, base.ReleaseTrack.GA
+)
+class SetIamPolicy(base.Command):
+  """Set IAM policy for a job.
+
+  Sets the IAM policy for a job, given a job ID and the policy.
+  """
+
+  detailed_help = iam_util.GetDetailedHelpForSetIamPolicy('job')
+
+  @classmethod
+  def Args(cls, parser):
+    dataproc = dp.Dataproc(cls.ReleaseTrack())
+    flags.AddJobResourceArg(parser, 'set the policy on', dataproc.api_version)
+    iam_util.AddArgForPolicyFile(parser)
+
+  def Run(self, args):
+    dataproc = dp.Dataproc(self.ReleaseTrack())
+    msgs = dataproc.messages
+
+    policy = iam_util.ParsePolicyFile(args.policy_file, msgs.Policy)
+    policy.version = iam_helpers.MAX_LIBRARY_IAM_SUPPORTED_VERSION
+    set_iam_policy_request = msgs.SetIamPolicyRequest(policy=policy)
+
+    job = args.CONCEPTS.job.Parse()
+    request = msgs.DataprocProjectsRegionsJobsSetIamPolicyRequest(
+        resource=job.RelativeName(),
+        setIamPolicyRequest=set_iam_policy_request)
+
+    return dataproc.client.projects_regions_jobs.SetIamPolicy(request)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/submit/__init__.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The command group for submitting cloud dataproc jobs."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc import flags
+
+
+class Submit(base.Group):
+  """Submit Dataproc jobs to execute on a cluster.
+
+  Submit Dataproc jobs to execute on a cluster.
+
+  ## EXAMPLES
+
+  To submit a Hadoop MapReduce job, run:
+
+    $ {command} hadoop --cluster my-cluster --jar my_jar.jar -- arg1 arg2
+
+  To submit a Spark Scala or Java job, run:
+
+    $ {command} spark --cluster my-cluster --jar my_jar.jar -- arg1 arg2
+
+  To submit a PySpark job, run:
+
+    $ {command} pyspark --cluster my-cluster my_script.py -- arg1 arg2
+
+  To submit a Spark SQL job, run:
+
+    $ {command} spark-sql --cluster my-cluster --file my_queries.q
+
+  To submit a Pig job, run:
+
+    $ {command} pig --cluster my-cluster --file my_script.pig
+
+  To submit a Hive job, run:
+
+    $ {command} hive --cluster my-cluster --file my_queries.q
+  """
+
+  @staticmethod
+  def Args(parser):
+    # Allow user specified Job ID, but don't expose it.
+    parser.add_argument(
+        '--id',
+        hidden=True,
+        help='Set the ID of the job, which is usually autogenerated')
+
+    flags.AddRegionFlag(parser)
+
+    base.ASYNC_FLAG.AddToParser(parser)
+
+    parser.add_argument(
+        '--bucket',
+        help=("The Cloud Storage bucket to stage files in. Defaults to the "
+              "cluster's configured bucket."))

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/submit/flink.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a Flink job to a cluster."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import util
+from googlecloudsdk.command_lib.dataproc.jobs import flink
+from googlecloudsdk.command_lib.dataproc.jobs import submitter
+
+
+class Flink(flink.FlinkBase, submitter.JobSubmitter):
+  # pylint: disable=line-too-long
+  r"""Submit a Flink job to a cluster.
+
+  Submit a Flink job to a cluster.
+
+  ## EXAMPLES
+
+  To submit a Flink job that runs the main class of a jar, run:
+
+    $ {command} --cluster=my-cluster --region=us-central1 --jar=my_jar.jar -- arg1 arg2
+
+  To submit a Flink job that runs a specific class as an entrypoint:
+
+    $ {command} --cluster=my-cluster --region=us-central1  --class=org.my.main.Class  \
+      --jars=my_jar.jar -- arg1 arg2
+
+  To submit a Flink job that runs a jar that is on the cluster, run:
+
+    $ {command} --cluster=my-cluster --region=us-central1 \
+        --jar=/usr/lib/flink/examples/streaming/TopSpeedWindowing.jar
+
+  """
+  # pylint: enable=line-too-long
+
+  @staticmethod
+  def Args(parser):
+    flink.FlinkBase.Args(parser)
+    submitter.JobSubmitter.Args(parser)
+    driver_group = parser.add_argument_group(required=True, mutex=True)
+    util.AddJvmDriverFlags(driver_group)
+
+  def ConfigureJob(self, messages, job, args):
+    flink.FlinkBase.ConfigureJob(
+        messages,
+        job,
+        self.files_by_type,
+        self.BuildLoggingConfig(messages, args.driver_log_levels),
+        args,
+    )
+    submitter.JobSubmitter.ConfigureJob(messages, job, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/submit/hadoop.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a Hadoop job to a cluster."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import util
+from googlecloudsdk.command_lib.dataproc.jobs import hadoop
+from googlecloudsdk.command_lib.dataproc.jobs import submitter
+
+
+class Hadoop(hadoop.HadoopBase, submitter.JobSubmitter):
+  r"""Submit a Hadoop job to a cluster.
+
+  Submit a Hadoop job to a cluster.
+
+  ## EXAMPLES
+
+  To submit a Hadoop job that runs the main class of a jar, run:
+
+    $ {command} --cluster=my-cluster --jar=my_jar.jar -- arg1 arg2
+
+  To submit a Hadoop job that runs a specific class of a jar, run:
+
+    $ {command} --cluster=my-cluster --class=org.my.main.Class \
+        --jars=my_jar1.jar,my_jar2.jar -- arg1 arg2
+
+  To submit a Hadoop job that runs a jar that is already on the cluster, run:
+
+    $ {command} --cluster=my-cluster \
+        --jar=file:///usr/lib/hadoop-op/hadoop-op-examples.jar \
+        -- wordcount gs://my_bucket/my_file.txt gs://my_bucket/output
+  """
+
+  @classmethod
+  def Args(cls, parser):
+    hadoop.HadoopBase.Args(parser)
+    submitter.JobSubmitter.Args(parser)
+    driver_group = parser.add_argument_group(required=True, mutex=True)
+    util.AddJvmDriverFlags(driver_group)
+
+  def ConfigureJob(self, messages, job, args):
+    hadoop.HadoopBase.ConfigureJob(
+        messages, job, self.files_by_type,
+        self.BuildLoggingConfig(messages, args.driver_log_levels), args)
+    submitter.JobSubmitter.ConfigureJob(messages, job, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/submit/hive.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a Hive job to a cluster."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.command_lib.dataproc.jobs import hive
+from googlecloudsdk.command_lib.dataproc.jobs import submitter
+
+
+class Hive(hive.HiveBase, submitter.JobSubmitter):
+  """Submit a Hive job to a cluster.
+
+  Submit a Hive job to a cluster.
+
+  ## EXAMPLES
+
+  To submit a Hive job with a local script, run:
+
+    $ {command} --cluster=my-cluster --file=my_queries.q
+
+  To submit a Hive job with inline queries, run:
+
+    $ {command} --cluster=my-cluster
+        -e="CREATE EXTERNAL TABLE foo(bar int) LOCATION 'gs://my_bucket/'"
+        -e="SELECT * FROM foo WHERE bar > 2"
+  """
+
+  @classmethod
+  def Args(cls, parser):
+    hive.HiveBase.Args(parser)
+    submitter.JobSubmitter.Args(parser)
+
+  def ConfigureJob(self, messages, job, args):
+    hive.HiveBase.ConfigureJob(messages, job, self.files_by_type, args)
+    submitter.JobSubmitter.ConfigureJob(messages, job, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/submit/pig.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a Pig job to a cluster."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.command_lib.dataproc.jobs import pig
+from googlecloudsdk.command_lib.dataproc.jobs import submitter
+
+
+class Pig(pig.PigBase, submitter.JobSubmitter):
+  """Submit a Pig job to a cluster.
+
+  Submit a Pig job to a cluster.
+
+  ## EXAMPLES
+
+  To submit a Pig job with a local script, run:
+
+    $ {command} --cluster=my-cluster --file=my_queries.pig
+
+  To submit a Pig job with inline queries, run:
+
+    $ {command} --cluster=my-cluster
+        -e="LNS = LOAD 'gs://my_bucket/my_file.txt' AS (line)"
+        -e="WORDS = FOREACH LNS GENERATE FLATTEN(TOKENIZE(line)) AS word"
+        -e="GROUPS = GROUP WORDS BY word"
+        -e="WORD_COUNTS = FOREACH GROUPS GENERATE group, COUNT(WORDS)"
+        -e="DUMP WORD_COUNTS"
+  """
+
+  @staticmethod
+  def Args(parser):
+    pig.PigBase.Args(parser)
+    submitter.JobSubmitter.Args(parser)
+
+  def ConfigureJob(self, messages, job, args):
+    pig.PigBase.ConfigureJob(messages, job, self.files_by_type,
+                             self.BuildLoggingConfig(
+                                 messages, args.driver_log_levels), args)
+    submitter.JobSubmitter.ConfigureJob(messages, job, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/submit/presto.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2019 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Submit a Presto job to a cluster."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.command_lib.dataproc.jobs import presto
+from googlecloudsdk.command_lib.dataproc.jobs import submitter
+
+
+class Presto(presto.PrestoBase, submitter.JobSubmitter):
+  r"""Submit a Presto job to a cluster.
+
+  Submit a Presto job to a cluster
+
+  ## EXAMPLES
+
+  To submit a Presto job with a local script, run:
+
+    $ {command} --cluster=my-cluster --file=my_script.R
+
+  To submit a Presto job with inline queries, run:
+
+    $ {command} --cluster=my-cluster -e="SELECT * FROM foo WHERE bar > 2"
+  """
+
+  @staticmethod
+  def Args(parser):
+    presto.PrestoBase.Args(parser)
+    submitter.JobSubmitter.Args(parser)
+
+  def ConfigureJob(self, messages, job, args):
+    presto.PrestoBase.ConfigureJob(
+        messages, job, self.files_by_type,
+        self.BuildLoggingConfig(messages, args.driver_log_levels), args)
+    submitter.JobSubmitter.ConfigureJob(messages, job, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/submit/pyflink.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a PyFlink job to a cluster."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.command_lib.dataproc.jobs import pyflink
+from googlecloudsdk.command_lib.dataproc.jobs import submitter
+
+
+class PyFlink(pyflink.PyFlinkBase, submitter.JobSubmitter):
+  # pylint: disable=line-too-long
+  r"""Submit a PyFlink job to a cluster.
+
+  Submit a PyFlink job to a cluster.
+
+  ## EXAMPLES
+
+    Submit a PyFlink job.
+
+    $ gcloud dataproc jobs submit pyflink my-pyflink.py --region=us-central1
+
+    Submit a PyFlink job with additional source and resource files.
+
+    $ gcloud dataproc jobs submit pyflink my-pyflink.py \
+      --region=us-central1 \
+      --py-files=my-python-file1.py,my-python-file2.py
+
+    Submit a PyFlink job with a jar file.
+
+    $ gcloud dataproc jobs submit pyflink my-pyflink.py \
+      --region=us-central1 \
+      --jars=my-jar-file.jar
+
+    Submit a PyFlink job with 'python-files' and 'python-module'.
+
+    $ gcloud dataproc jobs submit pyflink my-pyflink.py \
+      --region=us-central1 \
+      --py-files=my-python-file1.py,my-python-file2.py
+      --py-module=my-module
+
+  """
+  # pylint: enable=line-too-long
+
+  @staticmethod
+  def Args(parser):
+    pyflink.PyFlinkBase.Args(parser)
+    submitter.JobSubmitter.Args(parser)
+
+  def ConfigureJob(self, messages, job, args):
+    pyflink.PyFlinkBase.ConfigureJob(messages, job, self.files_by_type,
+                                     self.BuildLoggingConfig(
+                                         messages, args.driver_log_levels),
+                                     args)
+    submitter.JobSubmitter.ConfigureJob(messages, job, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/submit/pyspark.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a PySpark job to a cluster."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.command_lib.dataproc.jobs import pyspark
+from googlecloudsdk.command_lib.dataproc.jobs import submitter
+
+
+class PySpark(pyspark.PySparkBase, submitter.JobSubmitter):
+  # pylint: disable=line-too-long
+  """Submit a PySpark job to a cluster.
+
+  Submit a PySpark job to a cluster.
+
+  ## EXAMPLES
+
+  To submit a PySpark job with a local script and custom flags, run:
+
+    $ {command} --cluster=my-cluster my_script.py -- --custom-flag
+
+  To submit a Spark job that runs a script that is already on the cluster, run:
+
+    $ {command} --cluster=my-cluster file:///usr/lib/spark/examples/src/main/python/pi.py -- 100
+  """
+  # pylint: enable=line-too-long
+
+  @staticmethod
+  def Args(parser):
+    pyspark.PySparkBase.Args(parser)
+    submitter.JobSubmitter.Args(parser)
+
+  def ConfigureJob(self, messages, job, args):
+    pyspark.PySparkBase.ConfigureJob(messages, job, self.files_by_type,
+                                     self.BuildLoggingConfig(
+                                         messages, args.driver_log_levels),
+                                     args)
+    submitter.JobSubmitter.ConfigureJob(messages, job, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/submit/spark.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a Spark job to a cluster."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import util
+from googlecloudsdk.command_lib.dataproc.jobs import spark
+from googlecloudsdk.command_lib.dataproc.jobs import submitter
+
+
+class Spark(spark.SparkBase, submitter.JobSubmitter):
+  # pylint: disable=line-too-long
+  r"""Submit a Spark job to a cluster.
+
+  Submit a Spark job to a cluster.
+
+  ## EXAMPLES
+
+  To submit a Spark job that runs the main class of a jar, run:
+
+    $ {command} --cluster=my-cluster --region=us-central1 --jar=my_jar.jar -- arg1 arg2
+
+  To submit a Spark job that runs a specific class of a jar, run:
+
+    $ {command} --cluster=my-cluster --region=us-central1 --class=org.my.main.Class \
+        --jars=my_jar1.jar,my_jar2.jar -- arg1 arg2
+
+  To submit a Spark job that runs a jar that is already on the cluster, run:
+
+    $ {command} --cluster=my-cluster --region=us-central1 \
+        --class=org.apache.spark.examples.SparkPi \
+        --jars=file:///usr/lib/spark/examples/jars/spark-examples.jar \
+        -- 1000
+  """
+  # pylint: enable=line-too-long
+
+  @staticmethod
+  def Args(parser):
+    spark.SparkBase.Args(parser)
+    submitter.JobSubmitter.Args(parser)
+    driver_group = parser.add_argument_group(required=True, mutex=True)
+    util.AddJvmDriverFlags(driver_group)
+
+  def ConfigureJob(self, messages, job, args):
+    spark.SparkBase.ConfigureJob(messages, job, self.files_by_type,
+                                 self.BuildLoggingConfig(
+                                     messages, args.driver_log_levels), args)
+    submitter.JobSubmitter.ConfigureJob(messages, job, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/submit/spark_r.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a SparkR job to a cluster."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.command_lib.dataproc.jobs import spark_r
+from googlecloudsdk.command_lib.dataproc.jobs import submitter
+
+
+class SparkR(spark_r.SparkRBase, submitter.JobSubmitter):
+  r"""Submit a SparkR job to a cluster.
+
+  Submit a SparkR job to a cluster.
+
+  ## EXAMPLES
+
+  To submit a SparkR job with a local script, run:
+
+    $ {command} --cluster=my-cluster my_script.R
+
+  To submit a Spark job that runs a script already on the cluster, run:
+
+    $ {command} --cluster=my-cluster file:///.../my_script.R \
+        -- gs://my_bucket/data.csv
+  """
+
+  @staticmethod
+  def Args(parser):
+    spark_r.SparkRBase.Args(parser)
+    submitter.JobSubmitter.Args(parser)
+
+  def ConfigureJob(self, messages, job, args):
+    spark_r.SparkRBase.ConfigureJob(
+        messages, job, self.files_by_type,
+        self.BuildLoggingConfig(messages, args.driver_log_levels), args)
+    submitter.JobSubmitter.ConfigureJob(messages, job, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/submit/spark_sql.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Submit a Spark SQL job to a cluster."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.command_lib.dataproc.jobs import spark_sql
+from googlecloudsdk.command_lib.dataproc.jobs import submitter
+
+
+class SparkSql(spark_sql.SparkSqlBase, submitter.JobSubmitter):
+  """Submit a Spark SQL job to a cluster.
+
+  Submit a Spark SQL job to a cluster.
+
+  ## EXAMPLES
+
+  To submit a Spark SQL job with a local script, run:
+
+    $ {command} --cluster=my-cluster --file=my_queries.ql
+
+  To submit a Spark SQL job with inline queries, run:
+
+    $ {command} --cluster=my-cluster
+        -e="CREATE EXTERNAL TABLE foo(bar int) LOCATION 'gs://my_bucket/'"
+        -e="SELECT * FROM foo WHERE bar > 2"
+  """
+
+  @staticmethod
+  def Args(parser):
+    spark_sql.SparkSqlBase.Args(parser)
+    submitter.JobSubmitter.Args(parser)
+
+  def ConfigureJob(self, messages, job, args):
+    spark_sql.SparkSqlBase.ConfigureJob(messages, job, self.files_by_type,
+                                        self.BuildLoggingConfig(
+                                            messages, args.driver_log_levels),
+                                        args)
+    submitter.JobSubmitter.ConfigureJob(messages, job, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/submit/trino.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2022 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Submit a Trino job to a cluster."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.command_lib.dataproc.jobs import submitter
+from googlecloudsdk.command_lib.dataproc.jobs import trino
+
+
+class Trino(trino.TrinoBase, submitter.JobSubmitter):
+  r"""Submit a Trino job to a cluster.
+
+  Submit a Trino job to a cluster
+
+  ## EXAMPLES
+
+  To submit a Trino job with a local script, run:
+
+    $ {command} --cluster=my-cluster --file=my_script.R
+
+  To submit a Trino job with inline queries, run:
+
+    $ {command} --cluster=my-cluster -e="SELECT * FROM foo WHERE bar > 2"
+  """
+
+  @staticmethod
+  def Args(parser):
+    trino.TrinoBase.Args(parser)
+    submitter.JobSubmitter.Args(parser)
+
+  def ConfigureJob(self, messages, job, args):
+    trino.TrinoBase.ConfigureJob(
+        messages, job, self.files_by_type,
+        self.BuildLoggingConfig(messages, args.driver_log_levels), args)
+    submitter.JobSubmitter.ConfigureJob(messages, job, args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/update.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Update job command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc import flags
+from googlecloudsdk.command_lib.util.args import labels_util
+from googlecloudsdk.core import log
+
+
+class Update(base.UpdateCommand):
+  """Update the labels for a job.
+
+  Update the labels for a job.
+
+  ## EXAMPLES
+
+  To add the label 'customer=acme' to a job , run:
+
+    $ {command} job_id --update-labels=customer=acme
+
+  To update the label 'customer=ackme' to 'customer=acme', run:
+
+    $ {command} job_id --update-labels=customer=acme
+
+  To remove the label whose key is 'customer', run:
+
+    $ {command} job_id --remove-labels=customer
+  """
+
+  @classmethod
+  def Args(cls, parser):
+    dataproc = dp.Dataproc(cls.ReleaseTrack())
+    flags.AddJobResourceArg(parser, 'update', dataproc.api_version)
+    changes = parser.add_argument_group(required=True)
+    # Allow the user to specify new labels as well as update/remove existing
+    labels_util.AddUpdateLabelsFlags(changes)
+
+  def Run(self, args):
+    dataproc = dp.Dataproc(self.ReleaseTrack())
+
+    job_ref = args.CONCEPTS.job.Parse()
+
+    changed_fields = []
+
+    orig_job = dataproc.client.projects_regions_jobs.Get(
+        dataproc.messages.DataprocProjectsRegionsJobsGetRequest(
+            projectId=job_ref.projectId,
+            region=job_ref.region,
+            jobId=job_ref.jobId))
+
+    # Update labels if the user requested it
+    labels_update_result = labels_util.Diff.FromUpdateArgs(args).Apply(
+        dataproc.messages.Job.LabelsValue, orig_job.labels)
+    if labels_update_result.needs_update:
+      changed_fields.append('labels')
+
+    updated_job = orig_job
+    updated_job.labels = labels_update_result.GetOrNone()
+    request = dataproc.messages.DataprocProjectsRegionsJobsPatchRequest(
+        projectId=job_ref.projectId,
+        region=job_ref.region,
+        jobId=job_ref.jobId,
+        job=updated_job,
+        updateMask=','.join(changed_fields))
+
+    returned_job = dataproc.client.projects_regions_jobs.Patch(request)
+
+    log.UpdatedResource(returned_job)
+    return returned_job

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataproc/jobs/wait.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Wait for a job to complete."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataproc import dataproc as dp
+from googlecloudsdk.api_lib.dataproc import util
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataproc import flags
+from googlecloudsdk.core import log
+
+
+class Wait(base.Command):
+  r"""View the output of a job as it runs or after it completes.
+
+  View the output of a job as it runs or after it completes.
+
+  ## EXAMPLES
+
+  To see a list of all jobs, run:
+
+    $ gcloud dataproc jobs list
+
+  To display these jobs with their respective IDs and underlying REST calls,
+  run:
+
+    $ gcloud dataproc jobs list --format "table(reference.jobId)" \
+      --limit 1 --log-http
+
+  To view the output of a job as it runs, run:
+
+    $ {command} job_id
+  """
+
+  @classmethod
+  def Args(cls, parser):
+    dataproc = dp.Dataproc(cls.ReleaseTrack())
+    flags.AddJobResourceArg(parser, 'wait for', dataproc.api_version)
+
+  def Run(self, args):
+    dataproc = dp.Dataproc(self.ReleaseTrack())
+
+    job_ref = args.CONCEPTS.job.Parse()
+
+    job = dataproc.client.projects_regions_jobs.Get(
+        dataproc.messages.DataprocProjectsRegionsJobsGetRequest(
+            projectId=job_ref.projectId,
+            region=job_ref.region,
+            jobId=job_ref.jobId))
+
+    # TODO(b/36050945) Check if Job is still running and fail or handle 401.
+
+    job = util.WaitForJobTermination(
+        dataproc,
+        job,
+        job_ref,
+        message='Waiting for job completion',
+        goal_state=dataproc.messages.JobStatus.StateValueValuesEnum.DONE,
+        error_state=dataproc.messages.JobStatus.StateValueValuesEnum.ERROR,
+        stream_driver_log=True)
+
+    log.status.Print('Job [{0}] finished successfully.'.format(args.job))
+
+    return job