feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The command group for gcloud dataflow jobs.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+class Jobs(base.Group):
+  """A group of subcommands for working with Dataflow jobs.
+  """
+  pass

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/archive.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2025 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of gcloud dataflow jobs archive command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataflow import apis
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataflow import job_utils
+from googlecloudsdk.core import log
+from googlecloudsdk.core.console import console_io
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA, base.ReleaseTrack.BETA)
+@base.DefaultUniverseOnly
+class Archive(base.Command):
+  """Archives a job.
+
+  Archives a single job. The job must be in a terminal state, otherwise the
+  request will be rejected.
+
+  This command will require confirmation to run.
+
+  ## EXAMPLES
+
+  To archive job `2025-03-15_14_23_56-1234567890123456`, run:
+
+    $ {command} 2025-03-15_14_23_56-1234567890123456
+  """
+
+  @staticmethod
+  def Args(parser):
+    job_utils.ArgsForJobRef(parser)
+
+  def Run(self, args):
+    """Runs the command.
+
+    Args:
+      args: The arguments that were provided to this command invocation.
+
+    Returns:
+      A Job message.
+    """
+    job_ref = job_utils.ExtractJobRef(args)
+    job_id = job_ref.jobId
+
+    console_io.PromptContinue(
+        message='Job [{}] will be archived.'.format(job_id), cancel_on_no=True
+    )
+
+    messages = apis.GetMessagesModule()
+    job = messages.Job(
+        jobMetadata=messages.JobMetadata(
+            userDisplayProperties=messages.JobMetadata.UserDisplayPropertiesValue(
+                additionalProperties=[
+                    messages.JobMetadata.UserDisplayPropertiesValue.AdditionalProperty(
+                        key='archived', value='true'
+                    )
+                ]
+            )
+        )
+    )
+
+    request = messages.DataflowProjectsLocationsJobsUpdateRequest(
+        jobId=job_ref.jobId,
+        location=job_ref.location,
+        projectId=job_ref.projectId,
+        job=job,
+        updateMask='job_metadata.user_display_properties.archived',
+    )
+
+    result = apis.Jobs.GetService().Update(request)
+    log.status.Print('Archived job [{}].'.format(job_id))
+    return result

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/cancel.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of gcloud dataflow jobs cancel command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataflow import apis
+from googlecloudsdk.api_lib.util import exceptions
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataflow import job_utils
+from googlecloudsdk.core import log
+from googlecloudsdk.core.console import console_io
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+@base.DefaultUniverseOnly
+class Cancel(base.Command):
+  """Cancels all jobs that match the command line arguments."""
+
+  @staticmethod
+  def Args(parser):
+    """Register flags for this command."""
+    job_utils.ArgsForJobRefs(parser, nargs='+')
+    parser.add_argument(
+        '--force',
+        action='store_true',
+        help=(
+            'Forcibly cancels a Dataflow job. Regular cancel must have been'
+            ' attempted at least 30 minutes prior for a job to be force'
+            ' cancelled.'
+        ),
+    )
+
+  def Run(self, args):
+    """This is what gets called when the user runs this command.
+
+    Args:
+      args: all the arguments that were provided to this command invocation.
+    """
+    for job_ref in job_utils.ExtractJobRefs(args):
+      try:
+        if args.force:
+          console_io.PromptContinue(
+              message='Force cancellation will leak VMs the cancelled Dataflow job created that must be manually cleaned up.',
+              cancel_on_no=True,
+          )
+        apis.Jobs.Cancel(
+            job_ref.jobId,
+            args.force,
+            project_id=job_ref.projectId,
+            region_id=job_ref.location)
+        log.status.Print('Cancelled job [{0}]'.format(job_ref.jobId))
+      except exceptions.HttpException as error:
+        log.status.Print(
+            'Failed to cancel job [{0}]: {1} Ensure that you have permission '
+            'to access the job and that the `--region` flag, {2}, matches the'
+            " job's region.".format(
+                job_ref.jobId, error.payload.status_message, job_ref.location
+            )
+        )

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/config/__init__.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command group for managing Dataflow job configurations."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
+class Config(base.Group):
+  """Manage Dataflow job configurations."""

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/config/export.yaml

@@ -0,0 +1,38 @@
+release_tracks: [ALPHA]
+command_type: CONFIG_EXPORT
+help_text:
+  brief: Export the configuration for a Dataflow job.
+  description: |
+    *{command}* exports the configuration for a Dataflow job.
+
+    Job configurations can be exported in
+    Kubernetes Resource Model (krm) or Terraform HCL formats. The
+    default format is `krm`.
+
+    Specifying `--all` allows you to export the configurations for all
+    jobs within the project.
+
+    Specifying `--path` allows you to export the configuration(s) to
+    a local directory.
+  examples: |
+    To export the configuration for a job, run:
+
+      $ {command} my-job
+
+    To export the configuration for a job to a file, run:
+
+      $ {command} my-job --path=/path/to/dir/
+
+    To export the configuration for a job in Terraform
+    HCL format, run:
+
+      $ {command} my-job --resource-format=terraform
+
+    To export the configurations for all jobs within a
+    project, run:
+
+      $ {command} --all
+arguments:
+  resource:
+    help_text: Job to export the configuration for.
+    spec: !REF googlecloudsdk.command_lib.dataflow.resources:job

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/describe.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implementation of gcloud dataflow jobs describe command.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataflow import apis
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataflow import job_utils
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+class Describe(base.DescribeCommand):
+  """Outputs the Job object resulting from the Get API.
+
+  By default this will display the Summary view which includes:
+    - Project ID
+    - Regional Endpoint
+    - Job ID
+    - Job Name
+    - Job Type (Batch vs. Streaming)
+    - Job Create Time
+    - Job Status (Running, Done, Cancelled, Failed)
+    - Job Status Time
+
+  Notable values that are only in the full view:
+    - Environment (staging Jars, information about workers, etc.)
+    - Steps from the workflow graph
+  """
+
+  @staticmethod
+  def Args(parser):
+    """Register flags for this command.
+
+    Args:
+      parser: argparse.ArgumentParser to register arguments with.
+    """
+    job_utils.ArgsForJobRef(parser)
+
+    parser.add_argument(
+        '--full',
+        action='store_const',
+        const=apis.Jobs.GET_REQUEST.ViewValueValuesEnum.JOB_VIEW_ALL,
+        default=apis.Jobs.GET_REQUEST.ViewValueValuesEnum.JOB_VIEW_SUMMARY,
+        help='Retrieve the full Job rather than the summary view')
+
+  def Run(self, args):
+    """Runs the command.
+
+    Args:
+      args: The arguments that were provided to this command invocation.
+
+    Returns:
+      A Job message.
+    """
+    job_ref = job_utils.ExtractJobRef(args)
+    return apis.Jobs.Get(
+        job_ref.jobId,
+        project_id=job_ref.projectId,
+        region_id=job_ref.location,
+        view=args.full)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/drain.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implementation of gcloud dataflow jobs drain command.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataflow import apis
+from googlecloudsdk.api_lib.util import exceptions
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataflow import job_utils
+from googlecloudsdk.core import log
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+@base.DefaultUniverseOnly
+class Drain(base.Command):
+  """Drains all jobs that match the command line arguments.
+
+     Once Drain is triggered, the pipeline will stop accepting new inputs.
+     The input watermark will be advanced to infinity. Elements already in the
+     pipeline will continue to be processed. Drained jobs can safely be
+     cancelled.
+  """
+
+  @staticmethod
+  def Args(parser):
+    """Register flags for this command."""
+    job_utils.ArgsForJobRefs(parser, nargs='+')
+
+  def Run(self, args):
+    """This is what gets called when the user runs this command.
+
+    Args:
+      args: all the arguments that were provided to this command invocation.
+    """
+    for job_ref in job_utils.ExtractJobRefs(args):
+      try:
+        apis.Jobs.Drain(
+            job_ref.jobId,
+            project_id=job_ref.projectId,
+            region_id=job_ref.location)
+        log.status.Print('Started draining job [{0}]'.format(job_ref.jobId))
+      except exceptions.HttpException as error:
+        log.status.Print(
+            (
+                "Failed to drain job [{0}]: {1} Ensure that you have permission"
+                " to access the job and that the `--region` flag, {2}, matches"
+                " the job's region."
+            ).format(
+                job_ref.jobId, error.payload.status_message, job_ref.location
+            )
+        )

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/export_steps.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implementation of gcloud dataflow jobs export-steps command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataflow import apis
+from googlecloudsdk.api_lib.dataflow import step_graph
+from googlecloudsdk.api_lib.dataflow import step_json
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataflow import job_utils
+from googlecloudsdk.core import log
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA)
+class ExportSteps(base.Command):
+  """Exports information about the steps for the given job.
+
+  The only currently supported format is to a GraphViz dot file.
+  """
+
+  @staticmethod
+  def Args(parser):
+    """Register flags for this command.
+
+    Args:
+      parser: argparse.ArgumentParser to register arguments with.
+    """
+    job_utils.ArgsForJobRef(parser)
+
+  def Run(self, args):
+    """Runs the command.
+
+    Args:
+      args: All the arguments that were provided to this command invocation.
+
+    Returns:
+      An iterator over the steps in the given job.
+    """
+    job_ref = job_utils.ExtractJobRef(args)
+    return step_json.ExtractSteps(
+        apis.Jobs.Get(
+            job_ref.jobId,
+            project_id=job_ref.projectId,
+            region_id=job_ref.location,
+            view=apis.Jobs.GET_REQUEST.ViewValueValuesEnum.JOB_VIEW_ALL))
+
+  def Display(self, args, steps):
+    """This method is called to print the result of the Run() method.
+
+    Args:
+      args: all the arguments that were provided to this command invocation.
+      steps: The step information returned from Run().
+    """
+    if steps:
+      for line in step_graph.YieldGraphviz(steps, 'StepGraph'):
+        log.out.write(line)
+        log.out.write('\n')

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/list.py

@@ -0,0 +1,244 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of gcloud dataflow jobs list command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataflow import apis
+from googlecloudsdk.api_lib.dataflow import job_display
+from googlecloudsdk.calliope import arg_parsers
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataflow import dataflow_util
+from googlecloudsdk.core import log
+from googlecloudsdk.core import properties
+from googlecloudsdk.core.resource import resource_filter
+from googlecloudsdk.core.util import times
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+class List(base.ListCommand):
+  """Lists all jobs in a particular project, optionally filtered by region.
+
+  By default, 100 jobs in the current project are listed; this can be overridden
+  with the gcloud --project flag, and the --limit flag.
+
+  Using the --region flag will only list jobs from the given regional endpoint.
+
+  ## EXAMPLES
+
+  Filter jobs with the given name:
+
+    $ {command} --filter="name=my-wordcount"
+
+  List jobs with from a given region:
+
+    $ {command} --region="europe-west1"
+
+  List jobs created this year:
+
+    $ {command} --created-after=2018-01-01
+
+  List jobs created more than a week ago:
+
+    $ {command} --created-before=-P1W
+  """
+
+  @staticmethod
+  def Args(parser):
+    """Register flags for this command."""
+
+    base.ASYNC_FLAG.RemoveFromParser(parser)
+    # Set manageable limits on number of jobs that are listed.
+    base.LIMIT_FLAG.SetDefault(parser, 100)
+    base.PAGE_SIZE_FLAG.SetDefault(parser, 20)
+
+    # Flags for filtering jobs.
+    parser.add_argument(
+        '--status',
+        choices={
+            'all':
+                ('Returns running jobs first, ordered on creation timestamp, '
+                 'then, returns all terminated jobs ordered on the termination '
+                 'timestamp.'),
+            'terminated':
+                ('Filters the jobs that have a terminated state, ordered on '
+                 'the termination timestamp. Example terminated states: Done, '
+                 'Updated, Cancelled, etc.'),
+            'active':
+                ('Filters the jobs that are running ordered on the creation '
+                 'timestamp.'),
+        },
+        help='Filter the jobs to those with the selected status.')
+    parser.add_argument(
+        '--created-after',
+        type=arg_parsers.Datetime.Parse,
+        help=('Filter the jobs to those created after the given time. '
+              'See $ gcloud topic datetimes for information on time formats. '
+              'For example, `2018-01-01` is the first day of the year, and '
+              '`-P2W` is 2 weeks ago.'))
+    parser.add_argument(
+        '--created-before',
+        type=arg_parsers.Datetime.Parse,
+        help=('Filter the jobs to those created before the given time. '
+              'See $ gcloud topic datetimes for information on time formats.'))
+    parser.add_argument(
+        '--region',
+        metavar='REGION',
+        help=(
+            'Only resources from the given region are queried. '
+            'If not provided, an attempt will be made to query from all '
+            'available regions. In the event of an outage, jobs from certain '
+            'regions may not be available.'))
+
+    parser.display_info.AddFormat("""
+          table(
+            id:label=JOB_ID,
+            name:label=NAME,
+            type:label=TYPE,
+            creationTime.yesno(no="-"),
+            state,
+            location:label=REGION
+          )
+     """)
+    parser.display_info.AddUriFunc(dataflow_util.JobsUriFunc)
+
+  def Run(self, args):
+    """Runs the command.
+
+    Args:
+      args: All the arguments that were provided to this command invocation.
+
+    Returns:
+      An iterator over Job messages.
+    """
+    if args.filter:
+      filter_expr = resource_filter.Compile(args.filter)
+
+      def EvalFilter(x):
+        return (filter_expr.Evaluate(job_display.DisplayInfo(x)) and
+                _JobFilter(args)(x))
+
+      filter_pred = EvalFilter
+    else:
+      filter_pred = _JobFilter(args)
+
+    project_id = properties.VALUES.core.project.Get(required=True)
+    jobs = self._JobSummariesForProject(project_id, args, filter_pred)
+
+    return [job_display.DisplayInfo(job) for job in jobs]
+
+  def _JobSummariesForProject(self, project_id, args, filter_predicate):
+    """Get the list of job summaries that match the predicate.
+
+    Args:
+      project_id: The project ID to retrieve
+      args: parsed command line arguments
+      filter_predicate: The filter predicate to apply
+
+    Returns:
+      An iterator over all the matching jobs.
+    """
+    request = None
+    service = None
+    status_filter = self._StatusArgToFilter(args.status, args.region)
+    if args.region:
+      request = apis.Jobs.LIST_REQUEST(
+          projectId=project_id, location=args.region, filter=status_filter)
+      service = apis.Jobs.GetService()
+    else:
+      log.status.Print(
+          '`--region` not set; getting jobs from all available regions. ' +
+          'Some jobs may be missing in the event of an outage. ' +
+          'https://cloud.google.com/dataflow/docs/concepts/regional-endpoints')
+      request = apis.Jobs.AGGREGATED_LIST_REQUEST(
+          projectId=project_id, filter=status_filter)
+      service = apis.GetClientInstance().projects_jobs
+
+    return dataflow_util.YieldFromList(
+        project_id=project_id,
+        region_id=args.region,
+        service=service,
+        request=request,
+        limit=args.limit,
+        batch_size=args.page_size,
+        field='jobs',
+        batch_size_attribute='pageSize',
+        predicate=filter_predicate)
+
+  def _StatusArgToFilter(self, status, region=None):
+    """Return a string describing the job status.
+
+    Args:
+      status: The job status enum
+      region: The region argument, to select the correct wrapper message.
+
+    Returns:
+      string describing the job status
+    """
+
+    filter_value_enum = None
+    if region:
+      filter_value_enum = (
+          apis.GetMessagesModule().DataflowProjectsLocationsJobsListRequest
+          .FilterValueValuesEnum)
+    else:
+      filter_value_enum = (
+          apis.GetMessagesModule().DataflowProjectsJobsAggregatedRequest
+          .FilterValueValuesEnum)
+
+    value_map = {
+        'all': filter_value_enum.ALL,
+        'terminated': filter_value_enum.TERMINATED,
+        'active': filter_value_enum.ACTIVE,
+    }
+    return value_map.get(status, filter_value_enum.ALL)
+
+
+class _JobFilter(object):
+  """Predicate for filtering jobs."""
+
+  def __init__(self, args):
+    """Create a _JobFilter from the given args.
+
+    Args:
+      args: The argparse.Namespace containing the parsed arguments.
+    """
+    self.preds = []
+    if args.created_after or args.created_before:
+      self._ParseTimePredicate(args.created_after, args.created_before)
+
+  def __call__(self, job):
+    return all([pred(job) for pred in self.preds])
+
+  def _ParseTimePredicate(self, after, before):
+    """Return a predicate for filtering jobs by their creation time.
+
+    Args:
+      after: Only return true if the job was created after this time.
+      before: Only return true if the job was created before this time.
+    """
+    if after and (not before):
+      self.preds.append(lambda x: times.ParseDateTime(x.createTime) > after)
+    elif (not after) and before:
+      self.preds.append(lambda x: times.ParseDateTime(x.createTime) <= before)
+    elif after and before:
+
+      def _Predicate(x):
+        create_time = times.ParseDateTime(x.createTime)
+        return after < create_time and create_time <= before
+
+      self.preds.append(_Predicate)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/resume_unsupported_sdk.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2020 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implementation of gcloud dataflow jobs resume-unsupported-sdk command.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataflow import apis
+from googlecloudsdk.api_lib.util import exceptions
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataflow import job_utils
+from googlecloudsdk.core import log
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
+@base.DefaultUniverseOnly
+class Resume(base.Command):
+  """Resumes job running with the specified job id.
+
+     Resumes a pipeline job which is running on an unsupported SDK version.
+  """
+
+  @staticmethod
+  def Args(parser):
+    """Register flags for this command."""
+    job_utils.ArgsForJobRef(parser)
+
+    parser.add_argument(
+        "--token",
+        help=("The resume token unique to the job."),
+        required=True)
+
+  def Run(self, args):
+    """This is what gets called when the user runs this command.
+
+    Args:
+      args: all the arguments that were provided to this command invocation.
+    """
+    job_ref = job_utils.ExtractJobRef(args)
+    try:
+      apis.Jobs.ResumeUnsupportedSDK(
+          job_ref.jobId,
+          "unsupported_sdk_temporary_override_token=" + args.token,
+          project_id=job_ref.projectId,
+          region_id=job_ref.location)
+      log.status.Print("Resuming job running on unsupported SDK version [{0}]. "
+                       "This job may be cancelled in the future. For more "
+                       "details, see https://cloud.google.com/dataflow/docs/"
+                       "support/sdk-version-support-status.".format(
+                           job_ref.jobId))
+    except exceptions.HttpException as error:
+      log.status.Print(
+          (
+              "Failed to resume job [{0}]: {1} Ensure that you have permission "
+              "to access the job, the `--region` flag, {2}, is correct for the "
+              "job and the `--token` flag, {3}, corresponds to the job."
+          ).format(
+              job_ref.jobId,
+              error.payload.status_message,
+              job_ref.location,
+              args.token,
+          )
+      )

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/run.py

@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of gcloud dataflow jobs run command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataflow import apis
+from googlecloudsdk.calliope import actions
+from googlecloudsdk.calliope import arg_parsers
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataflow import dataflow_util
+from googlecloudsdk.command_lib.dataflow import job_utils
+from googlecloudsdk.core import properties
+
+
+def _CommonArgs(parser):
+  """Register flags for this command.
+
+  Args:
+    parser: argparse.ArgumentParser to register arguments with.
+  """
+  job_utils.CommonArgs(parser)
+
+  parser.add_argument(
+      'job_name',
+      metavar='JOB_NAME',
+      help='The unique name to assign to the job.')
+
+  parser.add_argument(
+      '--gcs-location',
+      help=('The Google Cloud Storage location of the job template to run. '
+            "(Must be a URL beginning with 'gs://'.)"),
+      type=arg_parsers.RegexpValidator(r'^gs://.*',
+                                       'Must begin with \'gs://\''),
+      required=True)
+
+  parser.add_argument(
+      '--staging-location',
+      help=('The Google Cloud Storage location to stage temporary files. '
+            "(Must be a URL beginning with 'gs://'.)"),
+      type=arg_parsers.RegexpValidator(r'^gs://.*',
+                                       'Must begin with \'gs://\''))
+
+  parser.add_argument(
+      '--parameters',
+      metavar='PARAMETERS',
+      type=arg_parsers.ArgDict(),
+      action=arg_parsers.UpdateAction,
+      help='The parameters to pass to the job.')
+
+  parser.add_argument(
+      '--enable-streaming-engine',
+      action=actions.StoreBooleanProperty(
+          properties.VALUES.dataflow.enable_streaming_engine),
+      help='Enabling Streaming Engine for the streaming job.')
+
+  parser.add_argument(
+      '--additional-experiments',
+      metavar='ADDITIONAL_EXPERIMENTS',
+      type=arg_parsers.ArgList(),
+      action=arg_parsers.UpdateAction,
+      help=('Additional experiments to pass to the job. These experiments are '
+            'appended to any experiments already set by the template.'))
+
+  parser.add_argument(
+      '--additional-user-labels',
+      metavar='ADDITIONAL_USER_LABELS',
+      type=arg_parsers.ArgDict(),
+      action=arg_parsers.UpdateAction,
+      help=(
+          'Additional user labels to pass to the job. Example: '
+          '--additional-user-labels=\'key1=value1,key2=value2\''
+      ),
+  )
+
+  # TODO(b/139889563): Mark as required when default region is removed
+  parser.add_argument(
+      '--region',
+      metavar='REGION_ID',
+      help=('Region ID of the job\'s regional endpoint. ' +
+            dataflow_util.DEFAULT_REGION_MESSAGE))
+
+  streaming_update_args = parser.add_argument_group()
+  streaming_update_args.add_argument(
+      '--update',
+      help='Set this to true for streaming update jobs.',
+      action=arg_parsers.StoreTrueFalseAction,
+      required=True,
+  )
+  streaming_update_args.add_argument(
+      '--transform-name-mappings',
+      metavar='TRANSFORM_NAME_MAPPINGS',
+      type=arg_parsers.ArgDict(),
+      action=arg_parsers.UpdateAction,
+      help='Transform name mappings for the streaming update job.',
+  )
+
+
+def _CommonRun(args):
+  """Runs the command.
+
+  Args:
+    args: The arguments that were provided to this command invocation.
+
+  Returns:
+    A Job message.
+  """
+  arguments = apis.TemplateArguments(
+      project_id=properties.VALUES.core.project.Get(required=True),
+      region_id=dataflow_util.GetRegion(args),
+      job_name=args.job_name,
+      gcs_location=args.gcs_location,
+      zone=args.zone,
+      max_workers=args.max_workers,
+      num_workers=args.num_workers,
+      network=args.network,
+      subnetwork=args.subnetwork,
+      worker_machine_type=args.worker_machine_type,
+      staging_location=args.staging_location,
+      kms_key_name=args.dataflow_kms_key,
+      disable_public_ips=properties.VALUES.dataflow.disable_public_ips.GetBool(),
+      parameters=args.parameters,
+      service_account_email=args.service_account_email,
+      worker_region=args.worker_region,
+      worker_zone=args.worker_zone,
+      enable_streaming_engine=properties.VALUES.dataflow.enable_streaming_engine.GetBool(),
+      streaming_update=args.update,
+      transform_name_mappings=args.transform_name_mappings,
+      additional_experiments=args.additional_experiments,
+      additional_user_labels=args.additional_user_labels,
+  )
+  if args.update:
+    return apis.Templates.LaunchDynamicTemplate(arguments)
+  else:
+    return apis.Templates.Create(arguments)
+
+
+@base.UniverseCompatible
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+class Run(base.Command):
+  """Runs a job from the specified path."""
+
+  @staticmethod
+  def Args(parser):
+    _CommonArgs(parser)
+
+  def Run(self, args):
+    return _CommonRun(args)
+
+
+@base.UniverseCompatible
+@base.ReleaseTracks(base.ReleaseTrack.BETA)
+class RunBeta(Run):
+  """Runs a job from the specified path."""
+
+  @staticmethod
+  def Args(parser):
+    _CommonArgs(parser)
+
+  def Run(self, args):
+    return _CommonRun(args)

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/show.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implementation of gcloud dataflow jobs show command.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataflow import apis
+from googlecloudsdk.api_lib.dataflow import job_display
+from googlecloudsdk.api_lib.dataflow import step_json
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.dataflow import job_utils
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.GA)
+class Show(base.Command):
+  """Shows a short description of the given job.
+  """
+
+  @staticmethod
+  def Args(parser):
+    """Register flags for this command.
+
+    Args:
+      parser: argparse.ArgumentParser to register arguments with.
+    """
+    job_utils.ArgsForJobRef(parser)
+
+    parser.add_argument(
+        '--environment', action='store_true',
+        help='If present, the environment will be listed.')
+    parser.add_argument(
+        '--steps', action='store_true',
+        help='If present, the steps will be listed.')
+
+  def Run(self, args):
+    """Runs the command.
+
+    Args:
+      args: The arguments that were provided to this command invocation.
+
+    Returns:
+      A Job message.
+    """
+    job_ref = job_utils.ExtractJobRef(args)
+    job = apis.Jobs.Get(
+        job_id=job_ref.jobId,
+        project_id=job_ref.projectId,
+        region_id=job_ref.location,
+        view=apis.Jobs.GET_REQUEST.ViewValueValuesEnum.JOB_VIEW_ALL)
+
+    # Extract the basic display information for the job
+    shown_job = job_display.DisplayInfo(job)
+
+    if args.environment:
+      shown_job.environment = job.environment
+
+    if args.steps:
+      shown_job.steps = [
+          self._PrettyStep(step) for step in step_json.ExtractSteps(job)]
+
+    return shown_job
+
+  def _PrettyStep(self, step):
+    """Prettify a given step, by only extracting certain pieces of info.
+
+    Args:
+      step: The step to prettify.
+    Returns:
+      A dictionary describing the step.
+    """
+    return {
+        'id': step['name'],
+        'user_name': step['properties']['user_name']
+    }

gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/update_options.py

@@ -0,0 +1,154 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2023 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of gcloud dataflow jobs update-options command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.dataflow import apis
+from googlecloudsdk.calliope import base
+from googlecloudsdk.calliope import exceptions
+from googlecloudsdk.command_lib.dataflow import job_utils
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA, base.ReleaseTrack.BETA)
+class UpdateOptions(base.Command):
+  """Update pipeline options on-the-fly for running Dataflow jobs.
+
+  This command can modify properties of running Dataflow jobs. Currently, only
+  updating autoscaling settings for Streaming Engine jobs is supported.
+
+  Adjust the autoscaling settings for Streaming Engine Dataflow jobs by
+  providing at-least one of --min-num-workers or --max-num-workers or
+  --worker-utilization-hint (or all 3), or --unset-worker-utilization-hint
+  (which cannot be run at the same time as --worker-utilization-hint but works
+  with the others).
+  Allow a few minutes for the changes to take effect.
+
+  Note that autoscaling settings can only be modified on-the-fly for Streaming
+  Engine jobs. Attempts to modify batch job or Streaming Appliance jobs will
+  fail.
+
+
+  ## EXAMPLES
+
+  Modify autoscaling settings to scale between 5-10 workers:
+
+    $ {command} --min-num-workers=5 --max-num-workers=10
+
+  Require a job to use at least 2 workers:
+
+    $ {command} --min-num-workers=2
+
+  Require a job to use at most 20 workers:
+
+    $ {command} --max-num-workers=20
+
+  Adjust the hint of target worker utilization to 70% for horizontal
+  autoscaling:
+
+    $ {command} --worker-utilization-hint=0.7
+
+  "Unset" worker utilization hint so that horizontal scaling will rely on its
+  default CPU utilization target:
+
+    $ {command} --unset-worker-utilization-hint
+  """
+
+  @staticmethod
+  def Args(parser):
+    """Register flags for this command."""
+    job_utils.ArgsForJobRef(parser)
+    parser.add_argument(
+        '--min-num-workers',
+        type=int,
+        help=(
+            'Lower-bound for autoscaling, between 1-1000. Only supported for'
+            ' streaming-engine jobs.'
+        ),
+    )
+    parser.add_argument(
+        '--max-num-workers',
+        type=int,
+        help=(
+            'Upper-bound for autoscaling, between 1-1000. Only supported for'
+            ' streaming-engine jobs.'
+        ),
+    )
+    parser.add_argument(
+        '--worker-utilization-hint',
+        type=float,
+        help=(
+            'Target CPU utilization for autoscaling, ranging from 0.1 to 0.9.'
+            ' Only supported for streaming-engine jobs with autoscaling'
+            ' enabled.'
+        ),
+    )
+    parser.add_argument(
+        '--unset-worker-utilization-hint',
+        action='store_true',
+        help=(
+            'Unset --worker-utilization-hint. This causes the'
+            ' job autoscaling to fall back to internal tunings'
+            ' if they exist, or otherwise use the default hint value.'
+        ),
+    )
+
+  def Run(self, args):
+    """Called when the user runs gcloud dataflow jobs update-options ...
+
+    Args:
+      args: all the arguments that were provided to this command invocation.
+
+    Returns:
+      The updated Job
+    """
+    if (
+        args.min_num_workers is None
+        and args.max_num_workers is None
+        and args.worker_utilization_hint is None
+        and not args.unset_worker_utilization_hint
+    ):
+      raise exceptions.OneOfArgumentsRequiredException(
+          [
+              '--min-num-workers',
+              '--max-num-workers',
+              '--worker-utilization-hint',
+              '--unset-worker-utilization-hint',
+          ],
+          'You must provide at-least one field to update',
+      )
+    elif (
+        args.worker_utilization_hint is not None
+        and args.unset_worker_utilization_hint
+    ):
+      raise exceptions.ConflictingArgumentsException(
+          'The arguments --worker-utilization-hint and'
+          ' --unset-worker-utilization-hint are mutually exclusive (as the'
+          ' unset command will unset the given hint), and must be called'
+          ' separately.',
+      )
+
+    job_ref = job_utils.ExtractJobRef(args)
+    return apis.Jobs.UpdateOptions(
+        job_ref.jobId,
+        project_id=job_ref.projectId,
+        region_id=job_ref.location,
+        min_num_workers=args.min_num_workers,
+        max_num_workers=args.max_num_workers,
+        worker_utilization_hint=args.worker_utilization_hint,
+        unset_worker_utilization_hint=args.unset_worker_utilization_hint,
+    )