novafarma/gcloud auth application-default login/google-cloud-sdk/lib/surface/dataflow/jobs/run.py

# -*- coding: utf-8 -*- #
# Copyright 2015 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Implementation of gcloud dataflow jobs run command."""

from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

from googlecloudsdk.api_lib.dataflow import apis
from googlecloudsdk.calliope import actions
from googlecloudsdk.calliope import arg_parsers
from googlecloudsdk.calliope import base
from googlecloudsdk.command_lib.dataflow import dataflow_util
from googlecloudsdk.command_lib.dataflow import job_utils
from googlecloudsdk.core import properties


def _CommonArgs(parser):
  """Register flags for this command.

  Args:
    parser: argparse.ArgumentParser to register arguments with.
  """
  job_utils.CommonArgs(parser)

  parser.add_argument(
      'job_name',
      metavar='JOB_NAME',
      help='The unique name to assign to the job.')

  parser.add_argument(
      '--gcs-location',
      help=('The Google Cloud Storage location of the job template to run. '
            "(Must be a URL beginning with 'gs://'.)"),
      type=arg_parsers.RegexpValidator(r'^gs://.*',
                                       'Must begin with \'gs://\''),
      required=True)

  parser.add_argument(
      '--staging-location',
      help=('The Google Cloud Storage location to stage temporary files. '
            "(Must be a URL beginning with 'gs://'.)"),
      type=arg_parsers.RegexpValidator(r'^gs://.*',
                                       'Must begin with \'gs://\''))

  parser.add_argument(
      '--parameters',
      metavar='PARAMETERS',
      type=arg_parsers.ArgDict(),
      action=arg_parsers.UpdateAction,
      help='The parameters to pass to the job.')

  parser.add_argument(
      '--enable-streaming-engine',
      action=actions.StoreBooleanProperty(
          properties.VALUES.dataflow.enable_streaming_engine),
      help='Enabling Streaming Engine for the streaming job.')

  parser.add_argument(
      '--additional-experiments',
      metavar='ADDITIONAL_EXPERIMENTS',
      type=arg_parsers.ArgList(),
      action=arg_parsers.UpdateAction,
      help=('Additional experiments to pass to the job. These experiments are '
            'appended to any experiments already set by the template.'))

  parser.add_argument(
      '--additional-user-labels',
      metavar='ADDITIONAL_USER_LABELS',
      type=arg_parsers.ArgDict(),
      action=arg_parsers.UpdateAction,
      help=(
          'Additional user labels to pass to the job. Example: '
          '--additional-user-labels=\'key1=value1,key2=value2\''
      ),
  )

  # TODO(b/139889563): Mark as required when default region is removed
  parser.add_argument(
      '--region',
      metavar='REGION_ID',
      help=('Region ID of the job\'s regional endpoint. ' +
            dataflow_util.DEFAULT_REGION_MESSAGE))

  streaming_update_args = parser.add_argument_group()
  streaming_update_args.add_argument(
      '--update',
      help='Set this to true for streaming update jobs.',
      action=arg_parsers.StoreTrueFalseAction,
      required=True,
  )
  streaming_update_args.add_argument(
      '--transform-name-mappings',
      metavar='TRANSFORM_NAME_MAPPINGS',
      type=arg_parsers.ArgDict(),
      action=arg_parsers.UpdateAction,
      help='Transform name mappings for the streaming update job.',
  )


def _CommonRun(args):
  """Runs the command.

  Args:
    args: The arguments that were provided to this command invocation.

  Returns:
    A Job message.
  """
  arguments = apis.TemplateArguments(
      project_id=properties.VALUES.core.project.Get(required=True),
      region_id=dataflow_util.GetRegion(args),
      job_name=args.job_name,
      gcs_location=args.gcs_location,
      zone=args.zone,
      max_workers=args.max_workers,
      num_workers=args.num_workers,
      network=args.network,
      subnetwork=args.subnetwork,
      worker_machine_type=args.worker_machine_type,
      staging_location=args.staging_location,
      kms_key_name=args.dataflow_kms_key,
      disable_public_ips=properties.VALUES.dataflow.disable_public_ips.GetBool(),
      parameters=args.parameters,
      service_account_email=args.service_account_email,
      worker_region=args.worker_region,
      worker_zone=args.worker_zone,
      enable_streaming_engine=properties.VALUES.dataflow.enable_streaming_engine.GetBool(),
      streaming_update=args.update,
      transform_name_mappings=args.transform_name_mappings,
      additional_experiments=args.additional_experiments,
      additional_user_labels=args.additional_user_labels,
  )
  if args.update:
    return apis.Templates.LaunchDynamicTemplate(arguments)
  else:
    return apis.Templates.Create(arguments)


@base.UniverseCompatible
@base.ReleaseTracks(base.ReleaseTrack.GA)
class Run(base.Command):
  """Runs a job from the specified path."""

  @staticmethod
  def Args(parser):
    _CommonArgs(parser)

  def Run(self, args):
    return _CommonRun(args)


@base.UniverseCompatible
@base.ReleaseTracks(base.ReleaseTrack.BETA)
class RunBeta(Run):
  """Runs a job from the specified path."""

  @staticmethod
  def Args(parser):
    _CommonArgs(parser)

  def Run(self, args):
    return _CommonRun(args)