feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/command_lib/managed_flink/flags.py

@@ -0,0 +1,361 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Flags and helpers for the Managed Flink CLI."""
+
+import argparse
+from googlecloudsdk.calliope import arg_parsers
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.util import completers
+from googlecloudsdk.command_lib.util import parameter_info_lib
+
+_AUTOTUNING_MODES = {
+    'fixed': 'The number of taskmanagers is fixed.',
+    'elastic': (
+        'The number of taskmanagers is scaled automatically according to'
+        ' workload.'
+    ),
+}
+
+# Completers
+
+
+class ListCommandParameterInfo(parameter_info_lib.ParameterInfoByConvention):
+  """Helper class for ListCommandCompleter."""
+
+  def GetFlag(
+      self,
+      parameter_name,
+      parameter_value=None,
+      check_properties=True,
+      for_update=False,
+  ):
+    return super(ListCommandParameterInfo, self).GetFlag(
+        parameter_name,
+        parameter_value,
+        check_properties=check_properties,
+        for_update=for_update,
+    )
+
+
+class ListCommandCompleter(completers.ListCommandCompleter):
+  """Helper class for DeploymentCompleter."""
+
+  def ParameterInfo(self, parsed_args, arguments):
+    return ListCommandParameterInfo(
+        parsed_args,
+        arguments,
+        self.collection,
+        updaters=COMPLETERS_BY_CONVENTION,
+    )
+
+
+class DeploymentCompleter(ListCommandCompleter):
+  """Completer for listing deployments."""
+
+  def __init__(self, **kwargs):
+    super(DeploymentCompleter, self).__init__(
+        collection='managedflink.projects.locations.deployments',
+        list_command='managed-flink deployments list',
+        **kwargs,
+    )
+
+
+COMPLETERS_BY_CONVENTION = {'deployment': (DeploymentCompleter, False)}
+
+
+# Flags
+def AddNetworkConfigArgs(parser):
+  """Adds network config arguments."""
+  parser.add_argument(
+      '--network-config-vpc',
+      metavar='NETWORK',
+      dest='network',
+      help='The network to use for the job.',
+  )
+  parser.add_argument(
+      '--network-config-subnetwork',
+      metavar='SUBNETWORK',
+      dest='subnetwork',
+      help='The subnetwork to use for the job.',
+  )
+
+
+def AddWorkloadIdentityArgument(parser):
+  """Adds workload identity argument."""
+  parser.add_argument(
+      '--workload-identity',
+      metavar='WORKLOAD_IDENTITY',
+      dest='workload_identity',
+      help=(
+          'The workload identity to use for the job. Managed Flink'
+          ' Default Workload Identity will be used if not specified.'
+      ),
+  )
+
+
+def AddLocationArgument(parser):
+  """Creates location argument."""
+  base.Argument(
+      '--location',
+      metavar='LOCATION',
+      required=True,
+      dest='location',
+      suggestion_aliases=['--region'],
+      help='The location to run the job in.',
+  ).AddToParser(parser)
+
+
+def AddJobTypeArgument(parser):
+  """Job type arguments."""
+  base.Argument(
+      '--job-type',
+      metavar='JOB_TYPE',
+      choices=['auto', 'jar', 'python', 'sql'],
+      default='auto',
+      help=(
+          'The type of job to run. If "auto" will be selected based on the file'
+          ' extension for the job argument.'
+      ),
+  ).AddToParser(parser)
+
+
+def AddJobJarArgument(parser):
+  """Creates the job argument."""
+  base.Argument(
+      'job',
+      metavar='JAR|PY|SQL',
+      help=(
+          'The file containing the Flink job to run. Can be a jar, python, or'
+          ' sql file.'
+      ),
+  ).AddToParser(parser)
+
+
+def AddExtraJarsArgument(parser):
+  """Creates the extra jars argument."""
+  base.Argument(
+      '--jars',
+      metavar='JAR',
+      type=arg_parsers.ArgList(),
+      dest='extra_jars',
+      help=(
+          'The extra jars to pass to the job. Can be a jar, python, or'
+          ' sql file.'
+      ),
+  ).AddToParser(parser)
+
+
+def AddDryRunArgument(parser):
+  """Creates dry run argument."""
+  base.Argument(
+      '--dry-run',
+      action='store_true',
+      dest='dry_run',
+      default=False,
+      required=False,
+      help='Return command used to submit a job without invoking API.',
+  ).AddToParser(parser)
+
+
+# This has been temporarily disabled. Commented out to avoid confusing
+# test coverage.
+# def AddManagedKafkaClustersArgument(parser):
+#  """Creates the managed flink argument."""
+#  base.Argument(
+#      '--managed-kafka-clusters',
+#      metavar='MANAGED_KAFKA_CLUSTERS',
+#      dest='managed_kafka_clusters',
+#      type=arg_parsers.ArgList(),
+#      help='Specifies managed kafka clusters to associate with this job.',
+#  ).AddToParser(parser)
+
+
+def AddMainClassArgument(parser):
+  """Creates main class argument."""
+  base.Argument(
+      '--class',
+      metavar='CLASS',
+      dest='main_class',
+      help=(
+          'The main class of the Flink job. Required if the jar file manifest'
+          ' does not contain a main class.'
+      ),
+  ).AddToParser(parser)
+
+
+def AddJobArgsCollector(parser):
+  """Collects extra arguments into the job_args list."""
+  parser.add_argument(
+      'job_args',
+      nargs=argparse.REMAINDER,
+      help='The job arguments to pass.',
+  )
+
+
+def AddNameArgument(parser):
+  """Creates name argument."""
+  base.Argument(
+      '--name',
+      metavar='NAME',
+      dest='name',
+      required=False,
+      help='The name of the job. The Flink job name will be used if not set.',
+  ).AddToParser(parser)
+
+
+def AddJobIdArgument(parser):
+  """Creates job id argument."""
+  base.Argument(
+      'job_id',
+      metavar='JOBID',
+      help='The id of the job.',
+  ).AddToParser(parser)
+
+
+def AddAsyncArgument(parser, default=False):
+  """Creates async argument."""
+  base.Argument(
+      '--async',
+      action='store_true',
+      dest='async_submit',
+      default=default,
+      required=False,
+      help='Return immediately after job submission.',
+  ).AddToParser(parser)
+
+
+def AddStagingLocationArgument(parser):
+  """Creates staging location argument."""
+  base.Argument(
+      '--staging-location',
+      metavar='STAGING_LOCATION',
+      dest='staging_location',
+      required=True,
+      help=(
+          'The Google Cloud Storage staging location for the job. Must start'
+          ' with gs://'
+      ),
+  ).AddToParser(parser)
+
+
+def AddDeploymentArgument(
+    parser,
+    help_text_to_prepend=None,
+    help_text_to_overwrite=None,
+    required=False,
+):
+  """Creates deployment argument."""
+
+  if help_text_to_overwrite:
+    help_text = help_text_to_overwrite
+  else:
+    help_text = """
+    The Flink Deployment to use for this invocation.
+    """
+
+  if help_text_to_prepend:
+    help_text = '\n\n'.join((help_text_to_prepend, help_text))
+
+  base.Argument(
+      '--deployment',
+      metavar='DEPLOYMENT_NAME',
+      required=required,
+      dest='deployment',
+      completer=DeploymentCompleter,
+      help=help_text,
+  ).AddToParser(parser)
+
+
+def AddAutotuningModeArgument(parser, default='elastic', required=False):
+  """Creates autotuning mode argument."""
+
+  base.Argument(
+      '--autotuning-mode',
+      metavar='AUTOTUNING_MODE',
+      choices=_AUTOTUNING_MODES,
+      default=default,
+      required=required,
+      dest='autotuning_mode',
+      help='Selects the autotuning mode for jobs.',
+  ).AddToParser(parser)
+
+
+def AddFixedParallelismArgs(parser):
+  """Adds fixed parallelism arguments."""
+  parser.add_argument(
+      '--parallelism',
+      type=arg_parsers.BoundedInt(lower_bound=1, upper_bound=10000),
+      help='The parallelism of the job when in "fixed" autotuning mode.',
+  )
+
+
+def AddElasticParallelismArgs(parser):
+  """Adds elastic parallelism arguments."""
+  parser.add_argument(
+      '--min-parallelism',
+      type=arg_parsers.BoundedInt(lower_bound=1, upper_bound=10000),
+      help=(
+          'The minimum parallelism of the job when in "elastic" autotuning'
+          ' mode. This will also be the initial parallelism of the job.'
+      ),
+  )
+  parser.add_argument(
+      '--max-parallelism',
+      type=arg_parsers.BoundedInt(lower_bound=1, upper_bound=10000),
+      help=(
+          'The maximum parallelism of the job when in "elastic" autotuning'
+          ' mode.'
+      ),
+  )
+
+
+def AddShowOutputArgument(parser):
+  """Creates show output argument."""
+  base.Argument(
+      '--enable-output',
+      action='store_true',
+      dest='show_output',
+      default=False,
+      required=False,
+      help='Shows the output of the Flink client.',
+  ).AddToParser(parser)
+
+
+def AddExtraArchivesArgument(parser):
+  """Creates the extra archives argument."""
+  base.Argument(
+      '--archives',
+      metavar='ZIP',
+      type=arg_parsers.ArgList(),
+      dest='archives',
+      help=(
+          'The extra archives to pass to the job. Can be a zip file containing'
+          ' resource files for the job.'
+      ),
+  ).AddToParser(parser)
+
+
+def AddPythonVirtualEnvArgument(parser):
+  """Creates main class argument."""
+  base.Argument(
+      '--python-venv',
+      metavar='ZIP',
+      dest='python_venv',
+      help=(
+          'The path to the zip file to manage the virtualenv for Python'
+          ' dependencies. Required if the job type is python. Must start with'
+          ' gs://.'
+      ),
+  ).AddToParser(parser)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/command_lib/managed_flink/flags.yaml

@@ -0,0 +1,78 @@
+deployment-name:
+  api_field: deploymentId
+  arg_name: deployment
+  help_text: |
+    deployment name.
+
+max-slots:
+  api_field: deployment.deploymentSpec.limits.maxSlots
+  arg_name: max-slots
+  type: googlecloudsdk.core.util.scaled_integer:ParseInteger
+  help_text: |
+    max slots of the Flink deployment.
+
+display-name:
+  api_field: deployment.displayName
+  arg_name: display-name
+  help_text: |
+    display name of the Flink deployment.
+
+workload-identity:
+  api_field: deployment.deploymentSpec.workloadIdentity
+  arg_name: workload-identity
+  help_text: |
+    The workload identity to use for the deployment. Managed Flink Default Workload Identity will be used if not specified.
+
+network-config-vpc:
+  api_field: deployment.deploymentSpec.networkConfig.vpc
+  arg_name: network-config-vpc
+  help_text: |
+    fully qualified VPC network for the Flink deployment network config.
+    Formatted as: projects/{project}/global/networks/{network_id}.
+
+network-config-subnetwork:
+  api_field: deployment.deploymentSpec.networkConfig.subnetwork
+  arg_name: network-config-subnetwork
+  help_text: |
+    subnetwork for the Flink deployment network config.
+
+secrets-paths:
+  api_field: deployment.deploymentSpec.secretsPaths
+  arg_name: secrets-paths
+  help_text: |
+    path to the secrets manager for the Flink deployment. (ie. projects/{my-project}/secrets/{my-secret}/versions/{1})
+
+## Job Specific Flags
+
+deployment:
+  arg_name: deployment
+  help_text: |
+    deployment name.
+
+staging-location:
+  arg_name: staging-location
+  help_text: |
+    staging location for artifacts related to Flink jobs. For example:
+    `gs://staging-bucket/flink`
+
+args:
+  arg_name: args
+  type: "googlecloudsdk.calliope.arg_parsers:ArgList:"
+  help_text: |
+    List of arguments to pass to the Flink job.
+
+class:
+  arg_name: class
+  help_text: |
+    Class with the program entry point (`main()` method). Only needed if the
+    JAR files does not specify the class in its manifest.
+
+# GCP "Better Together" Flags
+
+managed-kafka-clusters:
+  api_field: job.jobSpec.managedKafkaConfig.managedKafkaClusters
+  arg_name: managed-kafka-clusters
+  type: "googlecloudsdk.calliope.arg_parsers:ArgList:"
+  help_text: |
+    a list of Managed Kafka clusters for the Flink job to connect to. For example:
+    `projects/123456789/locations/us-central1/clusters/my-cluster`.

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/command_lib/managed_flink/flink_backend.py

@@ -0,0 +1,404 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Flink command library functions for the Flink cli binary."""
+
+import copy
+import os
+from urllib import parse
+
+from apitools.base.py import transfer
+from googlecloudsdk.api_lib.storage import storage_api
+from googlecloudsdk.api_lib.storage import storage_util
+from googlecloudsdk.calliope import base
+from googlecloudsdk.calliope import exceptions
+from googlecloudsdk.command_lib.artifacts import requests
+from googlecloudsdk.command_lib.util import java
+from googlecloudsdk.command_lib.util.anthos import binary_operations
+from googlecloudsdk.core import config
+from googlecloudsdk.core import exceptions as core_exceptions
+from googlecloudsdk.core import resources
+from googlecloudsdk.core.credentials import transports
+from googlecloudsdk.core.util import platforms
+
+DEFAULT_ENV_ARGS = {}
+
+DEFAULT_CONFIG_LOCATION = {
+    platforms.OperatingSystem.WINDOWS.id: os.path.join(
+        '%APPDATA%', 'google', 'flink', 'config.yaml'
+    ),
+    platforms.OperatingSystem.MACOSX.id: (
+        '~/Library/Preferences/google/flink/config.yaml'
+    ),
+    platforms.OperatingSystem.LINUX.id: '~/.config/google/flink/config.yaml',
+}
+
+_RELEASE_TRACK_TO_VERSION = {
+    base.ReleaseTrack.ALPHA: 'v1alpha',
+    base.ReleaseTrack.BETA: 'v1beta',
+    base.ReleaseTrack.GA: 'v1',
+}
+
+MISSING_BINARY = (
+    'Could not locate managed flink client executable [{binary}]'
+    ' on the system PATH. '
+    'Please ensure gcloud managed-flink-client component is properly '
+    'installed. '
+    'See https://cloud.google.com/sdk/docs/components for '
+    'more details.'
+)
+
+# 3 MiB as from the Artifact Registry default.
+DEFAULT_CHUNK_SIZE = 3 * 1024 * 1024
+
+
+class FileUploadError(core_exceptions.Error):
+  """Exception raised when a file upload fails."""
+
+
+class FileDownloadError(core_exceptions.Error):
+  """Exception raised when a file download fails."""
+
+
+def DummyJar():
+  """Get flink python jar location."""
+  return os.path.join(
+      config.Paths().sdk_root,
+      'platform',
+      'managed-flink-client',
+      'lib',
+      'flink-python-1.19.0.jar',
+  )
+
+
+def Upload(files, destination, storage_client=None):
+  """Uploads a list of files passed as strings to a Cloud Storage bucket."""
+  client = storage_client or storage_api.StorageClient()
+  destinations = dict()
+  for file_to_upload in files:
+    file_name = os.path.basename(file_to_upload)
+    dest_url = os.path.join(destination, file_name)
+    dest_object = storage_util.ObjectReference.FromUrl(dest_url)
+    try:
+      client.CopyFileToGCS(file_to_upload, dest_object)
+      destinations[file_to_upload] = dest_url
+    except exceptions.BadFileException as e:
+      raise FileUploadError(
+          'Failed to upload file ["{}"] to "{}": {}'.format(
+              ','.join(files), destination, e
+          )
+      )
+  return destinations
+
+
+def CreateRegistryFromArtifactUri(artifact_uri):
+  """Creates a registry from an artifact URI.
+
+  Args:
+    artifact_uri:
+      ar://<project>/<location>/<repository>/<file/path/version/file.jar>.
+
+  Returns:
+    Jar file name, The registry resource.
+  """
+  try:
+    parsed_url = parse.urlparse(artifact_uri)
+  except:
+    raise exceptions.InvalidArgumentException(
+        'JAR|PY|SQL',
+        'Artifact URI [{0}] is invalid. Must be in the format of'
+        ' ar://<project>/<location>/<repository>/<file/path/version/file.jar>.'
+        .format(artifact_uri),
+    )
+  split_path = parsed_url.path.split('/')
+  cleaned_split_path = [path for path in split_path if path]
+  if parsed_url.netloc:
+    cleaned_split_path = [parsed_url.netloc] + cleaned_split_path
+  if len(cleaned_split_path) < 4 or not cleaned_split_path[-1].endswith('.jar'):
+    raise exceptions.InvalidArgumentException(
+        'JAR|PY|SQL',
+        'Artifact URI [{0}] is invalid. Must be in the format of'
+        ' ar://<project>/<location>/<repository>/<file/path/version/file.jar>.'
+        .format(artifact_uri),
+    )
+  jar_file = '/'.join(cleaned_split_path[3:])
+  cleaned_jar_file = (
+      jar_file.replace('/', '%2F').replace('+', '%2B').replace('^', '%5E')
+  )
+
+  return jar_file, resources.REGISTRY.Create(
+      'artifactregistry.projects.locations.repositories.files',
+      projectsId=cleaned_split_path[0],
+      locationsId=cleaned_split_path[1],
+      repositoriesId=cleaned_split_path[2],
+      filesId=cleaned_jar_file,
+  )
+
+
+def DownloadJarFromArtifactRegistry(
+    dest_path, artifact_jar_path, artifact_client=None
+):
+  """Downloads a JAR file from Google Artifact Registry."""
+
+  # 1. Initialize Clients
+  client = artifact_client or requests.GetClient()
+  messages = requests.GetMessages()
+
+  # 2. Construct the Request
+  request = messages.ArtifactregistryProjectsLocationsRepositoriesFilesDownloadRequest(
+      name=artifact_jar_path
+  )
+
+  d = transfer.Download.FromFile(dest_path, True, chunksize=DEFAULT_CHUNK_SIZE)
+  d.bytes_http = transports.GetApitoolsTransport(response_encoding=None)
+  try:
+    client.projects_locations_repositories_files.Download(request, download=d)
+  except Exception as e:
+    raise FileDownloadError(
+        'Failed to download JAR from Artifact Registry: {}'.format(e)
+    )
+  finally:
+    d.stream.close()
+
+
+def CheckStagingLocation(staging_location):
+  dest = storage_util.ObjectReference.FromUrl(staging_location, True)
+  storage_util.ValidateBucketUrl(dest.bucket)
+  storage_api.StorageClient().GetBucket(dest.bucket)
+
+
+def GetEnvArgsForCommand(extra_vars=None, exclude_vars=None):
+  """Helper function to add our environment variables to the environment."""
+  env = copy.deepcopy(os.environ)
+  env.update(DEFAULT_ENV_ARGS)
+  if extra_vars:
+    env.update(extra_vars)
+  if exclude_vars:
+    for var in exclude_vars:
+      env.pop(var, None)
+  return env
+
+
+def PlatformExecutable():
+  """Get the platform executable location."""
+  return os.path.join(
+      config.Paths().sdk_root,
+      'platform',
+      'managed-flink-client',
+      'bin',
+      'managed-flink-client',
+  )
+
+
+def ValidateAutotuning(
+    autotuning_mode, min_parallelism, max_parallelism, parallelism
+):
+  """Validate autotuning configurations."""
+  if autotuning_mode == 'elastic':
+    if parallelism:
+      raise exceptions.InvalidArgumentException(
+          'parallelism',
+          'Parallelism must NOT be set for elastic autotuning mode.',
+      )
+    if not min_parallelism:
+      raise exceptions.InvalidArgumentException(
+          'min-parallelism',
+          'Min parallelism must be set for elastic autotuning mode.',
+      )
+    if not max_parallelism:
+      raise exceptions.InvalidArgumentException(
+          'max-parallelism',
+          'Max parallelism must be set for elastic autotuning mode.',
+      )
+    if min_parallelism > max_parallelism:
+      raise exceptions.InvalidArgumentException(
+          'min-parallelism',
+          'Min parallelism must be less than or equal to max parallelism.',
+      )
+  else:
+    if not parallelism:
+      raise exceptions.InvalidArgumentException(
+          'parallelism',
+          'Parallelism must be set to a value of 1 or greater for fixed'
+          ' autotuning mode.',
+      )
+    if min_parallelism:
+      raise exceptions.InvalidArgumentException(
+          'min-parallelism',
+          'Min parallelism must NOT be set for fixed autotuning mode.',
+      )
+    if max_parallelism:
+      raise exceptions.InvalidArgumentException(
+          'max-parallelism',
+          'Max parallelism must NOT be set for fixed autotuning mode.',
+      )
+
+
+class FlinkClientWrapper(binary_operations.BinaryBackedOperation):
+  """Wrapper for the Flink client binary."""
+
+  _java_path = None
+
+  def __init__(self, **kwargs):
+    custom_errors = {
+        'MISSING_EXEC': MISSING_BINARY.format(binary='managed-flink-client')
+    }
+    super(FlinkClientWrapper, self).__init__(
+        binary='managed-flink-client', custom_errors=custom_errors, **kwargs
+    )
+    self._java_path = java.RequireJavaInstalled('Managed Flink Client', 11)
+    # BinaryBackedOperation assumes the binary lives in bin, but that's
+    # not the case for managed-flink-client so we need to perform an
+    # additiona search. If it still doesn't exist then we can admit that
+    # it's not installed.
+    if not os.path.exists(self._executable):
+      component_executable = PlatformExecutable()
+      if os.path.exists(component_executable):
+        self._executable = component_executable
+
+  def _ParseArgsForCommand(
+      self,
+      command,
+      job_type,
+      jar,
+      staging_location,
+      temp_dir,
+      target='local',
+      release_track=base.ReleaseTrack.ALPHA,
+      location=None,
+      deployment=None,
+      network=None,
+      subnetwork=None,
+      name=None,
+      extra_jars=None,
+      managed_kafka_clusters=None,
+      main_class=None,
+      extra_args=None,
+      extra_archives=None,
+      python_venv=None,
+      **kwargs
+  ):
+    """Parses the arguments for the given command."""
+
+    if command != 'run':
+      raise binary_operations.InvalidOperationForBinary(
+          'Invalid operation [{}] for Flink CLI.'.format(command)
+      )
+
+    args = list()
+    if network:
+      args.append('-Dgcloud.network={0}'.format(network))
+    if subnetwork:
+      args.append('-Dgcloud.subnetwork={0}'.format(subnetwork))
+    if location:
+      args.append('-Dgcloud.region={0}'.format(location))
+    if deployment:
+      args.append('-Dgcloud.deployment={0}'.format(deployment))
+    if name:
+      args.append('-Dgcloud.job.display-name={0}'.format(name))
+    #   This has been temporarily disabled and commented out to avoid
+    #   confusing coverage.
+    #    if managed_kafka_clusters:
+    #      args.append(
+    #          '-Dgcloud.managed-kafka-clusters={0}'.format(
+    #              ','.join(managed_kafka_clusters)
+    #          )
+    #      )
+
+    if not extra_args:
+      extra_args = []
+
+    job_args = list()
+    for arg in extra_args:
+      if arg.startswith('-D'):
+        args.append(arg)
+      else:
+        job_args.append(arg)
+
+    if job_type == 'sql':
+      udfs = []
+      if extra_jars:
+        for j in extra_jars:
+          udfs.append('--jar')
+          udfs.append(j)
+
+      return (
+          args
+          + [
+              '-Dexecution.target=gcloud',
+              '-Dgcloud.output-path={0}'.format(temp_dir),
+              '-Dgcloud.api.staging-location={0}'.format(staging_location),
+              '--file',
+              jar,
+          ]
+          + udfs
+          + job_args
+      )
+    elif job_type == 'python':
+      udfs = []
+      if extra_jars:
+        udfs.append('-Dgcloud.pipeline.jars={0}'.format(','.join(extra_jars)))
+
+      env_folder = python_venv.split('/')[-1]
+      archives = ['-Dpython.archives={0}'.format(python_venv)]
+      if extra_archives:
+        for archive in extra_archives:
+          archives.append(',')
+          archives.append(archive)
+      return (
+          [
+              command,
+              '--target',
+              target,
+          ]
+          + args
+          + [
+              '-Dgcloud.output-path={0}'.format(temp_dir),
+              '-Dgcloud.api.staging-location={0}'.format(staging_location),
+              '-Dpython.client.executable={0}/bin/python3'.format(env_folder),
+              '-Dpython.executable={0}/bin/python3'.format(env_folder),
+              '-Dpython.pythonpath={0}/lib/python3.10/site-packages/'.format(
+                  env_folder
+              ),
+          ]
+          + archives
+          + udfs
+          + [
+              '--python',
+              jar,
+          ]
+          + job_args
+      )
+    else:
+      class_arg = []
+      if main_class:
+        class_arg = ['--class', main_class]
+      udfs = []
+      if extra_jars:
+        udfs.append('-Dgcloud.pipeline.jars={0}'.format(','.join(extra_jars)))
+
+      return (
+          [command, '--target', target]
+          + class_arg
+          + args
+          + [
+              '-Dgcloud.output-path={0}'.format(temp_dir),
+              '-Dgcloud.api.staging-location={0}'.format(staging_location),
+          ]
+          + udfs
+          + [
+              jar,
+          ]
+          + job_args
+      )

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/command_lib/managed_flink/resources.yaml

@@ -0,0 +1,47 @@
+project:
+  name: project
+  collection: managedflink.projects
+  attributes:
+  - &project
+    parameter_name: projectsId
+    attribute_name: project
+    help: The Google Cloud Platform project name.
+    property: core/project
+
+location:
+  name: location
+  collection: managedflink.projects.locations
+  disable_auto_completers: false
+  attributes:
+  - &location
+    parameter_name: locationsId
+    attribute_name: location
+    help: |
+      ID of the location of the Apache Flink for BigQuery resource. See
+      https://cloud.google.com/managed-flink/docs/locations for a list of supported
+      locations.
+
+
+deployment:
+  name: deployment
+  collection: managedflink.projects.locations.deployments
+  request_id_field: deploymentId
+  attributes:
+  - *location
+  - &deployment
+    parameter_name: deploymentsId
+    attribute_name: deployment
+    help: |
+      The deployment name.
+
+job:
+  name: job
+  collection: managedflink.projects.locations.jobs
+  request_id_field: jobId
+  attributes:
+  - *location
+  - &job
+    parameter_name: jobsId
+    attribute_name: job
+    help: |
+      The job name.