feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/platform/bq/gcloud_wrapper/bq_to_gcloud_adapter.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+"""An adapter that takes a bq command and executes it with gcloud."""
+
+import json
+import re
+import subprocess
+from typing import Dict, Optional
+
+from gcloud_wrapper import bq_to_gcloud_config_classes
+from gcloud_wrapper import gcloud_runner
+from gcloud_wrapper import supported_gcloud_commands
+
+GCLOUD_COMMAND_GENERATOR = bq_to_gcloud_config_classes.GcloudCommandGenerator(
+    global_flag_mappings=supported_gcloud_commands.SUPPORTED_GLOBAL_FLAGS,
+    command_mappings=supported_gcloud_commands.SUPPORTED_COMMANDS,
+)
+
+
+def _swap_gcloud_box_to_bq_pretty(gcloud_output: str) -> str:
+  # TODO(b/355324165): Either use `maketrans` and `translate` (for performance)
+  # or use a regex (both for performance and to be able to center some headers).
+  return (
+      gcloud_output.replace('┌', '+')
+      .replace('┐', '+')
+      .replace('└', '+')
+      .replace('┘', '+')
+      .replace('├', '+')
+      .replace('┤', '+')
+      .replace('┬', '+')
+      .replace('┼', '+')
+      .replace('┴', '+')
+      .replace('│', '|')
+      .replace('─', '-')
+  )
+
+
+def _swap_gcloud_box_to_bq_sparse(gcloud_output: str) -> str:
+  """Converts gcloud table output to bq sparse output."""
+  stripped_upper_border = re.sub(r'┌.*┐', '', gcloud_output)
+  stripped_lower_border = re.sub(r'└.*┘', '', stripped_upper_border)
+  mostly_stripped_side_borders = re.sub(
+      r'│(.*)│', r' \1 ', stripped_lower_border
+  )
+  stripped_side_borders = re.sub(r'[├┤]', r' ', mostly_stripped_side_borders)
+  no_vertical_bars = re.sub(r'[│┼]', r' ', stripped_side_borders)
+  return re.sub(r'─', '-', no_vertical_bars)
+
+
+def run_bq_command_using_gcloud(
+    resource: str,
+    bq_command: str,
+    bq_global_flags: Dict[str, str],
+    bq_command_flags: Dict[str, str],
+    identifier: Optional[str] = None,
+    dry_run: bool = False,
+) -> int:
+  """Takes a bq command and executes it with gcloud returning the exit code.
+
+  Args:
+    resource: The resource the command is being run on, named to align with
+      `gcloud` commands. For example, 'jobs' or 'datasets'.
+    bq_command: The bq command to run. For example, 'ls' or 'show'.
+    bq_global_flags: The BQ CLI global flags to use when running the command.
+    bq_command_flags: The BQ CLI command flags to use when running the command.
+    identifier: The identifier of the resource to act on.
+    dry_run: If true, the gcloud command will be printed instead of executed.
+
+  Returns:
+    The exit code of the gcloud command.
+  """
+  gcloud_command = GCLOUD_COMMAND_GENERATOR.get_gcloud_command(
+      resource=resource,
+      bq_command=bq_command,
+      bq_global_flags=bq_global_flags,
+      bq_command_flags=bq_command_flags,
+      identifier=identifier,
+  )
+  if dry_run:
+    print(
+        ' '.join(
+            ['gcloud']
+            + bq_to_gcloud_config_classes.quote_flag_values(gcloud_command)
+        )
+    )
+    return 0
+  proc = gcloud_runner.run_gcloud_command(
+      gcloud_command,
+      # TODO(b/355324165): Handle that create, and probably others, output their
+      # user messaging to stderr.
+      stderr=subprocess.STDOUT,
+  )
+  bq_format = bq_global_flags.get('format', 'sparse')
+  command_mapping = GCLOUD_COMMAND_GENERATOR.get_command_mapping(
+      resource=resource, bq_command=bq_command
+  )
+  if not proc.stdout:
+    return proc.returncode
+  # Print line-by-line unless for JSON output, where we first collect all the
+  # lines into a single JSON object before printing.
+  json_output = ''
+  for raw_line in iter(proc.stdout.readline, ''):
+    line_to_print = ''
+    output = str(raw_line).strip()
+    is_progress_message = command_mapping.synchronous_progress_message_matcher(
+        output
+    )
+    if is_progress_message:
+      line_to_print = output
+    elif not command_mapping.print_resource:
+      # If this command doesn't print the resource, then print the raw output.
+      line_to_print = command_mapping.status_mapping(
+          output, identifier, bq_global_flags.get('project_id')
+      )
+    elif 'json' in bq_format:
+      # Collect all the lines before printing them as a single JSON object.
+      json_output += output
+    elif bq_format == 'pretty':
+      line_to_print = _swap_gcloud_box_to_bq_pretty(output)
+    elif bq_format == 'sparse':
+      line_to_print = _swap_gcloud_box_to_bq_sparse(output)
+    else:
+      line_to_print = output
+    if line_to_print:
+      print(line_to_print)
+  if json_output:
+    try:
+      parsed_json = json.loads(json_output)
+      if isinstance(parsed_json, list):
+        json_object = []
+        for item_dict in parsed_json:
+          json_object.append(command_mapping.json_mapping(item_dict, bq_format))
+      else:
+        json_object = command_mapping.json_mapping(parsed_json, bq_format)
+      if 'json' == bq_format:
+        print(json.dumps(json_object, separators=(',', ':')))
+      elif 'prettyjson' == bq_format:
+        print(json.dumps(json_object, indent=2, sort_keys=True))
+    except json.JSONDecodeError:
+      # Print the raw output even if it cannot be parsed as json.
+      # This likely happens when the command returns an error like "not found".
+      print(json_output)
+  return proc.returncode

gcloud auth application-default login/google-cloud-sdk/platform/bq/gcloud_wrapper/bq_to_gcloud_command_executor.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+"""An adapter that takes a bq command with flags and executes it with gcloud."""
+
+from typing import Dict, Optional, Union
+
+from absl import flags
+from absl import logging
+
+import bq_flags
+from gcloud_wrapper import bq_to_gcloud_adapter
+
+
+# TODO(user): Close out design discussion in comment thread from cl/659752136.
+# Should this be here, or part of `GlobalFlagsMap.map_to_gcloud_global_flags`?
+def _unpack_bq_global_flags() -> Dict[str, Union[str, int, bool]]:
+  """Returns the bq_global_flags from the bq flags."""
+  bq_global_flags = {}
+
+  def unpack_flag_holder(flag_holder: flags.FlagHolder, key_name: str) -> None:
+    if flag_holder.present and flag_holder.value is not None:
+      bq_global_flags[key_name] = flag_holder.value
+
+  unpack_flag_holder(bq_flags.FORMAT, 'format')
+  unpack_flag_holder(bq_flags.PROJECT_ID, 'project_id')
+  unpack_flag_holder(bq_flags.HTTPLIB2_DEBUGLEVEL, 'httplib2_debuglevel')
+  try:
+    unpack_flag_holder(logging.VERBOSITY, 'verbosity')
+  except AttributeError:
+    # Some versions of absl.logging don't have VERBOSITY.
+    pass
+  unpack_flag_holder(bq_flags.APILOG, 'apilog')
+
+  # Unsupported flags
+  unpack_flag_holder(bq_flags.MTLS, 'mtls')
+
+  return bq_global_flags
+
+
+def run_bq_command_using_gcloud(
+    resource: str,
+    bq_command: str,
+    bq_command_flags: Dict[str, str],
+    identifier: Optional[str] = None,
+) -> int:
+  bq_global_flags = _unpack_bq_global_flags()
+  dry_run = False  # pylint: disable=unused-variable
+  return bq_to_gcloud_adapter.run_bq_command_using_gcloud(
+      resource=resource,
+      bq_command=bq_command,
+      bq_global_flags=bq_global_flags,
+      bq_command_flags=bq_command_flags,
+      identifier=identifier,
+      dry_run=dry_run,
+  )

gcloud auth application-default login/google-cloud-sdk/platform/bq/gcloud_wrapper/bq_to_gcloud_config_classes.py

@@ -0,0 +1,459 @@
+#!/usr/bin/env python
+"""The classes used to define config used to delegate BQ commands to gcloud."""
+
+from collections.abc import Callable
+import sys
+from typing import Dict, List, Optional, Union
+from typing_extensions import TypeAlias
+
+PrimitiveFlagValue: TypeAlias = Union[str, bool, int]
+NestedStrDict: TypeAlias = Dict[str, Union[str, 'NestedStrDict']]
+
+
+def _flatten_flag_dictionary(
+    mapped_flags: Dict[str, PrimitiveFlagValue],
+) -> List[str]:
+  """Returns the gcloud command flags as an array of strings."""
+  flag_array: List[str] = []
+  for name, value in mapped_flags.items():
+    if not isinstance(value, bool):
+      flag_array.append(f'--{name}={str(value)}')
+    elif value:
+      flag_array.append(f'--{name}')
+    else:
+      flag_array.append(f'--no-{name}')
+  return flag_array
+
+
+def quote_flag_values(command_array: List[str]) -> List[str]:
+  """Returns the gcloud command flags after quoting the flag values."""
+  result: List[str] = []
+  for command_or_flag in command_array:
+    if command_or_flag.startswith('--') and '=' in command_or_flag:
+      (name, _, value) = command_or_flag.partition('=')
+      result.append(f"{name}='{value}'")
+    else:
+      result.append(command_or_flag)
+  return result
+
+
+class BigqueryGcloudDelegationUserError(Exception):
+  """Class to represent a user error during gcloud delegation."""
+
+
+# This Callable annotation would cause a type error before Python 3.9.2, see
+# https://docs.python.org/3/whatsnew/3.9.html#notable-changes-in-python-3-9-2.
+if sys.version_info >= (3, 9, 2):
+  ConvertFlagValuesFunction: TypeAlias = Callable[
+      [PrimitiveFlagValue], PrimitiveFlagValue
+  ]
+  ConvertJsonFunction: TypeAlias = Callable[
+      [NestedStrDict, Optional[str]], NestedStrDict
+  ]
+  ConvertStatusFunction: TypeAlias = Callable[[str, str, str], str]
+  MatchOutputFunction: TypeAlias = Callable[[str], bool]
+else:
+  ConvertFlagValuesFunction: TypeAlias = Callable
+  ConvertJsonFunction: TypeAlias = Callable
+  ConvertStatusFunction: TypeAlias = Callable
+  MatchOutputFunction: TypeAlias = Callable
+
+
+class FlagMapping:
+  """Defines how to create a gcloud command flag from a bq flag.
+
+  For example this would return True:
+
+  FlagMapping(
+      bq_name='httplib2_debuglevel',
+      gcloud_name='log-http',
+      bq_to_gcloud_mapper=lambda x: x > 0,
+  ).bq_to_gcloud_mapper(1)
+  """
+
+  def __init__(
+      self,
+      bq_name: str,  # The name of the original bq flag.
+      gcloud_name: str,  # The gcloud flag that is mapped to.
+      bq_to_gcloud_mapper: Optional[ConvertFlagValuesFunction] = None,
+  ):
+    self.bq_name = bq_name
+    self.gcloud_name = gcloud_name
+    if bq_to_gcloud_mapper:
+      self.bq_to_gcloud_mapper = bq_to_gcloud_mapper
+    else:
+      self.bq_to_gcloud_mapper = self.default_map_bq_value_to_gcloud_value
+
+  # TODO(b/398206856): Simplify this function and validate behaviour.
+  def default_map_bq_value_to_gcloud_value(
+      self, bq_flag_value: PrimitiveFlagValue
+  ) -> PrimitiveFlagValue:
+    """Takes a bq flag value and returns the equivalent gcloud flag value."""
+    if isinstance(bq_flag_value, bool):
+      return bq_flag_value or False
+    elif isinstance(bq_flag_value, int):
+      return bq_flag_value
+    else:
+      return str(bq_flag_value)
+
+
+class UnsupportedFlagMapping(FlagMapping):
+  """Defines a bq global flag that is not supported in gcloud."""
+
+  def __init__(
+      self,
+      bq_name: str,
+      error_message: str,
+  ):
+    def raise_unsupported_flag_error(x: Union[str, bool]) -> Union[str, bool]:
+      raise BigqueryGcloudDelegationUserError(error_message)
+
+    super().__init__(bq_name, 'unsupported_flag', raise_unsupported_flag_error)
+
+
+def _convert_to_gcloud_flags(
+    flag_mappings: Dict[str, FlagMapping],
+    bq_flags: Dict[str, PrimitiveFlagValue],
+) -> Dict[str, PrimitiveFlagValue]:
+  """Returns the equivalent gcloud flags for a set of bq flags.
+
+  Args:
+    flag_mappings: The flag mappings to use. For example, {'project_id':
+      FlagMapping('project_id', 'project')}
+    bq_flags: The bq flags that will be mapped. For example, {'project_id':
+      'my_project'}
+
+  Returns:
+    The equivalent gcloud flags. For example,
+    {'project': 'my_project'}
+  """
+  gcloud_flags = {}
+  for bq_flag, bq_flag_value in bq_flags.items():
+    if bq_flag not in flag_mappings:
+      raise ValueError(f'Unsupported bq flag: {bq_flag}')
+    flag_mapper = flag_mappings[bq_flag]
+    gcloud_flags[flag_mapper.gcloud_name] = flag_mapper.bq_to_gcloud_mapper(
+        bq_flag_value
+    )
+  return gcloud_flags
+
+
+class CommandMapping:
+  """Stores the configuration to map a BQ CLI command to gcloud.
+
+  This class does not include the global flags. These are handled at a higher
+  level in the system.
+
+  Example usage:
+
+  CommandMapping(
+      resource='datasets',
+      bq_command='ls',
+      gcloud_command=['alpha', 'bq', 'datasets', 'list'],
+      flag_mapping_list=[
+          FlagMapping(
+              bq_name='max_results',
+              gcloud_name='limit',
+          ),
+      ],
+  ).get_gcloud_command_minus_global_flags(
+      bq_format='pretty',
+      bq_command_flags={'max_results': 5},
+  )
+
+  Results in:
+  ['alpha', 'bq', 'datasets', 'list', '--format=table[box]', '--limit=5']
+  """
+
+  def __init__(
+      self,
+      resource: str,
+      bq_command: str,
+      gcloud_command: List[str],
+      flag_mapping_list: Optional[List[FlagMapping]] = None,
+      table_projection: Optional[str] = None,
+      csv_projection: Optional[str] = None,
+      json_mapping: Optional[ConvertJsonFunction] = None,
+      status_mapping: Optional[ConvertStatusFunction] = None,
+      synchronous_progress_message_matcher: Optional[
+          MatchOutputFunction
+      ] = None,
+      print_resource: bool = True,
+      no_prompts: bool = False,
+  ):
+    """Initializes the CommandMapping.
+
+    Args:
+      resource: The resource this command targets. For example, 'datasets'.
+      bq_command: The bq command to map. For example, 'ls'.
+      gcloud_command: The gcloud command that will be mapped to. For example,
+        ['alpha', 'bq', 'datasets', 'list'].
+      flag_mapping_list: The flag mappings for this command. For example,
+        [FlagMapping('max_results', 'limit')]
+      table_projection: An optional projection to use for the command when a
+        table is displayed. For example:
+        'datasetReference.datasetId:label=datasetId'.
+      csv_projection: An optional projection to use for the command when the
+        output is in csv format. For example:
+        'datasetReference.datasetId:label=datasetId'.
+      json_mapping: A function to map the json output from gcloud to bq. For
+        example, lambda x: {'kind': 'bigquery#project', 'id': x['projectId']}
+      status_mapping: A function to map the status output from gcloud to bq. For
+        example, lambda orig, id, project: f'Dataset {project}:{id} deleted.'
+      synchronous_progress_message_matcher: A function to match a progress
+        message from gcloud when running a synchronous command. For example,
+        lambda message: 'Waiting for job' in message.
+      print_resource: If the command also prints the resource it is operating
+        on. For example, 'ls' will list resources but 'rm' usually prints status
+        and not the resource.
+      no_prompts: Some commands need a prompt to be disabled when they're run
+        and usually, the BQ CLI code flow will have done this already. For
+        example, the when `bq rm -d` is run, the BQ CLI will prompt the user
+        before deleting the dataset, so the gcloud prompt is not needed.
+    """
+    self.resource = resource
+    self.bq_command = bq_command
+    self.gcloud_command = gcloud_command
+    self.flag_mapping_list = flag_mapping_list or []
+    self._flag_mappings: Dict[str, FlagMapping] = None
+    self.table_projection = table_projection
+    self.csv_projection = csv_projection
+    self.json_mapping = json_mapping if json_mapping else lambda x, _: x
+    if status_mapping:
+      self.status_mapping = status_mapping
+    else:
+      self.status_mapping = lambda original_status, _, __: original_status
+    if synchronous_progress_message_matcher:
+      self.synchronous_progress_message_matcher = (
+          synchronous_progress_message_matcher
+      )
+    else:
+      self.synchronous_progress_message_matcher = lambda _: False
+    self.print_resource = print_resource
+    self.no_prompts = no_prompts
+
+  @property
+  def flag_mappings(self) -> Dict[str, FlagMapping]:
+    """Returns the command flag mappings as a dictionary."""
+    if not self._flag_mappings:
+      self._flag_mappings = {}
+      for flag_mapping in self.flag_mapping_list:
+        self._flag_mappings[flag_mapping.bq_name] = flag_mapping
+    return self._flag_mappings
+
+  def _add_fields_to_format(
+      self,
+      prefix: str,
+      labels: Optional[str] = None,
+  ) -> str:
+    """Returns the format from the map."""
+    if labels:
+      return f'{prefix}({labels})'
+    else:
+      return prefix
+
+  def get_gcloud_format(self, bq_format: Optional[str]) -> str:
+    """Returns the gcloud format for the given bq format."""
+    # TODO(b/355324165): Update the note on what happens when there is no flag,
+    # after we have better testing from the gcloud delegator.
+    if not bq_format or bq_format == 'pretty' or bq_format == 'sparse':
+      return self._add_fields_to_format('table[box]', self.table_projection)
+    elif 'json' in bq_format:
+      return 'json'
+    elif 'csv' in bq_format:
+      return self._add_fields_to_format('csv', self.csv_projection)
+    else:
+      raise ValueError(f'Unsupported format: {bq_format}')
+
+  def _get_gcloud_flags(
+      self,
+      bq_flags: Dict[str, PrimitiveFlagValue],
+  ) -> Dict[str, PrimitiveFlagValue]:
+    """Returns the gcloud flags for the given bq flags."""
+    return _convert_to_gcloud_flags(self.flag_mappings, bq_flags)
+
+  def get_gcloud_command_minus_global_flags(
+      self,
+      bq_format: Optional[str],
+      bq_command_flags: Dict[str, str],
+      identifier: Optional[str] = None,
+  ) -> List[str]:
+    """Returns the gcloud command to use for the given bq command.
+
+    Args:
+      bq_format: The `format` flag from the BQ CLI (eg. 'json').
+      bq_command_flags: The flags for this BQ command that will be mapped. For
+        example, {'max_results': 5}
+      identifier: An optional identifier of the resource this command will
+        operate on.
+
+    Returns:
+      The equivalent gcloud command array with the leading 'gcloud' removed,
+      with the format flag and command flags but no global flags. For example,
+      ['alpha', 'bq', 'datasets', 'list', '--format=json', '--limit=5']
+    """
+    gcloud_command: List[str] = self.gcloud_command.copy()
+    # If the resource is not being printed then don't add the format flag.
+    if self.print_resource:
+      gcloud_format = self.get_gcloud_format(bq_format)
+      gcloud_command.append(
+          f'--format={gcloud_format}',
+      )
+    gcloud_command.extend(
+        _flatten_flag_dictionary(self._get_gcloud_flags(bq_command_flags))
+    )
+    if self.no_prompts:
+      gcloud_command.append('--quiet')
+    if identifier:
+      gcloud_command.append(identifier)
+    return gcloud_command
+
+
+class GcloudCommandGenerator:
+  """Generates a gcloud command from a bq command."""
+
+  def __init__(
+      self,
+      command_mappings: List[CommandMapping],
+      global_flag_mappings: List[FlagMapping],
+  ):
+    self._command_mapping_list = command_mappings
+    self._global_flag_mapping_list = global_flag_mappings
+    self._command_dict: Optional[Dict[str, Dict[str, CommandMapping]]] = None
+    self._global_flag_dict: Optional[Dict[str, FlagMapping]] = None
+
+  @property
+  def command_dict(self) -> Dict[str, Dict[str, CommandMapping]]:
+    """Returns the commands as a map of resource to bq command to delegator."""
+    if not self._command_dict:
+      self._command_dict = {}
+      for command_mapping in self._command_mapping_list:
+        if command_mapping.resource not in self._command_dict:
+          self._command_dict[command_mapping.resource] = {}
+        resource_to_commands = self._command_dict[command_mapping.resource]
+        if command_mapping.bq_command in resource_to_commands:
+          raise ValueError(
+              f'Duplicate bq command: {command_mapping.bq_command}'
+          )
+        resource_to_commands[command_mapping.bq_command] = command_mapping
+    return self._command_dict
+
+  @property
+  def global_flag_dict(self) -> Dict[str, FlagMapping]:
+    if not self._global_flag_dict:
+      self._global_flag_dict = {}
+      for flag_mapping in self._global_flag_mapping_list:
+        bq_flag = flag_mapping.bq_name
+        if bq_flag in self._global_flag_dict:
+          raise ValueError(f'Duplicate bq flag: {bq_flag}')
+        self._global_flag_dict[bq_flag] = flag_mapping
+    return self._global_flag_dict
+
+  def map_to_gcloud_global_flags(
+      self, bq_global_flags: Dict[str, PrimitiveFlagValue]
+  ) -> Dict[str, PrimitiveFlagValue]:
+    """Returns the equivalent gcloud global flags for a set of bq flags.
+
+    In the Args and Returns below, this `GcloudCommandGenerator` is used:
+
+    GcloudCommandGenerator(
+      command_mappings=[],
+      global_flag_mappings=[
+        FlagMapping(
+            bq_name='project_id',
+            gcloud_name='project'),
+        FlagMapping(
+            bq_name='httplib2_debuglevel',
+            gcloud_name='log-http', lambda x: x > 0)
+    ])
+
+    Args:
+      bq_global_flags: The bq flags that will be mapped. For example,
+        {'project_id': 'my_project', 'httplib2_debuglevel': 1}
+
+    Returns:
+      The equivalent gcloud flags. For example,
+      {'project': 'my_project', 'log-http': True}
+    """
+    return _convert_to_gcloud_flags(self.global_flag_dict, bq_global_flags)
+
+  def get_command_mapping(
+      self, resource: str, bq_command: str
+  ) -> CommandMapping:
+    """Returns the gcloud delegator for the given resource and bq command."""
+    # Fail fast if there is no CommandMapping.
+    return self.command_dict[resource][bq_command]
+
+  def get_gcloud_command(
+      self,
+      resource: str,
+      bq_command: str,
+      bq_global_flags: Dict[str, str],
+      bq_command_flags: Dict[str, str],
+      identifier: Optional[str] = None,
+  ) -> List[str]:
+    """Returns the gcloud command to use for the given bq command.
+
+    As an example usage:
+
+    GcloudCommandGenerator(
+      command_mappings=[CommandMapping(
+        resource='datasets',
+        bq_command='ls',
+        gcloud_command=['alpha', 'bq', 'datasets', 'list'],
+        flag_mapping_list=[
+            FlagMapping(
+                bq_name='max_results',
+                gcloud_name='limit',
+            ),
+      ],
+      flag_mappings=[
+        FlagMapping(
+            bq_name='project_id',
+            gcloud_name='project'),
+    ]).get_gcloud_command(
+        resource='datasets',
+        bq_command='ls',
+        bq_global_flags={'project_id': 'bigquery-cli-e2e', 'format': 'pretty'},
+        bq_command_flags={'max_results': 5},
+    )
+
+    Will return:
+
+    ['--project=bigquery-cli-e2e', 'alpha', 'bq', 'datasets', 'list',
+    '--format=json', '--limit=5']
+
+    Args:
+      resource: The resource the command is being run on, named to align with
+        `gcloud` commands. For example, 'jobs' or 'datasets'.
+      bq_command: The bq command to run. For example, 'ls' or 'show'.
+      bq_global_flags: The BQ CLI global flags for the command.
+      bq_command_flags: The BQ CLI command flags for the command.
+      identifier: The identifier of the resource to act on.
+
+    Returns:
+      The gcloud command to run as an array of strings, minus the leading
+      'gcloud'. This can be parsed directly into
+      `gcloud_runner.run_gcloud_command`.
+    """
+    delegator = self.get_command_mapping(resource, bq_command)
+    if not delegator:
+      raise ValueError(f'Unsupported bq command: {bq_command}')
+
+    # TODO(b/355324165): Revisit how the format flag is passed.
+    # The format flag is handled separately so filter it out.
+    filtered_global_flags = bq_global_flags.copy()
+    bq_format = filtered_global_flags.pop('format', 'sparse')
+
+    gcloud_global_flags: List[str] = _flatten_flag_dictionary(
+        self.map_to_gcloud_global_flags(filtered_global_flags)
+    )
+
+    return (
+        gcloud_global_flags
+        + delegator.get_gcloud_command_minus_global_flags(
+            bq_format=bq_format,
+            bq_command_flags=bq_command_flags,
+            identifier=identifier,
+        )
+    )

gcloud auth application-default login/google-cloud-sdk/platform/bq/gcloud_wrapper/gcloud_runner.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+"""Utilities to run gcloud for the BQ CLI."""
+
+import logging
+import os
+import subprocess
+import sys
+from typing import List, Optional
+
+from typing_extensions import TypeAlias
+
+from pyglib import resources
+
+if sys.version_info >= (3, 9):
+  GcloudPopen: TypeAlias = subprocess.Popen[str]
+else:
+  # Before python 3.9 the type `subprocess.Popen[str]` is unsupported.
+  GcloudPopen: TypeAlias = subprocess.Popen  # pylint: disable=g-bare-generic
+
+_gcloud_path = None
+
+
+def _get_gcloud_path() -> str:
+  """Returns the string to use to call gcloud."""
+  global _gcloud_path
+  if _gcloud_path:
+    logging.info('Found cached gcloud path: %s', _gcloud_path)
+    return _gcloud_path
+
+  if 'nt' == os.name:
+    binary = 'gcloud.cmd'
+  else:
+    binary = 'gcloud'
+
+  # If a gcloud binary has been bundled with this code then use that version
+  # instead of the system installed version.
+  try:
+    binary = resources.GetResourceFilename(
+        'google3/cloud/sdk/gcloud/gcloud.par'
+    )
+  except FileNotFoundError:
+    pass
+
+  logging.info('Found gcloud path: %s', binary)
+  _gcloud_path = binary
+  return binary
+
+
+def run_gcloud_command(
+    cmd: List[str], stderr: Optional[int] = None
+) -> GcloudPopen:
+  """Runs the given gcloud command and returns the Popen object."""
+  gcloud_path = _get_gcloud_path()
+  logging.info('Running gcloud command: %s %s', gcloud_path, ' '.join(cmd))
+  return subprocess.Popen(
+      [gcloud_path] + cmd,
+      stdout=subprocess.PIPE,
+      stderr=stderr,
+      universal_newlines=True,
+  )

gcloud auth application-default login/google-cloud-sdk/platform/bq/gcloud_wrapper/supported_commands/supported_commands_dataset.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+"""The supported gcloud dataset commands in BQ CLI."""
+
+from typing import List
+
+from gcloud_wrapper import bq_to_gcloud_config_classes
+
+FlagMapping = bq_to_gcloud_config_classes.FlagMapping
+UnsupportedFlagMapping = bq_to_gcloud_config_classes.UnsupportedFlagMapping
+CommandMapping = bq_to_gcloud_config_classes.CommandMapping
+
+
+_ACLS_TABLE_LABEL = (
+    'access.format('
+    '"Owners:\n  {0}\nWriters:\n  {1}\nReaders:\n  {1}",'
+    '[].filter("role:OWNER").map(1).'
+    'extract("specialGroup","userByEmail").map(1).list()'
+    '.join(sep=\\",\n  \\"),'
+    '[].filter("role:WRITER").map(1).'
+    'extract("specialGroup","userByEmail").map(1).list()'
+    '.join(sep=\\",\n  \\"),'
+    '[].filter("role:READER").map(1).'
+    'extract("specialGroup","userByEmail").map(1).list()'
+    '.join(sep=\\",\n  \\")):label=ACLs:wrap=75'
+)
+
+
+def _json_mapping_list(
+    gcloud_json: bq_to_gcloud_config_classes.NestedStrDict,
+    _: str,
+) -> bq_to_gcloud_config_classes.NestedStrDict:
+  return {
+      'kind': 'bigquery#dataset',
+      'id': gcloud_json['id'],
+      'datasetReference': gcloud_json['datasetReference'],
+      'location': gcloud_json['location'],
+      'type': gcloud_json['type'],
+  }
+
+
+def _json_mapping_show(
+    gcloud_json: bq_to_gcloud_config_classes.NestedStrDict,
+    bq_format: str,
+) -> bq_to_gcloud_config_classes.NestedStrDict:
+  """Returns the dataset show json mapping."""
+  keys = [
+      'kind',
+      'etag',
+      'id',
+      'selfLink',
+      'datasetReference',
+      'access',
+      'creationTime',
+      'lastModifiedTime',
+      'location',
+      'type',
+      'maxTimeTravelHours',
+  ]
+  if bq_format == 'prettyjson':
+    keys.sort()
+  return {key: gcloud_json[key] for key in keys}
+
+
+def _create_status_mapping(
+    original_status: str, identifier: str, project_id: str
+) -> str:
+  if original_status.startswith('Created dataset'):
+    return f"Dataset '{project_id}:{identifier}' successfully created."
+  return original_status
+
+
+_DATASETS = 'datasets'
+
+
+SUPPORTED_COMMANDS_DATASET: List[CommandMapping] = [
+    CommandMapping(
+        resource=_DATASETS,
+        bq_command='ls',
+        gcloud_command=['alpha', 'bq', 'datasets', 'list'],
+        flag_mapping_list=[
+            FlagMapping('max_results', 'limit'),
+            FlagMapping('all', 'all'),
+        ],
+        table_projection='datasetReference.datasetId:label=datasetId',
+        csv_projection='datasetReference.datasetId:label=dataset_id',
+        json_mapping=_json_mapping_list,
+    ),
+    CommandMapping(
+        resource=_DATASETS,
+        bq_command='show',
+        gcloud_command=['alpha', 'bq', 'datasets', 'describe'],
+        table_projection=(
+            'lastModifiedTime.date('
+            'unit=1000,tz=LOCAL,format="%d %b %H:%M:%S"'
+            '):label="Last modified",'
+            f'{_ACLS_TABLE_LABEL},'
+            'labels:label=Labels,'
+            'type:label=Type,'
+            'maxTimeTravelHours:label="Max time travel (Hours)"'
+        ),
+        json_mapping=_json_mapping_show,
+    ),
+    CommandMapping(
+        resource=_DATASETS,
+        bq_command='mk',
+        gcloud_command=['alpha', 'bq', 'datasets', 'create'],
+        flag_mapping_list=[
+            FlagMapping('force', 'overwrite'),
+            FlagMapping('description', 'description'),
+            UnsupportedFlagMapping(
+                'location',
+                'The gcloud dataset create command does not support the'
+                ' location flag.',
+            ),
+        ],
+        status_mapping=_create_status_mapping,
+        print_resource=False,
+    ),
+    CommandMapping(
+        resource=_DATASETS,
+        bq_command='rm',
+        gcloud_command=['alpha', 'bq', 'datasets', 'delete'],
+        flag_mapping_list=[FlagMapping('recursive', 'remove-tables')],
+        print_resource=False,
+        no_prompts=True,
+        status_mapping=lambda input, _, __: '',
+    ),
+]

gcloud auth application-default login/google-cloud-sdk/platform/bq/gcloud_wrapper/supported_commands/supported_commands_migration_workflow.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+"""The supported gcloud migration workflow commands in BQ CLI."""
+
+from typing import List
+
+from gcloud_wrapper import bq_to_gcloud_config_classes
+
+FlagMapping = bq_to_gcloud_config_classes.FlagMapping
+UnsupportedFlagMapping = bq_to_gcloud_config_classes.UnsupportedFlagMapping
+CommandMapping = bq_to_gcloud_config_classes.CommandMapping
+
+
+def _synchronous_progress_message_matcher(raw_output: str) -> bool:
+  return raw_output.startswith(
+      'Running migration workflow'
+  ) or raw_output.startswith('......')
+
+
+# --synchronous_mode in BQ CLI should be mapped to --async in gcloud.
+_SYNCHRONOUS_MODE_MAPPER = lambda x: not x
+
+_MIGRATION_WORKFLOWS = 'migration_workflows'
+
+SUPPORTED_COMMANDS_MIGRATION_WORKFLOW: List[CommandMapping] = [
+    CommandMapping(
+        resource=_MIGRATION_WORKFLOWS,
+        bq_command='ls',
+        gcloud_command=['bq', 'migration-workflows', 'list'],
+        flag_mapping_list=[
+            FlagMapping('location', 'location'),
+            FlagMapping('max_results', 'limit'),
+        ],
+        table_projection='name,displayName,state,createTime,lastUpdateTime',
+        csv_projection='name,displayName:label=display_name,state,createTime:label=create_time,lastUpdateTime:label=last_update_time',
+    ),
+    CommandMapping(
+        resource=_MIGRATION_WORKFLOWS,
+        bq_command='show',
+        gcloud_command=['bq', 'migration-workflows', 'describe'],
+        table_projection=(
+            'name:label=Name,'
+            'displayName:label="Display Name",'
+            'state:label=State,'
+            'createTime:label="Create Time",'
+            'lastUpdateTime:label="Last Update Time"'
+        ),
+    ),
+    CommandMapping(
+        resource=_MIGRATION_WORKFLOWS,
+        bq_command='mk',
+        gcloud_command=['bq', 'migration-workflows', 'create'],
+        flag_mapping_list=[
+            FlagMapping('location', 'location'),
+            FlagMapping('config_file', 'config-file'),
+            FlagMapping(
+                'sync', 'async', bq_to_gcloud_mapper=_SYNCHRONOUS_MODE_MAPPER
+            ),
+            FlagMapping(
+                'synchronous_mode',
+                'async',
+                bq_to_gcloud_mapper=_SYNCHRONOUS_MODE_MAPPER,
+            ),
+        ],
+        table_projection=(
+            'name:label=Name,'
+            'displayName:label="Display Name",'
+            'state:label=State,'
+            'createTime:label="Create Time",'
+            'lastUpdateTime:label="Last Update Time"'
+        ),
+        synchronous_progress_message_matcher=_synchronous_progress_message_matcher,
+    ),
+    CommandMapping(
+        resource=_MIGRATION_WORKFLOWS,
+        bq_command='rm',
+        gcloud_command=['bq', 'migration-workflows', 'delete'],
+        print_resource=False,
+        no_prompts=True,
+        status_mapping=lambda *args: '',  # No message printed for deletion.
+    ),
+]

gcloud auth application-default login/google-cloud-sdk/platform/bq/gcloud_wrapper/supported_commands/supported_commands_project.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+"""The supported gcloud project commands in BQ CLI."""
+
+from typing import List
+
+from gcloud_wrapper import bq_to_gcloud_config_classes
+
+FlagMapping = bq_to_gcloud_config_classes.FlagMapping
+UnsupportedFlagMapping = bq_to_gcloud_config_classes.UnsupportedFlagMapping
+CommandMapping = bq_to_gcloud_config_classes.CommandMapping
+
+
+_PROJECTS = 'projects'
+
+
+def project_json_mapping(
+    gcloud_json: bq_to_gcloud_config_classes.NestedStrDict,
+    _: str,
+) -> bq_to_gcloud_config_classes.NestedStrDict:
+  return {
+      'kind': 'bigquery#project',
+      'id': gcloud_json['projectId'],
+      'numericId': gcloud_json['projectNumber'],
+      'projectReference': {
+          'projectId': gcloud_json['projectId'],
+      },
+      'friendlyName': gcloud_json['name'],
+  }
+
+
+SUPPORTED_COMMANDS_PROJECT: List[CommandMapping] = [
+    # Note: The API used by the BQ CLI is the BQ API and this has a different
+    # permissions structure that can cause issues during migration. This is
+    # similar to some issues seen when using the BQ UI vs using the Simba
+    # drivers.
+    CommandMapping(
+        resource=_PROJECTS,
+        bq_command='ls',
+        gcloud_command=[
+            'projects',
+            'list',
+            # The BQ CLI uses the BQ API to list projects and that lists
+            # projects in alphabetical order by project id.
+            '--sort-by=projectId',
+        ],
+        flag_mapping_list=[FlagMapping('max_results', 'limit')],
+        table_projection='projectId:label=projectId,name:label="friendlyName"',
+        csv_projection='projectId:label=project_id,name:label=friendly_name',
+        json_mapping=project_json_mapping,
+    ),
+]

gcloud auth application-default login/google-cloud-sdk/platform/bq/gcloud_wrapper/supported_gcloud_commands.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+"""The gcloud delegators supported by the BQ CLI."""
+
+import logging
+from typing import List
+
+from gcloud_wrapper import bq_to_gcloud_config_classes
+from gcloud_wrapper.supported_commands.supported_commands_dataset import SUPPORTED_COMMANDS_DATASET
+from gcloud_wrapper.supported_commands.supported_commands_migration_workflow import SUPPORTED_COMMANDS_MIGRATION_WORKFLOW
+from gcloud_wrapper.supported_commands.supported_commands_project import SUPPORTED_COMMANDS_PROJECT
+
+
+FlagMapping = bq_to_gcloud_config_classes.FlagMapping
+UnsupportedFlagMapping = bq_to_gcloud_config_classes.UnsupportedFlagMapping
+CommandMapping = bq_to_gcloud_config_classes.CommandMapping
+
+
+def _bq_apilog_to_gcloud_verbosity(apilog: str) -> str:
+  if apilog not in ('', '-', '1', 'true', 'stdout'):
+    logging.warning(
+        'Gcloud only supports logging to stdout and apilog is set to %s', apilog
+    )
+  return 'debug'
+
+
+def _bq_verbosity_to_gcloud_verbosity(verbosity: int) -> str:
+  """Returns the gcloud verbosity level for the given bq verbosity level."""
+  if verbosity <= -3:
+    # The `critical` value is used instead of `fatal` in gcloud.
+    return 'critical'
+  elif verbosity == -2:
+    return 'error'
+  elif verbosity == -1:
+    return 'warning'
+  elif verbosity == 0:
+    return 'info'
+  elif verbosity >= 1:
+    return 'debug'
+  raise ValueError(f'Unknown verbosity level: {verbosity}')
+
+
+# Note: Then `format` flag is not included here since it's mapping is a lot more
+# complicated and requires taking into account the command being executed. so it
+# is handled as part of the implementation.
+SUPPORTED_GLOBAL_FLAGS: List[FlagMapping] = [
+    FlagMapping('project_id', 'project'),
+    FlagMapping('httplib2_debuglevel', 'log-http', lambda x: x > 0),
+    # TODO(b/355324165): Handle condition when both flags are used.
+    FlagMapping('apilog', 'verbosity', _bq_apilog_to_gcloud_verbosity),
+    FlagMapping('verbosity', 'verbosity', _bq_verbosity_to_gcloud_verbosity),
+    # Unsupported flags.
+    UnsupportedFlagMapping(
+        'mtls',
+        'The `mtls` flag cannot be used directly when delegating to gcloud. It'
+        ' must be configured in the `gcloud` config and it will be loaded'
+        ' during execution',
+    ),
+]
+
+
+SUPPORTED_COMMANDS: List[CommandMapping] = (
+    SUPPORTED_COMMANDS_DATASET
+    + SUPPORTED_COMMANDS_PROJECT
+    + SUPPORTED_COMMANDS_MIGRATION_WORKFLOW
+)