feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/command_lib/emulators/bigtable_util.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility functions for gcloud bigtable emulator."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import os
+from googlecloudsdk.command_lib.emulators import util
+from googlecloudsdk.core import execution_utils
+from googlecloudsdk.core import log
+from googlecloudsdk.core.util import platforms
+
+BIGTABLE = 'bigtable'
+BIGTABLE_TITLE = 'Google Cloud Bigtable emulator'
+BIGTABLE_EXECUTABLE = 'cbtemulator'
+
+
+def GetDataDir():
+  return util.GetDataDir(BIGTABLE)
+
+
+def BuildStartArgs(args):
+  """Builds the command for starting the bigtable emulator.
+
+  Args:
+    args: (list of str) The arguments for the bigtable emulator, excluding the
+      program binary.
+
+  Returns:
+    A list of command arguments.
+  """
+  bigtable_dir = util.GetEmulatorRoot(BIGTABLE)
+  bigtable_executable = os.path.join(bigtable_dir, BIGTABLE_EXECUTABLE)
+  if platforms.OperatingSystem.Current() is platforms.OperatingSystem.WINDOWS:
+    bigtable_executable += '.exe'
+  return execution_utils.ArgsForExecutableTool(bigtable_executable, *args)
+
+
+def GetEnv(args):
+  """Returns an environment variable mapping from an argparse.Namespace."""
+  return {
+      'BIGTABLE_EMULATOR_HOST':
+          '%s:%s' % (args.host_port.host, args.host_port.port)
+  }
+
+
+def Start(args):
+  bigtable_args = BuildStartArgs(util.BuildArgsList(args))
+  log.status.Print('Executing: {0}'.format(' '.join(bigtable_args)))
+  with util.Exec(bigtable_args) as bigtable_process:
+    util.WriteEnvYaml(GetEnv(args), GetDataDir())
+    util.PrefixOutput(bigtable_process, BIGTABLE)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/command_lib/emulators/config.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Code related to proxy and emulator configuration."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import json
+from googlecloudsdk.command_lib.emulators import datastore_util
+from googlecloudsdk.command_lib.emulators import pubsub_util
+from googlecloudsdk.core.util import files
+import six
+
+
+# This is a list of all of the emulators currently supported, for use in
+# the code pertaining to launching emulators. New emulators should be
+# included here as they are added.
+EMULATORS = {}
+for emulator in [datastore_util.DatastoreEmulator(),
+                 pubsub_util.PubsubEmulator()]:
+  EMULATORS[emulator.service_name] = emulator
+
+
+def WriteRoutesConfig(emulators, output_file):
+  """This writes out the routes information to a file.
+
+  The routes will be written as json in the format
+  {service1: [route1, route2], service2: [route3, route4]}
+
+  Args:
+    emulators: [str], emulators to route the traffic of
+    output_file: str, file to write the configuration to
+  """
+  routes = {name: emulator.prefixes
+            for name, emulator in six.iteritems(emulators)}
+
+  files.WriteFileContents(output_file, json.dumps(routes, indent=2))
+
+
+# The configuration we get off the wire will be parsed and put into this object
+# This object will then help us generate the ClusterInfo objects that are
+# necessary, and other general route configuration
+class ProxyConfiguration(object):
+  """Configuration necessary to initialize the proxy."""
+
+  def __init__(self, local_emulators, should_proxy_to_gcp, proxy_port):
+    """Initializes object.
+
+    Args:
+      local_emulators: dict, the emulators and the ports they'll listen on
+      should_proxy_to_gcp: bool, whether traffic to other emulators should
+                           go to prod or not
+      proxy_port: int, the port the proxy should bind to
+    """
+    self._local_emulators = local_emulators
+    self._proxy_port = proxy_port
+    self._should_proxy_to_gcp = should_proxy_to_gcp
+
+  def WriteJsonToFile(self, output_file):
+    """Writes configuration to file.
+
+    The format will be
+    {"localEmulators": {emulator1: port1, emulator2: port2},
+     "proxyPort": port,
+     "shouldProxyToGcp": bool}
+
+    Args:
+      output_file: str, file to write to
+    """
+    data = {
+        'localEmulators': self._local_emulators,
+        'proxyPort': self._proxy_port,
+        'shouldProxyToGcp': self._should_proxy_to_gcp,
+    }
+    files.WriteFileContents(output_file, json.dumps(data, indent=2))

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/command_lib/emulators/datastore_util.py

@@ -0,0 +1,219 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utility functions for gcloud datastore emulator."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import os
+import tempfile
+from googlecloudsdk.command_lib.emulators import util
+from googlecloudsdk.core import exceptions
+from googlecloudsdk.core import execution_utils
+from googlecloudsdk.core import log
+from googlecloudsdk.core import properties
+from googlecloudsdk.core.util import platforms
+
+
+class UnableToPrepareDataDir(exceptions.Error):
+
+  def __init__(self):
+    super(UnableToPrepareDataDir, self).__init__(
+        'Unable to prepare the data directory for the emulator')
+
+
+def ArgsForGCDEmulator(emulator_args):
+  """Constructs an argument list for calling the GCD emulator.
+
+  Args:
+    emulator_args: args for the emulator.
+
+  Returns:
+    An argument list to execute the GCD emulator.
+  """
+  current_os = platforms.OperatingSystem.Current()
+  if current_os is platforms.OperatingSystem.WINDOWS:
+    cmd = 'cloud_datastore_emulator.cmd'
+    gcd_executable = os.path.join(util.GetEmulatorRoot(CLOUD_DATASTORE), cmd)
+    return execution_utils.ArgsForCMDTool(gcd_executable, *emulator_args)
+  else:
+    cmd = 'cloud_datastore_emulator'
+    gcd_executable = os.path.join(util.GetEmulatorRoot(CLOUD_DATASTORE), cmd)
+    return execution_utils.ArgsForExecutableTool(gcd_executable, *emulator_args)
+
+
+DATASTORE = 'datastore'
+CLOUD_DATASTORE = 'cloud-datastore'
+DATASTORE_TITLE = 'Google Cloud Datastore emulator'
+
+
+def PrepareGCDDataDir(args):
+  """Prepares the given directory using gcd create.
+
+  Raises:
+    UnableToPrepareDataDir: If the gcd create execution fails.
+
+  Args:
+    args: The arguments passed to the command.
+  """
+  data_dir = args.data_dir
+  if os.path.isdir(data_dir) and os.listdir(data_dir):
+    log.warning('Reusing existing data in [{0}].'.format(data_dir))
+    return
+
+  gcd_create_args = ['create']
+  project = properties.VALUES.core.project.Get(required=True)
+  gcd_create_args.append('--project_id={0}'.format(project))
+  gcd_create_args.append(data_dir)
+  exec_args = ArgsForGCDEmulator(gcd_create_args)
+
+  log.status.Print('Executing: {0}'.format(' '.join(exec_args)))
+  with util.Exec(exec_args) as process:
+    util.PrefixOutput(process, DATASTORE)
+    failed = process.poll()
+    if failed:
+      raise UnableToPrepareDataDir()
+
+
+def StartGCDEmulator(args, log_file=None):
+  """Starts the datastore emulator with the given arguments.
+
+  Args:
+    args: Arguments passed to the start command.
+    log_file: optional file argument to reroute process's output.
+
+  Returns:
+    process, The handle of the child process running the datastore emulator.
+  """
+  gcd_start_args = ['start']
+  gcd_start_args.append('--host={0}'.format(args.host_port.host))
+  gcd_start_args.append('--port={0}'.format(args.host_port.port))
+  gcd_start_args.append('--store_on_disk={0}'.format(args.store_on_disk))
+  gcd_start_args.append('--allow_remote_shutdown')
+  if args.use_firestore_in_datastore_mode:
+    gcd_start_args.append('--firestore_in_datastore_mode')
+  else:
+    gcd_start_args.append('--consistency={0}'.format(args.consistency))
+  gcd_start_args.append(args.data_dir)
+  exec_args = ArgsForGCDEmulator(gcd_start_args)
+
+  log.status.Print('Executing: {0}'.format(' '.join(exec_args)))
+  return util.Exec(exec_args, log_file=log_file)
+
+
+def WriteGCDEnvYaml(args):
+  """Writes the env.yaml file for the datastore emulator with provided args.
+
+  Args:
+    args: Arguments passed to the start command.
+  """
+  host_port = '{0}:{1}'.format(args.host_port.host, args.host_port.port)
+  project_id = properties.VALUES.core.project.Get(required=True)
+  env = {'DATASTORE_HOST': 'http://{0}'.format(host_port),
+         'DATASTORE_EMULATOR_HOST': host_port,
+         'DATASTORE_EMULATOR_HOST_PATH': '{0}/datastore'.format(host_port),
+         'DATASTORE_DATASET': project_id,
+         'DATASTORE_PROJECT_ID': project_id,
+        }
+  util.WriteEnvYaml(env, args.data_dir)
+
+
+def GetDataDir():
+  return util.GetDataDir(DATASTORE)
+
+
+def GetHostPort():
+  return util.GetHostPort(DATASTORE)
+
+
+class DatastoreEmulator(util.Emulator):
+  """Represents the ability to start and route datastore emulator."""
+
+  def Start(self, port):
+    args = util.AttrDict({
+        'host_port': {
+            'host': 'localhost',
+            'port': port,
+        },
+        'store_on_disk': True,
+        'consistency': 0.9,
+        'data_dir': tempfile.mkdtemp(),
+    })
+    PrepareGCDDataDir(args)
+    return StartGCDEmulator(args, self._GetLogNo())
+
+  @property
+  def prefixes(self):
+    # Taken Jan 1, 2017 from:
+    # https://cloud.google.com/datastore/docs/reference/rpc/google.datastore.v1
+    # Note that this should probably be updated to just be based off of the
+    # prefix, without enumerating all of the types.
+    return [
+        'google.datastore.v1.Datastore',
+        'google.datastore.v1.AllocateIdsRequest',
+        'google.datastore.v1.AllocateIdsResponse',
+        'google.datastore.v1.ArrayValue',
+        'google.datastore.v1.BeginTransactionRequest',
+        'google.datastore.v1.BeginTransactionResponse',
+        'google.datastore.v1.CommitRequest',
+        'google.datastore.v1.CommitRequest.Mode',
+        'google.datastore.v1.CommitResponse',
+        'google.datastore.v1.CompositeFilter',
+        'google.datastore.v1.CompositeFilter.Operator',
+        'google.datastore.v1.Entity',
+        'google.datastore.v1.EntityResult',
+        'google.datastore.v1.EntityResult.ResultType',
+        'google.datastore.v1.Filter',
+        'google.datastore.v1.GqlQuery',
+        'google.datastore.v1.GqlQueryParameter',
+        'google.datastore.v1.Key',
+        'google.datastore.v1.Key.PathElement',
+        'google.datastore.v1.KindExpression',
+        'google.datastore.v1.LookupRequest',
+        'google.datastore.v1.LookupResponse',
+        'google.datastore.v1.Mutation',
+        'google.datastore.v1.MutationResult',
+        'google.datastore.v1.PartitionId',
+        'google.datastore.v1.Projection',
+        'google.datastore.v1.PropertyFilter',
+        'google.datastore.v1.PropertyFilter.Operator',
+        'google.datastore.v1.PropertyOrder',
+        'google.datastore.v1.PropertyOrder.Direction',
+        'google.datastore.v1.PropertyReference',
+        'google.datastore.v1.Query',
+        'google.datastore.v1.QueryResultBatch',
+        'google.datastore.v1.QueryResultBatch.MoreResultsType',
+        'google.datastore.v1.ReadOptions',
+        'google.datastore.v1.ReadOptions.ReadConsistency'
+        'google.datastore.v1.RollbackRequest',
+        'google.datastore.v1.RollbackResponse',
+        'google.datastore.v1.RunQueryRequest',
+        'google.datastore.v1.RunQueryResponse',
+        'google.datastore.v1.Value',
+    ]
+
+  @property
+  def service_name(self):
+    return DATASTORE
+
+  @property
+  def emulator_title(self):
+    return DATASTORE_TITLE
+
+  @property
+  def emulator_component(self):
+    return 'cloud-datastore-emulator'

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/command_lib/emulators/firestore_util.py

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility functions for gcloud firestore emulator."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import os
+from googlecloudsdk.command_lib.emulators import util
+from googlecloudsdk.core import execution_utils
+from googlecloudsdk.core import log
+from googlecloudsdk.core.util import platforms
+
+
+def ArgsForFirestoreEmulator(emulator_args):
+  """Constructs an argument list for calling the Firestore emulator.
+
+  Args:
+    emulator_args: args for the emulator.
+
+  Returns:
+    An argument list to execute the Firestore emulator.
+  """
+  current_os = platforms.OperatingSystem.Current()
+  if current_os is platforms.OperatingSystem.WINDOWS:
+    cmd = 'cloud_firestore_emulator.cmd'
+    exe = os.path.join(util.GetEmulatorRoot(CLOUD_FIRESTORE), cmd)
+    return execution_utils.ArgsForCMDTool(exe, *emulator_args)
+  else:
+    cmd = 'cloud_firestore_emulator'
+    exe = os.path.join(util.GetEmulatorRoot(CLOUD_FIRESTORE), cmd)
+    return execution_utils.ArgsForExecutableTool(exe, *emulator_args)
+
+
+FIRESTORE = 'firestore'
+CLOUD_FIRESTORE = 'cloud-firestore'
+FIRESTORE_TITLE = 'Google Cloud Firestore emulator'
+
+
+def StartFirestoreEmulator(args, log_file=None):
+  """Starts the firestore emulator with the given arguments.
+
+  Args:
+    args: Arguments passed to the start command.
+    log_file: optional file argument to reroute process's output.
+
+  Returns:
+    process, The handle of the child process running the datastore emulator.
+  """
+  start_args = ['start']
+  start_args.append('--host={0}'.format(args.host_port.host))
+  start_args.append('--port={0}'.format(args.host_port.port))
+  if args.rules:
+    start_args.append('--rules={0}'.format(args.rules))
+  if args.use_firestore_in_datastore_mode:
+    start_args.append('--database-mode=datastore-mode')
+  else:
+    start_args.append('--database-mode={0}'.format(args.database_mode))
+  if args.import_data:
+    start_args.append('--import-data={0}'.format(args.import_data))
+  if args.export_on_exit:
+    start_args.append('--export-on-exit={0}'.format(args.export_on_exit))
+  if args.licenses:
+    start_args.append('--licenses')
+  exec_args = ArgsForFirestoreEmulator(start_args)
+
+  log.status.Print('Executing: {0}'.format(' '.join(exec_args)))
+  return util.Exec(exec_args, log_file=log_file)
+
+
+def ValidateStartArgs(args):  # pylint: disable=unused-argument
+  pass
+
+
+def GetHostPort():
+  return util.GetHostPort(FIRESTORE)
+
+
+class FirestoreEmulator(util.Emulator):
+  """Represents the ability to start and route firestore emulator."""
+
+  def Start(self, port):
+    args = util.AttrDict({
+        'host_port': {
+            'host': 'localhost',
+            'port': port,
+        }
+    })
+    return StartFirestoreEmulator(args, self._GetLogNo())
+
+  @property
+  def prefixes(self):
+    return ['google.firestore']
+
+  @property
+  def service_name(self):
+    return FIRESTORE
+
+  @property
+  def emulator_title(self):
+    return FIRESTORE_TITLE
+
+  @property
+  def emulator_component(self):
+    return 'cloud-firestore-emulator'

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/command_lib/emulators/flags.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Flags for gcloud emulators group."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+
+def AddDataDirFlag(parser, emulator_name):
+  parser.add_argument(
+      '--data-dir',
+      required=False,
+      help='The directory to be used to store/retrieve data/config for an '
+      'emulator run. The default value is `<USER_CONFIG_DIR>/emulators/{}`. '
+      'The value of USER_CONFIG_DIR can be found by running:\n\n'
+      '  $ gcloud info --format=\'get(config.paths.global_config_dir)\''
+      .format(emulator_name))

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/command_lib/emulators/pubsub_util.py

@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility functions for gcloud pubsub emulator."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import os
+from googlecloudsdk.command_lib.emulators import util
+from googlecloudsdk.core import exceptions
+from googlecloudsdk.core import execution_utils
+from googlecloudsdk.core import log
+from googlecloudsdk.core.util import platforms
+
+PUBSUB = 'pubsub'
+PUBSUB_TITLE = 'Google Cloud Pub/Sub emulator'
+
+
+class InvalidArgumentError(exceptions.Error):
+
+  pass
+
+
+def GetDataDir():
+  return util.GetDataDir(PUBSUB)
+
+
+def BuildStartArgs(args, current_os):
+  """Builds the command for starting the pubsub emulator.
+
+  Args:
+    args: (list of str) The arguments for the pubsub emulator, excluding the
+      program binary.
+    current_os: (platforms.OperatingSystem)
+
+  Returns:
+    A list of command arguments.
+  """
+  pubsub_dir = util.GetEmulatorRoot(PUBSUB)
+  if current_os is platforms.OperatingSystem.WINDOWS:
+    pubsub_executable = os.path.join(
+        pubsub_dir, r'bin\cloud-pubsub-emulator.bat')
+    return execution_utils.ArgsForCMDTool(pubsub_executable, *args)
+
+  pubsub_executable = os.path.join(pubsub_dir, 'bin/cloud-pubsub-emulator')
+  return execution_utils.ArgsForExecutableTool(pubsub_executable, *args)
+
+
+def GetEnv(args):
+  """Returns an environment variable mapping from an argparse.Namespace."""
+  return {'PUBSUB_EMULATOR_HOST': '%s:%s' %
+                                  (args.host_port.host, args.host_port.port)}
+
+
+def Start(args, log_file=None):
+  pubsub_args = BuildStartArgs(
+      util.BuildArgsList(args), platforms.OperatingSystem.Current())
+  log.status.Print('Executing: {0}'.format(' '.join(pubsub_args)))
+  return util.Exec(pubsub_args, log_file=log_file)
+
+
+class PubsubEmulator(util.Emulator):
+  """Represents the ability to start and route pubsub emulator."""
+
+  def Start(self, port):
+    args = util.AttrDict({'host_port': {'host': '::1', 'port': port}})
+    return Start(args, self._GetLogNo())
+
+  @property
+  def prefixes(self):
+    # Taken Jan 1, 2017 from:
+    # https://cloud.google.com/pubsub/docs/reference/rpc/google.pubsub.v1
+    # Note that this should probably be updated to just be based off of the
+    # prefix, without enumerating all of the types.
+    return [
+        'google.pubsub.v1.Publisher',
+        'google.pubsub.v1.Subscriber',
+        'google.pubsub.v1.AcknowledgeRequest',
+        'google.pubsub.v1.DeleteSubscriptionRequest',
+        'google.pubsub.v1.DeleteTopicRequest',
+        'google.pubsub.v1.GetSubscriptionRequest',
+        'google.pubsub.v1.GetTopicRequest',
+        'google.pubsub.v1.ListSubscriptionsRequest',
+        'google.pubsub.v1.ListSubscriptionsResponse',
+        'google.pubsub.v1.ListTopicSubscriptionsRequest',
+        'google.pubsub.v1.ListTopicSubscriptionsResponse',
+        'google.pubsub.v1.ListTopicsRequest',
+        'google.pubsub.v1.ListTopicsResponse',
+        'google.pubsub.v1.ModifyAckDeadlineRequest',
+        'google.pubsub.v1.ModifyPushConfigRequest',
+        'google.pubsub.v1.PublishRequest',
+        'google.pubsub.v1.PublishResponse',
+        'google.pubsub.v1.PubsubMessage',
+        'google.pubsub.v1.PullRequest',
+        'google.pubsub.v1.PullResponse',
+        'google.pubsub.v1.PushConfig',
+        'google.pubsub.v1.ReceivedMessage',
+        'google.pubsub.v1.Subscription',
+        'google.pubsub.v1.Topic',
+    ]
+
+  @property
+  def service_name(self):
+    return PUBSUB
+
+  @property
+  def emulator_title(self):
+    return PUBSUB_TITLE
+
+  @property
+  def emulator_component(self):
+    return 'pubsub-emulator'

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/command_lib/emulators/spanner_util.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2020 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility functions for gcloud spanner emulator."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import ipaddress
+import os
+
+from googlecloudsdk.command_lib.emulators import util
+from googlecloudsdk.core import exceptions
+from googlecloudsdk.core import execution_utils
+from googlecloudsdk.core import log
+from googlecloudsdk.core.util import platforms
+import six
+
+SPANNER_EMULATOR_PROPERTY_PREFIX = 'spanner'
+SPANNER_EMULATOR_COMPONENT_ID = 'cloud-spanner-emulator'
+SPANNER_EMULATOR_TITLE = 'Google Cloud Spanner Emulator'
+SPANNER_EMULATOR_EXECUTABLE_DIR = 'cloud_spanner_emulator'
+SPANNER_EMULATOR_EXECUTABLE_FILE = 'gateway_main'
+SPANNER_EMULATOR_DOCKER_IMAGE = 'gcr.io/cloud-spanner-emulator/emulator:1.5.28'
+SPANNER_EMULATOR_DEFAULT_GRPC_PORT = 9010
+SPANNER_EMULATOR_DEFAULT_REST_PORT = 9020
+
+
+class InvalidHostPortFormat(exceptions.Error):
+  pass
+
+
+def GetDataDir():
+  return util.GetDataDir(SPANNER_EMULATOR_PROPERTY_PREFIX)
+
+
+def _BuildStartArgsForDocker(args):
+  """Builds arguments for starting the spanner emulator under docker."""
+
+  # We use -p on Docker to enforce the specified hostname, but -p requires
+  # ip addresses. We handle the localhost case specifically as it is the
+  # common case.
+  host_ip = args.host_port.host
+  if host_ip == 'localhost':
+    host_ip = '127.0.0.1'
+  try:
+    ipaddress.ip_address(host_ip)
+  except ValueError:
+    raise InvalidHostPortFormat(
+        'When using docker, hostname specified via --host-port '
+        'must be an IPV4 or IPV6 address, found ' + host_ip)
+
+  docker_args = [
+      'docker', 'run', '-p',
+      '{}:{}:{}'.format(host_ip, args.host_port.port,
+                        SPANNER_EMULATOR_DEFAULT_GRPC_PORT), '-p',
+      '{}:{}:{}'.format(host_ip, args.rest_port,
+                        SPANNER_EMULATOR_DEFAULT_REST_PORT),
+      SPANNER_EMULATOR_DOCKER_IMAGE
+  ]
+
+  gateway_args = []
+  # Some arguments must be specified after the image name.
+  # If any are specified, we must specify the executable.
+  enable_fault_injection = getattr(args, 'enable_fault_injection', False)
+  print_notices = getattr(args, 'print_notices', False)
+  if enable_fault_injection or print_notices:
+    gateway_args.extend(['./gateway_main', '--hostname', '0.0.0.0'])
+    if enable_fault_injection:
+      gateway_args.append('--enable_fault_injection')
+    if print_notices:
+      gateway_args.append('--notices')
+
+  return execution_utils.ArgsForExecutableTool(*(docker_args + gateway_args))
+
+
+def _BuildStartArgsForNativeExecutable(args):
+  """Builds arguments for starting the spanner emulator as a native executable.
+
+  Args:
+    args: An argparse.Namespace object containing the command line arguments.
+
+  Returns:
+    A list of strings representing the command and its arguments.
+
+  Raises:
+    InvalidHostPortFormat: If the host_port is missing the port.
+  """
+  spanner_executable = os.path.join(util.GetCloudSDKRoot(), 'bin',
+                                    SPANNER_EMULATOR_EXECUTABLE_DIR,
+                                    SPANNER_EMULATOR_EXECUTABLE_FILE)
+  if args.host_port.port is None:
+    raise InvalidHostPortFormat(
+        'Invalid value for --host-port. Must be in the format host:port')
+  native_args = [
+      spanner_executable, '--hostname', args.host_port.host, '--grpc_port',
+      args.host_port.port, '--http_port',
+      six.text_type(args.rest_port)
+  ]
+  if getattr(args, 'enable_fault_injection', False):
+    native_args.append('--enable_fault_injection')
+  if getattr(args, 'print_notices', False):
+    native_args.append('--notices')
+  return execution_utils.ArgsForExecutableTool(*native_args)
+
+
+def _BuildStartArgs(args):
+  if (platforms.OperatingSystem.Current() is not platforms.OperatingSystem.LINUX
+      or args.use_docker):
+    return _BuildStartArgsForDocker(args)
+  else:
+    return _BuildStartArgsForNativeExecutable(args)
+
+
+def GetEnv(args):
+  """Returns an environment variable mapping from an argparse.Namespace."""
+  return {
+      'SPANNER_EMULATOR_HOST':
+          '{}:{}'.format(args.host_port.host, args.host_port.port)
+  }
+
+
+def Start(args):
+  spanner_args = _BuildStartArgs(args)
+  log.status.Print('Executing: {0}'.format(' '.join(spanner_args)))
+  with util.Exec(spanner_args) as spanner_process:
+    util.WriteEnvYaml(GetEnv(args), GetDataDir())
+    util.PrefixOutput(spanner_process, SPANNER_EMULATOR_COMPONENT_ID)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/command_lib/emulators/util.py

@@ -0,0 +1,481 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2015 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility functions for gcloud emulators datastore group."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import abc
+import contextlib
+import os
+import random
+import re
+import socket
+import subprocess
+import tempfile
+
+from googlecloudsdk.core import config
+from googlecloudsdk.core import exceptions
+from googlecloudsdk.core import log
+from googlecloudsdk.core import properties
+from googlecloudsdk.core import yaml
+from googlecloudsdk.core.resource import resource_printer
+from googlecloudsdk.core.updater import local_state
+from googlecloudsdk.core.updater import update_manager
+from googlecloudsdk.core.util import encoding
+from googlecloudsdk.core.util import files
+from googlecloudsdk.core.util import platforms
+import portpicker
+import six
+
+
+_IPV6_RE = re.compile(r'\[(.*)\]:(\d*)')
+
+
+class NoCloudSDKError(exceptions.Error):
+  """The module was unable to find Cloud SDK."""
+
+  def __init__(self):
+    super(NoCloudSDKError, self).__init__(
+        'Unable to find a Cloud SDK installation.')
+
+
+class NoEnvYamlError(exceptions.Error):
+  """Unable to find a env.yaml file."""
+
+  def __init__(self, data_dir):
+    super(NoEnvYamlError, self).__init__(
+        'Unable to find env.yaml in the data_dir [{0}]. Please ensure you have'
+        ' started the appropriate emulator.'.format(data_dir))
+
+
+class MissingProxyError(exceptions.Error):
+  pass
+
+
+class NoEmulatorError(exceptions.Error):
+  pass
+
+
+class InvalidHostError(exceptions.Error):
+  """The configured host:port is invalid."""
+
+  def __init__(self):
+    super(InvalidHostError, self).__init__(
+        'Emulator host-port must take the form ADDRESS:PORT where ADDRESS'
+        ' is a hostname, IPv4 or IPv6 address.')
+
+
+def EnsureComponentIsInstalled(component_id, for_text):
+  """Ensures that the specified component is installed.
+
+  Args:
+    component_id: str, the name of the component
+    for_text: str, the text explaining what the component is necessary for
+
+  Raises:
+    NoCloudSDKError: If a Cloud SDK installation is not found.
+  """
+  msg = ('You need the [{0}] component to use the {1}.'
+         .format(component_id, for_text))
+  try:
+    update_manager.UpdateManager.EnsureInstalledAndRestart([component_id],
+                                                           msg=msg)
+  except local_state.InvalidSDKRootError:
+    raise NoCloudSDKError()
+
+
+def GetCloudSDKRoot():
+  """Gets the directory of the root of the Cloud SDK, error if it doesn't exist.
+
+  Raises:
+    NoCloudSDKError: If there is no SDK root.
+
+  Returns:
+    str, The path to the root of the Cloud SDK.
+  """
+  sdk_root = config.Paths().sdk_root
+  if not sdk_root:
+    raise NoCloudSDKError()
+  log.debug('Found Cloud SDK root: %s', sdk_root)
+  return sdk_root
+
+
+def WriteEnvYaml(env, output_dir):
+  """Writes the given environment values into the output_dir/env.yaml file.
+
+  Args:
+    env: {str: str}, Dictonary of environment values.
+    output_dir: str, Path of directory to which env.yaml file should be written.
+  """
+  env_file_path = os.path.join(output_dir, 'env.yaml')
+  with files.FileWriter(env_file_path) as env_file:
+    resource_printer.Print([env], print_format='yaml', out=env_file)
+
+
+def ReadEnvYaml(output_dir):
+  """Reads and returns the environment values in output_dir/env.yaml file.
+
+  Args:
+    output_dir: str, Path of directory containing the env.yaml to be read.
+
+  Returns:
+    env: {str: str}
+  """
+  env_file_path = os.path.join(output_dir, 'env.yaml')
+  try:
+    with files.FileReader(env_file_path) as f:
+      return yaml.load(f)
+  except files.MissingFileError:
+    raise NoEnvYamlError(output_dir)
+
+
+def PrintEnvExport(env):
+  """Print OS specific export commands for the given environment values.
+
+  Args:
+    env: {str: str}, Dictonary of environment values.
+  """
+  current_os = platforms.OperatingSystem.Current()
+  export_command = 'export'
+  if current_os is platforms.OperatingSystem.WINDOWS:
+    export_command = 'set'
+  for var, value in six.iteritems(env):
+    if ' ' in value:
+      value = '"{value}"'.format(value=value)
+    log.Print('{export_command} {var}={value}'.format(
+        export_command=export_command,
+        var=var,
+        value=value))
+
+
+def PrintEnvUnset(env):
+  """Print OS specific unset commands for the given environment variables.
+
+  Args:
+    env: {str: str}, Dictionary of environment values, the value is ignored.
+  """
+  current_os = platforms.OperatingSystem.Current()
+  export_command = 'unset {var}'
+  if current_os is platforms.OperatingSystem.WINDOWS:
+    export_command = 'set {var}='
+  for var in six.iterkeys(env):
+    log.Print(export_command.format(var=var))
+
+
+def GetDataDir(prefix):
+  """If present, returns the configured data dir, else returns the default.
+
+  Args:
+    prefix: pubsub, datastore, bigtable, etc. The prefix for the *_data_dir
+    property of the emulators section.
+
+  Returns:
+    str, The configured or default data_dir path.
+  """
+  configured = _GetEmulatorProperty(prefix, 'data_dir')
+  if configured: return configured
+
+  config_root = config.Paths().global_config_dir
+  default_data_dir = os.path.join(config_root, 'emulators', prefix)
+  files.MakeDir(default_data_dir)
+  return default_data_dir
+
+
+def GetHostPort(prefix):
+  """If present, returns the configured host port, else returns the default.
+
+  Args:
+    prefix: str, The prefix for the *-emulator property group to look up.
+
+  Raises:
+    InvalidHostError: If configured host-port is not of the form
+    ADDRESS:PORT.
+
+  Returns:
+    str, Configured or default host_port if present, else an unused local port.
+  """
+  default_host = '[::1]' if socket.has_ipv6 else 'localhost'
+
+  # Can't use portpicker here, as it only finds free ports on localhost.
+  arbitrary_host_port = '{host}:{port}'.format(
+      host=default_host, port=random.randint(8000, 8999))
+  configured = _GetEmulatorProperty(prefix, 'host_port') or arbitrary_host_port
+  try:
+    host, port = _ParseHostPort(configured)
+    protocol = socket.AF_INET6 if _IPV6_RE.match(configured) else socket.AF_INET
+    sock = socket.socket(protocol, socket.SOCK_STREAM)
+    port = int(port)
+  except ValueError:
+    raise InvalidHostError()
+
+  if sock.connect_ex((host, port)) != 0:
+    return configured
+
+  return arbitrary_host_port
+
+
+def _ParseHostPort(hostport):
+  if _IPV6_RE.match(hostport):
+    return _IPV6_RE.match(hostport).groups()
+  else:
+    return hostport.split(':')
+
+
+def _GetEmulatorProperty(prefix, prop_name):
+  """Returns the value of the given property in the given emulator group.
+
+  Args:
+    prefix: str, The prefix for the *_emulator property group to look up.
+    prop_name: str, The name of the property to look up.
+
+  Returns:
+    str, The the value of the given property in the specified emulator group.
+  """
+  property_group = 'emulator'
+  full_name = '{}_{}'.format(prefix, prop_name)
+  for section in properties.VALUES:
+    if section.name == property_group and section.HasProperty(full_name):
+      return section.Property(full_name).Get()
+  return None
+
+
+@contextlib.contextmanager
+def Exec(args, log_file=None):
+  """Starts subprocess with given args and ensures its termination upon exit.
+
+  This starts a subprocess with the given args. The stdout and stderr of the
+  subprocess are piped. Note that this is a context manager, to ensure that
+  processes (and references to them) are not leaked.
+
+  Args:
+    args: [str], The arguments to execute. The first argument is the command.
+    log_file: optional file argument to reroute process's output. If given,
+      will be closed when the file is terminated.
+
+  Yields:
+    process, The process handle of the subprocess that has been started.
+  """
+  reroute_stdout = log_file or subprocess.PIPE
+  if not platforms.OperatingSystem.IsWindows():
+    # Check if pid is session leader.
+    if os.getsid(0) != os.getpid():
+      os.setpgid(0, 0)
+  process = subprocess.Popen(args,
+                             stdout=reroute_stdout,
+                             stderr=subprocess.STDOUT)
+  try:
+    yield process
+  finally:
+    if process.poll() is None:
+      process.terminate()
+      process.wait()
+
+
+def PrefixOutput(process, prefix):
+  """Prepends the given prefix to each line of the given process's output.
+
+  Args:
+    process: process, The handle to the process whose output should be prefixed
+    prefix: str, The prefix to be prepended to the process's output.
+  """
+  output_line = process.stdout.readline()
+  while output_line:
+    log.status.Print('[{0}] {1}'.format(prefix,
+                                        encoding.Decode(output_line.rstrip())))
+    log.status.flush()
+    output_line = process.stdout.readline()
+
+
+def BuildArgsList(args):
+  """Converts an argparse.Namespace to a list of arg strings."""
+  args_list = []
+  if args.host_port:
+    if _IPV6_RE.match(args.host_port.host):
+      host = '[{}]'.format(args.host_port.host)
+    else:
+      host = args.host_port.host
+
+    if args.host_port.host is not None:
+      args_list.append('--host={0}'.format(host))
+    if args.host_port.port is not None:
+      args_list.append('--port={0}'.format(args.host_port.port))
+  return args_list
+
+
+def GetEmulatorRoot(emulator):
+  emulator_dir = os.path.join(GetCloudSDKRoot(),
+                              'platform', '{0}-emulator'.format(emulator))
+  if not os.path.isdir(emulator_dir):
+    raise NoEmulatorError('No {0} directory found.'.format(emulator))
+  return emulator_dir
+
+
+def GetEmulatorProxyPath():
+  """Returns path to the emulator reverse proxy."""
+  path = os.path.join(GetCloudSDKRoot(), 'platform', 'emulator-reverse-proxy')
+  if not os.path.isdir(path):
+    # We shouldn't get here because the emulators package should ensure that
+    # this component installed. That's computers for you, though!
+    # TODO(b/36654459) We should potentially allow the user to specify a
+    # location via a property
+    raise MissingProxyError(
+        'emulator-reverse-proxy component must be installed. try running '
+        '`gcloud components install emulator-reverse-proxy`')
+  return path
+
+
+class AttrDict(object):
+  """Allows for a wrapped map to be indexed via attributes instead of keys.
+
+  Example:
+  m = {'a':'b', 'c':{'d':'e', 'f':'g'}}
+  a = AttrDict(m)
+  m['c']['d'] == a.c.d
+  """
+
+  def __init__(self, _dict, recurse=True):
+    """Initializes attributes dictionary.
+
+    Args:
+      _dict: dict, the map to convert into an attribute dictionary
+      recurse: bool, if True then any nested maps will also be treated as
+               attribute dictionaries
+    """
+    if recurse:
+      dict_copy = {}
+      for key, value in six.iteritems(_dict):
+        toset = value
+        if isinstance(value, dict):
+          toset = AttrDict(value, recurse)
+        dict_copy[key] = toset
+      self._dict = dict_copy
+    else:
+      self._dict = _dict
+    self._recurse = recurse
+
+  def __getattr__(self, attr):
+    return self._dict[attr]
+
+  def __setattr__(self, attr, value):
+    if attr in set(['_dict', '_recurse']):
+      super(AttrDict, self).__setattr__(attr, value)
+    else:
+      self._dict[attr] = value
+
+
+class Emulator(six.with_metaclass(abc.ABCMeta)):
+  """This organizes the information to expose an emulator."""
+
+  # TODO(b/35871640) Right now, there is no error handling contract with the
+  #   subclasses. This means that if the subclass process fails, there is no
+  #  way to detect that, surface that, etc. We could implement a contract as
+  #  well as liveness checks etc to ensure that we detect if a process has
+  #  failed and respond gracefully.
+  @abc.abstractmethod
+  def Start(self, port):
+    """Starts the emulator process on the given port.
+
+    Args:
+      port: int, port number for emulator to bind to
+
+    Returns:
+      subprocess.Popen, the emulator process
+    """
+    raise NotImplementedError()
+
+  @property
+  @abc.abstractproperty
+  def prefixes(self):
+    """Returns the grpc route prefixes to route to this service.
+
+    Returns:
+      list(str), list of prefixes.
+    """
+    raise NotImplementedError()
+
+  @property
+  @abc.abstractproperty
+  def service_name(self):
+    """Returns the service name this emulator corresponds to.
+
+    Note that it is assume that the production API this service is emulating
+    exists at <name>.googleapis.com
+
+    Returns:
+      str, the service name
+    """
+    raise NotImplementedError()
+
+  @property
+  @abc.abstractproperty
+  def emulator_title(self):
+    """Returns title of the emulator.
+
+    This is just for nice rendering in the cloud sdk.
+
+    Returns:
+      str, the emulator title
+    """
+    raise NotImplementedError()
+
+  @property
+  @abc.abstractproperty
+  def emulator_component(self):
+    """Returns cloud sdk component to install.
+
+    Returns:
+      str, cloud sdk component name
+    """
+    raise NotImplementedError()
+
+  def _GetLogNo(self):
+    """Returns the OS-level handle to log file.
+
+    This handle is the same as would be returned by os.open(). This is what the
+    subprocess interface expects. Note that the caller needs to make sure to
+    close this to avoid leaking file descriptors.
+
+    Returns:
+      int, OS-level handle to log file
+    """
+    log_file_no, log_file = tempfile.mkstemp()
+    log.status.Print('Logging {0} to: {1}'.format(self.service_name, log_file))
+    return log_file_no
+
+
+class EmulatorArgumentsError(exceptions.Error):
+  """Generic error for invalid arguments."""
+  pass
+
+
+_DEFAULT_PORT = 45763
+
+
+def DefaultPortIfAvailable():
+  """Returns default port if available.
+
+  Raises:
+    EmulatorArgumentsError: if port is not available.
+
+  Returns:
+    int, default port
+  """
+  if portpicker.is_port_free(_DEFAULT_PORT):
+    return _DEFAULT_PORT
+  else:
+    raise EmulatorArgumentsError(
+        'Default emulator port [{}] is already in use'.format(_DEFAULT_PORT))