feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/android/arg_manager.py

@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A shared library for processing and validating Android test arguments."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.firebase.test import arg_file
+from googlecloudsdk.api_lib.firebase.test import arg_util
+from googlecloudsdk.api_lib.firebase.test import arg_validate
+from googlecloudsdk.api_lib.firebase.test.android import catalog_manager
+from googlecloudsdk.calliope import exceptions
+
+
+def TypedArgRules():
+  """Returns the rules for Android test args which depend on the test type.
+
+  This dict is declared in a function rather than globally to avoid garbage
+  collection issues during unit tests.
+
+  Returns:
+    A dict keyed by whether type-specific args are required or optional, and
+    with a nested dict containing any default values for those args.
+  """
+  return {
+      'instrumentation': {
+          'required': ['test'],
+          'optional': [
+              'num_uniform_shards', 'test_targets_for_shard', 'test_package',
+              'test_runner_class', 'test_targets', 'use_orchestrator'
+          ],
+          'defaults': {}
+      },
+      'robo': {
+          'required': [],
+          'optional': ['robo_directives', 'robo_script', 'resign'],
+          'defaults': {
+              'resign': True,
+          }
+      },
+      'game-loop': {
+          'required': [],
+          'optional': ['scenario_numbers', 'scenario_labels'],
+          'defaults': {}
+      },
+  }
+
+
+def SharedArgRules():
+  """Returns the rules for Android test args which are shared by all test types.
+
+  This dict is declared in a function rather than globally to avoid garbage
+  collection issues during unit tests.
+
+  Returns:
+    A dict keyed by whether shared args are required or optional, and with a
+    nested dict containing any default values for those shared args.
+  """
+  return {
+      'required': ['type', 'app'],
+      'optional': [
+          'additional_apks',
+          'app_package',
+          'async_',
+          'auto_google_login',
+          'client_details',
+          'device',
+          'device_ids',
+          'directories_to_pull',
+          'environment_variables',
+          'grant_permissions',
+          'locales',
+          'network_profile',
+          'num_flaky_test_attempts',
+          'obb_files',
+          'orientations',
+          'os_version_ids',
+          'other_files',
+          'performance_metrics',
+          'record_video',
+          'results_bucket',
+          'results_dir',
+          'results_history_name',
+          'timeout',
+      ],
+      'defaults': {
+          'async_': False,
+          'auto_google_login': True,
+          'num_flaky_test_attempts': 0,
+          'performance_metrics': True,
+          'record_video': True,
+          'timeout': 900,  # 15 minutes
+          'grant_permissions': 'all',
+      }
+  }
+
+
+def AllArgsSet():
+  """Returns a set containing the names of every Android test arg."""
+  return arg_util.GetSetOfAllTestArgs(TypedArgRules(), SharedArgRules())
+
+
+class AndroidArgsManager(object):
+  """Manages test arguments for Android devices."""
+
+  def __init__(self,
+               catalog_mgr=None,
+               typed_arg_rules=None,
+               shared_arg_rules=None):
+    """Constructs an AndroidArgsManager for a single Android test matrix.
+
+    Args:
+      catalog_mgr: an AndroidCatalogManager object.
+      typed_arg_rules: a nested dict of dicts which are keyed first on the test
+        type, then by whether args are required or optional, and what their
+        default values are. If None, the default from TypedArgRules() is used.
+      shared_arg_rules: a dict keyed by whether shared args are required or
+        optional, and with a nested dict containing any default values for those
+        shared args. If None, the default dict from SharedArgRules() is used.
+    """
+    self._catalog_mgr = catalog_mgr or catalog_manager.AndroidCatalogManager()
+    self._typed_arg_rules = typed_arg_rules or TypedArgRules()
+    self._shared_arg_rules = shared_arg_rules or SharedArgRules()
+
+  def Prepare(self, args):
+    """Load, apply defaults, and perform validation on test arguments.
+
+    Args:
+      args: an argparse namespace. All the arguments that were provided to this
+        gcloud command invocation (i.e. group and command arguments combined).
+        Arg values from an optional arg-file and/or arg default values may be
+        added to this argparse namespace.
+
+    Raises:
+      InvalidArgumentException: If an argument name is unknown, an argument does
+        not contain a valid value, or an argument is not valid when used with
+        the given type of test.
+      RequiredArgumentException: If a required arg is missing.
+    """
+    all_test_args_set = arg_util.GetSetOfAllTestArgs(self._typed_arg_rules,
+                                                     self._shared_arg_rules)
+    args_from_file = arg_file.GetArgsFromArgFile(args.argspec,
+                                                 all_test_args_set)
+    arg_util.ApplyLowerPriorityArgs(args, args_from_file, True)
+
+    test_type = self.GetTestTypeOrRaise(args)
+    self._CheckForConflictingArgs(args)
+
+    typed_arg_defaults = self._typed_arg_rules[test_type]['defaults']
+    shared_arg_defaults = self._shared_arg_rules['defaults']
+    arg_util.ApplyLowerPriorityArgs(args, typed_arg_defaults)
+    arg_util.ApplyLowerPriorityArgs(args, shared_arg_defaults)
+    self._ApplyLegacyMatrixDimensionDefaults(args)
+
+    arg_validate.ValidateArgsForTestType(args, test_type, self._typed_arg_rules,
+                                         self._shared_arg_rules,
+                                         all_test_args_set)
+    arg_validate.ValidateOsVersions(args, self._catalog_mgr)
+    arg_validate.ValidateDeviceList(args, self._catalog_mgr)
+    arg_validate.ValidateResultsBucket(args)
+    arg_validate.ValidateResultsDir(args)
+    arg_validate.NormalizeAndValidateObbFileNames(args.obb_files)
+    arg_validate.ValidateRoboDirectivesList(args)
+    arg_validate.ValidateEnvironmentVariablesList(args)
+    arg_validate.ValidateTestTargetsForShard(args)
+    arg_validate.NormalizeAndValidateDirectoriesToPullList(
+        args.directories_to_pull)
+    arg_validate.ValidateScenarioNumbers(args)
+
+  def GetTestTypeOrRaise(self, args):
+    """If the test type is not user-specified, infer the most reasonable value.
+
+    Args:
+      args: an argparse namespace. All the arguments that were provided to this
+        gcloud command invocation (i.e. group and command arguments combined).
+
+    Returns:
+      The type of the test to be run (e.g. 'robo' or 'instrumentation') and
+      sets the 'type' arg if it was not user-specified.
+
+    Raises:
+      InvalidArgumentException if an explicit test type is invalid.
+    """
+    if not args.type:
+      args.type = 'instrumentation' if args.test else 'robo'
+    if args.type not in self._typed_arg_rules:
+      raise exceptions.InvalidArgumentException(
+          'type', "'{0}' is not a valid test type.".format(args.type))
+    return args.type
+
+  def _CheckForConflictingArgs(self, args):
+    """Check for any args that cannot appear simultaneously."""
+    if args.device:
+      # If using sparse matrix syntax, can't also use legacy dimension flags.
+      if args.device_ids:
+        raise exceptions.ConflictingArgumentsException('--device-ids',
+                                                       '--device')
+      if args.os_version_ids:
+        raise exceptions.ConflictingArgumentsException('--os-version-ids',
+                                                       '--device')
+      if args.locales:
+        raise exceptions.ConflictingArgumentsException('--locales', '--device')
+      if args.orientations:
+        raise exceptions.ConflictingArgumentsException('--orientations',
+                                                       '--device')
+
+  def _ApplyLegacyMatrixDimensionDefaults(self, args):
+    """Apply defaults to each dimension flag only if not using sparse matrix."""
+    if args.device:
+      return
+    # If --device is unset and all of the legacy dimension flags are unset,
+    # then we want to use --device by default. So don't apply any defaults to
+    # the legacy dimension flags here, which would cause conflicting args.
+    if not (args.device_ids or args.os_version_ids or args.locales or
+            args.orientations):
+      args.device = [{}]  # Default device dimensions will be filled in later.
+      return
+
+    if not args.device_ids:
+      args.device_ids = [self._catalog_mgr.GetDefaultModel()]
+    if not args.os_version_ids:
+      args.os_version_ids = [self._catalog_mgr.GetDefaultVersion()]
+    if not args.locales:
+      args.locales = [self._catalog_mgr.GetDefaultLocale()]
+    if not args.orientations:
+      args.orientations = [self._catalog_mgr.GetDefaultOrientation()]

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/android/catalog_manager.py

@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A wrapper for working with the Android Test Environment Catalog."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.firebase.test import exceptions
+from googlecloudsdk.api_lib.firebase.test import util
+
+
+class AndroidCatalogManager(object):
+  """Encapsulates operations on the Android TestEnvironmentCatalog."""
+
+  def __init__(self, catalog=None):
+    """Construct an AndroidCatalogManager object from a TestEnvironmentCatalog.
+
+    Args:
+      catalog: an Android TestEnvironmentCatalog from Testing API. If it is not
+        injected, the catalog is retrieved from the Testing service.
+
+    Attributes:
+      catalog: an Android TestEnvironmentCatalog.
+    """
+    self.catalog = catalog or util.GetAndroidCatalog()
+    models = self.catalog.models
+    versions = self.catalog.versions
+    locales = self.catalog.runtimeConfiguration.locales
+    orientations = self.catalog.runtimeConfiguration.orientations
+
+    self._model_ids = [m.id for m in models]
+    self._version_ids = [v.id for v in versions]
+    self._locale_ids = [l.id for l in locales]
+    self._orientation_ids = [o.id for o in orientations]
+
+    self._version_name_to_id = {v.versionString: v.id for v in versions}
+
+    # Dimension defaults are lazily computed and cached by GetDefault* methods.
+    self._default_model = None
+    self._default_version = None
+    self._default_locale = None
+    self._default_orientation = None
+
+  def GetDefaultModel(self):
+    """Return the default model listed in the Android environment catalog."""
+    model = (self._default_model if self._default_model else
+             self._FindDefaultDimension(self.catalog.models))
+    if not model:
+      raise exceptions.DefaultDimensionNotFoundError('model')
+    return model
+
+  def GetDefaultVersion(self):
+    """Return the default version listed in the Android environment catalog."""
+    version = (self._default_version if self._default_version else
+               self._FindDefaultDimension(self.catalog.versions))
+    if not version:
+      raise exceptions.DefaultDimensionNotFoundError('version')
+    return version
+
+  def GetDefaultLocale(self):
+    """Return the default locale listed in the Android environment catalog."""
+    locales = self.catalog.runtimeConfiguration.locales
+    locale = (self._default_locale
+              if self._default_locale else self._FindDefaultDimension(locales))
+    if not locale:
+      raise exceptions.DefaultDimensionNotFoundError('locale')
+    return locale
+
+  def GetDefaultOrientation(self):
+    """Return the default orientation in the Android environment catalog."""
+    orientations = self.catalog.runtimeConfiguration.orientations
+    orientation = (self._default_orientation if self._default_orientation else
+                   self._FindDefaultDimension(orientations))
+    if not orientation:
+      raise exceptions.DefaultDimensionNotFoundError('orientation')
+    return orientation
+
+  def _FindDefaultDimension(self, dimension_table):
+    for dimension in dimension_table:
+      if 'default' in dimension.tags:
+        return dimension.id
+    return None
+
+  def ValidateDimensionAndValue(self, dim_name, dim_value):
+    """Validates that a matrix dimension has a valid name and value."""
+    if dim_name == 'model':
+      if dim_value not in self._model_ids:
+        raise exceptions.ModelNotFoundError(dim_value)
+    elif dim_name == 'locale':
+      if dim_value not in self._locale_ids:
+        raise exceptions.LocaleNotFoundError(dim_value)
+    elif dim_name == 'orientation':
+      if dim_value not in self._orientation_ids:
+        raise exceptions.OrientationNotFoundError(dim_value)
+    elif dim_name == 'version':
+      if dim_value not in self._version_ids:
+        # Users are allowed to specify either version name or version ID.
+        version_id = self._version_name_to_id.get(dim_value, None)
+        if not version_id:
+          raise exceptions.VersionNotFoundError(dim_value)
+        return version_id
+    else:
+      raise exceptions.InvalidDimensionNameError(dim_name)
+    return dim_value

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/android/matrix_creator.py

@@ -0,0 +1,344 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Create Android test matrices in Firebase Test Lab."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import os
+import uuid
+
+from apitools.base.py import exceptions as apitools_exceptions
+
+from googlecloudsdk.api_lib.firebase.test import matrix_creator_common
+from googlecloudsdk.api_lib.firebase.test import matrix_ops
+from googlecloudsdk.api_lib.firebase.test import util
+from googlecloudsdk.calliope import exceptions
+from googlecloudsdk.core import log
+import six
+
+
+def CreateMatrix(args, context, history_id, gcs_results_root, release_track):
+  """Creates a new matrix test in Firebase Test Lab from the user's params.
+
+  Args:
+    args: an argparse namespace. All the arguments that were provided to this
+      gcloud command invocation (i.e. group and command arguments combined).
+    context: {str:obj} dict containing the gcloud command context, which
+      includes the Testing API client+messages libs generated by Apitools.
+    history_id: {str} A history ID to publish Tool Results to.
+    gcs_results_root: the root dir for a matrix within the GCS results bucket.
+    release_track: the release track that the command is invoked from.
+
+  Returns:
+    A TestMatrix object created from the supplied matrix configuration values.
+  """
+  creator = MatrixCreator(args, context, history_id, gcs_results_root,
+                          release_track)
+  return creator.CreateTestMatrix(uuid.uuid4().hex)
+
+
+class MatrixCreator(object):
+  """Creates a single test matrix based on user-supplied test arguments."""
+
+  def __init__(self, args, context, history_id, gcs_results_root,
+               release_track):
+    """Construct a MatrixCreator to be used to create a single test matrix.
+
+    Args:
+      args: an argparse namespace. All the arguments that were provided to this
+        gcloud command invocation (i.e. group and command arguments combined).
+      context: {str:obj} dict containing the gcloud command context, which
+        includes the Testing API client+messages libs generated by Apitools.
+      history_id: {str} A history ID to publish Tool Results to.
+      gcs_results_root: the root dir for a matrix within the GCS results bucket.
+      release_track: the release track that the command is invoked from.
+    """
+    self._project = util.GetProject()
+    self._args = args
+    self._history_id = history_id
+    self._gcs_results_root = gcs_results_root
+    self._client = context['testing_client']
+    self._messages = context['testing_messages']
+    self._release_track = release_track
+
+  def _BuildAppReference(self, filename):
+    """Builds either a FileReference or an AppBundle message for a file."""
+    if filename.endswith('.aab'):
+      return None, self._messages.AppBundle(
+          bundleLocation=self._BuildFileReference(os.path.basename(filename)))
+    else:
+      return self._BuildFileReference(os.path.basename(filename)), None
+
+  def _BuildFileReference(self, filename):
+    """Build a FileReference pointing to the GCS copy of a file."""
+    return self._messages.FileReference(
+        gcsPath=os.path.join(self._gcs_results_root, filename))
+
+  def _GetOrchestratorOption(self):
+    orchestrator_options = (
+        self._messages.AndroidInstrumentationTest
+        .OrchestratorOptionValueValuesEnum)
+    if self._args.use_orchestrator is None:
+      return orchestrator_options.ORCHESTRATOR_OPTION_UNSPECIFIED
+    elif self._args.use_orchestrator:
+      return orchestrator_options.USE_ORCHESTRATOR
+    else:
+      return orchestrator_options.DO_NOT_USE_ORCHESTRATOR
+
+  def _BuildRoboDirectives(self, robo_directives_dict):
+    """Build a list of RoboDirectives from the dictionary input."""
+    robo_directives = []
+    action_types = self._messages.RoboDirective.ActionTypeValueValuesEnum
+    action_type_mapping = {
+        'click': action_types.SINGLE_CLICK,
+        'text': action_types.ENTER_TEXT,
+        'ignore': action_types.IGNORE
+    }
+    for key, value in six.iteritems((robo_directives_dict or {})):
+      (action_type, resource_name) = util.ParseRoboDirectiveKey(key)
+      robo_directives.append(
+          self._messages.RoboDirective(
+              resourceName=resource_name,
+              inputText=value,
+              actionType=action_type_mapping.get(action_type)))
+    return robo_directives
+
+  def _BuildAndroidInstrumentationTestSpec(self):
+    """Build a TestSpecification for an AndroidInstrumentationTest."""
+    spec = self._BuildGenericTestSpec()
+    app_apk, app_bundle = self._BuildAppReference(self._args.app)
+    spec.androidInstrumentationTest = self._messages.AndroidInstrumentationTest(
+        appApk=app_apk,
+        appBundle=app_bundle,
+        testApk=self._BuildFileReference(os.path.basename(self._args.test)),
+        appPackageId=self._args.app_package,
+        testPackageId=self._args.test_package,
+        testRunnerClass=self._args.test_runner_class,
+        testTargets=(self._args.test_targets or []),
+        orchestratorOption=self._GetOrchestratorOption(),
+        shardingOption=self._BuildShardingOption())
+    return spec
+
+  def _BuildAndroidRoboTestSpec(self):
+    """Build a TestSpecification for an AndroidRoboTest."""
+    spec = self._BuildGenericTestSpec()
+    app_apk, app_bundle = self._BuildAppReference(self._args.app)
+    robo_modes = self._messages.AndroidRoboTest.RoboModeValueValuesEnum
+    robo_mode = robo_modes.ROBO_VERSION_2 if getattr(
+        self._args, 'resign', True) else robo_modes.ROBO_VERSION_1
+    spec.androidRoboTest = self._messages.AndroidRoboTest(
+        appApk=app_apk,
+        appBundle=app_bundle,
+        appPackageId=self._args.app_package,
+        roboDirectives=self._BuildRoboDirectives(self._args.robo_directives),
+        roboMode=robo_mode)
+    if getattr(self._args, 'robo_script', None):
+      spec.androidRoboTest.roboScript = self._BuildFileReference(
+          os.path.basename(self._args.robo_script))
+    return spec
+
+  def _BuildAndroidGameLoopTestSpec(self):
+    """Build a TestSpecification for an AndroidTestLoop."""
+    spec = self._BuildGenericTestSpec()
+    app_apk, app_bundle = self._BuildAppReference(self._args.app)
+    spec.androidTestLoop = self._messages.AndroidTestLoop(
+        appApk=app_apk,
+        appBundle=app_bundle,
+        appPackageId=self._args.app_package)
+    if self._args.scenario_numbers:
+      spec.androidTestLoop.scenarios = self._args.scenario_numbers
+    if self._args.scenario_labels:
+      spec.androidTestLoop.scenarioLabels = self._args.scenario_labels
+    return spec
+
+  def _BuildGenericTestSpec(self):
+    """Build a generic TestSpecification without test-type specifics."""
+    device_files = []
+    for obb_file in self._args.obb_files or []:
+      obb_file_name = os.path.basename(obb_file)
+      device_files.append(
+          self._messages.DeviceFile(
+              obbFile=self._messages.ObbFile(
+                  obbFileName=obb_file_name,
+                  obb=self._BuildFileReference(obb_file_name))))
+    other_files = getattr(self._args, 'other_files', None) or {}
+    for device_path in other_files.keys():
+      device_files.append(
+          self._messages.DeviceFile(
+              regularFile=self._messages.RegularFile(
+                  content=self._BuildFileReference(
+                      util.GetRelativeDevicePath(device_path)),
+                  devicePath=device_path)))
+    environment_variables = []
+    if self._args.environment_variables:
+      for key, value in six.iteritems(self._args.environment_variables):
+        environment_variables.append(
+            self._messages.EnvironmentVariable(key=key, value=value))
+
+    directories_to_pull = self._args.directories_to_pull or []
+
+    account = None
+    if self._args.auto_google_login:
+      account = self._messages.Account(googleAuto=self._messages.GoogleAuto())
+
+    additional_apks = [
+        self._messages.Apk(
+            location=self._BuildFileReference(os.path.basename(additional_apk)))
+        for additional_apk in getattr(self._args, 'additional_apks', []) or []
+    ]
+
+    grant_permissions = getattr(self._args, 'grant_permissions',
+                                'all') == 'all'
+
+    setup = self._messages.TestSetup(
+        filesToPush=device_files,
+        account=account,
+        environmentVariables=environment_variables,
+        directoriesToPull=directories_to_pull,
+        networkProfile=getattr(self._args, 'network_profile', None),
+        additionalApks=additional_apks,
+        dontAutograntPermissions=not grant_permissions)
+
+    return self._messages.TestSpecification(
+        testTimeout=matrix_ops.ReformatDuration(self._args.timeout),
+        testSetup=setup,
+        disableVideoRecording=not self._args.record_video,
+        disablePerformanceMetrics=not self._args.performance_metrics)
+
+  def _BuildShardingOption(self):
+    """Build a ShardingOption for an AndroidInstrumentationTest."""
+    if getattr(self._args, 'num_uniform_shards', {}):
+      return self._messages.ShardingOption(
+          uniformSharding=self._messages.UniformSharding(
+              numShards=self._args.num_uniform_shards))
+    elif getattr(self._args, 'test_targets_for_shard', {}):
+      return self._messages.ShardingOption(
+          manualSharding=self._BuildManualShard(
+              self._args.test_targets_for_shard))
+
+  def _BuildManualShard(self, test_targets_for_shard):
+    """Build a ManualShard for a ShardingOption."""
+    test_targets = [
+        self._BuildTestTargetsForShard(test_target)
+        for test_target in test_targets_for_shard
+    ]
+    return self._messages.ManualSharding(testTargetsForShard=test_targets)
+
+  def _BuildTestTargetsForShard(self, test_targets_for_each_shard):
+    return self._messages.TestTargetsForShard(testTargets=[
+        target for target in test_targets_for_each_shard.split(';')
+        if target is not None
+    ])
+
+  def _TestSpecFromType(self, test_type):
+    """Map a test type into its corresponding TestSpecification message."""
+    if test_type == 'instrumentation':
+      return self._BuildAndroidInstrumentationTestSpec()
+    elif test_type == 'robo':
+      return self._BuildAndroidRoboTestSpec()
+    elif test_type == 'game-loop':
+      return self._BuildAndroidGameLoopTestSpec()
+    else:  # It's a bug in our arg validation if we ever get here.
+      raise exceptions.InvalidArgumentException(
+          'type', 'Unknown test type "{}".'.format(test_type))
+
+  def _BuildTestMatrix(self, spec):
+    """Build just the user-specified parts of a TestMatrix message.
+
+    Args:
+      spec: a TestSpecification message corresponding to the test type.
+
+    Returns:
+      A TestMatrix message.
+    """
+    if self._args.device:
+      devices = [self._BuildAndroidDevice(d) for d in self._args.device]
+      environment_matrix = self._messages.EnvironmentMatrix(
+          androidDeviceList=self._messages.AndroidDeviceList(
+              androidDevices=devices))
+    else:
+      environment_matrix = self._messages.EnvironmentMatrix(
+          androidMatrix=self._messages.AndroidMatrix(
+              androidModelIds=self._args.device_ids,
+              androidVersionIds=self._args.os_version_ids,
+              locales=self._args.locales,
+              orientations=self._args.orientations))
+
+    gcs = self._messages.GoogleCloudStorage(gcsPath=self._gcs_results_root)
+    hist = self._messages.ToolResultsHistory(projectId=self._project,
+                                             historyId=self._history_id)
+    results = self._messages.ResultStorage(googleCloudStorage=gcs,
+                                           toolResultsHistory=hist)
+
+    client_info = matrix_creator_common.BuildClientInfo(
+        self._messages,
+        getattr(self._args, 'client_details', {}) or {}, self._release_track)
+
+    return self._messages.TestMatrix(
+        testSpecification=spec,
+        environmentMatrix=environment_matrix,
+        clientInfo=client_info,
+        resultStorage=results,
+        flakyTestAttempts=self._args.num_flaky_test_attempts or 0)
+
+  def _BuildAndroidDevice(self, device_map):
+    return self._messages.AndroidDevice(
+        androidModelId=device_map['model'],
+        androidVersionId=device_map['version'],
+        locale=device_map['locale'],
+        orientation=device_map['orientation'])
+
+  def _BuildTestMatrixRequest(self, request_id):
+    """Build a TestingProjectsTestMatricesCreateRequest for a test matrix.
+
+    Args:
+      request_id: {str} a unique ID for the CreateTestMatrixRequest.
+
+    Returns:
+      A TestingProjectsTestMatricesCreateRequest message.
+    """
+    spec = self._TestSpecFromType(self._args.type)
+    return self._messages.TestingProjectsTestMatricesCreateRequest(
+        projectId=self._project,
+        testMatrix=self._BuildTestMatrix(spec),
+        requestId=request_id)
+
+  def CreateTestMatrix(self, request_id):
+    """Invoke the Testing service to create a test matrix from the user's args.
+
+    Args:
+      request_id: {str} a unique ID for the CreateTestMatrixRequest.
+
+    Returns:
+      The TestMatrix response message from the TestMatrices.Create rpc.
+
+    Raises:
+      HttpException if the test service reports an HttpError.
+    """
+    request = self._BuildTestMatrixRequest(request_id)
+    log.debug('TestMatrices.Create request:\n{0}\n'.format(request))
+    try:
+      response = self._client.projects_testMatrices.Create(request)
+      log.debug('TestMatrices.Create response:\n{0}\n'.format(response))
+    except apitools_exceptions.HttpError as error:
+      msg = 'Http error while creating test matrix: ' + util.GetError(error)
+      raise exceptions.HttpException(msg)
+
+    log.status.Print('Test [{id}] has been created in the Google Cloud.'
+                     .format(id=response.testMatrixId))
+    return response

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/arg_file.py

@@ -0,0 +1,261 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A library to load and validate test arguments from a YAML argument file.
+
+  The optional, positional ARGSPEC argument on the command line is used to
+  specify an ARG_FILE:ARG_GROUP_NAME pair, where ARG_FILE is the path to the
+  YAML-format argument file, and ARG_GROUP_NAME is the name of the arg group
+  to load and parse.
+
+  The basic format of a YAML argument file is:
+
+  arg-group-1:
+    arg1: value1
+    arg2: value2
+
+  arg-group-2:
+    arg3: value3
+    ...
+
+  A special 'include: [<group-list>]' syntax allows composition/merging of
+  arg-groups (see example below). Included groups can include: other groups as
+  well, with unlimited nesting within one YAML file.
+
+  Precedence of arguments:
+    Args appearing on the command line will override any arg specified within
+    an argument file.
+    Args which are merged into a group using the 'include:' keyword have lower
+    precedence than an arg already defined in that group.
+
+  Example of a YAML argument file for use with 'gcloud test run ...' commands:
+
+  memegen-robo-args:
+    type: robo
+    app: path/to/memegen.apk
+    robo-script: crawl_init.json
+    include: [common-args, matrix-quick]
+    timeout: 5m
+
+  notepad-instr-args:
+    type: instrumentation
+    app: path/to/notepad.apk
+    test: path/to/notepad-test.apk
+    include: [common-args, matrix-large]
+
+  common-args:
+    results-bucket: gs://my-results-bucket
+    timeout: 600
+
+  matrix-quick:
+    device-ids: [Nexus5, Nexus6]
+    os-version-ids: 21
+    locales: en
+    orientation: landscape
+
+  matrix-large:
+    device-ids: [Nexus5, Nexus6, Nexus7, Nexus9, Nexus10]
+    os-version-ids: [18, 19, 21]
+    include: all-supported-locales
+
+  all-supported-locales:
+    locales: [de, en_US, en_GB, es, fr, it, ru, zh]
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import re
+
+from googlecloudsdk.api_lib.firebase.test import arg_validate
+from googlecloudsdk.api_lib.firebase.test import exceptions
+from googlecloudsdk.calliope import exceptions as calliope_exceptions
+from googlecloudsdk.core import log
+from googlecloudsdk.core import yaml
+import six
+
+
+_ARG_GROUP_PATTERN = re.compile(r'^[a-zA-Z0-9._\-]+\Z')
+
+_INCLUDE = 'include'
+
+
+def GetArgsFromArgFile(argspec, all_test_args_set):
+  """Loads a group of test args from an optional user-supplied arg file.
+
+  Args:
+    argspec: string containing an ARG_FILE:ARG_GROUP_NAME pair, where ARG_FILE
+      is the path to a file containing groups of test arguments in yaml format,
+      and ARG_GROUP_NAME is a yaml object name of a group of arg:value pairs.
+    all_test_args_set: a set of strings for every possible gcloud-test argument
+      name regardless of test type. Used for validation.
+
+  Returns:
+    A {str:str} dict created from the file which maps arg names to arg values.
+
+  Raises:
+    BadFileException: the YAML parser encountered an I/O error or syntax error
+      while reading the arg-file.
+    InvalidTestArgError: an argument name was not a valid gcloud test arg.
+    InvalidArgException: an argument has an invalid value or no value.
+  """
+  if argspec is None:
+    return {}
+
+  arg_file, group_name = _SplitArgFileAndGroup(argspec)
+  all_arg_groups = _ReadArgGroupsFromFile(arg_file)
+  _ValidateArgGroupNames(list(all_arg_groups.keys()))
+
+  args_from_file = {}
+  _MergeArgGroupIntoArgs(args_from_file, group_name, all_arg_groups,
+                         all_test_args_set)
+  log.info('Args loaded from file: ' + six.text_type(args_from_file))
+  return args_from_file
+
+
+def _SplitArgFileAndGroup(file_and_group_str):
+  """Parses an ARGSPEC and returns the arg filename and arg group name."""
+  index = file_and_group_str.rfind(':')
+  if index < 0 or (index == 2 and file_and_group_str.startswith('gs://')):
+    raise exceptions.InvalidArgException(
+        'arg-spec', 'Format must be ARG_FILE:ARG_GROUP_NAME')
+  return file_and_group_str[:index], file_and_group_str[index+1:]
+
+
+def _ReadArgGroupsFromFile(arg_file):
+  """Collects all the arg groups defined in the yaml file into a dictionary.
+
+  Each dictionary key is an arg-group name whose corresponding value is a nested
+  dictionary containing arg-name: arg-value pairs defined in that group.
+
+  Args:
+    arg_file: str, the name of the YAML argument file to open and parse.
+
+  Returns:
+    A dict containing all arg-groups found in the arg_file.
+
+  Raises:
+    yaml.Error: If the YAML file could not be read or parsed.
+    BadFileException: If the contents of the file are not valid.
+  """
+  all_groups = {}
+  for d in yaml.load_all_path(arg_file):
+    if d is None:
+      log.warning('Ignoring empty yaml document.')
+    elif isinstance(d, dict):
+      all_groups.update(d)
+    else:
+      raise calliope_exceptions.BadFileException(
+          'Failed to parse YAML file [{}]: [{}] is not a valid argument '
+          'group.'.format(arg_file, six.text_type(d)))
+  return all_groups
+
+
+def _ValidateArgGroupNames(group_names):
+  for group_name in group_names:
+    if not _ARG_GROUP_PATTERN.match(group_name):
+      raise calliope_exceptions.BadFileException(
+          'Invalid argument group name [{0}]. Names may only use a-zA-Z0-9._-'
+          .format(group_name))
+
+
+def _MergeArgGroupIntoArgs(
+    args_from_file, group_name, all_arg_groups, all_test_args_set,
+    already_included_set=None):
+  """Merges args from an arg group into the given args_from_file dictionary.
+
+  Args:
+    args_from_file: dict of arg:value pairs already loaded from the arg-file.
+    group_name: str, the name of the arg-group to merge into args_from_file.
+    all_arg_groups: dict containing all arg-groups loaded from the arg-file.
+    all_test_args_set: set of str, all possible test arg names.
+    already_included_set: set of str, all group names which were already
+      included. Used to detect 'include:' cycles.
+
+  Raises:
+    BadFileException: an undefined arg-group name was encountered.
+    InvalidArgException: a valid argument name has an invalid value, or
+      use of include: led to cyclic references.
+    InvalidTestArgError: an undefined argument name was encountered.
+  """
+  if already_included_set is None:
+    already_included_set = set()
+  elif group_name in already_included_set:
+    raise exceptions.InvalidArgException(
+        _INCLUDE,
+        'Detected cyclic reference to arg group [{g}]'.format(g=group_name))
+  if group_name not in all_arg_groups:
+    raise calliope_exceptions.BadFileException(
+        'Could not find argument group [{g}] in argument file.'
+        .format(g=group_name))
+
+  arg_group = all_arg_groups[group_name]
+  if not arg_group:
+    log.warning('Argument group [{0}] is empty.'.format(group_name))
+    return
+
+  for arg_name in arg_group:
+    arg = arg_validate.InternalArgNameFrom(arg_name)
+    # Must process include: groups last in order to follow precedence rules.
+    if arg == _INCLUDE:
+      continue
+
+    if arg not in all_test_args_set:
+      raise exceptions.InvalidTestArgError(arg_name)
+    if arg in args_from_file:
+      log.info(
+          'Skipping include: of arg [{0}] because it already had value [{1}].'
+          .format(arg_name, args_from_file[arg]))
+    else:
+      args_from_file[arg] = arg_validate.ValidateArgFromFile(
+          arg, arg_group[arg_name])
+
+  already_included_set.add(group_name)  # Prevent "include:" cycles
+
+  if _INCLUDE in arg_group:
+    included_groups = arg_validate.ValidateStringList(_INCLUDE,
+                                                      arg_group[_INCLUDE])
+    for included_group in included_groups:
+      _MergeArgGroupIntoArgs(args_from_file, included_group, all_arg_groups,
+                             all_test_args_set, already_included_set)
+
+
+# pylint: disable=unused-argument
+def ArgSpecCompleter(prefix, parsed_args, **kwargs):
+  """Tab-completion function for ARGSPECs in the format ARG_FILE:ARG_GROUP.
+
+  If the ARG_FILE exists, parse it on-the-fly to get the list of every ARG_GROUP
+  it contains. If the ARG_FILE does not exist or the ARGSPEC does not yet
+  contain a colon, then fall back to standard shell filename completion by
+  returning an empty list.
+
+  Args:
+    prefix: the partial ARGSPEC string typed by the user so far.
+    parsed_args: the argparse.Namespace for all args parsed so far.
+    **kwargs: keyword args, not used.
+
+  Returns:
+    The list of all ARG_FILE:ARG_GROUP strings which match the prefix.
+  """
+  try:
+    arg_file, group_prefix = _SplitArgFileAndGroup(prefix)
+  except exceptions.InvalidArgException:
+    return []
+  try:
+    groups = list(_ReadArgGroupsFromFile(arg_file).keys())
+  except yaml.FileLoadError:
+    return []
+  return [(arg_file + ':' + g) for g in groups if g.startswith(group_prefix)]

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/arg_util.py

@@ -0,0 +1,808 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A shared library for processing and validating test arguments."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.firebase.test import arg_file
+from googlecloudsdk.api_lib.firebase.test import arg_validate
+from googlecloudsdk.api_lib.firebase.test import exceptions
+from googlecloudsdk.calliope import actions
+from googlecloudsdk.calliope import arg_parsers
+from googlecloudsdk.calliope import base
+from googlecloudsdk.core import log
+import six
+
+ANDROID_INSTRUMENTATION_TEST = 'ANDROID INSTRUMENTATION TEST'
+ANDROID_ROBO_TEST = 'ANDROID ROBO TEST'
+ANDROID_GAME_LOOP_TEST = 'ANDROID GAME-LOOP TEST'
+DEPRECATED_DEVICE_DIMENSIONS = 'DEPRECATED DEVICE DIMENSIONS'
+
+
+def AddCommonTestRunArgs(parser):
+  """Register args which are common to all 'gcloud test run' commands.
+
+  Args:
+    parser: An argparse parser used to add arguments that follow a command
+        in the CLI.
+  """
+  parser.add_argument(
+      'argspec',
+      nargs='?',
+      completer=arg_file.ArgSpecCompleter,
+      help='An ARG_FILE:ARG_GROUP_NAME pair, where ARG_FILE is the path to a '
+      'file containing groups of test arguments in yaml format, and '
+      'ARG_GROUP_NAME is the particular yaml object holding a group of '
+      'arg:value pairs to use. Run *$ gcloud topic arg-files* for more '
+      'information and examples.')
+
+  parser.add_argument(
+      '--async',
+      action='store_true',
+      default=None,
+      dest='async_',
+      help='Invoke a test asynchronously without waiting for test results.')
+  parser.add_argument(
+      '--client-details',
+      type=arg_parsers.ArgDict(),
+      metavar='KEY=VALUE',
+      help="""\
+      Comma-separated, KEY=VALUE map of additional details to attach to the test
+      matrix. Arbitrary KEY=VALUE pairs may be attached to a test matrix to
+      provide additional context about the tests being run. When consuming the
+      test results, such as in Cloud Functions or a CI system, these details can
+      add additional context such as a link to the corresponding pull request.
+
+      Example:
+
+      ```
+      --client-details=buildNumber=1234,pullRequest=https://example.com/link/to/pull-request
+      ```
+
+      To help you identify and locate your test matrix in the Firebase console,
+      use the matrixLabel key.
+
+      Example:
+
+      ```
+      --client-details=matrixLabel="Example matrix label"
+      ```
+      """,
+  )
+  parser.add_argument(
+      '--num-flaky-test-attempts',
+      metavar='int',
+      type=arg_validate.NONNEGATIVE_INT_PARSER,
+      help="""\
+      Specifies the number of times a test execution should be reattempted if
+      one or more of its test cases fail for any reason. An execution that
+      initially fails but succeeds on any reattempt is reported as FLAKY.\n
+      The maximum number of reruns allowed is 10. (Default: 0, which implies
+      no reruns.) All additional attempts are executed in parallel.
+      """,
+  )
+  parser.add_argument(
+      '--record-video',
+      action='store_true',
+      default=None,
+      help='Enable video recording during the test. Enabled by default, use '
+      '--no-record-video to disable.')
+  parser.add_argument(
+      '--results-bucket',
+      help='The name of a Google Cloud Storage bucket where raw test results '
+      'will be stored (default: "test-lab-<random-UUID>"). Note that the '
+      'bucket must be owned by a billing-enabled project, and that using a '
+      'non-default bucket will result in billing charges for the storage used.')
+  parser.add_argument(
+      '--results-dir',
+      help='The name of a *unique* Google Cloud Storage object within the '
+      'results bucket where raw test results will be stored (default: a '
+      'timestamp with a random suffix). Caution: if specified, this argument '
+      '*must be unique* for each test matrix you create, otherwise results '
+      'from multiple test matrices will be overwritten or intermingled.')
+  parser.add_argument(
+      '--timeout',
+      category=base.COMMONLY_USED_FLAGS,
+      type=arg_validate.TIMEOUT_PARSER,
+      help='The max time this test execution can run before it is cancelled '
+      '(default: 15m). It does not include any time necessary to prepare and '
+      'clean up the target device. The maximum possible testing time is 45m '
+      'on physical devices and 60m on virtual devices. The _TIMEOUT_ units can '
+      'be h, m, or s. If no unit is given, seconds are assumed. Examples:\n'
+      '- *--timeout 1h* is 1 hour\n'
+      '- *--timeout 5m* is 5 minutes\n'
+      '- *--timeout 200s* is 200 seconds\n'
+      '- *--timeout 100* is 100 seconds')
+
+
+def AddAndroidTestArgs(parser):
+  """Register args which are specific to Android test commands.
+
+  Args:
+    parser: An argparse parser used to add arguments that follow a command in
+        the CLI.
+  """
+  parser.add_argument(
+      '--app',
+      category=base.COMMONLY_USED_FLAGS,
+      help='The path to the application binary file. The path may be in the '
+      'local filesystem or in Google Cloud Storage using gs:// notation. '
+      'Android App Bundles are specified as .aab, all other files are assumed '
+      'to be APKs.')
+  parser.add_argument(
+      '--app-package',
+      action=actions.DeprecationAction('--app-package', removed=True),
+      help='The Java package of the application under test. By default, the '
+      'application package name is parsed from the APK manifest.')
+  parser.add_argument(
+      '--additional-apks',
+      type=arg_parsers.ArgList(min_length=1, max_length=100),
+      metavar='APK',
+      help='A list of up to 100 additional APKs to install, in addition to '
+      'those being directly tested. The path may be in the local filesystem or '
+      'in Google Cloud Storage using gs:// notation.')
+  parser.add_argument(
+      '--auto-google-login',
+      action='store_true',
+      default=None,
+      help='Automatically log into the test device using a preconfigured '
+      'Google account before beginning the test. Enabled by default, use '
+      '--no-auto-google-login to disable.')
+  parser.add_argument(
+      '--directories-to-pull',
+      type=arg_parsers.ArgList(),
+      metavar='DIR_TO_PULL',
+      help='A list of paths that will be copied from the device\'s storage to '
+      'the designated results bucket after the test is complete. These must be '
+      'absolute paths under `/sdcard`, `/storage`, or `/data/local/tmp` (for '
+      'example, '
+      '`--directories-to-pull /sdcard/tempDir1,/data/local/tmp/tempDir2`). '
+      'Path names are restricted to the characters ```a-zA-Z0-9_-./+```. '
+      'The paths `/sdcard` and `/data` will be made available and treated as '
+      'implicit path substitutions. E.g. if `/sdcard` on a particular device '
+      'does not map to external storage, the system will replace it with the '
+      'external storage path prefix for that device. Note that access to some '
+      'directories on API levels 29 and later may also be limited by scoped '
+      'storage rules.')
+  parser.add_argument(
+      '--environment-variables',
+      type=arg_parsers.ArgDict(),
+      metavar='KEY=VALUE',
+      help="""\
+      A comma-separated, key=value map of environment variables and their
+      desired values. The environment variables are mirrored as extra options to
+      the `am instrument -e KEY1 VALUE1 ...` command and passed to your test
+      runner (typically AndroidJUnitRunner). Examples:
+
+      Enable code coverage and provide a directory to store the coverage
+      results when using Android Test Orchestrator (`--use-orchestrator`):
+
+      ```
+      --environment-variables clearPackageData=true,coverage=true,coverageFilePath=/sdcard/Download/
+      ```
+
+      Enable code coverage and provide a file path to store the coverage
+      results when *not* using Android Test Orchestrator
+      (`--no-use-orchestrator`):
+
+      ```
+      --environment-variables coverage=true,coverageFile=/sdcard/Download/coverage.ec
+      ```
+
+      Note: If you need to embed a comma into a `VALUE` string, please refer to
+      `gcloud topic escaping` for ways to change the default list delimiter.
+      """)
+  parser.add_argument(
+      '--network-profile',
+      metavar='PROFILE_ID',
+      help='The name of the network traffic profile, for example '
+      '`--network-profile=LTE`, which consists of a set of parameters to '
+      'emulate network conditions when running the test (default: no network '
+      'shaping; see available profiles listed by the '
+      '$ {grandparent_command} network-profiles list command). '
+      'This feature only works on physical devices.')
+  parser.add_argument(
+      '--obb-files',
+      type=arg_parsers.ArgList(min_length=1, max_length=2),
+      metavar='OBB_FILE',
+      help='A list of one or two Android OBB file names which will be copied '
+      'to each test device before the tests will run (default: None). Each '
+      'OBB file name must conform to the format as specified by Android (e.g. '
+      '[main|patch].0300110.com.example.android.obb) and will be installed '
+      'into <shared-storage>/Android/obb/<package-name>/ on the test device.')
+  parser.add_argument(
+      '--other-files',
+      type=arg_parsers.ArgDict(min_length=1),
+      metavar='DEVICE_PATH=FILE_PATH',
+      help="""\
+      A list of device-path=file-path pairs that indicate the device paths to
+      push files to the device before starting tests, and the paths of files to
+      push.\n
+      Device paths must be under absolute, approved paths
+      (${EXTERNAL_STORAGE}, or ${ANDROID_DATA}/local/tmp). Source file paths may
+      be in the local filesystem or in Google Cloud Storage (gs://...).\n
+      Examples:\n
+      ```
+      --other-files /sdcard/dir1/file1.txt=local/file.txt,/storage/dir2/file2.jpg=gs://bucket/file.jpg
+      ```\n
+      This flag only copies files to the device. To install files, like OBB or
+      APK files, see --obb-files and --additional-apks.
+      """)
+  parser.add_argument(
+      '--performance-metrics',
+      action='store_true',
+      default=None,
+      help='Monitor and record performance metrics: CPU, memory, network usage,'
+      ' and FPS (game-loop only). Enabled by default, use '
+      '--no-performance-metrics to disable.')
+  parser.add_argument(
+      '--results-history-name',
+      help='The history name for your test results (an arbitrary string label; '
+      'default: the application\'s label from the APK manifest). All tests '
+      'which use the same history name will have their results grouped '
+      'together in the Firebase console in a time-ordered test history list.')
+  parser.add_argument(
+      '--robo-script',
+      category=ANDROID_ROBO_TEST,
+      help='The path to a Robo Script JSON file. The path may be in the local '
+      'filesystem or in Google Cloud Storage using gs:// notation. You can '
+      'guide the Robo test to perform specific actions by recording a Robo '
+      'Script in Android Studio and then specifying this argument. Learn more '
+      'at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting.')
+  parser.add_argument(
+      '--type',
+      category=base.COMMONLY_USED_FLAGS,
+      choices=['instrumentation', 'robo', 'game-loop'],
+      help='The type of test to run.')
+
+  # The following args are specific to Android instrumentation tests.
+
+  parser.add_argument(
+      '--test',
+      category=base.COMMONLY_USED_FLAGS,
+      help='The path to the binary file containing instrumentation tests. The '
+      'given path may be in the local filesystem or in Google Cloud Storage '
+      'using a URL beginning with `gs://`.')
+  parser.add_argument(
+      '--test-package',
+      action=actions.DeprecationAction('--test-package', removed=True),
+      category=ANDROID_INSTRUMENTATION_TEST,
+      help='The Java package name of the instrumentation test. By default, the '
+      'test package name is parsed from the APK manifest.')
+  parser.add_argument(
+      '--test-runner-class',
+      category=ANDROID_INSTRUMENTATION_TEST,
+      help='The fully-qualified Java class name of the instrumentation test '
+      'runner (default: the last name extracted from the APK manifest).')
+  parser.add_argument(
+      '--test-targets',
+      category=ANDROID_INSTRUMENTATION_TEST,
+      type=arg_parsers.ArgList(min_length=1),
+      metavar='TEST_TARGET',
+      help="""\
+      A list of one or more test target filters to apply (default: run all test
+      targets). Each target filter must be fully qualified with the package
+      name, class name, or test annotation desired. Any test filter supported by
+      `am instrument -e ...` is supported. See
+       https://developer.android.com/reference/androidx/test/runner/AndroidJUnitRunner
+       for more information. Examples:
+
+         * `--test-targets "package com.my.package.name"`
+         * `--test-targets "notPackage com.package.to.skip"`
+         * `--test-targets "class com.foo.ClassName"`
+         * `--test-targets "notClass com.foo.ClassName#testMethodToSkip"`
+         * `--test-targets "annotation com.foo.AnnotationToRun"`
+         * `--test-targets "size large notAnnotation com.foo.AnnotationToSkip"`
+      """,
+  )
+  parser.add_argument(
+      '--use-orchestrator',
+      category=ANDROID_INSTRUMENTATION_TEST,
+      action='store_true',
+      default=None,
+      help='Whether each test runs in its own Instrumentation instance with '
+      'the Android Test Orchestrator (default: Orchestrator is not used, same '
+      'as specifying --no-use-orchestrator). Orchestrator is only compatible '
+      'with AndroidJUnitRunner v1.1 or higher. See '
+      'https://developer.android.com/training/testing/junit-runner.html'
+      '#using-android-test-orchestrator for more information about Android '
+      'Test Orchestrator.')
+
+  # The following args are specific to Android Robo tests.
+
+  parser.add_argument(
+      '--resign',
+      category=ANDROID_ROBO_TEST,
+      action='store_true',
+      default=None,
+      help=(
+          'Make Robo re-sign the app-under-test APK for a higher quality crawl.'
+          ' If your app cannot properly function when re-signed, disable this'
+          ' feature. When an app-under-test APK is not re-signed, Robo crawl is'
+          ' slower and Robo has access to less information about the state of'
+          ' the crawled app, which reduces crawl quality. Consequently, if your'
+          ' Roboscript has actions on elements of RecyclerView or AdapterView,'
+          ' and you disable APK re-signing, those actions might require manual'
+          ' tweaking because Robo does not identify RecyclerView and'
+          ' AdapterView in this mode. Enabled by default, use `--no-resign` to'
+          ' disable.'
+      ),
+  )
+  parser.add_argument(
+      '--robo-directives',
+      metavar='TYPE:RESOURCE_NAME=INPUT',
+      category=ANDROID_ROBO_TEST,
+      type=arg_parsers.ArgDict(),
+      help='A comma-separated (`<type>:<key>=<value>`) map of '
+      '`robo_directives` that you can use to customize the behavior of Robo '
+      'test. The `type` specifies the action type of the directive, which may '
+      'take on values `click`, `text` or `ignore`. If no `type` is provided, '
+      '`text` will be used by default. Each key should be the Android resource '
+      'name of a target UI element and each value should be the text input for '
+      'that element. Values are only permitted for `text` type elements, so no '
+      'value should be specified for `click` and `ignore` type elements. No '
+      'more than one `click` element is allowed.'
+      '\n\n'
+      'To provide custom login credentials for your app, use'
+      '\n\n'
+      '    --robo-directives text:username_resource=username,'
+      'text:password_resource=password'
+      '\n\n'
+      'To instruct Robo to click on the sign-in button, use'
+      '\n\n'
+      '    --robo-directives click:sign_in_button='
+      '\n\n'
+      'To instruct Robo to ignore any UI elements with resource names which '
+      'equal or start with the user-defined value, use'
+      '\n\n'
+      '  --robo-directives ignore:ignored_ui_element_resource_name='
+      '\n\n'
+      'To learn more about Robo test and robo_directives, see '
+      'https://firebase.google.com/docs/test-lab/android/command-line#custom_login_and_text_input_with_robo_test.'
+      '\n\n'
+      'Caution: You should only use credentials for test accounts that are not '
+      'associated with real users.')
+
+  # The following args are specific to Android game-loop tests.
+
+  parser.add_argument(
+      '--scenario-numbers',
+      metavar='int',
+      type=arg_parsers.ArgList(element_type=int, min_length=1, max_length=1024),
+      category=ANDROID_GAME_LOOP_TEST,
+      help='A list of game-loop scenario numbers which will be run as part of '
+      'the test (default: all scenarios). A maximum of 1024 scenarios may be '
+      'specified in one test matrix, but the maximum number may also be '
+      'limited by the overall test *--timeout* setting.')
+
+  parser.add_argument(
+      '--scenario-labels',
+      metavar='LABEL',
+      type=arg_parsers.ArgList(min_length=1),
+      category=ANDROID_GAME_LOOP_TEST,
+      help='A list of game-loop scenario labels (default: None). '
+      'Each game-loop scenario may be labeled in the APK manifest file with '
+      'one or more arbitrary strings, creating logical groupings (e.g. '
+      'GPU_COMPATIBILITY_TESTS). If *--scenario-numbers* and '
+      '*--scenario-labels* are specified together, Firebase Test Lab will '
+      'first execute each scenario from *--scenario-numbers*. It will then '
+      'expand each given scenario label into a list of scenario numbers marked '
+      'with that label, and execute those scenarios.')
+
+
+def AddIosTestArgs(parser):
+  """Register args which are specific to iOS test commands.
+
+  Args:
+    parser: An argparse parser used to add arguments that follow a command in
+        the CLI.
+  """
+  parser.add_argument(
+      '--type',
+      category=base.COMMONLY_USED_FLAGS,
+      choices=['xctest', 'game-loop', 'robo'],
+      # TODO(b/260103145): Include links to test documentation
+      help='The type of iOS test to run.')
+  parser.add_argument(
+      '--test',
+      category=base.COMMONLY_USED_FLAGS,
+      metavar='XCTEST_ZIP',
+      help='The path to the test package (a zip file containing the iOS app '
+      'and XCTest files). The given path may be in the local filesystem or in '
+      'Google Cloud Storage using a URL beginning with `gs://`. Note: any '
+      '.xctestrun file in this zip file will be ignored if *--xctestrun-file* '
+      'is specified.')
+  parser.add_argument(
+      '--xctestrun-file',
+      category=base.COMMONLY_USED_FLAGS,
+      metavar='XCTESTRUN_FILE',
+      help='The path to an .xctestrun file that will override any .xctestrun '
+      'file contained in the *--test* package. Because the .xctestrun file '
+      'contains environment variables along with test methods to run and/or '
+      'ignore, this can be useful for customizing or sharding test suites. The '
+      'given path may be in the local filesystem or in Google Cloud Storage '
+      'using a URL beginning with `gs://`.')
+  parser.add_argument(
+      '--xcode-version',
+      category=base.COMMONLY_USED_FLAGS,
+      help="""\
+      The version of Xcode that should be used to run an XCTest. Defaults to the
+      latest Xcode version supported in Firebase Test Lab. This Xcode version
+      must be supported by all iOS versions selected in the test matrix. The
+      list of Xcode versions supported by each version of iOS can be viewed by
+      running `$ {parent_command} versions list`.""")
+  parser.add_argument(
+      '--device',
+      category=base.COMMONLY_USED_FLAGS,
+      type=arg_parsers.ArgDict(min_length=1),
+      action='append',
+      metavar='DIMENSION=VALUE',
+      help="""\
+      A list of ``DIMENSION=VALUE'' pairs which specify a target device to test
+      against. This flag may be repeated to specify multiple devices. The device
+      dimensions are: *model*, *version*, *locale*, and *orientation*. If any
+      dimensions are omitted, they will use a default value. The default value,
+      and all possible values, for each dimension can be found with the
+      ``list'' command for that dimension, such as `$ {parent_command} models
+      list`. Omitting this flag entirely will run tests against a single device
+      using defaults for every dimension.
+
+      Examples:\n
+      ```
+      --device model=iphone8plus
+      --device version=11.2
+      --device model=ipadmini4,version=11.2,locale=zh_CN,orientation=landscape
+      ```
+      """)
+  parser.add_argument(
+      '--results-history-name',
+      help='The history name for your test results (an arbitrary string label; '
+      'default: the bundle ID for the iOS application). All tests '
+      'which use the same history name will have their results grouped '
+      'together in the Firebase console in a time-ordered test history list.')
+  parser.add_argument(
+      '--app',
+      help='The path to the application archive (.ipa file) for game-loop '
+           'testing. The path may be in the local filesystem or in Google '
+           'Cloud Storage using gs:// notation. This flag is only valid when '
+           '*--type* is *game-loop* or *robo*.'
+  )
+
+  # The following args are specific to iOS xctest tests.
+  parser.add_argument(
+      '--test-special-entitlements',
+      action='store_true',
+      default=None,
+      help="""\
+      Enables testing special app entitlements. Re-signs an app having special
+      entitlements with a new application-identifier. This currently supports
+      testing Push Notifications (aps-environment) entitlement for up to one
+      app in a project.
+
+      Note: Because this changes the app's identifier, make sure none of the
+      resources in your zip file contain direct references to the test app's
+      bundle id.
+      """)
+
+
+def AddBetaArgs(parser):
+  """Register args which are only available in the beta run commands.
+
+  Args:
+    parser: An argparse parser used to add args that follow a command.
+  """
+  del parser  # Unused by AddBetaArgs
+
+
+def AddGaArgs(parser):
+  """Register args which are only available in the GA run command.
+
+  Args:
+    parser: An argparse parser used to add args that follow a command.
+  """
+  del parser  # Unused by AddGaArgs
+
+
+def AddAndroidBetaArgs(parser):
+  """Register args which are only available in the Android beta run command.
+
+  Args:
+    parser: An argparse parser used to add args that follow a command.
+  """
+  # Mutually exclusive sharding options group.
+  sharding_options = parser.add_group(mutex=True, help='Sharding options.')
+  sharding_options.add_argument(
+      '--num-uniform-shards',
+      metavar='int',
+      type=arg_validate.POSITIVE_INT_PARSER,
+      help="""\
+      Specifies the number of shards across which to distribute test cases. The
+      shards are run in parallel on separate devices. For example, if your test
+      execution contains 20 test cases and you specify four shards, the
+      instrumentation command passes arguments of `-e numShards 4` to
+      AndroidJUnitRunner and each shard executes about five test cases. Based on
+      the sharding mechanism AndroidJUnitRunner uses, there is no guarantee that
+      test cases will be distributed with perfect uniformity.
+
+      The number of shards specified must always be a positive number that is no
+      greater than the total number of test cases. When you select one or more
+      physical devices, the number of shards specified must be <= 50. When you
+      select one or more Arm virtual devices, the number of shards specified
+      must be <= 200. When you select only x86 virtual devices, the number of
+      shards specified must be <= 500.
+      """)
+  sharding_options.add_argument(
+      '--test-targets-for-shard',
+      metavar='TEST_TARGETS_FOR_SHARD',
+      action='append',
+      help="""\
+      Specifies a group of packages, classes, and/or test cases to run in
+      each shard (a group of test cases). Each time this flag is repeated, it
+      creates a new shard. The shards are run in parallel on separate devices.
+      You can repeat this flag up to 50 times when you select one or more
+      physical devices, up to 200 times when you select one or more Arm virtual
+      devices, and up to 500 times when you select only x86 virtual devices.
+
+      Note: If you include the flags *--environment-variable* or
+      *--test-targets* when running *--test-targets-for-shard*, the former flags
+      are applied to all of the shards you create.
+
+      Examples:
+
+      You can also specify multiple packages, classes, or test cases in the
+      same shard by separating each item with a comma. For example:
+
+      ```
+      --test-targets-for-shard
+      "package com.package1.for.shard1,com.package2.for.shard1"
+      ```
+
+      ```
+      --test-targets-for-shard
+      "class com.foo.ClassForShard2#testMethod1,com.foo.ClassForShard2#testMethod2"
+      ```
+
+      To specify both package and class in the same shard, separate `package`
+      and `class` with semicolons:
+
+      ```
+      --test-targets-for-shard
+      "class com.foo.ClassForShard3;package com.package.for.shard3"
+      ```
+      """)
+  parser.add_argument(
+      '--grant-permissions',
+      metavar='PERMISSIONS',
+      help='Whether to grant runtime permissions on the device before the test '
+      'begins. By default, all permissions are granted.',
+      default=None,
+      choices=['all', 'none'])
+
+
+def AddIosBetaArgs(parser):
+  """Register args which are only available in the iOS beta run command.
+
+  Args:
+    parser: An argparse parser used to add args that follow a command.
+  """
+  parser.add_argument(
+      '--additional-ipas',
+      type=arg_parsers.ArgList(min_length=1, max_length=100),
+      metavar='IPA',
+      help='List of up to 100 additional IPAs to install, in addition to '
+      'the one being directly tested. The path may be in the local filesystem '
+      'or in Google Cloud Storage using gs:// notation.')
+  parser.add_argument(
+      '--other-files',
+      type=arg_parsers.ArgDict(min_length=1),
+      metavar='DEVICE_PATH=FILE_PATH',
+      help="""\
+      A list of device-path=file-path pairs that specify the paths of the test
+      device and the files you want pushed to the device prior to testing.\n
+      Device paths should either be under the Media shared folder (e.g. prefixed
+      with /private/var/mobile/Media) or within the documents directory of the
+      filesystem of an app under test (e.g. /Documents). Device paths to app
+      filesystems should be prefixed by the bundle ID and a colon. Source file
+      paths may be in the local filesystem or in Google Cloud Storage
+      (gs://...).\n
+      Examples:\n
+      ```
+      --other-files com.my.app:/Documents/file.txt=local/file.txt,/private/var/mobile/Media/file.jpg=gs://bucket/file.jpg
+      ```
+      """)
+  parser.add_argument(
+      '--directories-to-pull',
+      type=arg_parsers.ArgList(),
+      metavar='DIR_TO_PULL',
+      help="""\
+      A list of paths that will be copied from the device\'s storage to
+      the designated results bucket after the test is complete. These must be
+      absolute paths under `/private/var/mobile/Media` or `/Documents` of the
+      app under test. If the path is under an app\'s `/Documents`, it must be
+      prefixed with the app\'s bundle id and a colon.\n
+      Example:\n
+      ```
+      --directories-to-pull=com.my.app:/Documents/output,/private/var/mobile/Media/output
+      ```
+      """)
+
+  # The following args are specific to iOS game-loop tests.
+
+  parser.add_argument(
+      '--scenario-numbers',
+      metavar='int',
+      type=arg_parsers.ArgList(element_type=int, min_length=1, max_length=1024),
+      help='A list of game-loop scenario numbers which will be run as part of '
+           'the test (default: scenario 1). A maximum of 1024 scenarios may be '
+           'specified in one test matrix, but the maximum number may also be '
+           'limited by the overall test *--timeout* setting. This flag is only '
+           'valid when *--type=game-loop* is also set.'
+  )
+
+  # The following args are specific to iOS Robo tests.
+
+  parser.add_argument(
+      '--robo-script',
+      help="""\
+      The path to a Robo Script JSON file. The path may be in the local
+      filesystem or in Google Cloud Storage using gs:// notation. You can
+      guide the Robo test to perform specific actions by specifying a Robo
+      Script with this argument. Learn more at
+      https://firebase.google.com/docs/test-lab/robo-ux-test#scripting.
+      This flag is only valid when *--type=robo* is also set.
+      """)
+
+
+def AddMatrixArgs(parser):
+  """Register the repeatable args which define the axes for a test matrix.
+
+  Args:
+    parser: An argparse parser used to add arguments that follow a command
+        in the CLI.
+  """
+  parser.add_argument(
+      '--device',
+      category=base.COMMONLY_USED_FLAGS,
+      type=arg_parsers.ArgDict(min_length=1),
+      action='append',
+      metavar='DIMENSION=VALUE',
+      help="""\
+      A list of ``DIMENSION=VALUE'' pairs which specify a target device to test
+      against. This flag may be repeated to specify multiple devices. The four
+      device dimensions are: *model*, *version*, *locale*, and *orientation*. If
+      any dimensions are omitted, they will use a default value. The default
+      value, and all possible values, for each dimension can be found with the
+      ``list'' command for that dimension, such as `$ {parent_command} models
+      list`. *--device* is now the preferred way to specify test devices and may
+      not be used in conjunction with *--devices-ids*, *--os-version-ids*,
+      *--locales*, or *--orientations*. Omitting all of the preceding
+      dimension-related flags will run tests against a single device using
+      defaults for all four device dimensions.
+
+      Examples:\n
+      ```
+      --device model=Nexus6
+      --device version=23,orientation=portrait
+      --device model=shamu,version=22,locale=zh_CN,orientation=default
+      ```
+      """)
+  parser.add_argument(
+      '--device-ids',
+      '-d',
+      category=DEPRECATED_DEVICE_DIMENSIONS,
+      type=arg_parsers.ArgList(min_length=1),
+      metavar='MODEL_ID',
+      help='The list of MODEL_IDs to test against (default: one device model '
+      'determined by the Firebase Test Lab device catalog; see TAGS listed '
+      'by the `$ {parent_command} models list` command).')
+  parser.add_argument(
+      '--os-version-ids',
+      '-v',
+      category=DEPRECATED_DEVICE_DIMENSIONS,
+      type=arg_parsers.ArgList(min_length=1),
+      metavar='OS_VERSION_ID',
+      help='The list of OS_VERSION_IDs to test against (default: a version ID '
+      'determined by the Firebase Test Lab device catalog).')
+  parser.add_argument(
+      '--locales',
+      '-l',
+      category=DEPRECATED_DEVICE_DIMENSIONS,
+      type=arg_parsers.ArgList(min_length=1),
+      metavar='LOCALE',
+      help='The list of LOCALEs to test against (default: a single locale '
+      'determined by the Firebase Test Lab device catalog).')
+  parser.add_argument(
+      '--orientations',
+      '-o',
+      category=DEPRECATED_DEVICE_DIMENSIONS,
+      type=arg_parsers.ArgList(
+          min_length=1, max_length=2, choices=arg_validate.ORIENTATION_LIST),
+      completer=arg_parsers.GetMultiCompleter(OrientationsCompleter),
+      metavar='ORIENTATION',
+      help='The device orientation(s) to test against (default: portrait). '
+      'Specifying \'default\' will pick the preferred orientation '
+      'for the app.')
+
+
+def OrientationsCompleter(prefix, unused_parsed_args, unused_kwargs):
+  return [p for p in arg_validate.ORIENTATION_LIST if p.startswith(prefix)]
+
+
+def GetSetOfAllTestArgs(type_rules, shared_rules):
+  """Build a set of all possible 'gcloud test run' args.
+
+  We need this set to test for invalid arg combinations because gcloud core
+  adds many args to our args.Namespace that we don't care about and don't want
+  to validate. We also need this to validate args coming from an arg-file.
+
+  Args:
+    type_rules: a nested dictionary defining the required and optional args
+      per type of test, plus any default values.
+    shared_rules: a nested dictionary defining the required and optional args
+      shared among all test types, plus any default values.
+
+  Returns:
+    A set of strings for every gcloud-test argument.
+  """
+  all_test_args_list = (
+      shared_rules['required'] + shared_rules['optional'] + list(
+          shared_rules['defaults'].keys()))
+  for type_dict in type_rules.values():
+    all_test_args_list += (
+        type_dict['required'] + type_dict['optional'] + list(
+            type_dict['defaults'].keys()))
+  return set(all_test_args_list)
+
+
+def ApplyLowerPriorityArgs(args, lower_pri_args, issue_cli_warning=False):
+  """Apply lower-priority arg values from a dictionary to args without values.
+
+  May be used to apply arg default values, or to merge args from another source,
+  such as an arg-file. Args which already have a value are never modified by
+  this function. Thus, if there are multiple sets of lower-priority args, they
+  should be applied in order from highest-to-lowest precedence.
+
+  Args:
+    args: the existing argparse.Namespace. All the arguments that were provided
+      to the command invocation (i.e. group and command arguments combined),
+      plus any arg defaults already applied to the namespace. These args have
+      higher priority than the lower_pri_args.
+    lower_pri_args: a dict mapping lower-priority arg names to their values.
+    issue_cli_warning: (boolean) issue a warning if an arg already has a value
+      from the command line and we do not apply the lower-priority arg value
+      (used for arg-files where any args specified in the file are lower in
+      priority than the CLI args.).
+  """
+  for arg in lower_pri_args:
+    if getattr(args, arg, None) is None:
+      log.debug('Applying default {0}: {1}'
+                .format(arg, six.text_type(lower_pri_args[arg])))
+      setattr(args, arg, lower_pri_args[arg])
+    elif issue_cli_warning and getattr(args, arg) != lower_pri_args[arg]:
+      ext_name = exceptions.ExternalArgNameFrom(arg)
+      log.warning(
+          'Command-line argument "--{0} {1}" overrides file argument "{2}: {3}"'
+          .format(ext_name,
+                  _FormatArgValue(getattr(args, arg)), ext_name,
+                  _FormatArgValue(lower_pri_args[arg])))
+
+
+def _FormatArgValue(value):
+  if isinstance(value, list):
+    return ' '.join(value)
+  else:
+    return six.text_type(value)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/arg_validate.py

@@ -0,0 +1,611 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A shared library to validate 'gcloud test' CLI argument values."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import datetime
+import posixpath
+import random
+import re
+import string
+import sys
+
+from googlecloudsdk.api_lib.firebase.test import exceptions as test_exceptions
+from googlecloudsdk.api_lib.firebase.test import util as util
+from googlecloudsdk.api_lib.storage import storage_util
+from googlecloudsdk.calliope import arg_parsers
+from googlecloudsdk.calliope import exceptions
+from googlecloudsdk.core.util import files
+import six
+
+
+def ValidateArgFromFile(arg_internal_name, arg_value):
+  """Do checks/mutations on arg values parsed from YAML which need validation.
+
+  Any arg not appearing in the _FILE_ARG_VALIDATORS dictionary is assumed to be
+  a simple string to be validated by the default _ValidateString() function.
+
+  Mutations of the args are done in limited cases to improve ease-of-use.
+  This includes:
+  1) The YAML parser automatically converts attribute values into numeric types
+  where possible. The os-version-ids for Android devices happen to be integers,
+  but the Testing service expects them to be strings, so we automatically
+  convert them to strings so users don't have to quote each one.
+  2) The include: keyword, plus all test args that normally expect lists (e.g.
+  device-ids, os-version-ids, locales, orientations...), will also accept a
+  single value which is not specified using YAML list notation (e.g not enclosed
+  in []). Such single values are automatically converted into a list containing
+  one element.
+
+  Args:
+    arg_internal_name: the internal form of the arg name.
+    arg_value: the argument's value as parsed from the yaml file.
+
+  Returns:
+    The validated argument value.
+
+  Raises:
+    InvalidArgException: If the arg value is missing or is not valid.
+  """
+  if arg_value is None:
+    raise test_exceptions.InvalidArgException(arg_internal_name,
+                                              'no argument value found.')
+  if arg_internal_name in _FILE_ARG_VALIDATORS:
+    return _FILE_ARG_VALIDATORS[arg_internal_name](arg_internal_name, arg_value)
+  return _ValidateString(arg_internal_name, arg_value)
+
+
+# Constants shared between arg-file validation and CLI flag validation.
+POSITIVE_INT_PARSER = arg_parsers.BoundedInt(1, sys.maxsize)
+NONNEGATIVE_INT_PARSER = arg_parsers.BoundedInt(0, sys.maxsize)
+TIMEOUT_PARSER = arg_parsers.Duration(lower_bound='1m', upper_bound='6h')
+TIMEOUT_PARSER_US = arg_parsers.Duration(
+    lower_bound='1m', upper_bound='6h', parsed_unit='us')
+ORIENTATION_LIST = ['portrait', 'landscape', 'default']
+PERMISSIONS_LIST = ['all', 'none']
+
+
+def ValidateStringList(arg_internal_name, arg_value):
+  """Validates an arg whose value should be a list of strings.
+
+  Args:
+    arg_internal_name: the internal form of the arg name.
+    arg_value: the argument's value parsed from yaml file.
+
+  Returns:
+    The validated argument value.
+
+  Raises:
+    InvalidArgException: the argument's value is not valid.
+  """
+  # convert single str to a str list
+  if isinstance(arg_value, six.string_types):
+    return [arg_value]
+  if isinstance(arg_value, int):  # convert single int to a str list
+    return [str(arg_value)]
+  if isinstance(arg_value, list):
+    return [_ValidateString(arg_internal_name, value) for value in arg_value]
+  raise test_exceptions.InvalidArgException(arg_internal_name, arg_value)
+
+
+def _ValidateString(arg_internal_name, arg_value):
+  """Validates an arg whose value should be a simple string."""
+  if isinstance(arg_value, six.string_types):
+    return arg_value
+  if isinstance(arg_value, int):  # convert int->str if str is really expected
+    return str(arg_value)
+  raise test_exceptions.InvalidArgException(arg_internal_name, arg_value)
+
+
+def _ValidateBool(arg_internal_name, arg_value):
+  """Validates an argument which should have a boolean value."""
+  # Note: the python yaml parser automatically does string->bool conversion for
+  # true/True/TRUE/false/False/FALSE and also for variations of on/off/yes/no.
+  if isinstance(arg_value, bool):
+    return arg_value
+  raise test_exceptions.InvalidArgException(arg_internal_name, arg_value)
+
+
+def _ValidateDuration(arg_internal_name, arg_value):
+  """Validates an argument which should have a Duration value."""
+  try:
+    if isinstance(arg_value, six.string_types):
+      return TIMEOUT_PARSER(arg_value)
+    elif isinstance(arg_value, int):
+      return TIMEOUT_PARSER(str(arg_value))
+  except arg_parsers.ArgumentTypeError as e:
+    raise test_exceptions.InvalidArgException(arg_internal_name,
+                                              six.text_type(e))
+  raise test_exceptions.InvalidArgException(arg_internal_name, arg_value)
+
+
+def _ValidateDurationUs(arg_internal_name, arg_value):
+  """Validates an argument which should have Duration value in microseconds."""
+  try:
+    if isinstance(arg_value, six.string_types):
+      return TIMEOUT_PARSER_US(arg_value)
+    elif isinstance(arg_value, int):
+      return TIMEOUT_PARSER_US(str(arg_value))
+  except arg_parsers.ArgumentTypeError as e:
+    raise test_exceptions.InvalidArgException(arg_internal_name,
+                                              six.text_type(e))
+  raise test_exceptions.InvalidArgException(arg_internal_name, arg_value)
+
+
+def _ValidatePositiveInteger(arg_internal_name, arg_value):
+  """Validates an argument which should be an integer > 0."""
+  try:
+    if isinstance(arg_value, int):
+      return POSITIVE_INT_PARSER(str(arg_value))
+  except arg_parsers.ArgumentTypeError as e:
+    raise test_exceptions.InvalidArgException(arg_internal_name,
+                                              six.text_type(e))
+  raise test_exceptions.InvalidArgException(arg_internal_name, arg_value)
+
+
+def _ValidateNonNegativeInteger(arg_internal_name, arg_value):
+  """Validates an argument which should be an integer >= 0."""
+  try:
+    if isinstance(arg_value, int):
+      return NONNEGATIVE_INT_PARSER(str(arg_value))
+  except arg_parsers.ArgumentTypeError as e:
+    raise test_exceptions.InvalidArgException(arg_internal_name,
+                                              six.text_type(e))
+  raise test_exceptions.InvalidArgException(arg_internal_name, arg_value)
+
+
+def _ValidatePositiveIntList(arg_internal_name, arg_value):
+  """Validates an arg whose value should be a list of ints > 0.
+
+  Args:
+    arg_internal_name: the internal form of the arg name.
+    arg_value: the argument's value parsed from yaml file.
+
+  Returns:
+    The validated argument value.
+
+  Raises:
+    InvalidArgException: the argument's value is not valid.
+  """
+  if isinstance(arg_value, int):  # convert single int to an int list
+    arg_value = [arg_value]
+  if isinstance(arg_value, list):
+    return [_ValidatePositiveInteger(arg_internal_name, v) for v in arg_value]
+  raise test_exceptions.InvalidArgException(arg_internal_name, arg_value)
+
+
+def _ValidateOrientationList(arg_internal_name, arg_value):
+  """Validates that 'orientations' only contains allowable values."""
+  arg_value = ValidateStringList(arg_internal_name, arg_value)
+  for orientation in arg_value:
+    _ValidateOrientation(orientation)
+  if len(arg_value) != len(set(arg_value)):
+    raise test_exceptions.InvalidArgException(
+        arg_internal_name, 'orientations may not be repeated.')
+  return arg_value
+
+
+def _ValidateOrientation(orientation):
+  if orientation not in ORIENTATION_LIST:
+    raise test_exceptions.OrientationNotFoundError(orientation)
+
+
+def _ValidatePermissions(arg_internal_name, arg_value):
+  if arg_value not in PERMISSIONS_LIST:
+    raise test_exceptions.InvalidArgException(
+        arg_internal_name,
+        'Invalid permissions specified. Must be either "all" or "none"')
+  return arg_value
+
+
+def _ValidateObbFileList(arg_internal_name, arg_value):
+  """Validates that 'obb-files' contains at most 2 entries."""
+  arg_value = ValidateStringList(arg_internal_name, arg_value)
+  if len(arg_value) > 2:
+    raise test_exceptions.InvalidArgException(
+        arg_internal_name, 'At most two OBB files may be specified.')
+  return arg_value
+
+
+def _ValidateAdditionalApksList(arg_internal_name, arg_value):
+  """Validates that 'additional-apks' contains [1, 100] entries."""
+  arg_value = ValidateStringList(arg_internal_name, arg_value)
+  if len(arg_value) < 1:
+    raise test_exceptions.InvalidArgException(
+        arg_internal_name, 'At least 1 additional apk must be specified.')
+  if len(arg_value) > 100:
+    raise test_exceptions.InvalidArgException(
+        arg_internal_name, 'At most 100 additional apks may be specified.')
+  return arg_value
+
+
+def _ValidateAdditionalIpasList(arg_internal_name, arg_value):
+  """Validates that 'additional-ipas' contains [1, 100] entries."""
+  if len(arg_value) < 1:
+    raise test_exceptions.InvalidArgException(
+        arg_internal_name, 'At least 1 additional ipa must be specified.')
+  if len(arg_value) > 100:
+    raise test_exceptions.InvalidArgException(
+        arg_internal_name, 'At most 100 additional ipas may be specified.')
+  return arg_value
+
+
+def _ValidateKeyValueStringPairs(arg_internal_name, arg_value):
+  """Validates that an argument is a dict of string-type key-value pairs."""
+  if isinstance(arg_value, dict):
+    new_dict = {}
+    # Cannot use dict comprehension since it's not supported in Python 2.6.
+    for (key, value) in arg_value.items():
+      new_dict[str(key)] = _ValidateString(arg_internal_name, value)
+    return new_dict
+  else:
+    raise test_exceptions.InvalidArgException(arg_internal_name,
+                                              'Malformed key-value pairs.')
+
+
+def _ValidateListOfStringToStringDicts(arg_internal_name, arg_value):
+  """Validates that an argument is a list of dicts of key=value string pairs."""
+  if not isinstance(arg_value, list):
+    raise test_exceptions.InvalidArgException(
+        arg_internal_name, 'is not a list of maps of key-value pairs.')
+  new_list = []
+  for a_dict in arg_value:
+    if not isinstance(a_dict, dict):
+      raise test_exceptions.InvalidArgException(
+          arg_internal_name,
+          'Each list item must be a map of key-value string pairs.')
+    new_dict = {}
+    for (key, value) in a_dict.items():
+      new_dict[str(key)] = _ValidateString(key, value)
+    new_list.append(new_dict)
+  return new_list
+
+
+# Map of internal arg names to their appropriate validation functions.
+# Any arg not appearing in this map is assumed to be a simple string.
+_FILE_ARG_VALIDATORS = {
+    'additional_apks': _ValidateAdditionalApksList,
+    'additional_ipas': _ValidateAdditionalIpasList,
+    'async_': _ValidateBool,
+    'auto_google_login': _ValidateBool,
+    'client_details': _ValidateKeyValueStringPairs,
+    'device': _ValidateListOfStringToStringDicts,
+    'device_ids': ValidateStringList,
+    'directories_to_pull': ValidateStringList,
+    'environment_variables': _ValidateKeyValueStringPairs,
+    'grant_permissions': _ValidatePermissions,
+    'locales': ValidateStringList,
+    'orientations': _ValidateOrientationList,
+    'obb_files': _ValidateObbFileList,
+    'num_flaky_test_attempts': _ValidateNonNegativeInteger,
+    'num_uniform_shards': _ValidatePositiveInteger,
+    'test_targets_for_shard': ValidateStringList,
+    'test_special_entitlements': _ValidateBool,
+    'os_version_ids': ValidateStringList,
+    'other_files': _ValidateKeyValueStringPairs,
+    'performance_metrics': _ValidateBool,
+    'record_video': _ValidateBool,
+    'resign': _ValidateBool,
+    'robo_directives': _ValidateKeyValueStringPairs,
+    'scenario_labels': ValidateStringList,
+    'scenario_numbers': _ValidatePositiveIntList,
+    'test_targets': ValidateStringList,
+    'timeout': _ValidateDuration,
+    'timeout_us': _ValidateDurationUs,
+    'use_orchestrator': _ValidateBool,
+}
+
+
+def InternalArgNameFrom(arg_external_name):
+  """Converts a user-visible arg name into its corresponding internal name."""
+  if arg_external_name == 'async':
+    # The async flag has a special destination in the argparse namespace since
+    # 'async' is a reserved keyword as of Python 3.7.
+    return 'async_'
+  return arg_external_name.replace('-', '_')
+
+
+# Validation methods below this point are meant to be used on args regardless
+# of whether they came from the command-line or an arg-file, while the methods
+# above here are only for arg-file args, which bypass the standard validations
+# performed by the argparse package (which only works with CLI args).
+
+
+def ValidateArgsForTestType(args, test_type, type_rules, shared_rules,
+                            all_test_args_set):
+  """Raise errors if required args are missing or invalid args are present.
+
+  Args:
+    args: an argparse.Namespace object which contains attributes for all the
+      arguments that were provided to the command invocation (i.e. command
+      group and command arguments combined).
+    test_type: string containing the type of test to run.
+    type_rules: a nested dictionary defining the required and optional args
+      per type of test, plus any default values.
+    shared_rules: a nested dictionary defining the required and optional args
+      shared among all test types, plus any default values.
+    all_test_args_set: a set of strings for every gcloud-test argument to use
+      for validation.
+
+  Raises:
+    InvalidArgException: If an arg doesn't pair with the test type.
+    RequiredArgumentException: If a required arg for the test type is missing.
+  """
+  required_args = type_rules[test_type]['required'] + shared_rules['required']
+  optional_args = type_rules[test_type]['optional'] + shared_rules['optional']
+  allowable_args_for_type = required_args + optional_args
+
+  # Raise an error if an optional test arg is not allowed with this test_type.
+  for arg in all_test_args_set:
+    if getattr(args, arg, None) is not None:  # Ignore args equal to None
+      if arg not in allowable_args_for_type:
+        raise test_exceptions.InvalidArgException(
+            arg, 'may not be used with test type [{0}].'.format(test_type))
+  # Raise an error if a required test arg is missing or equal to None.
+  for arg in required_args:
+    if getattr(args, arg, None) is None:
+      raise exceptions.RequiredArgumentException(
+          '{0}'.format(test_exceptions.ExternalArgNameFrom(arg)),
+          'must be specified with test type [{0}].'.format(test_type))
+
+
+def ValidateResultsBucket(args):
+  """Do some basic sanity checks on the format of the results-bucket arg.
+
+  Args:
+    args: the argparse.Namespace containing all the args for the command.
+
+  Raises:
+    InvalidArgumentException: the bucket name is not valid or includes objects.
+  """
+  if args.results_bucket is None:
+    return
+  try:
+    bucket_ref = storage_util.BucketReference.FromArgument(args.results_bucket,
+                                                           require_prefix=False)
+  except Exception as err:
+    raise exceptions.InvalidArgumentException('results-bucket',
+                                              six.text_type(err))
+  args.results_bucket = bucket_ref.bucket
+
+
+def ValidateResultsDir(args):
+  """Sanity checks the results-dir arg and apply a default value if needed.
+
+  Args:
+    args: the argparse.Namespace containing all the args for the command.
+
+  Raises:
+    InvalidArgumentException: the arg value is not a valid cloud storage name.
+  """
+  if not args.results_dir:
+    args.results_dir = _GenerateUniqueGcsObjectName()
+    return
+
+  args.results_dir = args.results_dir.rstrip('/')
+  # See https://cloud.google.com/storage/docs/naming#objectnames for details.
+  if '\n' in args.results_dir or '\r' in args.results_dir:
+    raise exceptions.InvalidArgumentException(
+        'results-dir', 'Name may not contain newline or linefeed characters')
+  # Leave half of the max GCS object name length of 1024 for the backend to use.
+  if len(args.results_dir) > 512:
+    raise exceptions.InvalidArgumentException('results-dir', 'Name is too long')
+
+
+def _GenerateUniqueGcsObjectName():
+  """Create a unique GCS object name to hold test results in the results bucket.
+
+  The Testing back-end needs a unique GCS object name within the results bucket
+  to prevent race conditions while processing test results. By default, the
+  gcloud client uses the current time down to the microsecond in ISO format plus
+  a random 4-letter suffix. The format is: "YYYY-MM-DD_hh:mm:ss.ssssss_rrrr".
+
+  Returns:
+    A string with the unique GCS object name.
+  """
+  # In PY2, isoformat() argument 1 needs a char. But in PY3 it needs unicode.
+  return '{0}_{1}'.format(
+      datetime.datetime.now().isoformat(b'_' if six.PY2 else '_'), ''.join(
+          random.sample(string.ascii_letters, 4)))
+
+
+def ValidateOsVersions(args, catalog_mgr):
+  """Validate os-version-ids strings against the TestEnvironmentCatalog.
+
+  Also allow users to alternatively specify OS version strings (e.g. '5.1.x')
+  but translate them here to their corresponding version IDs (e.g. '22').
+  The final list of validated version IDs is sorted in ascending order.
+
+  Args:
+    args: an argparse namespace. All the arguments that were provided to the
+      command invocation (i.e. group and command arguments combined).
+    catalog_mgr: an AndroidCatalogManager object for working with the Android
+      TestEnvironmentCatalog.
+  """
+  if not args.os_version_ids:
+    return
+  validated_versions = set()  # Using a set will remove duplicates
+  for vers in args.os_version_ids:
+    version_id = catalog_mgr.ValidateDimensionAndValue('version', vers)
+    validated_versions.add(version_id)
+  args.os_version_ids = sorted(validated_versions)
+
+
+def ValidateXcodeVersion(args, catalog_mgr):
+  """Validates an Xcode version string against the TestEnvironmentCatalog."""
+  if args.xcode_version:
+    catalog_mgr.ValidateXcodeVersion(args.xcode_version)
+
+
+_OBB_FILE_REGEX = re.compile(
+    r'(.*[\\/:])?(main|patch)\.\d+(\.[a-zA-Z]\w*)+\.obb$')
+
+
+def NormalizeAndValidateObbFileNames(obb_files):
+  """Confirm that any OBB file names follow the required Android pattern.
+
+  Also expand local paths with "~"
+
+  Args:
+    obb_files: list of obb file references. Each one is either a filename on the
+      local FS or a gs:// reference.
+  """
+  if obb_files:
+    obb_files[:] = [
+        obb_file if not obb_file or
+        obb_file.startswith(storage_util.GSUTIL_BUCKET_PREFIX) else
+        files.ExpandHomeDir(obb_file) for obb_file in obb_files
+    ]
+  for obb_file in (obb_files or []):
+    if not _OBB_FILE_REGEX.match(obb_file):
+      raise test_exceptions.InvalidArgException(
+          'obb_files',
+          '[{0}] is not a valid OBB file name, which must have the format: '
+          '(main|patch).<versionCode>.<package.name>.obb'.format(obb_file))
+
+
+def ValidateRoboDirectivesList(args):
+  """Validates key-value pairs for 'robo_directives' flag."""
+  resource_names = set()
+  duplicates = set()
+  for key, value in six.iteritems((args.robo_directives or {})):
+    (action_type, resource_name) = util.ParseRoboDirectiveKey(key)
+    if action_type in ['click', 'ignore'] and value:
+      raise test_exceptions.InvalidArgException(
+          'robo_directives',
+          'Input value not allowed for click or ignore actions: [{0}={1}]'
+          .format(key, value))
+
+    # Validate resource_name validity
+    if not resource_name:
+      raise test_exceptions.InvalidArgException(
+          'robo_directives', 'Missing resource_name for key [{0}].'.format(key))
+
+    # Validate resource name uniqueness
+    if resource_name in resource_names:
+      duplicates.add(resource_name)
+    else:
+      resource_names.add(resource_name)
+
+  if duplicates:
+    raise test_exceptions.InvalidArgException(
+        'robo_directives',
+        'Duplicate resource names are not allowed: [{0}].'.format(
+            ', '.join(duplicates)))
+
+
+_ENVIRONMENT_VARIABLE_REGEX = re.compile(r'^[a-zA-Z][\w.-]+$')
+
+
+def ValidateEnvironmentVariablesList(args):
+  """Validates key-value pairs for 'environment-variables' flag."""
+  for key in (args.environment_variables or []):
+    # Check for illegal characters in the key.
+    if not _ENVIRONMENT_VARIABLE_REGEX.match(key):
+      raise test_exceptions.InvalidArgException(
+          'environment_variables',
+          'Invalid environment variable [{0}]'.format(key))
+
+
+_DIRECTORIES_TO_PULL_PATH_REGEX = re.compile(
+    r'^/?/(?:sdcard|data/local/tmp)(?:/[\w\-\.\+ /]+)*$')
+
+
+def NormalizeAndValidateDirectoriesToPullList(dirs):
+  """Validate list of file paths for 'directories-to-pull' flag.
+
+  Also collapse paths to remove "." ".." and "//".
+
+  Args:
+    dirs: list of directory names to pull from the device.
+  """
+  if dirs:
+    # Expand posix paths (Note: blank entries will fail in the next loop)
+    dirs[:] = [posixpath.abspath(path) if path else path for path in dirs]
+
+  for file_path in (dirs or []):
+    # Check for correct file path format.
+    if not _DIRECTORIES_TO_PULL_PATH_REGEX.match(file_path):
+      raise test_exceptions.InvalidArgException(
+          'directories_to_pull', 'Invalid path [{0}]'.format(file_path))
+
+
+_PACKAGE_OR_CLASS_FOLLOWED_BY_COMMA = \
+  re.compile(r'.*,(|\s+)(package |class ).*')
+
+_ANY_SPACE_AFTER_COMMA = re.compile(r'.*,(\s+).*')
+
+
+def ValidateTestTargetsForShard(args):
+  """Validates --test-targets-for-shard uses proper delimiter."""
+  if not getattr(args, 'test_targets_for_shard', {}):
+    return
+  for test_target in args.test_targets_for_shard:
+    if _PACKAGE_OR_CLASS_FOLLOWED_BY_COMMA.match(test_target):
+      raise test_exceptions.InvalidArgException(
+          'test_targets_for_shard',
+          '[{0}] is not a valid test_targets_for_shard argument. Multiple '
+          '"package" and "class" specifications should be separated by '
+          'a semicolon instead of a comma.'.format(test_target))
+    if _ANY_SPACE_AFTER_COMMA.match(test_target):
+      raise test_exceptions.InvalidArgException(
+          'test_targets_for_shard',
+          '[{0}] is not a valid test_targets_for_shard argument. No white '
+          'space is allowed after a comma.'.format(test_target))
+
+
+def ValidateScenarioNumbers(args):
+  """Validates list of game-loop scenario numbers, which must be > 0."""
+  if args.type != 'game-loop' or not args.scenario_numbers:
+    return
+  args.scenario_numbers = [_ValidatePositiveInteger('scenario_numbers', num)
+                           for num in args.scenario_numbers]
+
+
+def ValidateDeviceList(args, catalog_mgr):
+  """Validates that --device contains a valid set of dimensions and values."""
+  if not args.device:
+    return
+
+  for device_spec in args.device:
+    for (dim, val) in device_spec.items():
+      device_spec[dim] = catalog_mgr.ValidateDimensionAndValue(dim, val)
+
+    # Fill in any missing dimensions with default dimension values
+    if 'model' not in device_spec:
+      device_spec['model'] = catalog_mgr.GetDefaultModel()
+    if 'version' not in device_spec:
+      device_spec['version'] = catalog_mgr.GetDefaultVersion()
+    if 'locale' not in device_spec:
+      device_spec['locale'] = catalog_mgr.GetDefaultLocale()
+    if 'orientation' not in device_spec:
+      device_spec['orientation'] = catalog_mgr.GetDefaultOrientation()
+
+
+_IOS_DIRECTORIES_TO_PULL_PATH_REGEX = re.compile(
+    r'^(/private/var/mobile/Media.*|[a-zA-Z0-9.-]+:/Documents.*)')
+
+
+def ValidateIosDirectoriesToPullList(args):
+  if not getattr(args, 'directories_to_pull', []):
+    return
+  for file_path in args.directories_to_pull:
+    if not _IOS_DIRECTORIES_TO_PULL_PATH_REGEX.match(file_path):
+      raise test_exceptions.InvalidArgException(
+          'directories_to_pull', 'Invalid path [{0}]'.format(file_path))

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/ctrl_c_handler.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Context manager to help with Control-C handling during critical commands."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import signal
+
+from googlecloudsdk.api_lib.firebase.test import exit_code
+from googlecloudsdk.calliope import exceptions
+from googlecloudsdk.core import log
+
+
+class CancellableTestSection(object):
+  """Cancel a test matrix if CTRL-C is typed during a section of code.
+
+  While within this context manager, the CTRL-C signal is caught and a test
+  matrix is cancelled. This should only be used with a section of code where
+  the test matrix is running.
+  """
+
+  def __init__(self, matrix_monitor):
+    self._old_sigint_handler = None
+    self._old_sigterm_handler = None
+    self._matrix_monitor = matrix_monitor
+
+  def __enter__(self):
+    self._old_sigint_handler = signal.getsignal(signal.SIGINT)
+    self._old_sigterm_handler = signal.getsignal(signal.SIGTERM)
+    signal.signal(signal.SIGINT, self._Handler)
+    signal.signal(signal.SIGTERM, self._Handler)
+    return self
+
+  def __exit__(self, typ, value, traceback):
+    signal.signal(signal.SIGINT, self._old_sigint_handler)
+    signal.signal(signal.SIGTERM, self._old_sigterm_handler)
+    return False
+
+  def _Handler(self, unused_signal, unused_frame):
+    log.status.write('\n\nCancelling test [{id}]...\n\n'
+                     .format(id=self._matrix_monitor.matrix_id))
+    self._matrix_monitor.CancelTestMatrix()
+    log.status.write('\nTest matrix has been cancelled.\n')
+    raise exceptions.ExitCodeNoError(exit_code=exit_code.MATRIX_CANCELLED)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/device_capacities.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2022 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Common code for 'gcloud firebase test * list-device-capacities' commands."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import collections
+
+CapacityEntry = collections.namedtuple('CapacityEntry',
+                                       ['model', 'name', 'version', 'capacity'])
+
+DEVICE_CAPACITY_TABLE_FORMAT = """
+        table[box](
+          model:label=MODEL_ID,
+          name:label=MODEL_NAME,
+          version:label=OS_VERSION_ID,
+          capacity.color(red=None,yellow=Low,green=High):label=DEVICE_CAPACITY
+        )
+    """
+
+
+class DeviceCapacities(object):
+  """Common code for 'gcloud firebase test * list-device-capacities' commands."""
+
+  _capacity_messages_cache = None
+
+  @property
+  def capacity_messages(self):
+    """A map of enum to user-friendly message."""
+
+    if self._capacity_messages_cache is None:
+      device_capacity_enum_android = self.context[
+          'testing_messages'].PerAndroidVersionInfo.DeviceCapacityValueValuesEnum
+      # Note: PerIosVersionInfo.DeviceCapacityValueValuesEnum is the same enum
+      #  in the proto, but in gcloud client API it's an unique object
+      device_capacity_enum_ios = self.context[
+          'testing_messages'].PerIosVersionInfo.DeviceCapacityValueValuesEnum
+      self._capacity_messages_cache = {
+          device_capacity_enum_android.DEVICE_CAPACITY_UNSPECIFIED: 'None',
+          device_capacity_enum_android.DEVICE_CAPACITY_HIGH: 'High',
+          device_capacity_enum_android.DEVICE_CAPACITY_MEDIUM: 'Medium',
+          device_capacity_enum_android.DEVICE_CAPACITY_LOW: 'Low',
+          device_capacity_enum_android.DEVICE_CAPACITY_NONE: 'None',
+          device_capacity_enum_ios.DEVICE_CAPACITY_UNSPECIFIED: 'None',
+          device_capacity_enum_ios.DEVICE_CAPACITY_HIGH: 'High',
+          device_capacity_enum_ios.DEVICE_CAPACITY_MEDIUM: 'Medium',
+          device_capacity_enum_ios.DEVICE_CAPACITY_LOW: 'Low',
+          device_capacity_enum_ios.DEVICE_CAPACITY_NONE: 'None',
+      }
+
+    return self._capacity_messages_cache
+
+  def get_capacity_data(self, catalog):
+    """Generate a list of devices/OS versions & corresponding capacity info.
+
+    Args:
+      catalog: Android or iOS catalog
+
+    Returns:
+      The list of device models, versions, and capacity info we want to have
+      printed later. Obsolete (unsupported) devices, versions, and entries
+      missing capacity info are filtered out.
+    """
+
+    capacity_data = []
+    for model in catalog.models:
+      for version_info in model.perVersionInfo:
+        if version_info.versionId not in model.supportedVersionIds:
+          continue
+        if version_info.deviceCapacity is None:
+          continue
+        capacity_data.append(
+            CapacityEntry(
+                model=model.id,
+                name=model.name,
+                version=version_info.versionId,
+                capacity=self.capacity_messages[version_info.deviceCapacity]))
+
+    return capacity_data

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/endpoints.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Library for working with Firebase Test Lab service endpoints."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.firebase.test import exceptions
+from googlecloudsdk.core import log
+from googlecloudsdk.core import properties
+
+
+def ValidateTestServiceEndpoints():
+  """Ensure that test-service endpoints are compatible with each other.
+
+  A staging/test ToolResults API endpoint will not work correctly with a
+  production Testing API endpoint, and vice versa. This check is only relevant
+  for internal development.
+
+  Raises:
+    IncompatibleApiEndpointsError if the endpoints are not compatible.
+  """
+  testing_url = properties.VALUES.api_endpoint_overrides.testing.Get()
+  toolresults_url = properties.VALUES.api_endpoint_overrides.toolresults.Get()
+  log.info('Test Service endpoint: [{0}]'.format(testing_url))
+  log.info('Tool Results endpoint: [{0}]'.format(toolresults_url))
+  if ((toolresults_url is None or 'https://www.googleapis' in toolresults_url or
+       'https://toolresults' in toolresults_url) !=
+      (testing_url is None or 'https://testing' in testing_url)):
+    raise exceptions.IncompatibleApiEndpointsError(
+        testing_url, toolresults_url)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/exceptions.py

@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Exceptions raised by Testing API libs or commands."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.firebase.test import exit_code
+from googlecloudsdk.calliope import exceptions as calliope_exceptions
+from googlecloudsdk.core import exceptions as core_exceptions
+
+
+class TestingError(core_exceptions.Error):
+  """Base class for testing failures."""
+
+
+class MissingProjectError(TestingError):
+  """No GCP project was specified for a command."""
+
+
+class BadMatrixError(TestingError):
+  """BadMatrixException is for test matrices that fail prematurely."""
+
+
+class RestrictedServiceError(TestingError):
+  """RestrictedServiceError is for bad service request errors.
+
+  This is most likely due to allowlisted API features which are hidden behind a
+  visibility label.
+  """
+
+
+class ModelNotFoundError(TestingError):
+  """Failed to find a device model in the test environment catalog."""
+
+  def __init__(self, model_id):
+    super(ModelNotFoundError, self).__init__(
+        "'{id}' is not a valid model".format(id=model_id))
+
+
+class VersionNotFoundError(TestingError):
+  """Failed to find an OS version in the test environment catalog."""
+
+  def __init__(self, version):
+    super(VersionNotFoundError, self).__init__(
+        "'{v}' is not a valid OS version".format(v=version))
+
+
+class NetworkProfileNotFoundError(TestingError):
+  """Failed to find a network profile in the test environment catalog."""
+
+  def __init__(self, profile_id):
+    super(NetworkProfileNotFoundError, self).__init__(
+        "Could not find network profile ID '{id}'".format(id=profile_id))
+
+
+class LocaleNotFoundError(TestingError):
+  """Failed to find a locale in the test environment catalog."""
+
+  def __init__(self, locale):
+    super(LocaleNotFoundError, self).__init__(
+        "'{l}' is not a valid locale".format(l=locale))
+
+
+class OrientationNotFoundError(TestingError):
+  """Failed to find an orientation in the test environment catalog."""
+
+  def __init__(self, orientation):
+    super(OrientationNotFoundError, self).__init__(
+        "'{o}' is not a valid device orientation".format(o=orientation))
+
+
+class DefaultDimensionNotFoundError(TestingError):
+  """Failed to find a 'default' tag on any value for a device dimension."""
+
+  def __init__(self, dim_name):
+    super(DefaultDimensionNotFoundError, self).__init__(
+        "Test Lab did not provide a default value for '{d}'".format(d=dim_name))
+
+
+class InvalidDimensionNameError(TestingError):
+  """An invalid test matrix dimension name was encountered."""
+
+  def __init__(self, dim_name):
+    super(InvalidDimensionNameError, self).__init__(
+        "'{d}' is not a valid dimension name. Must be one of: "
+        "['model', 'version', 'locale', 'orientation']".format(d=dim_name))
+
+
+class XcodeVersionNotFoundError(TestingError):
+  """Failed to find an Xcode version in the test environment catalog."""
+
+  def __init__(self, version):
+    super(XcodeVersionNotFoundError, self).__init__(
+        "'{v}' is not a supported Xcode version".format(v=version))
+
+
+class TestExecutionNotFoundError(TestingError):
+  """A test execution ID was not found within a test matrix."""
+
+  def __init__(self, execution_id, matrix_id):
+    super(TestExecutionNotFoundError, self).__init__(
+        'Test execution [{e}] not found in matrix [{m}].'
+        .format(e=execution_id, m=matrix_id))
+
+
+class IncompatibleApiEndpointsError(TestingError):
+  """Two or more API endpoint overrides are incompatible with each other."""
+
+  def __init__(self, endpoint1, endpoint2):
+    super(IncompatibleApiEndpointsError, self).__init__(
+        'Service endpoints [{0}] and [{1}] are not compatible.'
+        .format(endpoint1, endpoint2))
+
+
+class InvalidTestArgError(TestingError):
+  """An invalid/unknown test argument was found in an argument file."""
+
+  def __init__(self, arg_name):
+    super(InvalidTestArgError, self).__init__(
+        '[{0}] is not a valid argument name for: gcloud test run.'
+        .format(arg_name))
+
+
+class TestLabInfrastructureError(TestingError):
+  """Encountered a Firebase Test Lab infrastructure error during testing."""
+
+  def __init__(self, error):
+    super(TestLabInfrastructureError, self).__init__(
+        'Firebase Test Lab infrastructure failure: {0}'.format(error),
+        exit_code=exit_code.INFRASTRUCTURE_ERR)
+
+
+class AllDimensionsIncompatibleError(TestingError):
+  """All device dimensions in a test matrix are incompatible."""
+
+  def __init__(self, msg):
+    super(AllDimensionsIncompatibleError, self).__init__(
+        msg, exit_code=exit_code.UNSUPPORTED_ENV)
+
+
+def ExternalArgNameFrom(arg_internal_name):
+  """Converts an internal arg name into its corresponding user-visible name.
+
+  This is used for creating exceptions using user-visible arg names.
+
+  Args:
+    arg_internal_name: the internal name of an argument.
+
+  Returns:
+    The user visible name for the argument.
+  """
+  if arg_internal_name == 'async_':
+    # The async flag has a special destination in the argparse namespace since
+    # 'async' is a reserved keyword as of Python 3.7.
+    return 'async'
+  return arg_internal_name.replace('_', '-')
+
+
+class InvalidArgException(calliope_exceptions.InvalidArgumentException):
+  """InvalidArgException is for malformed gcloud firebase test argument values.
+
+  It provides a wrapper around Calliope's InvalidArgumentException that
+  conveniently converts internal arg names with underscores into the external
+  arg names with hyphens.
+  """
+
+  def __init__(self, param_name, message):
+    super(InvalidArgException, self).__init__(
+        ExternalArgNameFrom(param_name), message)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/exit_code.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module to define and determine exit codes for 'gcloud test' commands.
+
+Note: Cloud-SDK-eng is reserving exit codes 1..9 for http errors, invalid args,
+bad filename, etc. Gcloud command surfaces are free to use exit codes 10..20.
+Gaps in exit_code numbering are left in case future expansion is needed.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.core import exceptions as core_exceptions
+from googlecloudsdk.core import log
+
+
+ROLLUP_SUCCESS = 0       # Every test case within an execution passed.
+ROLLUP_FAILURE = 10      # One or more test cases did not pass.
+INCONCLUSIVE = 15        # The test pass/fail status was not determined.
+UNSUPPORTED_ENV = 18     # The specified test environment is not supported.
+MATRIX_CANCELLED = 19    # The matrix execution was cancelled by the user.
+INFRASTRUCTURE_ERR = 20  # An infrastructure error occurred.
+
+
+class TestOutcomeError(core_exceptions.Error):
+  """The Tool Results backend did not return a valid test outcome."""
+
+  def __init__(self, msg):
+    super(TestOutcomeError, self).__init__(msg, exit_code=INFRASTRUCTURE_ERR)
+
+
+def ExitCodeFromRollupOutcome(outcome, summary_enum):
+  """Map a test roll-up outcome into the appropriate gcloud test exit_code.
+
+  Args:
+    outcome: a toolresults_v1.Outcome message.
+    summary_enum: a toolresults.Outcome.SummaryValueValuesEnum reference.
+
+  Returns:
+    The exit_code which corresponds to the test execution's rolled-up outcome.
+
+  Raises:
+    TestOutcomeError: If Tool Results service returns an invalid outcome value.
+  """
+  if not outcome or not outcome.summary:
+    log.warning('Tool Results service did not provide a roll-up test outcome.')
+    return INCONCLUSIVE
+  if (outcome.summary == summary_enum.success
+      or outcome.summary == summary_enum.flaky):
+    return ROLLUP_SUCCESS
+  if outcome.summary == summary_enum.failure:
+    return ROLLUP_FAILURE
+  if outcome.summary == summary_enum.skipped:
+    return UNSUPPORTED_ENV
+  if outcome.summary == summary_enum.inconclusive:
+    return INCONCLUSIVE
+  raise TestOutcomeError(
+      "Unknown test outcome summary value '{0}'".format(outcome.summary))

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/history_picker.py

@@ -0,0 +1,119 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A library to find a Tool Results History to publish results to."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from apitools.base.py import exceptions as apitools_exceptions
+
+from googlecloudsdk.api_lib.firebase.test import util
+from googlecloudsdk.calliope import exceptions
+from googlecloudsdk.core import log
+
+
+class ToolResultsHistoryPicker(object):
+  """Finds a History to publish mobile test results to.
+  """
+
+  def __init__(self, project, client, messages):
+    """Construct a ToolResultsHistoryPicker.
+
+    Args:
+      project: string containing the GCE project id.
+      client: ToolResults API client lib generated by apitools.
+      messages: ToolResults API message classes generated by apitools.
+    """
+    self._project = project
+    self._client = client
+    self._messages = messages
+
+  def _ListHistoriesByName(self, history_name, page_size):
+    """Lists histories by name using the Tool Results API.
+
+    Args:
+       history_name: string containing the history name.
+       page_size: maximum number of histories to return.
+
+    Returns:
+      A list of histories matching the name.
+
+    Raises:
+      HttpException if the Tool Results service reports a backend error.
+    """
+    request = self._messages.ToolresultsProjectsHistoriesListRequest(
+        projectId=self._project, filterByName=history_name, pageSize=page_size)
+    try:
+      response = self._client.projects_histories.List(request)
+      log.debug('\nToolResultsHistories.List response:\n{0}\n'.format(response))
+      return response
+    except apitools_exceptions.HttpError as error:
+      msg = ('Http error while getting list of Tool Results Histories:\n{0}'
+             .format(util.GetError(error)))
+      raise exceptions.HttpException(msg)
+
+  def _CreateHistory(self, history_name):
+    """Creates a History using the Tool Results API.
+
+    Args:
+       history_name: string containing the name of the history to create.
+
+    Returns:
+      The history id of the created history.
+
+    Raises:
+      HttpException if the Tool Results service reports a backend error.
+    """
+    history = self._messages.History(name=history_name,
+                                     displayName=history_name)
+    request = self._messages.ToolresultsProjectsHistoriesCreateRequest(
+        projectId=self._project, history=history)
+    try:
+      response = self._client.projects_histories.Create(request)
+      log.debug('\nToolResultsHistories.Create response:\n{0}\n'
+                .format(response))
+      return response
+    except apitools_exceptions.HttpError as error:
+      msg = ('Http error while creating a Tool Results History:\n{0}'
+             .format(util.GetError(error)))
+      raise exceptions.HttpException(msg)
+
+  def GetToolResultsHistoryId(self, history_name):
+    """Gets the history id associated with a given history name.
+
+    All the test executions for the same app should be in the same history. This
+    method will try to find an existing history for the application or create
+    one if this is the first time a particular history_name has been seen.
+
+    Args:
+       history_name: string containing the history's name (if the user supplied
+         one), else None.
+
+    Returns:
+      The id of the history to publish results to.
+    """
+    if not history_name:
+      return None
+
+    # There might be several histories with the same name. We only fetch the
+    # first history which is the history with the most recent results.
+    histories = self._ListHistoriesByName(history_name, 1).histories
+    if histories:
+      return histories[0].historyId
+    else:
+      new_history = self._CreateHistory(history_name)
+      return new_history.historyId

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/ios/__init__.py

@@ -0,0 +1 @@
+

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/ios/arg_manager.py

@@ -0,0 +1,187 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A shared library for processing and validating iOS test arguments."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.firebase.test import arg_file
+from googlecloudsdk.api_lib.firebase.test import arg_util
+from googlecloudsdk.api_lib.firebase.test import arg_validate
+from googlecloudsdk.api_lib.firebase.test.ios import catalog_manager
+from googlecloudsdk.calliope import exceptions
+
+
+def TypedArgRules():
+  """Returns the rules for iOS test args which depend on the test type.
+
+  This dict is declared in a function rather than globally to avoid garbage
+  collection issues during unit tests.
+
+  Returns:
+    A dict keyed by whether type-specific args are required or optional, and
+    with a nested dict containing any default values for those shared args.
+  """
+  return {
+      'xctest': {
+          'required': ['test'],
+          'optional': [
+              'xcode_version',
+              'xctestrun_file',
+              'test_special_entitlements',
+          ],
+          'defaults': {
+              'test_special_entitlements': False
+          }
+      },
+      'game-loop': {
+          'required': ['app'],
+          'optional': ['scenario_numbers'],
+          'defaults': {
+              'scenario_numbers': [1]
+          }
+      },
+      'robo': {
+          'required': ['app'],
+          'optional': ['test_special_entitlements', 'robo_script'],
+          'defaults': {
+              'test_special_entitlements': False
+          },
+      }
+  }
+
+
+def SharedArgRules():
+  """Returns the rules for iOS test args which are shared by all test types.
+
+  This dict is declared in a function rather than globally to avoid garbage
+  collection issues during unit tests.
+
+  Returns:
+    A dict keyed by whether shared args are required or optional, and with a
+    nested dict containing any default values for those shared args.
+  """
+  return {
+      'required': ['type'],
+      'optional': [
+          'additional_ipas',
+          'async_',
+          'client_details',
+          'device',
+          'directories_to_pull',
+          'network_profile',
+          'num_flaky_test_attempts',
+          'other_files',
+          'record_video',
+          'results_bucket',
+          'results_dir',
+          'results_history_name',
+          'timeout',
+      ],
+      'defaults': {
+          'async_': False,
+          'device': [{}],  # Default dimensions will come from the iOS catalog.
+          'num_flaky_test_attempts': 0,
+          'record_video': True,
+          'timeout': 900,  # 15 minutes
+      }
+  }
+
+
+def AllArgsSet():
+  """Returns a set containing the names of every iOS test arg."""
+  return arg_util.GetSetOfAllTestArgs(TypedArgRules(), SharedArgRules())
+
+
+class IosArgsManager(object):
+  """Manages test arguments for iOS devices."""
+
+  def __init__(self,
+               catalog_mgr=None,
+               typed_arg_rules=None,
+               shared_arg_rules=None):
+    """Constructs an IosArgsManager for a single iOS test matrix.
+
+    Args:
+      catalog_mgr: an IosCatalogManager object.
+      typed_arg_rules: a nested dict of dicts which are keyed first on the test
+        type, then by whether args are required or optional, and what their
+        default values are. If None, the default from TypedArgRules() is used.
+      shared_arg_rules: a dict keyed by whether shared args are required or
+        optional, and with a nested dict containing any default values for those
+        shared args. If None, the default dict from SharedArgRules() is used.
+    """
+    self._catalog_mgr = catalog_mgr or catalog_manager.IosCatalogManager()
+    self._typed_arg_rules = typed_arg_rules or TypedArgRules()
+    self._shared_arg_rules = shared_arg_rules or SharedArgRules()
+
+  def Prepare(self, args):
+    """Load, apply defaults, and perform validation on test arguments.
+
+    Args:
+      args: an argparse namespace. All the arguments that were provided to this
+        gcloud command invocation (i.e. group and command arguments combined).
+        Arg values from an optional arg-file and/or arg default values may be
+        added to this argparse namespace.
+
+    Raises:
+      InvalidArgumentException: If an argument name is unknown, an argument does
+        not contain a valid value, or an argument is not valid when used with
+        the given type of test.
+      RequiredArgumentException: If a required arg is missing.
+    """
+    all_test_args_set = arg_util.GetSetOfAllTestArgs(self._typed_arg_rules,
+                                                     self._shared_arg_rules)
+    args_from_file = arg_file.GetArgsFromArgFile(args.argspec,
+                                                 all_test_args_set)
+
+    arg_util.ApplyLowerPriorityArgs(args, args_from_file, True)
+
+    test_type = self.GetTestTypeOrRaise(args)
+    typed_arg_defaults = self._typed_arg_rules[test_type]['defaults']
+    shared_arg_defaults = self._shared_arg_rules['defaults']
+    arg_util.ApplyLowerPriorityArgs(args, typed_arg_defaults)
+    arg_util.ApplyLowerPriorityArgs(args, shared_arg_defaults)
+    arg_validate.ValidateArgsForTestType(args, test_type, self._typed_arg_rules,
+                                         self._shared_arg_rules,
+                                         all_test_args_set)
+    arg_validate.ValidateDeviceList(args, self._catalog_mgr)
+    arg_validate.ValidateXcodeVersion(args, self._catalog_mgr)
+    arg_validate.ValidateResultsBucket(args)
+    arg_validate.ValidateResultsDir(args)
+    arg_validate.ValidateScenarioNumbers(args)
+    arg_validate.ValidateIosDirectoriesToPullList(args)
+
+  def GetTestTypeOrRaise(self, args):
+    """If the test type is not user-specified, infer the most reasonable value.
+
+    Args:
+      args: an argparse namespace. All the arguments that were provided to this
+        gcloud command invocation (i.e. group and command arguments combined).
+
+    Returns:
+      The type of the test to be run (e.g. 'xctest'), and also sets the 'type'
+      arg if it was not user-specified.
+
+    Raises:
+      InvalidArgumentException if an explicit test type is invalid.
+    """
+    if not args.type:
+      args.type = 'xctest'
+    if args.type not in self._typed_arg_rules:
+      raise exceptions.InvalidArgumentException(
+          'type', "'{0}' is not a valid test type.".format(args.type))
+    return args.type

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/ios/catalog_manager.py

@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A wrapper for working with the iOS Test Environment Catalog."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.firebase.test import exceptions
+from googlecloudsdk.api_lib.firebase.test import util
+
+
+_MODEL_DIMENSION = 'model'
+_VERSION_DIMENSION = 'version'
+_LOCALE_DIMENSION = 'locale'
+_ORIENTATION_DIMENSION = 'orientation'
+
+
+class IosCatalogManager(object):
+  """Encapsulates operations on the iOS TestEnvironmentCatalog."""
+
+  def __init__(self, catalog=None):
+    """Construct an IosCatalogManager object from a TestEnvironmentCatalog.
+
+    Args:
+      catalog: an iOS TestEnvironmentCatalog from Testing API. If it is not
+        injected, the catalog is retrieved from the Testing service.
+
+    Attributes:
+      catalog: an iOS TestEnvironmentCatalog.
+    """
+    self.catalog = catalog or util.GetIosCatalog()
+    models = self.catalog.models
+    versions = self.catalog.versions
+    locales = self.catalog.runtimeConfiguration.locales
+    orientations = self.catalog.runtimeConfiguration.orientations
+
+    self._model_ids = [m.id for m in models]
+    self._version_ids = [v.id for v in versions]
+    self._locale_ids = [l.id for l in locales]
+    self._orientation_ids = [o.id for o in orientations]
+
+    # Dimension defaults are lazily computed and cached by GetDefault* methods.
+    self._default_model = None
+    self._default_version = None
+    self._default_locale = None
+    self._default_orientation = None
+
+  def GetDefaultModel(self):
+    """Return the default model listed in the iOS environment catalog."""
+    model = (self._default_model if self._default_model else
+             self._FindDefaultDimension(self.catalog.models))
+    if not model:
+      raise exceptions.DefaultDimensionNotFoundError(_MODEL_DIMENSION)
+    return model
+
+  def GetDefaultVersion(self):
+    """Return the default version listed in the iOS environment catalog."""
+    version = (self._default_version if self._default_version else
+               self._FindDefaultDimension(self.catalog.versions))
+    if not version:
+      raise exceptions.DefaultDimensionNotFoundError(_VERSION_DIMENSION)
+    return version
+
+  def GetDefaultLocale(self):
+    """Return the default iOS locale."""
+    locales = self.catalog.runtimeConfiguration.locales
+    locale = (
+        self._default_locale
+        if self._default_locale else self._FindDefaultDimension(locales))
+    if not locale:
+      raise exceptions.DefaultDimensionNotFoundError(_LOCALE_DIMENSION)
+    return locale
+
+  def GetDefaultOrientation(self):
+    """Return the default iOS orientation."""
+    orientations = self.catalog.runtimeConfiguration.orientations
+    orientation = (
+        self._default_orientation if self._default_orientation else
+        self._FindDefaultDimension(orientations))
+    if not orientation:
+      raise exceptions.DefaultDimensionNotFoundError(_ORIENTATION_DIMENSION)
+    return orientation
+
+  def _FindDefaultDimension(self, dimension_table):
+    for dimension in dimension_table:
+      if 'default' in dimension.tags:
+        return dimension.id
+    return None
+
+  def ValidateDimensionAndValue(self, dim_name, dim_value):
+    """Validates that a matrix dimension has a valid name and value."""
+    if dim_name == _MODEL_DIMENSION:
+      if dim_value not in self._model_ids:
+        raise exceptions.ModelNotFoundError(dim_value)
+    elif dim_name == _VERSION_DIMENSION:
+      if dim_value not in self._version_ids:
+        raise exceptions.VersionNotFoundError(dim_value)
+    elif dim_name == _LOCALE_DIMENSION:
+      if dim_value not in self._locale_ids:
+        raise exceptions.LocaleNotFoundError(dim_value)
+    elif dim_name == _ORIENTATION_DIMENSION:
+      if dim_value not in self._orientation_ids:
+        raise exceptions.OrientationNotFoundError(dim_value)
+    else:
+      raise exceptions.InvalidDimensionNameError(dim_name)
+    return dim_value
+
+  def ValidateXcodeVersion(self, xcode_version):
+    """Validates that an Xcode version is in the TestEnvironmentCatalog."""
+    if xcode_version not in [xv.version for xv in self.catalog.xcodeVersions]:
+      raise exceptions.XcodeVersionNotFoundError(xcode_version)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/ios/matrix_creator.py

@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Create iOS test matrices in Firebase Test Lab."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import os
+import uuid
+
+from apitools.base.py import exceptions as apitools_exceptions
+
+from googlecloudsdk.api_lib.firebase.test import matrix_creator_common
+from googlecloudsdk.api_lib.firebase.test import matrix_ops
+from googlecloudsdk.api_lib.firebase.test import util
+from googlecloudsdk.calliope import exceptions
+from googlecloudsdk.core import log
+
+
+def CreateMatrix(args, context, history_id, gcs_results_root, release_track):
+  """Creates a new iOS matrix test in Firebase Test Lab from the user's params.
+
+  Args:
+    args: an argparse namespace. All the arguments that were provided to this
+      gcloud command invocation (i.e. group and command arguments combined).
+    context: {str:obj} dict containing the gcloud command context, which
+      includes the Testing API client+messages libs generated by Apitools.
+    history_id: {str} A history ID to publish Tool Results to.
+    gcs_results_root: the root dir for a matrix within the GCS results bucket.
+    release_track: the release track that the command is invoked from.
+
+  Returns:
+    A TestMatrix object created from the supplied matrix configuration values.
+  """
+  creator = MatrixCreator(args, context, history_id, gcs_results_root,
+                          release_track)
+  return creator.CreateTestMatrix(uuid.uuid4().hex)
+
+
+class MatrixCreator(object):
+  """Creates a single iOS test matrix based on user-supplied test arguments."""
+
+  def __init__(self, args, context, history_id, gcs_results_root,
+               release_track):
+    """Construct an MatrixCreator to be used to create a single test matrix.
+
+    Args:
+      args: an argparse namespace. All the arguments that were provided to this
+        gcloud command invocation (i.e. group and command arguments combined).
+      context: {str:obj} dict containing the gcloud command context, which
+        includes the Testing API client+messages libs generated by Apitools.
+      history_id: {str} A history ID to publish Tool Results to.
+      gcs_results_root: the root dir for a matrix within the GCS results bucket.
+      release_track: the release track that the command is invoked from.
+    """
+    self._project = util.GetProject()
+    self._args = args
+    self._history_id = history_id
+    self._gcs_results_root = gcs_results_root
+    self._client = context['testing_client']
+    self._messages = context['testing_messages']
+    self._release_track = release_track
+
+  def _BuildFileReference(self, filename, use_basename=True):
+    """Build a FileReference pointing to a file in GCS."""
+    if not filename:
+      return None
+    if use_basename:
+      filename = os.path.basename(filename)
+    path = os.path.join(self._gcs_results_root, filename)
+    return self._messages.FileReference(gcsPath=path)
+
+  def _BuildGenericTestSetup(self):
+    """Build an IosTestSetup for an iOS test."""
+    additional_ipas = [
+        self._BuildFileReference(os.path.basename(additional_ipa))
+        for additional_ipa in getattr(self._args, 'additional_ipas', []) or []
+    ]
+    directories_to_pull = []
+    for directory in getattr(self._args, 'directories_to_pull', []) or []:
+      if ':' in directory:
+        bundle, path = directory.split(':')
+        directories_to_pull.append(
+            self._messages.IosDeviceFile(bundleId=bundle, devicePath=path))
+      else:
+        directories_to_pull.append(
+            self._messages.IosDeviceFile(devicePath=directory))
+    device_files = []
+    other_files = getattr(self._args, 'other_files', None) or {}
+    for device_path in other_files.keys():
+      # Device paths are be prefixed by the bundle ID if they refer to an app's
+      # sandboxed filesystem, separated with the device path by ':'
+      idx = device_path.find(':')
+      bundle_id = device_path[:idx] if idx != -1 else None
+      path = device_path[idx + 1:] if idx != -1 else device_path
+      device_files.append(
+          self._messages.IosDeviceFile(
+              content=self._BuildFileReference(
+                  util.GetRelativeDevicePath(path), use_basename=False),
+              bundleId=bundle_id,
+              devicePath=path))
+    return self._messages.IosTestSetup(
+        networkProfile=getattr(self._args, 'network_profile', None),
+        additionalIpas=additional_ipas,
+        pushFiles=device_files,
+        pullDirectories=directories_to_pull)
+
+  def _BuildIosXcTestSpec(self):
+    """Build a TestSpecification for an IosXcTest."""
+    spec = self._messages.TestSpecification(
+        disableVideoRecording=not self._args.record_video,
+        iosTestSetup=self._BuildGenericTestSetup(),
+        testTimeout=matrix_ops.ReformatDuration(self._args.timeout),
+        iosXcTest=self._messages.IosXcTest(
+            testsZip=self._BuildFileReference(self._args.test),
+            xctestrun=self._BuildFileReference(self._args.xctestrun_file),
+            xcodeVersion=self._args.xcode_version,
+            testSpecialEntitlements=getattr(self._args,
+                                            'test_special_entitlements',
+                                            False)))
+    return spec
+
+  def _BuildIosTestLoopTestSpec(self):
+    """Build a TestSpecification for an IosXcTest."""
+    spec = self._messages.TestSpecification(
+        disableVideoRecording=not self._args.record_video,
+        iosTestSetup=self._BuildGenericTestSetup(),
+        testTimeout=matrix_ops.ReformatDuration(self._args.timeout),
+        iosTestLoop=self._messages.IosTestLoop(
+            appIpa=self._BuildFileReference(self._args.app),
+            scenarios=self._args.scenario_numbers))
+    return spec
+
+  def _BuildIosRoboTestSpec(self):
+    """Build a TestSpecification for an iOS Robo test."""
+    spec = self._messages.TestSpecification(
+        disableVideoRecording=not self._args.record_video,
+        iosTestSetup=self._BuildGenericTestSetup(),
+        testTimeout=matrix_ops.ReformatDuration(self._args.timeout),
+        iosRoboTest=self._messages.IosRoboTest(
+            appIpa=self._BuildFileReference(self._args.app)))
+    if getattr(self._args, 'robo_script', None):
+      spec.iosRoboTest.roboScript = self._BuildFileReference(
+          os.path.basename(self._args.robo_script))
+    return spec
+
+  def _TestSpecFromType(self, test_type):
+    """Map a test type into its corresponding TestSpecification message ."""
+    if test_type == 'xctest':
+      return self._BuildIosXcTestSpec()
+    elif test_type == 'game-loop':
+      return self._BuildIosTestLoopTestSpec()
+    elif test_type == 'robo':
+      return self._BuildIosRoboTestSpec()
+    else:  # It's a bug in our arg validation if we ever get here.
+      raise exceptions.InvalidArgumentException(
+          'type', 'Unknown test type "{}".'.format(test_type))
+
+  def _BuildTestMatrix(self, spec):
+    """Build just the user-specified parts of an iOS TestMatrix message.
+
+    Args:
+      spec: a TestSpecification message corresponding to the test type.
+
+    Returns:
+      A TestMatrix message.
+    """
+    devices = [self._BuildIosDevice(d) for d in self._args.device]
+    environment_matrix = self._messages.EnvironmentMatrix(
+        iosDeviceList=self._messages.IosDeviceList(iosDevices=devices))
+
+    gcs = self._messages.GoogleCloudStorage(gcsPath=self._gcs_results_root)
+    hist = self._messages.ToolResultsHistory(
+        projectId=self._project, historyId=self._history_id)
+    results = self._messages.ResultStorage(
+        googleCloudStorage=gcs, toolResultsHistory=hist)
+
+    client_info = matrix_creator_common.BuildClientInfo(
+        self._messages,
+        getattr(self._args, 'client_details', {}) or {}, self._release_track)
+
+    return self._messages.TestMatrix(
+        testSpecification=spec,
+        environmentMatrix=environment_matrix,
+        clientInfo=client_info,
+        resultStorage=results,
+        flakyTestAttempts=self._args.num_flaky_test_attempts or 0)
+
+  def _BuildIosDevice(self, device_map):
+    return self._messages.IosDevice(
+        iosModelId=device_map['model'],
+        iosVersionId=device_map['version'],
+        locale=device_map['locale'],
+        orientation=device_map['orientation'])
+
+  def _BuildTestMatrixRequest(self, request_id):
+    """Build a TestingProjectsTestMatricesCreateRequest for a test matrix.
+
+    Args:
+      request_id: {str} a unique ID for the CreateTestMatrixRequest.
+
+    Returns:
+      A TestingProjectsTestMatricesCreateRequest message.
+    """
+    spec = self._TestSpecFromType(self._args.type)
+    return self._messages.TestingProjectsTestMatricesCreateRequest(
+        projectId=self._project,
+        testMatrix=self._BuildTestMatrix(spec),
+        requestId=request_id)
+
+  def CreateTestMatrix(self, request_id):
+    """Invoke the Testing service to create a test matrix from the user's args.
+
+    Args:
+      request_id: {str} a unique ID for the CreateTestMatrixRequest.
+
+    Returns:
+      The TestMatrix response message from the TestMatrices.Create rpc.
+
+    Raises:
+      HttpException if the test service reports an HttpError.
+    """
+    request = self._BuildTestMatrixRequest(request_id)
+    log.debug('TestMatrices.Create request:\n{0}\n'.format(request))
+    try:
+      response = self._client.projects_testMatrices.Create(request)
+      log.debug('TestMatrices.Create response:\n{0}\n'.format(response))
+    except apitools_exceptions.HttpError as error:
+      msg = 'Http error while creating test matrix: ' + util.GetError(error)
+      raise exceptions.HttpException(msg)
+
+    log.status.Print('Test [{id}] has been created in the Google Cloud.'.format(
+        id=response.testMatrixId))
+    return response

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/matrix_creator_common.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2019 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Shared code to create test matrices in Firebase Test Lab."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.core import config
+import six
+
+
+def BuildClientInfo(messages, client_details, release_track):
+  """Build the ClientInfo part of a TestMatrix message.
+
+  Sets the client name to 'gcloud' and attaches common and user-provided client
+  details to the ClientInfo message.
+
+  Args:
+    messages: Testing API messages generated by Apitools.
+    client_details: Dictionary of user-provided client_details.
+    release_track: Release track that the command is invoked from.
+
+  Returns:
+    ClientInfo message.
+  """
+  details_with_defaults = dict(client_details)
+  details_with_defaults['Cloud SDK Version'] = config.CLOUD_SDK_VERSION
+  details_with_defaults['Release Track'] = release_track
+  client_info_details = []
+
+  for key, value in six.iteritems(details_with_defaults):
+    client_info_details.append(messages.ClientInfoDetail(key=key, value=value))
+
+  # Sort by key to have a predictable order for testing
+  client_info_details.sort(key=lambda d: d.key)
+
+  return messages.ClientInfo(
+      name='gcloud', clientInfoDetails=client_info_details)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/matrix_ops.py

@@ -0,0 +1,340 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Common test matrix operations used by Firebase Test Lab commands."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import collections
+import datetime
+import time
+
+from apitools.base.py import exceptions as apitools_exceptions
+
+from googlecloudsdk.api_lib.firebase.test import exceptions
+from googlecloudsdk.api_lib.firebase.test import util
+from googlecloudsdk.calliope import exceptions as calliope_exceptions
+from googlecloudsdk.core import log
+from googlecloudsdk.core import properties
+from googlecloudsdk.core.console import console_io
+import six
+
+_DEFAULT_STATUS_INTERVAL_SECS = 6.0
+_TIMESTAMP_FORMAT = '%H:%M:%S'
+
+
+class MatrixMonitor(object):
+  """A monitor to follow and possibly cancel a single test matrix invocation.
+
+  Attributes:
+    matrix_id: {str} the unique ID of the matrix being monitored.
+    completed_matrix_states: the set of TestMatrix.State enums representing all
+      final matrix states.
+  """
+
+  def __init__(self,
+               matrix_id,
+               test_type,
+               context,
+               clock=datetime.datetime.now,
+               status_interval_secs=None):
+    """Construct a MatrixMonitor to monitor a single test matrix instance.
+
+    Args:
+      matrix_id: {str} the unique ID of the matrix being monitored.
+      test_type: {str} the type of matrix test being run (e.g. 'robo')
+      context: {str:obj} dict containing the gcloud command context, which
+        includes the Testing API client & messages libs generated by Apitools.
+      clock: injected function which returns a current datetime object when
+        called. Used to generate time-stamps on progress messages.
+      status_interval_secs: {float} how long to sleep between status checks.
+    """
+    self.matrix_id = matrix_id
+    self._test_type = test_type
+    self._client = context['testing_client']
+    self._messages = context['testing_messages']
+    self._clock = clock
+    self._project = util.GetProject()
+    self._max_status_length = 0
+
+    if status_interval_secs is not None:
+      self._status_interval_secs = status_interval_secs
+    else:
+      self._status_interval_secs = (
+          properties.VALUES.test.matrix_status_interval.GetInt() or
+          _DEFAULT_STATUS_INTERVAL_SECS)
+      # Poll for matrix status half as fast if the end user is not running in
+      # interactive mode (i.e. either sys.stdin or sys.stderr is not a terminal
+      # i/o stream) such as when gcloud is called by a CI system like Jenkins).
+      # This reduces Testing service load and API quota usage.
+      if not console_io.IsInteractive(error=True):
+        self._status_interval_secs *= 2
+
+    exec_states = self._messages.TestExecution.StateValueValuesEnum
+    self._state_names = {
+        exec_states.VALIDATING: 'Validating',
+        exec_states.PENDING: 'Pending',
+        exec_states.RUNNING: 'Running',
+        exec_states.FINISHED: 'Finished',
+        exec_states.ERROR: 'Error',
+        exec_states.UNSUPPORTED_ENVIRONMENT: 'Unsupported',
+        exec_states.INCOMPATIBLE_ENVIRONMENT: 'Incompatible Environment',
+        exec_states.INCOMPATIBLE_ARCHITECTURE: 'Incompatible Architecture',
+        exec_states.CANCELLED: 'Cancelled',
+        exec_states.INVALID: 'Invalid',
+        exec_states.TEST_STATE_UNSPECIFIED: '*Unspecified*',
+    }
+    self._completed_execution_states = set([
+        exec_states.FINISHED,
+        exec_states.ERROR,
+        exec_states.UNSUPPORTED_ENVIRONMENT,
+        exec_states.INCOMPATIBLE_ENVIRONMENT,
+        exec_states.INCOMPATIBLE_ARCHITECTURE,
+        exec_states.CANCELLED,
+        exec_states.INVALID,
+    ])
+    matrix_states = self._messages.TestMatrix.StateValueValuesEnum
+    self.completed_matrix_states = set([
+        matrix_states.FINISHED,
+        matrix_states.ERROR,
+        matrix_states.CANCELLED,
+        matrix_states.INVALID,
+    ])
+
+  def HandleUnsupportedExecutions(self, matrix):
+    """Report unsupported device dimensions and return supported test list.
+
+    Args:
+      matrix: a TestMatrix message.
+
+    Returns:
+      A list of TestExecution messages which have supported dimensions.
+    """
+    states = self._messages.TestExecution.StateValueValuesEnum
+    supported_tests = []
+    unsupported_dimensions = set()
+
+    for test in matrix.testExecutions:
+      if test.state == states.UNSUPPORTED_ENVIRONMENT:
+        unsupported_dimensions.add(_FormatInvalidDimension(test.environment))
+      else:
+        supported_tests.append(test)
+
+    if unsupported_dimensions:
+      log.status.Print(
+          'Some device dimensions are not compatible and will be skipped:'
+          '\n  {d}'.format(d='\n  '.join(unsupported_dimensions)))
+    log.status.Print(
+        'Firebase Test Lab will execute your {t} test on {n} device(s). More '
+        'devices may be added later if flaky test attempts are specified.'
+        .format(t=self._test_type, n=len(supported_tests)))
+    return supported_tests
+
+  def _GetTestExecutionStatus(self, test_id):
+    """Fetch the TestExecution state of a specific test within a matrix.
+
+    This method is only intended to be used for a TestMatrix with exactly one
+    supported TestExecution. It would be inefficient to use it iteratively on
+    a larger TestMatrix.
+
+    Args:
+      test_id: ID of the TestExecution status to find.
+
+    Returns:
+      The TestExecution message matching the unique test_id.
+    """
+    matrix = self.GetTestMatrixStatus()
+    for test in matrix.testExecutions:
+      if test.id == test_id:
+        return test
+    # We should never get here.
+    raise exceptions.TestExecutionNotFoundError(test_id, self.matrix_id)
+
+  def MonitorTestExecutionProgress(self, test_id):
+    """Monitor and report the progress of a single running test.
+
+    This method prints more detailed test progress messages for the case where
+    the matrix has exactly one supported test configuration.
+
+    Args:
+      test_id: str, the unique id of the single supported test in the matrix.
+
+    Raises:
+      TestLabInfrastructureError if the Test service reports a backend error.
+
+    """
+    states = self._messages.TestExecution.StateValueValuesEnum
+    last_state = ''
+    error = ''
+    progress = []
+    last_progress_len = 0
+
+    while True:
+      status = self._GetTestExecutionStatus(test_id)
+      timestamp = self._clock().strftime(_TIMESTAMP_FORMAT)
+      # Check for optional error and progress details
+      details = status.testDetails
+      if details:
+        error = details.errorMessage or ''
+        progress = details.progressMessages or []
+
+      # Progress is cumulative, so only print what's new since the last update.
+      for msg in progress[last_progress_len:]:
+        log.status.Print('{0} {1}'.format(timestamp, msg.rstrip()))
+      last_progress_len = len(progress)
+
+      if status.state == states.ERROR:
+        raise exceptions.TestLabInfrastructureError(error)
+
+      if status.state == states.UNSUPPORTED_ENVIRONMENT:
+        raise exceptions.AllDimensionsIncompatibleError(
+            'Device dimensions are not compatible: {d}. '
+            'Please use "gcloud firebase test android models list" to '
+            'determine which device dimensions are compatible.'.format(
+                d=_FormatInvalidDimension(status.environment)))
+
+      # Inform user of test progress, typically PENDING -> RUNNING -> FINISHED
+      if status.state != last_state:
+        last_state = status.state
+        log.status.Print('{0} Test is {1}'.format(
+            timestamp, self._state_names[last_state]))
+
+      if status.state in self._completed_execution_states:
+        break
+
+      self._SleepForStatusInterval()
+
+    # Even if the single TestExecution is done, we also need to wait for the
+    # matrix to reach a finalized state before monitoring is done.
+    matrix = self.GetTestMatrixStatus()
+    while matrix.state not in self.completed_matrix_states:
+      log.debug('Matrix not yet complete, still in state: %s', matrix.state)
+      self._SleepForStatusInterval()
+      matrix = self.GetTestMatrixStatus()
+    self._LogTestComplete(matrix.state)
+    return
+
+  def GetTestMatrixStatus(self):
+    """Fetch the response from the GetTestMatrix rpc.
+
+    Returns:
+      A TestMatrix message holding the current state of the created tests.
+
+    Raises:
+      HttpException if the Test service reports a backend error.
+    """
+    request = self._messages.TestingProjectsTestMatricesGetRequest(
+        projectId=self._project, testMatrixId=self.matrix_id)
+    try:
+      return self._client.projects_testMatrices.Get(request)
+    except apitools_exceptions.HttpError as e:
+      exc = calliope_exceptions.HttpException(e)
+      exc.error_format = (
+          'Http error {status_code} while monitoring test run: {message}')
+      raise exc
+
+  def MonitorTestMatrixProgress(self):
+    """Monitor and report the progress of multiple running tests in a matrix."""
+    while True:
+      matrix = self.GetTestMatrixStatus()
+
+      state_counts = collections.defaultdict(int)
+      for test in matrix.testExecutions:
+        state_counts[test.state] += 1
+
+      self._UpdateMatrixStatus(state_counts)
+
+      if matrix.state in self.completed_matrix_states:
+        self._LogTestComplete(matrix.state)
+        break
+      self._SleepForStatusInterval()
+
+  def _UpdateMatrixStatus(self, state_counts):
+    """Update the matrix status line with the current test state counts.
+
+    Example: 'Test matrix status: Finished:5 Running:3 Unsupported:2'
+
+    Args:
+      state_counts: {state:count} a dict mapping a test state to its frequency.
+    """
+    status = []
+    timestamp = self._clock().strftime(_TIMESTAMP_FORMAT)
+    for state, count in six.iteritems(state_counts):
+      if count > 0:
+        status.append('{s}:{c}'.format(s=self._state_names[state], c=count))
+    status.sort()
+    # Use \r so that the status summary will update on the same console line.
+    out = '\r{0} Test matrix status: {1} '.format(timestamp, ' '.join(status))
+
+    # Right-pad with blanks when the status line gets shorter.
+    self._max_status_length = max(len(out), self._max_status_length)
+    log.status.write(out.ljust(self._max_status_length))
+
+  def _LogTestComplete(self, matrix_state):
+    """Let the user know that their test matrix has completed running."""
+    log.info('Test matrix completed in state: {0}'.format(matrix_state))
+    log.status.Print('\n{0} testing complete.'.format(
+        self._test_type.capitalize()))
+
+  def _SleepForStatusInterval(self):
+    time.sleep(self._status_interval_secs)
+
+  def CancelTestMatrix(self):
+    """Cancels an in-progress TestMatrix.
+
+    Raises:
+      HttpException if the Test service reports a back-end error.
+    """
+    request = self._messages.TestingProjectsTestMatricesCancelRequest(
+        projectId=self._project, testMatrixId=self.matrix_id)
+    try:
+      self._client.projects_testMatrices.Cancel(request)
+    except apitools_exceptions.HttpError as e:
+      exc = calliope_exceptions.HttpException(e)
+      exc.error_format = 'CancelTestMatrix: {message}'
+      raise exc
+
+
+def _FormatInvalidDimension(environment):
+  """Return a human-readable string representing an invalid matrix dimension."""
+  if getattr(environment, 'androidDevice', None) is not None:
+    device = environment.androidDevice
+    return ('[OS-version {vers} on {model}]'.format(
+        model=device.androidModelId, vers=device.androidVersionId))
+  if getattr(environment, 'iosDevice', None) is not None:
+    device = environment.iosDevice
+    return ('[OS-version {vers} on {model}]'.format(
+        model=device.iosModelId, vers=device.iosVersionId))
+  # Handle any new device environments here.
+  return '[unknown-environment]'
+
+
+def ReformatDuration(duration):
+  """Reformat a Duration arg to work around ApiTools non-support of that type.
+
+  Duration args are normally converted to an int in seconds (e.g. --timeout 5m
+  becomes args.timeout with int value 300). Duration proto fields are converted
+  to type string during discovery doc creation, so we have to convert the int
+  back into a string-formatted Duration (i.e. append an 's') before
+  passing it to the Testing Service.
+
+  Args:
+    duration: {int} the number of seconds in the time duration.
+
+  Returns:
+    String representation of the Duration with units of seconds.
+  """
+  return '{secs}s'.format(secs=duration)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/results_bucket.py

@@ -0,0 +1,224 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utility methods to aid in interacting with a GCS results bucket."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import os
+
+from apitools.base.py import exceptions as apitools_exceptions
+from apitools.base.py import transfer
+
+from googlecloudsdk.api_lib.firebase.test import util
+from googlecloudsdk.api_lib.util import apis as core_apis
+from googlecloudsdk.calliope import exceptions
+from googlecloudsdk.core import log
+from googlecloudsdk.core import resources
+
+
+GCS_PREFIX = 'gs://'
+HTTP_FORBIDDEN = 403
+HTTP_NOT_FOUND = 404
+
+
+class ResultsBucketOps(object):
+  """A utility class to encapsulate operations on the results bucket."""
+
+  def __init__(self, project, bucket_name, unique_obj_name,
+               tr_client, tr_messages, storage_client):
+    """Construct a ResultsBucketOps object to be used with a single matrix run.
+
+    Args:
+      project: string containing the Google Cloud Platform project id.
+      bucket_name: string with the user-supplied name of a GCS bucket, or None.
+      unique_obj_name: a unique-per-matrix GCS object name which will prefix all
+        raw test result files within the supplied bucket_name.
+      tr_client: ToolResults API client library generated by Apitools.
+      tr_messages: ToolResults API messages library generated by Apitools.
+      storage_client: Cloud Storage API client library generated by Apitools.
+
+    Attributes:
+      gcs_results_root: string containing the root path for the test results in
+        'gs://{bucket}/{timestamp-suffix}' format.
+    """
+    self._project = project
+    self._storage_client = storage_client
+    self._storage_messages = core_apis.GetMessagesModule('storage', 'v1')
+    self._gcs_object_name = unique_obj_name
+
+    # If the user supplied a results bucket, make sure it exists. Otherwise,
+    # call the SettingsService to get the project's existing default bucket.
+    if bucket_name:
+      self.EnsureBucketExists(bucket_name)
+    else:
+      bucket_name = self._GetDefaultBucket(tr_client, tr_messages)
+
+    bucket_ref = resources.REGISTRY.Parse(
+        bucket_name, collection='storage.buckets')
+    self._results_bucket = bucket_ref.bucket
+
+    self._gcs_results_url = (
+        'https://console.developers.google.com/storage/browser/{b}/{t}/'
+        .format(b=bucket_name, t=self._gcs_object_name))
+    self.gcs_results_root = ('gs://{b}/{t}/'
+                             .format(b=bucket_name, t=self._gcs_object_name))
+    log.info('Raw results root path is: [{0}]'.format(self.gcs_results_root))
+
+  def _GetDefaultBucket(self, tr_client, tr_messages):
+    """Fetch the project's default GCS bucket name for storing tool results."""
+    request = tr_messages.ToolresultsProjectsInitializeSettingsRequest(
+        projectId=self._project)
+    try:
+      response = tr_client.projects.InitializeSettings(request)
+      return response.defaultBucket
+    except apitools_exceptions.HttpError as error:
+      code, err_msg = util.GetErrorCodeAndMessage(error)
+      if code == HTTP_FORBIDDEN:
+        msg = ('Permission denied while fetching the default results bucket '
+               '(Error {0}: {1}). Is billing enabled for project: [{2}]?'
+               .format(code, err_msg, self._project))
+      else:
+        msg = ('Http error while trying to fetch the default results bucket:\n'
+               'ResponseError {0}: {1}'
+               .format(code, err_msg))
+      raise exceptions.HttpException(msg)
+
+  def EnsureBucketExists(self, bucket_name):
+    """Create a GCS bucket if it doesn't already exist.
+
+    Args:
+      bucket_name: the name of the GCS bucket to create if it doesn't exist.
+
+    Raises:
+      BadFileException if the bucket name is malformed, the user does not
+        have access rights to the bucket, or the bucket can't be created.
+    """
+    get_req = self._storage_messages.StorageBucketsGetRequest(
+        bucket=bucket_name)
+    try:
+      self._storage_client.buckets.Get(get_req)
+      return  # The bucket exists and the user can access it.
+    except apitools_exceptions.HttpError as err:
+      code, err_msg = util.GetErrorCodeAndMessage(err)
+      if code != HTTP_NOT_FOUND:
+        raise exceptions.BadFileException(
+            'Could not access bucket [{b}]. Response error {c}: {e}. '
+            'Please supply a valid bucket name or use the default bucket '
+            'provided by Firebase Test Lab.'
+            .format(b=bucket_name, c=code, e=err_msg))
+
+    # The bucket does not exist in any project, so create it in user's project.
+    log.status.Print('Creating results bucket [{g}{b}] in project [{p}].'
+                     .format(g=GCS_PREFIX, b=bucket_name, p=self._project))
+
+    bucket_req = self._storage_messages.StorageBucketsInsertRequest
+    acl = bucket_req.PredefinedAclValueValuesEnum.projectPrivate
+    objacl = bucket_req.PredefinedDefaultObjectAclValueValuesEnum.projectPrivate
+
+    insert_req = self._storage_messages.StorageBucketsInsertRequest(
+        bucket=self._storage_messages.Bucket(name=bucket_name),
+        predefinedAcl=acl,
+        predefinedDefaultObjectAcl=objacl,
+        project=self._project)
+    try:
+      self._storage_client.buckets.Insert(insert_req)
+      return
+    except apitools_exceptions.HttpError as err:
+
+      code, err_msg = util.GetErrorCodeAndMessage(err)
+      if code == HTTP_FORBIDDEN:
+        msg = ('Permission denied while creating bucket [{b}]. '
+               'Is billing enabled for project: [{p}]?'
+               .format(b=bucket_name, p=self._project))
+      else:
+        msg = ('Failed to create bucket [{b}] {e}'
+               .format(b=bucket_name, e=util.GetError(err)))
+      raise exceptions.BadFileException(msg)
+
+  def UploadFileToGcs(self, path, mimetype=None, destination_object=None):
+    """Upload a file to the GCS results bucket using the storage API.
+
+    Args:
+      path: str, the absolute or relative path of the file to upload. File may
+        be in located in GCS or the local filesystem.
+      mimetype: str, the MIME type (aka Content-Type) that should be applied to
+        files being copied from a non-GCS source to GCS. MIME types for GCS->GCS
+        file uploads are not modified.
+      destination_object: str, the destination object path in GCS to upload to,
+        if it's different than the base name of the path argument.
+
+    Raises:
+      BadFileException if the file upload is not successful.
+    """
+    log.status.Print('Uploading [{f}] to Firebase Test Lab...'.format(f=path))
+    try:
+      if path.startswith(GCS_PREFIX):
+        # Perform a GCS object to GCS object copy
+        file_bucket, file_obj = _SplitBucketAndObject(path)
+        copy_req = self._storage_messages.StorageObjectsCopyRequest(
+            sourceBucket=file_bucket,
+            sourceObject=file_obj,
+            destinationBucket=self._results_bucket,
+            destinationObject='{obj}/{name}'.format(
+                obj=self._gcs_object_name,
+                name=destination_object or os.path.basename(file_obj)))
+        self._storage_client.objects.Copy(copy_req)
+      else:
+        # Perform a GCS insert of a file which is not in GCS
+        try:
+          file_size = os.path.getsize(path)
+        except os.error:
+          raise exceptions.BadFileException('[{0}] not found or not accessible'
+                                            .format(path))
+        src_obj = self._storage_messages.Object(size=file_size)
+        try:
+          upload = transfer.Upload.FromFile(path, mime_type=mimetype)
+        except apitools_exceptions.InvalidUserInputError:
+          upload = transfer.Upload.FromFile(
+              path, mime_type='application/octet-stream')
+
+        insert_req = self._storage_messages.StorageObjectsInsertRequest(
+            bucket=self._results_bucket,
+            name='{obj}/{name}'.format(
+                obj=self._gcs_object_name,
+                name=destination_object or os.path.basename(path)),
+            object=src_obj)
+        response = self._storage_client.objects.Insert(insert_req,
+                                                       upload=upload)
+        if response.size != file_size:
+          raise exceptions.BadFileException(
+              'Cloud storage upload failure: Insert response.size={0} bytes '
+              'but [{1}] contains {2} bytes.\nInsert response: {3}'
+              .format(response.size, path, file_size, repr(response)))
+    except apitools_exceptions.HttpError as err:
+      raise exceptions.BadFileException(
+          'Could not copy [{f}] to [{gcs}] {e}.'
+          .format(f=path, gcs=self.gcs_results_root, e=util.GetError(err)))
+
+  def LogGcsResultsUrl(self):
+    log.status.Print('Raw results will be stored in your GCS bucket at [{0}]\n'
+                     .format(self._gcs_results_url))
+
+
+def _SplitBucketAndObject(gcs_path):
+  """Split a GCS path into bucket & object tokens, or raise BadFileException."""
+  tokens = gcs_path[len(GCS_PREFIX):].strip('/').split('/', 1)
+  if len(tokens) != 2:
+    raise exceptions.BadFileException(
+        '[{0}] is not a valid Google Cloud Storage path'.format(gcs_path))
+  return tokens

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/results_summary.py

@@ -0,0 +1,477 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A library to build a test results summary."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import collections
+
+from apitools.base.py import exceptions as apitools_exceptions
+
+from googlecloudsdk.api_lib.firebase.test import util
+from googlecloudsdk.calliope import exceptions
+from googlecloudsdk.core import log
+
+_NATIVE_CRASH = 'Native crash'
+_NATIVE_CRASH_DETAILED_FORMAT = """\
+For test execution [{0}], a native process crashed on the device. This could \
+be caused by your app, by an app dependency, or by an unrelated cause."""
+_INFRASTRUCTURE_FAILURE = 'Infrastructure failure'
+_INFRASTRUCTURE_FAILURE_DETAILED_FORMAT = """\
+Need help for test execution [{0}]? Please join the #test-lab Slack channel \
+at https://firebase.community/ and include test matrix ID [{1}] with your \
+question."""
+
+
+class TestOutcome(
+    collections.namedtuple('TestOutcome',
+                           ['outcome', 'axis_value', 'test_details'])):
+  """A tuple to hold the outcome for a single test axis value.
+
+  Fields:
+    outcome: string containing the test outcome (e.g. 'Passed')
+    axis_value: string representing one axis value within the matrix.
+    test_details: string with extra details (e.g. "Incompatible architecture")
+  """
+
+
+# Human-friendly test outcome names
+_SUCCESS = 'Passed'
+_FLAKY = 'Flaky'
+_FAILURE = 'Failed'
+_INCONCLUSIVE = 'Inconclusive'
+_SKIPPED = 'Skipped'
+
+# Relative sort weightings for test outcomes
+_OUTCOME_SORTING = {
+    _FAILURE: 10,
+    _FLAKY: 20,
+    _SUCCESS: 30,
+    _INCONCLUSIVE: 40,
+    _SKIPPED: 50,
+}
+
+
+def _TestOutcomeSortKey(x):
+  """Transform a TestOutcome to a tuple yielding the desired sort order."""
+  return tuple([_OUTCOME_SORTING[x.outcome], x.test_details, x.axis_value])
+
+
+class ToolResultsSummaryFetcher(object):
+  """Creates Test Results summary using data from the ToolResults API."""
+
+  def __init__(self, project, client, messages, tool_results_ids,
+               test_matrix_id):
+    """Constructs a ToolResultsSummaryFetcher.
+
+    Args:
+      project: string containing the GCE project id.
+      client: ToolResults API client lib generated by apitools.
+      messages: ToolResults API message classes generated by apitools.
+      tool_results_ids: a ToolResultsIds object holding history & execution IDs.
+      test_matrix_id: test matrix ID from Testing API.
+    """
+    self._project = project
+    self._client = client
+    self._messages = messages
+    self._history_id = tool_results_ids.history_id
+    self._execution_id = tool_results_ids.execution_id
+    self._test_matrix_id = test_matrix_id
+    self._outcome_names = {
+        messages.Outcome.SummaryValueValuesEnum.success: _SUCCESS,
+        messages.Outcome.SummaryValueValuesEnum.failure: _FAILURE,
+        messages.Outcome.SummaryValueValuesEnum.flaky: _FLAKY,
+        messages.Outcome.SummaryValueValuesEnum.skipped: _SKIPPED,
+        messages.Outcome.SummaryValueValuesEnum.inconclusive: _INCONCLUSIVE,
+    }
+
+  def FetchMatrixRollupOutcome(self):
+    """Gets a test execution's rolled-up outcome from the ToolResults service.
+
+    Returns:
+      The rolled-up test execution outcome (type: toolresults_v1beta3.Outcome).
+
+    Raises:
+      HttpException if the ToolResults service reports a back-end error.
+    """
+    request = self._messages.ToolresultsProjectsHistoriesExecutionsGetRequest(
+        projectId=self._project,
+        historyId=self._history_id,
+        executionId=self._execution_id)
+    try:
+      response = self._client.projects_histories_executions.Get(request)
+      return response.outcome
+    except apitools_exceptions.HttpError as error:
+      msg = 'Http error fetching test roll-up outcome: ' + util.GetError(error)
+      raise exceptions.HttpException(msg)
+
+  def CreateMatrixOutcomeSummaryUsingSteps(self):
+    """Fetches test results and creates a test outcome summary.
+
+    Lists all the steps in an execution and creates a high-level outcome summary
+    for each step (pass/fail/inconclusive). Each step represents a test run on
+    a single device (e.g. running the tests on a Nexus 5 in portrait mode using
+    the en locale and API level 18).
+
+    Returns:
+      A list of TestOutcome objects.
+
+    Raises:
+      HttpException if the ToolResults service reports a back-end error.
+    """
+    outcomes = []
+    steps = self._ListAllSteps()
+    if not steps:
+      log.warning(
+          'No test results found, something went wrong. Try re-running the tests.'
+      )
+      return outcomes
+
+    for step in steps:
+      dimension_value = step.dimensionValue
+      axis_value = self._GetAxisValue(dimension_value)
+      # It's a bug in ToolResults if we get no outcome, but this guard
+      # prevents a stack trace if it should happen.
+      if not step.outcome:
+        log.warning('Step for [{0}] had no outcome value.'.format(axis_value))
+      else:
+        details = self._GetStepOutcomeDetails(step)
+        self._LogWarnings(details, axis_value)
+        outcome_summary = step.outcome.summary
+        outcome_str = self._GetOutcomeSummaryDisplayName(outcome_summary)
+        outcomes.append(
+            TestOutcome(
+                outcome=outcome_str,
+                axis_value=axis_value,
+                test_details=details))
+
+    return sorted(outcomes, key=_TestOutcomeSortKey)
+
+  def CreateMatrixOutcomeSummaryUsingEnvironments(self):
+    """Fetches test results and creates a test outcome summary.
+
+    Lists all the environments in an execution and creates a high-level outcome
+    summary for each environment (pass/flaky/fail/skipped/inconclusive). Each
+    environment represents a combination of one or more test executions with the
+    same device configuration (e.g. running the tests on a Nexus 5 in portrait
+    mode using the en locale and API level 18).
+
+    Returns:
+      A list of TestOutcome objects.
+
+    Raises:
+      HttpException if the ToolResults service reports a back-end error.
+    """
+    outcomes = []
+    environments = self._ListAllEnvironments()
+    # It's a bug in ToolResults if we get no environment, but this guard
+    # prevents a stack trace if it should happen.
+    if not environments:
+      log.warning(
+          'Environment has no results, something went wrong. Displaying step '
+          'outcomes instead.')
+      return self.CreateMatrixOutcomeSummaryUsingSteps()
+
+    for environment in environments:
+      dimension_value = environment.dimensionValue
+      axis_value = self._GetAxisValue(dimension_value)
+      # It's a bug in ToolResults if we get no outcome, but this guard
+      # prevents a stack trace if it should happen.
+      if not environment.environmentResult.outcome:
+        log.warning('Environment for [{0}] had no outcome value. Displaying '
+                    'step outcomes instead.'.format(axis_value))
+        return self.CreateMatrixOutcomeSummaryUsingSteps()
+
+      details = self._GetEnvironmentOutcomeDetails(environment)
+      self._LogWarnings(details, axis_value)
+      outcome_summary = environment.environmentResult.outcome.summary
+      outcome_str = self._GetOutcomeSummaryDisplayName(outcome_summary)
+      outcomes.append(
+          TestOutcome(
+              outcome=outcome_str, axis_value=axis_value, test_details=details))
+
+    return sorted(outcomes, key=_TestOutcomeSortKey)
+
+  def _LogWarnings(self, details, axis_value):
+    """Log warnings if there was native crash or infrustructure failure."""
+    if _NATIVE_CRASH in details:
+      log.warning(_NATIVE_CRASH_DETAILED_FORMAT.format(axis_value))
+    if _INFRASTRUCTURE_FAILURE in details:
+      log.warning(
+          _INFRASTRUCTURE_FAILURE_DETAILED_FORMAT.format(
+              axis_value, self._test_matrix_id))
+
+  def _GetAxisValue(self, dimensionvalue):
+    axes = {}
+    for dimension in dimensionvalue:
+      axes[dimension.key] = dimension.value
+    return ('{m}-{v}-{l}-{o}'.format(
+        m=axes.get('Model', '?'),
+        v=axes.get('Version', '?'),
+        l=axes.get('Locale', '?'),
+        o=axes.get('Orientation', '?')))
+
+  def _ListAllSteps(self):
+    """Lists all steps for a test execution using the ToolResults service.
+
+    Returns:
+      The full list of steps for a test execution.
+    """
+    response = self._ListSteps(None)
+    steps = []
+    steps.extend(response.steps)
+
+    while response.nextPageToken:
+      response = self._ListSteps(response.nextPageToken)
+      steps.extend(response.steps)
+
+    return steps
+
+  def _ListSteps(self, page_token):
+    """Lists one page of steps using the ToolResults service.
+
+    Args:
+      page_token: A page token to attach to the List request. If it's None, then
+        it returns at most the first 200 steps.
+
+    Returns:
+      A ListStepsResponse containing a single page's steps.
+
+    Raises:
+      HttpException if the ToolResults service reports a back-end error.
+    """
+    request = (
+        self._messages.ToolresultsProjectsHistoriesExecutionsStepsListRequest(
+            projectId=self._project,
+            historyId=self._history_id,
+            executionId=self._execution_id,
+            pageSize=100,
+            pageToken=page_token))
+    try:
+      return self._client.projects_histories_executions_steps.List(request)
+    except apitools_exceptions.HttpError as error:
+      msg = 'Http error while listing test results of steps: ' + util.GetError(
+          error)
+      raise exceptions.HttpException(msg)
+
+  def _ListAllEnvironments(self):
+    """Lists all environments of a test execution using the ToolResults service.
+
+    Returns:
+      A ListEnvironmentsResponse containing all environments within execution.
+    """
+    response = self._ListEnvironments(None)
+    environments = []
+    environments.extend(response.environments)
+
+    while response.nextPageToken:
+      response = self._ListEnvironments(response.nextPageToken)
+      environments.extend(response.environments)
+
+    return environments
+
+  def _ListEnvironments(self, page_token):
+    """Lists one page of environments using the ToolResults service.
+
+    Args:
+      page_token: A page token to attach to the List request. If it's None, then
+        it returns a maximum of 200 Environments.
+
+    Returns:
+      A ListEnvironmentsResponse containing a single page's environments.
+
+    Raises:
+      HttpException if the ToolResults service reports a back-end error.
+    """
+
+    request = (
+        self._messages
+        .ToolresultsProjectsHistoriesExecutionsEnvironmentsListRequest(
+            projectId=self._project,
+            historyId=self._history_id,
+            executionId=self._execution_id,
+            pageSize=100,
+            pageToken=page_token))
+    try:
+      return self._client.projects_histories_executions_environments.List(
+          request)
+    except apitools_exceptions.HttpError as error:
+      msg = 'Http error while listing test results: ' + util.GetError(error)
+      raise exceptions.HttpException(msg)
+
+  def _GetOutcomeSummaryDisplayName(self, outcome):
+    """Transforms the outcome enum to a human readable outcome.
+
+    Args:
+      outcome: An Outcome.SummaryValueValuesEnum value.
+
+    Returns:
+      A string containing a human readable outcome.
+    """
+    try:
+      return self._outcome_names[outcome]
+    except ValueError:
+      return 'Unknown'
+
+  def _GetStepOutcomeDetails(self, step):
+    """Turn test outcome counts and details into something human readable."""
+    outcome = step.outcome
+    summary_enum = self._messages.Outcome.SummaryValueValuesEnum
+    test_suite_overviews = step.testExecutionStep.testSuiteOverviews
+
+    if outcome.summary == summary_enum.success:
+      details = _GetSuccessCountDetails(test_suite_overviews)
+      if outcome.successDetail and outcome.successDetail.otherNativeCrash:
+        return '{d} ({c})'.format(d=details, c=_NATIVE_CRASH)
+      else:
+        return details
+
+    elif outcome.summary == summary_enum.failure:
+      if outcome.failureDetail:
+        return _GetFailureDetail(outcome, test_suite_overviews)
+      if not step.testExecutionStep:
+        return 'Unknown failure'
+      return _GetFailureOrFlakyCountDetails(test_suite_overviews)
+
+    elif outcome.summary == summary_enum.inconclusive:
+      return _GetInconclusiveDetail(outcome)
+
+    elif outcome.summary == summary_enum.skipped:
+      return _GetSkippedDetail(outcome)
+
+    else:
+      return 'Unknown outcome'
+
+  def _GetEnvironmentOutcomeDetails(self, environment):
+    """Turn test outcome counts and details into something human readable."""
+    outcome = environment.environmentResult.outcome
+    summary_enum = self._messages.Outcome.SummaryValueValuesEnum
+    test_suite_overviews = environment.environmentResult.testSuiteOverviews
+
+    if outcome.summary == summary_enum.success:
+      details = _GetSuccessCountDetails(test_suite_overviews)
+      if outcome.successDetail and outcome.successDetail.otherNativeCrash:
+        return '{d} ({c})'.format(d=details, c=_NATIVE_CRASH)
+      else:
+        return details
+
+    elif outcome.summary == summary_enum.failure or outcome.summary == summary_enum.flaky:
+      if outcome.failureDetail:
+        return _GetFailureDetail(outcome, test_suite_overviews)
+      return _GetFailureOrFlakyCountDetails(test_suite_overviews)
+
+    elif outcome.summary == summary_enum.inconclusive:
+      return _GetInconclusiveDetail(outcome)
+
+    elif outcome.summary == summary_enum.skipped:
+      return _GetSkippedDetail(outcome)
+
+    else:
+      return 'Unknown outcome'
+
+
+def _GetFailureDetail(outcome, test_suite_overviews):
+  """Build a string with failureDetail if present."""
+  details = ''
+  # Note: crashed/timedOut/notInstalled flags are mutually exclusive
+  if outcome.failureDetail.crashed:
+    details = 'Application crashed'
+  elif outcome.failureDetail.timedOut:
+    details = 'Test timed out'
+  elif outcome.failureDetail.notInstalled:
+    details = 'App failed to install'
+  # otherNativeCrash is not mutually exclusive to other failureDetails
+  crash = _NATIVE_CRASH if outcome.failureDetail.otherNativeCrash else ''
+  if details and crash:
+    return '{d} ({c})'.format(d=details, c=crash)
+  elif details:
+    return details
+  elif crash:
+    return crash
+  return _GetFailureOrFlakyCountDetails(test_suite_overviews)
+
+
+def _GetSkippedDetail(outcome):
+  """Build a string with skippedDetail if present."""
+  if outcome.skippedDetail:
+    if outcome.skippedDetail.incompatibleDevice:
+      return 'Incompatible device/OS combination'
+    if outcome.skippedDetail.incompatibleArchitecture:
+      return (
+          'App architecture or requested options are incompatible with this '
+          'device')
+    if outcome.skippedDetail.incompatibleAppVersion:
+      return 'App does not support the OS version'
+  return 'Unknown reason'
+
+
+def _GetInconclusiveDetail(outcome):
+  """Build a string with inconclusiveDetail if present."""
+  if outcome.inconclusiveDetail:
+    if outcome.inconclusiveDetail.infrastructureFailure:
+      return _INFRASTRUCTURE_FAILURE
+    if outcome.inconclusiveDetail.abortedByUser:
+      return 'Test run aborted by user'
+  return 'Unknown reason'
+
+
+def _GetSuccessCountDetails(test_suite_overviews):
+  """Build a string with status count sums for testSuiteOverviews."""
+  total = 0
+  skipped = 0
+  for overview in test_suite_overviews:
+    total += overview.totalCount or 0
+    skipped += overview.skippedCount or 0
+  passed = total - skipped
+  if passed:
+    msg = '{p} test cases passed'.format(p=passed)
+    if skipped:
+      msg = '{m}, {s} skipped'.format(m=msg, s=skipped)
+    return msg
+  return '--'
+
+
+def _GetFailureOrFlakyCountDetails(test_suite_overviews):
+  """Build a string with status count sums for testSuiteOverviews."""
+  total = 0
+  error = 0
+  failed = 0
+  skipped = 0
+  flaky = 0
+  for overview in test_suite_overviews:
+    total += overview.totalCount or 0
+    error += overview.errorCount or 0
+    failed += overview.failureCount or 0
+    skipped += overview.skippedCount or 0
+    flaky += overview.flakyCount or 0
+
+  if total:
+    msg = '{f} test cases failed'.format(f=failed)
+    passed = total - error - failed - skipped - flaky
+    if flaky and failed:
+      msg = '{m}, {f} flaky'.format(m=msg, f=flaky)
+    if flaky and not failed:
+      msg = '{f} test cases flaky'.format(f=flaky)
+    if passed:
+      msg = '{m}, {p} passed'.format(m=msg, p=passed)
+    if error:
+      msg = '{m}, {e} errors'.format(m=msg, e=error)
+    if skipped:
+      msg = '{m}, {s} skipped'.format(m=msg, s=skipped)
+    return msg
+  else:
+    return 'Test failed to run'

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/tool_results.py

@@ -0,0 +1,251 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A utility library to support interaction with the Tool Results service."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import collections
+import os
+import time
+
+from googlecloudsdk.api_lib.firebase.test import exceptions
+from googlecloudsdk.api_lib.util import apis
+from googlecloudsdk.core import properties
+from googlecloudsdk.core.console import progress_tracker
+from six.moves.urllib import parse
+import uritemplate
+
+_STATUS_INTERVAL_SECS = 3
+
+
+class ToolResultsIds(
+    collections.namedtuple('ToolResultsIds', ['history_id', 'execution_id'])):
+  """A tuple to hold the history & execution IDs returned from Tool Results.
+
+  Fields:
+    history_id: a string with the Tool Results history ID to publish to.
+    execution_id: a string with the ID of the Tool Results execution.
+  """
+
+
+def CreateToolResultsUiUrl(project_id, tool_results_ids):
+  """Create the URL for a test's Tool Results UI in the Firebase App Manager.
+
+  Args:
+    project_id: string containing the user's GCE project ID.
+    tool_results_ids: a ToolResultsIds object holding history & execution IDs.
+
+  Returns:
+    A url to the Tool Results UI.
+  """
+  url_base = properties.VALUES.test.results_base_url.Get()
+  if not url_base:
+    url_base = 'https://console.firebase.google.com'
+
+  url_end = uritemplate.expand(
+      'project/{project}/testlab/histories/{history}/matrices/{execution}', {
+          'project': project_id,
+          'history': tool_results_ids.history_id,
+          'execution': tool_results_ids.execution_id
+      })
+  return parse.urljoin(url_base, url_end)
+
+
+def GetToolResultsIds(matrix,
+                      matrix_monitor,
+                      status_interval=_STATUS_INTERVAL_SECS):
+  """Gets the Tool Results history ID and execution ID for a test matrix.
+
+  Sometimes the IDs are available immediately after a test matrix is created.
+  If not, we keep checking the matrix until the Testing and Tool Results
+  services have had enough time to create/assign the IDs, giving the user
+  continuous feedback using gcloud core's ProgressTracker class.
+
+  Args:
+    matrix: a TestMatrix which was just created by the Testing service.
+    matrix_monitor: a MatrixMonitor object.
+    status_interval: float, number of seconds to sleep between status checks.
+
+  Returns:
+    A ToolResultsIds tuple containing the history ID and execution ID, which
+    are shared by all TestExecutions in the TestMatrix.
+
+  Raises:
+    BadMatrixError: if the matrix finishes without both ToolResults IDs.
+  """
+  history_id = None
+  execution_id = None
+  msg = 'Creating individual test executions'
+  with progress_tracker.ProgressTracker(msg, autotick=True):
+    while True:
+      if matrix.resultStorage.toolResultsExecution:
+        history_id = matrix.resultStorage.toolResultsExecution.historyId
+        execution_id = matrix.resultStorage.toolResultsExecution.executionId
+        if history_id and execution_id:
+          break
+
+      if matrix.state in matrix_monitor.completed_matrix_states:
+        raise exceptions.BadMatrixError(_ErrorFromMatrixInFailedState(matrix))
+
+      time.sleep(status_interval)
+      matrix = matrix_monitor.GetTestMatrixStatus()
+
+  return ToolResultsIds(history_id=history_id, execution_id=execution_id)
+
+
+def _ErrorFromMatrixInFailedState(matrix):
+  """Produces a human-readable error message from an invalid matrix."""
+  messages = apis.GetMessagesModule('testing', 'v1')
+  if matrix.state == messages.TestMatrix.StateValueValuesEnum.INVALID:
+    return _ExtractInvalidMatrixDetails(matrix)
+
+  return _GenericErrorMessage(matrix)
+
+
+def _ExtractInvalidMatrixDetails(matrix):
+  invalid_details_for_user = []
+  for invalid_detail in matrix.extendedInvalidMatrixDetails:
+    invalid_details_for_user.append(
+        f'Reason: {invalid_detail.reason} Message: {invalid_detail.message}'
+    )
+  if invalid_details_for_user:
+    return 'Matrix [{m}] failed during validation.\n{msg}'.format(
+        m=matrix.testMatrixId, msg=os.linesep.join(invalid_details_for_user)
+    )
+  else:
+    return _GetLegacyInvalidMatrixDetails(matrix)
+
+
+def _GetLegacyInvalidMatrixDetails(matrix):
+  """Converts legacy invalid matrix enum to a descriptive message for the user.
+
+  Args:
+    matrix: A TestMatrix in a failed state
+
+  Returns:
+    A string containing the legacy error message when no message is available
+    from the API.
+
+  """
+  messages = apis.GetMessagesModule('testing', 'v1')
+  enum_values = messages.TestMatrix.InvalidMatrixDetailsValueValuesEnum
+  error_dict = {
+      enum_values.MALFORMED_APK:
+          'The app APK is not a valid Android application',
+      enum_values.MALFORMED_TEST_APK:
+          'The test APK is not a valid Android instrumentation test',
+      enum_values.NO_MANIFEST:
+          'The app APK is missing the manifest file',
+      enum_values.NO_PACKAGE_NAME:
+          'The APK manifest file is missing the package name',
+      enum_values.TEST_SAME_AS_APP:
+          'The test APK has the same package name as the app APK',
+      enum_values.NO_INSTRUMENTATION:
+          'The test APK declares no instrumentation tags in the manifest',
+      enum_values.NO_SIGNATURE:
+          'At least one supplied APK file has a missing or invalid signature',
+      enum_values.INSTRUMENTATION_ORCHESTRATOR_INCOMPATIBLE:
+          ("The test runner class specified by the user or the test APK's "
+           'manifest file is not compatible with Android Test Orchestrator. '
+           'Please use AndroidJUnitRunner version 1.1 or higher'),
+      enum_values.NO_TEST_RUNNER_CLASS:
+          ('The test APK does not contain the test runner class specified by '
+           'the user or the manifest file. The test runner class name may be '
+           'incorrect, or the class may be mislocated in the app APK.'),
+      enum_values.NO_LAUNCHER_ACTIVITY:
+          'The app APK does not specify a main launcher activity',
+      enum_values.FORBIDDEN_PERMISSIONS:
+          'The app declares one or more permissions that are not allowed',
+      enum_values.INVALID_ROBO_DIRECTIVES:
+          ('Robo directives are invalid: multiple robo-directives cannot have '
+           'the same resource name and there cannot be more than one `click:` '
+           'directive specified.'),
+      enum_values.INVALID_DIRECTIVE_ACTION:
+          'Robo Directive includes at least one invalid action definition.',
+      enum_values.INVALID_RESOURCE_NAME:
+          'Robo Directive resource name contains invalid characters: ":" '
+          ' (colon) or " " (space)',
+      enum_values.TEST_LOOP_INTENT_FILTER_NOT_FOUND:
+          'The app does not have a correctly formatted game-loop intent filter',
+      enum_values.SCENARIO_LABEL_NOT_DECLARED:
+          'A scenario-label was not declared in the manifest file',
+      enum_values.SCENARIO_LABEL_MALFORMED:
+          'A scenario-label in the manifest includes invalid numbers or ranges',
+      enum_values.SCENARIO_NOT_DECLARED:
+          'A scenario-number was not declared in the manifest file',
+      enum_values.DEVICE_ADMIN_RECEIVER:
+          'Device administrator applications are not allowed',
+      enum_values.MALFORMED_XC_TEST_ZIP:
+          'The XCTest zip file was malformed. The zip did not contain a single '
+          '.xctestrun file and the contents of the DerivedData/Build/Products '
+          'directory.',
+      enum_values.BUILT_FOR_IOS_SIMULATOR:
+          'The provided XCTest was built for the iOS simulator rather than for '
+          'a physical device',
+      enum_values.NO_TESTS_IN_XC_TEST_ZIP:
+          'The .xctestrun file did not specify any test targets to run',
+      enum_values.USE_DESTINATION_ARTIFACTS:
+          'One or more of the test targets defined in the .xctestrun file '
+          'specifies "UseDestinationArtifacts", which is not allowed',
+      enum_values.TEST_NOT_APP_HOSTED:
+          'One or more of the test targets defined in the .xctestrun file '
+          'does not have a host binary to run on the physical iOS device, '
+          'which may cause errors when running xcodebuild',
+      enum_values.NO_CODE_APK:
+          '"hasCode" is false in the Manifest. Tested APKs must contain code',
+      enum_values.INVALID_INPUT_APK:
+          'Either the provided input APK path was malformed, the APK file does '
+          'not exist, or the user does not have permission to access the file',
+      enum_values.INVALID_APK_PREVIEW_SDK:
+          "Your app targets a preview version of the Android SDK that's "
+          'incompatible with the selected devices.',
+      enum_values.PLIST_CANNOT_BE_PARSED:
+          'One or more of the Info.plist files in the zip could not be parsed',
+      enum_values.INVALID_PACKAGE_NAME:
+          'The APK application ID (aka package name) is invalid. See also '
+          'https://developer.android.com/studio/build/application-id',
+      enum_values.MALFORMED_IPA:
+          'The app IPA is not a valid iOS application',
+      enum_values.MISSING_URL_SCHEME:
+          'The iOS game loop application does not register the custom URL '
+          'scheme',
+      enum_values.MALFORMED_APP_BUNDLE:
+          'The iOS application bundle (.app) is invalid',
+      enum_values.MATRIX_TOO_LARGE:
+          'The matrix expanded to contain too many executions.',
+      enum_values.TEST_QUOTA_EXCEEDED:
+          'Not enough test quota to run the executions in this matrix.',
+      enum_values.SERVICE_NOT_ACTIVATED:
+          'A required cloud service api is not activated.',
+      enum_values.UNKNOWN_PERMISSION_ERROR:
+          'There was an unknown permission issue running this test.',
+  }
+  details_enum = matrix.invalidMatrixDetails
+  if details_enum in error_dict:
+    return ('\nMatrix [{m}] failed during validation: {e}.'.format(
+        m=matrix.testMatrixId, e=error_dict[details_enum]))
+  # Use a generic message if the enum is unknown or unspecified/unavailable.
+  return _GenericErrorMessage(matrix)
+
+
+def _GenericErrorMessage(matrix):
+  return (
+      '\nMatrix [{m}] unexpectedly reached final status {s} without returning '
+      'a URL to any test results in the Firebase console. Please re-check the '
+      'validity of your test files and parameters and try again.'.format(
+          m=matrix.testMatrixId, s=matrix.state))
+

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/util.py

@@ -0,0 +1,286 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A shared library to support implementation of Firebase Test Lab commands."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import json
+
+from apitools.base.py import exceptions as apitools_exceptions
+
+from googlecloudsdk.api_lib.firebase.test import exceptions
+from googlecloudsdk.api_lib.util import apis
+from googlecloudsdk.calliope import exceptions as calliope_exceptions
+from googlecloudsdk.core import log
+from googlecloudsdk.core import properties
+
+OUTCOMES_FORMAT = """
+          table[box](
+            outcome.color(red=Fail, green=Pass, blue=Flaky, yellow=Inconclusive),
+            axis_value:label=TEST_AXIS_VALUE,
+            test_details:label=TEST_DETAILS
+          )
+"""
+
+
+def GetError(error):
+  """Returns a ready-to-print string representation from the http response.
+
+  Args:
+    error: the Http error response, whose content is a JSON-format string for
+      most cases (e.g. invalid test dimension), but can be just a string other
+      times (e.g. invalid URI for CLOUDSDK_TEST_ENDPOINT).
+
+  Returns:
+    A ready-to-print string representation of the error.
+  """
+  try:
+    data = json.loads(error.content)
+  except ValueError:  # message is not JSON
+    return error.content
+
+  code = data['error']['code']
+  message = data['error']['message']
+  return 'ResponseError {0}: {1}'.format(code, message)
+
+
+def GetErrorCodeAndMessage(error):
+  """Returns the individual error code and message from a JSON http response.
+
+  Prefer using GetError(error) unless you need to examine the error code and
+  take specific action on it.
+
+  Args:
+    error: the Http error response, whose content is a JSON-format string.
+
+  Returns:
+    (code, msg) A tuple holding the error code and error message string.
+
+  Raises:
+    ValueError: if the error is not in JSON format.
+  """
+  data = json.loads(error.content)
+  return data['error']['code'], data['error']['message']
+
+
+def GetProject():
+  """Get the user's project id from the core project properties.
+
+  Returns:
+    The id of the GCE project to use while running the test.
+
+  Raises:
+    MissingProjectError: if the user did not specify a project id via the
+      --project flag or via running "gcloud config set project PROJECT_ID".
+  """
+  project = properties.VALUES.core.project.Get()
+  if not project:
+    raise exceptions.MissingProjectError(
+        'No project specified. Please add --project PROJECT_ID to the command'
+        ' line or first run\n  $ gcloud config set project PROJECT_ID')
+  return project
+
+
+def GetDeviceIpBlocks(context=None):
+  """Gets the device IP block catalog from the TestEnvironmentDiscoveryService.
+
+  Args:
+    context: {str:object}, The current context, which is a set of key-value
+      pairs that can be used for common initialization among commands.
+
+  Returns:
+    The device IP block catalog
+
+  Raises:
+    calliope_exceptions.HttpException: If it could not connect to the service.
+  """
+  if context:
+    client = context['testing_client']
+    messages = context['testing_messages']
+  else:
+    client = apis.GetClientInstance('testing', 'v1')
+    messages = apis.GetMessagesModule('testing', 'v1')
+
+  env_type = (
+      messages.TestingTestEnvironmentCatalogGetRequest
+      .EnvironmentTypeValueValuesEnum.DEVICE_IP_BLOCKS)
+  return _GetCatalog(client, messages, env_type).deviceIpBlockCatalog
+
+
+def GetAndroidCatalog(context=None):
+  """Gets the Android catalog from the TestEnvironmentDiscoveryService.
+
+  Args:
+    context: {str:object}, The current context, which is a set of key-value
+      pairs that can be used for common initialization among commands.
+
+  Returns:
+    The android catalog.
+
+  Raises:
+    calliope_exceptions.HttpException: If it could not connect to the service.
+  """
+  if context:
+    client = context['testing_client']
+    messages = context['testing_messages']
+  else:
+    client = apis.GetClientInstance('testing', 'v1')
+    messages = apis.GetMessagesModule('testing', 'v1')
+
+  env_type = (
+      messages.TestingTestEnvironmentCatalogGetRequest.
+      EnvironmentTypeValueValuesEnum.ANDROID)
+  return _GetCatalog(client, messages, env_type).androidDeviceCatalog
+
+
+def GetIosCatalog(context=None):
+  """Gets the iOS catalog from the TestEnvironmentDiscoveryService.
+
+  Args:
+    context: {str:object}, The current context, which is a set of key-value
+      pairs that can be used for common initialization among commands.
+
+  Returns:
+    The iOS catalog.
+
+  Raises:
+    calliope_exceptions.HttpException: If it could not connect to the service.
+  """
+  if context:
+    client = context['testing_client']
+    messages = context['testing_messages']
+  else:
+    client = apis.GetClientInstance('testing', 'v1')
+    messages = apis.GetMessagesModule('testing', 'v1')
+
+  env_type = (
+      messages.TestingTestEnvironmentCatalogGetRequest.
+      EnvironmentTypeValueValuesEnum.IOS)
+  return _GetCatalog(client, messages, env_type).iosDeviceCatalog
+
+
+def GetNetworkProfileCatalog(context=None):
+  """Gets the network profile catalog from the TestEnvironmentDiscoveryService.
+
+  Args:
+    context: {str:object}, The current context, which is a set of key-value
+      pairs that can be used for common initialization among commands.
+
+  Returns:
+    The network profile catalog.
+
+  Raises:
+    calliope_exceptions.HttpException: If it could not connect to the service.
+  """
+  if context:
+    client = context['testing_client']
+    messages = context['testing_messages']
+  else:
+    client = apis.GetClientInstance('testing', 'v1')
+    messages = apis.GetMessagesModule('testing', 'v1')
+
+  env_type = (
+      messages.TestingTestEnvironmentCatalogGetRequest.
+      EnvironmentTypeValueValuesEnum.NETWORK_CONFIGURATION)
+  return _GetCatalog(client, messages, env_type).networkConfigurationCatalog
+
+
+def _GetCatalog(client, messages, environment_type):
+  """Gets a test environment catalog from the TestEnvironmentDiscoveryService.
+
+  Args:
+    client: The Testing API client object.
+    messages: The Testing API messages object.
+    environment_type: {enum} which EnvironmentType catalog to get.
+
+  Returns:
+    The test environment catalog.
+
+  Raises:
+    calliope_exceptions.HttpException: If it could not connect to the service.
+  """
+  project_id = properties.VALUES.core.project.Get()
+  request = messages.TestingTestEnvironmentCatalogGetRequest(
+      environmentType=environment_type,
+      projectId=project_id)
+  try:
+    return client.testEnvironmentCatalog.Get(request)
+  except apitools_exceptions.HttpError as error:
+    raise calliope_exceptions.HttpException(
+        'Unable to access the test environment catalog: ' + GetError(error))
+  except:
+    # Give the user some explanation in case we get a vague/unexpected error,
+    # such as a socket.error from httplib2.
+    log.error('Unable to access the Firebase Test Lab environment catalog.')
+    raise  # Re-raise the error in case Calliope can do something with it.
+
+
+def ParseRoboDirectiveKey(key):
+  """Returns a tuple representing a directive's type and resource name.
+
+  Args:
+    key: the directive key, which can be "<type>:<resource>" or "<resource>"
+
+  Returns:
+    A tuple of the directive's parsed type and resource name. If no type is
+    specified, "text" will be returned as the default type.
+
+  Raises:
+    InvalidArgException: if the input format is incorrect or if the specified
+    type is unsupported.
+  """
+
+  parts = key.split(':')
+  resource_name = parts[-1]
+  if len(parts) > 2:
+    # Invalid format: at most one ':' is allowed.
+    raise exceptions.InvalidArgException(
+        'robo_directives', 'Invalid format for key [{0}]. '
+        'Use a colon only to separate action type and resource name.'.format(
+            key))
+
+  if len(parts) == 1:
+    # Format: '<resource_name>=<input_text>' defaults to 'text'
+    action_type = 'text'
+  else:
+    # Format: '<type>:<resource_name>=<input_value>'
+    action_type = parts[0]
+    supported_action_types = ['text', 'click', 'ignore']
+    if action_type not in supported_action_types:
+      raise exceptions.InvalidArgException(
+          'robo_directives',
+          'Unsupported action type [{0}]. Please choose one of [{1}]'.format(
+              action_type, ', '.join(supported_action_types)))
+
+  return (action_type, resource_name)
+
+
+def GetDeprecatedTagWarning(models, platform='android'):
+  """Returns a warning string iff any device model is marked deprecated."""
+  for model in models:
+    for tag in model.tags:
+      if 'deprecated' in tag:
+        return ('Some devices are deprecated. Learn more at https://firebase.'
+                'google.com/docs/test-lab/%s/'
+                'available-testing-devices#deprecated' % platform)
+  return None
+
+
+def GetRelativeDevicePath(device_path):
+  """Returns the relative device path that can be joined with GCS bucket."""
+  return device_path[1:] if device_path.startswith('/') else device_path