feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/ios/__init__.py

@@ -0,0 +1 @@
+

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/ios/arg_manager.py

@@ -0,0 +1,187 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A shared library for processing and validating iOS test arguments."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.firebase.test import arg_file
+from googlecloudsdk.api_lib.firebase.test import arg_util
+from googlecloudsdk.api_lib.firebase.test import arg_validate
+from googlecloudsdk.api_lib.firebase.test.ios import catalog_manager
+from googlecloudsdk.calliope import exceptions
+
+
+def TypedArgRules():
+  """Returns the rules for iOS test args which depend on the test type.
+
+  This dict is declared in a function rather than globally to avoid garbage
+  collection issues during unit tests.
+
+  Returns:
+    A dict keyed by whether type-specific args are required or optional, and
+    with a nested dict containing any default values for those shared args.
+  """
+  return {
+      'xctest': {
+          'required': ['test'],
+          'optional': [
+              'xcode_version',
+              'xctestrun_file',
+              'test_special_entitlements',
+          ],
+          'defaults': {
+              'test_special_entitlements': False
+          }
+      },
+      'game-loop': {
+          'required': ['app'],
+          'optional': ['scenario_numbers'],
+          'defaults': {
+              'scenario_numbers': [1]
+          }
+      },
+      'robo': {
+          'required': ['app'],
+          'optional': ['test_special_entitlements', 'robo_script'],
+          'defaults': {
+              'test_special_entitlements': False
+          },
+      }
+  }
+
+
+def SharedArgRules():
+  """Returns the rules for iOS test args which are shared by all test types.
+
+  This dict is declared in a function rather than globally to avoid garbage
+  collection issues during unit tests.
+
+  Returns:
+    A dict keyed by whether shared args are required or optional, and with a
+    nested dict containing any default values for those shared args.
+  """
+  return {
+      'required': ['type'],
+      'optional': [
+          'additional_ipas',
+          'async_',
+          'client_details',
+          'device',
+          'directories_to_pull',
+          'network_profile',
+          'num_flaky_test_attempts',
+          'other_files',
+          'record_video',
+          'results_bucket',
+          'results_dir',
+          'results_history_name',
+          'timeout',
+      ],
+      'defaults': {
+          'async_': False,
+          'device': [{}],  # Default dimensions will come from the iOS catalog.
+          'num_flaky_test_attempts': 0,
+          'record_video': True,
+          'timeout': 900,  # 15 minutes
+      }
+  }
+
+
+def AllArgsSet():
+  """Returns a set containing the names of every iOS test arg."""
+  return arg_util.GetSetOfAllTestArgs(TypedArgRules(), SharedArgRules())
+
+
+class IosArgsManager(object):
+  """Manages test arguments for iOS devices."""
+
+  def __init__(self,
+               catalog_mgr=None,
+               typed_arg_rules=None,
+               shared_arg_rules=None):
+    """Constructs an IosArgsManager for a single iOS test matrix.
+
+    Args:
+      catalog_mgr: an IosCatalogManager object.
+      typed_arg_rules: a nested dict of dicts which are keyed first on the test
+        type, then by whether args are required or optional, and what their
+        default values are. If None, the default from TypedArgRules() is used.
+      shared_arg_rules: a dict keyed by whether shared args are required or
+        optional, and with a nested dict containing any default values for those
+        shared args. If None, the default dict from SharedArgRules() is used.
+    """
+    self._catalog_mgr = catalog_mgr or catalog_manager.IosCatalogManager()
+    self._typed_arg_rules = typed_arg_rules or TypedArgRules()
+    self._shared_arg_rules = shared_arg_rules or SharedArgRules()
+
+  def Prepare(self, args):
+    """Load, apply defaults, and perform validation on test arguments.
+
+    Args:
+      args: an argparse namespace. All the arguments that were provided to this
+        gcloud command invocation (i.e. group and command arguments combined).
+        Arg values from an optional arg-file and/or arg default values may be
+        added to this argparse namespace.
+
+    Raises:
+      InvalidArgumentException: If an argument name is unknown, an argument does
+        not contain a valid value, or an argument is not valid when used with
+        the given type of test.
+      RequiredArgumentException: If a required arg is missing.
+    """
+    all_test_args_set = arg_util.GetSetOfAllTestArgs(self._typed_arg_rules,
+                                                     self._shared_arg_rules)
+    args_from_file = arg_file.GetArgsFromArgFile(args.argspec,
+                                                 all_test_args_set)
+
+    arg_util.ApplyLowerPriorityArgs(args, args_from_file, True)
+
+    test_type = self.GetTestTypeOrRaise(args)
+    typed_arg_defaults = self._typed_arg_rules[test_type]['defaults']
+    shared_arg_defaults = self._shared_arg_rules['defaults']
+    arg_util.ApplyLowerPriorityArgs(args, typed_arg_defaults)
+    arg_util.ApplyLowerPriorityArgs(args, shared_arg_defaults)
+    arg_validate.ValidateArgsForTestType(args, test_type, self._typed_arg_rules,
+                                         self._shared_arg_rules,
+                                         all_test_args_set)
+    arg_validate.ValidateDeviceList(args, self._catalog_mgr)
+    arg_validate.ValidateXcodeVersion(args, self._catalog_mgr)
+    arg_validate.ValidateResultsBucket(args)
+    arg_validate.ValidateResultsDir(args)
+    arg_validate.ValidateScenarioNumbers(args)
+    arg_validate.ValidateIosDirectoriesToPullList(args)
+
+  def GetTestTypeOrRaise(self, args):
+    """If the test type is not user-specified, infer the most reasonable value.
+
+    Args:
+      args: an argparse namespace. All the arguments that were provided to this
+        gcloud command invocation (i.e. group and command arguments combined).
+
+    Returns:
+      The type of the test to be run (e.g. 'xctest'), and also sets the 'type'
+      arg if it was not user-specified.
+
+    Raises:
+      InvalidArgumentException if an explicit test type is invalid.
+    """
+    if not args.type:
+      args.type = 'xctest'
+    if args.type not in self._typed_arg_rules:
+      raise exceptions.InvalidArgumentException(
+          'type', "'{0}' is not a valid test type.".format(args.type))
+    return args.type

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/ios/catalog_manager.py

@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A wrapper for working with the iOS Test Environment Catalog."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.firebase.test import exceptions
+from googlecloudsdk.api_lib.firebase.test import util
+
+
+_MODEL_DIMENSION = 'model'
+_VERSION_DIMENSION = 'version'
+_LOCALE_DIMENSION = 'locale'
+_ORIENTATION_DIMENSION = 'orientation'
+
+
+class IosCatalogManager(object):
+  """Encapsulates operations on the iOS TestEnvironmentCatalog."""
+
+  def __init__(self, catalog=None):
+    """Construct an IosCatalogManager object from a TestEnvironmentCatalog.
+
+    Args:
+      catalog: an iOS TestEnvironmentCatalog from Testing API. If it is not
+        injected, the catalog is retrieved from the Testing service.
+
+    Attributes:
+      catalog: an iOS TestEnvironmentCatalog.
+    """
+    self.catalog = catalog or util.GetIosCatalog()
+    models = self.catalog.models
+    versions = self.catalog.versions
+    locales = self.catalog.runtimeConfiguration.locales
+    orientations = self.catalog.runtimeConfiguration.orientations
+
+    self._model_ids = [m.id for m in models]
+    self._version_ids = [v.id for v in versions]
+    self._locale_ids = [l.id for l in locales]
+    self._orientation_ids = [o.id for o in orientations]
+
+    # Dimension defaults are lazily computed and cached by GetDefault* methods.
+    self._default_model = None
+    self._default_version = None
+    self._default_locale = None
+    self._default_orientation = None
+
+  def GetDefaultModel(self):
+    """Return the default model listed in the iOS environment catalog."""
+    model = (self._default_model if self._default_model else
+             self._FindDefaultDimension(self.catalog.models))
+    if not model:
+      raise exceptions.DefaultDimensionNotFoundError(_MODEL_DIMENSION)
+    return model
+
+  def GetDefaultVersion(self):
+    """Return the default version listed in the iOS environment catalog."""
+    version = (self._default_version if self._default_version else
+               self._FindDefaultDimension(self.catalog.versions))
+    if not version:
+      raise exceptions.DefaultDimensionNotFoundError(_VERSION_DIMENSION)
+    return version
+
+  def GetDefaultLocale(self):
+    """Return the default iOS locale."""
+    locales = self.catalog.runtimeConfiguration.locales
+    locale = (
+        self._default_locale
+        if self._default_locale else self._FindDefaultDimension(locales))
+    if not locale:
+      raise exceptions.DefaultDimensionNotFoundError(_LOCALE_DIMENSION)
+    return locale
+
+  def GetDefaultOrientation(self):
+    """Return the default iOS orientation."""
+    orientations = self.catalog.runtimeConfiguration.orientations
+    orientation = (
+        self._default_orientation if self._default_orientation else
+        self._FindDefaultDimension(orientations))
+    if not orientation:
+      raise exceptions.DefaultDimensionNotFoundError(_ORIENTATION_DIMENSION)
+    return orientation
+
+  def _FindDefaultDimension(self, dimension_table):
+    for dimension in dimension_table:
+      if 'default' in dimension.tags:
+        return dimension.id
+    return None
+
+  def ValidateDimensionAndValue(self, dim_name, dim_value):
+    """Validates that a matrix dimension has a valid name and value."""
+    if dim_name == _MODEL_DIMENSION:
+      if dim_value not in self._model_ids:
+        raise exceptions.ModelNotFoundError(dim_value)
+    elif dim_name == _VERSION_DIMENSION:
+      if dim_value not in self._version_ids:
+        raise exceptions.VersionNotFoundError(dim_value)
+    elif dim_name == _LOCALE_DIMENSION:
+      if dim_value not in self._locale_ids:
+        raise exceptions.LocaleNotFoundError(dim_value)
+    elif dim_name == _ORIENTATION_DIMENSION:
+      if dim_value not in self._orientation_ids:
+        raise exceptions.OrientationNotFoundError(dim_value)
+    else:
+      raise exceptions.InvalidDimensionNameError(dim_name)
+    return dim_value
+
+  def ValidateXcodeVersion(self, xcode_version):
+    """Validates that an Xcode version is in the TestEnvironmentCatalog."""
+    if xcode_version not in [xv.version for xv in self.catalog.xcodeVersions]:
+      raise exceptions.XcodeVersionNotFoundError(xcode_version)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/ios/matrix_creator.py

@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2018 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Create iOS test matrices in Firebase Test Lab."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import os
+import uuid
+
+from apitools.base.py import exceptions as apitools_exceptions
+
+from googlecloudsdk.api_lib.firebase.test import matrix_creator_common
+from googlecloudsdk.api_lib.firebase.test import matrix_ops
+from googlecloudsdk.api_lib.firebase.test import util
+from googlecloudsdk.calliope import exceptions
+from googlecloudsdk.core import log
+
+
+def CreateMatrix(args, context, history_id, gcs_results_root, release_track):
+  """Creates a new iOS matrix test in Firebase Test Lab from the user's params.
+
+  Args:
+    args: an argparse namespace. All the arguments that were provided to this
+      gcloud command invocation (i.e. group and command arguments combined).
+    context: {str:obj} dict containing the gcloud command context, which
+      includes the Testing API client+messages libs generated by Apitools.
+    history_id: {str} A history ID to publish Tool Results to.
+    gcs_results_root: the root dir for a matrix within the GCS results bucket.
+    release_track: the release track that the command is invoked from.
+
+  Returns:
+    A TestMatrix object created from the supplied matrix configuration values.
+  """
+  creator = MatrixCreator(args, context, history_id, gcs_results_root,
+                          release_track)
+  return creator.CreateTestMatrix(uuid.uuid4().hex)
+
+
+class MatrixCreator(object):
+  """Creates a single iOS test matrix based on user-supplied test arguments."""
+
+  def __init__(self, args, context, history_id, gcs_results_root,
+               release_track):
+    """Construct an MatrixCreator to be used to create a single test matrix.
+
+    Args:
+      args: an argparse namespace. All the arguments that were provided to this
+        gcloud command invocation (i.e. group and command arguments combined).
+      context: {str:obj} dict containing the gcloud command context, which
+        includes the Testing API client+messages libs generated by Apitools.
+      history_id: {str} A history ID to publish Tool Results to.
+      gcs_results_root: the root dir for a matrix within the GCS results bucket.
+      release_track: the release track that the command is invoked from.
+    """
+    self._project = util.GetProject()
+    self._args = args
+    self._history_id = history_id
+    self._gcs_results_root = gcs_results_root
+    self._client = context['testing_client']
+    self._messages = context['testing_messages']
+    self._release_track = release_track
+
+  def _BuildFileReference(self, filename, use_basename=True):
+    """Build a FileReference pointing to a file in GCS."""
+    if not filename:
+      return None
+    if use_basename:
+      filename = os.path.basename(filename)
+    path = os.path.join(self._gcs_results_root, filename)
+    return self._messages.FileReference(gcsPath=path)
+
+  def _BuildGenericTestSetup(self):
+    """Build an IosTestSetup for an iOS test."""
+    additional_ipas = [
+        self._BuildFileReference(os.path.basename(additional_ipa))
+        for additional_ipa in getattr(self._args, 'additional_ipas', []) or []
+    ]
+    directories_to_pull = []
+    for directory in getattr(self._args, 'directories_to_pull', []) or []:
+      if ':' in directory:
+        bundle, path = directory.split(':')
+        directories_to_pull.append(
+            self._messages.IosDeviceFile(bundleId=bundle, devicePath=path))
+      else:
+        directories_to_pull.append(
+            self._messages.IosDeviceFile(devicePath=directory))
+    device_files = []
+    other_files = getattr(self._args, 'other_files', None) or {}
+    for device_path in other_files.keys():
+      # Device paths are be prefixed by the bundle ID if they refer to an app's
+      # sandboxed filesystem, separated with the device path by ':'
+      idx = device_path.find(':')
+      bundle_id = device_path[:idx] if idx != -1 else None
+      path = device_path[idx + 1:] if idx != -1 else device_path
+      device_files.append(
+          self._messages.IosDeviceFile(
+              content=self._BuildFileReference(
+                  util.GetRelativeDevicePath(path), use_basename=False),
+              bundleId=bundle_id,
+              devicePath=path))
+    return self._messages.IosTestSetup(
+        networkProfile=getattr(self._args, 'network_profile', None),
+        additionalIpas=additional_ipas,
+        pushFiles=device_files,
+        pullDirectories=directories_to_pull)
+
+  def _BuildIosXcTestSpec(self):
+    """Build a TestSpecification for an IosXcTest."""
+    spec = self._messages.TestSpecification(
+        disableVideoRecording=not self._args.record_video,
+        iosTestSetup=self._BuildGenericTestSetup(),
+        testTimeout=matrix_ops.ReformatDuration(self._args.timeout),
+        iosXcTest=self._messages.IosXcTest(
+            testsZip=self._BuildFileReference(self._args.test),
+            xctestrun=self._BuildFileReference(self._args.xctestrun_file),
+            xcodeVersion=self._args.xcode_version,
+            testSpecialEntitlements=getattr(self._args,
+                                            'test_special_entitlements',
+                                            False)))
+    return spec
+
+  def _BuildIosTestLoopTestSpec(self):
+    """Build a TestSpecification for an IosXcTest."""
+    spec = self._messages.TestSpecification(
+        disableVideoRecording=not self._args.record_video,
+        iosTestSetup=self._BuildGenericTestSetup(),
+        testTimeout=matrix_ops.ReformatDuration(self._args.timeout),
+        iosTestLoop=self._messages.IosTestLoop(
+            appIpa=self._BuildFileReference(self._args.app),
+            scenarios=self._args.scenario_numbers))
+    return spec
+
+  def _BuildIosRoboTestSpec(self):
+    """Build a TestSpecification for an iOS Robo test."""
+    spec = self._messages.TestSpecification(
+        disableVideoRecording=not self._args.record_video,
+        iosTestSetup=self._BuildGenericTestSetup(),
+        testTimeout=matrix_ops.ReformatDuration(self._args.timeout),
+        iosRoboTest=self._messages.IosRoboTest(
+            appIpa=self._BuildFileReference(self._args.app)))
+    if getattr(self._args, 'robo_script', None):
+      spec.iosRoboTest.roboScript = self._BuildFileReference(
+          os.path.basename(self._args.robo_script))
+    return spec
+
+  def _TestSpecFromType(self, test_type):
+    """Map a test type into its corresponding TestSpecification message ."""
+    if test_type == 'xctest':
+      return self._BuildIosXcTestSpec()
+    elif test_type == 'game-loop':
+      return self._BuildIosTestLoopTestSpec()
+    elif test_type == 'robo':
+      return self._BuildIosRoboTestSpec()
+    else:  # It's a bug in our arg validation if we ever get here.
+      raise exceptions.InvalidArgumentException(
+          'type', 'Unknown test type "{}".'.format(test_type))
+
+  def _BuildTestMatrix(self, spec):
+    """Build just the user-specified parts of an iOS TestMatrix message.
+
+    Args:
+      spec: a TestSpecification message corresponding to the test type.
+
+    Returns:
+      A TestMatrix message.
+    """
+    devices = [self._BuildIosDevice(d) for d in self._args.device]
+    environment_matrix = self._messages.EnvironmentMatrix(
+        iosDeviceList=self._messages.IosDeviceList(iosDevices=devices))
+
+    gcs = self._messages.GoogleCloudStorage(gcsPath=self._gcs_results_root)
+    hist = self._messages.ToolResultsHistory(
+        projectId=self._project, historyId=self._history_id)
+    results = self._messages.ResultStorage(
+        googleCloudStorage=gcs, toolResultsHistory=hist)
+
+    client_info = matrix_creator_common.BuildClientInfo(
+        self._messages,
+        getattr(self._args, 'client_details', {}) or {}, self._release_track)
+
+    return self._messages.TestMatrix(
+        testSpecification=spec,
+        environmentMatrix=environment_matrix,
+        clientInfo=client_info,
+        resultStorage=results,
+        flakyTestAttempts=self._args.num_flaky_test_attempts or 0)
+
+  def _BuildIosDevice(self, device_map):
+    return self._messages.IosDevice(
+        iosModelId=device_map['model'],
+        iosVersionId=device_map['version'],
+        locale=device_map['locale'],
+        orientation=device_map['orientation'])
+
+  def _BuildTestMatrixRequest(self, request_id):
+    """Build a TestingProjectsTestMatricesCreateRequest for a test matrix.
+
+    Args:
+      request_id: {str} a unique ID for the CreateTestMatrixRequest.
+
+    Returns:
+      A TestingProjectsTestMatricesCreateRequest message.
+    """
+    spec = self._TestSpecFromType(self._args.type)
+    return self._messages.TestingProjectsTestMatricesCreateRequest(
+        projectId=self._project,
+        testMatrix=self._BuildTestMatrix(spec),
+        requestId=request_id)
+
+  def CreateTestMatrix(self, request_id):
+    """Invoke the Testing service to create a test matrix from the user's args.
+
+    Args:
+      request_id: {str} a unique ID for the CreateTestMatrixRequest.
+
+    Returns:
+      The TestMatrix response message from the TestMatrices.Create rpc.
+
+    Raises:
+      HttpException if the test service reports an HttpError.
+    """
+    request = self._BuildTestMatrixRequest(request_id)
+    log.debug('TestMatrices.Create request:\n{0}\n'.format(request))
+    try:
+      response = self._client.projects_testMatrices.Create(request)
+      log.debug('TestMatrices.Create response:\n{0}\n'.format(response))
+    except apitools_exceptions.HttpError as error:
+      msg = 'Http error while creating test matrix: ' + util.GetError(error)
+      raise exceptions.HttpException(msg)
+
+    log.status.Print('Test [{id}] has been created in the Google Cloud.'.format(
+        id=response.testMatrixId))
+    return response