feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/hp_tuning_jobs/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2020 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command group for Vertex AI hyperparameter tuning job."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA,
+                    base.ReleaseTrack.GA)
+class HpTuningJobs(base.Group):
+  """Manage Vertex AI hyperparameter tuning jobs."""
+
+  category = base.VERTEX_AI_CATEGORY

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/hp_tuning_jobs/cancel.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2020 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to cancel a hyperparameter tuning job in Vertex AI."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.ai.hp_tuning_jobs import client
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.ai import constants
+from googlecloudsdk.command_lib.ai import endpoint_util
+from googlecloudsdk.command_lib.ai import validation
+from googlecloudsdk.command_lib.ai.hp_tuning_jobs import flags
+from googlecloudsdk.command_lib.ai.hp_tuning_jobs import hp_tuning_jobs_util
+from googlecloudsdk.core import log
+
+_HPTUNING_JOB_CANCEL_DISPLAY_MESSAGE = """\
+Request to cancel hyperparameter tuning job [{id}] has been sent.
+
+You may view the status of your job with the command
+
+  $ gcloud{command_version} ai hp-tuning-jobs describe {id} --region={region}
+"""
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA, base.ReleaseTrack.BETA,
+                    base.ReleaseTrack.ALPHA)
+class Cancel(base.SilentCommand):
+  """Cancel a running hyperparameter tuning job.
+
+  If the job is already finished, the command will not perform any operation.
+
+  ## EXAMPLES
+
+  To cancel a job ``123'' under project ``example'' in region
+  ``us-central1'', run:
+
+    $ {command} 123 --project=example --region=us-central1
+  """
+
+  @staticmethod
+  def Args(parser):
+    flags.AddHptuningJobResourceArg(parser, 'to cancel')
+
+  def Run(self, args):
+    hptuning_job_ref = args.CONCEPTS.hptuning_job.Parse()
+    region = hptuning_job_ref.AsDict()['locationsId']
+    validation.ValidateRegion(
+        region, available_regions=constants.SUPPORTED_TRAINING_REGIONS)
+
+    version = constants.GA_VERSION if self.ReleaseTrack(
+    ) == base.ReleaseTrack.GA else constants.BETA_VERSION
+    with endpoint_util.AiplatformEndpointOverrides(
+        version=version, region=region):
+      response = client.HpTuningJobsClient(version=version).Cancel(
+          hptuning_job_ref.RelativeName())
+
+      log.status.Print(
+          _HPTUNING_JOB_CANCEL_DISPLAY_MESSAGE.format(
+              id=hptuning_job_ref.Name(),
+              command_version=hp_tuning_jobs_util.OutputCommandVersion(
+                  self.ReleaseTrack()),
+              region=region))
+      return response

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/hp_tuning_jobs/create.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2020 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to create a hyperparameter tuning job in Vertex AI."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.ai import util as api_util
+from googlecloudsdk.api_lib.ai.hp_tuning_jobs import client
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.ai import constants
+from googlecloudsdk.command_lib.ai import endpoint_util
+from googlecloudsdk.command_lib.ai import validation
+from googlecloudsdk.command_lib.ai.hp_tuning_jobs import flags
+from googlecloudsdk.command_lib.ai.hp_tuning_jobs import hp_tuning_jobs_util
+from googlecloudsdk.command_lib.util.apis import arg_utils
+from googlecloudsdk.command_lib.util.args import labels_util
+from googlecloudsdk.core import log
+
+_HPTUNING_JOB_CREATION_DISPLAY_MESSAGE = """\
+Hyperparameter tuning job [{id}] submitted successfully.
+
+Your job is still active. You may view the status of your job with the command
+
+  $ gcloud{command_version} ai hp-tuning-jobs describe {id} --region={region}
+
+Job State: {state}\
+"""
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+class CreateGa(base.CreateCommand):
+  """Create a hyperparameter tuning job."""
+
+  _api_version = constants.GA_VERSION
+
+  detailed_help = {
+      'EXAMPLES':
+          """\
+          To create a job named ``test'' under project ``example'' in region
+          ``us-central1'', run:
+
+            $ {command} --region=us-central1 --project=example --config=config.yaml --display-name=test
+          """
+  }
+
+  @classmethod
+  def Args(cls, parser):
+    flags.AddCreateHpTuningJobFlags(
+        parser,
+        api_util.GetMessage('StudySpec',
+                            version=cls._api_version).AlgorithmValueValuesEnum)
+
+  def Run(self, args):
+    region_ref = args.CONCEPTS.region.Parse()
+    region = region_ref.AsDict()['locationsId']
+    validation.ValidateRegion(
+        region, available_regions=constants.SUPPORTED_TRAINING_REGIONS)
+
+    with endpoint_util.AiplatformEndpointOverrides(
+        version=self._api_version, region=region):
+      api_client = client.HpTuningJobsClient(version=self._api_version)
+      algorithm = arg_utils.ChoiceToEnum(args.algorithm,
+                                         api_client.AlgorithmEnum())
+      labels = labels_util.ParseCreateArgs(
+          args,
+          api_client.HyperparameterTuningJobMessage().LabelsValue)
+
+      response = api_client.Create(
+          parent=region_ref.RelativeName(),
+          config_path=args.config,
+          display_name=args.display_name,
+          max_trial_count=args.max_trial_count,
+          parallel_trial_count=args.parallel_trial_count,
+          algorithm=algorithm,
+          kms_key_name=validation.GetAndValidateKmsKey(args),
+          network=args.network,
+          service_account=args.service_account,
+          enable_web_access=args.enable_web_access,
+          enable_dashboard_access=args.enable_dashboard_access,
+          labels=labels)
+
+      log.status.Print(
+          _HPTUNING_JOB_CREATION_DISPLAY_MESSAGE.format(
+              id=hp_tuning_jobs_util.ParseJobName(response.name),
+              command_version=hp_tuning_jobs_util.OutputCommandVersion(
+                  self.ReleaseTrack()),
+              region=region,
+              state=response.state))
+      return response
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.ALPHA)
+class CreatePreGa(CreateGa):
+  """Create a hyperparameter tuning job."""
+
+  _api_version = constants.BETA_VERSION

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/hp_tuning_jobs/describe.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2020 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to get a hyperparameter tuning job in Vertex AI."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.ai.hp_tuning_jobs import client
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.ai import constants
+from googlecloudsdk.command_lib.ai import endpoint_util
+from googlecloudsdk.command_lib.ai import validation
+from googlecloudsdk.command_lib.ai.hp_tuning_jobs import flags
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA, base.ReleaseTrack.BETA,
+                    base.ReleaseTrack.ALPHA)
+class Describe(base.DescribeCommand):
+  """Get detail information about the hyperparameter tuning job by given id.
+
+  ## EXAMPLES
+
+  To get a job ``123'' under project ``example'' in region
+  ``us-central1'', run:
+
+    $ {command} 123 --project=example --region=us-central1
+  """
+
+  @staticmethod
+  def Args(parser):
+    flags.AddHptuningJobResourceArg(parser, 'to describe')
+
+  def Run(self, args):
+    hptuning_job_ref = args.CONCEPTS.hptuning_job.Parse()
+    region = hptuning_job_ref.AsDict()['locationsId']
+    validation.ValidateRegion(
+        region, available_regions=constants.SUPPORTED_TRAINING_REGIONS)
+
+    version = constants.GA_VERSION if self.ReleaseTrack(
+    ) == base.ReleaseTrack.GA else constants.BETA_VERSION
+    with endpoint_util.AiplatformEndpointOverrides(
+        version=version, region=region):
+      response = client.HpTuningJobsClient(version=version).Get(
+          hptuning_job_ref.RelativeName())
+      return response

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/hp_tuning_jobs/list.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2020 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to list hyperparameter tuning jobs in Vertex AI."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.ai.hp_tuning_jobs import client
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.ai import constants
+from googlecloudsdk.command_lib.ai import endpoint_util
+from googlecloudsdk.command_lib.ai import flags
+from googlecloudsdk.command_lib.ai import region_util
+from googlecloudsdk.command_lib.ai import validation
+from googlecloudsdk.command_lib.ai.hp_tuning_jobs import hp_tuning_jobs_util
+
+_DETAILED_HELP = {
+    'EXAMPLES':
+        """ \
+        To list the jobs of project ``example'' in region
+        ``us-central1'', run:
+
+          $ {command} --project=example --region=us-central1
+        """,
+}
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+class ListGA(base.ListCommand):
+  """List existing hyperparameter tuning jobs."""
+  detailed_help = _DETAILED_HELP
+  _version = constants.GA_VERSION
+
+  @classmethod
+  def Args(cls, parser):
+    """Method called by Calliope to set up arguments for this command.
+
+    Args:
+      parser: A argparse.Parser to register accepted arguments in command input.
+    """
+    flags.AddRegionResourceArg(
+        parser,
+        'to list hyperparameter tuning jobs',
+        prompt_func=region_util.GetPromptForRegionFunc(
+            constants.SUPPORTED_TRAINING_REGIONS))
+    flags.AddUriFlags(parser, hp_tuning_jobs_util.HPTUNING_JOB_COLLECTION,
+                      constants.AI_PLATFORM_API_VERSION[cls._version])
+
+  def Run(self, args):
+    """Executes the list command.
+
+    Args:
+      args: an argparse.Namespace, it contains all arguments that this command
+        was invoked with.
+
+    Returns:
+      The list of resources
+    """
+
+    region_ref = args.CONCEPTS.region.Parse()
+    region = region_ref.AsDict()['locationsId']
+    validation.ValidateRegion(
+        region, available_regions=constants.SUPPORTED_TRAINING_REGIONS)
+
+    with endpoint_util.AiplatformEndpointOverrides(
+        version=self._version, region=region):
+      return client.HpTuningJobsClient(version=self._version).List(
+          region=region_ref.RelativeName())
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.ALPHA)
+class ListPreGA(ListGA):
+  """List existing hyperparameter tuning jobs."""
+  _version = constants.BETA_VERSION

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/hp_tuning_jobs/stream_logs.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2020 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command to check stream logs of a hyperparameter tuning job in Vertex AI."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.ai.hp_tuning_jobs import client
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.ai import constants
+from googlecloudsdk.command_lib.ai import endpoint_util
+from googlecloudsdk.command_lib.ai import flags as common_flags
+from googlecloudsdk.command_lib.ai import log_util
+from googlecloudsdk.command_lib.ai import validation
+from googlecloudsdk.command_lib.ai.hp_tuning_jobs import flags as hp_tuning_job_flags
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA, base.ReleaseTrack.BETA,
+                    base.ReleaseTrack.ALPHA)
+class StreamLogs(base.Command):
+  """Stream logs from a running Vertex AI hyperparameter tuning job.
+
+     ## EXAMPLES
+
+     To stream logs of a hyperparameter tuning job, run:
+
+     $ {command} stream-logs HP_TUNING_JOB
+  """
+
+  @staticmethod
+  def Args(parser):
+    hp_tuning_job_flags.AddHptuningJobResourceArg(parser, 'to fetch stream log')
+    common_flags.AddStreamLogsFlags(parser)
+    parser.display_info.AddFormat(log_util.LOG_FORMAT)
+
+  def Run(self, args):
+    hptuning_job_ref = args.CONCEPTS.hptuning_job.Parse()
+    region = hptuning_job_ref.AsDict()['locationsId']
+    validation.ValidateRegion(
+        region, available_regions=constants.SUPPORTED_TRAINING_REGIONS)
+
+    version = constants.GA_VERSION if self.ReleaseTrack(
+    ) == base.ReleaseTrack.GA else constants.BETA_VERSION
+    with endpoint_util.AiplatformEndpointOverrides(
+        version=version, region=region):
+      relative_name = hptuning_job_ref.RelativeName()
+      return log_util.StreamLogs(
+          hptuning_job_ref.AsDict()['hyperparameterTuningJobsId'],
+          continue_function=client.HpTuningJobsClient(
+              version=version).CheckJobComplete(relative_name),
+          polling_interval=args.polling_interval,
+          task_name=args.task_name,
+          allow_multiline=args.allow_multiline_logs)