feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/indexes/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2020 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command group for ai-platform index."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.calliope import base
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA,
+                    base.ReleaseTrack.GA)
+class Index(base.Group):
+  """Manage Vertex AI indexes."""
+
+  category = base.VERTEX_AI_CATEGORY

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/indexes/create.py

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Vertex AI indexes create command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.ai.indexes import client
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.ai import constants
+from googlecloudsdk.command_lib.ai import endpoint_util
+from googlecloudsdk.command_lib.ai import flags
+from googlecloudsdk.command_lib.ai import indexes_util
+from googlecloudsdk.command_lib.ai import region_util
+from googlecloudsdk.command_lib.ai import validation
+from googlecloudsdk.command_lib.util.args import labels_util
+from googlecloudsdk.core import log
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+class CreateV1(base.CreateCommand):
+  """Create a new Vertex AI index.
+
+  ## EXAMPLES
+
+  To create an index under project `example` in region
+  `us-central1`, encrypted with KMS key `kms-key-name`, run:
+
+    $ {command} --display-name=index --description=test
+    --metadata-file=path/to/your/metadata.json
+    --project=example --region=us-central1
+    --encryption-kms-key-name=kms-key-name
+  """
+
+  @staticmethod
+  def Args(parser):
+    flags.AddRegionResourceArg(
+        parser,
+        'to create index',
+        prompt_func=region_util.GetPromptForRegionFunc(
+            constants.SUPPORTED_OP_REGIONS
+        ),
+    )
+    flags.GetDisplayNameArg('index').AddToParser(parser)
+    flags.GetDescriptionArg('index').AddToParser(parser)
+    flags.GetMetadataFilePathArg('index', required=True).AddToParser(parser)
+    flags.GetMetadataSchemaUriArg('index').AddToParser(parser)
+    flags.GetIndexUpdateMethod().AddToParser(parser)
+    flags.GetEncryptionKmsKeyNameArg().AddToParser(parser)
+    labels_util.AddCreateLabelsFlags(parser)
+
+  def _Run(self, args, version):
+    validation.ValidateDisplayName(args.display_name)
+    region_ref = args.CONCEPTS.region.Parse()
+    region = region_ref.AsDict()['locationsId']
+    project_id = region_ref.AsDict()['projectsId']
+    with endpoint_util.AiplatformEndpointOverrides(version, region=region):
+      index_client = client.IndexesClient(version=version)
+      if version == constants.GA_VERSION:
+        operation = index_client.Create(region_ref, args)
+      else:
+        operation = index_client.CreateBeta(region_ref, args)
+
+      op_ref = indexes_util.ParseIndexOperation(operation.name)
+      index_id = op_ref.AsDict()['indexesId']
+      log.status.Print(
+          constants.OPERATION_CREATION_DISPLAY_MESSAGE.format(
+              name=operation.name,
+              verb='create index',
+              id=op_ref.Name(),
+              sub_commands='--index={} --region={} [--project={}]'.format(
+                  index_id, region, project_id)))
+      return operation
+
+  def Run(self, args):
+    return self._Run(args, constants.GA_VERSION)
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA)
+class CreateV1Beta1(CreateV1):
+  """Create a new Vertex AI index.
+
+  ## EXAMPLES
+
+  To create an index under project `example` in region
+  `us-central1`, encrypted with KMS key `kms-key-name`, run:
+
+    $ {command} --display-name=index --description=test
+    --metadata-file=path/to/your/metadata.json
+    --project=example --region=us-central1
+    --encryption-kms-key-name=kms-key-name
+  """
+
+  def Run(self, args):
+    return self._Run(args, constants.BETA_VERSION)

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/indexes/delete.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Vertex AI indexes delete command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.ai import operations
+from googlecloudsdk.api_lib.ai.indexes import client
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.ai import constants
+from googlecloudsdk.command_lib.ai import endpoint_util
+from googlecloudsdk.command_lib.ai import flags
+from googlecloudsdk.command_lib.ai import indexes_util
+from googlecloudsdk.command_lib.ai import operations_util
+from googlecloudsdk.core.console import console_io
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+class DeleteV1(base.DeleteCommand):
+  """Delete an existing Vertex AI index.
+
+  ## EXAMPLES
+
+  To delete an index `123` of project `example` in region `us-central1`, run:
+
+    $ {command} 123 --project=example --region=us-central1
+  """
+
+  @staticmethod
+  def Args(parser):
+    flags.AddIndexResourceArg(parser, 'to delete')
+
+  def _Run(self, args, version):
+    index_ref = args.CONCEPTS.index.Parse()
+    region = index_ref.AsDict()['locationsId']
+    index_id = index_ref.AsDict()['indexesId']
+    with endpoint_util.AiplatformEndpointOverrides(version, region=region):
+      console_io.PromptContinue(
+          'This will delete index [{}]...'.format(index_id), cancel_on_no=True)
+      operation = client.IndexesClient(version=version).Delete(index_ref)
+      return operations_util.WaitForOpMaybe(
+          operations_client=operations.OperationsClient(),
+          op=operation,
+          op_ref=indexes_util.ParseIndexOperation(operation.name))
+
+  def Run(self, args):
+    return self._Run(args, constants.GA_VERSION)
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA)
+class DeleteV1Beta1(DeleteV1):
+  """Delete an existing Vertex AI index.
+
+  ## EXAMPLES
+
+  To delete an index `123` of project `example` in region `us-central1`, run:
+
+    $ {command} 123 --project=example --region=us-central1
+  """
+
+  def Run(self, args):
+    return self._Run(args, constants.BETA_VERSION)

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/indexes/describe.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Vertex AI indexes describe command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.ai.indexes import client
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.ai import constants
+from googlecloudsdk.command_lib.ai import endpoint_util
+from googlecloudsdk.command_lib.ai import flags
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+class DescribeV1(base.DescribeCommand):
+  """Gets detailed index information about the given index id.
+
+  ## EXAMPLES
+
+  Describe an index `123` of project `example` in region `us-central1`, run:
+
+    $ {command} 123 --project=example --region=us-central1
+  """
+
+  @staticmethod
+  def Args(parser):
+    flags.AddIndexResourceArg(parser, 'to describe')
+
+  def _Run(self, args, version):
+    index_ref = args.CONCEPTS.index.Parse()
+    region = index_ref.AsDict()['locationsId']
+    with endpoint_util.AiplatformEndpointOverrides(version, region=region):
+      return client.IndexesClient(version=version).Get(index_ref)
+
+  def Run(self, args):
+    return self._Run(args, constants.GA_VERSION)
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA)
+class DescribeV1Beta1(DescribeV1):
+  """Gets detailed index information about the given index id.
+
+  ## EXAMPLES
+
+  Describe an index `123` of project `example` in region `us-central1`, run:
+
+    $ {command} 123 --project=example --region=us-central1
+  """
+
+  def Run(self, args):
+    return self._Run(args, constants.BETA_VERSION)

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/indexes/list.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Vertex AI indexes list command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.ai.indexes import client
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.ai import constants
+from googlecloudsdk.command_lib.ai import endpoint_util
+from googlecloudsdk.command_lib.ai import flags
+from googlecloudsdk.command_lib.ai import region_util
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+class ListV1(base.ListCommand):
+  """Lists the indexes of the given project and region.
+
+  ## EXAMPLES
+
+  Lists the indexes of project `example` in region `us-central1`, run:
+
+    $ {command} --project=example --region=us-central1
+  """
+
+  @staticmethod
+  def Args(parser):
+    flags.AddRegionResourceArg(
+        parser,
+        'to list indexes',
+        prompt_func=region_util.GetPromptForRegionFunc(
+            constants.SUPPORTED_OP_REGIONS
+        ),
+    )
+
+  def _Run(self, args, version):
+    region_ref = args.CONCEPTS.region.Parse()
+    region = region_ref.AsDict()['locationsId']
+    with endpoint_util.AiplatformEndpointOverrides(version, region=region):
+      return client.IndexesClient(version=version).List(region_ref=region_ref)
+
+  def Run(self, args):
+    return self._Run(args, constants.GA_VERSION)
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA)
+class ListV1Beta1(ListV1):
+  """Lists the indexes of the given project and region.
+
+  ## EXAMPLES
+
+  Lists the indexes of project `example` in region `us-central1`, run:
+
+    $ {command} --project=example --region=us-central1
+  """
+
+  def Run(self, args):
+    return self._Run(args, constants.BETA_VERSION)

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/indexes/remove_datapoints.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2022 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Vertex AI indexes remove datapoints command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.ai.indexes import client
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.ai import constants
+from googlecloudsdk.command_lib.ai import endpoint_util
+from googlecloudsdk.command_lib.ai import flags
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+class RemoveDatapointsV1(base.CreateCommand):
+  """Remove data points from the specified index.
+
+  ## EXAMPLES
+
+  To remove datapoints from an index '123', run:
+
+    $ {command} 123 --datapoint-ids=example1,example2
+    --project=example --region=us-central1
+
+  Or put datapoint ids in a json file and run:
+
+    $ {command} 123 --datapoints-from-file=example.json
+    --project=example --region=us-central1
+  """
+
+  @staticmethod
+  def Args(parser):
+    flags.AddIndexResourceArg(parser, 'to remove data points from')
+    flags.AddDatapointSourceGroupForStreamUpdate('index', parser, True)
+
+  def _Run(self, args, version):
+    index_ref = args.CONCEPTS.index.Parse()
+    region = index_ref.AsDict()['locationsId']
+    with endpoint_util.AiplatformEndpointOverrides(version, region=region):
+      index_client = client.IndexesClient(version=version)
+      if version == constants.GA_VERSION:
+        operation = index_client.RemoveDatapoints(index_ref, args)
+      else:
+        operation = index_client.RemoveDatapointsBeta(index_ref, args)
+      return operation
+
+  def Run(self, args):
+    return self._Run(args, constants.GA_VERSION)
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA)
+class RemoveDatapointsV1Beta(RemoveDatapointsV1):
+  """Remove data points from the specified index.
+
+  ## EXAMPLES
+
+  To remove data points from an index `123`, run:
+
+    $ {command} 123 --datapoint-ids=example1,example2
+    --project=example --region=us-central1
+
+  Or put datapoint ids in a JSON file and run:
+
+    $ {command} 123 --datapoints-from-file=example.json
+    --project=example --region=us-central1
+  """
+
+  def Run(self, args):
+    return self._Run(args, constants.BETA_VERSION)

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/indexes/update.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2021 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Vertex AI indexes update command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from apitools.base.py import encoding
+from googlecloudsdk.api_lib.ai import operations
+from googlecloudsdk.api_lib.ai.indexes import client
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.ai import constants
+from googlecloudsdk.command_lib.ai import endpoint_util
+from googlecloudsdk.command_lib.ai import flags
+from googlecloudsdk.command_lib.ai import indexes_util
+from googlecloudsdk.command_lib.ai import operations_util
+from googlecloudsdk.command_lib.util.args import labels_util
+from googlecloudsdk.core import log
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+class UpdateV1(base.UpdateCommand):
+  """Update an Vertex AI index.
+
+  ## EXAMPLES
+
+  To update index `123` under project `example` in region `us-central1`, run:
+
+    $ {command} --display-name=new-name
+    --metadata-file=path/to/your/metadata.json --project=example
+    --region=us-central1
+  """
+
+  @staticmethod
+  def Args(parser):
+    flags.AddIndexResourceArg(parser, 'to update')
+    flags.GetDisplayNameArg('index', required=False).AddToParser(parser)
+    flags.GetDescriptionArg('index').AddToParser(parser)
+    flags.GetMetadataFilePathArg('index').AddToParser(parser)
+    labels_util.AddUpdateLabelsFlags(parser)
+
+  def _Run(self, args, version):
+    index_ref = args.CONCEPTS.index.Parse()
+    region = index_ref.AsDict()['locationsId']
+    project_id = index_ref.AsDict()['projectsId']
+    with endpoint_util.AiplatformEndpointOverrides(version, region=region):
+      index_client = client.IndexesClient(version=version)
+      if version == constants.GA_VERSION:
+        operation = index_client.Patch(index_ref, args)
+      else:
+        operation = index_client.PatchBeta(index_ref, args)
+      if args.metadata_file is not None:
+        # Update index content.
+        op_ref = indexes_util.ParseIndexOperation(operation.name)
+        log.status.Print(
+            constants.OPERATION_CREATION_DISPLAY_MESSAGE.format(
+                name=operation.name,
+                verb='update index',
+                id=op_ref.Name(),
+                sub_commands='--index={} --region={} [--project={}]'.format(
+                    index_ref.Name(), region, project_id)))
+        # We will not wait for the operation since it can take up to hours.
+        return operation
+
+      # Update index meta data.
+      response_msg = operations_util.WaitForOpMaybe(
+          operations_client=operations.OperationsClient(version=version),
+          op=operation,
+          op_ref=indexes_util.ParseIndexOperation(operation.name),
+          log_method='update')
+      if response_msg is not None:
+        response = encoding.MessageToPyValue(response_msg)
+        if 'name' in response:
+          log.UpdatedResource(response['name'], kind='Vertex AI index')
+      return response_msg
+
+  def Run(self, args):
+    return self._Run(args, constants.GA_VERSION)
+
+
+@base.ReleaseTracks(base.ReleaseTrack.BETA, base.ReleaseTrack.ALPHA)
+class UpdateV1Beta1(UpdateV1):
+  """Update an Vertex AI index.
+
+  ## EXAMPLES
+
+  To update index `123` under project `example` in region `us-central1`, run:
+
+    $ {command} --display-name=new-name
+    --metadata-file=path/to/your/metadata.json --project=example
+    --region=us-central1
+  """
+
+  def Run(self, args):
+    return self._Run(args, constants.BETA_VERSION)
+

gcloud auth application-default login/google-cloud-sdk/lib/surface/ai/indexes/upsert_datapoints.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2022 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Vertex AI indexes upsert datapoints command."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.ai.indexes import client
+from googlecloudsdk.calliope import base
+from googlecloudsdk.command_lib.ai import constants
+from googlecloudsdk.command_lib.ai import endpoint_util
+from googlecloudsdk.command_lib.ai import flags
+
+
+@base.ReleaseTracks(base.ReleaseTrack.GA)
+class UpsertDatapointsV1(base.CreateCommand):
+  """Upsert data points into the specified index.
+
+  ## EXAMPLES
+
+  To upsert datapoints into an index '123', run:
+
+    $ {command} 123 --datapoints-from-file=example.json
+    --project=example --region=us-central1
+  """
+
+  @staticmethod
+  def Args(parser):
+    flags.AddIndexResourceArg(parser, 'to upsert data points from')
+    flags.GetDatapointsFilePathArg('index', required=True).AddToParser(parser)
+    flags.GetDynamicMetadataUpdateMaskArg(required=False).AddToParser(parser)
+
+  def _Run(self, args, version):
+    index_ref = args.CONCEPTS.index.Parse()
+    region = index_ref.AsDict()['locationsId']
+    with endpoint_util.AiplatformEndpointOverrides(version, region=region):
+      index_client = client.IndexesClient(version=version)
+      if version == constants.GA_VERSION:
+        operation = index_client.UpsertDatapoints(index_ref, args)
+      else:
+        operation = index_client.UpsertDatapointsBeta(index_ref, args)
+      return operation
+
+  def Run(self, args):
+    return self._Run(args, constants.GA_VERSION)
+
+
+@base.ReleaseTracks(base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA)
+class UpsertDatapointsV1Beta1(UpsertDatapointsV1):
+  """Upsert data points into the specified index.
+
+  ## EXAMPLES
+
+  To upsert datapoints into an index `123`, run:
+
+    $ {command} 123 --datapoints-from-file=example.json
+    --project=example --region=us-central1
+  """
+
+  def Run(self, args):
+    return self._Run(args, constants.BETA_VERSION)