feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/datastore/admin_api.py

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Useful commands for interacting with the Cloud Datastore Admin API."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.datastore import constants
+from googlecloudsdk.api_lib.datastore import util
+
+
+def GetExportEntitiesRequest(project,
+                             output_url_prefix,
+                             kinds=None,
+                             namespaces=None,
+                             labels=None):
+  """Returns a request for a Datastore Admin Export.
+
+  Args:
+    project: the project id to export, a string.
+    output_url_prefix: the output GCS path prefix, a string.
+    kinds: a string list of kinds to export.
+    namespaces:  a string list of namespaces to export.
+    labels: a string->string map of client labels.
+  Returns:
+    an ExportRequest message.
+  """
+  messages = util.GetMessages()
+  request_class = messages.GoogleDatastoreAdminV1ExportEntitiesRequest
+
+  labels_message = request_class.LabelsValue()
+  labels_message.additionalProperties = []
+  # We want label creation order to be deterministic.
+  labels = labels or {}
+  for key, value in sorted(labels.items()):
+    labels_message.additionalProperties.append(
+        request_class.LabelsValue.AdditionalProperty(key=key, value=value))
+
+  entity_filter = _MakeEntityFilter(namespaces, kinds)
+  export_request = request_class(
+      labels=labels_message,
+      entityFilter=entity_filter,
+      outputUrlPrefix=output_url_prefix)
+
+  request = messages.DatastoreProjectsExportRequest(
+      projectId=project,
+      googleDatastoreAdminV1ExportEntitiesRequest=export_request)
+  return request
+
+
+def GetImportEntitiesRequest(project,
+                             input_url,
+                             kinds=None,
+                             namespaces=None,
+                             labels=None):
+  """Returns a request for a Datastore Admin Import.
+
+  Args:
+    project: the project id to import, a string.
+    input_url: the location of the GCS overall export file, a string.
+    kinds: a string list of kinds to import.
+    namespaces:  a string list of namespaces to import.
+    labels: a string->string map of client labels.
+  Returns:
+    an ImportRequest message.
+  """
+  messages = util.GetMessages()
+  request_class = messages.GoogleDatastoreAdminV1ImportEntitiesRequest
+
+  entity_filter = _MakeEntityFilter(namespaces, kinds)
+
+  labels_message = request_class.LabelsValue()
+  labels_message.additionalProperties = []
+
+  # We want label creation order to be deterministic.
+  labels = labels or {}
+  for key, value in sorted(labels.items()):
+    labels_message.additionalProperties.append(
+        request_class.LabelsValue.AdditionalProperty(key=key, value=value))
+
+  import_request = request_class(
+      labels=labels_message, entityFilter=entity_filter, inputUrl=input_url)
+
+  return messages.DatastoreProjectsImportRequest(
+      projectId=project,
+      googleDatastoreAdminV1ImportEntitiesRequest=import_request)
+
+
+def Export(project, output_url_prefix, kinds=None, namespaces=None,
+           labels=None):
+  """Performs a Datastore Admin v1 Export.
+
+  Args:
+    project: the project id to export, a string.
+    output_url_prefix: the output GCS path prefix, a string.
+    kinds: a string list of kinds to export.
+    namespaces:  a string list of namespaces to export.
+    labels: a string->string map of client labels.
+  Returns:
+    a google.longrunning.Operation.
+  """
+  return util.GetService().Export(
+      GetExportEntitiesRequest(project, output_url_prefix, kinds, namespaces,
+                               labels))
+
+
+def Import(project, input_url, kinds=None, namespaces=None, labels=None):
+  """Performs a Datastore Admin v1 Import.
+
+  Args:
+    project: the project id to import, a string.
+    input_url: the input url of the GCS overall export file, a string.
+    kinds: a string list of kinds to import.
+    namespaces:  a string list of namespaces to import.
+    labels: a string->string map of client labels.
+  Returns:
+    a google.longrunning.Operation.
+  """
+  return util.GetService().Import(
+      GetImportEntitiesRequest(project, input_url, kinds, namespaces, labels))
+
+
+def _MakeEntityFilter(namespaces, kinds):
+  """Creates an entity filter for the given namespaces and kinds.
+
+  Args:
+    namespaces: a string list of the namespaces to include in the filter.
+    kinds: a string list of the kinds to include in the filter.
+  Returns:
+    a GetMessages().EntityFilter (proto).
+  """
+  namespaces = namespaces or []
+  namespaces = [_TransformNamespaceId(namespace) for namespace in namespaces]
+
+  return util.GetMessages().GoogleDatastoreAdminV1EntityFilter(
+      kinds=kinds or [], namespaceIds=namespaces)
+
+
+def _TransformNamespaceId(namespace_id):
+  """Transforms client namespace conventions into server conventions."""
+  if namespace_id == constants.DEFAULT_NAMESPACE:
+    return ''
+  return namespace_id

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/datastore/constants.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Constants used for Cloud Datastore."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+# Special marker used for the default namespace in gcloud commands.
+DEFAULT_NAMESPACE = '(default)'

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/datastore/index_api.py

@@ -0,0 +1,482 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2019 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for Cloud Datastore index management commands."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import re
+from typing import Sequence, Set, Tuple
+
+from googlecloudsdk.api_lib.datastore import util
+from googlecloudsdk.api_lib.firestore import api_utils as firestore_utils
+from googlecloudsdk.api_lib.firestore import indexes as firestore_indexes
+from googlecloudsdk.appengine.datastore import datastore_index
+from googlecloudsdk.calliope import exceptions
+from googlecloudsdk.core.console import progress_tracker
+from googlecloudsdk.generated_clients.apis.datastore.v1 import datastore_v1_client
+from googlecloudsdk.generated_clients.apis.datastore.v1 import datastore_v1_messages
+from googlecloudsdk.generated_clients.apis.firestore.v1 import firestore_v1_messages
+
+
+def GetIndexesService() -> (
+    datastore_v1_client.DatastoreV1.ProjectsIndexesService
+):
+  """Returns the service for interacting with the Datastore Admin Service.
+
+  This is used to manage the datastore indexes (create/delete).
+  """
+  return util.GetClient().projects_indexes
+
+
+ASCENDING = (
+    util.GetMessages().GoogleDatastoreAdminV1IndexedProperty.DirectionValueValuesEnum.ASCENDING
+)
+
+DESCENDING = (
+    util.GetMessages().GoogleDatastoreAdminV1IndexedProperty.DirectionValueValuesEnum.DESCENDING
+)
+
+NO_ANCESTOR = (
+    util.GetMessages().GoogleDatastoreAdminV1Index.AncestorValueValuesEnum.NONE
+)
+
+ALL_ANCESTORS = (
+    util.GetMessages().GoogleDatastoreAdminV1Index.AncestorValueValuesEnum.ALL_ANCESTORS
+)
+
+CREATING = (
+    util.GetMessages().GoogleDatastoreAdminV1Index.StateValueValuesEnum.CREATING
+)
+
+DATASTORE_API_SCOPE = (
+    firestore_utils.GetMessages().GoogleFirestoreAdminV1Index.ApiScopeValueValuesEnum.DATASTORE_MODE_API
+)
+
+COLLECTION_GROUP = (
+    firestore_utils.GetMessages().GoogleFirestoreAdminV1Index.QueryScopeValueValuesEnum.COLLECTION_GROUP
+)
+
+COLLECTION_RECURSIVE = (
+    firestore_utils.GetMessages().GoogleFirestoreAdminV1Index.QueryScopeValueValuesEnum.COLLECTION_RECURSIVE
+)
+
+FIRESTORE_ASCENDING = (
+    firestore_utils.GetMessages().GoogleFirestoreAdminV1IndexField.OrderValueValuesEnum.ASCENDING
+)
+
+FIRESTORE_DESCENDING = (
+    firestore_utils.GetMessages().GoogleFirestoreAdminV1IndexField.OrderValueValuesEnum.DESCENDING
+)
+
+
+def ApiMessageToIndexDefinition(
+    proto: datastore_v1_messages.GoogleDatastoreAdminV1Index,
+) -> Tuple[str, datastore_index.Index]:
+  """Converts a GoogleDatastoreAdminV1Index to an index definition structure."""
+  properties = []
+  for prop_proto in proto.properties:
+    prop_definition = datastore_index.Property(name=str(prop_proto.name))
+    if prop_proto.direction == DESCENDING:
+      prop_definition.direction = 'desc'
+    else:
+      prop_definition.direction = 'asc'
+    properties.append(prop_definition)
+  index = datastore_index.Index(kind=str(proto.kind), properties=properties)
+  if proto.ancestor is not NO_ANCESTOR:
+    index.ancestor = True
+  return proto.indexId, index
+
+
+def _Fullmatch(regex, string):
+  """Emulate python-3.4 re.fullmatch()."""
+  return re.match('(?:' + regex + r')\Z', string, flags=0)
+
+
+def CollectionIdAndIndexIdFromResourcePath(
+    resource_path: str,
+) -> Tuple[str, str]:
+  """Extracts collectionId and indexId from a collectionGroup resource path.
+
+  Args:
+    resource_path: A str to represent firestore resource path contains
+      collection group. ex: projects/p/databases/d/collectionGroups/c/indexes/i.
+
+  Returns:
+    collection_id: A str to represent the collection id in the resource path.
+    index_id: A str to represent the index id in the resource path.
+
+  Raises:
+    ValueError: If the resource path is invalid.
+  """
+  index_name_pattern = '^projects/([^/]*)/databases/([^/]*)/collectionGroups/([^/]*)/indexes/([^/]*)$'
+  match = _Fullmatch(regex=index_name_pattern, string=resource_path)
+  if not match:
+    raise ValueError('Invalid resource path: {}'.format(resource_path))
+
+  return match.group(3), match.group(4)
+
+
+def FirestoreApiMessageToIndexDefinition(
+    proto: firestore_v1_messages.GoogleFirestoreAdminV1Index,
+) -> Tuple[str, datastore_index.Index]:
+  """Converts a GoogleFirestoreAdminV1Index to an index definition structure.
+
+  Args:
+    proto: GoogleFirestoreAdminV1Index
+
+  Returns:
+    index_id: A str to represent the index id in the resource path.
+    index: A datastore_index.Index that contains index definition.
+
+  Raises:
+    ValueError: If GoogleFirestoreAdminV1Index cannot be converted to index
+    definition structure.
+  """
+  properties = []
+  for field_proto in proto.fields:
+    prop_definition = datastore_index.Property(name=str(field_proto.fieldPath))
+    if field_proto.vectorConfig is not None:
+      prop_definition.vectorConfig = datastore_index.VectorConfig(
+          dimension=field_proto.vectorConfig.dimension,
+          flat=datastore_index.VectorFlatIndex(),
+      )
+    elif field_proto.order == FIRESTORE_DESCENDING:
+      prop_definition.direction = 'desc'
+    else:
+      prop_definition.direction = 'asc'
+    properties.append(prop_definition)
+
+  collection_id, index_id = CollectionIdAndIndexIdFromResourcePath(proto.name)
+  index = datastore_index.Index(kind=str(collection_id), properties=properties)
+  if proto.apiScope != DATASTORE_API_SCOPE:
+    raise ValueError('Invalid api scope: {}'.format(proto.apiScope))
+
+  if proto.queryScope == COLLECTION_RECURSIVE:
+    index.ancestor = True
+  elif proto.queryScope == COLLECTION_GROUP:
+    index.ancestor = False
+  else:
+    raise ValueError('Invalid query scope: {}'.format(proto.queryScope))
+
+  return index_id, index
+
+
+def BuildIndexProto(
+    ancestor: datastore_v1_messages.GoogleDatastoreAdminV1Index.AncestorValueValuesEnum,
+    kind: str,
+    project_id: str,
+    properties: Sequence[datastore_index.Property],
+) -> datastore_v1_messages.GoogleDatastoreAdminV1Index:
+  """Builds and returns a GoogleDatastoreAdminV1Index."""
+  messages = util.GetMessages()
+  proto = messages.GoogleDatastoreAdminV1Index()
+  proto.projectId = project_id
+  proto.kind = kind
+  proto.ancestor = ancestor
+  proto.state = CREATING
+  props = []
+  for prop in properties:
+    prop_proto = messages.GoogleDatastoreAdminV1IndexedProperty()
+    prop_proto.name = prop.name
+    if prop.vectorConfig is not None:
+      raise ValueError(
+          'Vector Indexes cannot be created via the Datastore Admin API'
+      )
+    if prop.direction == 'asc':
+      prop_proto.direction = ASCENDING
+    else:
+      prop_proto.direction = DESCENDING
+    props.append(prop_proto)
+  proto.properties = props
+  return proto
+
+
+def BuildIndexFirestoreProto(
+    name: str,
+    is_ancestor: bool,
+    properties: Sequence[datastore_index.Property],
+    enable_vector: bool = True,
+) -> firestore_v1_messages.GoogleFirestoreAdminV1Index:
+  """Builds and returns a GoogleFirestoreAdminV1Index."""
+  messages = firestore_utils.GetMessages()
+  proto = messages.GoogleFirestoreAdminV1Index()
+
+  proto.name = name
+  proto.queryScope = COLLECTION_RECURSIVE if is_ancestor else COLLECTION_GROUP
+  proto.apiScope = DATASTORE_API_SCOPE
+  fields = []
+  for prop in properties:
+    field_proto = messages.GoogleFirestoreAdminV1IndexField()
+    field_proto.fieldPath = prop.name
+    if prop.vectorConfig is not None:
+      if not enable_vector:
+        raise exceptions.InvalidArgumentException(
+            'index.yaml',
+            'Vector Indexes are currently only supported in the Alpha Track',
+        )
+      field_proto.vectorConfig = messages.GoogleFirestoreAdminV1VectorConfig()
+      field_proto.vectorConfig.dimension = prop.vectorConfig.dimension
+      field_proto.vectorConfig.flat = messages.GoogleFirestoreAdminV1FlatIndex()
+    elif prop.direction == 'asc':
+      field_proto.order = FIRESTORE_ASCENDING
+    else:
+      field_proto.order = FIRESTORE_DESCENDING
+    fields.append(field_proto)
+  proto.fields = fields
+  return proto
+
+
+def BuildIndex(
+    is_ancestor: bool,
+    kind: str,
+    properties: Sequence[Tuple[str, str]],
+) -> datastore_index.Index:
+  """Builds and returns a datastore_index.Index YAML rep object."""
+  index = datastore_index.Index(
+      kind=str(kind),
+      properties=[
+          datastore_index.Property(name=str(prop[0]), direction=prop[1])
+          for prop in properties
+      ],
+  )
+  index.ancestor = is_ancestor
+  return index
+
+
+def NormalizeIndexesForDatastoreApi(
+    indexes: Sequence[datastore_index.Index],
+) -> Set[datastore_index.Index]:
+  """Removes the last index property if it is __key__:asc which is redundant."""
+  indexes = indexes or []
+  for index in indexes or []:
+    NormalizeIndexForDatastoreApi(index)
+  return set(indexes)
+
+
+def NormalizeIndexForDatastoreApi(
+    index: datastore_index.Index,
+) -> datastore_index.Index:
+  """Removes the last index property if it is __key__:asc which is redundant."""
+  if (
+      index.properties
+      # The key property path is represented as __key__ in Datastore API
+      # and __name__ in Firestore API.
+      and index.properties[-1].name in ('__key__', '__name__')
+      and index.properties[-1].direction == 'asc'
+  ):
+    index.properties.pop()
+  return index
+
+
+def NormalizeIndexesForFirestoreApi(
+    indexes: Sequence[datastore_index.Index],
+) -> Set[datastore_index.Index]:
+  """Removes the last index property if it is __name__:asc which is redundant."""
+  indexes = indexes or []
+  for index in indexes or []:
+    NormalizeIndexForFirestoreApi(index)
+  return set(indexes)
+
+
+def NormalizeIndexForFirestoreApi(
+    index: datastore_index.Index,
+) -> datastore_index.Index:
+  """Removes the last index property if it is __name__:asc which is redundant."""
+  # Firestore API returns index with '__name__' as opposed to Datastore which
+  # returns it as '__key__', normalize that here.
+  for prop in index.properties:
+    if prop.name == '__key__':
+      prop.name = '__name__'
+
+  # If the last property is '__name__ ASC', then we can remove it as the backend
+  # assumes that is the case.
+  if (
+      index.properties
+      # The key property path is represented as __key__ in Datastore API
+      # and __name__ in Firestore API.
+      and index.properties[-1].name in ('__key__', '__name__')
+      and index.properties[-1].direction == 'asc'
+  ):
+    index.properties.pop()
+  return index
+
+
+def ListIndexes(project_id: str) -> Sequence[datastore_index.Index]:
+  """Lists all datastore indexes under a database with Datastore Admin API."""
+  response = GetIndexesService().List(
+      util.GetMessages().DatastoreProjectsIndexesListRequest(
+          projectId=project_id
+      )
+  )
+  return {ApiMessageToIndexDefinition(index) for index in response.indexes}
+
+
+def ListDatastoreIndexesViaFirestoreApi(
+    project_id: str,
+    database_id: str,
+) -> Sequence[datastore_index.Index]:
+  """Lists all datastore indexes under a database with Firestore Admin API.
+
+  Args:
+    project_id: A str to represent the project id.
+    database_id: A str to represent the database id.
+
+  Returns:
+    List[index]: A list of datastore_index.Index that contains index definition.
+  """
+  response = firestore_indexes.ListIndexes(project_id, database_id)
+  return {
+      FirestoreApiMessageToIndexDefinition(index)
+      for index in response.indexes
+      if index.apiScope == DATASTORE_API_SCOPE
+  }
+
+
+def CreateIndexesViaDatastoreApi(
+    project_id: str,
+    indexes_to_create: Sequence[datastore_index.Index],
+) -> None:
+  """Sends the index creation requests via the Datastore Admin API."""
+  cnt = 0
+  detail_message = None
+  with progress_tracker.ProgressTracker(
+      '.', autotick=False, detail_message_callback=lambda: detail_message
+  ) as pt:
+    for index in indexes_to_create:
+      GetIndexesService().Create(
+          BuildIndexProto(
+              ALL_ANCESTORS if index.ancestor else NO_ANCESTOR,
+              kind=index.kind,
+              project_id=project_id,
+              properties=index.properties,
+          )
+      )
+      cnt = cnt + 1
+      detail_message = '{0:.0%}'.format(cnt / len(indexes_to_create))
+      pt.Tick()
+
+
+def CreateIndexesViaFirestoreApi(
+    project_id: str,
+    database_id: str,
+    indexes_to_create: Sequence[datastore_index.Index],
+    enable_vector: bool,
+) -> None:
+  """Sends the index creation requests via the Firestore Admin API."""
+  detail_message = None
+  with progress_tracker.ProgressTracker(
+      '.', autotick=False, detail_message_callback=lambda: detail_message
+  ) as pt:
+    for i, index in enumerate(indexes_to_create):
+      firestore_indexes.CreateIndex(
+          project_id,
+          database_id,
+          index.kind,
+          BuildIndexFirestoreProto(
+              name=None,
+              is_ancestor=index.ancestor,
+              properties=index.properties,
+              enable_vector=enable_vector,
+          ),
+      )
+      detail_message = '{0:.0%}'.format(i / len(indexes_to_create))
+      pt.Tick()
+
+
+def DeleteIndexes(
+    project_id: str,
+    indexes_to_delete_ids: Sequence[str],
+) -> None:
+  """Sends the index deletion requests via the Datastore Admin API."""
+  cnt = 0
+  detail_message = None
+  with progress_tracker.ProgressTracker(
+      '.',
+      autotick=False,
+      detail_message_callback=lambda: detail_message,
+  ) as pt:
+    for index_id in indexes_to_delete_ids:
+      GetIndexesService().Delete(
+          util.GetMessages().DatastoreProjectsIndexesDeleteRequest(
+              projectId=project_id, indexId=index_id
+          )
+      )
+      cnt = cnt + 1
+      detail_message = '{0:.0%}'.format(cnt / len(indexes_to_delete_ids))
+      pt.Tick()
+
+
+def DeleteIndexesViaFirestoreApi(
+    project_id: str,
+    database_id: str,
+    indexes_to_delete_ids: Sequence[str],
+) -> None:
+  """Sends the index deletion requests via the Firestore Admin API."""
+  cnt = 0
+  detail_message = None
+  delete_cnt = len(indexes_to_delete_ids)
+  with progress_tracker.ProgressTracker(
+      '.',
+      autotick=False,
+      detail_message_callback=lambda: detail_message,
+  ) as pt:
+    for index_id in indexes_to_delete_ids:
+      firestore_indexes.DeleteIndex(project_id, database_id, index_id)
+      cnt = cnt + 1
+      detail_message = '{0:.0%}'.format(cnt / delete_cnt)
+      pt.Tick()
+
+
+def CreateMissingIndexesViaDatastoreApi(
+    project_id: str,
+    index_definitions: datastore_index.IndexDefinitions,
+) -> None:
+  """Creates the indexes if the index configuration is not present."""
+  indexes = ListIndexes(project_id)
+  normalized_indexes = NormalizeIndexesForDatastoreApi(
+      index_definitions.indexes
+  )
+  new_indexes = normalized_indexes - {index for _, index in indexes}
+  CreateIndexesViaDatastoreApi(project_id, new_indexes)
+
+
+def CreateMissingIndexesViaFirestoreApi(
+    project_id: str,
+    database_id: str,
+    index_definitions: datastore_index.IndexDefinitions,
+    enable_vector: bool,
+) -> None:
+  """Creates the indexes via Firestore API if the index configuration is not present."""
+  existing_indexes = ListDatastoreIndexesViaFirestoreApi(
+      project_id, database_id
+  )
+  # Firestore API returns index with '__name__' field path. Normalizing the
+  # index is required.
+  existing_indexes_normalized = NormalizeIndexesForFirestoreApi(
+      [index for _, index in existing_indexes]
+  )
+  normalized_indexes = NormalizeIndexesForFirestoreApi(
+      index_definitions.indexes
+  )
+  new_indexes = normalized_indexes - existing_indexes_normalized
+
+  CreateIndexesViaFirestoreApi(
+      project_id=project_id,
+      database_id=database_id,
+      indexes_to_create=new_indexes,
+      enable_vector=enable_vector,
+  )

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/datastore/operations.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Useful commands for interacting with the Cloud Datastore Operations API."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from apitools.base.py import list_pager
+from googlecloudsdk.api_lib.util import apis
+from googlecloudsdk.api_lib.util import waiter
+from googlecloudsdk.core import resources
+
+
+_OPERATIONS_API_VERSION = 'v1'
+DEFAULT_PAGE_SIZE = 100
+
+
+def GetClient():
+  """Returns the Cloud Datastore client for operations."""
+  return apis.GetClientInstance('datastore', _OPERATIONS_API_VERSION)
+
+
+def GetService():
+  """Returns the service for interacting with the Operations service."""
+  return GetClient().projects_operations
+
+
+def GetMessages():
+  """Import and return the appropriate operations messages module."""
+  return apis.GetMessagesModule('datastore', _OPERATIONS_API_VERSION)
+
+
+def ListOperations(project, limit=None, operation_filter=None):
+  """Lists all of the Datastore operations.
+
+  Args:
+    project: the project to list operations for, a string.
+    limit: the maximum number of operations to return, an integer. Defaults to
+      positive infinity if unset.
+    operation_filter: a filter to apply to operations, a string.
+
+  Returns:
+    a generator of google.longrunning.Operations.
+  """
+  list_request = GetMessages().DatastoreProjectsOperationsListRequest(
+      filter=operation_filter, name='projects/{0}'.format(project))
+  batch_size = limit if limit else DEFAULT_PAGE_SIZE
+  return list_pager.YieldFromList(
+      GetService(),
+      list_request,
+      limit=limit,
+      batch_size=batch_size,
+      field='operations',
+      batch_size_attribute='pageSize')
+
+
+def GetOperation(name):
+  """Returns the google.longrunning.Operation with the given name."""
+  return GetService().Get(
+      GetMessages().DatastoreProjectsOperationsGetRequest(name=name))
+
+
+def CancelOperation(name):
+  """Cancels the google.longrunning.Operation with the given name."""
+  return GetService().Cancel(
+      GetMessages().DatastoreProjectsOperationsCancelRequest(name=name))
+
+
+def DeleteOperation(name):
+  """Deletes the google.longrunning.Operation with the given name."""
+  return GetService().Delete(
+      GetMessages().DatastoreProjectsOperationsDeleteRequest(name=name))
+
+
+def WaitForOperation(operation):
+  """Waits for the given google.longrunning.Operation to complete."""
+  operation_ref = resources.REGISTRY.Parse(
+      operation.name, collection='datastore.projects.operations')
+  poller = waiter.CloudOperationPollerNoResources(GetService())
+  return waiter.WaitFor(
+      poller, operation_ref,
+      'Waiting for [{0}] to finish'.format(operation_ref.RelativeName()))

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/datastore/rewrite_backend.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2017 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Backend rewrite tool for Cloud Datastore operations."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+import re
+
+from googlecloudsdk.api_lib.datastore import constants
+from googlecloudsdk.core.resource import resource_expr_rewrite
+import six
+
+
+class OperationsRewriteBackend(resource_expr_rewrite.Backend):
+  """Rewrites for Cloud Datastore server side filter expressions."""
+
+  _KEY_MAPPING = {
+      r'^label\.(.*)': r'metadata.common.labels.\1',
+      r'^labels\.(.*)': r'metadata.common.labels.\1',
+      '^namespace$': 'metadata.entity_filter.namespace_id',
+      '^namespaceId$': 'metadata.entity_filter.namespace_id',
+      '^type$': 'metadata.common.operation_type',
+      '^operationType$': 'metadata.common.operation_type',
+      '^kind$': 'metadata.entity_filter.kind',
+  }
+
+  _OPERATOR_MAPPING = {
+      # Datastore admin backends only supports EQ, not HAS
+      ':': '='
+  }
+
+  _KEY_OPERAND_MAPPING = {
+      'metadata.entity_filter.namespace_id': {
+          constants.DEFAULT_NAMESPACE: '',
+      },
+  }
+
+  def RewriteTerm(self, key, op, operand, key_type):
+    """Rewrites a <key op operand> term of a filter expression.
+
+    Args:
+      key: The key, a string.
+      op: The op, a string.
+      operand: The operand, a string or list of strings.
+      key_type: The key type, unknown if None.
+
+    Returns:
+      the new term, as a string.
+    """
+    key = self._RewriteKey(key)
+    op = self._RewriteOp(op)
+    operand = self._RewriteOperand(key, operand)
+    return super(OperationsRewriteBackend, self).RewriteTerm(
+        key, op, operand, key_type)
+
+  def Quote(self, value, always=False):
+    """Returns value or value "..." quoted with C-style escapes if needed.
+
+    Defers to BackendBase.Quote for everything but the empty string, which it
+    force quotes.
+
+    Args:
+      value: The string value to quote if needed.
+      always: Always quote non-numeric value if True.
+
+    Returns:
+      A string: value or value "..." quoted with C-style escapes if needed or
+      requested.
+    """
+    # The Cloud Datastore backend does not handle missing values. Always
+    # require quoting for the empty string.
+    always = always or not value
+    return super(OperationsRewriteBackend, self).Quote(value, always=always)
+
+  def _RewriteOperand(self, key, operand):
+    if isinstance(operand, list):
+      return [
+          self._RewriteOperand(key, operand_item) for operand_item in operand
+      ]
+    return self._KEY_OPERAND_MAPPING.get(key, {}).get(operand, operand)
+
+  def _RewriteKey(self, key):
+    for regex, replacement in six.iteritems(self._KEY_MAPPING):
+      if re.match(regex, key):
+        return re.sub(regex, replacement, key)
+    return key
+
+  def _RewriteOp(self, op):
+    return self._OPERATOR_MAPPING.get(op, op)

gcloud auth application-default login/google-cloud-sdk/lib/googlecloudsdk/api_lib/datastore/util.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*- #
+# Copyright 2019 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for Cloud Datastore datastore management commands."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from googlecloudsdk.api_lib.util import apis
+
+_DATASTORE_API_VERSION = 'v1'
+
+
+def GetMessages():
+  """Import and return the appropriate admin messages module."""
+  return apis.GetMessagesModule('datastore', _DATASTORE_API_VERSION)
+
+
+def GetClient():
+  """Returns the Cloud Datastore client for the appropriate release track."""
+  return apis.GetClientInstance('datastore', _DATASTORE_API_VERSION)
+
+
+def GetService():
+  """Returns the service for interacting with the Datastore Admin service.
+
+  This is used for import/export Datastore indexes.
+  """
+  return GetClient().projects