2989 lines
104 KiB
Python
2989 lines
104 KiB
Python
#
|
|
# Copyright 2006 Google LLC. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""The Python datastore API used by app developers.
|
|
|
|
Defines Entity, Query, and Iterator classes, as well as methods for all of the
|
|
datastore's calls. Also defines conversions between the Python classes and
|
|
their PB counterparts.
|
|
|
|
The datastore errors are defined in the datastore_errors module. That module is
|
|
only required to avoid circular imports. datastore imports datastore_types,
|
|
which needs BadValueError, so it can't be defined in datastore.
|
|
"""
|
|
|
|
|
|
# WARNING: This file is externally viewable by our users. All comments from
|
|
# this file will be stripped. The docstrings will NOT. Do not put sensitive
|
|
# information in docstrings. If you must communicate internal information in
|
|
# this source file, please place them in comments only.
|
|
|
|
# WARNING: Be careful when adding any new properties, since they can cause
|
|
# backward incompatibility! See http://b/1836832 for an example.
|
|
|
|
|
|
# See apphosting/datastore/datastore_v3.proto and
|
|
|
|
# TODO(user): Update doc strings to have a consistent syntax.
|
|
|
|
import heapq
|
|
import itertools
|
|
import logging
|
|
import os
|
|
import re
|
|
import sys
|
|
import threading # Knowing full well that this is a usually a placeholder.
|
|
import traceback
|
|
from xml.sax import saxutils
|
|
from googlecloudsdk.appengine.api import apiproxy_stub_map
|
|
from googlecloudsdk.appengine.api import capabilities
|
|
from googlecloudsdk.appengine.api import datastore_errors
|
|
from googlecloudsdk.appengine.api import datastore_types
|
|
from googlecloudsdk.appengine.datastore import datastore_pb
|
|
from googlecloudsdk.appengine.datastore import datastore_query
|
|
from googlecloudsdk.appengine.datastore import datastore_rpc
|
|
from googlecloudsdk.appengine.googlestorage.onestore.v3 import entity_pb
|
|
from googlecloudsdk.core.util import encoding
|
|
|
|
|
|
# The maximum number of subqueries to run in a multi-query.
|
|
MAX_ALLOWABLE_QUERIES = 30
|
|
|
|
# Maximum results allowed to be returned in a single rpc call (unused).
|
|
MAXIMUM_RESULTS = 1000
|
|
|
|
# the number of times to retry transactions. this isn't technically necessary;
|
|
# we could retry forever, and depend on the appserver's request deadline to
|
|
# cut us off. if we're retrying over and over, though, there's probably a
|
|
# reason, so we probably shouldn't keep hammering on the datastore.
|
|
DEFAULT_TRANSACTION_RETRIES = 3
|
|
|
|
# These capabilities can be used to determine if the datastore API is enabled.
|
|
READ_CAPABILITY = capabilities.CapabilitySet('datastore_v3')
|
|
WRITE_CAPABILITY = capabilities.CapabilitySet(
|
|
'datastore_v3',
|
|
capabilities=['write'])
|
|
|
|
# Maximum number of indexed property values per entity. This technically
|
|
# includes composite properties generated by composite indices, but we don't
|
|
# check against those here in the python API.
|
|
#
|
|
# and the docs. TODO(user,user): doc url.
|
|
_MAX_INDEXED_PROPERTIES = 20000
|
|
|
|
# The maximum number of ids that can be allocated at a time.
|
|
_MAX_ID_BATCH_SIZE = datastore_rpc._MAX_ID_BATCH_SIZE
|
|
|
|
Key = datastore_types.Key
|
|
typename = datastore_types.typename
|
|
|
|
# Flags to determine read policy.
|
|
STRONG_CONSISTENCY = datastore_rpc.Configuration.STRONG_CONSISTENCY
|
|
EVENTUAL_CONSISTENCY = datastore_rpc.Configuration.EVENTUAL_CONSISTENCY
|
|
|
|
# The largest possible signed 32-bit integer. Used to ensure that int32 proto
|
|
# fields do not overflow.
|
|
_MAX_INT_32 = 2**31-1
|
|
|
|
|
|
def NormalizeAndTypeCheck(arg, types):
|
|
"""Normalizes and type checks the given argument.
|
|
|
|
Args:
|
|
arg: an instance or iterable of the given type(s)
|
|
types: allowed type or tuple of types
|
|
|
|
Returns:
|
|
A (list, bool) tuple. The list is a normalized, shallow copy of the
|
|
argument. The boolean is True if the argument was a sequence, False
|
|
if it was a single object.
|
|
|
|
Raises:
|
|
AssertionError: types includes list or tuple.
|
|
BadArgumentError: arg is not an instance or sequence of one of the given
|
|
types.
|
|
"""
|
|
if not isinstance(types, (list, tuple)):
|
|
types = (types,)
|
|
|
|
assert list not in types and tuple not in types
|
|
|
|
if isinstance(arg, types):
|
|
# single instance
|
|
return [arg], False
|
|
else:
|
|
# Need to detect strings explicitly because iterating over the characters
|
|
# of a string will cause the wrong error message.
|
|
if isinstance(arg, basestring):
|
|
raise datastore_errors.BadArgumentError(
|
|
'Expected an instance or iterable of %s; received %s (a %s).' %
|
|
(types, arg, typename(arg)))
|
|
|
|
try:
|
|
# shallow copy
|
|
arg_list = list(arg)
|
|
except TypeError:
|
|
# not a sequence
|
|
raise datastore_errors.BadArgumentError(
|
|
'Expected an instance or iterable of %s; received %s (a %s).' %
|
|
(types, arg, typename(arg)))
|
|
|
|
# type check each element
|
|
for val in arg_list:
|
|
if not isinstance(val, types):
|
|
raise datastore_errors.BadArgumentError(
|
|
'Expected one of %s; received %s (a %s).' %
|
|
(types, val, typename(val)))
|
|
|
|
return arg_list, True
|
|
|
|
|
|
def NormalizeAndTypeCheckKeys(keys):
|
|
"""Normalizes and type checks that the given argument is a valid key or keys.
|
|
|
|
A wrapper around NormalizeAndTypeCheck() that accepts strings, Keys, and
|
|
Entities, and normalizes to Keys.
|
|
|
|
Args:
|
|
keys: a Key or sequence of Keys
|
|
|
|
Returns:
|
|
A (list of Keys, bool) tuple. See NormalizeAndTypeCheck.
|
|
|
|
Raises:
|
|
BadArgumentError: arg is not an instance or sequence of one of the given
|
|
types.
|
|
"""
|
|
keys, multiple = NormalizeAndTypeCheck(keys, (basestring, Entity, Key))
|
|
|
|
keys = [_GetCompleteKeyOrError(key) for key in keys]
|
|
|
|
return (keys, multiple)
|
|
|
|
|
|
def _GetConfigFromKwargs(kwargs, convert_rpc=False,
|
|
config_class=datastore_rpc.Configuration):
|
|
"""Get a Configuration object from the keyword arguments.
|
|
|
|
This is purely an internal helper for the various public APIs below
|
|
such as Get().
|
|
|
|
Args:
|
|
kwargs: A dict containing the keyword arguments passed to a public API.
|
|
convert_rpc: If the an rpc should be converted or passed on directly.
|
|
config_class: The config class that should be generated.
|
|
|
|
Returns:
|
|
A UserRPC instance, or a Configuration instance, or None.
|
|
|
|
Raises:
|
|
TypeError if unexpected keyword arguments are present.
|
|
"""
|
|
if not kwargs:
|
|
return None
|
|
|
|
# Legacy support for rpc= keyword
|
|
rpc = kwargs.pop('rpc', None)
|
|
if rpc is not None:
|
|
if not isinstance(rpc, apiproxy_stub_map.UserRPC):
|
|
raise datastore_errors.BadArgumentError(
|
|
'rpc= argument should be None or a UserRPC instance')
|
|
if 'config' in kwargs:
|
|
raise datastore_errors.BadArgumentError(
|
|
'Expected rpc= or config= argument but not both')
|
|
if not convert_rpc:
|
|
if kwargs:
|
|
raise datastore_errors.BadArgumentError(
|
|
'Unexpected keyword arguments: %s' % ', '.join(kwargs))
|
|
return rpc
|
|
|
|
# converting
|
|
read_policy = getattr(rpc, 'read_policy', None)
|
|
kwargs['config'] = datastore_rpc.Configuration(
|
|
deadline=rpc.deadline, read_policy=read_policy,
|
|
config=_GetConnection().config)
|
|
|
|
return config_class(**kwargs)
|
|
|
|
|
|
class _BaseIndex(object):
|
|
|
|
# Index states
|
|
BUILDING, SERVING, DELETING, ERROR = range(4)
|
|
|
|
# TODO(user): Support mode, and tolerate unspecified direction for
|
|
# index_list() API.
|
|
|
|
# Sort direction
|
|
ASCENDING = datastore_query.PropertyOrder.ASCENDING
|
|
DESCENDING = datastore_query.PropertyOrder.DESCENDING
|
|
|
|
def __init__(self, index_id, kind, has_ancestor, properties):
|
|
"""Construct a datastore index instance.
|
|
|
|
Args:
|
|
index_id: Required long; Uniquely identifies the index
|
|
kind: Required string; Specifies the kind of the entities to index
|
|
has_ancestor: Required boolean; indicates if the index supports a query
|
|
that filters entities by the entity group parent
|
|
properties: Required list of (string, int) tuples; The entity properties
|
|
to index. First item in a tuple is the property name and the second
|
|
item is the sorting direction (ASCENDING|DESCENDING).
|
|
The order of the properties is based on the order in the index.
|
|
"""
|
|
argument_error = datastore_errors.BadArgumentError
|
|
datastore_types.ValidateInteger(index_id, 'index_id', argument_error,
|
|
zero_ok=True)
|
|
datastore_types.ValidateString(kind, 'kind', argument_error, empty_ok=True)
|
|
if not isinstance(properties, (list, tuple)):
|
|
raise argument_error('properties must be a list or a tuple')
|
|
for idx, index_property in enumerate(properties):
|
|
if not isinstance(index_property, (list, tuple)):
|
|
raise argument_error('property[%d] must be a list or a tuple' % idx)
|
|
if len(index_property) != 2:
|
|
raise argument_error('property[%d] length should be 2 but was %d' %
|
|
(idx, len(index_property)))
|
|
datastore_types.ValidateString(index_property[0], 'property name',
|
|
argument_error)
|
|
_BaseIndex.__ValidateEnum(index_property[1],
|
|
(self.ASCENDING, self.DESCENDING),
|
|
'sort direction')
|
|
self.__id = long(index_id)
|
|
self.__kind = kind
|
|
self.__has_ancestor = bool(has_ancestor)
|
|
self.__properties = properties
|
|
|
|
@staticmethod
|
|
def __ValidateEnum(value, accepted_values, name='value',
|
|
exception=datastore_errors.BadArgumentError):
|
|
datastore_types.ValidateInteger(value, name, exception)
|
|
if not value in accepted_values:
|
|
raise exception('%s should be one of %s but was %d' %
|
|
(name, str(accepted_values), value))
|
|
|
|
def _Id(self):
|
|
"""Returns the index id, a long."""
|
|
return self.__id
|
|
|
|
def _Kind(self):
|
|
"""Returns the index kind, a string. Empty string ('') if none."""
|
|
return self.__kind
|
|
|
|
def _HasAncestor(self):
|
|
"""Indicates if this is an ancestor index, a boolean."""
|
|
return self.__has_ancestor
|
|
|
|
def _Properties(self):
|
|
"""Returns the index properties. a tuple of
|
|
(index name as a string, [ASCENDING|DESCENDING]) tuples.
|
|
"""
|
|
return self.__properties
|
|
|
|
def __eq__(self, other):
|
|
return self.__id == other.__id
|
|
|
|
def __ne__(self, other):
|
|
return self.__id != other.__id
|
|
|
|
def __hash__(self):
|
|
return hash(self.__id)
|
|
|
|
|
|
class Index(_BaseIndex):
|
|
"""A datastore index."""
|
|
|
|
Id = _BaseIndex._Id
|
|
Kind = _BaseIndex._Kind
|
|
HasAncestor = _BaseIndex._HasAncestor
|
|
Properties = _BaseIndex._Properties
|
|
|
|
|
|
class DatastoreAdapter(datastore_rpc.AbstractAdapter):
|
|
"""Adapter between datatypes defined here (Entity etc.) and protobufs.
|
|
|
|
See the base class in datastore_rpc.py for more docs.
|
|
"""
|
|
|
|
# mappings from pb index state to api index state
|
|
index_state_mappings = {
|
|
entity_pb.CompositeIndex.ERROR: Index.ERROR,
|
|
entity_pb.CompositeIndex.DELETED: Index.DELETING,
|
|
entity_pb.CompositeIndex.READ_WRITE: Index.SERVING,
|
|
entity_pb.CompositeIndex.WRITE_ONLY: Index.BUILDING
|
|
}
|
|
|
|
# mappings from pb index direction to api index direction
|
|
index_direction_mappings = {
|
|
entity_pb.Index_Property.ASCENDING: Index.ASCENDING,
|
|
entity_pb.Index_Property.DESCENDING: Index.DESCENDING
|
|
}
|
|
|
|
def __init__(self, _id_resolver=None):
|
|
super(DatastoreAdapter, self).__init__(_id_resolver)
|
|
|
|
def key_to_pb(self, key):
|
|
return key._Key__reference
|
|
|
|
def pb_to_key(self, pb):
|
|
return Key._FromPb(pb)
|
|
|
|
def entity_to_pb(self, entity):
|
|
return entity._ToPb()
|
|
|
|
def pb_to_entity(self, pb):
|
|
return Entity._FromPb(pb)
|
|
|
|
def pb_to_index(self, pb):
|
|
index_def = pb.definition()
|
|
properties = [(property.name().decode('utf-8'),
|
|
DatastoreAdapter.index_direction_mappings.get(property.direction()))
|
|
for property in index_def.property_list()]
|
|
index = Index(pb.id(), index_def.entity_type().decode('utf-8'),
|
|
index_def.ancestor(), properties)
|
|
state = DatastoreAdapter.index_state_mappings.get(pb.state())
|
|
return index, state
|
|
|
|
|
|
_adapter = DatastoreAdapter() # This does not need to be thread-local.
|
|
_thread_local = threading.local() # Our thread-local variables live here.
|
|
|
|
# Environment key used to detect new requests.
|
|
_ENV_KEY = '__DATASTORE_CONNECTION_INITIALIZED__'
|
|
|
|
|
|
def __InitConnection():
|
|
"""Internal method to make sure the connection state has been initialized."""
|
|
# While we do everything we can to restore the connection using a
|
|
# try/finally clause in RunInTransactionCustomRetries(), it is
|
|
# possible that a DeadlineExceededError prevents the finally clause
|
|
# from running, without killing the process. The next request would
|
|
# then find a TransactionalConnection containing transaction state
|
|
# from a previous request, and havoc would ensure (see b/3101260).
|
|
# So we need to detect the first __InitConnection() call for a new
|
|
# request. The most reliable way to do this is to check for an
|
|
# environment variable that we set: the runtime code clears out the
|
|
# environment at the start of each request.
|
|
# NOTE: In VMs and tests the env is shared by all threads, so _ENV_KEY alone
|
|
# is not a reliable way to check if the state has been initialized.
|
|
if encoding.GetEncodedValue(os.environ, _ENV_KEY) and\
|
|
hasattr(_thread_local, 'connection_stack'):
|
|
return
|
|
|
|
# pylint: disable=invalid-name,protected-access
|
|
def CreateConnection(adapter=None,
|
|
_id_resolver=None,
|
|
_api_version=datastore_rpc._DATASTORE_V3):
|
|
if _id_resolver:
|
|
adapter = DatastoreAdapter(_id_resolver=_id_resolver)
|
|
return datastore_rpc.Connection(adapter=adapter, _api_version=_api_version)
|
|
|
|
_thread_local.connection_stack = [
|
|
datastore_rpc._CreateDefaultConnection(CreateConnection,
|
|
adapter=_adapter)
|
|
]
|
|
|
|
# Record that we initialized the connection.
|
|
os.environ[_ENV_KEY] = '1'
|
|
|
|
|
|
def _GetConnection():
|
|
"""Internal method to retrieve a datastore connection local to the thread."""
|
|
__InitConnection()
|
|
return _thread_local.connection_stack[-1]
|
|
|
|
|
|
def _SetConnection(connection):
|
|
"""Internal method to replace the current thread local connection."""
|
|
__InitConnection()
|
|
_thread_local.connection_stack[-1] = connection
|
|
|
|
|
|
def _PushConnection(new_connection):
|
|
"""Internal method to save the current connection and sets a new one.
|
|
|
|
Args:
|
|
new_connection: The connection to set.
|
|
"""
|
|
__InitConnection()
|
|
_thread_local.connection_stack.append(new_connection)
|
|
|
|
|
|
def _PopConnection():
|
|
"""Internal method to restores the previous connection.
|
|
|
|
Returns:
|
|
The current connection.
|
|
"""
|
|
# The stack must have at least one connection after the pop.
|
|
assert len(_thread_local.connection_stack) >= 2
|
|
return _thread_local.connection_stack.pop()
|
|
|
|
|
|
# TODO(user): Get rid of all calls to _MakeSyncCall() in this file.
|
|
# (But keep the definition for backwards compatibility.)
|
|
|
|
def _MakeSyncCall(service, call, request, response, config=None):
|
|
"""The APIProxy entry point for a synchronous API call.
|
|
|
|
Args:
|
|
service: For backwards compatibility, must be 'datastore_v3'.
|
|
call: String representing which function to call.
|
|
request: Protocol buffer for the request.
|
|
response: Protocol buffer for the response.
|
|
config: Optional Configuration to use for this request.
|
|
|
|
Returns:
|
|
Response protocol buffer. Caller should always use returned value
|
|
which may or may not be same as passed in 'response'.
|
|
|
|
Raises:
|
|
apiproxy_errors.Error or a subclass.
|
|
"""
|
|
conn = _GetConnection()
|
|
if isinstance(request, datastore_pb.Query):
|
|
conn._set_request_read_policy(request, config)
|
|
conn._set_request_transaction(request)
|
|
rpc = conn._make_rpc_call(config, call, request, response)
|
|
conn.check_rpc_success(rpc)
|
|
return response
|
|
|
|
|
|
def CreateRPC(service='datastore_v3',
|
|
deadline=None, callback=None, read_policy=None):
|
|
"""Create an rpc for use in configuring datastore calls.
|
|
|
|
NOTE: This functions exists for backwards compatibility. Please use
|
|
CreateConfig() instead. NOTE: the latter uses 'on_completion',
|
|
which is a function taking an argument, wherease CreateRPC uses
|
|
'callback' which is a function without arguments.
|
|
|
|
Args:
|
|
service: Optional string; for backwards compatibility, must be
|
|
'datastore_v3'.
|
|
deadline: Optional int or float, deadline for calls in seconds.
|
|
callback: Optional callable, a callback triggered when this rpc
|
|
completes; takes no arguments.
|
|
read_policy: Optional read policy; set to EVENTUAL_CONSISTENCY to
|
|
enable eventually consistent reads (i.e. reads that may be
|
|
satisfied from an older version of the datastore in some cases).
|
|
The default read policy may have to wait until in-flight
|
|
transactions are committed.
|
|
|
|
Returns:
|
|
A UserRPC instance.
|
|
"""
|
|
assert service == 'datastore_v3' # Always leave this the default.
|
|
conn = _GetConnection()
|
|
config = None
|
|
if deadline is not None:
|
|
config = datastore_rpc.Configuration(deadline=deadline)
|
|
rpc = conn._create_rpc(config)
|
|
rpc.callback = callback # Never use the connection's callback
|
|
if read_policy is not None:
|
|
rpc.read_policy = read_policy
|
|
return rpc
|
|
|
|
|
|
def CreateConfig(**kwds):
|
|
"""Create a Configuration object for use in configuring datastore calls.
|
|
|
|
This configuration can be passed to most datastore calls using the
|
|
'config=...' argument.
|
|
|
|
Args:
|
|
deadline: Optional deadline; default None (which means the
|
|
system default deadline will be used, typically 5 seconds).
|
|
on_completion: Optional callback function; default None. If
|
|
specified, it will be called with a UserRPC object as argument
|
|
when an RPC completes.
|
|
read_policy: Optional read policy; set to EVENTUAL_CONSISTENCY to
|
|
enable eventually consistent reads (i.e. reads that may be
|
|
satisfied from an older version of the datastore in some cases).
|
|
The default read policy may have to wait until in-flight
|
|
transactions are committed.
|
|
**kwds: Other keyword arguments as long as they are supported by
|
|
datastore_rpc.Configuration().
|
|
|
|
Returns:
|
|
A datastore_rpc.Configuration instance.
|
|
"""
|
|
return datastore_rpc.Configuration(**kwds)
|
|
|
|
|
|
def CreateTransactionOptions(**kwds):
|
|
"""Create a configuration object for use in configuring transactions.
|
|
|
|
This configuration can be passed as run_in_transaction_option's first
|
|
argument.
|
|
|
|
Args:
|
|
deadline: Optional deadline; default None (which means the
|
|
system default deadline will be used, typically 5 seconds).
|
|
on_completion: Optional callback function; default None. If
|
|
specified, it will be called with a UserRPC object as argument
|
|
when an RPC completes.
|
|
xg: set to true to allow cross-group transactions (high replication
|
|
datastore only)
|
|
retries: set the number of retries for a transaction
|
|
**kwds: Other keyword arguments as long as they are supported by
|
|
datastore_rpc.TransactionOptions().
|
|
|
|
Returns:
|
|
A datastore_rpc.TransactionOptions instance.
|
|
"""
|
|
return datastore_rpc.TransactionOptions(**kwds)
|
|
|
|
|
|
def PutAsync(entities, **kwargs):
|
|
"""Asynchronously store one or more entities in the datastore.
|
|
|
|
Identical to datastore.Put() except returns an asynchronous object. Call
|
|
get_result() on the return value to block on the call and get the results.
|
|
"""
|
|
extra_hook = kwargs.pop('extra_hook', None)
|
|
config = _GetConfigFromKwargs(kwargs)
|
|
if getattr(config, 'read_policy', None) == EVENTUAL_CONSISTENCY:
|
|
raise datastore_errors.BadRequestError(
|
|
'read_policy is only supported on read operations.')
|
|
entities, multiple = NormalizeAndTypeCheck(entities, Entity)
|
|
|
|
for entity in entities:
|
|
if entity.is_projection():
|
|
raise datastore_errors.BadRequestError(
|
|
'Cannot put a partial entity: %s' % entity)
|
|
if not entity.kind() or not entity.app():
|
|
raise datastore_errors.BadRequestError(
|
|
'App and kind must not be empty, in entity: %s' % entity)
|
|
|
|
def local_extra_hook(keys):
|
|
num_keys = len(keys)
|
|
num_entities = len(entities)
|
|
if num_keys != num_entities:
|
|
raise datastore_errors.InternalError(
|
|
'Put accepted %d entities but returned %d keys.' %
|
|
(num_entities, num_keys))
|
|
|
|
for entity, key in zip(entities, keys):
|
|
if entity._Entity__key._Key__reference != key._Key__reference:
|
|
assert not entity._Entity__key.has_id_or_name()
|
|
entity._Entity__key._Key__reference.CopyFrom(key._Key__reference)
|
|
|
|
if multiple:
|
|
result = keys
|
|
else:
|
|
result = keys[0]
|
|
|
|
if extra_hook:
|
|
return extra_hook(result)
|
|
return result
|
|
|
|
return _GetConnection().async_put(config, entities, local_extra_hook)
|
|
|
|
|
|
def Put(entities, **kwargs):
|
|
"""Store one or more entities in the datastore.
|
|
|
|
The entities may be new or previously existing. For new entities, Put() will
|
|
fill in the app id and key assigned by the datastore.
|
|
|
|
If the argument is a single Entity, a single Key will be returned. If the
|
|
argument is a list of Entity, a list of Keys will be returned.
|
|
|
|
Args:
|
|
entities: Entity or list of Entities
|
|
config: Optional Configuration to use for this request, must be specified
|
|
as a keyword argument.
|
|
|
|
Returns:
|
|
Key or list of Keys
|
|
|
|
Raises:
|
|
TransactionFailedError, if the Put could not be committed.
|
|
"""
|
|
return PutAsync(entities, **kwargs).get_result()
|
|
|
|
|
|
def GetAsync(keys, **kwargs):
|
|
"""Asynchronously retrieves one or more entities from the datastore.
|
|
|
|
Identical to datastore.Get() except returns an asynchronous object. Call
|
|
get_result() on the return value to block on the call and get the results.
|
|
"""
|
|
extra_hook = kwargs.pop('extra_hook', None)
|
|
config = _GetConfigFromKwargs(kwargs)
|
|
keys, multiple = NormalizeAndTypeCheckKeys(keys)
|
|
|
|
def local_extra_hook(entities):
|
|
if multiple:
|
|
result = entities
|
|
else:
|
|
if entities[0] is None:
|
|
raise datastore_errors.EntityNotFoundError()
|
|
result = entities[0]
|
|
if extra_hook:
|
|
return extra_hook(result)
|
|
return result
|
|
|
|
return _GetConnection().async_get(config, keys, local_extra_hook)
|
|
|
|
|
|
def Get(keys, **kwargs):
|
|
"""Retrieves one or more entities from the datastore.
|
|
|
|
Retrieves the entity or entities with the given key(s) from the datastore
|
|
and returns them as fully populated Entity objects, as defined below. If
|
|
there is an error, raises a subclass of datastore_errors.Error.
|
|
|
|
If keys is a single key or string, an Entity will be returned, or
|
|
EntityNotFoundError will be raised if no existing entity matches the key.
|
|
|
|
However, if keys is a list or tuple, a list of entities will be returned
|
|
that corresponds to the sequence of keys. It will include entities for keys
|
|
that were found and None placeholders for keys that were not found.
|
|
|
|
Args:
|
|
keys: Key or string or list of Keys or strings
|
|
config: Optional Configuration to use for this request, must be specified
|
|
as a keyword argument.
|
|
|
|
Returns:
|
|
Entity or list of Entity objects
|
|
"""
|
|
return GetAsync(keys, **kwargs).get_result()
|
|
|
|
def GetIndexesAsync(**kwargs):
|
|
"""Asynchronously retrieves the application indexes and their states.
|
|
|
|
Identical to GetIndexes() except returns an asynchronous object. Call
|
|
get_result() on the return value to block on the call and get the results.
|
|
"""
|
|
extra_hook = kwargs.pop('extra_hook', None)
|
|
config = _GetConfigFromKwargs(kwargs)
|
|
|
|
def local_extra_hook(result):
|
|
if extra_hook:
|
|
return extra_hook(result)
|
|
return result
|
|
|
|
return _GetConnection().async_get_indexes(config, local_extra_hook)
|
|
|
|
|
|
def GetIndexes(**kwargs):
|
|
"""Retrieves the application indexes and their states.
|
|
|
|
Args:
|
|
config: Optional Configuration to use for this request, must be specified
|
|
as a keyword argument.
|
|
|
|
Returns:
|
|
A list of (Index, Index.[BUILDING|SERVING|DELETING|ERROR]) tuples.
|
|
An index can be in the following states:
|
|
Index.BUILDING: Index is being built and therefore can not serve queries
|
|
Index.SERVING: Index is ready to service queries
|
|
Index.DELETING: Index is being deleted
|
|
Index.ERROR: Index encounted an error in the BUILDING state
|
|
"""
|
|
return GetIndexesAsync(**kwargs).get_result()
|
|
|
|
def DeleteAsync(keys, **kwargs):
|
|
"""Asynchronously deletes one or more entities from the datastore.
|
|
|
|
Identical to datastore.Delete() except returns an asynchronous object. Call
|
|
get_result() on the return value to block on the call.
|
|
"""
|
|
config = _GetConfigFromKwargs(kwargs)
|
|
if getattr(config, 'read_policy', None) == EVENTUAL_CONSISTENCY:
|
|
raise datastore_errors.BadRequestError(
|
|
'read_policy is only supported on read operations.')
|
|
keys, _ = NormalizeAndTypeCheckKeys(keys)
|
|
|
|
return _GetConnection().async_delete(config, keys)
|
|
|
|
|
|
def Delete(keys, **kwargs):
|
|
"""Deletes one or more entities from the datastore. Use with care!
|
|
|
|
Deletes the given entity(ies) from the datastore. You can only delete
|
|
entities from your app. If there is an error, raises a subclass of
|
|
datastore_errors.Error.
|
|
|
|
Args:
|
|
# the primary key(s) of the entity(ies) to delete
|
|
keys: Key or string or list of Keys or strings
|
|
config: Optional Configuration to use for this request, must be specified
|
|
as a keyword argument.
|
|
|
|
Raises:
|
|
TransactionFailedError, if the Delete could not be committed.
|
|
"""
|
|
return DeleteAsync(keys, **kwargs).get_result()
|
|
|
|
|
|
class Entity(dict):
|
|
"""A datastore entity.
|
|
|
|
Includes read-only accessors for app id, kind, and primary key. Also
|
|
provides dictionary-style access to properties.
|
|
"""
|
|
|
|
# Instance variables with immutable defaults.
|
|
__projection = False
|
|
|
|
def __init__(self, kind, parent=None, _app=None, name=None, id=None,
|
|
unindexed_properties=[], namespace=None, **kwds):
|
|
"""Constructor. Takes the kind and transaction root, which cannot be
|
|
changed after the entity is constructed, and an optional parent. Raises
|
|
BadArgumentError or BadKeyError if kind is invalid or parent is not an
|
|
existing Entity or Key in the datastore.
|
|
|
|
Args:
|
|
# this entity's kind
|
|
kind: string
|
|
# if provided, this entity's parent. Its key must be complete.
|
|
parent: Entity or Key
|
|
# if provided, this entity's name.
|
|
name: string
|
|
# if provided, this entity's id.
|
|
id: integer
|
|
# if provided, a sequence of property names that should not be indexed
|
|
# by the built-in single property indices.
|
|
unindexed_properties: list or tuple of strings
|
|
namespace: string
|
|
# if provided, overrides the default namespace_manager setting.
|
|
"""
|
|
# the _app argument is intentionally not publicly documented. it's only
|
|
# intended to be used by the admin console, the datastore viewer, and
|
|
# other trusted apps. the datastore backend will prevent untrusted apps
|
|
# from using it.
|
|
# the _namespace argument is deprecated. Use namespace.
|
|
ref = entity_pb.Reference()
|
|
_app = datastore_types.ResolveAppId(_app)
|
|
ref.set_app(_app)
|
|
|
|
_namespace = kwds.pop('_namespace', None)
|
|
# Insist on only one keyword arg.
|
|
if kwds:
|
|
raise datastore_errors.BadArgumentError(
|
|
'Excess keyword arguments ' + repr(kwds))
|
|
|
|
# Use _namespace if namespace is None for legacy support.
|
|
# TODO(user) remove this code when we decide to remove the
|
|
# _namespace parameter.
|
|
if namespace is None:
|
|
namespace = _namespace
|
|
elif _namespace is not None:
|
|
raise datastore_errors.BadArgumentError(
|
|
"Must not set both _namespace and namespace parameters.")
|
|
|
|
datastore_types.ValidateString(kind, 'kind',
|
|
datastore_errors.BadArgumentError)
|
|
# Validate and set parent.
|
|
if parent is not None:
|
|
parent = _GetCompleteKeyOrError(parent)
|
|
if _app != parent.app():
|
|
raise datastore_errors.BadArgumentError(
|
|
" %s doesn't match parent's app %s" %
|
|
(_app, parent.app()))
|
|
# Use the parent's namespace if the namespace is not
|
|
# otherwise set.
|
|
if namespace is None:
|
|
namespace = parent.namespace()
|
|
elif namespace != parent.namespace():
|
|
raise datastore_errors.BadArgumentError(
|
|
" %s doesn't match parent's namespace %s" %
|
|
(namespace, parent.namespace()))
|
|
ref.CopyFrom(parent._Key__reference)
|
|
|
|
namespace = datastore_types.ResolveNamespace(namespace)
|
|
datastore_types.SetNamespace(ref, namespace)
|
|
|
|
last_path = ref.mutable_path().add_element()
|
|
last_path.set_type(kind.encode('utf-8'))
|
|
|
|
if name is not None and id is not None:
|
|
raise datastore_errors.BadArgumentError(
|
|
"Cannot set both name and id on an Entity")
|
|
|
|
# Validate and set key name.
|
|
if name is not None:
|
|
datastore_types.ValidateString(name, 'name')
|
|
last_path.set_name(name.encode('utf-8'))
|
|
|
|
if id is not None:
|
|
datastore_types.ValidateInteger(id, 'id')
|
|
last_path.set_id(id)
|
|
|
|
self.set_unindexed_properties(unindexed_properties)
|
|
|
|
self.__key = Key._FromPb(ref)
|
|
|
|
def app(self):
|
|
"""Returns the name of the application that created this entity, a
|
|
string or None if not set.
|
|
"""
|
|
return self.__key.app()
|
|
|
|
def namespace(self):
|
|
"""Returns the namespace of this entity, a string or None."""
|
|
return self.__key.namespace()
|
|
|
|
def kind(self):
|
|
"""Returns this entity's kind, a string."""
|
|
return self.__key.kind()
|
|
|
|
def is_saved(self):
|
|
"""Returns if this entity has been saved to the datastore."""
|
|
last_path = self.__key._Key__reference.path().element_list()[-1]
|
|
return ((last_path.has_name() ^ last_path.has_id()) and
|
|
self.__key.has_id_or_name())
|
|
|
|
def is_projection(self):
|
|
"""Returns if this entity is a projection from full entity.
|
|
|
|
Projected entities:
|
|
- may not contain all properties from the original entity;
|
|
- only contain single values for lists;
|
|
- may not contain values with the same type as the original entity.
|
|
"""
|
|
return self.__projection
|
|
|
|
def key(self):
|
|
"""Returns this entity's primary key, a Key instance."""
|
|
return self.__key
|
|
|
|
def parent(self):
|
|
"""Returns this entity's parent, as a Key. If this entity has no parent,
|
|
returns None.
|
|
"""
|
|
return self.key().parent()
|
|
|
|
def entity_group(self):
|
|
"""Returns this entity's entity group as a Key.
|
|
|
|
Note that the returned Key will be incomplete if this is a a root entity
|
|
and its key is incomplete.
|
|
"""
|
|
return self.key().entity_group()
|
|
|
|
def unindexed_properties(self):
|
|
"""Returns this entity's unindexed properties, as a frozenset of strings."""
|
|
# TODO(user): getattr is only for compatibility; remove eventually.
|
|
return getattr(self, '_Entity__unindexed_properties', [])
|
|
|
|
def set_unindexed_properties(self, unindexed_properties):
|
|
# validate and set unindexed_properties
|
|
unindexed_properties, multiple = \
|
|
NormalizeAndTypeCheck(unindexed_properties, basestring)
|
|
if not multiple:
|
|
raise datastore_errors.BadArgumentError(
|
|
'unindexed_properties must be a sequence; received %s (a %s).' %
|
|
(unindexed_properties, typename(unindexed_properties)))
|
|
for prop in unindexed_properties:
|
|
datastore_types.ValidateProperty(prop, None)
|
|
self.__unindexed_properties = frozenset(unindexed_properties)
|
|
|
|
def __setitem__(self, name, value):
|
|
"""Implements the [] operator. Used to set property value(s).
|
|
|
|
If the property name is the empty string or not a string, raises
|
|
BadPropertyError. If the value is not a supported type, raises
|
|
BadValueError.
|
|
"""
|
|
# If name or value are bad, this will raise an exception.
|
|
datastore_types.ValidateProperty(name, value)
|
|
dict.__setitem__(self, name, value)
|
|
|
|
def setdefault(self, name, value):
|
|
"""If the property exists, returns its value. Otherwise sets it to value.
|
|
|
|
If the property name is the empty string or not a string, raises
|
|
BadPropertyError. If the value is not a supported type, raises
|
|
BadValueError.
|
|
"""
|
|
# If name or value are bad, this will raise an exception.
|
|
datastore_types.ValidateProperty(name, value)
|
|
return dict.setdefault(self, name, value)
|
|
|
|
def update(self, other):
|
|
"""Updates this entity's properties from the values in other.
|
|
|
|
If any property name is the empty string or not a string, raises
|
|
BadPropertyError. If any value is not a supported type, raises
|
|
BadValueError.
|
|
"""
|
|
for name, value in other.items():
|
|
self.__setitem__(name, value)
|
|
|
|
def copy(self):
|
|
"""The copy method is not supported.
|
|
"""
|
|
raise NotImplementedError('Entity does not support the copy() method.')
|
|
|
|
def ToXml(self):
|
|
"""Returns an XML representation of this entity. Atom and gd:namespace
|
|
properties are converted to XML according to their respective schemas. For
|
|
more information, see:
|
|
|
|
http://www.atomenabled.org/developers/syndication/
|
|
https://developers.google.com/gdata/docs/1.0/elements
|
|
|
|
This is *not* optimized. It shouldn't be used anywhere near code that's
|
|
performance-critical.
|
|
"""
|
|
# write the outer entity element
|
|
xml = u'<entity kind=%s' % saxutils.quoteattr(self.kind())
|
|
if self.__key.has_id_or_name():
|
|
xml += ' key=%s' % saxutils.quoteattr(str(self.__key))
|
|
xml += '>'
|
|
if self.__key.has_id_or_name():
|
|
xml += '\n <key>%s</key>' % self.__key.ToTagUri()
|
|
|
|
|
|
# write a property element for each property value, in order of property
|
|
# name, so that the order is predictable for unit tests.
|
|
properties = self.keys()
|
|
if properties:
|
|
properties.sort()
|
|
xml += '\n ' + '\n '.join(self._PropertiesToXml(properties))
|
|
|
|
# close the entity element
|
|
xml += '\n</entity>\n'
|
|
return xml
|
|
|
|
def _PropertiesToXml(self, properties):
|
|
""" Returns a list of the XML representations of each of the given
|
|
properties. Ignores properties that don't exist in this entity.
|
|
|
|
Arg:
|
|
properties: string or list of strings
|
|
|
|
Returns:
|
|
list of strings
|
|
"""
|
|
xml_properties = []
|
|
|
|
for propname in properties:
|
|
if not self.has_key(propname):
|
|
continue
|
|
|
|
propname_xml = saxutils.quoteattr(propname)
|
|
|
|
values = self[propname]
|
|
if isinstance(values, list) and not values:
|
|
# Prior to empty list support empty lists produced no output. We
|
|
# continue doing that because there is no clear way to write an empty
|
|
# list in this particular XML schema.
|
|
continue
|
|
if not isinstance(values, list):
|
|
values = [values]
|
|
|
|
proptype = datastore_types.PropertyTypeName(values[0])
|
|
proptype_xml = saxutils.quoteattr(proptype)
|
|
escaped_values = self._XmlEscapeValues(propname)
|
|
|
|
open_tag = u'<property name=%s type=%s>' % (propname_xml, proptype_xml)
|
|
close_tag = u'</property>'
|
|
xml_properties += [open_tag + val + close_tag for val in escaped_values]
|
|
|
|
return xml_properties
|
|
|
|
def _XmlEscapeValues(self, property):
|
|
""" Returns a list of the XML-escaped string values for the given property.
|
|
Raises an AssertionError if the property doesn't exist.
|
|
|
|
Arg:
|
|
property: string
|
|
|
|
Returns:
|
|
list of strings
|
|
"""
|
|
assert self.has_key(property)
|
|
xml = []
|
|
|
|
values = self[property]
|
|
if not isinstance(values, list):
|
|
values = [values]
|
|
|
|
for val in values:
|
|
if hasattr(val, 'ToXml'):
|
|
xml.append(val.ToXml())
|
|
else:
|
|
if val is None:
|
|
xml.append('')
|
|
else:
|
|
xml.append(saxutils.escape(unicode(val)))
|
|
|
|
return xml
|
|
|
|
def ToPb(self):
|
|
"""Converts this Entity to its protocol buffer representation.
|
|
|
|
Returns:
|
|
entity_pb.Entity
|
|
"""
|
|
return self._ToPb(False)
|
|
|
|
def _ToPb(self, mark_key_as_saved=True):
|
|
"""Converts this Entity to its protocol buffer representation. Not
|
|
intended to be used by application developers.
|
|
|
|
Returns:
|
|
entity_pb.Entity
|
|
"""
|
|
|
|
# returns the PB representation of this Entity, as defined in
|
|
# storage/onestore/entity.proto
|
|
#
|
|
# construct the PB entity
|
|
pb = entity_pb.EntityProto()
|
|
pb.mutable_key().CopyFrom(self.key()._ToPb())
|
|
last_path = pb.key().path().element_list()[-1]
|
|
# TODO(user): What's the use of mark_key_as_saved?
|
|
if mark_key_as_saved and last_path.has_name() and last_path.has_id():
|
|
last_path.clear_id()
|
|
|
|
# populate the entity group
|
|
group = pb.mutable_entity_group()
|
|
if self.__key.has_id_or_name():
|
|
root = pb.key().path().element(0)
|
|
group.add_element().CopyFrom(root)
|
|
|
|
# construct and add the properties
|
|
properties = self.items()
|
|
properties.sort()
|
|
for (name, values) in properties:
|
|
properties = datastore_types.ToPropertyPb(name, values)
|
|
if not isinstance(properties, list):
|
|
properties = [properties]
|
|
|
|
for prop in properties:
|
|
if ((prop.has_meaning() and
|
|
prop.meaning() in datastore_types._RAW_PROPERTY_MEANINGS) or
|
|
name in self.unindexed_properties()):
|
|
pb.raw_property_list().append(prop)
|
|
else:
|
|
pb.property_list().append(prop)
|
|
|
|
# are there too many property values?
|
|
if pb.property_size() > _MAX_INDEXED_PROPERTIES:
|
|
raise datastore_errors.BadRequestError(
|
|
'Too many indexed properties for entity %r.' % self.key())
|
|
|
|
return pb
|
|
|
|
@staticmethod
|
|
def FromPb(pb, validate_reserved_properties=True,
|
|
default_kind='<not specified>'):
|
|
"""Static factory method. Returns the Entity representation of the
|
|
given protocol buffer (datastore_pb.Entity).
|
|
|
|
Args:
|
|
pb: datastore_pb.Entity or str encoding of a datastore_pb.Entity
|
|
validate_reserved_properties: deprecated
|
|
default_kind: str, the kind to use if the pb has no key.
|
|
|
|
Returns:
|
|
Entity: the Entity representation of pb
|
|
"""
|
|
# NOTE(user): validate_reserved_properties is unused
|
|
if isinstance(pb, str):
|
|
real_pb = entity_pb.EntityProto()
|
|
real_pb.ParsePartialFromString(pb)
|
|
pb = real_pb
|
|
|
|
return Entity._FromPb(
|
|
pb, require_valid_key=False, default_kind=default_kind)
|
|
|
|
@staticmethod
|
|
def _FromPb(pb, require_valid_key=True, default_kind='<not specified>'):
|
|
"""Static factory method. Returns the Entity representation of the
|
|
given protocol buffer (datastore_pb.Entity). Not intended to be used by
|
|
application developers.
|
|
|
|
The Entity PB's key must be complete. If it isn't, an AssertionError is
|
|
raised.
|
|
|
|
Args:
|
|
# a protocol buffer Entity
|
|
pb: datastore_pb.Entity
|
|
default_kind: str, the kind to use if the pb has no key.
|
|
|
|
Returns:
|
|
# the Entity representation of the argument
|
|
Entity
|
|
"""
|
|
# Ensure the Key is valid.
|
|
if not pb.key().path().element_size():
|
|
pb.mutable_key().CopyFrom(Key.from_path(default_kind, 0)._ToPb())
|
|
|
|
last_path = pb.key().path().element_list()[-1]
|
|
if require_valid_key:
|
|
assert last_path.has_id() ^ last_path.has_name() # xor
|
|
if last_path.has_id():
|
|
assert last_path.id() != 0
|
|
else:
|
|
assert last_path.has_name()
|
|
assert last_path.name()
|
|
|
|
# infer unindexed properties from the contents of raw_property
|
|
unindexed_properties = [unicode(p.name(), 'utf-8')
|
|
for p in pb.raw_property_list()]
|
|
|
|
# Construct the Entity with the valid key and kind.
|
|
if pb.key().has_name_space():
|
|
namespace = pb.key().name_space()
|
|
else:
|
|
namespace = ''
|
|
e = Entity(unicode(last_path.type(), 'utf-8'),
|
|
unindexed_properties=unindexed_properties,
|
|
_app=pb.key().app(), namespace=namespace)
|
|
ref = e.__key._Key__reference
|
|
ref.CopyFrom(pb.key())
|
|
|
|
# Unpack the properties into a temporary dictionary, collecting multiply
|
|
# valued properties with the same name into a list.
|
|
temporary_values = {}
|
|
|
|
for prop_list in (pb.property_list(), pb.raw_property_list()):
|
|
for prop in prop_list:
|
|
if prop.meaning() == entity_pb.Property.INDEX_VALUE:
|
|
e.__projection = True
|
|
try:
|
|
value = datastore_types.FromPropertyPb(prop)
|
|
except (AssertionError, AttributeError, TypeError, ValueError):
|
|
raise datastore_errors.Error(
|
|
'Property %s is corrupt in the datastore:\n%s' %
|
|
(prop.name(), traceback.format_exc()))
|
|
|
|
multiple = prop.multiple()
|
|
if multiple:
|
|
value = [value]
|
|
|
|
name = prop.name()
|
|
cur_value = temporary_values.get(name)
|
|
if cur_value is None:
|
|
temporary_values[name] = value
|
|
elif not multiple or not isinstance(cur_value, list):
|
|
raise datastore_errors.Error(
|
|
'Property %s is corrupt in the datastore; it has multiple '
|
|
'values, but is not marked as multiply valued.' % name)
|
|
else:
|
|
cur_value.extend(value)
|
|
|
|
# Decode the property names a single time, and insert the list values into
|
|
# the newly constructed entity. Specifically avoid any validation
|
|
for name, value in temporary_values.iteritems():
|
|
decoded_name = unicode(name, 'utf-8')
|
|
|
|
# NOTE: Removing this check gives us an extra 10-15% performance gain
|
|
# for reads. Application writers can decide to disable this check by
|
|
# overriding ValidateReadProperty to a useless function.
|
|
datastore_types.ValidateReadProperty(decoded_name, value)
|
|
|
|
dict.__setitem__(e, decoded_name, value)
|
|
|
|
return e
|
|
|
|
|
|
class Query(dict):
|
|
"""A datastore query.
|
|
|
|
(Instead of this, consider using appengine.ext.gql.Query! It provides a
|
|
query language interface on top of the same functionality.)
|
|
|
|
Queries are used to retrieve entities that match certain criteria, including
|
|
app id, kind, and property filters. Results may also be sorted by properties.
|
|
|
|
App id and kind are required. Only entities from the given app, of the given
|
|
type, are returned. If an ancestor is set, with Ancestor(), only entities
|
|
with that ancestor are returned.
|
|
|
|
Property filters are used to provide criteria based on individual property
|
|
values. A filter compares a specific property in each entity to a given
|
|
value or list of possible values.
|
|
|
|
An entity is returned if its property values match *all* of the query's
|
|
filters. In other words, filters are combined with AND, not OR. If an
|
|
entity does not have a value for a property used in a filter, it is not
|
|
returned.
|
|
|
|
Property filters map filter strings of the form '<property name> <operator>'
|
|
to filter values. Use dictionary accessors to set property filters, like so:
|
|
|
|
> query = Query('Person')
|
|
> query['name ='] = 'Ryan'
|
|
> query['age >='] = 21
|
|
|
|
This query returns all Person entities where the name property is 'Ryan'
|
|
and the age property is at least 21.
|
|
|
|
Another way to build this query is:
|
|
|
|
> query = Query('Person')
|
|
> query.update({'name =': 'Ryan', 'age >=': 21})
|
|
|
|
The supported operators are =, >, <, >=, and <=. Only one inequality
|
|
filter may be used per query. Any number of equals filters may be used in
|
|
a single Query.
|
|
|
|
A filter value may be a list or tuple of values. This is interpreted as
|
|
multiple filters with the same filter string and different values, all ANDed
|
|
together. For example, this query returns everyone with the tags "google"
|
|
and "app engine":
|
|
|
|
> Query('Person', {'tag =': ('google', 'app engine')})
|
|
|
|
Result entities can be returned in different orders. Use the Order()
|
|
method to specify properties that results will be sorted by, and in which
|
|
direction.
|
|
|
|
Note that filters and orderings may be provided at any time before the query
|
|
is run. When the query is fully specified, Run() runs the query and returns
|
|
an iterator. The query results can be accessed through the iterator.
|
|
|
|
A query object may be reused after it's been run. Its filters and
|
|
orderings can be changed to create a modified query.
|
|
|
|
If you know how many result entities you need, use Get() to fetch them:
|
|
|
|
> query = Query('Person', {'age >': 21})
|
|
> for person in query.Get(4):
|
|
> print 'I have four pints left. Have one on me, %s!' % person['name']
|
|
|
|
If you don't know how many results you need, or if you need them all, you
|
|
can get an iterator over the results by calling Run():
|
|
|
|
> for person in Query('Person', {'age >': 21}).Run():
|
|
> print 'Have a pint on me, %s!' % person['name']
|
|
|
|
Get() is more efficient than Run(), so use Get() whenever possible.
|
|
|
|
Finally, the Count() method returns the number of result entities matched by
|
|
the query. The returned count is cached; successive Count() calls will not
|
|
re-scan the datastore unless the query is changed.
|
|
"""
|
|
# ordering directions
|
|
ASCENDING = datastore_query.PropertyOrder.ASCENDING
|
|
DESCENDING = datastore_query.PropertyOrder.DESCENDING
|
|
|
|
# query hints
|
|
ORDER_FIRST = datastore_query.QueryOptions.ORDER_FIRST
|
|
ANCESTOR_FIRST = datastore_query.QueryOptions.ANCESTOR_FIRST
|
|
FILTER_FIRST = datastore_query.QueryOptions.FILTER_FIRST
|
|
|
|
# NOTE(user): datastore.Query accepts '==' while datastore_query does not.
|
|
OPERATORS = {'==': datastore_query.PropertyFilter._OPERATORS['=']}
|
|
OPERATORS.update(datastore_query.PropertyFilter._OPERATORS)
|
|
|
|
INEQUALITY_OPERATORS = datastore_query.PropertyFilter._INEQUALITY_OPERATORS
|
|
|
|
UPPERBOUND_INEQUALITY_OPERATORS = frozenset(['<', '<='])
|
|
FILTER_REGEX = re.compile(
|
|
'^\\s*([^\\s]+)(\\s+(%s)\\s*)?$' % '|'.join(OPERATORS),
|
|
re.IGNORECASE | re.UNICODE)
|
|
|
|
__kind = None
|
|
__app = None # initialized dynamically
|
|
__namespace = None # initialized dynamically
|
|
__orderings = None # a list of (property_name, direction) tuples
|
|
__ancestor_pb = None # a entity_pb.Reference
|
|
__distinct = False
|
|
__group_by = None
|
|
|
|
__index_list_source = None
|
|
__cursor_source = None
|
|
__compiled_query_source = None
|
|
|
|
# __filter_order is a dict that maps each filter to its index in order of
|
|
# insertion. indices may not be consecutive. __filter_counter stores the
|
|
# next index number to use.
|
|
__filter_order = None
|
|
__filter_counter = 0
|
|
|
|
# the property currently used in inequality filter(s), if any.
|
|
__inequality_prop = None
|
|
__inequality_count = 0
|
|
|
|
def __init__(self, kind=None, filters={}, _app=None, keys_only=False,
|
|
compile=True, cursor=None, namespace=None, end_cursor=None,
|
|
projection=None, distinct=None, _namespace=None,
|
|
_read_time_us=None): # pylint: disable=g-doc-args
|
|
"""Constructor.
|
|
|
|
Raises BadArgumentError if kind is not a string. Raises BadValueError or
|
|
BadFilterError if filters is not a dictionary of valid filters.
|
|
|
|
Args:
|
|
namespace: string, the namespace to query.
|
|
kind: string, the kind of entities to query, or None.
|
|
filters: dict, initial set of filters.
|
|
keys_only: boolean, if keys should be returned instead of entities.
|
|
projection: iterable of property names to project.
|
|
distinct: boolean, if projection should be distinct.
|
|
compile: boolean, if the query should generate cursors.
|
|
cursor: datastore_query.Cursor, the start cursor to use.
|
|
end_cursor: datastore_query.Cursor, the end cursor to use.
|
|
_namespace: deprecated, use namespace instead.
|
|
"""
|
|
# the _app argument is intentionally not publicly documented. it's only
|
|
# intended to be used by the admin console, the datastore viewer, and
|
|
# other trusted apps. the datastore backend will prevent untrusted apps
|
|
# from using it.
|
|
|
|
# the _read_time_us argument is intentionally not publicly documented. it's
|
|
# only intended to be used by apps with DATASTORE_SNAPSHOT_READ permission.
|
|
# appserver will prevent other apps from using it.
|
|
|
|
# Use _namespace if namespace is None for legacy support.
|
|
# TODO(user) remove this code when we decide to remove the
|
|
# _namespace parameter.
|
|
if namespace is None:
|
|
namespace = _namespace
|
|
elif _namespace is not None:
|
|
raise datastore_errors.BadArgumentError(
|
|
"Must not set both _namespace and namespace parameters.")
|
|
|
|
if kind is not None:
|
|
datastore_types.ValidateString(kind, 'kind',
|
|
datastore_errors.BadArgumentError)
|
|
|
|
self.__kind = kind
|
|
self.__orderings = []
|
|
self.__filter_order = {}
|
|
self.update(filters)
|
|
|
|
self.__app = datastore_types.ResolveAppId(_app)
|
|
self.__namespace = datastore_types.ResolveNamespace(namespace)
|
|
|
|
self.__read_time_us = _read_time_us
|
|
|
|
# QueryOptions validates these values
|
|
self.__query_options = datastore_query.QueryOptions(
|
|
keys_only=keys_only,
|
|
produce_cursors=compile,
|
|
start_cursor=cursor,
|
|
end_cursor=end_cursor,
|
|
projection=projection)
|
|
|
|
if distinct:
|
|
if not self.__query_options.projection:
|
|
raise datastore_errors.BadQueryError(
|
|
'cannot specify distinct without a projection')
|
|
self.__distinct = True
|
|
self.__group_by = self.__query_options.projection
|
|
|
|
def Order(self, *orderings):
|
|
"""Specify how the query results should be sorted.
|
|
|
|
Result entities will be sorted by the first property argument, then by the
|
|
second, and so on. For example, this:
|
|
|
|
> query = Query('Person')
|
|
> query.Order('bday', ('age', Query.DESCENDING))
|
|
|
|
sorts everyone in order of their birthday, starting with January 1.
|
|
People with the same birthday are sorted by age, oldest to youngest.
|
|
|
|
The direction for each sort property may be provided; if omitted, it
|
|
defaults to ascending.
|
|
|
|
Order() may be called multiple times. Each call resets the sort order
|
|
from scratch.
|
|
|
|
If an inequality filter exists in this Query it must be the first property
|
|
passed to Order. Any number of sort orders may be used after the
|
|
inequality filter property. Without inequality filters, any number of
|
|
filters with different orders may be specified.
|
|
|
|
Entities with multiple values for an order property are sorted by their
|
|
lowest value.
|
|
|
|
Note that a sort order implies an existence filter! In other words,
|
|
Entities without the sort order property are filtered out, and *not*
|
|
included in the query results.
|
|
|
|
If the sort order property has different types in different entities - ie,
|
|
if bob['id'] is an int and fred['id'] is a string - the entities will be
|
|
grouped first by the property type, then sorted within type. No attempt is
|
|
made to compare property values across types.
|
|
|
|
Raises BadArgumentError if any argument is of the wrong format.
|
|
|
|
Args:
|
|
# the properties to sort by, in sort order. each argument may be either a
|
|
# string or (string, direction) 2-tuple.
|
|
|
|
Returns:
|
|
# this query
|
|
Query
|
|
"""
|
|
orderings = list(orderings)
|
|
|
|
# normalize and sanity check arguments
|
|
for (order, i) in zip(orderings, range(len(orderings))):
|
|
if not (isinstance(order, basestring) or
|
|
(isinstance(order, tuple) and len(order) in [2, 3])):
|
|
raise datastore_errors.BadArgumentError(
|
|
'Order() expects strings or 2- or 3-tuples; received %s (a %s). ' %
|
|
(order, typename(order)))
|
|
|
|
# check property
|
|
if isinstance(order, basestring):
|
|
order = (order,)
|
|
|
|
datastore_types.ValidateString(order[0], 'sort order property',
|
|
datastore_errors.BadArgumentError)
|
|
property = order[0]
|
|
|
|
# check direction
|
|
direction = order[-1]
|
|
if direction not in (Query.ASCENDING, Query.DESCENDING):
|
|
if len(order) == 3:
|
|
raise datastore_errors.BadArgumentError(
|
|
'Order() expects Query.ASCENDING or DESCENDING; received %s' %
|
|
str(direction))
|
|
# default direction to ascending
|
|
direction = Query.ASCENDING
|
|
|
|
if (self.__kind is None and
|
|
(property != datastore_types.KEY_SPECIAL_PROPERTY or
|
|
direction != Query.ASCENDING)):
|
|
raise datastore_errors.BadArgumentError(
|
|
'Only %s ascending orders are supported on kindless queries' %
|
|
datastore_types.KEY_SPECIAL_PROPERTY)
|
|
|
|
orderings[i] = (property, direction)
|
|
|
|
# Ensure that the first ordering matches the inequality property, if any
|
|
if (orderings and self.__inequality_prop and
|
|
orderings[0][0] != self.__inequality_prop):
|
|
raise datastore_errors.BadArgumentError(
|
|
'First ordering property must be the same as inequality filter '
|
|
'property, if specified for this query; received %s, expected %s' %
|
|
(orderings[0][0], self.__inequality_prop))
|
|
|
|
self.__orderings = orderings
|
|
return self
|
|
|
|
def Hint(self, hint):
|
|
"""Sets a hint for how this query should run.
|
|
|
|
The query hint gives us information about how best to execute your query.
|
|
Currently, we can only do one index scan, so the query hint should be used
|
|
to indicates which index we should scan against.
|
|
|
|
Use FILTER_FIRST if your first filter will only match a few results. In
|
|
this case, it will be most efficient to scan against the index for this
|
|
property, load the results into memory, and apply the remaining filters
|
|
and sort orders there.
|
|
|
|
Similarly, use ANCESTOR_FIRST if the query's ancestor only has a few
|
|
descendants. In this case, it will be most efficient to scan all entities
|
|
below the ancestor and load them into memory first.
|
|
|
|
Use ORDER_FIRST if the query has a sort order and the result set is large
|
|
or you only plan to fetch the first few results. In that case, we
|
|
shouldn't try to load all of the results into memory; instead, we should
|
|
scan the index for this property, which is in sorted order.
|
|
|
|
Note that hints are currently ignored in the v3 datastore!
|
|
|
|
Arg:
|
|
one of datastore.Query.[ORDER_FIRST, ANCESTOR_FIRST, FILTER_FIRST]
|
|
|
|
Returns:
|
|
# this query
|
|
Query
|
|
"""
|
|
if hint is not self.__query_options.hint:
|
|
self.__query_options = datastore_query.QueryOptions(
|
|
hint=hint, config=self.__query_options)
|
|
return self
|
|
|
|
def Ancestor(self, ancestor):
|
|
"""Sets an ancestor for this query.
|
|
|
|
This restricts the query to only return result entities that are descended
|
|
from a given entity. In other words, all of the results will have the
|
|
ancestor as their parent, or parent's parent, or etc.
|
|
|
|
Raises BadArgumentError or BadKeyError if parent is not an existing Entity
|
|
or Key in the datastore.
|
|
|
|
Args:
|
|
# the key must be complete
|
|
ancestor: Entity or Key
|
|
|
|
Returns:
|
|
# this query
|
|
Query
|
|
"""
|
|
self.__ancestor_pb = _GetCompleteKeyOrError(ancestor)._ToPb()
|
|
return self
|
|
|
|
def IsKeysOnly(self):
|
|
"""Returns True if this query is keys only, false otherwise."""
|
|
return self.__query_options.keys_only
|
|
|
|
def GetQueryOptions(self):
|
|
"""Returns a datastore_query.QueryOptions for the current instance."""
|
|
return self.__query_options
|
|
|
|
def GetQuery(self):
|
|
"""Returns a datastore_query.Query for the current instance."""
|
|
return datastore_query.Query(app=self.__app,
|
|
namespace=self.__namespace,
|
|
kind=self.__kind,
|
|
ancestor=self.__ancestor_pb,
|
|
filter_predicate=self.GetFilterPredicate(),
|
|
order=self.GetOrder(),
|
|
group_by=self.__group_by,
|
|
read_time_us=self.__read_time_us)
|
|
|
|
def GetOrder(self):
|
|
"""Gets a datastore_query.Order for the current instance.
|
|
|
|
Returns:
|
|
datastore_query.Order or None if there are no sort orders set on the
|
|
current Query.
|
|
"""
|
|
# parse the orderings
|
|
|
|
orders = [datastore_query.PropertyOrder(property, direction)
|
|
for property, direction in self.__orderings]
|
|
if orders:
|
|
return datastore_query.CompositeOrder(orders)
|
|
return None
|
|
|
|
def GetFilterPredicate(self):
|
|
"""Returns a datastore_query.FilterPredicate for the current instance.
|
|
|
|
Returns:
|
|
datastore_query.FilterPredicate or None if no filters are set on the
|
|
current Query.
|
|
"""
|
|
# parse the filters, in the order they were inserted
|
|
ordered_filters = [(i, f) for f, i in self.__filter_order.iteritems()]
|
|
ordered_filters.sort()
|
|
|
|
property_filters = []
|
|
for _, filter_str in ordered_filters:
|
|
if filter_str not in self:
|
|
# this filter was deleted by the user; ignore it
|
|
continue
|
|
|
|
values = self[filter_str]
|
|
match = self._CheckFilter(filter_str, values)
|
|
name = match.group(1)
|
|
|
|
op = match.group(3)
|
|
if op is None or op == '==':
|
|
# no operator provided; default to EQUALS
|
|
op = '='
|
|
|
|
property_filters.append(datastore_query.make_filter(name, op, values))
|
|
|
|
if property_filters:
|
|
return datastore_query.CompositeFilter(
|
|
datastore_query.CompositeFilter.AND,
|
|
property_filters)
|
|
return None
|
|
|
|
def GetDistinct(self):
|
|
"""Returns True if the current instance is distinct.
|
|
|
|
Returns:
|
|
A boolean indicating if the distinct flag is set.
|
|
"""
|
|
return self.__distinct
|
|
|
|
def GetIndexList(self):
|
|
"""Get the index list from the last run of this query.
|
|
|
|
Returns:
|
|
A list of indexes used by the last run of this query.
|
|
|
|
Raises:
|
|
AssertionError: The query has not yet been run.
|
|
"""
|
|
index_list_function = self.__index_list_source
|
|
if index_list_function:
|
|
return index_list_function()
|
|
raise AssertionError('No index list available because this query has not '
|
|
'been executed')
|
|
|
|
def GetCursor(self):
|
|
"""Get the cursor from the last run of this query.
|
|
|
|
The source of this cursor varies depending on what the last call was:
|
|
- Run: A cursor that points immediately after the last result pulled off
|
|
the returned iterator.
|
|
- Get: A cursor that points immediately after the last result in the
|
|
returned list.
|
|
- Count: A cursor that points immediately after the last result counted.
|
|
|
|
Returns:
|
|
A datastore_query.Cursor object that can be used in subsequent query
|
|
requests.
|
|
|
|
Raises:
|
|
AssertionError: The query has not yet been run or cannot be compiled.
|
|
"""
|
|
# NOTE(user): this is stored as a function because its evaluation
|
|
# may block on an outstanding RPC.
|
|
cursor_function = self.__cursor_source
|
|
if cursor_function:
|
|
cursor = cursor_function()
|
|
if cursor:
|
|
return cursor
|
|
raise AssertionError('No cursor available, either this query has not '
|
|
'been executed or there is no compilation '
|
|
'available for this kind of query')
|
|
|
|
def GetBatcher(self, config=None):
|
|
"""Runs this query and returns a datastore_query.Batcher.
|
|
|
|
This is not intended to be used by application developers. Use Get()
|
|
instead!
|
|
|
|
Args:
|
|
config: Optional Configuration to use for this request.
|
|
|
|
Returns:
|
|
# an iterator that provides access to the query results
|
|
Iterator
|
|
"""
|
|
|
|
# NOTE(user): using merge to guarentee an inheritency priority of
|
|
# config over self
|
|
query_options = self.GetQueryOptions().merge(config)
|
|
if self.__distinct and query_options.projection != self.__group_by:
|
|
# NOTE(user): We don't allow overriding 'projection' in this case as
|
|
# 'distinct' is a high level short hand for 'group_by = projection' which
|
|
# means changing 'projection' would change the result set of the query
|
|
# (instead of just chaning how the query results are fetched).
|
|
raise datastore_errors.BadArgumentError(
|
|
'cannot override projection when distinct is set')
|
|
return self.GetQuery().run(_GetConnection(), query_options)
|
|
|
|
def Run(self, **kwargs):
|
|
"""Runs this query.
|
|
|
|
If a filter string is invalid, raises BadFilterError. If a filter value is
|
|
invalid, raises BadValueError. If an IN filter is provided, and a sort
|
|
order on another property is provided, raises BadQueryError.
|
|
|
|
If you know in advance how many results you want, use limit=#. It's
|
|
more efficient.
|
|
|
|
Args:
|
|
kwargs: Any keyword arguments accepted by datastore_query.QueryOptions().
|
|
|
|
Returns:
|
|
# an iterator that provides access to the query results
|
|
Iterator
|
|
"""
|
|
config = _GetConfigFromKwargs(kwargs, convert_rpc=True,
|
|
config_class=datastore_query.QueryOptions)
|
|
itr = Iterator(self.GetBatcher(config=config))
|
|
# Setting bound method as index list source (itr is bound as self).
|
|
self.__index_list_source = itr.GetIndexList
|
|
# Setting bound method as cursor source (itr is bound as self).
|
|
self.__cursor_source = itr.cursor
|
|
# Setting bound method as compiled query source (itr is bound as self).
|
|
self.__compiled_query_source = itr._compiled_query
|
|
return itr
|
|
|
|
def Get(self, limit, offset=0, **kwargs):
|
|
"""Deprecated, use list(Run(...)) instead.
|
|
|
|
Args:
|
|
limit: int or long representing the maximum number of entities to return.
|
|
offset: int or long representing the number of entities to skip
|
|
kwargs: Any keyword arguments accepted by datastore_query.QueryOptions().
|
|
|
|
Returns:
|
|
# a list of entities
|
|
[Entity, ...]
|
|
"""
|
|
if limit is None:
|
|
kwargs.setdefault('batch_size', _MAX_INT_32)
|
|
|
|
return list(self.Run(limit=limit, offset=offset, **kwargs))
|
|
|
|
def Count(self, limit=1000, **kwargs):
|
|
"""Returns the number of entities that this query matches.
|
|
|
|
Args:
|
|
limit, a number or None. If there are more results than this, stop short
|
|
and just return this number. Providing this argument makes the count
|
|
operation more efficient.
|
|
config: Optional Configuration to use for this request.
|
|
|
|
Returns:
|
|
The number of results.
|
|
"""
|
|
original_offset = kwargs.pop('offset', 0)
|
|
if limit is None:
|
|
offset = _MAX_INT_32
|
|
else:
|
|
offset = min(limit + original_offset, _MAX_INT_32)
|
|
kwargs['limit'] = 0
|
|
kwargs['offset'] = offset
|
|
config = _GetConfigFromKwargs(kwargs, convert_rpc=True,
|
|
config_class=datastore_query.QueryOptions)
|
|
|
|
batch = self.GetBatcher(config=config).next()
|
|
self.__index_list_source = (
|
|
lambda: [index for index, state in batch.index_list])
|
|
self.__cursor_source = lambda: batch.cursor(0)
|
|
self.__compiled_query_source = lambda: batch._compiled_query
|
|
return max(0, batch.skipped_results - original_offset)
|
|
|
|
def __iter__(self):
|
|
raise NotImplementedError(
|
|
'Query objects should not be used as iterators. Call Run() first.')
|
|
|
|
def __getstate__(self):
|
|
state = self.__dict__.copy()
|
|
state['_Query__index_list_source'] = None
|
|
state['_Query__cursor_source'] = None
|
|
state['_Query__compiled_query_source'] = None
|
|
return state
|
|
|
|
def __setstate__(self, state):
|
|
# Used to store query options directly on query.
|
|
if '_Query__query_options' not in state:
|
|
state['_Query__query_options'] = datastore_query.QueryOptions(
|
|
keys_only=state.pop('_Query__keys_only'),
|
|
produce_cursors=state.pop('_Query__compile'),
|
|
start_cursor=state.pop('_Query__cursor'),
|
|
end_cursor=state.pop('_Query__end_cursor'))
|
|
self.__dict__ = state
|
|
|
|
def __setitem__(self, filter, value):
|
|
"""Implements the [] operator. Used to set filters.
|
|
|
|
If the filter string is empty or not a string, raises BadFilterError. If
|
|
the value is not a supported type, raises BadValueError.
|
|
"""
|
|
if isinstance(value, tuple):
|
|
value = list(value)
|
|
|
|
datastore_types.ValidateProperty(' ', value)
|
|
match = self._CheckFilter(filter, value)
|
|
property = match.group(1)
|
|
operator = match.group(3)
|
|
|
|
dict.__setitem__(self, filter, value)
|
|
|
|
if (operator in self.INEQUALITY_OPERATORS and
|
|
property != datastore_types._UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY):
|
|
# record this inequality filter
|
|
if self.__inequality_prop is None:
|
|
self.__inequality_prop = property
|
|
else:
|
|
assert self.__inequality_prop == property
|
|
self.__inequality_count += 1
|
|
|
|
# remember this filter's insertion order
|
|
if filter not in self.__filter_order:
|
|
self.__filter_order[filter] = self.__filter_counter
|
|
self.__filter_counter += 1
|
|
|
|
def setdefault(self, filter, value):
|
|
"""If the filter exists, returns its value. Otherwise sets it to value.
|
|
|
|
If the property name is the empty string or not a string, raises
|
|
BadPropertyError. If the value is not a supported type, raises
|
|
BadValueError.
|
|
"""
|
|
datastore_types.ValidateProperty(' ', value) # may raise BadValueError
|
|
self._CheckFilter(filter, value)
|
|
return dict.setdefault(self, filter, value)
|
|
|
|
def __delitem__(self, filter):
|
|
"""Implements the del [] operator. Used to remove filters.
|
|
"""
|
|
dict.__delitem__(self, filter)
|
|
del self.__filter_order[filter]
|
|
|
|
# update the stored inequality filter property
|
|
match = Query.FILTER_REGEX.match(filter)
|
|
property = match.group(1)
|
|
operator = match.group(3)
|
|
|
|
if operator in self.INEQUALITY_OPERATORS:
|
|
assert self.__inequality_count >= 1
|
|
assert property == self.__inequality_prop
|
|
self.__inequality_count -= 1
|
|
if self.__inequality_count == 0:
|
|
self.__inequality_prop = None
|
|
|
|
def update(self, other):
|
|
"""Updates this query's filters from the ones in other.
|
|
|
|
If any filter string is invalid, raises BadFilterError. If any value is
|
|
not a supported type, raises BadValueError.
|
|
"""
|
|
for filter, value in other.items():
|
|
self.__setitem__(filter, value)
|
|
|
|
def copy(self):
|
|
"""The copy method is not supported.
|
|
"""
|
|
raise NotImplementedError('Query does not support the copy() method.')
|
|
|
|
def _CheckFilter(self, filter, values):
|
|
"""Type check a filter string and list of values.
|
|
|
|
Raises BadFilterError if the filter string is empty, not a string, or
|
|
invalid. Raises BadValueError if the value type is not supported.
|
|
|
|
Args:
|
|
filter: String containing the filter text.
|
|
values: List of associated filter values.
|
|
|
|
Returns:
|
|
re.MatchObject (never None) that matches the 'filter'. Group 1 is the
|
|
property name, group 3 is the operator. (Group 2 is unused.)
|
|
"""
|
|
if isinstance(values, list) and not values:
|
|
raise datastore_errors.BadValueError("Cannot filter on []")
|
|
try:
|
|
match = Query.FILTER_REGEX.match(filter)
|
|
if not match:
|
|
raise datastore_errors.BadFilterError(
|
|
'Could not parse filter string: %s' % str(filter))
|
|
except TypeError:
|
|
raise datastore_errors.BadFilterError(
|
|
'Could not parse filter string: %s' % str(filter))
|
|
|
|
property = match.group(1)
|
|
operator = match.group(3)
|
|
if operator is None:
|
|
operator = '='
|
|
|
|
if isinstance(values, tuple):
|
|
values = list(values)
|
|
elif not isinstance(values, list):
|
|
values = [values]
|
|
if isinstance(values[0], datastore_types._RAW_PROPERTY_TYPES):
|
|
raise datastore_errors.BadValueError(
|
|
'Filtering on %s properties is not supported.' % typename(values[0]))
|
|
|
|
if (operator in self.INEQUALITY_OPERATORS and
|
|
property != datastore_types._UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY):
|
|
if self.__inequality_prop and property != self.__inequality_prop:
|
|
raise datastore_errors.BadFilterError(
|
|
'Only one property per query may have inequality filters (%s).' %
|
|
', '.join(self.INEQUALITY_OPERATORS))
|
|
elif len(self.__orderings) >= 1 and self.__orderings[0][0] != property:
|
|
raise datastore_errors.BadFilterError(
|
|
'Inequality operators (%s) must be on the same property as the '
|
|
'first sort order, if any sort orders are supplied' %
|
|
', '.join(self.INEQUALITY_OPERATORS))
|
|
|
|
if (self.__kind is None and
|
|
property != datastore_types.KEY_SPECIAL_PROPERTY and
|
|
property != datastore_types._UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY):
|
|
raise datastore_errors.BadFilterError(
|
|
'Only %s filters are allowed on kindless queries.' %
|
|
datastore_types.KEY_SPECIAL_PROPERTY)
|
|
|
|
if property == datastore_types._UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY:
|
|
if self.__kind:
|
|
raise datastore_errors.BadFilterError(
|
|
'Only kindless queries can have %s filters.' %
|
|
datastore_types._UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY)
|
|
if not operator in self.UPPERBOUND_INEQUALITY_OPERATORS:
|
|
raise datastore_errors.BadFilterError(
|
|
'Only %s operators are supported with %s filters.' % (
|
|
self.UPPERBOUND_INEQUALITY_OPERATORS,
|
|
datastore_types._UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY))
|
|
|
|
if property in datastore_types._SPECIAL_PROPERTIES:
|
|
# special validation for special properties.
|
|
#
|
|
# NOTE(user): consider pulling this out if we add many more special
|
|
# properties.
|
|
if property == datastore_types.KEY_SPECIAL_PROPERTY:
|
|
for value in values:
|
|
if not isinstance(value, Key):
|
|
raise datastore_errors.BadFilterError(
|
|
'%s filter value must be a Key; received %s (a %s)' %
|
|
(datastore_types.KEY_SPECIAL_PROPERTY, value, typename(value)))
|
|
|
|
return match
|
|
|
|
def _Run(self, limit=None, offset=None,
|
|
prefetch_count=None, next_count=None, **kwargs):
|
|
"""Deprecated, use Run() instead."""
|
|
return self.Run(limit=limit, offset=offset,
|
|
prefetch_size=prefetch_count, batch_size=next_count,
|
|
**kwargs)
|
|
|
|
def _ToPb(self, limit=None, offset=None, count=None):
|
|
# This function is here soly for Internal test backwards compatability.
|
|
query_options = datastore_query.QueryOptions(
|
|
config=self.GetQueryOptions(),
|
|
limit=limit,
|
|
offset=offset,
|
|
batch_size=count)
|
|
return self.GetQuery()._to_pb(_GetConnection(), query_options)
|
|
|
|
def _GetCompiledQuery(self):
|
|
"""Returns the internal-only pb representation of the last query run.
|
|
|
|
Do not use.
|
|
|
|
Raises:
|
|
AssertionError: Query not compiled or not yet executed.
|
|
"""
|
|
compiled_query_function = self.__compiled_query_source
|
|
if compiled_query_function:
|
|
compiled_query = compiled_query_function()
|
|
if compiled_query:
|
|
return compiled_query
|
|
raise AssertionError('No compiled query available, either this query has '
|
|
'not been executed or there is no compilation '
|
|
'available for this kind of query')
|
|
|
|
GetCompiledQuery = _GetCompiledQuery # for backwards compatability
|
|
GetCompiledCursor = GetCursor # for backwards compatability
|
|
|
|
|
|
def AllocateIdsAsync(model_key, size=None, **kwargs):
|
|
"""Asynchronously allocates a range of IDs.
|
|
|
|
Identical to datastore.AllocateIds() except returns an asynchronous object.
|
|
Call get_result() on the return value to block on the call and get the
|
|
results.
|
|
"""
|
|
max = kwargs.pop('max', None)
|
|
config = _GetConfigFromKwargs(kwargs)
|
|
if getattr(config, 'read_policy', None) == EVENTUAL_CONSISTENCY:
|
|
raise datastore_errors.BadRequestError(
|
|
'read_policy is only supported on read operations.')
|
|
keys, _ = NormalizeAndTypeCheckKeys(model_key)
|
|
|
|
if len(keys) > 1:
|
|
raise datastore_errors.BadArgumentError(
|
|
'Cannot allocate IDs for more than one model key at a time')
|
|
|
|
rpc = _GetConnection().async_allocate_ids(config, keys[0], size, max)
|
|
return rpc
|
|
|
|
|
|
def AllocateIds(model_key, size=None, **kwargs):
|
|
"""Allocates a range of IDs of size or with max for the given key.
|
|
|
|
Allocates a range of IDs in the datastore such that those IDs will not
|
|
be automatically assigned to new entities. You can only allocate IDs
|
|
for model keys from your app. If there is an error, raises a subclass of
|
|
datastore_errors.Error.
|
|
|
|
Either size or max must be provided but not both. If size is provided then a
|
|
range of the given size is returned. If max is provided then the largest
|
|
range of ids that are safe to use with an upper bound of max is returned (can
|
|
be an empty range).
|
|
|
|
Max should only be provided if you have an existing numeric id range that you
|
|
want to reserve, e.g. bulk loading entities that already have IDs. If you
|
|
don't care about which IDs you receive, use size instead.
|
|
|
|
Args:
|
|
model_key: Key or string to serve as a model specifying the ID sequence
|
|
in which to allocate IDs
|
|
size: integer, number of IDs to allocate.
|
|
max: integer, upper bound of the range of IDs to allocate.
|
|
config: Optional Configuration to use for this request.
|
|
|
|
Returns:
|
|
(start, end) of the allocated range, inclusive.
|
|
"""
|
|
return AllocateIdsAsync(model_key, size, **kwargs).get_result()
|
|
|
|
|
|
# TODO(user): remove the inheritance from datastore.Query since the
|
|
# differences in implementation are too large to be meaningful.
|
|
class MultiQuery(Query):
|
|
"""Class representing a query which requires multiple datastore queries.
|
|
|
|
This class is actually a subclass of datastore.Query as it is intended to act
|
|
like a normal Query object (supporting the same interface).
|
|
|
|
Does not support keys only queries, since it needs whole entities in order
|
|
to merge sort them. (That's not true if there are no sort orders, or if the
|
|
sort order is on __key__, but allowing keys only queries in those cases, but
|
|
not in others, would be confusing.)
|
|
"""
|
|
|
|
def __init__(self, bound_queries, orderings):
|
|
if len(bound_queries) > MAX_ALLOWABLE_QUERIES:
|
|
raise datastore_errors.BadArgumentError(
|
|
'Cannot satisfy query -- too many subqueries (max: %d, got %d).'
|
|
' Probable cause: too many IN/!= filters in query.' %
|
|
(MAX_ALLOWABLE_QUERIES, len(bound_queries)))
|
|
|
|
projection = (bound_queries and
|
|
bound_queries[0].GetQueryOptions().projection)
|
|
|
|
for query in bound_queries:
|
|
if projection != query.GetQueryOptions().projection:
|
|
raise datastore_errors.BadQueryError(
|
|
'All queries must have the same projection.')
|
|
if query.IsKeysOnly():
|
|
raise datastore_errors.BadQueryError(
|
|
'MultiQuery does not support keys_only.')
|
|
|
|
self.__projection = projection
|
|
self.__bound_queries = bound_queries
|
|
self.__orderings = orderings
|
|
self.__compile = False
|
|
|
|
def __str__(self):
|
|
res = 'MultiQuery: '
|
|
for query in self.__bound_queries:
|
|
res = '%s %s' % (res, str(query))
|
|
return res
|
|
|
|
def Get(self, limit, offset=0, **kwargs):
|
|
"""Deprecated, use list(Run(...)) instead.
|
|
|
|
Args:
|
|
limit: int or long representing the maximum number of entities to return.
|
|
offset: int or long representing the number of entities to skip
|
|
kwargs: Any keyword arguments accepted by datastore_query.QueryOptions().
|
|
|
|
Returns:
|
|
A list of entities with at most "limit" entries (less if the query
|
|
completes before reading limit values).
|
|
"""
|
|
if limit is None:
|
|
kwargs.setdefault('batch_size', _MAX_INT_32)
|
|
return list(self.Run(limit=limit, offset=offset, **kwargs))
|
|
|
|
class SortOrderEntity(object):
|
|
"""Allow entity comparisons using provided orderings.
|
|
|
|
The iterator passed to the constructor is eventually consumed via
|
|
calls to GetNext(), which generate new SortOrderEntity s with the
|
|
same orderings.
|
|
"""
|
|
|
|
def __init__(self, entity_iterator, orderings):
|
|
"""Ctor.
|
|
|
|
Args:
|
|
entity_iterator: an iterator of entities which will be wrapped.
|
|
orderings: an iterable of (identifier, order) pairs. order
|
|
should be either Query.ASCENDING or Query.DESCENDING.
|
|
"""
|
|
self.__entity_iterator = entity_iterator
|
|
self.__entity = None
|
|
self.__min_max_value_cache = {}
|
|
try:
|
|
self.__entity = entity_iterator.next()
|
|
except StopIteration:
|
|
pass
|
|
else:
|
|
self.__orderings = orderings
|
|
|
|
def __str__(self):
|
|
return str(self.__entity)
|
|
|
|
def GetEntity(self):
|
|
"""Gets the wrapped entity."""
|
|
return self.__entity
|
|
|
|
def GetNext(self):
|
|
"""Wrap and return the next entity.
|
|
|
|
The entity is retrieved from the iterator given at construction time.
|
|
"""
|
|
return MultiQuery.SortOrderEntity(self.__entity_iterator,
|
|
self.__orderings)
|
|
|
|
def CmpProperties(self, that):
|
|
"""Compare two entities and return their relative order.
|
|
|
|
Compares self to that based on the current sort orderings and the
|
|
key orders between them. Returns negative, 0, or positive depending on
|
|
whether self is less, equal to, or greater than that. This
|
|
comparison returns as if all values were to be placed in ascending order
|
|
(highest value last). Only uses the sort orderings to compare (ignores
|
|
keys).
|
|
|
|
Args:
|
|
that: SortOrderEntity
|
|
|
|
Returns:
|
|
Negative if self < that
|
|
Zero if self == that
|
|
Positive if self > that
|
|
"""
|
|
# Empty less than everything but None -- use CMP
|
|
if not self.__entity:
|
|
return cmp(self.__entity, that.__entity)
|
|
|
|
# Otherwise, do the real comparisons
|
|
for (identifier, order) in self.__orderings:
|
|
# For lists, get the min element for that ID
|
|
value1 = self.__GetValueForId(self, identifier, order)
|
|
value2 = self.__GetValueForId(that, identifier, order)
|
|
|
|
result = cmp(value1, value2)
|
|
if order == Query.DESCENDING:
|
|
result = -result
|
|
if result:
|
|
return result
|
|
return 0
|
|
|
|
def __GetValueForId(self, sort_order_entity, identifier, sort_order):
|
|
# NOTE: list types are special in that they sort by first value when in
|
|
# ascending order and last value when in descending order. This is a
|
|
# little confusing, but at least it's simple to state.
|
|
value = _GetPropertyValue(sort_order_entity.__entity, identifier)
|
|
if isinstance(value, list):
|
|
entity_key = sort_order_entity.__entity.key()
|
|
if (entity_key, identifier) in self.__min_max_value_cache:
|
|
value = self.__min_max_value_cache[(entity_key, identifier)]
|
|
elif sort_order == Query.DESCENDING:
|
|
value = min(value)
|
|
else:
|
|
value = max(value)
|
|
self.__min_max_value_cache[(entity_key, identifier)] = value
|
|
|
|
return value
|
|
|
|
def __cmp__(self, that):
|
|
"""Compare self to that w.r.t. values defined in the sort order.
|
|
|
|
Compare an entity with another, using sort-order first, then the key
|
|
order to break ties. This can be used in a heap to have faster min-value
|
|
lookup.
|
|
|
|
Args:
|
|
that: other entity to compare to
|
|
Returns:
|
|
negative: if self is less than that in sort order
|
|
zero: if self is equal to that in sort order
|
|
positive: if self is greater than that in sort order
|
|
"""
|
|
property_compare = self.CmpProperties(that)
|
|
if property_compare:
|
|
return property_compare
|
|
else:
|
|
# If it's still a tie return the result of ordering by key
|
|
return cmp(self.__entity.key(), that.__entity.key())
|
|
|
|
def _ExtractBounds(self, config):
|
|
"""This function extracts the range of results to consider.
|
|
|
|
Since MultiQuery dedupes in memory, we must apply the offset and limit in
|
|
memory. The results that should be considered are
|
|
results[lower_bound:upper_bound].
|
|
|
|
We also pass the offset=0 and limit=upper_bound to the base queries to
|
|
optimize performance.
|
|
|
|
Args:
|
|
config: The base datastore_query.QueryOptions.
|
|
|
|
Returns:
|
|
a tuple consisting of the lower_bound and upper_bound to impose in memory
|
|
and the config to use with each bound query. The upper_bound may be None.
|
|
"""
|
|
if config is None:
|
|
return 0, None, None
|
|
|
|
lower_bound = config.offset or 0
|
|
upper_bound = config.limit
|
|
if lower_bound:
|
|
if upper_bound is not None:
|
|
upper_bound = min(lower_bound + upper_bound, _MAX_INT_32)
|
|
config = datastore_query.QueryOptions(offset=0,
|
|
limit=upper_bound,
|
|
config=config)
|
|
return lower_bound, upper_bound, config
|
|
|
|
def __GetProjectionOverride(self, config):
|
|
"""Returns a tuple of (original projection, projection override).
|
|
|
|
If projection is None, there is no projection. If override is None,
|
|
projection is sufficent for this query.
|
|
"""
|
|
projection = datastore_query.QueryOptions.projection(config)
|
|
if projection is None:
|
|
projection = self.__projection
|
|
else:
|
|
projection = projection
|
|
|
|
if not projection:
|
|
return None, None
|
|
|
|
# If there are sort orders, we need to make sure they are included in the
|
|
# projection as they are needed for sorting and deduping.
|
|
override = set()
|
|
for prop, _ in self.__orderings:
|
|
if prop not in projection:
|
|
override.add(prop)
|
|
if not override:
|
|
return projection, None
|
|
|
|
return projection, projection + tuple(override)
|
|
|
|
def Run(self, **kwargs):
|
|
"""Return an iterable output with all results in order.
|
|
|
|
Merge sort the results. First create a list of iterators, then walk
|
|
though them and yield results in order.
|
|
|
|
Args:
|
|
kwargs: Any keyword arguments accepted by datastore_query.QueryOptions().
|
|
|
|
Returns:
|
|
An iterator for the result set.
|
|
"""
|
|
config = _GetConfigFromKwargs(kwargs, convert_rpc=True,
|
|
config_class=datastore_query.QueryOptions)
|
|
if config and config.keys_only:
|
|
raise datastore_errors.BadRequestError(
|
|
'keys only queries are not supported by multi-query.')
|
|
|
|
# we have to apply the offset and limit locally
|
|
lower_bound, upper_bound, config = self._ExtractBounds(config)
|
|
|
|
projection, override = self.__GetProjectionOverride(config)
|
|
if override:
|
|
config = datastore_query.QueryOptions(projection=override, config=config)
|
|
|
|
results = []
|
|
count = 1
|
|
log_level = logging.DEBUG - 1 # log level copied from ext/gql/__init__.py
|
|
for bound_query in self.__bound_queries:
|
|
logging.log(log_level, 'Running query #%i' % count)
|
|
results.append(bound_query.Run(config=config))
|
|
count += 1
|
|
|
|
def GetDedupeKey(sort_order_entity):
|
|
if projection:
|
|
# Projections dedupe based on the projected values + key.
|
|
return (sort_order_entity.GetEntity().key(),
|
|
# Use a set so order is not considered.
|
|
frozenset(sort_order_entity.GetEntity().iteritems()))
|
|
else:
|
|
return sort_order_entity.GetEntity().key()
|
|
|
|
def IterateResults(results):
|
|
"""Iterator function to return all results in sorted order.
|
|
|
|
Iterate over the array of results, yielding the next element, in
|
|
sorted order. This function is destructive (results will be empty
|
|
when the operation is complete).
|
|
|
|
Args:
|
|
results: list of result iterators to merge and iterate through
|
|
|
|
Yields:
|
|
The next result in sorted order.
|
|
"""
|
|
# Gather a list of iterators and their first elements and iterate in
|
|
# sorted order on the top elements (merge sort)
|
|
result_heap = []
|
|
for result in results:
|
|
heap_value = MultiQuery.SortOrderEntity(result, self.__orderings)
|
|
if heap_value.GetEntity():
|
|
heapq.heappush(result_heap, heap_value)
|
|
|
|
# Keep a running set of used keys. This is needed because multiple
|
|
# queries may actually return results in different orders, even when
|
|
# accounting for sort order and key order. List types, in particular
|
|
# throw a wrench in the whole system.
|
|
used_keys = set()
|
|
|
|
# Always pop off the top then replace.
|
|
while result_heap:
|
|
if upper_bound is not None and len(used_keys) >= upper_bound:
|
|
# Limit reached
|
|
break
|
|
top_result = heapq.heappop(result_heap)
|
|
dedupe_key = GetDedupeKey(top_result)
|
|
if dedupe_key not in used_keys:
|
|
result = top_result.GetEntity()
|
|
if override:
|
|
# Filter out any fields not explicitly requested by the user.
|
|
for key in result.keys():
|
|
if key not in projection:
|
|
del result[key]
|
|
yield result
|
|
else:
|
|
# Skip this result since we've already seen it...
|
|
pass
|
|
|
|
used_keys.add(dedupe_key)
|
|
|
|
# Remove copies (which is klogn with k copies and n values)
|
|
results_to_push = []
|
|
while result_heap:
|
|
next = heapq.heappop(result_heap)
|
|
if dedupe_key != GetDedupeKey(next):
|
|
# heap has hit the end of equal values
|
|
results_to_push.append(next)
|
|
break
|
|
else:
|
|
# Each query should yield unique results, so we can call GetNext
|
|
# only once to advance the iterator
|
|
results_to_push.append(next.GetNext())
|
|
results_to_push.append(top_result.GetNext())
|
|
|
|
# Add top back into the heap
|
|
for popped_result in results_to_push:
|
|
# We may have pushed None values into the list, so we have to check
|
|
# each result before pushing back into the heap.
|
|
if popped_result.GetEntity():
|
|
heapq.heappush(result_heap, popped_result)
|
|
|
|
it = IterateResults(results)
|
|
|
|
# Apply offset
|
|
try:
|
|
for _ in xrange(lower_bound):
|
|
it.next()
|
|
except StopIteration:
|
|
pass
|
|
|
|
return it
|
|
|
|
def Count(self, limit=1000, **kwargs):
|
|
"""Return the number of matched entities for this query.
|
|
|
|
Will return the de-duplicated count of results. Will call the more
|
|
efficient Get() function if a limit is given.
|
|
|
|
Args:
|
|
limit: maximum number of entries to count (for any result > limit, return
|
|
limit).
|
|
config: Optional Configuration to use for this request.
|
|
|
|
Returns:
|
|
count of the number of entries returned.
|
|
"""
|
|
# Use a keys only query be cause order doesn't matter
|
|
kwargs['limit'] = limit
|
|
config = _GetConfigFromKwargs(kwargs, convert_rpc=True,
|
|
config_class=datastore_query.QueryOptions)
|
|
|
|
projection, override = self.__GetProjectionOverride(config)
|
|
|
|
if not projection:
|
|
config = datastore_query.QueryOptions(keys_only=True, config=config)
|
|
elif override:
|
|
config = datastore_query.QueryOptions(projection=override, config=config)
|
|
|
|
# Apply offset, limit locally
|
|
lower_bound, upper_bound, config = self._ExtractBounds(config)
|
|
|
|
# Keep a running set of used keys.
|
|
used_keys = set()
|
|
for bound_query in self.__bound_queries:
|
|
for result in bound_query.Run(config=config):
|
|
if projection:
|
|
# Projections dedupe based on the projected values + key.
|
|
dedupe_key = (result.key(),
|
|
tuple(result.iteritems()))
|
|
else:
|
|
dedupe_key = result
|
|
used_keys.add(dedupe_key)
|
|
if upper_bound and len(used_keys) >= upper_bound:
|
|
return upper_bound - lower_bound
|
|
|
|
return max(0, len(used_keys) - lower_bound)
|
|
|
|
def GetIndexList(self):
|
|
# TODO(user) Implement later by merging the individual index lists?
|
|
raise AssertionError('No index_list available for a MultiQuery (queries '
|
|
'using "IN" or "!=" operators)')
|
|
|
|
def GetCursor(self):
|
|
raise AssertionError('No cursor available for a MultiQuery (queries '
|
|
'using "IN" or "!=" operators)')
|
|
|
|
def _GetCompiledQuery(self):
|
|
"""Internal only, do not use."""
|
|
raise AssertionError('No compilation available for a MultiQuery (queries '
|
|
'using "IN" or "!=" operators)')
|
|
|
|
def __setitem__(self, query_filter, value):
|
|
"""Add a new filter by setting it on all subqueries.
|
|
|
|
If any of the setting operations raise an exception, the ones
|
|
that succeeded are undone and the exception is propagated
|
|
upward.
|
|
|
|
Args:
|
|
query_filter: a string of the form "property operand".
|
|
value: the value that the given property is compared against.
|
|
"""
|
|
saved_items = []
|
|
for index, query in enumerate(self.__bound_queries):
|
|
saved_items.append(query.get(query_filter, None))
|
|
try:
|
|
query[query_filter] = value
|
|
except:
|
|
for q, old_value in itertools.izip(self.__bound_queries[:index],
|
|
saved_items):
|
|
if old_value is not None:
|
|
q[query_filter] = old_value
|
|
else:
|
|
del q[query_filter]
|
|
raise
|
|
|
|
def __delitem__(self, query_filter):
|
|
"""Delete a filter by deleting it from all subqueries.
|
|
|
|
If a KeyError is raised during the attempt, it is ignored, unless
|
|
every subquery raised a KeyError. If any other exception is
|
|
raised, any deletes will be rolled back.
|
|
|
|
Args:
|
|
query_filter: the filter to delete.
|
|
|
|
Raises:
|
|
KeyError: No subquery had an entry containing query_filter.
|
|
"""
|
|
subquery_count = len(self.__bound_queries)
|
|
keyerror_count = 0
|
|
saved_items = []
|
|
for index, query in enumerate(self.__bound_queries):
|
|
try:
|
|
saved_items.append(query.get(query_filter, None))
|
|
del query[query_filter]
|
|
except KeyError:
|
|
keyerror_count += 1
|
|
except:
|
|
for q, old_value in itertools.izip(self.__bound_queries[:index],
|
|
saved_items):
|
|
if old_value is not None:
|
|
q[query_filter] = old_value
|
|
raise
|
|
|
|
if keyerror_count == subquery_count:
|
|
raise KeyError(query_filter)
|
|
|
|
def __iter__(self):
|
|
return iter(self.__bound_queries)
|
|
|
|
# Backwards compatability mappings
|
|
GetCompiledCursor = GetCursor
|
|
GetCompiledQuery = _GetCompiledQuery
|
|
|
|
|
|
def RunInTransaction(function, *args, **kwargs):
|
|
"""Runs a function inside a datastore transaction.
|
|
|
|
Runs the user-provided function inside transaction, retries default
|
|
number of times.
|
|
|
|
Args:
|
|
function: a function to be run inside the transaction on all remaining
|
|
arguments
|
|
*args: positional arguments for function.
|
|
**kwargs: keyword arguments for function.
|
|
|
|
Returns:
|
|
the function's return value, if any
|
|
|
|
Raises:
|
|
TransactionFailedError, if the transaction could not be committed.
|
|
"""
|
|
return RunInTransactionOptions(None, function, *args, **kwargs)
|
|
|
|
|
|
def RunInReadOnlyTransaction(function, *args, **kwargs):
|
|
"""Runs a function inside a read-only datastore transaction.
|
|
|
|
A read-only transaction cannot perform writes, but may be able to execute
|
|
more efficiently.
|
|
|
|
Runs the user-provided function inside a read-only transaction, retries
|
|
default number of times.
|
|
|
|
Args:
|
|
function: a function to be run inside the transaction on all remaining
|
|
arguments
|
|
*args: positional arguments for function.
|
|
**kwargs: keyword arguments for function.
|
|
|
|
Returns:
|
|
the function's return value, if any
|
|
|
|
Raises:
|
|
TransactionFailedError, if the transaction could not be committed.
|
|
"""
|
|
return RunInReadOnlyTransactionOptions(None, function, *args, **kwargs)
|
|
|
|
|
|
# TODO(user, ryanb): merge RunInTransaction and
|
|
# RunInTransactionCustomRetries into one coherent function in the next version
|
|
# of the Datastore API.
|
|
def RunInTransactionCustomRetries(retries, function, *args, **kwargs):
|
|
"""Runs a function inside a datastore transaction.
|
|
|
|
Runs the user-provided function inside transaction, with a specified
|
|
number of retries.
|
|
|
|
Args:
|
|
retries: number of retries (not counting the initial try)
|
|
function: a function to be run inside the transaction on all remaining
|
|
arguments
|
|
*args: positional arguments for function.
|
|
**kwargs: keyword arguments for function.
|
|
|
|
Returns:
|
|
the function's return value, if any
|
|
|
|
Raises:
|
|
TransactionFailedError, if the transaction could not be committed.
|
|
"""
|
|
options = datastore_rpc.TransactionOptions(retries=retries)
|
|
return RunInTransactionOptions(options, function, *args, **kwargs)
|
|
|
|
|
|
def RunInTransactionOptions(options, function, *args, **kwargs):
|
|
"""Runs a function inside a datastore transaction.
|
|
|
|
Runs the user-provided function inside a full-featured, ACID datastore
|
|
transaction. Every Put, Get, and Delete call in the function is made within
|
|
the transaction. All entities involved in these calls must belong to the
|
|
same entity group. Queries are supported as long as they specify an
|
|
ancestor belonging to the same entity group.
|
|
|
|
The trailing arguments are passed to the function as positional arguments.
|
|
If the function returns a value, that value will be returned by
|
|
RunInTransaction. Otherwise, it will return None.
|
|
|
|
The function may raise any exception to roll back the transaction instead of
|
|
committing it. If this happens, the transaction will be rolled back and the
|
|
exception will be re-raised up to RunInTransaction's caller.
|
|
|
|
If you want to roll back intentionally, but don't have an appropriate
|
|
exception to raise, you can raise an instance of datastore_errors.Rollback.
|
|
It will cause a rollback, but will *not* be re-raised up to the caller.
|
|
|
|
The function may be run more than once, so it should be idempotent. It
|
|
should avoid side effects, and it shouldn't have *any* side effects that
|
|
aren't safe to occur multiple times. This includes modifying the arguments,
|
|
since they persist across invocations of the function. However, this doesn't
|
|
include Put, Get, and Delete calls, of course.
|
|
|
|
Example usage:
|
|
|
|
> def decrement(key, amount=1):
|
|
> counter = datastore.Get(key)
|
|
> counter['count'] -= amount
|
|
> if counter['count'] < 0: # don't let the counter go negative
|
|
> raise datastore_errors.Rollback()
|
|
> datastore.Put(counter)
|
|
>
|
|
> counter = datastore.Query('Counter', {'name': 'foo'})
|
|
> datastore.RunInTransaction(decrement, counter.key(), amount=5)
|
|
|
|
Transactions satisfy the traditional ACID properties. They are:
|
|
|
|
- Atomic. All of a transaction's operations are executed or none of them are.
|
|
|
|
- Consistent. The datastore's state is consistent before and after a
|
|
transaction, whether it committed or rolled back. Invariants such as
|
|
"every entity has a primary key" are preserved.
|
|
|
|
- Isolated. Transactions operate on a snapshot of the datastore. Other
|
|
datastore operations do not see intermediated effects of the transaction;
|
|
they only see its effects after it has committed.
|
|
|
|
- Durable. On commit, all writes are persisted to the datastore.
|
|
|
|
Nested transactions are not supported.
|
|
|
|
Args:
|
|
options: TransactionOptions specifying options (number of retries, etc) for
|
|
this transaction
|
|
function: a function to be run inside the transaction on all remaining
|
|
arguments
|
|
*args: positional arguments for function.
|
|
**kwargs: keyword arguments for function.
|
|
|
|
Returns:
|
|
the function's return value, if any
|
|
|
|
Raises:
|
|
TransactionFailedError, if the transaction could not be committed.
|
|
"""
|
|
return _RunInTransactionInternal(options,
|
|
datastore_rpc.TransactionMode.READ_WRITE,
|
|
function, *args, **kwargs)
|
|
|
|
|
|
def RunInReadOnlyTransactionOptions(options, function, *args, **kwargs):
|
|
"""Runs a function inside a read-only datastore transaction.
|
|
|
|
A read-only transaction cannot perform writes, but may be able to execute
|
|
more efficiently.
|
|
|
|
Like RunInTransactionOptions, but with a read-only transaction.
|
|
|
|
Args:
|
|
options: TransactionOptions specifying options (number of retries, etc) for
|
|
this transaction
|
|
function: a function to be run inside the transaction on all remaining
|
|
arguments
|
|
*args: positional arguments for function.
|
|
**kwargs: keyword arguments for function.
|
|
|
|
Returns:
|
|
the function's return value, if any
|
|
|
|
Raises:
|
|
TransactionFailedError, if the transaction could not be committed.
|
|
"""
|
|
return _RunInTransactionInternal(options,
|
|
datastore_rpc.TransactionMode.READ_ONLY,
|
|
function, *args, **kwargs)
|
|
|
|
|
|
def _RunInTransactionInternal(options, mode, function, *args, **kwargs):
|
|
"""Runs a function inside a datastore transaction."""
|
|
# NOTE(user): right now, Put returns the stored entity (soon the key) along
|
|
# with its newly allocated id. if the Put fails, or if the datastore
|
|
# transaction rolls back, that id will no longer be valid. when that happens,
|
|
# the datastore API has to try to track down all of the entities that were Put
|
|
# inside the transaction and, if they were new entities, clear out their ids
|
|
# and entity groups. if the developer happens to grab the id and use it
|
|
# somewhere else, assuming that it was allocated, they're in for a rude
|
|
# surprise. :/
|
|
options = datastore_rpc.TransactionOptions(options)
|
|
if IsInTransaction():
|
|
if options.propagation in (None, datastore_rpc.TransactionOptions.NESTED):
|
|
# For now, disable nested transactions.
|
|
raise datastore_errors.BadRequestError(
|
|
'Nested transactions are not supported.')
|
|
elif options.propagation is datastore_rpc.TransactionOptions.INDEPENDENT:
|
|
# Run in a parallel transaction by temporarily exiting the current
|
|
# transaction.
|
|
txn_connection = _PopConnection()
|
|
try:
|
|
return _RunInTransactionInternal(options, mode,
|
|
function, *args, **kwargs)
|
|
finally:
|
|
_PushConnection(txn_connection)
|
|
return function(*args, **kwargs)
|
|
|
|
if options.propagation is datastore_rpc.TransactionOptions.MANDATORY:
|
|
raise datastore_errors.BadRequestError('Requires an existing transaction.')
|
|
|
|
# Starting a new transaction
|
|
retries = options.retries
|
|
if retries is None:
|
|
retries = DEFAULT_TRANSACTION_RETRIES
|
|
|
|
conn = _GetConnection()
|
|
_PushConnection(None) # We set the connection below.
|
|
previous_transaction = None
|
|
transactional_conn = None
|
|
try:
|
|
# Loop one extra time because the initial try is not counted.
|
|
for i in range(0, retries + 1):
|
|
transactional_conn = conn.new_transaction(options, previous_transaction,
|
|
mode)
|
|
_SetConnection(transactional_conn)
|
|
ok, result = _DoOneTry(function, args, kwargs)
|
|
if ok:
|
|
return result
|
|
|
|
if i < retries:
|
|
# Pass a second arg for the unit test. We may put the entity
|
|
# group there eventually.
|
|
logging.warning('Transaction collision. Retrying... %s', '')
|
|
|
|
if mode == datastore_rpc.TransactionMode.READ_WRITE:
|
|
# Only read-write transactions support setting previous_transaction.
|
|
# A rollback is not required, as the next begin transaction implicitly
|
|
# rolls back the previous transaction.
|
|
previous_transaction = transactional_conn.transaction
|
|
finally:
|
|
_PopConnection()
|
|
|
|
if transactional_conn is not None:
|
|
try:
|
|
transactional_conn.rollback()
|
|
except Exception: # pylint: disable=broad-except
|
|
# ignore errors on rollback
|
|
logging.exception('Exception sending Rollback:')
|
|
|
|
# We ran out of retries. give up. :(
|
|
raise datastore_errors.TransactionFailedError(
|
|
'The transaction could not be committed. Please try again.')
|
|
|
|
|
|
def _DoOneTry(function, args, kwargs):
|
|
"""Helper to call a function in a transaction, once.
|
|
|
|
Args:
|
|
function: The function to call.
|
|
*args: Tuple of positional arguments.
|
|
**kwargs: Dict of keyword arguments.
|
|
"""
|
|
try:
|
|
result = function(*args, **kwargs)
|
|
except original_exception:
|
|
try:
|
|
_GetConnection().rollback()
|
|
except Exception:
|
|
# If there was a problem sending the Rollback, just log
|
|
# it. We want to propagate the original exception.
|
|
# TODO(user): Maybe move this logic to datastore_rpc.py?
|
|
logging.exception('Exception sending Rollback:')
|
|
if isinstance(original_exception, datastore_errors.Rollback):
|
|
return True, None
|
|
else:
|
|
raise
|
|
else:
|
|
if _GetConnection().commit():
|
|
return True, result
|
|
else:
|
|
return False, None
|
|
|
|
|
|
def _MaybeSetupTransaction(request, keys):
|
|
"""Begin a transaction, if necessary, and populate it in the request.
|
|
|
|
This API exists for internal backwards compatibility, primarily with
|
|
api/taskqueue/taskqueue.py.
|
|
|
|
Args:
|
|
request: A protobuf with a mutable_transaction() method.
|
|
keys: Unused.
|
|
|
|
Returns:
|
|
A transaction if we're inside a transaction, otherwise None
|
|
"""
|
|
return _GetConnection()._set_request_transaction(request)
|
|
|
|
|
|
def IsInTransaction():
|
|
"""Determine whether already running in transaction.
|
|
|
|
Returns:
|
|
True if already running in transaction, else False.
|
|
"""
|
|
# TODO(user): Or maybe _GetConnection().get_transaction() is not None?
|
|
return isinstance(_GetConnection(), datastore_rpc.TransactionalConnection)
|
|
|
|
|
|
def Transactional(_func=None, **kwargs):
|
|
"""A decorator that makes sure a function is run in a transaction.
|
|
|
|
Defaults propagation to datastore_rpc.TransactionOptions.ALLOWED, which means
|
|
any existing transaction will be used in place of creating a new one.
|
|
|
|
WARNING: Reading from the datastore while in a transaction will not see any
|
|
changes made in the same transaction. If the function being decorated relies
|
|
on seeing all changes made in the calling scoope, set
|
|
propagation=datastore_rpc.TransactionOptions.NESTED.
|
|
|
|
Args:
|
|
_func: do not use.
|
|
**kwargs: TransactionOptions configuration options.
|
|
|
|
Returns:
|
|
A wrapper for the given function that creates a new transaction if needed.
|
|
"""
|
|
# Support no args
|
|
if _func is not None:
|
|
return Transactional()(_func)
|
|
|
|
# support deprecated keyword
|
|
if not kwargs.pop('require_new', None):
|
|
# Default to joining an existing transaction
|
|
kwargs.setdefault('propagation', datastore_rpc.TransactionOptions.ALLOWED)
|
|
|
|
options = datastore_rpc.TransactionOptions(**kwargs)
|
|
|
|
def outer_wrapper(func):
|
|
def inner_wrapper(*args, **kwds):
|
|
return RunInTransactionOptions(options, func, *args, **kwds)
|
|
return inner_wrapper
|
|
return outer_wrapper
|
|
|
|
|
|
@datastore_rpc._positional(1)
|
|
def NonTransactional(_func=None, allow_existing=True):
|
|
"""A decorator that insures a function is run outside a transaction.
|
|
|
|
If there is an existing transaction (and allow_existing=True), the existing
|
|
transaction is paused while the function is executed.
|
|
|
|
Args:
|
|
_func: do not use
|
|
allow_existing: If false, throw an exception if called from within a
|
|
transaction
|
|
|
|
Returns:
|
|
A wrapper for the decorated function that ensures it runs outside a
|
|
transaction.
|
|
"""
|
|
# Support no args
|
|
if _func is not None:
|
|
return NonTransactional()(_func)
|
|
|
|
def outer_wrapper(func):
|
|
def inner_wrapper(*args, **kwds):
|
|
if not IsInTransaction():
|
|
return func(*args, **kwds)
|
|
|
|
if not allow_existing:
|
|
raise datastore_errors.BadRequestError(
|
|
'Function cannot be called from within a transaction.')
|
|
|
|
# temporatily exit current transaction
|
|
|
|
txn_connection = _PopConnection()
|
|
try:
|
|
return func(*args, **kwds)
|
|
finally:
|
|
_PushConnection(txn_connection)
|
|
return inner_wrapper
|
|
return outer_wrapper
|
|
|
|
|
|
def _GetCompleteKeyOrError(arg):
|
|
"""Expects an Entity or a Key, and returns the corresponding Key. Raises
|
|
BadArgumentError or BadKeyError if arg is a different type or is incomplete.
|
|
|
|
Args:
|
|
arg: Entity or Key
|
|
|
|
Returns:
|
|
Key
|
|
"""
|
|
# type check
|
|
if isinstance(arg, Key):
|
|
key = arg
|
|
elif isinstance(arg, basestring):
|
|
# Also accept the string representation of a key.
|
|
key = Key(arg)
|
|
elif isinstance(arg, Entity):
|
|
key = arg.key()
|
|
elif not isinstance(arg, Key):
|
|
raise datastore_errors.BadArgumentError(
|
|
'Expects argument to be an Entity or Key; received %s (a %s).' %
|
|
(arg, typename(arg)))
|
|
assert isinstance(key, Key)
|
|
|
|
# check that it's complete
|
|
if not key.has_id_or_name():
|
|
raise datastore_errors.BadKeyError('Key %r is not complete.' % key)
|
|
|
|
return key
|
|
|
|
|
|
def _GetPropertyValue(entity, property):
|
|
"""Returns an entity's value for a given property name.
|
|
|
|
Handles special properties like __key__ as well as normal properties.
|
|
|
|
Args:
|
|
entity: datastore.Entity
|
|
property: str; the property name
|
|
|
|
Returns:
|
|
property value. For __key__, a datastore_types.Key.
|
|
|
|
Raises:
|
|
KeyError, if the entity does not have the given property.
|
|
"""
|
|
if property in datastore_types._SPECIAL_PROPERTIES:
|
|
if property == datastore_types._UNAPPLIED_LOG_TIMESTAMP_SPECIAL_PROPERTY:
|
|
raise KeyError(property)
|
|
# right now, __key__ is the only other special property
|
|
assert property == datastore_types.KEY_SPECIAL_PROPERTY
|
|
return entity.key()
|
|
else:
|
|
return entity[property]
|
|
|
|
|
|
def _AddOrAppend(dictionary, key, value):
|
|
"""Adds the value to the existing values in the dictionary, if any.
|
|
|
|
If dictionary[key] doesn't exist, sets dictionary[key] to value.
|
|
|
|
If dictionary[key] is not a list, sets dictionary[key] to [old_value, value].
|
|
|
|
If dictionary[key] is a list, appends value to that list.
|
|
|
|
Args:
|
|
dictionary: a dict
|
|
key, value: anything
|
|
"""
|
|
if key in dictionary:
|
|
existing_value = dictionary[key]
|
|
if isinstance(existing_value, list):
|
|
existing_value.append(value)
|
|
else:
|
|
dictionary[key] = [existing_value, value]
|
|
else:
|
|
dictionary[key] = value
|
|
|
|
|
|
class Iterator(datastore_query.ResultsIterator):
|
|
"""Thin wrapper of datastore_query.ResultsIterator.
|
|
|
|
Deprecated, do not use, only for backwards compatability.
|
|
"""
|
|
|
|
def _Next(self, count=None):
|
|
if count is None:
|
|
count = 20 # arbitrary batch size
|
|
result = []
|
|
for r in self:
|
|
if len(result) >= count:
|
|
break
|
|
result.append(r)
|
|
return result
|
|
|
|
def GetCompiledCursor(self, query):
|
|
return self.cursor()
|
|
|
|
def GetIndexList(self):
|
|
"""Returns the list of indexes used to perform the query."""
|
|
tuple_index_list = super(Iterator, self).index_list()
|
|
return [index for index, state in tuple_index_list]
|
|
|
|
_Get = _Next
|
|
# Inherited method index_list follows a different naming convention and
|
|
# returns a lower-level index type. Ensure that any use of it gets the
|
|
# right index type.
|
|
index_list = GetIndexList
|
|
|
|
|
|
# Legacy names for backwards compatibility.
|
|
DatastoreRPC = apiproxy_stub_map.UserRPC
|
|
GetRpcFromKwargs = _GetConfigFromKwargs
|
|
_CurrentTransactionKey = IsInTransaction
|
|
_ToDatastoreError = datastore_rpc._ToDatastoreError
|
|
_DatastoreExceptionFromErrorCodeAndDetail = \
|
|
datastore_rpc._DatastoreExceptionFromErrorCodeAndDetail
|