diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..e5d5556 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,14 @@ +Changelog +========= + +Version 0.X +----------- + +* Added a ``STORE_RELATIONS_AS_DB_KEYS`` database options, making it + possible to store foreign key values in the same way primary keys are + stored (using ``google.appengine.ext.db.db.Key``) +* Added an index for contrib.admin's ``LogEntry.object_id`` allowing + admin history to work +* Rewritten most of the code used for preparing fields' values for the + datastore / deconverting values from the database +* Allow "--allow_skipped_files" to be used diff --git a/appstats/__init__.py b/appstats/__init__.py index 2ca8b03..10e0f17 100644 --- a/appstats/__init__.py +++ b/appstats/__init__.py @@ -1,4 +1,4 @@ -# Initialize Django +# Initialize Django. from djangoappengine import main from google.appengine.ext.appstats.ui import app as application diff --git a/appstats/ui.py b/appstats/ui.py index d4914f2..b1e9b70 100644 --- a/appstats/ui.py +++ b/appstats/ui.py @@ -3,5 +3,6 @@ from google.appengine.ext.appstats.ui import app as application, main + if __name__ == '__main__': main() diff --git a/boot.py b/boot.py index 71c04cc..4abdc5e 100644 --- a/boot.py +++ b/boot.py @@ -2,14 +2,17 @@ import os import sys + PROJECT_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) DATA_ROOT = os.path.join(PROJECT_DIR, '.gaedata') -# Overrides for os.environ +# Overrides for os.environ. env_ext = {'DJANGO_SETTINGS_MODULE': 'settings'} + def setup_env(): - """Configures app engine environment for command-line apps.""" + """Configures GAE environment for command-line apps.""" + # Try to import the appengine code from the system path. try: from google.appengine.api import apiproxy_stub_map @@ -17,9 +20,9 @@ def setup_env(): for k in [k for k in sys.modules if k.startswith('google')]: del sys.modules[k] - # Not on the system path. Build a list of alternative paths where it - # may be. First look within the project for a local copy, then look for - # where the Mac OS SDK installs it. + # Not on the system path. Build a list of alternative paths + # where it may be. First look within the project for a local + # copy, then look for where the Mac OS SDK installs it. paths = [os.path.join(PROJECT_DIR, '.google_appengine'), os.environ.get('APP_ENGINE_SDK'), '/usr/local/google_appengine', @@ -31,6 +34,7 @@ def setup_env(): if os.name in ('nt', 'dos'): path = r'%(PROGRAMFILES)s\Google\google_appengine' % os.environ paths.append(path) + # Loop through all possible paths and look for the SDK dir. sdk_path = None for path in paths: @@ -41,12 +45,14 @@ def setup_env(): if os.path.exists(path): sdk_path = path break + + # The SDK could not be found in any known location. if sdk_path is None: - # The SDK could not be found in any known location. - sys.stderr.write('The Google App Engine SDK could not be found!\n' + sys.stderr.write("The Google App Engine SDK could not be found!\n" "Make sure it's accessible via your PATH " "environment and called google_appengine.\n") sys.exit(1) + # Add the SDK and the libraries within it to the system path. extra_paths = [sdk_path] lib = os.path.join(sdk_path, 'lib') @@ -54,8 +60,10 @@ def setup_env(): for name in os.listdir(lib): root = os.path.join(lib, name) subdir = name - # Package can be under 'lib///' or 'lib//lib//' - detect = (os.path.join(root, subdir), os.path.join(root, 'lib', subdir)) + # Package can be under 'lib///' or + # 'lib//lib//'. + detect = (os.path.join(root, subdir), + os.path.join(root, 'lib', subdir)) for path in detect: if os.path.isdir(path): extra_paths.append(os.path.dirname(path)) @@ -71,21 +79,23 @@ def setup_env(): setup_project() from .utils import have_appserver if have_appserver: - # App Engine's threading.local is broken + # App Engine's threading.local is broken. setup_threading() elif not os.path.exists(DATA_ROOT): os.mkdir(DATA_ROOT) setup_logging() if not have_appserver: - # Patch Django to support loading management commands from zip files + # Patch Django to support loading management commands from zip + # files. from django.core import management management.find_commands = find_commands + def find_commands(management_dir): """ - Given a path to a management directory, returns a list of all the command - names that are available. + Given a path to a management directory, returns a list of all the + command names that are available. This version works for django deployments which are file based or contained in a ZIP (in sys.path). @@ -95,10 +105,12 @@ def find_commands(management_dir): return [modname for importer, modname, ispkg in pkgutil.iter_modules( [os.path.join(management_dir, 'commands')]) if not ispkg] + def setup_threading(): if sys.version_info >= (2, 7): return - # XXX: On Python 2.5 GAE's threading.local doesn't work correctly with subclassing + # XXX: On Python 2.5 GAE's threading.local doesn't work correctly + # with subclassing. try: from django.utils._threading_local import local import threading @@ -106,47 +118,52 @@ def setup_threading(): except ImportError: pass + def setup_logging(): - # Fix Python 2.6 logging module + # Fix Python 2.6 logging module. logging.logMultiprocessing = 0 - # Enable logging + # Enable logging. level = logging.DEBUG from .utils import have_appserver if have_appserver: # We can't import settings at this point when running a normal - # manage.py command because this module gets imported from settings.py + # manage.py command because this module gets imported from + # settings.py. from django.conf import settings if not settings.DEBUG: level = logging.INFO logging.getLogger().setLevel(level) + def setup_project(): from .utils import have_appserver, on_production_server if have_appserver: - # This fixes a pwd import bug for os.path.expanduser() + # This fixes a pwd import bug for os.path.expanduser(). env_ext['HOME'] = PROJECT_DIR - # The dev_appserver creates a sandbox which restricts access to certain - # modules and builtins in order to emulate the production environment. - # Here we get the subprocess module back into the dev_appserver sandbox. - # This module is just too important for development. - # Also we add the compiler/parser module back and enable https connections - # (seem to be broken on Windows because the _ssl module is disallowed). + # The dev_appserver creates a sandbox which restricts access to + # certain modules and builtins in order to emulate the production + # environment. Here we get the subprocess module back into the + # dev_appserver sandbox.This module is just too important for + # development. Also we add the compiler/parser module back and + # enable https connections (seem to be broken on Windows because + # the _ssl module is disallowed). if not have_appserver: from google.appengine.tools import dev_appserver try: - # Backup os.environ. It gets overwritten by the dev_appserver, - # but it's needed by the subprocess module. + # Backup os.environ. It gets overwritten by the + # dev_appserver, but it's needed by the subprocess module. env = dev_appserver.DEFAULT_ENV dev_appserver.DEFAULT_ENV = os.environ.copy() dev_appserver.DEFAULT_ENV.update(env) - # Backup the buffer() builtin. The subprocess in Python 2.5 on - # Linux and OS X uses needs it, but the dev_appserver removes it. + # Backup the buffer() builtin. The subprocess in Python 2.5 + # on Linux and OS X uses needs it, but the dev_appserver + # removes it. dev_appserver.buffer = buffer except AttributeError: - logging.warn('Could not patch the default environment. ' - 'The subprocess module will not work correctly.') + logging.warn("Could not patch the default environment. " + "The subprocess module will not work correctly.") try: # Allow importing compiler/parser, _ssl (for https), @@ -154,19 +171,20 @@ def setup_project(): dev_appserver.HardenedModulesHook._WHITE_LIST_C_MODULES.extend( ('parser', '_ssl', '_io')) except AttributeError: - logging.warn('Could not patch modules whitelist. ' - 'The compiler and parser modules will not work and ' - 'SSL support is disabled.') + logging.warn("Could not patch modules whitelist. the compiler " + "and parser modules will not work and SSL support " + "is disabled.") elif not on_production_server: try: - # Restore the real subprocess module + # Restore the real subprocess module. from google.appengine.api.mail_stub import subprocess sys.modules['subprocess'] = subprocess - # Re-inject the buffer() builtin into the subprocess module + # Re-inject the buffer() builtin into the subprocess module. from google.appengine.tools import dev_appserver subprocess.buffer = dev_appserver.buffer except Exception, e: - logging.warn('Could not add the subprocess module to the sandbox: %s' % e) + logging.warn("Could not add the subprocess module to the " + "sandbox: %s" % e) os.environ.update(env_ext) @@ -178,10 +196,10 @@ def setup_project(): for zip_package in os.listdir(zip_packages_dir): extra_paths.append(os.path.join(zip_packages_dir, zip_package)) - # App Engine causes main.py to be reloaded if an exception gets raised - # on the first request of a main.py instance, so don't call setup_project() - # multiple times. We ensure this indirectly by checking if we've already - # modified sys.path, already. + # App Engine causes main.py to be reloaded if an exception gets + # raised on the first request of a main.py instance, so don't call + # setup_project() multiple times. We ensure this indirectly by + # checking if we've already modified sys.path, already. if len(sys.path) < len(extra_paths) or \ sys.path[:len(extra_paths)] != extra_paths: for path in extra_paths: diff --git a/db/base.py b/db/base.py index 2247c53..631bf36 100644 --- a/db/base.py +++ b/db/base.py @@ -1,17 +1,33 @@ +import datetime +import decimal +import logging +import os +import shutil + +from django.db.utils import DatabaseError + +from google.appengine.api.datastore import Delete, Query +from google.appengine.api.datastore_errors import BadArgumentError, \ + BadValueError +from google.appengine.api.datastore_types import Blob, Key, Text, \ + ValidateInteger +from google.appengine.api.namespace_manager import set_namespace +from google.appengine.ext.db.metadata import get_kinds, get_namespaces + +from djangotoolbox.db.base import ( + NonrelDatabaseClient, + NonrelDatabaseFeatures, + NonrelDatabaseIntrospection, + NonrelDatabaseOperations, + NonrelDatabaseValidation, + NonrelDatabaseWrapper) +from djangotoolbox.db.utils import decimal_to_string + from ..boot import DATA_ROOT from ..utils import appid, on_production_server from .creation import DatabaseCreation from .stubs import stub_manager -from django.db.backends.util import format_number -from djangotoolbox.db.base import NonrelDatabaseFeatures, \ - NonrelDatabaseOperations, NonrelDatabaseWrapper, NonrelDatabaseClient, \ - NonrelDatabaseValidation, NonrelDatabaseIntrospection -from google.appengine.ext.db.metadata import get_kinds, get_namespaces -from google.appengine.api.datastore import Query, Delete -from google.appengine.api.namespace_manager import set_namespace -import logging -import os -import shutil + DATASTORE_PATHS = { 'datastore_path': os.path.join(DATA_ROOT, 'datastore'), @@ -20,12 +36,26 @@ 'prospective_search_path': os.path.join(DATA_ROOT, 'prospective-search'), } + +def key_from_path(db_table, value): + """ + Workaround for GAE choosing not to validate integer ids when + creating keys. + + TODO: Should be removed if it gets fixed. + """ + if isinstance(value, (int, long)): + ValidateInteger(value, 'id') + return Key.from_path(db_table, value) + + def get_datastore_paths(options): paths = {} for key, path in DATASTORE_PATHS.items(): paths[key] = options.get(key, path) return paths + def destroy_datastore(paths): """Destroys the appengine datastore at the specified paths.""" for path in paths.values(): @@ -40,57 +70,221 @@ def destroy_datastore(paths): if error.errno != 2: logging.error("Failed to clear datastore: %s" % error) + class DatabaseFeatures(NonrelDatabaseFeatures): - allows_primary_key_0 = True - supports_dicts = True + + # GAE only allow strictly positive integers (and strings) to be + # used as key values. + allows_primary_key_0 = False + + # Anything that results in a something different than a positive + # integer or a string cannot be directly used as a key on GAE. + # Note that DecimalField values are encoded as strings, so can be + # used as keys. + # With some encoding, we could allow most fields to be used as a + # primary key, but for now only mark what can and what cannot be + # safely used. + supports_primary_key_on = \ + NonrelDatabaseFeatures.supports_primary_key_on - set(( + 'FloatField', 'DateField', 'DateTimeField', 'TimeField', + 'BooleanField', 'NullBooleanField', 'TextField', 'XMLField')) + class DatabaseOperations(NonrelDatabaseOperations): compiler_module = __name__.rsplit('.', 1)[0] + '.compiler' - DEFAULT_MAX_DIGITS = 16 + # Date used to store times as datetimes. + # TODO: Use just date()? + DEFAULT_DATE = datetime.date(1970, 1, 1) + + # Time used to store dates as datetimes. + DEFAULT_TIME = datetime.time() - def value_to_db_decimal(self, value, max_digits, decimal_places): + def sql_flush(self, style, tables, sequences): + self.connection.flush() + return [] + + def value_to_db_auto(self, value): + """ + New keys generated by the GAE datastore hold longs. + """ if value is None: return None + return long(value) - if value.is_signed(): - sign = u'-' - value = abs(value) - else: - sign = u'' + def value_for_db(self, value, field, lookup=None): + """ + We'll simulate `startswith` lookups with two inequalities: - if max_digits is None: - max_digits = self.DEFAULT_MAX_DIGITS + property >= value and property <= value + u'\ufffd', - if decimal_places is None: - value = unicode(value) - else: - value = format_number(value, max_digits, decimal_places) - decimal_places = decimal_places or 0 - n = value.find('.') + and need to "double" the value before passing it through the + actual datastore conversions. + """ + super_value_for_db = super(DatabaseOperations, self).value_for_db + if lookup == 'startswith': + return [super_value_for_db(value, field, lookup), + super_value_for_db(value + u'\ufffd', field, lookup)] + return super_value_for_db(value, field, lookup) - if n < 0: - n = len(value) - if n < max_digits - decimal_places: - value = u"0" * (max_digits - decimal_places - n) + value - return sign + value + def _value_for_db(self, value, field, field_kind, db_type, lookup): + """ + GAE database may store a restricted set of Python types, for + some cases it has its own types like Key, Text or Blob. + + TODO: Consider moving empty list handling here (from insert). + """ + + # Store Nones as Nones to handle nullable fields, even keys. + if value is None: + return None + + # Parent can handle iterable fields and Django wrappers. + value = super(DatabaseOperations, self)._value_for_db( + value, field, field_kind, db_type, lookup) + + # Convert decimals to strings preserving order. + if field_kind == 'DecimalField': + value = decimal_to_string( + value, field.max_digits, field.decimal_places) + + # Create GAE db.Keys from Django keys. + # We use model's table name as key kind (the table of the model + # of the instance that the key identifies, for ForeignKeys and + # other relations). + if db_type == 'key': +# value = self._value_for_db_key(value, field_kind) + try: + if not isinstance(value, Key): + value = key_from_path(field.model._meta.db_table, value) + except (BadArgumentError, BadValueError,): + raise DatabaseError("Only strings and positive integers " + "may be used as keys on GAE.") + + # Store all strings as unicode, use db.Text for longer content. + elif db_type == 'string' or db_type == 'text': + if isinstance(value, str): + value = value.decode('utf-8') + if db_type == 'text': + value = Text(value) + + # Store all date / time values as datetimes, by using some + # default time or date. + elif db_type == 'date': + value = datetime.datetime.combine(value, self.DEFAULT_TIME) + elif db_type == 'time': + value = datetime.datetime.combine(self.DEFAULT_DATE, value) + + # Store BlobField, DictField and EmbeddedModelField values as Blobs. + elif db_type == 'bytes': + value = Blob(value) + + return value + + def _value_from_db(self, value, field, field_kind, db_type): + """ + Undoes conversions done in value_for_db. + """ + + # We could have stored None for a null field. + if value is None: + return None + + # All keys were converted to the Key class. + if db_type == 'key': + assert isinstance(value, Key), \ + "GAE db.Key expected! Try changing to old storage, " \ + "dumping data, changing to new storage and reloading." + assert value.parent() is None, "Parents are not yet supported!" + value = value.id_or_name() +# value = self._value_from_db_key(value, field_kind) + + # Always retrieve strings as unicode (old datasets may + # contain non-unicode strings). + elif db_type == 'string' or db_type == 'text': + if isinstance(value, str): + value = value.decode('utf-8') + else: + value = unicode(value) + + # Dates and times are stored as datetimes, drop the added part. + elif db_type == 'date': + value = value.date() + elif db_type == 'time': + value = value.time() + + # Convert GAE Blobs to plain strings for Django. + elif db_type == 'bytes': + value = str(value) + + # Revert the decimal-to-string encoding. + if field_kind == 'DecimalField': + value = decimal.Decimal(value) + + return super(DatabaseOperations, self)._value_from_db( + value, field, field_kind, db_type) + +# def _value_for_db_key(self, value, field_kind): +# """ +# Converts values to be used as entity keys to strings, +# trying (but not fully succeeding) to preserve comparisons. +# """ + +# # Bools as positive integers. +# if field_kind == 'BooleanField': +# value = int(value) + 1 + +# # Encode floats as strings. +# elif field_kind == 'FloatField': +# value = self.value_to_db_decimal( +# decimal.Decimal(value), None, None) + +# # Integers as strings (string keys sort after int keys, so +# # all need to be encoded to preserve comparisons). +# elif field_kind in ('IntegerField', 'BigIntegerField', +# 'PositiveIntegerField', 'PositiveSmallIntegerField', +# 'SmallIntegerField'): +# value = self.value_to_db_decimal( +# decimal.Decimal(value), None, 0) + +# return value + +# def value_from_db_key(self, value, field_kind): +# """ +# Decodes value previously encoded in a key. +# """ +# if field_kind == 'BooleanField': +# value = bool(value - 1) +# elif field_kind == 'FloatField': +# value = float(value) +# elif field_kind in ('IntegerField', 'BigIntegerField', +# 'PositiveIntegerField', 'PositiveSmallIntegerField', +# 'SmallIntegerField'): +# value = int(value) + +# return value - def sql_flush(self, style, tables, sequences): - self.connection.flush() - return [] class DatabaseClient(NonrelDatabaseClient): pass + class DatabaseValidation(NonrelDatabaseValidation): pass + class DatabaseIntrospection(NonrelDatabaseIntrospection): + def table_names(self): - """Returns a list of names of all tables that exist in the database.""" + """ + Returns a list of names of all tables that exist in the + database. + """ return [kind.key().name() for kind in Query(kind='__kind__').Run()] + class DatabaseWrapper(NonrelDatabaseWrapper): + def __init__(self, *args, **kwds): super(DatabaseWrapper, self).__init__(*args, **kwds) self.features = DatabaseFeatures(self) @@ -114,29 +308,35 @@ def __init__(self, *args, **kwds): stub_manager.setup_stubs(self) def flush(self): - """Helper function to remove the current datastore and re-open the stubs""" + """ + Helper function to remove the current datastore and re-open the + stubs. + """ if stub_manager.active_stubs == 'remote': import random import string - code = ''.join([random.choice(string.ascii_letters) for x in range(4)]) - print '\n\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' - print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' + code = ''.join([random.choice(string.ascii_letters) + for x in range(4)]) + print "\n\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" print "Warning! You're about to delete the *production* datastore!" - print 'Only models defined in your INSTALLED_APPS can be removed!' - print 'If you want to clear the whole datastore you have to use the ' \ - 'datastore viewer in the dashboard. Also, in order to delete all '\ - 'unneeded indexes you have to run appcfg.py vacuum_indexes.' - print 'In order to proceed you have to enter the following code:' + print "Only models defined in your INSTALLED_APPS can be removed!" + print "If you want to clear the whole datastore you have to use " \ + "the datastore viewer in the dashboard. Also, in order to " \ + "delete all unneeded indexes you have to run appcfg.py " \ + "vacuum_indexes." + print "In order to proceed you have to enter the following code:" print code - response = raw_input('Repeat: ') + response = raw_input("Repeat: ") if code == response: - print 'Deleting...' + print "Deleting..." delete_all_entities() print "Datastore flushed! Please check your dashboard's " \ - 'datastore viewer for any remaining entities and remove ' \ - 'all unneeded indexes with appcfg.py vacuum_indexes.' + "datastore viewer for any remaining entities and " \ + "remove all unneeded indexes with appcfg.py " \ + "vacuum_indexes." else: - print 'Aborting' + print "Aborting." exit() elif stub_manager.active_stubs == 'test': stub_manager.deactivate_test_stubs() @@ -145,6 +345,7 @@ def flush(self): destroy_datastore(get_datastore_paths(self.settings_dict)) stub_manager.setup_local_stubs(self) + def delete_all_entities(): for namespace in get_namespaces(): set_namespace(namespace) diff --git a/db/compiler.py b/db/compiler.py index 7d9a249..0ce7d23 100644 --- a/db/compiler.py +++ b/db/compiler.py @@ -1,31 +1,29 @@ -from .db_settings import get_model_indexes -from .utils import commit_locked -from .expressions import ExpressionEvaluator - -import datetime +from functools import wraps import sys +from django.db.models.fields import AutoField from django.db.models.sql import aggregates as sqlaggregates from django.db.models.sql.constants import LOOKUP_SEP, MULTI, SINGLE from django.db.models.sql.where import AND, OR from django.db.utils import DatabaseError, IntegrityError from django.utils.tree import Node -from functools import wraps - from google.appengine.api.datastore import Entity, Query, MultiQuery, \ - Put, Get, Delete, Key + Put, Get, Delete from google.appengine.api.datastore_errors import Error as GAEError -from google.appengine.api.datastore_types import Text, Category, Email, Link, \ - PhoneNumber, PostalAddress, Text, Blob, ByteString, GeoPt, IM, Key, \ - Rating, BlobKey +from google.appengine.api.datastore_types import Key, Text -from djangotoolbox.db.basecompiler import NonrelQuery, NonrelCompiler, \ - NonrelInsertCompiler, NonrelUpdateCompiler, NonrelDeleteCompiler +from djangotoolbox.db.basecompiler import ( + NonrelQuery, + NonrelCompiler, + NonrelInsertCompiler, + NonrelUpdateCompiler, + NonrelDeleteCompiler) -import cPickle as pickle +from .db_settings import get_model_indexes +from .expressions import ExpressionEvaluator +from .utils import AncestorKey, commit_locked -import decimal # Valid query types (a dictionary is used for speedy lookups). OPERATORS_MAP = { @@ -35,7 +33,7 @@ 'lt': '<', 'lte': '<=', - # The following operators are supported with special code below: + # The following operators are supported with special code below. 'isnull': None, 'in': None, 'startswith': None, @@ -43,18 +41,27 @@ 'year': None, } +# GAE filters used for negated Django lookups. NEGATION_MAP = { 'gt': '<=', 'gte': '<', 'lt': '>=', 'lte': '>', - # TODO: support these filters - #'exact': '!=', # this might actually become individual '<' and '>' queries + # TODO: Support: "'exact': '!='" (it might actually become + # individual '<' and '>' queries). } +# In some places None is an allowed value, and we need to distinguish +# it from the lack of value. NOT_PROVIDED = object() + def safe_call(func): + """ + Causes the decorated function to reraise GAE datastore errors as + Django DatabaseErrors. + """ + @wraps(func) def _func(*args, **kwargs): try: @@ -63,29 +70,32 @@ def _func(*args, **kwargs): raise DatabaseError, DatabaseError(str(e)), sys.exc_info()[2] return _func + class GAEQuery(NonrelQuery): + """ + A simple App Engine query: no joins, no distinct, etc. + """ + # ---------------------------------------------- # Public API # ---------------------------------------------- + def __init__(self, compiler, fields): super(GAEQuery, self).__init__(compiler, fields) self.inequality_field = None - self.pk_filters = None + self.included_pks = None self.excluded_pks = () self.has_negated_exact_filter = False - self.ordering = () - self.gae_ordering = [] - pks_only = False - if len(fields) == 1 and fields[0].primary_key: - pks_only = True + self.ancestor_key = None + self.ordering = [] self.db_table = self.query.get_meta().db_table - self.pks_only = pks_only + self.pks_only = (len(fields) == 1 and fields[0].primary_key) start_cursor = getattr(self.query, '_gae_start_cursor', None) end_cursor = getattr(self.query, '_gae_end_cursor', None) self.gae_query = [Query(self.db_table, keys_only=self.pks_only, cursor=start_cursor, end_cursor=end_cursor)] - # This is needed for debugging + # This is needed for debugging. def __repr__(self): return '' % (self.gae_query, self.ordering) @@ -95,7 +105,7 @@ def fetch(self, low_mark, high_mark): executed = False if self.excluded_pks and high_mark is not None: high_mark += len(self.excluded_pks) - if self.pk_filters is not None: + if self.included_pks is not None: results = self.get_matching_pk(low_mark, high_mark) else: if high_mark is None: @@ -127,15 +137,16 @@ def fetch(self, low_mark, high_mark): @safe_call def count(self, limit=NOT_PROVIDED): - if self.pk_filters is not None: + if self.included_pks is not None: return len(self.get_matching_pk(0, limit)) if self.excluded_pks: return len(list(self.fetch(0, 2000))) # The datastore's Count() method has a 'limit' kwarg, which has - # a default value (obviously). This value can be overridden to anything - # you like, and importantly can be overridden to unlimited by passing - # a value of None. Hence *this* method has a default value of - # NOT_PROVIDED, rather than a default value of None + # a default value (obviously). This value can be overridden to + # anything you like, and importantly can be overridden to + # unlimited by passing a value of None. Hence *this* method + # has a default value of NOT_PROVIDED, rather than a default + # value of None kw = {} if limit is not NOT_PROVIDED: kw['limit'] = limit @@ -143,8 +154,8 @@ def count(self, limit=NOT_PROVIDED): @safe_call def delete(self): - if self.pk_filters is not None: - keys = [key for key in self.pk_filters if key is not None] + if self.included_pks is not None: + keys = [key for key in self.included_pks if key is not None] else: keys = self.fetch() if keys: @@ -152,68 +163,61 @@ def delete(self): @safe_call def order_by(self, ordering): - self.ordering = ordering - for order, descending in self.ordering: - direction = Query.DESCENDING if descending else Query.ASCENDING - if order == self.query.get_meta().pk.column: - order = '__key__' - self.gae_ordering.append((order, direction)) - - # This function is used by the default add_filters() implementation + + # GAE doesn't have any kind of natural ordering? + if not isinstance(ordering, bool): + for field, ascending in ordering: + column = '__key__' if field.primary_key else field.column + direction = Query.ASCENDING if ascending else Query.DESCENDING + self.ordering.append((column, direction)) + + @safe_call - def add_filter(self, column, lookup_type, negated, db_type, value): + def add_filter(self, field, lookup_type, negated, value): + """ + This function is used by the default add_filters() + implementation. + """ + if lookup_type not in OPERATORS_MAP: + raise DatabaseError("Lookup type %r isn't supported." % + lookup_type) + + # GAE does not let you store empty lists, so we can tell + # upfront that queriying for one will return nothing. if value in ([], ()): - self.pk_filters = [] + self.included_pks = [] return - # Emulated/converted lookups - if column == self.query.get_meta().pk.column: - column = '__key__' - db_table = self.query.get_meta().db_table - if lookup_type in ('exact', 'in'): - # Optimization: batch-get by key - if self.pk_filters is not None: - raise DatabaseError("You can't apply multiple AND filters " - "on the primary key. " - "Did you mean __in=[...]?") - if not isinstance(value, (tuple, list)): - value = [value] - pks = [create_key(db_table, pk) for pk in value if pk] + # Optimization: batch-get by key; this is only suitable for + # primary keys, not for anything that uses the key type. + if field.primary_key and lookup_type in ('exact', 'in'): + if lookup_type == 'exact' and isinstance(value, AncestorKey): if negated: - self.excluded_pks = pks - else: - self.pk_filters = pks + raise DatabaseError("You can't negate an ancestor operator.") + if self.ancestor_key is not None: + raise DatabaseError("You can't use more than one ancestor operator.") + self.ancestor_key = value.key return + + if self.included_pks is not None: + raise DatabaseError("You can't apply multiple AND " + "filters on the primary key. " + "Did you mean __in=[...]?") + if not isinstance(value, (tuple, list)): + value = [value] + pks = [pk for pk in value if pk is not None] + if negated: + self.excluded_pks = pks else: - # XXX: set db_type to 'gae_key' in order to allow - # convert_value_for_db to recognize the value to be a Key and - # not a str. Otherwise the key would be converted back to a - # unicode (see convert_value_for_db) - db_type = 'gae_key' - key_type_error = 'Lookup values on primary keys have to be' \ - 'a string or an integer.' - if lookup_type == 'range': - if isinstance(value, (list, tuple)) and not ( - isinstance(value[0], (basestring, int, long)) and - isinstance(value[1], (basestring, int, long))): - raise DatabaseError(key_type_error) - elif not isinstance(value, (basestring, int, long)): - raise DatabaseError(key_type_error) - # for lookup type range we have to deal with a list - if lookup_type == 'range': - value[0] = create_key(db_table, value[0]) - value[1] = create_key(db_table, value[1]) - else: - value = create_key(db_table, value) - if lookup_type not in OPERATORS_MAP: - raise DatabaseError("Lookup type %r isn't supported" % lookup_type) + self.included_pks = pks + return # We check for negation after lookup_type isnull because it # simplifies the code. All following lookup_type checks assume # that they're not negated. if lookup_type == 'isnull': if (negated and value) or not value: - # TODO/XXX: is everything greater than None? + # TODO/XXX: Is everything greater than None? op = '>' else: op = '=' @@ -221,66 +225,66 @@ def add_filter(self, column, lookup_type, negated, db_type, value): elif negated and lookup_type == 'exact': if self.has_negated_exact_filter: raise DatabaseError("You can't exclude more than one __exact " - "filter") + "filter.") self.has_negated_exact_filter = True - self._combine_filters(column, db_type, - (('<', value), ('>', value))) + self._combine_filters(field, (('<', value), ('>', value))) return elif negated: try: op = NEGATION_MAP[lookup_type] except KeyError: - raise DatabaseError("Lookup type %r can't be negated" % lookup_type) - if self.inequality_field and column != self.inequality_field: - raise DatabaseError("Can't have inequality filters on multiple " - "columns (here: %r and %r)" % (self.inequality_field, column)) - self.inequality_field = column + raise DatabaseError("Lookup type %r can't be negated." % + lookup_type) + if self.inequality_field and field != self.inequality_field: + raise DatabaseError("Can't have inequality filters on " + "multiple fields (here: %r and %r)." % + (field, self.inequality_field)) + self.inequality_field = field elif lookup_type == 'in': - # Create sub-query combinations, one for each value + # Create sub-query combinations, one for each value. if len(self.gae_query) * len(value) > 30: raise DatabaseError("You can't query against more than " - "30 __in filter value combinations") + "30 __in filter value combinations.") op_values = [('=', v) for v in value] - self._combine_filters(column, db_type, op_values) + self._combine_filters(field, op_values) return elif lookup_type == 'startswith': - self._add_filter(column, '>=', db_type, value) - if isinstance(value, str): - value = value.decode('utf8') - if isinstance(value, Key): - value = list(value.to_path()) - if isinstance(value[-1], str): - value[-1] = value[-1].decode('utf8') - value[-1] += u'\ufffd' - value = Key.from_path(*value) - else: - value += u'\ufffd' - self._add_filter(column, '<=', db_type, value) + # Lookup argument was converted to [arg, arg + u'\ufffd']. + self._add_filter(field, '>=', value[0]) + self._add_filter(field, '<=', value[1]) return elif lookup_type in ('range', 'year'): - self._add_filter(column, '>=', db_type, value[0]) + self._add_filter(field, '>=', value[0]) op = '<=' if lookup_type == 'range' else '<' - self._add_filter(column, op, db_type, value[1]) + self._add_filter(field, op, value[1]) return else: op = OPERATORS_MAP[lookup_type] - self._add_filter(column, op, db_type, value) + self._add_filter(field, op, value) # ---------------------------------------------- # Internal API # ---------------------------------------------- - def _add_filter(self, column, op, db_type, value): + + def _add_filter(self, field, op, value): for query in self.gae_query: + + # GAE uses a special property name for primary key filters. + if field.primary_key: + column = '__key__' + else: + column = field.column key = '%s %s' % (column, op) - value = self.convert_value_for_db(db_type, value) + if isinstance(value, Text): - raise DatabaseError('TextField is not indexed, by default, ' + raise DatabaseError("TextField is not indexed, by default, " "so you can't filter on it. Please add " - 'an index definition for the column %s ' - 'on the model %s.%s as described here:\n' - 'http://www.allbuttonspressed.com/blog/django/2010/07/Managing-per-field-indexes-on-App-Engine' - % (column, self.query.model.__module__, self.query.model.__name__)) + "an index definition for the field %s " + "on the model %s.%s as described here:\n" + "http://www.allbuttonspressed.com/blog/django/2010/07/Managing-per-field-indexes-on-App-Engine" % + (column, self.query.model.__module__, + self.query.model.__name__)) if key in query: existing_value = query[key] if isinstance(existing_value, list): @@ -290,7 +294,7 @@ def _add_filter(self, column, op, db_type, value): else: query[key] = value - def _combine_filters(self, column, db_type, op_values): + def _combine_filters(self, field, op_values): gae_query = self.gae_query combined = [] for query in gae_query: @@ -298,7 +302,7 @@ def _combine_filters(self, column, db_type, op_values): self.gae_query = [Query(self.db_table, keys_only=self.pks_only)] self.gae_query[0].update(query) - self._add_filter(column, op, db_type, value) + self._add_filter(field, op, value) combined.append(self.gae_query[0]) self.gae_query = combined @@ -315,16 +319,17 @@ def _make_entity(self, entity): @safe_call def _build_query(self): for query in self.gae_query: - query.Order(*self.gae_ordering) + query.Order(*self.ordering) + if self.ancestor_key: + query.Ancestor(self.ancestor_key) if len(self.gae_query) > 1: - return MultiQuery(self.gae_query, self.gae_ordering) + return MultiQuery(self.gae_query, self.ordering) return self.gae_query[0] def get_matching_pk(self, low_mark=0, high_mark=None): - if not self.pk_filters: + if not self.included_pks: return [] - - results = [result for result in Get(self.pk_filters) + results = [result for result in Get(self.included_pks) if result is not None and self.matches_filters(result)] if self.ordering: @@ -343,216 +348,105 @@ def order_pk_filtered(self, lhs, rhs): return self._order_in_memory(left, right) def matches_filters(self, entity): + """ + Checks if the GAE entity fetched from the database satisfies + the current query's constraints. + """ item = dict(entity) - pk = self.query.get_meta().pk - value = self.convert_value_from_db(pk.db_type(connection=self.connection), - entity.key()) - item[pk.column] = value - result = self._matches_filters(item, self.query.where) - return result + item[self.query.get_meta().pk.column] = entity.key() + return self._matches_filters(item, self.query.where) + class SQLCompiler(NonrelCompiler): """ - A simple App Engine query: no joins, no distinct, etc. + Base class for all GAE compilers. """ query_class = GAEQuery - def convert_value_from_db(self, db_type, value): - if isinstance(value, (list, tuple, set)) and \ - db_type.startswith(('ListField:', 'SetField:')): - db_sub_type = db_type.split(':', 1)[1] - value = [self.convert_value_from_db(db_sub_type, subvalue) - for subvalue in value] - - if db_type.startswith('SetField:') and value is not None: - value = set(value) - - if db_type.startswith('DictField:') and value is not None: - value = pickle.loads(value) - if ':' in db_type: - db_sub_type = db_type.split(':', 1)[1] - value = dict((key, self.convert_value_from_db(db_sub_type, value[key])) - for key in value) - - # the following GAE database types are all unicode subclasses, cast them - # to unicode so they appear like pure unicode instances for django - if isinstance(value, basestring) and value and db_type.startswith('decimal'): - value = decimal.Decimal(value) - elif isinstance(value, (Category, Email, Link, PhoneNumber, PostalAddress, - Text, unicode)): - value = unicode(value) - elif isinstance(value, Blob): - value = str(value) - elif isinstance(value, str): - # always retrieve strings as unicode (it is possible that old datasets - # contain non unicode strings, nevertheless work with unicode ones) - value = value.decode('utf-8') - elif isinstance(value, Key): - # for now we do not support KeyFields thus a Key has to be the own - # primary key - # TODO: GAE: support parents via GAEKeyField - assert value.parent() is None, "Parents are not yet supported!" - if db_type == 'integer': - if value.id() is None: - raise DatabaseError('Wrong type for Key. Expected integer, found' - 'None') - else: - value = value.id() - elif db_type == 'text': - if value.name() is None: - raise DatabaseError('Wrong type for Key. Expected string, found' - 'None') - else: - value = value.name() - else: - raise DatabaseError("%s fields cannot be keys on GAE" % db_type) - elif db_type == 'date' and isinstance(value, datetime.datetime): - value = value.date() - elif db_type == 'time' and isinstance(value, datetime.datetime): - value = value.time() - return value - - def convert_value_for_db(self, db_type, value): - if isinstance(value, unicode): - value = unicode(value) - elif isinstance(value, str): - value = str(value) - elif isinstance(value, (list, tuple, set)) and \ - db_type.startswith(('ListField:', 'SetField:')): - db_sub_type = db_type.split(':', 1)[1] - value = [self.convert_value_for_db(db_sub_type, subvalue) - for subvalue in value] - elif isinstance(value, decimal.Decimal) and db_type.startswith("decimal:"): - value = self.connection.ops.value_to_db_decimal(value, *eval(db_type[8:])) - elif isinstance(value, dict) and db_type.startswith('DictField:'): - if ':' in db_type: - db_sub_type = db_type.split(':', 1)[1] - value = dict([(key, self.convert_value_for_db(db_sub_type, value[key])) - for key in value]) - value = Blob(pickle.dumps(value)) - - if db_type == 'gae_key': - return value - elif db_type == 'longtext': - # long text fields cannot be indexed on GAE so use GAE's database - # type Text - if value is not None: - value = Text(value.decode('utf-8') if isinstance(value, str) else value) - elif db_type == 'text': - value = value.decode('utf-8') if isinstance(value, str) else value - elif db_type == 'blob': - if value is not None: - value = Blob(value) - elif type(value) is str: - # always store unicode strings - value = value.decode('utf-8') - elif db_type == 'date' or db_type == 'time' or db_type == 'datetime': - # here we have to check the db_type because GAE always stores datetimes - value = to_datetime(value) - return value class SQLInsertCompiler(NonrelInsertCompiler, SQLCompiler): + @safe_call - def insert(self, data, return_id=False): - gae_data = {} + def insert(self, data_list, return_id=False): opts = self.query.get_meta() unindexed_fields = get_model_indexes(self.query.model)['unindexed'] unindexed_cols = [opts.get_field(name).column for name in unindexed_fields] - kwds = {'unindexed_properties': unindexed_cols} - for column, value in data.items(): - if column == opts.pk.column: - if isinstance(value, basestring): - kwds['name'] = value + + entity_list = [] + for data in data_list: + properties = {} + kwds = {'unindexed_properties': unindexed_cols} + for column, value in data.items(): + # The value will already be a db.Key, but the Entity + # constructor takes a name or id of the key, and will + # automatically create a new key if neither is given. + if column == opts.pk.column: + if value is not None: + kwds['id'] = value.id() + kwds['name'] = value.name() + kwds['parent'] = value.parent() + + # GAE does not store empty lists (and even does not allow + # passing empty lists to Entity.update) so skip them. + elif isinstance(value, (tuple, list)) and not len(value): + continue + + # Use column names as property names. else: - kwds['id'] = value - elif isinstance(value, (tuple, list)) and not len(value): - # gae does not store emty lists (and even does not allow passing empty - # lists to Entity.update) so skip them - continue - else: - gae_data[column] = value + properties[column] = value + + entity = Entity(opts.db_table, **kwds) + entity.update(properties) + entity_list.append(entity) + + keys = Put(entity_list) + return keys[0] if isinstance(keys, list) else keys - entity = Entity(self.query.get_meta().db_table, **kwds) - entity.update(gae_data) - key = Put(entity) - return key.id_or_name() class SQLUpdateCompiler(NonrelUpdateCompiler, SQLCompiler): + def execute_sql(self, result_type=MULTI): - # modify query to fetch pks only and then execute the query - # to get all pks - pk = self.query.model._meta.pk.name - self.query.add_immediate_loading([pk]) + # Modify query to fetch pks only and then execute the query + # to get all pks. + pk_field = self.query.model._meta.pk + self.query.add_immediate_loading([pk_field.name]) pks = [row for row in self.results_iter()] - self.update_entities(pks) + self.update_entities(pks, pk_field) return len(pks) - def update_entities(self, pks): + def update_entities(self, pks, pk_field): for pk in pks: - self.update_entity(pk[0]) + self.update_entity(pk[0], pk_field) @commit_locked - def update_entity(self, pk): + def update_entity(self, pk, pk_field): gae_query = self.build_query() - key = create_key(self.query.get_meta().db_table, pk) - entity = Get(key) + entity = Get(self.ops.value_for_db(pk, pk_field)) + if not gae_query.matches_filters(entity): return - qn = self.quote_name_unless_alias - update_dict = {} - for field, o, value in self.query.values: + for field, _, value in self.query.values: if hasattr(value, 'prepare_database_save'): value = value.prepare_database_save(field) else: - value = field.get_db_prep_save(value, connection=self.connection) + value = field.get_db_prep_save(value, + connection=self.connection) - if hasattr(value, "evaluate"): + if hasattr(value, 'evaluate'): assert not value.negated assert not value.subtree_parents value = ExpressionEvaluator(value, self.query, entity, - allow_joins=False) + allow_joins=False) if hasattr(value, 'as_sql'): - # evaluate expression and return the new value - val = value.as_sql(qn, self.connection) - update_dict[field] = val - else: - update_dict[field] = value + value = value.as_sql(lambda n: n, self.connection) - for field, value in update_dict.iteritems(): - db_type = field.db_type(connection=self.connection) - entity[qn(field.column)] = self.convert_value_for_db(db_type, value) + entity[field.column] = self.ops.value_for_db(value, field) + + Put(entity) - key = Put(entity) class SQLDeleteCompiler(NonrelDeleteCompiler, SQLCompiler): pass - -def to_datetime(value): - """Convert a time or date to a datetime for datastore storage. - - Args: - value: A datetime.time, datetime.date or string object. - - Returns: - A datetime object with date set to 1970-01-01 if value is a datetime.time - A datetime object with date set to value.year - value.month - value.day and - time set to 0:00 if value is a datetime.date - """ - - if value is None: - return value - elif isinstance(value, datetime.datetime): - return value - elif isinstance(value, datetime.date): - return datetime.datetime(value.year, value.month, value.day) - elif isinstance(value, datetime.time): - return datetime.datetime(1970, 1, 1, value.hour, value.minute, - value.second, value.microsecond) - -def create_key(db_table, value): - if isinstance(value, (int, long)) and value < 1: - return None - return Key.from_path(db_table, value) diff --git a/db/creation.py b/db/creation.py index 912a52e..7a269dc 100644 --- a/db/creation.py +++ b/db/creation.py @@ -1,35 +1,75 @@ +from djangotoolbox.db.creation import NonrelDatabaseCreation + from .db_settings import get_model_indexes from .stubs import stub_manager -from djangotoolbox.db.creation import NonrelDatabaseCreation -class StringType(object): - def __init__(self, internal_type): - self.internal_type = internal_type - - def __mod__(self, field): - indexes = get_model_indexes(field['model']) - if field['name'] in indexes['indexed']: - return 'text' - elif field['name'] in indexes['unindexed']: - return 'longtext' - return self.internal_type - -def get_data_types(): - # TODO: Add GAEKeyField and a corresponding db_type - string_types = ('text', 'longtext') - data_types = NonrelDatabaseCreation.data_types.copy() - for name, field_type in data_types.items(): - if field_type in string_types: - data_types[name] = StringType(field_type) - return data_types class DatabaseCreation(NonrelDatabaseCreation): - # This dictionary maps Field objects to their associated GAE column - # types, as strings. Column-type strings can contain format strings; they'll - # be interpolated against the values of Field.__dict__ before being output. - # If a column type is set to None, it won't be included in the output. - data_types = get_data_types() + # For TextFields and XMLFields we'll default to the unindexable, + # but not length-limited, db.Text (db_type of "string" fields is + # overriden indexed / unindexed fields). + # GAE datastore cannot process sets directly, so we'll store them + # as lists, it also can't handle dicts so we'll store DictField and + # EmbeddedModelFields pickled as Blobs (pickled using the binary + # protocol 2, even though they used to be serialized with the ascii + # protocol 0 -- the deconversion is the same for both). + data_types = dict(NonrelDatabaseCreation.data_types, **{ + 'TextField': 'text', + 'XMLField': 'text', + 'SetField': 'list', + 'DictField': 'bytes', + 'EmbeddedModelField': 'bytes', + }) + + def db_type(self, field): + """ + Provides a choice to continue using db.Key just for primary key + storage or to use it for all references (ForeignKeys and other + relations). + + We also force the "string" db_type (plain string storage) if a + field is to be indexed, and the "text" db_type (db.Text) if + it's registered as unindexed. + """ + from djangoappengine.fields import DbKeyField + + # DBKeyField reads/stores db.Key objects directly + # so its treated as a special case + if isinstance(field, DbKeyField): + return field.db_type(connection=self.connection) + + if self.connection.settings_dict.get('STORE_RELATIONS_AS_DB_KEYS'): + if field.primary_key or field.rel is not None: + return 'key' + + # Primary keys were processed as db.Keys; for related fields + # the db_type of primary key of the referenced model was used, + # but RelatedAutoField type was not defined and resulted in + # "integer" being used for relations to models with AutoFields. + # TODO: Check with Positive/SmallIntegerField primary keys. + else: + if field.primary_key: + return 'key' + if field.rel is not None: + related_field = field.rel.get_related_field() + if related_field.get_internal_type() == 'AutoField': + return 'integer' + else: + return related_field.db_type(connection=self.connection) + + db_type = field.db_type(connection=self.connection) + + # Override db_type of "string" fields according to indexing. + if db_type in ('string', 'text'): + indexes = get_model_indexes(field.model) + if field.attname in indexes['indexed']: + return 'string' + elif field.attname in indexes['unindexed']: + return 'text' + + return db_type + def _create_test_db(self, *args, **kw): self._had_test_stubs = stub_manager.active_stubs != 'test' @@ -37,7 +77,8 @@ def _create_test_db(self, *args, **kw): stub_manager.activate_test_stubs() def _destroy_test_db(self, *args, **kw): - if self._had_test_stubs: - stub_manager.deactivate_test_stubs() - stub_manager.setup_stubs(self.connection) - del self._had_test_stubs + if hasattr(self, '_had_test_stubs'): + if self._had_test_stubs: + stub_manager.deactivate_test_stubs() + stub_manager.setup_stubs(self.connection) + del self._had_test_stubs diff --git a/db/db_settings.py b/db/db_settings.py index 6263b0c..262f7fb 100644 --- a/db/db_settings.py +++ b/db/db_settings.py @@ -1,13 +1,17 @@ from django.conf import settings from django.utils.importlib import import_module -# TODO: add autodiscover() and make API more like dbindexer's register_index +# TODO: Add autodiscover() and make API more like dbindexer's +# register_index. + +# TODO: Add support for eventual consistency setting on specific +# models. + _MODULE_NAMES = getattr(settings, 'GAE_SETTINGS_MODULES', ()) FIELD_INDEXES = None -# TODO: add support for eventual consistency setting on specific models def get_model_indexes(model): indexes = get_indexes() @@ -18,6 +22,7 @@ def get_model_indexes(model): model_index['unindexed'].extend(config.get('unindexed', ())) return model_index + def get_indexes(): global FIELD_INDEXES if FIELD_INDEXES is None: diff --git a/db/expressions.py b/db/expressions.py index d04d73a..9bfa895 100644 --- a/db/expressions.py +++ b/db/expressions.py @@ -1,21 +1,25 @@ from django.db.models.sql.expressions import SQLEvaluator from django.db.models.expressions import ExpressionNode + OPERATION_MAP = { - ExpressionNode.ADD: lambda x, y: x+y, - ExpressionNode.SUB: lambda x, y: x-y, - ExpressionNode.MUL: lambda x, y: x*y, - ExpressionNode.DIV: lambda x, y: x/y, - ExpressionNode.MOD: lambda x, y: x%y, - ExpressionNode.AND: lambda x, y: x&y, - ExpressionNode.OR: lambda x, y: x|y, + ExpressionNode.ADD: lambda x, y: x + y, + ExpressionNode.SUB: lambda x, y: x - y, + ExpressionNode.MUL: lambda x, y: x * y, + ExpressionNode.DIV: lambda x, y: x / y, + ExpressionNode.MOD: lambda x, y: x % y, + ExpressionNode.AND: lambda x, y: x & y, + ExpressionNode.OR: lambda x, y: x | y, } + class ExpressionEvaluator(SQLEvaluator): + def __init__(self, expression, query, entity, allow_joins=True): - super(ExpressionEvaluator, self).__init__(expression, query, allow_joins) + super(ExpressionEvaluator, self).__init__(expression, query, + allow_joins) self.entity = entity - + ################################################## # Vistor methods for final expression evaluation # ################################################## @@ -34,4 +38,4 @@ def evaluate_node(self, node, qn, connection): return OPERATION_MAP[node.connector](*values) def evaluate_leaf(self, node, qn, connection): - return self.entity[qn(self.cols[node][1])] \ No newline at end of file + return self.entity[qn(self.cols[node][1])] diff --git a/db/models/__init__.py b/db/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/db/models/manager.py b/db/models/manager.py new file mode 100644 index 0000000..5b5e732 --- /dev/null +++ b/db/models/manager.py @@ -0,0 +1,14 @@ +from django.db.models import Manager as _baseManager +from djangoappengine.db.utils import as_ancestor +from djangoappengine.db.models.query import QuerySet + +class Manager(_baseManager): + + def get_query_set(self): + """Returns a new QuerySet object. Subclasses can override this method + to easily customize the behavior of the Manager. + """ + return QuerySet(self.model, using=self._db) + + def ancestor(self, ancestor): + return self.get_query_set().ancestor(ancestor) \ No newline at end of file diff --git a/db/models/query.py b/db/models/query.py new file mode 100644 index 0000000..d81c661 --- /dev/null +++ b/db/models/query.py @@ -0,0 +1,17 @@ +from django.db.models.query import QuerySet as _baseQuerySet +from djangoappengine.db.utils import as_ancestor + +class QuerySet(_baseQuerySet): + def ancestor(self, ancestor): + """ + Returns a new QuerySet instance with the args ANDed to the existing + set. + """ + return self._filter_or_exclude(False, pk=as_ancestor(ancestor)) + +class EmptyQuerySet(QuerySet): + def ancestor(self, *args, **kwargs): + """ + Always returns EmptyQuerySet. + """ + return self \ No newline at end of file diff --git a/db/stubs.py b/db/stubs.py index 4ace516..f3c3258 100644 --- a/db/stubs.py +++ b/db/stubs.py @@ -1,25 +1,32 @@ -from ..utils import appid, have_appserver -from ..boot import PROJECT_DIR -from google.appengine.ext.testbed import Testbed -from urllib2 import HTTPError, URLError import logging import time +from urllib2 import HTTPError, URLError + +from google.appengine.ext.testbed import Testbed + +from ..boot import PROJECT_DIR +from ..utils import appid, have_appserver + REMOTE_API_SCRIPTS = ( '$PYTHON_LIB/google/appengine/ext/remote_api/handler.py', 'google.appengine.ext.remote_api.handler.application', ) + def auth_func(): import getpass - return raw_input('Login via Google Account (see note above if login fails): '), getpass.getpass('Password: ') + return raw_input("Login via Google Account (see note above if login fails): "), getpass.getpass("Password: ") + def rpc_server_factory(*args, ** kwargs): from google.appengine.tools import appengine_rpc kwargs['save_cookies'] = True return appengine_rpc.HttpRpcServer(*args, ** kwargs) + class StubManager(object): + def __init__(self): self.testbed = Testbed() self.active_stubs = None @@ -76,13 +83,14 @@ def setup_remote_stubs(self, connection): break server = '%s.%s' % (connection.remote_app_id, connection.domain) remote_url = 'https://%s%s' % (server, connection.remote_api_path) - logging.info('Setting up remote_api for "%s" at %s' % + logging.info("Setting up remote_api for '%s' at %s." % (connection.remote_app_id, remote_url)) if not have_appserver: - print('Connecting to remote_api handler.\n\n' - 'IMPORTANT: Check your login method settings in the ' - 'App Engine Dashboard if you have problems logging in. ' - 'Login is only supported for Google Accounts.\n') + logging.info( + "Connecting to remote_api handler.\n\n" + "IMPORTANT: Check your login method settings in the " + "App Engine Dashboard if you have problems logging in. " + "Login is only supported for Google Accounts.") from google.appengine.ext.remote_api import remote_api_stub remote_api_stub.ConfigureRemoteApi(None, connection.remote_api_path, auth_func, servername=server, @@ -94,7 +102,7 @@ def setup_remote_stubs(self, connection): remote_api_stub.MaybeInvokeAuthentication() except HTTPError, e: if not have_appserver: - print 'Retrying in %d seconds...' % retry_delay + logging.info("Retrying in %d seconds..." % retry_delay) time.sleep(retry_delay) retry_delay *= 2 else: @@ -110,10 +118,10 @@ def setup_remote_stubs(self, connection): "Note that login is only supported for " "Google Accounts. Make sure you've configured " "the correct authentication method in the " - "App Engine Dashboard." - % (e, remote_url)) - logging.info('Now using the remote datastore for "%s" at %s' % + "App Engine Dashboard." % (e, remote_url)) + logging.info("Now using the remote datastore for '%s' at %s." % (connection.remote_app_id, remote_url)) self.active_stubs = 'remote' + stub_manager = StubManager() diff --git a/db/utils.py b/db/utils.py index 76085c3..f7540be 100644 --- a/db/utils.py +++ b/db/utils.py @@ -1,22 +1,29 @@ +from google.appengine.api.datastore import Key from google.appengine.datastore.datastore_query import Cursor + from django.db import models, DEFAULT_DB_ALIAS + try: from functools import wraps except ImportError: from django.utils.functional import wraps # Python 2.3, 2.4 fallback. + class CursorQueryMixin(object): + def clone(self, *args, **kwargs): kwargs['_gae_cursor'] = getattr(self, '_gae_cursor', None) kwargs['_gae_start_cursor'] = getattr(self, '_gae_start_cursor', None) kwargs['_gae_end_cursor'] = getattr(self, '_gae_end_cursor', None) return super(CursorQueryMixin, self).clone(*args, **kwargs) + def get_cursor(queryset): - # Evaluate QuerySet + # Evaluate QuerySet. len(queryset) cursor = getattr(queryset.query, '_gae_cursor', None) - return Cursor.to_websafe_string(cursor) + return Cursor.to_websafe_string(cursor) if cursor else None + def set_cursor(queryset, start=None, end=None): queryset = queryset.all() @@ -33,18 +40,54 @@ class CursorQuery(CursorQueryMixin, queryset.query.__class__): queryset.query._gae_end_cursor = end return queryset + def commit_locked(func_or_using=None): """ Decorator that locks rows on DB reads. """ + def inner_commit_locked(func, using=None): + def _commit_locked(*args, **kw): from google.appengine.api.datastore import RunInTransaction return RunInTransaction(func, *args, **kw) + return wraps(func)(_commit_locked) + if func_or_using is None: func_or_using = DEFAULT_DB_ALIAS if callable(func_or_using): return inner_commit_locked(func_or_using, DEFAULT_DB_ALIAS) return lambda func: inner_commit_locked(func, func_or_using) +class AncestorKey(object): + def __init__(self, key): + self.key = key + +def as_ancestor(key_or_model): + if key_or_model is None: + raise ValueError("key_or_model must not be None") + + if isinstance(key_or_model, models.Model): + key_or_model = Key.from_path(key_or_model._meta.db_table, key_or_model.pk) + + return AncestorKey(key_or_model) + +def make_key(*args, **kwargs): + parent = kwargs.pop('parent', None) + + if kwargs: + raise AssertionError('Excess keyword arguments; received %s' % kwargs) + + if not args or len(args) % 2: + raise AssertionError('A non-zero even number of positional arguments is required; received %s' % args) + + if isinstance(parent, models.Model): + parent = Key.from_path(parent._meta.db_table, parent.pk) + + converted_args = [] + for i in xrange(0, len(args), 2): + model, id_or_name = args[i:i+2] + converted_args.extend((model._meta.db_table, id_or_name)) + + return Key.from_path(*converted_args, parent=parent) diff --git a/dbindexes.py b/dbindexes.py index abe0bf8..9a2e9ac 100644 --- a/dbindexes.py +++ b/dbindexes.py @@ -1,5 +1,6 @@ from django.conf import settings + if 'django.contrib.auth' in settings.INSTALLED_APPS: from dbindexer.api import register_index from django.contrib.auth.models import User @@ -8,3 +9,11 @@ 'username': 'iexact', 'email': 'iexact', }) + +if 'django.contrib.admin' in settings.INSTALLED_APPS: + from dbindexer.api import register_index + from django.contrib.admin.models import LogEntry + + register_index(LogEntry, { + 'object_id': 'exact', + }) diff --git a/deferred/handler.py b/deferred/handler.py index ff76125..b5a5ef8 100644 --- a/deferred/handler.py +++ b/deferred/handler.py @@ -1,19 +1,22 @@ -# Initialize Django +# Initialize Django. from djangoappengine import main from django.utils.importlib import import_module from django.conf import settings -# load all models.py to ensure signal handling installation or index loading -# of some apps + +# Load all models.py to ensure signal handling installation or index +# loading of some apps for app in settings.INSTALLED_APPS: try: import_module('%s.models' % (app)) except ImportError: pass + from google.appengine.ext.deferred.handler import main from google.appengine.ext.deferred.deferred import application + if __name__ == '__main__': main() diff --git a/docs/conf.py b/docs/conf.py index 08fbcc7..5724a46 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -214,7 +214,7 @@ # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'djangoappengine', u'Django App Engine Documentation', - [u'AllButtonsPressed, Potato London, Wilfred Hughes'], 1) + [u'AllButtonsPressed, Potato London, Wilfred Hughes'], 1), ] # If true, show URL addresses after external links. @@ -228,7 +228,8 @@ # dir menu entry, description, category) texinfo_documents = [ ('index', 'DjangoAppEngine', u'Django App Engine Documentation', - u'AllButtonsPressed, Potato London, Wilfred Hughes', 'DjangoAppEngine', 'One line description of project.', + u'AllButtonsPressed, Potato London, Wilfred Hughes', 'DjangoAppEngine', + 'One line description of project.', 'Miscellaneous'), ] diff --git a/fields.py b/fields.py new file mode 100644 index 0000000..54197d5 --- /dev/null +++ b/fields.py @@ -0,0 +1,87 @@ +from django.core.exceptions import ValidationError +from django.db import models +from django.utils.encoding import smart_unicode + +from djangoappengine.db.utils import AncestorKey + +from google.appengine.api.datastore import Key, datastore_errors + +import logging + +class DbKeyField(models.Field): + description = "A field for native database key objects" + __metaclass__ = models.SubfieldBase + + def __init__(self, *args, **kwargs): + kwargs['blank'] = True + + self.parent_key_attname = kwargs.pop('parent_key_name', None) + + if self.parent_key_attname is not None and kwargs.get('primary_key', None) is None: + raise ValueError("Primary key must be true to use parent_key_name") + + super(DbKeyField, self).__init__(*args, **kwargs) + + def contribute_to_class(self, cls, name): + if self.primary_key: + assert not cls._meta.has_auto_field, "A model can't have more than one auto field." + cls._meta.has_auto_field = True + cls._meta.auto_field = self + + if self.parent_key_attname is not None: + def get_parent_key(instance, instance_type=None): + if instance is None: + return self + + return instance.__dict__.get(self.parent_key_attname) + + def set_parent_key(instance, value): + if instance is None: + raise AttributeError("Attribute must be accessed via instance") + + if not isinstance(value, Key): + raise ValueError("'%s' must be a Key" % self.parent_key_attname) + + instance.__dict__[self.parent_key_attname] = value + + setattr(cls, self.parent_key_attname, property(get_parent_key, set_parent_key)) + + super(DbKeyField, self).contribute_to_class(cls, name) + + def to_python(self, value): + if value is None: + return None + if isinstance(value, Key): + return value + if isinstance(value, basestring): + if len(value) == 0: + return None + + try: + return Key(encoded=value) + except datastore_errors.BadKeyError: + return Key.from_path(self.model._meta.db_table, long(value)) + if isinstance(value, (int, long)): + return Key.from_path(self.model._meta.db_table, value) + + raise ValidationError("DbKeyField does not accept %s" % type(value)) + + def get_prep_value(self, value): + if isinstance(value, AncestorKey): + return value + return self.to_python(value) + + def pre_save(self, model_instance, add): + value = super(DbKeyField, self).pre_save(model_instance, add) + + if add and value is None and self.parent_key_attname is not None and hasattr(model_instance, self.parent_key_attname): + stashed_parent = getattr(model_instance, self.parent_key_attname) + value = Key.from_path(self.model._meta.db_table, 0, parent=stashed_parent) + + return value + + def formfield(self, **kwargs): + return None + + def value_to_string(self, obj): + return smart_unicode(self._get_val_from_obj(obj)) diff --git a/mail.py b/mail.py index 67e2e61..044bf6f 100644 --- a/mail.py +++ b/mail.py @@ -1,10 +1,13 @@ from email.MIMEBase import MIMEBase + from django.core.mail.backends.base import BaseEmailBackend from django.core.mail import EmailMultiAlternatives from django.core.exceptions import ImproperlyConfigured + from google.appengine.api import mail as aeemail from google.appengine.runtime import apiproxy_errors + def _send_deferred(message, fail_silently=False): try: message.send() @@ -12,6 +15,7 @@ def _send_deferred(message, fail_silently=False): if not fail_silently: raise + class EmailBackend(BaseEmailBackend): can_defer = False @@ -23,7 +27,9 @@ def send_messages(self, email_messages): return num_sent def _copy_message(self, message): - """Create and return App Engine EmailMessage class from message.""" + """ + Creates and returns App Engine EmailMessage class from message. + """ gmsg = aeemail.EmailMessage(sender=message.from_email, to=message.to, subject=message.subject, @@ -35,7 +41,7 @@ def _copy_message(self, message): if message.bcc: gmsg.bcc = list(message.bcc) if message.attachments: - # Must be populated with (filename, filecontents) tuples + # Must be populated with (filename, filecontents) tuples. attachments = [] for attachment in message.attachments: if isinstance(attachment, MIMEBase): @@ -44,7 +50,7 @@ def _copy_message(self, message): else: attachments.append((attachment[0], attachment[1])) gmsg.attachments = attachments - # Look for HTML alternative content + # Look for HTML alternative content. if isinstance(message, EmailMultiAlternatives): for content, mimetype in message.alternatives: if mimetype == 'text/html': @@ -81,5 +87,6 @@ def _defer_message(self, message): fail_silently=self.fail_silently, _queue=queue_name) + class AsyncEmailBackend(EmailBackend): can_defer = True diff --git a/main/__init__.py b/main/__init__.py index aaab44c..9bceeb2 100644 --- a/main/__init__.py +++ b/main/__init__.py @@ -1,11 +1,13 @@ import os import sys + # Add parent folder to sys.path, so we can import boot. # App Engine causes main.py to be reloaded if an exception gets raised -# on the first request of a main.py instance, so don't add project_dir multiple -# times. -project_dir = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) +# on the first request of a main.py instance, so don't add project_dir +# multiple times. +project_dir = os.path.abspath( + os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) if project_dir not in sys.path or sys.path.index(project_dir) > 0: while project_dir in sys.path: sys.path.remove(project_dir) @@ -25,11 +27,12 @@ from djangoappengine.boot import setup_env setup_env() + def validate_models(): - """Since BaseRunserverCommand is only run once, we need to call + """ + Since BaseRunserverCommand is only run once, we need to call model valdidation here to ensure it is run every time the code changes. - """ import logging from django.core.management.validation import get_validation_errors @@ -46,7 +49,8 @@ def validate_models(): if num_errors: s.seek(0) error_text = s.read() - logging.critical("One or more models did not validate:\n%s" % error_text) + logging.critical("One or more models did not validate:\n%s" % + error_text) else: logging.info("All models validated.") @@ -58,21 +62,24 @@ def validate_models(): from google.appengine.ext.webapp.util import run_wsgi_app from django.conf import settings + def log_traceback(*args, **kwargs): import logging - logging.exception('Exception in request:') + logging.exception("Exception in request:") from django.core import signals signals.got_request_exception.connect(log_traceback) -# Create a Django application for WSGI + +# Create a Django application for WSGI. application = WSGIHandler() -# Add the staticfiles handler if necessary +# Add the staticfiles handler if necessary. if settings.DEBUG and 'django.contrib.staticfiles' in settings.INSTALLED_APPS: from django.contrib.staticfiles.handlers import StaticFilesHandler application = StaticFilesHandler(application) if getattr(settings, 'ENABLE_APPSTATS', False): - from google.appengine.ext.appstats.recording import appstats_wsgi_middleware + from google.appengine.ext.appstats.recording import \ + appstats_wsgi_middleware application = appstats_wsgi_middleware(application) diff --git a/main/main.py b/main/main.py index 9a87981..9ee5251 100644 --- a/main/main.py +++ b/main/main.py @@ -1,4 +1,4 @@ -# Python 2.5 CGI handler +# Python 2.5 CGI handler. import os import sys @@ -8,10 +8,11 @@ from djangoappengine.boot import setup_logging, env_ext from django.conf import settings + path_backup = None def real_main(): - # Reset path and environment variables + # Reset path and environment variables. global path_backup try: sys.path = path_backup[:] @@ -23,6 +24,7 @@ def real_main(): # Run the WSGI CGI handler with that application. run_wsgi_app(application) + def profile_main(func): from cStringIO import StringIO import cProfile @@ -61,7 +63,8 @@ def profile_main(func): stats.print_callees() if 'callers' in extra_output: stats.print_callers() - logging.info('Profile data:\n%s', stream.getvalue()) + logging.info("Profile data:\n%s.", stream.getvalue()) + def make_profileable(func): if getattr(settings, 'ENABLE_PROFILER', False): diff --git a/management/commands/deploy.py b/management/commands/deploy.py index 2f57fe4..dcc250e 100644 --- a/management/commands/deploy.py +++ b/management/commands/deploy.py @@ -1,11 +1,14 @@ -from ...boot import PROJECT_DIR -from ...utils import appconfig +import logging +import time +import sys + from django.conf import settings from django.core.management import call_command from django.core.management.base import BaseCommand -import logging -import sys -import time + +from ...boot import PROJECT_DIR +from ...utils import appconfig + PRE_DEPLOY_COMMANDS = () if 'mediagenerator' in settings.INSTALLED_APPS: @@ -14,11 +17,14 @@ PRE_DEPLOY_COMMANDS) POST_DEPLOY_COMMANDS = getattr(settings, 'POST_DEPLOY_COMMANDS', ()) + def run_appcfg(argv): - # We don't really want to use that one though, it just executes this one + # We don't really want to use that one though, it just executes + # this one. from google.appengine.tools import appcfg - # Reset the logging level to WARN as appcfg will spew tons of logs on INFO + # Reset the logging level to WARN as appcfg will spew tons of logs + # on INFO. logging.getLogger().setLevel(logging.WARN) new_args = argv[:] @@ -33,8 +39,8 @@ def run_appcfg(argv): appcfg.main(new_args) if syncdb: - print 'Running syncdb.' - # Wait a little bit for deployment to finish + print "Running syncdb." + # Wait a little bit for deployment to finish. for countdown in range(9, 0, -1): sys.stdout.write('%s\r' % countdown) time.sleep(1) @@ -45,17 +51,19 @@ def run_appcfg(argv): call_command('syncdb', remote=True, interactive=True) if getattr(settings, 'ENABLE_PROFILER', False): - print '--------------------------\n' \ - 'WARNING: PROFILER ENABLED!\n' \ - '--------------------------' + print "--------------------------\n" \ + "WARNING: PROFILER ENABLED!\n" \ + "--------------------------" + class Command(BaseCommand): - """Deploys the website to the production server. + """ + Deploys the website to the production server. - Any additional arguments are passed directly to appcfg.py update + Any additional arguments are passed directly to appcfg.py update. """ - help = 'Calls appcfg.py update for the current project.' - args = '[any appcfg.py options]' + help = "Calls appcfg.py update for the current project." + args = "[any appcfg.py options]" def run_from_argv(self, argv): for command in PRE_DEPLOY_COMMANDS: diff --git a/management/commands/remote.py b/management/commands/remote.py index 1422308..c935a67 100644 --- a/management/commands/remote.py +++ b/management/commands/remote.py @@ -1,10 +1,11 @@ from django.core.management import execute_from_command_line from django.core.management.base import BaseCommand + class Command(BaseCommand): - help = 'Runs a command with access to the remote App Engine production ' \ - 'server (e.g. manage.py remote shell)' - args = 'remotecommand' + help = "Runs a command with access to the remote App Engine production " \ + "server (e.g. manage.py remote shell)." + args = "remotecommand" def run_from_argv(self, argv): from django.db import connections diff --git a/management/commands/runserver.py b/management/commands/runserver.py index a190692..4b5a020 100644 --- a/management/commands/runserver.py +++ b/management/commands/runserver.py @@ -1,92 +1,123 @@ -from optparse import make_option import logging +from optparse import make_option import sys from django.db import connections -from ...boot import PROJECT_DIR -from ...db.base import DatabaseWrapper, get_datastore_paths from django.core.management.base import BaseCommand from django.core.management.commands.runserver import BaseRunserverCommand from django.core.exceptions import ImproperlyConfigured from google.appengine.tools import dev_appserver_main +from ...boot import PROJECT_DIR +from ...db.base import DatabaseWrapper, get_datastore_paths + class Command(BaseRunserverCommand): - """Overrides the default Django runserver command. + """ + Overrides the default Django runserver command. - Instead of starting the default Django development server this command - fires up a copy of the full fledged App Engine dev_appserver that emulates - the live environment your application will be deployed to. + Instead of starting the default Django development server this + command fires up a copy of the full fledged App Engine + dev_appserver that emulates the live environment your application + will be deployed to. """ option_list = BaseCommand.option_list + ( - make_option('--debug', action='store_true', default=False, - help='Prints verbose debugging messages to the console while running.'), - make_option('--debug_imports', action='store_true', default=False, - help='Prints debugging messages related to importing modules, including \ - search paths and errors.'), - make_option('-c', '--clear_datastore', action='store_true', default=False, - help='Clears the datastore data and history files before starting the web server.'), - make_option('--high_replication', action='store_true', default=False, + make_option( + '--debug', action='store_true', default=False, + help="Prints verbose debugging messages to the console while " \ + "running."), + make_option( + '--debug_imports', action='store_true', default=False, + help="Prints debugging messages related to importing modules, " \ + "including search paths and errors."), + make_option( + '-c', '--clear_datastore', action='store_true', default=False, + help="Clears the datastore data and history files before " \ + "starting the web server."), + make_option( + '--high_replication', action='store_true', default=False, help='Use the high replication datastore consistency model.'), - make_option('--require_indexes', action='store_true', default=False, - help="""Disables automatic generation of entries in the index.yaml file. Instead, when - the application makes a query that requires that its index be defined in the - file and the index definition is not found, an exception will be raised, - similar to what would happen when running on App Engine."""), - make_option('--enable_sendmail', action='store_true', default=False, - help='Uses the local computer\'s Sendmail installation for sending email messages.'), - make_option('--datastore_path', - help="""The path to use for the local datastore data file. The server creates this file - if it does not exist."""), - make_option('--history_path', - help="""The path to use for the local datastore history file. The server uses the query - history file to generate entries for index.yaml."""), - make_option('--login_url', - help='The relative URL to use for the Users sign-in page. Default is /_ah/login.'), - make_option('--smtp_host', - help='The hostname of the SMTP server to use for sending email messages.'), - make_option('--smtp_port', - help='The port number of the SMTP server to use for sending email messages.'), - make_option('--smtp_user', - help='The username to use with the SMTP server for sending email messages.'), - make_option('--smtp_password', - help='The password to use with the SMTP server for sending email messages.'), - make_option('--use_sqlite', action='store_true', default=False, - help='Use the new, SQLite datastore stub.'), - make_option('--allow_skipped_files', action='store_true', default=False, - help='Allow access to files listed in skip_files.'), - make_option('--disable_task_running', action='store_true', default=False, - help='When supplied, tasks will not be automatically run after submission and must be run manually in the local admin console.'), + make_option( + '--require_indexes', action='store_true', default=False, + help="Disables automatic generation of entries in the " \ + "index.yaml file. Instead, when the application makes a " \ + "query that requires that its index be defined in the file " \ + "and the index definition is not found, an exception will " \ + "be raised, similar to what would happen when running on " \ + "App Engine."), + make_option( + '--enable_sendmail', action='store_true', default=False, + help="Uses the local computer's Sendmail installation for " \ + "sending email messages."), + make_option( + '--datastore_path', + help="The path to use for the local datastore data file. " \ + "The server creates this file if it does not exist."), + make_option( + '--history_path', + help="The path to use for the local datastore history file. " \ + "The server uses the query history file to generate " \ + "entries for index.yaml."), + make_option( + '--login_url', + help="The relative URL to use for the Users sign-in page. " \ + "Default is /_ah/login."), + make_option( + '--smtp_host', + help="The hostname of the SMTP server to use for sending email " \ + "messages."), + make_option( + '--smtp_port', + help="The port number of the SMTP server to use for sending " \ + "email messages."), + make_option( + '--smtp_user', + help="The username to use with the SMTP server for sending " \ + "email messages."), + make_option( + '--smtp_password', + help="The password to use with the SMTP server for sending " \ + "email messages."), + make_option( + '--use_sqlite', action='store_true', default=False, + help="Use the new, SQLite datastore stub."), + make_option( + '--allow_skipped_files', action='store_true', default=False, + help="Allow access to files listed in skip_files."), + make_option( + '--disable_task_running', action='store_true', default=False, + help="When supplied, tasks will not be automatically run after " \ + "submission and must be run manually in the local admin " \ + "console."), ) - help = 'Runs a copy of the App Engine development server.' - args = '[optional port number, or ipaddr:port]' + help = "Runs a copy of the App Engine development server." + args = "[optional port number, or ipaddr:port]" def create_parser(self, prog_name, subcommand): """ - Create and return the ``OptionParser`` which will be used to + Creates and returns the ``OptionParser`` which will be used to parse the arguments to this command. - """ - # hack __main__ so --help in dev_appserver_main works OK. + # Hack __main__ so --help in dev_appserver_main works OK. sys.modules['__main__'] = dev_appserver_main return super(Command, self).create_parser(prog_name, subcommand) def run_from_argv(self, argv): """ - Captures the program name, usually "manage.py" + Captures the program name, usually "manage.py". """ - self.progname = argv[0] super(Command, self).run_from_argv(argv) def run(self, *args, **options): """ - Starts the App Engine dev_appserver program for the Django project. - The appserver is run with default parameters. If you need to pass any special - parameters to the dev_appserver you will have to invoke it manually. + Starts the App Engine dev_appserver program for the Django + project. The appserver is run with default parameters. If you + need to pass any special parameters to the dev_appserver you + will have to invoke it manually. Unlike the normal devserver, does not use the autoreloader as App Engine dev_appserver needs to be run from the main thread @@ -95,63 +126,72 @@ def run(self, *args, **options): args = [] # Set bind ip/port if specified. if self.addr: - args.extend(["--address", self.addr]) + args.extend(['--address', self.addr]) if self.port: - args.extend(["--port", self.port]) + args.extend(['--port', self.port]) - # If runserver is called using handle(), progname will not be set + # If runserver is called using handle(), progname will not be + # set. if not hasattr(self, 'progname'): - self.progname = "manage.py" + self.progname = 'manage.py' - # Add email settings + # Add email settings. from django.conf import settings - if not options.get('smtp_host', None) and not options.get('enable_sendmail', None): + if not options.get('smtp_host', None) and \ + not options.get('enable_sendmail', None): args.extend(['--smtp_host', settings.EMAIL_HOST, - '--smtp_port', str(settings.EMAIL_PORT), - '--smtp_user', settings.EMAIL_HOST_USER, - '--smtp_password', settings.EMAIL_HOST_PASSWORD]) + '--smtp_port', str(settings.EMAIL_PORT), + '--smtp_user', settings.EMAIL_HOST_USER, + '--smtp_password', settings.EMAIL_HOST_PASSWORD]) - # Pass the application specific datastore location to the server. + # Pass the application specific datastore location to the + # server. preset_options = {} for name in connections: connection = connections[name] if isinstance(connection, DatabaseWrapper): - for key, path in get_datastore_paths(connection.settings_dict).items(): - # XXX/TODO: Remove this when SDK 1.4.3 is released + for key, path in get_datastore_paths( + connection.settings_dict).items(): + # XXX/TODO: Remove this when SDK 1.4.3 is released. if key == 'prospective_search_path': continue arg = '--' + key if arg not in args: args.extend([arg, path]) - # Get dev_appserver option presets, to be applied below - preset_options = connection.settings_dict.get('DEV_APPSERVER_OPTIONS', {}) + # Get dev_appserver option presets, to be applied below. + preset_options = connection.settings_dict.get( + 'DEV_APPSERVER_OPTIONS', {}) break - # Process the rest of the options here - bool_options = ['debug', 'debug_imports', 'clear_datastore', 'require_indexes', - 'high_replication', 'enable_sendmail', 'use_sqlite', 'allow_skipped_files','disable_task_running', ] + # Process the rest of the options here. + bool_options = [ + 'debug', 'debug_imports', 'clear_datastore', 'require_indexes', + 'high_replication', 'enable_sendmail', 'use_sqlite', + 'allow_skipped_files', 'disable_task_running', ] for opt in bool_options: if options[opt] != False: - args.append("--%s" % opt) + args.append('--%s' % opt) - str_options = ['datastore_path', 'history_path', 'login_url', 'smtp_host', 'smtp_port', - 'smtp_user', 'smtp_password',] + str_options = [ + 'datastore_path', 'history_path', 'login_url', 'smtp_host', + 'smtp_port', 'smtp_user', 'smtp_password', ] for opt in str_options: if options.get(opt, None) != None: - args.extend(["--%s" % opt, options[opt]]) + args.extend(['--%s' % opt, options[opt]]) - # Fill any non-overridden options with presets from settings + # Fill any non-overridden options with presets from settings. for opt, value in preset_options.items(): - arg = "--%s" % opt + arg = '--%s' % opt if arg not in args: if value and opt in bool_options: args.append(arg) elif opt in str_options: args.extend([arg, value]) - # TODO: issue warning about bogus option key(s)? + # TODO: Issue warning about bogus option key(s)? - # Reset logging level to INFO as dev_appserver will spew tons of debug logs + # Reset logging level to INFO as dev_appserver will spew tons + # of debug logs. logging.getLogger().setLevel(logging.INFO) # Append the current working directory to the arguments. diff --git a/mapreduce/handler.py b/mapreduce/handler.py index 427ada4..0d58088 100644 --- a/mapreduce/handler.py +++ b/mapreduce/handler.py @@ -1,10 +1,10 @@ -# Initialize Django +# Initialize Django. from djangoappengine import main from django.utils.importlib import import_module from django.conf import settings -# load all models.py to ensure signal handling installation or index loading -# of some apps +# Load all models.py to ensure signal handling installation or index +# loading of some apps. for app in settings.INSTALLED_APPS: try: import_module('%s.models' % app) @@ -13,5 +13,6 @@ from google.appengine.ext.mapreduce.main import APP as application, main + if __name__ == '__main__': main() diff --git a/settings_base.py b/settings_base.py index abdc89b..58e8d0c 100644 --- a/settings_base.py +++ b/settings_base.py @@ -1,4 +1,4 @@ -# Initialize App Engine SDK if necessary +# Initialize App Engine SDK if necessary. try: from google.appengine.api import apiproxy_stub_map except ImportError: @@ -7,6 +7,7 @@ from djangoappengine.utils import on_production_server, have_appserver + DEBUG = not on_production_server TEMPLATE_DEBUG = DEBUG @@ -16,26 +17,34 @@ 'default': { 'ENGINE': 'djangoappengine.db', - # Other settings which you might want to override in your settings.py + # Other settings which you might want to override in your + # settings.py. - # Activates high-replication support for remote_api + # Activates high-replication support for remote_api. # 'HIGH_REPLICATION': True, - # Switch to the App Engine for Business domain + # Switch to the App Engine for Business domain. # 'DOMAIN': 'googleplex.com', + # Store db.Keys as values of ForeignKey or other related + # fields. Warning: dump your data before, and reload it after + # changing! Defaults to False if not set. + # 'STORE_RELATIONS_AS_DB_KEYS': True, + 'DEV_APPSERVER_OPTIONS': { - # Optional parameters for development environment + # Optional parameters for development environment. - # Emulate the high-replication datastore locally + # Emulate the high-replication datastore locally. + # TODO: Likely to break loaddata (some records missing). # 'high_replication' : True, - # Use the SQLite backend for local storage (instead of default - # in-memory datastore). Useful for testing with larger datasets - # or when debugging concurrency/async issues (separate processes - # will share a common db state, rather than syncing on startup). + # Use the SQLite backend for local storage (instead of + # default in-memory datastore). Useful for testing with + # larger datasets or when debugging concurrency/async + # issues (separate processes will share a common db state, + # rather than syncing on startup). # 'use_sqlite': True, - } + }, }, } @@ -44,7 +53,7 @@ else: EMAIL_BACKEND = 'djangoappengine.mail.EmailBackend' -# Specify a queue name for the async. email backend +# Specify a queue name for the async. email backend. EMAIL_QUEUE_NAME = 'default' PREPARE_UPLOAD_BACKEND = 'djangoappengine.storage.prepare_upload' diff --git a/setup.py b/setup.py index 47ee825..d203670 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ -from setuptools import setup, find_packages +from setuptools import find_packages, setup + DESCRIPTION = 'App Engine backends for Django-nonrel' LONG_DESCRIPTION = None @@ -7,10 +8,12 @@ except: pass + setup(name='djangoappengine', version='1.0', package_dir={'djangoappengine': '.'}, - packages=['djangoappengine'] + ['djangoappengine.' + name for name in find_packages()], + packages=['djangoappengine'] + ['djangoappengine.' + name + for name in find_packages()], author='Waldemar Kornewald', author_email='wkornewald@gmail.com', url='http://www.allbuttonspressed.com/projects/djangoappengine', @@ -27,5 +30,5 @@ 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', 'License :: OSI Approved :: BSD License', - ], + ], ) diff --git a/storage.py b/storage.py index a26c712..e34af2e 100644 --- a/storage.py +++ b/storage.py @@ -18,9 +18,11 @@ from google.appengine.ext.blobstore import BlobInfo, BlobKey, delete, \ create_upload_url, BLOB_KEY_HEADER, BLOB_RANGE_HEADER, BlobReader + def prepare_upload(request, url, **kwargs): return create_upload_url(url), {} + def serve_file(request, file, save_as, content_type, **kwargs): if hasattr(file, 'file') and hasattr(file.file, 'blobstore_info'): blobkey = file.file.blobstore_info.key() @@ -36,20 +38,23 @@ def serve_file(request, file, save_as, content_type, **kwargs): if http_range is not None: response[BLOB_RANGE_HEADER] = http_range if save_as: - response['Content-Disposition'] = smart_str(u'attachment; filename=%s' % save_as) + response['Content-Disposition'] = smart_str( + u'attachment; filename=%s' % save_as) if file.size is not None: response['Content-Length'] = file.size return response + class BlobstoreStorage(Storage): - """Google App Engine Blobstore storage backend""" + """Google App Engine Blobstore storage backend.""" def _open(self, name, mode='rb'): return BlobstoreFile(name, mode, self) def _save(self, name, content): name = name.replace('\\', '/') - if hasattr(content, 'file') and hasattr(content.file, 'blobstore_info'): + if hasattr(content, 'file') and \ + hasattr(content.file, 'blobstore_info'): data = content.file.blobstore_info elif hasattr(content, 'blobstore_info'): data = content.blobstore_info @@ -59,7 +64,7 @@ def _save(self, name, content): "whose file attribute is a BlobstoreFile.") if isinstance(data, (BlobInfo, BlobKey)): - # We change the file name to the BlobKey's str() value + # We change the file name to the BlobKey's str() value. if isinstance(data, BlobInfo): data = data.key() return '%s/%s' % (data, name.lstrip('/')) @@ -93,7 +98,9 @@ def _get_key(self, name): def _get_blobinfo(self, name): return BlobInfo.get(self._get_key(name)) + class BlobstoreFile(File): + def __init__(self, name, mode, storage): self.name = name self._storage = storage @@ -113,9 +120,10 @@ def file(self): self._file = BlobReader(self.blobstore_info.key()) return self._file + class BlobstoreFileUploadHandler(FileUploadHandler): """ - File upload handler for the Google App Engine Blobstore + File upload handler for the Google App Engine Blobstore. """ def new_file(self, *args, **kwargs): @@ -144,10 +152,12 @@ def file_complete(self, file_size): blobinfo=BlobInfo(self.blobkey), charset=self.charset) + class BlobstoreUploadedFile(UploadedFile): """ A file uploaded into memory (i.e. stream-to-memory). """ + def __init__(self, blobinfo, charset): super(BlobstoreUploadedFile, self).__init__( BlobReader(blobinfo.key()), blobinfo.filename, @@ -157,7 +167,7 @@ def __init__(self, blobinfo, charset): def open(self, mode=None): pass - def chunks(self, chunk_size=1024*128): + def chunks(self, chunk_size=1024 * 128): self.file.seek(0) while True: content = self.read(chunk_size) @@ -165,5 +175,5 @@ def chunks(self, chunk_size=1024*128): break yield content - def multiple_chunks(self, chunk_size=1024*128): + def multiple_chunks(self, chunk_size=1024 * 128): return True diff --git a/test.py b/test.py new file mode 100644 index 0000000..ae8a773 --- /dev/null +++ b/test.py @@ -0,0 +1,32 @@ +from django.test import TestCase + +from google.appengine.datastore import datastore_stub_util + +from db.stubs import stub_manager + +class GAETestCase(TestCase): + def _pre_setup(self): + """Performs any pre-test setup. + * Set the dev_appserver consistency state. + """ + super(GAETestCase,self)._pre_setup() + + if hasattr(self, 'consistency_probability'): + datastore = stub_manager.testbed.get_stub('datastore_v3') + self._orig_policy = datastore._consistency_policy + + datastore.SetConsistencyPolicy(datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=self.consistency_probability)) + + + def _post_teardown(self): + """ Performs any post-test things. This includes: + + * Putting back the original ROOT_URLCONF if it was changed. + * Force closing the connection, so that the next test gets + a clean cursor. + """ + if hasattr(self, '_orig_policy'): + datastore = stub_manager.testbed.get_stub('datastore_v3') + datastore.SetConsistencyPolicy(self._orig_policy) + + super(GAETestCase,self)._post_teardown() diff --git a/tests/__init__.py b/tests/__init__.py index c5f867d..75b8a05 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,8 +1,10 @@ from .backend import BackendTest +#from .decimals import DecimalTest from .field_db_conversion import FieldDBConversionTest from .field_options import FieldOptionsTest from .filter import FilterTest -from .order import OrderTest +from .keys import KeysTest, DbKeyFieldTest, AncestorQueryTest, ParentKeyTest from .not_return_sets import NonReturnSetsTest -from .decimals import DecimalTest +from .order import OrderTest from .transactions import TransactionTest +from .ancestor import AncestorTest diff --git a/tests/ancestor.py b/tests/ancestor.py new file mode 100644 index 0000000..0f4b815 --- /dev/null +++ b/tests/ancestor.py @@ -0,0 +1,47 @@ +from django.test import TestCase +from django.utils import unittest +from django.db import models + +from djangoappengine.fields import DbKeyField + +from djangoappengine.db.models.manager import Manager + +#from djangotoolbox.fields import ListField +#from google.appengine.api.datastore import Key + +class ParentFoo(models.Model): + key = DbKeyField(primary_key=True) + foo = models.IntegerField() + objects = Manager() + +class ChildFoo(models.Model): + key = DbKeyField(primary_key=True, parent_key_name='parent_key') + foo = models.IntegerField() + objects = Manager() + +class AncestorTest(TestCase): + def test_simple(self): + px = ParentFoo(foo=5) + px.save() + px = ParentFoo(foo=2) + px.save() + + parents = ParentFoo.objects.all() + self.assertEqual(2, parents.count()) + + parents = ParentFoo.objects.filter(foo=2) + self.assertEqual(1, parents.count()) + + child = ChildFoo(foo=10, parent_key=px.pk) + orig_child_pk = child.pk + child.save() + + results = list(ChildFoo.objects.ancestor(px.pk)) + + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child.pk) + + results = list(ChildFoo.objects.all().ancestor(px.pk)) + + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child.pk) \ No newline at end of file diff --git a/tests/backend.py b/tests/backend.py index 066858a..326da6b 100644 --- a/tests/backend.py +++ b/tests/backend.py @@ -1,18 +1,23 @@ from django.db import models -from django.test import TestCase from django.db.utils import DatabaseError +from django.test import TestCase + class A(models.Model): value = models.IntegerField() + class B(A): other = models.IntegerField() + class BackendTest(TestCase): + def test_model_forms(self): from django import forms class F(forms.ModelForm): + class Meta: model = A diff --git a/tests/decimals.py b/tests/decimals.py deleted file mode 100644 index 7259533..0000000 --- a/tests/decimals.py +++ /dev/null @@ -1,40 +0,0 @@ -from .testmodels import DecimalModel -from django.test import TestCase - -from decimal import Decimal, InvalidOperation -D = Decimal - -class DecimalTest(TestCase): - DECIMALS = D("12345.6789"), D("5"), D("345.67"), D("45.6"), D("2345.678") - - def setUp(self): - for d in self.DECIMALS: - DecimalModel(decimal=d).save() - - def test_filter(self): - d = DecimalModel.objects.get(decimal=D("5.0")) - - self.assertTrue(isinstance(d.decimal, Decimal)) - self.assertEquals(str(d.decimal), "5.00") - - d = DecimalModel.objects.get(decimal=D("45.60")) - self.assertEquals(str(d.decimal), "45.60") - - # Filter argument should be converted to Decimal with 2 decimal_places - d = DecimalModel.objects.get(decimal="0000345.67333333333333333") - self.assertEquals(str(d.decimal), "345.67") - - def test_order(self): - rows = DecimalModel.objects.all().order_by('decimal') - values = list(d.decimal for d in rows) - self.assertEquals(values, sorted(values)) - - def test_sign_extend(self): - DecimalModel(decimal=D('-0.0')).save() - - try: - # if we've written a valid string we should be able to - # retrieve the DecimalModel object without error - DecimalModel.objects.filter(decimal__lt=1)[0] - except InvalidOperation: - self.assertTrue(False) diff --git a/tests/field_db_conversion.py b/tests/field_db_conversion.py index e8a50d5..29b91e9 100644 --- a/tests/field_db_conversion.py +++ b/tests/field_db_conversion.py @@ -1,63 +1,72 @@ -from .testmodels import FieldsWithoutOptionsModel +import datetime + from django.test import TestCase + from google.appengine.api.datastore import Get -from google.appengine.ext.db import Key -from google.appengine.api.datastore_types import Text, Category, Email, Link, \ - PhoneNumber, PostalAddress, Text, Blob, ByteString, GeoPt, IM, Key, \ - Rating, BlobKey -from google.appengine.api import users -import datetime +from google.appengine.api.datastore_types import Text, Category, Email, \ + Link, PhoneNumber, PostalAddress, Text, Blob, ByteString, GeoPt, IM, \ + Key, Rating, BlobKey + +from .testmodels import FieldsWithoutOptionsModel + +# TODO: Add field conversions for ForeignKeys? + class FieldDBConversionTest(TestCase): + def test_db_conversion(self): actual_datetime = datetime.datetime.now() entity = FieldsWithoutOptionsModel( datetime=actual_datetime, date=actual_datetime.date(), time=actual_datetime.time(), floating_point=5.97, boolean=True, null_boolean=False, text='Hallo', email='hallo@hallo.com', - comma_seperated_integer="5,4,3,2", + comma_seperated_integer='5,4,3,2', ip_address='194.167.1.1', slug='you slugy slut :)', - url='http://www.scholardocs.com', long_text=1000*'A', - indexed_text='hello', xml=2000*'B', - integer=-400, small_integer=-4, positiv_integer=400, - positiv_small_integer=4) + url='http://www.scholardocs.com', long_text=1000 * 'A', + indexed_text='hello', + integer=-400, small_integer=-4, positive_integer=400, + positive_small_integer=4) entity.save() - # get the gae entity (not the django model instance) and test if the - # fields have been converted right to the corresponding gae database types - gae_entity = Get(Key.from_path(FieldsWithoutOptionsModel._meta.db_table, + # Get the gae entity (not the django model instance) and test + # if the fields have been converted right to the corresponding + # GAE database types. + gae_entity = Get( + Key.from_path(FieldsWithoutOptionsModel._meta.db_table, entity.pk)) - - for name, gae_db_type in [('long_text', Text), - ('indexed_text', unicode), ('xml', Text), + opts = FieldsWithoutOptionsModel._meta + for name, types in [('long_text', Text), + ('indexed_text', unicode), ('text', unicode), ('ip_address', unicode), ('slug', unicode), ('email', unicode), ('comma_seperated_integer', unicode), ('url', unicode), ('time', datetime.datetime), ('datetime', datetime.datetime), ('date', datetime.datetime), ('floating_point', float), ('boolean', bool), ('null_boolean', bool), ('integer', (int, long)), - ('small_integer', (int, long)), ('positiv_integer', (int, long)), - ('positiv_small_integer', (int, long))]: - self.assertTrue(type(gae_entity[ - FieldsWithoutOptionsModel._meta.get_field_by_name( - name)[0].column]) in (isinstance(gae_db_type, (list, tuple)) and \ - gae_db_type or (gae_db_type, ))) - - # get the model instance and check if the fields convert back to the - # right types - entity = FieldsWithoutOptionsModel.objects.get() - for name, expected_type in [('long_text', unicode), - ('indexed_text', unicode), ('xml', unicode), - ('text', unicode), ('ip_address', unicode), ('slug', unicode), + ('small_integer', (int, long)), + ('positive_integer', (int, long)), + ('positive_small_integer', (int, long))]: + column = opts.get_field_by_name(name)[0].column + if not isinstance(types, (list, tuple)): + types = (types, ) + self.assertTrue(type(gae_entity[column]) in types) + + # Get the model instance and check if the fields convert back + # to the right types. + model = FieldsWithoutOptionsModel.objects.get() + for name, types in [ + ('long_text', unicode), + ('indexed_text', unicode), + ('text', unicode), ('ip_address', unicode), + ('slug', unicode), ('email', unicode), ('comma_seperated_integer', unicode), ('url', unicode), ('datetime', datetime.datetime), ('date', datetime.date), ('time', datetime.time), ('floating_point', float), ('boolean', bool), ('null_boolean', bool), ('integer', (int, long)), - ('small_integer', (int, long)), ('positiv_integer', (int, long)), - ('positiv_small_integer', (int, long))]: - self.assertTrue(type(getattr(entity, name)) in (isinstance( - expected_type, (list, tuple)) and expected_type or (expected_type, ))) - - -# TODO: Add field conversions for ForeignKeys? + ('small_integer', (int, long)), + ('positive_integer', (int, long)), + ('positive_small_integer', (int, long))]: + if not isinstance(types, (list, tuple)): + types = (types, ) + self.assertTrue(type(getattr(model, name)) in types) diff --git a/tests/field_options.py b/tests/field_options.py index 5e67b9b..1ce6377 100644 --- a/tests/field_options.py +++ b/tests/field_options.py @@ -1,46 +1,56 @@ +import datetime + from django.test import TestCase from django.db.utils import DatabaseError from django.db.models.fields import NOT_PROVIDED -from .testmodels import FieldsWithOptionsModel, NullableTextModel + +from google.appengine.api import users from google.appengine.api.datastore import Get -from google.appengine.ext.db import Key from google.appengine.api.datastore_types import Text, Category, Email, Link, \ PhoneNumber, PostalAddress, Text, Blob, ByteString, GeoPt, IM, Key, \ Rating, BlobKey -from google.appengine.api import users -import datetime +from google.appengine.ext.db import Key + +from .testmodels import FieldsWithOptionsModel, NullableTextModel + class FieldOptionsTest(TestCase): + def test_options(self): entity = FieldsWithOptionsModel() - # try to save the entity with non-nullable field time set to None, should - # raise an exception + # Try to save the entity with non-nullable field time set to + # None, should raise an exception. self.assertRaises(DatabaseError, entity.save) time = datetime.datetime.now().time() entity.time = time entity.save() - # check if primary_key=True is set correctly for the saved entity + # Check if primary_key=True is set correctly for the saved entity. self.assertEquals(entity.pk, u'app-engine@scholardocs.com') - gae_entity = Get(Key.from_path(FieldsWithOptionsModel._meta.db_table, - entity.pk)) + gae_entity = Get( + Key.from_path(FieldsWithOptionsModel._meta.db_table, entity.pk)) self.assertTrue(gae_entity is not None) - self.assertEquals(gae_entity.key().name(), u'app-engine@scholardocs.com') + self.assertEquals(gae_entity.key().name(), + u'app-engine@scholardocs.com') - # check if default values are set correctly on the db level, - # primary_key field is not stored at the db level + # Check if default values are set correctly on the db level, + # primary_key field is not stored at the db level. for field in FieldsWithOptionsModel._meta.local_fields: - if field.default and field.default != NOT_PROVIDED and not \ - field.primary_key: + if field.default and field.default != NOT_PROVIDED and \ + not field.primary_key: self.assertEquals(gae_entity[field.column], field.default) elif field.column == 'time': - self.assertEquals(gae_entity[field.column], datetime.datetime( - 1970, 1, 1, time.hour, time.minute, time.second, time.microsecond)) + self.assertEquals( + gae_entity[field.column], + datetime.datetime(1970, 1, 1, + time.hour, time.minute, time.second, + time.microsecond)) elif field.null and field.editable: self.assertEquals(gae_entity[field.column], None) - # check if default values are set correct on the model instance level + # Check if default values are set correct on the model instance + # level. entity = FieldsWithOptionsModel.objects.get() for field in FieldsWithOptionsModel._meta.local_fields: if field.default and field.default != NOT_PROVIDED: @@ -50,36 +60,48 @@ def test_options(self): elif field.null and field.editable: self.assertEquals(getattr(entity, field.column), None) - # check if nullable field with default values can be set to None + # Check if nullable field with default values can be set to + # None. entity.slug = None - entity.positiv_small_integer = None + entity.positive_small_integer = None try: entity.save() except: self.fail() - # check if slug and positiv_small_integer will be retrieved with values - # set to None (on db level and model instance level) - gae_entity = Get(Key.from_path(FieldsWithOptionsModel._meta.db_table, - entity.pk)) - self.assertEquals(gae_entity[FieldsWithOptionsModel._meta.get_field_by_name( - 'slug')[0].column], None) - self.assertEquals(gae_entity[FieldsWithOptionsModel._meta.get_field_by_name( - 'positiv_small_integer')[0].column], None) + # Check if slug and positive_small_integer will be retrieved + # with values set to None (on db level and model instance + # level). + gae_entity = Get(Key.from_path( + FieldsWithOptionsModel._meta.db_table, entity.pk)) + opts = FieldsWithOptionsModel._meta + self.assertEquals( + gae_entity[opts.get_field_by_name('slug')[0].column], + None) + self.assertEquals( + gae_entity[opts.get_field_by_name( + 'positive_small_integer')[0].column], + None) - # on the model instance level + # On the model instance level. entity = FieldsWithOptionsModel.objects.get() - self.assertEquals(getattr(entity, FieldsWithOptionsModel._meta.get_field_by_name( - 'slug')[0].column), None) - self.assertEquals(getattr(entity, FieldsWithOptionsModel._meta.get_field_by_name( - 'positiv_small_integer')[0].column), None) + self.assertEquals( + getattr(entity, opts.get_field_by_name('slug')[0].column), + None) + self.assertEquals( + getattr(entity, opts.get_field_by_name( + 'positive_small_integer')[0].column), + None) - # TODO: check db_column option - # TODO: change the primary key and check if a new instance with the - # changed primary key will be saved (not in this test class) + # TODO: Check db_column option. + # TODO: Change the primary key and check if a new instance with + # the changed primary key will be saved (not in this test + # class). def test_nullable_text(self): - # regression test for #48 + """ + Regression test for #48 (in old BitBucket repository). + """ entity = NullableTextModel(text=None) entity.save() diff --git a/tests/filter.py b/tests/filter.py index ab33d0b..110c34d 100644 --- a/tests/filter.py +++ b/tests/filter.py @@ -1,28 +1,32 @@ -from ..db.utils import get_cursor, set_cursor -from .testmodels import FieldsWithOptionsModel, EmailModel, DateTimeModel, \ - OrderedModel, BlobModel +import datetime +import time + from django.db import models from django.db.models import Q from django.db.utils import DatabaseError from django.test import TestCase from django.utils import unittest + from google.appengine.api.datastore import Get, Key -import datetime -import time + +from ..db.utils import get_cursor, set_cursor +from .testmodels import FieldsWithOptionsModel, EmailModel, DateTimeModel, \ + OrderedModel, BlobModel + class FilterTest(TestCase): floats = [5.3, 2.6, 9.1, 1.58] emails = ['app-engine@scholardocs.com', 'sharingan@uchias.com', - 'rinnengan@sage.de', 'rasengan@naruto.com'] + 'rinnengan@sage.de', 'rasengan@naruto.com'] datetimes = [datetime.datetime(2010, 1, 1, 0, 0, 0, 0), - datetime.datetime(2010, 12, 31, 23, 59, 59, 999999), - datetime.datetime(2011, 1, 1, 0, 0, 0, 0), - datetime.datetime(2013, 7, 28, 22, 30, 20, 50)] + datetime.datetime(2010, 12, 31, 23, 59, 59, 999999), + datetime.datetime(2011, 1, 1, 0, 0, 0, 0), + datetime.datetime(2013, 7, 28, 22, 30, 20, 50)] def setUp(self): - for index, (float, email, datetime_value) in enumerate(zip(FilterTest.floats, - FilterTest.emails, FilterTest.datetimes)): - # ensure distinct times when saving entities + for index, (float, email, datetime_value) in enumerate(zip( + FilterTest.floats, FilterTest.emails, FilterTest.datetimes)): + # Ensure distinct times when saving entities. time.sleep(0.01) self.last_save_datetime = datetime.datetime.now() self.last_save_time = self.last_save_datetime.time() @@ -37,241 +41,263 @@ def setUp(self): DateTimeModel(datetime=datetime_value).save() def test_startswith(self): - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - email__startswith='r').order_by('email')], - ['rasengan@naruto.com', 'rinnengan@sage.de']) - self.assertEquals([entity.email for entity in - EmailModel.objects.filter( - email__startswith='r').order_by('email')], - ['rasengan@naruto.com', 'rinnengan@sage.de']) + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(email__startswith='r').order_by('email')], + ['rasengan@naruto.com', 'rinnengan@sage.de']) + self.assertEquals( + [entity.email for entity in EmailModel.objects + .filter(email__startswith='r').order_by('email')], + ['rasengan@naruto.com', 'rinnengan@sage.de']) def test_gt(self): - # test gt on float - self.assertEquals([entity.floating_point for entity in - FieldsWithOptionsModel.objects.filter( - floating_point__gt=3.1).order_by('floating_point')], - [5.3, 9.1]) - - # test gt on integer - self.assertEquals([entity.integer for entity in - FieldsWithOptionsModel.objects.filter( - integer__gt=3).order_by('integer')], - [5, 9]) - - # test filter on primary_key field - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter(email__gt='as'). - order_by('email')], ['rasengan@naruto.com', - 'rinnengan@sage.de', 'sharingan@uchias.com', ]) - - # test ForeignKeys with id - self.assertEquals(sorted([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - foreign_key__gt=2)]), - ['rasengan@naruto.com', 'rinnengan@sage.de']) - - # and with instance + # Test gt on float. + self.assertEquals( + [entity.floating_point + for entity in FieldsWithOptionsModel.objects + .filter(floating_point__gt=3.1).order_by('floating_point')], + [5.3, 9.1]) + + # Test gt on integer. + self.assertEquals( + [entity.integer for entity in FieldsWithOptionsModel.objects + .filter(integer__gt=3).order_by('integer')], + [5, 9]) + + # Test filter on primary_key field. + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(email__gt='as').order_by('email')], + ['rasengan@naruto.com', 'rinnengan@sage.de', + 'sharingan@uchias.com', ]) + + # Test ForeignKeys with id. + self.assertEquals( + sorted([entity.email for entity in FieldsWithOptionsModel.objects + .filter(foreign_key__gt=2)]), + ['rasengan@naruto.com', 'rinnengan@sage.de']) + + # And with instance. ordered_instance = OrderedModel.objects.get(priority=1) - self.assertEquals(sorted([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - foreign_key__gt=ordered_instance)]), - ['rasengan@naruto.com', 'rinnengan@sage.de']) + self.assertEquals( + sorted([entity.email for entity in FieldsWithOptionsModel.objects + .filter(foreign_key__gt=ordered_instance)]), + ['rasengan@naruto.com', 'rinnengan@sage.de']) def test_lt(self): - # test lt on float - self.assertEquals([entity.floating_point for entity in - FieldsWithOptionsModel.objects.filter( - floating_point__lt=3.1).order_by('floating_point')], - [1.58, 2.6]) - - # test lt on integer - self.assertEquals([entity.integer for entity in - FieldsWithOptionsModel.objects.filter( - integer__lt=3).order_by('integer')], - [1, 2]) - - # test filter on primary_key field - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter(email__lt='as'). - order_by('email')], ['app-engine@scholardocs.com', ]) - - # filter on datetime - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - time__lt=self.last_save_time).order_by('time')], - ['app-engine@scholardocs.com', 'sharingan@uchias.com', - 'rinnengan@sage.de']) - - # test ForeignKeys with id - self.assertEquals(sorted([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - foreign_key__lt=3)]), - ['app-engine@scholardocs.com', 'sharingan@uchias.com']) - - # and with instance + # Test lt on float. + self.assertEquals( + [entity.floating_point + for entity in FieldsWithOptionsModel.objects + .filter(floating_point__lt=3.1).order_by('floating_point')], + [1.58, 2.6]) + + # Test lt on integer. + self.assertEquals( + [entity.integer for entity in FieldsWithOptionsModel.objects + .filter(integer__lt=3).order_by('integer')], + [1, 2]) + + # Test filter on primary_key field. + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(email__lt='as').order_by('email')], + ['app-engine@scholardocs.com', ]) + + # Filter on datetime. + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(time__lt=self.last_save_time).order_by('time')], + ['app-engine@scholardocs.com', 'sharingan@uchias.com', + 'rinnengan@sage.de']) + + # Test ForeignKeys with id. + self.assertEquals( + sorted([entity.email for entity in FieldsWithOptionsModel.objects + .filter(foreign_key__lt=3)]), + ['app-engine@scholardocs.com', 'sharingan@uchias.com']) + + # And with instance. ordered_instance = OrderedModel.objects.get(priority=2) - self.assertEquals(sorted([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - foreign_key__lt=ordered_instance)]), - ['app-engine@scholardocs.com', 'sharingan@uchias.com']) + self.assertEquals( + sorted([entity.email for entity in FieldsWithOptionsModel.objects + .filter(foreign_key__lt=ordered_instance)]), + ['app-engine@scholardocs.com', 'sharingan@uchias.com']) def test_gte(self): - # test gte on float - self.assertEquals([entity.floating_point for entity in - FieldsWithOptionsModel.objects.filter( - floating_point__gte=2.6).order_by('floating_point')], - [2.6, 5.3, 9.1]) - - # test gte on integer - self.assertEquals([entity.integer for entity in - FieldsWithOptionsModel.objects.filter( - integer__gte=2).order_by('integer')], - [2, 5, 9]) - - # test filter on primary_key field - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - email__gte='rinnengan@sage.de').order_by('email')], - ['rinnengan@sage.de', 'sharingan@uchias.com', ]) + # Test gte on float. + self.assertEquals( + [entity.floating_point + for entity in FieldsWithOptionsModel.objects + .filter(floating_point__gte=2.6).order_by('floating_point')], + [2.6, 5.3, 9.1]) + + # Test gte on integer. + self.assertEquals( + [entity.integer for entity in FieldsWithOptionsModel.objects + .filter(integer__gte=2).order_by('integer')], + [2, 5, 9]) + + # Test filter on primary_key field. + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(email__gte='rinnengan@sage.de').order_by('email')], + ['rinnengan@sage.de', 'sharingan@uchias.com', ]) def test_lte(self): - # test lte on float - self.assertEquals([entity.floating_point for entity in - FieldsWithOptionsModel.objects.filter( - floating_point__lte=5.3).order_by('floating_point')], - [1.58, 2.6, 5.3]) - - # test lte on integer - self.assertEquals([entity.integer for entity in - FieldsWithOptionsModel.objects.filter( - integer__lte=5).order_by('integer')], - [1, 2, 5]) - - # test filter on primary_key field - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - email__lte='rinnengan@sage.de').order_by('email')], - ['app-engine@scholardocs.com', 'rasengan@naruto.com', - 'rinnengan@sage.de']) + # Test lte on float. + self.assertEquals( + [entity.floating_point + for entity in FieldsWithOptionsModel.objects + .filter(floating_point__lte=5.3).order_by('floating_point')], + [1.58, 2.6, 5.3]) + + # Test lte on integer. + self.assertEquals( + [entity.integer for entity in FieldsWithOptionsModel.objects + .filter(integer__lte=5).order_by('integer')], + [1, 2, 5]) + + # Test filter on primary_key field. + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(email__lte='rinnengan@sage.de').order_by('email')], + ['app-engine@scholardocs.com', 'rasengan@naruto.com', + 'rinnengan@sage.de']) def test_equals(self): - # test equality filter on primary_key field - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - email='rinnengan@sage.de').order_by('email')], - ['rinnengan@sage.de']) + # Test equality filter on primary_key field. + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(email='rinnengan@sage.de').order_by('email')], + ['rinnengan@sage.de']) def test_is_null(self): self.assertEquals(FieldsWithOptionsModel.objects.filter( floating_point__isnull=True).count(), 0) - FieldsWithOptionsModel(integer=5.4, email='shinra.tensai@sixpaths.com', + FieldsWithOptionsModel( + integer=5.4, email='shinra.tensai@sixpaths.com', time=datetime.datetime.now().time()).save() self.assertEquals(FieldsWithOptionsModel.objects.filter( floating_point__isnull=True).count(), 1) - # XXX: These filters will not work because of a Django bug + # XXX: These filters will not work because of a Django bug. # self.assertEquals(FieldsWithOptionsModel.objects.filter( # foreign_key=None).count(), 1) - # (it uses left outer joins if checked against isnull + # (it uses left outer joins if checked against isnull) # self.assertEquals(FieldsWithOptionsModel.objects.filter( # foreign_key__isnull=True).count(), 1) def test_exclude(self): - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.all().exclude( - floating_point__lt=9.1).order_by('floating_point')], - ['rinnengan@sage.de', ]) + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .all().exclude(floating_point__lt=9.1) + .order_by('floating_point')], + ['rinnengan@sage.de', ]) - # test exclude with foreignKey + # Test exclude with ForeignKey. ordered_instance = OrderedModel.objects.get(priority=1) - self.assertEquals(sorted([entity.email for entity in - FieldsWithOptionsModel.objects.all().exclude( - foreign_key__gt=ordered_instance)]), - ['app-engine@scholardocs.com', 'sharingan@uchias.com']) + self.assertEquals( + sorted([entity.email for entity in FieldsWithOptionsModel.objects + .all().exclude(foreign_key__gt=ordered_instance)]), + ['app-engine@scholardocs.com', 'sharingan@uchias.com']) def test_exclude_pk(self): - self.assertEquals([entity.pk for entity in - OrderedModel.objects.exclude(pk__in=[2, 3]) - .order_by('pk')], - [1, 4]) + self.assertEquals( + [entity.pk for entity in OrderedModel.objects + .exclude(pk__in=[2, 3]).order_by('pk')], + [1, 4]) def test_chained_filter(self): - # additionally tests count :) + # Additionally tests count :) self.assertEquals(FieldsWithOptionsModel.objects.filter( - floating_point__lt=5.3, floating_point__gt=2.6). - count(), 0) - - # test across multiple columns. On app engine only one filter is allowed - # to be an inequality filter - self.assertEquals([(entity.floating_point, entity.integer) for entity in - FieldsWithOptionsModel.objects.filter( - floating_point__lte=5.3, integer=2).order_by( - 'floating_point')], [(2.6, 2), ]) - - # test multiple filters including the primary_key field - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - email__gte='rinnengan@sage.de', integer=2).order_by( - 'email')], ['sharingan@uchias.com', ]) - - # test in filter on primary key with another arbitrary filter - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - email__in=['rinnengan@sage.de', - 'sharingan@uchias.com'], integer__gt=2).order_by( - 'integer')], ['rinnengan@sage.de', ]) - - # Test exceptions - - # test multiple filters exception when filtered and not ordered against - # the first filter - self.assertRaises(DatabaseError, lambda: - FieldsWithOptionsModel.objects.filter( - email__gte='rinnengan@sage.de', floating_point=5.3).order_by( - 'floating_point')[0]) - - # test exception if filtered across multiple columns with inequality filter - self.assertRaises(DatabaseError, FieldsWithOptionsModel.objects.filter( - floating_point__lte=5.3, integer__gte=2).order_by( - 'floating_point').get) - - # test exception if filtered across multiple columns with inequality filter - # with exclude - self.assertRaises(DatabaseError, FieldsWithOptionsModel.objects.filter( - email__lte='rinnengan@sage.de').exclude( - floating_point__lt=9.1).order_by('email').get) - - self.assertRaises(DatabaseError, lambda: - FieldsWithOptionsModel.objects.all().exclude( - floating_point__lt=9.1).order_by('email')[0]) - - # TODO: Maybe check all possible exceptions + floating_point__lt=5.3, floating_point__gt=2.6).count(), 0) + + # Test across multiple columns. On App Engine only one filter + # is allowed to be an inequality filter. + self.assertEquals( + [(entity.floating_point, entity.integer) + for entity in FieldsWithOptionsModel.objects + .filter(floating_point__lte=5.3, integer=2) + .order_by('floating_point')], + [(2.6, 2), ]) + + # Test multiple filters including the primary_key field. + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(email__gte='rinnengan@sage.de', integer=2) + .order_by('email')], + ['sharingan@uchias.com', ]) + + # Test in filter on primary key with another arbitrary filter. + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(email__in=['rinnengan@sage.de', + 'sharingan@uchias.com'], + integer__gt=2) + .order_by('integer')], + ['rinnengan@sage.de', ]) + + # Test exceptions. + + # Test multiple filters exception when filtered and not ordered + # against the first filter. + self.assertRaises( + DatabaseError, + lambda: FieldsWithOptionsModel.objects + .filter(email__gte='rinnengan@sage.de', floating_point=5.3) + .order_by('floating_point')[0]) + + # Test exception if filtered across multiple columns with + # inequality filter. + self.assertRaises( + DatabaseError, + FieldsWithOptionsModel.objects + .filter(floating_point__lte=5.3, integer__gte=2) + .order_by('floating_point').get) + + # Test exception if filtered across multiple columns with + # inequality filter with exclude. + self.assertRaises( + DatabaseError, + FieldsWithOptionsModel.objects + .filter(email__lte='rinnengan@sage.de') + .exclude(floating_point__lt=9.1).order_by('email').get) + + self.assertRaises( + DatabaseError, + lambda: FieldsWithOptionsModel.objects + .all().exclude(floating_point__lt=9.1).order_by('email')[0]) + + # TODO: Maybe check all possible exceptions. def test_slicing(self): - # test slicing on filter with primary_key - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - email__lte='rinnengan@sage.de').order_by('email')[:2]], - ['app-engine@scholardocs.com', 'rasengan@naruto.com', ]) - - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - email__lte='rinnengan@sage.de').order_by('email')[1:2]], - ['rasengan@naruto.com', ]) - - # test on non pk field - self.assertEquals([entity.integer for entity in - FieldsWithOptionsModel.objects.all().order_by( - 'integer')[:2]], [1, 2, ]) - - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.all().order_by( - 'email')[::2]], - ['app-engine@scholardocs.com', 'rinnengan@sage.de']) + # Test slicing on filter with primary_key. + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(email__lte='rinnengan@sage.de') + .order_by('email')[:2]], + ['app-engine@scholardocs.com', 'rasengan@naruto.com', ]) + + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(email__lte='rinnengan@sage.de') + .order_by('email')[1:2]], + ['rasengan@naruto.com', ]) + + # Test on non pk field. + self.assertEquals( + [entity.integer for entity in FieldsWithOptionsModel.objects + .all().order_by('integer')[:2]], + [1, 2, ]) + + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .all().order_by('email')[::2]], + ['app-engine@scholardocs.com', 'rinnengan@sage.de']) def test_cursor(self): results = list(FieldsWithOptionsModel.objects.all()) @@ -287,46 +313,52 @@ def test_cursor(self): self.assertEqual(list(query[:1]), []) def test_Q_objects(self): - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - Q(email__lte='rinnengan@sage.de')).order_by('email')][:2], - ['app-engine@scholardocs.com', 'rasengan@naruto.com', ]) - - self.assertEquals([entity.integer for entity in - FieldsWithOptionsModel.objects.exclude(Q(integer__lt=5) | - Q(integer__gte=9)).order_by('integer')], - [5, ]) - - self.assertRaises(TypeError, FieldsWithOptionsModel.objects.filter( - Q(floating_point=9.1), Q(integer=9) | Q(integer=2))) + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(Q(email__lte='rinnengan@sage.de')) + .order_by('email')][:2], + ['app-engine@scholardocs.com', 'rasengan@naruto.com', ]) + + self.assertEquals( + [entity.integer for entity in FieldsWithOptionsModel.objects + .exclude(Q(integer__lt=5) | Q(integer__gte=9)) + .order_by('integer')], + [5, ]) + + self.assertRaises( + TypeError, + FieldsWithOptionsModel.objects + .filter(Q(floating_point=9.1), Q(integer=9) | Q(integer=2))) def test_pk_in(self): - # test pk__in with field name email - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - email__in=['app-engine@scholardocs.com', - 'rasengan@naruto.com'])], ['app-engine@scholardocs.com', - 'rasengan@naruto.com']) + # Test pk__in with field name email. + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(email__in=['app-engine@scholardocs.com', + 'rasengan@naruto.com'])], + ['app-engine@scholardocs.com', 'rasengan@naruto.com']) def test_in(self): - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - floating_point__in=[5.3, 2.6, 1.58]).filter( - integer__in=[1, 5, 9])], - ['app-engine@scholardocs.com', 'rasengan@naruto.com']) + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(floating_point__in=[5.3, 2.6, 1.58]) + .filter(integer__in=[1, 5, 9])], + ['app-engine@scholardocs.com', 'rasengan@naruto.com']) def test_in_with_pk_in(self): - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - floating_point__in=[5.3, 2.6, 1.58]).filter( - email__in=['app-engine@scholardocs.com', - 'rasengan@naruto.com'])], - ['app-engine@scholardocs.com', 'rasengan@naruto.com']) + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(floating_point__in=[5.3, 2.6, 1.58]) + .filter(email__in=['app-engine@scholardocs.com', + 'rasengan@naruto.com'])], + ['app-engine@scholardocs.com', 'rasengan@naruto.com']) def test_in_with_order_by(self): + class Post(models.Model): writer = models.IntegerField() order = models.IntegerField() + Post(writer=1, order=1).save() Post(writer=1, order=2).save() Post(writer=1, order=3).save() @@ -340,76 +372,82 @@ class Post(models.Model): self.assertEqual(orders, range(5, 0, -1)) def test_inequality(self): - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.exclude( - floating_point=5.3).filter( - integer__in=[1, 5, 9])], - ['rasengan@naruto.com', 'rinnengan@sage.de']) + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .exclude(floating_point=5.3).filter(integer__in=[1, 5, 9])], + ['rasengan@naruto.com', 'rinnengan@sage.de']) def test_values(self): - # test values() - self.assertEquals([entity['pk'] for entity in - FieldsWithOptionsModel.objects.filter(integer__gt=3). - order_by('integer').values('pk')], - ['app-engine@scholardocs.com', 'rinnengan@sage.de']) - - self.assertEquals(FieldsWithOptionsModel.objects.filter(integer__gt=3). - order_by('integer').values('pk').count(), 2) - - # these queries first fetch the whole entity and then only return the - # desired fields selected in .values - self.assertEquals([entity['integer'] for entity in - FieldsWithOptionsModel.objects.filter( - email__startswith='r').order_by('email').values( - 'integer')], [1, 9]) - - self.assertEquals([entity['floating_point'] for entity in - FieldsWithOptionsModel.objects.filter(integer__gt=3). - order_by('integer').values('floating_point')], - [5.3, 9.1]) - - # test values_list - self.assertEquals([entity[0] for entity in - FieldsWithOptionsModel.objects.filter(integer__gt=3). - order_by('integer').values_list('pk')], - ['app-engine@scholardocs.com', 'rinnengan@sage.de']) + # Test values(). + self.assertEquals( + [entity['pk'] for entity in FieldsWithOptionsModel.objects + .filter(integer__gt=3).order_by('integer').values('pk')], + ['app-engine@scholardocs.com', 'rinnengan@sage.de']) + + self.assertEquals(FieldsWithOptionsModel.objects + .filter(integer__gt=3).order_by('integer').values('pk').count(), 2) + + # These queries first fetch the whole entity and then only + # return the desired fields selected in .values. + self.assertEquals( + [entity['integer'] for entity in FieldsWithOptionsModel.objects + .filter(email__startswith='r') + .order_by('email').values('integer')], + [1, 9]) + + self.assertEquals( + [entity['floating_point'] + for entity in FieldsWithOptionsModel.objects + .filter(integer__gt=3) + .order_by('integer').values('floating_point')], + [5.3, 9.1]) + + # Test values_list. + self.assertEquals( + [entity[0] for entity in FieldsWithOptionsModel.objects + .filter(integer__gt=3).order_by('integer').values_list('pk')], + ['app-engine@scholardocs.com', 'rinnengan@sage.de']) def test_range(self): - # test range on float - self.assertEquals([entity.floating_point for entity in - FieldsWithOptionsModel.objects.filter( - floating_point__range=(2.6, 9.1)). - order_by('floating_point')], [2.6, 5.3, 9.1]) - - # test range on pk - self.assertEquals([entity.pk for entity in - FieldsWithOptionsModel.objects.filter( - pk__range=('app-engine@scholardocs.com', 'rinnengan@sage.de')). - order_by('pk')], - ['app-engine@scholardocs.com', - 'rasengan@naruto.com', 'rinnengan@sage.de']) - - # test range on date/datetime objects + # Test range on float. + self.assertEquals( + [entity.floating_point + for entity in FieldsWithOptionsModel.objects + .filter(floating_point__range=(2.6, 9.1)) + .order_by('floating_point')], + [2.6, 5.3, 9.1]) + + # Test range on pk. + self.assertEquals( + [entity.pk for entity in FieldsWithOptionsModel.objects + .filter(pk__range=('app-engine@scholardocs.com', + 'rinnengan@sage.de')) + .order_by('pk')], + ['app-engine@scholardocs.com', 'rasengan@naruto.com', + 'rinnengan@sage.de']) + + # Test range on date/datetime objects. start_time = self.last_save_datetime - datetime.timedelta(minutes=1) - self.assertEquals([entity.email for entity in - FieldsWithOptionsModel.objects.filter( - time__range=(start_time, self.last_save_time)).order_by('time')], - ['app-engine@scholardocs.com', 'sharingan@uchias.com', - 'rinnengan@sage.de', 'rasengan@naruto.com']) + self.assertEquals( + [entity.email for entity in FieldsWithOptionsModel.objects + .filter(time__range=(start_time, self.last_save_time)) + .order_by('time')], + ['app-engine@scholardocs.com', 'sharingan@uchias.com', + 'rinnengan@sage.de', 'rasengan@naruto.com']) def test_date(self): - # test year on date range boundaries - self.assertEquals([entity.datetime for entity in - DateTimeModel.objects.filter( - datetime__year=2010).order_by('datetime')], - [datetime.datetime(2010, 1, 1, 0, 0, 0, 0), - datetime.datetime(2010, 12, 31, 23, 59, 59, 999999)]) - - # test year on non boundary date - self.assertEquals([entity.datetime for entity in - DateTimeModel.objects.filter( - datetime__year=2013).order_by('datetime')], - [datetime.datetime(2013, 7, 28, 22, 30, 20, 50)]) + # Test year on date range boundaries. + self.assertEquals( + [entity.datetime for entity in DateTimeModel.objects + .filter(datetime__year=2010).order_by('datetime')], + [datetime.datetime(2010, 1, 1, 0, 0, 0, 0), + datetime.datetime(2010, 12, 31, 23, 59, 59, 999999)]) + + # Test year on non boundary date. + self.assertEquals( + [entity.datetime for entity in DateTimeModel.objects + .filter(datetime__year=2013).order_by('datetime')], + [datetime.datetime(2013, 7, 28, 22, 30, 20, 50)]) def test_auto_now(self): time.sleep(0.1) @@ -428,8 +466,8 @@ def test_auto_now_add(self): self.assertEqual(auto_now_add, entity.datetime_auto_now_add) def test_latest(self): - self.assertEquals(FieldsWithOptionsModel.objects.latest('time').floating_point, - 1.58) + self.assertEquals(FieldsWithOptionsModel.objects + .latest('time').floating_point, 1.58) def test_blob(self): x = BlobModel(data='lalala') diff --git a/tests/keys.py b/tests/keys.py new file mode 100644 index 0000000..4603435 --- /dev/null +++ b/tests/keys.py @@ -0,0 +1,455 @@ +from __future__ import with_statement +import warnings + +from django.db import connection, models +from django.db.utils import DatabaseError +from django.test import TestCase +from django.utils import unittest + +from djangoappengine.fields import DbKeyField +from djangoappengine.db.utils import as_ancestor +from djangotoolbox.fields import ListField + +from google.appengine.api.datastore import Key + + + +class AutoKey(models.Model): + pass + + +class CharKey(models.Model): + id = models.CharField(primary_key=True, max_length=10) + + +class IntegerKey(models.Model): + id = models.IntegerField(primary_key=True) + + +class Parent(models.Model): + pass + + +class Child(models.Model): + parent = models.ForeignKey(Parent, null=True) + + +class CharParent(models.Model): + id = models.CharField(primary_key=True, max_length=10) + + +class CharChild(models.Model): + parent = models.ForeignKey(CharParent) + + +class IntegerParent(models.Model): + id = models.IntegerField(primary_key=True) + + +class IntegerChild(models.Model): + parent = models.ForeignKey(IntegerParent) + + +class ParentKind(models.Model): + pass + + +class ChildKind(models.Model): + parent = models.ForeignKey(ParentKind) + parents = ListField(models.ForeignKey(ParentKind)) + + +class KeysTest(TestCase): + """ + GAE requires that keys are strings or positive integers, + keys also play a role in defining entity groups. + + Note: len() is a way of forcing evaluation of a QuerySet -- we + depend on the back-end to do some checks, so sometimes there is no + way to raise an exception earlier. + """ + + def setUp(self): + self.save_warnings_state() + + def tearDown(self): + self.restore_warnings_state() + + def test_auto_field(self): + """ + GAE keys may hold either strings or positive integers, however + Django uses integers as well as their string representations + for lookups, expecting both to be considered equivalent, so we + limit AutoFields to just ints and check that int or string(int) + may be used interchangably. + + Nonpositive keys are not allowed, and trying to use them to + create or look up objects should raise a database exception. + + See: http://code.google.com/appengine/docs/python/datastore/keyclass.html. + """ + AutoKey.objects.create() + o1 = AutoKey.objects.create(pk=1) + o2 = AutoKey.objects.create(pk='1') +# self.assertEqual(o1, o2) TODO: Not same for Django, same for the database. + with self.assertRaises(ValueError): + AutoKey.objects.create(pk='a') + self.assertEqual(AutoKey.objects.get(pk=1), o1) + self.assertEqual(AutoKey.objects.get(pk='1'), o1) + with self.assertRaises(ValueError): + AutoKey.objects.get(pk='a') + + with self.assertRaises(DatabaseError): + AutoKey.objects.create(id=-1) + with self.assertRaises(DatabaseError): + AutoKey.objects.create(id=0) + with self.assertRaises(DatabaseError): + AutoKey.objects.get(id=-1) + with self.assertRaises(DatabaseError): + AutoKey.objects.get(id__gt=-1) + with self.assertRaises(DatabaseError): + AutoKey.objects.get(id=0) + with self.assertRaises(DatabaseError): + AutoKey.objects.get(id__gt=0) + with self.assertRaises(DatabaseError): + len(AutoKey.objects.filter(id__gt=-1)) + with self.assertRaises(DatabaseError): + len(AutoKey.objects.filter(id__gt=0)) + + def test_primary_key(self): + """ + Specifying a field as primary_key should work as long as the + field values (after get_db_prep_*/value_to_db_* layer) can be + represented by the back-end key type. In case a value can be + represented, but lossy conversions, unexpected sorting, range + limitation or potential future ramifications are possible it + should warn the user (as early as possible). + + TODO: It may be even better to raise exceptions / issue + warnings during model validation. And make use of the new + supports_primary_key_on to prevent validation of models + using unsupported primary keys. + """ + + # TODO: Move to djangotoolbox or django.db.utils? + class Warning(StandardError): + """Database warning (name following PEP 249).""" + pass + + warnings.simplefilter('error', Warning) + + # This should just work. + class AutoFieldKey(models.Model): + key = models.AutoField(primary_key=True) + AutoFieldKey.objects.create() + + # This one can be exactly represented. + class CharKey(models.Model): + id = models.CharField(primary_key=True, max_length=10) + CharKey.objects.create(id='a') + + # Some rely on unstable assumptions or have other quirks and + # should warn. + +# # TODO: Warning with a range limitation. +# with self.assertRaises(Warning): +# +# class IntegerKey(models.Model): +# id = models.IntegerField(primary_key=True) +# IntegerKey.objects.create(id=1) + +# # TODO: date/times could be resonably encoded / decoded as +# # strings (in a reversible manner) for key usage, but +# # would need special handling and continue to raise an +# # exception for now +# with self.assertRaises(Warning): +# +# class DateKey(models.Model): +# id = models.DateField(primary_key=True, auto_now=True) +# DateKey.objects.create() + +# # TODO: There is a db.Email field that would be better to +# # store emails, but that may prevent them from being +# # used as keys. +# with self.assertRaises(Warning): +# +# class EmailKey(models.Model): +# id = models.EmailField(primary_key=True) +# EmailKey.objects.create(id='aaa@example.com') + +# # TODO: Warn that changing field parameters breaks sorting. +# # This applies to any DecimalField, so should belong to +# # the docs. +# with self.assertRaises(Warning): +# +# class DecimalKey(models.Model): +# id = models.DecimalField(primary_key=True, decimal_places=2, +# max_digits=5) +# DecimalKey.objects.create(id=1) + + # Some cannot be reasonably represented (e.g. binary or string + # encoding would prevent comparisons to work as expected). + with self.assertRaises(DatabaseError): + + class FloatKey(models.Model): + id = models.FloatField(primary_key=True) + FloatKey.objects.create(id=1.0) + + # TODO: Better fail during validation or creation than + # sometimes when filtering (False = 0 is a wrong key value). + with self.assertRaises(DatabaseError): + + class BooleanKey(models.Model): + id = models.BooleanField(primary_key=True) + BooleanKey.objects.create(id=True) + len(BooleanKey.objects.filter(id=False)) + + def test_primary_key_coercing(self): + """ + Creation and lookups should use the same type casting as + vanilla Django does, so CharField used as a key should cast + everything to a string, while IntegerField should cast to int. + """ + CharKey.objects.create(id=1) + CharKey.objects.create(id='a') + CharKey.objects.create(id=1.1) + CharKey.objects.get(id='1') + CharKey.objects.get(id='a') + CharKey.objects.get(id='1.1') + + IntegerKey.objects.create(id=1) + with self.assertRaises(ValueError): + IntegerKey.objects.create(id='a') + IntegerKey.objects.create(id=1.1) + IntegerKey.objects.get(id='1') + with self.assertRaises(ValueError): + IntegerKey.objects.get(id='a') + IntegerKey.objects.get(id=1.1) + + def test_foreign_key(self): + """ + Foreign key lookups may use parent instance or parent key value. + Using null foreign keys needs some special attention. + + TODO: In 1.4 one may also add _id suffix and use the key value. + """ + parent1 = Parent.objects.create(pk=1) + child1 = Child.objects.create(parent=parent1) + child2 = Child.objects.create(parent=None) + self.assertEqual(child1.parent, parent1) + self.assertEqual(child2.parent, None) + self.assertEqual(Child.objects.get(parent=parent1), child1) + self.assertEqual(Child.objects.get(parent=1), child1) + self.assertEqual(Child.objects.get(parent='1'), child1) + with self.assertRaises(ValueError): + Child.objects.get(parent='a') + self.assertEqual(Child.objects.get(parent=None), child2) + + def test_foreign_key_backwards(self): + """ + Following relationships backwards (_set syntax) with typed + parent key causes a unique problem for the legacy key storage. + """ + parent = CharParent.objects.create(id=1) + child = CharChild.objects.create(parent=parent) + self.assertEqual(list(parent.charchild_set.all()), [child]) + + parent = IntegerParent.objects.create(id=1) + child = IntegerChild.objects.create(parent=parent) + self.assertEqual(list(parent.integerchild_set.all()), [child]) + + @unittest.skipIf( + not connection.settings_dict.get('STORE_RELATIONS_AS_DB_KEYS'), + "No key kinds to check with the string/int foreign key storage.") + def test_key_kind(self): + """ + Checks that db.Keys stored in the database use proper kinds. + + Key kind should be the name of the table (db_table) of a model + for primary keys of entities, but for foreign keys, references + in general, it should be the db_table of the model the field + refers to. + + Note that Django hides the underlying db.Key objects well, and + it does work even with wrong kinds, but keeping the data + consistent may be significant for external tools. + + TODO: Add DictField / EmbeddedModelField and nesting checks. + """ + parent = ParentKind.objects.create(pk=1) + child = ChildKind.objects.create( + pk=2, parent=parent, parents=[parent.pk]) + self.assertEqual(child.parent.pk, parent.pk) + self.assertEqual(child.parents[0], parent.pk) + + from google.appengine.api.datastore import Get + from google.appengine.api.datastore_types import Key + parent_key = Key.from_path(parent._meta.db_table, 1) + child_key = Key.from_path(child._meta.db_table, 2) + parent_entity = Get(parent_key) + child_entity = Get(child_key) + parent_column = child._meta.get_field('parent').column + parents_column = child._meta.get_field('parents').column + self.assertEqual(child_entity[parent_column], parent_key) + self.assertEqual(child_entity[parents_column][0], parent_key) + + +class ParentModel(models.Model): + key = DbKeyField(primary_key=True) + +class NonDbKeyParentModel(models.Model): + id = models.AutoField(primary_key=True) + +class ChildModel(models.Model): + key = DbKeyField(primary_key=True, parent_key_name='parent_key') + +class AnotherChildModel(models.Model): + key = DbKeyField(primary_key=True, parent_key_name='parent_key') + +class ForeignKeyModel(models.Model): + id = models.AutoField(primary_key=True) + relation = models.ForeignKey(ParentModel) + +class DbKeyFieldTest(TestCase): + def testDbKeySave(self): + model = ParentModel() + model.save() + + self.assertIsNotNone(model.pk) + + def testForeignKeyWithGAEKey(self): + parent = ParentModel() + parent.save() + + fkm = ForeignKeyModel() + fkm.relation = parent + fkm.save() + + results = list(ForeignKeyModel.objects.filter(relation=parent)) + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, fkm.pk) + + def testPrimaryKeyQuery(self): + parent = ParentModel() + parent.save() + + db_parent = ParentModel.objects.get(pk=parent.pk) + + self.assertEquals(parent.pk, db_parent.pk) + +class ParentKeyTest(TestCase): + def testParentChildSave(self): + parent = ParentModel() + orig_parent_pk = parent.pk + parent.save() + + child = ChildModel(parent_key=parent.pk) + orig_child_pk = child.pk + child.save() + + self.assertNotEquals(parent.pk, orig_parent_pk) + self.assertNotEquals(child.pk, orig_child_pk) + self.assertEquals(child.pk.parent(), parent.pk) + + def testParentModelChildSave(self): + parent = ParentModel() + orig_parent_pk = parent.pk + parent.save() + + with self.assertRaises(ValueError): + child = ChildModel(parent_key=parent) + + def testNonDbKeyParent(self): + parent = NonDbKeyParentModel() + parent.save() + + with self.assertRaises(ValueError): + child = ChildModel(parent_key=parent.pk) + +class AncestorQueryTest(TestCase): + def testAncestorFilterQuery(self): + parent = ParentModel() + parent.save() + + child = ChildModel(parent_key=parent.pk) + child.save() + + results = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) + + self.assertEquals(1, len(results)) + self.assertEquals(results[0].pk, child.pk) + + def testAncestorGetQuery(self): + parent = ParentModel() + parent.save() + + child = ChildModel(parent_key=parent.pk) + child.save() + + result = ChildModel.objects.get(pk=as_ancestor(parent.pk)) + + self.assertEquals(result.pk, child.pk) + + def testEmptyAncestorQuery(self): + parent = ParentModel() + parent.save() + + results = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) + + self.assertEquals(0, len(results)) + + def testEmptyAncestorQueryWithUnsavedChild(self): + parent = ParentModel() + parent.save() + + child = ChildModel(parent_key=parent.pk) + + results = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) + + self.assertEquals(0, len(results)) + + def testUnsavedAncestorQuery(self): + parent = ParentModel() + + with self.assertRaises(ValueError): + results = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) + + def testDifferentChildrenAncestorQuery(self): + parent = ParentModel() + parent.save() + + child1 = ChildModel(parent_key=parent.pk) + child1.save() + child2 = AnotherChildModel(parent_key=parent.pk) + child2.save() + + results1 = list(ChildModel.objects.filter(pk=as_ancestor(parent.pk))) + + self.assertEquals(1, len(results1)) + self.assertEquals(results1[0].pk, child1.pk) + + results2 = list(AnotherChildModel.objects.filter(pk=as_ancestor(parent.pk))) + self.assertEquals(1, len(results2)) + self.assertEquals(results2[0].pk, child2.pk) + + def testDifferentParentsAncestorQuery(self): + parent1 = ParentModel() + parent1.save() + + child1 = ChildModel(parent_key=parent1.pk) + child1.save() + + parent2 = ParentModel() + parent2.save() + + child2 = ChildModel(parent_key=parent2.pk) + child2.save() + + results1 = list(ChildModel.objects.filter(pk=as_ancestor(parent1.pk))) + + self.assertEquals(1, len(results1)) + self.assertEquals(results1[0].pk, child1.pk) + + results2 = list(ChildModel.objects.filter(pk=as_ancestor(parent2.pk))) + self.assertEquals(1, len(results2)) + self.assertEquals(results2[0].pk, child2.pk) diff --git a/tests/not_return_sets.py b/tests/not_return_sets.py index 3302f30..e2d9810 100644 --- a/tests/not_return_sets.py +++ b/tests/not_return_sets.py @@ -1,13 +1,16 @@ -from .testmodels import FieldsWithOptionsModel, OrderedModel, SelfReferenceModel import datetime -from django.test import TestCase + from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned +from django.test import TestCase + +from .testmodels import FieldsWithOptionsModel, OrderedModel, \ + SelfReferenceModel class NonReturnSetsTest(TestCase): floats = [5.3, 2.6, 9.1, 1.58, 2.4] emails = ['app-engine@scholardocs.com', 'sharingan@uchias.com', - 'rinnengan@sage.de', 'rasengan@naruto.com', 'itachi@uchia.com'] + 'rinnengan@sage.de', 'rasengan@naruto.com', 'itachi@uchia.com'] def setUp(self): for index, (float, email) in enumerate(zip(NonReturnSetsTest.floats, @@ -22,48 +25,54 @@ def setUp(self): model.save() def test_get(self): - self.assertEquals(FieldsWithOptionsModel.objects.get( - email='itachi@uchia.com') - .email, 'itachi@uchia.com') + self.assertEquals( + FieldsWithOptionsModel.objects.get( + email='itachi@uchia.com').email, + 'itachi@uchia.com') - # test exception when matching multiple entities - self.assertRaises(MultipleObjectsReturned, FieldsWithOptionsModel.objects - .get, integer=2) + # Test exception when matching multiple entities. + self.assertRaises(MultipleObjectsReturned, + FieldsWithOptionsModel.objects.get, + integer=2) - # test exception when entity does not exist - self.assertRaises(ObjectDoesNotExist, FieldsWithOptionsModel.objects - .get, floating_point=5.2) + # Test exception when entity does not exist. + self.assertRaises(ObjectDoesNotExist, + FieldsWithOptionsModel.objects.get, + floating_point=5.2) - # TODO: test create when djangos model.save_base is refactored - # TODO: test get_or_create when refactored + # TODO: Test create when djangos model.save_base is refactored. + # TODO: Test get_or_create when refactored. def test_count(self): - self.assertEquals(FieldsWithOptionsModel.objects.filter( - integer=2).count(), 2) + self.assertEquals( + FieldsWithOptionsModel.objects.filter(integer=2).count(), 2) def test_in_bulk(self): - self.assertEquals([key in ['sharingan@uchias.com', 'itachi@uchia.com'] - for key in FieldsWithOptionsModel.objects.in_bulk( - ['sharingan@uchias.com', 'itachi@uchia.com']).keys()], - [True, ]*2) + self.assertEquals( + [key in ['sharingan@uchias.com', 'itachi@uchia.com'] + for key in FieldsWithOptionsModel.objects.in_bulk( + ['sharingan@uchias.com', 'itachi@uchia.com']).keys()], + [True, ] * 2) def test_latest(self): - self.assertEquals('itachi@uchia.com', FieldsWithOptionsModel.objects - .latest('time').email) + self.assertEquals( + FieldsWithOptionsModel.objects.latest('time').email, + 'itachi@uchia.com') def test_exists(self): - self.assertEquals(True, FieldsWithOptionsModel.objects.exists()) + self.assertEquals(FieldsWithOptionsModel.objects.exists(), True) def test_deletion(self): - # TODO: ForeignKeys will not be deleted! This has to be done via - # background tasks + # TODO: ForeignKeys will not be deleted! This has to be done + # via background tasks. self.assertEquals(FieldsWithOptionsModel.objects.count(), 5) FieldsWithOptionsModel.objects.get(email='itachi@uchia.com').delete() self.assertEquals(FieldsWithOptionsModel.objects.count(), 4) - FieldsWithOptionsModel.objects.filter(email__in=['sharingan@uchias.com', - 'itachi@uchia.com', 'rasengan@naruto.com', ]).delete() + FieldsWithOptionsModel.objects.filter(email__in=[ + 'sharingan@uchias.com', 'itachi@uchia.com', + 'rasengan@naruto.com', ]).delete() self.assertEquals(FieldsWithOptionsModel.objects.count(), 2) def test_selfref_deletion(self): @@ -72,21 +81,23 @@ def test_selfref_deletion(self): entity.delete() def test_foreign_key_fetch(self): - # test fetching the ForeignKey + # Test fetching the ForeignKey. ordered_instance = OrderedModel.objects.get(priority=2) - self.assertEquals(FieldsWithOptionsModel.objects.get(integer=9).foreign_key, - ordered_instance) + self.assertEquals( + FieldsWithOptionsModel.objects.get(integer=9).foreign_key, + ordered_instance) def test_foreign_key_backward(self): entity = OrderedModel.objects.all()[0] self.assertEquals(entity.keys.count(), 1) - # TODO: add should save the added instance transactional via for example - # force_insert - new_foreign_key = FieldsWithOptionsModel(floating_point=5.6, integer=3, + # TODO: Add should save the added instance transactional via for + # example force_insert. + new_foreign_key = FieldsWithOptionsModel( + floating_point=5.6, integer=3, email='temp@temp.com', time=datetime.datetime.now()) entity.keys.add(new_foreign_key) self.assertEquals(entity.keys.count(), 2) - # TODO: add test for create + # TODO: Add test for create. entity.keys.remove(new_foreign_key) self.assertEquals(entity.keys.count(), 1) entity.keys.clear() diff --git a/tests/order.py b/tests/order.py index 8e9bf96..467c8a7 100644 --- a/tests/order.py +++ b/tests/order.py @@ -1,7 +1,10 @@ -from .testmodels import OrderedModel from django.test import TestCase +from .testmodels import OrderedModel + + class OrderTest(TestCase): + def create_ordered_model_items(self): pks = [] priorities = [5, 2, 9, 1] @@ -14,43 +17,47 @@ def create_ordered_model_items(self): def test_default_order(self): pks, priorities = self.create_ordered_model_items() - self.assertEquals([item.priority - for item in OrderedModel.objects.all()], - sorted(priorities, reverse=True)) + self.assertEquals( + [item.priority for item in OrderedModel.objects.all()], + sorted(priorities, reverse=True)) def test_override_default_order(self): pks, priorities = self.create_ordered_model_items() - self.assertEquals([item.priority - for item in OrderedModel.objects.all().order_by('priority')], - sorted(priorities)) + self.assertEquals( + [item.priority for item in + OrderedModel.objects.all().order_by('priority')], + sorted(priorities)) def test_remove_default_order(self): pks, priorities = self.create_ordered_model_items() - self.assertEquals([item.pk - for item in OrderedModel.objects.all().order_by()], - sorted(pks)) + self.assertEquals( + [item.pk for item in OrderedModel.objects.all().order_by()], + sorted(pks)) def test_order_with_pk_filter(self): pks, priorities = self.create_ordered_model_items() - self.assertEquals([item.priority - for item in OrderedModel.objects.filter(pk__in=pks)], - sorted(priorities, reverse=True)) - - # test with id__in - self.assertEquals([item.priority - for item in OrderedModel.objects.filter(id__in=pks)], - sorted(priorities, reverse=True)) - - # test reverse - self.assertEquals([item.priority - for item in OrderedModel.objects.filter( - pk__in=pks).reverse()], sorted(priorities, - reverse=False)) + self.assertEquals( + [item.priority for item in + OrderedModel.objects.filter(pk__in=pks)], + sorted(priorities, reverse=True)) + + # Test with id__in. + self.assertEquals( + [item.priority for item in + OrderedModel.objects.filter(id__in=pks)], + sorted(priorities, reverse=True)) + + # Test reverse. + self.assertEquals( + [item.priority for item in + OrderedModel.objects.filter(pk__in=pks).reverse()], + sorted(priorities, reverse=False)) def test_remove_default_order_with_pk_filter(self): pks, priorities = self.create_ordered_model_items() - self.assertEquals([item.priority - for item in OrderedModel.objects.filter(pk__in=pks).order_by()], - priorities) + self.assertEquals( + [item.priority for item in + OrderedModel.objects.filter(pk__in=pks).order_by()], + priorities) - # TODO: test multiple orders + # TODO: Test multiple orders. diff --git a/tests/testmodels.py b/tests/testmodels.py index 125a30f..0b4bdc4 100644 --- a/tests/testmodels.py +++ b/tests/testmodels.py @@ -1,16 +1,21 @@ from django.db import models -from ..db.db_settings import get_indexes + from djangotoolbox.fields import BlobField +from ..db.db_settings import get_indexes + + class EmailModel(models.Model): email = models.EmailField() number = models.IntegerField(null=True) + class DateTimeModel(models.Model): datetime = models.DateTimeField() datetime_auto_now = models.DateTimeField(auto_now=True) datetime_auto_now_add = models.DateTimeField(auto_now_add=True) + class FieldsWithoutOptionsModel(models.Model): datetime = models.DateTimeField() date = models.DateField() @@ -28,11 +33,10 @@ class FieldsWithoutOptionsModel(models.Model): # file_path = models.FilePathField() long_text = models.TextField() indexed_text = models.TextField() - xml = models.XMLField() integer = models.IntegerField() small_integer = models.SmallIntegerField() - positiv_integer = models.PositiveIntegerField() - positiv_small_integer = models.PositiveSmallIntegerField() + positive_integer = models.PositiveIntegerField() + positive_small_integer = models.PositiveSmallIntegerField() # foreign_key = models.ForeignKey('FieldsWithOptionsModel') # foreign_key = models.ForeignKey('OrderedModel') # one_to_one = models.OneToOneField() @@ -41,38 +45,43 @@ class FieldsWithoutOptionsModel(models.Model): get_indexes()[FieldsWithoutOptionsModel] = {'indexed': ('indexed_text',)} + class FieldsWithOptionsModel(models.Model): - # any type of unique (unique_data, ...) is not supported on GAE, instead you - # can use primary_key=True for some special cases. But be carefull: changing - # the primary_key of an entity will not result in an updated entity, - # instead a new entity will be putted into the datastore. The old one will - # not be deleted and all references pointing to the old entitiy will not - # point to the new one either - datetime = models.DateTimeField(auto_now=True, db_column="birthday") + # Any type of unique (unique_data, ...) is not supported on GAE, + # instead you can use primary_key=True for some special cases. But + # be carefull: changing the primary_key of an entity will not + # result in an updated entity, instead a new entity will be putted + # into the datastore. The old one will not be deleted and all + # references pointing to the old entitiy will not point to the new + # one either. + datetime = models.DateTimeField(auto_now=True, db_column='birthday') date = models.DateField(auto_now_add=True) time = models.TimeField() floating_point = models.FloatField(null=True) boolean = models.BooleanField() null_boolean = models.NullBooleanField(default=True) text = models.CharField(default='Hallo', max_length=10) - email = models.EmailField(default='app-engine@scholardocs.com', primary_key=True) + email = models.EmailField(default='app-engine@scholardocs.com', + primary_key=True) comma_seperated_integer = models.CommaSeparatedIntegerField(max_length=10) - ip_address = models.IPAddressField(default="192.168.0.2") - slug = models.SlugField(default="GAGAA", null=True) + ip_address = models.IPAddressField(default='192.168.0.2') + slug = models.SlugField(default='GAGAA', null=True) url = models.URLField(default='http://www.scholardocs.com') # file = FileField() # file_path = FilePathField() - long_text = models.TextField(default=1000*'A') - xml = models.XMLField(default=2000*'B') + long_text = models.TextField(default=1000 * 'A') integer = models.IntegerField(default=100) small_integer = models.SmallIntegerField(default=-5) - positiv_integer = models.PositiveIntegerField(default=80) - positiv_small_integer = models.PositiveSmallIntegerField(default=3, null=True) - foreign_key = models.ForeignKey('OrderedModel', null=True, related_name='keys') + positive_integer = models.PositiveIntegerField(default=80) + positive_small_integer = models.PositiveSmallIntegerField(default=3, + null=True) + foreign_key = models.ForeignKey('OrderedModel', null=True, + related_name='keys') # one_to_one = OneToOneField() # decimal = DecimalField() # image = ImageField() + class OrderedModel(models.Model): id = models.IntegerField(primary_key=True) priority = models.IntegerField() @@ -80,14 +89,14 @@ class OrderedModel(models.Model): class Meta: ordering = ('-priority',) + class BlobModel(models.Model): data = BlobField() -class DecimalModel(models.Model): - decimal = models.DecimalField(max_digits=9, decimal_places=2) class SelfReferenceModel(models.Model): ref = models.ForeignKey('self', null=True) + class NullableTextModel(models.Model): text = models.TextField(null=True) diff --git a/tests/transactions.py b/tests/transactions.py index a727370..cd837b0 100644 --- a/tests/transactions.py +++ b/tests/transactions.py @@ -1,10 +1,12 @@ -from .testmodels import EmailModel from django.db.models import F from django.test import TestCase +from .testmodels import EmailModel + + class TransactionTest(TestCase): emails = ['app-engine@scholardocs.com', 'sharingan@uchias.com', - 'rinnengan@sage.de', 'rasengan@naruto.com'] + 'rinnengan@sage.de', 'rasengan@naruto.com'] def setUp(self): EmailModel(email=self.emails[0], number=1).save() @@ -14,47 +16,43 @@ def setUp(self): def test_update(self): self.assertEqual(2, len(EmailModel.objects.all().filter( email=self.emails[0]))) - + self.assertEqual(1, len(EmailModel.objects.all().filter( email=self.emails[1]))) - + EmailModel.objects.all().filter(email=self.emails[0]).update( email=self.emails[1]) - + self.assertEqual(0, len(EmailModel.objects.all().filter( email=self.emails[0]))) self.assertEqual(3, len(EmailModel.objects.all().filter( email=self.emails[1]))) - + def test_f_object_updates(self): self.assertEqual(1, len(EmailModel.objects.all().filter( number=1))) self.assertEqual(1, len(EmailModel.objects.all().filter( number=2))) - - # test add - EmailModel.objects.all().filter(email=self.emails[0]).update(number= - F('number') + F('number')) - + + # Test add. + EmailModel.objects.all().filter(email=self.emails[0]).update( + number=F('number') + F('number')) + self.assertEqual(1, len(EmailModel.objects.all().filter( number=2))) self.assertEqual(1, len(EmailModel.objects.all().filter( number=4))) - - EmailModel.objects.all().filter(email=self.emails[1]).update(number= - F('number') + 10, email=self.emails[0]) - + + EmailModel.objects.all().filter(email=self.emails[1]).update( + number=F('number') + 10, email=self.emails[0]) + self.assertEqual(1, len(EmailModel.objects.all().filter(number=13))) - self.assertEqual(self.emails[0], EmailModel.objects.all().get(number=13). - email) - - # complex expression test - EmailModel.objects.all().filter(number=13).update(number= - F('number')*(F('number') + 10) - 5, email=self.emails[0]) + self.assertEqual(self.emails[0], + EmailModel.objects.all().get(number=13).email) + + # Complex expression test. + EmailModel.objects.all().filter(number=13).update( + number=F('number') * (F('number') + 10) - 5, email=self.emails[0]) self.assertEqual(1, len(EmailModel.objects.all().filter(number=294))) - - # TODO: tests for - # test sub - # test muld - # test div - # test mod, .... \ No newline at end of file + + # TODO: Tests for: sub, muld, div, mod, .... diff --git a/utils.py b/utils.py index a181c36..f80b2ab 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,8 @@ +import os + from google.appengine.api import apiproxy_stub_map from google.appengine.api.app_identity import get_application_id -import os + have_appserver = bool(apiproxy_stub_map.apiproxy.GetStub('datastore_v3')) @@ -14,8 +16,8 @@ default_partition='dev')[0] appid = appconfig.application.split('~', 1)[-1] except ImportError, e: - raise Exception('Could not get appid. Is your app.yaml file missing? ' - 'Error was: %s' % e) + raise Exception("Could not get appid. Is your app.yaml file missing? " + "Error was: %s" % e) on_production_server = have_appserver and \ not os.environ.get('SERVER_SOFTWARE', '').lower().startswith('devel') diff --git a/views.py b/views.py index e7d06af..b3071ec 100644 --- a/views.py +++ b/views.py @@ -2,10 +2,11 @@ from django.http import HttpResponse from django.utils.importlib import import_module + def warmup(request): """ - Provides default procedure for handling warmup requests on App Engine. - Just add this view to your main urls.py. + Provides default procedure for handling warmup requests on App + Engine. Just add this view to your main urls.py. """ for app in settings.INSTALLED_APPS: for name in ('urls', 'views', 'models'): @@ -14,4 +15,4 @@ def warmup(request): except ImportError: pass content_type = 'text/plain; charset=%s' % settings.DEFAULT_CHARSET - return HttpResponse('Warmup done', content_type=content_type) + return HttpResponse("Warmup done.", content_type=content_type)