Skip to content

Commit

Permalink
Merge remote-tracking branch 'web2py' into Fix-belong-with-empty-set-…
Browse files Browse the repository at this point in the history
…for-MongoDB-and-GAE
  • Loading branch information
stephenrauch committed May 27, 2015
2 parents 33d330c + 4d36919 commit a887058
Show file tree
Hide file tree
Showing 14 changed files with 191 additions and 56 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,6 @@ build/*
.project
.pydevproject
.settings
.idea
.vslick

25 changes: 22 additions & 3 deletions CHANGES
Original file line number Diff line number Diff line change
@@ -1,10 +1,19 @@
pydal changelog
===============

Next
----
Version 15.05.26
----------------

Released on May 26th 2015

- Fixed `DAL.__getattr__`
- Fixed backward compatibility breaks introduced with 15.05


Version 15.05
-------------

(Release date to be announced)
Released on May 23rd 2015

- Fixed True/False expressions in MSSQL
- Introduced `iterselect()` and `IterRows`
Expand All @@ -13,6 +22,16 @@ Next
- Implemented JSON serialization for objects
- Refactored many internal objects to improve performance
- Added python 3.x support (experimental)
- Several fixes and improvements to `MongoDBAdapter`
- Implemented unicode handling in MSSQL (experimental) via mssql4n and mssql3n
adapters
Notes: These adapters will probably become the de-facto standard for MSSQL handling; any other adapter will continue to be supported just for legacy
databases
- Restricted table and field names to "valid" ones
Notes: the "dotted-notation-friendly" syntax for names means anything:
- alphanumeric
- not starting with underscore or an integer
`rname` attribute is intended to be used for anything else


Version 15.03
Expand Down
2 changes: 2 additions & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,6 @@ include LICENSE
include AUTHORS
include CHANGES
recursive-include tests *
recursive-exclude tests .DS_Store
recursive-include docs *
recursive-exclude docs .DS_Store
9 changes: 3 additions & 6 deletions docs/pydal.adapters.rst
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,10 @@ pydal.adapters.firebird module
:undoc-members:
:show-inheritance:

pydal.adapters.google module
--------------------------------
pydal.adapters.google_adapters module
-----------------------------------------

.. automodule:: pydal.adapters.google
:members:
:undoc-members:
:show-inheritance:
Adapter for GAE

pydal.adapters.imap module
------------------------------
Expand Down
2 changes: 1 addition & 1 deletion pydal/adapters/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -1454,7 +1454,7 @@ def represent_exceptions(self, obj, fieldtype):
return None

def lastrowid(self, table):
return None
return self.cursor.lastrowid

def rowslice(self, rows, minimum=0, maximum=None):
"""
Expand Down
56 changes: 49 additions & 7 deletions pydal/adapters/mongo.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,13 @@
from ..helpers.classes import SQLALL, Reference
from ..helpers.methods import use_common_filters, xorify
from .base import NoSQLAdapter
try:
from bson import Binary
from bson.binary import USER_DEFINED_SUBTYPE
except:
class Binary(object):
pass
USER_DEFINED_SUBTYPE = 0

long = integer_types[-1]

Expand Down Expand Up @@ -162,12 +169,7 @@ def represent(self, obj, fieldtype):
# string or integer
return datetime.datetime.combine(d, value)
elif fieldtype == "blob":
if value is None or isinstance(value, basestring):
return value
from bson import Binary
if not isinstance(value, Binary):
return Binary(value)
return value
return MongoBlob(value)
elif isinstance(fieldtype, basestring):
if fieldtype.startswith('list:'):
if fieldtype.startswith('list:reference'):
Expand All @@ -183,6 +185,9 @@ def represent(self, obj, fieldtype):
value = self.object_id(value)
return value

def parse_blob(self, value, field_type):
return MongoBlob.decode(value)

def _expand_query(self, query, tablename=None, safe=None):
""" Return a tuple containing query and ctable """
if not tablename:
Expand Down Expand Up @@ -363,7 +368,7 @@ def select(self, query, fields, attributes, snapshot=False):
row.append(value)
rows.append(row)
processor = attributes.get('processor', self.parse)
result = processor(rows, fields, newnames, False)
result = processor(rows, fields, newnames, blob_decode=True)
return result

def insert(self, table, fields, safe=None):
Expand Down Expand Up @@ -626,3 +631,40 @@ def CONTAINS(self, first, second, case_sensitive=True):
ret = {self.expand(first): val}

return ret

class MongoBlob(Binary):
MONGO_BLOB_BYTES = USER_DEFINED_SUBTYPE
MONGO_BLOB_NON_UTF8_STR = USER_DEFINED_SUBTYPE + 1

def __new__(cls, value):
# return None and Binary() unmolested
if value is None or isinstance(value, Binary):
return value

# bytearray is marked as MONGO_BLOB_BYTES
if isinstance(value, bytearray):
return Binary.__new__(cls, bytes(value), MongoBlob.MONGO_BLOB_BYTES)

# return non-strings as Binary(), eg: PY3 bytes()
if not isinstance(value, basestring):
return Binary(value)

# if string is encodable as UTF-8, then return as string
try:
value.encode('utf-8')
return value
except:
# string which can not be UTF-8 encoded, eg: pickle strings
return Binary.__new__(cls, value, MongoBlob.MONGO_BLOB_NON_UTF8_STR)

def __repr__(self):
return repr(MongoBlob.decode(self))

@staticmethod
def decode(value):
if isinstance(value, Binary):
if value.subtype == MongoBlob.MONGO_BLOB_BYTES:
return bytearray(value)
if value.subtype == MongoBlob.MONGO_BLOB_NON_UTF8_STR:
return str(value)
return value
4 changes: 0 additions & 4 deletions pydal/adapters/mysql.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,3 @@ def connector(driver_args=driver_args):
def after_connection(self):
self.execute('SET FOREIGN_KEY_CHECKS=1;')
self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")

def lastrowid(self,table):
self.execute('select last_insert_id();')
return int(self.cursor.fetchone()[0])
3 changes: 0 additions & 3 deletions pydal/adapters/sqlite.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,6 @@ def _truncate(self, table, mode=''):
return ['DELETE FROM %s;' % tablename,
"DELETE FROM sqlite_sequence WHERE name='%s';" % tablename]

def lastrowid(self, table):
return self.cursor.lastrowid

def REGEXP(self,first,second):
return '(%s REGEXP %s)' % (self.expand(first),
self.expand(second,'string'))
Expand Down
14 changes: 7 additions & 7 deletions pydal/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -899,31 +899,31 @@ def __getitem__(self, key):
return self.__getattr__(str(key))

def __getattr__(self, key):
if getattr(self, '_lazy_tables') and \
if object.__getattribute__(self, '_lazy_tables') and \
key in object.__getattribute__(self, '_LAZY_TABLES'):
tablename, fields, args = self._LAZY_TABLES.pop(key)
return self.lazy_define_table(tablename, *fields, **args)
return super(DAL, self).__getattr__(key)

def __setattr__(self, key, value):
if key[:1]!='_' and key in self:
if key[:1] != '_' and key in self:
raise SyntaxError(
'Object %s exists and cannot be redefined' % key)
return super(DAL, self).__setattr__(key, value)

def __repr__(self):
if hasattr(self,'_uri'):
if hasattr(self, '_uri'):
return '<DAL uri="%s">' % hide_password(self._adapter.uri)
else:
return '<DAL db_uid="%s">' % self._db_uid

def smart_query(self,fields,text):
return Set(self, smart_query(fields,text))
def smart_query(self, fields, text):
return Set(self, smart_query(fields, text))

def __call__(self, query=None, ignore_common_filters=None):
if isinstance(query,Table):
if isinstance(query, Table):
query = self._adapter.id_query(query)
elif isinstance(query,Field):
elif isinstance(query, Field):
query = query!=None
elif isinstance(query, dict):
icf = query.get("ignore_common_filters")
Expand Down
3 changes: 2 additions & 1 deletion pydal/helpers/regex.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
REGEX_UPLOAD_PATTERN = re.compile('(?P<table>[\w\-]+)\.(?P<field>[\w\-]+)\.(?P<uuidkey>[\w\-]+)(\.(?P<name>\w+))?\.\w+$')
REGEX_CLEANUP_FN = re.compile('[\'"\s;]+')
REGEX_UNPACK = re.compile('(?<!\|)\|(?!\|)')
REGEX_PYTHON_KEYWORDS = re.compile('^(and|del|from|not|while|as|elif|global|or|with|assert|else|if|pass|yield|break|except|import|print|class|exec|in|raise|continue|finally|is|return|def|for|lambda|try)$')
REGEX_PYTHON_KEYWORDS = re.compile('^(and|del|from|not|while|as|elif|global|or|with|assert|else|if|pass|yield|break|except|import|print|class|exec|in|raise|continue|finally|is|return|def|for|lambda|try|False|True|nonlocal)$')
REGEX_SELECT_AS_PARSER = re.compile("\s+AS\s+(\S+)")
REGEX_CONST_STRING = re.compile('(\"[^\"]*?\")|(\'[^\']*?\')')
REGEX_SEARCH_PATTERN = re.compile('^{[^\.]+\.[^\.]+(\.(lt|gt|le|ge|eq|ne|contains|startswith|year|month|day|hour|minute|second))?(\.not)?}$')
Expand All @@ -20,3 +20,4 @@
REGEX_ALPHANUMERIC = re.compile('^[0-9a-zA-Z]\w*$')
REGEX_PASSWORD = re.compile('\://([^:@]*)\:')
REGEX_NOPASSWD = re.compile('\/\/[\w\.\-]+[\:\/](.+)(?=@)') # was '(?<=[\:\/])([^:@/]+)(?=@.+)'
REGEX_VALID_TB_FLD = re.compile(r'^[^\d_][_0-9a-zA-Z]*\Z')
26 changes: 17 additions & 9 deletions pydal/objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from .exceptions import NotFoundException, NotAuthorizedException
from .helpers.regex import REGEX_TABLE_DOT_FIELD, REGEX_ALPHANUMERIC, \
REGEX_PYTHON_KEYWORDS, REGEX_STORE_PATTERN, REGEX_UPLOAD_PATTERN, \
REGEX_CLEANUP_FN
REGEX_CLEANUP_FN, REGEX_VALID_TB_FLD
from .helpers.classes import Reference, MethodAdder, SQLCallableList, SQLALL, \
Serializable, BasicStorage
from .helpers.methods import list_represent, bar_decode_integer, \
Expand All @@ -42,7 +42,7 @@ class Row(BasicStorage):

def __getitem__(self, k):
key = str(k)
_extra = self.get('_extra', None)
_extra = super(Row, self).get('_extra', None)
if _extra is not None:
v = _extra.get(key, DEFAULT)
if v != DEFAULT:
Expand All @@ -66,13 +66,14 @@ def __getitem__(self, k):
except KeyError:
pass
try:
e = self.get('__get_lazy_reference__')
e = super(Row, self).get('__get_lazy_reference__')
if e is not None and callable(e):
self[key] = e(key)
return self[key]
except Exception as e:
raise e
return None

raise KeyError

__str__ = __repr__ = lambda self: '<Row %s>' % self.as_dict(custom_types=[LazySet])

Expand All @@ -91,6 +92,12 @@ def __eq__(self, other):
except AttributeError:
return False

def get(self, key, default=None):
try:
return self.__getitem__(key)
except(KeyError, AttributeError, TypeError):
return default

def as_dict(self, datetime_to_str=False, custom_types=None):
SERIALIZABLE_TYPES = [str, int, float, bool, list, dict]
if PY2:
Expand Down Expand Up @@ -205,8 +212,8 @@ def __init__(
self._actual = False # set to True by define_table()
self._db = db
self._tablename = tablename
if (not isinstance(tablename, str) or tablename[0] == '_'
or hasattr(DAL, tablename) or '.' in tablename
if (not isinstance(tablename, str) or hasattr(DAL, tablename)
or not REGEX_VALID_TB_FLD.match(tablename)
or REGEX_PYTHON_KEYWORDS.match(tablename)
):
raise SyntaxError('Field: invalid table name: %s, '
Expand Down Expand Up @@ -1396,9 +1403,10 @@ def __init__(
except UnicodeEncodeError:
raise SyntaxError('Field: invalid unicode field name')
self.name = fieldname = cleanup(fieldname)
if not isinstance(fieldname, str) or hasattr(Table, fieldname) or \
fieldname[0] == '_' or '.' in fieldname or \
REGEX_PYTHON_KEYWORDS.match(fieldname):
if (not isinstance(fieldname, str) or hasattr(Table, fieldname)
or not REGEX_VALID_TB_FLD.match(fieldname)
or REGEX_PYTHON_KEYWORDS.match(fieldname)
):
raise SyntaxError('Field: invalid field name: %s, '
'use rname for "funny" names' % fieldname)

Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from setuptools import setup
setup(
name='pyDAL',
version='15.04-dev',
version='15.05.26',
url='https://github.com/web2py/pydal',
license='BSD',
author='Massimo Di Pierro',
Expand Down
12 changes: 12 additions & 0 deletions tests/nosql.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,18 @@ def testRun(self):
self.assertEqual(isinstance(db.tt.insert(aa=b'xyzzy'), long), True)
self.assertEqual(db().select(db.tt.aa)[0].aa, b'xyzzy')
drop(db.tt)
# pickling a tuple will create a string which is not UTF-8 able.
import pickle
insert_val = pickle.dumps((0,), pickle.HIGHEST_PROTOCOL)
db.define_table('tt', Field('aa', 'blob', default=''))
self.assertEqual(isinstance(db.tt.insert(aa=insert_val), long), True)
self.assertEqual(db().select(db.tt.aa)[0].aa, insert_val)
drop(db.tt)
insert_val = bytearray('a','utf-8')
db.define_table('tt', Field('aa', 'blob', default=''))
self.assertEqual(isinstance(db.tt.insert(aa=insert_val), long), True)
self.assertEqual(db().select(db.tt.aa)[0].aa, insert_val)
drop(db.tt)
db.define_table('tt', Field('aa', 'integer', default=1))
self.assertEqual(isinstance(db.tt.insert(aa=3), long), True)
self.assertEqual(db().select(db.tt.aa)[0].aa, 3)
Expand Down
Loading

0 comments on commit a887058

Please sign in to comment.