Fixes issues with security.create_token() and Unicode problems

.gitignore

@@ -10,3 +10,5 @@
 # repository level.
 *.egg-info
 
+# Editor-specific files.
+.idea/

tests/extras_json_test.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
-from webapp2_extras import json
+
+from webapp2_extras import escape as json
 
 import test_base
 

tests/extras_security_test.py

@@ -13,19 +13,21 @@ def test_create_token(self):
         self.assertRaises(ValueError, security.create_token, 0)
         self.assertRaises(ValueError, security.create_token, -1)
 
-        token = security.create_token(16)
+        token = security.create_token(16, 16)
         self.assertTrue(re.match(r'^[a-f0-9]{4}$', token) is not None)
 
-        token = security.create_token(32)
+        token = security.create_token(16, 10)
+        self.assertTrue(int(token, 10) >= 0)
+
+        token = security.create_token(32, 16)
         self.assertTrue(re.match(r'^[a-f0-9]{8}$', token) is not None)
 
-        token = security.create_token(64)
+        token = security.create_token(64, 16)
         self.assertTrue(re.match(r'^[a-f0-9]{16}$', token) is not None)
 
-        token = security.create_token(128)
+        token = security.create_token(128, 16)
         self.assertTrue(re.match(r'^[a-f0-9]{32}$', token) is not None)
 
-        token = security.create_token(16, True)
 
     def test_create_check_password_hash(self):
         self.assertRaises(TypeError, security.create_password_hash, 'foo',

tests/misc_test.py

@@ -64,10 +64,10 @@ def test_import_string(self):
         self.assertRaises(AttributeError, webapp2.import_string, 'webob.dfasfasdfdsfsd')
 
     def test_to_utf8(self):
-        res = webapp2._to_utf8('ábcdéf'.decode('utf-8'))
+        res = webapp2.types.unicode_to_utf8('ábcdéf'.decode('utf-8'))
         self.assertEqual(isinstance(res, str), True)
 
-        res = webapp2._to_utf8('abcdef')
+        res = webapp2.types.unicode_to_utf8('abcdef')
         self.assertEqual(isinstance(res, str), True)
 
     '''

webapp2.py → webapp2/__init__.py

@@ -14,7 +14,13 @@
 import logging
 import re
 import urllib
-import urlparse
+
+try:
+    # Python 3.
+    from urllib.parse import urljoin, urlunsplit
+except ImportError:
+    # Python 2.x
+    from urlparse import urljoin, urlunsplit
 
 import webob
 from webob import exc
@@ -44,6 +50,10 @@ def _run(self, app):
 
         run_wsgi_app = run_bare_wsgi_app = classmethod(_run)
 
+from webapp2.types import \
+    bytes, is_bytes, is_bytes_or_unicode, bytes_to_unicode, \
+    is_unicode, is_list, is_tuple, is_dict, unicode_to_utf8, to_unicode_if_bytes, unicode_string
+
 __version_info__ = ('1', '8', '1')
 __version__ = '.'.join(__version_info__)
 
@@ -234,11 +244,11 @@ def blank(cls, path, environ=None, base_url=None,
             environ['REQUEST_METHOD'] = 'POST'
             if hasattr(data, 'items'):
                 data = data.items()
-            if not isinstance(data, str):
+            if not is_bytes(data):
                 data = urllib.urlencode(data)
             environ['wsgi.input'] = StringIO(data)
             environ['webob.is_body_seekable'] = True
-            environ['CONTENT_LENGTH'] = str(len(data))
+            environ['CONTENT_LENGTH'] = bytes(len(data))
             environ['CONTENT_TYPE'] = 'application/x-www-form-urlencoded'
 
         base = super(Request, cls).blank(path, environ=environ,
@@ -326,10 +336,12 @@ def write(self, text):
         """Appends a text to the response body."""
         # webapp uses StringIO as Response.out, so we need to convert anything
         # that is not str or unicode to string to keep same behavior.
-        if not isinstance(text, basestring):
-            text = unicode(text)
+        #if not is_bytes_or_unicode(text):
+        #    text = bytes(text)
+        if not is_bytes_or_unicode(text):
+            text = unicode_string(text)
 
-        if isinstance(text, unicode) and not self.charset:
+        if is_unicode(text) and not self.charset:
             self.charset = self.default_charset
 
         super(Response, self).write(text)
@@ -341,11 +353,11 @@ def _set_status(self, value):
         if isinstance(value, (int, long)):
             code = int(value)
         else:
-            if isinstance(value, unicode):
+            if is_unicode(value):
                 # Status messages have to be ASCII safe, so this is OK.
-                value = str(value)
+                value = bytes(value)
 
-            if not isinstance(value, str):
+            if not is_bytes(value):
                 raise TypeError(
                     'You must set status to a string or integer (not %s)' %
                     type(value))
@@ -397,7 +409,7 @@ def _get_headers(self):
     def _set_headers(self, value):
         if hasattr(value, 'items'):
             value = value.items()
-        elif not isinstance(value, list):
+        elif not is_list(value):
             raise TypeError('Response headers must be a list or dictionary.')
 
         self.headerlist = value
@@ -424,7 +436,7 @@ def wsgi_write(self, start_response):
         if (self.headers.get('Cache-Control') == 'no-cache' and
             not self.headers.get('Expires')):
             self.headers['Expires'] = 'Fri, 01 Jan 1990 00:00:00 GMT'
-            self.headers['Content-Length'] = str(len(self.body))
+            self.headers['Content-Length'] = bytes(len(self.body))
 
         write = start_response(self.status, self.headerlist)
         write(self.body)
@@ -895,7 +907,7 @@ def __init__(self, template, handler=None, name=None, defaults=None,
         self.defaults = defaults or {}
         self.methods = methods
         self.schemes = schemes
-        if isinstance(handler, basestring) and ':' in handler:
+        if is_bytes_or_unicode(handler) and ':' in handler:
             if handler_method:
                 raise ValueError(
                     "If handler_method is defined in a Route, handler "
@@ -974,8 +986,8 @@ def _build(self, args, kwargs):
                 raise KeyError('Missing argument "%s" to build URI.' % \
                     name.strip('_'))
 
-            if not isinstance(value, basestring):
-                value = str(value)
+            if not is_bytes_or_unicode(value):
+                value = bytes(value)
 
             if not regex.match(value):
                 raise ValueError('URI buiding error: Value "%s" is not '
@@ -1092,7 +1104,7 @@ def add(self, route):
             A :class:`Route` instance or, for compatibility with webapp, a
             tuple ``(regex, handler_class)``.
         """
-        if isinstance(route, tuple):
+        if is_tuple(route):
             # Exceptional compatibility case: route compatible with webapp.
             route = self.route_class(*route)
 
@@ -1224,7 +1236,7 @@ def default_dispatcher(self, request, response):
 
         if route.handler_adapter is None:
             handler = route.handler
-            if isinstance(handler, basestring):
+            if is_bytes_or_unicode(handler):
                 if handler not in self.handlers:
                     self.handlers[handler] = handler = import_string(handler)
                 else:
@@ -1527,7 +1539,7 @@ def handle_exception(self, request, response, e):
 
         handler = self.error_handlers.get(code)
         if handler:
-            if isinstance(handler, basestring):
+            if is_bytes_or_unicode(handler):
                 self.error_handlers[code] = handler = import_string(handler)
 
             return handler(request, response, e)
@@ -1644,7 +1656,7 @@ def redirect(uri, permanent=False, abort=False, code=None, body=None,
     """
     if uri.startswith(('.', '/')):
         request = request or get_request()
-        uri = str(urlparse.urljoin(request.url, uri))
+        uri = bytes(urljoin(request.url, uri))
 
     if code is None:
         if permanent:
@@ -1724,7 +1736,7 @@ def import_string(import_name, silent=False):
     :returns:
         The imported object.
     """
-    import_name = _to_utf8(import_name)
+    import_name = unicode_to_utf8(import_name)
     try:
         if '.' in import_name:
             module, obj = import_name.rsplit('.', 1)
@@ -1760,19 +1772,19 @@ def _urlunsplit(scheme=None, netloc=None, path=None, query=None,
         netloc = None
 
     if path:
-        path = urllib.quote(_to_utf8(path))
+        path = urllib.quote(unicode_to_utf8(path))
 
-    if query and not isinstance(query, basestring):
-        if isinstance(query, dict):
+    if query and not is_bytes_or_unicode(query):
+        if is_dict(query):
             query = query.iteritems()
 
         # Sort args: commonly needed to build signatures for services.
         query = urllib.urlencode(sorted(query))
 
     if fragment:
-        fragment = urllib.quote(_to_utf8(fragment))
+        fragment = urllib.quote(unicode_to_utf8(fragment))
 
-    return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
+    return urlunsplit((scheme, netloc, path, query, fragment))
 
 
 def _get_handler_methods(handler):
@@ -1796,14 +1808,6 @@ def _normalize_handler_method(method):
     return method.lower().replace('-', '_')
 
 
-def _to_utf8(value):
-    """Encodes a unicode value to UTF-8 if not yet encoded."""
-    if isinstance(value, str):
-        return value
-
-    return value.encode('utf-8')
-
-
 def _parse_route_template(template, default_sufix=''):
     """Lazy route template parser."""
     variables = {}