From 828affc919064c986b360a9e02f5eceb85cf6f4b Mon Sep 17 00:00:00 2001 From: Florent Messa Date: Wed, 29 Jul 2015 15:14:29 +0200 Subject: [PATCH 01/10] Add compat layer for python 3 --- nydus/compat.py | 32 ++++++++++++++++++++++++++ nydus/conf.py | 4 +++- nydus/contrib/ketama.py | 43 +++++++++++++++++++---------------- nydus/db/__init__.py | 15 ++++++------ nydus/db/backends/base.py | 4 ++-- nydus/db/backends/memcache.py | 4 ++-- nydus/db/backends/redis.py | 2 +- nydus/db/backends/riak.py | 2 +- nydus/db/base.py | 13 +++++++---- nydus/db/map.py | 12 ++++++---- nydus/db/routers/keyvalue.py | 3 ++- nydus/utils.py | 8 ++++--- 12 files changed, 94 insertions(+), 48 deletions(-) create mode 100644 nydus/compat.py diff --git a/nydus/compat.py b/nydus/compat.py new file mode 100644 index 0000000..cea76b3 --- /dev/null +++ b/nydus/compat.py @@ -0,0 +1,32 @@ +import sys + +try: + from Queue import Queue, Empty +except ImportError: + from queue import Queue, Empty # noqa + + +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + + +if PY3: + string_types = str, +else: + string_types = basestring, + + +try: + # Python 2 + from itertools import izip +except ImportError: + # Python 3 + izip = zip + + +if PY3: + def iteritems(d, **kw): + return iter(d.items(**kw)) +else: + def iteritems(d, **kw): + return iter(d.iteritems(**kw)) diff --git a/nydus/conf.py b/nydus/conf.py index a846da5..405704c 100644 --- a/nydus/conf.py +++ b/nydus/conf.py @@ -8,11 +8,13 @@ import warnings +from nydus.compat import iteritems + CONNECTIONS = {} def configure(kwargs): - for k, v in kwargs.iteritems(): + for k, v in iteritems(kwargs): if k.upper() != k: warnings.warn('Invalid setting, \'%s\' which is not defined by Nydus' % k) elif k not in globals(): diff --git a/nydus/contrib/ketama.py b/nydus/contrib/ketama.py index fc12474..004b078 100644 --- a/nydus/contrib/ketama.py +++ b/nydus/contrib/ketama.py @@ -4,6 +4,13 @@ Rewrited from the original source: http://www.audioscrobbler.net/development/ketama/ """ +from __future__ import print_function + +import hashlib +import math +from bisect import bisect + + __author__ = "Andrey Nikishaev" __email__ = "creotiv@gmail.com" __version__ = 0.1 @@ -11,10 +18,6 @@ __all__ = ['Ketama'] -import hashlib -import math -from bisect import bisect - class Ketama(object): @@ -49,10 +52,10 @@ def _build_circle(self): b_key = self._md5_digest('%s-%s-salt' % (node, i)) for l in xrange(0, 4): - key = ((b_key[3 + l * 4] << 24) - | (b_key[2 + l * 4] << 16) - | (b_key[1 + l * 4] << 8) - | b_key[l * 4]) + key = ((b_key[3 + l * 4] << 24) | + (b_key[2 + l * 4] << 16) | + (b_key[1 + l * 4] << 8) | + b_key[l * 4]) self._hashring[key] = node self._sorted_keys.append(key) @@ -84,10 +87,10 @@ def _gen_key(self, key): return self._hashi(b_key, lambda x: x) def _hashi(self, b_key, fn): - return ((b_key[fn(3)] << 24) - | (b_key[fn(2)] << 16) - | (b_key[fn(1)] << 8) - | b_key[fn(0)]) + return ((b_key[fn(3)] << 24) | + (b_key[fn(2)] << 16) | + (b_key[fn(1)] << 8) | + b_key[fn(0)]) def _md5_digest(self, key): return map(ord, hashlib.md5(key).digest()) @@ -134,14 +137,14 @@ def test(k): tower = k.get_node('a' + str(i)) data.setdefault(tower, 0) data[tower] += 1 - print 'Number of caches on each node: ' - print data - print '' - - print k.get_node('Aplple') - print k.get_node('Hello') - print k.get_node('Data') - print k.get_node('Computer') + print('Number of caches on each node: ') + print(data) + print('') + + print(k.get_node('Aplple')) + print(k.get_node('Hello')) + print(k.get_node('Data')) + print(k.get_node('Computer')) NODES = [ '192.168.0.1:6000', '192.168.0.1:6001', '192.168.0.1:6002', diff --git a/nydus/db/__init__.py b/nydus/db/__init__.py index 1a2afb7..94c0518 100644 --- a/nydus/db/__init__.py +++ b/nydus/db/__init__.py @@ -14,15 +14,16 @@ :copyright: (c) 2011-2012 DISQUS. :license: Apache License 2.0, see LICENSE for more details. """ - -__all__ = ('create_cluster', 'connections', 'Cluster') - import copy from nydus import conf +from nydus.compat import string_types from nydus.db.base import LazyConnectionHandler from nydus.db.routers.base import BaseRouter -from nydus.utils import import_string, apply_defaults +from nydus.utils import import_string + + +__all__ = ('create_cluster', 'connections', 'Cluster') def create_cluster(settings): @@ -49,7 +50,7 @@ def create_cluster(settings): # Pull in our client settings = copy.deepcopy(settings) backend = settings.pop('engine', settings.pop('backend', None)) - if isinstance(backend, basestring): + if isinstance(backend, string_types): Conn = import_string(backend) elif backend: Conn = backend @@ -60,7 +61,7 @@ def create_cluster(settings): cluster = settings.pop('cluster', None) if not cluster: Cluster = Conn.get_cluster() - elif isinstance(cluster, basestring): + elif isinstance(cluster, string_types): Cluster = import_string(cluster) else: Cluster = cluster @@ -69,7 +70,7 @@ def create_cluster(settings): router = settings.pop('router', None) if not router: Router = BaseRouter - elif isinstance(router, basestring): + elif isinstance(router, string_types): Router = import_string(router) else: Router = router diff --git a/nydus/db/backends/base.py b/nydus/db/backends/base.py index f20d720..d41acc9 100644 --- a/nydus/db/backends/base.py +++ b/nydus/db/backends/base.py @@ -5,10 +5,10 @@ :copyright: (c) 2011-2012 DISQUS. :license: Apache License 2.0, see LICENSE for more details. """ +from nydus.db.base import BaseCluster -__all__ = ('BaseConnection',) -from nydus.db.base import BaseCluster +__all__ = ('BaseConnection',) class BasePipeline(object): diff --git a/nydus/db/backends/memcache.py b/nydus/db/backends/memcache.py index e1ac718..dd08037 100644 --- a/nydus/db/backends/memcache.py +++ b/nydus/db/backends/memcache.py @@ -10,10 +10,10 @@ import pylibmc -from itertools import izip from nydus.db.backends import BaseConnection, BasePipeline from nydus.db.promise import EventualCommand from nydus.utils import peek +from nydus.compat import izip class Memcache(BaseConnection): @@ -22,7 +22,7 @@ class Memcache(BaseConnection): supports_pipelines = True def __init__(self, num, host='localhost', port=11211, binary=True, - behaviors=None, **options): + behaviors=None, **options): self.host = host self.port = port self.binary = binary diff --git a/nydus/db/backends/redis.py b/nydus/db/backends/redis.py index 3a93805..648ab29 100644 --- a/nydus/db/backends/redis.py +++ b/nydus/db/backends/redis.py @@ -8,11 +8,11 @@ from __future__ import absolute_import -from itertools import izip from redis import Redis as RedisClient, StrictRedis from redis import ConnectionError, InvalidResponse from nydus.db.backends import BaseConnection, BasePipeline +from nydus.compat import izip class RedisPipeline(BasePipeline): diff --git a/nydus/db/backends/riak.py b/nydus/db/backends/riak.py index 71156ad..aa0f196 100644 --- a/nydus/db/backends/riak.py +++ b/nydus/db/backends/riak.py @@ -22,7 +22,7 @@ class Riak(BaseConnection): supports_pipelines = False def __init__(self, num, host='127.0.0.1', port=8098, prefix='riak', mapred_prefix='mapred', client_id=None, - transport_class=None, solr_transport_class=None, transport_options=None, **options): + transport_class=None, solr_transport_class=None, transport_options=None, **options): self.host = host self.port = port diff --git a/nydus/db/base.py b/nydus/db/base.py index 4ef6051..371e07a 100644 --- a/nydus/db/base.py +++ b/nydus/db/base.py @@ -6,19 +6,22 @@ :license: Apache License 2.0, see LICENSE for more details. """ -__all__ = ('LazyConnectionHandler', 'BaseCluster') - import collections + from nydus.db.map import DistributedContextManager from nydus.db.routers import BaseRouter, routing_params from nydus.utils import apply_defaults +from nydus.compat import iteritems + + +__all__ = ('LazyConnectionHandler', 'BaseCluster') def iter_hosts(hosts): # this can either be a dictionary (with the key acting as the numeric # index) or it can be a sorted list. if isinstance(hosts, collections.Mapping): - return hosts.iteritems() + return iteritems(hosts) return enumerate(hosts) @@ -76,7 +79,7 @@ def execute(self, path, args, kwargs): func = getattr(func, piece) try: results.append(func(*args, **kwargs)) - except tuple(conn.retryable_exceptions), e: + except tuple(conn.retryable_exceptions) as e: if not self.router.retryable: raise e elif retry == self.max_connection_retries - 1: @@ -155,7 +158,7 @@ def is_ready(self): def reload(self): from nydus.db import create_cluster - for conn_alias, conn_settings in self.conf_callback().iteritems(): + for conn_alias, conn_settings in iteritems(self.conf_callback()): self[conn_alias] = create_cluster(conn_settings) self._is_ready = True diff --git a/nydus/db/map.py b/nydus/db/map.py index f049c32..0e71b51 100644 --- a/nydus/db/map.py +++ b/nydus/db/map.py @@ -7,9 +7,11 @@ """ from collections import defaultdict + from nydus.utils import ThreadPool from nydus.db.exceptions import CommandError from nydus.db.promise import EventualCommand, change_resolution +from nydus.compat import iteritems class BaseDistributedConnection(object): @@ -122,7 +124,7 @@ def execute(self, cluster, commands): pool = self.get_pool(commands) # execute our pending commands either in the pool, or using a pipeline - for db_num, command_list in commands.iteritems(): + for db_num, command_list in iteritems(commands): for command in command_list: # XXX: its important that we clone the command here so we dont override anything # in the EventualCommand proxy (it can only resolve once) @@ -144,14 +146,14 @@ def execute(self, cluster, commands): pool = self.get_pool(commands) # execute our pending commands either in the pool, or using a pipeline - for db_num, command_list in commands.iteritems(): + for db_num, command_list in iteritems(commands): pipes[db_num] = cluster[db_num].get_pipeline() for command in command_list: # add to pipeline pipes[db_num].add(command.clone()) # We need to finalize our commands with a single execute in pipelines - for db_num, pipe in pipes.iteritems(): + for db_num, pipe in iteritems(pipes): pool.add(db_num, pipe.execute, (), {}) # Consolidate commands with their appropriate results @@ -160,7 +162,7 @@ def execute(self, cluster, commands): # Results get grouped by their command signature, so we have to separate the logic results = defaultdict(list) - for db_num, db_results in db_result_map.iteritems(): + for db_num, db_results in iteritems(db_result_map): # Pipelines always execute on a single database assert len(db_results) == 1 db_results = db_results[0] @@ -171,7 +173,7 @@ def execute(self, cluster, commands): results[command].append(db_results) continue - for command, result in db_results.iteritems(): + for command, result in iteritems(db_results): results[command].append(result) return results diff --git a/nydus/db/routers/keyvalue.py b/nydus/db/routers/keyvalue.py index af9cd69..55ea12e 100644 --- a/nydus/db/routers/keyvalue.py +++ b/nydus/db/routers/keyvalue.py @@ -10,6 +10,7 @@ from nydus.contrib.ketama import Ketama from nydus.db.routers import BaseRouter, RoundRobinRouter, routing_params +from nydus import compat __all__ = ('ConsistentHashingRouter', 'PartitionRouter') @@ -87,4 +88,4 @@ def _route(self, attr, args, kwargs, **fkwargs): """ key = get_key(args, kwargs) - return [crc32(str(key)) % len(self.cluster)] + return [crc32(str(key) if compat.PY2 else key.encode('utf-8')) % len(self.cluster)] diff --git a/nydus/utils.py b/nydus/utils.py index ff0ba9e..0cb99f0 100644 --- a/nydus/utils.py +++ b/nydus/utils.py @@ -1,7 +1,9 @@ from collections import defaultdict -from Queue import Queue, Empty + from threading import Thread +from nydus.compat import Queue, Empty, iteritems + # import_string comes form Werkzeug # http://werkzeug.pocoo.org @@ -31,7 +33,7 @@ def import_string(import_name, silent=False): def apply_defaults(host, defaults): - for key, value in defaults.iteritems(): + for key, value in iteritems(defaults): if key not in host: host[key] = value return host @@ -94,6 +96,6 @@ def join(self): results = defaultdict(list) for worker in self.workers: worker.join() - for k, v in worker.results.iteritems(): + for k, v in iteritems(worker.results): results[k].extend(v) return results From 59bd807c0487f72d2989ed04d0494dbd94ea8897 Mon Sep 17 00:00:00 2001 From: Florent Messa Date: Wed, 29 Jul 2015 15:18:17 +0200 Subject: [PATCH 02/10] Add iterkeys and itervalues to compat --- nydus/compat.py | 12 ++++++++++++ nydus/db/base.py | 4 ++-- nydus/db/map.py | 4 ++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/nydus/compat.py b/nydus/compat.py index cea76b3..e1bef66 100644 --- a/nydus/compat.py +++ b/nydus/compat.py @@ -27,6 +27,18 @@ if PY3: def iteritems(d, **kw): return iter(d.items(**kw)) + + def itervalues(d, **kw): + return iter(d.values(**kw)) + + def iterkeys(d, **kw): + return iter(d.keys(**kw)) else: def iteritems(d, **kw): return iter(d.iteritems(**kw)) + + def iterkeys(d, **kw): + return iter(d.iterkeys(**kw)) + + def itervalues(d, **kw): + return iter(d.itervalues(**kw)) diff --git a/nydus/db/base.py b/nydus/db/base.py index 371e07a..c2b74b9 100644 --- a/nydus/db/base.py +++ b/nydus/db/base.py @@ -11,7 +11,7 @@ from nydus.db.map import DistributedContextManager from nydus.db.routers import BaseRouter, routing_params from nydus.utils import apply_defaults -from nydus.compat import iteritems +from nydus.compat import iteritems, iterkeys __all__ = ('LazyConnectionHandler', 'BaseCluster') @@ -62,7 +62,7 @@ def __getattr__(self, name): return CallProxy(self, name) def __iter__(self): - for name in self.hosts.iterkeys(): + for name in iterkeys(self.hosts): yield name def install_router(self, router): diff --git a/nydus/db/map.py b/nydus/db/map.py index 0e71b51..068ca0b 100644 --- a/nydus/db/map.py +++ b/nydus/db/map.py @@ -11,7 +11,7 @@ from nydus.utils import ThreadPool from nydus.db.exceptions import CommandError from nydus.db.promise import EventualCommand, change_resolution -from nydus.compat import iteritems +from nydus.compat import iteritems, itervalues class BaseDistributedConnection(object): @@ -60,7 +60,7 @@ def get_pool(self, commands): def resolve(self): pending_commands = self._build_pending_commands() - num_commands = sum(len(v) for v in pending_commands.itervalues()) + num_commands = sum(len(v) for v in itervalues(pending_commands)) # Don't bother with the pooling if we only need to do one operation on a single machine if num_commands == 1: db_num, (command,) = pending_commands.items()[0] From cda557dd9906b60fe8ca520ac39743ec7513b235 Mon Sep 17 00:00:00 2001 From: Florent Messa Date: Wed, 29 Jul 2015 15:28:41 +0200 Subject: [PATCH 03/10] Add xrange and __unicode__ compatibility --- nydus/compat.py | 21 ++++++++++++++++++++- nydus/contrib/ketama.py | 2 ++ nydus/db/backends/__init__.py | 2 +- nydus/db/base.py | 2 +- nydus/db/promise.py | 16 +++++++++------- nydus/db/routers/__init__.py | 2 +- nydus/db/routers/base.py | 8 +++++--- nydus/utils.py | 2 +- 8 files changed, 40 insertions(+), 15 deletions(-) diff --git a/nydus/compat.py b/nydus/compat.py index e1bef66..636cede 100644 --- a/nydus/compat.py +++ b/nydus/compat.py @@ -13,7 +13,7 @@ if PY3: string_types = str, else: - string_types = basestring, + string_types = basestring, # noqa try: @@ -42,3 +42,22 @@ def iterkeys(d, **kw): def itervalues(d, **kw): return iter(d.itervalues(**kw)) + + +try: + xrange +except NameError: + xrange = range + + +def python_2_unicode_compatible(klass): + """ + A decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if PY2: + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass diff --git a/nydus/contrib/ketama.py b/nydus/contrib/ketama.py index 004b078..145f83a 100644 --- a/nydus/contrib/ketama.py +++ b/nydus/contrib/ketama.py @@ -10,6 +10,8 @@ import math from bisect import bisect +from nydus.compat import xrange + __author__ = "Andrey Nikishaev" __email__ = "creotiv@gmail.com" diff --git a/nydus/db/backends/__init__.py b/nydus/db/backends/__init__.py index 9300632..3fe189a 100644 --- a/nydus/db/backends/__init__.py +++ b/nydus/db/backends/__init__.py @@ -6,4 +6,4 @@ :license: Apache License 2.0, see LICENSE for more details. """ -from .base import BaseConnection, BasePipeline +from .base import BaseConnection, BasePipeline # noqa diff --git a/nydus/db/base.py b/nydus/db/base.py index c2b74b9..36ebdbe 100644 --- a/nydus/db/base.py +++ b/nydus/db/base.py @@ -11,7 +11,7 @@ from nydus.db.map import DistributedContextManager from nydus.db.routers import BaseRouter, routing_params from nydus.utils import apply_defaults -from nydus.compat import iteritems, iterkeys +from nydus.compat import iteritems, iterkeys, xrange __all__ = ('LazyConnectionHandler', 'BaseCluster') diff --git a/nydus/db/promise.py b/nydus/db/promise.py index 71c3916..36d4ee9 100644 --- a/nydus/db/promise.py +++ b/nydus/db/promise.py @@ -7,6 +7,8 @@ """ from nydus.db.exceptions import CommandError +from nydus.compat import python_2_unicode_compatible, PY2 + from functools import wraps @@ -35,6 +37,7 @@ def change_resolution(command, value): command._EventualCommand__resolved = True +@python_2_unicode_compatible class EventualCommand(object): # introspection support: __members__ = property(lambda self: self.__dir__()) @@ -69,11 +72,6 @@ def __str__(self): return str(self.__wrapped) return repr(self) - def __unicode__(self): - if self.__resolved: - return unicode(self.__wrapped) - return unicode(repr(self)) - def __getattr__(self, name): return getattr(self.__wrapped, name) @@ -127,7 +125,11 @@ def __instancecheck__(self, cls): __ne__ = lambda x, o: x.__wrapped != o __gt__ = lambda x, o: x.__wrapped > o __ge__ = lambda x, o: x.__wrapped >= o - __cmp__ = lambda x, o: cmp(x.__wrapped, o) + + if PY2: + __cmp__ = lambda x, o: cmp(x.__wrapped, o) + __long__ = lambda x: long(x.__wrapped) + # attributes are currently not callable # __call__ = lambda x, *a, **kw: x.__wrapped(*a, **kw) __nonzero__ = lambda x: bool(x.__wrapped) @@ -156,7 +158,7 @@ def __instancecheck__(self, cls): __invert__ = lambda x: ~(x.__wrapped) __complex__ = lambda x: complex(x.__wrapped) __int__ = lambda x: int(x.__wrapped) - __long__ = lambda x: long(x.__wrapped) + __float__ = lambda x: float(x.__wrapped) __oct__ = lambda x: oct(x.__wrapped) __hex__ = lambda x: hex(x.__wrapped) diff --git a/nydus/db/routers/__init__.py b/nydus/db/routers/__init__.py index 747c57e..0b62709 100644 --- a/nydus/db/routers/__init__.py +++ b/nydus/db/routers/__init__.py @@ -6,4 +6,4 @@ :license: Apache License 2.0, see LICENSE for more details. """ -from .base import * +from .base import * # noqa diff --git a/nydus/db/routers/base.py b/nydus/db/routers/base.py index 5b677c4..4d797c7 100644 --- a/nydus/db/routers/base.py +++ b/nydus/db/routers/base.py @@ -5,14 +5,16 @@ :copyright: (c) 2011-2012 DISQUS. :license: Apache License 2.0, see LICENSE for more details. """ - -__all__ = ('BaseRouter', 'RoundRobinRouter', 'routing_params') - import time from functools import wraps from itertools import cycle +from nydus.compat import xrange + + +__all__ = ('BaseRouter', 'RoundRobinRouter', 'routing_params') + def routing_params(func): @wraps(func) diff --git a/nydus/utils.py b/nydus/utils.py index 0cb99f0..a1a4813 100644 --- a/nydus/utils.py +++ b/nydus/utils.py @@ -2,7 +2,7 @@ from threading import Thread -from nydus.compat import Queue, Empty, iteritems +from nydus.compat import Queue, Empty, iteritems, xrange # import_string comes form Werkzeug From 7c042c9b828d29b8f50e064b65913044b52e8e98 Mon Sep 17 00:00:00 2001 From: Florent Messa Date: Wed, 29 Jul 2015 15:33:01 +0200 Subject: [PATCH 04/10] Fix xrange import --- nydus/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nydus/compat.py b/nydus/compat.py index 636cede..a01fb45 100644 --- a/nydus/compat.py +++ b/nydus/compat.py @@ -45,7 +45,7 @@ def itervalues(d, **kw): try: - xrange + xrange = xrange except NameError: xrange = range From 1ed06f45f154519c5ca53dc06c1b75371ab2bae6 Mon Sep 17 00:00:00 2001 From: Florent Messa Date: Thu, 30 Jul 2015 10:01:02 +0200 Subject: [PATCH 05/10] Skip pycassa tests on py3 --- tests/nydus/db/backends/pycassa/tests.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/nydus/db/backends/pycassa/tests.py b/tests/nydus/db/backends/pycassa/tests.py index 93405fe..e8d665d 100644 --- a/tests/nydus/db/backends/pycassa/tests.py +++ b/tests/nydus/db/backends/pycassa/tests.py @@ -1,5 +1,12 @@ from __future__ import absolute_import +import unittest + +try: + import pycassa # noqa +except ImportError: + raise unittest.SkipTest("Skip pycassa tests as it's not compatible with py3") + from nydus.db import create_cluster from nydus.db.backends.pycassa import Pycassa, PycassaCluster from nydus.testutils import BaseTest, fixture @@ -16,7 +23,7 @@ def cluster(self): }) def test_is_pycassa_cluster(self): - self.assertEquals(type(self.cluster), PycassaCluster) + self.assertEqual(type(self.cluster), PycassaCluster) class PycassClusterTest(BaseTest): @@ -28,10 +35,10 @@ def cluster(self): ) def test_has_one_connection(self): - self.assertEquals(len(self.cluster), 1) + self.assertEqual(len(self.cluster), 1) def test_backend_is_pycassa(self): - self.assertEquals(type(self.cluster[0]), Pycassa) + self.assertEqual(type(self.cluster[0]), Pycassa) class PycassaTest(BaseTest): From 56dd22b4c72ac64faadd3b85d64a8ca7c42c789f Mon Sep 17 00:00:00 2001 From: Florent Messa Date: Thu, 30 Jul 2015 10:01:10 +0200 Subject: [PATCH 06/10] Add more compatibility * Rename assertEquals to assertEqual * Rename assertNotEquals to assertNotEqual * Add httplib and next in compat * Fix map use cases which does not return a list anymore in py3 --- nydus/compat.py | 14 +++ nydus/contrib/ketama.py | 4 +- nydus/db/backends/riak.py | 2 +- nydus/db/base.py | 4 +- nydus/db/map.py | 2 +- nydus/db/promise.py | 5 +- nydus/db/routers/base.py | 4 +- nydus/db/routers/keyvalue.py | 4 +- nydus/testutils.py | 5 +- nydus/utils.py | 4 +- setup.py | 6 +- tests/nydus/db/backends/memcache/tests.py | 128 +++++++++++----------- tests/nydus/db/backends/redis/tests.py | 34 +++--- tests/nydus/db/backends/riak/tests.py | 26 +++-- tests/nydus/db/connections/tests.py | 86 +++++++-------- tests/nydus/db/routers/tests.py | 17 +-- 16 files changed, 186 insertions(+), 159 deletions(-) diff --git a/nydus/compat.py b/nydus/compat.py index a01fb45..9347cba 100644 --- a/nydus/compat.py +++ b/nydus/compat.py @@ -16,6 +16,20 @@ string_types = basestring, # noqa +try: + import httplib +except ImportError: + from http import client as httplib # noqa + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + try: # Python 2 from itertools import izip diff --git a/nydus/contrib/ketama.py b/nydus/contrib/ketama.py index 145f83a..da37344 100644 --- a/nydus/contrib/ketama.py +++ b/nydus/contrib/ketama.py @@ -4,7 +4,7 @@ Rewrited from the original source: http://www.audioscrobbler.net/development/ketama/ """ -from __future__ import print_function +from __future__ import print_function, unicode_literals import hashlib import math @@ -95,7 +95,7 @@ def _hashi(self, b_key, fn): b_key[fn(0)]) def _md5_digest(self, key): - return map(ord, hashlib.md5(key).digest()) + return list(map(ord, '%s' % hashlib.md5(key.encode('utf-8')).digest())) def remove_node(self, node): """ diff --git a/nydus/db/backends/riak.py b/nydus/db/backends/riak.py index aa0f196..eeabf5d 100644 --- a/nydus/db/backends/riak.py +++ b/nydus/db/backends/riak.py @@ -9,11 +9,11 @@ from __future__ import absolute_import import socket -import httplib from riak import RiakClient, RiakError from nydus.db.backends import BaseConnection +from nydus.compat import httplib class Riak(BaseConnection): diff --git a/nydus/db/base.py b/nydus/db/base.py index 36ebdbe..3e90060 100644 --- a/nydus/db/base.py +++ b/nydus/db/base.py @@ -11,7 +11,7 @@ from nydus.db.map import DistributedContextManager from nydus.db.routers import BaseRouter, routing_params from nydus.utils import apply_defaults -from nydus.compat import iteritems, iterkeys, xrange +from nydus.compat import iteritems, iterkeys, xrange, itervalues __all__ = ('LazyConnectionHandler', 'BaseCluster') @@ -97,7 +97,7 @@ def execute(self, path, args, kwargs): def disconnect(self): """Disconnects all connections in cluster""" - for connection in self.hosts.itervalues(): + for connection in itervalues(self.hosts): connection.disconnect() def get_conn(self, *args, **kwargs): diff --git a/nydus/db/map.py b/nydus/db/map.py index 068ca0b..7dcf576 100644 --- a/nydus/db/map.py +++ b/nydus/db/map.py @@ -63,7 +63,7 @@ def resolve(self): num_commands = sum(len(v) for v in itervalues(pending_commands)) # Don't bother with the pooling if we only need to do one operation on a single machine if num_commands == 1: - db_num, (command,) = pending_commands.items()[0] + db_num, (command,) = list(pending_commands.items())[0] self._commands = [command.resolve(self._cluster[db_num])] elif num_commands > 1: diff --git a/nydus/db/promise.py b/nydus/db/promise.py index 36d4ee9..8e6d9ea 100644 --- a/nydus/db/promise.py +++ b/nydus/db/promise.py @@ -133,7 +133,10 @@ def __instancecheck__(self, cls): # attributes are currently not callable # __call__ = lambda x, *a, **kw: x.__wrapped(*a, **kw) __nonzero__ = lambda x: bool(x.__wrapped) - __len__ = lambda x: len(x.__wrapped) + + def __len__(self): + return len(self.__wrapped) + __getitem__ = lambda x, i: x.__wrapped[i] __iter__ = lambda x: iter(x.__wrapped) __contains__ = lambda x, i: i in x.__wrapped diff --git a/nydus/db/routers/base.py b/nydus/db/routers/base.py index 4d797c7..b6e3626 100644 --- a/nydus/db/routers/base.py +++ b/nydus/db/routers/base.py @@ -10,7 +10,7 @@ from functools import wraps from itertools import cycle -from nydus.compat import xrange +from nydus.compat import xrange, next __all__ = ('BaseRouter', 'RoundRobinRouter', 'routing_params') @@ -224,7 +224,7 @@ def _route(self, attr, args, kwargs, **fkwargs): now = time.time() for i in xrange(len(self.cluster)): - db_num = self._hosts_cycler.next() + db_num = next(self._hosts_cycler) marked_down_at = self._down_connections.get(db_num, False) diff --git a/nydus/db/routers/keyvalue.py b/nydus/db/routers/keyvalue.py index 55ea12e..af45bf3 100644 --- a/nydus/db/routers/keyvalue.py +++ b/nydus/db/routers/keyvalue.py @@ -52,7 +52,7 @@ def mark_connection_up(self, db_num): @routing_params def _setup_router(self, args, kwargs, **fkwargs): - self._db_num_id_map = dict([(db_num, host.identifier) for db_num, host in self.cluster.hosts.iteritems()]) + self._db_num_id_map = dict([(db_num, host.identifier) for db_num, host in compat.iteritems(self.cluster.hosts)]) self._hash = Ketama(self._db_num_id_map.values()) return True @@ -76,7 +76,7 @@ def _route(self, attr, args, kwargs, **fkwargs): if not found and len(self._down_connections) > 0: raise self.HostListExhausted() - return [i for i, h in self.cluster.hosts.iteritems() + return [i for i, h in compat.iteritems(self.cluster.hosts) if h.identifier == found] diff --git a/nydus/testutils.py b/nydus/testutils.py index 03d0d3c..6753730 100644 --- a/nydus/testutils.py +++ b/nydus/testutils.py @@ -1,4 +1,7 @@ -import unittest2 +try: + import unittest2 +except ImportError: + import unittest as unittest2 # noqa NOTSET = object() diff --git a/nydus/utils.py b/nydus/utils.py index a1a4813..80aa1c5 100644 --- a/nydus/utils.py +++ b/nydus/utils.py @@ -2,7 +2,7 @@ from threading import Thread -from nydus.compat import Queue, Empty, iteritems, xrange +from nydus.compat import Queue, Empty, iteritems, xrange, next # import_string comes form Werkzeug @@ -41,7 +41,7 @@ def apply_defaults(host, defaults): def peek(value): generator = iter(value) - prev = generator.next() + prev = next(generator) for item in generator: yield prev, item prev = item diff --git a/setup.py b/setup.py index 2012299..bc98d93 100644 --- a/setup.py +++ b/setup.py @@ -13,10 +13,11 @@ else: setup_requires = [] +PY3 = sys.version_info[0] == 3 + tests_require = [ 'mock', 'nose', - 'pycassa', 'pylibmc', 'redis', 'riak', @@ -24,6 +25,9 @@ 'unittest2', ] +if not PY3: + tests_require += ['pycassa', ] + dependency_links = [ 'https://github.com/andyet/thoonk.py/tarball/master#egg=thoonk', ] diff --git a/tests/nydus/db/backends/memcache/tests.py b/tests/nydus/db/backends/memcache/tests.py index 3aa34de..0ed71af 100644 --- a/tests/nydus/db/backends/memcache/tests.py +++ b/tests/nydus/db/backends/memcache/tests.py @@ -2,8 +2,8 @@ from nydus.db import create_cluster from nydus.db.base import BaseCluster -from nydus.db.backends.memcache import Memcache, regroup_commands, grouped_args_for_command, \ - can_group_commands +from nydus.db.backends.memcache import (Memcache, regroup_commands, grouped_args_for_command, + can_group_commands) from nydus.db.promise import EventualCommand from nydus.testutils import BaseTest, fixture @@ -15,55 +15,55 @@ class CanGroupCommandsTest(BaseTest): def test_groupable_set_commands(self): command = EventualCommand('set', ['foo', 1]) other = EventualCommand('set', ['bar', 2]) - self.assertEquals(can_group_commands(command, other), True) + self.assertEqual(can_group_commands(command, other), True) def test_ungroupable_set_commands(self): command = EventualCommand('set', ['foo', 1], {'timeout': 1}) other = EventualCommand('set', ['bar', 2], {'timeout': 2}) - self.assertEquals(can_group_commands(command, other), False) + self.assertEqual(can_group_commands(command, other), False) def test_groupable_get_commands(self): command = EventualCommand('get', ['foo']) other = EventualCommand('get', ['bar']) - self.assertEquals(can_group_commands(command, other), True) + self.assertEqual(can_group_commands(command, other), True) def test_ungroupable_get_commands(self): command = EventualCommand('get', ['foo'], {'timeout': 1}) other = EventualCommand('get', ['bar'], {'timeout': 2}) - self.assertEquals(can_group_commands(command, other), False) + self.assertEqual(can_group_commands(command, other), False) def test_groupable_delete_commands(self): command = EventualCommand('delete', ['foo']) other = EventualCommand('delete', ['bar']) - self.assertEquals(can_group_commands(command, other), True) + self.assertEqual(can_group_commands(command, other), True) def test_ungroupable_delete_commands(self): command = EventualCommand('delete', ['foo'], {'timeout': 1}) other = EventualCommand('delete', ['bar'], {'timeout': 2}) - self.assertEquals(can_group_commands(command, other), False) + self.assertEqual(can_group_commands(command, other), False) class GroupedArgsForCommandTest(BaseTest): def test_set_excludes_first_two_args(self): command = EventualCommand('set', ['foo', 1, 'biz']) result = grouped_args_for_command(command) - self.assertEquals(result, ['biz']) + self.assertEqual(result, ['biz']) def test_get_excludes_first_arg(self): command = EventualCommand('get', ['foo', 1]) result = grouped_args_for_command(command) - self.assertEquals(result, [1]) + self.assertEqual(result, [1]) def test_delete_excludes_first_arg(self): command = EventualCommand('delete', ['foo', 1]) result = grouped_args_for_command(command) - self.assertEquals(result, [1]) + self.assertEqual(result, [1]) class RegroupCommandsTest(BaseTest): def get_grouped_results(self, commands, num_expected): grouped = regroup_commands(commands) - self.assertEquals(len(grouped), num_expected, grouped) + self.assertEqual(len(grouped), num_expected, grouped) return grouped def test_set_basic(self): @@ -76,23 +76,23 @@ def test_set_basic(self): items = self.get_grouped_results(commands, 2) new_command, grouped_commands = items[0] - self.assertEquals(len(grouped_commands), 2) - self.assertEquals(grouped_commands, commands[0:2]) - self.assertEquals(new_command.get_name(), 'set_multi') - self.assertEquals(new_command.get_args(), ({ + self.assertEqual(len(grouped_commands), 2) + self.assertEqual(grouped_commands, commands[0:2]) + self.assertEqual(new_command.get_name(), 'set_multi') + self.assertEqual(new_command.get_args(), ({ 'foo': 1, 'bar': 2, },)) - self.assertEquals(new_command.get_kwargs(), { + self.assertEqual(new_command.get_kwargs(), { 'timeout': 1, }) new_command, grouped_commands = items[1] - self.assertEquals(len(grouped_commands), 1) - self.assertEquals(grouped_commands, commands[2:3]) - self.assertEquals(new_command.get_name(), 'set') - self.assertEquals(new_command.get_args(), ['baz', 3]) - self.assertEquals(new_command.get_kwargs(), { + self.assertEqual(len(grouped_commands), 1) + self.assertEqual(grouped_commands, commands[2:3]) + self.assertEqual(new_command.get_name(), 'set') + self.assertEqual(new_command.get_args(), ['baz', 3]) + self.assertEqual(new_command.get_kwargs(), { 'timeout': 2, }) @@ -105,20 +105,20 @@ def test_get_basic(self): items = self.get_grouped_results(commands, 2) new_command, grouped_commands = items[0] - self.assertEquals(len(grouped_commands), 2) - self.assertEquals(grouped_commands, commands[0:2]) - self.assertEquals(new_command.get_name(), 'get_multi') - self.assertEquals(new_command.get_args(), (['foo', 'bar'],)) - self.assertEquals(new_command.get_kwargs(), { + self.assertEqual(len(grouped_commands), 2) + self.assertEqual(grouped_commands, commands[0:2]) + self.assertEqual(new_command.get_name(), 'get_multi') + self.assertEqual(new_command.get_args(), (['foo', 'bar'],)) + self.assertEqual(new_command.get_kwargs(), { 'key_prefix': 1, }) new_command, grouped_commands = items[1] - self.assertEquals(len(grouped_commands), 1) - self.assertEquals(grouped_commands, commands[2:3]) - self.assertEquals(new_command.get_name(), 'get') - self.assertEquals(new_command.get_args(), ['baz']) - self.assertEquals(new_command.get_kwargs(), { + self.assertEqual(len(grouped_commands), 1) + self.assertEqual(grouped_commands, commands[2:3]) + self.assertEqual(new_command.get_name(), 'get') + self.assertEqual(new_command.get_args(), ['baz']) + self.assertEqual(new_command.get_kwargs(), { 'key_prefix': 2, }) @@ -131,20 +131,20 @@ def test_delete_basic(self): items = self.get_grouped_results(commands, 2) new_command, grouped_commands = items[0] - self.assertEquals(len(grouped_commands), 2) - self.assertEquals(grouped_commands, commands[0:2]) - self.assertEquals(new_command.get_name(), 'delete_multi') - self.assertEquals(new_command.get_args(), (['foo', 'bar'],)) - self.assertEquals(new_command.get_kwargs(), { + self.assertEqual(len(grouped_commands), 2) + self.assertEqual(grouped_commands, commands[0:2]) + self.assertEqual(new_command.get_name(), 'delete_multi') + self.assertEqual(new_command.get_args(), (['foo', 'bar'],)) + self.assertEqual(new_command.get_kwargs(), { 'key_prefix': 1, }) new_command, grouped_commands = items[1] - self.assertEquals(len(grouped_commands), 1) - self.assertEquals(grouped_commands, commands[2:3]) - self.assertEquals(new_command.get_name(), 'delete') - self.assertEquals(new_command.get_args(), ['baz']) - self.assertEquals(new_command.get_kwargs(), { + self.assertEqual(len(grouped_commands), 1) + self.assertEqual(grouped_commands, commands[2:3]) + self.assertEqual(new_command.get_name(), 'delete') + self.assertEqual(new_command.get_args(), ['baz']) + self.assertEqual(new_command.get_kwargs(), { 'key_prefix': 2, }) @@ -159,30 +159,30 @@ def test_mixed_commands(self): items = self.get_grouped_results(commands, 3) new_command, grouped_commands = items[0] - self.assertEquals(len(grouped_commands), 1) - self.assertEquals(grouped_commands, commands[2:3]) - self.assertEquals(new_command.get_name(), 'get') - self.assertEquals(new_command.get_args(), ['foo']) - self.assertEquals(new_command.get_kwargs(), {}) + self.assertEqual(len(grouped_commands), 1) + self.assertEqual(grouped_commands, commands[2:3]) + self.assertEqual(new_command.get_name(), 'get') + self.assertEqual(new_command.get_args(), ['foo']) + self.assertEqual(new_command.get_kwargs(), {}) new_command, grouped_commands = items[1] - self.assertEquals(len(grouped_commands), 2) - self.assertEquals(grouped_commands, commands[0:2]) - self.assertEquals(new_command.get_name(), 'set_multi') - self.assertEquals(new_command.get_args(), ({ + self.assertEqual(len(grouped_commands), 2) + self.assertEqual(grouped_commands, commands[0:2]) + self.assertEqual(new_command.get_name(), 'set_multi') + self.assertEqual(new_command.get_args(), ({ 'bar': 1, 'baz': 2, },)) - self.assertEquals(new_command.get_kwargs(), { + self.assertEqual(new_command.get_kwargs(), { 'timeout': 1, }) new_command, grouped_commands = items[2] - self.assertEquals(len(grouped_commands), 1) - self.assertEquals(grouped_commands, commands[2:3]) - self.assertEquals(new_command.get_name(), 'get') - self.assertEquals(new_command.get_args(), ['bar']) - self.assertEquals(new_command.get_kwargs(), {}) + self.assertEqual(len(grouped_commands), 1) + self.assertEqual(grouped_commands, commands[2:3]) + self.assertEqual(new_command.get_name(), 'get') + self.assertEqual(new_command.get_args(), ['bar']) + self.assertEqual(new_command.get_kwargs(), {}) class MemcacheTest(BaseTest): @@ -192,17 +192,17 @@ def memcache(self): return Memcache(num=0) def test_provides_retryable_exceptions(self): - self.assertEquals(Memcache.retryable_exceptions, frozenset([pylibmc.Error])) + self.assertEqual(Memcache.retryable_exceptions, frozenset([pylibmc.Error])) def test_provides_identifier(self): - self.assertEquals(self.memcache.identifier, str(self.memcache.identifier)) + self.assertEqual(self.memcache.identifier, str(self.memcache.identifier)) @mock.patch('pylibmc.Client') def test_client_instantiates_with_kwargs(self, Client): client = Memcache(num=0) client.connect() - self.assertEquals(Client.call_count, 1) + self.assertEqual(Client.call_count, 1) Client.assert_any_call(['localhost:11211'], binary=True, behaviors=None) @mock.patch('pylibmc.Client.get') @@ -213,7 +213,7 @@ def test_with_cluster(self, get): ) result = p.get('MemcacheTest_with_cluster') get.assert_called_once_with('MemcacheTest_with_cluster') - self.assertEquals(result, get.return_value) + self.assertEqual(result, get.return_value) @mock.patch('pylibmc.Client') def test_pipeline_behavior(self, Client): @@ -252,6 +252,6 @@ def test_pipeline_integration(self): conn.get('c') results = conn.get_results() - self.assertEquals(len(results), 6, results) - self.assertEquals(results[0:3], [None, None, None]) - self.assertEquals(results[3:6], [1, 2, 3]) + self.assertEqual(len(results), 6, results) + self.assertEqual(results[0:3], [None, None, None]) + self.assertEqual(results[3:6], [1, 2, 3]) diff --git a/tests/nydus/db/backends/redis/tests.py b/tests/nydus/db/backends/redis/tests.py index 2713a64..72167ef 100644 --- a/tests/nydus/db/backends/redis/tests.py +++ b/tests/nydus/db/backends/redis/tests.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals from nydus.db import create_cluster from nydus.db.base import BaseCluster @@ -30,12 +30,12 @@ def test_pipelined_map(self): with self.cluster.map() as conn: [conn.set(c, i) for i, c in enumerate(chars)] res = [conn.get(c) for c in chars] - self.assertEqual(range(len(chars)), [int(r) for r in res]) + self.assertEqual(list(range(len(chars))), [int(r) for r in res]) def test_map_single_connection(self): with self.cluster.map() as conn: conn.set('a', '1') - self.assertEquals(self.cluster.get('a'), '1') + self.assertEqual(self.cluster.get('a').decode('utf-8'), '1') def test_no_proxy_without_call_on_map(self): with self.cluster.map() as conn: @@ -52,29 +52,29 @@ def setUp(self): self.redis.flushdb() def test_proxy(self): - self.assertEquals(self.redis.incr('RedisTest_proxy'), 1) + self.assertEqual(self.redis.incr('RedisTest_proxy'), 1) def test_with_cluster(self): p = BaseCluster( backend=Redis, hosts={0: {'db': 1}}, ) - self.assertEquals(p.incr('RedisTest_with_cluster'), 1) + self.assertEqual(p.incr('RedisTest_with_cluster'), 1) def test_provides_retryable_exceptions(self): - self.assertEquals(Redis.retryable_exceptions, frozenset([redis_.ConnectionError, redis_.InvalidResponse])) + self.assertEqual(Redis.retryable_exceptions, frozenset([redis_.ConnectionError, redis_.InvalidResponse])) def test_provides_identifier(self): - self.assertEquals(self.redis.identifier, str(self.redis.identifier)) + self.assertEqual(self.redis.identifier, str(self.redis.identifier)) @mock.patch('nydus.db.backends.redis.StrictRedis') def test_client_instantiates_with_kwargs(self, RedisClient): client = Redis(num=0) client.connect() - self.assertEquals(RedisClient.call_count, 1) + self.assertEqual(RedisClient.call_count, 1) RedisClient.assert_any_call(host='localhost', port=6379, db=0, socket_timeout=None, - password=None, unix_socket_path=None) + password=None, unix_socket_path=None) @mock.patch('nydus.db.backends.redis.StrictRedis') def test_map_does_pipeline(self, RedisClient): @@ -94,14 +94,14 @@ def test_map_does_pipeline(self, RedisClient): # ensure this was actually called through the pipeline self.assertFalse(RedisClient().set.called) - self.assertEquals(RedisClient().pipeline.call_count, 2) + self.assertEqual(RedisClient().pipeline.call_count, 2) RedisClient().pipeline.assert_called_with() - self.assertEquals(RedisClient().pipeline().set.call_count, 2) + self.assertEqual(RedisClient().pipeline().set.call_count, 2) RedisClient().pipeline().set.assert_any_call('a', 0) RedisClient().pipeline().set.assert_any_call('d', 1) - self.assertEquals(RedisClient().pipeline().execute.call_count, 2) + self.assertEqual(RedisClient().pipeline().execute.call_count, 2) RedisClient().pipeline().execute.assert_called_with() @mock.patch('nydus.db.backends.redis.StrictRedis') @@ -121,14 +121,14 @@ def test_map_only_runs_on_required_nodes(self, RedisClient): # ensure this was actually called through the pipeline self.assertFalse(RedisClient().set.called) - self.assertEquals(RedisClient().pipeline.call_count, 1) + self.assertEqual(RedisClient().pipeline.call_count, 1) RedisClient().pipeline.assert_called_with() - self.assertEquals(RedisClient().pipeline().set.call_count, 2) + self.assertEqual(RedisClient().pipeline().set.call_count, 2) RedisClient().pipeline().set.assert_any_call('a', 0) RedisClient().pipeline().set.assert_any_call('b', 1) - self.assertEquals(RedisClient().pipeline().execute.call_count, 1) + self.assertEqual(RedisClient().pipeline().execute.call_count, 1) RedisClient().pipeline().execute.assert_called_with() def test_normal_exceptions_dont_break_the_cluster(self): @@ -164,5 +164,5 @@ def test_custom_identifier_specified(self): redis = create_cluster(cluster_config) for idx in cluster_config['hosts'].keys(): - self.assertEquals(redis.hosts[idx].identifier, - cluster_config['hosts'][idx]['identifier']) + self.assertEqual(redis.hosts[idx].identifier, + cluster_config['hosts'][idx]['identifier']) diff --git a/tests/nydus/db/backends/riak/tests.py b/tests/nydus/db/backends/riak/tests.py index 847c9d2..2164e0b 100644 --- a/tests/nydus/db/backends/riak/tests.py +++ b/tests/nydus/db/backends/riak/tests.py @@ -1,9 +1,11 @@ from __future__ import absolute_import import mock -from httplib import HTTPException + from nydus.db.backends.riak import Riak from nydus.testutils import BaseTest +from nydus.compat import httplib + from riak import RiakClient, RiakError from socket import error as SocketError @@ -40,29 +42,29 @@ def test_init_properties(self): def test_identifier(self): expected_identifier = 'http://%(host)s:%(port)s/%(prefix)s' % self.conn.__dict__ - self.assertEquals(expected_identifier, self.conn.identifier) + self.assertEqual(expected_identifier, self.conn.identifier) def test_identifier_properties(self): expected_identifier = 'http://%(host)s:%(port)s/%(prefix)s' % self.modified_props - self.assertEquals(expected_identifier, self.modified_conn.identifier) + self.assertEqual(expected_identifier, self.modified_conn.identifier) @mock.patch('nydus.db.backends.riak.RiakClient') def test_connect_riakclient_options(self, _RiakClient): self.conn.connect() - _RiakClient.assert_called_with(host=self.conn.host, port=self.conn.port, prefix=self.conn.prefix, \ - mapred_prefix=self.conn.mapred_prefix, client_id=self.conn.client_id, \ - transport_options=self.conn.transport_options, transport_class=self.conn.transport_class, \ - solr_transport_class=self.conn.solr_transport_class) + _RiakClient.assert_called_with(host=self.conn.host, port=self.conn.port, prefix=self.conn.prefix, + mapred_prefix=self.conn.mapred_prefix, client_id=self.conn.client_id, + transport_options=self.conn.transport_options, transport_class=self.conn.transport_class, + solr_transport_class=self.conn.solr_transport_class) @mock.patch('nydus.db.backends.riak.RiakClient') def test_connect_riakclient_modified_options(self, _RiakClient): self.modified_conn.connect() - _RiakClient.assert_called_with(host=self.modified_conn.host, port=self.modified_conn.port, prefix=self.modified_conn.prefix, \ - mapred_prefix=self.modified_conn.mapred_prefix, client_id=self.modified_conn.client_id, \ - transport_options=self.modified_conn.transport_options, transport_class=self.modified_conn.transport_class, \ - solr_transport_class=self.modified_conn.solr_transport_class) + _RiakClient.assert_called_with(host=self.modified_conn.host, port=self.modified_conn.port, prefix=self.modified_conn.prefix, + mapred_prefix=self.modified_conn.mapred_prefix, client_id=self.modified_conn.client_id, + transport_options=self.modified_conn.transport_options, transport_class=self.modified_conn.transport_class, + solr_transport_class=self.modified_conn.solr_transport_class) def test_connect_returns_riakclient(self): client = self.conn.connect() @@ -70,4 +72,4 @@ def test_connect_returns_riakclient(self): self.assertIsInstance(client, RiakClient) def test_provides_retryable_exceptions(self): - self.assertItemsEqual([RiakError, HTTPException, SocketError], self.conn.retryable_exceptions) + self.assertItemsEqual([RiakError, httplib.HTTPException, SocketError], self.conn.retryable_exceptions) diff --git a/tests/nydus/db/connections/tests.py b/tests/nydus/db/connections/tests.py index 1735991..5be71ee 100644 --- a/tests/nydus/db/connections/tests.py +++ b/tests/nydus/db/connections/tests.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import mock @@ -51,7 +51,7 @@ def test_close_calls_disconnect(self, disconnect): @mock.patch('nydus.db.backends.base.BaseConnection.disconnect', mock.Mock(return_value=None)) def test_close_unsets_connection(self): self.connection.close() - self.assertEquals(self.connection._connection, None) + self.assertEqual(self.connection._connection, None) @mock.patch('nydus.db.backends.base.BaseConnection.disconnect') def test_close_propagates_noops_if_not_connected(self, disconnect): @@ -72,14 +72,14 @@ def test_connection_doesnt_reconnect_with_existing_connection(self, connect): @mock.patch('nydus.db.backends.base.BaseConnection.connect') def test_connection_returns_result_of_connect(self, connect): val = self.connection.connection - self.assertEquals(val, connect.return_value) + self.assertEqual(val, connect.return_value) def test_attrs_proxy(self): conn = mock.Mock() self.connection._connection = conn val = self.connection.foo(biz='baz') conn.foo.assert_called_once_with(biz='baz') - self.assertEquals(val, conn.foo.return_value) + self.assertEqual(val, conn.foo.return_value) class CreateConnectionTest(BaseTest): @@ -103,7 +103,7 @@ def test_creates_cluster(self): 0: {'resp': 'bar'}, } }) - self.assertEquals(len(c), 1) + self.assertEqual(len(c), 1) @mock.patch('nydus.db.base.create_connection') def test_does_create_connection_with_defaults(self, create_connection): @@ -123,14 +123,14 @@ def test_len_returns_num_backends(self): backend=BaseConnection, hosts={0: {}}, ) - self.assertEquals(len(p), 1) + self.assertEqual(len(p), 1) def test_proxy(self): p = BaseCluster( backend=DummyConnection, hosts={0: {'resp': 'bar'}}, ) - self.assertEquals(p.foo(), 'bar') + self.assertEqual(p.foo(), 'bar') def test_disconnect(self): c = mock.Mock() @@ -150,8 +150,8 @@ def test_with_split_router(self): 1: {'resp': 'bar'}, }, ) - self.assertEquals(p.foo(), 'foo') - self.assertEquals(p.foo('foo'), 'bar') + self.assertEqual(p.foo(), 'foo') + self.assertEqual(p.foo('foo'), 'bar') def test_default_routing_with_multiple_hosts(self): p = BaseCluster( @@ -161,8 +161,8 @@ def test_default_routing_with_multiple_hosts(self): 1: {'resp': 'bar'}, }, ) - self.assertEquals(p.foo(), ['foo', 'bar']) - self.assertEquals(p.foo('foo'), ['foo', 'bar']) + self.assertEqual(p.foo(), ['foo', 'bar']) + self.assertEqual(p.foo('foo'), ['foo', 'bar']) def test_get_conn_with_split_router(self): # test dummy router @@ -174,8 +174,8 @@ def test_get_conn_with_split_router(self): }, router=DummyRouter, ) - self.assertEquals(p.get_conn().num, 0) - self.assertEquals(p.get_conn('foo').num, 1) + self.assertEqual(p.get_conn().num, 0) + self.assertEqual(p.get_conn('foo').num, 1) def test_get_conn_default_routing_with_multiple_hosts(self): # test default routing behavior @@ -186,8 +186,8 @@ def test_get_conn_default_routing_with_multiple_hosts(self): 1: {'resp': 'bar'}, }, ) - self.assertEquals(map(lambda x: x.num, p.get_conn()), [0, 1]) - self.assertEquals(map(lambda x: x.num, p.get_conn('foo')), [0, 1]) + self.assertEqual(list(map(lambda x: x.num, p.get_conn())), [0, 1]) + self.assertEqual(list(map(lambda x: x.num, p.get_conn('foo'))), [0, 1]) class MapTest(BaseTest): @@ -207,21 +207,21 @@ def test_handles_single_routing_results(self): with self.cluster.map() as conn: foo = conn.foo() bar = conn.foo('foo') - self.assertEquals(foo, None) - self.assertEquals(bar, None) + self.assertEqual(foo, None) + self.assertEqual(bar, None) - self.assertEquals(bar, 'bar') - self.assertEquals(foo, 'foo') + self.assertEqual(bar, 'bar') + self.assertEqual(foo, 'foo') def test_handles_groups_of_results(self): with self.cluster.map() as conn: foo = conn.foo() bar = conn.foo('foo') - self.assertEquals(foo, None) - self.assertEquals(bar, None) + self.assertEqual(foo, None) + self.assertEqual(bar, None) - self.assertEquals(foo, ['foo', 'bar']) - self.assertEquals(bar, ['foo', 'bar']) + self.assertEqual(foo, ['foo', 'bar']) + self.assertEqual(bar, ['foo', 'bar']) class MapWithFailuresTest(BaseTest): @@ -241,21 +241,21 @@ def test_propagates_errors(self): with self.cluster.map() as conn: foo = conn.foo() bar = conn.foo('foo') - self.assertEquals(foo, None) - self.assertEquals(bar, None) + self.assertEqual(foo, None) + self.assertEqual(bar, None) def test_fail_silenlty(self): with self.cluster.map(fail_silently=True) as conn: foo = conn.foo() bar = conn.foo('foo') - self.assertEquals(foo, None) - self.assertEquals(bar, None) + self.assertEqual(foo, None) + self.assertEqual(bar, None) - self.assertEquals(len(conn.get_errors()), 1, conn.get_errors()) - self.assertEquals(type(conn.get_errors()[0][1]), ValueError) + self.assertEqual(len(conn.get_errors()), 1, conn.get_errors()) + self.assertEqual(type(conn.get_errors()[0][1]), ValueError) - self.assertEquals(foo, 'foo') - self.assertNotEquals(foo, 'bar') + self.assertEqual(foo, 'foo') + self.assertNotEqual(foo, 'bar') class FlakeyConnection(DummyConnection): @@ -306,13 +306,13 @@ def build_cluster(self, connection=FlakeyConnection, router=RetryableRouter): def test_returns_correctly(self): cluster = self.build_cluster(connection=DummyConnection) - self.assertEquals(cluster.foo(), 'bar') + self.assertEqual(cluster.foo(), 'bar') def test_retry_router_when_receives_error(self): cluster = self.build_cluster() cluster.foo() - self.assertEquals({'retry_for': 0}, cluster.router.kwargs_seen.pop()) + self.assertEqual({'retry_for': 0}, cluster.router.kwargs_seen.pop()) def test_protection_from_infinate_loops(self): cluster = self.build_cluster(connection=ScumbagConnection) @@ -325,48 +325,48 @@ def test_unevaled_repr(self): ec = EventualCommand('foo') ec('bar', baz='foo') - self.assertEquals(repr(ec), u"") + self.assertEqual(repr(ec), u"") def test_evaled_repr(self): ec = EventualCommand('foo') ec('bar', baz='foo') ec.resolve_as('biz') - self.assertEquals(repr(ec), u"'biz'") + self.assertEqual(repr(ec), u"'biz'") def test_coersion(self): ec = EventualCommand('foo')() ec.resolve_as('5') - self.assertEquals(int(ec), 5) + self.assertEqual(int(ec), 5) def test_nonzero(self): ec = EventualCommand('foo')() ec.resolve_as(None) - self.assertEquals(int(ec or 0), 0) + self.assertEqual(int(ec or 0), 0) def test_evaled_unicode(self): ec = EventualCommand('foo') ec.resolve_as('biz') - self.assertEquals(unicode(ec), u'biz') + self.assertEqual('%s' % ec, 'biz') def test_command_error_returns_as_error(self): ec = EventualCommand('foo') ec.resolve_as(CommandError([ValueError('test')])) - self.assertEquals(ec.is_error, True) + self.assertEqual(ec.is_error, True) def test_other_error_does_not_return_as_error(self): ec = EventualCommand('foo') ec.resolve_as(ValueError('test')) - self.assertEquals(ec.is_error, False) + self.assertEqual(ec.is_error, False) def test_isinstance_check(self): ec = EventualCommand('foo') ec.resolve_as(['foo', 'bar']) - self.assertEquals(isinstance(ec, list), True) + self.assertEqual(isinstance(ec, list), True) class ApplyDefaultsTest(BaseTest): @@ -374,7 +374,7 @@ def test_does_apply(self): host = {'port': 6379} defaults = {'host': 'localhost'} results = apply_defaults(host, defaults) - self.assertEquals(results, { + self.assertEqual(results, { 'port': 6379, 'host': 'localhost', }) @@ -383,6 +383,6 @@ def test_does_not_overwrite(self): host = {'port': 6379} defaults = {'port': 9000} results = apply_defaults(host, defaults) - self.assertEquals(results, { + self.assertEqual(results, { 'port': 6379, }) diff --git a/tests/nydus/db/routers/tests.py b/tests/nydus/db/routers/tests.py index 3224ec4..d87ef7e 100644 --- a/tests/nydus/db/routers/tests.py +++ b/tests/nydus/db/routers/tests.py @@ -11,6 +11,7 @@ from nydus.db.routers import BaseRouter, RoundRobinRouter from nydus.db.routers.keyvalue import ConsistentHashingRouter from nydus.testutils import BaseTest +from nydus.compat import xrange def _get_func(func): @@ -71,7 +72,7 @@ def test_offers_router_interface(self): self.assertIsNone(setupdefaults) def test_returns_whole_cluster_without_key(self): - self.assertEquals(self.hosts.keys(), self.get_dbs(attr='test')) + self.assertEqual(self.hosts.keys(), self.get_dbs(attr='test')) def test_get_dbs_handles_exception(self): with mock.patch.object(self.router, '_route') as _route: @@ -91,7 +92,7 @@ def test__pre_routing_returns_args_and_kwargs(self): self.assertEqual((('foo',), {}), self.router._pre_routing(attr='test', args=('foo',))) def test__route_returns_first_db_num(self): - self.assertEqual(self.cluster.hosts.keys()[0], self.router._route(attr='test', args=('foo',))[0]) + self.assertEqual(list(self.cluster.hosts.keys())[0], list(self.router._route(attr='test', args=('foo',)))[0]) def test__post_routing_returns_db_nums(self): db_nums = self.hosts.keys() @@ -196,7 +197,7 @@ def test__setup_router(self): self.assertIsInstance(self.router._hosts_cycler, Iterable) def test__route_cycles_through_keys(self): - db_nums = self.hosts.keys() * 2 + db_nums = list(self.hosts.keys()) * 2 results = [self.router._route(attr='test', args=('foo',))[0] for _ in db_nums] self.assertEqual(results, db_nums) @@ -237,14 +238,14 @@ def get_dbs(self, *args, **kwargs): return super(ConsistentHashingRouterTest, self).get_dbs(*args, **kwargs) def test_retry_gives_next_host_if_primary_is_offline(self): - self.assertEquals([2], self.get_dbs(args=('foo',))) - self.assertEquals([4], self.get_dbs(args=('foo',), retry_for=2)) + self.assertEqual([2], self.get_dbs(args=('foo',))) + self.assertEqual([4], self.get_dbs(args=('foo',), retry_for=2)) def test_retry_host_change_is_sticky(self): - self.assertEquals([2], self.get_dbs(args=('foo',))) - self.assertEquals([4], self.get_dbs(args=('foo',), retry_for=2)) + self.assertEqual([2], self.get_dbs(args=('foo',))) + self.assertEqual([4], self.get_dbs(args=('foo',), retry_for=2)) - self.assertEquals([4], self.get_dbs(args=('foo',))) + self.assertEqual([4], self.get_dbs(args=('foo',))) def test_raises_host_list_exhaused_if_no_host_can_be_found(self): # Kill the first 4 From 1832b79cad463bc4119d7754da6dc64a00e82734 Mon Sep 17 00:00:00 2001 From: Florent Messa Date: Thu, 30 Jul 2015 11:09:31 +0200 Subject: [PATCH 07/10] Fix repr on py2/py3 --- nydus/contrib/ketama.py | 15 +++++++++++++-- nydus/db/promise.py | 6 ++++-- tests/nydus/db/backends/thoonk/tests.py | 7 ++++++- tests/nydus/db/connections/tests.py | 11 +++++++++-- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/nydus/contrib/ketama.py b/nydus/contrib/ketama.py index da37344..f623748 100644 --- a/nydus/contrib/ketama.py +++ b/nydus/contrib/ketama.py @@ -10,7 +10,7 @@ import math from bisect import bisect -from nydus.compat import xrange +from nydus.compat import xrange, PY3 __author__ = "Andrey Nikishaev" @@ -95,7 +95,18 @@ def _hashi(self, b_key, fn): b_key[fn(0)]) def _md5_digest(self, key): - return list(map(ord, '%s' % hashlib.md5(key.encode('utf-8')).digest())) + if PY3: + key = key.encode('utf-8') + + m = hashlib.md5() + m.update(key) + + digest = m.digest() + + if PY3: + digest = digest.decode('latin-1') + + return list(map(ord, digest)) def remove_node(self, node): """ diff --git a/nydus/db/promise.py b/nydus/db/promise.py index 8e6d9ea..ce80105 100644 --- a/nydus/db/promise.py +++ b/nydus/db/promise.py @@ -5,6 +5,7 @@ :copyright: (c) 2011 DISQUS. :license: Apache License 2.0, see LICENSE for more details. """ +from __future__ import unicode_literals from nydus.db.exceptions import CommandError from nydus.compat import python_2_unicode_compatible, PY2 @@ -65,11 +66,12 @@ def __hash__(self): def __repr__(self): if self.__resolved: return repr(self.__wrapped) - return u'' % (self.__attr, self.__args, self.__kwargs) + return '' % (self.__attr, self.__args, self.__kwargs) def __str__(self): if self.__resolved: - return str(self.__wrapped) + return '%s' % self.__wrapped + return repr(self) def __getattr__(self, name): diff --git a/tests/nydus/db/backends/thoonk/tests.py b/tests/nydus/db/backends/thoonk/tests.py index 75c1008..0df88aa 100644 --- a/tests/nydus/db/backends/thoonk/tests.py +++ b/tests/nydus/db/backends/thoonk/tests.py @@ -1,8 +1,13 @@ from __future__ import absolute_import -import unittest2 +try: + import unittest2 +except ImportError: + import unittest as unittest2 + from nydus.db.backends.thoonk import Thoonk from nydus.db import create_cluster +from nydus.compat import xrange class ThoonkTest(unittest2.TestCase): diff --git a/tests/nydus/db/connections/tests.py b/tests/nydus/db/connections/tests.py index 5be71ee..cd716ed 100644 --- a/tests/nydus/db/connections/tests.py +++ b/tests/nydus/db/connections/tests.py @@ -11,6 +11,7 @@ from nydus.db.promise import EventualCommand from nydus.testutils import BaseTest, fixture from nydus.utils import apply_defaults +from nydus.compat import PY2 class DummyConnection(BaseConnection): @@ -325,14 +326,20 @@ def test_unevaled_repr(self): ec = EventualCommand('foo') ec('bar', baz='foo') - self.assertEqual(repr(ec), u"") + if PY2: + self.assertEqual(repr(ec), "") + else: + self.assertEqual(repr(ec), "") def test_evaled_repr(self): ec = EventualCommand('foo') ec('bar', baz='foo') ec.resolve_as('biz') - self.assertEqual(repr(ec), u"'biz'") + if PY2: + self.assertEqual(repr(ec), "u'biz'") + else: + self.assertEqual(repr(ec), "'biz'") def test_coersion(self): ec = EventualCommand('foo')() From 59d7df7c42f9871d83e8be242f7ac7fea11d4174 Mon Sep 17 00:00:00 2001 From: Florent Messa Date: Mon, 3 Aug 2015 11:45:38 +0200 Subject: [PATCH 08/10] Fix __len__ error when __wrapped is NoneType --- nydus/db/promise.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nydus/db/promise.py b/nydus/db/promise.py index ce80105..926a64d 100644 --- a/nydus/db/promise.py +++ b/nydus/db/promise.py @@ -137,7 +137,10 @@ def __instancecheck__(self, cls): __nonzero__ = lambda x: bool(x.__wrapped) def __len__(self): - return len(self.__wrapped) + if self.__wrapped is not None: + return len(self.__wrapped) + + return 0 __getitem__ = lambda x, i: x.__wrapped[i] __iter__ = lambda x: iter(x.__wrapped) From 95ed277d432784634f76e22f9edeb4f2aba08522 Mon Sep 17 00:00:00 2001 From: Florent Messa Date: Mon, 3 Aug 2015 11:46:00 +0200 Subject: [PATCH 09/10] Skip thoonk tests as the library is not compatible at all with py3 --- tests/nydus/db/backends/thoonk/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/nydus/db/backends/thoonk/tests.py b/tests/nydus/db/backends/thoonk/tests.py index 0df88aa..e8a0ba5 100644 --- a/tests/nydus/db/backends/thoonk/tests.py +++ b/tests/nydus/db/backends/thoonk/tests.py @@ -1,10 +1,16 @@ from __future__ import absolute_import +from nydus.compat import PY3 + try: import unittest2 except ImportError: import unittest as unittest2 +if PY3: + import unittest + raise unittest.SkipTest("Skip thoonk tests as it's not compatible with py3") + from nydus.db.backends.thoonk import Thoonk from nydus.db import create_cluster from nydus.compat import xrange From 04aed00280b90159de34e97b9fe55427a5f97c42 Mon Sep 17 00:00:00 2001 From: Florent Messa Date: Tue, 4 Aug 2015 10:47:07 +0200 Subject: [PATCH 10/10] Do not raise NotImplementedError in DummyConnection --- tests/nydus/db/connections/tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/nydus/db/connections/tests.py b/tests/nydus/db/connections/tests.py index cd716ed..0453a5f 100644 --- a/tests/nydus/db/connections/tests.py +++ b/tests/nydus/db/connections/tests.py @@ -19,6 +19,9 @@ def __init__(self, num, resp='foo', **kwargs): self.resp = resp super(DummyConnection, self).__init__(num, **kwargs) + def connect(self): + pass + def foo(self, *args, **kwargs): return self.resp