From bf0a5990720c679dfcb004426782cfd5607cec83 Mon Sep 17 00:00:00 2001 From: Dulmandakh Date: Tue, 20 Aug 2024 23:54:59 +0800 Subject: [PATCH 1/3] remove dbm, dyamodb, redis KVStores --- .github/workflows/test.yml | 2 +- docs/reference/settings.rst | 115 -------------------- sorl/thumbnail/conf/defaults.py | 19 ---- sorl/thumbnail/default.py | 11 +- sorl/thumbnail/kvstores/dbm_kvstore.py | 93 ---------------- sorl/thumbnail/kvstores/dynamodb_kvstore.py | 37 ------- sorl/thumbnail/kvstores/redis_kvstore.py | 36 ------ tests/settings/dbm.py | 4 - tests/settings/default.py | 2 - tests/settings/dynamodb.py | 8 -- tests/settings/redis.py | 4 - tests/thumbnail_tests/utils.py | 3 +- tox.ini | 9 +- 13 files changed, 8 insertions(+), 335 deletions(-) delete mode 100644 sorl/thumbnail/kvstores/dbm_kvstore.py delete mode 100644 sorl/thumbnail/kvstores/dynamodb_kvstore.py delete mode 100644 sorl/thumbnail/kvstores/redis_kvstore.py delete mode 100644 tests/settings/dbm.py delete mode 100644 tests/settings/dynamodb.py delete mode 100644 tests/settings/redis.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5b1355760..87c0b310d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - target: [pil, imagemagick, graphicsmagick, redis, wand, dbm] + target: [pil, imagemagick, graphicsmagick, wand] include: - python-version: '3.8' diff --git a/docs/reference/settings.rst b/docs/reference/settings.rst index 7d34c2f12..4094d9329 100644 --- a/docs/reference/settings.rst +++ b/docs/reference/settings.rst @@ -25,54 +25,6 @@ default one but just in case you would like to generate thumbnails filenames differently or need some special functionality you can override this and use your own implementation. - -``THUMBNAIL_KVSTORE`` -===================== - -- Default: ``'sorl.thumbnail.kvstores.cached_db_kvstore.KVStore'`` - -sorl-thumbnail needs a Key Value Store to :doc:`/operation`. -sorl-thumbnail ships with support for three Key Value Stores: - -Cached DB ---------- -``sorl.thumbnail.kvstores.cached_db_kvstore.KVStore``. This is the default and -preferred Key Value Store. - -Features -^^^^^^^^ -* Fast persistent storage -* First query uses database which is slow. Successive queries are cached and if - you use memcached this is very fast. -* Easy to transfer data between environments since the data is in the default - database. -* If you get the database and fast cache out of sync there could be problems. - -Redis ------ -``sorl.thumbnail.kvstores.redis_kvstore.KVStore``. It requires you to install a -Redis server as well as a `redis python client -`_. - -Features -^^^^^^^^ -* Fast persistent storage -* More dependencies -* Requires a little extra work to transfer data between environments - -Dbm ---- -``sorl.thumbnail.kvstores.dbm_kvstore.KVStore``. A simple Key Value Store has no -dependencies outside the standard Python library and uses the DBM modules to -store the data. - -Features -^^^^^^^^ -* No external dependencies, besides the standard library -* No extra components required, e.g., database or cache -* Specially indicated for local development environments - - ``THUMBNAIL_KEY_DBCOLUMN`` ========================== @@ -159,73 +111,6 @@ Only applicable for the convert Engine. The storage class to use for the generated thumbnails. -``THUMBNAIL_REDIS_URL`` -======================= - -The Redis database URL to connect as used by `redis-py `_ - -When specified, other ``THUMBNAIL_REDIS_*`` connection settings will be ignored. - - -``THUMBNAIL_REDIS_DB`` -====================== - -- Default: ``0`` - -The Redis database. Only applicable for the Redis Key Value Store - - -``THUMBNAIL_REDIS_PASSWORD`` -============================ - -- Default: ``''`` - -The password for Redis server. Only applicable for the Redis Key Value Store - - -``THUMBNAIL_REDIS_HOST`` -======================== - -- Default: ``'localhost'`` - -The host for Redis server. Only applicable for the Redis Key Value Store - - -``THUMBNAIL_REDIS_PORT`` -======================== - -- Default: ``6379`` - -The port for Redis server. Only applicable for the Redis Key Value Store - - -``THUMBNAIL_REDIS_TIMEOUT`` -=========================== - -- Default: ``3600 * 24 * 365 * 10`` - -Cache timeout for Redis Key Value Store in seconds. You should probably keep this -at maximum or ``None``. - - -``THUMBNAIL_DBM_FILE`` -====================== - -- Default: ``thumbnail_kvstore`` - -Filename of the DBM database. Depending on the DBM engine selected by your -Python installation, this will be used as a prefix because multiple files may be -created. This can be an absolute path. - - -``THUMBNAIL_DBM_MODE`` -====================== - -- Default: ``0x644`` - -Permission bits to use when creating new DBM files - - ``THUMBNAIL_CACHE_TIMEOUT`` =========================== diff --git a/sorl/thumbnail/conf/defaults.py b/sorl/thumbnail/conf/defaults.py index 5618622ed..00a74a889 100644 --- a/sorl/thumbnail/conf/defaults.py +++ b/sorl/thumbnail/conf/defaults.py @@ -6,12 +6,6 @@ # Backend THUMBNAIL_BACKEND = 'sorl.thumbnail.base.ThumbnailBackend' -# Key-value store, ships with: -# sorl.thumbnail.kvstores.cached_db_kvstore.KVStore -# sorl.thumbnail.kvstores.redis_kvstore.KVStore -# Redis requires some more work, see docs -THUMBNAIL_KVSTORE = 'sorl.thumbnail.kvstores.cached_db_kvstore.KVStore' - # Change this to something else for MSSQL THUMBNAIL_KEY_DBCOLUMN = 'key' @@ -33,19 +27,6 @@ # Storage for the generated thumbnails THUMBNAIL_STORAGE = settings.STORAGES['default']['BACKEND'] -# Redis settings -THUMBNAIL_REDIS_DB = 0 -THUMBNAIL_REDIS_PASSWORD = '' -THUMBNAIL_REDIS_HOST = 'localhost' -THUMBNAIL_REDIS_PORT = 6379 -THUMBNAIL_REDIS_UNIX_SOCKET_PATH = None -THUMBNAIL_REDIS_SSL = False -THUMBNAIL_REDIS_TIMEOUT = 3600 * 24 * 365 * 10 # 10 years - -# DBM settings -THUMBNAIL_DBM_FILE = "thumbnail_kvstore" -THUMBNAIL_DBM_MODE = 0o644 - # Cache timeout for ``cached_db`` store. You should probably keep this at # maximum or ``0`` if your caching backend can handle that as infinite. THUMBNAIL_CACHE_TIMEOUT = 3600 * 24 * 365 * 10 # 10 years diff --git a/sorl/thumbnail/default.py b/sorl/thumbnail/default.py index fbbccc40e..2706490fc 100644 --- a/sorl/thumbnail/default.py +++ b/sorl/thumbnail/default.py @@ -1,19 +1,14 @@ -from django.utils.functional import LazyObject +from django.utils.functional import LazyObject, SimpleLazyObject from sorl.thumbnail.conf import settings from sorl.thumbnail.helpers import get_module_class - +from sorl.thumbnail.kvstores.cached_db_kvstore import KVStore class Backend(LazyObject): def _setup(self): self._wrapped = get_module_class(settings.THUMBNAIL_BACKEND)() -class KVStore(LazyObject): - def _setup(self): - self._wrapped = get_module_class(settings.THUMBNAIL_KVSTORE)() - - class Engine(LazyObject): def _setup(self): self._wrapped = get_module_class(settings.THUMBNAIL_ENGINE)() @@ -25,6 +20,6 @@ def _setup(self): backend = Backend() -kvstore = KVStore() +kvstore = SimpleLazyObject(lambda: KVStore()) engine = Engine() storage = Storage() diff --git a/sorl/thumbnail/kvstores/dbm_kvstore.py b/sorl/thumbnail/kvstores/dbm_kvstore.py deleted file mode 100644 index bda4c80a2..000000000 --- a/sorl/thumbnail/kvstores/dbm_kvstore.py +++ /dev/null @@ -1,93 +0,0 @@ -import os - -from sorl.thumbnail.conf import settings -from sorl.thumbnail.kvstores.base import KVStoreBase - -try: - import anydbm as dbm -except KeyError: - import dbm -except ImportError: - # Python 3, hopefully - import dbm - -# -# OS filesystem locking primitives. TODO: Test Windows versions -# -if os.name == 'nt': - import msvcrt - - def lock(f, readonly): - msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) - - def unlock(f): - msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1) -else: - import fcntl - - def lock(f, readonly): - fcntl.lockf(f.fileno(), fcntl.LOCK_SH if readonly else fcntl.LOCK_EX) - - def unlock(f): - fcntl.lockf(f.fileno(), fcntl.LOCK_UN) - - -class DBMContext: - """ - A context manager to access the key-value store in a concurrent-safe manner. - """ - __slots__ = ('filename', 'mode', 'readonly', 'lockfile', 'db') - - def __init__(self, filename, mode, readonly): - self.filename = filename - self.mode = mode - self.readonly = readonly - self.lockfile = open(filename + ".lock", 'w+b') - - def __enter__(self): - lock(self.lockfile, self.readonly) - self.db = dbm.open(self.filename, 'c', self.mode) - return self.db - - def __exit__(self, exval, extype, tb): - self.db.close() - unlock(self.lockfile) - self.lockfile.close() - - -class KVStore(KVStoreBase): - # Please note that all the coding effort is devoted to provide correct - # semantics, not performance. Therefore, use this store only in development - # environments. - - def __init__(self): - super().__init__() - self.filename = settings.THUMBNAIL_DBM_FILE - self.mode = settings.THUMBNAIL_DBM_MODE - - def _cast_key(self, key): - return key if isinstance(key, bytes) else key.encode('utf-8') - - def _get_raw(self, key): - with DBMContext(self.filename, self.mode, True) as db: - try: - return db[self._cast_key(key)] - except KeyError: - return None - - def _set_raw(self, key, value): - with DBMContext(self.filename, self.mode, False) as db: - db[self._cast_key(key)] = value - - def _delete_raw(self, *keys): - with DBMContext(self.filename, self.mode, False) as db: - for key in keys: - try: - del db[self._cast_key(key)] - except KeyError: - pass - - def _find_keys_raw(self, prefix): - with DBMContext(self.filename, self.mode, True) as db: - p = self._cast_key(prefix) - return [k.decode('utf-8') for k in db.keys() if k.startswith(p)] diff --git a/sorl/thumbnail/kvstores/dynamodb_kvstore.py b/sorl/thumbnail/kvstores/dynamodb_kvstore.py deleted file mode 100644 index fed9e24a2..000000000 --- a/sorl/thumbnail/kvstores/dynamodb_kvstore.py +++ /dev/null @@ -1,37 +0,0 @@ -import boto -from boto.dynamodb2.table import Table - -from sorl.thumbnail.conf import settings -from sorl.thumbnail.kvstores.base import KVStoreBase - - -class KVStore(KVStoreBase): - def __init__(self): - super().__init__() - region = settings.AWS_REGION_NAME - access_key = settings.AWS_ACCESS_KEY_ID - secret = settings.AWS_SECRET_ACCESS_KEY - conn = boto.dynamodb2.connect_to_region(region, aws_access_key_id=access_key, - aws_secret_access_key=secret) - self.table = Table(settings.THUMBNAIL_DYNAMODB_NAME, connection=conn) - - def _get_raw(self, key): - try: - return self.table.get_item(key=key)['value'] - except boto.dynamodb2.exceptions.ItemNotFound: - pass - - def _set_raw(self, key, value): - try: - item = self.table.get_item(key=key) - except boto.dynamodb2.exceptions.ItemNotFound: - item = self.table.new_item() - item['key'] = key - item['value'] = value - item.save(overwrite=True) - - def _delete_raw(self, *keys): - [self.table.delete_item(key=k) for k in keys] - - def _find_keys_raw(self, prefix): - return [i['key'] for i in self.table.scan(key__beginswith=prefix)] diff --git a/sorl/thumbnail/kvstores/redis_kvstore.py b/sorl/thumbnail/kvstores/redis_kvstore.py deleted file mode 100644 index 4ab0e77d5..000000000 --- a/sorl/thumbnail/kvstores/redis_kvstore.py +++ /dev/null @@ -1,36 +0,0 @@ -import redis - -from sorl.thumbnail.conf import settings -from sorl.thumbnail.kvstores.base import KVStoreBase - - -class KVStore(KVStoreBase): - def __init__(self): - super().__init__() - - if hasattr(settings, 'THUMBNAIL_REDIS_URL'): - self.connection = redis.from_url(settings.THUMBNAIL_REDIS_URL) - else: - self.connection = redis.Redis( - host=settings.THUMBNAIL_REDIS_HOST, - port=settings.THUMBNAIL_REDIS_PORT, - db=settings.THUMBNAIL_REDIS_DB, - ssl=settings.THUMBNAIL_REDIS_SSL, - password=settings.THUMBNAIL_REDIS_PASSWORD, - unix_socket_path=settings.THUMBNAIL_REDIS_UNIX_SOCKET_PATH, - ) - - def _get_raw(self, key): - return self.connection.get(key) - - def _set_raw(self, key, value): - return self.connection.set( - key, value, ex=settings.THUMBNAIL_REDIS_TIMEOUT) - - def _delete_raw(self, *keys): - return self.connection.delete(*keys) - - def _find_keys_raw(self, prefix): - pattern = prefix + '*' - return list(map(lambda key: key.decode('utf-8'), - self.connection.keys(pattern=pattern))) diff --git a/tests/settings/dbm.py b/tests/settings/dbm.py deleted file mode 100644 index 2a04e96b2..000000000 --- a/tests/settings/dbm.py +++ /dev/null @@ -1,4 +0,0 @@ -from .default import * - - -THUMBNAIL_KVSTORE = 'sorl.thumbnail.kvstores.dbm_kvstore.KVStore' diff --git a/tests/settings/default.py b/tests/settings/default.py index 2c98c5319..b6e1f079f 100644 --- a/tests/settings/default.py +++ b/tests/settings/default.py @@ -11,7 +11,6 @@ 'class': 'sorl.thumbnail.log.ThumbnailLogHandler', 'level': 'ERROR', } -THUMBNAIL_KVSTORE = 'tests.thumbnail_tests.kvstore.TestKVStore' THUMBNAIL_STORAGE = 'tests.thumbnail_tests.storage.TestStorage' STORAGES = { "default": { @@ -51,4 +50,3 @@ 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ) -THUMBNAIL_REDIS_SSL = False diff --git a/tests/settings/dynamodb.py b/tests/settings/dynamodb.py deleted file mode 100644 index 17afdee34..000000000 --- a/tests/settings/dynamodb.py +++ /dev/null @@ -1,8 +0,0 @@ -from .default import * - - -THUMBNAIL_KVSTORE = 'sorl.thumbnail.kvstores.dynamodb_kvstore.KVStore' -THUMBNAIL_DYNAMODB_NAME = 'test' -AWS_REGION_NAME = 'eu-central-1' -AWS_ACCESS_KEY_ID = 'use' -AWS_SECRET_ACCESS_KEY = 'yours' diff --git a/tests/settings/redis.py b/tests/settings/redis.py deleted file mode 100644 index 3e9cc1bee..000000000 --- a/tests/settings/redis.py +++ /dev/null @@ -1,4 +0,0 @@ -from .default import * - - -THUMBNAIL_KVSTORE = 'sorl.thumbnail.kvstores.redis_kvstore.KVStore' diff --git a/tests/thumbnail_tests/utils.py b/tests/thumbnail_tests/utils.py index 443c35cac..5dd8fb29e 100644 --- a/tests/thumbnail_tests/utils.py +++ b/tests/thumbnail_tests/utils.py @@ -11,6 +11,7 @@ from sorl.thumbnail.helpers import get_module_class from sorl.thumbnail.images import ImageFile from sorl.thumbnail.log import ThumbnailLogHandler +from sorl.thumbnail.kvstores.cached_db_kvstore import KVStore from .models import Item from .storage import MockLoggingHandler @@ -88,7 +89,7 @@ def is_transparent(self, img): def setUp(self): self.BACKEND = get_module_class(settings.THUMBNAIL_BACKEND)() self.ENGINE = get_module_class(settings.THUMBNAIL_ENGINE)() - self.KVSTORE = get_module_class(settings.THUMBNAIL_KVSTORE)() + self.KVSTORE = KVStore() if not os.path.exists(settings.MEDIA_ROOT): os.makedirs(settings.MEDIA_ROOT) diff --git a/tox.ini b/tox.ini index 3d03c7abc..aaf170ba7 100644 --- a/tox.ini +++ b/tox.ini @@ -17,15 +17,14 @@ TARGET = graphicsmagick: graphicsmagick redis: redis wand: wand - dbm: dbm qa: qa [tox] skipsdist = True envlist = py38-qa, - py{38,39,310,311,312}-django{42}-{pil,imagemagick,graphicsmagick,redis,dynamodb,wand,pgmagick,dbm,vipsthumbnail} - py{310,311,312}-django{50,51}-{pil,imagemagick,graphicsmagick,redis,dynamodb,wand,pgmagick,dbm,vipsthumbnail} + py{38,39,310,311,312}-django{42}-{pil,imagemagick,graphicsmagick,wand,pgmagick,vipsthumbnail} + py{310,311,312}-django{50,51}-{pil,imagemagick,graphicsmagick,wand,pgmagick,vipsthumbnail} [testenv] deps = @@ -34,7 +33,6 @@ deps = pytest-django pillow redis: redis - dynamodb: boto pgmagick: pgmagick wand: wand django42: django>=4.2,<4.3 @@ -47,11 +45,8 @@ setenv = imagemagick: DJANGO_SETTINGS_MODULE=tests.settings.imagemagick graphicsmagick: DJANGO_SETTINGS_MODULE=tests.settings.graphicsmagick vipsthumbnail: DJANGO_SETTINGS_MODULE=tests.settings.vipsthumbnail - redis: DJANGO_SETTINGS_MODULE=tests.settings.redis - dynamodb: DJANGO_SETTINGS_MODULE=tests.settings.dynamodb wand: DJANGO_SETTINGS_MODULE=tests.settings.wand pgmagick: DJANGO_SETTINGS_MODULE=tests.settings.pgmagick - dbm: DJANGO_SETTINGS_MODULE=tests.settings.dbm commands = python -m pytest -rw --cov-append --cov-config setup.cfg --cov sorl --cov-report=xml {posargs:} From b8b423a0b770bd8ac035c62f0f8aad4805e0aea7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:56:18 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- sorl/thumbnail/default.py | 1 + tests/thumbnail_tests/utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sorl/thumbnail/default.py b/sorl/thumbnail/default.py index 2706490fc..9706a9374 100644 --- a/sorl/thumbnail/default.py +++ b/sorl/thumbnail/default.py @@ -4,6 +4,7 @@ from sorl.thumbnail.helpers import get_module_class from sorl.thumbnail.kvstores.cached_db_kvstore import KVStore + class Backend(LazyObject): def _setup(self): self._wrapped = get_module_class(settings.THUMBNAIL_BACKEND)() diff --git a/tests/thumbnail_tests/utils.py b/tests/thumbnail_tests/utils.py index 5dd8fb29e..5d09f5db0 100644 --- a/tests/thumbnail_tests/utils.py +++ b/tests/thumbnail_tests/utils.py @@ -10,8 +10,8 @@ from sorl.thumbnail.conf import settings from sorl.thumbnail.helpers import get_module_class from sorl.thumbnail.images import ImageFile -from sorl.thumbnail.log import ThumbnailLogHandler from sorl.thumbnail.kvstores.cached_db_kvstore import KVStore +from sorl.thumbnail.log import ThumbnailLogHandler from .models import Item from .storage import MockLoggingHandler From c9edbcdb5a5d5f72fa087431617b0cafb4b5a899 Mon Sep 17 00:00:00 2001 From: Dulmandakh Date: Wed, 21 Aug 2024 00:00:43 +0800 Subject: [PATCH 3/3] fix circular import --- sorl/thumbnail/default.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sorl/thumbnail/default.py b/sorl/thumbnail/default.py index 9706a9374..0ea584671 100644 --- a/sorl/thumbnail/default.py +++ b/sorl/thumbnail/default.py @@ -1,14 +1,17 @@ -from django.utils.functional import LazyObject, SimpleLazyObject +from django.utils.functional import LazyObject from sorl.thumbnail.conf import settings from sorl.thumbnail.helpers import get_module_class -from sorl.thumbnail.kvstores.cached_db_kvstore import KVStore class Backend(LazyObject): def _setup(self): self._wrapped = get_module_class(settings.THUMBNAIL_BACKEND)() +class KVStore(LazyObject): + def _setup(self): + self._wrapped = get_module_class("sorl.thumbnail.kvstores.cached_db_kvstore.KVStore")() + class Engine(LazyObject): def _setup(self): @@ -21,6 +24,6 @@ def _setup(self): backend = Backend() -kvstore = SimpleLazyObject(lambda: KVStore()) +kvstore = KVStore() engine = Engine() storage = Storage()