diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5db25a9a..5ab0af38 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,8 +26,7 @@ jobs: - {name: Linux, python: '3.9', os: ubuntu-latest, tox: py39} - {name: '3.8', python: '3.8', os: ubuntu-latest, tox: py38} - {name: '3.7', python: '3.7', os: ubuntu-latest, tox: py37} - - {name: '3.6', python: '3.6', os: ubuntu-latest, tox: py36} - - {name: 'PyPy', python: pypy3, os: ubuntu-latest, tox: pypy3} + - {name: 'PyPy', python: pypy-3.7, os: ubuntu-latest, tox: pypy37} steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7b2b746a..fe46316e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,28 +2,28 @@ ci: autoupdate_schedule: monthly repos: - repo: https://github.com/asottile/pyupgrade - rev: v2.25.0 + rev: v2.32.0 hooks: - id: pyupgrade args: ["--py36-plus"] - repo: https://github.com/asottile/reorder_python_imports - rev: v2.6.0 + rev: v3.0.1 hooks: - id: reorder-python-imports args: ["--application-directories", "src"] - repo: https://github.com/psf/black - rev: 21.8b0 + rev: 22.3.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 3.9.2 + rev: 4.0.1 hooks: - id: flake8 additional_dependencies: - flake8-bugbear - flake8-implicit-str-concat - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.2.0 hooks: - id: fix-byte-order-marker - id: trailing-whitespace diff --git a/CHANGES.rst b/CHANGES.rst index 1836e2bc..14469478 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,7 +6,9 @@ Version 1.2.0 Unreleased -- cachelib is now used as backend. PR `#308 `_. +- Cachelib is now used as backend. PR `#308 `_. +- Drop support for python 3.6 +- A ``DynamoDbCache`` backend has been add to the user contributed backends. Version 1.10.1 diff --git a/requirements/dev.txt b/requirements/dev.txt index 541eef87..b0f1eafa 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,6 +6,8 @@ # alabaster==0.7.12 # via sphinx +async-timeout==4.0.2 + # via redis attrs==21.4.0 # via pytest babel==2.9.1 @@ -13,7 +15,6 @@ babel==2.9.1 cachelib==0.6.0 # via -r tests.in certifi==2021.10.8 -backports-entry-points-selectable==1.1.0 # via requests cfgv==3.3.1 # via pre-commit @@ -27,7 +28,7 @@ deprecated==1.2.13 # via redis distlib==0.3.4 # via virtualenv -docutils==0.16 +docutils==0.17.1 # via # sphinx # sphinx-tabs @@ -35,7 +36,7 @@ filelock==3.4.2 # via # tox # virtualenv -flask==2.0.3 +flask==2.1.1 # via -r tests.in identify==2.4.4 # via pre-commit @@ -43,8 +44,6 @@ idna==3.3 # via requests imagesize==1.3.0 # via sphinx -importlib-metadata==4.10.1 - # via sphinx iniconfig==1.1.1 # via pytest itsdangerous==2.0.1 @@ -92,7 +91,7 @@ pylibmc==1.6.1 # via -r tests.in pyparsing==3.0.6 # via packaging -pytest==6.2.5 +pytest==7.1.1 # via # -r tests.in # pytest-xprocess @@ -102,7 +101,7 @@ pytz==2021.3 # via babel pyyaml==6.0 # via pre-commit -redis==4.1.1 +redis==4.2.1 # via -r tests.in requests==2.27.1 # via sphinx @@ -112,7 +111,7 @@ six==1.16.0 # virtualenv snowballstemmer==2.2.0 # via sphinx -sphinx==4.4.0 +sphinx==4.5.0 # via # -r docs.in # pallets-sphinx-themes @@ -121,7 +120,7 @@ sphinx==4.4.0 # sphinxcontrib-log-cabinet sphinx-issues==3.0.1 # via -r docs.in -sphinx-tabs==3.2.0 +sphinx-tabs==3.3.1 # via -r docs.in sphinxcontrib-applehelp==1.0.2 # via sphinx @@ -140,10 +139,11 @@ sphinxcontrib-serializinghtml==1.1.5 toml==0.10.2 # via # pre-commit - # pytest # tox tomli==2.0.0 - # via pep517 + # via + # pep517 + # pytest tox==3.24.5 # via -r dev.in urllib3==1.26.8 @@ -158,8 +158,6 @@ wheel==0.37.1 # via pip-tools wrapt==1.13.3 # via deprecated -zipp==3.7.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/docs.txt b/requirements/docs.txt index 493e506e..e0d5b93e 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -12,7 +12,7 @@ certifi==2021.10.8 # via requests charset-normalizer==2.0.10 # via requests -docutils==0.16 +docutils==0.17.1 # via # sphinx # sphinx-tabs @@ -20,8 +20,6 @@ idna==3.3 # via requests imagesize==1.3.0 # via sphinx -importlib-metadata==4.10.1 - # via sphinx jinja2==3.0.3 # via sphinx markupsafe==2.0.1 @@ -44,7 +42,7 @@ requests==2.27.1 # via sphinx snowballstemmer==2.2.0 # via sphinx -sphinx==4.4.0 +sphinx==4.5.0 # via # -r docs.in # pallets-sphinx-themes @@ -53,7 +51,7 @@ sphinx==4.4.0 # sphinxcontrib-log-cabinet sphinx-issues==3.0.1 # via -r docs.in -sphinx-tabs==3.2.0 +sphinx-tabs==3.3.1 # via -r docs.in sphinxcontrib-applehelp==1.0.2 # via sphinx @@ -71,5 +69,3 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx urllib3==1.26.8 # via requests -zipp==3.7.0 - # via importlib-metadata diff --git a/requirements/tests.txt b/requirements/tests.txt index f02651f7..974c4ba9 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -4,6 +4,8 @@ # # pip-compile tests.in # +async-timeout==4.0.2 + # via redis attrs==21.4.0 # via pytest cachelib==0.6.0 @@ -12,7 +14,7 @@ click==8.0.3 # via flask deprecated==1.2.13 # via redis -flask==2.0.3 +flask==2.1.1 # via -r tests.in iniconfig==1.1.1 # via pytest @@ -36,15 +38,15 @@ pylibmc==1.6.1 # via -r tests.in pyparsing==3.0.6 # via packaging -pytest==6.2.5 +pytest==7.1.1 # via # -r tests.in # pytest-xprocess pytest-xprocess==0.18.1 # via -r tests.in -redis==4.1.1 +redis==4.2.1 # via -r tests.in -toml==0.10.2 +tomli==2.0.1 # via pytest werkzeug==2.0.2 # via flask diff --git a/setup.cfg b/setup.cfg index f2b25fa9..f794c4e7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ classifiers = packages = find: package_dir = = src include_package_data = true -python_requires = >= 3.6 +python_requires = >= 3.7 # Dependencies are in setup.py for GitHub's dependency graph. [options.packages.find] diff --git a/src/flask_caching/contrib/dynamodbcache.py b/src/flask_caching/contrib/dynamodbcache.py new file mode 100644 index 00000000..aa9fcb73 --- /dev/null +++ b/src/flask_caching/contrib/dynamodbcache.py @@ -0,0 +1,254 @@ +import datetime + +import flask + +from flask_caching.backends.base import BaseCache + +try: + import boto3 + from boto3.dynamodb.conditions import Attr +except ImportError as e: + raise RuntimeError('No boto3 package found') from e + +CREATED_AT_FIELD = 'created_at' +RESPONSE_FIELD = 'response' + + +def utcnow(): + """Return a tz-aware UTC datetime representing the current time""" + return datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc) + + +class DynamoDbCache(BaseCache): + """ + Implementation of flask_caching.BaseCache that uses an AWS DynamoDb table + as the backend. + + The DynamoDB table is required to already exist. The table must be + defined with a hash_key of type string, and no sort key. Additionally, + you'll probably want to enable the TTL feature on the table, so that + DynamoDB will automatically delete expired cache items. The hash_key + attribute name defaults to 'cache_key', and the ttl attribute name + defaults to 'expiration_time'. These defaults can be changed via + constructor parameter, or via app config properties. + + Your server process will require dynamodb:GetItem and dynamodb:PutItem + IAM permissions on the cache table. + + App config: The factory method for this class uses the following app + config attributes: + + CACHE_DYNAMODB_TABLE: (Required) the name of the DynamoDB table to use + CACHE_DYNAMODB_KEY_FIELD: The name of the hash key attribute of the + table. Defaults to 'cache_key' + CACHE_DYNAMODB_EXPIRATION_TIME_FIELD: The name of the TTL field. Defaults + to 'expiration_time' + + Additionally, the CACHE_DEFAULT_TIMEOUT attribute can be used to override + default the cache timeout. + + Here is how you could create a DynamoDB table suitable for use by this + class using the AWS CLI. + + TABLE_NAME=cache-table + KEY_ATTRIBUTE=cache_key + TTL_ATTRIBUTE=expiration_time + + aws dynamodb create-table \ + --table-name $TABLE_NAME \ + --attribute-definitions AttributeName=$KEY_ATTRIBUTE,AttributeType=S \ + --key-schema AttributeName=$KEY_ATTRIBUTE,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST + + aws dynamodb update-time-to-live \ + --table-name $TABLE_NAME \ + --time-to-live-specification Enabled=true,AttributeName=$TTL_ATTRIBUTE + + If you use anything other than the default key and TTL attribute names, + be sure to update the app config appropriately. + + Limitations: DynamoDB table items are limited to 400 KB in size. Since + this class stores cached items in a table, the max size of a cache entry + will be slightly less than 400 KB, since the cache key and expiration + time fields are also part of the item. + + :param table_name: The name of the DynamoDB table to use + :param default_timeout: Set the timeout in seconds after which cache entries + expire + :param key_field: The name of the hash_key attribute in the DynamoDb + table. This must be a string attribute. + :param expiration_time_field: The name of the table attribute to store the + expiration time in. This will be an int + attribute. The timestamp will be stored as + seconds past the epoch. If you configure + this as the TTL field, then DynamoDB will + automatically delete expired entries. + :param dynamo: A boto3 dynamodb resource object. This is mainly for testing; + by default the class will create its own resource object. + """ + + def __init__( + self, + table_name, + default_timeout=300, + key_field='cache_key', + expiration_time_field='expiration_time', + dynamo=None + ): + super().__init__(default_timeout) + self._table_name = table_name + + self._key_field = key_field + self._expiration_time_field = expiration_time_field + + if dynamo is None: + self._dynamo = boto3.resource('dynamodb') + else: + self._dynamo = dynamo + + self._table = self._dynamo.Table(self._table_name) + + @classmethod + def factory(cls, app, config, args: list, kwargs: dict): + args.insert(0, config['CACHE_DYNAMODB_TABLE']) + key_field = config.get('CACHE_DYNAMODB_KEY_FIELD') + expiration_time_field = config.get( + 'CACHE_DYNAMODB_EXPIRATION_TIME_FIELD') + + if key_field: + kwargs.setdefault('key_field', key_field) + if expiration_time_field: + kwargs.setdefault('expiration_time_field', expiration_time_field) + + return cls(*args, **kwargs) + + def _get_item(self, key, attributes=None): + """ + Get an item from the cache table, optionally limiting the returned + attributes. + + :param key: The cache key of the item to fetch + + :param attributes: An optional list of attributes to fetch. If not + given, all attributes are fetched. The + expiration_time field will always be added to the + list of fetched attributes. + :return: The table item for key if it exists and is not expired, else + None + """ + kwargs = {} + if attributes: + if self._expiration_time_field not in attributes: + attributes = list(attributes) + [self._expiration_time_field] + kwargs = dict(ProjectionExpression=','.join(attributes)) + + response = self._table.get_item(Key={self._key_field: key}, **kwargs) + cache_item = response.get('Item') + + if cache_item: + now = int(utcnow().timestamp()) + if cache_item[self._expiration_time_field] > now: + return cache_item + + return None + + def get(self, key): + """ + Get a cache item as a Flask Response + + :param key: The cache key of the item to fetch + :return: If key is found and the item isn't expired, returns a Flask + response object containing the cached response body, status + code, and headers. Else returns None + """ + cache_item = self._get_item(key) + if cache_item: + response = cache_item[RESPONSE_FIELD] + return flask.make_response( + bytes(response['body']), + int(response['statusCode']), + response['headers']) + + return None + + def delete(self, key): + """ + Deletes an item from the cache. This is a no-op if the item doesn't + exist + + :param key: Key of the item to delete. + :return: True if the key existed and was deleted + """ + try: + self._table.delete_item( + Key={self._key_field: key}, + ConditionExpression=Attr(self._key_field).exists() + ) + return True + except self._dynamo.meta.client.exceptions.ConditionalCheckFailedException: + return False + + def _set(self, key, value, timeout=None, overwrite=True): + """ + Store a cache item, with the option to not overwrite existing items + + :param key: Cache key to use + :param value: A value returned by a flask view function + :param timeout: The timeout in seconds for the cached item, to override + the default + :param overwrite: If true, overwrite any existing cache item with key. + If false, the new value will only be stored if no + non-expired cache item exists with key. + :return: True if the new item was stored. + """ + now = utcnow() + expiration_time = now + datetime.timedelta( + seconds=self._normalize_timeout(timeout)) + response_obj = flask.make_response(value) + + cached_response = { + 'body': response_obj.get_data(), + 'statusCode': response_obj.status_code, + 'headers': dict(response_obj.headers) + } + + kwargs = {} + if not overwrite: + # Cause the put to fail if a non-expired item with this key + # already exists + cond = Attr(self._key_field).not_exists() \ + | Attr(self._expiration_time_field).lte(int(now.timestamp())) + kwargs = dict(ConditionExpression=cond) + + try: + item = { + self._key_field: key, + self._expiration_time_field: int(expiration_time.timestamp()), + CREATED_AT_FIELD: now.isoformat(), + RESPONSE_FIELD: cached_response + } + self._table.put_item(Item=item, **kwargs) + return True + except self._dynamo.meta.client.exceptions.ConditionalCheckFailedException: + return False + + def set(self, key, value, timeout=None): + return self._set(key, value, timeout=timeout, overwrite=True) + + def add(self, key, value, timeout=None): + return self._set(key, value, timeout=timeout, overwrite=False) + + def has(self, key): + return self._get_item(key, [self._expiration_time_field]) is not None + + def clear(self): + paginator = self._dynamo.meta.client.get_paginator('scan') + pages = paginator.paginate(TableName=self._table_name, + ProjectionExpression=self._key_field) + + with self._table.batch_writer() as batch: + for page in pages: + for item in page['Items']: + batch.delete_item(Key=item) + + return True diff --git a/tox.ini b/tox.ini index 4d186718..64b27447 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{39,38,37,36,py3} + py{39,38,37,py3} style typing docs