From e9ae584fa1f14535d2a13602c78fcf9c0c7821ae Mon Sep 17 00:00:00 2001 From: Laurin Schmidt Date: Tue, 12 Nov 2024 16:18:51 +0100 Subject: [PATCH 1/2] fixture postrequisites (inverse of prerequisite) and docs. uses graphlib(backport) for topological sort. --- docs/chapter-06.rst | 59 ++++++++++++++++++++++++++++++------ py4web/core.py | 73 +++++++++++++++++++++++++++++++++++++-------- requirements.txt | 1 + 3 files changed, 112 insertions(+), 21 deletions(-) diff --git a/docs/chapter-06.rst b/docs/chapter-06.rst index dc1b0dd0..a312e70c 100644 --- a/docs/chapter-06.rst +++ b/docs/chapter-06.rst @@ -61,13 +61,17 @@ Then you can apply all of them at once with: return dict() Usually, it's not important the order you use to specify the fixtures, because py4web -knows well how to manage them if they have explicit dependencies. For example auth -depends explicitly on db and session and flash, so you do not even needs to list them. +knows well how to manage them if they have explicit dependencies +(using the ``__prerequisites__`` attribute). For example auth depends explicitly on +db and session and flash, so you do not even needs to list them. But there is an important exception: the Template fixture must always be the **first one**. Otherwise, it will not have access to various things it should need from the other fixtures, especially Inject() and Flash() that we'll see later. +If all your templates depend on a specific fixture, you can also list them as +``__postrequisites__`` of Template. This is explained in detail below. + The Template fixture -------------------- @@ -146,6 +150,12 @@ And then: This syntax has no performance implications: it's just for avoiding to replicate a decorator logic in multiple places. In this way you'll have cleaner code and if needed you'll be able to change it later in one place only. +Alternatively, you can also specify fixtures as a global dependency of Template: +.. code:: python + + Template.dependencies.append(flash) + +This ensures flash always runs before Template. See :ref:`Fixtures with dependencies` for more info. The Inject fixture ------------------ @@ -926,11 +936,11 @@ Here is a fixture that logs exceptions tracebacks to a file: @action.uses(errlog) def index(): return 1/0 -Fixtures also have a ``__prerequisite__`` attribute. If a fixture -takes another fixture as an argument, its value must be appended -to the list of ``__prerequisites__``. This guarantees that they are +Fixtures also have ``__prerequisites__`` and ``__postrequisites__`` attributes. +If a fixture takes another fixture as an argument, its value must be appended +to either of these depending on the required behaviour. This guarantees that they are always executed in the proper order even if listed in the wrong order. -It also makes it optional to declare prerequisite fixtures in ``action.uses``. +It also makes it optional to declare requisite fixtures in ``action.uses``. For example ``Auth`` depends on ``db``, ``session``, and ``flash``. ``db`` and ``session`` are indeed arguments. ``flash`` is a special singleton fixture declared within ``Auth``. @@ -964,9 +974,14 @@ retrieve that data. Fixtures with dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~ -If a fixture depends on another fixture, it needs to be passed that fixture in the initializer, -and the fixture must be listed in the ``__prerequisites__`` attribute. -For example, suppose we want to create a fixture that grants access to a controller only +If a fixture depends on another fixture, it needs to be passed that fixture +in the initializer, and the fixture must be listed in either the ``__prerequisites__`` +or ``__postrequisites__`` attributes. Specifically, if the dependencies +``on_request`` should run before your ``on_request``, list it in ``__prerequisites__``. +If instead the dependencies ``on_success`` should run before your ``on_success``, +list it in ``__postrequisites``. These are different as ``on_success`` is called in +reverse order compared to ``on_request``. For example, suppose we want to create a +fixture that grants access to a controller only to users whose email address is included in an ADMIN_EMAILS list. We can write the following fixture: @@ -1003,6 +1018,32 @@ The fixture can be created and used as follows: def admin_only(): return dict() +``__postrequisites__`` on the other hand inverts this dependency relation. +Essentially, instead of requiring other fixtures to run before yours, +it requires your fixture to run before another. +This is mostly useful since ``on_success`` is called in reverse order compared to +``on_request``. + +This is useful for example for ``Template``, as the ``on_success`` of ``Template`` +has to run last. So if ``Template`` always needs ``flash`` instead of having to +add ``flash`` to each controller or define a custom decorator, it can be added +to the ``__postrequisites__`` of ``Template`` instead. + +Note: ``Template`` has a class variable called ``dependencies`` which +you should use, as ``dependencies`` is copied into ``__postrequisites__`` +each time a new instance of ``Template`` is created. + +Internally, adding ``flash`` to the ``__postrequisites__`` of ``Template`` +is the same as adding all ``Template`` instances to the ``__prerequisites__`` +of ``flash``. + +.. warning:: + It's possible to cause dependency cycles using this. + Never add the same dependency to both ``__prerequisites__`` and + ``__postrequisites__``, as both are parts of the same dependency + graph (which is a Directed Acyclic Graph). + Doing this will lead to a ``graphlib.CycleError``. + Using local storage ~~~~~~~~~~~~~~~~~~~ diff --git a/py4web/core.py b/py4web/core.py index da825f29..3b1b16be 100644 --- a/py4web/core.py +++ b/py4web/core.py @@ -35,6 +35,7 @@ import urllib.parse import uuid import zipfile +import graphlib # provided by backport when py < 3.9 from collections import OrderedDict from contextlib import redirect_stderr, redirect_stdout @@ -590,12 +591,14 @@ class Template(Fixture): """The Template Fixture class""" cache = Cache(100) + dependencies = [] def __init__(self, filename, path=None, delimiters="[[ ]]"): """Initialized the template object""" self.filename = filename self.path = path self.delimiters = delimiters + self.__postrequisites__ = Template.dependencies[:] def on_success(self, context): """ @@ -942,6 +945,62 @@ def redirect(location): raise HTTP(303) +def fixture_topological_sort(fixtures_in): + # deduplicated list of all fixtures + all_fixtures = [] + + # dependency graph based only on indices into all_fixtures + graph = {} + + # recursively create the graph by adding fixtures and their dependencies to it + def recursive_create_graph(fixt): + if fixt not in all_fixtures: + all_fixtures.append(fixt) + graph[all_fixtures.index(fixt)] = set() + + prereqs = getattr(fixt, "__prerequisites__", ()) or () + postreqs = getattr(fixt, "__postrequisites__", ()) or () + + # prerequisites are added as dependencies of the source fixture + for pre in prereqs: + recursive_create_graph(pre) + graph[all_fixtures.index(fixt)].add(all_fixtures.index(pre)) + + # postrequisites imply the source fixture is a dependency of the postrequisite + for post in postreqs: + recursive_create_graph(post) + graph[all_fixtures.index(post)].add(all_fixtures.index(fixt)) + + # populate graph/all_fixtures + for fixt in fixtures_in: + recursive_create_graph(fixt) + + sorter = graphlib.TopologicalSorter(graph) + try: + # convert from indices back into their fixtures + return [all_fixtures[i] for i in sorter.static_order()] + except graphlib.CycleError as e: + # nicely format the error displaying the cycle + nodes = set() + cycle = [] + # e.args[1] contains dependency cycles: A -> B -> A + for i in e.args[1]: + fixt = all_fixtures[i] + cycle.append(str(fixt)) + prereqs = getattr(fixt, "__prerequisites__", ()) + postreqs = getattr(fixt, "__postrequisites__", ()) + nodes.add(f""" +{fixt}: + - prerequisites: {prereqs} + - postrequisites: {postreqs}""") + raise graphlib.CycleError(f""" +Involving the following fixtures: +{''.join(nodes)} + +Fixture dependency cycle: {' -> '.join(cycle)}""",) from None + + + class action: # pylint: disable=invalid-name """@action(...) is a decorator for functions to be exposed as actions""" @@ -956,18 +1015,8 @@ def __init__(self, path, **kwargs): @staticmethod def uses(*fixtures_in): """Used to declare needed fixtures, they will be topologically sorted""" - fixtures = [] - reversed_fixtures = [] - stack = list(fixtures_in) - while stack: - fixture = stack.pop() - reversed_fixtures.append(fixture) - stack.extend(getattr(fixture, "__prerequisites__", ())) - for fixture in reversed(reversed_fixtures): - if isinstance(fixture, str): - fixture = Template(fixture) - if fixture not in fixtures: - fixtures.append(fixture) + fixtures = [Template(f) if isinstance(f, str) else f for f in fixtures_in] + fixtures = fixture_topological_sort(fixtures) def decorator(func): if DEBUG: diff --git a/requirements.txt b/requirements.txt index b3447db1..bde82a39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ rocket3 >= 20241019.1 yatl >= 20230507.3 pydal >= 20241027.1 watchgod >= 0.6 +graphlib_backport>=1.0;python_version<"3.9" # optional modules: # gunicorn From 4506e6ef8b7c3a31fcf21ccf802042d34c181907 Mon Sep 17 00:00:00 2001 From: Laurin Schmidt Date: Tue, 12 Nov 2024 16:19:09 +0100 Subject: [PATCH 2/2] fix tests on windows (partly) --- tests/test_action.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_action.py b/tests/test_action.py index 1e229eb2..34917fef 100644 --- a/tests/test_action.py +++ b/tests/test_action.py @@ -2,6 +2,7 @@ import copy import multiprocessing import os +import sys import threading import time import unittest @@ -18,7 +19,18 @@ ) SECRET = str(uuid.uuid4()) -db = DAL("sqlite://storage_%s" % uuid.uuid4(), folder="/tmp/") +if sys.platform == "win32": + path = "./tmp/" +else: + path = "/tmp/" + +try: + os.mkdir(path) +except Exception: + pass +with open(path + "sql.log", "w"): + pass +db = DAL("sqlite://storage_%s" % uuid.uuid4(), folder=path) db.define_table("thing", Field("name")) session = Session(secret=SECRET) cache = Cache()