From ca356d12f9fb94342a85112dbe0dcf98eb1037ea Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Tue, 19 Apr 2022 19:37:28 +0200 Subject: [PATCH 01/29] Bump dev version --- emmett/__version__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/emmett/__version__.py b/emmett/__version__.py index 72ebbb39..bee6b3e5 100644 --- a/emmett/__version__.py +++ b/emmett/__version__.py @@ -1 +1 @@ -__version__ = "2.4.12" +__version__ = "2.5.0.dev0" diff --git a/pyproject.toml b/pyproject.toml index 9b4677ea..2a79d045 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "Emmett" -version = "2.4.12" +version = "2.5.0.dev0" description = "The web framework for inventors" authors = ["Giovanni Barillari "] license = "BSD-3-Clause" From 179fabc3bcad5d81ff237f91898f6f52ee5e2d25 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Mon, 27 Dec 2021 16:58:38 +0100 Subject: [PATCH 02/29] rsgi: add handlers and wrappers --- emmett/_reloader.py | 17 +- emmett/app.py | 20 ++- emmett/asgi/handlers.py | 44 +++-- emmett/asgi/helpers.py | 4 + emmett/asgi/workers.py | 5 +- emmett/asgi/wrappers.py | 259 +++++++++++++++++++++++++++++ emmett/cli.py | 83 +++------- emmett/ctx.py | 25 +-- emmett/http.py | 53 +++++- emmett/rsgi/__init__.py | 0 emmett/rsgi/handlers.py | 305 +++++++++++++++++++++++++++++++++++ emmett/rsgi/helpers.py | 42 +++++ emmett/rsgi/wrappers.py | 135 ++++++++++++++++ emmett/server.py | 48 ++++++ emmett/sessions.py | 4 +- emmett/testing/client.py | 19 +-- emmett/wrappers/__init__.py | 53 ++---- emmett/wrappers/helpers.py | 60 +------ emmett/wrappers/request.py | 125 +++----------- emmett/wrappers/websocket.py | 68 ++------ pyproject.toml | 3 +- tests/helpers.py | 6 +- tests/test_session.py | 5 +- tests/test_wrappers.py | 2 +- 24 files changed, 976 insertions(+), 409 deletions(-) create mode 100644 emmett/asgi/wrappers.py create mode 100644 emmett/rsgi/__init__.py create mode 100644 emmett/rsgi/handlers.py create mode 100644 emmett/rsgi/helpers.py create mode 100644 emmett/rsgi/wrappers.py create mode 100644 emmett/server.py diff --git a/emmett/_reloader.py b/emmett/_reloader.py index 3276829a..058f08ed 100644 --- a/emmett/_reloader.py +++ b/emmett/_reloader.py @@ -27,7 +27,7 @@ import click from ._internal import locate_app -from .asgi.server import run as _asgi_run +from .server import run as _server_run def _iter_module_files(): @@ -149,18 +149,14 @@ def run(self, process): def run_with_reloader( + interface, app_target, host, port, loop='auto', - proto_http='auto', - proto_ws='auto', log_level=None, - access_log=None, ssl_certfile: Optional[str] = None, ssl_keyfile: Optional[str] = None, - ssl_cert_reqs: int = ssl.CERT_NONE, - ssl_ca_certs: Optional[str] = None, extra_files=None, interval=1, reloader_type='auto' @@ -174,18 +170,13 @@ def run_with_reloader( locate_app(*app_target) process = multiprocessing.Process( - target=_asgi_run, - args=(app_target, host, port), + target=_server_run, + args=(interface, app_target, host, port), kwargs={ "loop": loop, - "proto_http": proto_http, - "proto_ws": proto_ws, "log_level": log_level, - "access_log": access_log, "ssl_certfile": ssl_certfile, "ssl_keyfile": ssl_keyfile, - "ssl_cert_reqs": ssl_cert_reqs, - "ssl_ca_certs": ssl_ca_certs } ) process.start() diff --git a/emmett/app.py b/emmett/app.py index d54b5b2c..4f8b43c8 100644 --- a/emmett/app.py +++ b/emmett/app.py @@ -22,7 +22,7 @@ from yaml import SafeLoader as ymlLoader, load as ymlload from ._internal import get_root_path, create_missing_app_folders, warn_of_deprecation -from .asgi.handlers import HTTPHandler, LifeSpanHandler, WSHandler +from .asgi import handlers as asgi_handlers from .cache import RouteCacheRule from .ctx import current from .datastructures import sdict, ConfigData @@ -34,6 +34,7 @@ from .pipeline import Pipe, Injector from .routing.router import HTTPRouter, WebsocketRouter, RoutingCtx from .routing.urls import url +from .rsgi import handlers as rsgi_handlers from .templating.templater import Templater from .testing import EmmettTestClient from .typing import ErrorHandlerType @@ -186,9 +187,13 @@ def __init__( self._router_http = HTTPRouter(self, url_prefix=url_prefix) self._router_ws = WebsocketRouter(self, url_prefix=url_prefix) self._asgi_handlers = { - 'http': HTTPHandler(self), - 'lifespan': LifeSpanHandler(self), - 'websocket': WSHandler(self) + 'http': asgi_handlers.HTTPHandler(self), + 'lifespan': asgi_handlers.LifeSpanHandler(self), + 'websocket': asgi_handlers.WSHandler(self) + } + self._rsgi_handlers = { + 'http': rsgi_handlers.HTTPHandler(self), + 'ws': rsgi_handlers.WSHandler(self) } self.error_handlers: Dict[int, Callable[[], Awaitable[str]]] = {} self.template_default_extension = '.html' @@ -215,6 +220,7 @@ def __init__( def _configure_asgi_handlers(self): self._asgi_handlers['http']._configure_methods() + self._rsgi_handlers['http']._configure_methods() @cachedprop def name(self): @@ -427,6 +433,12 @@ def test_client(self, use_cookies: bool = True, **kwargs) -> EmmettTestClient: def __call__(self, scope, receive, send): return self._asgi_handlers[scope['type']](scope, receive, send) + def __rsgi__(self, scope, protocol): + return self._rsgi_handlers[scope.proto](scope, protocol) + + def __rsgi_init__(self, loop): + self.send_signal(Signals.after_loop, loop=loop) + def module( self, import_name: str, diff --git a/emmett/asgi/handlers.py b/emmett/asgi/handlers.py index 6a3373b7..0a9a9425 100644 --- a/emmett/asgi/handlers.py +++ b/emmett/asgi/handlers.py @@ -24,14 +24,14 @@ from ..ctx import RequestContext, WSContext, current from ..debug import smart_traceback, debug_handler +from ..extensions import Signals from ..http import HTTPBytes, HTTPResponse, HTTPFile, HTTP from ..libs.contenttype import contenttype from ..utils import cachedprop -from ..wrappers.helpers import RequestCancelled -from ..wrappers.request import Request from ..wrappers.response import Response -from ..wrappers.websocket import Websocket +from .helpers import RequestCancelled from .typing import Event, EventHandler, EventLooper, Receive, Scope, Send +from .wrappers import Request, Websocket REGEX_STATIC = re.compile( r'^/static/(?P__[\w\-\.]+__/)?(?P_\d+\.\d+\.\d+/)?(?P.*?)$' @@ -131,6 +131,7 @@ async def event_startup( send: Send, event: Event ) -> EventLooper: + self.app.send_signal(Signals.after_loop, loop=asyncio.get_event_loop()) await send({'type': 'lifespan.startup.complete'}) return _event_looper @@ -191,7 +192,7 @@ async def __call__( try: http = await self.pre_handler(scope, receive, send) await asyncio.wait_for( - http.send(scope, send), + http.asgi(scope, send), self.app.config.response_timeout ) except RequestCancelled: @@ -312,17 +313,18 @@ async def dynamic_handler( receive: Receive, send: Send ) -> HTTPResponse: - ctx = RequestContext( - self.app, + request = Request( scope, receive, send, - Request, - Response + max_content_length=self.app.config.request_max_content_length, + body_timeout=self.app.config.request_body_timeout ) + response = Response() + ctx = RequestContext(self.app, request, response) ctx_token = current._init_(ctx) try: - http = await self.router.dispatch(ctx.request, ctx.response) + http = await self.router.dispatch(request, response) except HTTPResponse as http_exception: http = http_exception #: render error with handlers if in app @@ -331,8 +333,8 @@ async def dynamic_handler( http = HTTP( http.status_code, await error_handler(), - headers=ctx.response.headers, - cookies=ctx.response.cookies + headers=response.headers, + cookies=response.cookies ) except RequestCancelled: raise @@ -341,7 +343,7 @@ async def dynamic_handler( http = HTTP( 500, await self.error_handler(), - headers=ctx.response.headers + headers=response.headers ) finally: current._close_(ctx_token) @@ -376,10 +378,8 @@ async def __call__( send: Send ): scope['emt.input'] = asyncio.Queue() - task_events = asyncio.create_task( - self.handle_events(scope, receive, send)) - task_request = asyncio.create_task( - self.handle_request(scope, receive, send)) + task_events = asyncio.create_task(self.handle_events(scope, receive, send)) + task_request = asyncio.create_task(self.handle_request(scope, send)) _, pending = await asyncio.wait( [task_request, task_events], return_when=asyncio.FIRST_COMPLETED ) @@ -420,13 +420,12 @@ async def event_receive( async def handle_request( self, scope: Scope, - receive: Receive, send: Send ): scope['emt.path'] = scope['path'] or '/' scope['emt._ws_closed'] = False try: - await self.pre_handler(scope, receive, send) + await self.pre_handler(scope, send) except HTTPResponse: if not scope['emt._ws_closed']: await send({'type': 'websocket.close', 'code': 1006}) @@ -441,27 +440,22 @@ async def handle_request( def _prefix_handler( self, scope: Scope, - receive: Receive, send: Send ) -> Awaitable[None]: path = scope['emt.path'] if not path.startswith(self.router._prefix_main): raise HTTP(404) scope['emt.path'] = path[self.router._prefix_main_len:] or '/' - return self.dynamic_handler(scope, receive, send) + return self.dynamic_handler(scope, send) async def dynamic_handler( self, scope: Scope, - receive: Receive, send: Send ): ctx = WSContext( self.app, - scope, - scope['emt.input'].get, - send, - Websocket + Websocket(scope, scope['emt.input'].get, send) ) ctx_token = current._init_(ctx) try: diff --git a/emmett/asgi/helpers.py b/emmett/asgi/helpers.py index 76d05078..9156e7e1 100644 --- a/emmett/asgi/helpers.py +++ b/emmett/asgi/helpers.py @@ -125,6 +125,10 @@ def is_ssl(self) -> bool: return self.ssl_certfile is not None and self.ssl_keyfile is not None +class RequestCancelled(Exception): + ... + + def _create_ssl_context( certfile: str, keyfile: str, diff --git a/emmett/asgi/workers.py b/emmett/asgi/workers.py index 2f531bb9..94d799d3 100644 --- a/emmett/asgi/workers.py +++ b/emmett/asgi/workers.py @@ -14,11 +14,11 @@ import signal from gunicorn.workers.base import Worker as _Worker +from uvicorn.server import Server -from ..extensions import Signals +from .helpers import Config from .loops import loops from .protocols import protocols_http, protocols_ws -from .server import Config, Server class Worker(_Worker): @@ -82,7 +82,6 @@ def init_signals(self): def run(self): self.config.app = self.wsgi - self.config.app.send_signal(Signals.after_loop, loop=self.config.loop) server = Server(config=self.config) loop = asyncio.get_event_loop() loop.run_until_complete(server.serve(sockets=self.sockets)) diff --git a/emmett/asgi/wrappers.py b/emmett/asgi/wrappers.py new file mode 100644 index 00000000..99935b98 --- /dev/null +++ b/emmett/asgi/wrappers.py @@ -0,0 +1,259 @@ +import asyncio + +from datetime import datetime +from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Tuple, Union +from urllib.parse import parse_qs + +from ..datastructures import sdict +from ..http import HTTP +from ..utils import cachedprop +from ..wrappers.helpers import regex_client +from ..wrappers.request import Request as _Request +from ..wrappers.websocket import Websocket as _Websocket +from .helpers import RequestCancelled +from .typing import Scope, Receive, Send + +_push_headers = { + "accept", + "accept-encoding", + "accept-language", + "cache-control", + "user-agent" +} + + +class Headers(Mapping[str, str]): + __slots__ = ["_data"] + + def __init__(self, scope: Dict[str, Any]): + self._data: Dict[bytes, bytes] = { + key: val for key, val in scope["headers"] + } + + __hash__ = None # type: ignore + + def __getitem__(self, key: str) -> str: + return self._data[key.lower().encode("latin-1")].decode("latin-1") + + def __contains__(self, key: str) -> bool: # type: ignore + return key.lower().encode("latin-1") in self._data + + def __iter__(self) -> Iterator[str]: + for key in self._data.keys(): + yield key.decode("latin-1") + + def __len__(self) -> int: + return len(self._data) + + def get( + self, + key: str, + default: Optional[Any] = None, + cast: Optional[Callable[[Any], Any]] = None + ) -> Any: + rv = self._data.get(key.lower().encode("latin-1")) + rv = rv.decode() if rv is not None else default # type: ignore + if cast is None: + return rv + try: + return cast(rv) + except ValueError: + return default + + def items(self) -> Iterator[Tuple[str, str]]: # type: ignore + for key, value in self._data.items(): + yield key.decode("latin-1"), value.decode("latin-1") + + def keys(self) -> Iterator[str]: # type: ignore + for key in self._data.keys(): + yield key.decode("latin-1") + + def values(self) -> Iterator[str]: # type: ignore + for value in self._data.values(): + yield value.decode("latin-1") + + +class Body: + __slots__ = ('_data', '_receive', '_max_content_length') + + def __init__(self, receive, max_content_length=None): + self._data = bytearray() + self._receive = receive + self._max_content_length = max_content_length + + def append(self, data: bytes): + if data == b'': + return + self._data.extend(data) + if ( + self._max_content_length is not None and + len(self._data) > self._max_content_length + ): + raise HTTP(413, 'Request entity too large') + + async def __load(self) -> bytes: + while True: + event = await self._receive() + if event['type'] == 'http.request': + self.append(event['body']) + if not event.get('more_body', False): + break + elif event['type'] == 'http.disconnect': + raise RequestCancelled + return bytes(self._data) + + def __await__(self): + return self.__load().__await__() + + +class ASGIIngressMixin: + def __init__( + self, + scope: Scope, + receive: Receive, + send: Send + ): + self._scope = scope + self._receive = receive + self._send = send + self.scheme = scope['scheme'] + self.path = scope['emt.path'] + + @cachedprop + def headers(self) -> Headers: + return Headers(self._scope) + + @cachedprop + def query_params(self) -> sdict[str, Union[str, List[str]]]: + rv: sdict[str, Any] = sdict() + for key, values in parse_qs( + self._scope['query_string'].decode('latin-1'), keep_blank_values=True + ).items(): + if len(values) == 1: + rv[key] = values[0] + continue + rv[key] = values + return rv + + @cachedprop + def client(self) -> str: + g = regex_client.search(self.headers.get('x-forwarded-for', '')) + client = ( + (g.group() or '').split(',')[0] if g else ( + self._scope['client'][0] if self._scope['client'] else None + ) + ) + if client in (None, '', 'unknown', 'localhost'): + client = '::1' if self.host.startswith('[') else '127.0.0.1' + return client # type: ignore + + +class Request(ASGIIngressMixin, _Request): + __slots__ = ['_scope', '_receive', '_send'] + + def __init__( + self, + scope: Scope, + receive: Receive, + send: Send, + max_content_length: Optional[int] = None, + body_timeout: Optional[int] = None + ): + super().__init__(scope, receive, send) + self.max_content_length = max_content_length + self.body_timeout = body_timeout + self._now = datetime.utcnow() + self.method = scope['method'] + + @cachedprop + def _input(self): + return Body(self._receive, self.max_content_length) + + @cachedprop + async def body(self) -> bytes: + if ( + self.max_content_length and + self.content_length > self.max_content_length + ): + raise HTTP(413, 'Request entity too large') + try: + rv = await asyncio.wait_for(self._input, timeout=self.body_timeout) + except asyncio.TimeoutError: + raise HTTP(408, 'Request timeout') + return rv + + async def push_promise(self, path: str): + if "http.response.push" not in self._scope.get("extensions", {}): + return + await self._send({ + "type": "http.response.push", + "path": path, + "headers": [ + (key.encode("latin-1"), self.headers[key].encode("latin-1")) + for key in _push_headers & set(self.headers.keys()) + ] + }) + + +class Websocket(ASGIIngressMixin, _Websocket): + __slots__ = ['_scope', '_receive', '_send', '_accepted'] + + def __init__( + self, + scope: Scope, + receive: Receive, + send: Send + ): + super().__init__(scope, receive, send) + self._accepted = False + self._flow_receive = None + self._flow_send = None + self.receive = self._accept_and_receive + self.send = self._accept_and_send + + @property + def _asgi_spec_version(self) -> int: + return int(''.join( + self._scope.get('asgi', {}).get('spec_version', '2.0').split('.') + )) + + def _encode_headers( + self, + headers: Dict[str, str] + ) -> List[Tuple[bytes, bytes]]: + return [ + (key.encode('utf-8'), val.encode('utf-8')) + for key, val in headers.items() + ] + + async def accept( + self, + headers: Optional[Dict[str, str]] = None, + subprotocol: Optional[str] = None + ): + if self._accepted: + return + message: Dict[str, Any] = { + 'type': 'websocket.accept', + 'subprotocol': subprotocol + } + if headers and self._asgi_spec_version > 20: + message['headers'] = self._encode_headers(headers) + await self._send(message) + self._accepted = True + self.receive = self._wrapped_receive + self.send = self._wrapped_send + + async def _wrapped_receive(self) -> Any: + data = await self._receive() + for method in self._flow_receive: # type: ignore + data = method(data) + return data + + async def _wrapped_send(self, data: Any): + for method in self._flow_send: # type: ignore + data = method(data) + if isinstance(data, str): + await self._send({'type': 'websocket.send', 'text': data}) + else: + await self._send({'type': 'websocket.send', 'bytes': data}) diff --git a/emmett/cli.py b/emmett/cli.py index c85c2a78..f9e9d80f 100644 --- a/emmett/cli.py +++ b/emmett/cli.py @@ -16,7 +16,6 @@ import code import os import re -import ssl import sys import types @@ -25,9 +24,8 @@ from .__version__ import __version__ as fw_version from ._internal import locate_app, get_app_module from .asgi.loops import loops -from .asgi.protocols import protocols_http, protocols_ws -from .asgi.server import run as asgi_run from .logger import LOG_LEVELS +from .server import run as sgi_run def find_app_module(): @@ -244,33 +242,22 @@ def main(self, *args, **kwargs): '--host', '-h', default='127.0.0.1', help='The interface to bind to.') @click.option( '--port', '-p', type=int, default=8000, help='The port to bind to.') +@click.option( + '--interface', type=click.Choice(['rsgi', 'asgi']), default='rsgi', + help='Application interface.') @click.option( '--loop', type=click.Choice(loops.keys()), default='auto', help='Event loop implementation.') -@click.option( - '--http-protocol', type=click.Choice(protocols_http.keys()), - default='auto', help='HTTP protocol implementation.') -@click.option( - '--ws-protocol', type=click.Choice(protocols_ws.keys()), - default='auto', help='Websocket protocol implementation.') @click.option( '--ssl-certfile', type=str, default=None, help='SSL certificate file') @click.option( '--ssl-keyfile', type=str, default=None, help='SSL key file') -@click.option( - '--ssl-cert-reqs', type=int, default=ssl.CERT_NONE, - help='Whether client certificate is required (see ssl module)') -@click.option( - '--ssl-ca-certs', type=str, default=None, help='CA certificates file') @click.option( '--reloader/--no-reloader', is_flag=True, default=True, help='Runs with reloader.') @pass_script_info def develop_command( - info, host, port, - loop, http_protocol, ws_protocol, - ssl_certfile, ssl_keyfile, ssl_cert_reqs, ssl_ca_certs, - reloader + info, host, port, interface, loop, ssl_certfile, ssl_keyfile, reloader ): os.environ["EMMETT_RUN_ENV"] = 'true' app_target = info._get_import_name() @@ -296,21 +283,17 @@ def develop_command( from ._reloader import run_with_reloader runner = run_with_reloader else: - runner = asgi_run + runner = sgi_run runner( + interface, app_target, host, port, loop=loop, - proto_http=http_protocol, - proto_ws=ws_protocol, log_level='debug', - access_log=True, ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile, - ssl_cert_reqs=ssl_cert_reqs, - ssl_ca_certs=ssl_ca_certs ) @@ -321,69 +304,39 @@ def develop_command( '--port', '-p', type=int, default=8000, help='The port to bind to.') @click.option( "--workers", type=int, default=1, help="Number of worker processes. Defaults to 1.") +@click.option( + '--interface', type=click.Choice(['rsgi', 'asgi']), default='rsgi', + help='Application interface.') @click.option( '--loop', type=click.Choice(loops.keys()), default='auto', help='Event loop implementation.') -@click.option( - '--http-protocol', type=click.Choice(protocols_http.keys()), - default='auto', help='HTTP protocol implementation.') -@click.option( - '--ws-protocol', type=click.Choice(protocols_ws.keys()), - default='auto', help='Websocket protocol implementation.') @click.option( '--log-level', type=click.Choice(LOG_LEVELS.keys()), default='info', help='Logging level.') -@click.option( - '--access-log/--no-access-log', is_flag=True, default=True, - help='Enable/Disable access log.') -@click.option( - '--proxy-headers/--no-proxy-headers', is_flag=True, default=False, - help='Enable/Disable proxy headers.') -@click.option( - '--proxy-trust-ips', type=str, default=None, - help='Comma seperated list of IPs to trust with proxy headers') -@click.option( - '--max-concurrency', type=int, - help='The maximum number of concurrent connections.') @click.option( '--backlog', type=int, default=2048, help='Maximum number of connections to hold in backlog') -@click.option( - '--keep-alive-timeout', type=int, default=0, - help='Keep alive timeout for connections.') @click.option( '--ssl-certfile', type=str, default=None, help='SSL certificate file') @click.option( '--ssl-keyfile', type=str, default=None, help='SSL key file') -@click.option( - '--ssl-cert-reqs', type=int, default=ssl.CERT_NONE, - help='Whether client certificate is required (see ssl module)') -@click.option( - '--ssl-ca-certs', type=str, default=None, help='CA certificates file') @pass_script_info def serve_command( - info, host, port, workers, - loop, http_protocol, ws_protocol, - log_level, access_log, - proxy_headers, proxy_trust_ips, - max_concurrency, backlog, keep_alive_timeout, - ssl_certfile, ssl_keyfile, ssl_cert_reqs, ssl_ca_certs + info, host, port, workers, interface, loop, log_level, backlog, + ssl_certfile, ssl_keyfile ): app_target = info._get_import_name() - asgi_run( + sgi_run( + interface, app_target, - host=host, port=port, - loop=loop, proto_http=http_protocol, proto_ws=ws_protocol, - log_level=log_level, access_log=access_log, - proxy_headers=proxy_headers, proxy_trust_ips=proxy_trust_ips, + host=host, + port=port, + loop=loop, + log_level=log_level, workers=workers, - limit_concurrency=max_concurrency, backlog=backlog, - timeout_keep_alive=keep_alive_timeout, ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile, - ssl_cert_reqs=ssl_cert_reqs, - ssl_ca_certs=ssl_ca_certs ) diff --git a/emmett/ctx.py b/emmett/ctx.py index ee0e09f2..73bee75f 100644 --- a/emmett/ctx.py +++ b/emmett/ctx.py @@ -39,21 +39,12 @@ class RequestContext(Context): def __init__( self, app, - scope, - receive, - send, - wrapper_request, - wrapper_response + request, + response ): self.app = app - self.request = wrapper_request( - scope, - receive, - send, - app.config.request_max_content_length, - app.config.request_body_timeout - ) - self.response = wrapper_response() + self.request = request + self.response = response self.session = None @property @@ -70,13 +61,9 @@ def language(self): class WSContext(Context): __slots__ = ["websocket", "session"] - def __init__(self, app, scope, receive, send, wrapper_websocket): + def __init__(self, app, websocket): self.app = app - self.websocket = wrapper_websocket( - scope, - receive, - send - ) + self.websocket = websocket self.session = None @cachedprop diff --git a/emmett/http.py b/emmett/http.py index c63f6851..0fc22b98 100644 --- a/emmett/http.py +++ b/emmett/http.py @@ -19,10 +19,13 @@ from hashlib import md5 from typing import Any, BinaryIO, Dict, Generator, Tuple +from granian.rsgi import Response as RSGIResponse + from ._internal import loop_open_file from .ctx import current from .libs.contenttype import contenttype + status_codes = { 100: '100 CONTINUE', 101: '101 SWITCHING PROTOCOLS', @@ -87,6 +90,13 @@ def headers(self) -> Generator[Tuple[bytes, bytes], None, None]: for cookie in self._cookies.values(): yield b'set-cookie', str(cookie)[12:].encode('latin-1') + @property + def rsgi_headers(self) -> Generator[Tuple[str, str], None, None]: + for key, val in self._headers.items(): + yield key, val + for cookie in self._cookies.values(): + yield 'set-cookie', str(cookie)[12:] + async def _send_headers(self, send): await send({ 'type': 'http.response.start', @@ -97,10 +107,16 @@ async def _send_headers(self, send): async def _send_body(self, send): await send({'type': 'http.response.body'}) - async def send(self, scope, send): + async def asgi(self, scope, send): await self._send_headers(send) await self._send_body(send) + def rsgi(self): + return RSGIResponse.empty( + self.status_code, + list(self.rsgi_headers) + ) + class HTTPBytes(HTTPResponse): def __init__( @@ -120,6 +136,13 @@ async def _send_body(self, send): 'more_body': False }) + def rsgi(self): + return RSGIResponse.bytes( + self.body, + self.status_code, + list(self.rsgi_headers) + ) + class HTTP(HTTPResponse): def __init__( @@ -143,6 +166,13 @@ async def _send_body(self, send): 'more_body': False }) + def rsgi(self): + return RSGIResponse.str( + self.body, + self.status_code, + list(self.rsgi_headers) + ) + class HTTPRedirect(HTTPResponse): def __init__( @@ -183,7 +213,7 @@ def _get_stat_headers(self, stat_data): 'etag': etag } - async def send(self, scope, send): + async def asgi(self, scope, send): try: stat_data = os.stat(self.file_path) if not stat.S_ISREG(stat_data.st_mode): @@ -210,6 +240,23 @@ async def _send_body(self, send): 'more_body': more_body, }) + def rsgi(self): + try: + stat_data = os.stat(self.file_path) + if not stat.S_ISREG(stat_data.st_mode): + return HTTP(403).rsgi() + self._headers.update(self._get_stat_headers(stat_data)) + except IOError as e: + if e.errno == errno.EACCES: + return HTTP(403).rsgi() + return HTTP(404).rsgi() + + return RSGIResponse.file( + self.file_path, + self.status_code, + list(self.rsgi_headers) + ) + class HTTPIO(HTTPResponse): def __init__( @@ -229,7 +276,7 @@ def _get_io_headers(self): 'content-length': content_length } - async def send(self, scope, send): + async def asgi(self, scope, send): self._headers.update(self._get_io_headers()) await self._send_headers(send) await self._send_body(send) diff --git a/emmett/rsgi/__init__.py b/emmett/rsgi/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/emmett/rsgi/handlers.py b/emmett/rsgi/handlers.py new file mode 100644 index 00000000..70e9bffd --- /dev/null +++ b/emmett/rsgi/handlers.py @@ -0,0 +1,305 @@ +# -*- coding: utf-8 -*- +""" + emmett.rsgi.handlers + -------------------- + + Provides RSGI handlers. + + :copyright: 2014 Giovanni Barillari + :license: BSD-3-Clause +""" + +from __future__ import annotations + +import asyncio +import os +import re + +from typing import Awaitable, Callable, Optional, Tuple + +from granian.rsgi import Scope, HTTPProtocol, WebsocketProtocol, WebsocketMessageType + +from ..ctx import RequestContext, WSContext, current +from ..debug import smart_traceback, debug_handler +from ..http import HTTPResponse, HTTPFile, HTTP +from ..utils import cachedprop +from ..wrappers.response import Response + +from .helpers import WSTransport +from .wrappers import Request, Websocket + +REGEX_STATIC = re.compile( + r'^/static/(?P__[\w\-\.]+__/)?(?P_\d+\.\d+\.\d+/)?(?P.*?)$' +) +REGEX_STATIC_LANG = re.compile( + r'^/(?P\w{2}/)?static/(?P__[\w\-\.]__+/)?(?P_\d+\.\d+\.\d+/)?(?P.*?)$' +) + + +class Handler: + __slots__ = ['app'] + + def __init__(self, app): + self.app = app + + +class RequestHandler(Handler): + __slots__ = ['router'] + + def __init__(self, app): + super().__init__(app) + self._bind_router() + self._configure_methods() + + def _bind_router(self): + raise NotImplementedError + + def _configure_methods(self): + raise NotImplementedError + + +class HTTPHandler(RequestHandler): + __slots__ = ['pre_handler', 'static_handler', 'static_matcher', '__dict__'] + + def _bind_router(self): + self.router = self.app._router_http + + def _configure_methods(self): + self.static_matcher = ( + self._static_lang_matcher if self.app.language_force_on_url else + self._static_nolang_matcher + ) + self.static_handler = ( + self._static_handler if self.app.config.handle_static else + self.dynamic_handler + ) + self.pre_handler = ( + self._prefix_handler if self.router._prefix_main else + self.static_handler + ) + + async def __call__( + self, + scope: Scope, + protocol: HTTPProtocol + ): + try: + http = await self.pre_handler(scope, protocol, scope.path) + except asyncio.TimeoutError: + self.app.log.warn( + f"Timeout sending response: ({scope.path})" + ) + return http.rsgi() + + @cachedprop + def error_handler(self) -> Callable[[], Awaitable[str]]: + return ( + self._debug_handler if self.app.debug else self.exception_handler + ) + + @cachedprop + def exception_handler(self) -> Callable[[], Awaitable[str]]: + return self.app.error_handlers.get(500, self._exception_handler) + + @staticmethod + async def _http_response(code: int) -> HTTPResponse: + return HTTP(code) + + def _prefix_handler( + self, + scope: Scope, + protocol: HTTPProtocol, + path: str + ) -> Awaitable[HTTPResponse]: + if not path.startswith(self.router._prefix_main): + return self._http_response(404) + path = path[self.router._prefix_main_len:] or '/' + return self.static_handler(scope, protocol, path) + + def _static_lang_matcher( + self, path: str + ) -> Tuple[Optional[str], Optional[str]]: + match = REGEX_STATIC_LANG.match(path) + if match: + lang, mname, version, file_name = match.group('l', 'm', 'v', 'f') + if mname: + mod = self.app._modules.get(mname) + spath = mod._static_path if mod else self.app.static_path + else: + spath = self.app.static_path + static_file = os.path.join(spath, file_name) + if lang: + lang_file = os.path.join(spath, lang, file_name) + if os.path.exists(lang_file): + static_file = lang_file + return static_file, version + return None, None + + def _static_nolang_matcher( + self, path: str + ) -> Tuple[Optional[str], Optional[str]]: + if path.startswith('/static'): + mname, version, file_name = REGEX_STATIC.match(path).group('m', 'v', 'f') + if mname: + mod = self.app._modules.get(mname[2:-3]) + static_file = os.path.join(mod._static_path, file_name) if mod else None + else: + static_file = os.path.join(self.app.static_path, file_name) + return static_file, version + return None, None + + async def _static_response(self, file_path: str) -> HTTPFile: + return HTTPFile(file_path) + + def _static_handler( + self, + scope: Scope, + protocol: HTTPProtocol, + path: str + ) -> Awaitable[HTTPResponse]: + #: handle internal assets + if path.startswith('/__emmett__'): + file_name = path[12:] + static_file = os.path.join( + os.path.dirname(__file__), '..', 'assets', file_name) + if os.path.splitext(static_file)[1] == 'html': + return self._http_response(404) + return self._static_response(static_file) + #: handle app assets + static_file, _ = self.static_matcher(path) + if static_file: + return self._static_response(static_file) + return self.dynamic_handler(scope, protocol, path) + + async def dynamic_handler( + self, + scope: Scope, + protocol: HTTPProtocol, + path: str + ) -> HTTPResponse: + request = Request( + scope, + path, + protocol, + max_content_length=self.app.config.request_max_content_length, + body_timeout=self.app.config.request_body_timeout + ) + response = Response() + ctx = RequestContext(self.app, request, response) + ctx_token = current._init_(ctx) + try: + http = await self.router.dispatch(request, response) + except HTTPResponse as http_exception: + http = http_exception + #: render error with handlers if in app + error_handler = self.app.error_handlers.get(http.status_code) + if error_handler: + http = HTTP( + http.status_code, + await error_handler(), + headers=response.headers, + cookies=response.cookies + ) + except Exception: + self.app.log.exception('Application exception:') + http = HTTP( + 500, + await self.error_handler(), + headers=response.headers + ) + finally: + current._close_(ctx_token) + return http + + async def _debug_handler(self) -> str: + current.response.headers._data['content-type'] = ( + 'text/html; charset=utf-8' + ) + return debug_handler(smart_traceback(self.app)) + + async def _exception_handler(self) -> str: + current.response.headers._data['content-type'] = 'text/plain' + return 'Internal error' + + +class WSHandler(RequestHandler): + __slots__ = ['pre_handler', '__dict__'] + + def _bind_router(self): + self.router = self.app._router_ws + + def _configure_methods(self): + self.pre_handler = ( + self._prefix_handler if self.router._prefix_main else + self.dynamic_handler + ) + + async def __call__( + self, + scope: Scope, + protocol: WebsocketProtocol + ): + transport = WSTransport(protocol) + task_transport = asyncio.create_task(self.handle_transport(transport)) + task_request = asyncio.create_task(self.handle_request(scope, transport)) + _, pending = await asyncio.wait( + [task_request, task_transport], return_when=asyncio.FIRST_COMPLETED + ) + for task in pending: + task.cancel() + return self._close_connection(transport) + + async def handle_transport(self, transport: WSTransport): + await transport.accepted.wait() + while True: + msg = await transport.transport.receive() + if msg.kind == WebsocketMessageType.close: + transport.interrupted + break + await transport.input.put(msg) + + def handle_request( + self, + scope: Scope, + transport: WSTransport + ): + return self.pre_handler(scope, transport, scope.path) + + async def _empty_awaitable(self): + return + + def _prefix_handler( + self, + scope: Scope, + transport: WSTransport, + path: str + ) -> Awaitable[None]: + if not path.startswith(self.router._prefix_main): + transport.status = 404 + return self._empty_awaitable() + path = path[self.router._prefix_main_len:] or '/' + return self.dynamic_handler(scope, transport, path) + + async def dynamic_handler( + self, + scope: Scope, + transport: WSTransport, + path: str + ): + ctx = WSContext(self.app, Websocket(scope, path, transport)) + ctx_token = current._init_(ctx) + try: + await self.router.dispatch(ctx.websocket) + except HTTPResponse as http: + transport.status = http.status_code + except asyncio.CancelledError: + if not transport.interrupted: + self.app.log.exception('Application exception:') + except Exception: + transport.status = 500 + self.app.log.exception('Application exception:') + finally: + current._close_(ctx_token) + + async def _close_connection(self, transport: WSTransport): + return transport.protocol.close(transport.status) diff --git a/emmett/rsgi/helpers.py b/emmett/rsgi/helpers.py new file mode 100644 index 00000000..f23fc4b2 --- /dev/null +++ b/emmett/rsgi/helpers.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +""" + emmett.rsgi.helpers + ------------------- + + Provides RSGI helpers + + :copyright: 2014 Giovanni Barillari + :license: BSD-3-Clause +""" + +import asyncio + +from granian.rsgi import WebsocketProtocol + + +class WSTransport: + __slots__ = [ + 'protocol', 'transport', + 'accepted', 'closed', + 'interrupted', 'status' + ] + + def __init__( + self, + protocol: WebsocketProtocol + ) -> None: + self.protocol = protocol + self.transport = None + self.accepted = asyncio.Event() + self.closed = asyncio.Event() + self.input = asyncio.Queue() + self.interrupted = False + self.status = 200 + + async def init(self): + self.transport = await self.protocol.accept() + self.accepted.set() + + @property + def receive(self): + return self.input.get diff --git a/emmett/rsgi/wrappers.py b/emmett/rsgi/wrappers.py new file mode 100644 index 00000000..9e2079e3 --- /dev/null +++ b/emmett/rsgi/wrappers.py @@ -0,0 +1,135 @@ +import asyncio + +from datetime import datetime +from typing import Any, Dict, List, Union, Optional +from urllib.parse import parse_qs + +from granian.rsgi import Scope, HTTPProtocol, WebsocketMessageType + +from .helpers import WSTransport +from ..datastructures import sdict +from ..http import HTTP +from ..utils import cachedprop +from ..wrappers.helpers import regex_client +from ..wrappers.request import Request as _Request +from ..wrappers.websocket import Websocket as _Websocket + + +class RSGIIngressMixin: + def __init__( + self, + scope: Scope, + path: str, + protocol: Union[HTTPProtocol, WSTransport] + ): + self._scope = scope + self._proto = protocol + self.scheme = scope.scheme + self.path = path + + @property + def headers(self): + return self._scope.headers + + @cachedprop + def query_params(self) -> sdict[str, Union[str, List[str]]]: + rv: sdict[str, Any] = sdict() + for key, values in parse_qs( + self._scope.query_string, keep_blank_values=True + ).items(): + if len(values) == 1: + rv[key] = values[0] + continue + rv[key] = values + return rv + + +class Request(RSGIIngressMixin, _Request): + __slots__ = ['_scope', '_proto'] + + def __init__( + self, + scope: Scope, + path: str, + protocol: HTTPProtocol, + max_content_length: Optional[int] = None, + body_timeout: Optional[int] = None + ): + super().__init__(scope, path, protocol) + self.max_content_length = max_content_length + self.body_timeout = body_timeout + self._now = datetime.utcnow() + self.method = scope.method + + @property + def _multipart_headers(self): + return dict(self.headers.items()) + + @cachedprop + async def body(self) -> bytes: + if ( + self.max_content_length and + self.content_length > self.max_content_length + ): + raise HTTP(413, 'Request entity too large') + try: + rv = await asyncio.wait_for(self._proto(), timeout=self.body_timeout) + except asyncio.TimeoutError: + raise HTTP(408, 'Request timeout') + return rv + + @cachedprop + def client(self) -> str: + g = regex_client.search(self.headers.get('x-forwarded-for', '')) + client = ( + (g.group() or '').split(',')[0] if g else ( + self._scope.client[0] if self._scope.client else None + ) + ) + if client in (None, '', 'unknown', 'localhost'): + client = '::1' if self.host.startswith('[') else '127.0.0.1' + return client # type: ignore + + async def push_promise(self, path: str): + raise NotImplementedError("RSGI protocol doesn't support HTTP2 push.") + + +class Websocket(RSGIIngressMixin, _Websocket): + __slots__ = ['_scope', '_proto'] + + def __init__( + self, + scope: Scope, + path: str, + protocol: WSTransport + ): + super().__init__(scope, path, protocol) + self._flow_receive = None + self._flow_send = None + self.receive = self._accept_and_receive + self.send = self._accept_and_send + + async def accept( + self, + headers: Optional[Dict[str, str]] = None, + subprotocol: Optional[str] = None + ): + if self._proto.transport: + return + await self._proto.init() + self.receive = self._wrapped_receive + self.send = self._wrapped_send + + async def _wrapped_receive(self) -> Any: + data = (await self._proto.receive()).data + for method in self._flow_receive: # type: ignore + data = method(data) + return data + + async def _wrapped_send(self, data: Any): + for method in self._flow_send: # type: ignore + data = method(data) + if isinstance(data, str): + await self._proto.transport.send_str(data) + else: + await self._proto.transport.send_bytes(data) diff --git a/emmett/server.py b/emmett/server.py new file mode 100644 index 00000000..31f5e6b3 --- /dev/null +++ b/emmett/server.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +""" + emmett.server + ------------- + + Provides server wrapper over granian + + :copyright: 2014 Giovanni Barillari + :license: BSD-3-Clause +""" + +from typing import Optional + +from granian import Granian + + +def run( + interface, + app, + host='127.0.0.1', + port=8000, + loop='auto', + log_level=None, + workers=1, + threads=None, + threading_mode='runtime', + backlog=2048, + enable_websockets=True, + ssl_certfile: Optional[str] = None, + ssl_keyfile: Optional[str] = None +): + app_path = ":".join[app[0], app[1] or "app"] + runner = Granian( + app_path, + address=host, + port=port, + interface=interface, + workers=workers, + threads=threads, + threading_mode=threading_mode, + loop=loop, + websockets=enable_websockets, + backlog=backlog, + log_level=log_level, + ssl_cert=ssl_certfile, + ssl_key=ssl_keyfile + ) + runner.serve() diff --git a/emmett/sessions.py b/emmett/sessions.py index 74e67631..c55ab10c 100644 --- a/emmett/sessions.py +++ b/emmett/sessions.py @@ -24,7 +24,7 @@ from .datastructures import sdict, SessionData from .pipeline import Pipe from .security import secure_loads, secure_dumps, uuid -from .wrappers import ScopeWrapper +from .wrappers import IngressWrapper try: from emmett_crypto import symmetric as crypto_symmetric @@ -51,7 +51,7 @@ def __init__( ) self.cookie_data = cookie_data or {} - def _load_session(self, wrapper: ScopeWrapper): + def _load_session(self, wrapper: IngressWrapper): raise NotImplementedError def _new_session(self) -> SessionData: diff --git a/emmett/testing/client.py b/emmett/testing/client.py index 38a6a86f..cd5dad65 100644 --- a/emmett/testing/client.py +++ b/emmett/testing/client.py @@ -20,9 +20,9 @@ from io import BytesIO from ..asgi.handlers import HTTPHandler +from ..asgi.wrappers import Request from ..ctx import RequestContext, current from ..http import HTTP, HTTPResponse -from ..wrappers.request import Request from ..wrappers.response import Response from ..utils import cachedprop from .env import ScopeBuilder @@ -55,17 +55,18 @@ def __exit__(self, exc_type, exc_value, tb): class ClientHTTPHandler(HTTPHandler): async def dynamic_handler(self, scope, receive, send): - ctx = RequestContext( - self.app, + request = Request( scope, receive, send, - Request, - Response + max_content_length=self.app.config.request_max_content_length, + body_timeout=self.app.config.request_body_timeout ) + response = Response() + ctx = RequestContext(self.app, request, response) ctx_token = current._init_(ctx) try: - http = await self.router.dispatch(ctx.request, ctx.response) + http = await self.router.dispatch(request, response) except HTTPResponse as http_exception: http = http_exception #: render error with handlers if in app @@ -74,15 +75,15 @@ async def dynamic_handler(self, scope, receive, send): http = HTTP( http.status_code, await error_handler(), - headers=ctx.response.headers, - cookies=ctx.response.cookies + headers=response.headers, + cookies=response.cookies ) except Exception: self.app.log.exception('Application exception:') http = HTTP( 500, await self.error_handler(), - headers=ctx.response.headers + headers=response.headers ) finally: scope['emt.ctx'] = ClientContext(ctx) diff --git a/emmett/wrappers/__init__.py b/emmett/wrappers/__init__.py index f8404485..6d6abd35 100644 --- a/emmett/wrappers/__init__.py +++ b/emmett/wrappers/__init__.py @@ -13,16 +13,14 @@ import re +from abc import ABCMeta, abstractmethod from http.cookies import SimpleCookie -from typing import Any, List, Type, TypeVar, Union -from urllib.parse import parse_qs +from typing import Any, List, Mapping, Type, TypeVar, Union -from ..asgi.typing import Scope, Receive, Send from ..datastructures import Accept, sdict from ..language.helpers import LanguageAccept from ..typing import T from ..utils import cachedprop -from .helpers import Headers AcceptType = TypeVar("AcceptType", bound=Accept) @@ -39,20 +37,19 @@ def __setitem__(self, name: str, value: Any): setattr(self, name, value) -class ScopeWrapper(Wrapper): - __slots__ = ('_scope', '_receive', '_send', 'scheme', 'path') +class IngressWrapper(Wrapper, metaclass=ABCMeta): + __slots__ = ['scheme', 'path'] - def __init__( - self, - scope: Scope, - receive: Receive, - send: Send - ): - self._scope = scope - self._receive = receive - self._send = send - self.scheme: str = scope['scheme'] - self.path: str = scope['emt.path'] + scheme: str + path: str + + @property + @abstractmethod + def headers(self) -> Mapping[str, str]: ... + + @cachedprop + def host(self) -> str: + return self.headers.get('host') def __parse_accept_header( self, @@ -71,14 +68,6 @@ def __parse_accept_header( result.append((match.group(1), quality)) return cls(result) - @cachedprop - def headers(self) -> Headers: - return Headers(self._scope) - - @cachedprop - def host(self) -> str: - return self.headers.get('host') - @cachedprop def accept_language(self) -> LanguageAccept: return self.__parse_accept_header( @@ -92,14 +81,6 @@ def cookies(self) -> SimpleCookie: cookies.load(cookie) return cookies - @cachedprop - def query_params(self) -> sdict[str, Union[str, List[str]]]: - rv: sdict[str, Any] = sdict() - for key, values in parse_qs( - self._scope['query_string'].decode('latin-1'), keep_blank_values=True - ).items(): - if len(values) == 1: - rv[key] = values[0] - continue - rv[key] = values - return rv + @property + @abstractmethod + def query_params(self) -> sdict[str, Union[str, List[str]]]: ... diff --git a/emmett/wrappers/helpers.py b/emmett/wrappers/helpers.py index f990d080..c3ae3a9b 100644 --- a/emmett/wrappers/helpers.py +++ b/emmett/wrappers/helpers.py @@ -9,14 +9,13 @@ :license: BSD-3-Clause """ +import re + from typing import ( - Any, BinaryIO, - Callable, Dict, Iterable, Iterator, - Mapping, MutableMapping, Optional, Tuple, @@ -25,56 +24,7 @@ from .._internal import loop_copyfileobj - -class Headers(Mapping[str, str]): - __slots__ = ["_data"] - - def __init__(self, scope: Dict[str, Any]): - self._data: Dict[bytes, bytes] = { - key: val for key, val in scope["headers"] - } - - __hash__ = None # type: ignore - - def __getitem__(self, key: str) -> str: - return self._data[key.lower().encode("latin-1")].decode("latin-1") - - def __contains__(self, key: str) -> bool: # type: ignore - return key.lower().encode("latin-1") in self._data - - def __iter__(self) -> Iterator[str]: - for key in self._data.keys(): - yield key.decode("latin-1") - - def __len__(self) -> int: - return len(self._data) - - def get( - self, - key: str, - default: Optional[Any] = None, - cast: Optional[Callable[[Any], Any]] = None - ) -> Any: - rv = self._data.get(key.lower().encode("latin-1")) - rv = rv.decode() if rv is not None else default # type: ignore - if cast is None: - return rv - try: - return cast(rv) - except ValueError: - return default - - def items(self) -> Iterator[Tuple[str, str]]: # type: ignore - for key, value in self._data.items(): - yield key.decode("latin-1"), value.decode("latin-1") - - def keys(self) -> Iterator[str]: # type: ignore - for key in self._data.keys(): - yield key.decode("latin-1") - - def values(self) -> Iterator[str]: # type: ignore - for value in self._data.values(): - yield value.decode("latin-1") +regex_client = re.compile(r'[\w\-:]+(\.[\w\-]+)*\.?') class ResponseHeaders(MutableMapping[str, str]): @@ -163,7 +113,3 @@ def __repr__(self) -> str: return ( f'<{self.__class__.__name__}: ' f'{self.filename} ({self.content_type})') - - -class RequestCancelled(Exception): - pass diff --git a/emmett/wrappers/request.py b/emmett/wrappers/request.py index e3bea436..2aa2ccb7 100644 --- a/emmett/wrappers/request.py +++ b/emmett/wrappers/request.py @@ -9,101 +9,29 @@ :license: BSD-3-Clause """ -import asyncio -import re - +from abc import abstractmethod from cgi import FieldStorage, parse_header -from datetime import datetime from io import BytesIO from urllib.parse import parse_qs -from typing import Any, Optional +from typing import Any import pendulum -from ..asgi.typing import Scope, Receive, Send from ..datastructures import sdict -from ..http import HTTP from ..parsers import Parsers from ..utils import cachedprop -from . import ScopeWrapper -from .helpers import FileStorage, RequestCancelled - -_regex_client = re.compile(r'[\w\-:]+(\.[\w\-]+)*\.?') -_push_headers = { - "accept", - "accept-encoding", - "accept-language", - "cache-control", - "user-agent" -} - - -class Body: - __slots__ = ('_data', '_receive', '_max_content_length') - - def __init__(self, receive, max_content_length=None): - self._data = bytearray() - self._receive = receive - self._max_content_length = max_content_length - - def append(self, data: bytes): - if data == b'': - return - self._data.extend(data) - if ( - self._max_content_length is not None and - len(self._data) > self._max_content_length - ): - raise HTTP(413, 'Request entity too large') - - async def __load(self) -> bytes: - while True: - event = await self._receive() - if event['type'] == 'http.request': - self.append(event['body']) - if not event.get('more_body', False): - break - elif event['type'] == 'http.disconnect': - raise RequestCancelled - return bytes(self._data) - - def __await__(self): - return self.__load().__await__() - - -class Request(ScopeWrapper): - __slots__ = ['_now', 'method'] +from . import IngressWrapper +from .helpers import FileStorage - def __init__( - self, - scope: Scope, - receive: Receive, - send: Send, - max_content_length: Optional[int] = None, - body_timeout: Optional[int] = None - ): - super().__init__(scope, receive, send) - self.max_content_length = max_content_length - self.body_timeout = body_timeout - self._now = datetime.utcnow() - self.method = scope['method'] - @cachedprop - def _input(self): - return Body(self._receive, self.max_content_length) +class Request(IngressWrapper): + __slots__ = ['_now', 'method'] - @cachedprop - async def body(self) -> bytes: - if ( - self.max_content_length and - self.content_length > self.max_content_length - ): - raise HTTP(413, 'Request entity too large') - try: - rv = await asyncio.wait_for(self._input, timeout=self.body_timeout) - except asyncio.TimeoutError: - raise HTTP(408, 'Request timeout') - return rv + method: str + + @property + @abstractmethod + async def body(self) -> bytes: ... @cachedprop def now(self) -> pendulum.DateTime: @@ -160,11 +88,15 @@ def _load_params_form_urlencoded(self, data): rv[key] = values return rv, sdict() + @property + def _multipart_headers(self): + return self.headers + def _load_params_form_multipart(self, data): params, files = sdict(), sdict() field_storage = FieldStorage( BytesIO(data), - headers=self.headers, + headers=self._multipart_headers, environ={'REQUEST_METHOD': self.method}, keep_blank_values=True ) @@ -204,26 +136,5 @@ async def _load_params(self): self.content_type, self._load_params_missing) return loader(self, await self.body) - @cachedprop - def client(self) -> str: - g = _regex_client.search(self.headers.get('x-forwarded-for', '')) - client = ( - (g.group() or '').split(',')[0] if g else ( - self._scope['client'][0] if self._scope['client'] else None - ) - ) - if client in (None, '', 'unknown', 'localhost'): - client = '::1' if self.host.startswith('[') else '127.0.0.1' - return client # type: ignore - - async def push_promise(self, path: str): - if "http.response.push" not in self._scope.get("extensions", {}): - return - await self._send({ - "type": "http.response.push", - "path": path, - "headers": [ - (key.encode("latin-1"), self.headers[key].encode("latin-1")) - for key in _push_headers & set(self.headers.keys()) - ] - }) + @abstractmethod + async def push_promise(self, path: str): ... diff --git a/emmett/wrappers/websocket.py b/emmett/wrappers/websocket.py index 9b9b82cb..75ebf6de 100644 --- a/emmett/wrappers/websocket.py +++ b/emmett/wrappers/websocket.py @@ -9,64 +9,26 @@ :license: BSD-3-Clause """ -from typing import Any, Dict, List, Optional, Tuple +from abc import abstractmethod +from typing import Any, Dict, Optional -from ..asgi.typing import Scope, Receive, Send -from . import ScopeWrapper +from . import IngressWrapper -class Websocket(ScopeWrapper): - __slots__ = ('_accepted', 'receive', 'send', '_flow_receive', '_flow_send') - - def __init__( - self, - scope: Scope, - receive: Receive, - send: Send - ): - super().__init__(scope, receive, send) - self._accepted = False - self._flow_receive = None - self._flow_send = None - self.receive = self._accept_and_receive - self.send = self._accept_and_send +class Websocket(IngressWrapper): + __slots__ = ['_flow_receive', '_flow_send', 'receive', 'send'] def _bind_flow(self, flow_receive, flow_send): self._flow_receive = flow_receive self._flow_send = flow_send - @property - def _asgi_spec_version(self) -> int: - return int(''.join( - self._scope.get('asgi', {}).get('spec_version', '2.0').split('.') - )) - - def _encode_headers( - self, - headers: Dict[str, str] - ) -> List[Tuple[bytes, bytes]]: - return [ - (key.encode('utf-8'), val.encode('utf-8')) - for key, val in headers.items() - ] - + @abstractmethod async def accept( self, headers: Optional[Dict[str, str]] = None, subprotocol: Optional[str] = None ): - if self._accepted: - return - message: Dict[str, Any] = { - 'type': 'websocket.accept', - 'subprotocol': subprotocol - } - if headers and self._asgi_spec_version > 20: - message['headers'] = self._encode_headers(headers) - await self._send(message) - self._accepted = True - self.receive = self._wrapped_receive - self.send = self._wrapped_send + ... async def _accept_and_receive(self) -> Any: await self.accept() @@ -76,16 +38,8 @@ async def _accept_and_send(self, data: Any): await self.accept() await self.send(data) - async def _wrapped_receive(self) -> Any: - data = await self._receive() - for method in self._flow_receive: # type: ignore - data = method(data) - return data + @abstractmethod + async def _wrapped_receive(self) -> Any: ... - async def _wrapped_send(self, data: Any): - for method in self._flow_send: # type: ignore - data = method(data) - if isinstance(data, str): - await self._send({'type': 'websocket.send', 'text': data}) - else: - await self._send({'type': 'websocket.send', 'bytes': data}) + @abstractmethod + async def _wrapped_send(self, data: Any): ... diff --git a/pyproject.toml b/pyproject.toml index 2a79d045..b11cbf48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ emmett = "emmett.cli:main" [tool.poetry.dependencies] python = "^3.7" click = ">=6.0" +granian = "0.1.0b2" h11 = ">= 0.12.0" h2 = ">= 3.2.0, < 4.1.0" pendulum = "~2.1.2" @@ -69,8 +70,8 @@ pytest-asyncio = "^0.15" psycopg2-binary = "^2.9.3" [tool.poetry.extras] -orjson = ["orjson"] crypto = ["emmett-crypto"] +orjson = ["orjson"] [tool.poetry.urls] "Issue Tracker" = "https://github.com/emmett-framework/emmett/issues" diff --git a/tests/helpers.py b/tests/helpers.py index 0e05659a..0be3146d 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -8,13 +8,11 @@ from contextlib import contextmanager -from emmett.asgi.handlers import RequestContext, WSContext -from emmett.ctx import current +from emmett.asgi.wrappers import Request, Websocket +from emmett.ctx import RequestContext, WSContext, current from emmett.serializers import Serializers from emmett.testing.env import ScopeBuilder -from emmett.wrappers.request import Request from emmett.wrappers.response import Response -from emmett.wrappers.websocket import Websocket json_dump = Serializers.get_for('json') diff --git a/tests/test_session.py b/tests/test_session.py index 5a4766aa..334af1cc 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -8,11 +8,10 @@ import pytest -from emmett.asgi.handlers import RequestContext -from emmett.ctx import current +from emmett.asgi.wrappers import Request +from emmett.ctx import RequestContext, current from emmett.sessions import SessionManager from emmett.testing.env import ScopeBuilder -from emmett.wrappers.request import Request from emmett.wrappers.response import Response diff --git a/tests/test_wrappers.py b/tests/test_wrappers.py index a52eb716..be7704f2 100644 --- a/tests/test_wrappers.py +++ b/tests/test_wrappers.py @@ -7,8 +7,8 @@ """ from helpers import current_ctx +from emmett.asgi.wrappers import Request from emmett.testing.env import ScopeBuilder -from emmett.wrappers.request import Request from emmett.wrappers.response import Response From 1d889d68c1114feb3d77cca3295269336c78903e Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Tue, 18 Oct 2022 22:35:23 +0200 Subject: [PATCH 03/29] rsgi: make uvicorn dependencies optional --- emmett/asgi/helpers.py | 50 ---------- emmett/asgi/server.py | 205 ----------------------------------------- emmett/asgi/workers.py | 54 ++++++++++- pyproject.toml | 14 +-- 4 files changed, 60 insertions(+), 263 deletions(-) delete mode 100644 emmett/asgi/server.py diff --git a/emmett/asgi/helpers.py b/emmett/asgi/helpers.py index 9156e7e1..d1518e31 100644 --- a/emmett/asgi/helpers.py +++ b/emmett/asgi/helpers.py @@ -14,10 +14,6 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple -from uvicorn.config import Config as UvicornConfig -from uvicorn.lifespan.on import LifespanOn -from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware - class Registry: __slots__ = ["_data"] @@ -79,52 +75,6 @@ def get(self, key: str) -> Callable[..., Any]: return builder(**packages) -class Config(UvicornConfig): - def setup_event_loop(self): - pass - - def load(self): - assert not self.loaded - - if self.is_ssl: - self.ssl = _create_ssl_context( - keyfile=self.ssl_keyfile, - certfile=self.ssl_certfile, - cert_reqs=self.ssl_cert_reqs, - ca_certs=self.ssl_ca_certs, - alpn_protocols=self.http.alpn_protocols - ) - else: - self.ssl = None - - encoded_headers = [ - (key.lower().encode("latin1"), value.encode("latin1")) - for key, value in self.headers - ] - self.encoded_headers = ( - encoded_headers if b"server" in dict(encoded_headers) else - [(b"server", b"Emmett")] + encoded_headers - ) - - self.http_protocol_class = self.http - self.ws_protocol_class = self.ws - self.lifespan_class = LifespanOn - - self.loaded_app = self.app - self.interface = "asgi3" - - if self.proxy_headers: - self.loaded_app = ProxyHeadersMiddleware( - self.loaded_app, trusted_hosts=self.forwarded_allow_ips - ) - - self.loaded = True - - @property - def is_ssl(self) -> bool: - return self.ssl_certfile is not None and self.ssl_keyfile is not None - - class RequestCancelled(Exception): ... diff --git a/emmett/asgi/server.py b/emmett/asgi/server.py deleted file mode 100644 index 33c13155..00000000 --- a/emmett/asgi/server.py +++ /dev/null @@ -1,205 +0,0 @@ -# -*- coding: utf-8 -*- -""" - emmett.asgi.server - ------------------ - - Provides ASGI server wrapper over uvicorn - - :copyright: 2014 Giovanni Barillari - :license: BSD-3-Clause -""" - -import logging -import multiprocessing -import os -import signal -import socket -import ssl -import sys -import threading - -from abc import ABCMeta, abstractmethod -from typing import Any, Optional, Tuple - -from uvicorn.config import logger as uvlogger -from uvicorn.server import Server - -from .._internal import locate_app -from ..extensions import Signals -from ..logger import LOG_LEVELS -from .helpers import Config -from .loops import loops -from .protocols import protocols_http, protocols_ws - -multiprocessing.allow_connection_pickling() - - -class Runner(metaclass=ABCMeta): - def __init__( - self, - app_target: Tuple[str, Optional[str]], - workers: int, - **kwargs: Any - ): - self.app_target = app_target - self.workers = workers - self.cfgdata = {**kwargs} - - @staticmethod - def serve(app_target, cfgdata, sockets=None): - cfg = {**cfgdata} - cfg["loop"] = loops.get(cfg.pop("loop")) - cfg["http"] = protocols_http.get(cfg.pop("proto_http")) - cfg["ws"] = protocols_ws.get(cfg.pop("proto_ws")) - - app = locate_app(*app_target) - app.send_signal(Signals.after_loop, loop=cfg["loop"]) - - cfg["access_log"] = ( - cfg["access_log"] is not None and cfg["access_log"] or - bool(app.debug) - ) - cfg["log_level"] = LOG_LEVELS[cfg["log_level"]] if cfg["log_level"] else ( - logging.DEBUG if app.debug else logging.WARNING - ) - cfg["forwarded_allow_ips"] = cfg.pop("proxy_trust_ips") - - Server(Config(app=app, **cfg)).run(sockets=sockets) - - @abstractmethod - def run(self): - ... - - -class SingleRunner(Runner): - def run(self): - self.serve(self.app_target, self.cfgdata) - - -class MultiRunner(Runner): - SIGNALS = { - signal.SIGINT, - signal.SIGTERM - } - - def bind_socket(self): - family = socket.AF_INET - if self.cfgdata["host"] and ":" in self.cfgdata["host"]: - family = socket.AF_INET6 - - sock = socket.socket(family=family) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - sock.bind((self.cfgdata["host"], self.cfgdata["port"])) - except OSError as exc: - uvlogger.error(exc) - sys.exit(1) - sock.set_inheritable(True) - return sock - - def signal_handler(self, *args, **kwargs): - self.exit_event.set() - - @staticmethod - def _subprocess_target(target, data, stdin_fileno): - if stdin_fileno is not None: - sys.stdin = os.fdopen(stdin_fileno) - target(**data) - - def _subprocess_spawn(self, target, **data): - try: - stdin_fileno = sys.stdin.fileno() - except OSError: - stdin_fileno = None - - return multiprocessing.get_context("spawn").Process( - target=self._subprocess_target, - kwargs={ - "target": target, - "data": data, - "stdin_fileno": stdin_fileno - } - ) - - def startup(self): - for sig in self.SIGNALS: - signal.signal(sig, self.signal_handler) - - for _ in range(self.workers): - proc = self._subprocess_spawn( - target=self.serve, - cfgdata=self.cfgdata, - app_target=self.app_target, - sockets=self.sockets - ) - proc.start() - self.processes.append(proc) - - def shutdown(self): - for proc in self.processes: - proc.join() - - def run(self): - self.processes = [] - self.sockets = [self.bind_socket()] - self.exit_event = threading.Event() - - self.startup() - self.exit_event.wait() - self.shutdown() - - -def run( - app, - host='127.0.0.1', - port=8000, - uds=None, - fd=None, - loop='auto', - proto_http='auto', - proto_ws='auto', - log_level=None, - access_log=None, - proxy_headers=False, - proxy_trust_ips=None, - workers=1, - limit_concurrency=None, - # limit_max_requests=None, - backlog=2048, - timeout_keep_alive=0, - # timeout_notify=30, - ssl_certfile: Optional[str] = None, - ssl_keyfile: Optional[str] = None, - ssl_cert_reqs: int = ssl.CERT_NONE, - ssl_ca_certs: Optional[str] = None -): - if proxy_trust_ips is None: - proxy_trust_ips = os.environ.get("PROXY_TRUST_IPS", "*") - - runner_cls = MultiRunner if workers > 1 else SingleRunner - - runner = runner_cls( - app, - workers, - host=host, - port=port, - uds=uds, - fd=fd, - loop=loop, - proto_http=proto_http, - proto_ws=proto_ws, - log_level=log_level, - access_log=access_log, - proxy_headers=proxy_headers, - proxy_trust_ips=proxy_trust_ips, - limit_concurrency=limit_concurrency, - # limit_max_requests=None, - backlog=backlog, - timeout_keep_alive=timeout_keep_alive, - # timeout_notify=30, - ssl_certfile=ssl_certfile, - ssl_keyfile=ssl_keyfile, - ssl_cert_reqs=ssl_cert_reqs, - ssl_ca_certs=ssl_ca_certs - ) - runner.run() diff --git a/emmett/asgi/workers.py b/emmett/asgi/workers.py index 94d799d3..ec3100e6 100644 --- a/emmett/asgi/workers.py +++ b/emmett/asgi/workers.py @@ -14,13 +14,65 @@ import signal from gunicorn.workers.base import Worker as _Worker +from uvicorn.config import Config as UvicornConfig +from uvicorn.lifespan.on import LifespanOn +from uvicorn.middleware.debug import DebugMiddleware +from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware from uvicorn.server import Server -from .helpers import Config +from .helpers import _create_ssl_context from .loops import loops from .protocols import protocols_http, protocols_ws +class Config(UvicornConfig): + def setup_event_loop(self): + pass + + def load(self): + assert not self.loaded + + if self.is_ssl: + self.ssl = _create_ssl_context( + keyfile=self.ssl_keyfile, + certfile=self.ssl_certfile, + cert_reqs=self.ssl_cert_reqs, + ca_certs=self.ssl_ca_certs, + alpn_protocols=self.http.alpn_protocols + ) + else: + self.ssl = None + + encoded_headers = [ + (key.lower().encode("latin1"), value.encode("latin1")) + for key, value in self.headers + ] + self.encoded_headers = ( + encoded_headers if b"server" in dict(encoded_headers) else + [(b"server", b"Emmett")] + encoded_headers + ) + + self.http_protocol_class = self.http + self.ws_protocol_class = self.ws + self.lifespan_class = LifespanOn + + self.loaded_app = self.app + self.interface = "asgi3" + + if self.debug: + self.loaded_app = DebugMiddleware(self.loaded_app) + if self.proxy_headers: + self.loaded_app = ProxyHeadersMiddleware( + self.loaded_app, trusted_hosts=self.forwarded_allow_ips + ) + + self.loaded = True + + @property + def is_ssl(self) -> bool: + return self.ssl_certfile is not None and self.ssl_keyfile is not None + + class Worker(_Worker): EMMETT_CONFIG = {} diff --git a/pyproject.toml b/pyproject.toml index b11cbf48..22d47716 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,8 +44,6 @@ emmett = "emmett.cli:main" python = "^3.7" click = ">=6.0" granian = "0.1.0b2" -h11 = ">= 0.12.0" -h2 = ">= 3.2.0, < 4.1.0" pendulum = "~2.1.2" pyaes = "~1.6.1" pyDAL = "17.3" @@ -53,15 +51,16 @@ python-rapidjson = "^1.0" pyyaml = "^5.4" renoir = "^1.5" severus = "^1.1" -uvicorn = "~0.19.0" -websockets = "^10.0" - -httptools = { version = "~0.5.0", markers = "sys_platform != 'win32'" } -uvloop = { version = "~0.17.0", markers = "sys_platform != 'win32'" } orjson = { version = "~3.8", optional = true } emmett-crypto = { version = "^0.2.0", optional = true } +uvicorn = { version = "^0.19.0", optional = true } +h11 = { version = ">= 0.12.0", optional = true } +h2 = { version = ">= 3.2.0, < 4.1.0", optional = true } +websockets = { version = "^10.0", optional = true } +httptools = { version = "~0.5.0", optional = true, markers = "sys_platform != 'win32'" } + [tool.poetry.dev-dependencies] ipaddress = "^1.0" pylint = "^2.4.4" @@ -72,6 +71,7 @@ psycopg2-binary = "^2.9.3" [tool.poetry.extras] crypto = ["emmett-crypto"] orjson = ["orjson"] +uvicorn = ["uvicorn", "h11", "h2", "httptools", "websockets"] [tool.poetry.urls] "Issue Tracker" = "https://github.com/emmett-framework/emmett/issues" From f84694d017eb59ff45d8537f20032313102b10e5 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Wed, 2 Nov 2022 01:24:31 +0100 Subject: [PATCH 04/29] rsgi: bump granian version --- emmett/http.py | 30 +++++++++++++++--------------- emmett/rsgi/handlers.py | 4 ++-- emmett/server.py | 2 +- pyproject.toml | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/emmett/http.py b/emmett/http.py index 0fc22b98..bdc4b5e7 100644 --- a/emmett/http.py +++ b/emmett/http.py @@ -19,7 +19,7 @@ from hashlib import md5 from typing import Any, BinaryIO, Dict, Generator, Tuple -from granian.rsgi import Response as RSGIResponse +from granian.rsgi import HTTPProtocol from ._internal import loop_open_file from .ctx import current @@ -111,8 +111,8 @@ async def asgi(self, scope, send): await self._send_headers(send) await self._send_body(send) - def rsgi(self): - return RSGIResponse.empty( + def rsgi(self, protocol: HTTPProtocol): + protocol.response_empty( self.status_code, list(self.rsgi_headers) ) @@ -136,11 +136,11 @@ async def _send_body(self, send): 'more_body': False }) - def rsgi(self): - return RSGIResponse.bytes( - self.body, + def rsgi(self, protocol: HTTPProtocol): + protocol.response_bytes( self.status_code, - list(self.rsgi_headers) + list(self.rsgi_headers), + self.body ) @@ -166,11 +166,11 @@ async def _send_body(self, send): 'more_body': False }) - def rsgi(self): - return RSGIResponse.str( - self.body, + def rsgi(self, protocol: HTTPProtocol): + protocol.response_str( self.status_code, - list(self.rsgi_headers) + list(self.rsgi_headers), + self.body ) @@ -240,7 +240,7 @@ async def _send_body(self, send): 'more_body': more_body, }) - def rsgi(self): + def rsgi(self, protocol: HTTPProtocol): try: stat_data = os.stat(self.file_path) if not stat.S_ISREG(stat_data.st_mode): @@ -251,10 +251,10 @@ def rsgi(self): return HTTP(403).rsgi() return HTTP(404).rsgi() - return RSGIResponse.file( - self.file_path, + protocol.response_file( self.status_code, - list(self.rsgi_headers) + list(self.rsgi_headers), + self.file_path ) diff --git a/emmett/rsgi/handlers.py b/emmett/rsgi/handlers.py index 70e9bffd..6ad9bc70 100644 --- a/emmett/rsgi/handlers.py +++ b/emmett/rsgi/handlers.py @@ -89,7 +89,7 @@ async def __call__( self.app.log.warn( f"Timeout sending response: ({scope.path})" ) - return http.rsgi() + http.rsgi(protocol) @cachedprop def error_handler(self) -> Callable[[], Awaitable[str]]: @@ -247,7 +247,7 @@ async def __call__( ) for task in pending: task.cancel() - return self._close_connection(transport) + self._close_connection(transport) async def handle_transport(self, transport: WSTransport): await transport.accepted.wait() diff --git a/emmett/server.py b/emmett/server.py index 31f5e6b3..1c67f04b 100644 --- a/emmett/server.py +++ b/emmett/server.py @@ -29,7 +29,7 @@ def run( ssl_certfile: Optional[str] = None, ssl_keyfile: Optional[str] = None ): - app_path = ":".join[app[0], app[1] or "app"] + app_path = ":".join([app[0], app[1] or "app"]) runner = Granian( app_path, address=host, diff --git a/pyproject.toml b/pyproject.toml index 22d47716..7fe30391 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ emmett = "emmett.cli:main" [tool.poetry.dependencies] python = "^3.7" click = ">=6.0" -granian = "0.1.0b2" +granian = "~0.1" pendulum = "~2.1.2" pyaes = "~1.6.1" pyDAL = "17.3" From 8d15f450a21c42e9499283e719cb219346a19523 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Wed, 2 Nov 2022 18:13:18 +0100 Subject: [PATCH 05/29] Drop custom uvicorn implementations --- emmett/asgi/helpers.py | 89 ------ emmett/asgi/loops/__init__.py | 18 -- emmett/asgi/loops/asyncio.py | 19 -- emmett/asgi/loops/auto.py | 17 -- emmett/asgi/loops/uvloop.py | 19 -- emmett/asgi/protocols/__init__.py | 2 - emmett/asgi/protocols/http/__init__.py | 12 - emmett/asgi/protocols/http/auto.py | 18 -- emmett/asgi/protocols/http/h11.py | 189 ------------ emmett/asgi/protocols/http/h2.py | 363 ------------------------ emmett/asgi/protocols/http/helpers.py | 302 -------------------- emmett/asgi/protocols/http/httptools.py | 61 ---- emmett/asgi/protocols/ws/__init__.py | 12 - emmett/asgi/protocols/ws/auto.py | 18 -- emmett/asgi/protocols/ws/websockets.py | 17 -- emmett/asgi/protocols/ws/wsproto.py | 17 -- emmett/asgi/workers.py | 84 +----- emmett/cli.py | 5 +- pyproject.toml | 3 +- 19 files changed, 18 insertions(+), 1247 deletions(-) delete mode 100644 emmett/asgi/loops/__init__.py delete mode 100644 emmett/asgi/loops/asyncio.py delete mode 100644 emmett/asgi/loops/auto.py delete mode 100644 emmett/asgi/loops/uvloop.py delete mode 100644 emmett/asgi/protocols/__init__.py delete mode 100644 emmett/asgi/protocols/http/__init__.py delete mode 100644 emmett/asgi/protocols/http/auto.py delete mode 100644 emmett/asgi/protocols/http/h2.py delete mode 100644 emmett/asgi/protocols/http/helpers.py delete mode 100644 emmett/asgi/protocols/ws/__init__.py delete mode 100644 emmett/asgi/protocols/ws/auto.py delete mode 100644 emmett/asgi/protocols/ws/websockets.py delete mode 100644 emmett/asgi/protocols/ws/wsproto.py diff --git a/emmett/asgi/helpers.py b/emmett/asgi/helpers.py index d1518e31..1a41792a 100644 --- a/emmett/asgi/helpers.py +++ b/emmett/asgi/helpers.py @@ -9,95 +9,6 @@ :license: BSD-3-Clause """ -import ssl -import sys - -from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple - - -class Registry: - __slots__ = ["_data"] - - def __init__(self): - self._data: Dict[str, Callable[..., Any]] = {} - - def __contains__(self, key: str) -> bool: - return key in self._data - - def keys(self) -> Iterable[str]: - return self._data.keys() - - def register(self, key: str) -> Callable[[], Callable[..., Any]]: - def wrap(builder: Callable[..., Any]) -> Callable[..., Any]: - self._data[key] = builder - return builder - return wrap - - def get(self, key: str) -> Callable[..., Any]: - try: - return self._data[key] - except KeyError: - raise RuntimeError(f"'{key}' implementation not available.") - - - -class BuilderRegistry(Registry): - __slots__ = [] - - def __init__(self): - self._data: Dict[str, Tuple[Callable[..., Any], List[str]]] = {} - - def register( - self, - key: str, - packages: Optional[List[str]] = None - ) -> Callable[[], Callable[..., Any]]: - packages = packages or [] - - def wrap(builder: Callable[..., Any]) -> Callable[..., Any]: - loaded_packages, implemented = {}, True - try: - for package in packages: - __import__(package) - loaded_packages[package] = sys.modules[package] - except ImportError: - implemented = False - if implemented: - self._data[key] = (builder, loaded_packages) - return builder - return wrap - - def get(self, key: str) -> Callable[..., Any]: - try: - builder, packages = self._data[key] - except KeyError: - raise RuntimeError(f"'{key}' implementation not available.") - return builder(**packages) - class RequestCancelled(Exception): ... - - -def _create_ssl_context( - certfile: str, - keyfile: str, - cert_reqs: int, - ca_certs: Optional[str], - alpn_protocols: List[str] -) -> ssl.SSLContext: - ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ctx.set_ciphers("ECDHE+AESGCM") - ctx.options |= ( - ssl.OP_NO_SSLv2 | - ssl.OP_NO_SSLv3 | - ssl.OP_NO_TLSv1 | - ssl.OP_NO_TLSv1_1 | - ssl.OP_NO_COMPRESSION - ) - ctx.set_alpn_protocols(alpn_protocols) - ctx.load_cert_chain(certfile, keyfile) - ctx.verify_mode = cert_reqs - if ca_certs: - ctx.load_verify_locations(ca_certs) - return ctx diff --git a/emmett/asgi/loops/__init__.py b/emmett/asgi/loops/__init__.py deleted file mode 100644 index 301178bb..00000000 --- a/emmett/asgi/loops/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -""" - emmett.asgi.loops - ----------------- - - :copyright: 2014 Giovanni Barillari - :license: BSD-3-Clause -""" - -from ..helpers import BuilderRegistry - -loops = BuilderRegistry() - -from . import ( - asyncio, - auto, - uvloop -) diff --git a/emmett/asgi/loops/asyncio.py b/emmett/asgi/loops/asyncio.py deleted file mode 100644 index 31f5ced5..00000000 --- a/emmett/asgi/loops/asyncio.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -""" - emmett.asgi.loops.asyncio - ------------------------- - - :copyright: 2014 Giovanni Barillari - :license: BSD-3-Clause -""" - -import asyncio - -from . import loops - - -@loops.register('asyncio') -def build_asyncio_loop(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - return loop diff --git a/emmett/asgi/loops/auto.py b/emmett/asgi/loops/auto.py deleted file mode 100644 index 5a2982e9..00000000 --- a/emmett/asgi/loops/auto.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- -""" - emmett.asgi.loops.auto - ---------------------- - - :copyright: 2014 Giovanni Barillari - :license: BSD-3-Clause -""" - -from . import loops - - -@loops.register('auto') -def build_auto_loop(): - if 'uvloop' in loops: - return loops.get('uvloop') - return loops.get('asyncio') diff --git a/emmett/asgi/loops/uvloop.py b/emmett/asgi/loops/uvloop.py deleted file mode 100644 index bb3b3017..00000000 --- a/emmett/asgi/loops/uvloop.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -""" - emmett.asgi.loops.uvloop - ------------------------ - - :copyright: 2014 Giovanni Barillari - :license: BSD-3-Clause -""" - -import asyncio - -from . import loops - - -@loops.register('uvloop', packages=['uvloop']) -def build_uv_loop(uvloop): - asyncio.get_event_loop().close() - asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) - return asyncio.get_event_loop() diff --git a/emmett/asgi/protocols/__init__.py b/emmett/asgi/protocols/__init__.py deleted file mode 100644 index 40afee61..00000000 --- a/emmett/asgi/protocols/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .http import protocols as protocols_http -from .ws import protocols as protocols_ws diff --git a/emmett/asgi/protocols/http/__init__.py b/emmett/asgi/protocols/http/__init__.py deleted file mode 100644 index ac22b902..00000000 --- a/emmett/asgi/protocols/http/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from ...helpers import Registry - -protocols = Registry() - -from . import h11 - -try: - from . import httptools -except ImportError: - pass - -from . import auto diff --git a/emmett/asgi/protocols/http/auto.py b/emmett/asgi/protocols/http/auto.py deleted file mode 100644 index 001f9628..00000000 --- a/emmett/asgi/protocols/http/auto.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -""" - emmett.asgi.protocols.http.auto - ------------------------------- - - Provides HTTP auto protocol loader - - :copyright: 2014 Giovanni Barillari - :license: BSD-3-Clause -""" - -from . import protocols - - -if "httptools" in protocols: - protocols.register("auto")(protocols.get("httptools")) -else: - protocols.register("auto")(protocols.get("h11")) diff --git a/emmett/asgi/protocols/http/h11.py b/emmett/asgi/protocols/http/h11.py index 5ca13068..e69de29b 100644 --- a/emmett/asgi/protocols/http/h11.py +++ b/emmett/asgi/protocols/http/h11.py @@ -1,189 +0,0 @@ -# -*- coding: utf-8 -*- -""" - emmett.asgi.protocols.http.h11 - ------------------------------ - - Provides HTTP h11 protocol implementation - - :copyright: 2014 Giovanni Barillari - :license: BSD-3-Clause -""" - -import asyncio - -import h11 - -from uvicorn.protocols.http.h11_impl import ( - HIGH_WATER_LIMIT, - H11Protocol as _H11Protocol, - RequestResponseCycle, - service_unavailable, - unquote -) - -from . import protocols -from .h2 import H2Protocol - - -@protocols.register("h11") -class H11Protocol(_H11Protocol): - alpn_protocols = ["h2", "http/1.1"] - h2_protocol_class = H2Protocol - - def handle_upgrade(self, event: h11.Request): - upgrade_value = None - for name, value in self.headers: - if name == b"upgrade": - upgrade_value = value.lower() - break - - if upgrade_value == b"websocket" and self.ws_protocol_class: - self.connections.discard(self) - output = [event.method, b" ", event.target, b" HTTP/1.1\r\n"] - for name, value in self.headers: - output += [name, b": ", value, b"\r\n"] - output.append(b"\r\n") - protocol = self.ws_protocol_class( - config=self.config, - server_state=self.server_state - ) - protocol.connection_made(self.transport) - protocol.data_received(b"".join(output)) - self.transport.set_protocol(protocol) - elif upgrade_value == b"h2c": - self.connections.discard(self) - self.transport.write( - self.conn.send( - h11.InformationalResponse( - status_code=101, - headers=self.headers - ) - ) - ) - protocol = self.h2_protocol_class( - config=self.config, - server_state=self.server_state, - _loop=self.loop - ) - protocol.handle_upgrade_from_h11(self.transport, event, self.headers) - self.transport.set_protocol(protocol) - else: - msg = "Unsupported upgrade request." - self.logger.warning(msg) - self.send_400_response(msg) - - def handle_h2_assumed(self): - self.connections.discard(self) - protocol = self.h2_protocol_class( - config=self.config, - server_state=self.server_state, - _loop=self.loop - ) - protocol.connection_made(self.transport) - protocol.data_received( - b"PRI * HTTP/2.0\r\n\r\n" + self.conn.trailing_data[0] - ) - self.transport.set_protocol(protocol) - - def handle_events(self): - while True: - try: - event = self.conn.next_event() - except h11.RemoteProtocolError: - msg = "Invalid HTTP request received." - self.logger.warning(msg) - self.send_400_response(msg) - return - event_type = type(event) - - if event_type is h11.NEED_DATA: - break - - elif event_type is h11.PAUSED: - # This case can occur in HTTP pipelining, so we need to - # stop reading any more data, and ensure that at the end - # of the active request/response cycle we handle any - # events that have been buffered up. - self.flow.pause_reading() - break - - elif event_type is h11.Request: - self.headers, upgrade_value = [], None - for name, value in event.headers: - lname = name.lower() - self.headers.append((lname, value)) - if lname == b"upgrade": - upgrade_value = value - - if upgrade_value: - self.handle_upgrade(event) - return - elif ( - event.method == b"PRI" and - event.target == b"*" and - event.http_version == b"2.0" - ): - self.handle_h2_assumed() - return - - raw_path, _, query_string = event.target.partition(b"?") - self.scope = { - "type": "http", - "asgi": { - "version": self.config.asgi_version, - "spec_version": "2.3", - }, - "http_version": event.http_version.decode("ascii"), - "server": self.server, - "client": self.client, - "scheme": self.scheme, - "method": event.method.decode("ascii"), - "root_path": self.root_path, - "path": unquote(raw_path.decode("ascii")), - "raw_path": raw_path, - "query_string": query_string, - "headers": self.headers - } - - # Handle 503 responses when 'limit_concurrency' is exceeded. - if self.limit_concurrency is not None and ( - len(self.connections) >= self.limit_concurrency - or len(self.tasks) >= self.limit_concurrency - ): - app = service_unavailable - message = "Exceeded concurrency limit." - self.logger.warning(message) - else: - app = self.app - - self.cycle = RequestResponseCycle( - scope=self.scope, - conn=self.conn, - transport=self.transport, - flow=self.flow, - logger=self.logger, - access_logger=self.access_logger, - access_log=self.access_log, - default_headers=self.server_state.default_headers, - message_event=asyncio.Event(), - on_response=self.on_response_complete, - ) - task = self.loop.create_task(self.cycle.run_asgi(app)) - task.add_done_callback(self.tasks.discard) - self.tasks.add(task) - - elif event_type is h11.Data: - if self.conn.our_state is h11.DONE: - continue - self.cycle.body += event.data - if len(self.cycle.body) > HIGH_WATER_LIMIT: - self.flow.pause_reading() - self.cycle.message_event.set() - - elif event_type is h11.EndOfMessage: - if self.conn.our_state is h11.DONE: - self.transport.resume_reading() - self.conn.start_next_cycle() - continue - self.cycle.more_body = False - self.cycle.message_event.set() diff --git a/emmett/asgi/protocols/http/h2.py b/emmett/asgi/protocols/http/h2.py deleted file mode 100644 index e3aab4cc..00000000 --- a/emmett/asgi/protocols/http/h2.py +++ /dev/null @@ -1,363 +0,0 @@ -# -*- coding: utf-8 -*- -""" - emmett.asgi.protocols.http.h2 - ----------------------------- - - Provides HTTP h2 protocol implementation - - :copyright: 2014 Giovanni Barillari - :license: BSD-3-Clause -""" - -from __future__ import annotations - -import asyncio - -from collections import defaultdict -from functools import partial -from typing import Any, List, Tuple -from urllib.parse import unquote - -import h11 -import h2.config -import h2.connection -import h2.events -import h2.exceptions - -from uvicorn.protocols.http.flow_control import HIGH_WATER_LIMIT -from uvicorn.protocols.utils import get_client_addr, get_path_with_query_string - -from .helpers import ( - TRACE_LOG_LEVEL, - ASGICycle, - Config, - HTTPProtocol, - ServerState, - _service_unavailable -) - - -class EventsRegistry(defaultdict): - def __init__(self): - super().__init__(lambda: (lambda *args, **kwargs: None)) - - def register(self, key: Any): - def wrap(f): - self[key] = f - return f - return wrap - - -class H2Protocol(HTTPProtocol): - __slots__ = ["conn", "streams"] - - alpn_protocols = ["h2"] - - def __init__( - self, - config: Config, - server_state: ServerState, - _loop=None - ): - super().__init__( - config=config, - server_state=server_state, - _loop=_loop - ) - self.conn = h2.connection.H2Connection( - config=h2.config.H2Configuration( - client_side=False, - header_encoding=None - ) - ) - self.streams = {} - - def connection_made(self, transport: asyncio.Transport, init: bool = False): - super().connection_made(transport) - if init: - self.conn.initiate_connection() - self.transport.write(self.conn.data_to_send()) - - def connection_lost(self, exc): - self.connections.discard(self) - - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % tuple(self.addr_remote) if self.addr_remote else "" - self.logger.log(TRACE_LOG_LEVEL, "%sConnection lost", prefix) - - for stream in self.streams.values(): - stream.message_event.set() - if self.flow is not None: - self.flow.resume_writing() - if exc is None: - self.transport.close() - - def handle_upgrade_from_h11( - self, - transport: asyncio.Protocol, - upgrade_event: h11.Request, - headers: List[Tuple[bytes, bytes]] - ): - self.connection_made(transport, init=False) - - settings = "" - headers = [ - (b":method", upgrade_event.method.encode("ascii")), - (b":path", upgrade_event.target), - (b":scheme", self.scheme.encode("ascii")) - ] - for name, value in headers: - if name == b"http2-settings": - settings = value.decode("latin-1") - elif name == b"host": - headers.append((b":authority", value)) - else: - headers.append((name, value)) - - self.conn.initiate_upgrade_connection(settings) - self.transport.write(self.conn.data_to_send()) - event = h2.events.RequestReceived() - event.stream_id = 1 - event.headers = headers - on_request_received(self, event) - - def data_received(self, data: bytes): - self._might_unset_keepalive() - - try: - events = self.conn.receive_data(data) - except h2.exceptions.ProtocolError: - self.transport.write(self.conn.data_to_send()) - self.transport.close() - return - - for event in events: - eventsreg[type(event)](self, event) - - def shutdown(self): - self.transport.write(self.conn.data_to_send()) - self.transport.close() - - def timeout_keep_alive_handler(self): - if self.transport.is_closing(): - return - - self.conn.close_connection() - self.transport.write(self.conn.data_to_send()) - self.transport.close() - - def on_response_complete(self, stream_id): - self.server_state.total_requests += 1 - self.streams.pop(stream_id, None) - if self.transport.is_closing(): - return - - if not self.streams: - self._might_unset_keepalive() - self.timeout_keep_alive_task = self.loop.call_later( - self.timeout_keep_alive, self.timeout_keep_alive_handler - ) - self.flow.resume_reading() - - -class H2ASGICycle(ASGICycle): - __slots__ = ["scheme", "host", "stream_id", "new_request"] - - def __init__( - self, - scope, - conn, - protocol: H2Protocol, - stream_id: int, - host: bytes - ): - super().__init__(scope, conn, protocol) - self.scheme = protocol.scheme.encode("ascii") - self.host = host - self.stream_id = stream_id - self.new_request = partial(on_request_received, protocol) - - async def send(self, message): - message_type = message["type"] - - if self.flow.write_paused and not self.disconnected: - await self.flow.drain() - - if self.disconnected: - return - - if message_type == "http.response.push": - push_stream_id = self.conn.get_next_available_stream_id() - headers = [ - (b":authority", self.host), - (b":method", b"GET"), - (b":path", message["path"].encode("ascii")), - (b":scheme", self.scheme) - ] + message["headers"] - - try: - self.conn.push_stream( - stream_id=self.stream_id, - promised_stream_id=push_stream_id, - request_headers=headers - ) - self.transport.write(self.conn.data_to_send()) - except h2.exceptions.ProtocolError: - self.logger.debug("h2 protocol error.", exc_info=True) - else: - event = h2.events.RequestReceived() - event.stream_id = push_stream_id - event.headers = headers - self.new_request(event) - elif not self.response_started: - # Sending response status line and headers - if message_type != "http.response.start": - msg = "Expected ASGI message 'http.response.start', but got '%s'." - raise RuntimeError(msg % message_type) - - self.response_started = True - - status_code = message["status"] - headers = ( - [(":status", str(status_code))] + - self.default_headers + - message.get("headers", []) - ) - - if self.access_log_enabled: - self.access_logger.info( - '%s - "%s %s HTTP/%s" %d', - get_client_addr(self.scope), - self.scope["method"], - get_path_with_query_string(self.scope), - self.scope["http_version"], - status_code - ) - - self.conn.send_headers(self.stream_id, headers, end_stream=False) - self.transport.write(self.conn.data_to_send()) - elif not self.response_completed: - if message_type == "http.response.body": - more_body = message.get("more_body", False) - if self.scope["method"] == "HEAD": - body = b"" - else: - body = message.get("body", b"") - self.conn.send_data(self.stream_id, body, end_stream=not more_body) - self.transport.write(self.conn.data_to_send()) - if not more_body: - self.response_completed = True - self.message_event.set() - else: - msg = "Got unexpected ASGI message '%s'." - raise RuntimeError(msg % message_type) - - if self.response_completed: - self.on_response(self.stream_id) - - -eventsreg = EventsRegistry() - - -@eventsreg.register(h2.events.RequestReceived) -def on_request_received(protocol: H2Protocol, event: h2.events.RequestReceived): - headers, pseudo_headers = [], {} - for key, value in event.headers: - if key[0] == b":"[0]: - pseudo_headers[key] = value - else: - headers.append((key.lower(), value)) - host = pseudo_headers[b":authority"] - headers.append((b"host", host)) - - raw_path, _, query_string = pseudo_headers[b":path"].partition(b"?") - scope = { - "type": "http", - "asgi": { - "version": protocol.config.asgi_version, - "spec_version": "2.3" - }, - "http_version": "2", - "server": protocol.addr_local, - "client": protocol.addr_remote, - "scheme": protocol.scheme, - "method": pseudo_headers[b":method"].decode("ascii"), - "root_path": protocol.root_path, - "path": unquote(raw_path.decode("ascii")), - "raw_path": raw_path, - "query_string": query_string, - "headers": headers, - "extensions": {"http.response.push": {}} - } - - if protocol.limit_concurrency is not None and ( - len(protocol.connections) >= protocol.limit_concurrency or - len(protocol.tasks) >= protocol.limit_concurrency - ): - app = _service_unavailable - message = "Exceeded concurrency limit." - protocol.logger.warning(message) - else: - app = protocol.app - - protocol.streams[event.stream_id] = cycle = H2ASGICycle( - scope=scope, - conn=protocol.conn, - protocol=protocol, - stream_id=event.stream_id, - host=host - ) - task = protocol.loop.create_task(cycle.run_asgi(app)) - task.add_done_callback(protocol.tasks.discard) - protocol.tasks.add(task) - - -@eventsreg.register(h2.events.DataReceived) -def on_data_received(protocol: H2Protocol, event: h2.events.DataReceived): - try: - stream = protocol.streams[event.stream_id] - except KeyError: - protocol.conn.reset_stream( - event.stream_id, - error_code=h2.errors.ErrorCodes.PROTOCOL_ERROR - ) - return - - stream.body += event.data - if len(stream.body) > HIGH_WATER_LIMIT: - protocol.flow.pause_reading() - stream.message_event.set() - - -@eventsreg.register(h2.events.StreamEnded) -def on_stream_ended(protocol: H2Protocol, event: h2.events.StreamEnded): - try: - stream = protocol.streams[event.stream_id] - except KeyError: - protocol.conn.reset_stream( - event.stream_id, - error_code=h2.errors.ErrorCodes.STREAM_CLOSED - ) - return - - stream.transport.resume_reading() - stream.more_body = False - stream.message_event.set() - - -@eventsreg.register(h2.events.ConnectionTerminated) -def on_connection_terminated( - protocol: H2Protocol, - event: h2.events.ConnectionTerminated -): - stream = protocol.streams.pop(event.last_stream_id, None) - if stream: - stream.disconnected = True - protocol.conn.close_connection(last_stream_id=event.last_stream_id) - protocol.transport.write(protocol.conn.data_to_send()) - protocol.transport.close() - - -@eventsreg.register(h2.events.StreamReset) -def on_stream_reset(protocol: H2Protocol, event: h2.events.StreamReset): - protocol.streams.pop(event.stream_id, None) diff --git a/emmett/asgi/protocols/http/helpers.py b/emmett/asgi/protocols/http/helpers.py deleted file mode 100644 index 6cd06a73..00000000 --- a/emmett/asgi/protocols/http/helpers.py +++ /dev/null @@ -1,302 +0,0 @@ -# -*- coding: utf-8 -*- -""" - emmett.asgi.protocols.helpers - ----------------------------- - - Provides HTTP protocols helpers - - :copyright: 2014 Giovanni Barillari - :license: BSD-3-Clause -""" - -import asyncio -import logging - -from uvicorn.main import ServerState -from uvicorn.protocols.utils import ( - get_local_addr, - get_remote_addr, - is_ssl -) - -from ...helpers import Config - -TRACE_LOG_LEVEL = 5 - - -class FlowControl: - __slots__ = [ - "_is_writable_event", - "_transport", - "read_paused", - "write_paused" - ] - - def __init__(self, transport: asyncio.Transport): - self._transport = transport - self.read_paused = False - self.write_paused = False - self._is_writable_event = asyncio.Event() - self._is_writable_event.set() - - async def drain(self): - await self._is_writable_event.wait() - - def pause_reading(self): - if not self.read_paused: - self.read_paused = True - self._transport.pause_reading() - - def resume_reading(self): - if self.read_paused: - self.read_paused = False - self._transport.resume_reading() - - def pause_writing(self): - if not self.write_paused: - self.write_paused = True - self._is_writable_event.clear() - - def resume_writing(self): - if self.write_paused: - self.write_paused = False - self._is_writable_event.set() - - -class HTTPProtocol(asyncio.Protocol): - __slots__ = [ - "access_log_enabled", - "access_logger", - "addr_local", - "addr_remote", - "app", - "config", - "connections", - "default_headers", - "flow", - "limit_concurrency", - "logger", - "loop", - "root_path", - "scheme", - "server_state", - "tasks", - "timeout_keep_alive_task", - "timeout_keep_alive", - "transport", - "ws_protocol_class" - ] - - def __init__( - self, - config: Config, - server_state: ServerState, - _loop=None - ): - self.config = config - self.app = config.loaded_app - self.loop = _loop or asyncio.get_event_loop() - self.logger = logging.getLogger("uvicorn.error") - self.access_logger = logging.getLogger("uvicorn.access") - self.access_log_enabled = self.access_logger.hasHandlers() - self.ws_protocol_class = config.ws_protocol_class - self.root_path = config.root_path - self.limit_concurrency = config.limit_concurrency - - # Timeouts - self.timeout_keep_alive_task = None - self.timeout_keep_alive = config.timeout_keep_alive - - # Shared server state - self.server_state = server_state - self.connections = server_state.connections - self.tasks = server_state.tasks - self.default_headers = server_state.default_headers - - # Per-connection state - self.transport = None - self.flow = None - self.addr_local = None - self.addr_remote = None - self.scheme = None - - def connection_made(self, transport: asyncio.Transport): - self.connections.add(self) - - self.transport = transport - self.flow = FlowControl(transport) - self.addr_local = get_local_addr(transport) - self.addr_remote = get_remote_addr(transport) - self.scheme = "https" if is_ssl(transport) else "http" - - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % tuple(self.addr_remote) if self.addr_remote else "" - self.logger.log(TRACE_LOG_LEVEL, "%sConnection made", prefix) - - def connection_lost(self, exc): - self.connections.discard(self) - - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % tuple(self.addr_remote) if self.addr_remote else "" - self.logger.log(TRACE_LOG_LEVEL, "%sConnection lost", prefix) - - if self.flow is not None: - self.flow.resume_writing() - - def eof_received(self): - pass - - def _might_unset_keepalive(self): - if self.timeout_keep_alive_task is not None: - self.timeout_keep_alive_task.cancel() - self.timeout_keep_alive_task = None - - def data_received(self, data: bytes): - self._might_unset_keepalive() - - def on_response_complete(self): - self.server_state.total_requests += 1 - if self.transport.is_closing(): - return - - self._might_unset_keepalive() - self.timeout_keep_alive_task = self.loop.call_later( - self.timeout_keep_alive, self.timeout_keep_alive_handler - ) - self.flow.resume_reading() - self.unblock_on_completed() - - def unblock_on_completed(self): - pass - - def shutdown(self): - pass - - def pause_writing(self): - self.flow.pause_writing() - - def resume_writing(self): - self.flow.resume_writing() - - def timeout_keep_alive_handler(self): - pass - - -class ASGICycle: - __slots__ = [ - "access_log_enabled", - "access_logger", - "body", - "conn", - "default_headers", - "disconnected", - "flow", - "logger", - "message_event", - "more_body", - "on_response", - "response_completed", - "response_started", - "scope", - "transport" - ] - - def __init__( - self, - scope, - conn, - protocol: HTTPProtocol - ): - self.scope = scope - self.conn = conn - self.transport = protocol.transport - self.flow = protocol.flow - self.logger = protocol.logger - self.access_logger = protocol.access_logger - self.access_log_enabled = protocol.access_log_enabled - self.default_headers = protocol.default_headers - self.message_event = asyncio.Event() - self.on_response = protocol.on_response_complete - - self.disconnected = False - self.response_started = False - self.response_completed = False - - self.body = b"" - self.more_body = True - - async def run_asgi(self, app): - try: - result = await app(self.scope, self.receive, self.send) - except Exception as exc: - msg = "Exception in ASGI application\n" - self.logger.error(msg, exc_info=exc) - if not self.response_started: - await self.send_500_response() - else: - self.transport.close() - else: - if result is not None: - msg = "ASGI callable should return None, but returned '%s'." - self.logger.error(msg, result) - self.transport.close() - elif not self.response_started and not self.disconnected: - msg = "ASGI callable returned without starting response." - self.logger.error(msg) - await self.send_500_response() - elif not self.response_completed and not self.disconnected: - msg = "ASGI callable returned without completing response." - self.logger.error(msg) - self.transport.close() - finally: - self.on_response = None - - async def receive(self): - if not self.disconnected and not self.response_completed: - self.flow.resume_reading() - await self.message_event.wait() - self.message_event.clear() - - if self.disconnected or self.response_completed: - message = {"type": "http.disconnect"} - else: - message = { - "type": "http.request", - "body": self.body, - "more_body": self.more_body, - } - self.body = b"" - - return message - - async def send(self, message): - raise NotImplementedError - - async def send_500_response(self): - await self.send( - { - "type": "http.response.start", - "status": 500, - "headers": [ - (b"content-type", b"text/plain; charset=utf-8"), - (b"connection", b"close") - ] - } - ) - await self.send( - {"type": "http.response.body", "body": b"Internal Server Error"} - ) - - -async def _service_unavailable(scope, receive, send): - await send( - { - "type": "http.response.start", - "status": 503, - "headers": [ - (b"content-type", b"text/plain; charset=utf-8"), - (b"connection", b"close") - ] - } - ) - await send({"type": "http.response.body", "body": b"Service Unavailable"}) diff --git a/emmett/asgi/protocols/http/httptools.py b/emmett/asgi/protocols/http/httptools.py index 494ad4d3..e69de29b 100644 --- a/emmett/asgi/protocols/http/httptools.py +++ b/emmett/asgi/protocols/http/httptools.py @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- -""" - emmett.asgi.protocols.http.httptools - ------------------------------------ - - Provides HTTP httptools protocol implementation - - :copyright: 2014 Giovanni Barillari - :license: BSD-3-Clause -""" - -from uvicorn.protocols.http.httptools_impl import ( - HttpToolsProtocol as _HttpToolsProtocol, - httptools -) - -from . import protocols - - -@protocols.register("httptools") -class HTTPToolsProtocol(_HttpToolsProtocol): - alpn_protocols = ["http/1.1"] - - def data_received(self, data: bytes) -> None: - self._unset_keepalive_if_required() - - try: - self.parser.feed_data(data) - except httptools.HttpParserError: - msg = "Invalid HTTP request received." - self.logger.warning(msg) - self.send_400_response(msg) - return - except httptools.HttpParserUpgrade: - self.handle_upgrade() - - def handle_upgrade(self): - upgrade_value = None - for name, value in self.headers: - if name == b"upgrade": - upgrade_value = value.lower() - - if upgrade_value != b"websocket" or self.ws_protocol_class is None: - msg = "Unsupported upgrade request." - self.logger.warning(msg) - self.send_400_response(msg) - return - - self.connections.discard(self) - method = self.scope["method"].encode() - output = [method, b" ", self.url, b" HTTP/1.1\r\n"] - for name, value in self.scope["headers"]: - output += [name, b": ", value, b"\r\n"] - output.append(b"\r\n") - protocol = self.ws_protocol_class( - config=self.config, - server_state=self.server_state - ) - protocol.connection_made(self.transport) - protocol.data_received(b"".join(output)) - self.transport.set_protocol(protocol) diff --git a/emmett/asgi/protocols/ws/__init__.py b/emmett/asgi/protocols/ws/__init__.py deleted file mode 100644 index 04a70613..00000000 --- a/emmett/asgi/protocols/ws/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from ...helpers import Registry - -protocols = Registry() - -from . import websockets - -try: - from . import wsproto -except ImportError: - pass - -from . import auto diff --git a/emmett/asgi/protocols/ws/auto.py b/emmett/asgi/protocols/ws/auto.py deleted file mode 100644 index 77b74429..00000000 --- a/emmett/asgi/protocols/ws/auto.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -""" - emmett.asgi.protocols.ws.auto - ----------------------------- - - Provides websocket auto protocol loader - - :copyright: 2014 Giovanni Barillari - :license: BSD-3-Clause -""" - -from . import protocols - - -if "websockets" in protocols: - protocols.register("auto")(protocols.get("websockets")) -else: - protocols.register("auto")(protocols.get("wsproto")) diff --git a/emmett/asgi/protocols/ws/websockets.py b/emmett/asgi/protocols/ws/websockets.py deleted file mode 100644 index 27b82888..00000000 --- a/emmett/asgi/protocols/ws/websockets.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- -""" - emmett.asgi.protocols.ws.websockets - ----------------------------------- - - Provides websocket websockets protocol implementation - - :copyright: 2014 Giovanni Barillari - :license: BSD-3-Clause -""" - -from uvicorn.protocols.websockets.websockets_impl import WebSocketProtocol - -from . import protocols - - -protocols.register("websockets")(WebSocketProtocol) diff --git a/emmett/asgi/protocols/ws/wsproto.py b/emmett/asgi/protocols/ws/wsproto.py deleted file mode 100644 index 5edb0e1e..00000000 --- a/emmett/asgi/protocols/ws/wsproto.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- -""" - emmett.asgi.protocols.ws.wsproto - -------------------------------- - - Provides websocket wsproto protocol implementation - - :copyright: 2014 Giovanni Barillari - :license: BSD-3-Clause -""" - -from uvicorn.protocols.websockets.wsproto_impl import WSProtocol - -from . import protocols - - -protocols.register("wsproto")(WSProtocol) diff --git a/emmett/asgi/workers.py b/emmett/asgi/workers.py index ec3100e6..fc335cbc 100644 --- a/emmett/asgi/workers.py +++ b/emmett/asgi/workers.py @@ -12,66 +12,13 @@ import asyncio import logging import signal +import sys +from gunicorn.arbiter import Arbiter from gunicorn.workers.base import Worker as _Worker -from uvicorn.config import Config as UvicornConfig -from uvicorn.lifespan.on import LifespanOn -from uvicorn.middleware.debug import DebugMiddleware -from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware +from uvicorn.config import Config from uvicorn.server import Server -from .helpers import _create_ssl_context -from .loops import loops -from .protocols import protocols_http, protocols_ws - - -class Config(UvicornConfig): - def setup_event_loop(self): - pass - - def load(self): - assert not self.loaded - - if self.is_ssl: - self.ssl = _create_ssl_context( - keyfile=self.ssl_keyfile, - certfile=self.ssl_certfile, - cert_reqs=self.ssl_cert_reqs, - ca_certs=self.ssl_ca_certs, - alpn_protocols=self.http.alpn_protocols - ) - else: - self.ssl = None - - encoded_headers = [ - (key.lower().encode("latin1"), value.encode("latin1")) - for key, value in self.headers - ] - self.encoded_headers = ( - encoded_headers if b"server" in dict(encoded_headers) else - [(b"server", b"Emmett")] + encoded_headers - ) - - self.http_protocol_class = self.http - self.ws_protocol_class = self.ws - self.lifespan_class = LifespanOn - - self.loaded_app = self.app - self.interface = "asgi3" - - if self.debug: - self.loaded_app = DebugMiddleware(self.loaded_app) - if self.proxy_headers: - self.loaded_app = ProxyHeadersMiddleware( - self.loaded_app, trusted_hosts=self.forwarded_allow_ips - ) - - self.loaded = True - - @property - def is_ssl(self) -> bool: - return self.ssl_certfile is not None and self.ssl_keyfile is not None - class Worker(_Worker): EMMETT_CONFIG = {} @@ -103,6 +50,7 @@ def __init__(self, *args, **kwargs): config.update( ssl_keyfile=self.cfg.ssl_options.get("keyfile"), ssl_certfile=self.cfg.ssl_options.get("certfile"), + ssl_keyfile_password=self.cfg.ssl_options.get("password"), ssl_version=self.cfg.ssl_options.get("ssl_version"), ssl_cert_reqs=self.cfg.ssl_options.get("cert_reqs"), ssl_ca_certs=self.cfg.ssl_options.get("ca_certs"), @@ -113,32 +61,30 @@ def __init__(self, *args, **kwargs): config["backlog"] = self.cfg.settings["backlog"].value config.update(self.EMMETT_CONFIG) - config.update( - http=protocols_http.get(config.get('http', 'auto')), - ws=protocols_ws.get(config.get('ws', 'auto')) - ) self.config = Config(**config) def init_process(self): - self.config.loop = loops.get(self.config.loop) + self.config.setup_event_loop() super().init_process() - def init_signals(self): + def init_signals(self) -> None: for s in self.SIGNALS: signal.signal(s, signal.SIG_DFL) - signal.signal(signal.SIGUSR1, self.handle_usr1) - # Don't let SIGUSR1 disturb active requests by interrupting system calls signal.siginterrupt(signal.SIGUSR1, False) - def run(self): + async def _serve(self) -> None: self.config.app = self.wsgi server = Server(config=self.config) - loop = asyncio.get_event_loop() - loop.run_until_complete(server.serve(sockets=self.sockets)) + await server.serve(sockets=self.sockets) + if not server.started: + sys.exit(Arbiter.WORKER_BOOT_ERROR) + + def run(self) -> None: + return asyncio.run(self._serve()) - async def callback_notify(self): + async def callback_notify(self) -> None: self.notify() @@ -153,7 +99,7 @@ class EmmettWorker(Worker): class EmmettH11Worker(EmmettWorker): EMMETT_CONFIG = { - "loop": "asyncio", + "loop": "auto", "http": "h11", "proxy_headers": False, "interface": "asgi3" diff --git a/emmett/cli.py b/emmett/cli.py index f9e9d80f..12fec614 100644 --- a/emmett/cli.py +++ b/emmett/cli.py @@ -23,7 +23,6 @@ from .__version__ import __version__ as fw_version from ._internal import locate_app, get_app_module -from .asgi.loops import loops from .logger import LOG_LEVELS from .server import run as sgi_run @@ -246,7 +245,7 @@ def main(self, *args, **kwargs): '--interface', type=click.Choice(['rsgi', 'asgi']), default='rsgi', help='Application interface.') @click.option( - '--loop', type=click.Choice(loops.keys()), default='auto', + '--loop', type=click.Choice(['auto', 'asyncio', 'uvloop']), default='auto', help='Event loop implementation.') @click.option( '--ssl-certfile', type=str, default=None, help='SSL certificate file') @@ -308,7 +307,7 @@ def develop_command( '--interface', type=click.Choice(['rsgi', 'asgi']), default='rsgi', help='Application interface.') @click.option( - '--loop', type=click.Choice(loops.keys()), default='auto', + '--loop', type=click.Choice(['auto', 'asyncio', 'uvloop']), default='auto', help='Event loop implementation.') @click.option( '--log-level', type=click.Choice(LOG_LEVELS.keys()), default='info', diff --git a/pyproject.toml b/pyproject.toml index 7fe30391..c9f274a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,6 @@ emmett-crypto = { version = "^0.2.0", optional = true } uvicorn = { version = "^0.19.0", optional = true } h11 = { version = ">= 0.12.0", optional = true } -h2 = { version = ">= 3.2.0, < 4.1.0", optional = true } websockets = { version = "^10.0", optional = true } httptools = { version = "~0.5.0", optional = true, markers = "sys_platform != 'win32'" } @@ -71,7 +70,7 @@ psycopg2-binary = "^2.9.3" [tool.poetry.extras] crypto = ["emmett-crypto"] orjson = ["orjson"] -uvicorn = ["uvicorn", "h11", "h2", "httptools", "websockets"] +uvicorn = ["uvicorn", "h11", "httptools", "websockets"] [tool.poetry.urls] "Issue Tracker" = "https://github.com/emmett-framework/emmett/issues" From bf6703b58b14e51ea0f8182e8b3553c6bfb1e47b Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Wed, 2 Nov 2022 19:36:56 +0100 Subject: [PATCH 06/29] fix typing in sessions --- emmett/sessions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/emmett/sessions.py b/emmett/sessions.py index c55ab10c..63ff10df 100644 --- a/emmett/sessions.py +++ b/emmett/sessions.py @@ -159,7 +159,7 @@ def _decrypt_data_modern(self, data: str) -> SessionData: rv = None return SessionData(rv, expires=self.expire) - def _load_session(self, wrapper: ScopeWrapper) -> SessionData: + def _load_session(self, wrapper: IngressWrapper) -> SessionData: cookie_data = wrapper.cookies[self.cookie_name].value return self._decrypt_data(cookie_data) @@ -183,7 +183,7 @@ def _new_session(self): def _session_cookie_data(self) -> str: return current.session._sid - def _load_session(self, wrapper: ScopeWrapper) -> Optional[SessionData]: + def _load_session(self, wrapper: IngressWrapper) -> Optional[SessionData]: sid = wrapper.cookies[self.cookie_name].value data = self._load(sid) if data is not None: From 9f73bc3652c9ed947b0f548d921b52570ce19121 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Wed, 2 Nov 2022 20:09:06 +0100 Subject: [PATCH 07/29] Drop legacy encryption support --- .github/workflows/tests.yml | 6 +- emmett/libs/pbkdf2.py | 140 ------------------------------------ emmett/security.py | 76 +++----------------- emmett/sessions.py | 46 +++--------- pyproject.toml | 4 +- tests/test_session.py | 6 +- 6 files changed, 24 insertions(+), 254 deletions(-) delete mode 100644 emmett/libs/pbkdf2.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 47912af3..39a89b08 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,7 @@ jobs: virtualenvs-in-project: true - name: Install dependencies run: | - poetry install -v --extras crypto + poetry install -v - name: Test env: POSTGRES_URI: postgres:postgres@localhost:5432/test @@ -61,7 +61,7 @@ jobs: virtualenvs-in-project: true - name: Install dependencies run: | - poetry install -v --extras crypto + poetry install -v - name: Test run: | poetry run pytest -v tests @@ -85,7 +85,7 @@ jobs: - name: Install dependencies shell: bash run: | - poetry install -v --extras crypto + poetry install -v - name: Test shell: bash run: | diff --git a/emmett/libs/pbkdf2.py b/emmett/libs/pbkdf2.py deleted file mode 100644 index 114354cf..00000000 --- a/emmett/libs/pbkdf2.py +++ /dev/null @@ -1,140 +0,0 @@ -# -*- coding: utf-8 -*- -""" - pbkdf2 - ~~~~~~ - - This module implements pbkdf2 for Python. It also has some basic - tests that ensure that it works. The implementation is straightforward - and uses stdlib only stuff and can be easily be copy/pasted into - your favourite application. - - Use this as replacement for bcrypt that does not need a c implementation - of a modified blowfish crypto algo. - - Example usage: - - >>> pbkdf2_hex('what i want to hash', 'the random salt') - 'fa7cc8a2b0a932f8e6ea42f9787e9d36e592e0c222ada6a9' - - How to use this: - - 1. Use a constant time string compare function to compare the stored hash - with the one you're generating:: - - def safe_str_cmp(a, b): - if len(a) != len(b): - return False - rv = 0 - for x, y in izip(a, b): - rv |= ord(x) ^ ord(y) - return rv == 0 - - 2. Use `os.urandom` to generate a proper salt of at least 8 byte. - Use a unique salt per hashed password. - - 3. Store ``algorithm$salt:costfactor$hash`` in the database so that - you can upgrade later easily to a different algorithm if you need - one. For instance ``PBKDF2-256$thesalt:10000$deadbeef...``. - - - :copyright: (c) Copyright 2011 by Armin Ronacher. - :license: BSD-3-Clause -""" -import codecs -import sys -import hmac -import hashlib - -from itertools import starmap -from struct import Struct -from operator import xor -from emmett._shortcuts import to_bytes - -izip = zip -xrange = range -_pack_int = Struct('>I').pack - - -def pbkdf2_hex(data, salt, iterations=1000, keylen=24, hashfunc=None): - """Like :func:`pbkdf2_bin` but returns a hex encoded string.""" - rv = pbkdf2_bin(data, salt, iterations, keylen, hashfunc) - return codecs.encode(rv, 'hex_codec').decode("utf8") - - -def pbkdf2_bin(data, salt, iterations=1000, keylen=24, hashfunc=None): - """Returns a binary digest for the PBKDF2 hash algorithm of `data` - with the given `salt`. It iterates `iterations` time and produces a - key of `keylen` bytes. By default SHA-1 is used as hash function, - a different hashlib `hashfunc` can be provided. - """ - hashfunc = hashfunc or hashlib.sha1 - mac = hmac.HMAC(data, None, hashfunc) - if not keylen: - keylen = mac.digest_size - - def _pseudorandom(x, mac=mac): - h = mac.copy() - h.update(x) - return bytearray(h.digest()) - buf = bytearray() - for block in xrange(1, -(-keylen // mac.digest_size) + 1): - rv = u = _pseudorandom(salt + _pack_int(block)) - for i in xrange(iterations - 1): - u = _pseudorandom(bytes(u)) - rv = bytearray(starmap(xor, izip(rv, u))) - buf.extend(rv) - return bytes(buf[:keylen]) - - -def test(): - failed = [] - - def check(data, salt, iterations, keylen, expected): - rv = pbkdf2_hex(to_bytes(data), to_bytes(salt), iterations, keylen) - if rv != expected: - sys.stdout.write('Test failed:\n') - sys.stdout.write(' Expected: %s\n' % expected) - sys.stdout.write(' Got: %s\n' % rv) - sys.stdout.write(' Parameters:\n') - sys.stdout.write(' data=%s\n' % data) - sys.stdout.write(' salt=%s\n' % salt) - sys.stdout.write(' iterations=%d\n' % iterations) - sys.stdout.write('\n') - failed.append(1) - - # From RFC 6070 - check('password', 'salt', 1, 20, - '0c60c80f961f0e71f3a9b524af6012062fe037a6') - check('password', 'salt', 2, 20, - 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957') - check('password', 'salt', 4096, 20, - '4b007901b765489abead49d926f721d065a429c1') - check('passwordPASSWORDpassword', 'saltSALTsaltSALTsaltSALTsaltSALTsalt', - 4096, 25, '3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038') - check('pass\x00word', 'sa\x00lt', 4096, 16, - '56fa6aa75548099dcc37d7f03425e0c3') - # This one is from the RFC but it just takes for ages - # check('password', 'salt', 16777216, 20, - # 'eefe3d61cd4da4e4e9945b3d6ba2158c2634e984') - - # From Crypt-PBKDF2 - check('password', 'ATHENA.MIT.EDUraeburn', 1, 16, - 'cdedb5281bb2f801565a1122b2563515') - check('password', 'ATHENA.MIT.EDUraeburn', 1, 32, - 'cdedb5281bb2f801565a1122b25635150ad1f7a04bb9f3a333ecc0e2e1f70837') - check('password', 'ATHENA.MIT.EDUraeburn', 2, 16, - '01dbee7f4a9e243e988b62c73cda935d') - check('password', 'ATHENA.MIT.EDUraeburn', 2, 32, - '01dbee7f4a9e243e988b62c73cda935da05378b93244ec8f48a99e61ad799d86') - check('password', 'ATHENA.MIT.EDUraeburn', 1200, 32, - '5c08eb61fdf71e4e4ec3cf6ba1f5512ba7e52ddbc5e5142f708a31e2e62b1e13') - check('X' * 64, 'pass phrase equals block size', 1200, 32, - '139c30c0966bc32ba55fdbf212530ac9c5ec59f1a452f5cc9ad940fea0598ed1') - check('X' * 65, 'pass phrase exceeds block size', 1200, 32, - '9ccad6d468770cd51b10e6a68721be611a8b4d282601db3b36be9246915ec82a') - - raise SystemExit(bool(failed)) - - -if __name__ == '__main__': - test() diff --git a/emmett/security.py b/emmett/security.py index 9b54ad5f..3cbfbdc0 100644 --- a/emmett/security.py +++ b/emmett/security.py @@ -13,29 +13,21 @@ :license: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) """ -import base64 import hashlib import hmac import os -import pickle -import pyaes import random import struct import threading import time import uuid as uuidm -import zlib from collections import OrderedDict -# TODO: check bytes conversions -from ._shortcuts import hashlib_sha1, to_bytes -from .libs.pbkdf2 import pbkdf2_hex +from emmett_crypto import kdf -try: - from emmett_crypto import kdf -except: - kdf = None +# TODO: check bytes conversions +from ._shortcuts import to_bytes class CSRFStorage(OrderedDict): @@ -69,20 +61,12 @@ def simple_hash(text, key='', salt='', digest_alg='md5'): h = digest_alg(text + key + salt) elif digest_alg.startswith('pbkdf2'): # latest and coolest! iterations, keylen, alg = digest_alg[7:-1].split(',') - if kdf: - return kdf.pbkdf2_hex( - text, - salt, - iterations=int(iterations), - keylen=int(keylen), - hash_algorithm=kdf.PBKDF2_HMAC[alg] - ) - return pbkdf2_hex( - to_bytes(text), - to_bytes(salt), - int(iterations), - int(keylen), - get_digest(alg) + return kdf.pbkdf2_hex( + text, + salt, + iterations=int(iterations), + keylen=int(keylen), + hash_algorithm=kdf.PBKDF2_HMAC[alg] ) elif key: # use hmac digest_alg = get_digest(digest_alg) @@ -126,48 +110,6 @@ def get_digest(value): } -def _pad(s, n=32, padchar='.'): - expected_len = ((len(s) + n) - len(s) % n) - return s.ljust(expected_len, to_bytes(padchar)) - - -# DEPRECATED: remove this method in future versions -def secure_dumps(data, encryption_key, hash_key=None, compression_level=None): - if not hash_key: - hash_key = hashlib_sha1(encryption_key).hexdigest() - dump = pickle.dumps(data) - if compression_level: - dump = zlib.compress(dump, compression_level) - key = _pad(to_bytes(encryption_key[:32])) - aes = pyaes.AESModeOfOperationCFB(key, iv=key[:16], segment_size=8) - encrypted_data = base64.urlsafe_b64encode(aes.encrypt(_pad(dump))) - signature = hmac.new(to_bytes(hash_key), msg=encrypted_data, digestmod='md5').hexdigest() - return signature + ':' + encrypted_data.decode('utf8') - - -# DEPRECATED: remove this method in future versions -def secure_loads(data, encryption_key, hash_key=None, compression_level=None): - if ':' not in data: - return None - if not hash_key: - hash_key = hashlib_sha1(encryption_key).hexdigest() - signature, encrypted_data = data.split(':', 1) - actual_signature = hmac.new( - to_bytes(hash_key), msg=to_bytes(encrypted_data), digestmod='md5').hexdigest() - if signature != actual_signature: - return None - key = _pad(to_bytes(encryption_key[:32])) - aes = pyaes.AESModeOfOperationCFB(key, iv=key[:16], segment_size=8) - try: - data = aes.decrypt(base64.urlsafe_b64decode(to_bytes(encrypted_data))) - data = data.rstrip(to_bytes(' ')) - if compression_level: - data = zlib.decompress(data) - return pickle.loads(data) - except (TypeError, pickle.UnpicklingError): - return None - - def _init_urandom(): """ This function and the web2py_uuid follow from the following discussion: diff --git a/emmett/sessions.py b/emmett/sessions.py index 63ff10df..e96dc512 100644 --- a/emmett/sessions.py +++ b/emmett/sessions.py @@ -19,18 +19,14 @@ from typing import Any, Dict, Optional, Type, TypeVar -from ._internal import warn_of_deprecation +from emmett_crypto import symmetric as crypto_symmetric + from .ctx import current from .datastructures import sdict, SessionData from .pipeline import Pipe -from .security import secure_loads, secure_dumps, uuid +from .security import uuid from .wrappers import IngressWrapper -try: - from emmett_crypto import symmetric as crypto_symmetric -except Exception: - crypto_symmetric = None - class SessionPipe(Pipe): def __init__( @@ -103,7 +99,7 @@ def __init__( domain=None, cookie_name=None, cookie_data=None, - encryption_mode="legacy", + encryption_mode="modern", compression_level=0 ): super().__init__( @@ -115,41 +111,17 @@ def __init__( cookie_data=cookie_data ) self.key = key - if encryption_mode == "legacy": - warn_of_deprecation("legacy encryption_mode", "modern", stack=5) - self._encrypt_data = self._encrypt_data_legacy - self._decrypt_data = self._decrypt_data_legacy - elif encryption_mode == "modern": - if not crypto_symmetric: - raise RuntimeError( - "You need emmett-crypto to use modern encryption mode" - ) - self._encrypt_data = self._encrypt_data_modern - self._decrypt_data = self._decrypt_data_modern - else: - raise ValueError("Invalid encryption_mode") + if encryption_mode != "modern": + raise ValueError("Unsupported encryption_mode") self.compression_level = compression_level - def _encrypt_data_legacy(self) -> str: - return secure_dumps( - sdict(current.session), - self.key, - compression_level=self.compression_level - ) - - def _encrypt_data_modern(self) -> str: + def _encrypt_data(self) -> str: data = pickle.dumps(sdict(current.session)) if self.compression_level: data = zlib.compress(data, self.compression_level) return crypto_symmetric.encrypt_b64(data, self.key) - def _decrypt_data_legacy(self, data: str) -> SessionData: - return SessionData( - secure_loads(data, self.key, compression_level=self.compression_level), - expires=self.expire - ) - - def _decrypt_data_modern(self, data: str) -> SessionData: + def _decrypt_data(self, data: str) -> SessionData: try: ddata = crypto_symmetric.decrypt_b64(data, self.key) if self.compression_level: @@ -364,7 +336,7 @@ def cookies( domain: Optional[str] = None, cookie_name: Optional[str] = None, cookie_data: Optional[Dict[str, Any]] = None, - encryption_mode: str = "legacy", + encryption_mode: str = "modern", compression_level: int = 0 ) -> CookieSessionPipe: return cls._build_pipe( diff --git a/pyproject.toml b/pyproject.toml index c9f274a0..64f54e0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,8 +44,8 @@ emmett = "emmett.cli:main" python = "^3.7" click = ">=6.0" granian = "~0.1" +emmett-crypto = "~0.3" pendulum = "~2.1.2" -pyaes = "~1.6.1" pyDAL = "17.3" python-rapidjson = "^1.0" pyyaml = "^5.4" @@ -53,7 +53,6 @@ renoir = "^1.5" severus = "^1.1" orjson = { version = "~3.8", optional = true } -emmett-crypto = { version = "^0.2.0", optional = true } uvicorn = { version = "^0.19.0", optional = true } h11 = { version = ">= 0.12.0", optional = true } @@ -68,7 +67,6 @@ pytest-asyncio = "^0.15" psycopg2-binary = "^2.9.3" [tool.poetry.extras] -crypto = ["emmett-crypto"] orjson = ["orjson"] uvicorn = ["uvicorn", "h11", "httptools", "websockets"] diff --git a/tests/test_session.py b/tests/test_session.py index 334af1cc..91ec0cc0 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -30,15 +30,13 @@ def ctx(): current._close_(token) -@pytest.mark.parametrize("encryption_mode", ["legacy", "modern"]) @pytest.mark.asyncio -async def test_session_cookie(ctx, encryption_mode): +async def test_session_cookie(ctx): session_cookie = SessionManager.cookies( key='sid', secure=True, domain='localhost', - cookie_name='foo_session', - encryption_mode=encryption_mode + cookie_name='foo_session' ) assert session_cookie.key == 'sid' assert session_cookie.secure is True From fa97d91041a58190ec4c2678892936d911644a29 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Wed, 9 Nov 2022 02:07:15 +0100 Subject: [PATCH 08/29] rsgi: fix websockets protocol handlers --- emmett/routing/router.py | 12 ++++++++++-- emmett/rsgi/handlers.py | 4 ++-- emmett/rsgi/helpers.py | 8 ++++---- emmett/rsgi/wrappers.py | 20 +++++++++++++------- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/emmett/routing/router.py b/emmett/routing/router.py index eda83927..6bf2623a 100644 --- a/emmett/routing/router.py +++ b/emmett/routing/router.py @@ -262,10 +262,18 @@ def __init__(self, *args, **kwargs): @staticmethod def _build_routing_dict(): rv = {} - for scheme in ['ws', 'wss']: + for scheme in ['http', 'https', 'ws', 'wss']: rv[scheme] = {'static': {}, 'match': {}} return rv + @staticmethod + def _all_schemes_from_route(schemes): + auto = {'ws': ['ws', 'http'], 'wss': ['wss', 'https']} + rv = [] + for scheme in schemes: + rv.extend(auto[scheme]) + return rv + def add_route_str(self, route): self._routes_str[route.name] = "%s://%s%s%s -> %s" % ( "|".join(route.schemes), @@ -281,7 +289,7 @@ def add_route(self, route): if host not in self.routes_in: self.routes_in[host] = self._build_routing_dict() self._get_routes_in_for_host = self._get_routes_in_for_host_match - for scheme in route.schemes: + for scheme in self._all_schemes_from_route(route.schemes): routing_dict = self.routes_in[host][scheme] slot, key = ( ('static', route.path) if route.is_static else diff --git a/emmett/rsgi/handlers.py b/emmett/rsgi/handlers.py index 6ad9bc70..fee0286b 100644 --- a/emmett/rsgi/handlers.py +++ b/emmett/rsgi/handlers.py @@ -301,5 +301,5 @@ async def dynamic_handler( finally: current._close_(ctx_token) - async def _close_connection(self, transport: WSTransport): - return transport.protocol.close(transport.status) + def _close_connection(self, transport: WSTransport): + transport.protocol.close(transport.status) diff --git a/emmett/rsgi/helpers.py b/emmett/rsgi/helpers.py index f23fc4b2..dd11f988 100644 --- a/emmett/rsgi/helpers.py +++ b/emmett/rsgi/helpers.py @@ -11,14 +11,14 @@ import asyncio -from granian.rsgi import WebsocketProtocol +from granian.rsgi import WebsocketMessageType, WebsocketProtocol class WSTransport: __slots__ = [ 'protocol', 'transport', - 'accepted', 'closed', - 'interrupted', 'status' + 'accepted', 'interrupted', + 'input', 'status', 'noop' ] def __init__( @@ -28,10 +28,10 @@ def __init__( self.protocol = protocol self.transport = None self.accepted = asyncio.Event() - self.closed = asyncio.Event() self.input = asyncio.Queue() self.interrupted = False self.status = 200 + self.noop = asyncio.Event() async def init(self): self.transport = await self.protocol.accept() diff --git a/emmett/rsgi/wrappers.py b/emmett/rsgi/wrappers.py index 9e2079e3..a9a5a32a 100644 --- a/emmett/rsgi/wrappers.py +++ b/emmett/rsgi/wrappers.py @@ -4,7 +4,7 @@ from typing import Any, Dict, List, Union, Optional from urllib.parse import parse_qs -from granian.rsgi import Scope, HTTPProtocol, WebsocketMessageType +from granian.rsgi import Scope, HTTPProtocol, WebsocketMessageType, ProtocolClosed from .helpers import WSTransport from ..datastructures import sdict @@ -122,14 +122,20 @@ async def accept( async def _wrapped_receive(self) -> Any: data = (await self._proto.receive()).data - for method in self._flow_receive: # type: ignore + for method in self._flow_receive: data = method(data) return data async def _wrapped_send(self, data: Any): - for method in self._flow_send: # type: ignore + for method in self._flow_send: data = method(data) - if isinstance(data, str): - await self._proto.transport.send_str(data) - else: - await self._proto.transport.send_bytes(data) + trx = ( + self._proto.transport.send_str if isinstance(data, str) else + self._proto.transport.send_bytes + ) + try: + await trx(data) + except ProtocolClosed: + if not self._proto.interrupted: + raise + await self._proto.noop.wait() From 56d01f49c0489cbd3325bb78358473a664633b29 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Wed, 9 Nov 2022 12:16:54 +0100 Subject: [PATCH 09/29] Update CLI options for `serve` command --- emmett/_reloader.py | 4 ++++ emmett/cli.py | 16 +++++++++++++--- emmett/server.py | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/emmett/_reloader.py b/emmett/_reloader.py index 058f08ed..dcaf34d0 100644 --- a/emmett/_reloader.py +++ b/emmett/_reloader.py @@ -155,6 +155,8 @@ def run_with_reloader( port, loop='auto', log_level=None, + threads=1, + threading_mode="workers", ssl_certfile: Optional[str] = None, ssl_keyfile: Optional[str] = None, extra_files=None, @@ -175,6 +177,8 @@ def run_with_reloader( kwargs={ "loop": loop, "log_level": log_level, + "threads": threads, + "threading_mode": threading_mode, "ssl_certfile": ssl_certfile, "ssl_keyfile": ssl_keyfile, } diff --git a/emmett/cli.py b/emmett/cli.py index 12fec614..1c1ce314 100644 --- a/emmett/cli.py +++ b/emmett/cli.py @@ -291,6 +291,8 @@ def develop_command( port, loop=loop, log_level='debug', + threads=1, + threading_mode="workers", ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile, ) @@ -302,7 +304,13 @@ def develop_command( @click.option( '--port', '-p', type=int, default=8000, help='The port to bind to.') @click.option( - "--workers", type=int, default=1, help="Number of worker processes. Defaults to 1.") + "--workers", '-w', type=int, default=1, + help="Number of worker processes. Defaults to 1.") +@click.option( + "--threads", type=int, default=None, help="Number of worker threads.") +@click.option( + "--threading-mode", type=click.Choice(['runtime', 'workers']), default='runtime', + help="Server threading mode.") @click.option( '--interface', type=click.Choice(['rsgi', 'asgi']), default='rsgi', help='Application interface.') @@ -321,8 +329,8 @@ def develop_command( '--ssl-keyfile', type=str, default=None, help='SSL key file') @pass_script_info def serve_command( - info, host, port, workers, interface, loop, log_level, backlog, - ssl_certfile, ssl_keyfile + info, host, port, workers, threads, threading_mode, interface, loop, log_level, + backlog, ssl_certfile, ssl_keyfile ): app_target = info._get_import_name() sgi_run( @@ -333,6 +341,8 @@ def serve_command( loop=loop, log_level=log_level, workers=workers, + threads=threads, + threading_mode=threading_mode, backlog=backlog, ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile, diff --git a/emmett/server.py b/emmett/server.py index 1c67f04b..90511ae9 100644 --- a/emmett/server.py +++ b/emmett/server.py @@ -24,7 +24,7 @@ def run( workers=1, threads=None, threading_mode='runtime', - backlog=2048, + backlog=1024, enable_websockets=True, ssl_certfile: Optional[str] = None, ssl_keyfile: Optional[str] = None From b0fe31b156c4939d821c6a5162094abb1a9c9c11 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Thu, 24 Nov 2022 14:24:10 +0100 Subject: [PATCH 10/29] orm: ensure transaction ops clean state on rollbacks --- emmett/orm/transactions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/emmett/orm/transactions.py b/emmett/orm/transactions.py index f253edb1..d7e7160d 100644 --- a/emmett/orm/transactions.py +++ b/emmett/orm/transactions.py @@ -77,6 +77,7 @@ def commit(self, begin=True): self._begin() def rollback(self, begin=True): + self._ops.clear() self.adapter.rollback() if begin: self._begin() @@ -124,6 +125,7 @@ def commit(self, begin=True): self._begin() def rollback(self): + self._ops.clear() self.adapter.execute('ROLLBACK TO SAVEPOINT %s;' % self.quoted_sid) def __enter__(self): From 8fbca5d32005b4e05e3f61421bff82f17b64b436 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Mon, 19 Dec 2022 17:05:21 +0100 Subject: [PATCH 11/29] Bump `renoir` to 1.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 64f54e0e..15fc982c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ pendulum = "~2.1.2" pyDAL = "17.3" python-rapidjson = "^1.0" pyyaml = "^5.4" -renoir = "^1.5" +renoir = "^1.6" severus = "^1.1" orjson = { version = "~3.8", optional = true } From dbea9e54d7d4775b1f7dbe1d370be581542f49c4 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Mon, 19 Dec 2022 17:06:03 +0100 Subject: [PATCH 12/29] Remove setup.py placeholder --- setup.py | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index 8304e207..00000000 --- a/setup.py +++ /dev/null @@ -1,13 +0,0 @@ -import io -import re - -from setuptools import setup - - -with io.open("emmett/__version__.py", "rt", encoding="utf8") as f: - version = re.search(r'__version__ = "(.*?)"', f.read()).group(1) - -setup( - name="Emmett", - version=version -) From 5b68e547e19560f0b34b346ed44fb42b5a07ac17 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Tue, 20 Dec 2022 18:13:27 +0100 Subject: [PATCH 13/29] Add Python 3.11 support (#447) --- .github/workflows/tests.yml | 4 ++-- pyproject.toml | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 39a89b08..a25ab5ee 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,7 +47,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, '3.10'] + python-version: [3.7, 3.8, 3.9, '3.10', '3.11'] steps: - uses: actions/checkout@v2 @@ -70,7 +70,7 @@ jobs: runs-on: windows-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, '3.10'] + python-version: [3.7, 3.8, 3.9, '3.10', '3.11'] steps: - uses: actions/checkout@v2 diff --git a/pyproject.toml b/pyproject.toml index 15fc982c..445418c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.poetry] -name = "Emmett" +name = "emmett" version = "2.5.0.dev0" description = "The web framework for inventors" authors = ["Giovanni Barillari "] @@ -23,6 +23,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Internet :: WWW/HTTP :: Dynamic Content", "Topic :: Software Development :: Libraries :: Python Modules" ] From 8f823f19c2a9599208ae9676169cbd0c51e93739 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Tue, 20 Dec 2022 18:19:10 +0100 Subject: [PATCH 14/29] Remove Python 3.7 support on Windows due to granian dependency --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a25ab5ee..5a3de2f1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -70,7 +70,7 @@ jobs: runs-on: windows-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, '3.10', '3.11'] + python-version: [3.8, 3.9, '3.10', '3.11'] steps: - uses: actions/checkout@v2 From 8e5026023e26f9a96892584c47ebc8ca29bc2e20 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Tue, 20 Dec 2022 18:19:37 +0100 Subject: [PATCH 15/29] Cleanup backers --- BACKERS.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/BACKERS.md b/BACKERS.md index 5b14d027..6ae7da9b 100644 --- a/BACKERS.md +++ b/BACKERS.md @@ -3,7 +3,3 @@ Emmett is a BSD-licensed open source project. It's an independent project with its ongoing development made possible entirely thanks to the support by these awesome backers. If you'd like to join them, please consider: - [Become a backer or sponsor on Github](https://github.com/sponsors/gi0baro). - -## Sponsors - -- Kkeller83 From 45150144a16b442a909f9256baaad78f08a8aa7a Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Sun, 25 Dec 2022 14:39:38 +0100 Subject: [PATCH 16/29] Bump `granian` to 0.2 --- emmett/cli.py | 4 ++-- emmett/server.py | 5 +++-- pyproject.toml | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/emmett/cli.py b/emmett/cli.py index 1c1ce314..cb663aa7 100644 --- a/emmett/cli.py +++ b/emmett/cli.py @@ -307,9 +307,9 @@ def develop_command( "--workers", '-w', type=int, default=1, help="Number of worker processes. Defaults to 1.") @click.option( - "--threads", type=int, default=None, help="Number of worker threads.") + "--threads", type=int, default=1, help="Number of worker threads.") @click.option( - "--threading-mode", type=click.Choice(['runtime', 'workers']), default='runtime', + "--threading-mode", type=click.Choice(['runtime', 'workers']), default='workers', help="Server threading mode.") @click.option( '--interface', type=click.Choice(['rsgi', 'asgi']), default='rsgi', diff --git a/emmett/server.py b/emmett/server.py index 90511ae9..2122e2bb 100644 --- a/emmett/server.py +++ b/emmett/server.py @@ -22,8 +22,8 @@ def run( loop='auto', log_level=None, workers=1, - threads=None, - threading_mode='runtime', + threads=1, + threading_mode='workers', backlog=1024, enable_websockets=True, ssl_certfile: Optional[str] = None, @@ -37,6 +37,7 @@ def run( interface=interface, workers=workers, threads=threads, + pthreads=threads, threading_mode=threading_mode, loop=loop, websockets=enable_websockets, diff --git a/pyproject.toml b/pyproject.toml index 445418c6..556f0fda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ emmett = "emmett.cli:main" [tool.poetry.dependencies] python = "^3.7" click = ">=6.0" -granian = "~0.1" +granian = "~0.2" emmett-crypto = "~0.3" pendulum = "~2.1.2" pyDAL = "17.3" From 12c0d8ad07264fe447c68092784b840806ec4515 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Wed, 2 Nov 2022 22:54:06 +0100 Subject: [PATCH 17/29] add `AppModuleGroup` class (#436) --- emmett/app.py | 34 +++++++++++++++++++++++++++++++++- emmett/routing/router.py | 13 +++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/emmett/app.py b/emmett/app.py index 4f8b43c8..2da8d520 100644 --- a/emmett/app.py +++ b/emmett/app.py @@ -32,7 +32,7 @@ from .language.helpers import Tstr from .language.translator import Translator from .pipeline import Pipe, Injector -from .routing.router import HTTPRouter, WebsocketRouter, RoutingCtx +from .routing.router import HTTPRouter, WebsocketRouter, RoutingCtx, RoutingCtxGroup from .routing.urls import url from .rsgi import handlers as rsgi_handlers from .templating.templater import Templater @@ -474,6 +474,9 @@ def module( opts=kwargs ) + def module_group(self, *modules: AppModule) -> AppModuleGroup: + return AppModuleGroup(self, *modules) + class AppModule: @classmethod @@ -702,3 +705,32 @@ def websocket( hostname=self.hostname, **kwargs ) + + +class AppModuleGroup: + def __init__(self, app: App, *modules: AppModule) -> None: + self.app = app + self.modules = modules + + def route( + self, + paths: Optional[Union[str, List[str]]] = None, + name: Optional[str] = None, + template: Optional[str] = None, + **kwargs + ) -> RoutingCtxGroup: + return RoutingCtxGroup([ + mod.route(paths=paths, name=name, template=template, **kwargs) + for mod in self.modules + ]) + + def websocket( + self, + paths: Optional[Union[str, List[str]]] = None, + name: Optional[str] = None, + **kwargs + ): + return RoutingCtxGroup([ + mod.websocket(paths=paths, name=name, **kwargs) + for mod in self.modules + ]) diff --git a/emmett/routing/router.py b/emmett/routing/router.py index 6bf2623a..0b1b1fdd 100644 --- a/emmett/routing/router.py +++ b/emmett/routing/router.py @@ -356,3 +356,16 @@ def __call__(self, f: Callable[..., Any]) -> Callable[..., Any]: self.router.app.send_signal(Signals.after_route, route=self.rule) self.router._routing_stack.pop() return rv + + +class RoutingCtxGroup: + __slots__ = ['ctxs'] + + def __init__(self, ctxs: List[RoutingCtx]): + self.ctxs = ctxs + + def __call__(self, f: Callable[..., Any]) -> Callable[..., Any]: + rv = f + for ctx in self.ctxs: + rv = ctx(f) + return rv From 3bcf2635ad21ed44256062dfc2d9b7fd5b8ad9f6 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Sun, 25 Dec 2022 18:41:16 +0100 Subject: [PATCH 18/29] add `AppModulesGrouped` class (#436) --- emmett/app.py | 91 ++++++++++++++++++++++++++++++++++++++++-- tests/test_pipeline.py | 79 ++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 3 deletions(-) diff --git a/emmett/app.py b/emmett/app.py index 2da8d520..86c4b65b 100644 --- a/emmett/app.py +++ b/emmett/app.py @@ -475,7 +475,7 @@ def module( ) def module_group(self, *modules: AppModule) -> AppModuleGroup: - return AppModuleGroup(self, *modules) + return AppModuleGroup(*modules) class AppModule: @@ -558,6 +558,41 @@ def from_module( **opts ) + @classmethod + def from_module_group( + cls, + appmodgroup: AppModuleGroup, + import_name: str, + name: str, + template_folder: Optional[str], + template_path: Optional[str], + static_folder: Optional[str], + static_path: Optional[str], + url_prefix: Optional[str], + hostname: Optional[str], + cache: Optional[RouteCacheRule], + root_path: Optional[str], + opts: Dict[str, Any] = {} + ) -> AppModulesGrouped: + mods = [] + for module in appmodgroup.modules: + mod = cls.from_module( + module, + import_name, + name, + template_folder=template_folder, + template_path=template_path, + static_folder=static_folder, + static_path=static_path, + url_prefix=url_prefix, + hostname=hostname, + cache=cache, + root_path=root_path, + opts=opts + ) + mods.append(mod) + return AppModulesGrouped(*mods) + def module( self, import_name: str, @@ -708,10 +743,40 @@ def websocket( class AppModuleGroup: - def __init__(self, app: App, *modules: AppModule) -> None: - self.app = app + def __init__(self, *modules: AppModule): self.modules = modules + def module( + self, + import_name: str, + name: str, + template_folder: Optional[str] = None, + template_path: Optional[str] = None, + static_folder: Optional[str] = None, + static_path: Optional[str] = None, + url_prefix: Optional[str] = None, + hostname: Optional[str] = None, + cache: Optional[RouteCacheRule] = None, + root_path: Optional[str] = None, + module_class: Optional[Type[AppModule]] = None, + **kwargs: Any + ) -> AppModulesGrouped: + module_class = module_class or AppModule + return module_class.from_module_group( + self, + import_name, + name, + template_folder=template_folder, + template_path=template_path, + static_folder=static_folder, + static_path=static_path, + url_prefix=url_prefix, + hostname=hostname, + cache=cache, + root_path=root_path, + opts=kwargs + ) + def route( self, paths: Optional[Union[str, List[str]]] = None, @@ -734,3 +799,23 @@ def websocket( mod.websocket(paths=paths, name=name, **kwargs) for mod in self.modules ]) + + +class AppModulesGrouped(AppModuleGroup): + @property + def pipeline(self) -> List[List[Pipe]]: + return [module.pipeline for module in self.modules] + + @pipeline.setter + def pipeline(self, pipeline: List[Pipe]): + for module in self.modules: + module.pipeline = pipeline + + @property + def injectors(self) -> List[List[Injector]]: + return [module.injectors for module in self.modules] + + @injectors.setter + def injectors(self, injectors: List[Injector]): + for module in self.modules: + module.injectors = injectors diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index b948d766..559f3385 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -122,6 +122,10 @@ class Pipe6(FlowStorePipeCommon): pass +class Pipe7(FlowStorePipeCommon): + pass + + class ExcPipeOpen(FlowStorePipeCommon): async def open(self): raise PipeException(self) @@ -327,6 +331,23 @@ def ws_pipe6(): def injpipe(): return {'posts': []} + mg1 = app.module(__name__, 'mg1', url_prefix='mg1') + mg2 = app.module(__name__, 'mg2', url_prefix='mg2') + mg1.pipeline = [Pipe5()] + mg2.pipeline = [Pipe6()] + mg = app.module_group(mg1, mg2) + + @mg.route() + async def pipe_mg(): + return "mg" + + mgc = mg.module(__name__, 'mgc', url_prefix='mgc') + mgc.pipeline = [Pipe7()] + + @mgc.route() + async def pipe_mgc(): + return "mgc" + return app @@ -573,6 +594,64 @@ async def test_module_pipeline_composition(app): assert parallel_flows_are_equal(parallel_flow, ctx.ctx) +@pytest.mark.asyncio +async def test_module_group_pipeline(app): + with request_ctx(app, '/mg1/pipe_mg') as ctx: + parallel_flow = [ + 'Pipe1.open', 'Pipe2.open_request', 'Pipe3.open', 'Pipe5.open', + 'Pipe5.close', 'Pipe3.close', 'Pipe2.close_request', 'Pipe1.close'] + linear_flow = [ + 'Pipe1.pipe', 'Pipe2.pipe_request', 'Pipe3.pipe', 'Pipe5.pipe', + 'Pipe5.success', 'Pipe3.success', 'Pipe2.success', 'Pipe1.success'] + await ctx.dispatch() + assert linear_flows_are_equal(linear_flow, ctx.ctx) + assert parallel_flows_are_equal(parallel_flow, ctx.ctx) + + with request_ctx(app, '/mg2/pipe_mg') as ctx: + parallel_flow = [ + 'Pipe1.open', 'Pipe2.open_request', 'Pipe3.open', 'Pipe6.open', + 'Pipe6.close', 'Pipe3.close', 'Pipe2.close_request', 'Pipe1.close'] + linear_flow = [ + 'Pipe1.pipe', 'Pipe2.pipe_request', 'Pipe3.pipe', 'Pipe6.pipe', + 'Pipe6.success', 'Pipe3.success', 'Pipe2.success', 'Pipe1.success'] + await ctx.dispatch() + assert linear_flows_are_equal(linear_flow, ctx.ctx) + assert parallel_flows_are_equal(parallel_flow, ctx.ctx) + + +@pytest.mark.asyncio +async def test_module_group_pipeline_composition(app): + with request_ctx(app, '/mg1/mgc/pipe_mgc') as ctx: + parallel_flow = [ + 'Pipe1.open', 'Pipe2.open_request', 'Pipe3.open', 'Pipe5.open', + 'Pipe7.open', + 'Pipe7.close', 'Pipe5.close', 'Pipe3.close', 'Pipe2.close_request', + 'Pipe1.close'] + linear_flow = [ + 'Pipe1.pipe', 'Pipe2.pipe_request', 'Pipe3.pipe', 'Pipe5.pipe', + 'Pipe7.pipe', + 'Pipe7.success', 'Pipe5.success', 'Pipe3.success', 'Pipe2.success', + 'Pipe1.success'] + await ctx.dispatch() + assert linear_flows_are_equal(linear_flow, ctx.ctx) + assert parallel_flows_are_equal(parallel_flow, ctx.ctx) + + with request_ctx(app, '/mg2/mgc/pipe_mgc') as ctx: + parallel_flow = [ + 'Pipe1.open', 'Pipe2.open_request', 'Pipe3.open', 'Pipe6.open', + 'Pipe7.open', + 'Pipe7.close', 'Pipe6.close', 'Pipe3.close', 'Pipe2.close_request', + 'Pipe1.close'] + linear_flow = [ + 'Pipe1.pipe', 'Pipe2.pipe_request', 'Pipe3.pipe', 'Pipe6.pipe', + 'Pipe7.pipe', + 'Pipe7.success', 'Pipe6.success', 'Pipe3.success', 'Pipe2.success', + 'Pipe1.success'] + await ctx.dispatch() + assert linear_flows_are_equal(linear_flow, ctx.ctx) + assert parallel_flows_are_equal(parallel_flow, ctx.ctx) + + @pytest.mark.asyncio async def test_receive_send_flow(app): send_storage_key = { From b58453b694f1d94c2240498385853826be63d5b5 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Sun, 25 Dec 2022 19:00:32 +0100 Subject: [PATCH 19/29] add module groups documentation (#436) --- docs/app_and_modules.md | 46 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/app_and_modules.md b/docs/app_and_modules.md index c40a844a..bc73276e 100644 --- a/docs/app_and_modules.md +++ b/docs/app_and_modules.md @@ -212,3 +212,49 @@ v2_apis.pipeline = [AnotherAuthPipe()] ``` Then all the routes defined in these modules or in sub-modules of these modules will have a final pipeline composed by the one of the `apis` module, and the one of the sub-module. + +Modules groups +-------------- + +*New in version 2.5* + +Once your application structure gets more complex, you might encounter the need of exposing the same routes with different pipelines: for example, you might want to expose the same APIs with different authentication policies over different endpoint. + +In order to avoid code duplication, Emmett provides you modules groups. Groups can be created from several modules and provide you the `route` and `websocket` method, so you can write routes a single time from the upper previous example: + +```python +from emmett.tools import ServicePipe + +apis = app.module(__name__, 'apis', url_prefix='apis') +apis.pipeline = [ServicePipe('json')] + +v1_apis = apis.module(__name__, 'v1', url_prefix='v1') +v1_apis.pipeline = [SomeAuthPipe()] + +v2_apis = apis.module(__name__, 'v2', url_prefix='v2') +v2_apis.pipeline = [AnotherAuthPipe()] + +apis_group = app.module_group(v1_apis, v2_apis) + +@apis_group.route("/users") +async def users(): + ... +``` + +> **Note:** even if the resulting route code is the same, Emmett `route` and `websocket` decorators will produce a number of routes equal to the number of modules defined in the group + +### Nest modules into groups + +Modules groups also let you define additional sub-modules. The resulting object will be a wrapper over the nested modules, so you can still customise their pipelines, and use the `route` and `websocket` decorators: + +```python +apis_group = app.module_group(v1_apis, v2_apis) +users = apis_group.module(__name__, 'users', url_prefix='users') +users.pipeline = [SomePipe()] + +@users.route("/") +async def index(): + ... +``` + +> **Note:** *under the hood* Emmett will produce a nested module for every module defined in the parent group From 73bc5a3e8ea36516d1e851eccb11677926f440de Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Sun, 25 Dec 2022 19:08:39 +0100 Subject: [PATCH 20/29] Update changelog --- CHANGES.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index f2518b89..cb89618a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,17 @@ Emmett changelog ================ +Next +---- + +(Release date to be defined, codename to be selected) + +- Added official Python 3.11 support +- Removed support for legacy encryption stack +- Added RSGI protocol support +- Use Granian as default web server in place of uvicorn +- Added application modules groups + Version 2.4 ----------- From 0061e2715794d09166314067e5a1bf99a193480b Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Mon, 30 Jan 2023 16:19:54 +0100 Subject: [PATCH 21/29] Fix ORM self references (#454) --- emmett/orm/base.py | 7 ++++--- emmett/orm/models.py | 26 ++++++++++++++++++-------- tests/test_orm.py | 9 ++++++++- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/emmett/orm/base.py b/emmett/orm/base.py index 006f2a3c..7c235acd 100644 --- a/emmett/orm/base.py +++ b/emmett/orm/base.py @@ -192,7 +192,6 @@ def define_models(self, *models): obj._define_props_() obj._define_relations_() obj._define_virtuals_() - obj._build_rowclass_() # define table and store in model args = dict( migrate=obj.migrate, @@ -204,10 +203,12 @@ def define_models(self, *models): obj.tablename, *obj.fields, **args ) model.table._model_ = obj - # load user's definitions - obj._define_() # set reference in db for model name self.__setattr__(model.__name__, obj.table) + # configure structured rows + obj._build_rowclass_() + # load user's definitions + obj._define_() if self._auto_migrate and not self._do_connect: self.connection_close() diff --git a/emmett/orm/models.py b/emmett/orm/models.py index 6b209fa9..15b92897 100644 --- a/emmett/orm/models.py +++ b/emmett/orm/models.py @@ -17,7 +17,6 @@ from typing import Any, Callable from ..datastructures import sdict -from ..utils import cachedprop from .apis import ( compute, rowattr, @@ -244,6 +243,7 @@ def __init__(self): self.format = None if not hasattr(self, 'primary_keys'): self.primary_keys = [] + self._fieldset_pk = set(self.primary_keys or ['id']) @property def config(self): @@ -401,7 +401,11 @@ def _define_relations_(self): raise RuntimeError(bad_args_error) reference = self.__parse_belongs_relation(item, ondelete) reference.is_refers = not is_belongs - refmodel = self.db[reference.model]._model_ + refmodel = ( + self.db[reference.model]._model_ + if reference.model != self.__class__.__name__ + else self + ) ref_multi_pk = len(refmodel._fieldset_pk) > 1 fk_def_key, fks_data, multi_fk = None, {}, [] if ref_multi_pk and reference.fk: @@ -418,7 +422,7 @@ def _define_relations_(self): elif ref_multi_pk and not reference.fk: multi_fk = list(refmodel.primary_keys) elif not reference.fk: - reference.fk = refmodel.table._id.name + reference.fk = list(refmodel._fieldset_pk)[0] if multi_fk: references = [] fks_data["fields"] = [] @@ -426,7 +430,7 @@ def _define_relations_(self): for fk in multi_fk: refclone = sdict(reference) refclone.fk = fk - refclone.ftype = refmodel.table[refclone.fk].type + refclone.ftype = getattr(refmodel, refclone.fk).type refclone.name = f"{refclone.name}_{refclone.fk}" refclone.compound = reference.name references.append(refclone) @@ -450,7 +454,7 @@ def _define_relations_(self): coupled_fields=belongs_fks[reference.name].coupled_fields, ) else: - reference.ftype = refmodel.table[reference.fk].type + reference.ftype = getattr(refmodel, reference.fk).type references = [reference] belongs_fks[reference.name] = sdict( model=reference.model, @@ -531,8 +535,15 @@ def __define_fks(self): implicit_defs = {} grouped_rels = {} for rname, rel in self._belongs_ref_.items(): - rmodel = self.db[rel.model]._model_ - if not rmodel.primary_keys and rmodel.table._id.type == 'id': + rmodel = ( + self.db[rel.model]._model_ + if rel.model != self.__class__.__name__ + else self + ) + if ( + not rmodel.primary_keys and + getattr(rmodel, list(rmodel._fieldset_pk)[0]).type == 'id' + ): continue if len(rmodel._fieldset_pk) > 1: match = self.__find_matching_fk_definition([rel.fk], rmodel) @@ -619,7 +630,6 @@ def _unset_row_persistence(self, row): def _build_rowclass_(self): #: build helpers for rows - self._fieldset_pk = set(self.primary_keys or ['id']) save_excluded_fields = ( set( field.name for field in self.fields if diff --git a/tests/test_orm.py b/tests/test_orm.py index c31b3691..b99dc303 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -24,7 +24,7 @@ before_destroy, after_destroy, before_commit, after_commit, rowattr, rowmethod, - has_one, has_many, belongs_to, + has_one, has_many, belongs_to, refers_to, scope ) from emmett.orm.migrations.utils import generate_runtime_migration @@ -361,6 +361,12 @@ def _compute_price(self, row): return row.quantity * row.product.price +class SelfRef(Model): + refers_to({'parent': 'self'}) + + name = Field.string() + + class CustomPKType(Model): id = Field.string() @@ -421,6 +427,7 @@ def _db(): User, Organization, Membership, House, Mouse, NeedSplit, Zoo, Animal, Elephant, Product, Cart, CartElement, + SelfRef, CustomPKType, CustomPKName, CustomPKMulti, CommitWatcher ) From e66c071974903e7bd1de5428f72488b7205a91ec Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Tue, 14 Mar 2023 14:03:20 +0100 Subject: [PATCH 22/29] Bump `granian` to 0.3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 556f0fda..0cd30b37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ emmett = "emmett.cli:main" [tool.poetry.dependencies] python = "^3.7" click = ">=6.0" -granian = "~0.2" +granian = "~0.3.0" emmett-crypto = "~0.3" pendulum = "~2.1.2" pyDAL = "17.3" From 4ee83fa68a6f84186b3c67daf1e66b9dd361542d Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Tue, 14 Mar 2023 14:05:25 +0100 Subject: [PATCH 23/29] Drop Python 3.7 support --- .github/workflows/tests.yml | 4 ++-- CHANGES.md | 1 + README.md | 2 +- docs/installation.md | 2 +- pyproject.toml | 3 +-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5a3de2f1..52f6330e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, '3.10', '3.11'] + python-version: [3.8, 3.9, '3.10', '3.11'] services: postgres: @@ -47,7 +47,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, '3.10', '3.11'] + python-version: [3.8, 3.9, '3.10', '3.11'] steps: - uses: actions/checkout@v2 diff --git a/CHANGES.md b/CHANGES.md index cb89618a..e23254c6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,7 @@ Next - Added RSGI protocol support - Use Granian as default web server in place of uvicorn - Added application modules groups +- Dropped Python 3.7 support Version 2.4 ----------- diff --git a/README.md b/README.md index 6fca0f64..aa1fa084 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ The *bloggy* example described in the [Tutorial](https://emmett.sh/docs/latest/t ## Status of the project -Emmett is production ready and is compatible with Python 3.7 and above versions. +Emmett is production ready and is compatible with Python 3.8 and above versions. Emmett follows a *semantic versioning* for its releases, with a `{major}.{minor}.{patch}` scheme for versions numbers, where: diff --git a/docs/installation.md b/docs/installation.md index e2108b73..fc169f3c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -4,7 +4,7 @@ Installation So, how do you get Emmett on your computer quickly? There are many ways you could do that, but the most kick-ass method is virtualenv, so let’s have a look at that first. -You will need Python version 3.7 or higher in order to get Emmett working. +You will need Python version 3.8 or higher in order to get Emmett working. virtualenv ---------- diff --git a/pyproject.toml b/pyproject.toml index 0cd30b37..d1aa801d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ classifiers = [ "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -42,7 +41,7 @@ include = [ emmett = "emmett.cli:main" [tool.poetry.dependencies] -python = "^3.7" +python = "^3.8" click = ">=6.0" granian = "~0.3.0" emmett-crypto = "~0.3" From 4a42de4d440566bb868b4fc94dd4489c8aea7a46 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Tue, 14 Mar 2023 14:08:50 +0100 Subject: [PATCH 24/29] Bump `pyyaml` dependency --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d1aa801d..4dcce682 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ emmett-crypto = "~0.3" pendulum = "~2.1.2" pyDAL = "17.3" python-rapidjson = "^1.0" -pyyaml = "^5.4" +pyyaml = "^6.0" renoir = "^1.6" severus = "^1.1" From f6f383404b48d827edf974fd25413d15aaeecd1c Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Tue, 14 Mar 2023 14:32:52 +0100 Subject: [PATCH 25/29] Update dev dependencies --- .github/workflows/tests.yml | 6 +++--- pyproject.toml | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 52f6330e..ccb9c897 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,7 +31,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install and configure Poetry - uses: gi0baro/setup-poetry-bin@v1 + uses: gi0baro/setup-poetry-bin@v1.3 with: virtualenvs-in-project: true - name: Install dependencies @@ -56,7 +56,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install and configure Poetry - uses: gi0baro/setup-poetry-bin@v1 + uses: gi0baro/setup-poetry-bin@v1.3 with: virtualenvs-in-project: true - name: Install dependencies @@ -79,7 +79,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install and configure Poetry - uses: gi0baro/setup-poetry-bin@v1 + uses: gi0baro/setup-poetry-bin@v1.3 with: virtualenvs-in-project: true - name: Install dependencies diff --git a/pyproject.toml b/pyproject.toml index 4dcce682..f406724c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,8 +61,7 @@ httptools = { version = "~0.5.0", optional = true, markers = "sys_platform != 'w [tool.poetry.dev-dependencies] ipaddress = "^1.0" -pylint = "^2.4.4" -pytest = "^6.2" +pytest = "^7.1" pytest-asyncio = "^0.15" psycopg2-binary = "^2.9.3" From e8238d2890f84a7b4c72b3479c25cb16dd757524 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Sun, 19 Mar 2023 13:36:04 +0100 Subject: [PATCH 26/29] Add websockets option to CLI `serve` command --- emmett/cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/emmett/cli.py b/emmett/cli.py index cb663aa7..ad912f58 100644 --- a/emmett/cli.py +++ b/emmett/cli.py @@ -314,6 +314,9 @@ def develop_command( @click.option( '--interface', type=click.Choice(['rsgi', 'asgi']), default='rsgi', help='Application interface.') +@click.option( + '--ws/--no-ws', is_flag=True, default=True, + help='Enable websockets support.') @click.option( '--loop', type=click.Choice(['auto', 'asyncio', 'uvloop']), default='auto', help='Event loop implementation.') @@ -329,7 +332,7 @@ def develop_command( '--ssl-keyfile', type=str, default=None, help='SSL key file') @pass_script_info def serve_command( - info, host, port, workers, threads, threading_mode, interface, loop, log_level, + info, host, port, workers, threads, threading_mode, interface, ws, loop, log_level, backlog, ssl_certfile, ssl_keyfile ): app_target = info._get_import_name() @@ -344,6 +347,7 @@ def serve_command( threads=threads, threading_mode=threading_mode, backlog=backlog, + enable_websockets=ws, ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile, ) From 953a5915fc190c3f24bf25c11e2a008c97f6ed22 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Sun, 19 Mar 2023 13:37:08 +0100 Subject: [PATCH 27/29] Drop deprecations, code cleaning --- emmett/asgi/protocols/http/h11.py | 0 emmett/asgi/protocols/http/httptools.py | 0 emmett/asgi/wrappers.py | 11 +++++++++++ emmett/html.py | 16 +--------------- emmett/rsgi/helpers.py | 2 +- emmett/rsgi/wrappers.py | 13 ++++++++++++- 6 files changed, 25 insertions(+), 17 deletions(-) delete mode 100644 emmett/asgi/protocols/http/h11.py delete mode 100644 emmett/asgi/protocols/http/httptools.py diff --git a/emmett/asgi/protocols/http/h11.py b/emmett/asgi/protocols/http/h11.py deleted file mode 100644 index e69de29b..00000000 diff --git a/emmett/asgi/protocols/http/httptools.py b/emmett/asgi/protocols/http/httptools.py deleted file mode 100644 index e69de29b..00000000 diff --git a/emmett/asgi/wrappers.py b/emmett/asgi/wrappers.py index 99935b98..90251d87 100644 --- a/emmett/asgi/wrappers.py +++ b/emmett/asgi/wrappers.py @@ -1,3 +1,14 @@ +# -*- coding: utf-8 -*- +""" + emmett.asgi.wrappers + -------------------- + + Provides ASGI request and websocket wrappers + + :copyright: 2014 Giovanni Barillari + :license: BSD-3-Clause +""" + import asyncio from datetime import datetime diff --git a/emmett/html.py b/emmett/html.py index 744b28e3..25e11c97 100644 --- a/emmett/html.py +++ b/emmett/html.py @@ -15,9 +15,7 @@ from functools import reduce -from ._internal import deprecated, warnings - -__all__ = ['tag', 'cat', 'safe', 'asis'] +__all__ = ['tag', 'cat', 'asis'] class TagStack(threading.local): @@ -216,18 +214,6 @@ def __html__(self): return _to_str(self.text) -class safe(asis): - @deprecated("html.safe", "html.asis") - def __init__(self, text, sanitize=False, allowed_tags=None): - super().__init__(text) - if sanitize: - warnings.warn( - "HTML sanitizer is no longer available. " - "Please switch to html.asis or implement your own policy." - ) - self.sanitize = False - - def _to_str(obj): if not isinstance(obj, str): return str(obj) diff --git a/emmett/rsgi/helpers.py b/emmett/rsgi/helpers.py index dd11f988..f1f25971 100644 --- a/emmett/rsgi/helpers.py +++ b/emmett/rsgi/helpers.py @@ -11,7 +11,7 @@ import asyncio -from granian.rsgi import WebsocketMessageType, WebsocketProtocol +from granian.rsgi import WebsocketProtocol class WSTransport: diff --git a/emmett/rsgi/wrappers.py b/emmett/rsgi/wrappers.py index a9a5a32a..e6a189b6 100644 --- a/emmett/rsgi/wrappers.py +++ b/emmett/rsgi/wrappers.py @@ -1,10 +1,21 @@ +# -*- coding: utf-8 -*- +""" + emmett.rsgi.wrappers + -------------------- + + Provides RSGI request and websocket wrappers + + :copyright: 2014 Giovanni Barillari + :license: BSD-3-Clause +""" + import asyncio from datetime import datetime from typing import Any, Dict, List, Union, Optional from urllib.parse import parse_qs -from granian.rsgi import Scope, HTTPProtocol, WebsocketMessageType, ProtocolClosed +from granian.rsgi import Scope, HTTPProtocol, ProtocolClosed from .helpers import WSTransport from ..datastructures import sdict From 20f1ac6aceac101a1682f7f9221cbfb05225a5db Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Sun, 19 Mar 2023 13:37:36 +0100 Subject: [PATCH 28/29] Update docs --- docs/deployment.md | 31 ++++++++++++++++++------------- docs/installation.md | 2 +- docs/sessions.md | 5 +---- docs/tutorial.md | 2 +- docs/upgrading.md | 21 +++++++++++++++++++++ examples/bloggy/app.py | 4 ++-- 6 files changed, 44 insertions(+), 21 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index d41f5065..5e44acbb 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -8,9 +8,9 @@ If you want to use an ASGI server not listed in this section, please refer to it Included server --------------- -*Changed in version 2.2* +*Changed in version 2.5* -Emmett comes with an included server based on [uvicorn](https://www.uvicorn.org/). In order to run your application in production you can just use the included `serve` command: +Emmett comes with [Granian](https://github.com/emmett-framework/granian) as its HTTP server. In order to run your application in production you can just use the included `serve` command: emmett serve --host 0.0.0.0 --port 80 @@ -20,26 +20,31 @@ You can inspect all the available options of the `serve` command using the `--he | --- | --- | --- | | host | 0.0.0.0 | Bind address | | port | 8000 | Bind port | -| workers | 1 | Number of worker processes | +| workers | 1 | Number of worker processes | +| threads | 1 | Number of threads | +| threading-mode | workers | Threading implementation (possible values: runtime,workers) | +| interface | rsgi | Server interface (possible values: rsgi,asgi) | +| http | auto | HTTP protocol version (possible values: auto,1,2) | +| ws/no-ws | ws | Enable/disable websockets support | | loop | auto | Loop implementation (possible values: auto,asyncio,uvloop) | -| http-protocol | auto | HTTP protocol implementation (possible values: auto,h11,httptools) | -| ws-protocol | auto | Websocket protocol implementation (possible values: auto,websockets,wsproto) | | log-level | info | Logging level (possible values: debug,info,warning,error,critical) | -| access-log | (flag) enabled | Enable/disable access log | -| proxy-headers | (flag) disabled | Enable/disable proxy headers | -| proxy-trust-ips | | Comma separated list of IPs to trust for proxy headers | -| max-concurrency | | Limit number of concurrent connections | | backlog | 2048 | Maximum connection queue | -| keep-alive-timeout | 0 | Connection keep-alive timeout | | ssl-certfile | | Path to SSL certificate file | | ssl-keyfile | | Path to SSL key file | -| ssl-cert-reqs | | SSL client certificate requirements (see `ssl` module) | -| ssl-ca-certs | | SSL CA allowed certificates | + +Uvicorn +------- + +*Changed in version 2.5* + +In case you want to stick with a more popular option, Emmett also comes with included support for [Uvicorn](https://github.com/encode/uvicorn). + +You can just use the `emmett[uvicorn]` extra during installation and rely on the `uvicorn` command to serve your application. Gunicorn -------- -The included server might suit most of the common demands, but whenever you need a fully featured server, you can use [Gunicorn](https://gunicorn.org). +The included server might suit most of the common demands, but whenever you need additional features, you can use [Gunicorn](https://gunicorn.org). Emmett includes a Gunicorn worker class allowing you to run ASGI applications with the Emmett's environment, while also giving you Gunicorn's fully-featured process management: diff --git a/docs/installation.md b/docs/installation.md index fc169f3c..08e87521 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -39,7 +39,7 @@ You should now be using your virtualenv (notice how the prompt of your shell has Now you can just enter the following command to get Emmett activated in your virtualenv: ```bash -$ pip install emmett[crypto] +$ pip install emmett ``` And now you are good to go. diff --git a/docs/sessions.md b/docs/sessions.md index 32572aa2..5c2fbae2 100644 --- a/docs/sessions.md +++ b/docs/sessions.md @@ -18,7 +18,7 @@ Basically, you can use `session` object to store and retrieve data, but before y Storing sessions in cookies --------------------------- -*Changed in version 2.3* +*Changed in version 2.5* You can store session contents directly in the cookies of the client using the Emmett's `SessionManager.cookies` pipe: @@ -43,11 +43,8 @@ As you can see, `SessionManager.cookies` needs a secret key to crypt the session | domain | | allows to set a specific domain for the cookie | | cookie\_name | | allows to set a specific name for the cookie | | cookie\_data | | allows to pass additional cookie data to the manager | -| encryption\_mode | legacy | allows to set the encryption method (`legacy` or `modern`) | | compression\_level | 0 | allows to set the compression level for the data stored (0 means disabled) | -> **Note:** in order to use *modern* value for `encryption_mode`, you need to install Emmett with `crypto` extra. This module is written in Rust language, so in case no pre-build wheel is available for your platform, you will need Rust toolchain on your system to compile it from source. - Storing sessions on filesystem ------------------------------ diff --git a/docs/tutorial.md b/docs/tutorial.md index 36cf1ba6..3a1ece9b 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -237,7 +237,7 @@ Moreover, to use the authorization module, we need to add a **session manager** ```python from emmett.sessions import SessionManager app.pipeline = [ - SessionManager.cookies('GreatScott', encryption_mode='modern'), + SessionManager.cookies('GreatScott'), db.pipe, auth.pipe ] diff --git a/docs/upgrading.md b/docs/upgrading.md index c0134a06..f1960472 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -13,6 +13,27 @@ Just as a remind, you can update Emmett using *pip*: $ pip install -U emmett ``` +Version 2.5 +----------- + +Emmett 2.5 release is highly focused on the included HTTP server. + +This release drops previous 2.x deprecations and support for Python 3.7. + +### New features + +#### Granian HTTP server and RSGI protocol + +Emmett 2.5 drops its dependency on uvicorn as HTTP server and replaces it with [Granian](https://github.com/emmett-framework/granian), a modern, Rust based HTTP server. + +Granian provides [RSGI](https://github.com/emmett-framework/granian/blob/master/docs/spec/RSGI.md), an alternative protocol to ASGI implementation, which is now the default protocol used in Emmett 2.5. This should give you roughly 50% more performance out of the box on your application, with no need to change its code. + +Emmett 2.5 applications still preserve an ASGI interface, so in case you want to stick with this protocol you can use the `--interface` option [in the included server](./deployment#included-server) or still use Uvicorn with the `emmett[uvicorn]` extra notation in your dependencies. + +#### Other new features + +Emmett 2.5 also introduces support for [application modules groups](./app_and_modules#modules-groups) and Python 3.11 + Version 2.4 ----------- diff --git a/examples/bloggy/app.py b/examples/bloggy/app.py index 75dde0a4..62211370 100644 --- a/examples/bloggy/app.py +++ b/examples/bloggy/app.py @@ -91,8 +91,8 @@ def setup(): #: pipeline app.pipeline = [ - SessionManager.cookies('GreatScott', encryption_mode='modern'), - db.pipe, + SessionManager.cookies('GreatScott'), + db.pipe, auth.pipe ] From 8c270a311516a2cb02e4e67f7752ef94224879e0 Mon Sep 17 00:00:00 2001 From: Giovanni Barillari Date: Sun, 19 Mar 2023 13:51:31 +0100 Subject: [PATCH 29/29] Release 2.5.0 --- CHANGES.md | 6 +++--- emmett/__version__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e23254c6..5364d5b7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,10 +1,10 @@ Emmett changelog ================ -Next ----- +Version 2.5 +----------- -(Release date to be defined, codename to be selected) +Released on March 19th 2023, codename Fermi - Added official Python 3.11 support - Removed support for legacy encryption stack diff --git a/emmett/__version__.py b/emmett/__version__.py index bee6b3e5..50062f87 100644 --- a/emmett/__version__.py +++ b/emmett/__version__.py @@ -1 +1 @@ -__version__ = "2.5.0.dev0" +__version__ = "2.5.0" diff --git a/pyproject.toml b/pyproject.toml index f406724c..94103cae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "emmett" -version = "2.5.0.dev0" +version = "2.5.0" description = "The web framework for inventors" authors = ["Giovanni Barillari "] license = "BSD-3-Clause"