From 5a1d57ba96d0b55f95d9f4da724f5c82f0129dc4 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 1 Dec 2022 15:26:07 +0100 Subject: [PATCH 1/6] Allow to set SQLiteYStore's database path and document time-to-live --- jupyter_server_ydoc/app.py | 41 ++++++++++++++++++++++--- jupyter_server_ydoc/handlers.py | 53 ++++++++++++++++++++------------- pyproject.toml | 2 +- 3 files changed, 71 insertions(+), 25 deletions(-) diff --git a/jupyter_server_ydoc/app.py b/jupyter_server_ydoc/app.py index 13b3c991..23791cd9 100644 --- a/jupyter_server_ydoc/app.py +++ b/jupyter_server_ydoc/app.py @@ -6,10 +6,10 @@ except ModuleNotFoundError: raise ModuleNotFoundError("Jupyter Server must be installed to use this extension.") -from traitlets import Float, Int, Type -from ypy_websocket.ystore import BaseYStore # type: ignore +from traitlets import Float, Int, Type, Unicode, observe +from ypy_websocket.ystore import BaseYStore, SQLiteYStore # type: ignore -from .handlers import JupyterSQLiteYStore, YDocRoomIdHandler, YDocWebSocketHandler +from .handlers import YDocRoomIdHandler, YDocWebSocketHandler, sqlite_ystore_factory class YDocExtension(ExtensionApp): @@ -41,7 +41,7 @@ class YDocExtension(ExtensionApp): ) ystore_class = Type( - default_value=JupyterSQLiteYStore, + default_value=sqlite_ystore_factory(), klass=BaseYStore, config=True, help="""The YStore class to use for storing Y updates. Defaults to JupyterSQLiteYStore, @@ -49,6 +49,39 @@ class YDocExtension(ExtensionApp): directory, and clears history every 24 hours.""", ) + sqlite_ystore_db_path = Unicode( + ".jupyter_ystore.db", + config=True, + help="""The path to the YStore database. Defaults to '.jupyter_ystore.db' in the current + directory. Only applicable if the YStore is an SQLiteYStore.""", + ) + + @observe("sqlite_ystore_db_path") + def _observe_sqlite_ystore_db_path(self, change): + if issubclass(self.ystore_class, SQLiteYStore): + self.ystore_class.db_path = change["new"] + else: + raise RuntimeError( + f"ystore_class must be an SQLiteYStore to be able to set sqlite_ystore_db_path, not {self.ystore_class}" + ) + + sqlite_ystore_document_ttl = Int( + None, + allow_none=True, + config=True, + help="""The document time-to-live in seconds. Defaults to None (document history is never + cleared). Only applicable if the YStore is an SQLiteYStore.""", + ) + + @observe("sqlite_ystore_document_ttl") + def _observe_sqlite_ystore_document_ttl(self, change): + if issubclass(self.ystore_class, SQLiteYStore): + self.ystore_class.document_ttl = change["new"] + else: + raise RuntimeError( + f"ystore_class must be an SQLiteYStore to be able to set sqlite_ystore_document_ttl, not {self.ystore_class}" + ) + def initialize_settings(self): self.settings.update( { diff --git a/jupyter_server_ydoc/handlers.py b/jupyter_server_ydoc/handlers.py index c1f3c971..39857796 100644 --- a/jupyter_server_ydoc/handlers.py +++ b/jupyter_server_ydoc/handlers.py @@ -5,7 +5,7 @@ import json from logging import Logger from pathlib import Path -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional, Tuple, Type from jupyter_server.auth import authorized from jupyter_server.base.handlers import APIHandler, JupyterHandler @@ -13,13 +13,14 @@ from jupyter_ydoc import ydocs as YDOCS # type: ignore from tornado import web from tornado.websocket import WebSocketHandler -from ypy_websocket import WebsocketServer, YMessageType, YRoom # type: ignore +from ypy_websocket.websocket_server import WebsocketServer, YRoom # type: ignore from ypy_websocket.ystore import ( # type: ignore BaseYStore, SQLiteYStore, TempFileYStore, YDocNotFound, ) +from ypy_websocket.yutils import YMessageType # type: ignore YFILE = YDOCS["file"] @@ -28,16 +29,22 @@ class JupyterTempFileYStore(TempFileYStore): prefix_dir = "jupyter_ystore_" -class JupyterSQLiteYStore(SQLiteYStore): - db_path = ".jupyter_ystore.db" - document_ttl = 24 * 60 * 60 +def sqlite_ystore_factory( + db_path: str = ".jupyter_ystore.db", document_ttl: Optional[int] = None +) -> Type[SQLiteYStore]: + _db_path = db_path + _document_ttl = document_ttl + + class JupyterSQLiteYStore(SQLiteYStore): + db_path = _db_path + document_ttl = _document_ttl + + return JupyterSQLiteYStore class DocumentRoom(YRoom): """A Y room for a possibly stored document (e.g. a notebook).""" - is_transient = False - def __init__(self, type: str, ystore: BaseYStore, log: Optional[Logger]): super().__init__(ready=False, ystore=ystore, log=log) self.type = type @@ -49,8 +56,6 @@ def __init__(self, type: str, ystore: BaseYStore, log: Optional[Logger]): class TransientRoom(YRoom): """A Y room for sharing state (e.g. awareness).""" - is_transient = True - def __init__(self, log: Optional[Logger]): super().__init__(log=log) @@ -132,6 +137,7 @@ async def __anext__(self): def get_file_info(self) -> Tuple[str, str, str]: assert self.websocket_server is not None + assert isinstance(self.room, DocumentRoom) room_name = self.websocket_server.get_room_name(self.room) file_format: str file_type: str @@ -175,10 +181,10 @@ async def open(self, path): asyncio.create_task(self.websocket_server.serve(self)) # cancel the deletion of the room if it was scheduled - if not self.room.is_transient and self.room.cleaner is not None: + if isinstance(self.room, DocumentRoom) and self.room.cleaner is not None: self.room.cleaner.cancel() - if not self.room.is_transient and not self.room.ready: + if isinstance(self.room, DocumentRoom) and not self.room.ready: file_format, file_type, file_path = self.get_file_info() self.log.debug("Opening Y document from disk: %s", file_path) model = await ensure_async( @@ -188,19 +194,22 @@ async def open(self, path): # check again if ready, because loading the file can be async if not self.room.ready: # try to apply Y updates from the YStore for this document - try: - await self.room.ystore.apply_updates(self.room.ydoc) - read_from_source = False - except YDocNotFound: - # YDoc not found in the YStore, create the document from the source file (no change history) - read_from_source = True + read_from_source = True + if self.room.ystore is not None: + try: + await self.room.ystore.apply_updates(self.room.ydoc) + read_from_source = False + except YDocNotFound: + # YDoc not found in the YStore, create the document from the source file (no change history) + pass if not read_from_source: # if YStore updates and source file are out-of-sync, resync updates with source if self.room.document.source != model["content"]: read_from_source = True if read_from_source: self.room.document.source = model["content"] - await self.room.ystore.encode_state_as_update(self.room.ydoc) + if self.room.ystore: + await self.room.ystore.encode_state_as_update(self.room.ydoc) self.room.document.dirty = False self.room.ready = True self.room.watcher = asyncio.create_task(self.watch_file()) @@ -208,6 +217,7 @@ async def open(self, path): self.room.document.observe(self.on_document_change) async def watch_file(self): + assert isinstance(self.room, DocumentRoom) poll_interval = self.settings["collaborative_file_poll_interval"] if not poll_interval: self.room.watcher = None @@ -217,6 +227,7 @@ async def watch_file(self): await self.maybe_load_document() async def maybe_load_document(self): + assert isinstance(self.room, DocumentRoom) file_format, file_type, file_path = self.get_file_info() async with self.lock: model = await ensure_async( @@ -267,7 +278,7 @@ def on_message(self, message): # filter out message depending on changes if skip: self.log.debug( - "Filtered out Y message of type: %s", YMessageType(message_type).raw_str() + "Filtered out Y message of type: %s", YMessageType(message_type).name ) return skip self._message_queue.put_nowait(message) @@ -276,12 +287,13 @@ def on_message(self, message): def on_close(self) -> None: # stop serving this client self._message_queue.put_nowait(b"") - if not self.room.is_transient and self.room.clients == [self]: + if isinstance(self.room, DocumentRoom) and self.room.clients == [self]: # no client in this room after we disconnect # keep the document for a while in case someone reconnects self.room.cleaner = asyncio.create_task(self.clean_room()) async def clean_room(self) -> None: + assert isinstance(self.room, DocumentRoom) seconds = self.settings["collaborative_document_cleanup_delay"] if seconds is None: return @@ -309,6 +321,7 @@ def on_document_change(self, event): self.saving_document = asyncio.create_task(self.maybe_save_document()) async def maybe_save_document(self): + assert isinstance(self.room, DocumentRoom) seconds = self.settings["collaborative_document_save_delay"] if seconds is None: return diff --git a/pyproject.toml b/pyproject.toml index f75a38e0..14a06b34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ requires-python = ">=3.7" dependencies = [ "jupyter_ydoc>=0.2.0,<0.4.0", - "ypy-websocket>=0.8.0,<0.9.0", + "ypy-websocket>=0.8.1,<0.9.0", "jupyter_server_fileid >=0.6.0,<1" ] From 127aeba55fadef54d24846d41667c35a9e2c6042 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Fri, 2 Dec 2022 09:18:33 +0100 Subject: [PATCH 2/6] No need for sqlite_ystore_factory() --- jupyter_server_ydoc/app.py | 4 ++-- jupyter_server_ydoc/handlers.py | 16 ++++------------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/jupyter_server_ydoc/app.py b/jupyter_server_ydoc/app.py index 23791cd9..c56a70f2 100644 --- a/jupyter_server_ydoc/app.py +++ b/jupyter_server_ydoc/app.py @@ -9,7 +9,7 @@ from traitlets import Float, Int, Type, Unicode, observe from ypy_websocket.ystore import BaseYStore, SQLiteYStore # type: ignore -from .handlers import YDocRoomIdHandler, YDocWebSocketHandler, sqlite_ystore_factory +from .handlers import JupyterSQLiteYStore, YDocRoomIdHandler, YDocWebSocketHandler class YDocExtension(ExtensionApp): @@ -41,7 +41,7 @@ class YDocExtension(ExtensionApp): ) ystore_class = Type( - default_value=sqlite_ystore_factory(), + default_value=JupyterSQLiteYStore, klass=BaseYStore, config=True, help="""The YStore class to use for storing Y updates. Defaults to JupyterSQLiteYStore, diff --git a/jupyter_server_ydoc/handlers.py b/jupyter_server_ydoc/handlers.py index 39857796..6dd9689a 100644 --- a/jupyter_server_ydoc/handlers.py +++ b/jupyter_server_ydoc/handlers.py @@ -5,7 +5,7 @@ import json from logging import Logger from pathlib import Path -from typing import Any, Dict, Optional, Tuple, Type +from typing import Any, Dict, Optional, Tuple from jupyter_server.auth import authorized from jupyter_server.base.handlers import APIHandler, JupyterHandler @@ -29,17 +29,9 @@ class JupyterTempFileYStore(TempFileYStore): prefix_dir = "jupyter_ystore_" -def sqlite_ystore_factory( - db_path: str = ".jupyter_ystore.db", document_ttl: Optional[int] = None -) -> Type[SQLiteYStore]: - _db_path = db_path - _document_ttl = document_ttl - - class JupyterSQLiteYStore(SQLiteYStore): - db_path = _db_path - document_ttl = _document_ttl - - return JupyterSQLiteYStore +class JupyterSQLiteYStore(SQLiteYStore): + db_path = ".jupyter_ystore.db" + document_ttl = None class DocumentRoom(YRoom): From c8ed335d90a13b3f9af431fa097bcc9f9616b72d Mon Sep 17 00:00:00 2001 From: david qiu Date: Tue, 6 Dec 2022 01:27:41 -0800 Subject: [PATCH 3/6] make configurable traits local to JupyterSQLiteYStore (#1) * make configurable traits local to JupyterSQLiteYStore * Lint * Set YStore config on class before instantiation Co-authored-by: David Brochart --- jupyter_server_ydoc/app.py | 37 ++---------------------------- jupyter_server_ydoc/handlers.py | 40 ++++++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 41 deletions(-) diff --git a/jupyter_server_ydoc/app.py b/jupyter_server_ydoc/app.py index c56a70f2..13b3c991 100644 --- a/jupyter_server_ydoc/app.py +++ b/jupyter_server_ydoc/app.py @@ -6,8 +6,8 @@ except ModuleNotFoundError: raise ModuleNotFoundError("Jupyter Server must be installed to use this extension.") -from traitlets import Float, Int, Type, Unicode, observe -from ypy_websocket.ystore import BaseYStore, SQLiteYStore # type: ignore +from traitlets import Float, Int, Type +from ypy_websocket.ystore import BaseYStore # type: ignore from .handlers import JupyterSQLiteYStore, YDocRoomIdHandler, YDocWebSocketHandler @@ -49,39 +49,6 @@ class YDocExtension(ExtensionApp): directory, and clears history every 24 hours.""", ) - sqlite_ystore_db_path = Unicode( - ".jupyter_ystore.db", - config=True, - help="""The path to the YStore database. Defaults to '.jupyter_ystore.db' in the current - directory. Only applicable if the YStore is an SQLiteYStore.""", - ) - - @observe("sqlite_ystore_db_path") - def _observe_sqlite_ystore_db_path(self, change): - if issubclass(self.ystore_class, SQLiteYStore): - self.ystore_class.db_path = change["new"] - else: - raise RuntimeError( - f"ystore_class must be an SQLiteYStore to be able to set sqlite_ystore_db_path, not {self.ystore_class}" - ) - - sqlite_ystore_document_ttl = Int( - None, - allow_none=True, - config=True, - help="""The document time-to-live in seconds. Defaults to None (document history is never - cleared). Only applicable if the YStore is an SQLiteYStore.""", - ) - - @observe("sqlite_ystore_document_ttl") - def _observe_sqlite_ystore_document_ttl(self, change): - if issubclass(self.ystore_class, SQLiteYStore): - self.ystore_class.document_ttl = change["new"] - else: - raise RuntimeError( - f"ystore_class must be an SQLiteYStore to be able to set sqlite_ystore_document_ttl, not {self.ystore_class}" - ) - def initialize_settings(self): self.settings.update( { diff --git a/jupyter_server_ydoc/handlers.py b/jupyter_server_ydoc/handlers.py index 6dd9689a..6fabd8a7 100644 --- a/jupyter_server_ydoc/handlers.py +++ b/jupyter_server_ydoc/handlers.py @@ -13,6 +13,8 @@ from jupyter_ydoc import ydocs as YDOCS # type: ignore from tornado import web from tornado.websocket import WebSocketHandler +from traitlets import Int, Unicode +from traitlets.config import LoggingConfigurable from ypy_websocket.websocket_server import WebsocketServer, YRoom # type: ignore from ypy_websocket.ystore import ( # type: ignore BaseYStore, @@ -29,9 +31,27 @@ class JupyterTempFileYStore(TempFileYStore): prefix_dir = "jupyter_ystore_" -class JupyterSQLiteYStore(SQLiteYStore): - db_path = ".jupyter_ystore.db" - document_ttl = None +class JupyterSQLiteYStoreMetaclass(type(LoggingConfigurable), type(SQLiteYStore)): # type: ignore + pass + + +class JupyterSQLiteYStore( + LoggingConfigurable, SQLiteYStore, metaclass=JupyterSQLiteYStoreMetaclass +): + db_path = Unicode( + ".jupyter_ystore.db", + config=True, + help="""The path to the YStore database. Defaults to '.jupyter_ystore.db' in the current + directory.""", + ) + + document_ttl = Int( + None, + allow_none=True, + config=True, + help="""The document time-to-live in seconds. Defaults to None (document history is never + cleared).""", + ) class DocumentRoom(YRoom): @@ -142,7 +162,9 @@ def get_file_info(self) -> Tuple[str, str, str]: assert file_path is not None if file_path != self.room.document.path: self.log.debug( - "File with ID %s was moved from %s to %s", self.room.document.path, file_path + "File with ID %s was moved from %s to %s", + self.room.document.path, + file_path, ) self.room.document.path = file_path return file_format, file_type, file_path @@ -161,8 +183,13 @@ async def get(self, *args, **kwargs): async def open(self, path): ystore_class = self.settings["collaborative_ystore_class"] if self.websocket_server is None: + for k, v in self.config.get(ystore_class.__name__, {}).items(): + setattr(ystore_class, k, v) YDocWebSocketHandler.websocket_server = JupyterWebsocketServer( - rooms_ready=False, auto_clean_rooms=False, ystore_class=ystore_class, log=self.log + rooms_ready=False, + auto_clean_rooms=False, + ystore_class=ystore_class, + log=self.log, ) self._message_queue = asyncio.Queue() self.lock = asyncio.Lock() @@ -270,7 +297,8 @@ def on_message(self, message): # filter out message depending on changes if skip: self.log.debug( - "Filtered out Y message of type: %s", YMessageType(message_type).name + "Filtered out Y message of type: %s", + YMessageType(message_type).name, ) return skip self._message_queue.put_nowait(message) From 554c19d2679c8145af77606403bbe9db8d58764e Mon Sep 17 00:00:00 2001 From: David Brochart Date: Tue, 6 Dec 2022 11:05:28 +0100 Subject: [PATCH 4/6] Change JupyterSQLiteYStore to SQLiteYStore --- jupyter_server_ydoc/app.py | 8 ++++---- jupyter_server_ydoc/handlers.py | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/jupyter_server_ydoc/app.py b/jupyter_server_ydoc/app.py index 13b3c991..ea43084e 100644 --- a/jupyter_server_ydoc/app.py +++ b/jupyter_server_ydoc/app.py @@ -9,7 +9,7 @@ from traitlets import Float, Int, Type from ypy_websocket.ystore import BaseYStore # type: ignore -from .handlers import JupyterSQLiteYStore, YDocRoomIdHandler, YDocWebSocketHandler +from .handlers import SQLiteYStore, YDocRoomIdHandler, YDocWebSocketHandler class YDocExtension(ExtensionApp): @@ -41,12 +41,12 @@ class YDocExtension(ExtensionApp): ) ystore_class = Type( - default_value=JupyterSQLiteYStore, + default_value=SQLiteYStore, klass=BaseYStore, config=True, - help="""The YStore class to use for storing Y updates. Defaults to JupyterSQLiteYStore, + help="""The YStore class to use for storing Y updates. Defaults to an SQLiteYStore, which stores Y updates in a '.jupyter_ystore.db' SQLite database in the current - directory, and clears history every 24 hours.""", + directory.""", ) def initialize_settings(self): diff --git a/jupyter_server_ydoc/handlers.py b/jupyter_server_ydoc/handlers.py index 6fabd8a7..1dc2f2f9 100644 --- a/jupyter_server_ydoc/handlers.py +++ b/jupyter_server_ydoc/handlers.py @@ -18,8 +18,8 @@ from ypy_websocket.websocket_server import WebsocketServer, YRoom # type: ignore from ypy_websocket.ystore import ( # type: ignore BaseYStore, - SQLiteYStore, - TempFileYStore, + SQLiteYStore as _SQLiteYStore, + TempFileYStore as _TempFileYStore, YDocNotFound, ) from ypy_websocket.yutils import YMessageType # type: ignore @@ -27,16 +27,16 @@ YFILE = YDOCS["file"] -class JupyterTempFileYStore(TempFileYStore): +class TempFileYStore(_TempFileYStore): prefix_dir = "jupyter_ystore_" -class JupyterSQLiteYStoreMetaclass(type(LoggingConfigurable), type(SQLiteYStore)): # type: ignore +class SQLiteYStoreMetaclass(type(LoggingConfigurable), type(_SQLiteYStore)): # type: ignore pass -class JupyterSQLiteYStore( - LoggingConfigurable, SQLiteYStore, metaclass=JupyterSQLiteYStoreMetaclass +class SQLiteYStore( + LoggingConfigurable, _SQLiteYStore, metaclass=SQLiteYStoreMetaclass ): db_path = Unicode( ".jupyter_ystore.db", From 3ef148a0d0f62953404b839feb8768da9eb3f64a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Dec 2022 10:07:34 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyter_server_ydoc/handlers.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/jupyter_server_ydoc/handlers.py b/jupyter_server_ydoc/handlers.py index 1dc2f2f9..9db4f9ad 100644 --- a/jupyter_server_ydoc/handlers.py +++ b/jupyter_server_ydoc/handlers.py @@ -16,12 +16,10 @@ from traitlets import Int, Unicode from traitlets.config import LoggingConfigurable from ypy_websocket.websocket_server import WebsocketServer, YRoom # type: ignore -from ypy_websocket.ystore import ( # type: ignore - BaseYStore, - SQLiteYStore as _SQLiteYStore, - TempFileYStore as _TempFileYStore, - YDocNotFound, -) +from ypy_websocket.ystore import BaseYStore +from ypy_websocket.ystore import SQLiteYStore as _SQLiteYStore # type: ignore +from ypy_websocket.ystore import TempFileYStore as _TempFileYStore +from ypy_websocket.ystore import YDocNotFound from ypy_websocket.yutils import YMessageType # type: ignore YFILE = YDOCS["file"] @@ -35,9 +33,7 @@ class SQLiteYStoreMetaclass(type(LoggingConfigurable), type(_SQLiteYStore)): # pass -class SQLiteYStore( - LoggingConfigurable, _SQLiteYStore, metaclass=SQLiteYStoreMetaclass -): +class SQLiteYStore(LoggingConfigurable, _SQLiteYStore, metaclass=SQLiteYStoreMetaclass): db_path = Unicode( ".jupyter_ystore.db", config=True, From a9e9ec6cfd9b2f848b52dc8e357d5dbc15e106f4 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Tue, 6 Dec 2022 11:57:06 +0100 Subject: [PATCH 6/6] Pin ypy-websocket>=0.8.2 --- jupyter_server_ydoc/handlers.py | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jupyter_server_ydoc/handlers.py b/jupyter_server_ydoc/handlers.py index 9db4f9ad..517b983a 100644 --- a/jupyter_server_ydoc/handlers.py +++ b/jupyter_server_ydoc/handlers.py @@ -16,8 +16,8 @@ from traitlets import Int, Unicode from traitlets.config import LoggingConfigurable from ypy_websocket.websocket_server import WebsocketServer, YRoom # type: ignore -from ypy_websocket.ystore import BaseYStore -from ypy_websocket.ystore import SQLiteYStore as _SQLiteYStore # type: ignore +from ypy_websocket.ystore import BaseYStore # type: ignore +from ypy_websocket.ystore import SQLiteYStore as _SQLiteYStore from ypy_websocket.ystore import TempFileYStore as _TempFileYStore from ypy_websocket.ystore import YDocNotFound from ypy_websocket.yutils import YMessageType # type: ignore diff --git a/pyproject.toml b/pyproject.toml index 14a06b34..3e1e9e51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ requires-python = ">=3.7" dependencies = [ "jupyter_ydoc>=0.2.0,<0.4.0", - "ypy-websocket>=0.8.1,<0.9.0", + "ypy-websocket>=0.8.2,<0.9.0", "jupyter_server_fileid >=0.6.0,<1" ]