From 4d009c20d5cac3d8abae07c2a39082d87149d2b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Krassowski?= <5832902+krassowski@users.noreply.github.com> Date: Tue, 28 May 2024 14:11:57 +0100 Subject: [PATCH] Implement configuration of exhibits via traitlets (#8) * [WiP] Implement configuration of exhibits via traitlets * Rename, add docs, add title as configurable --- README.md | 32 +++++ conftest.py | 2 +- jupyterlab_gallery/__init__.py | 17 +-- jupyterlab_gallery/app.py | 30 +++++ jupyterlab_gallery/gitpuller.py | 94 ++++++++------ jupyterlab_gallery/handlers.py | 146 ++++++---------------- jupyterlab_gallery/manager.py | 107 ++++++++++++++++ jupyterlab_gallery/tests/test_handlers.py | 8 +- src/gallery.tsx | 6 - src/index.ts | 12 +- src/types.ts | 6 +- ui-tests/jupyter_server_test_config.py | 1 + 12 files changed, 285 insertions(+), 176 deletions(-) create mode 100644 jupyterlab_gallery/app.py create mode 100644 jupyterlab_gallery/manager.py diff --git a/README.md b/README.md index 8ff71f3..169b3de 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,38 @@ This extension is composed of a Python package named `jupyterlab-gallery` for the server extension and a NPM package named `jupyterlab-gallery` for the frontend extension. +When [`jupyterlab-new-launcher`](https://github.com/nebari-dev/jupyterlab-new-launcher) is installed, the gallery will be added as a "Gallery" section in the launcher; otherwise it will be shown in the left sidebar. + +## Configuration + +You can configure the gallery with the following traitlets: + +- `GalleryManager.exhibits`: controls the tiles shown in the gallery +- `GalleryManager.destination`: defined the path into which the exhibits will be cloned (by default `/gallery`) +- `GalleryManager.title`: the display name of the widget (by default "Gallery") + +These traitlets can be passed from the command line, a JSON file (`.json`) or a Python file (`.py`). + +You must name the file `jupyter_gallery_config.py` or `jupyter_gallery_config.json` and place it in one of the paths returned by `jupyter --paths` under the `config` section. + +An example Python file would include: + +```python +c.GalleryManager.title = "Examples" +c.GalleryManager.destination = "examples" +c.GalleryManager.exhibits = [ + { + "git": "https://github.com/jupyterlab/jupyterlab.git", + "repository": "https://github.com/jupyterlab/jupyterlab/", + "title": "JupyterLab", + "description": "JupyterLab", + "icon": "https://raw.githubusercontent.com/jupyterlab/jupyterlab/main/packages/ui-components/style/icons/jupyter/jupyter.svg" + } +] +``` + +Using the Python file enables including the PAT access token in the `git` stanza (note: while the `git` value is never exposed to the user, the `repository` is and should not contain the secret if you do not want it to be shared with the users). + ## Requirements - JupyterLab >= 4.0.0 diff --git a/conftest.py b/conftest.py index f97eeb2..c8f07ea 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,6 @@ import pytest -pytest_plugins = ("pytest_jupyter.jupyter_server", ) +pytest_plugins = ("pytest_jupyter.jupyter_server",) @pytest.fixture diff --git a/jupyterlab_gallery/__init__.py b/jupyterlab_gallery/__init__.py index 0a2b86d..5c61b37 100644 --- a/jupyterlab_gallery/__init__.py +++ b/jupyterlab_gallery/__init__.py @@ -8,7 +8,7 @@ warnings.warn("Importing 'jupyterlab_gallery' outside a proper installation.") __version__ = "dev" -from .handlers import setup_handlers +from .app import GalleryApp def _jupyter_labextension_paths(): @@ -16,17 +16,4 @@ def _jupyter_labextension_paths(): def _jupyter_server_extension_points(): - return [{"module": "jupyterlab_gallery"}] - - -def _load_jupyter_server_extension(server_app): - """Registers the API handler to receive HTTP requests from the frontend extension. - - Parameters - ---------- - server_app: jupyterlab.labapp.LabApp - JupyterLab application instance - """ - setup_handlers(server_app.web_app, server_app) - name = "jupyterlab_gallery" - server_app.log.info(f"Registered {name} server extension") + return [{"module": "jupyterlab_gallery", "app": GalleryApp}] diff --git a/jupyterlab_gallery/app.py b/jupyterlab_gallery/app.py new file mode 100644 index 0000000..9610d9d --- /dev/null +++ b/jupyterlab_gallery/app.py @@ -0,0 +1,30 @@ +from jupyter_server.extension.application import ExtensionApp +from .handlers import ExhibitsHandler, GalleryHandler, PullHandler +from .manager import GalleryManager + + +class GalleryApp(ExtensionApp): + name = "gallery" + + handlers = [ + ("jupyterlab-gallery/gallery", GalleryHandler), + ("jupyterlab-gallery/exhibits", ExhibitsHandler), + ("jupyterlab-gallery/pull", PullHandler), + ] + + # default_url = "/jupyterlab-gallery" + # load_other_extensions = True + # file_url_prefix = "/gallery" + + def initialize_settings(self): + self.log.info("Configured gallery manager") + gallery_manager = GalleryManager( + log=self.log, root_dir=self.serverapp.root_dir, config=self.config + ) + self.settings.update({"gallery_manager": gallery_manager}) + + def initialize_handlers(self): + # setting nbapp is needed for nbgitpuller + self.serverapp.web_app.settings["nbapp"] = self.serverapp + + self.log.info(f"Registered {self.name} server extension") diff --git a/jupyterlab_gallery/gitpuller.py b/jupyterlab_gallery/gitpuller.py index 7f37b0b..c9510f4 100644 --- a/jupyterlab_gallery/gitpuller.py +++ b/jupyterlab_gallery/gitpuller.py @@ -20,20 +20,19 @@ class SyncHandlerBase(JupyterHandler): - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if 'pull_status_queues' not in self.settings: - self.settings['pull_status_queues'] = defaultdict(Queue) + if "pull_status_queues" not in self.settings: + self.settings["pull_status_queues"] = defaultdict(Queue) # store the most recent message from each queue to re-emit when client re-connects self.last_message = {} # We use this lock to make sure that only one sync operation # can be happening at a time. Git doesn't like concurrent use! - if 'git_lock' not in self.settings: - self.settings['git_lock'] = locks.Lock() + if "git_lock" not in self.settings: + self.settings["git_lock"] = locks.Lock() def get_login_url(self): # raise on failed auth, not redirect @@ -43,26 +42,25 @@ def get_login_url(self): @property def git_lock(self): - return self.settings['git_lock'] + return self.settings["git_lock"] async def _pull(self, repo: str, targetpath: str, exhibit_id: int): - q = self.settings['pull_status_queues'][exhibit_id] + q = self.settings["pull_status_queues"][exhibit_id] try: - q.put_nowait({ - 'phase': 'waiting', - 'message': 'Waiting for a git lock' - }) + q.put_nowait({"phase": "waiting", "message": "Waiting for a git lock"}) await self.git_lock.acquire(1) except gen.TimeoutError: - q.put_nowait({ - 'phase': 'error', - 'message': 'Another git operations is currently running, try again in a few minutes' - }) + q.put_nowait( + { + "phase": "error", + "message": "Another git operations is currently running, try again in a few minutes", + } + ) return try: - branch = self.get_argument('branch', None) - depth = self.get_argument('depth', None) + branch = self.get_argument("branch", None) + depth = self.get_argument("depth", None) if depth: depth = int(depth) # The default working directory is the directory from which Jupyter @@ -74,11 +72,19 @@ async def _pull(self, repo: str, targetpath: str, exhibit_id: int): # so that all repos are always in scope after cloning. Sometimes # server_root_dir will include things like `~` and so the path # must be expanded. - repo_parent_dir = os.path.join(os.path.expanduser(self.settings['server_root_dir']), - os.getenv('NBGITPULLER_PARENTPATH', '')) - repo_dir = os.path.join(repo_parent_dir, targetpath or repo.split('/')[-1]) - - gp = GitPuller(repo, repo_dir, branch=branch, depth=depth, parent=self.settings['nbapp']) + repo_parent_dir = os.path.join( + os.path.expanduser(self.settings["server_root_dir"]), + os.getenv("NBGITPULLER_PARENTPATH", ""), + ) + repo_dir = os.path.join(repo_parent_dir, targetpath or repo.split("/")[-1]) + + gp = GitPuller( + repo, + repo_dir, + branch=branch, + depth=depth, + parent=self.settings["nbapp"], + ) def pull(): try: @@ -88,6 +94,7 @@ def pull(): q.put_nowait(None) except Exception as e: raise e + self.gp_thread = threading.Thread(target=pull) self.gp_thread.start() except Exception as e: @@ -97,23 +104,23 @@ def pull(): async def emit(self, data: dict): serialized_data = json.dumps(data) - if 'output' in data: - self.log.info(data['output']) + if "output" in data: + self.log.info(data["output"]) else: self.log.info(data) - self.write('data: {}\n\n'.format(serialized_data)) + self.write("data: {}\n\n".format(serialized_data)) await self.flush() async def _stream(self): # We gonna send out event streams! - self.set_header('content-type', 'text/event-stream') - self.set_header('cache-control', 'no-cache') + self.set_header("content-type", "text/event-stream") + self.set_header("cache-control", "no-cache") # start by re-emitting last message so that client can catch up after reconnecting for _exhibit_id, msg in self.last_message.items(): await self.emit(msg) - queues = self.settings['pull_status_queues'] + queues = self.settings["pull_status_queues"] # stream new messages as they are put on respective queues while True: @@ -128,26 +135,31 @@ async def _stream(self): continue if progress is None: - msg = {'phase': 'finished', 'exhibit_id': exhibit_id} - del self.settings['pull_status_queues'][exhibit_id] + msg = {"phase": "finished", "exhibit_id": exhibit_id} + del self.settings["pull_status_queues"][exhibit_id] elif isinstance(progress, Exception): msg = { - 'phase': 'error', - 'exhibit_id': exhibit_id, - 'message': str(progress), - 'output': '\n'.join([ - line.strip() - for line in traceback.format_exception( - type(progress), progress, progress.__traceback__ - ) - ]) + "phase": "error", + "exhibit_id": exhibit_id, + "message": str(progress), + "output": "\n".join( + [ + line.strip() + for line in traceback.format_exception( + type(progress), progress, progress.__traceback__ + ) + ] + ), } else: - msg = {'output': progress, 'phase': 'syncing', 'exhibit_id': exhibit_id} + msg = { + "output": progress, + "phase": "syncing", + "exhibit_id": exhibit_id, + } self.last_message[exhibit_id] = msg await self.emit(msg) if empty_queues == len(queues_view): await gen.sleep(0.5) - diff --git a/jupyterlab_gallery/handlers.py b/jupyterlab_gallery/handlers.py index d5f3761..0cd57c6 100644 --- a/jupyterlab_gallery/handlers.py +++ b/jupyterlab_gallery/handlers.py @@ -1,150 +1,76 @@ import json -from pathlib import Path +from typing import cast from jupyter_server.base.handlers import APIHandler -from jupyter_server.utils import url_path_join from .gitpuller import SyncHandlerBase +from .manager import GalleryManager import tornado -exhibits = [ - { - # repository URL can include branch, PAT, basically anything that git will digest - # TODO: if we embed PAT in repository URL we need another repository URL that is - #"git": "git@github.com:nebari-dev/nebari.git", - "git": "https://github.com/nebari-dev/nebari.git", - "repository": "https://github.com/nebari-dev/nebari/", - # "documentation": "https://github.com/nebari-dev/nebari/", - "title": "Nebari", - "description": "🪴 Nebari - your open source data science platform", - # we may want to pin the repository to specific revision? - # "revision": "2a2f2ee779ac21b70339da6551c2f6b0b00f6efe", - # icon should be optional to allow for fast prototyping; we may want to allow allow relative paths and then host the assets from `/static` - #"icon": "test.svg", - # we may want to allow checking in a single directory from a repo - # "path_in_repository": "" - # "" - }, - { - #"git": "git@github.com:nebari-dev/nebari-docker-images.git", - "git": "https://github.com/nebari-dev/nebari-docker-images.git", - "repository": "https://github.com/nebari-dev/nebari-docker-images/", - "title": "Nebari docker images", - "description": "Nebari Docker images", - } -] - # We do not want to expose `git_url` as it may contain PAT; # we want an allow-list over block-list to avoid exposing PAT in case # if the author of the config makes a typo like `giit` instead of `git`. EXPOSED_EXHIBIT_KEYS = ["repository", "title", "description", "icon"] -def extract_repository_owner(git_url: str) -> str: - fragments = git_url.strip("/").split("/") - return fragments[-2] if len(fragments) >= 2 else '' - - -def extract_repository_name(git_url: str) -> str: - fragment = git_url.split("/")[-1] - if fragment.endswith(".git"): - return fragment[:-4] - return fragment +class BaseHandler(APIHandler): + @property + def gallery_manager(self) -> GalleryManager: + return cast(GalleryManager, self.settings["gallery_manager"]) -def prepare_exhibit(exhibit_config, exhibit_id: int) -> dict: - exposed_config = { - k: v for k, v in exhibit_config.items() if k in EXPOSED_EXHIBIT_KEYS - } - clone_destination = Path("examples") - repository_name = extract_repository_name(exhibit_config["git"]) - repository_owner = extract_repository_owner(exhibit_config["repository"]) - local_path = clone_destination / repository_name - - if "icon" not in exposed_config: - if exposed_config["repository"].startswith('https://github.com/'): - exposed_config["icon"] = f"https://opengraph.githubassets.com/1/{repository_owner}/{repository_name}" - - # we probably want a tratilet to configure path into which the exhibits should be cloned - # path/relative/to/root/if/cloned - exposed_config["localPath"] = str(local_path) # e.g. "examples/nebari" - exposed_config["revision"] = "2a2f2ee779ac21b70339da6551c2f6b0b00f6efe" - # timestamp from .git/FETCH_HEAD of the cloned repo - exposed_config["lastUpdated"] = "2024-05-01" - exposed_config["currentTag"] = "v3.2.4" - # the UI can show that there are X updates available; it could also show - # a summary of the commits available, or tags available; possibly the name - # of the most recent tag and would be sufficient over sending the list of commits, - # which can be long and delay the initialization. - exposed_config["updatesAvailable"] = False - exposed_config["isCloned"] = local_path.exists() - exposed_config["newestTag"] = "v3.2.5" - exposed_config["updates"] = [ - { - "revision": "02f04c339f880540064d2223176830afdd02f5fa", - "title": "commit description", - "description": "long commit description", - "date": "date in format returned by git", - } - ] - exposed_config["id"] = exhibit_id - - return exposed_config +class GalleryHandler(BaseHandler): + @tornado.web.authenticated + def get(self): + self.finish( + json.dumps( + { + "title": self.gallery_manager.title, + "apiVersion": "1.0", + } + ) + ) -class ExhibitsHandler(APIHandler): - # The following decorator should be present on all verb methods (head, get, post, - # patch, put, delete, options) to ensure only authorized user can request the - # Jupyter server +class ExhibitsHandler(BaseHandler): @tornado.web.authenticated def get(self): - # TODO: - # - move it to a configurable app? - # - decide if we want to read the config from a file (json/yaml) or just use traitlets (so we could read in from the same json/py as used to configure jupyter-server) - # - populate PATs in he repository url field by using env variables - # - implement validation on file reading (either with traitlets or schema) self.finish( json.dumps( { "exhibits": [ - prepare_exhibit(exhibit_config, exhibit_id=i) - for i, exhibit_config in enumerate(exhibits) - ], - "apiVersion": "1.0", + self._prepare_exhibit(exhibit_config, exhibit_id=i) + for i, exhibit_config in enumerate( + self.gallery_manager.exhibits + ) + ] } ) ) + def _prepare_exhibit(self, exhibit, exhibit_id: int) -> dict: + exposed_config = {k: v for k, v in exhibit.items() if k in EXPOSED_EXHIBIT_KEYS} + return { + **self.gallery_manager.get_exhibit_data(exhibit), + **exposed_config, + "id": exhibit_id, + } + -class PullHandler(SyncHandlerBase): +class PullHandler(BaseHandler, SyncHandlerBase): @tornado.web.authenticated async def post(self): data = self.get_json_body() - exhibit_id = data['exhibit_id'] - raw_exhibit = exhibits[exhibit_id] - exhibit = prepare_exhibit(raw_exhibit, exhibit_id=exhibit_id) + exhibit_id = data["exhibit_id"] + exhibit = self.gallery_manager.exhibits[exhibit_id] return await super()._pull( - repo=raw_exhibit["git"], + repo=exhibit["git"], exhibit_id=exhibit_id, # branch # depth - targetpath=exhibit["localPath"] + targetpath=str(self.gallery_manager.get_local_path(exhibit)), ) @tornado.web.authenticated async def get(self): return await super()._stream() - - -def setup_handlers(web_app, server_app): - host_pattern = ".*$" - - base_url = web_app.settings["base_url"] - exhibits_pattern = url_path_join(base_url, "jupyterlab-gallery", "exhibits") - download_pattern = url_path_join(base_url, "jupyterlab-gallery", "pull") - handlers = [ - (exhibits_pattern, ExhibitsHandler), - (download_pattern, PullHandler) - ] - web_app.settings['nbapp'] = server_app - web_app.add_handlers(host_pattern, handlers) diff --git a/jupyterlab_gallery/manager.py b/jupyterlab_gallery/manager.py new file mode 100644 index 0000000..30b084f --- /dev/null +++ b/jupyterlab_gallery/manager.py @@ -0,0 +1,107 @@ +from pathlib import Path + +from traitlets.config.configurable import LoggingConfigurable +from traitlets import Dict, List, Unicode + + +def extract_repository_owner(git_url: str) -> str: + fragments = git_url.strip("/").split("/") + return fragments[-2] if len(fragments) >= 2 else "" + + +def extract_repository_name(git_url: str) -> str: + fragment = git_url.split("/")[-1] + if fragment.endswith(".git"): + return fragment[:-4] + return fragment + + +class GalleryManager(LoggingConfigurable): + root_dir = Unicode( + config=False, + allow_none=True, + ) + + exhibits = List( + Dict( + per_key_traits={ + "git": Unicode( + help="Git URL used for cloning (can include branch, PAT) - not show to the user" + ), + "repository": Unicode(help="User-facing URL of the repository"), + "title": Unicode(help="Name of the exhibit"), + "description": Unicode(help="Short description"), + # TODO: validate path exists + "icon": Unicode(help="Path to an svg or png, or base64 encoded string"), + # other ideas: `path_in_repository`, `documentation_url` + } + ), + config=True, + allow_none=False, + default_value=[ + { + "git": "https://github.com/nebari-dev/nebari.git", + "repository": "https://github.com/nebari-dev/nebari/", + "title": "Nebari", + "description": "🪴 Nebari - your open source data science platform", + }, + { + "git": "https://github.com/nebari-dev/nebari-docker-images.git", + "repository": "https://github.com/nebari-dev/nebari-docker-images/", + "title": "Nebari docker images", + "description": "Nebari Docker images", + }, + ], + ) + + destination = Unicode( + help="The directory into which the exhibits will be cloned", + default_value="gallery", + config=True, + ) + + title = Unicode( + help="The the display name of the Gallery widget", + default_value="Gallery", + config=True, + ) + + def get_local_path(self, exhibit) -> Path: + clone_destination = Path(self.destination) + repository_name = extract_repository_name(exhibit["git"]) + return clone_destination / repository_name + + def get_exhibit_data(self, exhibit): + data = {} + + if "icon" not in exhibit: + if exhibit["repository"].startswith("https://github.com/"): + repository_name = extract_repository_name(exhibit["git"]) + repository_owner = extract_repository_owner(exhibit["repository"]) + data["icon"] = ( + f"https://opengraph.githubassets.com/1/{repository_owner}/{repository_name}" + ) + + local_path = self.get_local_path(exhibit) + + data["localPath"] = str(local_path) + data["revision"] = "2a2f2ee779ac21b70339da6551c2f6b0b00f6efe" + # timestamp from .git/FETCH_HEAD of the cloned repo + data["lastUpdated"] = "2024-05-01" + data["currentTag"] = "v3.2.4" + # the UI can show that there are X updates available; it could also show + # a summary of the commits available, or tags available; possibly the name + # of the most recent tag and would be sufficient over sending the list of commits, + # which can be long and delay the initialization. + data["updatesAvailable"] = False + data["isCloned"] = local_path.exists() + data["newestTag"] = "v3.2.5" + data["updates"] = [ + { + "revision": "02f04c339f880540064d2223176830afdd02f5fa", + "title": "commit description", + "description": "long commit description", + "date": "date in format returned by git", + } + ] + return data diff --git a/jupyterlab_gallery/tests/test_handlers.py b/jupyterlab_gallery/tests/test_handlers.py index a62442f..6403dae 100644 --- a/jupyterlab_gallery/tests/test_handlers.py +++ b/jupyterlab_gallery/tests/test_handlers.py @@ -5,5 +5,11 @@ async def test_exhibits(jp_fetch): response = await jp_fetch("jupyterlab-gallery", "exhibits") assert response.code == 200 payload = json.loads(response.body) - assert payload["exhibits"] + assert isinstance(payload["exhibits"], list) + + +async def test_gallery(jp_fetch): + response = await jp_fetch("jupyterlab-gallery", "gallery") + assert response.code == 200 + payload = json.loads(response.body) assert payload["apiVersion"] == "1.0" diff --git a/src/gallery.tsx b/src/gallery.tsx index f20060b..2d3720b 100644 --- a/src/gallery.tsx +++ b/src/gallery.tsx @@ -79,12 +79,6 @@ export class GalleryWidget extends ReactWidget { private async _load() { try { const data = await requestAPI('exhibits'); - const expectedVersion = '1.0'; - if (data.apiVersion !== expectedVersion) { - console.warn( - `jupyter-gallery API version out of sync, expected ${expectedVersion}, got ${data.apiVersion}` - ); - } this.exhibits = data.exhibits; } catch (reason) { this._status = `jupyterlab_gallery server failed:\n${reason}`; diff --git a/src/index.ts b/src/index.ts index 241c6b3..d4b3796 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,6 +11,8 @@ import type { INewLauncher } from 'jupyterlab-new-launcher/lib/types'; import { GalleryWidget } from './gallery'; import { galleryIcon } from './icons'; +import { IGalleryReply } from './types'; +import { requestAPI } from './handler'; function isNewLauncher(launcher: ILauncher): launcher is INewLauncher { return 'addSection' in launcher; @@ -52,7 +54,15 @@ const plugin: JupyterFrontEndPlugin = { } }); - const title = trans.__('Gallery'); + const data = await requestAPI('gallery'); + const expectedVersion = '1.0'; + if (data.apiVersion !== expectedVersion) { + console.warn( + `jupyter-gallery API version out of sync, expected ${expectedVersion}, got ${data.apiVersion}` + ); + } + + const title = data.title === 'Gallery' ? trans.__('Gallery') : data.title; // add the widget to sidebar before waiting for server reply to reduce UI jitter if (launcher && isNewLauncher(launcher)) { launcher.addSection({ diff --git a/src/types.ts b/src/types.ts index 4f4f6fa..989bff0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,10 @@ +export interface IGalleryReply { + title: string; + apiVersion: string; +} + export interface IExhibitReply { exhibits: IExhibit[]; - apiVersion: string; } export interface IExhibit { diff --git a/ui-tests/jupyter_server_test_config.py b/ui-tests/jupyter_server_test_config.py index f2a9478..4b3f4db 100644 --- a/ui-tests/jupyter_server_test_config.py +++ b/ui-tests/jupyter_server_test_config.py @@ -4,6 +4,7 @@ opens the server to the world and provide access to JupyterLab JavaScript objects through the global window variable. """ + from jupyterlab.galata import configure_jupyter_server configure_jupyter_server(c)