From 2324e3176a32667230b64615b17e3134e5c01554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Krassowski?= <5832902+krassowski@users.noreply.github.com> Date: Tue, 28 May 2024 18:58:50 +0100 Subject: [PATCH] Add progress bar and fetch button (#9) * Implement progress bar and refresh button * Bump version * Fix for initial state * Do not fail if stream was cloased * Lint --- README.md | 2 +- jupyterlab_gallery/gitpuller.py | 81 +++++++++++++++++++++-- jupyterlab_gallery/manager.py | 51 +++++++++------ package.json | 2 +- pyproject.toml | 3 +- src/gallery.tsx | 112 ++++++++++++++++++++++++++------ src/handler.ts | 17 ++++- style/base.css | 44 ++++++++++++- 8 files changed, 262 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 169b3de..6330be9 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ c.GalleryManager.exhibits = [ "git": "https://github.com/jupyterlab/jupyterlab.git", "repository": "https://github.com/jupyterlab/jupyterlab/", "title": "JupyterLab", - "description": "JupyterLab", + "description": "JupyterLab is a highly extensible, feature-rich notebook authoring application and editing environment.", "icon": "https://raw.githubusercontent.com/jupyterlab/jupyterlab/main/packages/ui-components/style/icons/jupyter/jupyter.svg" } ] diff --git a/jupyterlab_gallery/gitpuller.py b/jupyterlab_gallery/gitpuller.py index c9510f4..908770e 100644 --- a/jupyterlab_gallery/gitpuller.py +++ b/jupyterlab_gallery/gitpuller.py @@ -7,6 +7,7 @@ # - reconnecting to the event stream when refreshing the browser # - handling multiple waiting pulls from tornado import gen, web, locks +import logging import traceback import threading @@ -14,9 +15,71 @@ import os from queue import Queue, Empty from collections import defaultdict +from numbers import Number + +import git from jupyter_server.base.handlers import JupyterHandler from nbgitpuller.pull import GitPuller +from tornado.iostream import StreamClosedError + + +class CloneProgress(git.RemoteProgress): + def __init__(self): + self.queue = Queue() + self.max_stage = 0.01 + self.prev_stage = 0 + super().__init__() + + def update(self, op_code: int, cur_count, max_count=None, message=""): + if op_code & git.RemoteProgress.BEGIN: + new_stage = None + if op_code & git.RemoteProgress.COUNTING: + new_stage = 0.05 + elif op_code & git.RemoteProgress.COMPRESSING: + new_stage = 0.10 + elif op_code & git.RemoteProgress.RECEIVING: + new_stage = 0.90 + elif op_code & git.RemoteProgress.RESOLVING: + new_stage = 1 + + if new_stage: + self.prev_stage = self.max_stage + self.max_stage = new_stage + + if isinstance(cur_count, Number) and isinstance(max_count, Number): + self.queue.put( + { + "progress": self.prev_stage + + cur_count / max_count * (self.max_stage - self.prev_stage), + "message": message, + } + ) + # self.queue.join() + + +class ProgressGitPuller(GitPuller): + def initialize_repo(self): + logging.info("Repo {} doesn't exist. Cloning...".format(self.repo_dir)) + progress = CloneProgress() + + def clone_task(): + git.Repo.clone_from( + self.git_url, self.repo_dir, branch=self.branch_name, progress=progress + ) + progress.queue.put(None) + + threading.Thread(target=clone_task).start() + + timeout = 60 + + while True: + item = progress.queue.get(True) # , timeout) + if item is None: + break + yield item + + logging.info("Repo {} initialized".format(self.repo_dir)) class SyncHandlerBase(JupyterHandler): @@ -78,7 +141,7 @@ async def _pull(self, repo: str, targetpath: str, exhibit_id: int): ) repo_dir = os.path.join(repo_parent_dir, targetpath or repo.split("/")[-1]) - gp = GitPuller( + gp = ProgressGitPuller( repo, repo_dir, branch=branch, @@ -104,10 +167,6 @@ def pull(): async def emit(self, data: dict): serialized_data = json.dumps(data) - if "output" in data: - self.log.info(data["output"]) - else: - self.log.info(data) self.write("data: {}\n\n".format(serialized_data)) await self.flush() @@ -137,6 +196,12 @@ async def _stream(self): if progress is None: msg = {"phase": "finished", "exhibit_id": exhibit_id} del self.settings["pull_status_queues"][exhibit_id] + elif isinstance(progress, dict): + msg = { + "output": progress, + "phase": "progress", + "exhibit_id": exhibit_id, + } elif isinstance(progress, Exception): msg = { "phase": "error", @@ -159,7 +224,11 @@ async def _stream(self): } self.last_message[exhibit_id] = msg - await self.emit(msg) + try: + await self.emit(msg) + except StreamClosedError: + self.log.warn("git puller stream got closed") + pass if empty_queues == len(queues_view): await gen.sleep(0.5) diff --git a/jupyterlab_gallery/manager.py b/jupyterlab_gallery/manager.py index 30b084f..fc7b809 100644 --- a/jupyterlab_gallery/manager.py +++ b/jupyterlab_gallery/manager.py @@ -1,7 +1,10 @@ +from datetime import datetime from pathlib import Path from traitlets.config.configurable import LoggingConfigurable from traitlets import Dict, List, Unicode +from subprocess import run +import re def extract_repository_owner(git_url: str) -> str: @@ -16,6 +19,25 @@ def extract_repository_name(git_url: str) -> str: return fragment +def has_updates(repo_path: Path) -> bool: + try: + result = run( + "git status -b --porcelain -u n --ignored n", + cwd=repo_path, + capture_output=True, + shell=True, + ) + except FileNotFoundError: + return False + data = re.match( + r"^## (.*?)( \[(ahead (?P\d+))?(, )?(behind (?P\d+))?\])?$", + result.stdout.decode("utf-8"), + ) + if not data: + return False + return data["behind"] is not None + + class GalleryManager(LoggingConfigurable): root_dir = Unicode( config=False, @@ -85,23 +107,14 @@ def get_exhibit_data(self, exhibit): local_path = self.get_local_path(exhibit) data["localPath"] = str(local_path) - data["revision"] = "2a2f2ee779ac21b70339da6551c2f6b0b00f6efe" - # timestamp from .git/FETCH_HEAD of the cloned repo - data["lastUpdated"] = "2024-05-01" - data["currentTag"] = "v3.2.4" - # the UI can show that there are X updates available; it could also show - # a summary of the commits available, or tags available; possibly the name - # of the most recent tag and would be sufficient over sending the list of commits, - # which can be long and delay the initialization. - data["updatesAvailable"] = False - data["isCloned"] = local_path.exists() - data["newestTag"] = "v3.2.5" - data["updates"] = [ - { - "revision": "02f04c339f880540064d2223176830afdd02f5fa", - "title": "commit description", - "description": "long commit description", - "date": "date in format returned by git", - } - ] + exists = local_path.exists() + data["isCloned"] = exists + if exists: + fetch_head = local_path / ".git" / "FETCH_HEAD" + if fetch_head.exists(): + data["lastUpdated"] = datetime.fromtimestamp( + fetch_head.stat().st_mtime + ).isoformat() + data["updatesAvailable"] = has_updates(local_path) + return data diff --git a/package.json b/package.json index 44e0e6c..aebffc8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jupyterlab-gallery", - "version": "0.1.2", + "version": "0.1.3", "description": "A JupyterLab gallery extension for presenting and downloading examples from remote repositories", "keywords": [ "jupyter", diff --git a/pyproject.toml b/pyproject.toml index dd35fd7..22475c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,8 @@ classifiers = [ ] dependencies = [ "jupyter_server>=2.0.1,<3", - "nbgitpuller>=1.2.1" + "nbgitpuller>=1.2.1", + "GitPython>=3.1.43" ] dynamic = ["version", "description", "authors", "urls", "keywords"] diff --git a/src/gallery.tsx b/src/gallery.tsx index 2d3720b..9ed9f9f 100644 --- a/src/gallery.tsx +++ b/src/gallery.tsx @@ -1,12 +1,17 @@ import * as React from 'react'; import { ReactWidget, showErrorMessage } from '@jupyterlab/apputils'; -import { Button, UseSignal } from '@jupyterlab/ui-components'; +import { + Button, + UseSignal, + folderIcon, + downloadIcon +} from '@jupyterlab/ui-components'; import { Contents } from '@jupyterlab/services'; import { IStream, Stream, Signal } from '@lumino/signaling'; import { TranslationBundle } from '@jupyterlab/translation'; import { IExhibit } from './types'; import { IExhibitReply } from './types'; -import { requestAPI, eventStream, IStreamMessage } from './handler'; +import { requestAPI, eventStream, IStreamMessage, IProgress } from './handler'; interface IActions { download(exhibit: IExhibit): Promise; @@ -20,8 +25,9 @@ export class GalleryWidget extends ReactWidget { fileChanged: Contents.IManager['fileChanged']; refreshFileBrowser: () => Promise; }) { - const { trans, fileChanged } = options; super(); + const { trans, fileChanged } = options; + this._trans = trans; this._status = trans.__('Gallery loading...'); this._actions = { open: async (exhibit: IExhibit) => { @@ -109,6 +115,7 @@ export class GalleryWidget extends ReactWidget { exhibits={this.exhibits} actions={this._actions} progressStream={this._stream} + trans={this._trans} /> ); } @@ -119,6 +126,7 @@ export class GalleryWidget extends ReactWidget { ); } + private _trans: TranslationBundle; private _update = new Signal(this); private _exhibits: IExhibit[] | null = null; private _status: string; @@ -130,11 +138,13 @@ function Gallery(props: { exhibits: IExhibit[]; actions: IActions; progressStream: IStream; + trans: TranslationBundle; }): JSX.Element { return (
{props.exhibits.map(exhibit => ( ; + trans: TranslationBundle; }): JSX.Element { const { exhibit, actions } = props; - const [progressMessage, setProgressMessage] = React.useState(); + const [progress, setProgress] = React.useState(null); + const [progressMessage, setProgressMessage] = React.useState(''); React.useEffect(() => { const listenToStreams = (_: GalleryWidget, message: IStreamMessage) => { @@ -164,9 +180,13 @@ function Exhibit(props: { 'Could not download', message.output ?? 'Unknown error' ); + } + if (message.phase === 'progress') { + setProgress(message.output); + setProgressMessage(message.output.message); } else { const { output, phase } = message; - setProgressMessage(output ? phase + ': ' + output : phase); + console.log(output + phase); } }; props.progressStream.connect(listenToStreams); @@ -177,31 +197,85 @@ function Exhibit(props: { return (

{exhibit.title}

- {exhibit.title} +
+ {exhibit.title} +
{exhibit.description}
- {progressMessage} + {progress ? ( +
+
+
{progressMessage}
+
+ ) : null}
{!exhibit.isCloned ? ( ) : ( - + <> + + + )} - {exhibit.isCloned && exhibit.updatesAvailable ? ( - - ) : null}
); diff --git a/src/handler.ts b/src/handler.ts index 2677412..973b9b7 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -45,12 +45,25 @@ export async function requestAPI( return data; } -export interface IStreamMessage { +export interface IProgress { + progress: number; + message: string; +} + +export interface IProgressStreamMessage { + output: IProgress; + phase: 'progress'; + exhibit_id: number; +} + +export interface ITextStreamMessage { output?: string; - phase: string; + phase: 'error' | 'finished' | 'syncing'; exhibit_id: number; } +export type IStreamMessage = IProgressStreamMessage | ITextStreamMessage; + export function eventStream( endPoint = '', onStream: (message: IStreamMessage) => void, diff --git a/style/base.css b/style/base.css index 201086a..5d7a724 100644 --- a/style/base.css +++ b/style/base.css @@ -17,12 +17,54 @@ .jp-Exhibit-description { padding: 2px; + height: 2.15em; + overflow: hidden; } -.jp-Exhibit-icon { +.jp-Exhibit-icon > img { max-width: 100%; } +.jp-Exhibit-icon { + display: flex; + justify-content: center; + align-items: center; +} + .jp-Launcher-openExample .jp-Gallery { display: contents; } + +.jp-Exhibit-progressbar-filler { + background: var(--jp-success-color2); + height: 1em; + transition: width 1s; +} + +.jp-Exhibit-progressbar { + border: 1px solid var(--jp-layout-color3); + background: var(--jp-layout-color2); + width: 100%; + margin-bottom: 10px; + border-radius: 2px; + position: relative; +} + +.jp-Exhibit-progressbar-filler > .jp-Exhibit-progressbar-error { + background: var(--jp-error-color1); +} + +.jp-Exhibit-progressMessage { + color: black; + max-height: 1.2em; + font-size: 85%; + position: absolute; + top: 0; + padding: 0 2px; + overflow: hidden; + text-overflow: ellipsis; +} + +.jp-Exhibit-buttons .jp-Button:disabled { + opacity: 0.3; +}