Webrtc refactor

.github/workflows/main.yml

@@ -58,6 +58,12 @@ jobs:
           npm run test
         working-directory: js
 
+      - name: Run pytest
+        run: |
+          pip install ".[test]"
+          playwright install chromium
+          pytest --no-solara-vuetify-warmup
+
       - name: Build docs (Only on MacOS for build speed)
         if: matrix.os == 'macos-latest'
         run: |

.gitignore

@@ -12,3 +12,7 @@ share
 
 # OS X
 .DS_Store
+.yarn/
+.vscode 
+labextension 
+yarn.lock

ipywebrtc/__init__.py

@@ -3,8 +3,21 @@
 import ipywidgets as widgets
 from IPython.display import display
 
-from ._version import __version__, version_info  # noqa
-from .webrtc import CameraStream, WebRTCRoomMqtt  # noqa
+from ._version import __version__, version_info
+from .webrtc import (
+    AudioRecorder,
+    AudioStream,
+    CameraStream,
+    ImageRecorder,
+    ImageStream,
+    VideoRecorder,
+    VideoStream,
+    WebRTCPeer,
+    WebRTCRoom,
+    WebRTCRoomLocal,
+    WebRTCRoomMqtt,
+    WidgetStream,
+)
 
 
 def _prefix():

js/src/embed.js

@@ -5,5 +5,5 @@
 // already be loaded by the notebook otherwise.
 
 // Export widget models and views, and the npm package version number.
-module.exports = require("./webrtc.js");
+module.exports = require("./jupyter-webrtc.js");
 module.exports["version"] = require("../package.json").version;

js/src/index.js

@@ -9,5 +9,5 @@
 // __webpack_public_path__ = document.querySelector('body').getAttribute('data-base-url') + 'nbextensions/jupyter-webrtc/';
 
 // Export widget models and views, and the npm package version number.
-module.exports = require("./webrtc.js");
-module.exports["version"] = require("../package.json").version;
+export * from "./jupyter-webrtc";
+// module.exports['version'] = require('../package.json').version;

js/src/jupyter-webrtc.js

@@ -0,0 +1,8 @@
+export * from "./streams/Audio";
+export * from "./streams/Camera";
+export * from "./streams/Image";
+export * from "./streams/Media";
+export * from "./streams/Recorder";
+export * from "./streams/Video";
+export * from "./streams/Webrtc";
+export * from "./streams/Widget";

js/src/labplugin.js

@@ -1,15 +1,17 @@
-const jupyter_webrtc = require("./index");
-const base = require("@jupyter-widgets/base");
+import { IJupyterWidgetRegistry } from "@jupyter-widgets/base";
+import { version } from "../package.json";
 
-module.exports = {
+const extension = {
   id: "jupyter-webrtc",
-  requires: [base.IJupyterWidgetRegistry],
-  activate: function (app, widgets) {
+  requires: [IJupyterWidgetRegistry],
+  activate: (app, widgets) => {
     widgets.registerWidget({
       name: "jupyter-webrtc",
-      version: jupyter_webrtc.version,
-      exports: jupyter_webrtc,
+      version: version,
+      exports: async () => import("./index"),
     });
   },
   autoStart: true,
 };
+
+export default extension;

js/src/streams/Audio.js

@@ -0,0 +1,89 @@
+import { DOMWidgetView, unpack_models } from "@jupyter-widgets/base";
+import { RecorderModel, RecorderView } from "./Recorder";
+import { StreamModel } from "./Media";
+
+export class AudioStreamModel extends StreamModel {
+  defaults() {
+    return {
+      ...super.defaults(),
+      _model_name: "AudioStreamModel",
+      _view_name: "AudioStreamView",
+      audio: undefined,
+    };
+  }
+
+  initialize() {
+    super.initialize.apply(this, arguments);
+    window.last_audio_stream = this;
+
+    this.type = "audio";
+  }
+}
+
+AudioStreamModel.serializers = {
+  ...StreamModel.serializers,
+  audio: { deserialize: unpack_models },
+};
+
+export class AudioStreamView extends DOMWidgetView {
+  render() {
+    super.render.apply(this, arguments);
+    window.last_audio_stream_view = this;
+    this.audio = document.createElement("audio");
+    this.audio.controls = true;
+    this.pWidget.addClass("jupyter-widgets");
+
+    this.model.captureStream().then(
+      (stream) => {
+        this.audio.srcObject = stream;
+        this.el.appendChild(this.audio);
+        this.audio.play();
+      },
+      (error) => {
+        const text = document.createElement("div");
+        text.innerHTML =
+          "Error creating view for mediastream: " + error.message;
+        this.el.appendChild(text);
+      },
+    );
+  }
+
+  remove() {
+    this.model.captureStream().then((stream) => {
+      this.audio.pause();
+      this.audio.srcObject = null;
+    });
+    return super.remove.apply(this, arguments);
+  }
+}
+
+export class AudioRecorderModel extends RecorderModel {
+  defaults() {
+    return {
+      ...super.defaults(),
+      _model_name: "AudioRecorderModel",
+      _view_name: "AudioRecorderView",
+      audio: null,
+    };
+  }
+
+  initialize() {
+    super.initialize.apply(this, arguments);
+    window.last_audio_recorder = this;
+
+    this.type = "audio";
+  }
+}
+
+AudioRecorderModel.serializers = {
+  ...RecorderModel.serializers,
+  audio: { deserialize: unpack_models },
+};
+
+export class AudioRecorderView extends RecorderView {
+  initialize() {
+    super.initialize.apply(this, arguments);
+    this.tag = "audio";
+    this.recordIconClass = "fa fa-circle";
+  }
+}

js/src/streams/Camera.js

@@ -0,0 +1,31 @@
+import { MediaStreamModel } from "./Media";
+
+export class CameraStreamModel extends MediaStreamModel {
+  defaults() {
+    return {
+      ...super.defaults(),
+      _model_name: "CameraStreamModel",
+      constraints: { audio: true, video: true },
+    };
+  }
+
+  captureStream() {
+    if (!this.cameraStream) {
+      this.cameraStream = navigator.mediaDevices.getUserMedia(
+        this.get("constraints"),
+      );
+    }
+    return this.cameraStream;
+  }
+
+  close() {
+    if (this.cameraStream) {
+      this.cameraStream.then((stream) => {
+        stream.getTracks().forEach((track) => {
+          track.stop();
+        });
+      });
+    }
+    return super.close.apply(this, arguments);
+  }
+}

js/src/streams/Image.js

@@ -0,0 +1,145 @@
+import { unpack_models } from "@jupyter-widgets/base";
+import * as utils from "../utils";
+import { imageWidgetToCanvas } from "../utils";
+import { MediaStreamModel } from "./Media";
+import { RecorderModel, RecorderView } from "./Recorder";
+
+const captureStream = function (widget) {
+  if (widget.captureStream) {
+    return widget.captureStream();
+  } else {
+    return widget.stream;
+  }
+};
+
+export class ImageStreamModel extends MediaStreamModel {
+  defaults() {
+    return {
+      ...super.defaults(),
+      _model_name: "ImageStreamModel",
+      image: null,
+    };
+  }
+
+  initialize() {
+    super.initialize.apply(this, arguments);
+    window.last_image_stream = this;
+    this.canvas = document.createElement("canvas");
+    this.context = this.canvas.getContext("2d");
+
+    this.canvas.width = this.get("width");
+    this.canvas.height = this.get("height");
+    // I was hoping this should do it
+    imageWidgetToCanvas(this.get("image"), this.canvas);
+    this.get("image").on("change:value", this.sync_image, this);
+  }
+
+  sync_image() {
+    // not sure if firefox uses moz prefix also on a canvas
+    if (this.canvas.captureStream) {
+      // TODO: add a fps trait
+      // but for some reason we need to do it again
+      imageWidgetToCanvas(this.get("image"), this.canvas);
+    } else {
+      throw new Error("captureStream not supported for this browser");
+    }
+  }
+
+  async captureStream() {
+    this.sync_image();
+    return this.canvas.captureStream();
+  }
+}
+
+ImageStreamModel.serializers = {
+  ...MediaStreamModel.serializers,
+  image: { deserialize: unpack_models },
+};
+
+export class ImageRecorderModel extends RecorderModel {
+  defaults() {
+    return {
+      ...super.defaults(),
+      _model_name: "ImageRecorderModel",
+      _view_name: "ImageRecorderView",
+      image: null,
+      _height: "",
+      _width: "",
+    };
+  }
+
+  initialize() {
+    super.initialize.apply(this, arguments);
+    window.last_image_recorder = this;
+
+    this.type = "image";
+  }
+
+  async snapshot() {
+    const mimeType = this.type + "/" + this.get("format");
+    const mediaStream = await captureStream(this.get("stream"));
+    // turn the mediastream into a video element
+    let video = document.createElement("video");
+    video.srcObject = mediaStream;
+    video.play();
+    await utils.onCanPlay(video);
+    await utils.onLoadedMetaData(video);
+    // and the video element can be drawn onto a canvas
+    let canvas = document.createElement("canvas");
+    let context = canvas.getContext("2d");
+    let height = video.videoHeight;
+    let width = video.videoWidth;
+    canvas.height = height;
+    canvas.width = width;
+    context.drawImage(video, 0, 0, canvas.width, canvas.height);
+
+    // from the canvas we can get the underlying encoded data
+    // TODO: check support for toBlob, or find a polyfill
+    const blob = await utils.canvasToBlob(canvas, mimeType);
+    this.set("_data_src", window.URL.createObjectURL(blob));
+    this._last_blob = blob;
+
+    const bytes = await utils.blobToBytes(blob);
+
+    this.get(this.type).set("value", new DataView(bytes.buffer));
+    this.get(this.type).save_changes();
+    this.set("_height", height.toString() + "px");
+    this.set("_width", width.toString() + "px");
+    this.set("recording", false);
+    this.save_changes();
+  }
+
+  updateRecord() {
+    const source = this.get("stream");
+    if (!source) {
+      throw new Error("No stream specified");
+    }
+
+    if (this.get("_data_src") !== "") {
+      URL.revokeObjectURL(this.get("_data_src"));
+    }
+    if (this.get("recording")) this.snapshot();
+  }
+
+  download() {
+    let filename = this.get("filename");
+    let format = this.get("format");
+    if (filename.indexOf(".") < 0) {
+      filename = this.get("filename") + "." + format;
+    }
+    utils.downloadBlob(this._last_blob, filename);
+  }
+}
+
+ImageRecorderModel.serializers = {
+  ...RecorderModel.serializers,
+  image: { deserialize: unpack_models },
+};
+
+export class ImageRecorderView extends RecorderView {
+  initialize() {
+    super.initialize.apply(this, arguments);
+    this.tag = "img";
+    this.recordIconClass = "fa fa-camera";
+  }
+}