Merge pull request #663 from SAGE-3/dev-webview

[App] Web streaming
SAGE-3 · Aug 3, 2023 · 5ac7d1f · 5ac7d1f
2 parents a2ea611 + 1cbcf83
commit 5ac7d1f
Show file tree

Hide file tree

Showing 13 changed files with 372 additions and 140,070 deletions.
diff --git a/sage_sbom.json b/sage_sbom.json
diff --git a/webstack/apps/homebase/src/main.ts b/webstack/apps/homebase/src/main.ts
@@ -189,64 +189,47 @@ async function startServer() {
   });
 
   // Websocket API for WebRTC
-  const clients: Record<string, WebSocket> = {};
+  const clients: Map<string, WebSocket[]> = new Map();
 
-  function emitRTC(name: string, socket: WebSocket, data: any) {
-    for (const k in clients) {
-      const sock = clients[k];
-      if (sock !== socket) {
-        sock.send(JSON.stringify({ type: name, data: data }));
-      }
-    }
+  // Broadcast to all clients in the room
+  function emitRTC(room: string, type: string, params: any) {
+    const msg = JSON.stringify({ type, params });
+    clients.get(room)?.forEach((ws) => ws.send(msg));
   }
-  async function sendRTC(name: string, socket: WebSocket, data: any) {
-    socket.send(JSON.stringify({ type: name, data: data }));
-  }
-
-  rtcWebSocketServer.on('connection', (socket: WebSocket, request: IncomingMessage) => {
-    console.log('WebRTC> connection', request.url);
-
-    if (request.url) {
-      const parts = request.url.split('/');
-      const roomID = parts[parts.length - 1];
-      console.log('WebRTC> roomID', roomID);
-    }
 
+  rtcWebSocketServer.on('connection', (socket: WebSocket, _request: IncomingMessage) => {
+    // new message
     socket.on('message', (data) => {
       const datastr = data.toString();
       const msg = JSON.parse(datastr);
-      if (msg.type === 'join') {
-        clients[msg.user] = socket;
-        emitRTC('join', socket, msg);
-        console.log('WebRTC> connection #', Object.keys(clients).length);
-        sendRTC('clients', socket, Object.keys(clients));
-      } else if (msg.type === 'create') {
-        clients[msg.user] = socket;
-        console.log('WebRTC> new group for', msg.app);
-      } else if (msg.type === 'paint') {
-        emitRTC('paint', socket, msg.data);
+      switch (msg.type) {
+        case 'join':
+          if (!clients.has(msg.params.room)) {
+            clients.set(msg.params.room, []);
+          }
+          clients.get(msg.params.room)?.push(socket);
+          break;
+        case 'pixels':
+          // broadcast to all clients in the room
+          emitRTC(msg.params.room, 'data', msg.params);
+          break;
+        case 'leave':
+          clients.get(msg.params.room)?.splice(clients.get(msg.params.room)?.indexOf(socket) || 0, 1);
+          break;
       }
     });
-    socket.on('close', (_msg) => {
-      console.log('WebRTC> close');
-      // Delete the socket from the clients array
-      for (const [key, value] of Object.entries(clients)) {
-        if (value === socket) {
-          delete clients[key];
-          emitRTC('left', socket, key);
-        }
-      }
-      console.log('WebRTC> connection #', Object.keys(clients).length);
+    // close handler
+    socket.on('close', () => {
+      clients.forEach((sockets) => {
+        sockets.splice(sockets.indexOf(socket) || 0, 1);
+      });
     });
+    // error handler
     socket.on('error', (msg) => {
       console.log('WebRTC> error', msg);
-      // Delete the socket from the clients array
-      for (const [key, value] of Object.entries(clients)) {
-        if (value === socket) {
-          delete clients[key];
-        }
-      }
-      console.log('WebRTC> connection #', Object.keys(clients).length);
+      clients.forEach((sockets) => {
+        sockets.splice(sockets.indexOf(socket) || 0, 1);
+      });
     });
   });
 

diff --git a/webstack/apps/webapp/src/app/pages/board/layers/background/components/Background.tsx b/webstack/apps/webapp/src/app/pages/board/layers/background/components/Background.tsx
@@ -198,7 +198,7 @@ export function Background(props: BackgroundProps) {
             const valid = isValidURL(pastedText);
             if (valid) {
               // Create a link app
-              createApp(setupApp('', 'WebpageLink', xdrop, ydrop, props.roomId, props.boardId, { w: 400, h: 400 }, { url: pastedText }));
+              createApp(setupApp('WebpageLink', 'WebpageLink', xdrop, ydrop, props.roomId, props.boardId, { w: 400, h: 400 }, { url: pastedText }));
             }
           }
         }

diff --git a/webstack/clients/electron/electron.js b/webstack/clients/electron/electron.js
@@ -649,27 +649,40 @@ function createWindow() {
     // webPreferences.nodeIntegration = true;
     // params.nodeIntegration = true;
 
+    ipcMain.on('streamview_stop', (e, args) => {
+      // Message for the webview pixel streaming
+      const viewContent = electron.webContents.fromId(args.id);
+      if (viewContent) viewContent.endFrameSubscription();
+    });
     ipcMain.on('streamview', (e, args) => {
-      // console.log('Webview> IPC Message', evt.frameId, evt.processId, evt.reply);
       // console.log('Webview>    message', channel, args);
+      // console.log('Webview> IPC Message', args.id, args.width, args.height);
+
       // Message for the webview pixel streaming
       const viewContent = electron.webContents.fromId(args.id);
-      viewContent.beginFrameSubscription(true, (image, dirty) => {
+
+      viewContent.enableDeviceEmulation({
+        screenPosition: 'mobile',
+        screenSize: { width: args.width, height: args.height },
+      });
+
+      viewContent.beginFrameSubscription(false, (image, dirty) => {
         let dataenc;
         let neww, newh;
         const devicePixelRatio = 2;
-        const quality = 50;
+        const quality = 60;
         if (devicePixelRatio > 1) {
           neww = dirty.width / devicePixelRatio;
           newh = dirty.height / devicePixelRatio;
-          const resizedimage = image.resize({ width: neww, height: newh });
+          const resizedimage = image.resize({ width: neww, height: newh, quality: 'better' });
           dataenc = resizedimage.toJPEG(quality);
         } else {
           dataenc = image.toJPEG(quality);
           neww = dirty.width;
           newh = dirty.height;
         }
         mainWindow.webContents.send('paint', {
+          id: args.id,
           buf: dataenc.toString('base64'),
           dirty: { ...dirty, width: neww, height: newh },
         });

diff --git a/webstack/clients/electron/package.json b/webstack/clients/electron/package.json
@@ -1,6 +1,6 @@
 {
   "name": "SAGE3",
-  "version": "1.0.5",
+  "version": "1.0.6",
   "description": "Electron-based SAGE3 client",
   "main": "electron.js",
   "repository": {
@@ -18,7 +18,7 @@
     "uuid": "8.3.0"
   },
   "devDependencies": {
-    "electron": "^25.3.0",
+    "electron": "^25.4.0",
     "electron-notarize": "^1",
     "electron-packager": "latest"
   },

diff --git a/webstack/clients/electron/preload.js b/webstack/clients/electron/preload.js
@@ -19,6 +19,7 @@ const validChannels = [
   'close-connect-page',
   'take-screenshot',
   'streamview',
+  'streamview_stop',
   'paint',
   'load-landing',
   'store-interface',

diff --git a/webstack/clients/electron/yarn.lock b/webstack/clients/electron/yarn.lock
@@ -620,10 +620,10 @@ electron-winstaller@latest:
     lodash.template "^4.2.2"
     temp "^0.9.0"
 
-electron@^25.3.0:
-  version "25.3.0"
-  resolved "https://registry.yarnpkg.com/electron/-/electron-25.3.0.tgz#e818ab3ebd3e7a45f8fca0f47e607c9af2dc92c7"
-  integrity sha512-cyqotxN+AroP5h2IxUsJsmehYwP5LrFAOO7O7k9tILME3Sa1/POAg3shrhx4XEnaAMyMqMLxzGvkzCVxzEErnA==
+electron@^25.4.0:
+  version "25.4.0"
+  resolved "https://registry.yarnpkg.com/electron/-/electron-25.4.0.tgz#d45b1cf3e4e96eb5bff5fee704d7aa13b532f3a5"
+  integrity sha512-VLTRxDhL4UvQbqM7pTNENnJo62cdAPZT92N+B7BZQ5Xfok1wuVPEewIjBot4K7U3EpLUuHn1veeLzho3ihiP+Q==
   dependencies:
     "@electron/get" "^2.0.0"
     "@types/node" "^18.11.18"