feat(wt_video): implement command and control

worldcoin · Feb 25, 2025 · 08e24ff · 08e24ff
1 parent 10d2817
commit 08e24ff
Show file tree

Hide file tree

Showing 9 changed files with 325 additions and 54 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -67,7 +67,6 @@ ed25519-dalek = { version = "2.1.1", default-features = false, features = ["std"
 eyre = "0.6.12"
 ftdi-embedded-hal = { version = "0.22.0", features = ["libftd2xx", "libftd2xx-static"] }
 futures = "0.3.30"
-futures-lite = "2.6.0"
 hex = "0.4.3"
 hex-literal = "0.4.1"
 http = "1.2.0"
@@ -77,8 +76,6 @@ nix = { version = "0.28", default-features = false, features = [] }
 opentelemetry = { version = "0.27", features = ["trace"] }
 opentelemetry-otlp = { version = "0.27", default-features = false }
 opentelemetry_sdk = "0.27"
-prost = "0.13.4"
-prost-build = "0.13.4"
 reqwest = { version = "0.12.9", default-features = false, features = ["rustls-tls", "stream"] }
 ring = "0.16"
 rustix = "0.38.37"

diff --git a/experiments/webtransport_video/Cargo.toml b/experiments/webtransport_video/Cargo.toml
@@ -14,11 +14,15 @@ base64.workspace = true
 clap = { workspace = true, features = ["derive"] }
 color-eyre.workspace = true
 derive_more = { workspace = true, features = ["into", "as_ref", "deref"] }
+futures.workspace = true
+keycode = { version = "0.4.0", features = ["serde"] }
 orb-telemetry.workspace = true
 png = "0.17.16"
 rcgen = "0.13.2"
+serde = { workspace = true, features = ["derive"] }
 tokio = { workspace = true, features = ["full"] }
-tokio-util.workspace = true
+tokio-serde = { version = "0.9.0", features = ["json"] }
+tokio-util = { workspace = true, features = ["codec"] }
 tower = "0.5.2"
 tower-http = { version = "0.6.2", features = ["fs"] }
 tracing.workspace = true

diff --git a/experiments/webtransport_video/index.html b/experiments/webtransport_video/index.html
@@ -7,6 +7,7 @@
   <body>
     <h1>WebTransport Video Frames</h1>
     <canvas id="videoFrame" width="640" height="480"></canvas>
+    <div id="position">Position: x=0, y=0</div>
     <script type="module" src="./index.js"></script>
   </body>
 </html>
diff --git a/experiments/webtransport_video/index.js b/experiments/webtransport_video/index.js
@@ -2,6 +2,8 @@
 // e.g., "https://192.168.0.100:443" or the appropriate path
 const transportUrl = "https://localhost:1337";
 
+const COMMAND_EVENT_NAME = "commandEvt";
+
 console.log("running script");
 
 // Helper function to convert a reader to an async iterable
@@ -43,7 +45,83 @@ async function handleIncomingStream(stream) {
   return blob;
 }
 
+// Function to get cursor position relative to canvas
+function getCursorPosition(canvas, event) {
+  // Get the bounding rectangle of the canvas
+  const rect = canvas.getBoundingClientRect();
+  const x = (event.clientX - rect.left) / rect.width;
+  const y = (event.clientY - rect.top) / rect.height;
+  return { x, y };
+}
+
+function setUpElements(canvas, positionDisplay) {
+  // Style the canvas to make it visible
+  canvas.style.border = "1px solid black";
+  canvas.style.backgroundColor = "#f0f0f0";
+  canvas.style.padding = "0";
+  canvas.style.margin = "0";
+
+  canvas.addEventListener("mousemove", function (event) {
+    const position = getCursorPosition(canvas, event);
+    const x = position.x.toFixed(2);
+    const y = position.y.toFixed(2);
+    positionDisplay.textContent = `Position: x=${x}, y=${y}`;
+    const obj = {
+      MouseEvent: {
+        Move: {
+          x: position.x,
+          y: position.y,
+        },
+      },
+    };
+    const commandEvt = new CustomEvent(COMMAND_EVENT_NAME, {
+      detail: obj,
+    });
+    canvas.dispatchEvent(commandEvt);
+  });
+
+  canvas.addEventListener("mouseleave", function () {
+    positionDisplay.textContent = "Position: x=-, y=-";
+    const obj = {
+      MouseEvent: "Unfocus",
+    };
+    const commandEvt = new CustomEvent(COMMAND_EVENT_NAME, {
+      detail: obj,
+    });
+    canvas.dispatchEvent(commandEvt);
+  });
+}
+
+function encodeWithLengthPrefix(obj) {
+  const jsonString = JSON.stringify(obj);
+
+  // Convert JSON string to UTF-8 encoded bytes
+  const encoder = new TextEncoder();
+  const jsonBytes = encoder.encode(jsonString);
+
+  // Create a buffer with enough space for the 32-bit length + JSON content
+  const buffer = new ArrayBuffer(4 + jsonBytes.byteLength);
+
+  // Create a view to write the 32-bit length prefix
+  const view = new DataView(buffer);
+  view.setUint32(0, jsonBytes.byteLength, false); // false = big endian
+
+  // Create a view for the entire buffer
+  const uint8View = new Uint8Array(buffer);
+
+  // Copy the JSON bytes after the length prefix
+  uint8View.set(jsonBytes, 4);
+
+  return buffer;
+}
+
 async function main() {
+  const canvas = document.getElementById("videoFrame");
+  const ctx = canvas.getContext("2d");
+  const positionDisplay = document.getElementById("position");
+
+  setUpElements(canvas, positionDisplay);
+
   const hash_response = await fetch("/cert_hash");
   if (!hash_response.ok) {
     throw new Error(`Response status: ${hash_response.status}`);
@@ -69,8 +147,21 @@ async function main() {
     return;
   }
 
-  const canvas = document.getElementById("videoFrame");
-  const ctx = canvas.getContext("2d");
+  const controlStream = await transport.createUnidirectionalStream();
+  const controlWriter = controlStream.getWriter();
+  canvas.addEventListener(COMMAND_EVENT_NAME, async (evt) => {
+    const obj = evt.detail;
+    console.log("Got command event:", obj);
+    const encodedBuffer = encodeWithLengthPrefix(obj);
+
+    // Convert ArrayBuffer to Uint8Array if not already
+    const dataToSend =
+      encodedBuffer instanceof ArrayBuffer
+        ? new Uint8Array(encodedBuffer)
+        : encodedBuffer;
+    await controlWriter.write(dataToSend);
+    console.log("Data successfully sent");
+  });
 
   const streamReader = transport.incomingUnidirectionalStreams.getReader();
   try {
@@ -79,8 +170,6 @@ async function main() {
       try {
         // Handle the incoming stream
         const blob = await handleIncomingStream(stream);
-        // Now you can use the blob
-        console.log("Received blob size:", blob.size);
 
         // Create an image bitmap from the blob
         const bitmap = await createImageBitmap(blob);