Merge pull request #114 from Nerimity/Push-To-talk

Prepare event handling

src/chat-api/events/voiceEvents.ts

@@ -4,13 +4,16 @@ import useAccount from "../store/useAccount";
 import useVoiceUsers from "../store/useVoiceUsers";
 
 export function onVoiceUserJoined(payload: RawVoice) {
-  const {createVoiceUser} = useVoiceUsers();
+  const { createVoiceUser } = useVoiceUsers();
 
   createVoiceUser(payload);
 }
-export function onVoiceUserLeft(payload: {userId: string, channelId: string}) {
-  const {removeVoiceUser, setCurrentChannelId} = useVoiceUsers();
-  const {user} = useAccount();
+export function onVoiceUserLeft(payload: {
+  userId: string;
+  channelId: string;
+}) {
+  const { removeVoiceUser, setCurrentChannelId } = useVoiceUsers();
+  const { user } = useAccount();
 
   if (user()?.id === payload.userId) {
     setCurrentChannelId(null);
@@ -27,12 +30,16 @@ interface VoiceSignalReceivedPayload {
 
 export function onVoiceSignalReceived(payload: VoiceSignalReceivedPayload) {
   const voiceUsers = useVoiceUsers();
-  const voiceUser = voiceUsers.getVoiceUser(payload.channelId, payload.fromUserId);
+
+  const voiceUser = voiceUsers.getVoiceUser(
+    payload.channelId,
+    payload.fromUserId
+  );
   if (!voiceUser) return;
 
   if (!voiceUser.peer) {
     return voiceUsers.createPeer(voiceUser, payload.signal);
   }
 
   voiceUsers.signal(voiceUser, payload.signal);
-}
+}

src/chat-api/store/useVoiceUsers.ts

@@ -1,16 +1,23 @@
 import { createStore, reconcile } from "solid-js/store";
 import { RawVoice } from "../RawData";
-import { batch, createEffect, createSignal } from "solid-js";
+import { batch, createEffect, createMemo, createSignal, on } from "solid-js";
 import { getCachedCredentials } from "../services/VoiceService";
 import { emitVoiceSignal } from "../emits/voiceEmits";
 
 import type SimplePeer from "@thaunknown/simple-peer";
 import useUsers, { User } from "./useUsers";
 import LazySimplePeer from "@/components/LazySimplePeer";
-import { getStorageString, StorageKeys } from "@/common/localStorage";
+import {
+  getStorageObject,
+  getStorageString,
+  StorageKeys,
+  useVoiceInputMode,
+} from "@/common/localStorage";
 import useAccount from "./useAccount";
 import { set } from "idb-keyval";
 import vad from "voice-activity-detection";
+import { downKeys, useGlobalKey } from "@/common/GlobalKey";
+import { arrayEquals } from "@/common/arrayEquals";
 
 const createIceServers = () => [
   getCachedCredentials(),
@@ -73,6 +80,49 @@ const [currentVoiceUser, setCurrentVoiceUser] = createSignal<
   CurrentVoiceUser | undefined
 >(undefined);
 
+const { start, stop } = useGlobalKey();
+const [voiceMode] = useVoiceInputMode();
+
+createEffect(
+  on(currentVoiceUser, (current) => {
+    stop();
+    if (!current?.channelId) return;
+    if (voiceMode() !== "PTT") return;
+    start();
+  })
+);
+
+const micTrack = createMemo(() => {
+  const current = currentVoiceUser();
+  return current?.audioStream?.getAudioTracks()[0];
+});
+
+createEffect(
+  on(
+    () => downKeys.length,
+    () => {
+      const bound = getStorageObject(StorageKeys.PTTBoundKeys, []);
+      if (!bound.length) return;
+      const mic = micTrack();
+      if (!mic) return;
+      const current = currentVoiceUser();
+      if (!current) return;
+
+      if (!arrayEquals(downKeys, bound)) {
+        mic.enabled = false;
+        setVoiceUsers(current.channelId, useAccount().user()?.id!, {
+          voiceActivity: false,
+        });
+        return;
+      }
+      mic.enabled = true;
+      setVoiceUsers(current.channelId, useAccount().user()?.id!, {
+        voiceActivity: true,
+      });
+    }
+  )
+);
+
 const setCurrentChannelId = (channelId: string | null) => {
   const current = currentVoiceUser();
   if (current?.channelId) {
@@ -328,8 +378,10 @@ function createVadInstance(
         }
       : {
           minNoiseLevel: 0,
-          noiseCaptureDuration: 0,
+          noiseCaptureDuration: 100,
           avgNoiseMultiplier: 0.1,
+          maxNoiseLevel: 0.01,
+          onUpdate: console.log,
         }),
 
     onVoiceStart: function () {
@@ -413,12 +465,30 @@ const toggleMic = async () => {
     audio: !deviceId ? true : { deviceId: JSON.parse(deviceId) },
     video: false,
   });
-  const vadStream = await navigator.mediaDevices.getUserMedia({
-    audio: !deviceId ? true : { deviceId: JSON.parse(deviceId) },
-    video: false,
-  });
+
+  let vadStream: MediaStream | undefined;
+  let vadInstance: ReturnType<typeof vad> | undefined;
+
+  if (voiceMode() === "OPEN") {
+    setVoiceUsers(current.channelId, useAccount().user()?.id!, {
+      voiceActivity: true,
+    });
+  }
+
+  if (voiceMode() !== "OPEN") {
+    stream.getAudioTracks()[0]!.enabled = false;
+  }
+
+  if (voiceMode() === "VOICE_ACTIVITY") {
+    vadStream = await navigator.mediaDevices.getUserMedia({
+      audio: !deviceId ? true : { deviceId: JSON.parse(deviceId) },
+      video: false,
+    });
+    vadInstance = createVadInstance(vadStream, stream);
+  }
+
   addStreamToPeers(stream);
-  const vadInstance = createVadInstance(vadStream, stream);
+
   setCurrentVoiceUser({
     ...current,
     audioStream: stream,

src/common/Electron.ts

@@ -26,6 +26,17 @@ export const [spellcheckSuggestions, setSpellcheckSuggestions] = createSignal<
   string[]
 >([]);
 
+type KeyState = "DOWN" | "UP";
+
+interface GlobalKeyEvent {
+  event: {
+    name: string;
+    vKey: number;
+    state: KeyState;
+  };
+  down: Record<string, boolean>;
+}
+
 interface WindowAPI {
   isElectron: boolean;
   minimize(): void;
@@ -58,6 +69,10 @@ interface WindowAPI {
   clipboardPaste(): void;
   clipboardCopy(text: string): void;
   clipboardCut(): void;
+
+  startGlobalKeyListener: () => void;
+  stopGlobalKeyListener: () => void;
+  onGlobalKey: (callback: (event: GlobalKeyEvent) => void) => void;
 }
 
 export function electronWindowAPI(): WindowAPI | undefined {

src/common/GlobalKey.ts

@@ -0,0 +1,84 @@
+import { onCleanup, onMount } from "solid-js";
+import { electronWindowAPI } from "./Electron";
+import { createStore, reconcile } from "solid-js/store";
+
+export const [downKeys, setDownKeys] = createStore<(string | number)[]>([]);
+
+const onMouseDown = (e: MouseEvent) => {
+  if (e.button === 0) return;
+  const code = `MOUSE ${e.button}`;
+  if (!downKeys.includes(code)) {
+    setDownKeys([...downKeys, code]);
+  }
+};
+const onMouseUp = (e: MouseEvent) => {
+  if (e.button === 0) return;
+  const code = `MOUSE ${e.button}`;
+  setDownKeys(downKeys.filter((k) => k !== code));
+};
+
+const onKeyDown = (e: KeyboardEvent) => {
+  let code = e.code || e.key;
+  if (code.startsWith("Key")) {
+    code = code.slice(3);
+  }
+  if (!downKeys.includes(code)) {
+    setDownKeys([...downKeys, code]);
+  }
+};
+
+const onKeyUp = (e: KeyboardEvent) => {
+  let code = e.code || e.key;
+  if (code.startsWith("Key")) {
+    code = code.slice(3);
+  }
+  setDownKeys(downKeys.filter((k) => k !== code));
+};
+
+if (electronWindowAPI()?.isElectron) {
+  electronWindowAPI()?.onGlobalKey(({ event }) => {
+    const key = event.name || event.vKey;
+    if (event.name === "MOUSE LEFT") return;
+    if (event.state === "DOWN") {
+      if (!downKeys.includes(key)) {
+        setDownKeys([...downKeys, key]);
+      }
+    } else {
+      setDownKeys(downKeys.filter((k) => k !== key));
+    }
+  });
+}
+
+export const useGlobalKey = () => {
+  let started = false;
+  const start = () => {
+    started = true;
+    if (!electronWindowAPI()?.isElectron) {
+      document.addEventListener("mousedown", onMouseDown);
+      document.addEventListener("mouseup", onMouseUp);
+      document.addEventListener("keydown", onKeyDown);
+      document.addEventListener("keyup", onKeyUp);
+      return;
+    }
+    electronWindowAPI()?.startGlobalKeyListener();
+  };
+
+  const stop = () => {
+    if (!started) return;
+    setDownKeys(reconcile([]));
+    if (!electronWindowAPI()?.isElectron) {
+      document.removeEventListener("mousedown", onMouseDown);
+      document.removeEventListener("mouseup", onMouseUp);
+      document.removeEventListener("keydown", onKeyDown);
+      document.removeEventListener("keyup", onKeyUp);
+      return;
+    }
+    electronWindowAPI()?.stopGlobalKeyListener();
+  };
+
+  onCleanup(() => {
+    stop();
+  });
+
+  return { start, stop };
+};

src/common/arrayEquals.ts

@@ -0,0 +1,6 @@
+export function arrayEquals<A extends unknown[], B extends unknown[]>(
+  a: A,
+  b: B
+) {
+  return a.length === b.length && a.every((v, i) => v === b[i]);
+}

src/common/localStorage.ts

@@ -21,6 +21,8 @@ export enum StorageKeys {
   CUSTOM_COLORS = "customColors",
   inputDeviceId = "inputDeviceId",
   outputDeviceId = "outputDeviceId",
+  voiceInputMode = "voiceInputMode",
+  PTTBoundKeys = "pttBoundKeys",
 }
 
 export function getStorageBoolean(
@@ -85,3 +87,9 @@ export function useReactiveLocalStorage<T>(key: StorageKeys, defaultValue: T) {
 
   return [value, setCustomValue] as const;
 }
+
+const voiceInputMode = useReactiveLocalStorage<
+  "OPEN" | "VOICE_ACTIVITY" | "PTT"
+>(StorageKeys.voiceInputMode, "VOICE_ACTIVITY");
+
+export const useVoiceInputMode = () => voiceInputMode;