[WIP] [UI] Add voice dialog

bundles/org.openhab.ui/web/build/webpack.config.js

@@ -60,8 +60,9 @@ module.exports = {
     allowedHosts: "all",
     historyApiFallback: true,
     proxy: [{
-      context: ['/auth', '/rest', '/chart', '/proxy', '/icon', '/static', '/changePassword', '/createApiToken', '/audio'],
-      target: apiBaseUrl
+      context: ['/auth', '/rest', '/chart', '/proxy', '/icon', '/static', '/changePassword', '/createApiToken', '/audio', '/ws/pcm-audio'],
+      target: apiBaseUrl,
+      ws: true
     }]
   },
   performance: {

bundles/org.openhab.ui/web/package.json

@@ -93,6 +93,8 @@
     "path-browserify": "^1.0.1",
     "pkce-challenge": "^2.1.0",
     "qrcode": "^1.0.0",
+    "reentrant-lock": "^3.0.0",
+    "rustpotter-worklet": "^3.0.3",
     "scope-css": "^1.2.1",
     "sprintf-js": "^1.1.2",
     "stream-browserify": "^3.0.0",
@@ -115,6 +117,7 @@
     "vue2-leaflet": "^2.5.2",
     "vuetrend": "^0.3.4",
     "vuex": "^3.5.1",
+    "wave-resampler": "^1.0.0",
     "yaml": "^2.3.4"
   },
   "devDependencies": {

bundles/org.openhab.ui/web/src/assets/i18n/theme-switcher/en.json

@@ -14,5 +14,15 @@
   "about.miscellaneous.theme.disablePageTransition": "Disable page transition animations",
   "about.miscellaneous.webaudio.enable": "Enable Web Audio sink support",
   "about.miscellaneous.commandItem.title": "Listen for UI commands to ",
-  "about.miscellaneous.commandItem.selectItem": "Item"
+  "about.miscellaneous.commandItem.selectItem": "Item",
+  "about.dialog": "Voice Support",
+  "about.dialog.enable": "Enable Voice Dialog",
+  "about.dialog.id": "Connection Id",
+  "about.dialog.listeningItem": "Listening Item",
+  "about.dialog.locationItem": "Location Item",
+  "about.dialog.keyword": "Keyword File",
+  "about.dialog.keyword.choose": "Choose",
+  "about.dialog.keyword.remove": "Remove",
+  "about.dialog.keyword.threshold": "Keyword Threshold",
+  "about.dialog.keyword.minScores": "Keyword Min. Detections"
 }

bundles/org.openhab.ui/web/src/components/app.vue

@@ -256,12 +256,13 @@ import auth from './auth-mixin'
 import i18n from './i18n-mixin'
 import connectionHealth from './connection-health-mixin'
 import sseEvents from './sse-events-mixin'
+import dialog from './dialog-mixin'
 
 import dayjs from 'dayjs'
 import dayjsLocales from 'dayjs/locale.json'
 
 export default {
-  mixins: [auth, i18n, connectionHealth, sseEvents],
+  mixins: [auth, i18n, connectionHealth, sseEvents, dialog],
   components: {
     EmptyStatePlaceholder,
     PanelRight,
@@ -715,11 +716,16 @@ export default {
         }
       })
 
+      this.$f7.on('triggerDialog', () => {
+        this.triggerDialog()
+      })
+
       if (window) {
         window.addEventListener('keydown', this.keyDown)
       }
 
       this.startEventSource()
+      this.startAudioWebSocket()
     })
   }
 }

bundles/org.openhab.ui/web/src/components/dialog-mixin.js

@@ -0,0 +1,65 @@
+export default {
+  data () {
+    return {
+      audioMain: null
+    }
+  },
+  methods: {
+    startAudioWebSocket () {
+      if (this.audioMain) {
+        return
+      }
+      const dialogEnabled = localStorage.getItem('openhab.ui:dialog.enabled') === 'true'
+      if (!dialogEnabled) {
+        console.warn('CHECK audio not enabled')
+        return
+      }
+      const identifier = localStorage.getItem('openhab.ui:dialog.id') ?? 'anonymous'
+      const dialogListeningItem = localStorage.getItem('openhab.ui:dialog.listeningItem') ?? ''
+      const dialogLocationItem = localStorage.getItem('openhab.ui:dialog.locationItem') ?? ''
+      const dialogKeywordThreshold = Number(localStorage.getItem('openhab.ui:dialog.keyword.threshold') ?? '0.75')
+      const dialogKeywordMinScores = Number(localStorage.getItem('openhab.ui:dialog.keyword.minScores') ?? '5')
+      import('../js/voice/audio-main.js').then(({ AudioMain }) => {
+        if (this.audioMain) {
+          return
+        }
+        let port = ''
+        if (!((location.protocol === 'https:' && location.port === '443') || (location.protocol === 'http:' && location.port === '80'))) {
+          port = `:${location.port}`
+        }
+        const ohURL = `${location.protocol}//${location.hostname}${port}`
+        this.audioMain = new AudioMain(ohURL, {
+          onMessage: (...args) => {
+            console.warn('MESSAGE: ' + args[0])
+          },
+          onRunningChange (io) {
+            console.warn('CHECK ONLINE: ' + io.isRunning())
+          },
+          onListeningChange (io) {
+            console.warn('CHECK LISTENING: ' + io.isListening())
+          },
+          onSpeakingChange (io) {
+            console.warn('CHECK SPEAKING: ' + io.isSpeaking())
+          }
+        })
+        const events = ['touchstart', 'touchend', 'mousedown', 'keydown']
+        const startAudio = () => {
+          clean()
+          this.audioMain.initialize(identifier, dialogListeningItem, dialogLocationItem, { threshold: dialogKeywordThreshold, minScores: dialogKeywordMinScores })
+        }
+        const clean = () => events.forEach(e => document.body.removeEventListener(e, startAudio))
+        events.forEach(e => document.body.addEventListener(e, startAudio, false))
+      })
+    },
+    stopAudioWebSocket () {
+      // TODO
+      this.audioMain.close()
+      this.audioMain = null
+    },
+    triggerDialog () {
+      if (this.audioMain != null) {
+        this.audioMain.sendSpot()
+      }
+    }
+  }
+}