Voice

client/package.json

@@ -122,6 +122,7 @@
     "@types/styled-components": "^5.1.26",
     "@web3modal/ethereum": "^2.1.1",
     "@web3modal/react": "^2.1.1",
+    "audio-recorder-polyfill": "^0.4.1",
     "axios": "^0.21.1",
     "babel-plugin-styled-components": "^2.0.7",
     "bn.js": "^5.2.1",
@@ -149,6 +150,7 @@
     "react-text-mask": "^5.5.0",
     "react-toastify": "^9.0.8",
     "react-twitter-embed": "^4.0.4",
+    "socket.io-client": "^4.6.1",
     "styled-components": "^5.3.0",
     "text-mask-addons": "^3.8.0",
     "three-dots": "^0.3.2",
@@ -158,6 +160,7 @@
     "web3": "1.8.2",
     "web3-eth-abi": "^1.7.4",
     "web3-eth-contract": "1.8.2",
+    "webm-to-wav-converter": "^1.1.0",
     "zerg": "^2.1.2"
   },
   "resolutions": {

client/src/Routes.tsx

@@ -2,6 +2,7 @@ import React, { lazy, Suspense } from 'react'
 import { Routes, Route } from 'react-router-dom'
 import { HomePage } from './routes/home/HomePage'
 import { HomePageLoader } from './routes/home/components/HomePageLoader'
+import { VoicePage } from './routes/voice/VoicePage'
 
 const StatsPage = lazy(
   () => import(/* webpackChunkName: "Others" */ './routes/stats/Stats')
@@ -37,6 +38,7 @@ const AppRoutes = () => {
         <Route path="stats/" element={<StatsPage />} />
         <Route path="details/" element={<DetailsPage />} />
         <Route path="live/" element={<LiveStreamPage />} />
+        <Route path="voice" element={<VoicePage />} />
         <Route path="*" element={<HomePage />} />
       </Routes>
     </Suspense>

client/src/custom.d.ts

@@ -13,5 +13,6 @@ declare module '*.gif'
 declare module 'react-video-thumbnail-image'
 declare module 'react-fb-image-video-grid'
 declare module 'grommet-icons/icons/FormClose'
+declare module 'grommet-icons/icons/Microphone'
 declare module 'grommet-icons/icons/FormSearch'
 declare module 'use-lodash-debounce'

client/src/routes/voice/VoicePage.tsx

@@ -0,0 +1,225 @@
+import React, { useEffect, useRef, useState } from 'react'
+import { Container } from '../home/Home.styles'
+import { Box } from 'grommet/components/Box'
+import { observable } from 'mobx'
+import { observer } from 'mobx-react-lite'
+import { RecordStatus, Translation } from './types'
+import { TranslationItem } from './components/TranslationItem'
+import { RecordButton } from './components/RecordButton'
+import config from '../../../config'
+
+import { io } from 'socket.io-client'
+
+import AudioRecorder from 'audio-recorder-polyfill'
+import styled from 'styled-components'
+window.MediaRecorder = AudioRecorder
+
+const translationList = observable<Translation>([])
+const tList = observable<any>([])
+
+const TranslationText = styled.span<{ temp: boolean }>`
+  color: ${(props) => (props.temp ? '#c1c1c1' : '#1f1f1f')};
+`
+
+const socket = io(config.backendHost)
+
+socket.on('ReadyToTranslation', (data) => {
+  console.log('### ReadyToTranslation')
+})
+
+// [].reduce((acc, item) => {
+//
+//   // перезависыавем последний елемент
+//   // case 1: пришел AddPartialTranscript
+//   // case 2: пришел AddTranscript
+//   if(acc[acc.length - 1].start_time === item.start_time) {
+//     acc.push(item)
+//   }
+//
+//   return acc;
+// });
+
+export const VoicePage: React.FC = observer(() => {
+  const [mediaRecorder, setMediaRecorder] = useState<MediaRecorder>(null)
+  const recordedChunks = useRef<Blob[]>([])
+  const [status, setStatus] = useState<RecordStatus>('stopped')
+  const [transcription, setTranscription] = useState('')
+
+  useEffect(() => {
+    socket.on('AddTranscript', (data) => {
+      console.log('### AddTranscript data', data)
+      tList.push(data)
+    })
+
+    socket.on('AddPartialTranscript', (data) => {
+      console.log('### AddPartialTranscript data', data)
+      tList.push(data)
+    })
+  }, [])
+
+  useEffect(() => {
+    return () => {
+      socket.disconnect()
+    }
+  }, [])
+
+  const handleStartRecording = async () => {
+    const stream = await navigator.mediaDevices.getUserMedia({
+      audio: true,
+    })
+
+    socket.emit('StartRecognition')
+
+    const recorder = new window.MediaRecorder(stream, {})
+
+    recorder.addEventListener('dataavailable', (event) => {
+      socket.emit('AddAudio', event.data)
+      if (event.data.size > 0) {
+        recordedChunks.current.push(event.data)
+      }
+    })
+
+    recorder.addEventListener('stop', () => {
+      // const blob = new Blob(recordedChunks.current, {
+      //   type: recorder.mimeType,
+      // })
+      //
+
+      socket.emit('EndOfStream')
+      stream.getTracks().forEach(function (track) {
+        if (track.readyState == 'live') {
+          track.stop()
+        }
+      })
+      //
+      // console.log('### recorder', recorder.mimeType)
+      // const url = URL.createObjectURL(blob)
+      //
+      // // const a = document.createElement('a')
+      // // document.body.appendChild(a)
+      // // a.href = url
+      // // a.download = `test${Date.now()}.wav`
+      // // a.click()
+      // //
+      // // return
+      //
+      // const dateTime = Date.now()
+      //
+      // const translation: Translation = {
+      //   translation: '',
+      //   audio: url,
+      //   date: dateTime,
+      //   inProgress: true,
+      // }
+      //
+      // translationList.unshift(translation)
+      //
+      // const formData = new FormData()
+      //
+      // const wrapper = (blob: Blob) => {
+      //   if (recorder.mimeType.includes('audio/webm')) {
+      //     console.log('### to wav')
+      //
+      //     return getWaveBlob(blob, false)
+      //   }
+      //
+      //   return Promise.resolve(blob)
+      // }
+      //
+      // wrapper(blob)
+      //   .then((blob) => {
+      //     formData.append('audio', blob, 'voice_record' + translation.date)
+      //
+      //     console.log('### blob', blob)
+      //
+      //     return axios.post<{ result: string }>(
+      //       config.backendHost + '/translations',
+      //       formData
+      //     )
+      //   })
+      //   .then((t) => {
+      //     const item = translationList.find((item) => {
+      //       return item.date === dateTime
+      //     })
+      //
+      //     item.translation = t.data.result
+      //     item.inProgress = false
+      //
+      //     console.log('### translationList', translationList)
+      //   })
+      //
+      // recordedChunks.current = []
+    })
+
+    recorder.start(2000)
+    setMediaRecorder(recorder)
+    setStatus('recording')
+  }
+
+  const handleStopRecording = () => {
+    if (mediaRecorder) {
+      mediaRecorder.stop()
+      setStatus('stopped')
+    }
+  }
+
+  return (
+    <Container>
+      <Box pad="20px" align="center">
+        <div style={{ position: 'fixed', bottom: '50px', zIndex: 100 }}>
+          <RecordButton
+            status={status}
+            onClick={
+              status === 'recording'
+                ? handleStopRecording
+                : handleStartRecording
+            }
+          />
+        </div>
+        <Box justify="center" gap="8px">
+          <div>{transcription}</div>
+
+          <div>{tList.length}</div>
+          <div>
+            {tList
+              .reduce((acc, item) => {
+                if (acc.length === 0) {
+                  acc.push(item)
+                  console.log('### first')
+                  return acc
+                }
+                // перезависыавем последний елемент
+                // case 1: пришел AddPartialTranscript
+                // case 2: пришел AddTranscript
+                if (
+                  acc[acc.length - 1].metadata.start_time ===
+                  item.metadata.start_time
+                ) {
+                  console.log('### push next')
+
+                  acc[acc.length - 1] = item
+
+                  return acc
+                }
+
+                acc.push(item)
+
+                return acc
+              }, [])
+              .map((item: any, index: number) => {
+                console.log('### item', item)
+                const key = `${index}${item.metadata.start_time}${item.metadata.end_time}`
+                const temp = item.message === 'AddPartialTranscript'
+
+                return (
+                  <TranslationText key={key} temp={temp}>
+                    {item.metadata.transcript}
+                  </TranslationText>
+                )
+              })}
+          </div>
+        </Box>
+      </Box>
+    </Container>
+  )
+})

client/src/routes/voice/components/RecordButton.tsx

@@ -0,0 +1,62 @@
+import React from 'react'
+import styled, { css, keyframes } from 'styled-components'
+import { Button } from 'grommet/components/Button'
+import { RecordStatus } from '../types'
+import { Box } from 'grommet/components/Box'
+import { Microphone } from 'grommet-icons/icons/Microphone'
+
+export const pulseAnimation = keyframes`
+  0% {
+    transform: scale(0.95);
+    box-shadow: 0 0 0 0 rgba(255, 0, 0, 0.7);
+  }
+
+  70% {
+    transform: scale(1);
+    box-shadow: 0 0 0 10px rgba(255, 0, 0, 0);
+  }
+
+  100% {
+    transform: scale(0.95);
+    box-shadow: 0 0 0 0 rgba(255, 0, 0, 0);
+  }
+`
+
+const StyledButton = styled(Button)<{ status: RecordStatus }>`
+  width: 60px;
+  height: 60px;
+  border-radius: 50%;
+  background-color: #00aee9;
+
+  ${(props) => {
+    if (props.status === 'recording') {
+      return css`
+        animation-name: ${pulseAnimation};
+        animation-duration: 1.5s;
+        animation-iteration-count: infinite;
+        background-color: rgb(255, 0, 0);
+      `
+    }
+
+    if (props.status === 'stopped') {
+      return css`
+        background-color: #00aee9;
+      `
+    }
+
+    return ''
+  }}
+`
+
+export const RecordButton: React.FC<{
+  status: RecordStatus
+  onClick: () => void
+}> = ({ status, onClick }) => {
+  return (
+    <StyledButton status={status} justify="center" onClick={onClick}>
+      <Box fill align="center" justify="center">
+        <Microphone color="white" />
+      </Box>
+    </StyledButton>
+  )
+}

client/src/routes/voice/components/TranslationItem.tsx

@@ -0,0 +1,31 @@
+import React from 'react'
+import { Box } from 'grommet/components/Box'
+import { Spinner } from 'grommet/components/Spinner'
+import { BaseText, SmallText } from '../../../components/Text'
+import { Translation } from '../types'
+import styled from 'styled-components'
+import { observer } from 'mobx-react-lite'
+
+const StyleBox = styled(Box)`
+  border-radius: 12px;
+  padding: 16px;
+  border: 1px solid #758796;
+`
+
+const dateFormat = new Intl.DateTimeFormat('en-US', {
+  dateStyle: 'medium',
+  timeStyle: 'short',
+})
+
+export const TranslationItem: React.FC<{
+  item: Translation
+}> = observer(({ item }) => {
+  return (
+    <StyleBox align="center">
+      <SmallText>{dateFormat.format(item.date)}</SmallText>
+      {item.inProgress && <Spinner />}
+      <BaseText>{item.translation}</BaseText>
+      <audio src={item.audio} controls />
+    </StyleBox>
+  )
+})

client/src/routes/voice/types.ts

@@ -0,0 +1,8 @@
+export type RecordStatus = 'recording' | 'stopped'
+
+export interface Translation {
+  audio: string
+  translation: string
+  date: number
+  inProgress: boolean
+}