diff --git a/brave/outputs/webrtc.py b/brave/outputs/webrtc.py index 53a26e6..7d2d3e5 100644 --- a/brave/outputs/webrtc.py +++ b/brave/outputs/webrtc.py @@ -74,7 +74,7 @@ def _create_pipeline(self): if config.enable_audio(): # bandwidth=superwideband allows the encoder to focus a little more on the important audio # (Basic testing showed 'wideband' to be quite poor poor) - pipeline_string += (' interaudiosrc name=interaudiosrc ! audioconvert ! ' + pipeline_string += (' interaudiosrc name=interaudiosrc ! audioconvert ! level message=true ! ' 'audioresample name=webrtc-audioresample ! opusenc bandwidth=superwideband ! ' 'rtpopuspay ! application/x-rtp,media=audio,encoding-name=OPUS,payload=96 ! ' 'tee name=webrtc_audio_tee webrtc_audio_tee. ! fakesink') @@ -104,6 +104,29 @@ async def new_peer_request(self, ws): self.peers[ws]['webrtcbin'].connect('on-ice-candidate', self._send_ice_candidate_message) # In the future, use connect('pad-added' here if the client's return video is wanted + loop = asyncio.get_event_loop() + + def on_message(bus, message): + t = message.type + if t == Gst.MessageType.ELEMENT: + if message.get_structure().get_name() == 'level': + s = message.get_structure() + channels = len(s['peak']) + data = [] + + for i in range(0, channels): + data.append(json.dumps({ + 'peak': message.get_structure().get_value('peak')[i], + 'rms': message.get_structure().get_value('rms')[i], + 'decay': message.get_structure().get_value('decay')[i] + })) + + jsonData = json.dumps({'msg_type': 'volume', 'channels': channels, 'data': data}) + loop.create_task(ws.send(jsonData)) + + self.pipeline.get_bus().add_signal_watch() + self.pipeline.get_bus().connect('message::element', on_message) + if not self.pipeline.set_state(Gst.State.PLAYING): self.logger.warn('Unable to enter PLAYING state now that we have a peer') else: diff --git a/public/js/preview.js b/public/js/preview.js index b08a732..2242687 100644 --- a/public/js/preview.js +++ b/public/js/preview.js @@ -110,6 +110,104 @@ preview._handlePreviewRequest = function(request) { } } +preview._clamp = function(value, min = 0, max = 1) { + return Math.min(Math.max(value, min), max); +} + +preview._normaliseDb = function(db) { + // -60db -> 1.00 (very quiet) + // -30db -> 0.75 + // -15db -> 0.50 + // -5db -> 0.25 + // -0db -> 0.00 (very loud) + var logscale = 1 - Math.log10(-0.15 * db + 1); + return preview._clamp(logscale); +} + +preview._gradient = function(context, brightness, darkness, height) { + var gradient = context.createLinearGradient(0, 0, 0, height); + + gradient.addColorStop(0.0, `rgb(${brightness}, ${darkness}, ${darkness})`); + gradient.addColorStop(0.22, `rgb(${brightness}, ${brightness}, ${darkness})`); + gradient.addColorStop(0.25, `rgb(${brightness}, ${brightness}, ${darkness})`); + gradient.addColorStop(0.35, `rgb(${darkness}, ${brightness}, ${darkness})`); + gradient.addColorStop(1.0, `rgb(${darkness}, ${brightness}, ${darkness})`); + + return gradient; +} + +preview.updateAudioLevels = function() { + var context = document.getElementById("audio_levels").getContext("2d"); + var channels = websocket.volume.channels; + var channel; + var margin = 2; + var width = 80; + var height = 360; + var channelWidth = parseInt((width - (margin * (channels - 1))) / channels); + var peak; + + //console.log(`width: ${width} filled with ${channels} channels of each ${channelWidth} and ${channels - 1} margin of ${margin}`); + + var bgFill = preview._gradient(context, 64, 0, height); + var rmsFill = preview._gradient(context, 255, 0, height); + var peakFill = preview._gradient(context, 192, 0, height); + var decayFill = preview._gradient(context, 255, 127, height); + + //Clear the canvas + context.clearRect(0,0, width, height); + + for (channel = 0; channel < channels; channel++) { + var audioData = JSON.parse(websocket.volume.data[channel]); + + var rms = preview._normaliseDb(audioData.rms) * height; + peak = preview._normaliseDb(audioData.peak) * height; + var decay = preview._normaliseDb(audioData.decay) * height; + + var x = (channel * channelWidth) + (channel * margin); + + //draw background + context.fillStyle = bgFill; + context.fillRect(x, 0, channelWidth, height - peak); + + // draw peak bar + context.fillStyle = peakFill; + context.fillRect(x, height - peak, channelWidth, peak); + + // draw rms bar below + context.fillStyle = rmsFill; + context.fillRect(x, height - rms, channelWidth, rms - peak); + + // draw decay bar + context.fillStyle = decayFill; + context.fillRect(x, height - decay, channelWidth, 2); + + // draw medium grey margin bar + context.fillStyle = "gray"; + context.fillRect(x + channelWidth, 0, margin, height); + } + + context.fillStyle= "white" + context.font = "10px Arial"; + + var dbMarkers = [-40, -20, -10, -5, -4, -3, -2, -1]; + dbMarkers.forEach(function(db) { + var text = db.toString(); + var y = preview._normaliseDb(db) * height; + var textwidth = context.measureText(text).width; + var textheight = 10; + + if (y > peak) { + context.fillStyle= "white"; + } else { + context.fillStyle= "black"; + } + + context.fillText(text, (width - textwidth) - 2, height - y - textheight); + }); + + window.requestAnimationFrame( preview.updateAudioLevels ); +} + preview.setVideoSrc = function(src) { preview._createVideoPlayer() getVideoElement().srcObject = src; @@ -127,9 +225,18 @@ preview._createVideoPlayer = function() { var video = document.createElement('video') video.id = 'stream' video.autoplay = true + + var audio_canvas = document.createElement('canvas') + audio_canvas.id = 'audio_levels'; + audio_canvas.width = 80; + audio_canvas.height = 360; + $('#preview-bar').empty() $('#preview-bar').append(video) + $('#preview-bar').append(audio_canvas) + preview._showMuteButton() + preview.updateAudioLevels(); } preview._createImage = function() { diff --git a/public/js/websocket.js b/public/js/websocket.js index 5956ad3..f6b53a9 100644 --- a/public/js/websocket.js +++ b/public/js/websocket.js @@ -3,7 +3,8 @@ // websocket = { - setupErrorCount: 0 + setupErrorCount: 0, + volume: {channels: 0, data: [] }, } websocket.setup = function() { @@ -57,6 +58,10 @@ websocket._onMessageReceived = event => { else if (dataParsed.msg_type === 'webrtc-initialising') { if (dataParsed.ice_servers) webrtc.setIceServers(dataParsed.ice_servers) } + else if (dataParsed.msg_type === 'volume') { + websocket.volume.channels = dataParsed.channels; + websocket.volume.data = dataParsed.data; + } else if (dataParsed.sdp != null) { webrtc.onIncomingSDP(dataParsed.sdp); } else if (dataParsed.ice != null) { diff --git a/public/style.css b/public/style.css index 6b82107..2b63ff4 100644 --- a/public/style.css +++ b/public/style.css @@ -114,6 +114,12 @@ body { } } +#preview-bar canvas { + background-color: #222; + width: 80px; + height: 360px; +} + .preview-select { border: 1px solid #ccc; /* padding: 0 4px 0 4px; */