Merge pull request #7 from bbc/feat/audio_levels

WebRTC audio levels

brave/outputs/webrtc.py

@@ -74,7 +74,7 @@ def _create_pipeline(self):
         if config.enable_audio():
             # bandwidth=superwideband allows the encoder to focus a little more on the important audio
             # (Basic testing showed 'wideband' to be quite poor poor)
-            pipeline_string += (' interaudiosrc name=interaudiosrc ! audioconvert ! '
+            pipeline_string += (' interaudiosrc name=interaudiosrc ! audioconvert ! level message=true ! '
                                 'audioresample name=webrtc-audioresample ! opusenc bandwidth=superwideband  ! '
                                 'rtpopuspay ! application/x-rtp,media=audio,encoding-name=OPUS,payload=96 ! '
                                 'tee name=webrtc_audio_tee webrtc_audio_tee. ! fakesink')
@@ -104,6 +104,29 @@ async def new_peer_request(self, ws):
         self.peers[ws]['webrtcbin'].connect('on-ice-candidate', self._send_ice_candidate_message)
         # In the future, use connect('pad-added' here if the client's return video is wanted
 
+        loop = asyncio.get_event_loop()
+
+        def on_message(bus, message):
+            t = message.type
+            if t == Gst.MessageType.ELEMENT:
+                if  message.get_structure().get_name() == 'level':
+                    s = message.get_structure()
+                    channels = len(s['peak'])
+                    data = []
+
+                    for i in range(0, channels):
+                        data.append(json.dumps({
+                            'peak': message.get_structure().get_value('peak')[i],
+                            'rms': message.get_structure().get_value('rms')[i],
+                            'decay': message.get_structure().get_value('decay')[i]
+                        }))
+
+                    jsonData = json.dumps({'msg_type': 'volume', 'channels': channels, 'data': data})
+                    loop.create_task(ws.send(jsonData))
+
+        self.pipeline.get_bus().add_signal_watch()
+        self.pipeline.get_bus().connect('message::element', on_message)
+
         if not self.pipeline.set_state(Gst.State.PLAYING):
             self.logger.warn('Unable to enter PLAYING state now that we have a peer')
         else:

public/js/preview.js

@@ -110,6 +110,104 @@ preview._handlePreviewRequest = function(request) {
     }
 }
 
+preview._clamp = function(value, min = 0, max = 1) {
+    return Math.min(Math.max(value, min), max);
+}
+
+preview._normaliseDb = function(db) {
+    //  -60db -> 1.00 (very quiet)
+    //  -30db -> 0.75
+    //  -15db -> 0.50
+    //  -5db -> 0.25
+    //  -0db -> 0.00 (very loud)
+    var logscale = 1 - Math.log10(-0.15 * db + 1);
+    return preview._clamp(logscale);
+}
+
+preview._gradient = function(context, brightness, darkness, height) {
+    var gradient = context.createLinearGradient(0, 0, 0, height);
+
+    gradient.addColorStop(0.0, `rgb(${brightness}, ${darkness}, ${darkness})`);
+    gradient.addColorStop(0.22, `rgb(${brightness}, ${brightness}, ${darkness})`);
+    gradient.addColorStop(0.25, `rgb(${brightness}, ${brightness}, ${darkness})`);
+    gradient.addColorStop(0.35, `rgb(${darkness}, ${brightness}, ${darkness})`);
+    gradient.addColorStop(1.0, `rgb(${darkness}, ${brightness}, ${darkness})`);
+
+    return gradient;
+}
+
+preview.updateAudioLevels = function() {
+    var context = document.getElementById("audio_levels").getContext("2d");
+    var channels = websocket.volume.channels;
+    var channel;
+    var margin = 2;
+    var width = 80;
+    var height = 360;
+    var channelWidth = parseInt((width - (margin * (channels - 1))) / channels);
+    var peak;
+
+    //console.log(`width: ${width} filled with ${channels} channels of each ${channelWidth} and ${channels - 1} margin of ${margin}`);
+
+    var bgFill = preview._gradient(context, 64, 0, height);
+    var rmsFill =  preview._gradient(context, 255, 0, height);
+    var peakFill = preview._gradient(context,  192, 0, height);
+    var decayFill = preview._gradient(context,  255, 127, height);
+
+    //Clear the canvas
+    context.clearRect(0,0, width, height);
+
+    for (channel = 0; channel < channels; channel++) {
+        var audioData = JSON.parse(websocket.volume.data[channel]);
+
+        var rms = preview._normaliseDb(audioData.rms) * height;
+        peak = preview._normaliseDb(audioData.peak) * height;
+        var decay = preview._normaliseDb(audioData.decay) * height;
+
+        var x = (channel * channelWidth) + (channel * margin);
+
+        //draw background
+        context.fillStyle = bgFill;
+        context.fillRect(x, 0, channelWidth, height - peak);
+
+        // draw peak bar
+        context.fillStyle = peakFill;
+        context.fillRect(x, height - peak, channelWidth, peak);
+
+        // draw rms bar below
+        context.fillStyle = rmsFill;
+        context.fillRect(x, height - rms, channelWidth, rms - peak);
+
+        // draw decay bar
+        context.fillStyle = decayFill;
+        context.fillRect(x, height - decay, channelWidth, 2);
+
+        // draw medium grey margin bar
+        context.fillStyle = "gray";
+        context.fillRect(x + channelWidth, 0, margin, height);
+    }
+
+    context.fillStyle= "white"
+    context.font = "10px Arial";
+
+    var dbMarkers = [-40, -20, -10, -5, -4, -3, -2, -1];
+    dbMarkers.forEach(function(db) {
+        var text = db.toString();
+        var y = preview._normaliseDb(db) * height;
+        var textwidth = context.measureText(text).width;
+        var textheight = 10;
+
+        if (y > peak) {
+            context.fillStyle= "white";
+        } else {
+            context.fillStyle= "black";
+        }
+
+        context.fillText(text, (width - textwidth) - 2, height - y - textheight);
+    });
+
+    window.requestAnimationFrame( preview.updateAudioLevels );
+}
+
 preview.setVideoSrc = function(src) {
     preview._createVideoPlayer()
     getVideoElement().srcObject = src;
@@ -127,9 +225,18 @@ preview._createVideoPlayer = function() {
     var video = document.createElement('video')
     video.id = 'stream'
     video.autoplay = true
+
+    var audio_canvas = document.createElement('canvas')
+    audio_canvas.id = 'audio_levels';
+    audio_canvas.width  = 80;
+    audio_canvas.height = 360;
+
     $('#preview-bar').empty()
     $('#preview-bar').append(video)
+    $('#preview-bar').append(audio_canvas)
+
     preview._showMuteButton()
+    preview.updateAudioLevels();
 }
 
 preview._createImage = function() {

public/js/websocket.js

@@ -3,7 +3,8 @@
 //
 
 websocket = {
-    setupErrorCount: 0
+    setupErrorCount: 0,
+    volume: {channels: 0, data: [] },
 }
 
 websocket.setup = function() {
@@ -57,6 +58,10 @@ websocket._onMessageReceived = event => {
     else if (dataParsed.msg_type === 'webrtc-initialising') {
         if (dataParsed.ice_servers) webrtc.setIceServers(dataParsed.ice_servers)
     }
+    else if (dataParsed.msg_type === 'volume') {
+        websocket.volume.channels = dataParsed.channels;
+        websocket.volume.data = dataParsed.data;
+    }
     else if (dataParsed.sdp != null) {
         webrtc.onIncomingSDP(dataParsed.sdp);
     } else if (dataParsed.ice != null) {

public/style.css

@@ -114,6 +114,12 @@ body {
     }
 }
 
+#preview-bar canvas {
+    background-color: #222;
+    width: 80px;
+    height: 360px;
+}
+
 .preview-select {
     border: 1px solid #ccc;
     /* padding: 0 4px 0 4px; */