feat: native audio processing apis and various useTrackVolume hooks

android/build.gradle

@@ -131,6 +131,7 @@ dependencies {
   implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
   api 'com.github.davidliu:audioswitch:89582c47c9a04c62f90aa5e57251af4800a62c9a'
   api 'io.github.webrtc-sdk:android:125.6422.02'
+  implementation "com.github.paramsen:noise:2.0.0"
   implementation project(':livekit_react-native-webrtc')
-  implementation "androidx.annotation:annotation:1.4.0"
+  implementation "androidx.annotation:annotation:1.9.1"
 }

android/src/main/java/com/livekit/reactnative/LiveKitReactNative.kt

@@ -5,10 +5,14 @@ import android.app.Application
 import android.content.Context
 import android.os.Build
 import com.livekit.reactnative.audio.AudioType
+import com.livekit.reactnative.audio.processing.AudioProcessingController
+import com.livekit.reactnative.audio.processing.AudioRecordSamplesDispatcher
+import com.livekit.reactnative.audio.processing.CustomAudioProcessingController
 import com.livekit.reactnative.video.CustomVideoDecoderFactory
 import com.livekit.reactnative.video.CustomVideoEncoderFactory
 import com.oney.WebRTCModule.WebRTCModuleOptions
 import org.webrtc.audio.JavaAudioDeviceModule
+import java.util.concurrent.Callable
 
 object LiveKitReactNative {
 
@@ -25,6 +29,28 @@ object LiveKitReactNative {
             return adm
         }
 
+    private lateinit var _audioProcessingController: AudioProcessingController
+
+    val audioProcessingController: AudioProcessingController
+        get() {
+            if (!::_audioProcessingController.isInitialized) {
+                throw IllegalStateException("audioProcessingController is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?")
+            }
+            return _audioProcessingController
+        }
+
+
+    lateinit var _audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher
+
+    val audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher
+        get() {
+            if (!::_audioRecordSamplesDispatcher.isInitialized) {
+                throw IllegalStateException("audioRecordSamplesDispatcher is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?")
+            }
+            return _audioRecordSamplesDispatcher
+        }
+
+
     /**
      * Initializes components required for LiveKit to work on Android.
      *
@@ -37,6 +63,8 @@ object LiveKitReactNative {
         context: Context,
         audioType: AudioType = AudioType.CommunicationAudioType()
     ) {
+        _audioRecordSamplesDispatcher = AudioRecordSamplesDispatcher()
+
         this.audioType = audioType
         val options = WebRTCModuleOptions.getInstance()
         options.videoEncoderFactory = CustomVideoEncoderFactory(null, true, true)
@@ -54,6 +82,7 @@ object LiveKitReactNative {
             .setUseHardwareAcousticEchoCanceler(useHardwareAudioProcessing)
             .setUseHardwareNoiseSuppressor(useHardwareAudioProcessing)
             .setAudioAttributes(audioType.audioAttributes)
+            .setSamplesReadyCallback(audioRecordSamplesDispatcher)
             .createAudioDeviceModule()
     }
 
@@ -67,5 +96,13 @@ object LiveKitReactNative {
 
         setupAdm(context)
         options.audioDeviceModule = adm
+
+        // CustomAudioProcessingController can't be instantiated before WebRTC is loaded.
+        options.audioProcessingFactoryFactory = Callable {
+            val apc = CustomAudioProcessingController()
+            _audioProcessingController = apc
+            return@Callable apc.externalAudioProcessor
+        }
+
     }
 }

android/src/main/java/com/livekit/reactnative/LivekitReactNativeModule.kt

@@ -1,6 +1,7 @@
 package com.livekit.reactnative
 
 import android.media.AudioAttributes
+import android.util.Log
 import com.facebook.react.bridge.Arguments
 import com.facebook.react.bridge.Promise
 import com.facebook.react.bridge.ReactApplicationContext
@@ -10,11 +11,16 @@ import com.facebook.react.bridge.ReadableMap
 import com.livekit.reactnative.audio.AudioDeviceKind
 import com.livekit.reactnative.audio.AudioManagerUtils
 import com.livekit.reactnative.audio.AudioSwitchManager
+import com.livekit.reactnative.audio.processing.AudioSinkManager
+import com.livekit.reactnative.audio.processing.MultibandVolumeProcessor
+import com.livekit.reactnative.audio.processing.VolumeProcessor
 import org.webrtc.audio.WebRtcAudioTrackHelper
+import kotlin.time.Duration.Companion.milliseconds
 
 
 class LivekitReactNativeModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) {
 
+    val audioSinkManager = AudioSinkManager(reactContext)
     val audioManager = AudioSwitchManager(reactContext.applicationContext)
     override fun getName(): String {
         return "LivekitReactNative"
@@ -118,12 +124,76 @@ class LivekitReactNativeModule(reactContext: ReactApplicationContext) : ReactCon
         promise.resolve(Arguments.makeNativeArray(deviceIds))
     }
 
-    @ReactMethod
+    @ReactMethod(isBlockingSynchronousMethod = true)
     fun selectAudioOutput(deviceId: String, promise: Promise) {
         audioManager.selectAudioOutput(AudioDeviceKind.fromTypeName(deviceId))
         promise.resolve(null)
     }
 
+    @ReactMethod(isBlockingSynchronousMethod = true)
+    fun createVolumeProcessor(pcId: Int, trackId: String): String {
+        val processor = VolumeProcessor(reactApplicationContext)
+        val reactTag = audioSinkManager.registerSink(processor)
+        audioSinkManager.attachSinkToTrack(processor, pcId, trackId)
+        processor.reactTag = reactTag
+
+        return reactTag
+    }
+
+    @ReactMethod(isBlockingSynchronousMethod = true)
+    fun deleteVolumeProcessor(reactTag: String, pcId: Int, trackId: String) {
+        audioSinkManager.detachSinkFromTrack(reactTag, pcId, trackId)
+        audioSinkManager.unregisterSink(reactTag)
+    }
+
+    @ReactMethod(isBlockingSynchronousMethod = true)
+    fun createMultibandVolumeProcessor(options: ReadableMap, pcId: Int, trackId: String): String {
+        val bands = options.getInt("bands")
+        val minFrequency = options.getDouble("minFrequency")
+        val maxFrequency = options.getDouble("maxFrequency")
+        val intervalMs = options.getDouble("updateInterval")
+
+        val processor = MultibandVolumeProcessor(
+            minFrequency = minFrequency.toFloat(),
+            maxFrequency = maxFrequency.toFloat(),
+            barCount = bands,
+            interval = intervalMs.milliseconds,
+            reactContext = reactApplicationContext
+        )
+        val reactTag = audioSinkManager.registerSink(processor)
+        processor.reactTag = reactTag
+        audioSinkManager.attachSinkToTrack(processor, pcId, trackId)
+
+        processor.start()
+
+        return reactTag
+    }
+
+    @ReactMethod(isBlockingSynchronousMethod = true)
+    fun deleteMultibandVolumeProcessor(reactTag: String, pcId: Int, trackId: String) {
+        val volumeProcessor =
+            audioSinkManager.getSink(reactTag) ?: throw IllegalArgumentException("Can't find volume processor for $reactTag")
+        audioSinkManager.detachSinkFromTrack(volumeProcessor, pcId, trackId)
+        audioSinkManager.unregisterSink(volumeProcessor)
+        val multibandVolumeProcessor = volumeProcessor as? MultibandVolumeProcessor
+
+        if (multibandVolumeProcessor != null) {
+            multibandVolumeProcessor.release()
+        } else {
+            Log.w(name, "deleteMultibandVolumeProcessor called, but non-MultibandVolumeProcessor found?!")
+        }
+    }
+
+    @ReactMethod
+    fun addListener(eventName: String?) {
+        // Keep: Required for RN built in Event Emitter Calls.
+    }
+
+    @ReactMethod
+    fun removeListeners(count: Int?) {
+        // Keep: Required for RN built in Event Emitter Calls.
+    }
+
     override fun invalidate() {
         LiveKitReactNative.invalidate(reactApplicationContext)
     }

android/src/main/java/com/livekit/reactnative/audio/events/Events.kt

@@ -0,0 +1,6 @@
+package com.livekit.reactnative.audio.events
+
+enum class Events {
+    LK_VOLUME_PROCESSED,
+    LK_MULTIBAND_PROCESSED,
+}

android/src/main/java/com/livekit/reactnative/audio/processing/AudioFormat.kt

@@ -0,0 +1,2 @@
+package com.livekit.reactnative.audio.processing
+data class AudioFormat(val bitsPerSample: Int, val sampleRate: Int, val numberOfChannels: Int)

android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessingController.kt

@@ -0,0 +1,27 @@
+package com.livekit.reactnative.audio.processing
+
+/**
+ * Interface for controlling external audio processing.
+ */
+interface AudioProcessingController {
+    /**
+     * the audio processor to be used for capture post processing.
+     */
+    var capturePostProcessor: AudioProcessorInterface?
+
+    /**
+     * the audio processor to be used for render pre processing.
+     */
+    var renderPreProcessor: AudioProcessorInterface?
+
+    /**
+     * whether to bypass mode the render pre processing.
+     */
+    var bypassRenderPreProcessing: Boolean
+
+    /**
+     * whether to bypass the capture post processing.
+     */
+    var bypassCapturePostProcessing: Boolean
+
+}

android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessorInterface.kt

@@ -0,0 +1,52 @@
+package com.livekit.reactnative.audio.processing
+
+import java.nio.ByteBuffer
+
+/**
+ * Interface for external audio processing.
+ */
+interface AudioProcessorInterface {
+    /**
+     * Check if the audio processing is enabled.
+     */
+    fun isEnabled(): Boolean
+
+    /**
+     * Get the name of the audio processing.
+     */
+    fun getName(): String
+
+    /**
+     * Initialize the audio processing.
+     *
+     * Note: audio processing methods will be called regardless of whether
+     * [isEnabled] returns true or not.
+     */
+    fun initializeAudioProcessing(sampleRateHz: Int, numChannels: Int)
+
+    /**
+     * Called when the sample rate has changed.
+     *
+     * Note: audio processing methods will be called regardless of whether
+     * [isEnabled] returns true or not.
+     */
+    fun resetAudioProcessing(newRate: Int)
+
+    /**
+     * Process the audio frame (10ms).
+     *
+     * Note: audio processing methods will be called regardless of whether
+     * [isEnabled] returns true or not.
+     */
+    fun processAudio(numBands: Int, numFrames: Int, buffer: ByteBuffer)
+}
+
+/**
+ * @suppress
+ */
+interface AuthedAudioProcessorInterface : AudioProcessorInterface {
+    /**
+     * @suppress
+     */
+    fun authenticate(url: String, token: String)
+}

android/src/main/java/com/livekit/reactnative/audio/processing/AudioRecordSamplesDispatcher.kt

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.livekit.reactnative.audio.processing
+
+import android.media.AudioFormat
+import android.os.SystemClock
+import org.webrtc.AudioTrackSink
+import org.webrtc.audio.JavaAudioDeviceModule
+import java.nio.ByteBuffer
+
+/**
+ * Dispatches recorded audio samples from the local microphone.
+ */
+class AudioRecordSamplesDispatcher : JavaAudioDeviceModule.SamplesReadyCallback {
+
+    private val sinks = mutableSetOf<AudioTrackSink>()
+
+    @Synchronized
+    fun registerSink(sink: AudioTrackSink) {
+        sinks.add(sink)
+    }
+
+    @Synchronized
+    fun unregisterSink(sink: AudioTrackSink) {
+        sinks.remove(sink)
+    }
+
+    // Reference from Android code, AudioFormat.getBytesPerSample. BitPerSample / 8
+    // Default audio data format is PCM 16 bits per sample.
+    // Guaranteed to be supported by all devices
+    private fun getBytesPerSample(audioFormat: Int): Int {
+        return when (audioFormat) {
+            AudioFormat.ENCODING_PCM_8BIT -> 1
+            AudioFormat.ENCODING_PCM_16BIT, AudioFormat.ENCODING_IEC61937, AudioFormat.ENCODING_DEFAULT -> 2
+            AudioFormat.ENCODING_PCM_FLOAT -> 4
+            AudioFormat.ENCODING_INVALID -> throw IllegalArgumentException("Bad audio format $audioFormat")
+            else -> throw IllegalArgumentException("Bad audio format $audioFormat")
+        }
+    }
+
+    @Synchronized
+    override fun onWebRtcAudioRecordSamplesReady(samples: JavaAudioDeviceModule.AudioSamples) {
+        val bitsPerSample = getBytesPerSample(samples.audioFormat) * 8
+        val numFrames = samples.sampleRate / 100 // 10ms worth of samples.
+        val timestamp = SystemClock.elapsedRealtime()
+        for (sink in sinks) {
+            val byteBuffer = ByteBuffer.wrap(samples.data)
+            sink.onData(
+                byteBuffer,
+                bitsPerSample,
+                samples.sampleRate,
+                samples.channelCount,
+                numFrames,
+                timestamp,
+            )
+        }
+    }
+}