diff --git a/android/build.gradle b/android/build.gradle index a9d1886..a4a6d42 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -131,6 +131,7 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" api 'com.github.davidliu:audioswitch:89582c47c9a04c62f90aa5e57251af4800a62c9a' api 'io.github.webrtc-sdk:android:125.6422.02' + implementation "com.github.paramsen:noise:2.0.0" implementation project(':livekit_react-native-webrtc') - implementation "androidx.annotation:annotation:1.4.0" + implementation "androidx.annotation:annotation:1.9.1" } diff --git a/android/src/main/java/com/livekit/reactnative/LiveKitReactNative.kt b/android/src/main/java/com/livekit/reactnative/LiveKitReactNative.kt index f2cbe7b..f2d975c 100644 --- a/android/src/main/java/com/livekit/reactnative/LiveKitReactNative.kt +++ b/android/src/main/java/com/livekit/reactnative/LiveKitReactNative.kt @@ -5,10 +5,14 @@ import android.app.Application import android.content.Context import android.os.Build import com.livekit.reactnative.audio.AudioType +import com.livekit.reactnative.audio.processing.AudioProcessingController +import com.livekit.reactnative.audio.processing.AudioRecordSamplesDispatcher +import com.livekit.reactnative.audio.processing.CustomAudioProcessingController import com.livekit.reactnative.video.CustomVideoDecoderFactory import com.livekit.reactnative.video.CustomVideoEncoderFactory import com.oney.WebRTCModule.WebRTCModuleOptions import org.webrtc.audio.JavaAudioDeviceModule +import java.util.concurrent.Callable object LiveKitReactNative { @@ -25,6 +29,28 @@ object LiveKitReactNative { return adm } + private lateinit var _audioProcessingController: AudioProcessingController + + val audioProcessingController: AudioProcessingController + get() { + if (!::_audioProcessingController.isInitialized) { + throw IllegalStateException("audioProcessingController is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?") + } + return _audioProcessingController + } + + + lateinit var _audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher + + val audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher + get() { + if (!::_audioRecordSamplesDispatcher.isInitialized) { + throw IllegalStateException("audioRecordSamplesDispatcher is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?") + } + return _audioRecordSamplesDispatcher + } + + /** * Initializes components required for LiveKit to work on Android. * @@ -37,6 +63,8 @@ object LiveKitReactNative { context: Context, audioType: AudioType = AudioType.CommunicationAudioType() ) { + _audioRecordSamplesDispatcher = AudioRecordSamplesDispatcher() + this.audioType = audioType val options = WebRTCModuleOptions.getInstance() options.videoEncoderFactory = CustomVideoEncoderFactory(null, true, true) @@ -54,6 +82,7 @@ object LiveKitReactNative { .setUseHardwareAcousticEchoCanceler(useHardwareAudioProcessing) .setUseHardwareNoiseSuppressor(useHardwareAudioProcessing) .setAudioAttributes(audioType.audioAttributes) + .setSamplesReadyCallback(audioRecordSamplesDispatcher) .createAudioDeviceModule() } @@ -67,5 +96,13 @@ object LiveKitReactNative { setupAdm(context) options.audioDeviceModule = adm + + // CustomAudioProcessingController can't be instantiated before WebRTC is loaded. + options.audioProcessingFactoryFactory = Callable { + val apc = CustomAudioProcessingController() + _audioProcessingController = apc + return@Callable apc.externalAudioProcessor + } + } } \ No newline at end of file diff --git a/android/src/main/java/com/livekit/reactnative/LivekitReactNativeModule.kt b/android/src/main/java/com/livekit/reactnative/LivekitReactNativeModule.kt index 745f1cc..fc1970e 100644 --- a/android/src/main/java/com/livekit/reactnative/LivekitReactNativeModule.kt +++ b/android/src/main/java/com/livekit/reactnative/LivekitReactNativeModule.kt @@ -1,6 +1,7 @@ package com.livekit.reactnative import android.media.AudioAttributes +import android.util.Log import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.Promise import com.facebook.react.bridge.ReactApplicationContext @@ -10,11 +11,16 @@ import com.facebook.react.bridge.ReadableMap import com.livekit.reactnative.audio.AudioDeviceKind import com.livekit.reactnative.audio.AudioManagerUtils import com.livekit.reactnative.audio.AudioSwitchManager +import com.livekit.reactnative.audio.processing.AudioSinkManager +import com.livekit.reactnative.audio.processing.MultibandVolumeProcessor +import com.livekit.reactnative.audio.processing.VolumeProcessor import org.webrtc.audio.WebRtcAudioTrackHelper +import kotlin.time.Duration.Companion.milliseconds class LivekitReactNativeModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) { + val audioSinkManager = AudioSinkManager(reactContext) val audioManager = AudioSwitchManager(reactContext.applicationContext) override fun getName(): String { return "LivekitReactNative" @@ -118,12 +124,76 @@ class LivekitReactNativeModule(reactContext: ReactApplicationContext) : ReactCon promise.resolve(Arguments.makeNativeArray(deviceIds)) } - @ReactMethod + @ReactMethod(isBlockingSynchronousMethod = true) fun selectAudioOutput(deviceId: String, promise: Promise) { audioManager.selectAudioOutput(AudioDeviceKind.fromTypeName(deviceId)) promise.resolve(null) } + @ReactMethod(isBlockingSynchronousMethod = true) + fun createVolumeProcessor(pcId: Int, trackId: String): String { + val processor = VolumeProcessor(reactApplicationContext) + val reactTag = audioSinkManager.registerSink(processor) + audioSinkManager.attachSinkToTrack(processor, pcId, trackId) + processor.reactTag = reactTag + + return reactTag + } + + @ReactMethod(isBlockingSynchronousMethod = true) + fun deleteVolumeProcessor(reactTag: String, pcId: Int, trackId: String) { + audioSinkManager.detachSinkFromTrack(reactTag, pcId, trackId) + audioSinkManager.unregisterSink(reactTag) + } + + @ReactMethod(isBlockingSynchronousMethod = true) + fun createMultibandVolumeProcessor(options: ReadableMap, pcId: Int, trackId: String): String { + val bands = options.getInt("bands") + val minFrequency = options.getDouble("minFrequency") + val maxFrequency = options.getDouble("maxFrequency") + val intervalMs = options.getDouble("updateInterval") + + val processor = MultibandVolumeProcessor( + minFrequency = minFrequency.toFloat(), + maxFrequency = maxFrequency.toFloat(), + barCount = bands, + interval = intervalMs.milliseconds, + reactContext = reactApplicationContext + ) + val reactTag = audioSinkManager.registerSink(processor) + processor.reactTag = reactTag + audioSinkManager.attachSinkToTrack(processor, pcId, trackId) + + processor.start() + + return reactTag + } + + @ReactMethod(isBlockingSynchronousMethod = true) + fun deleteMultibandVolumeProcessor(reactTag: String, pcId: Int, trackId: String) { + val volumeProcessor = + audioSinkManager.getSink(reactTag) ?: throw IllegalArgumentException("Can't find volume processor for $reactTag") + audioSinkManager.detachSinkFromTrack(volumeProcessor, pcId, trackId) + audioSinkManager.unregisterSink(volumeProcessor) + val multibandVolumeProcessor = volumeProcessor as? MultibandVolumeProcessor + + if (multibandVolumeProcessor != null) { + multibandVolumeProcessor.release() + } else { + Log.w(name, "deleteMultibandVolumeProcessor called, but non-MultibandVolumeProcessor found?!") + } + } + + @ReactMethod + fun addListener(eventName: String?) { + // Keep: Required for RN built in Event Emitter Calls. + } + + @ReactMethod + fun removeListeners(count: Int?) { + // Keep: Required for RN built in Event Emitter Calls. + } + override fun invalidate() { LiveKitReactNative.invalidate(reactApplicationContext) } diff --git a/android/src/main/java/com/livekit/reactnative/audio/events/Events.kt b/android/src/main/java/com/livekit/reactnative/audio/events/Events.kt new file mode 100644 index 0000000..25c1b65 --- /dev/null +++ b/android/src/main/java/com/livekit/reactnative/audio/events/Events.kt @@ -0,0 +1,6 @@ +package com.livekit.reactnative.audio.events + +enum class Events { + LK_VOLUME_PROCESSED, + LK_MULTIBAND_PROCESSED, +} \ No newline at end of file diff --git a/android/src/main/java/com/livekit/reactnative/audio/processing/AudioFormat.kt b/android/src/main/java/com/livekit/reactnative/audio/processing/AudioFormat.kt new file mode 100644 index 0000000..019b5a3 --- /dev/null +++ b/android/src/main/java/com/livekit/reactnative/audio/processing/AudioFormat.kt @@ -0,0 +1,2 @@ +package com.livekit.reactnative.audio.processing +data class AudioFormat(val bitsPerSample: Int, val sampleRate: Int, val numberOfChannels: Int) \ No newline at end of file diff --git a/android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessingController.kt b/android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessingController.kt new file mode 100644 index 0000000..efd7289 --- /dev/null +++ b/android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessingController.kt @@ -0,0 +1,27 @@ +package com.livekit.reactnative.audio.processing + +/** + * Interface for controlling external audio processing. + */ +interface AudioProcessingController { + /** + * the audio processor to be used for capture post processing. + */ + var capturePostProcessor: AudioProcessorInterface? + + /** + * the audio processor to be used for render pre processing. + */ + var renderPreProcessor: AudioProcessorInterface? + + /** + * whether to bypass mode the render pre processing. + */ + var bypassRenderPreProcessing: Boolean + + /** + * whether to bypass the capture post processing. + */ + var bypassCapturePostProcessing: Boolean + +} diff --git a/android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessorInterface.kt b/android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessorInterface.kt new file mode 100644 index 0000000..0783af3 --- /dev/null +++ b/android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessorInterface.kt @@ -0,0 +1,52 @@ +package com.livekit.reactnative.audio.processing + +import java.nio.ByteBuffer + +/** + * Interface for external audio processing. + */ +interface AudioProcessorInterface { + /** + * Check if the audio processing is enabled. + */ + fun isEnabled(): Boolean + + /** + * Get the name of the audio processing. + */ + fun getName(): String + + /** + * Initialize the audio processing. + * + * Note: audio processing methods will be called regardless of whether + * [isEnabled] returns true or not. + */ + fun initializeAudioProcessing(sampleRateHz: Int, numChannels: Int) + + /** + * Called when the sample rate has changed. + * + * Note: audio processing methods will be called regardless of whether + * [isEnabled] returns true or not. + */ + fun resetAudioProcessing(newRate: Int) + + /** + * Process the audio frame (10ms). + * + * Note: audio processing methods will be called regardless of whether + * [isEnabled] returns true or not. + */ + fun processAudio(numBands: Int, numFrames: Int, buffer: ByteBuffer) +} + +/** + * @suppress + */ +interface AuthedAudioProcessorInterface : AudioProcessorInterface { + /** + * @suppress + */ + fun authenticate(url: String, token: String) +} diff --git a/android/src/main/java/com/livekit/reactnative/audio/processing/AudioRecordSamplesDispatcher.kt b/android/src/main/java/com/livekit/reactnative/audio/processing/AudioRecordSamplesDispatcher.kt new file mode 100644 index 0000000..2406f48 --- /dev/null +++ b/android/src/main/java/com/livekit/reactnative/audio/processing/AudioRecordSamplesDispatcher.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2024 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.livekit.reactnative.audio.processing + +import android.media.AudioFormat +import android.os.SystemClock +import org.webrtc.AudioTrackSink +import org.webrtc.audio.JavaAudioDeviceModule +import java.nio.ByteBuffer + +/** + * Dispatches recorded audio samples from the local microphone. + */ +class AudioRecordSamplesDispatcher : JavaAudioDeviceModule.SamplesReadyCallback { + + private val sinks = mutableSetOf() + + @Synchronized + fun registerSink(sink: AudioTrackSink) { + sinks.add(sink) + } + + @Synchronized + fun unregisterSink(sink: AudioTrackSink) { + sinks.remove(sink) + } + + // Reference from Android code, AudioFormat.getBytesPerSample. BitPerSample / 8 + // Default audio data format is PCM 16 bits per sample. + // Guaranteed to be supported by all devices + private fun getBytesPerSample(audioFormat: Int): Int { + return when (audioFormat) { + AudioFormat.ENCODING_PCM_8BIT -> 1 + AudioFormat.ENCODING_PCM_16BIT, AudioFormat.ENCODING_IEC61937, AudioFormat.ENCODING_DEFAULT -> 2 + AudioFormat.ENCODING_PCM_FLOAT -> 4 + AudioFormat.ENCODING_INVALID -> throw IllegalArgumentException("Bad audio format $audioFormat") + else -> throw IllegalArgumentException("Bad audio format $audioFormat") + } + } + + @Synchronized + override fun onWebRtcAudioRecordSamplesReady(samples: JavaAudioDeviceModule.AudioSamples) { + val bitsPerSample = getBytesPerSample(samples.audioFormat) * 8 + val numFrames = samples.sampleRate / 100 // 10ms worth of samples. + val timestamp = SystemClock.elapsedRealtime() + for (sink in sinks) { + val byteBuffer = ByteBuffer.wrap(samples.data) + sink.onData( + byteBuffer, + bitsPerSample, + samples.sampleRate, + samples.channelCount, + numFrames, + timestamp, + ) + } + } +} diff --git a/android/src/main/java/com/livekit/reactnative/audio/processing/AudioSinkManager.kt b/android/src/main/java/com/livekit/reactnative/audio/processing/AudioSinkManager.kt new file mode 100644 index 0000000..5900edf --- /dev/null +++ b/android/src/main/java/com/livekit/reactnative/audio/processing/AudioSinkManager.kt @@ -0,0 +1,75 @@ +package com.livekit.reactnative.audio.processing + +import com.facebook.react.bridge.ReactContext +import com.livekit.reactnative.LiveKitReactNative +import com.oney.WebRTCModule.WebRTCModule +import org.webrtc.AudioTrack +import org.webrtc.AudioTrackSink +import java.util.UUID + +private const val LOCAL_PC_ID = -1 + +class AudioSinkManager(val reactContext: ReactContext) { + + private val sinks = mutableMapOf() + + /** + * Registers a sink to this manager. + * @return the tag to identify this sink in future calls, such as [getSink] or [unregisterSink] + */ + fun registerSink(sink: AudioTrackSink): String { + val reactTag = UUID.randomUUID().toString() + sinks[reactTag] = sink + + return reactTag + } + + /** + * Unregisters a sink from this manager. Does not detach the sink from tracks. + */ + fun unregisterSink(reactTag: String) { + sinks.remove(reactTag) + } + + /** + * Unregisters a sink from this manager. Does not detach the sink from tracks. + */ + fun unregisterSink(sink: AudioTrackSink) { + sinks.filterNot { entry -> entry.value == sink } + } + + fun getSink(reactTag: String) = sinks[reactTag] + + fun attachSinkToTrack(sink: AudioTrackSink, pcId: Int, trackId: String) { + val webRTCModule = + reactContext.getNativeModule(WebRTCModule::class.java) ?: throw IllegalArgumentException("Couldn't find WebRTC module!") + + val track = webRTCModule.getTrack(pcId, trackId) as? AudioTrack + ?: throw IllegalArgumentException("Couldn't find audio track for pcID:${pcId}, trackId:${trackId}") + + if (pcId == LOCAL_PC_ID) { + LiveKitReactNative.audioRecordSamplesDispatcher.registerSink(sink) + } else { + track.addSink(sink) + } + } + + fun detachSinkFromTrack(sink: AudioTrackSink, pcId: Int, trackId: String) { + val webRTCModule = + reactContext.getNativeModule(WebRTCModule::class.java) ?: throw IllegalArgumentException("Couldn't find WebRTC module!") + val track = webRTCModule.getTrack(pcId, trackId) as? AudioTrack + ?: return // fail silently + + if (pcId == LOCAL_PC_ID) { + LiveKitReactNative.audioRecordSamplesDispatcher.unregisterSink(sink) + } else { + track.removeSink(sink) + } + } + + fun detachSinkFromTrack(sinkReactTag: String, pcId: Int, trackId: String) { + val sink = sinks[sinkReactTag] + ?: throw IllegalArgumentException("Couldn't find audio sink for react tag: $sinkReactTag") + detachSinkFromTrack(sink, pcId, trackId) + } +} \ No newline at end of file diff --git a/android/src/main/java/com/livekit/reactnative/audio/processing/CustomAudioProcessingFactory.kt b/android/src/main/java/com/livekit/reactnative/audio/processing/CustomAudioProcessingFactory.kt new file mode 100644 index 0000000..17f506d --- /dev/null +++ b/android/src/main/java/com/livekit/reactnative/audio/processing/CustomAudioProcessingFactory.kt @@ -0,0 +1,78 @@ +package com.livekit.reactnative.audio.processing + +import org.webrtc.ExternalAudioProcessingFactory +import java.nio.ByteBuffer + +/** + * @suppress + */ +class CustomAudioProcessingController( + /** + * the audio processor to be used for capture post processing. + */ + capturePostProcessor: AudioProcessorInterface? = null, + + /** + * the audio processor to be used for render pre processing. + */ + renderPreProcessor: AudioProcessorInterface? = null, + + /** + * whether to bypass mode the render pre processing. + */ + bypassRenderPreProcessing: Boolean = false, + + /** + * whether to bypass the capture post processing. + */ + bypassCapturePostProcessing: Boolean = false, +) : AudioProcessingController { + + val externalAudioProcessor = ExternalAudioProcessingFactory() + + override var capturePostProcessor: AudioProcessorInterface? = capturePostProcessor + set(value) { + field = value + externalAudioProcessor.setCapturePostProcessing( + value.toAudioProcessing(), + ) + } + + override var renderPreProcessor: AudioProcessorInterface? = renderPreProcessor + set(value) { + field = value + externalAudioProcessor.setRenderPreProcessing( + value.toAudioProcessing(), + ) + } + + override var bypassCapturePostProcessing: Boolean = bypassCapturePostProcessing + set(value) { + externalAudioProcessor.setBypassFlagForCapturePost(value) + } + + override var bypassRenderPreProcessing: Boolean = bypassRenderPreProcessing + set(value) { + externalAudioProcessor.setBypassFlagForRenderPre(value) + } + + private class AudioProcessingBridge( + var audioProcessing: AudioProcessorInterface? = null, + ) : ExternalAudioProcessingFactory.AudioProcessing { + override fun initialize(sampleRateHz: Int, numChannels: Int) { + audioProcessing?.initializeAudioProcessing(sampleRateHz, numChannels) + } + + override fun reset(newRate: Int) { + audioProcessing?.resetAudioProcessing(newRate) + } + + override fun process(numBands: Int, numFrames: Int, buffer: ByteBuffer?) { + audioProcessing?.processAudio(numBands, numFrames, buffer!!) + } + } + + private fun AudioProcessorInterface?.toAudioProcessing(): ExternalAudioProcessingFactory.AudioProcessing { + return AudioProcessingBridge(this) + } +} diff --git a/android/src/main/java/com/livekit/reactnative/audio/processing/MultibandVolumeProcessor.kt b/android/src/main/java/com/livekit/reactnative/audio/processing/MultibandVolumeProcessor.kt new file mode 100644 index 0000000..b41314e --- /dev/null +++ b/android/src/main/java/com/livekit/reactnative/audio/processing/MultibandVolumeProcessor.kt @@ -0,0 +1,181 @@ +package com.livekit.reactnative.audio.processing + +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.ReactContext +import com.facebook.react.modules.core.DeviceEventManagerModule +import com.livekit.reactnative.audio.events.Events +import com.livekit.reactnative.audio.processing.fft.FFTAudioAnalyzer +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.conflate +import kotlinx.coroutines.flow.transform +import kotlinx.coroutines.launch +import org.webrtc.AudioTrackSink +import java.nio.ByteBuffer +import kotlin.math.pow +import kotlin.math.round +import kotlin.math.roundToInt +import kotlin.math.sqrt +import kotlin.time.Duration + +class MultibandVolumeProcessor( + minFrequency: Float = 1000f, + maxFrequency: Float = 8000f, + barCount: Int, + interval: Duration, + private val reactContext: ReactContext, +) : BaseMultibandVolumeProcessor(minFrequency, maxFrequency, barCount, interval) { + + var reactTag: String? = null + override fun onMagnitudesCollected(magnitudes: FloatArray) { + val reactTag = this.reactTag ?: return + val event = Arguments.createMap().apply { + putArray("magnitudes", Arguments.fromArray(magnitudes)) + putString("id", reactTag) + } + reactContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java) + .emit(Events.LK_MULTIBAND_PROCESSED.name, event) + } +} + +abstract class BaseMultibandVolumeProcessor( + val minFrequency: Float = 1000f, + val maxFrequency: Float = 8000f, + val barCount: Int, + val interval: Duration, +) : AudioTrackSink { + + private val audioProcessor = FFTAudioAnalyzer() + private var coroutineScope: CoroutineScope? = null + + abstract fun onMagnitudesCollected(magnitudes: FloatArray) + + fun start() { + coroutineScope?.cancel() + val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + coroutineScope = scope + scope.launch { + val averages = FloatArray(barCount) + audioProcessor.fftFlow.throttleLatest(interval).collect { fft -> + val loPass: Int + val hiPass: Int + val audioFormat = audioProcessor.configuredInputFormat + + if (audioFormat != null) { + loPass = (minFrequency * fft.size / (audioFormat.sampleRate / 2)).roundToInt().coerceIn(fft.indices) + hiPass = (maxFrequency * fft.size / (audioFormat.sampleRate / 2)).roundToInt().coerceIn(fft.indices) + } else { + loPass = 0 + hiPass = fft.size + } + + val sliced = fft.slice(loPass until hiPass) + val magnitudes = calculateAmplitudeBarsFromFFT(sliced, averages, barCount) + + onMagnitudesCollected(magnitudes) + } + } + } + + fun stop() { + coroutineScope?.cancel() + coroutineScope = null + } + + fun release() { + stop() + audioProcessor.release() + } + + override fun onData( + audioData: ByteBuffer, + bitsPerSample: Int, + sampleRate: Int, + numberOfChannels: Int, + numberOfFrames: Int, + absoluteCaptureTimestampMs: Long + ) { + val curAudioFormat = audioProcessor.configuredInputFormat + if (curAudioFormat == null || + curAudioFormat.bitsPerSample != bitsPerSample || + curAudioFormat.sampleRate != sampleRate || + curAudioFormat.numberOfChannels != numberOfChannels + ) { + audioProcessor.configure(AudioFormat(bitsPerSample, sampleRate, numberOfChannels)) + } + + audioProcessor.queueInput(audioData) + } +} + +fun Flow.throttleLatest(interval: Duration): Flow = this + .conflate() + .transform { + emit(it) + delay(interval) + } + + +private const val MIN_CONST = 2f +private const val MAX_CONST = 25f + +private fun calculateAmplitudeBarsFromFFT( + fft: List, + averages: FloatArray, + barCount: Int, +): FloatArray { + val amplitudes = FloatArray(barCount) + if (fft.isEmpty()) { + return amplitudes + } + + // We average out the values over 3 occurrences (plus the current one), so big jumps are smoothed out + // Iterate over the entire FFT result array. + for (barIndex in 0 until barCount) { + // Note: each FFT is a real and imaginary pair. + // Scale down by 2 and scale back up to ensure we get an even number. + val prevLimit = (round(fft.size.toFloat() / 2 * barIndex / barCount).toInt() * 2) + .coerceIn(0, fft.size - 1) + val nextLimit = (round(fft.size.toFloat() / 2 * (barIndex + 1) / barCount).toInt() * 2) + .coerceIn(0, fft.size - 1) + + var accum = 0f + // Here we iterate within this single band + for (i in prevLimit until nextLimit step 2) { + // Convert real and imaginary part to get energy + + val realSq = fft[i] + .toDouble() + .pow(2.0) + val imaginarySq = fft[i + 1] + .toDouble() + .pow(2.0) + val raw = sqrt(realSq + imaginarySq).toFloat() + + accum += raw + } + + // A window might be empty which would result in a 0 division + if ((nextLimit - prevLimit) != 0) { + accum /= (nextLimit - prevLimit) + } else { + accum = 0.0f + } + + val smoothingFactor = 5 + var avg = averages[barIndex] + avg += (accum - avg / smoothingFactor) + averages[barIndex] = avg + + var amplitude = avg.coerceIn(MIN_CONST, MAX_CONST) + amplitude -= MIN_CONST + amplitude /= (MAX_CONST - MIN_CONST) + amplitudes[barIndex] = amplitude + } + + return amplitudes +} diff --git a/android/src/main/java/com/livekit/reactnative/audio/processing/VolumeProcessor.kt b/android/src/main/java/com/livekit/reactnative/audio/processing/VolumeProcessor.kt new file mode 100644 index 0000000..802eae8 --- /dev/null +++ b/android/src/main/java/com/livekit/reactnative/audio/processing/VolumeProcessor.kt @@ -0,0 +1,67 @@ +package com.livekit.reactnative.audio.processing + +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.ReactContext +import com.facebook.react.modules.core.DeviceEventManagerModule +import com.livekit.reactnative.audio.events.Events +import org.webrtc.AudioTrackSink +import java.nio.ByteBuffer +import kotlin.math.round +import kotlin.math.sqrt + +class VolumeProcessor(private val reactContext: ReactContext) : BaseVolumeProcessor() { + var reactTag: String? = null + + override fun onVolumeCalculated(volume: Double) { + val reactTag = this.reactTag ?: return + val event = Arguments.createMap().apply { + putDouble("volume", volume) + putString("id", reactTag) + } + reactContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java) + .emit(Events.LK_VOLUME_PROCESSED.name, event) + } +} + +abstract class BaseVolumeProcessor : AudioTrackSink { + abstract fun onVolumeCalculated(volume: Double) + + override fun onData( + audioData: ByteBuffer, + bitsPerSample: Int, + sampleRate: Int, + numberOfChannels: Int, + numberOfFrames: Int, + absoluteCaptureTimestampMs: Long + ) { + audioData.mark() + audioData.position(0) + var average = 0L + val bytesPerSample = bitsPerSample / 8 + + // RMS average calculation + for (i in 0 until numberOfFrames) { + val value = when (bytesPerSample) { + 1 -> audioData.get().toLong() + 2 -> audioData.getShort().toLong() + 4 -> audioData.getInt().toLong() + else -> throw IllegalArgumentException() + } + + average += value * value + } + + average /= numberOfFrames + + val volume = round(sqrt(average.toDouble())) + val volumeNormalized = when (bytesPerSample) { + 1 -> volume / Byte.MAX_VALUE + 2 -> volume / Short.MAX_VALUE + 4 -> volume / Int.MAX_VALUE + else -> throw IllegalArgumentException() + } + audioData.reset() + + onVolumeCalculated(volumeNormalized) + } +} \ No newline at end of file diff --git a/android/src/main/java/com/livekit/reactnative/audio/processing/fft/FFTAudioAnalyzer.kt b/android/src/main/java/com/livekit/reactnative/audio/processing/fft/FFTAudioAnalyzer.kt new file mode 100644 index 0000000..9f06f4b --- /dev/null +++ b/android/src/main/java/com/livekit/reactnative/audio/processing/fft/FFTAudioAnalyzer.kt @@ -0,0 +1,224 @@ +/* + * Copyright 2024 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Originally adapted from: https://github.com/dzolnai/ExoVisualizer + * + * MIT License + * + * Copyright (c) 2019 Dániel Zolnai + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.livekit.reactnative.audio.processing.fft; + +import android.media.AudioTrack +import com.livekit.reactnative.audio.processing.AudioFormat +import com.paramsen.noise.Noise +import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableSharedFlow +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.TimeUnit +import kotlin.math.max + +/** + * A Fast Fourier Transform analyzer for audio bytes. + * + * Use [queueInput] to add audio bytes, and collect on [fftFlow] + * to receive the analyzed frequencies. + */ +class FFTAudioAnalyzer { + + companion object { + const val SAMPLE_SIZE = 1024 + private val EMPTY_BUFFER = ByteBuffer.allocateDirect(0).order(ByteOrder.nativeOrder()) + + // Extra size next in addition to the AudioTrack buffer size + private const val BUFFER_EXTRA_SIZE = SAMPLE_SIZE * 8 + + // Size of short in bytes. + private const val SHORT_SIZE = 2 + } + + val isActive: Boolean + get() = noise != null + + private var noise: Noise? = null + private lateinit var inputAudioFormat: AudioFormat + val configuredInputFormat: AudioFormat? + get() { + if (::inputAudioFormat.isInitialized) { + return inputAudioFormat + } else { + return null + } + } + + private var audioTrackBufferSize = 0 + + private var fftBuffer: ByteBuffer = EMPTY_BUFFER + private lateinit var srcBuffer: ByteBuffer + private var srcBufferPosition = 0 + private val tempShortArray = ShortArray(SAMPLE_SIZE) + private val src = FloatArray(SAMPLE_SIZE) + + private val mutableFftFlow = MutableSharedFlow(replay = 1, onBufferOverflow = BufferOverflow.DROP_OLDEST) + + /** + * A flow of frequencies for the audio bytes given through [queueInput]. + */ + val fftFlow: Flow = mutableFftFlow + + fun configure(inputAudioFormat: AudioFormat) { + this.inputAudioFormat = inputAudioFormat + + noise = Noise.real(SAMPLE_SIZE) + + audioTrackBufferSize = getDefaultBufferSizeInBytes(inputAudioFormat) + + srcBuffer = ByteBuffer.allocate(audioTrackBufferSize + BUFFER_EXTRA_SIZE) + } + + fun release() { + noise?.close() + noise = null + } + + /** + * Add audio bytes to be processed. + */ + fun queueInput(inputBuffer: ByteBuffer) { + if (!isActive) { + return + } + var position = inputBuffer.position() + val limit = inputBuffer.limit() + val frameCount = (limit - position) / (SHORT_SIZE * inputAudioFormat.numberOfChannels) + val singleChannelOutputSize = frameCount * SHORT_SIZE + + // Setup buffer + if (fftBuffer.capacity() < singleChannelOutputSize) { + fftBuffer = + ByteBuffer.allocateDirect(singleChannelOutputSize).order(ByteOrder.nativeOrder()) + } else { + fftBuffer.clear() + } + + // Process inputBuffer + while (position < limit) { + var summedUp = 0 + for (channelIndex in 0 until inputAudioFormat.numberOfChannels) { + val current = inputBuffer.getShort(position + 2 * channelIndex) + summedUp += current + } + // For the FFT, we use an average of all the channels and put into a single short. + fftBuffer.putShort((summedUp / inputAudioFormat.numberOfChannels).toShort()) + position += inputAudioFormat.numberOfChannels * 2 + } + + // Reset input buffer to original position. + inputBuffer.position(position) + + processFFT(this.fftBuffer) + } + + private fun processFFT(buffer: ByteBuffer) { + if (noise == null) { + return + } + srcBuffer.put(buffer.array()) + srcBufferPosition += buffer.array().size + // Since this is PCM 16 bit, each sample will be 2 bytes. + // So to get the sample size in the end, we need to take twice as many bytes off the buffer + val bytesToProcess = SAMPLE_SIZE * 2 + while (srcBufferPosition > bytesToProcess) { + // Move to start of + srcBuffer.position(0) + + srcBuffer.asShortBuffer().get(tempShortArray, 0, SAMPLE_SIZE) + tempShortArray.forEachIndexed { index, sample -> + // Normalize to value between -1.0 and 1.0 + src[index] = sample.toFloat() / Short.MAX_VALUE + } + + srcBuffer.position(bytesToProcess) + srcBuffer.compact() + srcBufferPosition -= bytesToProcess + srcBuffer.position(srcBufferPosition) + val dst = FloatArray(SAMPLE_SIZE + 2) + val fft = noise?.fft(src, dst)!! + + mutableFftFlow.tryEmit(fft) + } + } + + private fun durationUsToFrames(sampleRate: Int, durationUs: Long): Long { + return durationUs * sampleRate / TimeUnit.MICROSECONDS.convert(1, TimeUnit.SECONDS) + } + + private fun getPcmFrameSize(channelCount: Int): Int { + // assumes PCM_16BIT + return channelCount * 2 + } + + private fun getAudioTrackChannelConfig(channelCount: Int): Int { + return when (channelCount) { + 1 -> android.media.AudioFormat.CHANNEL_OUT_MONO + 2 -> android.media.AudioFormat.CHANNEL_OUT_STEREO + // ignore other channel counts that aren't used in LiveKit + else -> android.media.AudioFormat.CHANNEL_INVALID + } + } + + private fun getDefaultBufferSizeInBytes(audioFormat: AudioFormat): Int { + val outputPcmFrameSize = getPcmFrameSize(audioFormat.numberOfChannels) + val minBufferSize = + AudioTrack.getMinBufferSize( + audioFormat.sampleRate, + getAudioTrackChannelConfig(audioFormat.numberOfChannels), + android.media.AudioFormat.ENCODING_PCM_16BIT + ) + + check(minBufferSize != AudioTrack.ERROR_BAD_VALUE) + val multipliedBufferSize = minBufferSize * 4 + val minAppBufferSize = + durationUsToFrames(audioFormat.sampleRate, 30 * 1000).toInt() * outputPcmFrameSize + val maxAppBufferSize = max( + minBufferSize.toLong(), + durationUsToFrames(audioFormat.sampleRate, 500 * 1000) * outputPcmFrameSize + ).toInt() + val bufferSizeInFrames = + multipliedBufferSize.coerceIn(minAppBufferSize, maxAppBufferSize) / outputPcmFrameSize + return bufferSizeInFrames * outputPcmFrameSize + } +} diff --git a/ci/ios/Podfile.lock b/ci/ios/Podfile.lock index 552029d..84e3c96 100644 --- a/ci/ios/Podfile.lock +++ b/ci/ios/Podfile.lock @@ -7,12 +7,12 @@ PODS: - hermes-engine (0.74.2): - hermes-engine/Pre-built (= 0.74.2) - hermes-engine/Pre-built (0.74.2) - - livekit-react-native (2.4.3): + - livekit-react-native (2.5.1): - livekit-react-native-webrtc - React-Core - - livekit-react-native-webrtc (125.0.6): + - livekit-react-native-webrtc (125.0.8): - React-Core - - WebRTC-SDK (~> 125.6422.06) + - WebRTC-SDK (~> 125.6422.07) - RCT-Folly (2024.01.01.00): - boost - DoubleConversion @@ -1171,7 +1171,7 @@ PODS: - React-perflogger (= 0.74.2) - React-utils (= 0.74.2) - SocketRocket (0.7.0) - - WebRTC-SDK (125.6422.06) + - WebRTC-SDK (125.6422.07) - Yoga (0.0.0) DEPENDENCIES: @@ -1361,8 +1361,8 @@ SPEC CHECKSUMS: fmt: 4c2741a687cc09f0634a2e2c72a838b99f1ff120 glog: c5d68082e772fa1c511173d6b30a9de2c05a69a2 hermes-engine: 01d3e052018c2a13937aca1860fbedbccd4a41b7 - livekit-react-native: b93bfc9038869dcbd5cee53b419c956c28e44949 - livekit-react-native-webrtc: 7f040d0557b0e0903d6cfe4977ddf608462988c6 + livekit-react-native: 08264d81497fe9d18eb513e1db05d0539ddb9417 + livekit-react-native-webrtc: c456181c7c6f9f2b0a79ea14d0d3c97215266ba0 RCT-Folly: 02617c592a293bd6d418e0a88ff4ee1f88329b47 RCTDeprecation: b03c35057846b685b3ccadc9bfe43e349989cdb2 RCTRequired: 194626909cfa8d39ca6663138c417bc6c431648c @@ -1411,7 +1411,7 @@ SPEC CHECKSUMS: React-utils: 4476b7fcbbd95cfd002f3e778616155241d86e31 ReactCommon: ecad995f26e0d1e24061f60f4e5d74782f003f12 SocketRocket: abac6f5de4d4d62d24e11868d7a2f427e0ef940d - WebRTC-SDK: 79942c006ea64f6fb48d7da8a4786dfc820bc1db + WebRTC-SDK: dff00a3892bc570b6014e046297782084071657e Yoga: ae3c32c514802d30f687a04a6a35b348506d411f PODFILE CHECKSUM: 7e787510e5e3fbe259a5a7507ea2e7e1b1ff65ef diff --git a/ci/package.json b/ci/package.json index c005afe..72fcde2 100644 --- a/ci/package.json +++ b/ci/package.json @@ -12,7 +12,7 @@ }, "dependencies": { "@livekit/react-native": "*", - "@livekit/react-native-webrtc": "^125.0.7", + "@livekit/react-native-webrtc": "^125.0.8", "react": "18.2.0", "react-native": "0.74.2" }, diff --git a/ci/yarn.lock b/ci/yarn.lock index 02c2112..fcbf851 100644 --- a/ci/yarn.lock +++ b/ci/yarn.lock @@ -2158,16 +2158,16 @@ __metadata: languageName: node linkType: hard -"@livekit/react-native-webrtc@npm:^125.0.7": - version: 125.0.7 - resolution: "@livekit/react-native-webrtc@npm:125.0.7" +"@livekit/react-native-webrtc@npm:^125.0.8": + version: 125.0.8 + resolution: "@livekit/react-native-webrtc@npm:125.0.8" dependencies: base64-js: 1.5.1 debug: 4.3.4 event-target-shim: 6.0.2 peerDependencies: react-native: ">=0.60.0" - checksum: 20a9359cfe6d1570e97d545f1fc958b748432744cc55233a8fbda199c9906ef11e7cc81576080a3aba59575fd1b75b677b78f283eb96d932c10e936f5e80c9a4 + checksum: addf528f9d5fdc18b5f03134cec9d0adc1f0837d727af3117c9da04c56a709875be8d1e360fb65409297a8137bb2c6933fc8aaf527da49fc86764f4ab55e5cb6 languageName: node linkType: hard @@ -3856,7 +3856,7 @@ __metadata: "@babel/preset-env": ^7.20.0 "@babel/runtime": ^7.20.0 "@livekit/react-native": "*" - "@livekit/react-native-webrtc": ^125.0.7 + "@livekit/react-native-webrtc": ^125.0.8 "@react-native/babel-preset": 0.74.84 "@react-native/eslint-config": 0.74.84 "@react-native/metro-config": 0.74.84 diff --git a/example/android/app/build.gradle b/example/android/app/build.gradle index 6dab3fd..9f08e51 100644 --- a/example/android/app/build.gradle +++ b/example/android/app/build.gradle @@ -107,7 +107,7 @@ android { dependencies { // The version of react-native is set by the React Native Gradle Plugin implementation("com.facebook.react:react-android") - implementation project(':livekitreactnative') + implementation project(':livekit_react-native') if (hermesEnabled.toBoolean()) { implementation("com.facebook.react:hermes-android") } else { diff --git a/example/android/settings.gradle b/example/android/settings.gradle index 40ba1bc..ee60fac 100644 --- a/example/android/settings.gradle +++ b/example/android/settings.gradle @@ -2,6 +2,6 @@ rootProject.name = 'LivekitReactNativeExample' apply from: file("../node_modules/@react-native-community/cli-platform-android/native_modules.gradle"); applyNativeModulesSettingsGradle(settings) include ':app' -include ':livekitreactnative' -project(':livekitreactnative').projectDir = new File(rootProject.projectDir, '../../android') +include ':livekit_react-native' +project(':livekit_react-native').projectDir = new File(rootProject.projectDir, '../../android') includeBuild('../node_modules/@react-native/gradle-plugin') \ No newline at end of file diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 853809f..dffc91d 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -10,9 +10,9 @@ PODS: - livekit-react-native (2.5.1): - livekit-react-native-webrtc - React-Core - - livekit-react-native-webrtc (125.0.7): + - livekit-react-native-webrtc (125.0.8): - React-Core - - WebRTC-SDK (~> 125.6422.06) + - WebRTC-SDK (~> 125.6422.07) - RCT-Folly (2024.01.01.00): - boost - DoubleConversion @@ -1201,7 +1201,7 @@ PODS: - ReactCommon/turbomodule/core - Yoga - SocketRocket (0.7.0) - - WebRTC-SDK (125.6422.06) + - WebRTC-SDK (125.6422.07) - Yoga (0.0.0) DEPENDENCIES: @@ -1406,8 +1406,8 @@ SPEC CHECKSUMS: fmt: 4c2741a687cc09f0634a2e2c72a838b99f1ff120 glog: fdfdfe5479092de0c4bdbebedd9056951f092c4f hermes-engine: 01d3e052018c2a13937aca1860fbedbccd4a41b7 - livekit-react-native: 4d654dffad48942f2d5f636a5bd30a203a7676e3 - livekit-react-native-webrtc: 5b7ce2ff4b44961f9363519de098bcba7dfa213f + livekit-react-native: 08264d81497fe9d18eb513e1db05d0539ddb9417 + livekit-react-native-webrtc: c456181c7c6f9f2b0a79ea14d0d3c97215266ba0 RCT-Folly: 02617c592a293bd6d418e0a88ff4ee1f88329b47 RCTDeprecation: b03c35057846b685b3ccadc9bfe43e349989cdb2 RCTRequired: 194626909cfa8d39ca6663138c417bc6c431648c @@ -1461,7 +1461,7 @@ SPEC CHECKSUMS: RNCAsyncStorage: 0c357f3156fcb16c8589ede67cc036330b6698ca RNScreens: b32a9ff15bea7fcdbe5dff6477bc503f792b1208 SocketRocket: abac6f5de4d4d62d24e11868d7a2f427e0ef940d - WebRTC-SDK: 79942c006ea64f6fb48d7da8a4786dfc820bc1db + WebRTC-SDK: dff00a3892bc570b6014e046297782084071657e Yoga: ae3c32c514802d30f687a04a6a35b348506d411f PODFILE CHECKSUM: b5aad0c7d12b2ea501eb822f98f00ca01d154bd9 diff --git a/example/package.json b/example/package.json index 6200bc8..a574b98 100644 --- a/example/package.json +++ b/example/package.json @@ -10,7 +10,7 @@ "postinstall": "patch-package" }, "dependencies": { - "@livekit/react-native-webrtc": "^125.0.7", + "@livekit/react-native-webrtc": "^125.0.8", "@react-native-async-storage/async-storage": "^1.17.10", "@react-navigation/native": "^6.0.8", "@react-navigation/native-stack": "^6.5.0", diff --git a/example/src/RoomPage.tsx b/example/src/RoomPage.tsx index 080581d..3568eae 100644 --- a/example/src/RoomPage.tsx +++ b/example/src/RoomPage.tsx @@ -82,25 +82,28 @@ export const RoomPage = ({ audio={true} video={true} > - + ); }; interface RoomViewProps { navigation: NativeStackNavigationProp; + e2ee: boolean; } -const RoomView = ({ navigation }: RoomViewProps) => { +const RoomView = ({ navigation, e2ee }: RoomViewProps) => { const [isCameraFrontFacing, setCameraFrontFacing] = useState(true); const room = useRoomContext(); useEffect(() => { let setup = async () => { - await room.setE2EEEnabled(true); + if (e2ee) { + await room.setE2EEEnabled(true); + } }; setup(); return () => {}; - }, [room]); + }, [room, e2ee]); useIOSAudioManagement(room, true); // Setup room listeners diff --git a/example/yarn.lock b/example/yarn.lock index 60f6047..e10fb27 100644 --- a/example/yarn.lock +++ b/example/yarn.lock @@ -2149,16 +2149,16 @@ __metadata: languageName: node linkType: hard -"@livekit/react-native-webrtc@npm:^125.0.7": - version: 125.0.7 - resolution: "@livekit/react-native-webrtc@npm:125.0.7" +"@livekit/react-native-webrtc@npm:^125.0.8": + version: 125.0.8 + resolution: "@livekit/react-native-webrtc@npm:125.0.8" dependencies: base64-js: 1.5.1 debug: 4.3.4 event-target-shim: 6.0.2 peerDependencies: react-native: ">=0.60.0" - checksum: 20a9359cfe6d1570e97d545f1fc958b748432744cc55233a8fbda199c9906ef11e7cc81576080a3aba59575fd1b75b677b78f283eb96d932c10e936f5e80c9a4 + checksum: addf528f9d5fdc18b5f03134cec9d0adc1f0837d727af3117c9da04c56a709875be8d1e360fb65409297a8137bb2c6933fc8aaf527da49fc86764f4ab55e5cb6 languageName: node linkType: hard @@ -6079,7 +6079,7 @@ __metadata: "@babel/core": ^7.20.0 "@babel/preset-env": ^7.20.0 "@babel/runtime": ^7.20.0 - "@livekit/react-native-webrtc": ^125.0.7 + "@livekit/react-native-webrtc": ^125.0.8 "@react-native-async-storage/async-storage": ^1.17.10 "@react-native/babel-preset": 0.74.84 "@react-native/eslint-config": 0.74.84 diff --git a/ios/LKAudioProcessingAdapter.h b/ios/LKAudioProcessingAdapter.h new file mode 100644 index 0000000..f732708 --- /dev/null +++ b/ios/LKAudioProcessingAdapter.h @@ -0,0 +1,26 @@ +#import +#import + +@protocol LKExternalAudioProcessingDelegate + +- (void)audioProcessingInitializeWithSampleRate:(size_t)sampleRateHz channels:(size_t)channels; + +- (void)audioProcessingProcess:(RTC_OBJC_TYPE(RTCAudioBuffer) * _Nonnull)audioBuffer; + +- (void)audioProcessingRelease; + +@end + +@interface LKAudioProcessingAdapter : NSObject + +- (nonnull instancetype)init; + +- (void)addProcessing:(id _Nonnull)processor; + +- (void)removeProcessing:(id _Nonnull)processor; + +- (void)addAudioRenderer:(nonnull id)renderer; + +- (void)removeAudioRenderer:(nonnull id)renderer; + +@end \ No newline at end of file diff --git a/ios/LKAudioProcessingAdapter.m b/ios/LKAudioProcessingAdapter.m new file mode 100644 index 0000000..1d63824 --- /dev/null +++ b/ios/LKAudioProcessingAdapter.m @@ -0,0 +1,117 @@ +#import "LKAudioProcessingAdapter.h" +#import +#import + +@implementation LKAudioProcessingAdapter { + NSMutableArray>* _renderers; + NSMutableArray>* _processors; + os_unfair_lock _lock; + BOOL _isAudioProcessingInitialized; + size_t _sampleRateHz; + size_t _channels; +} + +- (instancetype)init { + self = [super init]; + if (self) { + _isAudioProcessingInitialized = NO; + _lock = OS_UNFAIR_LOCK_INIT; + _renderers = [[NSMutableArray> alloc] init]; + _processors = [[NSMutableArray> alloc] init]; + } + return self; +} + +- (void)addProcessing:(id _Nonnull)processor { + os_unfair_lock_lock(&_lock); + [_processors addObject:processor]; + if (_isAudioProcessingInitialized) { + [processor audioProcessingInitializeWithSampleRate:_sampleRateHz channels:_channels]; + } + os_unfair_lock_unlock(&_lock); +} + +- (void)removeProcessing:(id _Nonnull)processor { + os_unfair_lock_lock(&_lock); + _processors = [[_processors + filteredArrayUsingPredicate:[NSPredicate predicateWithBlock:^BOOL(id evaluatedObject, + NSDictionary* bindings) { + return evaluatedObject != processor; + }]] mutableCopy]; + os_unfair_lock_unlock(&_lock); +} + +- (void)addAudioRenderer:(nonnull id)renderer { + os_unfair_lock_lock(&_lock); + [_renderers addObject:renderer]; + os_unfair_lock_unlock(&_lock); +} + +- (void)removeAudioRenderer:(nonnull id)renderer { + os_unfair_lock_lock(&_lock); + _renderers = [[_renderers + filteredArrayUsingPredicate:[NSPredicate predicateWithBlock:^BOOL(id evaluatedObject, + NSDictionary* bindings) { + return evaluatedObject != renderer; + }]] mutableCopy]; + os_unfair_lock_unlock(&_lock); +} + +- (void)audioProcessingInitializeWithSampleRate:(size_t)sampleRateHz channels:(size_t)channels { + os_unfair_lock_lock(&_lock); + _isAudioProcessingInitialized = YES; + _sampleRateHz = sampleRateHz; + _channels = channels; + + for (id processor in _processors) { + [processor audioProcessingInitializeWithSampleRate:sampleRateHz channels:channels]; + } + os_unfair_lock_unlock(&_lock); +} + +- (AVAudioPCMBuffer*)toPCMBuffer:(RTC_OBJC_TYPE(RTCAudioBuffer) *)audioBuffer { + AVAudioFormat* format = + [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 + sampleRate:audioBuffer.frames * 100.0 + channels:(AVAudioChannelCount)audioBuffer.channels + interleaved:NO]; + AVAudioPCMBuffer* pcmBuffer = + [[AVAudioPCMBuffer alloc] initWithPCMFormat:format + frameCapacity:(AVAudioFrameCount)audioBuffer.frames]; + if (!pcmBuffer) { + NSLog(@"Failed to create AVAudioPCMBuffer"); + return nil; + } + pcmBuffer.frameLength = (AVAudioFrameCount)audioBuffer.frames; + for (int i = 0; i < audioBuffer.channels; i++) { + float* sourceBuffer = [audioBuffer rawBufferForChannel:i]; + int16_t* targetBuffer = (int16_t*)pcmBuffer.int16ChannelData[i]; + for (int frame = 0; frame < audioBuffer.frames; frame++) { + targetBuffer[frame] = sourceBuffer[frame]; + } + } + return pcmBuffer; +} + +- (void)audioProcessingProcess:(RTC_OBJC_TYPE(RTCAudioBuffer) *)audioBuffer { + os_unfair_lock_lock(&_lock); + for (id processor in _processors) { + [processor audioProcessingProcess:audioBuffer]; + } + + for (id renderer in _renderers) { + [renderer renderPCMBuffer:[self toPCMBuffer:audioBuffer]]; + } + os_unfair_lock_unlock(&_lock); +} + +- (void)audioProcessingRelease { + os_unfair_lock_lock(&_lock); + for (id processor in _processors) { + [processor audioProcessingRelease]; + } + _isAudioProcessingInitialized = NO; + os_unfair_lock_unlock(&_lock); +} + +@end diff --git a/ios/LKAudioProcessingManager.h b/ios/LKAudioProcessingManager.h new file mode 100644 index 0000000..332b85c --- /dev/null +++ b/ios/LKAudioProcessingManager.h @@ -0,0 +1,34 @@ +#import +#import +#import "LKAudioProcessingAdapter.h" + +@interface LKAudioProcessingManager : NSObject + +@property(nonatomic, strong) RTCDefaultAudioProcessingModule* _Nonnull audioProcessingModule; + +@property(nonatomic, strong) LKAudioProcessingAdapter* _Nonnull capturePostProcessingAdapter; + +@property(nonatomic, strong) LKAudioProcessingAdapter* _Nonnull renderPreProcessingAdapter; + ++ (_Nonnull instancetype)sharedInstance; + + +- (void)addLocalAudioRenderer:(nonnull id)renderer; + +- (void)removeLocalAudioRenderer:(nonnull id)renderer; + +- (void)addRemoteAudioRenderer:(nonnull id)renderer; + +- (void)removeRemoteAudioRenderer:(nonnull id)renderer; + +- (void)addCapturePostProcessor:(nonnull id)processor; + +- (void)removeCapturePostProcessor:(nonnull id)processor; + +- (void)addRenderPreProcessor:(nonnull id)renderer; + +- (void)removeRenderPreProcessor:(nonnull id)renderer; + +- (void)clearProcessors; + +@end diff --git a/ios/LKAudioProcessingManager.m b/ios/LKAudioProcessingManager.m new file mode 100644 index 0000000..083fa77 --- /dev/null +++ b/ios/LKAudioProcessingManager.m @@ -0,0 +1,63 @@ +#import "LKAudioProcessingManager.h" +#import "LKAudioProcessingAdapter.h" + +@implementation LKAudioProcessingManager + ++ (instancetype)sharedInstance { + static dispatch_once_t onceToken; + static LKAudioProcessingManager* sharedInstance = nil; + dispatch_once(&onceToken, ^{ + sharedInstance = [[self alloc] init]; + }); + return sharedInstance; +} + +- (instancetype)init { + if (self = [super init]) { + _audioProcessingModule = [[RTCDefaultAudioProcessingModule alloc] init]; + _capturePostProcessingAdapter = [[LKAudioProcessingAdapter alloc] init]; + _renderPreProcessingAdapter = [[LKAudioProcessingAdapter alloc] init]; + _audioProcessingModule.capturePostProcessingDelegate = _capturePostProcessingAdapter; + _audioProcessingModule.renderPreProcessingDelegate = _renderPreProcessingAdapter; + } + return self; +} + +- (void)addLocalAudioRenderer:(nonnull id)renderer { + [_capturePostProcessingAdapter addAudioRenderer:renderer]; +} + +- (void)removeLocalAudioRenderer:(nonnull id)renderer { + [_capturePostProcessingAdapter removeAudioRenderer:renderer]; +} + +- (void)addRemoteAudioRenderer:(nonnull id)renderer { + [_renderPreProcessingAdapter addAudioRenderer:renderer]; +} + +- (void)removeRemoteAudioRenderer:(nonnull id)renderer { + [_renderPreProcessingAdapter removeAudioRenderer:renderer]; +} + +- (void)addCapturePostProcessor:(nonnull id)processor { + [_capturePostProcessingAdapter addProcessing:processor]; +} + +- (void)removeCapturePostProcessor:(nonnull id)processor { + [_capturePostProcessingAdapter removeProcessing:processor]; +} + +- (void)addRenderPreProcessor:(nonnull id)processor { + [_renderPreProcessingAdapter addProcessing:processor]; +} + +- (void)removeRenderPreProcessor:(nonnull id)processor { + [_renderPreProcessingAdapter removeProcessing:processor]; +} + +- (void)clearProcessors { + // TODO +} + + +@end diff --git a/ios/LivekitReactNative-Bridging-Header.h b/ios/LivekitReactNative-Bridging-Header.h index dea7ff6..846f301 100644 --- a/ios/LivekitReactNative-Bridging-Header.h +++ b/ios/LivekitReactNative-Bridging-Header.h @@ -1,2 +1,4 @@ #import #import +#import "WebRTCModule.h" +#import "WebRTCModule+RTCMediaStream.h" diff --git a/ios/LivekitReactNative.h b/ios/LivekitReactNative.h index e084769..fa43ffc 100644 --- a/ios/LivekitReactNative.h +++ b/ios/LivekitReactNative.h @@ -3,12 +3,17 @@ // LivekitReactNative // // Created by David Liu on 9/4/22. -// Copyright © 2022 LiveKit. All rights reserved. +// Copyright © 2022-2025 LiveKit. All rights reserved. // #import +#import +#import -@interface LivekitReactNative : NSObject - +@class AudioRendererManager; +@interface LivekitReactNative : RCTEventEmitter +@property(nonatomic, strong) AudioRendererManager* _Nonnull audioRendererManager; +(void)setup; - @end + +extern NSString * _Nonnull const kEventVolumeProcessed; +extern NSString * _Nonnull const kEventMultibandProcessed; diff --git a/ios/LivekitReactNative.m b/ios/LivekitReactNative.m index ee9600e..8232de4 100644 --- a/ios/LivekitReactNative.m +++ b/ios/LivekitReactNative.m @@ -1,15 +1,23 @@ #import "AudioUtils.h" #import "LivekitReactNative.h" +#import "LKAudioProcessingManager.h" #import "WebRTCModule.h" #import "WebRTCModuleOptions.h" #import #import #import #import +#import "livekit_react_native-Swift.h" + +NSString *const kEventVolumeProcessed = @"LK_VOLUME_PROCESSED"; +NSString *const kEventMultibandProcessed = @"LK_MULTIBAND_PROCESSED"; @implementation LivekitReactNative + + RCT_EXPORT_MODULE(); + -(instancetype)init { if(self = [super init]) { @@ -38,6 +46,7 @@ +(void)setup { RTCVideoEncoderFactorySimulcast *simulcastVideoEncoderFactory = [[RTCVideoEncoderFactorySimulcast alloc] initWithPrimary:videoEncoderFactory fallback:videoEncoderFactory]; WebRTCModuleOptions *options = [WebRTCModuleOptions sharedInstance]; options.videoEncoderFactory = simulcastVideoEncoderFactory; + options.audioProcessingModule = LKAudioProcessingManager.sharedInstance.audioProcessingModule; } /// Configure default audio config for WebRTC @@ -123,13 +132,13 @@ +(void)setup { RCT_EXPORT_METHOD(setAppleAudioConfiguration:(NSDictionary *) configuration){ RTCAudioSession* session = [RTCAudioSession sharedInstance]; RTCAudioSessionConfiguration* config = [RTCAudioSessionConfiguration webRTCConfiguration]; - + NSString* appleAudioCategory = configuration[@"audioCategory"]; NSArray* appleAudioCategoryOptions = configuration[@"audioCategoryOptions"]; NSString* appleAudioMode = configuration[@"audioMode"]; [session lockForConfiguration]; - + NSError* error = nil; BOOL categoryChanged = NO; if(appleAudioCategoryOptions != nil) { @@ -151,7 +160,7 @@ +(void)setup { } } } - + if(appleAudioCategory != nil) { categoryChanged = YES; config.category = [AudioUtils audioSessionCategoryFromString:appleAudioCategory]; @@ -164,7 +173,7 @@ +(void)setup { error = nil; } } - + if(appleAudioMode != nil) { config.mode = [AudioUtils audioSessionModeFromString:appleAudioMode]; [session setMode:config.mode error:&error]; @@ -173,7 +182,76 @@ +(void)setup { error = nil; } } - + [session unlockForConfiguration]; } + +-(AudioRendererManager *)audioRendererManager { + if(!_audioRendererManager) { + _audioRendererManager = [[AudioRendererManager alloc] initWithBridge:self.bridge]; + } + + return _audioRendererManager; +} + +RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(createVolumeProcessor:(nonnull NSNumber *)pcId + trackId:(nonnull NSString *)trackId) { + + + VolumeAudioRenderer *renderer = [[VolumeAudioRenderer alloc] initWithIntervalMs:40.0 eventEmitter:self]; + + NSString *reactTag = [self.audioRendererManager registerRenderer:renderer]; + renderer.reactTag = reactTag; + [self.audioRendererManager attachWithRenderer:renderer pcId:pcId trackId:trackId]; + return reactTag; +} + +RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(deleteVolumeProcessor:(nonnull NSString *)reactTag + pcId:(nonnull NSNumber *)pcId + trackId:(nonnull NSString *)trackId) { + + [self.audioRendererManager detachWithRendererByTag:reactTag pcId:pcId trackId:trackId]; + [self.audioRendererManager unregisterRendererForReactTag:reactTag]; + + return nil; +} + +RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(createMultibandVolumeProcessor:(NSDictionary *)options + pcId:(nonnull NSNumber *)pcId + trackId:(nonnull NSString *)trackId) { + + NSInteger bands = [(NSNumber *)options[@"bands"] integerValue]; + float minFrequency = [(NSNumber *)options[@"minFrequency"] floatValue]; + float maxFrequency = [(NSNumber *)options[@"maxFrequency"] floatValue]; + float intervalMs = [(NSNumber *)options[@"updateInterval"] floatValue]; + MultibandVolumeAudioRenderer *renderer = [[MultibandVolumeAudioRenderer alloc] initWithBands:bands + minFrequency:minFrequency + maxFrequency:maxFrequency + intervalMs:intervalMs + eventEmitter:self]; + + NSString *reactTag = [self.audioRendererManager registerRenderer:renderer]; + renderer.reactTag = reactTag; + [self.audioRendererManager attachWithRenderer:renderer pcId:pcId trackId:trackId]; + return reactTag; +} + +RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(deleteMultibandVolumeProcessor:(nonnull NSString *)reactTag + pcId:(nonnull NSNumber *)pcId + trackId:(nonnull NSString *)trackId) { + + [self.audioRendererManager detachWithRendererByTag:reactTag pcId:pcId trackId:trackId]; + [self.audioRendererManager unregisterRendererForReactTag:reactTag]; + + return nil; +} + + +- (NSArray *)supportedEvents { + return @[ + kEventVolumeProcessed, + kEventMultibandProcessed, + ]; +} + @end diff --git a/ios/Logging.swift b/ios/Logging.swift new file mode 100644 index 0000000..b964ca0 --- /dev/null +++ b/ios/Logging.swift @@ -0,0 +1,4 @@ +public func lklog(_ object: Any, functionName: String = #function, fileName: String = #file, lineNumber: Int = #line) { + let className = (fileName as NSString).lastPathComponent + print("\(className).\(functionName):\(lineNumber) : \(object)\n") +} diff --git a/ios/audio/AVAudioPCMBuffer.swift b/ios/audio/AVAudioPCMBuffer.swift new file mode 100644 index 0000000..5e8fd49 --- /dev/null +++ b/ios/audio/AVAudioPCMBuffer.swift @@ -0,0 +1,136 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Accelerate +import AVFoundation + +public extension AVAudioPCMBuffer { + func resample(toSampleRate targetSampleRate: Double) -> AVAudioPCMBuffer? { + let sourceFormat = format + + if sourceFormat.sampleRate == targetSampleRate { + // Already targetSampleRate. + return self + } + + // Define the source format (from the input buffer) and the target format. + guard let targetFormat = AVAudioFormat(commonFormat: sourceFormat.commonFormat, + sampleRate: targetSampleRate, + channels: sourceFormat.channelCount, + interleaved: sourceFormat.isInterleaved) + else { + print("Failed to create target format.") + return nil + } + + guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else { + print("Failed to create audio converter.") + return nil + } + + let capacity = targetFormat.sampleRate * Double(frameLength) / sourceFormat.sampleRate + + guard let convertedBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: AVAudioFrameCount(capacity)) else { + print("Failed to create converted buffer.") + return nil + } + + var isDone = false + let inputBlock: AVAudioConverterInputBlock = { _, outStatus in + if isDone { + outStatus.pointee = .noDataNow + return nil + } + outStatus.pointee = .haveData + isDone = true + return self + } + + var error: NSError? + let status = converter.convert(to: convertedBuffer, error: &error, withInputFrom: inputBlock) + + if status == .error { + print("Conversion failed: \(error?.localizedDescription ?? "Unknown error")") + return nil + } + + // Adjust frame length to the actual amount of data written + convertedBuffer.frameLength = convertedBuffer.frameCapacity + + return convertedBuffer + } + + /// Convert PCM buffer to specified common format. + /// Currently supports conversion from Int16 to Float32. + func convert(toCommonFormat commonFormat: AVAudioCommonFormat) -> AVAudioPCMBuffer? { + // Check if conversion is needed + guard format.commonFormat != commonFormat else { + return self + } + + // Check if the conversion is supported + guard format.commonFormat == .pcmFormatInt16, commonFormat == .pcmFormatFloat32 else { + print("Unsupported conversion: only Int16 to Float32 is supported") + return nil + } + + // Create output format + guard let outputFormat = AVAudioFormat(commonFormat: commonFormat, + sampleRate: format.sampleRate, + channels: format.channelCount, + interleaved: false) + else { + print("Failed to create output audio format") + return nil + } + + // Create output buffer + guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: outputFormat, + frameCapacity: frameCapacity) + else { + print("Failed to create output PCM buffer") + return nil + } + + outputBuffer.frameLength = frameLength + + let channelCount = Int(format.channelCount) + let frameCount = Int(frameLength) + + // Ensure the source buffer has Int16 data + guard let int16Data = int16ChannelData else { + print("Source buffer doesn't contain Int16 data") + return nil + } + + // Ensure the output buffer has Float32 data + guard let floatData = outputBuffer.floatChannelData else { + print("Failed to get float channel data from output buffer") + return nil + } + + // Convert Int16 to Float32 and normalize to [-1.0, 1.0] + let scale = Float(Int16.max) + var scalar = 1.0 / scale + + for channel in 0 ..< channelCount { + vDSP_vflt16(int16Data[channel], 1, floatData[channel], 1, vDSP_Length(frameCount)) + vDSP_vsmul(floatData[channel], 1, &scalar, floatData[channel], 1, vDSP_Length(frameCount)) + } + + return outputBuffer + } +} diff --git a/ios/audio/AudioProcessing.swift b/ios/audio/AudioProcessing.swift new file mode 100644 index 0000000..2b3b8cb --- /dev/null +++ b/ios/audio/AudioProcessing.swift @@ -0,0 +1,163 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Accelerate +import AVFoundation +import Foundation +import WebRTC + +public struct AudioLevel { + /// Linear Scale RMS Value + public let average: Float + public let peak: Float +} + +public extension RTCAudioBuffer { + /// Convert to AVAudioPCMBuffer Int16 format. + @objc + func toAVAudioPCMBuffer() -> AVAudioPCMBuffer? { + guard let audioFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: Double(frames * 100), + channels: AVAudioChannelCount(channels), + interleaved: false), + let pcmBuffer = AVAudioPCMBuffer(pcmFormat: audioFormat, + frameCapacity: AVAudioFrameCount(frames)) + else { return nil } + + pcmBuffer.frameLength = AVAudioFrameCount(frames) + + guard let targetBufferPointer = pcmBuffer.int16ChannelData else { return nil } + + for i in 0 ..< channels { + let sourceBuffer = rawBuffer(forChannel: i) + let targetBuffer = targetBufferPointer[i] + // sourceBuffer is in the format of [Int16] but is stored in 32-bit alignment, we need to pack the Int16 data correctly. + + for frame in 0 ..< frames { + // Cast and pack the source 32-bit Int16 data into the target 16-bit buffer + let clampedValue = max(Float(Int16.min), min(Float(Int16.max), sourceBuffer[frame])) + targetBuffer[frame] = Int16(clampedValue) + } + } + + return pcmBuffer + } +} + +public extension AVAudioPCMBuffer { + /// Computes Peak and Linear Scale RMS Value (Average) for all channels. + func audioLevels() -> [AudioLevel] { + var result: [AudioLevel] = [] + guard let data = floatChannelData else { + // Not containing float data + return result + } + + for i in 0 ..< Int(format.channelCount) { + let channelData = data[i] + var max: Float = 0.0 + vDSP_maxv(channelData, stride, &max, vDSP_Length(frameLength)) + var rms: Float = 0.0 + vDSP_rmsqv(channelData, stride, &rms, vDSP_Length(frameLength)) + + // No conversion to dB, return linear scale values directly + result.append(AudioLevel(average: rms, peak: max)) + } + + return result + } +} + +public extension Sequence where Iterator.Element == AudioLevel { + /// Combines all elements into a single audio level by computing the average value of all elements. + func combine() -> AudioLevel? { + var count = 0 + let totalSums: (averageSum: Float, peakSum: Float) = reduce((averageSum: 0.0, peakSum: 0.0)) { totals, audioLevel in + count += 1 + return (totals.averageSum + audioLevel.average, + totals.peakSum + audioLevel.peak) + } + + guard count > 0 else { return nil } + + return AudioLevel(average: totalSums.averageSum / Float(count), + peak: totalSums.peakSum / Float(count)) + } +} + +public class AudioVisualizeProcessor { + static let bufferSize = 1024 + + // MARK: - Public + + public let minFrequency: Float + public let maxFrequency: Float + public let minDB: Float + public let maxDB: Float + public let bandsCount: Int + + private var bands: [Float]? + + // MARK: - Private + + private let ringBuffer = RingBuffer(size: AudioVisualizeProcessor.bufferSize) + private let processor: FFTProcessor + + public init(minFrequency: Float = 10, + maxFrequency: Float = 8000, + minDB: Float = -32.0, + maxDB: Float = 32.0, + bandsCount: Int = 100) + { + self.minFrequency = minFrequency + self.maxFrequency = maxFrequency + self.minDB = minDB + self.maxDB = maxDB + self.bandsCount = bandsCount + + processor = FFTProcessor(bufferSize: Self.bufferSize) + bands = [Float](repeating: 0.0, count: bandsCount) + } + + public func process(pcmBuffer: AVAudioPCMBuffer) -> [Float]? { + guard let pcmBuffer = pcmBuffer.convert(toCommonFormat: .pcmFormatFloat32) else { return nil } + guard let floatChannelData = pcmBuffer.floatChannelData else { return nil } + + // Get the float array. + let floats = Array(UnsafeBufferPointer(start: floatChannelData[0], count: Int(pcmBuffer.frameLength))) + ringBuffer.write(floats) + + // Get full-size buffer if available, otherwise return + guard let buffer = ringBuffer.read() else { return nil } + + // Process FFT and compute frequency bands + let fftRes = processor.process(buffer: buffer) + let bands = fftRes.computeBands( + minFrequency: minFrequency, + maxFrequency: maxFrequency, + bandsCount: bandsCount, + sampleRate: Float(pcmBuffer.format.sampleRate) + ) + + let headroom = maxDB - minDB + + // Normalize magnitudes (already in decibels) + return bands.magnitudes.map { magnitude in + let adjustedMagnitude = max(0, magnitude + abs(minDB)) + return min(1.0, adjustedMagnitude / headroom) + } + } +} diff --git a/ios/audio/AudioRendererManager.swift b/ios/audio/AudioRendererManager.swift new file mode 100644 index 0000000..36acf33 --- /dev/null +++ b/ios/audio/AudioRendererManager.swift @@ -0,0 +1,72 @@ +import Foundation +import WebRTC + +public class AudioRendererManager: NSObject { + private let bridge: RCTBridge + public private(set) var renderers: [String: RTCAudioRenderer] = [:] + + @objc + public init(bridge: RCTBridge) { + self.bridge = bridge + } + + @objc + public func registerRenderer(_ audioRenderer: RTCAudioRenderer) -> String { + let reactTag = NSUUID().uuidString + self.renderers[reactTag] = audioRenderer + return reactTag + } + + @objc + public func unregisterRenderer(forReactTag: String) { + self.renderers.removeValue(forKey: forReactTag) + } + + @objc + public func unregisterRenderer(_ audioRenderer: RTCAudioRenderer) { + self.renderers = self.renderers.filter({ $0.value !== audioRenderer }) + } + + @objc + public func attach(renderer: RTCAudioRenderer, pcId: NSNumber, trackId: String) { + let webrtcModule = self.bridge.module(for: WebRTCModule.self) as! WebRTCModule + guard let track = webrtcModule.track(forId: trackId, pcId: pcId) as? RTCAudioTrack + else { + lklog("couldn't find audio track: pcId: \(pcId), trackId: \(trackId)") + return + } + + if (pcId == -1) { + LKAudioProcessingManager.sharedInstance().addLocalAudioRenderer(renderer); + } else { + track.add(renderer) + } + } + + @objc + public func detach(rendererByTag reactTag:String, pcId: NSNumber, trackId: String){ + guard let renderer = self.renderers[reactTag] + else { + lklog("couldn't find renderer: tag: \(reactTag)") + return + } + + detach(renderer: renderer, pcId: pcId, trackId: trackId) + } + + @objc + public func detach(renderer: RTCAudioRenderer, pcId: NSNumber, trackId: String) { + let webrtcModule = self.bridge.module(for: WebRTCModule.self) as! WebRTCModule + guard let track = webrtcModule.track(forId: trackId, pcId: pcId) as? RTCAudioTrack + else { + lklog("couldn't find audio track: pcId: \(pcId), trackId: \(trackId)") + return + } + + if (pcId == -1) { + LKAudioProcessingManager.sharedInstance().removeLocalAudioRenderer(renderer); + } else { + track.remove(renderer) + } + } +} diff --git a/ios/audio/FFTProcessor.swift b/ios/audio/FFTProcessor.swift new file mode 100755 index 0000000..2ee1017 --- /dev/null +++ b/ios/audio/FFTProcessor.swift @@ -0,0 +1,147 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Accelerate +import AVFoundation + +extension Float { + var nyquistFrequency: Float { self / 2.0 } +} + +public struct FFTComputeBandsResult { + let count: Int + let magnitudes: [Float] + let frequencies: [Float] +} + +public class FFTResult { + public let magnitudes: [Float] + + init(magnitudes: [Float]) { + self.magnitudes = magnitudes + } + + func computeBands(minFrequency: Float, maxFrequency: Float, bandsCount: Int, sampleRate: Float) -> FFTComputeBandsResult { + let actualMaxFrequency = min(sampleRate.nyquistFrequency, maxFrequency) + var bandMagnitudes = [Float](repeating: 0.0, count: bandsCount) + var bandFrequencies = [Float](repeating: 0.0, count: bandsCount) + + let magLowerRange = _magnitudeIndex(for: minFrequency, sampleRate: sampleRate) + let magUpperRange = _magnitudeIndex(for: actualMaxFrequency, sampleRate: sampleRate) + let ratio = Float(magUpperRange - magLowerRange) / Float(bandsCount) + + return magnitudes.withUnsafeBufferPointer { magnitudesPtr in + for i in 0 ..< bandsCount { + let magsStartIdx = vDSP_Length(floorf(Float(i) * ratio)) + magLowerRange + let magsEndIdx = vDSP_Length(floorf(Float(i + 1) * ratio)) + magLowerRange + + let count = magsEndIdx - magsStartIdx + if count > 0 { + var sum: Float = 0 + vDSP_sve(magnitudesPtr.baseAddress! + Int(magsStartIdx), 1, &sum, count) + bandMagnitudes[i] = sum / Float(count) + } else { + bandMagnitudes[i] = magnitudes[Int(magsStartIdx)] + } + + // Compute average frequency + let bandwidth = sampleRate.nyquistFrequency / Float(magnitudes.count) + bandFrequencies[i] = (bandwidth * Float(magsStartIdx) + bandwidth * Float(magsEndIdx)) / 2 + } + + return FFTComputeBandsResult(count: bandsCount, magnitudes: bandMagnitudes, frequencies: bandFrequencies) + } + } + + @inline(__always) private func _magnitudeIndex(for frequency: Float, sampleRate: Float) -> vDSP_Length { + vDSP_Length(Float(magnitudes.count) * frequency / sampleRate.nyquistFrequency) + } +} + +class FFTProcessor { + public enum WindowType { + case none + case hanning + case hamming + } + + public let bufferSize: vDSP_Length + public let windowType: WindowType + + private let bufferHalfSize: vDSP_Length + private let bufferLog2Size: vDSP_Length + private var window: [Float] = [] + private var fftSetup: FFTSetup + private var realBuffer: [Float] + private var imaginaryBuffer: [Float] + private var zeroDBReference: Float = 1.0 + + init(bufferSize: Int, windowType: WindowType = .hanning) { + self.bufferSize = vDSP_Length(bufferSize) + self.windowType = windowType + + bufferHalfSize = vDSP_Length(bufferSize / 2) + bufferLog2Size = vDSP_Length(log2f(Float(bufferSize))) + + realBuffer = [Float](repeating: 0.0, count: Int(bufferHalfSize)) + imaginaryBuffer = [Float](repeating: 0.0, count: Int(bufferHalfSize)) + window = [Float](repeating: 1.0, count: Int(bufferSize)) + + fftSetup = vDSP_create_fftsetup(UInt(bufferLog2Size), FFTRadix(FFT_RADIX2))! + + switch windowType { + case .none: + break + case .hanning: + vDSP_hann_window(&window, vDSP_Length(bufferSize), Int32(vDSP_HANN_NORM)) + case .hamming: + vDSP_hamm_window(&window, vDSP_Length(bufferSize), 0) + } + } + + deinit { + vDSP_destroy_fftsetup(fftSetup) + } + + func process(buffer: [Float]) -> FFTResult { + precondition(buffer.count == Int(bufferSize), "Input buffer size mismatch.") + + var windowedBuffer = [Float](repeating: 0.0, count: Int(bufferSize)) + + vDSP_vmul(buffer, 1, window, 1, &windowedBuffer, 1, bufferSize) + + return realBuffer.withUnsafeMutableBufferPointer { realPtr in + imaginaryBuffer.withUnsafeMutableBufferPointer { imagPtr in + var complexBuffer = DSPSplitComplex(realp: realPtr.baseAddress!, imagp: imagPtr.baseAddress!) + + windowedBuffer.withUnsafeBufferPointer { bufferPtr in + let complexPtr = UnsafeRawPointer(bufferPtr.baseAddress!).bindMemory(to: DSPComplex.self, capacity: Int(bufferHalfSize)) + vDSP_ctoz(complexPtr, 2, &complexBuffer, 1, bufferHalfSize) + } + + vDSP_fft_zrip(fftSetup, &complexBuffer, 1, bufferLog2Size, FFTDirection(FFT_FORWARD)) + + var magnitudes = [Float](repeating: 0.0, count: Int(bufferHalfSize)) + vDSP_zvabs(&complexBuffer, 1, &magnitudes, 1, bufferHalfSize) + + // Convert magnitudes to decibels + vDSP_vdbcon(magnitudes, 1, &zeroDBReference, &magnitudes, 1, vDSP_Length(magnitudes.count), 1) + + return FFTResult(magnitudes: magnitudes) + } + } + } +} diff --git a/ios/audio/MultibandVolumeAudioRenderer.swift b/ios/audio/MultibandVolumeAudioRenderer.swift new file mode 100644 index 0000000..282ad5a --- /dev/null +++ b/ios/audio/MultibandVolumeAudioRenderer.swift @@ -0,0 +1,65 @@ +import WebRTC + +public class MultibandVolumeAudioRenderer: BaseMultibandVolumeAudioRenderer { + private let eventEmitter: RCTEventEmitter + + @objc + public var reactTag: String? = nil + + @objc + public init( + bands: Int, + minFrequency: Float, + maxFrequency: Float, + intervalMs: Float, + eventEmitter: RCTEventEmitter + ) { + self.eventEmitter = eventEmitter + super.init(bands: bands, + minFrequency: minFrequency, + maxFrequency: maxFrequency, + intervalMs: intervalMs) + } + + override func onMagnitudesCalculated(_ magnitudes: [Float]) { + guard !magnitudes.isEmpty, let reactTag = self.reactTag + else { return } + eventEmitter.sendEvent(withName: kEventMultibandProcessed, body: [ + "magnitudes": magnitudes, + "id": reactTag + ]) + } + +} + +public class BaseMultibandVolumeAudioRenderer: NSObject, RTCAudioRenderer { + private let frameInterval: Int + private var skippedFrames = 0 + private let audioProcessor: AudioVisualizeProcessor + + init( + bands: Int, + minFrequency: Float, + maxFrequency: Float, + intervalMs: Float + ) { + self.frameInterval = Int((intervalMs / 10.0).rounded()) + self.audioProcessor = AudioVisualizeProcessor(minFrequency: minFrequency, maxFrequency: maxFrequency, bandsCount: bands) + } + + public func render(pcmBuffer: AVAudioPCMBuffer) { + if(skippedFrames < frameInterval - 1) { + skippedFrames += 1 + return + } + + skippedFrames = 0 + guard let magnitudes = audioProcessor.process(pcmBuffer: pcmBuffer) + else { + return + } + onMagnitudesCalculated(magnitudes) + } + + func onMagnitudesCalculated(_ magnitudes: [Float]) { } +} diff --git a/ios/audio/RingBuffer.swift b/ios/audio/RingBuffer.swift new file mode 100644 index 0000000..7f9847a --- /dev/null +++ b/ios/audio/RingBuffer.swift @@ -0,0 +1,51 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +// Simple ring-buffer used for internal audio processing. Not thread-safe. +class RingBuffer { + private var _isFull = false + private var _buffer: [T] + private var _head: Int = 0 + + init(size: Int) { + _buffer = [T](repeating: 0, count: size) + } + + func write(_ value: T) { + _buffer[_head] = value + _head = (_head + 1) % _buffer.count + if _head == 0 { _isFull = true } + } + + func write(_ sequence: [T]) { + for value in sequence { + write(value) + } + } + + func read() -> [T]? { + guard _isFull else { return nil } + + if _head == 0 { + return _buffer // Return the entire buffer if _head is at the start + } else { + // Return the buffer in the correct order + return Array(_buffer[_head ..< _buffer.count] + _buffer[0 ..< _head]) + } + } +} diff --git a/ios/audio/VolumeAudioRenderer.swift b/ios/audio/VolumeAudioRenderer.swift new file mode 100644 index 0000000..86a4c81 --- /dev/null +++ b/ios/audio/VolumeAudioRenderer.swift @@ -0,0 +1,48 @@ +import WebRTC + +public class VolumeAudioRenderer: BaseVolumeAudioRenderer { + private let eventEmitter: RCTEventEmitter + + @objc + public var reactTag: String? = nil + + @objc + public init(intervalMs: Double, eventEmitter: RCTEventEmitter) { + self.eventEmitter = eventEmitter + super.init(intervalMs: intervalMs) + } + + override public func onVolumeCalculated(_ audioLevels: [AudioLevel]) { + guard let rmsAvg = audioLevels.combine()?.average, + let reactTag = self.reactTag + else { return } + eventEmitter.sendEvent(withName: kEventVolumeProcessed, body: [ + "volume": rmsAvg, + "id": reactTag + ]) + } +} + +public class BaseVolumeAudioRenderer: NSObject, RTCAudioRenderer { + private let frameInterval: Int + private var skippedFrames = 0 + public init(intervalMs: Double = 30) { + self.frameInterval = Int((intervalMs / 10.0).rounded()) + } + + public func render(pcmBuffer: AVAudioPCMBuffer) { + if(skippedFrames < frameInterval - 1) { + skippedFrames += 1 + return + } + + skippedFrames = 0 + guard let pcmBuffer = pcmBuffer.convert(toCommonFormat: .pcmFormatFloat32) else { return } + let audioLevels = pcmBuffer.audioLevels() + onVolumeCalculated(audioLevels) + } + + public func onVolumeCalculated(_ audioLevels: [AudioLevel]) { + + } +} diff --git a/livekit-react-native.podspec b/livekit-react-native.podspec index b5e3a73..319698f 100644 --- a/livekit-react-native.podspec +++ b/livekit-react-native.podspec @@ -13,7 +13,7 @@ Pod::Spec.new do |s| s.platforms = { :ios => "10.0" } s.source = { :git => "https://github.com/livekit/client-sdk-react-native.git", :tag => "#{s.version}" } - s.source_files = "ios/**/*.{h,m,mm}" + s.source_files = "ios/**/*.{h,m,mm,swift}" s.framework = 'AVFAudio' diff --git a/package.json b/package.json index dfbe5bd..0630b5c 100644 --- a/package.json +++ b/package.json @@ -42,10 +42,10 @@ "android" ], "dependencies": { - "@livekit/components-react": "^2.0.6", + "@livekit/components-react": "^2.8.1", "array.prototype.at": "^1.1.1", "events": "^3.3.0", - "livekit-client": "^2.7.5", + "livekit-client": "^2.9.0", "loglevel": "^1.8.0", "promise.allsettled": "^1.0.5", "react-native-url-polyfill": "^1.3.0", @@ -57,7 +57,7 @@ "@babel/preset-env": "^7.20.0", "@babel/runtime": "^7.20.0", "@commitlint/config-conventional": "^16.2.1", - "@livekit/react-native-webrtc": "^125.0.7", + "@livekit/react-native-webrtc": "^125.0.8", "@react-native/babel-preset": "0.74.84", "@react-native/eslint-config": "0.74.84", "@react-native/metro-config": "0.74.84", @@ -83,7 +83,7 @@ "typescript": "5.0.4" }, "peerDependencies": { - "@livekit/react-native-webrtc": "^125.0.7", + "@livekit/react-native-webrtc": "^125.0.8", "react": "*", "react-native": "*" }, diff --git a/src/LKNativeModule.ts b/src/LKNativeModule.ts new file mode 100644 index 0000000..4c28097 --- /dev/null +++ b/src/LKNativeModule.ts @@ -0,0 +1,19 @@ +import { NativeModules, Platform } from 'react-native'; +const LINKING_ERROR = + `The package '@livekit/react-native' doesn't seem to be linked. Make sure: \n\n` + + Platform.select({ ios: "- You have run 'pod install'\n", default: '' }) + + '- You rebuilt the app after installing the package\n' + + '- You are not using Expo managed workflow\n'; + +const LiveKitModule = NativeModules.LivekitReactNative + ? NativeModules.LivekitReactNative + : new Proxy( + {}, + { + get() { + throw new Error(LINKING_ERROR); + }, + } + ); + +export default LiveKitModule; diff --git a/src/components/BarVisualizer.tsx b/src/components/BarVisualizer.tsx new file mode 100644 index 0000000..5a2bc33 --- /dev/null +++ b/src/components/BarVisualizer.tsx @@ -0,0 +1,252 @@ +import { + type AgentState, + type TrackReferenceOrPlaceholder, + useMaybeTrackRefContext, +} from '@livekit/components-react'; +import { + Animated, + StyleSheet, + View, + type ColorValue, + type DimensionValue, + type ViewStyle, +} from 'react-native'; +import { useMultibandTrackVolume } from '../hooks'; +import React, { useEffect, useRef, useState } from 'react'; +export type BarVisualizerOptions = { + /** decimal values from 0 to 1 */ + maxHeight?: number; + /** decimal values from 0 to 1 */ + minHeight?: number; + + barColor?: ColorValue; + barWidth?: DimensionValue; + barBorderRadius?: number; +}; + +const defaultBarOptions = { + maxHeight: 1, + minHeight: 0.2, + barColor: '#888888', + barWidth: 24, + barBorderRadius: 12, +} as const satisfies BarVisualizerOptions; + +const sequencerIntervals = new Map([ + ['connecting', 2000], + ['initializing', 2000], + ['listening', 500], + ['thinking', 150], +]); + +const getSequencerInterval = ( + state: AgentState | undefined, + barCount: number +): number | undefined => { + if (state === undefined) { + return 1000; + } + let interval = sequencerIntervals.get(state); + if (interval) { + switch (state) { + case 'connecting': + // case 'thinking': + interval /= barCount; + break; + + default: + break; + } + } + return interval; +}; +/** + * @beta + */ +export interface BarVisualizerProps { + /** If set, the visualizer will transition between different voice assistant states */ + state?: AgentState; + /** Number of bars that show up in the visualizer */ + barCount?: number; + trackRef?: TrackReferenceOrPlaceholder; + options?: BarVisualizerOptions; + /** + * Custom React Native styles for the container. + */ + style?: ViewStyle; +} + +/** + * Visualizes audio signals from a TrackReference as bars. + * If the `state` prop is set, it automatically transitions between VoiceAssistant states. + * @beta + * + * @remarks For VoiceAssistant state transitions this component requires a voice assistant agent running with livekit-agents \>= 0.9.0 + * + * @example + * ```tsx + * function SimpleVoiceAssistant() { + * const { state, audioTrack } = useVoiceAssistant(); + * return ( + * + * ); + * } + * ``` + */ +export const BarVisualizer = ({ + style = {}, + state, + barCount = 5, + trackRef, + options, +}: BarVisualizerProps) => { + let trackReference = useMaybeTrackRefContext(); + + if (trackRef) { + trackReference = trackRef; + } + + const opacityAnimations = useRef([]).current; + let magnitudes = useMultibandTrackVolume(trackReference, { bands: barCount }); + + let opts = { ...defaultBarOptions, ...options }; + + const highlightedIndices = useBarAnimator( + state, + barCount, + getSequencerInterval(state, barCount) ?? 100 + ); + + useEffect(() => { + let animations = []; + for (let i = 0; i < barCount; i++) { + if (!opacityAnimations[i]) { + opacityAnimations[i] = new Animated.Value(0.3); + } + let targetOpacity = 0.3; + if (highlightedIndices.includes(i)) { + targetOpacity = 1; + } + animations.push( + Animated.timing(opacityAnimations[i], { + toValue: targetOpacity, + duration: 250, + useNativeDriver: true, + }) + ); + } + + let parallel = Animated.parallel(animations); + parallel.start(); + return () => { + parallel.stop(); + }; + }, [highlightedIndices, barCount, opacityAnimations]); + + let bars: React.ReactNode[] = []; + magnitudes.forEach((value, index) => { + let coerced = Math.min(opts.maxHeight, Math.max(opts.minHeight, value)); + let coercedPercent = Math.min(100, Math.max(0, coerced * 100 + 5)); + let opacity = opacityAnimations[index] ?? new Animated.Value(0.3); + let barStyle = { + opacity: opacity, + backgroundColor: opts.barColor, + borderRadius: opts.barBorderRadius, + width: opts.barWidth, + }; + bars.push( + + ); + }); + + return {bars}; +}; +const styles = StyleSheet.create({ + container: { + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'space-evenly', + }, + volumeIndicator: { + borderRadius: 12, + }, +}); + +export const useBarAnimator = ( + state: AgentState | undefined, + columns: number, + interval: number +): number[] => { + const [index, setIndex] = useState(0); + const [sequence, setSequence] = useState([[]]); + + useEffect(() => { + if (state === 'thinking') { + setSequence(generateListeningSequenceBar(columns)); + } else if (state === 'connecting' || state === 'initializing') { + const seq = [...generateConnectingSequenceBar(columns)]; + setSequence(seq); + } else if (state === 'listening') { + setSequence(generateListeningSequenceBar(columns)); + } else if (state === undefined) { + // highlight everything + setSequence([new Array(columns).fill(0).map((_, idx) => idx)]); + } else { + setSequence([[]]); + } + setIndex(0); + }, [state, columns]); + + const animationFrameId = useRef(null); + useEffect(() => { + let startTime = performance.now(); + + const animate = (time: number) => { + const timeElapsed = time - startTime; + + if (timeElapsed >= interval) { + setIndex((prev) => prev + 1); + startTime = time; + } + + animationFrameId.current = requestAnimationFrame(animate); + }; + + animationFrameId.current = requestAnimationFrame(animate); + + return () => { + if (animationFrameId.current !== null) { + cancelAnimationFrame(animationFrameId.current); + } + }; + }, [interval, columns, state, sequence.length]); + + return sequence[index % sequence.length]; +}; + +const generateListeningSequenceBar = (columns: number): number[][] => { + const center = Math.floor(columns / 2); + const noIndex = -1; + + return [[center], [noIndex]]; +}; + +const generateConnectingSequenceBar = (columns: number): number[][] => { + const seq: number[][] = [[]]; + + for (let x = 0; x < columns; x++) { + seq.push([x, columns - 1 - x]); + } + + return seq; +}; diff --git a/src/events/EventEmitter.ts b/src/events/EventEmitter.ts new file mode 100644 index 0000000..01eba0d --- /dev/null +++ b/src/events/EventEmitter.ts @@ -0,0 +1,51 @@ +import { NativeEventEmitter, type EmitterSubscription } from 'react-native'; +// @ts-ignore +import EventEmitter from 'react-native/Libraries/vendor/emitter/EventEmitter'; +import LiveKitModule from '../LKNativeModule'; + +// This emitter is going to be used to listen to all the native events (once) and then +// re-emit them on a JS-only emitter. +const nativeEmitter = new NativeEventEmitter(LiveKitModule); + +const NATIVE_EVENTS = ['LK_VOLUME_PROCESSED', 'LK_MULTIBAND_PROCESSED']; + +const eventEmitter = new EventEmitter(); + +export function setupNativeEvents() { + for (const eventName of NATIVE_EVENTS) { + nativeEmitter.addListener(eventName, (...args) => { + eventEmitter.emit(eventName, ...args); + }); + } +} + +type EventHandler = (event: unknown) => void; +type Listener = unknown; + +const _subscriptions: Map = new Map(); + +export function addListener( + listener: Listener, + eventName: string, + eventHandler: EventHandler +): void { + if (!NATIVE_EVENTS.includes(eventName)) { + throw new Error(`Invalid event: ${eventName}`); + } + + if (!_subscriptions.has(listener)) { + _subscriptions.set(listener, []); + } + + _subscriptions + .get(listener) + ?.push(eventEmitter.addListener(eventName, eventHandler)); +} + +export function removeListener(listener: Listener): void { + _subscriptions.get(listener)?.forEach((sub) => { + sub.remove(); + }); + + _subscriptions.delete(listener); +} diff --git a/src/hooks.ts b/src/hooks.ts index 036e4bf..70539dc 100644 --- a/src/hooks.ts +++ b/src/hooks.ts @@ -42,4 +42,6 @@ export type { export type { ReceivedDataMessage } from '@livekit/components-core'; export * from './hooks/useE2EEManager'; +export * from './hooks/useTrackVolume'; +export * from './hooks/useMultibandTrackVolume'; export type { UseRNE2EEManagerOptions } from './hooks/useE2EEManager'; diff --git a/src/hooks/useMultibandTrackVolume.ts b/src/hooks/useMultibandTrackVolume.ts new file mode 100644 index 0000000..7f8f242 --- /dev/null +++ b/src/hooks/useMultibandTrackVolume.ts @@ -0,0 +1,97 @@ +import { type TrackReferenceOrPlaceholder } from '@livekit/components-react'; +import { + Track, + type LocalAudioTrack, + type RemoteAudioTrack, +} from 'livekit-client'; +import { useEffect, useMemo, useState } from 'react'; +import { addListener, removeListener } from '../events/EventEmitter'; +import LiveKitModule from '../LKNativeModule'; + +/** + * Interface for configuring options for the useMultibandTrackVolume hook. + * @alpha + */ +export interface MultiBandTrackVolumeOptions { + /** + * the number of bands to split the audio into + */ + bands?: number; + /** + * cut off frequency on the lower end + */ + minFrequency?: number; + /** + * cut off frequency on the higher end + */ + maxFrequency?: number; + /** + * update should run every x ms + */ + updateInterval?: number; +} + +const multibandDefaults = { + bands: 5, + minFrequency: 1000, + maxFrequency: 8000, + updateInterval: 40, +} as const satisfies MultiBandTrackVolumeOptions; + +/** + * A hook for tracking the volume of an audio track across multiple frequency bands. + * + * @param trackOrTrackReference + * @returns A number array containing the volume for each frequency band. + */ +export function useMultibandTrackVolume( + trackOrTrackReference?: + | LocalAudioTrack + | RemoteAudioTrack + | TrackReferenceOrPlaceholder, + options: MultiBandTrackVolumeOptions = {} +) { + const track = + trackOrTrackReference instanceof Track + ? trackOrTrackReference + : ( + trackOrTrackReference?.publication?.track + ); + const opts = useMemo(() => { + return { ...multibandDefaults, ...options }; + }, [options]); + const mediaStreamTrack = track?.mediaStreamTrack; + + let [magnitudes, setMagnitudes] = useState([]); + useEffect(() => { + let listener = Object(); + let reactTag: string | null = null; + if (mediaStreamTrack) { + reactTag = LiveKitModule.createMultibandVolumeProcessor( + opts, + mediaStreamTrack._peerConnectionId ?? -1, + mediaStreamTrack.id + ); + addListener(listener, 'LK_MULTIBAND_PROCESSED', (event: any) => { + if (event.magnitudes && reactTag && event.id === reactTag) { + console.log('event received: ', event.magnitudes[0]); + setMagnitudes(event.magnitudes); + } + }); + } + return () => { + if (mediaStreamTrack) { + removeListener(listener); + if (reactTag) { + LiveKitModule.deleteMultibandVolumeProcessor( + reactTag, + mediaStreamTrack._peerConnectionId ?? -1, + mediaStreamTrack.id + ); + } + } + }; + }, [mediaStreamTrack, opts]); + + return magnitudes; +} diff --git a/src/hooks/useTrackVolume.ts b/src/hooks/useTrackVolume.ts new file mode 100644 index 0000000..4aa985a --- /dev/null +++ b/src/hooks/useTrackVolume.ts @@ -0,0 +1,62 @@ +import { type TrackReferenceOrPlaceholder } from '@livekit/components-react'; +import { + Track, + type LocalAudioTrack, + type RemoteAudioTrack, +} from 'livekit-client'; +import { useEffect, useState } from 'react'; +import { addListener, removeListener } from '../events/EventEmitter'; +import LiveKitModule from '../LKNativeModule'; + +/** + * A hook for tracking the volume of an audio track. + * + * @param trackOrTrackReference + * @returns A number between 0-1 representing the volume. + */ +export function useTrackVolume( + trackOrTrackReference?: + | LocalAudioTrack + | RemoteAudioTrack + | TrackReferenceOrPlaceholder +) { + const track = + trackOrTrackReference instanceof Track + ? trackOrTrackReference + : ( + trackOrTrackReference?.publication?.track + ); + + const mediaStreamTrack = track?.mediaStreamTrack; + + let [volume, setVolume] = useState(0.0); + useEffect(() => { + let listener = Object(); + let reactTag: string | null = null; + if (mediaStreamTrack) { + reactTag = LiveKitModule.createVolumeProcessor( + mediaStreamTrack._peerConnectionId ?? -1, + mediaStreamTrack.id + ); + addListener(listener, 'LK_VOLUME_PROCESSED', (event: any) => { + if (event.volume && reactTag && event.id === reactTag) { + setVolume(event.volume); + } + }); + } + return () => { + if (mediaStreamTrack) { + removeListener(listener); + if (reactTag) { + LiveKitModule.deleteVolumeProcessor( + reactTag, + mediaStreamTrack._peerConnectionId ?? -1, + mediaStreamTrack.id + ); + } + } + }; + }, [mediaStreamTrack]); + + return volume; +} diff --git a/src/index.tsx b/src/index.tsx index 5eeadfd..76ebd02 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -17,6 +17,7 @@ import { type LiveKitReactNativeInfo } from 'livekit-client'; import type { LogLevel, SetLogLevelOptions } from './logger'; import RNE2EEManager from './e2ee/RNE2EEManager'; import RNKeyProvider, { type RNKeyProviderOptions } from './e2ee/RNKeyProvider'; +import { setupNativeEvents } from './events/EventEmitter'; /** * Registers the required globals needed for LiveKit to work. @@ -33,6 +34,7 @@ export function registerGlobals() { shimArrayAt(); shimAsyncIterator(); shimIterator(); + setupNativeEvents(); } /** @@ -99,6 +101,7 @@ function shimIterator() { shim(); } export * from './hooks'; +export * from './components/BarVisualizer'; export * from './components/LiveKitRoom'; export * from './components/VideoTrack'; export * from './components/VideoView'; // deprecated diff --git a/yarn.lock b/yarn.lock index 4dd169f..41f3fae 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1947,29 +1947,29 @@ __metadata: languageName: node linkType: hard -"@floating-ui/core@npm:^1.0.0": - version: 1.6.2 - resolution: "@floating-ui/core@npm:1.6.2" +"@floating-ui/core@npm:^1.6.0": + version: 1.6.9 + resolution: "@floating-ui/core@npm:1.6.9" dependencies: - "@floating-ui/utils": ^0.2.0 - checksum: a161b2c8e14b6e185960ec19398f4b893ef3cd6620d535c348c1dc877fb4ffc9f701eb7156f6a30a89b7826093ba28ea223fc2fd1996c0b2464741208725ac8f + "@floating-ui/utils": ^0.2.9 + checksum: 21cbcac72a40172399570dedf0eb96e4f24b0d829980160e8d14edf08c2955ac6feffb7b94e1530c78fb7944635e52669c9257ad08570e0295efead3b5a9af91 languageName: node linkType: hard -"@floating-ui/dom@npm:1.6.5": - version: 1.6.5 - resolution: "@floating-ui/dom@npm:1.6.5" +"@floating-ui/dom@npm:1.6.11": + version: 1.6.11 + resolution: "@floating-ui/dom@npm:1.6.11" dependencies: - "@floating-ui/core": ^1.0.0 - "@floating-ui/utils": ^0.2.0 - checksum: 767295173cfc9024b2187b65d3c1a0c8d8596a1f827d57c86288e52edf91b41508b3679643e24e0ef9f522d86aab59ef97354b456b39be4f6f5159d819cc807d + "@floating-ui/core": ^1.6.0 + "@floating-ui/utils": ^0.2.8 + checksum: d6413759abd06a541edfad829c45313f930310fe76a3322e74a00eb655e283db33fe3e65b5265c4072eb54db7447e11225acd355a9a02cabd1d1b0d5fc8fc21d languageName: node linkType: hard -"@floating-ui/utils@npm:^0.2.0": - version: 0.2.2 - resolution: "@floating-ui/utils@npm:0.2.2" - checksum: 3d8d46fd1b071c98e10d374e2dcf54d1eb9de0aa75ed2b994c9132ebf6f783f896f979053be71450bdb6d60021120cfc24d25a5c84ebb3db0994080e13d9762f +"@floating-ui/utils@npm:^0.2.8, @floating-ui/utils@npm:^0.2.9": + version: 0.2.9 + resolution: "@floating-ui/utils@npm:0.2.9" + checksum: d518b80cec5a323e54a069a1dd99a20f8221a4853ed98ac16c75275a0cc22f75de4f8ac5b121b4f8990bd45da7ad1fb015b9a1e4bac27bb1cd62444af84e9784 languageName: node linkType: hard @@ -2383,65 +2383,66 @@ __metadata: languageName: node linkType: hard -"@livekit/components-core@npm:0.10.3": - version: 0.10.3 - resolution: "@livekit/components-core@npm:0.10.3" +"@livekit/components-core@npm:0.12.1": + version: 0.12.1 + resolution: "@livekit/components-core@npm:0.12.1" dependencies: - "@floating-ui/dom": 1.6.5 + "@floating-ui/dom": 1.6.11 loglevel: 1.9.1 rxjs: 7.8.1 peerDependencies: - "@livekit/protocol": ^1.16.0 - livekit-client: ^2.1.5 + livekit-client: ^2.8.1 tslib: ^2.6.2 - checksum: e024b3584d17869ce0a393393614e1834baa89a57159a38553a8e92bf998b777dda4699bb458338ee6778ff0de7b5354ea9aecae78cf0410922178fc1f481c82 + checksum: 293123096de6daba12e0c6a57b6eecec5fd4dc06caf0333db37d5c17c7c890bddb2d7bfa1425ad3e66cb304ae646a4f537a31fa130227382a23202fcc272674a languageName: node linkType: hard -"@livekit/components-react@npm:^2.0.6": - version: 2.3.3 - resolution: "@livekit/components-react@npm:2.3.3" +"@livekit/components-react@npm:^2.8.1": + version: 2.8.1 + resolution: "@livekit/components-react@npm:2.8.1" dependencies: - "@livekit/components-core": 0.10.3 - "@react-hook/latest": 1.0.3 + "@livekit/components-core": 0.12.1 clsx: 2.1.1 - usehooks-ts: 2.16.0 + usehooks-ts: 3.1.0 peerDependencies: - "@livekit/protocol": ^1.16.0 - livekit-client: ^2.1.5 + "@livekit/krisp-noise-filter": ^0.2.12 + livekit-client: ^2.8.1 react: ">=18" react-dom: ">=18" tslib: ^2.6.2 - checksum: 142ab57e8cc7a5af167669df4c78a94a48926187ad2386a7bd8dbf635c1a900c9996b48dcddafda0c823dc1078f157306d12c1ce53a7f94561877d305e6e5957 + peerDependenciesMeta: + "@livekit/krisp-noise-filter": + optional: true + checksum: 031f168aed8c3d210133974a9e1d5945cebf15fd64afd8579db16ce22eb431cb026305489707a5ce4afbf7fca5965a19ca20e38eeec2c944467f093b35291c2f languageName: node linkType: hard -"@livekit/mutex@npm:1.0.0": - version: 1.0.0 - resolution: "@livekit/mutex@npm:1.0.0" - checksum: 9085325d7fc988d4bb595a51f5672ed92867a3c58ae6a6e06ebf0a967c038f64b665dae88d2d46e445792cd27f117796e7bdc69bef29e08c34c00f1682065e34 +"@livekit/mutex@npm:1.1.1": + version: 1.1.1 + resolution: "@livekit/mutex@npm:1.1.1" + checksum: 44a31eb7a913357ffb57d04eaa20f7507c0a659638c6dfaba9a413c21a3397aa351497f7c77bca3c06a29ac2cbe83698a5f96b9230012a24b86ac8366e9b8666 languageName: node linkType: hard -"@livekit/protocol@npm:1.29.4": - version: 1.29.4 - resolution: "@livekit/protocol@npm:1.29.4" +"@livekit/protocol@npm:1.33.0": + version: 1.33.0 + resolution: "@livekit/protocol@npm:1.33.0" dependencies: "@bufbuild/protobuf": ^1.10.0 - checksum: f9113aafb559d0a924593dc95d7d71819ed06c6ac7da622073a3a78ad2cec8f7a6a3df9f53352308b515ed6eb3617aa0a217e58f54cb4ca2f65f525b7f5b6e5b + checksum: 00609412a17326b1d6c145d137e4ca97448b1fc6b295ed0c1d9714b029cd5a004b96baa7ee7fb53fb5569180f7af63546299224c03f3ce9899d30e8d41a7a8b7 languageName: node linkType: hard -"@livekit/react-native-webrtc@npm:^125.0.7": - version: 125.0.7 - resolution: "@livekit/react-native-webrtc@npm:125.0.7" +"@livekit/react-native-webrtc@npm:^125.0.8": + version: 125.0.8 + resolution: "@livekit/react-native-webrtc@npm:125.0.8" dependencies: base64-js: 1.5.1 debug: 4.3.4 event-target-shim: 6.0.2 peerDependencies: react-native: ">=0.60.0" - checksum: 20a9359cfe6d1570e97d545f1fc958b748432744cc55233a8fbda199c9906ef11e7cc81576080a3aba59575fd1b75b677b78f283eb96d932c10e936f5e80c9a4 + checksum: addf528f9d5fdc18b5f03134cec9d0adc1f0837d727af3117c9da04c56a709875be8d1e360fb65409297a8137bb2c6933fc8aaf527da49fc86764f4ab55e5cb6 languageName: node linkType: hard @@ -2453,8 +2454,8 @@ __metadata: "@babel/preset-env": ^7.20.0 "@babel/runtime": ^7.20.0 "@commitlint/config-conventional": ^16.2.1 - "@livekit/components-react": ^2.0.6 - "@livekit/react-native-webrtc": ^125.0.7 + "@livekit/components-react": ^2.8.1 + "@livekit/react-native-webrtc": ^125.0.8 "@react-native/babel-preset": 0.74.84 "@react-native/eslint-config": 0.74.84 "@react-native/metro-config": 0.74.84 @@ -2472,7 +2473,7 @@ __metadata: events: ^3.3.0 husky: ^7.0.4 jest: ^29.6.3 - livekit-client: ^2.7.5 + livekit-client: ^2.9.0 loglevel: ^1.8.0 pod-install: ^0.2.2 prettier: 2.8.8 @@ -2487,7 +2488,7 @@ __metadata: typescript: 5.0.4 well-known-symbols: ^4.0.0 peerDependencies: - "@livekit/react-native-webrtc": ^125.0.7 + "@livekit/react-native-webrtc": ^125.0.8 react: "*" react-native: "*" languageName: unknown @@ -2689,15 +2690,6 @@ __metadata: languageName: node linkType: hard -"@react-hook/latest@npm:1.0.3": - version: 1.0.3 - resolution: "@react-hook/latest@npm:1.0.3" - peerDependencies: - react: ">=16.8" - checksum: 2408c9cd35c5cfa7697b6da3bc5eebef254a932ade70955074c474f23be7dd3e2f81bbba12edcc9208bd0f89c6ed366d6b11d4f6d7b1052877a0bac8f74afad4 - languageName: node - linkType: hard - "@react-native-community/cli-clean@npm:13.6.8": version: 13.6.8 resolution: "@react-native-community/cli-clean@npm:13.6.8" @@ -8724,20 +8716,20 @@ __metadata: languageName: node linkType: hard -"livekit-client@npm:^2.7.5": - version: 2.7.5 - resolution: "livekit-client@npm:2.7.5" +"livekit-client@npm:^2.9.0": + version: 2.9.0 + resolution: "livekit-client@npm:2.9.0" dependencies: - "@livekit/mutex": 1.0.0 - "@livekit/protocol": 1.29.4 + "@livekit/mutex": 1.1.1 + "@livekit/protocol": 1.33.0 events: ^3.3.0 loglevel: ^1.8.0 sdp-transform: ^2.14.1 ts-debounce: ^4.0.0 - tslib: 2.7.0 + tslib: 2.8.1 typed-emitter: ^2.1.0 webrtc-adapter: ^9.0.0 - checksum: 8586862fae8545971fb106438e7de14452da6bd6bfc87da0ee7dab29005971e0bac72e13ca05619f48034cf727907583e5998fd481f04cb5ad498a2a3f227e14 + checksum: d30237d9d079c40e1a05f9d13f643c7036cfa2c8cb22a9b0471579421a522ed3728e42e848a34a4456b22b9879af30b3747bed7dd27ef32407437c1921c24957 languageName: node linkType: hard @@ -12195,10 +12187,10 @@ __metadata: languageName: node linkType: hard -"tslib@npm:2.7.0": - version: 2.7.0 - resolution: "tslib@npm:2.7.0" - checksum: 1606d5c89f88d466889def78653f3aab0f88692e80bb2066d090ca6112ae250ec1cfa9dbfaab0d17b60da15a4186e8ec4d893801c67896b277c17374e36e1d28 +"tslib@npm:2.8.1": + version: 2.8.1 + resolution: "tslib@npm:2.8.1" + checksum: e4aba30e632b8c8902b47587fd13345e2827fa639e7c3121074d5ee0880723282411a8838f830b55100cbe4517672f84a2472667d355b81e8af165a55dc6203a languageName: node linkType: hard @@ -12619,14 +12611,14 @@ __metadata: languageName: node linkType: hard -"usehooks-ts@npm:2.16.0": - version: 2.16.0 - resolution: "usehooks-ts@npm:2.16.0" +"usehooks-ts@npm:3.1.0": + version: 3.1.0 + resolution: "usehooks-ts@npm:3.1.0" dependencies: lodash.debounce: ^4.0.8 peerDependencies: react: ^16.8.0 || ^17 || ^18 - checksum: 43f23923dd0ea4bf4401cada035301572ea3f251ec045a48640180255437c0c5424edf71a24666ff9ceafbc6adc39b0faf7000eab673e84411868165740f0906 + checksum: 4f850c0c5ab408afa52fa2ea2c93c488cd7065c82679eb1fb62cba12ca4c57ff62d52375acc6738823421fe6579ce3adcea1e2dc345ce4f549c593d2e51455b3 languageName: node linkType: hard