diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index 01dc112abb..afd5dd8444 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -939,6 +939,25 @@ if (is_ios || is_mac) { ] } + rtc_library("audiorendereradapter_objc") { + visibility = [ "*" ] + sources = [ + "objc/api/RTCAudioRendererAdapter+Private.h", + "objc/api/RTCAudioRendererAdapter.h", + "objc/api/RTCAudioRendererAdapter.mm", + ] + + configs += [ "..:common_objc" ] + public_configs = [ ":common_config_objc" ] + + deps = [ + ":base_objc", + ":native_api", + "../api:libjingle_peerconnection_api", + "../api:media_stream_interface", + ] + } + rtc_library("mediasource_objc") { sources = [ "objc/api/peerconnection/RTCMediaSource+Private.h", @@ -1150,6 +1169,7 @@ if (is_ios || is_mac) { ":objc_audio_device_module", ":videoframebuffer_objc", ":videorendereradapter_objc", + ":audiorendereradapter_objc", ":videosource_objc", ":videotoolbox_objc", "../api/crypto:frame_crypto_transformer", diff --git a/sdk/objc/api/RTCAudioRendererAdapter+Private.h b/sdk/objc/api/RTCAudioRendererAdapter+Private.h new file mode 100644 index 0000000000..8a914138de --- /dev/null +++ b/sdk/objc/api/RTCAudioRendererAdapter+Private.h @@ -0,0 +1,36 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCAudioRendererAdapter.h" + +#import "base/RTCAudioRenderer.h" + +#include "api/media_stream_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE(RTCAudioRendererAdapter) () + +@property(nonatomic, readonly) id audioRenderer; + +@property(nonatomic, readonly) webrtc::AudioTrackSinkInterface *nativeAudioRenderer; + +- (instancetype)initWithNativeRenderer:(id)audioRenderer + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/RTCAudioRendererAdapter.h b/sdk/objc/api/RTCAudioRendererAdapter.h new file mode 100644 index 0000000000..5753257182 --- /dev/null +++ b/sdk/objc/api/RTCAudioRendererAdapter.h @@ -0,0 +1,29 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCAudioRendererAdapter): NSObject + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/RTCAudioRendererAdapter.mm b/sdk/objc/api/RTCAudioRendererAdapter.mm new file mode 100644 index 0000000000..4788522ead --- /dev/null +++ b/sdk/objc/api/RTCAudioRendererAdapter.mm @@ -0,0 +1,82 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCAudioRendererAdapter+Private.h" + +#include + +namespace webrtc { + +class AudioRendererAdapter : public webrtc::AudioTrackSinkInterface { + public: + AudioRendererAdapter(RTC_OBJC_TYPE(RTCAudioRendererAdapter) * adapter) { adapter_ = adapter; } + + private: + __weak RTC_OBJC_TYPE(RTCAudioRendererAdapter) * adapter_; + + void OnData(const void *audio_data, int bits_per_sample, int sample_rate, + size_t number_of_channels, size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) override { + // Create AVAudioFormat + AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 + sampleRate:sample_rate + channels:number_of_channels + interleaved:YES]; + + // Calculate the buffer length in frames + AVAudioFrameCount frameCount = (AVAudioFrameCount)number_of_frames; + + // Create AVAudioPCMBuffer + AVAudioPCMBuffer *pcmBuffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format + frameCapacity:frameCount]; + + // Ensure we have enough space in the buffer + if (pcmBuffer == nil) { + NSLog(@"Failed to create AVAudioPCMBuffer"); + return; + } + + pcmBuffer.frameLength = frameCount; + + // Copy the audio data to the PCM buffer + memcpy(pcmBuffer.int16ChannelData[0], audio_data, + number_of_frames * sizeof(int16_t) * number_of_channels); + + [adapter_.audioRenderer renderPCMBuffer:pcmBuffer]; + } +}; +} // namespace webrtc + +@implementation RTC_OBJC_TYPE (RTCAudioRendererAdapter) { + std::unique_ptr _adapter; +} + +@synthesize audioRenderer = _audioRenderer; + +- (instancetype)initWithNativeRenderer:(id)audioRenderer { + NSParameterAssert(audioRenderer); + if (self = [super init]) { + _audioRenderer = audioRenderer; + _adapter.reset(new webrtc::AudioRendererAdapter(self)); + } + return self; +} + +- (webrtc::AudioTrackSinkInterface *)nativeAudioRenderer { + return _adapter.get(); +} + +@end diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h index 38c0bd3b1b..731bc1d56d 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h +++ b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h @@ -26,8 +26,6 @@ NS_ASSUME_NONNULL_BEGIN source:(RTC_OBJC_TYPE(RTCAudioSource) *)source trackId:(NSString *)trackId; -- (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer; - @end NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index d79972eae1..48a4f31a9a 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -17,173 +17,14 @@ #import "RTCAudioSource+Private.h" #import "RTCMediaStreamTrack+Private.h" #import "RTCPeerConnectionFactory+Private.h" +#import "api/RTCAudioRendererAdapter+Private.h" #import "helpers/NSString+StdString.h" #include "rtc_base/checks.h" -namespace webrtc { -/** - * Captures audio data and converts to CMSampleBuffers - */ -class AudioSinkConverter : public rtc::RefCountInterface, public webrtc::AudioTrackSinkInterface { - private: - os_unfair_lock *lock_; - __weak RTC_OBJC_TYPE(RTCAudioTrack) *audio_track_; - int64_t total_frames_ = 0; - bool attached_ = false; - - public: - AudioSinkConverter(RTC_OBJC_TYPE(RTCAudioTrack) *audioTrack, os_unfair_lock *lock) { - RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter init"; - audio_track_ = audioTrack; - lock_ = lock; - } - - ~AudioSinkConverter() { - // - RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter dealloc"; - } - - // Must be called while locked - void TryAttach() { - if (attached_) { - // Already attached - return; - } - RTC_LOG(LS_INFO) << "RTCAudioTrack attaching sink..."; - // Reset for creating CMSampleTimingInfo correctly - audio_track_.nativeAudioTrack->AddSink(this); - total_frames_ = 0; - attached_ = true; - } - - // Must be called while locked - void TryDetach() { - if (!attached_) { - // Already detached - return; - } - RTC_LOG(LS_INFO) << "RTCAudioTrack detaching sink..."; - audio_track_.nativeAudioTrack->RemoveSink(this); - attached_ = false; - } - - void OnData(const void *audio_data, - int bits_per_sample, - int sample_rate, - size_t number_of_channels, - size_t number_of_frames, - absl::optional absolute_capture_timestamp_ms) override { - RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData bits_per_sample: " - << bits_per_sample << " sample_rate: " << sample_rate - << " number_of_channels: " << number_of_channels - << " number_of_frames: " << number_of_frames - << " absolute_capture_timestamp_ms: " - << (absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : 0); - - bool is_locked = os_unfair_lock_trylock(lock_); - if (!is_locked) { - RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData already locked, skipping..."; - return; - } - bool is_attached = attached_; - os_unfair_lock_unlock(lock_); - - if (!is_attached) { - RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData already detached, skipping..."; - return; - } - - /* - * Convert to CMSampleBuffer - */ - - if (!(number_of_channels == 1 || number_of_channels == 2)) { - NSLog(@"RTCAudioTrack: Only mono or stereo is supported currently. numberOfChannels: %zu", - number_of_channels); - return; - } - - OSStatus status; - - AudioChannelLayout acl; - bzero(&acl, sizeof(acl)); - acl.mChannelLayoutTag = - number_of_channels == 2 ? kAudioChannelLayoutTag_Stereo : kAudioChannelLayoutTag_Mono; - - AudioStreamBasicDescription sd; - sd.mSampleRate = sample_rate; - sd.mFormatID = kAudioFormatLinearPCM; - sd.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; - sd.mFramesPerPacket = 1; - sd.mChannelsPerFrame = number_of_channels; - sd.mBitsPerChannel = bits_per_sample; /* 16 */ - sd.mBytesPerFrame = sd.mChannelsPerFrame * (sd.mBitsPerChannel / 8); - sd.mBytesPerPacket = sd.mBytesPerFrame; - - CMSampleTimingInfo timing = { - CMTimeMake(1, sample_rate), - CMTimeMake(total_frames_, sample_rate), - kCMTimeInvalid, - }; - - total_frames_ += number_of_frames; // update the total - - CMFormatDescriptionRef format = NULL; - status = CMAudioFormatDescriptionCreate( - kCFAllocatorDefault, &sd, sizeof(acl), &acl, 0, NULL, NULL, &format); - - if (status != 0) { - NSLog(@"RTCAudioTrack: Failed to create audio format description"); - return; - } - - CMSampleBufferRef buffer; - status = CMSampleBufferCreate(kCFAllocatorDefault, - NULL, - false, - NULL, - NULL, - format, - (CMItemCount)number_of_frames, - 1, - &timing, - 0, - NULL, - &buffer); - // format is no longer required - CFRelease(format); - - if (status != 0) { - NSLog(@"RTCAudioTrack: Failed to allocate sample buffer"); - return; - } - - AudioBufferList bufferList; - bufferList.mNumberBuffers = 1; - bufferList.mBuffers[0].mNumberChannels = sd.mChannelsPerFrame; - bufferList.mBuffers[0].mDataByteSize = (UInt32)(number_of_frames * sd.mBytesPerFrame); - bufferList.mBuffers[0].mData = (void *)audio_data; - status = CMSampleBufferSetDataBufferFromAudioBufferList( - buffer, kCFAllocatorDefault, kCFAllocatorDefault, 0, &bufferList); - if (status != 0) { - NSLog(@"RTCAudioTrack: Failed to convert audio buffer list into sample buffer"); - return; - } - - // Report back to RTCAudioTrack - [audio_track_ didCaptureSampleBuffer:buffer]; - - CFRelease(buffer); - } -}; -} // namespace webrtc - @implementation RTC_OBJC_TYPE (RTCAudioTrack) { - rtc::scoped_refptr _audioConverter; - // Stores weak references to renderers - NSHashTable *_renderers; - os_unfair_lock _lock; + rtc::Thread *_workerThread; + NSMutableArray *_adapters; } @synthesize source = _source; @@ -201,6 +42,7 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto if (self = [self initWithFactory:factory nativeTrack:track type:RTCMediaStreamTrackTypeAudio]) { _source = source; } + return self; } @@ -211,20 +53,17 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto NSParameterAssert(nativeTrack); NSParameterAssert(type == RTCMediaStreamTrackTypeAudio); if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) { - RTC_LOG(LS_INFO) << "RTCAudioTrack init"; - _renderers = [NSHashTable weakObjectsHashTable]; - _audioConverter = new rtc::RefCountedObject(self, &_lock); + _adapters = [NSMutableArray array]; + _workerThread = factory.workerThread; } return self; } - (void)dealloc { - os_unfair_lock_lock(&_lock); - _audioConverter->TryDetach(); - os_unfair_lock_unlock(&_lock); - - RTC_LOG(LS_INFO) << "RTCAudioTrack dealloc"; + for (RTC_OBJC_TYPE(RTCAudioRendererAdapter) * adapter in _adapters) { + self.nativeAudioTrack->RemoveSink(adapter.nativeAudioRenderer); + } } - (RTC_OBJC_TYPE(RTCAudioSource) *)source { @@ -239,22 +78,45 @@ - (void)dealloc { } - (void)addRenderer:(id)renderer { - os_unfair_lock_lock(&_lock); - [_renderers addObject:renderer]; - _audioConverter->TryAttach(); - os_unfair_lock_unlock(&_lock); + if (!_workerThread->IsCurrent()) { + _workerThread->BlockingCall([renderer, self] { [self addRenderer:renderer]; }); + return; + } + + // Make sure we don't have this renderer yet. + for (RTC_OBJC_TYPE(RTCAudioRendererAdapter) * adapter in _adapters) { + if (adapter.audioRenderer == renderer) { + RTC_LOG(LS_INFO) << "|renderer| is already attached to this track"; + return; + } + } + // Create a wrapper that provides a native pointer for us. + RTC_OBJC_TYPE(RTCAudioRendererAdapter) *adapter = + [[RTC_OBJC_TYPE(RTCAudioRendererAdapter) alloc] initWithNativeRenderer:renderer]; + [_adapters addObject:adapter]; + self.nativeAudioTrack->AddSink(adapter.nativeAudioRenderer); } - (void)removeRenderer:(id)renderer { - os_unfair_lock_lock(&_lock); - [_renderers removeObject:renderer]; - NSUInteger renderersCount = _renderers.allObjects.count; - - if (renderersCount == 0) { - // Detach if no more renderers... - _audioConverter->TryDetach(); + if (!_workerThread->IsCurrent()) { + _workerThread->BlockingCall([renderer, self] { [self removeRenderer:renderer]; }); + return; + } + __block NSUInteger indexToRemove = NSNotFound; + [_adapters enumerateObjectsUsingBlock:^(RTC_OBJC_TYPE(RTCAudioRendererAdapter) * adapter, + NSUInteger idx, BOOL * stop) { + if (adapter.audioRenderer == renderer) { + indexToRemove = idx; + *stop = YES; + } + }]; + if (indexToRemove == NSNotFound) { + RTC_LOG(LS_INFO) << "removeRenderer called with a renderer that has not been previously added"; + return; } - os_unfair_lock_unlock(&_lock); + RTC_OBJC_TYPE(RTCAudioRendererAdapter) *adapterToRemove = [_adapters objectAtIndex:indexToRemove]; + self.nativeAudioTrack->RemoveSink(adapterToRemove.nativeAudioRenderer); + [_adapters removeObjectAtIndex:indexToRemove]; } #pragma mark - Private @@ -264,18 +126,4 @@ - (void)removeRenderer:(id)renderer { static_cast(self.nativeTrack.get())); } -- (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer { - bool is_locked = os_unfair_lock_trylock(&_lock); - if (!is_locked) { - RTC_LOG(LS_INFO) << "RTCAudioTrack didCaptureSampleBuffer already locked, skipping..."; - return; - } - NSArray *renderers = [_renderers allObjects]; - os_unfair_lock_unlock(&_lock); - - for (id renderer in renderers) { - [renderer renderSampleBuffer:sampleBuffer]; - } -} - @end diff --git a/sdk/objc/base/RTCAudioRenderer.h b/sdk/objc/base/RTCAudioRenderer.h index 3669831fca..73bad7d39c 100644 --- a/sdk/objc/base/RTCAudioRenderer.h +++ b/sdk/objc/base/RTCAudioRenderer.h @@ -15,6 +15,9 @@ */ #import +#import +#import + #if TARGET_OS_IPHONE #import #endif @@ -25,7 +28,7 @@ NS_ASSUME_NONNULL_BEGIN RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE(RTCAudioRenderer) -- (void)renderSampleBuffer: (CMSampleBufferRef)sampleBuffer NS_SWIFT_NAME(render(sampleBuffer:)); +- (void)renderPCMBuffer: (AVAudioPCMBuffer *)pcmBuffer NS_SWIFT_NAME(render(pcmBuffer:)); @end