From 8eb35325891fb5cbd374e73e3c59884bde636ad3 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:34:48 +0900 Subject: [PATCH 1/7] Fix --- sdk/BUILD.gn | 20 ++ .../api/RTCAudioRendererAdapter+Private.h | 30 +++ sdk/objc/api/RTCAudioRendererAdapter.h | 23 ++ sdk/objc/api/RTCAudioRendererAdapter.mm | 121 +++++++++ .../peerconnection/RTCAudioTrack+Private.h | 2 - sdk/objc/api/peerconnection/RTCAudioTrack.mm | 240 ++++-------------- sdk/objc/base/RTCAudioRenderer.h | 2 + 7 files changed, 240 insertions(+), 198 deletions(-) create mode 100644 sdk/objc/api/RTCAudioRendererAdapter+Private.h create mode 100644 sdk/objc/api/RTCAudioRendererAdapter.h create mode 100644 sdk/objc/api/RTCAudioRendererAdapter.mm diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index 01dc112abb..afd5dd8444 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -939,6 +939,25 @@ if (is_ios || is_mac) { ] } + rtc_library("audiorendereradapter_objc") { + visibility = [ "*" ] + sources = [ + "objc/api/RTCAudioRendererAdapter+Private.h", + "objc/api/RTCAudioRendererAdapter.h", + "objc/api/RTCAudioRendererAdapter.mm", + ] + + configs += [ "..:common_objc" ] + public_configs = [ ":common_config_objc" ] + + deps = [ + ":base_objc", + ":native_api", + "../api:libjingle_peerconnection_api", + "../api:media_stream_interface", + ] + } + rtc_library("mediasource_objc") { sources = [ "objc/api/peerconnection/RTCMediaSource+Private.h", @@ -1150,6 +1169,7 @@ if (is_ios || is_mac) { ":objc_audio_device_module", ":videoframebuffer_objc", ":videorendereradapter_objc", + ":audiorendereradapter_objc", ":videosource_objc", ":videotoolbox_objc", "../api/crypto:frame_crypto_transformer", diff --git a/sdk/objc/api/RTCAudioRendererAdapter+Private.h b/sdk/objc/api/RTCAudioRendererAdapter+Private.h new file mode 100644 index 0000000000..a6a0c27e1a --- /dev/null +++ b/sdk/objc/api/RTCAudioRendererAdapter+Private.h @@ -0,0 +1,30 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioRendererAdapter.h" + +#import "base/RTCAudioRenderer.h" + +#include "api/media_stream_interface.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE(RTCAudioRendererAdapter) () + +@property(nonatomic, readonly) id audioRenderer; + +@property(nonatomic, readonly) webrtc::AudioTrackSinkInterface *nativeAudioRenderer; + +- (instancetype)initWithNativeRenderer:(id)audioRenderer + NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/RTCAudioRendererAdapter.h b/sdk/objc/api/RTCAudioRendererAdapter.h new file mode 100644 index 0000000000..240279e198 --- /dev/null +++ b/sdk/objc/api/RTCAudioRendererAdapter.h @@ -0,0 +1,23 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCAudioRendererAdapter): NSObject + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/RTCAudioRendererAdapter.mm b/sdk/objc/api/RTCAudioRendererAdapter.mm new file mode 100644 index 0000000000..e99823c17d --- /dev/null +++ b/sdk/objc/api/RTCAudioRendererAdapter.mm @@ -0,0 +1,121 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCAudioRendererAdapter+Private.h" + +#include + +namespace webrtc { + +class AudioRendererAdapter : public webrtc::AudioTrackSinkInterface { + public: + AudioRendererAdapter(RTC_OBJC_TYPE(RTCAudioRendererAdapter) * adapter) { adapter_ = adapter; } + + private: + __weak RTC_OBJC_TYPE(RTCAudioRendererAdapter) * adapter_; + int64_t total_frames_ = 0; + + void OnData(const void *audio_data, int bits_per_sample, int sample_rate, + size_t number_of_channels, size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) override { + /* + * Convert to CMSampleBuffer + */ + + if (!(number_of_channels == 1 || number_of_channels == 2)) { + NSLog(@"RTCAudioTrack: Only mono or stereo is supported currently. numberOfChannels: %zu", + number_of_channels); + return; + } + + OSStatus status; + + AudioChannelLayout acl; + bzero(&acl, sizeof(acl)); + acl.mChannelLayoutTag = + number_of_channels == 2 ? kAudioChannelLayoutTag_Stereo : kAudioChannelLayoutTag_Mono; + + AudioStreamBasicDescription sd; + sd.mSampleRate = sample_rate; + sd.mFormatID = kAudioFormatLinearPCM; + sd.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; + sd.mFramesPerPacket = 1; + sd.mChannelsPerFrame = number_of_channels; + sd.mBitsPerChannel = bits_per_sample; /* 16 */ + sd.mBytesPerFrame = sd.mChannelsPerFrame * (sd.mBitsPerChannel / 8); + sd.mBytesPerPacket = sd.mBytesPerFrame; + + CMSampleTimingInfo timing = { + CMTimeMake(1, sample_rate), + CMTimeMake(total_frames_, sample_rate), + kCMTimeInvalid, + }; + + total_frames_ += number_of_frames; // update the total + + CMFormatDescriptionRef format = NULL; + status = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &sd, sizeof(acl), &acl, 0, NULL, + NULL, &format); + + if (status != 0) { + NSLog(@"RTCAudioTrack: Failed to create audio format description"); + return; + } + + CMSampleBufferRef buffer; + status = CMSampleBufferCreate(kCFAllocatorDefault, NULL, false, NULL, NULL, format, + (CMItemCount)number_of_frames, 1, &timing, 0, NULL, &buffer); + // format is no longer required + CFRelease(format); + + if (status != 0) { + NSLog(@"RTCAudioTrack: Failed to allocate sample buffer"); + return; + } + + AudioBufferList bufferList; + bufferList.mNumberBuffers = 1; + bufferList.mBuffers[0].mNumberChannels = sd.mChannelsPerFrame; + bufferList.mBuffers[0].mDataByteSize = (UInt32)(number_of_frames * sd.mBytesPerFrame); + bufferList.mBuffers[0].mData = (void *)audio_data; + status = CMSampleBufferSetDataBufferFromAudioBufferList(buffer, kCFAllocatorDefault, + kCFAllocatorDefault, 0, &bufferList); + if (status != 0) { + NSLog(@"RTCAudioTrack: Failed to convert audio buffer list into sample buffer"); + return; + } + + // Report back to RTCAudioTrack + [adapter_.audioRenderer renderSampleBuffer:buffer]; + CFRelease(buffer); + } +}; +} // namespace webrtc + +@implementation RTC_OBJC_TYPE (RTCAudioRendererAdapter) { + std::unique_ptr _adapter; +} + +@synthesize audioRenderer = _audioRenderer; + +- (instancetype)initWithNativeRenderer:(id)audioRenderer { + NSParameterAssert(audioRenderer); + if (self = [super init]) { + _audioRenderer = audioRenderer; + _adapter.reset(new webrtc::AudioRendererAdapter(self)); + } + return self; +} + +- (webrtc::AudioTrackSinkInterface *)nativeAudioRenderer { + return _adapter.get(); +} + +@end diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h index 38c0bd3b1b..731bc1d56d 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h +++ b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h @@ -26,8 +26,6 @@ NS_ASSUME_NONNULL_BEGIN source:(RTC_OBJC_TYPE(RTCAudioSource) *)source trackId:(NSString *)trackId; -- (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer; - @end NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index d79972eae1..48a4f31a9a 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -17,173 +17,14 @@ #import "RTCAudioSource+Private.h" #import "RTCMediaStreamTrack+Private.h" #import "RTCPeerConnectionFactory+Private.h" +#import "api/RTCAudioRendererAdapter+Private.h" #import "helpers/NSString+StdString.h" #include "rtc_base/checks.h" -namespace webrtc { -/** - * Captures audio data and converts to CMSampleBuffers - */ -class AudioSinkConverter : public rtc::RefCountInterface, public webrtc::AudioTrackSinkInterface { - private: - os_unfair_lock *lock_; - __weak RTC_OBJC_TYPE(RTCAudioTrack) *audio_track_; - int64_t total_frames_ = 0; - bool attached_ = false; - - public: - AudioSinkConverter(RTC_OBJC_TYPE(RTCAudioTrack) *audioTrack, os_unfair_lock *lock) { - RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter init"; - audio_track_ = audioTrack; - lock_ = lock; - } - - ~AudioSinkConverter() { - // - RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter dealloc"; - } - - // Must be called while locked - void TryAttach() { - if (attached_) { - // Already attached - return; - } - RTC_LOG(LS_INFO) << "RTCAudioTrack attaching sink..."; - // Reset for creating CMSampleTimingInfo correctly - audio_track_.nativeAudioTrack->AddSink(this); - total_frames_ = 0; - attached_ = true; - } - - // Must be called while locked - void TryDetach() { - if (!attached_) { - // Already detached - return; - } - RTC_LOG(LS_INFO) << "RTCAudioTrack detaching sink..."; - audio_track_.nativeAudioTrack->RemoveSink(this); - attached_ = false; - } - - void OnData(const void *audio_data, - int bits_per_sample, - int sample_rate, - size_t number_of_channels, - size_t number_of_frames, - absl::optional absolute_capture_timestamp_ms) override { - RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData bits_per_sample: " - << bits_per_sample << " sample_rate: " << sample_rate - << " number_of_channels: " << number_of_channels - << " number_of_frames: " << number_of_frames - << " absolute_capture_timestamp_ms: " - << (absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : 0); - - bool is_locked = os_unfair_lock_trylock(lock_); - if (!is_locked) { - RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData already locked, skipping..."; - return; - } - bool is_attached = attached_; - os_unfair_lock_unlock(lock_); - - if (!is_attached) { - RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData already detached, skipping..."; - return; - } - - /* - * Convert to CMSampleBuffer - */ - - if (!(number_of_channels == 1 || number_of_channels == 2)) { - NSLog(@"RTCAudioTrack: Only mono or stereo is supported currently. numberOfChannels: %zu", - number_of_channels); - return; - } - - OSStatus status; - - AudioChannelLayout acl; - bzero(&acl, sizeof(acl)); - acl.mChannelLayoutTag = - number_of_channels == 2 ? kAudioChannelLayoutTag_Stereo : kAudioChannelLayoutTag_Mono; - - AudioStreamBasicDescription sd; - sd.mSampleRate = sample_rate; - sd.mFormatID = kAudioFormatLinearPCM; - sd.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; - sd.mFramesPerPacket = 1; - sd.mChannelsPerFrame = number_of_channels; - sd.mBitsPerChannel = bits_per_sample; /* 16 */ - sd.mBytesPerFrame = sd.mChannelsPerFrame * (sd.mBitsPerChannel / 8); - sd.mBytesPerPacket = sd.mBytesPerFrame; - - CMSampleTimingInfo timing = { - CMTimeMake(1, sample_rate), - CMTimeMake(total_frames_, sample_rate), - kCMTimeInvalid, - }; - - total_frames_ += number_of_frames; // update the total - - CMFormatDescriptionRef format = NULL; - status = CMAudioFormatDescriptionCreate( - kCFAllocatorDefault, &sd, sizeof(acl), &acl, 0, NULL, NULL, &format); - - if (status != 0) { - NSLog(@"RTCAudioTrack: Failed to create audio format description"); - return; - } - - CMSampleBufferRef buffer; - status = CMSampleBufferCreate(kCFAllocatorDefault, - NULL, - false, - NULL, - NULL, - format, - (CMItemCount)number_of_frames, - 1, - &timing, - 0, - NULL, - &buffer); - // format is no longer required - CFRelease(format); - - if (status != 0) { - NSLog(@"RTCAudioTrack: Failed to allocate sample buffer"); - return; - } - - AudioBufferList bufferList; - bufferList.mNumberBuffers = 1; - bufferList.mBuffers[0].mNumberChannels = sd.mChannelsPerFrame; - bufferList.mBuffers[0].mDataByteSize = (UInt32)(number_of_frames * sd.mBytesPerFrame); - bufferList.mBuffers[0].mData = (void *)audio_data; - status = CMSampleBufferSetDataBufferFromAudioBufferList( - buffer, kCFAllocatorDefault, kCFAllocatorDefault, 0, &bufferList); - if (status != 0) { - NSLog(@"RTCAudioTrack: Failed to convert audio buffer list into sample buffer"); - return; - } - - // Report back to RTCAudioTrack - [audio_track_ didCaptureSampleBuffer:buffer]; - - CFRelease(buffer); - } -}; -} // namespace webrtc - @implementation RTC_OBJC_TYPE (RTCAudioTrack) { - rtc::scoped_refptr _audioConverter; - // Stores weak references to renderers - NSHashTable *_renderers; - os_unfair_lock _lock; + rtc::Thread *_workerThread; + NSMutableArray *_adapters; } @synthesize source = _source; @@ -201,6 +42,7 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto if (self = [self initWithFactory:factory nativeTrack:track type:RTCMediaStreamTrackTypeAudio]) { _source = source; } + return self; } @@ -211,20 +53,17 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto NSParameterAssert(nativeTrack); NSParameterAssert(type == RTCMediaStreamTrackTypeAudio); if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) { - RTC_LOG(LS_INFO) << "RTCAudioTrack init"; - _renderers = [NSHashTable weakObjectsHashTable]; - _audioConverter = new rtc::RefCountedObject(self, &_lock); + _adapters = [NSMutableArray array]; + _workerThread = factory.workerThread; } return self; } - (void)dealloc { - os_unfair_lock_lock(&_lock); - _audioConverter->TryDetach(); - os_unfair_lock_unlock(&_lock); - - RTC_LOG(LS_INFO) << "RTCAudioTrack dealloc"; + for (RTC_OBJC_TYPE(RTCAudioRendererAdapter) * adapter in _adapters) { + self.nativeAudioTrack->RemoveSink(adapter.nativeAudioRenderer); + } } - (RTC_OBJC_TYPE(RTCAudioSource) *)source { @@ -239,22 +78,45 @@ - (void)dealloc { } - (void)addRenderer:(id)renderer { - os_unfair_lock_lock(&_lock); - [_renderers addObject:renderer]; - _audioConverter->TryAttach(); - os_unfair_lock_unlock(&_lock); + if (!_workerThread->IsCurrent()) { + _workerThread->BlockingCall([renderer, self] { [self addRenderer:renderer]; }); + return; + } + + // Make sure we don't have this renderer yet. + for (RTC_OBJC_TYPE(RTCAudioRendererAdapter) * adapter in _adapters) { + if (adapter.audioRenderer == renderer) { + RTC_LOG(LS_INFO) << "|renderer| is already attached to this track"; + return; + } + } + // Create a wrapper that provides a native pointer for us. + RTC_OBJC_TYPE(RTCAudioRendererAdapter) *adapter = + [[RTC_OBJC_TYPE(RTCAudioRendererAdapter) alloc] initWithNativeRenderer:renderer]; + [_adapters addObject:adapter]; + self.nativeAudioTrack->AddSink(adapter.nativeAudioRenderer); } - (void)removeRenderer:(id)renderer { - os_unfair_lock_lock(&_lock); - [_renderers removeObject:renderer]; - NSUInteger renderersCount = _renderers.allObjects.count; - - if (renderersCount == 0) { - // Detach if no more renderers... - _audioConverter->TryDetach(); + if (!_workerThread->IsCurrent()) { + _workerThread->BlockingCall([renderer, self] { [self removeRenderer:renderer]; }); + return; + } + __block NSUInteger indexToRemove = NSNotFound; + [_adapters enumerateObjectsUsingBlock:^(RTC_OBJC_TYPE(RTCAudioRendererAdapter) * adapter, + NSUInteger idx, BOOL * stop) { + if (adapter.audioRenderer == renderer) { + indexToRemove = idx; + *stop = YES; + } + }]; + if (indexToRemove == NSNotFound) { + RTC_LOG(LS_INFO) << "removeRenderer called with a renderer that has not been previously added"; + return; } - os_unfair_lock_unlock(&_lock); + RTC_OBJC_TYPE(RTCAudioRendererAdapter) *adapterToRemove = [_adapters objectAtIndex:indexToRemove]; + self.nativeAudioTrack->RemoveSink(adapterToRemove.nativeAudioRenderer); + [_adapters removeObjectAtIndex:indexToRemove]; } #pragma mark - Private @@ -264,18 +126,4 @@ - (void)removeRenderer:(id)renderer { static_cast(self.nativeTrack.get())); } -- (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer { - bool is_locked = os_unfair_lock_trylock(&_lock); - if (!is_locked) { - RTC_LOG(LS_INFO) << "RTCAudioTrack didCaptureSampleBuffer already locked, skipping..."; - return; - } - NSArray *renderers = [_renderers allObjects]; - os_unfair_lock_unlock(&_lock); - - for (id renderer in renderers) { - [renderer renderSampleBuffer:sampleBuffer]; - } -} - @end diff --git a/sdk/objc/base/RTCAudioRenderer.h b/sdk/objc/base/RTCAudioRenderer.h index 3669831fca..aabbbececb 100644 --- a/sdk/objc/base/RTCAudioRenderer.h +++ b/sdk/objc/base/RTCAudioRenderer.h @@ -15,6 +15,8 @@ */ #import +#import + #if TARGET_OS_IPHONE #import #endif From cd0a3b817ce87a48437214816921492843f33094 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:40:02 +0900 Subject: [PATCH 2/7] License for new files --- sdk/objc/api/RTCAudioRendererAdapter+Private.h | 18 ++++++++++++------ sdk/objc/api/RTCAudioRendererAdapter.h | 18 ++++++++++++------ sdk/objc/api/RTCAudioRendererAdapter.mm | 18 ++++++++++++------ 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/sdk/objc/api/RTCAudioRendererAdapter+Private.h b/sdk/objc/api/RTCAudioRendererAdapter+Private.h index a6a0c27e1a..8a914138de 100644 --- a/sdk/objc/api/RTCAudioRendererAdapter+Private.h +++ b/sdk/objc/api/RTCAudioRendererAdapter+Private.h @@ -1,11 +1,17 @@ /* - * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * Copyright 2024 LiveKit * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #import "RTCAudioRendererAdapter.h" diff --git a/sdk/objc/api/RTCAudioRendererAdapter.h b/sdk/objc/api/RTCAudioRendererAdapter.h index 240279e198..5753257182 100644 --- a/sdk/objc/api/RTCAudioRendererAdapter.h +++ b/sdk/objc/api/RTCAudioRendererAdapter.h @@ -1,11 +1,17 @@ /* - * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * Copyright 2024 LiveKit * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #import diff --git a/sdk/objc/api/RTCAudioRendererAdapter.mm b/sdk/objc/api/RTCAudioRendererAdapter.mm index e99823c17d..5fd7a0a012 100644 --- a/sdk/objc/api/RTCAudioRendererAdapter.mm +++ b/sdk/objc/api/RTCAudioRendererAdapter.mm @@ -1,11 +1,17 @@ /* - * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * Copyright 2024 LiveKit * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #import "RTCAudioRendererAdapter+Private.h" From 6ea932ae1b33426f0a476c1d8abaf216b2fa1a94 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:59:35 +0900 Subject: [PATCH 3/7] PCM --- sdk/objc/api/RTCAudioRendererAdapter.mm | 87 ++++++------------------- sdk/objc/base/RTCAudioRenderer.h | 3 +- 2 files changed, 23 insertions(+), 67 deletions(-) diff --git a/sdk/objc/api/RTCAudioRendererAdapter.mm b/sdk/objc/api/RTCAudioRendererAdapter.mm index 5fd7a0a012..4788522ead 100644 --- a/sdk/objc/api/RTCAudioRendererAdapter.mm +++ b/sdk/objc/api/RTCAudioRendererAdapter.mm @@ -26,81 +26,36 @@ private: __weak RTC_OBJC_TYPE(RTCAudioRendererAdapter) * adapter_; - int64_t total_frames_ = 0; void OnData(const void *audio_data, int bits_per_sample, int sample_rate, size_t number_of_channels, size_t number_of_frames, absl::optional absolute_capture_timestamp_ms) override { - /* - * Convert to CMSampleBuffer - */ - - if (!(number_of_channels == 1 || number_of_channels == 2)) { - NSLog(@"RTCAudioTrack: Only mono or stereo is supported currently. numberOfChannels: %zu", - number_of_channels); + // Create AVAudioFormat + AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 + sampleRate:sample_rate + channels:number_of_channels + interleaved:YES]; + + // Calculate the buffer length in frames + AVAudioFrameCount frameCount = (AVAudioFrameCount)number_of_frames; + + // Create AVAudioPCMBuffer + AVAudioPCMBuffer *pcmBuffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format + frameCapacity:frameCount]; + + // Ensure we have enough space in the buffer + if (pcmBuffer == nil) { + NSLog(@"Failed to create AVAudioPCMBuffer"); return; } - OSStatus status; - - AudioChannelLayout acl; - bzero(&acl, sizeof(acl)); - acl.mChannelLayoutTag = - number_of_channels == 2 ? kAudioChannelLayoutTag_Stereo : kAudioChannelLayoutTag_Mono; - - AudioStreamBasicDescription sd; - sd.mSampleRate = sample_rate; - sd.mFormatID = kAudioFormatLinearPCM; - sd.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; - sd.mFramesPerPacket = 1; - sd.mChannelsPerFrame = number_of_channels; - sd.mBitsPerChannel = bits_per_sample; /* 16 */ - sd.mBytesPerFrame = sd.mChannelsPerFrame * (sd.mBitsPerChannel / 8); - sd.mBytesPerPacket = sd.mBytesPerFrame; - - CMSampleTimingInfo timing = { - CMTimeMake(1, sample_rate), - CMTimeMake(total_frames_, sample_rate), - kCMTimeInvalid, - }; - - total_frames_ += number_of_frames; // update the total - - CMFormatDescriptionRef format = NULL; - status = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &sd, sizeof(acl), &acl, 0, NULL, - NULL, &format); - - if (status != 0) { - NSLog(@"RTCAudioTrack: Failed to create audio format description"); - return; - } + pcmBuffer.frameLength = frameCount; - CMSampleBufferRef buffer; - status = CMSampleBufferCreate(kCFAllocatorDefault, NULL, false, NULL, NULL, format, - (CMItemCount)number_of_frames, 1, &timing, 0, NULL, &buffer); - // format is no longer required - CFRelease(format); - - if (status != 0) { - NSLog(@"RTCAudioTrack: Failed to allocate sample buffer"); - return; - } - - AudioBufferList bufferList; - bufferList.mNumberBuffers = 1; - bufferList.mBuffers[0].mNumberChannels = sd.mChannelsPerFrame; - bufferList.mBuffers[0].mDataByteSize = (UInt32)(number_of_frames * sd.mBytesPerFrame); - bufferList.mBuffers[0].mData = (void *)audio_data; - status = CMSampleBufferSetDataBufferFromAudioBufferList(buffer, kCFAllocatorDefault, - kCFAllocatorDefault, 0, &bufferList); - if (status != 0) { - NSLog(@"RTCAudioTrack: Failed to convert audio buffer list into sample buffer"); - return; - } + // Copy the audio data to the PCM buffer + memcpy(pcmBuffer.int16ChannelData[0], audio_data, + number_of_frames * sizeof(int16_t) * number_of_channels); - // Report back to RTCAudioTrack - [adapter_.audioRenderer renderSampleBuffer:buffer]; - CFRelease(buffer); + [adapter_.audioRenderer renderPCMBuffer:pcmBuffer]; } }; } // namespace webrtc diff --git a/sdk/objc/base/RTCAudioRenderer.h b/sdk/objc/base/RTCAudioRenderer.h index aabbbececb..73bad7d39c 100644 --- a/sdk/objc/base/RTCAudioRenderer.h +++ b/sdk/objc/base/RTCAudioRenderer.h @@ -15,6 +15,7 @@ */ #import +#import #import #if TARGET_OS_IPHONE @@ -27,7 +28,7 @@ NS_ASSUME_NONNULL_BEGIN RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE(RTCAudioRenderer) -- (void)renderSampleBuffer: (CMSampleBufferRef)sampleBuffer NS_SWIFT_NAME(render(sampleBuffer:)); +- (void)renderPCMBuffer: (AVAudioPCMBuffer *)pcmBuffer NS_SWIFT_NAME(render(pcmBuffer:)); @end From 06445335654078b1584eb1e44b5f0f79f99aebc0 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sat, 21 Sep 2024 01:07:45 +0900 Subject: [PATCH 4/7] Optimize --- sdk/BUILD.gn | 1 + sdk/objc/api/RTCAudioRendererAdapter.mm | 51 ++++++++++++++++++------- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index afd5dd8444..33bf72df8e 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -1915,6 +1915,7 @@ if (is_ios || is_mac) { "CoreMedia.framework", "CoreVideo.framework", "VideoToolbox.framework", + "Accelerate.framework", ] } } diff --git a/sdk/objc/api/RTCAudioRendererAdapter.mm b/sdk/objc/api/RTCAudioRendererAdapter.mm index 4788522ead..97bc65c7a7 100644 --- a/sdk/objc/api/RTCAudioRendererAdapter.mm +++ b/sdk/objc/api/RTCAudioRendererAdapter.mm @@ -14,6 +14,7 @@ * limitations under the License. */ +#import #import "RTCAudioRendererAdapter+Private.h" #include @@ -30,20 +31,38 @@ void OnData(const void *audio_data, int bits_per_sample, int sample_rate, size_t number_of_channels, size_t number_of_frames, absl::optional absolute_capture_timestamp_ms) override { - // Create AVAudioFormat - AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 - sampleRate:sample_rate - channels:number_of_channels - interleaved:YES]; + OSStatus status; + AudioChannelLayout acl; + bzero(&acl, sizeof(acl)); + acl.mChannelLayoutTag = + number_of_channels == 2 ? kAudioChannelLayoutTag_Stereo : kAudioChannelLayoutTag_Mono; + + // Create AudioStreamBasicDescription for float format + AudioStreamBasicDescription sd; + sd.mSampleRate = sample_rate; + sd.mFormatID = kAudioFormatLinearPCM; + sd.mFormatFlags = kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked; + sd.mFramesPerPacket = 1; + sd.mChannelsPerFrame = number_of_channels; + sd.mBitsPerChannel = 32; // 32-bit float + sd.mBytesPerFrame = sd.mChannelsPerFrame * (sd.mBitsPerChannel / 8); + sd.mBytesPerPacket = sd.mBytesPerFrame; + + CMFormatDescriptionRef format = NULL; + status = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &sd, sizeof(acl), &acl, 0, NULL, + NULL, &format); + if (status != 0) { + NSLog(@"RTCAudioTrack: Failed to create audio format description"); + return; + } - // Calculate the buffer length in frames - AVAudioFrameCount frameCount = (AVAudioFrameCount)number_of_frames; + AVAudioFormat *format2 = [[AVAudioFormat alloc] initWithCMAudioFormatDescription:format]; + CFRelease(format); - // Create AVAudioPCMBuffer - AVAudioPCMBuffer *pcmBuffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format + AVAudioFrameCount frameCount = (AVAudioFrameCount)number_of_frames; + AVAudioPCMBuffer *pcmBuffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format2 frameCapacity:frameCount]; - // Ensure we have enough space in the buffer if (pcmBuffer == nil) { NSLog(@"Failed to create AVAudioPCMBuffer"); return; @@ -51,9 +70,15 @@ void OnData(const void *audio_data, int bits_per_sample, int sample_rate, pcmBuffer.frameLength = frameCount; - // Copy the audio data to the PCM buffer - memcpy(pcmBuffer.int16ChannelData[0], audio_data, - number_of_frames * sizeof(int16_t) * number_of_channels); + const int16_t *inputData = static_cast(audio_data); + float scale = 1.0f / 32768.0f; + + for (size_t channel = 0; channel < number_of_channels; ++channel) { + vDSP_vflt16(inputData + channel * number_of_frames, 1, pcmBuffer.floatChannelData[channel], 1, + frameCount); + vDSP_vsmul(pcmBuffer.floatChannelData[channel], 1, &scale, + pcmBuffer.floatChannelData[channel], 1, frameCount); + } [adapter_.audioRenderer renderPCMBuffer:pcmBuffer]; } From e028994b47c890cbbb5061c86396eba9acaf48f1 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sat, 21 Sep 2024 19:51:30 +0900 Subject: [PATCH 5/7] Optimize --- sdk/objc/api/RTCAudioRendererAdapter.mm | 58 ++++++++++++------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/sdk/objc/api/RTCAudioRendererAdapter.mm b/sdk/objc/api/RTCAudioRendererAdapter.mm index 97bc65c7a7..478d2f5984 100644 --- a/sdk/objc/api/RTCAudioRendererAdapter.mm +++ b/sdk/objc/api/RTCAudioRendererAdapter.mm @@ -32,53 +32,51 @@ void OnData(const void *audio_data, int bits_per_sample, int sample_rate, size_t number_of_channels, size_t number_of_frames, absl::optional absolute_capture_timestamp_ms) override { OSStatus status; - AudioChannelLayout acl; - bzero(&acl, sizeof(acl)); + AudioChannelLayout acl = {}; acl.mChannelLayoutTag = - number_of_channels == 2 ? kAudioChannelLayoutTag_Stereo : kAudioChannelLayoutTag_Mono; - - // Create AudioStreamBasicDescription for float format - AudioStreamBasicDescription sd; - sd.mSampleRate = sample_rate; - sd.mFormatID = kAudioFormatLinearPCM; - sd.mFormatFlags = kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked; - sd.mFramesPerPacket = 1; - sd.mChannelsPerFrame = number_of_channels; - sd.mBitsPerChannel = 32; // 32-bit float - sd.mBytesPerFrame = sd.mChannelsPerFrame * (sd.mBitsPerChannel / 8); - sd.mBytesPerPacket = sd.mBytesPerFrame; - - CMFormatDescriptionRef format = NULL; + (number_of_channels == 2) ? kAudioChannelLayoutTag_Stereo : kAudioChannelLayoutTag_Mono; + + AudioStreamBasicDescription sd = { + .mSampleRate = static_cast(sample_rate), + .mFormatID = kAudioFormatLinearPCM, + .mFormatFlags = kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked, + .mBytesPerPacket = static_cast(number_of_channels * 4), + .mFramesPerPacket = 1, + .mBytesPerFrame = static_cast(number_of_channels * 4), + .mChannelsPerFrame = static_cast(number_of_channels), + .mBitsPerChannel = 32, + .mReserved = 0}; + + CMFormatDescriptionRef format = nullptr; status = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &sd, sizeof(acl), &acl, 0, NULL, NULL, &format); - if (status != 0) { - NSLog(@"RTCAudioTrack: Failed to create audio format description"); + if (status != noErr) { + NSLog(@"RTCAudioTrack: Failed to create audio format description. Error: %d", (int)status); return; } AVAudioFormat *format2 = [[AVAudioFormat alloc] initWithCMAudioFormatDescription:format]; CFRelease(format); - AVAudioFrameCount frameCount = (AVAudioFrameCount)number_of_frames; + AVAudioFrameCount frameCount = static_cast(number_of_frames); AVAudioPCMBuffer *pcmBuffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format2 frameCapacity:frameCount]; - - if (pcmBuffer == nil) { + if (!pcmBuffer) { NSLog(@"Failed to create AVAudioPCMBuffer"); return; } pcmBuffer.frameLength = frameCount; - const int16_t *inputData = static_cast(audio_data); - float scale = 1.0f / 32768.0f; - - for (size_t channel = 0; channel < number_of_channels; ++channel) { - vDSP_vflt16(inputData + channel * number_of_frames, 1, pcmBuffer.floatChannelData[channel], 1, - frameCount); - vDSP_vsmul(pcmBuffer.floatChannelData[channel], 1, &scale, - pcmBuffer.floatChannelData[channel], 1, frameCount); - } + const float scale = 1.0f / 32768.0f; + + dispatch_apply(number_of_channels, dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), + ^(size_t channel) { + vDSP_vflt16(inputData + channel * number_of_frames, 1, + pcmBuffer.floatChannelData[channel], 1, frameCount); + vDSP_vsmul(pcmBuffer.floatChannelData[channel], 1, &scale, + pcmBuffer.floatChannelData[channel], 1, frameCount); + }); [adapter_.audioRenderer renderPCMBuffer:pcmBuffer]; } From 06a79e78afb257dcd74056c4e2e301decd08b370 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 24 Sep 2024 02:53:09 +0900 Subject: [PATCH 6/7] Use signaling thread --- sdk/objc/api/peerconnection/RTCAudioTrack.h | 5 ++- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 32 ++++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.h b/sdk/objc/api/peerconnection/RTCAudioTrack.h index c8218ad926..6505686043 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.h +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.h @@ -24,13 +24,12 @@ RTC_OBJC_EXPORT /** The audio source for this audio track. */ @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCAudioSource) * source; -/** Register a renderer that will receive all audio CMSampleBuffers on this track. - * Does not retain. */ - (void)addRenderer:(id)renderer; -/** Deregister a renderer */ - (void)removeRenderer:(id)renderer; +- (void)removeAllRenderers; + @end NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index 48a4f31a9a..dcdf9d93e8 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -23,7 +23,7 @@ #include "rtc_base/checks.h" @implementation RTC_OBJC_TYPE (RTCAudioTrack) { - rtc::Thread *_workerThread; + rtc::Thread *_signalingThread; NSMutableArray *_adapters; } @@ -54,16 +54,14 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto NSParameterAssert(type == RTCMediaStreamTrackTypeAudio); if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) { _adapters = [NSMutableArray array]; - _workerThread = factory.workerThread; + _signalingThread = factory.signalingThread; } return self; } - (void)dealloc { - for (RTC_OBJC_TYPE(RTCAudioRendererAdapter) * adapter in _adapters) { - self.nativeAudioTrack->RemoveSink(adapter.nativeAudioRenderer); - } + [self removeAllRenderers]; } - (RTC_OBJC_TYPE(RTCAudioSource) *)source { @@ -78,8 +76,8 @@ - (void)dealloc { } - (void)addRenderer:(id)renderer { - if (!_workerThread->IsCurrent()) { - _workerThread->BlockingCall([renderer, self] { [self addRenderer:renderer]; }); + if (!_signalingThread->IsCurrent()) { + _signalingThread->BlockingCall([renderer, self] { [self addRenderer:renderer]; }); return; } @@ -98,8 +96,8 @@ - (void)addRenderer:(id)renderer { } - (void)removeRenderer:(id)renderer { - if (!_workerThread->IsCurrent()) { - _workerThread->BlockingCall([renderer, self] { [self removeRenderer:renderer]; }); + if (!_signalingThread->IsCurrent()) { + _signalingThread->BlockingCall([renderer, self] { [self removeRenderer:renderer]; }); return; } __block NSUInteger indexToRemove = NSNotFound; @@ -119,6 +117,22 @@ - (void)removeRenderer:(id)renderer { [_adapters removeObjectAtIndex:indexToRemove]; } +- (void)removeAllRenderers { + // Ensure the method is executed on the signaling thread. + if (!_signalingThread->IsCurrent()) { + _signalingThread->BlockingCall([self] { [self removeAllRenderers]; }); + return; + } + + // Iterate over all adapters and remove each one from the native audio track. + for (RTC_OBJC_TYPE(RTCAudioRendererAdapter) * adapter in _adapters) { + self.nativeAudioTrack->RemoveSink(adapter.nativeAudioRenderer); + } + + // Clear the adapters array after all sinks have been removed. + [_adapters removeAllObjects]; +} + #pragma mark - Private - (rtc::scoped_refptr)nativeAudioTrack { From 0039a2fd72ebce56f3bd209c6f80cab1759ca437 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 24 Sep 2024 02:55:40 +0900 Subject: [PATCH 7/7] Refactor --- sdk/objc/api/RTCAudioRendererAdapter.mm | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sdk/objc/api/RTCAudioRendererAdapter.mm b/sdk/objc/api/RTCAudioRendererAdapter.mm index 478d2f5984..67c05792ad 100644 --- a/sdk/objc/api/RTCAudioRendererAdapter.mm +++ b/sdk/objc/api/RTCAudioRendererAdapter.mm @@ -47,19 +47,21 @@ void OnData(const void *audio_data, int bits_per_sample, int sample_rate, .mBitsPerChannel = 32, .mReserved = 0}; - CMFormatDescriptionRef format = nullptr; + CMFormatDescriptionRef formatDescription = nullptr; status = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &sd, sizeof(acl), &acl, 0, NULL, - NULL, &format); + NULL, &formatDescription); if (status != noErr) { - NSLog(@"RTCAudioTrack: Failed to create audio format description. Error: %d", (int)status); + NSLog(@"RTCAudioTrack: Failed to create audio formatDescription description. Error: %d", + (int)status); return; } - AVAudioFormat *format2 = [[AVAudioFormat alloc] initWithCMAudioFormatDescription:format]; - CFRelease(format); + AVAudioFormat *format = + [[AVAudioFormat alloc] initWithCMAudioFormatDescription:formatDescription]; + CFRelease(formatDescription); AVAudioFrameCount frameCount = static_cast(number_of_frames); - AVAudioPCMBuffer *pcmBuffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format2 + AVAudioPCMBuffer *pcmBuffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:frameCount]; if (!pcmBuffer) { NSLog(@"Failed to create AVAudioPCMBuffer");