diff --git a/BUILD.gn b/BUILD.gn index bcb7ce7ac1..b9898a5f17 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -69,6 +69,7 @@ rtc_shared_library("libwebrtc") { "include/base/scoped_ref_ptr.h", "include/libwebrtc.h", "include/rtc_audio_device.h", + "include/rtc_audio_processing.h", "include/rtc_audio_source.h", "include/rtc_audio_track.h", "include/rtc_data_channel.h", @@ -96,6 +97,16 @@ rtc_shared_library("libwebrtc") { "include/helper.h", "src/helper.cc", "src/base/portable.cc", + "src/internal/custom_audio_state.cc", + "src/internal/custom_audio_state.h", + "src/internal/custom_audio_transport_impl.cc", + "src/internal/custom_audio_transport_impl.h", + "src/internal/local_audio_track.cc", + "src/internal/local_audio_track.h", + "src/internal/custom_media_context.cc", + "src/internal/custom_media_context.h", + "src/internal/custom_webrtc_voice_engine.cc", + "src/internal/custom_webrtc_voice_engine.h", "src/internal/vcm_capturer.cc", "src/internal/vcm_capturer.h", "src/internal/video_capturer.cc", @@ -103,6 +114,8 @@ rtc_shared_library("libwebrtc") { "src/libwebrtc.cc", "src/rtc_audio_device_impl.cc", "src/rtc_audio_device_impl.h", + "src/rtc_audio_processing_impl.cc", + "src/rtc_audio_processing_impl.h", "src/rtc_audio_source_impl.cc", "src/rtc_audio_source_impl.h", "src/rtc_audio_track_impl.cc", diff --git a/include/rtc_audio_frame.h b/include/rtc_audio_frame.h index 3f276a1676..7774f0c787 100644 --- a/include/rtc_audio_frame.h +++ b/include/rtc_audio_frame.h @@ -1,17 +1,17 @@ -#ifndef AUDIO_FRAME_HXX -#define AUDIO_FRAME_HXX +#ifndef LIB_WEBRTC_RTC_AUDIO_FRAME_HXX +#define LIB_WEBRTC_RTC_AUDIO_FRAME_HXX -#include "media_manager_types.h" +#include "rtc_types.h" -namespace b2bua { +namespace libwebrtc { -class AudioFrame { +class AudioFrame : public RefCountInterface { public: /** * @brief Creates a new instance of AudioFrame. * @return AudioFrame*: a pointer to the newly created AudioFrame. */ - MEDIA_MANAGER_API static AudioFrame* Create(); + LIB_WEBRTC_API static AudioFrame* Create(); /** * @brief Creates a new instance of AudioFrame with specified parameters. @@ -23,16 +23,11 @@ class AudioFrame { * @param num_channels: the number of audio channels. * @return AudioFrame*: a pointer to the newly created AudioFrame. */ - MEDIA_MANAGER_API static AudioFrame* Create(int id, uint32_t timestamp, - const int16_t* data, - size_t samples_per_channel, - int sample_rate_hz, - size_t num_channels = 1); - - /** - * @brief Releases the memory of this AudioFrame. - */ - virtual void Release() = 0; + LIB_WEBRTC_API static AudioFrame* Create(int id, uint32_t timestamp, + const int16_t* data, + size_t samples_per_channel, + int sample_rate_hz, + size_t num_channels = 1); public: /** @@ -103,6 +98,6 @@ class AudioFrame { virtual int id() = 0; }; -}; // namespace b2bua +}; // namespace libwebrtc -#endif +#endif // LIB_WEBRTC_RTC_AUDIO_FRAME_HXX diff --git a/include/rtc_audio_processing.h b/include/rtc_audio_processing.h new file mode 100644 index 0000000000..908a16a380 --- /dev/null +++ b/include/rtc_audio_processing.h @@ -0,0 +1,35 @@ +#ifndef LIB_WEBRTC_RTC_AUDIO_PROCESSING_HXX +#define LIB_WEBRTC_RTC_AUDIO_PROCESSING_HXX + +#include "rtc_types.h" + +namespace libwebrtc { + +class RTCAudioProcessing : public RefCountInterface { + public: + class CustomProcessing { + public: + virtual void Initialize(int sample_rate_hz, int num_channels) = 0; + + virtual void Process(int num_bands, int num_frames, int buffer_size, + float* buffer) = 0; + + virtual void Reset(int new_rate) = 0; + + virtual void Release() = 0; + + protected: + virtual ~CustomProcessing() {} + }; + + public: + virtual void SetCapturePostProcessing( + CustomProcessing* capture_post_processing) = 0; + + virtual void SetRenderPreProcessing( + CustomProcessing* render_pre_processing) = 0; +}; + +} // namespace libwebrtc + +#endif // LIB_WEBRTC_RTC_AUDIO_PROCESSING_HXX \ No newline at end of file diff --git a/include/rtc_audio_source.h b/include/rtc_audio_source.h index 43e39fd801..e2f98fca0f 100644 --- a/include/rtc_audio_source.h +++ b/include/rtc_audio_source.h @@ -13,6 +13,16 @@ namespace libwebrtc { * processing and transmission mechanisms. */ class RTCAudioSource : public RefCountInterface { + public: + enum SourceType { kMicrophone, kCustom }; + + public: + virtual void CaptureFrame(const void* audio_data, int bits_per_sample, + int sample_rate, size_t number_of_channels, + size_t number_of_frames) = 0; + + virtual SourceType GetSourceType() const = 0; + protected: /** * The destructor for the RTCAudioSource class. diff --git a/include/rtc_audio_track.h b/include/rtc_audio_track.h index c64e4bc4a5..82459005d1 100644 --- a/include/rtc_audio_track.h +++ b/include/rtc_audio_track.h @@ -17,6 +17,10 @@ class RTCAudioTrack : public RTCMediaTrack { // volume in [0-10] virtual void SetVolume(double volume) = 0; + virtual void AddSink(AudioTrackSink* sink) = 0; + + virtual void RemoveSink(AudioTrackSink* sink) = 0; + protected: /** * The destructor for the RTCAudioTrack class. diff --git a/include/rtc_media_track.h b/include/rtc_media_track.h index ff5a2f743e..d971c89d27 100644 --- a/include/rtc_media_track.h +++ b/include/rtc_media_track.h @@ -5,6 +5,15 @@ namespace libwebrtc { +class AudioTrackSink { + public: + virtual void OnData(const void* audio_data, int bits_per_sample, + int sample_rate, size_t number_of_channels, + size_t number_of_frames) = 0; + protected: + virtual ~AudioTrackSink() {} +}; + /*Media Track interface*/ class RTCMediaTrack : public RefCountInterface { public: diff --git a/include/rtc_peerconnection_factory.h b/include/rtc_peerconnection_factory.h index cb024672c2..1b6b8c1aec 100644 --- a/include/rtc_peerconnection_factory.h +++ b/include/rtc_peerconnection_factory.h @@ -16,6 +16,7 @@ namespace libwebrtc { class RTCPeerConnection; class RTCAudioDevice; +class RTCAudioProcessing; class RTCVideoDevice; class RTCRtpCapabilities; @@ -33,12 +34,16 @@ class RTCPeerConnectionFactory : public RefCountInterface { virtual scoped_refptr GetAudioDevice() = 0; + virtual scoped_refptr GetAudioProcessing() = 0; + virtual scoped_refptr GetVideoDevice() = 0; #ifdef RTC_DESKTOP_DEVICE virtual scoped_refptr GetDesktopDevice() = 0; #endif virtual scoped_refptr CreateAudioSource( - const string audio_source_label) = 0; + const string audio_source_label, + RTCAudioSource::SourceType source_type = + RTCAudioSource::SourceType::kMicrophone) = 0; virtual scoped_refptr CreateVideoSource( scoped_refptr capturer, const string video_source_label, diff --git a/patchs/custom_audio_source.patch b/patchs/custom_audio_source.patch new file mode 100644 index 0000000000..4759364a04 --- /dev/null +++ b/patchs/custom_audio_source.patch @@ -0,0 +1,92 @@ +diff --git a/audio/audio_receive_stream.cc b/audio/audio_receive_stream.cc +index 415ad0640a..1467860a71 100644 +--- a/audio/audio_receive_stream.cc ++++ b/audio/audio_receive_stream.cc +@@ -467,8 +467,8 @@ AudioReceiveStreamImpl::GetAssociatedSendStreamForTesting() const { + return associated_send_stream_; + } + +-internal::AudioState* AudioReceiveStreamImpl::audio_state() const { +- auto* audio_state = static_cast(audio_state_.get()); ++webrtc::AudioState* AudioReceiveStreamImpl::audio_state() const { ++ auto* audio_state = static_cast(audio_state_.get()); + RTC_DCHECK(audio_state); + return audio_state; + } +diff --git a/audio/audio_receive_stream.h b/audio/audio_receive_stream.h +index db49631638..982cad39a4 100644 +--- a/audio/audio_receive_stream.h ++++ b/audio/audio_receive_stream.h +@@ -144,7 +144,7 @@ class AudioReceiveStreamImpl final : public webrtc::AudioReceiveStreamInterface, + const webrtc::AudioReceiveStreamInterface::Config& config); + + private: +- internal::AudioState* audio_state() const; ++ webrtc::AudioState* audio_state() const; + + RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_thread_checker_; + // TODO(bugs.webrtc.org/11993): This checker conceptually represents +diff --git a/audio/audio_state.h b/audio/audio_state.h +index f21cca771e..a4eb0fedf1 100644 +--- a/audio/audio_state.h ++++ b/audio/audio_state.h +@@ -54,13 +54,14 @@ class AudioState : public webrtc::AudioState { + return config_.audio_device_module.get(); + } + +- void AddReceivingStream(webrtc::AudioReceiveStreamInterface* stream); +- void RemoveReceivingStream(webrtc::AudioReceiveStreamInterface* stream); ++ void AddReceivingStream(webrtc::AudioReceiveStreamInterface* stream) override; ++ void RemoveReceivingStream( ++ webrtc::AudioReceiveStreamInterface* stream) override; + + void AddSendingStream(webrtc::AudioSendStream* stream, + int sample_rate_hz, +- size_t num_channels); +- void RemoveSendingStream(webrtc::AudioSendStream* stream); ++ size_t num_channels) override; ++ void RemoveSendingStream(webrtc::AudioSendStream* stream) override; + + private: + void UpdateAudioTransportWithSendingStreams(); +diff --git a/audio/audio_transport_impl.h b/audio/audio_transport_impl.h +index 24b09d2140..1b39e52869 100644 +--- a/audio/audio_transport_impl.h ++++ b/audio/audio_transport_impl.h +@@ -82,7 +82,7 @@ class AudioTransportImpl : public AudioTransport { + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) override; + +- void UpdateAudioSenders(std::vector senders, ++ virtual void UpdateAudioSenders(std::vector senders, + int send_sample_rate_hz, + size_t send_num_channels); + void SetStereoChannelSwapping(bool enable); +diff --git a/call/audio_state.h b/call/audio_state.h +index 85f04758dd..7bd7ce25f1 100644 +--- a/call/audio_state.h ++++ b/call/audio_state.h +@@ -20,6 +20,8 @@ + namespace webrtc { + + class AudioTransport; ++class AudioReceiveStreamInterface; ++class AudioSendStream; + + // AudioState holds the state which must be shared between multiple instances of + // webrtc::Call for audio processing purposes. +@@ -62,6 +64,14 @@ class AudioState : public rtc::RefCountInterface { + // Notify the AudioState that a stream updated it's mute state. + virtual void OnMuteStreamChanged() = 0; + ++ virtual void AddReceivingStream(AudioReceiveStreamInterface* stream) = 0; ++ virtual void RemoveReceivingStream(AudioReceiveStreamInterface* stream) = 0; ++ ++ virtual void AddSendingStream(AudioSendStream* stream, ++ int sample_rate_hz, ++ size_t num_channels) = 0; ++ virtual void RemoveSendingStream(AudioSendStream* stream) = 0; ++ + static rtc::scoped_refptr Create( + const AudioState::Config& config); + diff --git a/patchs/fix_emplace.patch b/patchs/fix_emplace.patch new file mode 100644 index 0000000000..afb35b9b1c --- /dev/null +++ b/patchs/fix_emplace.patch @@ -0,0 +1,33 @@ +diff --git a/modules/congestion_controller/goog_cc/loss_based_bwe_v2.cc b/modules/congestion_controller/goog_cc/loss_based_bwe_v2.cc +index 7deeb7ad64..f913557f8c 100644 +--- a/modules/congestion_controller/goog_cc/loss_based_bwe_v2.cc ++++ b/modules/congestion_controller/goog_cc/loss_based_bwe_v2.cc +@@ -545,11 +545,14 @@ absl::optional LossBasedBweV2::CreateConfig( + key_value_config->Lookup("WebRTC-Bwe-LossBasedBweV2")); + } + +- absl::optional config; ++ // absl::optional config; + if (!enabled.Get()) { +- return config; ++ return std::nullopt; + } +- config.emplace(); ++ ++ Config _config; ++ Config* config = &_config; ++ + config->bandwidth_rampup_upper_bound_factor = + bandwidth_rampup_upper_bound_factor.Get(); + config->bandwidth_rampup_upper_bound_factor_in_hold = +@@ -608,7 +611,7 @@ absl::optional LossBasedBweV2::CreateConfig( + config->padding_duration = padding_duration.Get(); + config->bound_best_candidate = bound_best_candidate.Get(); + config->pace_at_loss_based_estimate = pace_at_loss_based_estimate.Get(); +- return config; ++ return _config; + } + + bool LossBasedBweV2::IsConfigValid() const { + + diff --git a/src/internal/custom_audio_state.cc b/src/internal/custom_audio_state.cc new file mode 100644 index 0000000000..008f4eec56 --- /dev/null +++ b/src/internal/custom_audio_state.cc @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "src/internal/custom_audio_state.h" + +#include +#include +#include +#include + +#include "api/sequence_checker.h" +#include "api/task_queue/task_queue_base.h" +#include "api/units/time_delta.h" +#include "audio/audio_receive_stream.h" +#include "audio/audio_send_stream.h" +#include "modules/audio_device/include/audio_device.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "src/internal/custom_audio_transport_impl.h" +#include "src/internal/custom_media_context.h" + +namespace webrtc { + +CustomAudioState::CustomAudioState( + const AudioState::Config& config, + std::unique_ptr custom_audio_transport) + : config_(config), audio_transport_(std::move(custom_audio_transport)) { + RTC_DCHECK(config_.audio_mixer); + RTC_DCHECK(config_.audio_device_module); +} + +CustomAudioState::~CustomAudioState() { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK(receiving_streams_.empty()); + RTC_DCHECK(sending_streams_.empty()); + RTC_DCHECK(!null_audio_poller_.Running()); +} + +AudioProcessing* CustomAudioState::audio_processing() { + return config_.audio_processing.get(); +} + +AudioTransport* CustomAudioState::audio_transport() { + return audio_transport_.get(); +} + +void CustomAudioState::AddReceivingStream( + webrtc::AudioReceiveStreamInterface* stream) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK_EQ(0, receiving_streams_.count(stream)); + receiving_streams_.insert(stream); + if (!config_.audio_mixer->AddSource( + static_cast(stream))) { + RTC_DLOG(LS_ERROR) << "Failed to add source to mixer."; + } + + // Make sure playback is initialized; start playing if enabled. + UpdateNullAudioPollerState(); + auto* adm = config_.audio_device_module.get(); + if (!adm->Playing()) { + if (adm->InitPlayout() == 0) { + if (playout_enabled_) { + adm->StartPlayout(); + } + } else { + RTC_DLOG_F(LS_ERROR) << "Failed to initialize playout."; + } + } +} + +void CustomAudioState::RemoveReceivingStream( + webrtc::AudioReceiveStreamInterface* stream) { + RTC_DCHECK_RUN_ON(&thread_checker_); + auto count = receiving_streams_.erase(stream); + RTC_DCHECK_EQ(1, count); + config_.audio_mixer->RemoveSource( + static_cast(stream)); + UpdateNullAudioPollerState(); + if (receiving_streams_.empty()) { + config_.audio_device_module->StopPlayout(); + } +} + +void CustomAudioState::AddSendingStream(webrtc::AudioSendStream* stream, + int sample_rate_hz, + size_t num_channels) { + RTC_DCHECK_RUN_ON(&thread_checker_); + auto& properties = sending_streams_[stream]; + properties.sample_rate_hz = sample_rate_hz; + properties.num_channels = num_channels; + UpdateAudioTransportWithSendingStreams(); + + // Make sure recording is initialized; start recording if enabled. + if (ShouldRecord()) { + auto* adm = config_.audio_device_module.get(); + if (!adm->Recording()) { + if (adm->InitRecording() == 0) { + if (recording_enabled_) { + // TODO: Verify if the following windows only logic is still required. +#if defined(WEBRTC_WIN) + if (adm->BuiltInAECIsAvailable() && !adm->Playing()) { + if (!adm->PlayoutIsInitialized()) { + adm->InitPlayout(); + } + adm->StartPlayout(); + } +#endif + adm->StartRecording(); + } + } else { + RTC_DLOG_F(LS_ERROR) << "Failed to initialize recording."; + } + } + } +} + +void CustomAudioState::RemoveSendingStream(webrtc::AudioSendStream* stream) { + RTC_DCHECK_RUN_ON(&thread_checker_); + auto count = sending_streams_.erase(stream); + RTC_DCHECK_EQ(1, count); + UpdateAudioTransportWithSendingStreams(); + + bool should_record = ShouldRecord(); + RTC_LOG(LS_INFO) << "RemoveSendingStream: should_record = " << should_record; + if (!should_record) { + config_.audio_device_module->StopRecording(); + } +} + +void CustomAudioState::SetPlayout(bool enabled) { + RTC_LOG(LS_INFO) << "SetPlayout(" << enabled << ")"; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (playout_enabled_ != enabled) { + playout_enabled_ = enabled; + if (enabled) { + UpdateNullAudioPollerState(); + if (!receiving_streams_.empty()) { + config_.audio_device_module->StartPlayout(); + } + } else { + config_.audio_device_module->StopPlayout(); + UpdateNullAudioPollerState(); + } + } +} + +void CustomAudioState::SetRecording(bool enabled) { + RTC_LOG(LS_INFO) << "SetRecording(" << enabled << ")"; + RTC_DCHECK_RUN_ON(&thread_checker_); + if (recording_enabled_ != enabled) { + recording_enabled_ = enabled; + if (enabled) { + if (ShouldRecord()) { + config_.audio_device_module->StartRecording(); + } + } else { + config_.audio_device_module->StopRecording(); + } + } +} + +void CustomAudioState::SetStereoChannelSwapping(bool enable) { + RTC_DCHECK(thread_checker_.IsCurrent()); + audio_transport_->SetStereoChannelSwapping(enable); +} + +void CustomAudioState::UpdateAudioTransportWithSendingStreams() { + RTC_DCHECK(thread_checker_.IsCurrent()); + std::vector audio_senders; + int max_sample_rate_hz = 8000; + size_t max_num_channels = 1; + for (const auto& kv : sending_streams_) { + audio_senders.push_back(kv.first); + max_sample_rate_hz = std::max(max_sample_rate_hz, kv.second.sample_rate_hz); + max_num_channels = std::max(max_num_channels, kv.second.num_channels); + } + audio_transport_->UpdateAudioSenders(std::move(audio_senders), + max_sample_rate_hz, max_num_channels); +} + +void CustomAudioState::UpdateNullAudioPollerState() { + // Run NullAudioPoller when there are receiving streams and playout is + // disabled. + if (!receiving_streams_.empty() && !playout_enabled_) { + if (!null_audio_poller_.Running()) { + AudioTransport* audio_transport = audio_transport_.get(); + null_audio_poller_ = RepeatingTaskHandle::Start( + TaskQueueBase::Current(), [audio_transport] { + static constexpr size_t kNumChannels = 1; + static constexpr uint32_t kSamplesPerSecond = 48'000; + // 10ms of samples + static constexpr size_t kNumSamples = kSamplesPerSecond / 100; + + // Buffer to hold the audio samples. + int16_t buffer[kNumSamples * kNumChannels]; + + // Output variables from `NeedMorePlayData`. + size_t n_samples; + int64_t elapsed_time_ms; + int64_t ntp_time_ms; + audio_transport->NeedMorePlayData( + kNumSamples, sizeof(int16_t), kNumChannels, kSamplesPerSecond, + buffer, n_samples, &elapsed_time_ms, &ntp_time_ms); + + // Reschedule the next poll iteration. + return TimeDelta::Millis(10); + }); + } + } else { + null_audio_poller_.Stop(); + } +} + +void CustomAudioState::OnMuteStreamChanged() { + auto* adm = config_.audio_device_module.get(); + bool should_record = ShouldRecord(); + + RTC_LOG(LS_INFO) << "OnMuteStreamChanged: should_record = " << should_record; + if (should_record && !adm->Recording()) { + if (adm->InitRecording() == 0) { + adm->StartRecording(); + } + } else if (!should_record && adm->Recording()) { + adm->StopRecording(); + } +} + +bool CustomAudioState::ShouldRecord() { + RTC_LOG(LS_INFO) << "ShouldRecord"; + // no streams to send + if (sending_streams_.empty()) { + RTC_LOG(LS_INFO) << "ShouldRecord: send stream = empty"; + return false; + } + + int stream_count = sending_streams_.size(); + + int muted_count = 0; + for (const auto& kv : sending_streams_) { + if (kv.first->GetMuted()) { + muted_count++; + } + } + + RTC_LOG(LS_INFO) << "ShouldRecord: " << muted_count << " muted, " + << stream_count << " sending"; + return muted_count != stream_count; +} + +} // namespace webrtc diff --git a/src/internal/custom_audio_state.h b/src/internal/custom_audio_state.h new file mode 100644 index 0000000000..aedde0083a --- /dev/null +++ b/src/internal/custom_audio_state.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef INTERNAL_CUSTOM_AUDIO_STATE_H_ +#define INTERNAL_CUSTOM_AUDIO_STATE_H_ + +#include +#include + +#include "api/sequence_checker.h" +#include "audio/audio_transport_impl.h" +#include "call/audio_sender.h" +#include "call/audio_state.h" +#include "rtc_base/containers/flat_set.h" +#include "rtc_base/ref_count.h" +#include "rtc_base/task_utils/repeating_task.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +class CustomAudioTransportImpl; +class AudioSendStream; +class AudioReceiveStreamInterface; + +class CustomAudioState : public webrtc::AudioState { + public: + explicit CustomAudioState( + const AudioState::Config& config, + std::unique_ptr custom_audio_transport); + + CustomAudioState() = delete; + CustomAudioState(const CustomAudioState&) = delete; + CustomAudioState& operator=(const CustomAudioState&) = delete; + + ~CustomAudioState() override; + + AudioProcessing* audio_processing() override; + AudioTransport* audio_transport() override; + + void SetPlayout(bool enabled) override; + void SetRecording(bool enabled) override; + + void SetStereoChannelSwapping(bool enable) override; + + void OnMuteStreamChanged() override; + + AudioDeviceModule* audio_device_module() { + RTC_DCHECK(config_.audio_device_module); + return config_.audio_device_module.get(); + } + + void AddReceivingStream(webrtc::AudioReceiveStreamInterface* stream) override; + void RemoveReceivingStream( + webrtc::AudioReceiveStreamInterface* stream) override; + + void AddSendingStream(webrtc::AudioSendStream* stream, int sample_rate_hz, + size_t num_channels) override; + void RemoveSendingStream(webrtc::AudioSendStream* stream) override; + + private: + void UpdateAudioTransportWithSendingStreams(); + void UpdateNullAudioPollerState() RTC_RUN_ON(&thread_checker_); + + // Returns true when at least 1 stream exists and all streams are not muted. + bool ShouldRecord(); + + SequenceChecker thread_checker_; + SequenceChecker process_thread_checker_{SequenceChecker::kDetached}; + const webrtc::AudioState::Config config_; + bool recording_enabled_ = true; + bool playout_enabled_ = true; + + // Transports mixed audio from the mixer to the audio device and + // recorded audio to the sending streams. + std::unique_ptr audio_transport_; + + // Null audio poller is used to continue polling the audio streams if audio + // playout is disabled so that audio processing still happens and the audio + // stats are still updated. + RepeatingTaskHandle null_audio_poller_ RTC_GUARDED_BY(&thread_checker_); + + webrtc::flat_set receiving_streams_; + struct StreamProperties { + int sample_rate_hz = 0; + size_t num_channels = 0; + }; + std::map sending_streams_; +}; + +} // namespace webrtc + +#endif // INTERNAL_CUSTOM_AUDIO_STATE_H_ diff --git a/src/internal/custom_audio_transport_impl.cc b/src/internal/custom_audio_transport_impl.cc new file mode 100644 index 0000000000..77631c21c8 --- /dev/null +++ b/src/internal/custom_audio_transport_impl.cc @@ -0,0 +1,98 @@ +#include "src/internal/custom_audio_transport_impl.h" + +namespace webrtc { + +CustomAudioTransportImpl::CustomAudioTransportImpl( + AudioMixer* mixer, AudioProcessing* audio_processing, + AsyncAudioProcessing::Factory* async_audio_processing_factory) + : audio_transport_impl_(mixer, audio_processing, + async_audio_processing_factory) {} + +// TODO(bugs.webrtc.org/13620) Deprecate this function +int32_t CustomAudioTransportImpl::RecordedDataIsAvailable( + const void* audioSamples, size_t nSamples, size_t nBytesPerSample, + size_t nChannels, uint32_t samplesPerSec, uint32_t totalDelayMS, + int32_t clockDrift, uint32_t currentMicLevel, bool keyPressed, + uint32_t& newMicLevel) { + return audio_transport_impl_.RecordedDataIsAvailable( + audioSamples, nSamples, nBytesPerSample, nChannels, samplesPerSec, + totalDelayMS, clockDrift, currentMicLevel, keyPressed, newMicLevel); +} + +int32_t CustomAudioTransportImpl::RecordedDataIsAvailable( + const void* audioSamples, size_t nSamples, size_t nBytesPerSample, + size_t nChannels, uint32_t samplesPerSec, uint32_t totalDelayMS, + int32_t clockDrift, uint32_t currentMicLevel, bool keyPressed, + uint32_t& newMicLevel, absl::optional estimated_capture_time_ns) { + return audio_transport_impl_.RecordedDataIsAvailable( + audioSamples, nSamples, nBytesPerSample, nChannels, samplesPerSec, + totalDelayMS, clockDrift, currentMicLevel, keyPressed, newMicLevel, + estimated_capture_time_ns); +} + +int32_t CustomAudioTransportImpl::NeedMorePlayData( + size_t nSamples, size_t nBytesPerSample, size_t nChannels, + uint32_t samplesPerSec, void* audioSamples, size_t& nSamplesOut, + int64_t* elapsed_time_ms, int64_t* ntp_time_ms) { + return audio_transport_impl_.NeedMorePlayData( + nSamples, nBytesPerSample, nChannels, samplesPerSec, audioSamples, + nSamplesOut, elapsed_time_ms, ntp_time_ms); +} + +void CustomAudioTransportImpl::PullRenderData( + int bits_per_sample, int sample_rate, size_t number_of_channels, + size_t number_of_frames, void* audio_data, int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) { + audio_transport_impl_.PullRenderData( + bits_per_sample, sample_rate, number_of_channels, number_of_frames, + audio_data, elapsed_time_ms, ntp_time_ms); +} + +void CustomAudioTransportImpl::UpdateAudioSenders( + std::vector senders, int send_sample_rate_hz, + size_t send_num_channels) { + if (senders.size() > 0) { + std::vector snds = std::vector(); + snds.push_back(this); + audio_transport_impl_.UpdateAudioSenders( + std::move(snds), send_sample_rate_hz, send_num_channels); + } else { + std::vector snds = std::vector(); + audio_transport_impl_.UpdateAudioSenders( + std::move(snds), send_sample_rate_hz, send_num_channels); + } +} + +void CustomAudioTransportImpl::AddAudioSender(AudioSender* sender) { + MutexLock lock(&capture_lock_); + audio_senders_.push_back(sender); +} + +void CustomAudioTransportImpl::RemoveAudioSender(AudioSender* sender) { + MutexLock lock(&capture_lock_); + auto it = std::remove(audio_senders_.begin(), audio_senders_.end(), sender); + if (it != audio_senders_.end()) { + audio_senders_.erase(it, audio_senders_.end()); + } +} + +void CustomAudioTransportImpl::SetStereoChannelSwapping(bool enable) { + audio_transport_impl_.SetStereoChannelSwapping(enable); +} + +void CustomAudioTransportImpl::SendAudioData( + std::unique_ptr audio_frame) { + RTC_DCHECK_GT(audio_frame->samples_per_channel_, 0); + MutexLock lock(&capture_lock_); + if (audio_senders_.empty()) return; + + auto it = audio_senders_.begin(); + while (++it != audio_senders_.end()) { + auto audio_frame_copy = std::make_unique(); + audio_frame_copy->CopyFrom(*audio_frame); + (*it)->SendAudioData(std::move(audio_frame_copy)); + } + // Send the original frame to the first stream w/o copying. + (*audio_senders_.begin())->SendAudioData(std::move(audio_frame)); +} +} // namespace webrtc \ No newline at end of file diff --git a/src/internal/custom_audio_transport_impl.h b/src/internal/custom_audio_transport_impl.h new file mode 100644 index 0000000000..80da05103b --- /dev/null +++ b/src/internal/custom_audio_transport_impl.h @@ -0,0 +1,69 @@ +#ifndef INTERNAL_CUSTOM_AUDIO_TRANSPORT_STATE_H_ +#define INTERNAL_CUSTOM_AUDIO_TRANSPORT_STATE_H_ + +#include +#include + +#include "api/sequence_checker.h" +#include "audio/audio_transport_impl.h" +#include "call/audio_sender.h" +#include "call/audio_state.h" +#include "rtc_base/containers/flat_set.h" +#include "rtc_base/ref_count.h" +#include "rtc_base/task_utils/repeating_task.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +class CustomAudioTransportImpl : public AudioTransport, public AudioSender { + public: + CustomAudioTransportImpl( + AudioMixer* mixer, AudioProcessing* audio_processing, + AsyncAudioProcessing::Factory* async_audio_processing_factory); + + int32_t RecordedDataIsAvailable(const void* audioSamples, size_t nSamples, + size_t nBytesPerSample, size_t nChannels, + uint32_t samplesPerSec, uint32_t totalDelayMS, + int32_t clockDrift, uint32_t currentMicLevel, + bool keyPressed, + uint32_t& newMicLevel) override; + + int32_t RecordedDataIsAvailable( + const void* audioSamples, size_t nSamples, size_t nBytesPerSample, + size_t nChannels, uint32_t samplesPerSec, uint32_t totalDelayMS, + int32_t clockDrift, uint32_t currentMicLevel, bool keyPressed, + uint32_t& newMicLevel, + absl::optional estimated_capture_time_ns) override; + + int32_t NeedMorePlayData(size_t nSamples, size_t nBytesPerSample, + size_t nChannels, uint32_t samplesPerSec, + void* audioSamples, size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) override; + + void PullRenderData(int bits_per_sample, int sample_rate, + size_t number_of_channels, size_t number_of_frames, + void* audio_data, int64_t* elapsed_time_ms, + int64_t* ntp_time_ms) override; + + virtual void UpdateAudioSenders(std::vector senders, + int send_sample_rate_hz, + size_t send_num_channels); + + void AddAudioSender(AudioSender* sender); + + void RemoveAudioSender(AudioSender* sender); + + void SetStereoChannelSwapping(bool enable); + + void SendAudioData(std::unique_ptr audio_frame) override; + + private: + webrtc::AudioTransportImpl audio_transport_impl_; + mutable Mutex capture_lock_; + std::vector audio_senders_ RTC_GUARDED_BY(capture_lock_); +}; + +} // namespace libwebrtc + +#endif // INTERNAL_CUSTOM_AUDIO_TRANSPORT_STATE_H_ diff --git a/src/internal/custom_media_context.cc b/src/internal/custom_media_context.cc new file mode 100644 index 0000000000..415d706dde --- /dev/null +++ b/src/internal/custom_media_context.cc @@ -0,0 +1,151 @@ +#include "src/internal/custom_media_context.h" + +#include + +#include +#include + +#include "api/create_peerconnection_factory.h" +#include "api/enable_media.h" +#include "api/environment/environment.h" +#include "api/peer_connection_interface.h" +#include "api/rtc_event_log/rtc_event_log_factory.h" +#include "api/scoped_refptr.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "api/transport/field_trial_based_config.h" +#include "call/create_call.h" +#include "media/base/media_engine.h" +#include "media/engine/webrtc_media_engine.h" +#include "media/engine/webrtc_video_engine.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "pc/media_factory.h" +#include "rtc_base/thread.h" +#include "src/internal/custom_audio_state.h" +#include "src/internal/custom_audio_transport_impl.h" +#include "src/internal/custom_webrtc_voice_engine.h" +#include "src/internal/local_audio_track.h" + +namespace webrtc { + +class TaskQueueFactory; +class AudioDeviceModule; +using ::cricket::AudioCodec; +using ::cricket::CompositeMediaEngine; +using ::cricket::CustomWebRtcVoiceEngine; +using ::cricket::MediaEngineInterface; +using ::cricket::WebRtcVideoEngine; + +class CustomMediaFactoryImpl : public MediaFactory { + public: + CustomMediaFactoryImpl( + webrtc::CreateAudioStateFactory* create_audio_state_factory) + : create_audio_state_factory_(create_audio_state_factory) { + RTC_DCHECK(create_audio_state_factory_); + } + ~CustomMediaFactoryImpl() override = default; + + std::unique_ptr CreateCall(const CallConfig& config) override { + return webrtc::CreateCall(config); + } + + std::unique_ptr CreateMediaEngine( + const Environment& env, + PeerConnectionFactoryDependencies& deps) override { + auto audio_engine = std::make_unique( + create_audio_state_factory_, &env.task_queue_factory(), deps.adm.get(), + std::move(deps.audio_encoder_factory), + std::move(deps.audio_decoder_factory), std::move(deps.audio_mixer), + std::move(deps.audio_processing), std::move(deps.audio_frame_processor), + env.field_trials()); + auto video_engine = std::make_unique( + std::move(deps.video_encoder_factory), + std::move(deps.video_decoder_factory), env.field_trials()); + return std::make_unique(std::move(audio_engine), + std::move(video_engine)); + } + + private: + webrtc::CreateAudioStateFactory* create_audio_state_factory_ = nullptr; +}; + +rtc::scoped_refptr +CustomMediaContext::CreateAudioSource(cricket::AudioOptions* options, + bool is_custom_source) { + RTC_DCHECK(options); + // if is_custom_source == true, not using the default audio transport, + // you can put costom audio frame via LocalAudioSource::CaptureFrame(...) + // and the audio transport will be null. + // otherwise, use the default audio transport, audio transport will + // put audio frame from your platform adm to your + // LocalAudioSource::SendAudioData(...). + if (rtc::Thread::Current() != signaling_thread_) { + return signaling_thread_->BlockingCall([this, options, is_custom_source] { + return libwebrtc::LocalAudioSource::Create( + options, is_custom_source ? nullptr : audio_transport_); + }); + } + return libwebrtc::LocalAudioSource::Create( + options, is_custom_source ? nullptr : audio_transport_); +} + +rtc::scoped_refptr CustomMediaContext::CreateAudioState( + const webrtc::AudioState::Config& config) { + auto audio_transport = std::make_unique( + config.audio_mixer.get(), config.audio_processing.get(), + config.async_audio_processing_factory.get()); + audio_transport_ = audio_transport.get(); + return rtc::make_ref_counted(config, + std::move(audio_transport)); +} + +rtc::scoped_refptr +CustomMediaContext::CreatePeerConnectionFactory( + rtc::Thread* network_thread, rtc::Thread* worker_thread, + rtc::Thread* signaling_thread, + rtc::scoped_refptr default_adm, + rtc::scoped_refptr audio_encoder_factory, + rtc::scoped_refptr audio_decoder_factory, + std::unique_ptr video_encoder_factory, + std::unique_ptr video_decoder_factory, + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing, + std::unique_ptr audio_frame_processor, + std::unique_ptr field_trials, + MediaFactory* media_factory) { + if (!field_trials) { + field_trials = std::make_unique(); + } + + PeerConnectionFactoryDependencies dependencies; + dependencies.network_thread = network_thread; + dependencies.worker_thread = worker_thread; + dependencies.signaling_thread = signaling_thread; + dependencies.task_queue_factory = + CreateDefaultTaskQueueFactory(field_trials.get()); + dependencies.event_log_factory = std::make_unique(); + dependencies.trials = std::move(field_trials); + + if (network_thread) { + // TODO(bugs.webrtc.org/13145): Add an rtc::SocketFactory* argument. + dependencies.socket_factory = network_thread->socketserver(); + } + dependencies.adm = std::move(default_adm); + dependencies.audio_encoder_factory = std::move(audio_encoder_factory); + dependencies.audio_decoder_factory = std::move(audio_decoder_factory); + dependencies.audio_frame_processor = std::move(audio_frame_processor); + if (audio_processing) { + dependencies.audio_processing = std::move(audio_processing); + } else { + dependencies.audio_processing = AudioProcessingBuilder().Create(); + } + dependencies.audio_mixer = std::move(audio_mixer); + dependencies.video_encoder_factory = std::move(video_encoder_factory); + dependencies.video_decoder_factory = std::move(video_decoder_factory); + + dependencies.media_factory = std::make_unique(this); + + return CreateModularPeerConnectionFactory(std::move(dependencies)); +} + +} // namespace webrtc diff --git a/src/internal/custom_media_context.h b/src/internal/custom_media_context.h new file mode 100644 index 0000000000..9c19bc37f9 --- /dev/null +++ b/src/internal/custom_media_context.h @@ -0,0 +1,59 @@ +#ifndef INTERNAL_CUSTOM_MEDIA_BRIDGE_FACTORY_H_ +#define INTERNAL_CUSTOM_MEDIA_BRIDGE_FACTORY_H_ + +#include + +#include "api/audio/audio_mixer.h" +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_encoder_factory.h" +#include "api/peer_connection_interface.h" +#include "api/ref_count.h" +#include "api/scoped_refptr.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "call/audio_state.h" +#include "pc/media_factory.h" +#include "rtc_base/thread.h" +#include "src/internal/custom_webrtc_voice_engine.h" +#include "src/internal/local_audio_track.h" + +namespace webrtc { + +class CustomAudioTransportImpl; + +class CustomMediaContext : public webrtc::CreateAudioStateFactory, + public webrtc::RefCountInterface { + public: + CustomMediaContext(rtc::Thread* signaling_thread) { + RTC_DCHECK(signaling_thread); + signaling_thread_ = signaling_thread; + } + rtc::scoped_refptr CreateAudioSource( + cricket::AudioOptions* options, bool is_custom_source = false); + + rtc::scoped_refptr CreateAudioState( + const webrtc::AudioState::Config& config) override; + + rtc::scoped_refptr + CreatePeerConnectionFactory( + rtc::Thread* network_thread, rtc::Thread* worker_thread, + rtc::Thread* signaling_thread, + rtc::scoped_refptr default_adm, + rtc::scoped_refptr audio_encoder_factory, + rtc::scoped_refptr audio_decoder_factory, + std::unique_ptr video_encoder_factory, + std::unique_ptr video_decoder_factory, + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing, + std::unique_ptr audio_frame_processor = nullptr, + std::unique_ptr field_trials = nullptr, + MediaFactory* media_factory = nullptr); + + private: + rtc::Thread* signaling_thread_; + webrtc::CustomAudioTransportImpl* audio_transport_; +}; + +} // namespace webrtc + +#endif // INTERNAL_CUSTOM_MEDIA_BRIDGE_FACTORY_H_ diff --git a/src/internal/custom_webrtc_voice_engine.cc b/src/internal/custom_webrtc_voice_engine.cc new file mode 100644 index 0000000000..0f8f2424bf --- /dev/null +++ b/src/internal/custom_webrtc_voice_engine.cc @@ -0,0 +1,2674 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "custom_webrtc_voice_engine.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/algorithm/algorithm.h" +#include "absl/algorithm/container.h" +#include "absl/functional/bind_front.h" +#include "absl/strings/match.h" +#include "absl/types/optional.h" +#include "api/audio/audio_frame.h" +#include "api/audio/audio_frame_processor.h" +#include "api/audio_codecs/audio_codec_pair_id.h" +#include "api/audio_codecs/audio_encoder.h" +#include "api/call/audio_sink.h" +#include "api/field_trials_view.h" +#include "api/make_ref_counted.h" +#include "api/media_types.h" +#include "api/priority.h" +#include "api/rtp_headers.h" +#include "api/rtp_parameters.h" +#include "api/rtp_transceiver_direction.h" +#include "api/task_queue/pending_task_safety_flag.h" +#include "api/transport/bitrate_settings.h" +#include "api/units/data_rate.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "call/audio_receive_stream.h" +#include "call/packet_receiver.h" +#include "call/rtp_config.h" +#include "call/rtp_transport_controller_send_interface.h" +#include "media/base/audio_source.h" +#include "media/base/codec.h" +#include "media/base/media_constants.h" +#include "media/base/stream_params.h" +#include "media/engine/adm_helpers.h" +#include "media/engine/payload_type_mapper.h" +#include "media/engine/webrtc_media_engine.h" +#include "modules/async_audio_processing/async_audio_processing.h" +#include "modules/audio_mixer/audio_mixer_impl.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/include/audio_processing_statistics.h" +#include "modules/rtp_rtcp/include/report_block_data.h" +#include "modules/rtp_rtcp/include/rtp_header_extension_map.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_util.h" +#include "rtc_base/checks.h" +#include "rtc_base/dscp.h" +#include "rtc_base/experiments/struct_parameters_parser.h" +#include "rtc_base/logging.h" +#include "rtc_base/race_checker.h" +#include "rtc_base/string_encode.h" +#include "rtc_base/strings/audio_format_to_string.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/strings/string_format.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/time_utils.h" +#include "rtc_base/trace_event.h" +#include "src/internal/custom_audio_state.h" +#include "src/internal/custom_media_context.h" +#include "system_wrappers/include/metrics.h" + +namespace cricket { +namespace { + +using ::webrtc::ParseRtpSsrc; + +constexpr size_t kMaxUnsignaledRecvStreams = 4; + +constexpr int kNackRtpHistoryMs = 5000; + +const int kMinTelephoneEventCode = 0; // RFC4733 (Section 2.3.1) +const int kMaxTelephoneEventCode = 255; + +const int kMinPayloadType = 0; +const int kMaxPayloadType = 127; + +class ProxySink : public webrtc::AudioSinkInterface { + public: + explicit ProxySink(AudioSinkInterface* sink) : sink_(sink) { + RTC_DCHECK(sink); + } + + void OnData(const Data& audio) override { sink_->OnData(audio); } + + private: + webrtc::AudioSinkInterface* sink_; +}; + +bool ValidateStreamParams(const StreamParams& sp) { + if (sp.ssrcs.empty()) { + RTC_DLOG(LS_ERROR) << "No SSRCs in stream parameters: " << sp.ToString(); + return false; + } + if (sp.ssrcs.size() > 1) { + RTC_DLOG(LS_ERROR) << "Multiple SSRCs in stream parameters: " + << sp.ToString(); + return false; + } + return true; +} + +// Dumps an AudioCodec in RFC 2327-ish format. +std::string ToString(const AudioCodec& codec) { + rtc::StringBuilder ss; + ss << codec.name << "/" << codec.clockrate << "/" << codec.channels; + if (!codec.params.empty()) { + ss << " {"; + for (const auto& param : codec.params) { + ss << " " << param.first << "=" << param.second; + } + ss << " }"; + } + ss << " (" << codec.id << ")"; + return ss.Release(); +} + +bool IsCodec(const AudioCodec& codec, const char* ref_name) { + return absl::EqualsIgnoreCase(codec.name, ref_name); +} + +absl::optional FindCodec(const std::vector& codecs, + const AudioCodec& codec) { + for (const AudioCodec& c : codecs) { + if (c.Matches(codec)) { + return c; + } + } + return absl::nullopt; +} + +bool VerifyUniquePayloadTypes(const std::vector& codecs) { + if (codecs.empty()) { + return true; + } + std::vector payload_types; + absl::c_transform(codecs, std::back_inserter(payload_types), + [](const AudioCodec& codec) { return codec.id; }); + absl::c_sort(payload_types); + return absl::c_adjacent_find(payload_types) == payload_types.end(); +} + +absl::optional GetAudioNetworkAdaptorConfig( + const AudioOptions& options) { + if (options.audio_network_adaptor && *options.audio_network_adaptor && + options.audio_network_adaptor_config) { + // Turn on audio network adaptor only when `options_.audio_network_adaptor` + // equals true and `options_.audio_network_adaptor_config` has a value. + return options.audio_network_adaptor_config; + } + return absl::nullopt; +} + +// Returns its smallest positive argument. If neither argument is positive, +// returns an arbitrary nonpositive value. +int MinPositive(int a, int b) { + if (a <= 0) { + return b; + } + if (b <= 0) { + return a; + } + return std::min(a, b); +} + +// `max_send_bitrate_bps` is the bitrate from "b=" in SDP. +// `rtp_max_bitrate_bps` is the bitrate from RtpSender::SetParameters. +absl::optional ComputeSendBitrate(int max_send_bitrate_bps, + absl::optional rtp_max_bitrate_bps, + const webrtc::AudioCodecSpec& spec) { + // If application-configured bitrate is set, take minimum of that and SDP + // bitrate. + const int bps = rtp_max_bitrate_bps + ? MinPositive(max_send_bitrate_bps, *rtp_max_bitrate_bps) + : max_send_bitrate_bps; + if (bps <= 0) { + return spec.info.default_bitrate_bps; + } + + if (bps < spec.info.min_bitrate_bps) { + // If codec is not multi-rate and `bps` is less than the fixed bitrate then + // fail. If codec is not multi-rate and `bps` exceeds or equal the fixed + // bitrate then ignore. + RTC_LOG(LS_ERROR) << "Failed to set codec " << spec.format.name + << " to bitrate " << bps + << " bps" + ", requires at least " + << spec.info.min_bitrate_bps << " bps."; + return absl::nullopt; + } + + if (spec.info.HasFixedBitrate()) { + return spec.info.default_bitrate_bps; + } else { + // If codec is multi-rate then just set the bitrate. + return std::min(bps, spec.info.max_bitrate_bps); + } +} + +bool IsEnabled(const webrtc::FieldTrialsView& config, absl::string_view trial) { + return absl::StartsWith(config.Lookup(trial), "Enabled"); +} + +struct AdaptivePtimeConfig { + bool enabled = false; + webrtc::DataRate min_payload_bitrate = webrtc::DataRate::KilobitsPerSec(16); + // Value is chosen to ensure FEC can be encoded, see LBRR_WB_MIN_RATE_BPS in + // libopus. + webrtc::DataRate min_encoder_bitrate = webrtc::DataRate::KilobitsPerSec(16); + bool use_slow_adaptation = true; + + absl::optional audio_network_adaptor_config; + + std::unique_ptr Parser() { + return webrtc::StructParametersParser::Create( // + "enabled", &enabled, // + "min_payload_bitrate", &min_payload_bitrate, // + "min_encoder_bitrate", &min_encoder_bitrate, // + "use_slow_adaptation", &use_slow_adaptation); + } + + explicit AdaptivePtimeConfig(const webrtc::FieldTrialsView& trials) { + Parser()->Parse(trials.Lookup("WebRTC-Audio-AdaptivePtime")); + } +}; + +// TODO(tommi): Constructing a receive stream could be made simpler. +// Move some of this boiler plate code into the config structs themselves. +webrtc::AudioReceiveStreamInterface::Config BuildReceiveStreamConfig( + uint32_t remote_ssrc, uint32_t local_ssrc, bool use_nack, + bool enable_non_sender_rtt, const std::vector& stream_ids, + const std::vector& extensions, + webrtc::Transport* rtcp_send_transport, + const rtc::scoped_refptr& decoder_factory, + const std::map& decoder_map, + absl::optional codec_pair_id, + size_t jitter_buffer_max_packets, bool jitter_buffer_fast_accelerate, + int jitter_buffer_min_delay_ms, + rtc::scoped_refptr frame_decryptor, + const webrtc::CryptoOptions& crypto_options, + rtc::scoped_refptr frame_transformer) { + webrtc::AudioReceiveStreamInterface::Config config; + config.rtp.remote_ssrc = remote_ssrc; + config.rtp.local_ssrc = local_ssrc; + config.rtp.nack.rtp_history_ms = use_nack ? kNackRtpHistoryMs : 0; + if (!stream_ids.empty()) { + config.sync_group = stream_ids[0]; + } + config.rtcp_send_transport = rtcp_send_transport; + config.enable_non_sender_rtt = enable_non_sender_rtt; + config.decoder_factory = decoder_factory; + config.decoder_map = decoder_map; + config.codec_pair_id = codec_pair_id; + config.jitter_buffer_max_packets = jitter_buffer_max_packets; + config.jitter_buffer_fast_accelerate = jitter_buffer_fast_accelerate; + config.jitter_buffer_min_delay_ms = jitter_buffer_min_delay_ms; + config.frame_decryptor = std::move(frame_decryptor); + config.crypto_options = crypto_options; + config.frame_transformer = std::move(frame_transformer); + return config; +} + +// Utility function to check if RED codec and its parameters match a codec spec. +bool CheckRedParameters( + const AudioCodec& red_codec, + const webrtc::AudioSendStream::Config::SendCodecSpec& send_codec_spec) { + if (red_codec.clockrate != send_codec_spec.format.clockrate_hz || + red_codec.channels != send_codec_spec.format.num_channels) { + return false; + } + + // Check the FMTP line for the empty parameter which should match + // /[/...] + auto red_parameters = red_codec.params.find(""); + if (red_parameters == red_codec.params.end()) { + RTC_LOG(LS_WARNING) << "audio/RED missing fmtp parameters."; + return false; + } + std::vector redundant_payloads = + rtc::split(red_parameters->second, '/'); + // 32 is chosen as a maximum upper bound for consistency with the + // red payload splitter. + if (redundant_payloads.size() < 2 || redundant_payloads.size() > 32) { + return false; + } + for (auto pt : redundant_payloads) { + if (pt != rtc::ToString(send_codec_spec.payload_type)) { + return false; + } + } + return true; +} + +} // namespace + +CustomWebRtcVoiceEngine::CustomWebRtcVoiceEngine( + webrtc::CreateAudioStateFactory* create_audio_state_factory, + webrtc::TaskQueueFactory* task_queue_factory, + webrtc::AudioDeviceModule* adm, + const rtc::scoped_refptr& encoder_factory, + const rtc::scoped_refptr& decoder_factory, + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing, + std::unique_ptr audio_frame_processor, + const webrtc::FieldTrialsView& trials) + : create_audio_state_factory_(create_audio_state_factory), + task_queue_factory_(task_queue_factory), + adm_(adm), + encoder_factory_(encoder_factory), + decoder_factory_(decoder_factory), + audio_mixer_(audio_mixer), + apm_(audio_processing), + audio_frame_processor_(std::move(audio_frame_processor)), + minimized_remsampling_on_mobile_trial_enabled_( + IsEnabled(trials, "WebRTC-Audio-MinimizeResamplingOnMobile")) { + RTC_LOG(LS_INFO) << "CustomWebRtcVoiceEngine::CustomWebRtcVoiceEngine"; + RTC_DCHECK(decoder_factory); + RTC_DCHECK(encoder_factory); + // The rest of our initialization will happen in Init. +} + +CustomWebRtcVoiceEngine::~CustomWebRtcVoiceEngine() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_LOG(LS_INFO) << "CustomWebRtcVoiceEngine::~CustomWebRtcVoiceEngine"; + if (initialized_) { + StopAecDump(); + + // Stop AudioDevice. + adm()->StopPlayout(); + adm()->StopRecording(); + adm()->RegisterAudioCallback(nullptr); + adm()->Terminate(); + } +} + +void CustomWebRtcVoiceEngine::Init() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_LOG(LS_INFO) << "CustomWebRtcVoiceEngine::Init"; + + // TaskQueue expects to be created/destroyed on the same thread. + RTC_DCHECK(!low_priority_worker_queue_); + low_priority_worker_queue_ = task_queue_factory_->CreateTaskQueue( + "rtc-low-prio", webrtc::TaskQueueFactory::Priority::LOW); + + // Load our audio codec lists. + RTC_LOG(LS_VERBOSE) << "Supported send codecs in order of preference:"; + send_codecs_ = CollectCodecs(encoder_factory_->GetSupportedEncoders()); + for (const AudioCodec& codec : send_codecs_) { + RTC_LOG(LS_VERBOSE) << ToString(codec); + } + + RTC_LOG(LS_VERBOSE) << "Supported recv codecs in order of preference:"; + recv_codecs_ = CollectCodecs(decoder_factory_->GetSupportedDecoders()); + for (const AudioCodec& codec : recv_codecs_) { + RTC_LOG(LS_VERBOSE) << ToString(codec); + } + +#if defined(WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE) + // No ADM supplied? Create a default one. + if (!adm_) { + adm_ = webrtc::AudioDeviceModule::Create( + webrtc::AudioDeviceModule::kPlatformDefaultAudio, task_queue_factory_); + } +#endif // WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE + RTC_CHECK(adm()); + webrtc::adm_helpers::Init(adm()); + + // Set up AudioState. + { + webrtc::AudioState::Config config; + if (audio_mixer_) { + config.audio_mixer = audio_mixer_; + } else { + config.audio_mixer = webrtc::AudioMixerImpl::Create(); + } + config.audio_processing = apm_; + config.audio_device_module = adm_; + if (audio_frame_processor_) { + config.async_audio_processing_factory = + rtc::make_ref_counted( + std::move(audio_frame_processor_), *task_queue_factory_); + } + audio_state_ = create_audio_state_factory_->CreateAudioState(config); + } + + // Connect the ADM to our audio path. + adm()->RegisterAudioCallback(audio_state()->audio_transport()); + + // Set default engine options. + { + AudioOptions options; + options.echo_cancellation = true; + options.auto_gain_control = true; +#if defined(WEBRTC_IOS) + // On iOS, VPIO provides built-in NS. + options.noise_suppression = false; +#else + options.noise_suppression = true; +#endif + options.highpass_filter = true; + options.stereo_swapping = false; + options.audio_jitter_buffer_max_packets = 200; + options.audio_jitter_buffer_fast_accelerate = false; + options.audio_jitter_buffer_min_delay_ms = 0; + ApplyOptions(options); + } + initialized_ = true; +} + +rtc::scoped_refptr CustomWebRtcVoiceEngine::GetAudioState() + const { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return audio_state_; +} + +std::unique_ptr +CustomWebRtcVoiceEngine::CreateSendChannel( + webrtc::Call* call, const MediaConfig& config, const AudioOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::AudioCodecPairId codec_pair_id) { + return std::make_unique( + this, config, options, crypto_options, call, codec_pair_id); +} + +std::unique_ptr +CustomWebRtcVoiceEngine::CreateReceiveChannel( + webrtc::Call* call, const MediaConfig& config, const AudioOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::AudioCodecPairId codec_pair_id) { + return std::make_unique( + this, config, options, crypto_options, call, codec_pair_id); +} + +void CustomWebRtcVoiceEngine::ApplyOptions(const AudioOptions& options_in) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_LOG(LS_INFO) << "CustomWebRtcVoiceEngine::ApplyOptions: " + << options_in.ToString(); + AudioOptions options = options_in; // The options are modified below. + + // Set and adjust echo canceller options. + // Use desktop AEC by default, when not using hardware AEC. + bool use_mobile_software_aec = false; + +#if defined(WEBRTC_IOS) && !TARGET_OS_SIMULATOR + if (options.ios_force_software_aec_HACK && + *options.ios_force_software_aec_HACK) { + // EC may be forced on for a device known to have non-functioning platform + // AEC. + options.echo_cancellation = true; + RTC_LOG(LS_WARNING) + << "Force software AEC on iOS. May conflict with platform AEC."; + } else { + // On iOS, VPIO provides built-in EC. + options.echo_cancellation = false; + RTC_LOG(LS_INFO) << "Always disable AEC on iOS. Use built-in instead."; + } +#elif defined(WEBRTC_ANDROID) + use_mobile_software_aec = true; +#endif + +// Set and adjust gain control options. +#if defined(WEBRTC_IOS) && !TARGET_OS_SIMULATOR + // On iOS, VPIO provides built-in AGC. + options.auto_gain_control = false; + RTC_LOG(LS_INFO) << "Always disable AGC on iOS. Use built-in instead."; +#endif + +#if defined(WEBRTC_IOS) || defined(WEBRTC_ANDROID) + // Turn off the gain control if specified by the field trial. + // The purpose of the field trial is to reduce the amount of resampling + // performed inside the audio processing module on mobile platforms by + // whenever possible turning off the fixed AGC mode and the high-pass filter. + // (https://bugs.chromium.org/p/webrtc/issues/detail?id=6181). + if (minimized_remsampling_on_mobile_trial_enabled_) { + options.auto_gain_control = false; + RTC_LOG(LS_INFO) << "Disable AGC according to field trial."; + if (!(options.noise_suppression.value_or(false) || + options.echo_cancellation.value_or(false))) { + // If possible, turn off the high-pass filter. + RTC_LOG(LS_INFO) + << "Disable high-pass filter in response to field trial."; + options.highpass_filter = false; + } + } +#endif + + if (options.echo_cancellation) { + // Check if platform supports built-in EC. Currently only supported on + // Android and in combination with Java based audio layer. + // TODO(henrika): investigate possibility to support built-in EC also + // in combination with Open SL ES audio. + const bool built_in_aec = adm()->BuiltInAECIsAvailable(); + if (built_in_aec) { + // Built-in EC exists on this device. Enable/Disable it according to the + // echo_cancellation audio option. + const bool enable_built_in_aec = *options.echo_cancellation; + if (adm()->EnableBuiltInAEC(enable_built_in_aec) == 0 && + enable_built_in_aec) { + // Disable internal software EC if built-in EC is enabled, + // i.e., replace the software EC with the built-in EC. + options.echo_cancellation = false; + RTC_LOG(LS_INFO) + << "Disabling EC since built-in EC will be used instead"; + } + } + } + + if (options.auto_gain_control) { + bool built_in_agc_avaliable = adm()->BuiltInAGCIsAvailable(); + if (built_in_agc_avaliable) { + if (adm()->EnableBuiltInAGC(*options.auto_gain_control) == 0 && + *options.auto_gain_control) { + // Disable internal software AGC if built-in AGC is enabled, + // i.e., replace the software AGC with the built-in AGC. + options.auto_gain_control = false; + RTC_LOG(LS_INFO) + << "Disabling AGC since built-in AGC will be used instead"; + } + } + } + + if (options.noise_suppression) { + if (adm()->BuiltInNSIsAvailable()) { + bool builtin_ns = *options.noise_suppression; + if (adm()->EnableBuiltInNS(builtin_ns) == 0 && builtin_ns) { + // Disable internal software NS if built-in NS is enabled, + // i.e., replace the software NS with the built-in NS. + options.noise_suppression = false; + RTC_LOG(LS_INFO) + << "Disabling NS since built-in NS will be used instead"; + } + } + } + + if (options.stereo_swapping) { + audio_state()->SetStereoChannelSwapping(*options.stereo_swapping); + } + + if (options.audio_jitter_buffer_max_packets) { + audio_jitter_buffer_max_packets_ = + std::max(20, *options.audio_jitter_buffer_max_packets); + } + if (options.audio_jitter_buffer_fast_accelerate) { + audio_jitter_buffer_fast_accelerate_ = + *options.audio_jitter_buffer_fast_accelerate; + } + if (options.audio_jitter_buffer_min_delay_ms) { + audio_jitter_buffer_min_delay_ms_ = + *options.audio_jitter_buffer_min_delay_ms; + } + + webrtc::AudioProcessing* ap = apm(); + if (!ap) { + return; + } + + webrtc::AudioProcessing::Config apm_config = ap->GetConfig(); + + if (options.echo_cancellation) { + apm_config.echo_canceller.enabled = *options.echo_cancellation; + apm_config.echo_canceller.mobile_mode = use_mobile_software_aec; + } + + if (options.auto_gain_control) { + const bool enabled = *options.auto_gain_control; + apm_config.gain_controller1.enabled = enabled; +#if defined(WEBRTC_IOS) || defined(WEBRTC_ANDROID) + apm_config.gain_controller1.mode = + apm_config.gain_controller1.kFixedDigital; +#else + apm_config.gain_controller1.mode = + apm_config.gain_controller1.kAdaptiveAnalog; +#endif + } + + if (options.highpass_filter) { + apm_config.high_pass_filter.enabled = *options.highpass_filter; + } + + if (options.noise_suppression) { + const bool enabled = *options.noise_suppression; + apm_config.noise_suppression.enabled = enabled; + apm_config.noise_suppression.level = + webrtc::AudioProcessing::Config::NoiseSuppression::Level::kHigh; + } + + ap->ApplyConfig(apm_config); +} + +const std::vector& CustomWebRtcVoiceEngine::send_codecs() const { + RTC_DCHECK(signal_thread_checker_.IsCurrent()); + return send_codecs_; +} + +const std::vector& CustomWebRtcVoiceEngine::recv_codecs() const { + RTC_DCHECK(signal_thread_checker_.IsCurrent()); + return recv_codecs_; +} + +std::vector +CustomWebRtcVoiceEngine::GetRtpHeaderExtensions() const { + RTC_DCHECK(signal_thread_checker_.IsCurrent()); + std::vector result; + // id is *not* incremented for non-default extensions, UsedIds needs to + // resolve conflicts. + int id = 1; + for (const auto& uri : {webrtc::RtpExtension::kAudioLevelUri, + webrtc::RtpExtension::kAbsSendTimeUri, + webrtc::RtpExtension::kTransportSequenceNumberUri, + webrtc::RtpExtension::kMidUri}) { + result.emplace_back(uri, id++, webrtc::RtpTransceiverDirection::kSendRecv); + } + for (const auto& uri : {webrtc::RtpExtension::kAbsoluteCaptureTimeUri}) { + result.emplace_back(uri, id, webrtc::RtpTransceiverDirection::kStopped); + } + return result; +} + +bool CustomWebRtcVoiceEngine::StartAecDump(webrtc::FileWrapper file, + int64_t max_size_bytes) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + + webrtc::AudioProcessing* ap = apm(); + if (!ap) { + RTC_LOG(LS_WARNING) + << "Attempting to start aecdump when no audio processing module is " + "present, hence no aecdump is started."; + return false; + } + + return ap->CreateAndAttachAecDump(file.Release(), max_size_bytes, + low_priority_worker_queue_.get()); +} + +void CustomWebRtcVoiceEngine::StopAecDump() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + webrtc::AudioProcessing* ap = apm(); + if (ap) { + ap->DetachAecDump(); + } else { + RTC_LOG(LS_WARNING) << "Attempting to stop aecdump when no audio " + "processing module is present"; + } +} + +absl::optional +CustomWebRtcVoiceEngine::GetAudioDeviceStats() { + return adm()->GetStats(); +} + +webrtc::AudioDeviceModule* CustomWebRtcVoiceEngine::adm() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_DCHECK(adm_); + return adm_.get(); +} + +webrtc::AudioProcessing* CustomWebRtcVoiceEngine::apm() const { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return apm_.get(); +} + +webrtc::AudioState* CustomWebRtcVoiceEngine::audio_state() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_DCHECK(audio_state_); + return audio_state_.get(); +} + +std::vector CustomWebRtcVoiceEngine::CollectCodecs( + const std::vector& specs) const { + PayloadTypeMapper mapper; + std::vector out; + + // Only generate CN payload types for these clockrates: + std::map> generate_cn = { + {8000, false}, {16000, false}, {32000, false}}; + // Only generate telephone-event payload types for these clockrates: + std::map> generate_dtmf = { + {8000, false}, {16000, false}, {32000, false}, {48000, false}}; + + auto map_format = [&mapper](const webrtc::SdpAudioFormat& format, + std::vector* out) { + absl::optional opt_codec = mapper.ToAudioCodec(format); + if (opt_codec) { + if (out) { + out->push_back(*opt_codec); + } + } else { + RTC_LOG(LS_ERROR) << "Unable to assign payload type to format: " + << rtc::ToString(format); + } + + return opt_codec; + }; + + for (const auto& spec : specs) { + // We need to do some extra stuff before adding the main codecs to out. + absl::optional opt_codec = map_format(spec.format, nullptr); + if (opt_codec) { + AudioCodec& codec = *opt_codec; + if (spec.info.supports_network_adaption) { + codec.AddFeedbackParam( + FeedbackParam(kRtcpFbParamTransportCc, kParamValueEmpty)); + } + + if (spec.info.allow_comfort_noise) { + // Generate a CN entry if the decoder allows it and we support the + // clockrate. + auto cn = generate_cn.find(spec.format.clockrate_hz); + if (cn != generate_cn.end()) { + cn->second = true; + } + } + + // Generate a telephone-event entry if we support the clockrate. + auto dtmf = generate_dtmf.find(spec.format.clockrate_hz); + if (dtmf != generate_dtmf.end()) { + dtmf->second = true; + } + + out.push_back(codec); + + if (codec.name == kOpusCodecName) { + std::string red_fmtp = + rtc::ToString(codec.id) + "/" + rtc::ToString(codec.id); + map_format({kRedCodecName, 48000, 2, {{"", red_fmtp}}}, &out); + } + } + } + + // Add CN codecs after "proper" audio codecs. + for (const auto& cn : generate_cn) { + if (cn.second) { + map_format({kCnCodecName, cn.first, 1}, &out); + } + } + + // Add telephone-event codecs last. + for (const auto& dtmf : generate_dtmf) { + if (dtmf.second) { + map_format({kDtmfCodecName, dtmf.first, 1}, &out); + } + } + + return out; +} + +// --------------------------------- WebRtcVoiceSendChannel2 ------------------ + +class WebRtcVoiceSendChannel2::WebRtcAudioSendStream + : public AudioSource::Sink { + public: + WebRtcAudioSendStream( + uint32_t ssrc, const std::string& mid, const std::string& c_name, + const std::string track_id, + const absl::optional& + send_codec_spec, + bool extmap_allow_mixed, + const std::vector& extensions, + int max_send_bitrate_bps, int rtcp_report_interval_ms, + const absl::optional& audio_network_adaptor_config, + webrtc::Call* call, webrtc::Transport* send_transport, + const rtc::scoped_refptr& encoder_factory, + const absl::optional codec_pair_id, + rtc::scoped_refptr frame_encryptor, + const webrtc::CryptoOptions& crypto_options) + : adaptive_ptime_config_(call->trials()), + call_(call), + config_(send_transport), + max_send_bitrate_bps_(max_send_bitrate_bps), + rtp_parameters_(CreateRtpParametersWithOneEncoding()) { + RTC_DCHECK(call); + RTC_DCHECK(encoder_factory); + config_.rtp.ssrc = ssrc; + config_.rtp.mid = mid; + config_.rtp.c_name = c_name; + config_.rtp.extmap_allow_mixed = extmap_allow_mixed; + config_.rtp.extensions = extensions; + config_.has_dscp = + rtp_parameters_.encodings[0].network_priority != webrtc::Priority::kLow; + config_.encoder_factory = encoder_factory; + config_.codec_pair_id = codec_pair_id; + config_.track_id = track_id; + config_.frame_encryptor = frame_encryptor; + config_.crypto_options = crypto_options; + config_.rtcp_report_interval_ms = rtcp_report_interval_ms; + rtp_parameters_.encodings[0].ssrc = ssrc; + rtp_parameters_.rtcp.cname = c_name; + rtp_parameters_.header_extensions = extensions; + + audio_network_adaptor_config_from_options_ = audio_network_adaptor_config; + UpdateAudioNetworkAdaptorConfig(); + + if (send_codec_spec) { + UpdateSendCodecSpec(*send_codec_spec); + } + + stream_ = call_->CreateAudioSendStream(config_); + } + + WebRtcAudioSendStream() = delete; + WebRtcAudioSendStream(const WebRtcAudioSendStream&) = delete; + WebRtcAudioSendStream& operator=(const WebRtcAudioSendStream&) = delete; + + ~WebRtcAudioSendStream() override { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + ClearSource(); + call_->DestroyAudioSendStream(stream_); + } + + void SetSendCodecSpec( + const webrtc::AudioSendStream::Config::SendCodecSpec& send_codec_spec) { + UpdateSendCodecSpec(send_codec_spec); + ReconfigureAudioSendStream(nullptr); + } + + void SetRtpExtensions(const std::vector& extensions) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + config_.rtp.extensions = extensions; + rtp_parameters_.header_extensions = extensions; + ReconfigureAudioSendStream(nullptr); + } + + void SetExtmapAllowMixed(bool extmap_allow_mixed) { + config_.rtp.extmap_allow_mixed = extmap_allow_mixed; + ReconfigureAudioSendStream(nullptr); + } + + void SetMid(const std::string& mid) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + if (config_.rtp.mid == mid) { + return; + } + config_.rtp.mid = mid; + ReconfigureAudioSendStream(nullptr); + } + + void SetFrameEncryptor( + rtc::scoped_refptr frame_encryptor) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + config_.frame_encryptor = frame_encryptor; + ReconfigureAudioSendStream(nullptr); + } + + void SetAudioNetworkAdaptorConfig( + const absl::optional& audio_network_adaptor_config) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + if (audio_network_adaptor_config_from_options_ == + audio_network_adaptor_config) { + return; + } + audio_network_adaptor_config_from_options_ = audio_network_adaptor_config; + UpdateAudioNetworkAdaptorConfig(); + UpdateAllowedBitrateRange(); + ReconfigureAudioSendStream(nullptr); + } + + bool SetMaxSendBitrate(int bps) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_DCHECK(config_.send_codec_spec); + RTC_DCHECK(audio_codec_spec_); + auto send_rate = ComputeSendBitrate( + bps, rtp_parameters_.encodings[0].max_bitrate_bps, *audio_codec_spec_); + + if (!send_rate) { + return false; + } + + max_send_bitrate_bps_ = bps; + + if (send_rate != config_.send_codec_spec->target_bitrate_bps) { + config_.send_codec_spec->target_bitrate_bps = send_rate; + ReconfigureAudioSendStream(nullptr); + } + return true; + } + + bool SendTelephoneEvent(int payload_type, int payload_freq, int event, + int duration_ms) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_DCHECK(stream_); + return stream_->SendTelephoneEvent(payload_type, payload_freq, event, + duration_ms); + } + + void SetSend(bool send) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + send_ = send; + UpdateSendState(); + } + + void SetMuted(bool muted) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_DCHECK(stream_); + stream_->SetMuted(muted); + muted_ = muted; + } + + bool muted() const { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return muted_; + } + + webrtc::AudioSendStream::Stats GetStats(bool has_remote_tracks) const { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_DCHECK(stream_); + return stream_->GetStats(has_remote_tracks); + } + + // Starts the sending by setting ourselves as a sink to the AudioSource to + // get data callbacks. + // This method is called on the libjingle worker thread. + // TODO(xians): Make sure Start() is called only once. + void SetSource(AudioSource* source) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_DCHECK(source); + if (source_) { + RTC_DCHECK(source_ == source); + return; + } + source->SetSink(this); + source_ = source; + UpdateSendState(); + } + + // Stops sending by setting the sink of the AudioSource to nullptr. No data + // callback will be received after this method. + // This method is called on the libjingle worker thread. + void ClearSource() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + if (source_) { + source_->SetSink(nullptr); + source_ = nullptr; + } + UpdateSendState(); + } + + // AudioSource::Sink implementation. + // This method is called on the audio thread. + void OnData(const void* audio_data, int bits_per_sample, int sample_rate, + size_t number_of_channels, size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) override { + TRACE_EVENT_BEGIN2("webrtc", "WebRtcAudioSendStream::OnData", "sample_rate", + sample_rate, "number_of_frames", number_of_frames); + RTC_DCHECK_EQ(16, bits_per_sample); + RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_); + RTC_DCHECK(stream_); + std::unique_ptr audio_frame(new webrtc::AudioFrame()); + audio_frame->UpdateFrame( + audio_frame->timestamp_, static_cast(audio_data), + number_of_frames, sample_rate, audio_frame->speech_type_, + audio_frame->vad_activity_, number_of_channels); + // TODO(bugs.webrtc.org/10739): add dcheck that + // `absolute_capture_timestamp_ms` always receives a value. + if (absolute_capture_timestamp_ms) { + audio_frame->set_absolute_capture_timestamp_ms( + *absolute_capture_timestamp_ms); + } + stream_->SendAudioData(std::move(audio_frame)); + TRACE_EVENT_END1("webrtc", "WebRtcAudioSendStream::OnData", + "number_of_channels", number_of_channels); + } + + // Callback from the `source_` when it is going away. In case Start() has + // never been called, this callback won't be triggered. + void OnClose() override { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + // Set `source_` to nullptr to make sure no more callback will get into + // the source. + source_ = nullptr; + UpdateSendState(); + } + + const webrtc::RtpParameters& rtp_parameters() const { + return rtp_parameters_; + } + + webrtc::RTCError SetRtpParameters(const webrtc::RtpParameters& parameters, + webrtc::SetParametersCallback callback) { + webrtc::RTCError error = CheckRtpParametersInvalidModificationAndValues( + rtp_parameters_, parameters); + if (!error.ok()) { + return webrtc::InvokeSetParametersCallback(callback, error); + } + + absl::optional send_rate; + if (audio_codec_spec_) { + send_rate = ComputeSendBitrate(max_send_bitrate_bps_, + parameters.encodings[0].max_bitrate_bps, + *audio_codec_spec_); + if (!send_rate) { + return webrtc::InvokeSetParametersCallback( + callback, webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR)); + } + } + + const absl::optional old_rtp_max_bitrate = + rtp_parameters_.encodings[0].max_bitrate_bps; + double old_priority = rtp_parameters_.encodings[0].bitrate_priority; + webrtc::Priority old_dscp = rtp_parameters_.encodings[0].network_priority; + bool old_adaptive_ptime = rtp_parameters_.encodings[0].adaptive_ptime; + rtp_parameters_ = parameters; + config_.bitrate_priority = rtp_parameters_.encodings[0].bitrate_priority; + config_.has_dscp = (rtp_parameters_.encodings[0].network_priority != + webrtc::Priority::kLow); + + bool reconfigure_send_stream = + (rtp_parameters_.encodings[0].max_bitrate_bps != old_rtp_max_bitrate) || + (rtp_parameters_.encodings[0].bitrate_priority != old_priority) || + (rtp_parameters_.encodings[0].network_priority != old_dscp) || + (rtp_parameters_.encodings[0].adaptive_ptime != old_adaptive_ptime); + if (rtp_parameters_.encodings[0].max_bitrate_bps != old_rtp_max_bitrate) { + // Update the bitrate range. + if (send_rate) { + config_.send_codec_spec->target_bitrate_bps = send_rate; + } + } + if (reconfigure_send_stream) { + // Changing adaptive_ptime may update the audio network adaptor config + // used. + UpdateAudioNetworkAdaptorConfig(); + UpdateAllowedBitrateRange(); + ReconfigureAudioSendStream(std::move(callback)); + } else { + webrtc::InvokeSetParametersCallback(callback, webrtc::RTCError::OK()); + } + + rtp_parameters_.rtcp.cname = config_.rtp.c_name; + rtp_parameters_.rtcp.reduced_size = false; + + // parameters.encodings[0].active could have changed. + UpdateSendState(); + return webrtc::RTCError::OK(); + } + + void SetEncoderToPacketizerFrameTransformer( + rtc::scoped_refptr frame_transformer) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + config_.frame_transformer = std::move(frame_transformer); + ReconfigureAudioSendStream(nullptr); + } + + private: + void UpdateSendState() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_DCHECK(stream_); + RTC_DCHECK_EQ(1UL, rtp_parameters_.encodings.size()); + // Stream can be started without |source_| being set. + if (send_ && rtp_parameters_.encodings[0].active) { + stream_->Start(); + } else { + stream_->Stop(); + } + } + + void UpdateAllowedBitrateRange() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + // The order of precedence, from lowest to highest is: + // - a reasonable default of 32kbps min/max + // - fixed target bitrate from codec spec + // - lower min bitrate if adaptive ptime is enabled + const int kDefaultBitrateBps = 32000; + config_.min_bitrate_bps = kDefaultBitrateBps; + config_.max_bitrate_bps = kDefaultBitrateBps; + + if (config_.send_codec_spec && + config_.send_codec_spec->target_bitrate_bps) { + config_.min_bitrate_bps = *config_.send_codec_spec->target_bitrate_bps; + config_.max_bitrate_bps = *config_.send_codec_spec->target_bitrate_bps; + } + + if (rtp_parameters_.encodings[0].adaptive_ptime) { + config_.min_bitrate_bps = std::min( + config_.min_bitrate_bps, + static_cast(adaptive_ptime_config_.min_encoder_bitrate.bps())); + } + } + + void UpdateSendCodecSpec( + const webrtc::AudioSendStream::Config::SendCodecSpec& send_codec_spec) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + config_.send_codec_spec = send_codec_spec; + auto info = + config_.encoder_factory->QueryAudioEncoder(send_codec_spec.format); + RTC_DCHECK(info); + // If a specific target bitrate has been set for the stream, use that as + // the new default bitrate when computing send bitrate. + if (send_codec_spec.target_bitrate_bps) { + info->default_bitrate_bps = std::max( + info->min_bitrate_bps, + std::min(info->max_bitrate_bps, *send_codec_spec.target_bitrate_bps)); + } + + audio_codec_spec_.emplace( + webrtc::AudioCodecSpec{send_codec_spec.format, *info}); + + config_.send_codec_spec->target_bitrate_bps = ComputeSendBitrate( + max_send_bitrate_bps_, rtp_parameters_.encodings[0].max_bitrate_bps, + *audio_codec_spec_); + + UpdateAllowedBitrateRange(); + + // Encoder will only use two channels if the stereo parameter is set. + const auto& it = send_codec_spec.format.parameters.find("stereo"); + if (it != send_codec_spec.format.parameters.end() && it->second == "1") { + num_encoded_channels_ = 2; + } else { + num_encoded_channels_ = 1; + } + } + + void UpdateAudioNetworkAdaptorConfig() { + if (adaptive_ptime_config_.enabled || + rtp_parameters_.encodings[0].adaptive_ptime) { + config_.audio_network_adaptor_config = + adaptive_ptime_config_.audio_network_adaptor_config; + return; + } + config_.audio_network_adaptor_config = + audio_network_adaptor_config_from_options_; + } + + void ReconfigureAudioSendStream(webrtc::SetParametersCallback callback) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + RTC_DCHECK(stream_); + stream_->Reconfigure(config_, std::move(callback)); + } + + int NumPreferredChannels() const override { return num_encoded_channels_; } + + const AdaptivePtimeConfig adaptive_ptime_config_; + webrtc::SequenceChecker worker_thread_checker_; + rtc::RaceChecker audio_capture_race_checker_; + webrtc::Call* call_ = nullptr; + webrtc::AudioSendStream::Config config_; + // The stream is owned by WebRtcAudioSendStream and may be reallocated if + // configuration changes. + webrtc::AudioSendStream* stream_ = nullptr; + + // Raw pointer to AudioSource owned by LocalAudioTrackHandler. + // PeerConnection will make sure invalidating the pointer before the object + // goes away. + AudioSource* source_ = nullptr; + bool send_ = false; + bool muted_ = false; + int max_send_bitrate_bps_; + webrtc::RtpParameters rtp_parameters_; + absl::optional audio_codec_spec_; + // TODO(webrtc:11717): Remove this once audio_network_adaptor in AudioOptions + // has been removed. + absl::optional audio_network_adaptor_config_from_options_; + std::atomic num_encoded_channels_{-1}; +}; + +WebRtcVoiceSendChannel2::WebRtcVoiceSendChannel2( + CustomWebRtcVoiceEngine* engine, const MediaConfig& config, + const AudioOptions& options, const webrtc::CryptoOptions& crypto_options, + webrtc::Call* call, webrtc::AudioCodecPairId codec_pair_id) + : MediaChannelUtil(call->network_thread(), config.enable_dscp), + worker_thread_(call->worker_thread()), + engine_(engine), + call_(call), + audio_config_(config.audio), + codec_pair_id_(codec_pair_id), + crypto_options_(crypto_options) { + RTC_LOG(LS_VERBOSE) << "WebRtcVoiceSendChannel2::WebRtcVoiceSendChannel2"; + RTC_DCHECK(call); + SetOptions(options); +} + +WebRtcVoiceSendChannel2::~WebRtcVoiceSendChannel2() { + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_DLOG(LS_VERBOSE) << "WebRtcVoiceSendChannel2::~WebRtcVoiceSendChannel2"; + // TODO(solenberg): Should be able to delete the streams directly, without + // going through RemoveNnStream(), once stream objects handle + // all (de)configuration. + while (!send_streams_.empty()) { + RemoveSendStream(send_streams_.begin()->first); + } +} + +bool WebRtcVoiceSendChannel2::SetOptions(const AudioOptions& options) { + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_LOG(LS_INFO) << "Setting voice channel options: " << options.ToString(); + + // We retain all of the existing options, and apply the given ones + // on top. This means there is no way to "clear" options such that + // they go back to the engine default. + options_.SetAll(options); + engine()->ApplyOptions(options_); + + absl::optional audio_network_adaptor_config = + GetAudioNetworkAdaptorConfig(options_); + for (auto& it : send_streams_) { + it.second->SetAudioNetworkAdaptorConfig(audio_network_adaptor_config); + } + + RTC_LOG(LS_INFO) << "Set voice send channel options. Current options: " + << options_.ToString(); + return true; +} + +bool WebRtcVoiceSendChannel2::SetSenderParameters( + const AudioSenderParameter& params) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::SetSenderParameters"); + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_LOG(LS_INFO) << "WebRtcVoiceMediaChannel::SetSenderParameters: " + << params.ToString(); + // TODO(pthatcher): Refactor this to be more clean now that we have + // all the information at once. + + // Finding if the RtpParameters force a specific codec + absl::optional force_codec; + if (send_streams_.size() == 1) { + // Since audio simulcast is not supported, currently, only PlanB + // has multiple tracks and we don't care about getting the + // functionality working there properly. + auto rtp_parameters = send_streams_.begin()->second->rtp_parameters(); + if (rtp_parameters.encodings[0].codec) { + auto matched_codec = + absl::c_find_if(params.codecs, [&](auto negotiated_codec) { + return negotiated_codec.MatchesRtpCodec( + *rtp_parameters.encodings[0].codec); + }); + if (matched_codec != params.codecs.end()) { + force_codec = *matched_codec; + } else { + // The requested codec has been negotiated away, we clear it from the + // parameters. + for (auto& encoding : rtp_parameters.encodings) { + encoding.codec.reset(); + } + send_streams_.begin()->second->SetRtpParameters(rtp_parameters, + nullptr); + } + } + } + + if (!SetSendCodecs(params.codecs, force_codec)) { + return false; + } + + if (!ValidateRtpExtensions(params.extensions, send_rtp_extensions_)) { + return false; + } + + if (ExtmapAllowMixed() != params.extmap_allow_mixed) { + SetExtmapAllowMixed(params.extmap_allow_mixed); + for (auto& it : send_streams_) { + it.second->SetExtmapAllowMixed(params.extmap_allow_mixed); + } + } + + std::vector filtered_extensions = FilterRtpExtensions( + params.extensions, webrtc::RtpExtension::IsSupportedForAudio, true, + call_->trials()); + if (send_rtp_extensions_ != filtered_extensions) { + send_rtp_extensions_.swap(filtered_extensions); + for (auto& it : send_streams_) { + it.second->SetRtpExtensions(send_rtp_extensions_); + } + } + if (!params.mid.empty()) { + mid_ = params.mid; + for (auto& it : send_streams_) { + it.second->SetMid(params.mid); + } + } + + if (send_codec_spec_ && !SetMaxSendBitrate(params.max_bandwidth_bps)) { + return false; + } + return SetOptions(params.options); +} + +absl::optional WebRtcVoiceSendChannel2::GetSendCodec() const { + if (send_codec_spec_) { + return CreateAudioCodec(send_codec_spec_->format); + } + return absl::nullopt; +} + +// Utility function called from SetSenderParameters() to extract current send +// codec settings from the given list of codecs (originally from SDP). Both send +// and receive streams may be reconfigured based on the new settings. +bool WebRtcVoiceSendChannel2::SetSendCodecs( + const std::vector& codecs, absl::optional preferred_codec) { + RTC_DCHECK_RUN_ON(worker_thread_); + dtmf_payload_type_ = absl::nullopt; + dtmf_payload_freq_ = -1; + + // Validate supplied codecs list. + for (const Codec& codec : codecs) { + // TODO(solenberg): Validate more aspects of input - that payload types + // don't overlap, remove redundant/unsupported codecs etc - + // the same way it is done for RtpHeaderExtensions. + if (codec.id < kMinPayloadType || codec.id > kMaxPayloadType) { + RTC_LOG(LS_WARNING) << "Codec payload type out of range: " + << ToString(codec); + return false; + } + } + + // Find PT of telephone-event codec with lowest clockrate, as a fallback, in + // case we don't have a DTMF codec with a rate matching the send codec's, or + // if this function returns early. + std::vector dtmf_codecs; + for (const Codec& codec : codecs) { + if (IsCodec(codec, kDtmfCodecName)) { + dtmf_codecs.push_back(codec); + if (!dtmf_payload_type_ || codec.clockrate < dtmf_payload_freq_) { + dtmf_payload_type_ = codec.id; + dtmf_payload_freq_ = codec.clockrate; + } + } + } + + // Scan through the list to figure out the codec to use for sending. + absl::optional + send_codec_spec; + webrtc::BitrateConstraints bitrate_config; + absl::optional voice_codec_info; + size_t send_codec_position = 0; + for (const Codec& voice_codec : codecs) { + if (!(IsCodec(voice_codec, kCnCodecName) || + IsCodec(voice_codec, kDtmfCodecName) || + IsCodec(voice_codec, kRedCodecName)) && + (!preferred_codec || preferred_codec->Matches(voice_codec))) { + webrtc::SdpAudioFormat format(voice_codec.name, voice_codec.clockrate, + voice_codec.channels, voice_codec.params); + + voice_codec_info = engine()->encoder_factory_->QueryAudioEncoder(format); + if (!voice_codec_info) { + RTC_LOG(LS_WARNING) << "Unknown codec " << ToString(voice_codec); + continue; + } + + send_codec_spec = webrtc::AudioSendStream::Config::SendCodecSpec( + voice_codec.id, format); + if (voice_codec.bitrate > 0) { + send_codec_spec->target_bitrate_bps = voice_codec.bitrate; + } + send_codec_spec->transport_cc_enabled = HasTransportCc(voice_codec); + send_codec_spec->nack_enabled = HasNack(voice_codec); + send_codec_spec->enable_non_sender_rtt = HasRrtr(voice_codec); + bitrate_config = GetBitrateConfigForCodec(voice_codec); + break; + } + send_codec_position++; + } + + if (!send_codec_spec) { + // No codecs in common, bail out early. + return true; + } + + RTC_DCHECK(voice_codec_info); + if (voice_codec_info->allow_comfort_noise) { + // Loop through the codecs list again to find the CN codec. + // TODO(solenberg): Break out into a separate function? + for (const Codec& cn_codec : codecs) { + if (IsCodec(cn_codec, kCnCodecName) && + cn_codec.clockrate == send_codec_spec->format.clockrate_hz && + cn_codec.channels == voice_codec_info->num_channels) { + if (cn_codec.channels != 1) { + RTC_LOG(LS_WARNING) + << "CN #channels " << cn_codec.channels << " not supported."; + } else if (cn_codec.clockrate != 8000 && cn_codec.clockrate != 16000 && + cn_codec.clockrate != 32000) { + RTC_LOG(LS_WARNING) + << "CN frequency " << cn_codec.clockrate << " not supported."; + } else { + send_codec_spec->cng_payload_type = cn_codec.id; + } + break; + } + } + + // Find the telephone-event PT exactly matching the preferred send codec. + for (const Codec& dtmf_codec : dtmf_codecs) { + if (dtmf_codec.clockrate == send_codec_spec->format.clockrate_hz) { + dtmf_payload_type_ = dtmf_codec.id; + dtmf_payload_freq_ = dtmf_codec.clockrate; + break; + } + } + } + + // Loop through the codecs to find the RED codec that matches opus + // with respect to clockrate and number of channels. + // RED codec needs to be negotiated before the actual codec they + // reference. + for (size_t i = 0; i < send_codec_position; ++i) { + const Codec& red_codec = codecs[i]; + if (IsCodec(red_codec, kRedCodecName) && + CheckRedParameters(red_codec, *send_codec_spec)) { + send_codec_spec->red_payload_type = red_codec.id; + break; + } + } + + if (send_codec_spec_ != send_codec_spec) { + send_codec_spec_ = std::move(send_codec_spec); + // Apply new settings to all streams. + for (const auto& kv : send_streams_) { + kv.second->SetSendCodecSpec(*send_codec_spec_); + } + } else { + // If the codec isn't changing, set the start bitrate to -1 which means + // "unchanged" so that BWE isn't affected. + bitrate_config.start_bitrate_bps = -1; + } + call_->GetTransportControllerSend()->SetSdpBitrateParameters(bitrate_config); + + send_codecs_ = codecs; + + if (send_codec_changed_callback_) { + send_codec_changed_callback_(); + } + + return true; +} + +void WebRtcVoiceSendChannel2::SetSend(bool send) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::SetSend"); + if (send_ == send) { + return; + } + + // Apply channel specific options. + if (send) { + engine()->ApplyOptions(options_); + + // Initialize the ADM for recording (this may take time on some platforms, + // e.g. Android). + if (options_.init_recording_on_send.value_or(true) && + // InitRecording() may return an error if the ADM is already recording. + !engine()->adm()->RecordingIsInitialized() && + !engine()->adm()->Recording()) { + if (engine()->adm()->InitRecording() != 0) { + RTC_LOG(LS_WARNING) << "Failed to initialize recording"; + } + } + } + + // Change the settings on each send channel. + for (auto& kv : send_streams_) { + kv.second->SetSend(send); + } + + send_ = send; +} + +bool WebRtcVoiceSendChannel2::SetAudioSend(uint32_t ssrc, bool enable, + const AudioOptions* options, + AudioSource* source) { + RTC_DCHECK_RUN_ON(worker_thread_); + // TODO(solenberg): The state change should be fully rolled back if any one of + // these calls fail. + if (!SetLocalSource(ssrc, source)) { + return false; + } + if (!MuteStream(ssrc, !enable)) { + return false; + } + if (enable && options) { + return SetOptions(*options); + } + return true; +} + +bool WebRtcVoiceSendChannel2::AddSendStream(const StreamParams& sp) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::AddSendStream"); + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_LOG(LS_INFO) << "AddSendStream: " << sp.ToString(); + + uint32_t ssrc = sp.first_ssrc(); + RTC_DCHECK(0 != ssrc); + + if (send_streams_.find(ssrc) != send_streams_.end()) { + RTC_LOG(LS_ERROR) << "Stream already exists with ssrc " << ssrc; + return false; + } + + absl::optional audio_network_adaptor_config = + GetAudioNetworkAdaptorConfig(options_); + WebRtcAudioSendStream* stream = new WebRtcAudioSendStream( + ssrc, mid_, sp.cname, sp.id, send_codec_spec_, ExtmapAllowMixed(), + send_rtp_extensions_, max_send_bitrate_bps_, + audio_config_.rtcp_report_interval_ms, audio_network_adaptor_config, + call_, transport(), engine()->encoder_factory_, codec_pair_id_, nullptr, + crypto_options_); + send_streams_.insert(std::make_pair(ssrc, stream)); + if (ssrc_list_changed_callback_) { + std::set ssrcs_in_use; + for (auto it : send_streams_) { + ssrcs_in_use.insert(it.first); + } + ssrc_list_changed_callback_(ssrcs_in_use); + } + + send_streams_[ssrc]->SetSend(send_); + return true; +} + +bool WebRtcVoiceSendChannel2::RemoveSendStream(uint32_t ssrc) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::RemoveSendStream"); + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_LOG(LS_INFO) << "RemoveSendStream: " << ssrc; + + auto it = send_streams_.find(ssrc); + if (it == send_streams_.end()) { + RTC_LOG(LS_WARNING) << "Try to remove stream with ssrc " << ssrc + << " which doesn't exist."; + return false; + } + + it->second->SetSend(false); + + // TODO(solenberg): If we're removing the receiver_reports_ssrc_ stream, find + // the first active send stream and use that instead, reassociating receive + // streams. + + delete it->second; + send_streams_.erase(it); + if (send_streams_.empty()) { + SetSend(false); + } + return true; +} + +void WebRtcVoiceSendChannel2::SetSsrcListChangedCallback( + absl::AnyInvocable&)> callback) { + ssrc_list_changed_callback_ = std::move(callback); +} + +bool WebRtcVoiceSendChannel2::SetLocalSource(uint32_t ssrc, + AudioSource* source) { + auto it = send_streams_.find(ssrc); + if (it == send_streams_.end()) { + if (source) { + // Return an error if trying to set a valid source with an invalid ssrc. + RTC_LOG(LS_ERROR) << "SetLocalSource failed with ssrc " << ssrc; + return false; + } + + // The channel likely has gone away, do nothing. + return true; + } + + if (source) { + it->second->SetSource(source); + } else { + it->second->ClearSource(); + } + + return true; +} + +bool WebRtcVoiceSendChannel2::CanInsertDtmf() { + return dtmf_payload_type_.has_value() && send_; +} + +void WebRtcVoiceSendChannel2::SetFrameEncryptor( + uint32_t ssrc, + rtc::scoped_refptr frame_encryptor) { + RTC_DCHECK_RUN_ON(worker_thread_); + auto matching_stream = send_streams_.find(ssrc); + if (matching_stream != send_streams_.end()) { + matching_stream->second->SetFrameEncryptor(frame_encryptor); + } +} + +bool WebRtcVoiceSendChannel2::InsertDtmf(uint32_t ssrc, int event, + int duration) { + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_LOG(LS_INFO) << "WebRtcVoiceMediaChannel::InsertDtmf"; + if (!CanInsertDtmf()) { + return false; + } + + // Figure out which WebRtcAudioSendStream to send the event on. + auto it = ssrc != 0 ? send_streams_.find(ssrc) : send_streams_.begin(); + if (it == send_streams_.end()) { + RTC_LOG(LS_WARNING) << "The specified ssrc " << ssrc << " is not in use."; + return false; + } + if (event < kMinTelephoneEventCode || event > kMaxTelephoneEventCode) { + RTC_LOG(LS_WARNING) << "DTMF event code " << event << " out of range."; + return false; + } + RTC_DCHECK_NE(-1, dtmf_payload_freq_); + return it->second->SendTelephoneEvent(*dtmf_payload_type_, dtmf_payload_freq_, + event, duration); +} + +void WebRtcVoiceSendChannel2::OnPacketSent(const rtc::SentPacket& sent_packet) { + RTC_DCHECK_RUN_ON(&network_thread_checker_); + // TODO(tommi): We shouldn't need to go through call_ to deliver this + // notification. We should already have direct access to + // video_send_delay_stats_ and transport_send_ptr_ via `stream_`. + // So we should be able to remove OnSentPacket from Call and handle this per + // channel instead. At the moment Call::OnSentPacket calls OnSentPacket for + // the video stats, which we should be able to skip. + call_->OnSentPacket(sent_packet); +} + +void WebRtcVoiceSendChannel2::OnNetworkRouteChanged( + absl::string_view transport_name, const rtc::NetworkRoute& network_route) { + RTC_DCHECK_RUN_ON(&network_thread_checker_); + + call_->OnAudioTransportOverheadChanged(network_route.packet_overhead); + + worker_thread_->PostTask(SafeTask( + task_safety_.flag(), + [this, name = std::string(transport_name), route = network_route] { + RTC_DCHECK_RUN_ON(worker_thread_); + call_->GetTransportControllerSend()->OnNetworkRouteChanged(name, route); + })); +} + +bool WebRtcVoiceSendChannel2::MuteStream(uint32_t ssrc, bool muted) { + RTC_DCHECK_RUN_ON(worker_thread_); + const auto it = send_streams_.find(ssrc); + if (it == send_streams_.end()) { + RTC_LOG(LS_WARNING) << "The specified ssrc " << ssrc << " is not in use."; + return false; + } + it->second->SetMuted(muted); + + // TODO(solenberg): + // We set the AGC to mute state only when all the channels are muted. + // This implementation is not ideal, instead we should signal the AGC when + // the mic channel is muted/unmuted. We can't do it today because there + // is no good way to know which stream is mapping to the mic channel. + bool all_muted = muted; + for (const auto& kv : send_streams_) { + all_muted = all_muted && kv.second->muted(); + } + webrtc::AudioProcessing* ap = engine()->apm(); + if (ap) { + ap->set_output_will_be_muted(all_muted); + } + + // Notfy the AudioState that the mute state has updated. + engine_->audio_state()->OnMuteStreamChanged(); + + return true; +} + +bool WebRtcVoiceSendChannel2::SetMaxSendBitrate(int bps) { + RTC_LOG(LS_INFO) << "WebRtcVoiceMediaChannel::SetMaxSendBitrate."; + max_send_bitrate_bps_ = bps; + bool success = true; + for (const auto& kv : send_streams_) { + if (!kv.second->SetMaxSendBitrate(max_send_bitrate_bps_)) { + success = false; + } + } + return success; +} + +void WebRtcVoiceSendChannel2::OnReadyToSend(bool ready) { + RTC_DCHECK_RUN_ON(&network_thread_checker_); + RTC_LOG(LS_VERBOSE) << "OnReadyToSend: " << (ready ? "Ready." : "Not ready."); + call_->SignalChannelNetworkState( + webrtc::MediaType::AUDIO, + ready ? webrtc::kNetworkUp : webrtc::kNetworkDown); +} + +bool WebRtcVoiceSendChannel2::GetStats(VoiceMediaSendInfo* info) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::GetSendStats"); + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_DCHECK(info); + + // Get SSRC and stats for each sender. + // With separate send and receive channels, we expect GetStats to be called on + // both, and accumulate info, but only one channel (the send one) should have + // senders. + RTC_DCHECK(info->senders.size() == 0U || send_streams_.size() == 0); + for (const auto& stream : send_streams_) { + webrtc::AudioSendStream::Stats stats = stream.second->GetStats(false); + VoiceSenderInfo sinfo; + sinfo.add_ssrc(stats.local_ssrc); + sinfo.payload_bytes_sent = stats.payload_bytes_sent; + sinfo.header_and_padding_bytes_sent = stats.header_and_padding_bytes_sent; + sinfo.retransmitted_bytes_sent = stats.retransmitted_bytes_sent; + sinfo.packets_sent = stats.packets_sent; + sinfo.total_packet_send_delay = stats.total_packet_send_delay; + sinfo.retransmitted_packets_sent = stats.retransmitted_packets_sent; + sinfo.packets_lost = stats.packets_lost; + sinfo.fraction_lost = stats.fraction_lost; + sinfo.nacks_received = stats.nacks_received; + sinfo.target_bitrate = stats.target_bitrate_bps; + sinfo.codec_name = stats.codec_name; + sinfo.codec_payload_type = stats.codec_payload_type; + sinfo.jitter_ms = stats.jitter_ms; + sinfo.rtt_ms = stats.rtt_ms; + sinfo.audio_level = stats.audio_level; + sinfo.total_input_energy = stats.total_input_energy; + sinfo.total_input_duration = stats.total_input_duration; + sinfo.ana_statistics = stats.ana_statistics; + sinfo.apm_statistics = stats.apm_statistics; + sinfo.report_block_datas = std::move(stats.report_block_datas); + + auto encodings = stream.second->rtp_parameters().encodings; + if (!encodings.empty()) { + sinfo.active = encodings[0].active; + } + + info->senders.push_back(sinfo); + } + + FillSendCodecStats(info); + + return true; +} + +void WebRtcVoiceSendChannel2::FillSendCodecStats( + VoiceMediaSendInfo* voice_media_info) { + for (const auto& sender : voice_media_info->senders) { + auto codec = absl::c_find_if(send_codecs_, [&sender](const AudioCodec& c) { + return sender.codec_payload_type && *sender.codec_payload_type == c.id; + }); + if (codec != send_codecs_.end()) { + voice_media_info->send_codecs.insert( + std::make_pair(codec->id, codec->ToCodecParameters())); + } + } +} + +void WebRtcVoiceSendChannel2::SetEncoderToPacketizerFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) { + RTC_DCHECK_RUN_ON(worker_thread_); + auto matching_stream = send_streams_.find(ssrc); + if (matching_stream == send_streams_.end()) { + RTC_LOG(LS_INFO) << "Attempting to set frame transformer for SSRC:" << ssrc + << " which doesn't exist."; + return; + } + matching_stream->second->SetEncoderToPacketizerFrameTransformer( + std::move(frame_transformer)); +} + +webrtc::RtpParameters WebRtcVoiceSendChannel2::GetRtpSendParameters( + uint32_t ssrc) const { + RTC_DCHECK_RUN_ON(worker_thread_); + auto it = send_streams_.find(ssrc); + if (it == send_streams_.end()) { + RTC_LOG(LS_WARNING) << "Attempting to get RTP send parameters for stream " + "with ssrc " + << ssrc << " which doesn't exist."; + return webrtc::RtpParameters(); + } + + webrtc::RtpParameters rtp_params = it->second->rtp_parameters(); + // Need to add the common list of codecs to the send stream-specific + // RTP parameters. + for (const AudioCodec& codec : send_codecs_) { + rtp_params.codecs.push_back(codec.ToCodecParameters()); + } + return rtp_params; +} + +webrtc::RTCError WebRtcVoiceSendChannel2::SetRtpSendParameters( + uint32_t ssrc, const webrtc::RtpParameters& parameters, + webrtc::SetParametersCallback callback) { + RTC_DCHECK_RUN_ON(worker_thread_); + auto it = send_streams_.find(ssrc); + if (it == send_streams_.end()) { + RTC_LOG(LS_WARNING) << "Attempting to set RTP send parameters for stream " + "with ssrc " + << ssrc << " which doesn't exist."; + return webrtc::InvokeSetParametersCallback( + callback, webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR)); + } + + // TODO(deadbeef): Handle setting parameters with a list of codecs in a + // different order (which should change the send codec). + webrtc::RtpParameters current_parameters = GetRtpSendParameters(ssrc); + if (current_parameters.codecs != parameters.codecs) { + RTC_DLOG(LS_ERROR) << "Using SetParameters to change the set of codecs " + "is not currently supported."; + return webrtc::InvokeSetParametersCallback( + callback, webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR)); + } + + if (!parameters.encodings.empty()) { + // Note that these values come from: + // https://tools.ietf.org/html/draft-ietf-tsvwg-rtcweb-qos-16#section-5 + rtc::DiffServCodePoint new_dscp = rtc::DSCP_DEFAULT; + switch (parameters.encodings[0].network_priority) { + case webrtc::Priority::kVeryLow: + new_dscp = rtc::DSCP_CS1; + break; + case webrtc::Priority::kLow: + new_dscp = rtc::DSCP_DEFAULT; + break; + case webrtc::Priority::kMedium: + new_dscp = rtc::DSCP_EF; + break; + case webrtc::Priority::kHigh: + new_dscp = rtc::DSCP_EF; + break; + } + SetPreferredDscp(new_dscp); + + absl::optional send_codec = GetSendCodec(); + // Since we validate that all layers have the same value, we can just check + // the first layer. + // TODO(orphis): Support mixed-codec simulcast + if (parameters.encodings[0].codec && send_codec && + !send_codec->MatchesRtpCodec(*parameters.encodings[0].codec)) { + RTC_LOG(LS_VERBOSE) << "Trying to change codec to " + << parameters.encodings[0].codec->name; + auto matched_codec = + absl::c_find_if(send_codecs_, [&](auto negotiated_codec) { + return negotiated_codec.MatchesRtpCodec( + *parameters.encodings[0].codec); + }); + + if (matched_codec == send_codecs_.end()) { + return webrtc::InvokeSetParametersCallback( + callback, webrtc::RTCError( + webrtc::RTCErrorType::INVALID_MODIFICATION, + "Attempted to use an unsupported codec for layer 0")); + } + + SetSendCodecs(send_codecs_, *matched_codec); + } + } + + // TODO(minyue): The following legacy actions go into + // `WebRtcAudioSendStream::SetRtpParameters()` which is called at the end, + // though there are two difference: + // 1. `WebRtcVoiceMediaChannel::SetChannelSendParameters()` only calls + // `SetSendCodec` while `WebRtcAudioSendStream::SetRtpParameters()` calls + // `SetSendCodecs`. The outcome should be the same. + // 2. AudioSendStream can be recreated. + + // Codecs are handled at the WebRtcVoiceMediaChannel level. + webrtc::RtpParameters reduced_params = parameters; + reduced_params.codecs.clear(); + return it->second->SetRtpParameters(reduced_params, std::move(callback)); +} + +// -------------------------- WebRtcVoiceReceiveChannel2 ---------------------- + +class WebRtcVoiceReceiveChannel2::WebRtcAudioReceiveStream { + public: + WebRtcAudioReceiveStream(webrtc::AudioReceiveStreamInterface::Config config, + webrtc::Call* call) + : call_(call), stream_(call_->CreateAudioReceiveStream(config)) { + RTC_DCHECK(call); + RTC_DCHECK(stream_); + } + + WebRtcAudioReceiveStream() = delete; + WebRtcAudioReceiveStream(const WebRtcAudioReceiveStream&) = delete; + WebRtcAudioReceiveStream& operator=(const WebRtcAudioReceiveStream&) = delete; + + ~WebRtcAudioReceiveStream() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + call_->DestroyAudioReceiveStream(stream_); + } + + webrtc::AudioReceiveStreamInterface& stream() { + RTC_DCHECK(stream_); + return *stream_; + } + + void SetFrameDecryptor( + rtc::scoped_refptr frame_decryptor) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + stream_->SetFrameDecryptor(std::move(frame_decryptor)); + } + + void SetUseNack(bool use_nack) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + stream_->SetNackHistory(use_nack ? kNackRtpHistoryMs : 0); + } + + void SetNonSenderRttMeasurement(bool enabled) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + stream_->SetNonSenderRttMeasurement(enabled); + } + + // Set a new payload type -> decoder map. + void SetDecoderMap(const std::map& decoder_map) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + stream_->SetDecoderMap(decoder_map); + } + + webrtc::AudioReceiveStreamInterface::Stats GetStats( + bool get_and_clear_legacy_stats) const { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return stream_->GetStats(get_and_clear_legacy_stats); + } + + void SetRawAudioSink(std::unique_ptr sink) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + // Need to update the stream's sink first; once raw_audio_sink_ is + // reassigned, whatever was in there before is destroyed. + stream_->SetSink(sink.get()); + raw_audio_sink_ = std::move(sink); + } + + void SetOutputVolume(double volume) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + stream_->SetGain(volume); + } + + void SetPlayout(bool playout) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + if (playout) { + stream_->Start(); + } else { + stream_->Stop(); + } + } + + bool SetBaseMinimumPlayoutDelayMs(int delay_ms) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + if (stream_->SetBaseMinimumPlayoutDelayMs(delay_ms)) return true; + + RTC_LOG(LS_ERROR) << "Failed to SetBaseMinimumPlayoutDelayMs" + " on AudioReceiveStreamInterface on SSRC=" + << stream_->remote_ssrc() + << " with delay_ms=" << delay_ms; + return false; + } + + int GetBaseMinimumPlayoutDelayMs() const { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return stream_->GetBaseMinimumPlayoutDelayMs(); + } + + std::vector GetSources() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return stream_->GetSources(); + } + + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + stream_->SetDepacketizerToDecoderFrameTransformer(frame_transformer); + } + + private: + webrtc::SequenceChecker worker_thread_checker_; + webrtc::Call* call_ = nullptr; + webrtc::AudioReceiveStreamInterface* const stream_ = nullptr; + std::unique_ptr raw_audio_sink_ + RTC_GUARDED_BY(worker_thread_checker_); +}; + +WebRtcVoiceReceiveChannel2::WebRtcVoiceReceiveChannel2( + CustomWebRtcVoiceEngine* engine, const MediaConfig& config, + const AudioOptions& options, const webrtc::CryptoOptions& crypto_options, + webrtc::Call* call, webrtc::AudioCodecPairId codec_pair_id) + : MediaChannelUtil(call->network_thread(), config.enable_dscp), + worker_thread_(call->worker_thread()), + engine_(engine), + call_(call), + audio_config_(config.audio), + codec_pair_id_(codec_pair_id), + crypto_options_(crypto_options) { + RTC_LOG(LS_VERBOSE) + << "WebRtcVoiceReceiveChannel2::WebRtcVoiceReceiveChannel2"; + RTC_DCHECK(call); + SetOptions(options); +} + +WebRtcVoiceReceiveChannel2::~WebRtcVoiceReceiveChannel2() { + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_DLOG(LS_VERBOSE) + << "WebRtcVoiceReceiveChannel2::~WebRtcVoiceReceiveChannel2"; + // TODO(solenberg): Should be able to delete the streams directly, without + // going through RemoveNnStream(), once stream objects handle + // all (de)configuration. + while (!recv_streams_.empty()) { + RemoveRecvStream(recv_streams_.begin()->first); + } +} + +bool WebRtcVoiceReceiveChannel2::SetReceiverParameters( + const AudioReceiverParameters& params) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::SetReceiverParameters"); + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_LOG(LS_INFO) << "WebRtcVoiceMediaChannel::SetReceiverParameters: " + << params.ToString(); + // TODO(pthatcher): Refactor this to be more clean now that we have + // all the information at once. + + if (!SetRecvCodecs(params.codecs)) { + return false; + } + + if (!ValidateRtpExtensions(params.extensions, recv_rtp_extensions_)) { + return false; + } + std::vector filtered_extensions = FilterRtpExtensions( + params.extensions, webrtc::RtpExtension::IsSupportedForAudio, false, + call_->trials()); + if (recv_rtp_extensions_ != filtered_extensions) { + recv_rtp_extensions_.swap(filtered_extensions); + recv_rtp_extension_map_ = + webrtc::RtpHeaderExtensionMap(recv_rtp_extensions_); + } + return true; +} + +webrtc::RtpParameters WebRtcVoiceReceiveChannel2::GetRtpReceiverParameters( + uint32_t ssrc) const { + RTC_DCHECK_RUN_ON(worker_thread_); + webrtc::RtpParameters rtp_params; + auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_WARNING) + << "Attempting to get RTP receive parameters for stream " + "with ssrc " + << ssrc << " which doesn't exist."; + return webrtc::RtpParameters(); + } + rtp_params.encodings.emplace_back(); + rtp_params.encodings.back().ssrc = it->second->stream().remote_ssrc(); + rtp_params.header_extensions = recv_rtp_extensions_; + + for (const AudioCodec& codec : recv_codecs_) { + rtp_params.codecs.push_back(codec.ToCodecParameters()); + } + return rtp_params; +} + +webrtc::RtpParameters +WebRtcVoiceReceiveChannel2::GetDefaultRtpReceiveParameters() const { + RTC_DCHECK_RUN_ON(worker_thread_); + webrtc::RtpParameters rtp_params; + if (!default_sink_) { + // Getting parameters on a default, unsignaled audio receive stream but + // because we've not configured to receive such a stream, `encodings` is + // empty. + return rtp_params; + } + rtp_params.encodings.emplace_back(); + + for (const AudioCodec& codec : recv_codecs_) { + rtp_params.codecs.push_back(codec.ToCodecParameters()); + } + return rtp_params; +} + +bool WebRtcVoiceReceiveChannel2::SetOptions(const AudioOptions& options) { + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_LOG(LS_INFO) << "Setting voice channel options: " << options.ToString(); + + // We retain all of the existing options, and apply the given ones + // on top. This means there is no way to "clear" options such that + // they go back to the engine default. + options_.SetAll(options); + engine()->ApplyOptions(options_); + + RTC_LOG(LS_INFO) << "Set voice receive channel options. Current options: " + << options_.ToString(); + return true; +} + +bool WebRtcVoiceReceiveChannel2::SetRecvCodecs( + const std::vector& codecs) { + RTC_DCHECK_RUN_ON(worker_thread_); + + // Set the payload types to be used for incoming media. + RTC_LOG(LS_INFO) << "Setting receive voice codecs."; + + if (!VerifyUniquePayloadTypes(codecs)) { + RTC_LOG(LS_ERROR) << "Codec payload types overlap."; + return false; + } + + // Create a payload type -> SdpAudioFormat map with all the decoders. Fail + // unless the factory claims to support all decoders. + std::map decoder_map; + for (const AudioCodec& codec : codecs) { + // Log a warning if a codec's payload type is changing. This used to be + // treated as an error. It's abnormal, but not really illegal. + absl::optional old_codec = FindCodec(recv_codecs_, codec); + if (old_codec && old_codec->id != codec.id) { + RTC_LOG(LS_WARNING) << codec.name << " mapped to a second payload type (" + << codec.id << ", was already mapped to " + << old_codec->id << ")"; + } + auto format = AudioCodecToSdpAudioFormat(codec); + if (!IsCodec(codec, kCnCodecName) && !IsCodec(codec, kDtmfCodecName) && + !IsCodec(codec, kRedCodecName) && + !engine()->decoder_factory_->IsSupportedDecoder(format)) { + RTC_LOG(LS_ERROR) << "Unsupported codec: " << rtc::ToString(format); + return false; + } + // We allow adding new codecs but don't allow changing the payload type of + // codecs that are already configured since we might already be receiving + // packets with that payload type. See RFC3264, Section 8.3.2. + // TODO(deadbeef): Also need to check for clashes with previously mapped + // payload types, and not just currently mapped ones. For example, this + // should be illegal: + // 1. {100: opus/48000/2, 101: ISAC/16000} + // 2. {100: opus/48000/2} + // 3. {100: opus/48000/2, 101: ISAC/32000} + // Though this check really should happen at a higher level, since this + // conflict could happen between audio and video codecs. + auto existing = decoder_map_.find(codec.id); + if (existing != decoder_map_.end() && !existing->second.Matches(format)) { + RTC_LOG(LS_ERROR) << "Attempting to use payload type " << codec.id + << " for " << codec.name + << ", but it is already used for " + << existing->second.name; + return false; + } + decoder_map.insert({codec.id, std::move(format)}); + } + + if (decoder_map == decoder_map_) { + // There's nothing new to configure. + return true; + } + + bool playout_enabled = playout_; + // Receive codecs can not be changed while playing. So we temporarily + // pause playout. + SetPlayout(false); + RTC_DCHECK(!playout_); + + decoder_map_ = std::move(decoder_map); + for (auto& kv : recv_streams_) { + kv.second->SetDecoderMap(decoder_map_); + } + + recv_codecs_ = codecs; + + SetPlayout(playout_enabled); + RTC_DCHECK_EQ(playout_, playout_enabled); + + return true; +} + +void WebRtcVoiceReceiveChannel2::SetReceiveNackEnabled(bool enabled) { + // Check if the NACK status has changed on the + // preferred send codec, and in that case reconfigure all receive streams. + if (recv_nack_enabled_ != enabled) { + RTC_LOG(LS_INFO) << "Changing NACK status on receive streams."; + recv_nack_enabled_ = enabled; + for (auto& kv : recv_streams_) { + kv.second->SetUseNack(recv_nack_enabled_); + } + } +} + +void WebRtcVoiceReceiveChannel2::SetReceiveNonSenderRttEnabled(bool enabled) { + // Check if the receive-side RTT status has changed on the preferred send + // codec, in that case reconfigure all receive streams. + if (enable_non_sender_rtt_ != enabled) { + RTC_LOG(LS_INFO) << "Changing receive-side RTT status on receive streams."; + enable_non_sender_rtt_ = enabled; + for (auto& kv : recv_streams_) { + kv.second->SetNonSenderRttMeasurement(enable_non_sender_rtt_); + } + } +} + +void WebRtcVoiceReceiveChannel2::SetPlayout(bool playout) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::SetPlayout"); + RTC_DCHECK_RUN_ON(worker_thread_); + if (playout_ == playout) { + return; + } + + for (const auto& kv : recv_streams_) { + kv.second->SetPlayout(playout); + } + playout_ = playout; +} + +bool WebRtcVoiceReceiveChannel2::AddRecvStream(const StreamParams& sp) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::AddRecvStream"); + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_LOG(LS_INFO) << "AddRecvStream: " << sp.ToString(); + + if (!sp.has_ssrcs()) { + // This is a StreamParam with unsignaled SSRCs. Store it, so it can be used + // later when we know the SSRCs on the first packet arrival. + unsignaled_stream_params_ = sp; + return true; + } + + if (!ValidateStreamParams(sp)) { + return false; + } + + const uint32_t ssrc = sp.first_ssrc(); + + // If this stream was previously received unsignaled, we promote it, possibly + // updating the sync group if stream ids have changed. + if (MaybeDeregisterUnsignaledRecvStream(ssrc)) { + auto stream_ids = sp.stream_ids(); + std::string sync_group = stream_ids.empty() ? std::string() : stream_ids[0]; + call_->OnUpdateSyncGroup(recv_streams_[ssrc]->stream(), + std::move(sync_group)); + return true; + } + + if (recv_streams_.find(ssrc) != recv_streams_.end()) { + RTC_LOG(LS_ERROR) << "Stream already exists with ssrc " << ssrc; + return false; + } + + // Create a new channel for receiving audio data. + auto config = BuildReceiveStreamConfig( + ssrc, receiver_reports_ssrc_, recv_nack_enabled_, enable_non_sender_rtt_, + sp.stream_ids(), recv_rtp_extensions_, transport(), + engine()->decoder_factory_, decoder_map_, codec_pair_id_, + engine()->audio_jitter_buffer_max_packets_, + engine()->audio_jitter_buffer_fast_accelerate_, + engine()->audio_jitter_buffer_min_delay_ms_, unsignaled_frame_decryptor_, + crypto_options_, unsignaled_frame_transformer_); + + recv_streams_.insert(std::make_pair( + ssrc, new WebRtcAudioReceiveStream(std::move(config), call_))); + recv_streams_[ssrc]->SetPlayout(playout_); + + return true; +} + +bool WebRtcVoiceReceiveChannel2::RemoveRecvStream(uint32_t ssrc) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::RemoveRecvStream"); + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_LOG(LS_INFO) << "RemoveRecvStream: " << ssrc; + + const auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_WARNING) << "Try to remove stream with ssrc " << ssrc + << " which doesn't exist."; + return false; + } + + MaybeDeregisterUnsignaledRecvStream(ssrc); + + it->second->SetRawAudioSink(nullptr); + delete it->second; + recv_streams_.erase(it); + return true; +} + +void WebRtcVoiceReceiveChannel2::ResetUnsignaledRecvStream() { + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_LOG(LS_INFO) << "ResetUnsignaledRecvStream."; + unsignaled_stream_params_ = StreamParams(); + // Create a copy since RemoveRecvStream will modify `unsignaled_recv_ssrcs_`. + std::vector to_remove = unsignaled_recv_ssrcs_; + for (uint32_t ssrc : to_remove) { + RemoveRecvStream(ssrc); + } +} + +absl::optional WebRtcVoiceReceiveChannel2::GetUnsignaledSsrc() const { + if (unsignaled_recv_ssrcs_.empty()) { + return absl::nullopt; + } + // In the event of multiple unsignaled ssrcs, the last in the vector will be + // the most recent one (the one forwarded to the MediaStreamTrack). + return unsignaled_recv_ssrcs_.back(); +} + +void WebRtcVoiceReceiveChannel2::ChooseReceiverReportSsrc( + const std::set& choices) { + // Don't change SSRC if set is empty. Note that this differs from + // the behavior of video. + if (choices.empty()) { + return; + } + if (choices.find(receiver_reports_ssrc_) != choices.end()) { + return; + } + uint32_t ssrc = *(choices.begin()); + receiver_reports_ssrc_ = ssrc; + for (auto& kv : recv_streams_) { + call_->OnLocalSsrcUpdated(kv.second->stream(), ssrc); + } +} + +// Not implemented. +// TODO(https://crbug.com/webrtc/12676): Implement a fix for the unsignalled +// SSRC race that can happen when an m= section goes from receiving to not +// receiving. +void WebRtcVoiceReceiveChannel2::OnDemuxerCriteriaUpdatePending() {} +void WebRtcVoiceReceiveChannel2::OnDemuxerCriteriaUpdateComplete() {} + +bool WebRtcVoiceReceiveChannel2::SetOutputVolume(uint32_t ssrc, double volume) { + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_LOG(LS_INFO) << rtc::StringFormat("WRVMC::%s({ssrc=%u}, {volume=%.2f})", + __func__, ssrc, volume); + const auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_WARNING) << rtc::StringFormat( + "WRVMC::%s => (WARNING: no receive stream for SSRC %u)", __func__, + ssrc); + return false; + } + it->second->SetOutputVolume(volume); + RTC_LOG(LS_INFO) << rtc::StringFormat( + "WRVMC::%s => (stream with SSRC %u now uses volume %.2f)", __func__, ssrc, + volume); + return true; +} + +bool WebRtcVoiceReceiveChannel2::SetDefaultOutputVolume(double volume) { + RTC_DCHECK_RUN_ON(worker_thread_); + default_recv_volume_ = volume; + for (uint32_t ssrc : unsignaled_recv_ssrcs_) { + const auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_WARNING) << "SetDefaultOutputVolume: no recv stream " << ssrc; + return false; + } + it->second->SetOutputVolume(volume); + RTC_LOG(LS_INFO) << "SetDefaultOutputVolume() to " << volume + << " for recv stream with ssrc " << ssrc; + } + return true; +} + +bool WebRtcVoiceReceiveChannel2::SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, + int delay_ms) { + RTC_DCHECK_RUN_ON(worker_thread_); + std::vector ssrcs(1, ssrc); + // SSRC of 0 represents the default receive stream. + if (ssrc == 0) { + default_recv_base_minimum_delay_ms_ = delay_ms; + ssrcs = unsignaled_recv_ssrcs_; + } + for (uint32_t ssrc : ssrcs) { + const auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_WARNING) << "SetBaseMinimumPlayoutDelayMs: no recv stream " + << ssrc; + return false; + } + it->second->SetBaseMinimumPlayoutDelayMs(delay_ms); + RTC_LOG(LS_INFO) << "SetBaseMinimumPlayoutDelayMs() to " << delay_ms + << " for recv stream with ssrc " << ssrc; + } + return true; +} + +absl::optional WebRtcVoiceReceiveChannel2::GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const { + // SSRC of 0 represents the default receive stream. + if (ssrc == 0) { + return default_recv_base_minimum_delay_ms_; + } + + const auto it = recv_streams_.find(ssrc); + + if (it != recv_streams_.end()) { + return it->second->GetBaseMinimumPlayoutDelayMs(); + } + return absl::nullopt; +} + +void WebRtcVoiceReceiveChannel2::SetFrameDecryptor( + uint32_t ssrc, + rtc::scoped_refptr frame_decryptor) { + RTC_DCHECK_RUN_ON(worker_thread_); + auto matching_stream = recv_streams_.find(ssrc); + if (matching_stream != recv_streams_.end()) { + matching_stream->second->SetFrameDecryptor(frame_decryptor); + } + // Handle unsignaled frame decryptors. + if (ssrc == 0) { + unsignaled_frame_decryptor_ = frame_decryptor; + } +} + +void WebRtcVoiceReceiveChannel2::OnPacketReceived( + const webrtc::RtpPacketReceived& packet) { + RTC_DCHECK_RUN_ON(&network_thread_checker_); + + // TODO(bugs.webrtc.org/11993): This code is very similar to what + // WebRtcVideoChannel::OnPacketReceived does. For maintainability and + // consistency it would be good to move the interaction with + // call_->Receiver() to a common implementation and provide a callback on + // the worker thread for the exception case (DELIVERY_UNKNOWN_SSRC) and + // how retry is attempted. + worker_thread_->PostTask( + SafeTask(task_safety_.flag(), [this, packet = packet]() mutable { + RTC_DCHECK_RUN_ON(worker_thread_); + + // TODO(bugs.webrtc.org/7135): extensions in `packet` is currently set + // in RtpTransport and does not neccessarily include extensions specific + // to this channel/MID. Also see comment in + // BaseChannel::MaybeUpdateDemuxerAndRtpExtensions_w. + // It would likely be good if extensions where merged per BUNDLE and + // applied directly in RtpTransport::DemuxPacket; + packet.IdentifyExtensions(recv_rtp_extension_map_); + if (!packet.arrival_time().IsFinite()) { + packet.set_arrival_time(webrtc::Timestamp::Micros(rtc::TimeMicros())); + } + + call_->Receiver()->DeliverRtpPacket( + webrtc::MediaType::AUDIO, std::move(packet), + absl::bind_front( + &WebRtcVoiceReceiveChannel2::MaybeCreateDefaultReceiveStream, + this)); + })); +} + +bool WebRtcVoiceReceiveChannel2::MaybeCreateDefaultReceiveStream( + const webrtc::RtpPacketReceived& packet) { + // Create an unsignaled receive stream for this previously not received + // ssrc. If there already is N unsignaled receive streams, delete the + // oldest. See: https://bugs.chromium.org/p/webrtc/issues/detail?id=5208 + uint32_t ssrc = packet.Ssrc(); + RTC_DCHECK(!absl::c_linear_search(unsignaled_recv_ssrcs_, ssrc)); + + // Add new stream. + StreamParams sp = unsignaled_stream_params_; + sp.ssrcs.push_back(ssrc); + RTC_LOG(LS_INFO) << "Creating unsignaled receive stream for SSRC=" << ssrc; + if (!AddRecvStream(sp)) { + RTC_LOG(LS_WARNING) << "Could not create unsignaled receive stream."; + return false; + } + unsignaled_recv_ssrcs_.push_back(ssrc); + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.NumOfUnsignaledStreams", + unsignaled_recv_ssrcs_.size(), 1, 100, 101); + + // Remove oldest unsignaled stream, if we have too many. + if (unsignaled_recv_ssrcs_.size() > kMaxUnsignaledRecvStreams) { + uint32_t remove_ssrc = unsignaled_recv_ssrcs_.front(); + RTC_DLOG(LS_INFO) << "Removing unsignaled receive stream with SSRC=" + << remove_ssrc; + RemoveRecvStream(remove_ssrc); + } + RTC_DCHECK_GE(kMaxUnsignaledRecvStreams, unsignaled_recv_ssrcs_.size()); + + SetOutputVolume(ssrc, default_recv_volume_); + SetBaseMinimumPlayoutDelayMs(ssrc, default_recv_base_minimum_delay_ms_); + + // The default sink can only be attached to one stream at a time, so we hook + // it up to the *latest* unsignaled stream we've seen, in order to support + // the case where the SSRC of one unsignaled stream changes. + if (default_sink_) { + for (uint32_t drop_ssrc : unsignaled_recv_ssrcs_) { + auto it = recv_streams_.find(drop_ssrc); + it->second->SetRawAudioSink(nullptr); + } + std::unique_ptr proxy_sink( + new ProxySink(default_sink_.get())); + SetRawAudioSink(ssrc, std::move(proxy_sink)); + } + return true; +} + +bool WebRtcVoiceReceiveChannel2::GetStats(VoiceMediaReceiveInfo* info, + bool get_and_clear_legacy_stats) { + TRACE_EVENT0("webrtc", "WebRtcVoiceMediaChannel::GetReceiveStats"); + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_DCHECK(info); + + // Get SSRC and stats for each receiver. + RTC_DCHECK_EQ(info->receivers.size(), 0U); + for (const auto& stream : recv_streams_) { + uint32_t ssrc = stream.first; + // When SSRCs are unsignaled, there's only one audio MediaStreamTrack, but + // multiple RTP streams can be received over time (if the SSRC changes for + // whatever reason). We only want the RTCMediaStreamTrackStats to represent + // the stats for the most recent stream (the one whose audio is actually + // routed to the MediaStreamTrack), so here we ignore any unsignaled SSRCs + // except for the most recent one (last in the vector). This is somewhat of + // a hack, and means you don't get *any* stats for these inactive streams, + // but it's slightly better than the previous behavior, which was "highest + // SSRC wins". + // See: https://bugs.chromium.org/p/webrtc/issues/detail?id=8158 + if (!unsignaled_recv_ssrcs_.empty()) { + auto end_it = --unsignaled_recv_ssrcs_.end(); + if (absl::linear_search(unsignaled_recv_ssrcs_.begin(), end_it, ssrc)) { + continue; + } + } + webrtc::AudioReceiveStreamInterface::Stats stats = + stream.second->GetStats(get_and_clear_legacy_stats); + VoiceReceiverInfo rinfo; + rinfo.add_ssrc(stats.remote_ssrc); + rinfo.payload_bytes_received = stats.payload_bytes_received; + rinfo.header_and_padding_bytes_received = + stats.header_and_padding_bytes_received; + rinfo.packets_received = stats.packets_received; + rinfo.fec_packets_received = stats.fec_packets_received; + rinfo.fec_packets_discarded = stats.fec_packets_discarded; + rinfo.packets_lost = stats.packets_lost; + rinfo.packets_discarded = stats.packets_discarded; + rinfo.codec_name = stats.codec_name; + rinfo.codec_payload_type = stats.codec_payload_type; + rinfo.jitter_ms = stats.jitter_ms; + rinfo.jitter_buffer_ms = stats.jitter_buffer_ms; + rinfo.jitter_buffer_preferred_ms = stats.jitter_buffer_preferred_ms; + rinfo.delay_estimate_ms = stats.delay_estimate_ms; + rinfo.audio_level = stats.audio_level; + rinfo.total_output_energy = stats.total_output_energy; + rinfo.total_samples_received = stats.total_samples_received; + rinfo.total_output_duration = stats.total_output_duration; + rinfo.concealed_samples = stats.concealed_samples; + rinfo.silent_concealed_samples = stats.silent_concealed_samples; + rinfo.concealment_events = stats.concealment_events; + rinfo.jitter_buffer_delay_seconds = stats.jitter_buffer_delay_seconds; + rinfo.jitter_buffer_emitted_count = stats.jitter_buffer_emitted_count; + rinfo.jitter_buffer_target_delay_seconds = + stats.jitter_buffer_target_delay_seconds; + rinfo.jitter_buffer_minimum_delay_seconds = + stats.jitter_buffer_minimum_delay_seconds; + rinfo.inserted_samples_for_deceleration = + stats.inserted_samples_for_deceleration; + rinfo.removed_samples_for_acceleration = + stats.removed_samples_for_acceleration; + rinfo.expand_rate = stats.expand_rate; + rinfo.speech_expand_rate = stats.speech_expand_rate; + rinfo.secondary_decoded_rate = stats.secondary_decoded_rate; + rinfo.secondary_discarded_rate = stats.secondary_discarded_rate; + rinfo.accelerate_rate = stats.accelerate_rate; + rinfo.preemptive_expand_rate = stats.preemptive_expand_rate; + rinfo.delayed_packet_outage_samples = stats.delayed_packet_outage_samples; + rinfo.decoding_calls_to_silence_generator = + stats.decoding_calls_to_silence_generator; + rinfo.decoding_calls_to_neteq = stats.decoding_calls_to_neteq; + rinfo.decoding_normal = stats.decoding_normal; + rinfo.decoding_plc = stats.decoding_plc; + rinfo.decoding_codec_plc = stats.decoding_codec_plc; + rinfo.decoding_cng = stats.decoding_cng; + rinfo.decoding_plc_cng = stats.decoding_plc_cng; + rinfo.decoding_muted_output = stats.decoding_muted_output; + rinfo.capture_start_ntp_time_ms = stats.capture_start_ntp_time_ms; + rinfo.last_packet_received = stats.last_packet_received; + rinfo.estimated_playout_ntp_timestamp_ms = + stats.estimated_playout_ntp_timestamp_ms; + rinfo.jitter_buffer_flushes = stats.jitter_buffer_flushes; + rinfo.relative_packet_arrival_delay_seconds = + stats.relative_packet_arrival_delay_seconds; + rinfo.interruption_count = stats.interruption_count; + rinfo.total_interruption_duration_ms = stats.total_interruption_duration_ms; + rinfo.last_sender_report_timestamp_ms = + stats.last_sender_report_timestamp_ms; + rinfo.last_sender_report_remote_timestamp_ms = + stats.last_sender_report_remote_timestamp_ms; + rinfo.sender_reports_packets_sent = stats.sender_reports_packets_sent; + rinfo.sender_reports_bytes_sent = stats.sender_reports_bytes_sent; + rinfo.sender_reports_reports_count = stats.sender_reports_reports_count; + rinfo.round_trip_time = stats.round_trip_time; + rinfo.round_trip_time_measurements = stats.round_trip_time_measurements; + rinfo.total_round_trip_time = stats.total_round_trip_time; + + if (recv_nack_enabled_) { + rinfo.nacks_sent = stats.nacks_sent; + } + + info->receivers.push_back(rinfo); + } + + FillReceiveCodecStats(info); + + info->device_underrun_count = engine_->adm()->GetPlayoutUnderrunCount(); + + return true; +} + +void WebRtcVoiceReceiveChannel2::FillReceiveCodecStats( + VoiceMediaReceiveInfo* voice_media_info) { + for (const auto& receiver : voice_media_info->receivers) { + auto codec = + absl::c_find_if(recv_codecs_, [&receiver](const AudioCodec& c) { + return receiver.codec_payload_type && + *receiver.codec_payload_type == c.id; + }); + if (codec != recv_codecs_.end()) { + voice_media_info->receive_codecs.insert( + std::make_pair(codec->id, codec->ToCodecParameters())); + } + } +} + +void WebRtcVoiceReceiveChannel2::SetRawAudioSink( + uint32_t ssrc, std::unique_ptr sink) { + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_LOG(LS_VERBOSE) << "WebRtcVoiceMediaChannel::SetRawAudioSink: ssrc:" + << ssrc << " " << (sink ? "(ptr)" : "NULL"); + const auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_WARNING) << "SetRawAudioSink: no recv stream " << ssrc; + return; + } + it->second->SetRawAudioSink(std::move(sink)); +} + +void WebRtcVoiceReceiveChannel2::SetDefaultRawAudioSink( + std::unique_ptr sink) { + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_LOG(LS_VERBOSE) << "WebRtcVoiceMediaChannel::SetDefaultRawAudioSink:"; + if (!unsignaled_recv_ssrcs_.empty()) { + std::unique_ptr proxy_sink( + sink ? new ProxySink(sink.get()) : nullptr); + SetRawAudioSink(unsignaled_recv_ssrcs_.back(), std::move(proxy_sink)); + } + default_sink_ = std::move(sink); +} + +std::vector WebRtcVoiceReceiveChannel2::GetSources( + uint32_t ssrc) const { + auto it = recv_streams_.find(ssrc); + if (it == recv_streams_.end()) { + RTC_LOG(LS_ERROR) << "Attempting to get contributing sources for SSRC:" + << ssrc << " which doesn't exist."; + return std::vector(); + } + return it->second->GetSources(); +} + +void WebRtcVoiceReceiveChannel2::SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) { + RTC_DCHECK_RUN_ON(worker_thread_); + if (ssrc == 0) { + // If the receiver is unsignaled, save the frame transformer and set it when + // the stream is associated with an ssrc. + unsignaled_frame_transformer_ = std::move(frame_transformer); + return; + } + + auto matching_stream = recv_streams_.find(ssrc); + if (matching_stream == recv_streams_.end()) { + RTC_LOG(LS_INFO) << "Attempting to set frame transformer for SSRC:" << ssrc + << " which doesn't exist."; + return; + } + matching_stream->second->SetDepacketizerToDecoderFrameTransformer( + std::move(frame_transformer)); +} + +bool WebRtcVoiceReceiveChannel2::MaybeDeregisterUnsignaledRecvStream( + uint32_t ssrc) { + RTC_DCHECK_RUN_ON(worker_thread_); + auto it = absl::c_find(unsignaled_recv_ssrcs_, ssrc); + if (it != unsignaled_recv_ssrcs_.end()) { + unsignaled_recv_ssrcs_.erase(it); + return true; + } + return false; +} +} // namespace cricket diff --git a/src/internal/custom_webrtc_voice_engine.h b/src/internal/custom_webrtc_voice_engine.h new file mode 100644 index 0000000000..fee5ecb816 --- /dev/null +++ b/src/internal/custom_webrtc_voice_engine.h @@ -0,0 +1,519 @@ +/* + * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef INTERNAL_CUSTOM_MEDIA_ENGINE_WEBRTC_VOICE_ENGINE_H_ +#define INTERNAL_CUSTOM_MEDIA_ENGINE_WEBRTC_VOICE_ENGINE_H_ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "absl/functional/any_invocable.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/audio/audio_frame_processor.h" +#include "api/audio/audio_mixer.h" +#include "api/audio_codecs/audio_codec_pair_id.h" +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_encoder_factory.h" +#include "api/audio_codecs/audio_format.h" +#include "api/audio_options.h" +#include "api/call/audio_sink.h" +#include "api/call/transport.h" +#include "api/crypto/crypto_options.h" +#include "api/crypto/frame_decryptor_interface.h" +#include "api/crypto/frame_encryptor_interface.h" +#include "api/field_trials_view.h" +#include "api/frame_transformer_interface.h" +#include "api/rtc_error.h" +#include "api/rtp_parameters.h" +#include "api/rtp_sender_interface.h" +#include "api/scoped_refptr.h" +#include "api/sequence_checker.h" +#include "api/task_queue/pending_task_safety_flag.h" +#include "api/task_queue/task_queue_base.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/transport/rtp/rtp_source.h" +#include "call/audio_send_stream.h" +#include "call/audio_state.h" +#include "call/call.h" +#include "media/base/codec.h" +#include "media/base/media_channel.h" +#include "media/base/media_channel_impl.h" +#include "media/base/media_config.h" +#include "media/base/media_engine.h" +#include "media/base/rtp_utils.h" +#include "media/base/stream_params.h" +#include "modules/async_audio_processing/async_audio_processing.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/rtp_rtcp/include/rtp_header_extension_map.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "rtc_base/buffer.h" +#include "rtc_base/network/sent_packet.h" +#include "rtc_base/network_route.h" +#include "rtc_base/system/file_wrapper.h" + +namespace webrtc { +class AudioFrameProcessor; + +class CreateAudioStateFactory { + public: + virtual ~CreateAudioStateFactory() = default; + virtual rtc::scoped_refptr CreateAudioState( + const webrtc::AudioState::Config& config) = 0; +}; +} // namespace webrtc + +namespace cricket { + +class AudioSource; + +// CustomWebRtcVoiceEngine is a class to be used with CompositeMediaEngine. +// It uses the WebRtc VoiceEngine library for audio handling. +class CustomWebRtcVoiceEngine final : public VoiceEngineInterface { + friend class WebRtcVoiceSendChannel2; + friend class WebRtcVoiceReceiveChannel2; + + public: + CustomWebRtcVoiceEngine( + webrtc::CreateAudioStateFactory* create_audio_state_factory, + webrtc::TaskQueueFactory* task_queue_factory, + webrtc::AudioDeviceModule* adm, + const rtc::scoped_refptr& encoder_factory, + const rtc::scoped_refptr& decoder_factory, + rtc::scoped_refptr audio_mixer, + rtc::scoped_refptr audio_processing, + std::unique_ptr owned_audio_frame_processor, + const webrtc::FieldTrialsView& trials); + + CustomWebRtcVoiceEngine() = delete; + CustomWebRtcVoiceEngine(const CustomWebRtcVoiceEngine&) = delete; + CustomWebRtcVoiceEngine& operator=(const CustomWebRtcVoiceEngine&) = delete; + + ~CustomWebRtcVoiceEngine() override; + + // Does initialization that needs to occur on the worker thread. + void Init() override; + rtc::scoped_refptr GetAudioState() const override; + + std::unique_ptr CreateSendChannel( + webrtc::Call* call, const MediaConfig& config, + const AudioOptions& options, const webrtc::CryptoOptions& crypto_options, + webrtc::AudioCodecPairId codec_pair_id) override; + + std::unique_ptr CreateReceiveChannel( + webrtc::Call* call, const MediaConfig& config, + const AudioOptions& options, const webrtc::CryptoOptions& crypto_options, + webrtc::AudioCodecPairId codec_pair_id) override; + + const std::vector& send_codecs() const override; + const std::vector& recv_codecs() const override; + std::vector GetRtpHeaderExtensions() + const override; + + // Starts AEC dump using an existing file. A maximum file size in bytes can be + // specified. When the maximum file size is reached, logging is stopped and + // the file is closed. If max_size_bytes is set to <= 0, no limit will be + // used. + bool StartAecDump(webrtc::FileWrapper file, int64_t max_size_bytes) override; + + // Stops AEC dump. + void StopAecDump() override; + + absl::optional GetAudioDeviceStats() + override; + // Moved to public so WebRtcVoiceMediaChannel can access it. + webrtc::AudioState* audio_state(); + + private: + // Every option that is "set" will be applied. Every option not "set" will be + // ignored. This allows us to selectively turn on and off different options + // easily at any time. + void ApplyOptions(const AudioOptions& options); + + webrtc::CreateAudioStateFactory* create_audio_state_factory_; + + webrtc::TaskQueueFactory* const task_queue_factory_; + std::unique_ptr + low_priority_worker_queue_; + + webrtc::AudioDeviceModule* adm(); + webrtc::AudioProcessing* apm() const; + + std::vector CollectCodecs( + const std::vector& specs) const; + + webrtc::SequenceChecker signal_thread_checker_{ + webrtc::SequenceChecker::kDetached}; + webrtc::SequenceChecker worker_thread_checker_{ + webrtc::SequenceChecker::kDetached}; + + // The audio device module. + rtc::scoped_refptr adm_; + rtc::scoped_refptr encoder_factory_; + rtc::scoped_refptr decoder_factory_; + rtc::scoped_refptr audio_mixer_; + // The audio processing module. + rtc::scoped_refptr apm_; + // Asynchronous audio processing. + std::unique_ptr audio_frame_processor_; + // The primary instance of WebRtc VoiceEngine. + rtc::scoped_refptr audio_state_; + std::vector send_codecs_; + std::vector recv_codecs_; + bool is_dumping_aec_ = false; + bool initialized_ = false; + + // Jitter buffer settings for new streams. + size_t audio_jitter_buffer_max_packets_ = 200; + bool audio_jitter_buffer_fast_accelerate_ = false; + int audio_jitter_buffer_min_delay_ms_ = 0; + + const bool minimized_remsampling_on_mobile_trial_enabled_; +}; + +class WebRtcVoiceSendChannel2 final : public MediaChannelUtil, + public VoiceMediaSendChannelInterface { + public: + WebRtcVoiceSendChannel2(CustomWebRtcVoiceEngine* engine, + const MediaConfig& config, + const AudioOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::Call* call, + webrtc::AudioCodecPairId codec_pair_id); + + WebRtcVoiceSendChannel2() = delete; + WebRtcVoiceSendChannel2(const WebRtcVoiceSendChannel2&) = delete; + WebRtcVoiceSendChannel2& operator=(const WebRtcVoiceSendChannel2&) = delete; + + ~WebRtcVoiceSendChannel2() override; + + MediaType media_type() const override { return MEDIA_TYPE_AUDIO; } + VideoMediaSendChannelInterface* AsVideoSendChannel() override { + RTC_CHECK_NOTREACHED(); + return nullptr; + } + VoiceMediaSendChannelInterface* AsVoiceSendChannel() override { return this; } + + absl::optional GetSendCodec() const override; + + // Functions imported from MediaChannelUtil + void SetInterface(MediaChannelNetworkInterface* iface) override { + MediaChannelUtil::SetInterface(iface); + } + + bool HasNetworkInterface() const override { + return MediaChannelUtil::HasNetworkInterface(); + } + void SetExtmapAllowMixed(bool extmap_allow_mixed) override { + MediaChannelUtil::SetExtmapAllowMixed(extmap_allow_mixed); + } + bool ExtmapAllowMixed() const override { + return MediaChannelUtil::ExtmapAllowMixed(); + } + + const AudioOptions& options() const { return options_; } + + bool SetSenderParameters(const AudioSenderParameter& params) override; + webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override; + webrtc::RTCError SetRtpSendParameters( + uint32_t ssrc, const webrtc::RtpParameters& parameters, + webrtc::SetParametersCallback callback) override; + + void SetSend(bool send) override; + bool SetAudioSend(uint32_t ssrc, bool enable, const AudioOptions* options, + AudioSource* source) override; + bool AddSendStream(const StreamParams& sp) override; + bool RemoveSendStream(uint32_t ssrc) override; + + void SetSsrcListChangedCallback( + absl::AnyInvocable&)> callback) override; + + // E2EE Frame API + // Set a frame encryptor to a particular ssrc that will intercept all + // outgoing audio payloads frames and attempt to encrypt them and forward the + // result to the packetizer. + void SetFrameEncryptor(uint32_t ssrc, + rtc::scoped_refptr + frame_encryptor) override; + + bool CanInsertDtmf() override; + bool InsertDtmf(uint32_t ssrc, int event, int duration) override; + + void OnPacketSent(const rtc::SentPacket& sent_packet) override; + void OnNetworkRouteChanged(absl::string_view transport_name, + const rtc::NetworkRoute& network_route) override; + void OnReadyToSend(bool ready) override; + bool GetStats(VoiceMediaSendInfo* info) override; + + // Sets a frame transformer between encoder and packetizer, to transform + // encoded frames before sending them out the network. + void SetEncoderToPacketizerFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) + override; + + bool SenderNackEnabled() const override { + if (!send_codec_spec_) { + return false; + } + return send_codec_spec_->nack_enabled; + } + bool SenderNonSenderRttEnabled() const override { + if (!send_codec_spec_) { + return false; + } + return send_codec_spec_->enable_non_sender_rtt; + } + bool SendCodecHasNack() const override { return SenderNackEnabled(); } + + void SetSendCodecChangedCallback( + absl::AnyInvocable callback) override { + send_codec_changed_callback_ = std::move(callback); + } + + private: + bool SetOptions(const AudioOptions& options); + bool SetSendCodecs(const std::vector& codecs, + absl::optional preferred_codec); + bool SetLocalSource(uint32_t ssrc, AudioSource* source); + bool MuteStream(uint32_t ssrc, bool mute); + + CustomWebRtcVoiceEngine* engine() { return engine_; } + bool SetMaxSendBitrate(int bps); + void SetupRecording(); + + webrtc::TaskQueueBase* const worker_thread_; + webrtc::ScopedTaskSafety task_safety_; + webrtc::SequenceChecker network_thread_checker_{ + webrtc::SequenceChecker::kDetached}; + + CustomWebRtcVoiceEngine* const engine_ = nullptr; + std::vector send_codecs_; + + int max_send_bitrate_bps_ = 0; + AudioOptions options_; + absl::optional dtmf_payload_type_; + int dtmf_payload_freq_ = -1; + bool enable_non_sender_rtt_ = false; + bool send_ = false; + webrtc::Call* const call_ = nullptr; + + const MediaConfig::Audio audio_config_; + + class WebRtcAudioSendStream; + + std::map send_streams_; + std::vector send_rtp_extensions_; + std::string mid_; + + absl::optional + send_codec_spec_; + + // TODO(kwiberg): Per-SSRC codec pair IDs? + const webrtc::AudioCodecPairId codec_pair_id_; + + // Per peer connection crypto options that last for the lifetime of the peer + // connection. + const webrtc::CryptoOptions crypto_options_; + rtc::scoped_refptr + unsignaled_frame_transformer_; + + void FillSendCodecStats(VoiceMediaSendInfo* voice_media_info); + + // Callback invoked whenever the send codec changes. + // TODO(bugs.webrtc.org/13931): Remove again when coupling isn't needed. + absl::AnyInvocable send_codec_changed_callback_; + // Callback invoked whenever the list of SSRCs changes. + absl::AnyInvocable&)> + ssrc_list_changed_callback_; +}; + +class WebRtcVoiceReceiveChannel2 final + : public MediaChannelUtil, + public VoiceMediaReceiveChannelInterface { + public: + WebRtcVoiceReceiveChannel2(CustomWebRtcVoiceEngine* engine, + const MediaConfig& config, + const AudioOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::Call* call, + webrtc::AudioCodecPairId codec_pair_id); + + WebRtcVoiceReceiveChannel2() = delete; + WebRtcVoiceReceiveChannel2(const WebRtcVoiceReceiveChannel2&) = delete; + WebRtcVoiceReceiveChannel2& operator=(const WebRtcVoiceReceiveChannel2&) = + delete; + + ~WebRtcVoiceReceiveChannel2() override; + + MediaType media_type() const override { return MEDIA_TYPE_AUDIO; } + + VideoMediaReceiveChannelInterface* AsVideoReceiveChannel() override { + RTC_CHECK_NOTREACHED(); + return nullptr; + } + VoiceMediaReceiveChannelInterface* AsVoiceReceiveChannel() override { + return this; + } + + const AudioOptions& options() const { return options_; } + + void SetInterface(MediaChannelNetworkInterface* iface) override { + MediaChannelUtil::SetInterface(iface); + } + bool SetReceiverParameters(const AudioReceiverParameters& params) override; + webrtc::RtpParameters GetRtpReceiverParameters(uint32_t ssrc) const override; + webrtc::RtpParameters GetDefaultRtpReceiveParameters() const override; + + void SetPlayout(bool playout) override; + bool AddRecvStream(const StreamParams& sp) override; + bool RemoveRecvStream(uint32_t ssrc) override; + void ResetUnsignaledRecvStream() override; + absl::optional GetUnsignaledSsrc() const override; + + void ChooseReceiverReportSsrc(const std::set& choices) override; + + void OnDemuxerCriteriaUpdatePending() override; + void OnDemuxerCriteriaUpdateComplete() override; + + // E2EE Frame API + // Set a frame decryptor to a particular ssrc that will intercept all + // incoming audio payloads and attempt to decrypt them before forwarding the + // result. + void SetFrameDecryptor(uint32_t ssrc, + rtc::scoped_refptr + frame_decryptor) override; + + bool SetOutputVolume(uint32_t ssrc, double volume) override; + // Applies the new volume to current and future unsignaled streams. + bool SetDefaultOutputVolume(double volume) override; + + bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override; + absl::optional GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const override; + + void OnPacketReceived(const webrtc::RtpPacketReceived& packet) override; + bool GetStats(VoiceMediaReceiveInfo* info, + bool get_and_clear_legacy_stats) override; + + // Set the audio sink for an existing stream. + void SetRawAudioSink( + uint32_t ssrc, std::unique_ptr sink) override; + // Will set the audio sink on the latest unsignaled stream, future or + // current. Only one stream at a time will use the sink. + void SetDefaultRawAudioSink( + std::unique_ptr sink) override; + + std::vector GetSources(uint32_t ssrc) const override; + + void SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) + override; + + void SetReceiveNackEnabled(bool enabled) override; + void SetReceiveNonSenderRttEnabled(bool enabled) override; + + private: + bool SetOptions(const AudioOptions& options); + bool SetRecvCodecs(const std::vector& codecs); + bool SetLocalSource(uint32_t ssrc, AudioSource* source); + bool MuteStream(uint32_t ssrc, bool mute); + + CustomWebRtcVoiceEngine* engine() { return engine_; } + void SetupRecording(); + + // Expected to be invoked once per packet that belongs to this channel that + // can not be demuxed. Returns true if a default receive stream has been + // created. + bool MaybeCreateDefaultReceiveStream(const webrtc::RtpPacketReceived& packet); + // Check if 'ssrc' is an unsignaled stream, and if so mark it as not being + // unsignaled anymore (i.e. it is now removed, or signaled), and return true. + bool MaybeDeregisterUnsignaledRecvStream(uint32_t ssrc); + + webrtc::TaskQueueBase* const worker_thread_; + webrtc::ScopedTaskSafety task_safety_; + webrtc::SequenceChecker network_thread_checker_{ + webrtc::SequenceChecker::kDetached}; + + CustomWebRtcVoiceEngine* const engine_ = nullptr; + + // TODO(kwiberg): decoder_map_ and recv_codecs_ store the exact same + // information, in slightly different formats. Eliminate recv_codecs_. + std::map decoder_map_; + std::vector recv_codecs_; + + AudioOptions options_; + bool recv_nack_enabled_ = false; + bool enable_non_sender_rtt_ = false; + bool playout_ = false; + webrtc::Call* const call_ = nullptr; + + const MediaConfig::Audio audio_config_; + + // Queue of unsignaled SSRCs; oldest at the beginning. + std::vector unsignaled_recv_ssrcs_; + + // This is a stream param that comes from the remote description, but wasn't + // signaled with any a=ssrc lines. It holds the information that was signaled + // before the unsignaled receive stream is created when the first packet is + // received. + StreamParams unsignaled_stream_params_; + + // Volume for unsignaled streams, which may be set before the stream exists. + double default_recv_volume_ = 1.0; + + // Delay for unsignaled streams, which may be set before the stream exists. + int default_recv_base_minimum_delay_ms_ = 0; + + // Sink for latest unsignaled stream - may be set before the stream exists. + std::unique_ptr default_sink_; + // Default SSRC to use for RTCP receiver reports in case of no signaled + // send streams. See: https://code.google.com/p/webrtc/issues/detail?id=4740 + // and https://code.google.com/p/chromium/issues/detail?id=547661 + uint32_t receiver_reports_ssrc_ = 0xFA17FA17u; + + std::string mid_; + + class WebRtcAudioReceiveStream; + + std::map recv_streams_; + std::vector recv_rtp_extensions_; + webrtc::RtpHeaderExtensionMap recv_rtp_extension_map_; + + absl::optional + send_codec_spec_; + + // TODO(kwiberg): Per-SSRC codec pair IDs? + const webrtc::AudioCodecPairId codec_pair_id_; + + // Per peer connection crypto options that last for the lifetime of the peer + // connection. + const webrtc::CryptoOptions crypto_options_; + // Unsignaled streams have an option to have a frame decryptor set on them. + rtc::scoped_refptr + unsignaled_frame_decryptor_; + rtc::scoped_refptr + unsignaled_frame_transformer_; + + void FillReceiveCodecStats(VoiceMediaReceiveInfo* voice_media_info); +}; + +} // namespace cricket + +#endif // INTERNAL_CUSTOM_MEDIA_ENGINE_WEBRTC_VOICE_ENGINE_H_ diff --git a/src/internal/local_audio_track.cc b/src/internal/local_audio_track.cc new file mode 100644 index 0000000000..e7cf09ff65 --- /dev/null +++ b/src/internal/local_audio_track.cc @@ -0,0 +1,21 @@ +#include "src/internal/local_audio_track.h" + +using webrtc::MediaSourceInterface; + +namespace libwebrtc { + +rtc::scoped_refptr LocalAudioSource::Create( + const cricket::AudioOptions* audio_options, + webrtc::CustomAudioTransportImpl* audio_transport) { + auto source = rtc::make_ref_counted(audio_transport); + source->Initialize(audio_options); + return source; +} + +void LocalAudioSource::Initialize(const cricket::AudioOptions* audio_options) { + if (!audio_options) return; + + options_ = *audio_options; +} + +} // namespace libwebrtc diff --git a/src/internal/local_audio_track.h b/src/internal/local_audio_track.h new file mode 100644 index 0000000000..a8592a6df3 --- /dev/null +++ b/src/internal/local_audio_track.h @@ -0,0 +1,81 @@ +#ifndef INTERNAL_CUSTOM_LOCAL_AUDIO_TRACK_HXX +#define INTERNAL_CUSTOM_LOCAL_AUDIO_TRACK_HXX + +#include "api/audio_options.h" +#include "api/media_stream_interface.h" +#include "api/notifier.h" +#include "api/scoped_refptr.h" +#include "call/audio_sender.h" +#include "src/internal/custom_audio_transport_impl.h" + +namespace libwebrtc { + +using namespace webrtc; + +class LocalAudioSource : public Notifier, AudioSender { + public: + // Creates an instance of CustomLocalAudioSource. + static rtc::scoped_refptr Create( + const cricket::AudioOptions* audio_options, + webrtc::CustomAudioTransportImpl* audio_transport); + + SourceState state() const override { return kLive; } + bool remote() const override { return false; } + + const cricket::AudioOptions options() const override { return options_; } + + void AddSink(AudioTrackSinkInterface* sink) override { + webrtc::MutexLock lock(&sink_lock_); + if (std::find(sinks_.begin(), sinks_.end(), sink) != sinks_.end()) { + return; // Already added. + } + sinks_.push_back(sink); + } + + void RemoveSink(AudioTrackSinkInterface* sink) override { + webrtc::MutexLock lock(&sink_lock_); + auto it = std::remove(sinks_.begin(), sinks_.end(), sink); + if (it != sinks_.end()) { + sinks_.erase(it, sinks_.end()); + } + } + + void SendAudioData(std::unique_ptr audio_frame) override { + OnData((const void*)audio_frame->data(), 16, audio_frame->sample_rate_hz(), + audio_frame->num_channels(), audio_frame->samples_per_channel()); + } + + void OnData(const void* audio_data, int bits_per_sample, int sample_rate, + size_t number_of_channels, size_t number_of_frames) { + webrtc::MutexLock lock(&sink_lock_); + for (auto* sink : sinks_) { + sink->OnData(audio_data, bits_per_sample, sample_rate, number_of_channels, + number_of_frames); + } + } + + protected: + LocalAudioSource(webrtc::CustomAudioTransportImpl* audio_transport) + : audio_transport_(audio_transport) { + if (audio_transport_) { + audio_transport_->AddAudioSender(this); + } + } + ~LocalAudioSource() override { + webrtc::MutexLock lock(&sink_lock_); + if (audio_transport_) { + audio_transport_->RemoveAudioSender(this); + } + sinks_.clear(); + } + + private: + void Initialize(const cricket::AudioOptions* audio_options); + mutable webrtc::Mutex sink_lock_; + std::vector sinks_ RTC_GUARDED_BY(sink_lock_); + cricket::AudioOptions options_; + webrtc::CustomAudioTransportImpl* audio_transport_; +}; +} // namespace libwebrtc + +#endif // INTERNAL_CUSTOM_LOCAL_AUDIO_TRACK_HXX diff --git a/src/rtc_audio_processing_impl.cc b/src/rtc_audio_processing_impl.cc new file mode 100644 index 0000000000..ffed8051e4 --- /dev/null +++ b/src/rtc_audio_processing_impl.cc @@ -0,0 +1,105 @@ +#include "rtc_audio_processing_impl.h" + +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/ns/ns_common.h" +#include "rtc_base/logging.h" +#include "rtc_base/synchronization/mutex.h" + +namespace libwebrtc { + +class CustomProcessingAdapter : public webrtc::CustomProcessing { + public: + CustomProcessingAdapter() = default; + ~CustomProcessingAdapter() override = default; + + void SetExternalAudioProcessing( + RTCAudioProcessing::CustomProcessing* processor) { + webrtc::MutexLock lock(&mutex_); + custom_processor_ = processor; + if (initialized_) { + custom_processor_->Initialize(sample_rate_hz_, num_channels_); + } + } + + void SetBypassFlag(bool bypass) { + webrtc::MutexLock lock(&mutex_); + bypass_flag_ = bypass; + } + + private: + void Initialize(int sample_rate_hz, int num_channels) override { + webrtc::MutexLock lock(&mutex_); + sample_rate_hz_ = sample_rate_hz; + num_channels_ = num_channels; + if (custom_processor_) { + custom_processor_->Initialize(sample_rate_hz, num_channels); + } + initialized_ = true; + } + + void Process(webrtc::AudioBuffer* audio) override { + webrtc::MutexLock lock(&mutex_); + if (!custom_processor_ || bypass_flag_ || !initialized_) { + return; + } + + size_t num_frames = audio->num_frames(); + size_t num_bands = audio->num_bands(); + + // 1 buffer = 10ms of frames + int rate = num_frames * 100; + + if (rate != sample_rate_hz_) { + custom_processor_->Reset(rate); + sample_rate_hz_ = rate; + } + + custom_processor_->Process(num_bands, num_frames, + webrtc::kNsFrameSize * num_bands, + audio->channels()[0]); + } + + std::string ToString() const override { return "ExternalAudioProcessor"; } + void SetRuntimeSetting( + webrtc::AudioProcessing::RuntimeSetting setting) override {} + + private: + mutable webrtc::Mutex mutex_; + RTCAudioProcessing::CustomProcessing* custom_processor_; + bool bypass_flag_ = false; + bool initialized_ = false; + int sample_rate_hz_ = 0; + int num_channels_ = 0; +}; + +RTCAudioProcessingImpl::RTCAudioProcessingImpl() { + capture_post_processor_ = new CustomProcessingAdapter(); + std::unique_ptr capture_post_processor( + capture_post_processor_); + + render_pre_processor_ = new CustomProcessingAdapter(); + std::unique_ptr render_pre_processor( + render_pre_processor_); + + apm_ = webrtc::AudioProcessingBuilder() + .SetCapturePostProcessing(std::move(capture_post_processor)) + .SetRenderPreProcessing(std::move(render_pre_processor)) + .Create(); + + webrtc::AudioProcessing::Config config; + apm_->ApplyConfig(config); +} + +RTCAudioProcessingImpl::~RTCAudioProcessingImpl() {} + +void RTCAudioProcessingImpl::SetCapturePostProcessing( + RTCAudioProcessing::CustomProcessing* processor) { + capture_post_processor_->SetExternalAudioProcessing(processor); +} + +void RTCAudioProcessingImpl::SetRenderPreProcessing( + RTCAudioProcessing::CustomProcessing* processor) { + render_pre_processor_->SetExternalAudioProcessing(processor); +} + +} // namespace libwebrtc \ No newline at end of file diff --git a/src/rtc_audio_processing_impl.h b/src/rtc_audio_processing_impl.h new file mode 100644 index 0000000000..574c3e881f --- /dev/null +++ b/src/rtc_audio_processing_impl.h @@ -0,0 +1,35 @@ +#ifndef LIB_WEBRTC_RTC_AUDIO_PROCESSING_IMPL_HXX +#define LIB_WEBRTC_RTC_AUDIO_PROCESSING_IMPL_HXX + +#include + +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_audio_processing.h" + +namespace libwebrtc { + +class CustomProcessingAdapter; + +class RTCAudioProcessingImpl : public RTCAudioProcessing { + public: + RTCAudioProcessingImpl(); + ~RTCAudioProcessingImpl(); + void SetCapturePostProcessing( + RTCAudioProcessing::CustomProcessing* capture_post_processing) override; + + void SetRenderPreProcessing( + RTCAudioProcessing::CustomProcessing* render_pre_processing) override; + + virtual rtc::scoped_refptr GetAudioProcessing() { + return apm_; + } + + private: + CustomProcessingAdapter* capture_post_processor_; + CustomProcessingAdapter* render_pre_processor_; + rtc::scoped_refptr apm_; +}; + +} // namespace libwebrtc + +#endif // LIB_WEBRTC_RTC_AUDIO_PROCESSING_IMPL_HXX diff --git a/src/rtc_audio_source_impl.cc b/src/rtc_audio_source_impl.cc index a9e2526c80..e75513af0f 100644 --- a/src/rtc_audio_source_impl.cc +++ b/src/rtc_audio_source_impl.cc @@ -3,8 +3,9 @@ namespace libwebrtc { RTCAudioSourceImpl::RTCAudioSourceImpl( - rtc::scoped_refptr rtc_audio_source) - : rtc_audio_source_(rtc_audio_source) { + rtc::scoped_refptr rtc_audio_source, + SourceType source_type) + : rtc_audio_source_(rtc_audio_source), source_type_(source_type) { RTC_LOG(LS_INFO) << __FUNCTION__ << ": ctor "; } diff --git a/src/rtc_audio_source_impl.h b/src/rtc_audio_source_impl.h index 61bd922f5e..d295755241 100644 --- a/src/rtc_audio_source_impl.h +++ b/src/rtc_audio_source_impl.h @@ -11,13 +11,26 @@ #include "rtc_audio_source.h" #include "rtc_base/logging.h" #include "rtc_base/synchronization/mutex.h" +#include "src/internal/local_audio_track.h" namespace libwebrtc { class RTCAudioSourceImpl : public RTCAudioSource { public: RTCAudioSourceImpl( - rtc::scoped_refptr rtc_audio_source); + rtc::scoped_refptr rtc_audio_source, + SourceType source_type); + + void CaptureFrame(const void* audio_data, int bits_per_sample, + int sample_rate, size_t number_of_channels, + size_t number_of_frames) override { + RTC_DCHECK(rtc_audio_source_); + RTC_DCHECK(audio_data); + rtc_audio_source_->OnData(audio_data, bits_per_sample, sample_rate, + number_of_channels, number_of_frames); + } + + SourceType GetSourceType() const override { return source_type_; } virtual ~RTCAudioSourceImpl(); @@ -26,7 +39,8 @@ class RTCAudioSourceImpl : public RTCAudioSource { } private: - rtc::scoped_refptr rtc_audio_source_; + rtc::scoped_refptr rtc_audio_source_; + SourceType source_type_; }; } // namespace libwebrtc diff --git a/src/rtc_audio_track_impl.h b/src/rtc_audio_track_impl.h index 1fd120ea9f..7f5ed19f18 100644 --- a/src/rtc_audio_track_impl.h +++ b/src/rtc_audio_track_impl.h @@ -14,6 +14,22 @@ namespace libwebrtc { +class AudioTrackSinkAdapter : public webrtc::AudioTrackSinkInterface { + public: + AudioTrackSinkAdapter(AudioTrackSink* sink) : sink_(sink) {} + + void OnData(const void* audio_data, int bits_per_sample, int sample_rate, + size_t number_of_channels, size_t number_of_frames) override { + if (sink_) { + sink_->OnData(audio_data, bits_per_sample, sample_rate, + number_of_channels, number_of_frames); + } + } + + private: + AudioTrackSink* sink_; +}; + class AudioTrackImpl : public RTCAudioTrack { public: AudioTrackImpl(rtc::scoped_refptr audio_track); @@ -32,6 +48,26 @@ class AudioTrackImpl : public RTCAudioTrack { return rtc_track_->set_enabled(enable); } + virtual void AddSink(AudioTrackSink* sink) override { + webrtc::MutexLock lock(&mutex_); + if (sinks_.find(sink) != sinks_.end()) { + return; + } + auto adapter = std::make_unique(sink); + rtc_track_->AddSink(adapter.get()); + sinks_[sink] = std::move(adapter); + } + + virtual void RemoveSink(AudioTrackSink* sink) override { + webrtc::MutexLock lock(&mutex_); + auto it = sinks_.find(sink); + if (it == sinks_.end()) { + return; + } + rtc_track_->RemoveSink(it->second.get()); + sinks_.erase(it); + } + rtc::scoped_refptr rtc_track() { return rtc_track_; } @@ -42,6 +78,8 @@ class AudioTrackImpl : public RTCAudioTrack { private: rtc::scoped_refptr rtc_track_; + std::map> sinks_; + webrtc::Mutex mutex_; string id_, kind_; }; diff --git a/src/rtc_peerconnection_factory_impl.cc b/src/rtc_peerconnection_factory_impl.cc index 7c7fe2d452..b9388d6ba3 100644 --- a/src/rtc_peerconnection_factory_impl.cc +++ b/src/rtc_peerconnection_factory_impl.cc @@ -24,6 +24,8 @@ #endif #include +#include "src/internal/custom_media_context.h" + namespace libwebrtc { #if defined(USE_INTEL_MEDIA_SDK) @@ -55,6 +57,9 @@ bool RTCPeerConnectionFactoryImpl::Initialize() { signaling_thread_->SetName("signaling_thread", nullptr); RTC_CHECK(signaling_thread_->Start()) << "Failed to start thread"; + custom_media_context_ = rtc::make_ref_counted( + signaling_thread_.get()); + network_thread_ = rtc::Thread::CreateWithSocketServer(); network_thread_->SetName("network_thread", nullptr); RTC_CHECK(network_thread_->Start()) << "Failed to start thread"; @@ -63,18 +68,26 @@ bool RTCPeerConnectionFactoryImpl::Initialize() { worker_thread_->BlockingCall([=] { CreateAudioDeviceModule_w(); }); } + if (!audio_processing_impl_) { + worker_thread_->BlockingCall([this] { + audio_processing_impl_ = new RefCountedObject(); + }); + } + if (!rtc_peerconnection_factory_) { - rtc_peerconnection_factory_ = webrtc::CreatePeerConnectionFactory( - network_thread_.get(), worker_thread_.get(), signaling_thread_.get(), - audio_device_module_, webrtc::CreateBuiltinAudioEncoderFactory(), - webrtc::CreateBuiltinAudioDecoderFactory(), + rtc_peerconnection_factory_ = + custom_media_context_->CreatePeerConnectionFactory( + network_thread_.get(), worker_thread_.get(), + signaling_thread_.get(), audio_device_module_, + webrtc::CreateBuiltinAudioEncoderFactory(), + webrtc::CreateBuiltinAudioDecoderFactory(), #if defined(USE_INTEL_MEDIA_SDK) - CreateIntelVideoEncoderFactory(), CreateIntelVideoDecoderFactory(), + CreateIntelVideoEncoderFactory(), CreateIntelVideoDecoderFactory(), #else - webrtc::CreateBuiltinVideoEncoderFactory(), - webrtc::CreateBuiltinVideoDecoderFactory(), + webrtc::CreateBuiltinVideoEncoderFactory(), + webrtc::CreateBuiltinVideoDecoderFactory(), #endif - nullptr, nullptr); + nullptr, audio_processing_impl_->GetAudioProcessing(), nullptr); } if (!rtc_peerconnection_factory_.get()) { @@ -89,6 +102,7 @@ bool RTCPeerConnectionFactoryImpl::Terminate() { worker_thread_->BlockingCall([&] { audio_device_impl_ = nullptr; video_device_impl_ = nullptr; + audio_processing_impl_ = nullptr; }); rtc_peerconnection_factory_ = NULL; if (audio_device_module_) { @@ -144,6 +158,17 @@ scoped_refptr RTCPeerConnectionFactoryImpl::GetAudioDevice() { return audio_device_impl_; } +scoped_refptr +RTCPeerConnectionFactoryImpl::GetAudioProcessing() { + if (!audio_processing_impl_) { + worker_thread_->BlockingCall([this] { + audio_processing_impl_ = new RefCountedObject(); + }); + } + + return audio_processing_impl_; +} + scoped_refptr RTCPeerConnectionFactoryImpl::GetVideoDevice() { if (!video_device_impl_) video_device_impl_ = scoped_refptr( @@ -153,12 +178,12 @@ scoped_refptr RTCPeerConnectionFactoryImpl::GetVideoDevice() { } scoped_refptr RTCPeerConnectionFactoryImpl::CreateAudioSource( - const string audio_source_label) { - rtc::scoped_refptr rtc_source_track = - rtc_peerconnection_factory_->CreateAudioSource(cricket::AudioOptions()); - + const string audio_source_label, RTCAudioSource::SourceType source_type) { + auto options = cricket::AudioOptions(); + rtc::scoped_refptr rtc_source_track = + custom_media_context_->CreateAudioSource(&options); scoped_refptr source = scoped_refptr( - new RefCountedObject(rtc_source_track)); + new RefCountedObject(rtc_source_track, source_type)); return source; } diff --git a/src/rtc_peerconnection_factory_impl.h b/src/rtc_peerconnection_factory_impl.h index f87bba67ff..4c62b3703c 100644 --- a/src/rtc_peerconnection_factory_impl.h +++ b/src/rtc_peerconnection_factory_impl.h @@ -7,6 +7,7 @@ #include "api/peer_connection_interface.h" #include "api/task_queue/task_queue_factory.h" #include "rtc_audio_device_impl.h" +#include "rtc_audio_processing_impl.h" #include "rtc_base/thread.h" #include "rtc_peerconnection.h" #include "rtc_peerconnection_factory.h" @@ -18,6 +19,8 @@ #include "src/internal/desktop_capturer.h" #endif +#include "src/internal/custom_media_context.h" + namespace libwebrtc { class RTCPeerConnectionFactoryImpl : public RTCPeerConnectionFactory { @@ -40,8 +43,11 @@ class RTCPeerConnectionFactoryImpl : public RTCPeerConnectionFactory { scoped_refptr GetVideoDevice() override; + scoped_refptr GetAudioProcessing() override; + virtual scoped_refptr CreateAudioSource( - const string audio_source_label) override; + const string audio_source_label, + RTCAudioSource::SourceType source_type) override; virtual scoped_refptr CreateVideoSource( scoped_refptr capturer, const string video_source_label, @@ -93,10 +99,12 @@ class RTCPeerConnectionFactoryImpl : public RTCPeerConnectionFactory { std::unique_ptr worker_thread_; std::unique_ptr signaling_thread_; std::unique_ptr network_thread_; + rtc::scoped_refptr custom_media_context_; rtc::scoped_refptr rtc_peerconnection_factory_; rtc::scoped_refptr audio_device_module_; scoped_refptr audio_device_impl_; + scoped_refptr audio_processing_impl_; scoped_refptr video_device_impl_; #ifdef RTC_DESKTOP_DEVICE scoped_refptr desktop_device_impl_;