feat: audio processing api.

BUILD.gn

@@ -69,6 +69,7 @@ rtc_shared_library("libwebrtc") {
     "include/base/scoped_ref_ptr.h",
     "include/libwebrtc.h",
     "include/rtc_audio_device.h",
+    "include/rtc_audio_processing.h",
     "include/rtc_audio_source.h",
     "include/rtc_audio_track.h",
     "include/rtc_data_channel.h",
@@ -96,13 +97,25 @@ rtc_shared_library("libwebrtc") {
     "include/helper.h",
     "src/helper.cc",
     "src/base/portable.cc",
+    "src/internal/custom_audio_state.cc",
+    "src/internal/custom_audio_state.h",
+    "src/internal/custom_audio_transport_impl.cc",
+    "src/internal/custom_audio_transport_impl.h",
+    "src/internal/local_audio_track.cc",
+    "src/internal/local_audio_track.h",
+    "src/internal/custom_media_context.cc",
+    "src/internal/custom_media_context.h",
+    "src/internal/custom_webrtc_voice_engine.cc",
+    "src/internal/custom_webrtc_voice_engine.h",
     "src/internal/vcm_capturer.cc",
     "src/internal/vcm_capturer.h",
     "src/internal/video_capturer.cc",
     "src/internal/video_capturer.h",
     "src/libwebrtc.cc",
     "src/rtc_audio_device_impl.cc",
     "src/rtc_audio_device_impl.h",
+    "src/rtc_audio_processing_impl.cc",
+    "src/rtc_audio_processing_impl.h",
     "src/rtc_audio_source_impl.cc",
     "src/rtc_audio_source_impl.h",
     "src/rtc_audio_track_impl.cc",

include/rtc_audio_frame.h

@@ -1,17 +1,17 @@
-#ifndef AUDIO_FRAME_HXX
-#define AUDIO_FRAME_HXX
+#ifndef LIB_WEBRTC_RTC_AUDIO_FRAME_HXX
+#define LIB_WEBRTC_RTC_AUDIO_FRAME_HXX
 
-#include "media_manager_types.h"
+#include "rtc_types.h"
 
-namespace b2bua {
+namespace libwebrtc {
 
-class AudioFrame {
+class AudioFrame : public RefCountInterface {
  public:
   /**
    * @brief Creates a new instance of AudioFrame.
    * @return AudioFrame*: a pointer to the newly created AudioFrame.
    */
-  MEDIA_MANAGER_API static AudioFrame* Create();
+  LIB_WEBRTC_API static AudioFrame* Create();
 
   /**
    * @brief Creates a new instance of AudioFrame with specified parameters.
@@ -23,16 +23,11 @@ class AudioFrame {
    * @param num_channels: the number of audio channels.
    * @return AudioFrame*: a pointer to the newly created AudioFrame.
    */
-  MEDIA_MANAGER_API static AudioFrame* Create(int id, uint32_t timestamp,
-                                              const int16_t* data,
-                                              size_t samples_per_channel,
-                                              int sample_rate_hz,
-                                              size_t num_channels = 1);
-
-  /**
-   * @brief Releases the memory of this AudioFrame.
-   */
-  virtual void Release() = 0;
+  LIB_WEBRTC_API static AudioFrame* Create(int id, uint32_t timestamp,
+                                           const int16_t* data,
+                                           size_t samples_per_channel,
+                                           int sample_rate_hz,
+                                           size_t num_channels = 1);
 
  public:
   /**
@@ -103,6 +98,6 @@ class AudioFrame {
   virtual int id() = 0;
 };
 
-};  // namespace b2bua
+};  // namespace libwebrtc
 
-#endif
+#endif  // LIB_WEBRTC_RTC_AUDIO_FRAME_HXX

include/rtc_audio_processing.h

@@ -0,0 +1,35 @@
+#ifndef LIB_WEBRTC_RTC_AUDIO_PROCESSING_HXX
+#define LIB_WEBRTC_RTC_AUDIO_PROCESSING_HXX
+
+#include "rtc_types.h"
+
+namespace libwebrtc {
+
+class RTCAudioProcessing : public RefCountInterface {
+ public:
+  class CustomProcessing {
+   public:
+    virtual void Initialize(int sample_rate_hz, int num_channels) = 0;
+
+    virtual void Process(int num_bands, int num_frames, int buffer_size,
+                         float* buffer) = 0;
+
+    virtual void Reset(int new_rate) = 0;
+
+    virtual void Release() = 0;
+
+   protected:
+    virtual ~CustomProcessing() {}
+  };
+
+ public:
+  virtual void SetCapturePostProcessing(
+      CustomProcessing* capture_post_processing) = 0;
+
+  virtual void SetRenderPreProcessing(
+      CustomProcessing* render_pre_processing) = 0;
+};
+
+}  // namespace libwebrtc
+
+#endif  // LIB_WEBRTC_RTC_AUDIO_PROCESSING_HXX

include/rtc_audio_source.h

@@ -13,6 +13,16 @@ namespace libwebrtc {
  * processing and transmission mechanisms.
  */
 class RTCAudioSource : public RefCountInterface {
+ public:
+  enum SourceType { kMicrophone, kCustom };
+
+ public:
+  virtual void CaptureFrame(const void* audio_data, int bits_per_sample,
+                            int sample_rate, size_t number_of_channels,
+                            size_t number_of_frames) = 0;
+
+  virtual SourceType GetSourceType() const = 0;
+
  protected:
   /**
    * The destructor for the RTCAudioSource class.

include/rtc_audio_track.h

@@ -17,6 +17,10 @@ class RTCAudioTrack : public RTCMediaTrack {
   // volume in [0-10]
   virtual void SetVolume(double volume) = 0;
 
+  virtual void AddSink(AudioTrackSink* sink) = 0;
+
+  virtual void RemoveSink(AudioTrackSink* sink) = 0;
+
  protected:
   /**
    * The destructor for the RTCAudioTrack class.

include/rtc_media_track.h

@@ -5,6 +5,15 @@
 
 namespace libwebrtc {
 
+class AudioTrackSink {
+ public:
+  virtual void OnData(const void* audio_data, int bits_per_sample,
+                      int sample_rate, size_t number_of_channels,
+                      size_t number_of_frames) = 0;
+ protected:
+  virtual ~AudioTrackSink() {}
+};
+
 /*Media Track interface*/
 class RTCMediaTrack : public RefCountInterface {
  public:

include/rtc_peerconnection_factory.h

@@ -16,6 +16,7 @@ namespace libwebrtc {
 
 class RTCPeerConnection;
 class RTCAudioDevice;
+class RTCAudioProcessing;
 class RTCVideoDevice;
 class RTCRtpCapabilities;
 
@@ -33,12 +34,16 @@ class RTCPeerConnectionFactory : public RefCountInterface {
 
   virtual scoped_refptr<RTCAudioDevice> GetAudioDevice() = 0;
 
+  virtual scoped_refptr<RTCAudioProcessing> GetAudioProcessing() = 0;
+
   virtual scoped_refptr<RTCVideoDevice> GetVideoDevice() = 0;
 #ifdef RTC_DESKTOP_DEVICE
   virtual scoped_refptr<RTCDesktopDevice> GetDesktopDevice() = 0;
 #endif
   virtual scoped_refptr<RTCAudioSource> CreateAudioSource(
-      const string audio_source_label) = 0;
+      const string audio_source_label,
+      RTCAudioSource::SourceType source_type =
+          RTCAudioSource::SourceType::kMicrophone) = 0;
 
   virtual scoped_refptr<RTCVideoSource> CreateVideoSource(
       scoped_refptr<RTCVideoCapturer> capturer, const string video_source_label,

patchs/custom_audio_source.patch

@@ -0,0 +1,92 @@
+diff --git a/audio/audio_receive_stream.cc b/audio/audio_receive_stream.cc
+index 415ad0640a..1467860a71 100644
+--- a/audio/audio_receive_stream.cc
++++ b/audio/audio_receive_stream.cc
+@@ -467,8 +467,8 @@ AudioReceiveStreamImpl::GetAssociatedSendStreamForTesting() const {
+   return associated_send_stream_;
+ }
+
+-internal::AudioState* AudioReceiveStreamImpl::audio_state() const {
+-  auto* audio_state = static_cast<internal::AudioState*>(audio_state_.get());
++webrtc::AudioState* AudioReceiveStreamImpl::audio_state() const {
++  auto* audio_state = static_cast<webrtc::AudioState*>(audio_state_.get());
+   RTC_DCHECK(audio_state);
+   return audio_state;
+ }
+diff --git a/audio/audio_receive_stream.h b/audio/audio_receive_stream.h
+index db49631638..982cad39a4 100644
+--- a/audio/audio_receive_stream.h
++++ b/audio/audio_receive_stream.h
+@@ -144,7 +144,7 @@ class AudioReceiveStreamImpl final : public webrtc::AudioReceiveStreamInterface,
+       const webrtc::AudioReceiveStreamInterface::Config& config);
+
+  private:
+-  internal::AudioState* audio_state() const;
++  webrtc::AudioState* audio_state() const;
+
+   RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_thread_checker_;
+   // TODO(bugs.webrtc.org/11993): This checker conceptually represents
+diff --git a/audio/audio_state.h b/audio/audio_state.h
+index f21cca771e..a4eb0fedf1 100644
+--- a/audio/audio_state.h
++++ b/audio/audio_state.h
+@@ -54,13 +54,14 @@ class AudioState : public webrtc::AudioState {
+     return config_.audio_device_module.get();
+   }
+
+-  void AddReceivingStream(webrtc::AudioReceiveStreamInterface* stream);
+-  void RemoveReceivingStream(webrtc::AudioReceiveStreamInterface* stream);
++  void AddReceivingStream(webrtc::AudioReceiveStreamInterface* stream) override;
++  void RemoveReceivingStream(
++      webrtc::AudioReceiveStreamInterface* stream) override;
+
+   void AddSendingStream(webrtc::AudioSendStream* stream,
+                         int sample_rate_hz,
+-                        size_t num_channels);
+-  void RemoveSendingStream(webrtc::AudioSendStream* stream);
++                        size_t num_channels) override;
++  void RemoveSendingStream(webrtc::AudioSendStream* stream) override;
+
+  private:
+   void UpdateAudioTransportWithSendingStreams();
+diff --git a/audio/audio_transport_impl.h b/audio/audio_transport_impl.h
+index 24b09d2140..1b39e52869 100644
+--- a/audio/audio_transport_impl.h
++++ b/audio/audio_transport_impl.h
+@@ -82,7 +82,7 @@ class AudioTransportImpl : public AudioTransport {
+                       int64_t* elapsed_time_ms,
+                       int64_t* ntp_time_ms) override;
+
+-  void UpdateAudioSenders(std::vector<AudioSender*> senders,
++  virtual void UpdateAudioSenders(std::vector<AudioSender*> senders,
+                           int send_sample_rate_hz,
+                           size_t send_num_channels);
+   void SetStereoChannelSwapping(bool enable);
+diff --git a/call/audio_state.h b/call/audio_state.h
+index 85f04758dd..7bd7ce25f1 100644
+--- a/call/audio_state.h
++++ b/call/audio_state.h
+@@ -20,6 +20,8 @@
+ namespace webrtc {
+
+ class AudioTransport;
++class AudioReceiveStreamInterface;
++class AudioSendStream;
+
+ // AudioState holds the state which must be shared between multiple instances of
+ // webrtc::Call for audio processing purposes.
+@@ -62,6 +64,14 @@ class AudioState : public rtc::RefCountInterface {
+   // Notify the AudioState that a stream updated it's mute state.
+   virtual void OnMuteStreamChanged() = 0;
+
++  virtual void AddReceivingStream(AudioReceiveStreamInterface* stream) = 0;
++  virtual void RemoveReceivingStream(AudioReceiveStreamInterface* stream) = 0;
++
++  virtual void AddSendingStream(AudioSendStream* stream,
++                        int sample_rate_hz,
++                        size_t num_channels) = 0;
++  virtual void RemoveSendingStream(AudioSendStream* stream) = 0;
++
+   static rtc::scoped_refptr<AudioState> Create(
+       const AudioState::Config& config);
+

patchs/fix_emplace.patch

@@ -0,0 +1,33 @@
+diff --git a/modules/congestion_controller/goog_cc/loss_based_bwe_v2.cc b/modules/congestion_controller/goog_cc/loss_based_bwe_v2.cc
+index 7deeb7ad64..f913557f8c 100644
+--- a/modules/congestion_controller/goog_cc/loss_based_bwe_v2.cc
++++ b/modules/congestion_controller/goog_cc/loss_based_bwe_v2.cc
+@@ -545,11 +545,14 @@ absl::optional<LossBasedBweV2::Config> LossBasedBweV2::CreateConfig(
+                     key_value_config->Lookup("WebRTC-Bwe-LossBasedBweV2"));
+   }
+
+-  absl::optional<Config> config;
++  // absl::optional<Config> config;
+   if (!enabled.Get()) {
+-    return config;
++    return std::nullopt;
+   }
+-  config.emplace();
++
++  Config _config;
++  Config* config = &_config;
++
+   config->bandwidth_rampup_upper_bound_factor =
+       bandwidth_rampup_upper_bound_factor.Get();
+   config->bandwidth_rampup_upper_bound_factor_in_hold =
+@@ -608,7 +611,7 @@ absl::optional<LossBasedBweV2::Config> LossBasedBweV2::CreateConfig(
+   config->padding_duration = padding_duration.Get();
+   config->bound_best_candidate = bound_best_candidate.Get();
+   config->pace_at_loss_based_estimate = pace_at_loss_based_estimate.Get();
+-  return config;
++  return _config;
+ }
+
+ bool LossBasedBweV2::IsConfigValid() const {
+
+