From 9865b027b10d786007ba937bd4a3324893aaf5a2 Mon Sep 17 00:00:00 2001 From: melpon Date: Tue, 28 May 2024 13:50:51 +0900 Subject: [PATCH] =?UTF-8?q?=E3=83=93=E3=83=AB=E3=83=89=E3=82=92=E9=80=9A?= =?UTF-8?q?=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/dummy_audio_mixer.cpp | 8 +++---- src/dummy_audio_mixer.h | 5 +++-- src/dynamic_h264_decoder.cpp | 4 ++-- src/dynamic_h264_encoder.cpp | 42 +++++++++++++++++++++--------------- src/dynamic_h264_encoder.h | 17 +++++++++------ src/sora_factory.cpp | 4 +++- 6 files changed, 46 insertions(+), 34 deletions(-) diff --git a/src/dummy_audio_mixer.cpp b/src/dummy_audio_mixer.cpp index d8413fd8..8a4b49d9 100644 --- a/src/dummy_audio_mixer.cpp +++ b/src/dummy_audio_mixer.cpp @@ -52,12 +52,10 @@ DummyAudioMixer::DummyAudioMixer(webrtc::TaskQueueFactory* task_queue_factory) * sora::SoraClientContextConfig::use_audio_device を false にした際に設定される、 * webrtc::AudioDeviceDummy はループを回さないため、ここでループを作ることとした。 */ - task_queue_ = - std::make_unique(task_queue_factory_->CreateTaskQueue( - "TestAudioDeviceModuleImpl", - webrtc::TaskQueueFactory::Priority::NORMAL)); + task_queue_ = task_queue_factory_->CreateTaskQueue( + "TestAudioDeviceModuleImpl", webrtc::TaskQueueFactory::Priority::NORMAL); - webrtc::RepeatingTaskHandle::Start(task_queue_->Get(), [this]() { + webrtc::RepeatingTaskHandle::Start(task_queue_.get(), [this]() { ProcessAudio(); // オーディオフレームは 10 ms ごとに処理するため 10000 us を指定する return webrtc::TimeDelta::Micros(10000); diff --git a/src/dummy_audio_mixer.h b/src/dummy_audio_mixer.h index 6d2e0b0d..eb63bc71 100644 --- a/src/dummy_audio_mixer.h +++ b/src/dummy_audio_mixer.h @@ -7,8 +7,9 @@ #include #include #include +#include +#include #include -#include #include #include @@ -44,7 +45,7 @@ class DummyAudioMixer : public webrtc::AudioMixer { private: void ProcessAudio(); const webrtc::TaskQueueFactory* task_queue_factory_; - std::unique_ptr task_queue_; + std::unique_ptr task_queue_; mutable webrtc::Mutex mutex_; std::vector> audio_source_list_ diff --git a/src/dynamic_h264_decoder.cpp b/src/dynamic_h264_decoder.cpp index 6da57e35..8b265072 100644 --- a/src/dynamic_h264_decoder.cpp +++ b/src/dynamic_h264_decoder.cpp @@ -4,6 +4,7 @@ // WebRTC #include +#include #include #include @@ -33,8 +34,7 @@ bool DynamicH264Decoder::Configure(const Settings& settings) { Release(); return false; } - destroy_decoder_ = - (DestroyDecoderFunc)::dlsym(handle, "WelsDestroyDecoder"); + destroy_decoder_ = (DestroyDecoderFunc)::dlsym(handle, "WelsDestroyDecoder"); if (destroy_decoder_ == nullptr) { RTC_LOG(LS_ERROR) << "Failed to dlsym(WelsDestroyDecoder)"; Release(); diff --git a/src/dynamic_h264_encoder.cpp b/src/dynamic_h264_encoder.cpp index 44cd5712..f2c6b277 100644 --- a/src/dynamic_h264_encoder.cpp +++ b/src/dynamic_h264_encoder.cpp @@ -14,18 +14,21 @@ #include "dynamic_h264_encoder.h" -#include - #include #include #include +// Linux +#include + // WebRTC #include #include #include #include #include +#include +#include #include #include #include @@ -53,7 +56,7 @@ static const int kLowH264QpThreshold = 24; static const int kHighH264QpThreshold = 37; // Used by histograms. Values of entries should not be changed. -enum DynamicH264EncoderEvent { +enum H264EncoderImplEvent { kH264EncoderEventInit = 0, kH264EncoderEventError = 1, kH264EncoderEventMax = 16, @@ -173,22 +176,17 @@ static void RtpFragmentize(EncodedImage* encoded_image, SFrameBSInfo* info) { } } -DynamicH264Encoder::DynamicH264Encoder(const cricket::VideoCodec& codec, +DynamicH264Encoder::DynamicH264Encoder(const Environment& env, + H264EncoderSettings settings, std::string openh264) - : packetization_mode_(H264PacketizationMode::SingleNalUnit), + : env_(env), + packetization_mode_(settings.packetization_mode), max_payload_size_(0), number_of_cores_(0), encoded_image_callback_(nullptr), has_reported_init_(false), has_reported_error_(false), openh264_(std::move(openh264)) { - RTC_CHECK(absl::EqualsIgnoreCase(codec.name, cricket::kH264CodecName)); - std::string packetization_mode_string; - if (codec.GetParam(cricket::kH264FmtpPacketizationMode, - &packetization_mode_string) && - packetization_mode_string == "1") { - packetization_mode_ = H264PacketizationMode::NonInterleaved; - } downscaled_buffers_.reserve(kMaxSimulcastStreams - 1); encoded_images_.reserve(kMaxSimulcastStreams); encoders_.reserve(kMaxSimulcastStreams); @@ -260,7 +258,7 @@ int32_t DynamicH264Encoder::InitEncode(const VideoCodec* inst, ++i, --idx) { ISVCEncoder* openh264_encoder; // Create encoder. - if (create_encoder_(&openh264_encoder) != 0) { + if (WelsCreateSVCEncoder(&openh264_encoder) != 0) { // Failed to create encoder. RTC_LOG(LS_ERROR) << "Failed to create OpenH264 encoder"; RTC_DCHECK(!openh264_encoder); @@ -353,7 +351,7 @@ int32_t DynamicH264Encoder::Release() { ISVCEncoder* openh264_encoder = encoders_.back(); if (openh264_encoder) { RTC_CHECK_EQ(0, openh264_encoder->Uninitialize()); - destroy_encoder_(openh264_encoder); + WelsDestroySVCEncoder(openh264_encoder); } encoders_.pop_back(); } @@ -544,7 +542,7 @@ int32_t DynamicH264Encoder::Encode( encoded_images_[i]._encodedWidth = configurations_[i].width; encoded_images_[i]._encodedHeight = configurations_[i].height; - encoded_images_[i].SetRtpTimestamp(input_frame.timestamp()); + encoded_images_[i].SetRtpTimestamp(input_frame.rtp_timestamp()); encoded_images_[i].SetColorSpace(input_frame.color_space()); encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType); encoded_images_[i].SetSimulcastIndex(configurations_[i].simulcast_idx); @@ -576,10 +574,20 @@ int32_t DynamicH264Encoder::Encode( codec_specific.codecSpecific.H264.base_layer_sync = tid > 0 && tid < tl0sync_limit_[i]; if (svc_controllers_[i]) { + if (encoded_images_[i]._frameType == VideoFrameType::kVideoFrameKey) { + // Reset the ScalableVideoController on key frame + // to reset the expected dependency structure. + layer_frames = + svc_controllers_[i]->NextFrameConfig(/* restart= */ true); + RTC_CHECK_EQ(layer_frames.size(), 1); + RTC_DCHECK_EQ(layer_frames[0].TemporalId(), 0); + RTC_DCHECK_EQ(layer_frames[0].IsKeyframe(), true); + } + if (layer_frames[0].TemporalId() != tid) { RTC_LOG(LS_WARNING) - << "Encoder produced a frame for layer S" << (i + 1) << "T" - << tid + 1 << " that wasn't requested."; + << "Encoder produced a frame with temporal id " << tid + << ", expected " << layer_frames[0].TemporalId() << "."; continue; } encoded_images_[i].SetTemporalIndex(tid); diff --git a/src/dynamic_h264_encoder.h b/src/dynamic_h264_encoder.h index 28f6c8b1..65382c11 100644 --- a/src/dynamic_h264_encoder.h +++ b/src/dynamic_h264_encoder.h @@ -41,12 +41,13 @@ class ISVCEncoder; namespace webrtc { -class DynamicH264Encoder : public H264Encoder { +class DynamicH264Encoder : public VideoEncoder { public: - static std::unique_ptr Create(const cricket::VideoCodec& codec, + static std::unique_ptr Create(const Environment& env, + H264EncoderSettings settings, std::string openh264) { return std::unique_ptr( - new DynamicH264Encoder(codec, std::move(openh264))); + new DynamicH264Encoder(env, settings, std::move(openh264))); } public: @@ -66,9 +67,10 @@ class DynamicH264Encoder : public H264Encoder { void SetStreamState(bool send_stream); }; - public: - explicit DynamicH264Encoder(const cricket::VideoCodec& codec, - const std::string openh264); + DynamicH264Encoder(const Environment& env, + H264EncoderSettings settings, + std::string openh264); + ~DynamicH264Encoder() override; // `settings.max_payload_size` is ignored. @@ -115,6 +117,7 @@ class DynamicH264Encoder : public H264Encoder { absl::InlinedVector, kMaxSimulcastStreams> scalability_modes_; + const Environment env_; VideoCodec codec_; H264PacketizationMode packetization_mode_; size_t max_payload_size_; @@ -141,4 +144,4 @@ class DynamicH264Encoder : public H264Encoder { } // namespace webrtc -#endif // MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_ +#endif // MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_ \ No newline at end of file diff --git a/src/sora_factory.cpp b/src/sora_factory.cpp index 6db9e2b0..aa9b03ea 100644 --- a/src/sora_factory.cpp +++ b/src/sora_factory.cpp @@ -5,6 +5,7 @@ // WebRTC #include +#include #include #include #include @@ -54,7 +55,8 @@ SoraFactory::SoraFactory(std::optional use_hardware_encoder, [openh264 = openh264]( auto format) -> std::unique_ptr { return webrtc::DynamicH264Encoder::Create( - cricket::CreateVideoCodec(format), *openh264); + webrtc::CreateEnvironment(), + webrtc::H264EncoderSettings(), *openh264); })); dependencies.video_encoder_factory = absl::make_unique(