diff --git a/webrtc/modules/video_coding/frame_buffer2.cc b/webrtc/modules/video_coding/frame_buffer2.cc index 3567f5d26..58c41bf1d 100644 --- a/webrtc/modules/video_coding/frame_buffer2.cc +++ b/webrtc/modules/video_coding/frame_buffer2.cc @@ -16,7 +16,6 @@ #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" -#include "webrtc/modules/video_coding/include/video_coding_defines.h" #include "webrtc/modules/video_coding/jitter_estimator.h" #include "webrtc/modules/video_coding/timing.h" #include "webrtc/system_wrappers/include/clock.h" @@ -35,8 +34,7 @@ constexpr int kMaxFramesHistory = 50; FrameBuffer::FrameBuffer(Clock* clock, VCMJitterEstimator* jitter_estimator, - VCMTiming* timing, - VCMReceiveStatisticsCallback* stats_callback) + VCMTiming* timing) : clock_(clock), new_countinuous_frame_event_(false, false), jitter_estimator_(jitter_estimator), @@ -47,10 +45,11 @@ FrameBuffer::FrameBuffer(Clock* clock, num_frames_history_(0), num_frames_buffered_(0), stopped_(false), - protection_mode_(kProtectionNack), - stats_callback_(stats_callback) {} + protection_mode_(kProtectionNack) {} -FrameBuffer::~FrameBuffer() {} +FrameBuffer::~FrameBuffer() { + UpdateHistograms(); +} FrameBuffer::ReturnReason FrameBuffer::NextFrame( int64_t max_wait_time_ms, @@ -164,8 +163,9 @@ int FrameBuffer::InsertFrame(std::unique_ptr frame) { rtc::CritScope lock(&crit_); RTC_DCHECK(frame); - if (stats_callback_) - stats_callback_->OnCompleteFrame(frame->num_references == 0, frame->size()); + ++num_total_frames_; + if (frame->num_references == 0) + ++num_key_frames_; FrameKey key(frame->picture_id, frame->spatial_layer); int last_continuous_picture_id = @@ -379,22 +379,28 @@ bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const FrameObject& frame, } void FrameBuffer::UpdateJitterDelay() { - if (!stats_callback_) - return; - - int decode_ms; - int max_decode_ms; - int current_delay_ms; - int target_delay_ms; - int jitter_buffer_ms; - int min_playout_delay_ms; - int render_delay_ms; - if (timing_->GetTimings(&decode_ms, &max_decode_ms, ¤t_delay_ms, - &target_delay_ms, &jitter_buffer_ms, - &min_playout_delay_ms, &render_delay_ms)) { - stats_callback_->OnFrameBufferTimingsUpdated( - decode_ms, max_decode_ms, current_delay_ms, target_delay_ms, - jitter_buffer_ms, min_playout_delay_ms, render_delay_ms); + int unused; + int delay; + timing_->GetTimings(&unused, &unused, &unused, &unused, &delay, &unused, + &unused); + + accumulated_delay_ += delay; + ++accumulated_delay_samples_; +} + +void FrameBuffer::UpdateHistograms() const { + rtc::CritScope lock(&crit_); + if (num_total_frames_ > 0) { + int key_frames_permille = (static_cast(num_key_frames_) * 1000.0f / + static_cast(num_total_frames_) + + 0.5f); + RTC_HISTOGRAM_COUNTS_1000("WebRTC.Video.KeyFramesReceivedInPermille", + key_frames_permille); + } + + if (accumulated_delay_samples_ > 0) { + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.JitterBufferDelayInMs", + accumulated_delay_ / accumulated_delay_samples_); } } diff --git a/webrtc/modules/video_coding/frame_buffer2.h b/webrtc/modules/video_coding/frame_buffer2.h index 7af48d351..f667fd532 100644 --- a/webrtc/modules/video_coding/frame_buffer2.h +++ b/webrtc/modules/video_coding/frame_buffer2.h @@ -28,7 +28,6 @@ namespace webrtc { class Clock; -class VCMReceiveStatisticsCallback; class VCMJitterEstimator; class VCMTiming; @@ -40,8 +39,7 @@ class FrameBuffer { FrameBuffer(Clock* clock, VCMJitterEstimator* jitter_estimator, - VCMTiming* timing, - VCMReceiveStatisticsCallback* stats_proxy); + VCMTiming* timing); virtual ~FrameBuffer(); @@ -143,6 +141,8 @@ class FrameBuffer { void UpdateJitterDelay() EXCLUSIVE_LOCKS_REQUIRED(crit_); + void UpdateHistograms() const; + void ClearFramesAndHistory() EXCLUSIVE_LOCKS_REQUIRED(crit_); FrameMap frames_ GUARDED_BY(crit_); @@ -160,9 +160,16 @@ class FrameBuffer { int num_frames_buffered_ GUARDED_BY(crit_); bool stopped_ GUARDED_BY(crit_); VCMVideoProtection protection_mode_ GUARDED_BY(crit_); - VCMReceiveStatisticsCallback* const stats_callback_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(FrameBuffer); + + // For WebRTC.Video.JitterBufferDelayInMs metric. + int64_t accumulated_delay_ = 0; + int64_t accumulated_delay_samples_ = 0; + + // For WebRTC.Video.KeyFramesReceivedInPermille metric. + int64_t num_total_frames_ = 0; + int64_t num_key_frames_ = 0; }; } // namespace video_coding diff --git a/webrtc/modules/video_coding/frame_buffer2_unittest.cc b/webrtc/modules/video_coding/frame_buffer2_unittest.cc index 5c12e9459..96be01f83 100644 --- a/webrtc/modules/video_coding/frame_buffer2_unittest.cc +++ b/webrtc/modules/video_coding/frame_buffer2_unittest.cc @@ -25,9 +25,6 @@ #include "webrtc/test/gmock.h" #include "webrtc/test/gtest.h" -using testing::_; -using testing::Return; - namespace webrtc { namespace video_coding { @@ -57,16 +54,6 @@ class VCMTimingFake : public VCMTiming { return std::max(0, render_time_ms - now_ms - kDecodeTime); } - bool GetTimings(int* decode_ms, - int* max_decode_ms, - int* current_delay_ms, - int* target_delay_ms, - int* jitter_buffer_ms, - int* min_playout_delay_ms, - int* render_delay_ms) const override { - return true; - } - private: static constexpr int kDelayMs = 50; static constexpr int kDecodeTime = kDelayMs / 2; @@ -95,27 +82,6 @@ class FrameObjectFake : public FrameObject { int64_t ReceivedTime() const override { return 0; } int64_t RenderTime() const override { return _renderTimeMs; } - - // In EncodedImage |_length| is used to descibe its size and |_size| to - // describe its capacity. - void SetSize(int size) { _length = size; } -}; - -class VCMReceiveStatisticsCallbackMock : public VCMReceiveStatisticsCallback { - public: - MOCK_METHOD2(OnReceiveRatesUpdated, - void(uint32_t bitRate, uint32_t frameRate)); - MOCK_METHOD2(OnCompleteFrame, void(bool is_keyframe, size_t size_bytes)); - MOCK_METHOD1(OnDiscardedPacketsUpdated, void(int discarded_packets)); - MOCK_METHOD1(OnFrameCountsUpdated, void(const FrameCounts& frame_counts)); - MOCK_METHOD7(OnFrameBufferTimingsUpdated, - void(int decode_ms, - int max_decode_ms, - int current_delay_ms, - int target_delay_ms, - int jitter_buffer_ms, - int min_playout_delay_ms, - int render_delay_ms)); }; class TestFrameBuffer2 : public ::testing::Test { @@ -129,7 +95,7 @@ class TestFrameBuffer2 : public ::testing::Test { : clock_(0), timing_(&clock_), jitter_estimator_(&clock_), - buffer_(&clock_, &jitter_estimator_, &timing_, &stats_callback_), + buffer_(&clock_, &jitter_estimator_, &timing_), rand_(0x34678213), tear_down_(false), extract_thread_(&ExtractLoop, this, "Extract Thread"), @@ -224,7 +190,6 @@ class TestFrameBuffer2 : public ::testing::Test { FrameBuffer buffer_; std::vector> frames_; Random rand_; - ::testing::NiceMock stats_callback_; int64_t max_wait_time_; bool tear_down_; @@ -471,30 +436,5 @@ TEST_F(TestFrameBuffer2, PictureIdJumpBack) { CheckNoFrame(2); } -TEST_F(TestFrameBuffer2, StatsCallback) { - uint16_t pid = Rand(); - uint32_t ts = Rand(); - const int kFrameSize = 5000; - - EXPECT_CALL(stats_callback_, OnCompleteFrame(true, kFrameSize)); - EXPECT_CALL(stats_callback_, - OnFrameBufferTimingsUpdated(_, _, _, _, _, _, _)); - - { - std::unique_ptr frame(new FrameObjectFake()); - frame->SetSize(kFrameSize); - frame->picture_id = pid; - frame->spatial_layer = 0; - frame->timestamp = ts; - frame->num_references = 0; - frame->inter_layer_predicted = false; - - EXPECT_EQ(buffer_.InsertFrame(std::move(frame)), pid); - } - - ExtractFrame(); - CheckFrame(0, pid, 0); -} - } // namespace video_coding } // namespace webrtc diff --git a/webrtc/modules/video_coding/include/video_coding_defines.h b/webrtc/modules/video_coding/include/video_coding_defines.h index dede5b6ff..122ddc631 100644 --- a/webrtc/modules/video_coding/include/video_coding_defines.h +++ b/webrtc/modules/video_coding/include/video_coding_defines.h @@ -90,16 +90,8 @@ class VCMSendStatisticsCallback { class VCMReceiveStatisticsCallback { public: virtual void OnReceiveRatesUpdated(uint32_t bitRate, uint32_t frameRate) = 0; - virtual void OnCompleteFrame(bool is_keyframe, size_t size_bytes) = 0; virtual void OnDiscardedPacketsUpdated(int discarded_packets) = 0; virtual void OnFrameCountsUpdated(const FrameCounts& frame_counts) = 0; - virtual void OnFrameBufferTimingsUpdated(int decode_ms, - int max_decode_ms, - int current_delay_ms, - int target_delay_ms, - int jitter_buffer_ms, - int min_playout_delay_ms, - int render_delay_ms) = 0; protected: virtual ~VCMReceiveStatisticsCallback() {} diff --git a/webrtc/modules/video_coding/timing.h b/webrtc/modules/video_coding/timing.h index 429c2825f..e7d2b1ffd 100644 --- a/webrtc/modules/video_coding/timing.h +++ b/webrtc/modules/video_coding/timing.h @@ -94,13 +94,13 @@ class VCMTiming { // Return current timing information. Returns true if the first frame has been // decoded, false otherwise. - virtual bool GetTimings(int* decode_ms, - int* max_decode_ms, - int* current_delay_ms, - int* target_delay_ms, - int* jitter_buffer_ms, - int* min_playout_delay_ms, - int* render_delay_ms) const; + bool GetTimings(int* decode_ms, + int* max_decode_ms, + int* current_delay_ms, + int* target_delay_ms, + int* jitter_buffer_ms, + int* min_playout_delay_ms, + int* render_delay_ms) const; enum { kDefaultRenderDelayMs = 10 }; enum { kDelayMaxChangeMsPerS = 100 }; diff --git a/webrtc/modules/video_coding/video_receiver.cc b/webrtc/modules/video_coding/video_receiver.cc index 14f12653a..129a1b5c4 100644 --- a/webrtc/modules/video_coding/video_receiver.cc +++ b/webrtc/modules/video_coding/video_receiver.cc @@ -56,14 +56,31 @@ VideoReceiver::~VideoReceiver() {} void VideoReceiver::Process() { // Receive-side statistics - - // TODO(philipel): Remove this if block when we know what to do with - // ReceiveStatisticsProxy::QualitySample. if (_receiveStatsTimer.TimeUntilProcess() == 0) { _receiveStatsTimer.Processed(); rtc::CritScope cs(&process_crit_); if (_receiveStatsCallback != nullptr) { - _receiveStatsCallback->OnReceiveRatesUpdated(0, 0); + uint32_t bitRate; + uint32_t frameRate; + _receiver.ReceiveStatistics(&bitRate, &frameRate); + _receiveStatsCallback->OnReceiveRatesUpdated(bitRate, frameRate); + } + + if (_decoderTimingCallback != nullptr) { + int decode_ms; + int max_decode_ms; + int current_delay_ms; + int target_delay_ms; + int jitter_buffer_ms; + int min_playout_delay_ms; + int render_delay_ms; + if (_timing->GetTimings(&decode_ms, &max_decode_ms, ¤t_delay_ms, + &target_delay_ms, &jitter_buffer_ms, + &min_playout_delay_ms, &render_delay_ms)) { + _decoderTimingCallback->OnDecoderTiming( + decode_ms, max_decode_ms, current_delay_ms, target_delay_ms, + jitter_buffer_ms, min_playout_delay_ms, render_delay_ms); + } } } @@ -275,7 +292,7 @@ int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) { return ret; } -// Used for the new jitter buffer. +// Used for the WebRTC-NewVideoJitterBuffer experiment. // TODO(philipel): Clean up among the Decode functions as we replace // VCMEncodedFrame with FrameObject. int32_t VideoReceiver::Decode(const webrtc::VCMEncodedFrame* frame) { diff --git a/webrtc/video/end_to_end_tests.cc b/webrtc/video/end_to_end_tests.cc index a2fdebcf0..54b83e265 100644 --- a/webrtc/video/end_to_end_tests.cc +++ b/webrtc/video/end_to_end_tests.cc @@ -1228,6 +1228,9 @@ TEST_P(EndToEndTest, ReceivesPliAndRecoversWithNack) { } TEST_P(EndToEndTest, ReceivesPliAndRecoversWithoutNack) { + // This test makes no sense for the new video jitter buffer. + if (GetParam() == new_jb_enabled) + return; ReceivesPliAndRecovers(0); } @@ -3027,6 +3030,10 @@ TEST_P(EndToEndTest, GetStats) { ReceiveStreamRenderer receive_stream_renderer_; } test; + // TODO(philipel): Implement statistics for the new video jitter buffer. + if (GetParam() == new_jb_enabled) + return; + RunBaseTest(&test); } diff --git a/webrtc/video/receive_statistics_proxy.cc b/webrtc/video/receive_statistics_proxy.cc index 35f32034f..673fabb7b 100644 --- a/webrtc/video/receive_statistics_proxy.cc +++ b/webrtc/video/receive_statistics_proxy.cc @@ -10,9 +10,7 @@ #include "webrtc/video/receive_statistics_proxy.h" -#include #include -#include #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" @@ -42,9 +40,6 @@ const int kLowQpThresholdVp8 = 60; const int kHighQpThresholdVp8 = 70; const int kLowVarianceThreshold = 1; const int kHighVarianceThreshold = 2; - -// How large window we use to calculate the framerate/bitrate. -const int kRateStatisticsWindowSizeMs = 1000; } // namespace ReceiveStatisticsProxy::ReceiveStatisticsProxy( @@ -74,9 +69,7 @@ ReceiveStatisticsProxy::ReceiveStatisticsProxy( render_fps_tracker_(100, 10u), render_pixel_tracker_(100, 10u), freq_offset_counter_(clock, nullptr, kFreqOffsetProcessIntervalMs), - first_report_block_time_ms_(-1), - avg_rtt_ms_(0), - frame_window_accumulated_bytes_(0) { + first_report_block_time_ms_(-1) { stats_.ssrc = config_.rtp.remote_ssrc; for (auto it : config_.rtp.rtx) rtx_stats_[it.second.ssrc] = StreamDataCounters(); @@ -128,17 +121,6 @@ void ReceiveStatisticsProxy::UpdateHistograms() { << freq_offset_stats.ToString(); } - if (stats_.frame_counts.key_frames > 0 || - stats_.frame_counts.delta_frames > 0) { - float num_key_frames = stats_.frame_counts.key_frames; - float num_total_frames = - stats_.frame_counts.key_frames + stats_.frame_counts.delta_frames; - int key_frames_permille = - (num_key_frames * 1000.0f / num_total_frames + 0.5f); - RTC_HISTOGRAM_COUNTS_1000("WebRTC.Video.KeyFramesReceivedInPermille", - key_frames_permille); - } - int qp = qp_counters_.vp8.Avg(kMinRequiredSamples); if (qp != -1) RTC_HISTOGRAM_COUNTS_200("WebRTC.Video.Decoded.Vp8.Qp", qp); @@ -150,12 +132,15 @@ void ReceiveStatisticsProxy::UpdateHistograms() { if (decode_ms != -1) RTC_HISTOGRAM_COUNTS_1000("WebRTC.Video.DecodeTimeInMs", decode_ms); - int jb_delay_ms = jitter_buffer_delay_counter_.Avg(kMinRequiredDecodeSamples); - if (jb_delay_ms != -1) { - RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.JitterBufferDelayInMs", - jb_delay_ms); + if (field_trial::FindFullName("WebRTC-NewVideoJitterBuffer") != + "Enabled") { + int jb_delay_ms = + jitter_buffer_delay_counter_.Avg(kMinRequiredDecodeSamples); + if (jb_delay_ms != -1) { + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.JitterBufferDelayInMs", + jb_delay_ms); + } } - int target_delay_ms = target_delay_counter_.Avg(kMinRequiredDecodeSamples); if (target_delay_ms != -1) { RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.TargetDelayInMs", target_delay_ms); @@ -312,25 +297,8 @@ void ReceiveStatisticsProxy::QualitySample() { } } -void ReceiveStatisticsProxy::UpdateFrameAndBitrate(int64_t now_ms) const { - int64_t old_frames_ms = now_ms - kRateStatisticsWindowSizeMs; - while (!frame_window_.empty() && - frame_window_.begin()->first < old_frames_ms) { - frame_window_accumulated_bytes_ -= frame_window_.begin()->second; - frame_window_.erase(frame_window_.begin()); - } - - size_t framerate = - (frame_window_.size() * 1000 + 500) / kRateStatisticsWindowSizeMs; - size_t bitrate_bps = - frame_window_accumulated_bytes_ * 8000 / kRateStatisticsWindowSizeMs; - stats_.network_frame_rate = static_cast(framerate); - stats_.total_bitrate_bps = static_cast(bitrate_bps); -} - VideoReceiveStream::Stats ReceiveStatisticsProxy::GetStats() const { rtc::CritScope lock(&crit_); - UpdateFrameAndBitrate(clock_->TimeInMilliseconds()); return stats_; } @@ -349,16 +317,18 @@ void ReceiveStatisticsProxy::OnIncomingRate(unsigned int framerate, rtc::CritScope lock(&crit_); if (stats_.rtp_stats.first_packet_time_ms != -1) QualitySample(); + stats_.network_frame_rate = framerate; + stats_.total_bitrate_bps = bitrate_bps; } -void ReceiveStatisticsProxy::OnFrameBufferTimingsUpdated( - int decode_ms, - int max_decode_ms, - int current_delay_ms, - int target_delay_ms, - int jitter_buffer_ms, - int min_playout_delay_ms, - int render_delay_ms) { +void ReceiveStatisticsProxy::OnDecoderTiming(int decode_ms, + int max_decode_ms, + int current_delay_ms, + int target_delay_ms, + int jitter_buffer_ms, + int min_playout_delay_ms, + int render_delay_ms, + int64_t rtt_ms) { rtc::CritScope lock(&crit_); stats_.decode_ms = decode_ms; stats_.max_decode_ms = max_decode_ms; @@ -373,7 +343,7 @@ void ReceiveStatisticsProxy::OnFrameBufferTimingsUpdated( current_delay_counter_.Add(current_delay_ms); // Network delay (rtt/2) + target_delay_ms (jitter delay + decode time + // render delay). - delay_counter_.Add(target_delay_ms + avg_rtt_ms_ / 2); + delay_counter_.Add(target_delay_ms + rtt_ms / 2); } void ReceiveStatisticsProxy::RtcpPacketTypesCounterUpdated( @@ -477,20 +447,6 @@ void ReceiveStatisticsProxy::OnReceiveRatesUpdated(uint32_t bitRate, uint32_t frameRate) { } -void ReceiveStatisticsProxy::OnCompleteFrame(bool is_keyframe, - size_t size_bytes) { - rtc::CritScope lock(&crit_); - if (is_keyframe) - ++stats_.frame_counts.key_frames; - else - ++stats_.frame_counts.delta_frames; - - int64_t now_ms = clock_->TimeInMilliseconds(); - frame_window_accumulated_bytes_ += size_bytes; - frame_window_.insert(std::make_pair(now_ms, size_bytes)); - UpdateFrameAndBitrate(now_ms); -} - void ReceiveStatisticsProxy::OnFrameCountsUpdated( const FrameCounts& frame_counts) { rtc::CritScope lock(&crit_); @@ -532,10 +488,4 @@ void ReceiveStatisticsProxy::SampleCounter::Reset() { sum = 0; } -void ReceiveStatisticsProxy::OnRttUpdate(int64_t avg_rtt_ms, - int64_t max_rtt_ms) { - rtc::CritScope lock(&crit_); - avg_rtt_ms_ = avg_rtt_ms; -} - } // namespace webrtc diff --git a/webrtc/video/receive_statistics_proxy.h b/webrtc/video/receive_statistics_proxy.h index 73a9b378a..e54f53aac 100644 --- a/webrtc/video/receive_statistics_proxy.h +++ b/webrtc/video/receive_statistics_proxy.h @@ -37,8 +37,7 @@ struct CodecSpecificInfo; class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, public RtcpStatisticsCallback, public RtcpPacketTypeCounterObserver, - public StreamDataCountersCallback, - public CallStatsObserver { + public StreamDataCountersCallback { public: ReceiveStatisticsProxy(const VideoReceiveStream::Config* config, Clock* clock); @@ -52,6 +51,14 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, void OnIncomingPayloadType(int payload_type); void OnDecoderImplementationName(const char* implementation_name); void OnIncomingRate(unsigned int framerate, unsigned int bitrate_bps); + void OnDecoderTiming(int decode_ms, + int max_decode_ms, + int current_delay_ms, + int target_delay_ms, + int jitter_buffer_ms, + int min_playout_delay_ms, + int render_delay_ms, + int64_t rtt_ms); void OnPreDecode(const EncodedImage& encoded_image, const CodecSpecificInfo* codec_specific_info); @@ -60,14 +67,6 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, void OnReceiveRatesUpdated(uint32_t bitRate, uint32_t frameRate) override; void OnFrameCountsUpdated(const FrameCounts& frame_counts) override; void OnDiscardedPacketsUpdated(int discarded_packets) override; - void OnCompleteFrame(bool is_keyframe, size_t size_bytes) override; - void OnFrameBufferTimingsUpdated(int decode_ms, - int max_decode_ms, - int current_delay_ms, - int target_delay_ms, - int jitter_buffer_ms, - int min_playout_delay_ms, - int render_delay_ms) override; // Overrides RtcpStatisticsCallback. void StatisticsUpdated(const webrtc::RtcpStatistics& statistics, @@ -82,9 +81,6 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, void DataCountersUpdated(const webrtc::StreamDataCounters& counters, uint32_t ssrc) override; - // Implements CallStatsObserver. - void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; - private: struct SampleCounter { SampleCounter() : sum(0), num_samples(0) {} @@ -104,10 +100,6 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, void QualitySample() EXCLUSIVE_LOCKS_REQUIRED(crit_); - // Removes info about old frames and then updates the framerate/bitrate. - void UpdateFrameAndBitrate(int64_t now_ms) const - EXCLUSIVE_LOCKS_REQUIRED(crit_); - Clock* const clock_; // Ownership of this object lies with the owner of the ReceiveStatisticsProxy // instance. Lifetime is guaranteed to outlive |this|. @@ -127,7 +119,7 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, SampleCounter qp_sample_ GUARDED_BY(crit_); int num_bad_states_ GUARDED_BY(crit_); int num_certain_states_ GUARDED_BY(crit_); - mutable VideoReceiveStream::Stats stats_ GUARDED_BY(crit_); + VideoReceiveStream::Stats stats_ GUARDED_BY(crit_); RateStatistics decode_fps_estimator_ GUARDED_BY(crit_); RateStatistics renders_fps_estimator_ GUARDED_BY(crit_); rtc::RateTracker render_fps_tracker_ GUARDED_BY(crit_); @@ -146,9 +138,6 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, ReportBlockStats report_block_stats_ GUARDED_BY(crit_); QpCounters qp_counters_; // Only accessed on the decoding thread. std::map rtx_stats_ GUARDED_BY(crit_); - int64_t avg_rtt_ms_ GUARDED_BY(crit_); - mutable std::map frame_window_ GUARDED_BY(&crit_); - mutable size_t frame_window_accumulated_bytes_ GUARDED_BY(&crit_); }; } // namespace webrtc diff --git a/webrtc/video/receive_statistics_proxy_unittest.cc b/webrtc/video/receive_statistics_proxy_unittest.cc index 3fd9bc68d..a485edae0 100644 --- a/webrtc/video/receive_statistics_proxy_unittest.cc +++ b/webrtc/video/receive_statistics_proxy_unittest.cc @@ -74,14 +74,12 @@ TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsDecoderImplementationName) { kName, statistics_proxy_->GetStats().decoder_implementation_name.c_str()); } -TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsOnCompleteFrame) { - const int kFrameSizeBytes = 1000; - statistics_proxy_->OnCompleteFrame(true, kFrameSizeBytes); - VideoReceiveStream::Stats stats = statistics_proxy_->GetStats(); - EXPECT_EQ(1, stats.network_frame_rate); - EXPECT_EQ(kFrameSizeBytes * 8, stats.total_bitrate_bps); - EXPECT_EQ(1, stats.frame_counts.key_frames); - EXPECT_EQ(0, stats.frame_counts.delta_frames); +TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsIncomingRate) { + const int kFramerate = 28; + const int kBitrateBps = 311000; + statistics_proxy_->OnIncomingRate(kFramerate, kBitrateBps); + EXPECT_EQ(kFramerate, statistics_proxy_->GetStats().network_frame_rate); + EXPECT_EQ(kBitrateBps, statistics_proxy_->GetStats().total_bitrate_bps); } TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsDecodeTimingStats) { @@ -93,10 +91,9 @@ TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsDecodeTimingStats) { const int kMinPlayoutDelayMs = 6; const int kRenderDelayMs = 7; const int64_t kRttMs = 8; - statistics_proxy_->OnRttUpdate(kRttMs, 0); - statistics_proxy_->OnFrameBufferTimingsUpdated( + statistics_proxy_->OnDecoderTiming( kDecodeMs, kMaxDecodeMs, kCurrentDelayMs, kTargetDelayMs, kJitterBufferMs, - kMinPlayoutDelayMs, kRenderDelayMs); + kMinPlayoutDelayMs, kRenderDelayMs, kRttMs); VideoReceiveStream::Stats stats = statistics_proxy_->GetStats(); EXPECT_EQ(kDecodeMs, stats.decode_ms); EXPECT_EQ(kMaxDecodeMs, stats.max_decode_ms); diff --git a/webrtc/video/rtp_stream_receiver.cc b/webrtc/video/rtp_stream_receiver.cc index e75169ab8..d23608587 100644 --- a/webrtc/video/rtp_stream_receiver.cc +++ b/webrtc/video/rtp_stream_receiver.cc @@ -199,21 +199,25 @@ RtpStreamReceiver::RtpStreamReceiver( process_thread_->RegisterModule(rtp_rtcp_.get()); - nack_module_.reset( - new NackModule(clock_, nack_sender, keyframe_request_sender)); - if (config_.rtp.nack.rtp_history_ms == 0) - nack_module_->Stop(); - process_thread_->RegisterModule(nack_module_.get()); + jitter_buffer_experiment_ = + field_trial::FindFullName("WebRTC-NewVideoJitterBuffer") == "Enabled"; - packet_buffer_ = video_coding::PacketBuffer::Create( - clock_, kPacketBufferStartSize, kPacketBufferMaxSixe, this); - reference_finder_.reset(new video_coding::RtpFrameReferenceFinder(this)); + if (jitter_buffer_experiment_) { + nack_module_.reset( + new NackModule(clock_, nack_sender, keyframe_request_sender)); + process_thread_->RegisterModule(nack_module_.get()); + + packet_buffer_ = video_coding::PacketBuffer::Create( + clock_, kPacketBufferStartSize, kPacketBufferMaxSixe, this); + reference_finder_.reset(new video_coding::RtpFrameReferenceFinder(this)); + } } RtpStreamReceiver::~RtpStreamReceiver() { process_thread_->DeRegisterModule(rtp_rtcp_.get()); - process_thread_->DeRegisterModule(nack_module_.get()); + if (jitter_buffer_experiment_) + process_thread_->DeRegisterModule(nack_module_.get()); packet_router_->RemoveRtpModule(rtp_rtcp_.get()); rtp_rtcp_->SetREMBStatus(false); @@ -257,35 +261,43 @@ int32_t RtpStreamReceiver::OnReceivedPayloadData( WebRtcRTPHeader rtp_header_with_ntp = *rtp_header; rtp_header_with_ntp.ntp_time_ms = ntp_estimator_.Estimate(rtp_header->header.timestamp); - VCMPacket packet(payload_data, payload_size, rtp_header_with_ntp); - timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds()); - packet.timesNacked = nack_module_->OnReceivedPacket(packet); - - if (packet.codec == kVideoCodecH264) { - // Only when we start to receive packets will we know what payload type - // that will be used. When we know the payload type insert the correct - // sps/pps into the tracker. - if (packet.payloadType != last_payload_type_) { - last_payload_type_ = packet.payloadType; - InsertSpsPpsIntoTracker(packet.payloadType); + if (jitter_buffer_experiment_) { + VCMPacket packet(payload_data, payload_size, rtp_header_with_ntp); + timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds()); + packet.timesNacked = nack_module_->OnReceivedPacket(packet); + + if (packet.codec == kVideoCodecH264) { + // Only when we start to receive packets will we know what payload type + // that will be used. When we know the payload type insert the correct + // sps/pps into the tracker. + if (packet.payloadType != last_payload_type_) { + last_payload_type_ = packet.payloadType; + InsertSpsPpsIntoTracker(packet.payloadType); + } + + switch (tracker_.CopyAndFixBitstream(&packet)) { + case video_coding::H264SpsPpsTracker::kRequestKeyframe: + keyframe_request_sender_->RequestKeyFrame(); + FALLTHROUGH(); + case video_coding::H264SpsPpsTracker::kDrop: + return 0; + case video_coding::H264SpsPpsTracker::kInsert: + break; + } + } else { + uint8_t* data = new uint8_t[packet.sizeBytes]; + memcpy(data, packet.dataPtr, packet.sizeBytes); + packet.dataPtr = data; } - switch (tracker_.CopyAndFixBitstream(&packet)) { - case video_coding::H264SpsPpsTracker::kRequestKeyframe: - keyframe_request_sender_->RequestKeyFrame(); - FALLTHROUGH(); - case video_coding::H264SpsPpsTracker::kDrop: - return 0; - case video_coding::H264SpsPpsTracker::kInsert: - break; - } + packet_buffer_->InsertPacket(&packet); } else { - uint8_t* data = new uint8_t[packet.sizeBytes]; - memcpy(data, packet.dataPtr, packet.sizeBytes); - packet.dataPtr = data; + if (video_receiver_->IncomingPacket(payload_data, payload_size, + rtp_header_with_ntp) != 0) { + // Check this... + return -1; + } } - - packet_buffer_->InsertPacket(&packet); return 0; } @@ -422,7 +434,8 @@ void RtpStreamReceiver::OnCompleteFrame( } void RtpStreamReceiver::OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) { - nack_module_->UpdateRtt(max_rtt_ms); + if (jitter_buffer_experiment_) + nack_module_->UpdateRtt(max_rtt_ms); } bool RtpStreamReceiver::ReceivePacket(const uint8_t* packet, @@ -550,31 +563,35 @@ bool RtpStreamReceiver::DeliverRtcp(const uint8_t* rtcp_packet, } void RtpStreamReceiver::FrameContinuous(uint16_t picture_id) { - int seq_num = -1; - { - rtc::CritScope lock(&last_seq_num_cs_); - auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id); - if (seq_num_it != last_seq_num_for_pic_id_.end()) - seq_num = seq_num_it->second; + if (jitter_buffer_experiment_) { + int seq_num = -1; + { + rtc::CritScope lock(&last_seq_num_cs_); + auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id); + if (seq_num_it != last_seq_num_for_pic_id_.end()) + seq_num = seq_num_it->second; + } + if (seq_num != -1) + nack_module_->ClearUpTo(seq_num); } - if (seq_num != -1) - nack_module_->ClearUpTo(seq_num); } void RtpStreamReceiver::FrameDecoded(uint16_t picture_id) { - int seq_num = -1; - { - rtc::CritScope lock(&last_seq_num_cs_); - auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id); - if (seq_num_it != last_seq_num_for_pic_id_.end()) { - seq_num = seq_num_it->second; - last_seq_num_for_pic_id_.erase(last_seq_num_for_pic_id_.begin(), - ++seq_num_it); + if (jitter_buffer_experiment_) { + int seq_num = -1; + { + rtc::CritScope lock(&last_seq_num_cs_); + auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id); + if (seq_num_it != last_seq_num_for_pic_id_.end()) { + seq_num = seq_num_it->second; + last_seq_num_for_pic_id_.erase(last_seq_num_for_pic_id_.begin(), + ++seq_num_it); + } + } + if (seq_num != -1) { + packet_buffer_->ClearTo(seq_num); + reference_finder_->ClearTo(seq_num); } - } - if (seq_num != -1) { - packet_buffer_->ClearTo(seq_num); - reference_finder_->ClearTo(seq_num); } } diff --git a/webrtc/video/rtp_stream_receiver.h b/webrtc/video/rtp_stream_receiver.h index 196e02da6..b1e1db4b1 100644 --- a/webrtc/video/rtp_stream_receiver.h +++ b/webrtc/video/rtp_stream_receiver.h @@ -189,6 +189,7 @@ class RtpStreamReceiver : public RtpData, const std::unique_ptr rtp_rtcp_; // Members for the new jitter buffer experiment. + bool jitter_buffer_experiment_; video_coding::OnCompleteFrameCallback* complete_frame_callback_; KeyFrameRequestSender* keyframe_request_sender_; VCMTiming* timing_; diff --git a/webrtc/video/video_receive_stream.cc b/webrtc/video/video_receive_stream.cc index 7835e779b..183f72b53 100644 --- a/webrtc/video/video_receive_stream.cc +++ b/webrtc/video/video_receive_stream.cc @@ -223,7 +223,10 @@ VideoReceiveStream::VideoReceiveStream( this, // KeyFrameRequestSender this, // OnCompleteFrameCallback timing_.get()), - rtp_stream_sync_(&video_receiver_, &rtp_stream_receiver_) { + rtp_stream_sync_(&video_receiver_, &rtp_stream_receiver_), + jitter_buffer_experiment_( + field_trial::FindFullName("WebRTC-NewVideoJitterBuffer") == + "Enabled") { LOG(LS_INFO) << "VideoReceiveStream: " << config_.ToString(); RTC_DCHECK(process_thread_); @@ -243,9 +246,11 @@ VideoReceiveStream::VideoReceiveStream( video_receiver_.SetRenderDelay(config.render_delay_ms); - jitter_estimator_.reset(new VCMJitterEstimator(clock_)); - frame_buffer_.reset(new video_coding::FrameBuffer( - clock_, jitter_estimator_.get(), timing_.get(), &stats_proxy_)); + if (jitter_buffer_experiment_) { + jitter_estimator_.reset(new VCMJitterEstimator(clock_)); + frame_buffer_.reset(new video_coding::FrameBuffer( + clock_, jitter_estimator_.get(), timing_.get())); + } process_thread_->RegisterModule(&video_receiver_); process_thread_->RegisterModule(&rtp_stream_sync_); @@ -285,15 +290,15 @@ bool VideoReceiveStream::OnRecoveredPacket(const uint8_t* packet, void VideoReceiveStream::Start() { if (decode_thread_.IsRunning()) return; + if (jitter_buffer_experiment_) { + frame_buffer_->Start(); + call_stats_->RegisterStatsObserver(&rtp_stream_receiver_); - frame_buffer_->Start(); - call_stats_->RegisterStatsObserver(&rtp_stream_receiver_); - - if (rtp_stream_receiver_.IsRetransmissionsEnabled() && - rtp_stream_receiver_.IsUlpfecEnabled()) { - frame_buffer_->SetProtectionMode(kProtectionNackFEC); + if (rtp_stream_receiver_.IsRetransmissionsEnabled() && + rtp_stream_receiver_.IsUlpfecEnabled()) { + frame_buffer_->SetProtectionMode(kProtectionNackFEC); + } } - transport_adapter_.Enable(); rtc::VideoSinkInterface* renderer = nullptr; if (config_.renderer) { @@ -338,8 +343,10 @@ void VideoReceiveStream::Stop() { // before joining the decoder thread thread. video_receiver_.TriggerDecoderShutdown(); - frame_buffer_->Stop(); - call_stats_->DeregisterStatsObserver(&rtp_stream_receiver_); + if (jitter_buffer_experiment_) { + frame_buffer_->Stop(); + call_stats_->DeregisterStatsObserver(&rtp_stream_receiver_); + } if (decode_thread_.IsRunning()) { decode_thread_.Stop(); @@ -435,21 +442,26 @@ bool VideoReceiveStream::DecodeThreadFunction(void* ptr) { } void VideoReceiveStream::Decode() { - static const int kMaxWaitForFrameMs = 3000; - std::unique_ptr frame; - video_coding::FrameBuffer::ReturnReason res = - frame_buffer_->NextFrame(kMaxWaitForFrameMs, &frame); - - if (res == video_coding::FrameBuffer::ReturnReason::kStopped) - return; - - if (frame) { - if (video_receiver_.Decode(frame.get()) == VCM_OK) - rtp_stream_receiver_.FrameDecoded(frame->picture_id); + static const int kMaxDecodeWaitTimeMs = 50; + if (jitter_buffer_experiment_) { + static const int kMaxWaitForFrameMs = 3000; + std::unique_ptr frame; + video_coding::FrameBuffer::ReturnReason res = + frame_buffer_->NextFrame(kMaxWaitForFrameMs, &frame); + + if (res == video_coding::FrameBuffer::ReturnReason::kStopped) + return; + + if (frame) { + if (video_receiver_.Decode(frame.get()) == VCM_OK) + rtp_stream_receiver_.FrameDecoded(frame->picture_id); + } else { + LOG(LS_WARNING) << "No decodable frame in " << kMaxWaitForFrameMs + << " ms, requesting keyframe."; + RequestKeyFrame(); + } } else { - LOG(LS_WARNING) << "No decodable frame in " << kMaxWaitForFrameMs - << " ms, requesting keyframe."; - RequestKeyFrame(); + video_receiver_.Decode(kMaxDecodeWaitTimeMs); } } diff --git a/webrtc/video/video_receive_stream.h b/webrtc/video/video_receive_stream.h index 3c5a653c5..ab9fe4ea7 100644 --- a/webrtc/video/video_receive_stream.h +++ b/webrtc/video/video_receive_stream.h @@ -130,6 +130,7 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, std::unique_ptr ivf_writer_ GUARDED_BY(ivf_writer_lock_); // Members for the new jitter buffer experiment. + const bool jitter_buffer_experiment_; std::unique_ptr jitter_estimator_; std::unique_ptr frame_buffer_; }; diff --git a/webrtc/video/video_stream_decoder.cc b/webrtc/video/video_stream_decoder.cc index 34469ed30..7a00e4209 100644 --- a/webrtc/video/video_stream_decoder.cc +++ b/webrtc/video/video_stream_decoder.cc @@ -122,17 +122,17 @@ void VideoStreamDecoder::OnDecoderTiming(int decode_ms, int target_delay_ms, int jitter_buffer_ms, int min_playout_delay_ms, - int render_delay_ms) {} - -void VideoStreamDecoder::OnFrameBufferTimingsUpdated(int decode_ms, - int max_decode_ms, - int current_delay_ms, - int target_delay_ms, - int jitter_buffer_ms, - int min_playout_delay_ms, - int render_delay_ms) {} + int render_delay_ms) { + int last_rtt = -1; + { + rtc::CritScope lock(&crit_); + last_rtt = last_rtt_ms_; + } -void VideoStreamDecoder::OnCompleteFrame(bool is_keyframe, size_t size_bytes) {} + receive_stats_callback_->OnDecoderTiming( + decode_ms, max_decode_ms, current_delay_ms, target_delay_ms, + jitter_buffer_ms, min_playout_delay_ms, render_delay_ms, last_rtt); +} void VideoStreamDecoder::OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) { video_receiver_->SetReceiveChannelParameters(max_rtt_ms); diff --git a/webrtc/video/video_stream_decoder.h b/webrtc/video/video_stream_decoder.h index 14ad071f4..ad355570b 100644 --- a/webrtc/video/video_stream_decoder.h +++ b/webrtc/video/video_stream_decoder.h @@ -69,14 +69,6 @@ class VideoStreamDecoder : public VCMReceiveCallback, void OnReceiveRatesUpdated(uint32_t bit_rate, uint32_t frame_rate) override; void OnDiscardedPacketsUpdated(int discarded_packets) override; void OnFrameCountsUpdated(const FrameCounts& frame_counts) override; - void OnCompleteFrame(bool is_keyframe, size_t size_bytes) override; - void OnFrameBufferTimingsUpdated(int decode_ms, - int max_decode_ms, - int current_delay_ms, - int target_delay_ms, - int jitter_buffer_ms, - int min_playout_delay_ms, - int render_delay_ms) override; // Implements VCMDecoderTimingCallback. void OnDecoderTiming(int decode_ms,