From c622c51234ae7fb0a0c6c45c74a8a8ee073482d5 Mon Sep 17 00:00:00 2001 From: Sergey Ulanov Date: Mon, 13 Mar 2017 11:50:13 -0700 Subject: [PATCH] BlankDetectorDesktopCapturerWrapper to detect a blank DesktopFrame DXGI capturer highly depends on video adapter and its driver, as well as Windows itself. I recently found it cannot work on my virtualbox instance any more, which indicates it may not work well on some specific systems. What worse is, the APIs do not return a failure in such case. So this change adds a BlankDetectorDesktopCapturerWrapper, which samples several pixels in the frame returned by a DesktopCapturer implementation. If all the pixels selected are blank, this wrapper returns a failure. A typical usage is to combine BlankDetectorDesktopCapturerWrapper with FallbackDesktopCapturerWrapper, and use GDI capturer in case of failure. Usually less than 10000 pixels are checked, so the BlankDetectorDesktopCapturerWrapper should not significant impact the capturer performance. This change is expected to resolve bug 682112 in another dimension. BUG=chromium:682112 Review-Url: https://codereview.webrtc.org/2709523003 Cr-Original-Commit-Position: refs/heads/master@{#16984} Committed: https://chromium.googlesource.com/external/webrtc/+/c4e9d210b3516c7b2faa32f24409a2e626599255 Review-Url: https://codereview.webrtc.org/2709523003 Cr-Commit-Position: refs/heads/master@{#17024} (cherry picked from commit ccf57a71eb2a550744367a4cbec285d85494c3c2) Review-Url: https://codereview.webrtc.org/2748813002 . Cr-Commit-Position: refs/branch-heads/58@{#2} Cr-Branched-From: f31969a584bcafe9406c214a9d4c3afb49d19650-refs/heads/master@{#16937} --- webrtc/modules/desktop_capture/BUILD.gn | 31 +--- ...blank_detector_desktop_capturer_wrapper.cc | 115 ++++++++++++ .../blank_detector_desktop_capturer_wrapper.h | 74 ++++++++ ...ector_desktop_capturer_wrapper_unittest.cc | 163 ++++++++++++++++++ webrtc/modules/desktop_capture/rgba_color.h | 3 - .../desktop_capture/screen_capturer_win.cc | 22 ++- 6 files changed, 378 insertions(+), 30 deletions(-) create mode 100644 webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc create mode 100644 webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h create mode 100644 webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper_unittest.cc diff --git a/webrtc/modules/desktop_capture/BUILD.gn b/webrtc/modules/desktop_capture/BUILD.gn index 151627de3..f8d529340 100644 --- a/webrtc/modules/desktop_capture/BUILD.gn +++ b/webrtc/modules/desktop_capture/BUILD.gn @@ -44,7 +44,7 @@ if (rtc_include_tests) { ":screen_drawer", "../../base:rtc_base", "../../base:rtc_base_approved", - "../../system_wrappers:system_wrappers", + "../../system_wrappers", "../../test:test_support", "../../test:video_test_support", ] @@ -58,6 +58,7 @@ if (rtc_include_tests) { rtc_source_set("desktop_capture_unittests") { testonly = true sources = [ + "blank_detector_desktop_capturer_wrapper_unittest.cc", "desktop_and_cursor_composer_unittest.cc", "desktop_capturer_differ_wrapper_unittest.cc", "desktop_frame_rotation_unittest.cc", @@ -77,10 +78,9 @@ if (rtc_include_tests) { ":desktop_capture", ":desktop_capture_mock", ":primitives", - ":rgba_color", "../..:webrtc_common", "../../base:rtc_base_approved", - "../../system_wrappers:system_wrappers", + "../../system_wrappers", "../../test:test_support", "//testing/gmock", ] @@ -99,29 +99,11 @@ if (rtc_include_tests) { } } - source_set("rgba_color") { - testonly = true - - public_deps = [ - ":desktop_capture", - ] - - sources = [ - "rgba_color.cc", - "rgba_color.h", - ] - - deps = [ - ":primitives", - "../..:webrtc_common", - ] - } - source_set("screen_drawer") { testonly = true public_deps = [ - ":rgba_color", + ":desktop_capture", ] sources = [ @@ -144,7 +126,6 @@ if (rtc_include_tests) { public_deps = [ ":desktop_capture", - ":rgba_color", "//testing/gmock", ] @@ -167,6 +148,8 @@ if (rtc_include_tests) { rtc_static_library("desktop_capture") { sources = [ + "blank_detector_desktop_capturer_wrapper.cc", + "blank_detector_desktop_capturer_wrapper.h", "cropped_desktop_frame.cc", "cropped_desktop_frame.h", "cropping_window_capturer.cc", @@ -205,6 +188,8 @@ rtc_static_library("desktop_capture") { "mouse_cursor_monitor_win.cc", "resolution_change_detector.cc", "resolution_change_detector.h", + "rgba_color.cc", + "rgba_color.h", "screen_capture_frame_queue.h", "screen_capturer_helper.cc", "screen_capturer_helper.h", diff --git a/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc b/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc new file mode 100644 index 000000000..d909db4f2 --- /dev/null +++ b/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.cc @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h" + +#include +#include + +#include "webrtc/base/checks.h" +#include "webrtc/modules/desktop_capture/desktop_geometry.h" + +namespace webrtc { + +BlankDetectorDesktopCapturerWrapper::BlankDetectorDesktopCapturerWrapper( + std::unique_ptr capturer, + RgbaColor blank_pixel) + : capturer_(std::move(capturer)), + blank_pixel_(blank_pixel) { + RTC_DCHECK(capturer_); +} + +BlankDetectorDesktopCapturerWrapper::~BlankDetectorDesktopCapturerWrapper() = + default; + +void BlankDetectorDesktopCapturerWrapper::Start( + DesktopCapturer::Callback* callback) { + capturer_->Start(this); + callback_ = callback; +} + +void BlankDetectorDesktopCapturerWrapper::SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory) { + capturer_->SetSharedMemoryFactory(std::move(shared_memory_factory)); +} + +void BlankDetectorDesktopCapturerWrapper::CaptureFrame() { + RTC_DCHECK(callback_); + capturer_->CaptureFrame(); +} + +void BlankDetectorDesktopCapturerWrapper::SetExcludedWindow(WindowId window) { + capturer_->SetExcludedWindow(window); +} + +bool BlankDetectorDesktopCapturerWrapper::GetSourceList(SourceList* sources) { + return capturer_->GetSourceList(sources); +} + +bool BlankDetectorDesktopCapturerWrapper::SelectSource(SourceId id) { + return capturer_->SelectSource(id); +} + +bool BlankDetectorDesktopCapturerWrapper::FocusOnSelectedSource() { + return capturer_->FocusOnSelectedSource(); +} + +void BlankDetectorDesktopCapturerWrapper::OnCaptureResult( + Result result, + std::unique_ptr frame) { + RTC_DCHECK(callback_); + if (result != Result::SUCCESS || non_blank_frame_received_) { + callback_->OnCaptureResult(result, std::move(frame)); + return; + } + + RTC_DCHECK(frame); + + // If nothing has been changed in current frame, we do not need to check it + // again. + if (!frame->updated_region().is_empty() || is_first_frame_) { + last_frame_is_blank_ = IsBlankFrame(*frame); + is_first_frame_ = false; + } + if (!last_frame_is_blank_) { + non_blank_frame_received_ = true; + callback_->OnCaptureResult(Result::SUCCESS, std::move(frame)); + return; + } + + callback_->OnCaptureResult(Result::ERROR_TEMPORARY, + std::unique_ptr()); +} + +bool BlankDetectorDesktopCapturerWrapper::IsBlankFrame( + const DesktopFrame& frame) const { + // We will check 7489 pixels for a frame with 1024 x 768 resolution. + for (int i = 0; i < frame.size().width() * frame.size().height(); i += 105) { + const int x = i % frame.size().width(); + const int y = i / frame.size().width(); + if (!IsBlankPixel(frame, x, y)) { + return false; + } + } + + // We are verifying the pixel in the center as well. + return IsBlankPixel(frame, frame.size().width() / 2, + frame.size().height() / 2); +} + +bool BlankDetectorDesktopCapturerWrapper::IsBlankPixel( + const DesktopFrame& frame, + int x, + int y) const { + uint8_t* pixel_data = frame.GetFrameDataAtPos(DesktopVector(x, y)); + return RgbaColor(pixel_data) == blank_pixel_; +} + +} // namespace webrtc diff --git a/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h b/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h new file mode 100644 index 000000000..0501cae14 --- /dev/null +++ b/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_DESKTOP_CAPTURE_BLANK_DETECTOR_DESKTOP_CAPTURER_WRAPPER_H_ +#define WEBRTC_MODULES_DESKTOP_CAPTURE_BLANK_DETECTOR_DESKTOP_CAPTURER_WRAPPER_H_ + +#include + +#include "webrtc/modules/desktop_capture/desktop_capturer.h" +#include "webrtc/modules/desktop_capture/rgba_color.h" + +namespace webrtc { + +// A DesktopCapturer wrapper detects the return value of its owned +// DesktopCapturer implementation. If sampled pixels returned by the +// DesktopCapturer implementation all equal to the blank pixel, this wrapper +// returns ERROR_TEMPORARY. If the DesktopCapturer implementation fails for too +// many times, this wrapper returns ERROR_PERMANENT. +class BlankDetectorDesktopCapturerWrapper final + : public DesktopCapturer, + public DesktopCapturer::Callback { + public: + // Creates BlankDetectorDesktopCapturerWrapper. BlankDesktopCapturerWrapper + // takes ownership of |capturer|. The |blank_pixel| is the unmodified color + // returned by the |capturer|. + BlankDetectorDesktopCapturerWrapper(std::unique_ptr capturer, + RgbaColor blank_pixel); + ~BlankDetectorDesktopCapturerWrapper() override; + + // DesktopCapturer interface. + void Start(DesktopCapturer::Callback* callback) override; + void SetSharedMemoryFactory( + std::unique_ptr shared_memory_factory) override; + void CaptureFrame() override; + void SetExcludedWindow(WindowId window) override; + bool GetSourceList(SourceList* sources) override; + bool SelectSource(SourceId id) override; + bool FocusOnSelectedSource() override; + + private: + // DesktopCapturer::Callback interface. + void OnCaptureResult(Result result, + std::unique_ptr frame) override; + + bool IsBlankFrame(const DesktopFrame& frame) const; + + // Detects whether pixel at (x, y) equals to |blank_pixel_|. + bool IsBlankPixel(const DesktopFrame& frame, int x, int y) const; + + const std::unique_ptr capturer_; + const RgbaColor blank_pixel_; + + // Whether a non-blank frame has been received. + bool non_blank_frame_received_ = false; + + // Whether the last frame is blank. + bool last_frame_is_blank_ = false; + + // Whether current frame is the first frame. + bool is_first_frame_ = true; + + DesktopCapturer::Callback* callback_ = nullptr; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_DESKTOP_CAPTURE_BLANK_DETECTOR_DESKTOP_CAPTURER_WRAPPER_H_ diff --git a/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper_unittest.cc b/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper_unittest.cc new file mode 100644 index 000000000..bce82ddc2 --- /dev/null +++ b/webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper_unittest.cc @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h" + +#include +#include + +#include "webrtc/modules/desktop_capture/desktop_capturer.h" +#include "webrtc/modules/desktop_capture/desktop_frame.h" +#include "webrtc/modules/desktop_capture/desktop_frame_generator.h" +#include "webrtc/modules/desktop_capture/fake_desktop_capturer.h" +#include "webrtc/test/gtest.h" + +namespace webrtc { + +class BlankDetectorDesktopCapturerWrapperTest + : public testing::Test, + public DesktopCapturer::Callback { + public: + BlankDetectorDesktopCapturerWrapperTest(); + ~BlankDetectorDesktopCapturerWrapperTest() override; + + protected: + void PerfTest(DesktopCapturer* capturer); + + const int frame_width_ = 1024; + const int frame_height_ = 768; + std::unique_ptr wrapper_; + DesktopCapturer* capturer_ = nullptr; + BlackWhiteDesktopFramePainter painter_; + int num_frames_captured_ = 0; + DesktopCapturer::Result last_result_ = DesktopCapturer::Result::SUCCESS; + std::unique_ptr last_frame_; + + private: + // DesktopCapturer::Callback interface. + void OnCaptureResult(DesktopCapturer::Result result, + std::unique_ptr frame) override; + + PainterDesktopFrameGenerator frame_generator_; +}; + +BlankDetectorDesktopCapturerWrapperTest:: +BlankDetectorDesktopCapturerWrapperTest() { + frame_generator_.size()->set(frame_width_, frame_height_); + frame_generator_.set_desktop_frame_painter(&painter_); + std::unique_ptr capturer(new FakeDesktopCapturer()); + FakeDesktopCapturer* fake_capturer = + static_cast(capturer.get()); + fake_capturer->set_frame_generator(&frame_generator_); + capturer_ = fake_capturer; + wrapper_.reset(new BlankDetectorDesktopCapturerWrapper( + std::move(capturer), RgbaColor(0, 0, 0, 0))); + wrapper_->Start(this); +} + +BlankDetectorDesktopCapturerWrapperTest:: +~BlankDetectorDesktopCapturerWrapperTest() = default; + +void BlankDetectorDesktopCapturerWrapperTest::OnCaptureResult( + DesktopCapturer::Result result, + std::unique_ptr frame) { + last_result_ = result; + last_frame_ = std::move(frame); + num_frames_captured_++; +} + +void BlankDetectorDesktopCapturerWrapperTest::PerfTest( + DesktopCapturer* capturer) { + for (int i = 0; i < 10000; i++) { + capturer->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, i + 1); + } +} + +TEST_F(BlankDetectorDesktopCapturerWrapperTest, ShouldDetectBlankFrame) { + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 1); + ASSERT_EQ(last_result_, DesktopCapturer::Result::ERROR_TEMPORARY); + ASSERT_FALSE(last_frame_); +} + +TEST_F(BlankDetectorDesktopCapturerWrapperTest, ShouldPassBlankDetection) { + painter_.updated_region()->AddRect(DesktopRect::MakeXYWH(0, 0, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 1); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + painter_.updated_region()->AddRect( + DesktopRect::MakeXYWH(frame_width_ - 100, frame_height_ - 100, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 2); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + painter_.updated_region()->AddRect( + DesktopRect::MakeXYWH(0, frame_height_ - 100, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 3); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + painter_.updated_region()->AddRect( + DesktopRect::MakeXYWH(frame_width_ - 100, 0, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 4); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + painter_.updated_region()->AddRect(DesktopRect::MakeXYWH( + (frame_width_ >> 1) - 50, (frame_height_ >> 1) - 50, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 5); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); +} + +TEST_F(BlankDetectorDesktopCapturerWrapperTest, + ShouldNotCheckAfterANonBlankFrameReceived) { + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 1); + ASSERT_EQ(last_result_, DesktopCapturer::Result::ERROR_TEMPORARY); + ASSERT_FALSE(last_frame_); + + painter_.updated_region()->AddRect( + DesktopRect::MakeXYWH(frame_width_ - 100, 0, 100, 100)); + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, 2); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + + for (int i = 0; i < 100; i++) { + wrapper_->CaptureFrame(); + ASSERT_EQ(num_frames_captured_, i + 3); + ASSERT_EQ(last_result_, DesktopCapturer::Result::SUCCESS); + ASSERT_TRUE(last_frame_); + } +} + +// There is no perceptible impact by using BlankDetectorDesktopCapturerWrapper. +// i.e. less than 0.2ms per frame. +// [ OK ] DISABLED_Performance (10210 ms) +// [ OK ] DISABLED_PerformanceComparison (8791 ms) +TEST_F(BlankDetectorDesktopCapturerWrapperTest, DISABLED_Performance) { + PerfTest(wrapper_.get()); +} + +TEST_F(BlankDetectorDesktopCapturerWrapperTest, + DISABLED_PerformanceComparison) { + capturer_->Start(this); + PerfTest(capturer_); +} + +} // namespace webrtc diff --git a/webrtc/modules/desktop_capture/rgba_color.h b/webrtc/modules/desktop_capture/rgba_color.h index 1236d03ee..11e8d44f5 100644 --- a/webrtc/modules/desktop_capture/rgba_color.h +++ b/webrtc/modules/desktop_capture/rgba_color.h @@ -21,9 +21,6 @@ namespace webrtc { // provides functions to be created from uint8_t array, say, // DesktopFrame::data(). It always uses BGRA order for internal storage to match // DesktopFrame::data(). -// -// This struct is for testing purpose only, and should not be used in production -// logic. struct RgbaColor final { // Creates a color with BGRA channels. RgbaColor(uint8_t blue, uint8_t green, uint8_t red, uint8_t alpha); diff --git a/webrtc/modules/desktop_capture/screen_capturer_win.cc b/webrtc/modules/desktop_capture/screen_capturer_win.cc index 602600be8..d8aecb15c 100644 --- a/webrtc/modules/desktop_capture/screen_capturer_win.cc +++ b/webrtc/modules/desktop_capture/screen_capturer_win.cc @@ -11,24 +11,38 @@ #include #include +#include "webrtc/modules/desktop_capture/blank_detector_desktop_capturer_wrapper.h" #include "webrtc/modules/desktop_capture/desktop_capturer.h" #include "webrtc/modules/desktop_capture/desktop_capture_options.h" #include "webrtc/modules/desktop_capture/fallback_desktop_capturer_wrapper.h" +#include "webrtc/modules/desktop_capture/rgba_color.h" #include "webrtc/modules/desktop_capture/win/screen_capturer_win_directx.h" #include "webrtc/modules/desktop_capture/win/screen_capturer_win_gdi.h" #include "webrtc/modules/desktop_capture/win/screen_capturer_win_magnifier.h" namespace webrtc { +namespace { + +std::unique_ptr CreateScreenCapturerWinDirectx( + const DesktopCaptureOptions& options) { + std::unique_ptr capturer( + new ScreenCapturerWinDirectx(options)); + capturer.reset(new BlankDetectorDesktopCapturerWrapper( + std::move(capturer), RgbaColor(0, 0, 0, 0))); + return capturer; +} + +} // namespace + // static std::unique_ptr DesktopCapturer::CreateRawScreenCapturer( const DesktopCaptureOptions& options) { - std::unique_ptr capturer; + std::unique_ptr capturer(new ScreenCapturerWinGdi(options)); if (options.allow_directx_capturer() && ScreenCapturerWinDirectx::IsSupported()) { - capturer.reset(new ScreenCapturerWinDirectx(options)); - } else { - capturer.reset(new ScreenCapturerWinGdi(options)); + capturer.reset(new FallbackDesktopCapturerWrapper( + CreateScreenCapturerWinDirectx(options), std::move(capturer))); } if (options.allow_use_magnification_api()) {