From 0442f7012b26064a1cc8ff3fa0171dbbb453543f Mon Sep 17 00:00:00 2001 From: 21pages Date: Mon, 27 May 2024 09:27:30 +0800 Subject: [PATCH 1/3] fix mac render memory, dispose old decoded image (#8140) Signed-off-by: 21pages --- flutter/lib/desktop/pages/remote_page.dart | 2 + flutter/lib/mobile/pages/remote_page.dart | 2 + flutter/lib/models/model.dart | 16 +++++ flutter/lib/utils/image.dart | 82 ++++++++++++++++------ 4 files changed, 80 insertions(+), 22 deletions(-) diff --git a/flutter/lib/desktop/pages/remote_page.dart b/flutter/lib/desktop/pages/remote_page.dart index fce0243f2f..d8a3c5b60c 100644 --- a/flutter/lib/desktop/pages/remote_page.dart +++ b/flutter/lib/desktop/pages/remote_page.dart @@ -237,6 +237,8 @@ class _RemotePageState extends State _ffi.inputModel.enterOrLeave(false); DesktopMultiWindow.removeListener(this); _ffi.dialogManager.hideMobileActionsOverlay(); + _ffi.imageModel.disposeImage(); + _ffi.cursorModel.disposeImages(); _ffi.recordingModel.onClose(); _rawKeyFocusNode.dispose(); await _ffi.close(closeSession: closeSession); diff --git a/flutter/lib/mobile/pages/remote_page.dart b/flutter/lib/mobile/pages/remote_page.dart index ce46f5d43e..a73f45fee5 100644 --- a/flutter/lib/mobile/pages/remote_page.dart +++ b/flutter/lib/mobile/pages/remote_page.dart @@ -90,6 +90,8 @@ class _RemotePageState extends State { super.dispose(); gFFI.dialogManager.hideMobileActionsOverlay(); gFFI.inputModel.listenToMouse(false); + gFFI.imageModel.disposeImage(); + gFFI.cursorModel.disposeImages(); await gFFI.invokeMethod("enable_soft_keyboard", true); _mobileFocusNode.dispose(); _physicalFocusNode.dispose(); diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart index d897a9ec28..a83c381ed9 100644 --- a/flutter/lib/models/model.dart +++ b/flutter/lib/models/model.dart @@ -1208,6 +1208,7 @@ class ImageModel with ChangeNotifier { parent.target?.canvasModel.updateViewStyle(); } } + _image?.dispose(); _image = image; if (image != null) notifyListeners(); } @@ -1231,6 +1232,11 @@ class ImageModel with ChangeNotifier { final yscale = size.height / _image!.height; return min(xscale, yscale) / 1.5; } + + void disposeImage() { + _image?.dispose(); + _image = null; + } } enum ScrollStyle { @@ -1702,6 +1708,7 @@ class PredefinedCursor { final defaultImg = _image2!; // This function is called only one time, no need to care about the performance. Uint8List data = defaultImg.getBytes(order: img2.ChannelOrder.rgba); + _image?.dispose(); _image = await img.decodeImageFromPixels( data, defaultImg.width, defaultImg.height, ui.PixelFormat.rgba8888); @@ -1937,6 +1944,11 @@ class CursorModel with ChangeNotifier { notifyListeners(); } + disposeImages() { + _images.forEach((_, v) => v.item1.dispose()); + _images.clear(); + } + updateCursorData(Map evt) async { final id = int.parse(evt['id']); final hotx = double.parse(evt['hotx']); @@ -1947,7 +1959,11 @@ class CursorModel with ChangeNotifier { final rgba = Uint8List.fromList(colors.map((s) => s as int).toList()); final image = await img.decodeImageFromPixels( rgba, width, height, ui.PixelFormat.rgba8888); + if (image == null) { + return; + } if (await _updateCache(rgba, image, id, hotx, hoty, width, height)) { + _images[id]?.item1.dispose(); _images[id] = Tuple3(image, hotx, hoty); } diff --git a/flutter/lib/utils/image.dart b/flutter/lib/utils/image.dart index 2a208a6831..e622395013 100644 --- a/flutter/lib/utils/image.dart +++ b/flutter/lib/utils/image.dart @@ -5,7 +5,7 @@ import 'package:flutter/widgets.dart'; import 'package:flutter_hbb/common.dart'; -Future decodeImageFromPixels( +Future decodeImageFromPixels( Uint8List pixels, int width, int height, @@ -18,36 +18,74 @@ Future decodeImageFromPixels( }) async { if (targetWidth != null) { assert(allowUpscaling || targetWidth <= width); + if (!(allowUpscaling || targetWidth <= width)) { + print("not allow upscaling but targetWidth > width"); + return null; + } } if (targetHeight != null) { assert(allowUpscaling || targetHeight <= height); + if (!(allowUpscaling || targetHeight <= height)) { + print("not allow upscaling but targetHeight > height"); + return null; + } } - final ui.ImmutableBuffer buffer = - await ui.ImmutableBuffer.fromUint8List(pixels); - onPixelsCopied?.call(); - final ui.ImageDescriptor descriptor = ui.ImageDescriptor.raw( - buffer, - width: width, - height: height, - rowBytes: rowBytes, - pixelFormat: format, - ); - if (!allowUpscaling) { - if (targetWidth != null && targetWidth > descriptor.width) { - targetWidth = descriptor.width; - } - if (targetHeight != null && targetHeight > descriptor.height) { - targetHeight = descriptor.height; + final ui.ImmutableBuffer buffer; + try { + buffer = await ui.ImmutableBuffer.fromUint8List(pixels); + onPixelsCopied?.call(); + } catch (e) { + return null; + } + + final ui.ImageDescriptor descriptor; + try { + descriptor = ui.ImageDescriptor.raw( + buffer, + width: width, + height: height, + rowBytes: rowBytes, + pixelFormat: format, + ); + if (!allowUpscaling) { + if (targetWidth != null && targetWidth > descriptor.width) { + targetWidth = descriptor.width; + } + if (targetHeight != null && targetHeight > descriptor.height) { + targetHeight = descriptor.height; + } } + } catch (e) { + print("ImageDescriptor.raw failed: $e"); + buffer.dispose(); + return null; } - final ui.Codec codec = await descriptor.instantiateCodec( - targetWidth: targetWidth, - targetHeight: targetHeight, - ); + final ui.Codec codec; + try { + codec = await descriptor.instantiateCodec( + targetWidth: targetWidth, + targetHeight: targetHeight, + ); + } catch (e) { + print("instantiateCodec failed: $e"); + buffer.dispose(); + descriptor.dispose(); + return null; + } + + final ui.FrameInfo frameInfo; + try { + frameInfo = await codec.getNextFrame(); + } catch (e) { + print("getNextFrame failed: $e"); + codec.dispose(); + buffer.dispose(); + descriptor.dispose(); + return null; + } - final ui.FrameInfo frameInfo = await codec.getNextFrame(); codec.dispose(); buffer.dispose(); descriptor.dispose(); From 9ce62dc5843b772cdeec71adfdc15210b2bddcc1 Mon Sep 17 00:00:00 2001 From: fufesou <13586388+fufesou@users.noreply.github.com> Date: Mon, 27 May 2024 09:29:31 +0800 Subject: [PATCH 2/3] fix: window backgroud color (#8155) Signed-off-by: fufesou --- flutter/lib/consts.dart | 1 + flutter/lib/desktop/pages/remote_page.dart | 2 +- flutter/lib/desktop/widgets/tabbar_widget.dart | 7 ++++++- flutter/lib/mobile/pages/remote_page.dart | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/flutter/lib/consts.dart b/flutter/lib/consts.dart index f9caac94d4..be8ee3c17e 100644 --- a/flutter/lib/consts.dart +++ b/flutter/lib/consts.dart @@ -157,6 +157,7 @@ const String kKeyReverseMouseWheel = "reverse_mouse_wheel"; const String kEnvPortableExecutable = "RUSTDESK_APPNAME"; const Color kColorWarn = Color.fromARGB(255, 245, 133, 59); +const Color kColorCanvas = Colors.black; const int kMobileDefaultDisplayWidth = 720; const int kMobileDefaultDisplayHeight = 1280; diff --git a/flutter/lib/desktop/pages/remote_page.dart b/flutter/lib/desktop/pages/remote_page.dart index d8a3c5b60c..6fe256cf67 100644 --- a/flutter/lib/desktop/pages/remote_page.dart +++ b/flutter/lib/desktop/pages/remote_page.dart @@ -279,7 +279,7 @@ class _RemotePageState extends State return Stack( children: [ Container( - color: Colors.black, + color: kColorCanvas, child: RawKeyFocusScope( focusNode: _rawKeyFocusNode, onFocusChange: (bool imageFocused) { diff --git a/flutter/lib/desktop/widgets/tabbar_widget.dart b/flutter/lib/desktop/widgets/tabbar_widget.dart index 27bae1a295..0016f8fb35 100644 --- a/flutter/lib/desktop/widgets/tabbar_widget.dart +++ b/flutter/lib/desktop/widgets/tabbar_widget.dart @@ -334,7 +334,7 @@ class DesktopTab extends StatelessWidget { List _tabWidgets = []; Widget _buildPageView() { - return _buildBlock( + final child = _buildBlock( child: Obx(() => PageView( controller: state.value.pageController, physics: NeverScrollableScrollPhysics(), @@ -358,6 +358,11 @@ class DesktopTab extends StatelessWidget { return newList; } }()))); + if (tabType == DesktopTabType.remoteScreen) { + return Container(color: kColorCanvas, child: child); + } else { + return child; + } } /// Check whether to show ListView diff --git a/flutter/lib/mobile/pages/remote_page.dart b/flutter/lib/mobile/pages/remote_page.dart index a73f45fee5..ca7b909e80 100644 --- a/flutter/lib/mobile/pages/remote_page.dart +++ b/flutter/lib/mobile/pages/remote_page.dart @@ -312,7 +312,7 @@ class _RemotePageState extends State { initialEntries: [ OverlayEntry(builder: (context) { return Container( - color: Colors.black, + color: kColorCanvas, child: isWebDesktop ? getBodyForDesktopWithListener(keyboard) : SafeArea( From c7308dbbc9c8e882667b4de8b7bd3a852dfc3675 Mon Sep 17 00:00:00 2001 From: 21pages Date: Mon, 27 May 2024 19:34:40 +0800 Subject: [PATCH 3/3] fix mediacodec bad encoding quality (#8159) Signed-off-by: 21pages --- Cargo.lock | 4 ++-- libs/scrap/src/common/hwcodec.rs | 35 +++++++++++++++++++------------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f7f0be6a3..bb948aa79d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3037,8 +3037,8 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hwcodec" -version = "0.4.11" -source = "git+https://github.com/21pages/hwcodec#a5864080e41836b94feb9f73732280c651162fec" +version = "0.4.12" +source = "git+https://github.com/21pages/hwcodec#6fbafe7dd25adad5e897b3192b2b40b720a9b118" dependencies = [ "bindgen 0.59.2", "cc", diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index ddc20cb951..2c00182458 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -14,14 +14,16 @@ use hbb_common::{ serde_json, ResultType, }; use hwcodec::{ - common::DataFormat, + common::{ + DataFormat, + Quality::{self, *}, + RateControl::{self, *}, + }, ffmpeg::AVPixelFormat, ffmpeg_ram::{ decode::{DecodeContext, DecodeFrame, Decoder}, encode::{EncodeContext, EncodeFrame, Encoder}, CodecInfo, - Quality::{self, *}, - RateControl::{self, *}, }, }; @@ -29,10 +31,6 @@ const DEFAULT_PIXFMT: AVPixelFormat = AVPixelFormat::AV_PIX_FMT_NV12; pub const DEFAULT_TIME_BASE: [i32; 2] = [1, 30]; const DEFAULT_GOP: i32 = i32::MAX; const DEFAULT_HW_QUALITY: Quality = Quality_Default; -#[cfg(target_os = "android")] -const DEFAULT_RC: RateControl = RC_VBR; // android cbr poor quality -#[cfg(not(target_os = "android"))] -const DEFAULT_RC: RateControl = RC_CBR; #[derive(Debug, Clone)] pub struct HwRamEncoderConfig { @@ -59,6 +57,7 @@ impl EncoderApi for HwRamEncoder { { match cfg { EncoderCfg::HWRAM(config) => { + let rc = Self::rate_control(&config); let b = Self::convert_quality(&config.name, config.quality); let base_bitrate = base_bitrate(config.width as _, config.height as _); let mut bitrate = base_bitrate * b / 100; @@ -78,7 +77,8 @@ impl EncoderApi for HwRamEncoder { timebase: DEFAULT_TIME_BASE, gop, quality: DEFAULT_HW_QUALITY, - rc: DEFAULT_RC, + rc, + q: -1, thread_count: codec_thread_num(16) as _, // ffmpeg's thread_count is used for cpu }; let format = match Encoder::format_from_name(config.name.clone()) { @@ -175,6 +175,7 @@ impl EncoderApi for HwRamEncoder { self.encoder.set_bitrate(bitrate as _).ok(); self.bitrate = bitrate; } + self.config.quality = quality; Ok(()) } @@ -232,6 +233,14 @@ impl HwRamEncoder { } } + fn rate_control(config: &HwRamEncoderConfig) -> RateControl { + #[cfg(target_os = "android")] + if config.name.contains("mediacodec") { + return RC_VBR; + } + RC_CBR + } + pub fn convert_quality(name: &str, quality: crate::codec::Quality) -> u32 { use crate::codec::Quality; let quality = match quality { @@ -241,11 +250,8 @@ impl HwRamEncoder { Quality::Custom(b) => b, }; let factor = if name.contains("mediacodec") { - if name.contains("h264") { - 6 - } else { - 3 - } + // https://stackoverflow.com/questions/26110337/what-are-valid-bit-rates-to-set-for-mediacodec?rq=3 + 5 } else { 1 }; @@ -510,7 +516,8 @@ pub fn check_available_hwcodec() { timebase: DEFAULT_TIME_BASE, gop: DEFAULT_GOP, quality: DEFAULT_HW_QUALITY, - rc: DEFAULT_RC, + rc: RC_CBR, + q: -1, thread_count: 4, }; #[cfg(feature = "vram")]