Skip to content

Commit

Permalink
add ffmpeg mediacodec h264/h265 encode (rustdesk#8028)
Browse files Browse the repository at this point in the history 
* Check available when app start from kotlin via get codec info
* For latency free, repeat encode 10 frame at most when capture return WouldBlock
* For changing quality, kotlin support but jni doesn't support, rerun video service when quality is manualy
  changed
* 3 or 6 times bitrate for mediacodec because its quality is poor

Signed-off-by: 21pages <[email protected]>
  • Loading branch information
@21pages
21pages authored May 13, 2024
1 parent 4c99b8c commit a7499c2
Show file tree
Hide file tree
Showing 13 changed files with 380 additions and 31 deletions.
4 changes: 2 additions & 2 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

89 changes: 89 additions & 0 deletions flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ package com.carriez.flutter_hbb
* Inspired by [droidVNC-NG] https://github.com/bk138/droidVNC-NG
*/

import ffi.FFI

import android.content.ComponentName
import android.content.Context
import android.content.Intent
Expand All @@ -15,10 +17,20 @@ import android.os.Build
import android.os.IBinder
import android.util.Log
import android.view.WindowManager
import android.media.MediaCodecInfo
import android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface
import android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar
import android.media.MediaCodecList
import android.media.MediaFormat
import android.util.DisplayMetrics
import androidx.annotation.RequiresApi
import org.json.JSONArray
import org.json.JSONObject
import com.hjq.permissions.XXPermissions
import io.flutter.embedding.android.FlutterActivity
import io.flutter.embedding.engine.FlutterEngine
import io.flutter.plugin.common.MethodChannel
import kotlin.concurrent.thread


class MainActivity : FlutterActivity() {
Expand All @@ -42,6 +54,7 @@ class MainActivity : FlutterActivity() {
channelTag
)
initFlutterChannel(flutterMethodChannel!!)
thread { setCodecInfo() }
}

override fun onResume() {
Expand Down Expand Up @@ -223,4 +236,80 @@ class MainActivity : FlutterActivity() {
}
}
}

private fun setCodecInfo() {
val codecList = MediaCodecList(MediaCodecList.REGULAR_CODECS)
val codecs = codecList.codecInfos
val codecArray = JSONArray()

val windowManager = getSystemService(Context.WINDOW_SERVICE) as WindowManager
var w = 0
var h = 0
@Suppress("DEPRECATION")
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
val m = windowManager.maximumWindowMetrics
w = m.bounds.width()
h = m.bounds.height()
} else {
val dm = DisplayMetrics()
windowManager.defaultDisplay.getRealMetrics(dm)
w = dm.widthPixels
h = dm.heightPixels
}
codecs.forEach { codec ->
val codecObject = JSONObject()
codecObject.put("name", codec.name)
codecObject.put("is_encoder", codec.isEncoder)
var hw: Boolean? = null;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
hw = codec.isHardwareAccelerated
} else {
if (listOf("OMX.google.", "OMX.SEC.", "c2.android").any { codec.name.startsWith(it) }) {
hw = false
}
}
codecObject.put("hw", hw)
var mime_type = ""
codec.supportedTypes.forEach { type ->
if (listOf("video/avc", "video/hevc", "video/x-vnd.on2.vp8", "video/x-vnd.on2.vp9", "video/av01").contains(type)) {
mime_type = type;
}
}
if (mime_type.isNotEmpty()) {
codecObject.put("mime_type", mime_type)
val caps = codec.getCapabilitiesForType(mime_type)
var usable = true;
if (codec.isEncoder) {
if (!caps.videoCapabilities.isSizeSupported(w,h) || !caps.videoCapabilities.isSizeSupported(h,w)) {
usable = false
}
}
codecObject.put("min_width", caps.videoCapabilities.supportedWidths.lower)
codecObject.put("max_width", caps.videoCapabilities.supportedWidths.upper)
codecObject.put("min_height", caps.videoCapabilities.supportedHeights.lower)
codecObject.put("max_height", caps.videoCapabilities.supportedHeights.upper)
val surface = caps.colorFormats.contains(COLOR_FormatSurface);
codecObject.put("surface", surface)
val nv12 = caps.colorFormats.contains(COLOR_FormatYUV420SemiPlanar)
codecObject.put("nv12", nv12)
if (!(nv12 || surface)) {
usable = false
}
codecObject.put("min_bitrate", caps.videoCapabilities.bitrateRange.lower / 1000)
codecObject.put("max_bitrate", caps.videoCapabilities.bitrateRange.upper / 1000)
if (!codec.isEncoder) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
codecObject.put("low_latency", caps.isFeatureSupported(MediaCodecInfo.CodecCapabilities.FEATURE_LowLatency))
}
}
if (usable) {
codecArray.put(codecObject)
}
}
}
val result = JSONObject()
result.put("version", Build.VERSION.SDK_INT)
result.put("codecs", codecArray)
FFI.setCodecInfo(result.toString())
}
}
1 change: 1 addition & 0 deletions flutter/android/app/src/main/kotlin/ffi.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,5 @@ object FFI {
external fun translateLocale(localeName: String, input: String): String
external fun refreshScreen()
external fun setFrameRawEnable(name: String, value: Boolean)
external fun setCodecInfo(info: String)
}
24 changes: 24 additions & 0 deletions flutter/lib/mobile/pages/settings_page.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ class _SettingsState extends State<SettingsPage> with WidgetsBindingObserver {
var _onlyWhiteList = false;
var _enableDirectIPAccess = false;
var _enableRecordSession = false;
var _enableHardwareCodec = false;
var _autoRecordIncomingSession = false;
var _allowAutoDisconnect = false;
var _localIP = "";
Expand Down Expand Up @@ -120,6 +121,13 @@ class _SettingsState extends State<SettingsPage> with WidgetsBindingObserver {
_enableRecordSession = enableRecordSession;
}

final enableHardwareCodec = option2bool(
'enable-hwcodec', await bind.mainGetOption(key: 'enable-hwcodec'));
if (_enableHardwareCodec != enableHardwareCodec) {
update = true;
_enableHardwareCodec = enableHardwareCodec;
}

final autoRecordIncomingSession = option2bool(
'allow-auto-record-incoming',
await bind.mainGetOption(key: 'allow-auto-record-incoming'));
Expand Down Expand Up @@ -513,6 +521,22 @@ class _SettingsState extends State<SettingsPage> with WidgetsBindingObserver {
},
)
]),
if (isAndroid)
SettingsSection(title: Text(translate('Hardware Codec')), tiles: [
SettingsTile.switchTile(
title: Text(translate('Enable hardware codec')),
initialValue: _enableHardwareCodec,
onToggle: (v) async {
await bind.mainSetOption(
key: "enable-hwcodec", value: v ? "" : "N");
final newValue =
await bind.mainGetOption(key: "enable-hwcodec") != "N";
setState(() {
_enableHardwareCodec = newValue;
});
},
),
]),
if (isAndroid && !outgoingOnly)
SettingsSection(
title: Text(translate("Recording")),
Expand Down
1 change: 1 addition & 0 deletions libs/scrap/examples/benchmark.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ mod hw {
let mut encoder = HwRamEncoder::new(
EncoderCfg::HWRAM(HwRamEncoderConfig {
name: info.name.clone(),
mc_name: None,
width,
height,
quality,
Expand Down
46 changes: 46 additions & 0 deletions libs/scrap/src/android/ffi.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use jni::{

use jni::errors::{Error as JniError, Result as JniResult};
use lazy_static::lazy_static;
use serde::Deserialize;
use std::ops::Not;
use std::sync::atomic::{AtomicPtr, Ordering::SeqCst};
use std::sync::{Mutex, RwLock};
Expand All @@ -20,6 +21,7 @@ lazy_static! {
static ref VIDEO_RAW: Mutex<FrameRaw> = Mutex::new(FrameRaw::new("video", MAX_VIDEO_FRAME_TIMEOUT));
static ref AUDIO_RAW: Mutex<FrameRaw> = Mutex::new(FrameRaw::new("audio", MAX_AUDIO_FRAME_TIMEOUT));
static ref NDK_CONTEXT_INITED: Mutex<bool> = Default::default();
static ref MEDIA_CODEC_INFOS: RwLock<Option<MediaCodecInfos>> = RwLock::new(None);
}

const MAX_VIDEO_FRAME_TIMEOUT: Duration = Duration::from_millis(100);
Expand Down Expand Up @@ -154,6 +156,50 @@ pub extern "system" fn Java_ffi_FFI_init(env: JNIEnv, _class: JClass, ctx: JObje
}
}

#[derive(Debug, Deserialize, Clone)]
pub struct MediaCodecInfo {
pub name: String,
pub is_encoder: bool,
#[serde(default)]
pub hw: Option<bool>, // api 29+
pub mime_type: String,
pub surface: bool,
pub nv12: bool,
#[serde(default)]
pub low_latency: Option<bool>, // api 30+, decoder
pub min_bitrate: u32,
pub max_bitrate: u32,
pub min_width: usize,
pub max_width: usize,
pub min_height: usize,
pub max_height: usize,
}

#[derive(Debug, Deserialize, Clone)]
pub struct MediaCodecInfos {
pub version: usize,
pub codecs: Vec<MediaCodecInfo>,
}

#[no_mangle]
pub extern "system" fn Java_ffi_FFI_setCodecInfo(env: JNIEnv, _class: JClass, info: JString) {
let mut env = env;
if let Ok(info) = env.get_string(&info) {
let info: String = info.into();
if let Ok(infos) = serde_json::from_str::<MediaCodecInfos>(&info) {
*MEDIA_CODEC_INFOS.write().unwrap() = Some(infos);
}
}
}

pub fn get_codec_info() -> Option<MediaCodecInfos> {
MEDIA_CODEC_INFOS.read().unwrap().as_ref().cloned()
}

pub fn clear_codec_info() {
*MEDIA_CODEC_INFOS.write().unwrap() = None;
}

pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) -> JniResult<()> {
if let (Some(jvm), Some(ctx)) = (
JVM.read().unwrap().as_ref(),
Expand Down
8 changes: 8 additions & 0 deletions libs/scrap/src/common/aom.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,14 @@ impl EncoderApi for AomEncoder {
fn support_abr(&self) -> bool {
true
}

fn support_changing_quality(&self) -> bool {
true
}

fn latency_free(&self) -> bool {
true
}
}

impl AomEncoder {
Expand Down
27 changes: 25 additions & 2 deletions libs/scrap/src/common/codec.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,10 @@ pub trait EncoderApi {
fn bitrate(&self) -> u32;

fn support_abr(&self) -> bool;

fn support_changing_quality(&self) -> bool;

fn latency_free(&self) -> bool;
}

pub struct Encoder {
Expand Down Expand Up @@ -138,6 +142,9 @@ impl Encoder {
}),
Err(e) => {
log::error!("new hw encoder failed: {e:?}, clear config");
#[cfg(target_os = "android")]
crate::android::ffi::clear_codec_info();
#[cfg(not(target_os = "android"))]
hbb_common::config::HwCodecConfig::clear_ram();
Self::update(EncodingUpdate::Check);
*ENCODE_CODEC_FORMAT.lock().unwrap() = CodecFormat::VP9;
Expand Down Expand Up @@ -346,7 +353,14 @@ impl Encoder {
EncoderCfg::AOM(_) => CodecFormat::AV1,
#[cfg(feature = "hwcodec")]
EncoderCfg::HWRAM(hw) => {
if hw.name.to_lowercase().contains("h264") {
let name = hw.name.to_lowercase();
if name.contains("vp8") {
CodecFormat::VP8
} else if name.contains("vp9") {
CodecFormat::VP9
} else if name.contains("av1") {
CodecFormat::AV1
} else if name.contains("h264") {
CodecFormat::H264
} else {
CodecFormat::H265
Expand Down Expand Up @@ -817,7 +831,7 @@ impl Decoder {

#[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
pub fn enable_hwcodec_option() -> bool {
if cfg!(windows) || cfg!(target_os = "linux") || cfg!(feature = "mediacodec") {
if cfg!(windows) || cfg!(target_os = "linux") || cfg!(target_os = "android") {
if let Some(v) = Config2::get().options.get("enable-hwcodec") {
return v != "N";
}
Expand Down Expand Up @@ -847,6 +861,15 @@ impl Default for Quality {
}
}

impl Quality {
pub fn is_custom(&self) -> bool {
match self {
Quality::Custom(_) => true,
_ => false,
}
}
}

pub fn base_bitrate(width: u32, height: u32) -> u32 {
#[allow(unused_mut)]
let mut base_bitrate = ((width * height) / 1000) as u32; // same as 1.1.9
Expand Down
Loading

0 comments on commit a7499c2

Please sign in to comment.