sync

.github/workflows/bridge.yml

@@ -38,10 +38,8 @@ jobs:
             git \
             g++ \
             libclang-10-dev \
-            libclang-dev \
             libgtk-3-dev \
             llvm-10-dev \
-            llvm-dev \
             nasm \
             ninja-build \
             pkg-config \

.github/workflows/flutter-build.yml

@@ -783,7 +783,6 @@ jobs:
                libasound2-dev \
                libc6-dev \
                libclang-10-dev \
-               libclang-dev \
                libgstreamer1.0-dev \
                libgstreamer-plugins-base1.0-dev \
                libgtk-3-dev \
@@ -797,7 +796,6 @@ jobs:
                libxdo-dev \
                libxfixes-dev \
                llvm-10-dev \
-               llvm-dev \
                nasm \
                ninja-build \
                openjdk-11-jdk-headless \
@@ -1073,7 +1071,6 @@ jobs:
                libappindicator3-dev \
                libasound2-dev \
                libclang-10-dev \
-               libclang-dev \
                libgstreamer1.0-dev \
                libgstreamer-plugins-base1.0-dev \
                libgtk-3-dev \
@@ -1087,7 +1084,6 @@ jobs:
                libxdo-dev \
                libxfixes-dev \
                llvm-10-dev \
-               llvm-dev \
                nasm \
                ninja-build \
                pkg-config \

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/AudioRecordHandle.kt

@@ -0,0 +1,200 @@
+package com.carriez.flutter_hbb
+
+import ffi.FFI
+
+import android.Manifest
+import android.content.Context
+import android.media.*
+import android.content.pm.PackageManager
+import android.media.projection.MediaProjection
+import androidx.annotation.RequiresApi
+import androidx.core.app.ActivityCompat
+import android.os.Build
+import android.util.Log
+import kotlin.concurrent.thread
+
+const val AUDIO_ENCODING = AudioFormat.ENCODING_PCM_FLOAT //  ENCODING_OPUS need API 30
+const val AUDIO_SAMPLE_RATE = 48000
+const val AUDIO_CHANNEL_MASK = AudioFormat.CHANNEL_IN_STEREO
+
+class AudioRecordHandle(private var context: Context, private var isVideoStart: ()->Boolean, private var isAudioStart: ()->Boolean) {
+    private val logTag = "LOG_AUDIO_RECORD_HANDLE"
+
+    private var audioRecorder: AudioRecord? = null
+    private var audioReader: AudioReader? = null
+    private var minBufferSize = 0
+    private var audioRecordStat = false
+    private var audioThread: Thread? = null
+
+    @RequiresApi(Build.VERSION_CODES.M)
+    fun createAudioRecorder(inVoiceCall: Boolean, mediaProjection: MediaProjection?): Boolean {
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
+            return false
+        }
+        if (ActivityCompat.checkSelfPermission(
+            context,
+            Manifest.permission.RECORD_AUDIO
+        ) != PackageManager.PERMISSION_GRANTED
+        ) {
+            Log.d(logTag, "createAudioRecorder failed, no RECORD_AUDIO permission")
+            return false
+        }
+
+        var builder = AudioRecord.Builder()
+        .setAudioFormat(
+            AudioFormat.Builder()
+                .setEncoding(AUDIO_ENCODING)
+                .setSampleRate(AUDIO_SAMPLE_RATE)
+                .setChannelMask(AUDIO_CHANNEL_MASK).build()
+        );
+        if (inVoiceCall) {
+            builder.setAudioSource(MediaRecorder.AudioSource.VOICE_COMMUNICATION)
+        } else {
+            mediaProjection?.let {
+                var apcc = AudioPlaybackCaptureConfiguration.Builder(it)
+                .addMatchingUsage(AudioAttributes.USAGE_MEDIA)
+                .addMatchingUsage(AudioAttributes.USAGE_ALARM)
+                .addMatchingUsage(AudioAttributes.USAGE_GAME)
+                .addMatchingUsage(AudioAttributes.USAGE_UNKNOWN).build();
+                builder.setAudioPlaybackCaptureConfig(apcc);
+            } ?: let {
+                Log.d(logTag, "createAudioRecorder failed, mediaProjection null")
+                return false
+            }
+        }
+        audioRecorder = builder.build()
+        Log.d(logTag, "createAudioRecorder done,minBufferSize:$minBufferSize")
+        return true
+    }
+
+    @RequiresApi(Build.VERSION_CODES.M)
+    private fun checkAudioReader() {
+        if (audioReader != null && minBufferSize != 0) {
+            return
+        }
+        // read f32 to byte , length * 4
+        minBufferSize = 2 * 4 * AudioRecord.getMinBufferSize(
+            AUDIO_SAMPLE_RATE,
+            AUDIO_CHANNEL_MASK,
+            AUDIO_ENCODING
+        )
+        if (minBufferSize == 0) {
+            Log.d(logTag, "get min buffer size fail!")
+            return
+        }
+        audioReader = AudioReader(minBufferSize, 4)
+        Log.d(logTag, "init audioData len:$minBufferSize")
+    }
+
+    @RequiresApi(Build.VERSION_CODES.M)
+    fun startAudioRecorder() {
+        checkAudioReader()
+        if (audioReader != null && audioRecorder != null && minBufferSize != 0) {
+            try {
+                FFI.setFrameRawEnable("audio", true)
+                audioRecorder!!.startRecording()
+                audioRecordStat = true
+                audioThread = thread {
+                    while (audioRecordStat) {
+                        audioReader!!.readSync(audioRecorder!!)?.let {
+                            FFI.onAudioFrameUpdate(it)
+                        }
+                    }
+                    // let's release here rather than onDestroy to avoid threading issue
+                    audioRecorder?.release()
+                    audioRecorder = null
+                    minBufferSize = 0
+                    FFI.setFrameRawEnable("audio", false)
+                    Log.d(logTag, "Exit audio thread")
+                }
+            } catch (e: Exception) {
+                Log.d(logTag, "startAudioRecorder fail:$e")
+            }
+        } else {
+            Log.d(logTag, "startAudioRecorder fail")
+        }
+    }
+
+    fun onVoiceCallStarted(mediaProjection: MediaProjection?): Boolean {
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
+            return false
+        }
+        if (isVideoStart() || isAudioStart()) {
+            if (!switchToVoiceCall(mediaProjection)) {
+                return false
+            }
+        } else {
+            if (!switchToVoiceCall(mediaProjection)) {
+                return false
+            }
+        }
+        return true
+    }
+
+    fun onVoiceCallClosed(mediaProjection: MediaProjection?): Boolean {
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
+            return false
+        }
+        if (isVideoStart()) {
+            switchOutVoiceCall(mediaProjection)
+        }
+        tryReleaseAudio()
+        return true
+    }
+
+    @RequiresApi(Build.VERSION_CODES.M)
+    fun switchToVoiceCall(mediaProjection: MediaProjection?): Boolean {
+        audioRecorder?.let {
+            if (it.getAudioSource() == MediaRecorder.AudioSource.VOICE_COMMUNICATION) {
+                return true
+            }
+        }
+        audioRecordStat = false
+        audioThread?.join()
+        audioThread = null
+
+        if (!createAudioRecorder(true, mediaProjection)) {
+            Log.e(logTag, "createAudioRecorder fail")
+            return false
+        }
+        startAudioRecorder()
+        return true
+    }
+
+    @RequiresApi(Build.VERSION_CODES.M)
+    fun switchOutVoiceCall(mediaProjection: MediaProjection?): Boolean {
+        audioRecorder?.let {
+            if (it.getAudioSource() != MediaRecorder.AudioSource.VOICE_COMMUNICATION) {
+                return true
+            }
+        }
+        audioRecordStat = false
+        audioThread?.join()
+
+        if (!createAudioRecorder(false, mediaProjection)) {
+            Log.e(logTag, "createAudioRecorder fail")
+            return false
+        }
+        startAudioRecorder()
+        return true
+    }
+
+    fun tryReleaseAudio() {
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
+            return
+        }
+        if (isAudioStart() || isVideoStart()) {
+            return
+        }
+        audioRecordStat = false
+        audioThread?.join()
+        audioThread = null
+    }
+
+    fun destroy() {
+        Log.d(logTag, "destroy audio record handle")
+
+        audioRecordStat = false
+        audioThread?.join()
+    }
+}

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt

@@ -42,6 +42,9 @@ class MainActivity : FlutterActivity() {
     private val logTag = "mMainActivity"
     private var mainService: MainService? = null
 
+    private var isAudioStart = false
+    private val audioRecordHandle = AudioRecordHandle(this, { false }, { isAudioStart })
+
     override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
         super.configureFlutterEngine(flutterEngine)
         if (MainService.isReady) {
@@ -230,6 +233,12 @@ class MainActivity : FlutterActivity() {
                         result.success(false)
                     }
                 }
+                "on_voice_call_started" -> {
+                    onVoiceCallStarted()
+                }
+                "on_voice_call_closed" -> {
+                    onVoiceCallClosed()
+                }
                 else -> {
                     result.error("-1", "No such method", null)
                 }
@@ -264,8 +273,12 @@ class MainActivity : FlutterActivity() {
             if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
                 hw = codec.isHardwareAccelerated
             } else {
-                if (listOf("OMX.google.", "OMX.SEC.", "c2.android").any { codec.name.startsWith(it) }) {
+                // https://chromium.googlesource.com/external/webrtc/+/HEAD/sdk/android/src/java/org/webrtc/MediaCodecUtils.java#29
+                // https://chromium.googlesource.com/external/webrtc/+/master/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java#229
+                if (listOf("OMX.google.", "OMX.SEC.", "c2.android").any { codec.name.startsWith(it, true) }) {
                     hw = false
+                } else if (listOf("c2.qti", "OMX.qcom.video", "OMX.Exynos", "OMX.hisi", "OMX.MTK", "OMX.Intel", "OMX.Nvidia").any { codec.name.startsWith(it, true) }) {
+                    hw = true
                 }
             }
             codecObject.put("hw", hw)
@@ -280,7 +293,8 @@ class MainActivity : FlutterActivity() {
                 val caps = codec.getCapabilitiesForType(mime_type)
                 var usable = true;
                 if (codec.isEncoder) {
-                    if (!caps.videoCapabilities.isSizeSupported(w,h) || !caps.videoCapabilities.isSizeSupported(h,w)) {
+                    // Encoder‘s max_height and max_width are interchangeable
+                    if (!caps.videoCapabilities.isSizeSupported(w,h) && !caps.videoCapabilities.isSizeSupported(h,w)) {
                         usable = false
                     }
                 }
@@ -309,7 +323,49 @@ class MainActivity : FlutterActivity() {
         }
         val result = JSONObject()
         result.put("version", Build.VERSION.SDK_INT)
+        result.put("w", w)
+        result.put("h", h)
         result.put("codecs", codecArray)
         FFI.setCodecInfo(result.toString())
     }
+
+    private fun onVoiceCallStarted() {
+        var ok = false
+        mainService?.let {
+            ok = it.onVoiceCallStarted()
+        } ?: let {
+            isAudioStart = true
+            ok = audioRecordHandle.onVoiceCallStarted(null)
+        }
+        if (!ok) {
+            // Rarely happens, So we just add log and msgbox here.
+            Log.e(logTag, "onVoiceCallStarted fail")
+            flutterMethodChannel?.invokeMethod("msgbox", mapOf(
+                "type" to "custom-nook-nocancel-hasclose-error",
+                "title" to "Voice call",
+                "text" to "Failed to start voice call."))
+        } else {
+            Log.d(logTag, "onVoiceCallStarted success")
+        }
+    }
+
+    private fun onVoiceCallClosed() {
+        var ok = false
+        mainService?.let {
+            ok = it.onVoiceCallClosed()
+        } ?: let {
+            isAudioStart = false
+            ok = audioRecordHandle.onVoiceCallClosed(null)
+        }
+        if (!ok) {
+            // Rarely happens, So we just add log and msgbox here.
+            Log.e(logTag, "onVoiceCallClosed fail")
+            flutterMethodChannel?.invokeMethod("msgbox", mapOf(
+                "type" to "custom-nook-nocancel-hasclose-error",
+                "title" to "Voice call",
+                "text" to "Failed to stop voice call."))
+        } else {
+            Log.d(logTag, "onVoiceCallClosed success")
+        }
+    }
 }