Skip to content

Commit

Permalink
integrate ggml's excellent and amazing whisper.cpp to kantv ---- step4
Browse files Browse the repository at this point in the history 
add asr subystem in kantv-core

refine directory hierarchy of whisper.cpp(put local whisper.cpp to directory external)

fix conflict between open source native libs and prebuilt proprietary native libs(IDE would complain more than one file was found)

study ggml_abort_callback in ggml_graph_compute
  • Loading branch information
@zhouwg
zhouwg committed Mar 7, 2024
1 parent 31989a5 commit 2d7febb
Show file tree
Hide file tree
Showing 39 changed files with 669 additions and 310 deletions.
2 changes: 1 addition & 1 deletion build/envsetup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ export PROJECT_BRANCH=`git branch | grep "*" | cut -f 2 -d ' ' `
export PROJECT_ROOT_PATH=${PROJECT_HOME_PATH}


export LOCAL_WHISPERCPP_PATH=${PROJECT_ROOT_PATH}/cdeosplayer/kantv/src/main/jni/whispercpp
export LOCAL_WHISPERCPP_PATH=${PROJECT_ROOT_PATH}/external/whispercpp

#modify following lines to adapt to local dev envs
export ANDROID_NDK=/opt/kantv-toolchains/android-ndk-r21e
Expand Down
2 changes: 1 addition & 1 deletion build/public.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ fi
function check_local_whispercpp()
{
if [ "x${LOCAL_WHISPERCPP_PATH}" == "x" ]; then
echo "pls checkout LOCAL_WHISPERCPP_PATH"
echo "pls check LOCAL_WHISPERCPP_PATH"
exit 1
fi

Expand Down
84 changes: 49 additions & 35 deletions cdeosplayer/cdeosplayer-lib/src/main/java/cdeos/media/player/KANTVEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,38 +16,52 @@
* limitations under the License.
*/

package cdeos.media.player;


final public class KANTVEvent
{
//keep sync with KANTV_event.h in native layer
public final static int KANTV_ERROR = 100;
public final static int KANTV_INFO = 200;

public final static int KANTV_INFO_PREVIEW_START = 0;
public final static int KANTV_INFO_PREVIEW_STOP = 1;
public final static int KANTV_INFO_SETPREVIEWDISPLAY = 2;
public final static int KANTV_INFO_SETRENDERID = 3;
public final static int KANTV_INFO_OPEN = 4;
public final static int KANTV_INFO_CLOSE = 5;
public final static int KANTV_INFO_PREVIEW = 8;
public final static int KANTV_INFO_GRAPHIC_BENCHMARK_START = 9;
public final static int KANTV_INFO_GRAPHIC_BENCHMARK_STOP = 10;
public final static int KANTV_INFO_ENCODE_BENCHMARK_START = 11;
public final static int KANTV_INFO_ENCODE_BENCHMARK_STOP = 12;
public final static int KANTV_INFO_ENCODE_BENCHMARK_INFO = 13;
public final static int KANTV_INFO_STATUS = 14;

public final static int KANTV_ERROR_PREVIEW_START = 0;
public final static int KANTV_ERROR_PREVIEW_STOP = 1;
public final static int KANTV_ERROR_SETPREVIEWDISPLAY = 2;
public final static int KANTV_ERROR_SETRENDERID = 3;
public final static int KANTV_ERROR_OPEN = 4;
public final static int KANTV_ERROR_CLOSE = 5;
public final static int KANTV_ERROR_PREVIEW = 8;
public final static int KANTV_ERROR_GRAPHIC_BENCHMARK = 9;
public final static int KANTV_ERROR_ENCODE_BENCHMARK = 10;

public static native int kantv_anti_tamper();
}
package cdeos.media.player;


final public class KANTVEvent {
//keep sync with KANTV_event.h in native layer
public final static int KANTV_ERROR = 100;
public final static int KANTV_INFO = 200;

public final static int KANTV_INFO_PREVIEW_START = 0;
public final static int KANTV_INFO_PREVIEW_STOP = 1;
public final static int KANTV_INFO_SETPREVIEWDISPLAY = 2;
public final static int KANTV_INFO_SETRENDERID = 3;
public final static int KANTV_INFO_OPEN = 4;
public final static int KANTV_INFO_CLOSE = 5;
public final static int KANTV_INFO_PREVIEW = 8;
public final static int KANTV_INFO_GRAPHIC_BENCHMARK_START = 9;
public final static int KANTV_INFO_GRAPHIC_BENCHMARK_STOP = 10;
public final static int KANTV_INFO_ENCODE_BENCHMARK_START = 11;
public final static int KANTV_INFO_ENCODE_BENCHMARK_STOP = 12;
public final static int KANTV_INFO_ENCODE_BENCHMARK_INFO = 13;
public final static int KANTV_INFO_STATUS = 14;

public final static int KANTV_INFO_ASR_INIT = 15;
public final static int KANTV_INFO_ASR_FINALiZE = 16;

public final static int KANTV_INFO_ASR_START = 17;
public final static int KANTV_INFO_ASR_STOP = 18;
public final static int KANTV_INFO_GGML_INTERNAL = 19;
public final static int KANTV_INFO_WHISPERCPP_INTERNAL = 20;

public final static int KANTV_ERROR_PREVIEW_START = 0;
public final static int KANTV_ERROR_PREVIEW_STOP = 1;
public final static int KANTV_ERROR_SETPREVIEWDISPLAY = 2;
public final static int KANTV_ERROR_SETRENDERID = 3;
public final static int KANTV_ERROR_OPEN = 4;
public final static int KANTV_ERROR_CLOSE = 5;
public final static int KANTV_ERROR_PREVIEW = 8;
public final static int KANTV_ERROR_GRAPHIC_BENCHMARK = 9;
public final static int KANTV_ERROR_ENCODE_BENCHMARK = 10;

public final static int KANTV_ERROR_ASR_INIT = 11;
public final static int KANTV_ERROR_ASR_FINALiZE = 12;
public final static int KANTV_ERROR_ASR_START = 13;
public final static int KANTV_ERROR_ASR_STOP = 14;
public final static int KANTV_ERROR_GGML_INTERNAL = 15;
public final static int KANTV_ERROR_WHISPERCPP_INTERNAL = 16;

public static native int kantv_anti_tamper();
}
7 changes: 6 additions & 1 deletion cdeosplayer/cdeosplayer-lib/src/main/java/cdeos/media/player/KANTVMgr.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,9 +215,14 @@ public void close()
public native void startGraphicBenchmarkPreview();
public native void stopGraphicBenchmarkPreview();

public native void initASR();
public native void finalizeASR();
public native void startASR();
public native void stopASR();

private native static int getVersionDescription(byte[] byteSecurityToken, int bufSize);
private native static final void native_init();
private native final void native_setup(Object KANTVmgr_this);
private native final void native_setup(Object KANTVMgr_this);
private native final void native_release();


Expand Down
2 changes: 2 additions & 0 deletions cdeosplayer/cdeosplayer-lib/src/main/java/org/ggml/whispercpp/WhisperLib.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,6 @@ public class WhisperLib {
public static native String benchMemcpy(int nthread);

public static native String benchGgmlMulMat(int nthread);

public static native void set_mulmat_benchmark_status(int bExit);
}
10 changes: 9 additions & 1 deletion cdeosplayer/kantv/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ android {
//abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"
abiFilters "arm64-v8a"
}

//fix conflict between open source native libs and prebuilt proprietary native libs(IDE would complain more than one file was found)
packagingOptions {
pickFirst 'lib/arm64-v8a/libkantv-core.so'
pickFirst 'lib/arm64-v8a/libwhispercpp.so'
}

versionName project.ext.releaseVersion
versionCode project.ext.releaseVersionCode
minSdkVersion project.ext.minSdkVersion
Expand All @@ -32,7 +39,8 @@ android {

externalNativeBuild {
cmake {
path "src/main/jni/whispercpp/CMakeLists.txt"
//path "src/main/jni/whispercpp/CMakeLists.txt"
path "../../external/whispercpp/CMakeLists.txt"
}
}

Expand Down
4 changes: 2 additions & 2 deletions ...splayer/kantv/src/main/java/com/cdeos/kantv/ui/activities/personal/BenchmarkActivity.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -819,11 +819,11 @@ public void release() {
mKANTVMgr.release();
mKANTVMgr = null;
_bPreviewing = false;
_btnEncodeBenchmark.setText("编码测试");
_btnEncodeBenchmark.setText("video encode benchmark");
}
} catch (Exception ex) {
String errorMsg = "An Exception was thrown because:\n" + " " + ex.getMessage();
CDELog.j(TAG, "error occured: " + errorMsg);
CDELog.j(TAG, "error occurred: " + errorMsg);
ex.printStackTrace();
}
}
Expand Down
Loading

3 comments on commit 2d7febb

@zhouwg
Copy link
Owner Author

@zhouwg zhouwg commented on 2d7febb Mar 7, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

KANTV_ERROR_ASR_FINALiZE: this is typo and should be fixed in next commit

@zhouwg
Copy link
Owner Author

@zhouwg zhouwg commented on 2d7febb Mar 7, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

external/whispercpp/diff-with-upstream-whispercpp.sh

echo -e "upstream whispercpp path: ${UPSTREAM_WHISPERCPP_PATH}\n"
-echo -e "local whispercpp path: ${UPSTREAM_WHISPERCPP_PATH}\n"
+echo -e "local whispercpp path: ${LOCAL_WHISPERCPP_PATH}\n"

should be fixed in next commit

@zhouwg
Copy link
Owner Author

@zhouwg zhouwg commented on 2d7febb Mar 7, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the JNI in original Android sample should be rewritten. I have a good idea to implement it.

Please sign in to comment.