fsmn-vad bugfix (#1270)

* funasr1.0 funetine

* funasr1.0 pbar

* update with main (#1260)

* Update websocket_protocol_zh.md

* update

---------

Co-authored-by: Yabin Li <[email protected]>
Co-authored-by: shixian.shi <[email protected]>

* update with main (#1264)

* Funasr1.0 (#1261)

* funasr1.0 funetine

* funasr1.0 pbar

* update with main (#1260)

* Update websocket_protocol_zh.md

* update

---------

Co-authored-by: Yabin Li <[email protected]>
Co-authored-by: shixian.shi <[email protected]>

---------

Co-authored-by: Yabin Li <[email protected]>
Co-authored-by: shixian.shi <[email protected]>

* bug fix

---------

Co-authored-by: Yabin Li <[email protected]>
Co-authored-by: shixian.shi <[email protected]>

* funasr1.0 sanm scama

* funasr1.0 infer_after_finetune

* funasr1.0 fsmn-vad bug fix

* funasr1.0 fsmn-vad bug fix

---------

Co-authored-by: Yabin Li <[email protected]>
Co-authored-by: shixian.shi <[email protected]>

Author: 3 people

README.md

@@ -91,12 +91,13 @@ Notes: Support recognition of single audio file, as well as file list in Kaldi-s
 from funasr import AutoModel
 # paraformer-zh is a multi-functional asr model
 # use vad, punc, spk or not as you need
-model = AutoModel(model="paraformer-zh", model_revision="v2.0.2", \
-                  vad_model="fsmn-vad", vad_model_revision="v2.0.2", \
-                  punc_model="ct-punc-c", punc_model_revision="v2.0.2", \
-                  spk_model="cam++", spk_model_revision="v2.0.2")
+model = AutoModel(model="paraformer-zh", model_revision="v2.0.2",
+                  vad_model="fsmn-vad", vad_model_revision="v2.0.2",
+                  punc_model="ct-punc-c", punc_model_revision="v2.0.2",
+                  # spk_model="cam++", spk_model_revision="v2.0.2",
+                  )
 res = model.generate(input=f"{model.model_path}/example/asr_example.wav", 
-                     batch_size=64, 
+                     batch_size_s=300, 
                      hotword='魔搭')
 print(res)
 ```

README_zh.md

@@ -87,12 +87,13 @@ funasr +model=paraformer-zh +vad_model="fsmn-vad" +punc_model="ct-punc" +input=a
 from funasr import AutoModel
 # paraformer-zh is a multi-functional asr model
 # use vad, punc, spk or not as you need
-model = AutoModel(model="paraformer-zh", model_revision="v2.0.2", \
-                  vad_model="fsmn-vad", vad_model_revision="v2.0.2", \
-                  punc_model="ct-punc-c", punc_model_revision="v2.0.2", \
-                  spk_model="cam++", spk_model_revision="v2.0.2")
+model = AutoModel(model="paraformer-zh", model_revision="v2.0.2",
+                  vad_model="fsmn-vad", vad_model_revision="v2.0.2",
+                  punc_model="ct-punc-c", punc_model_revision="v2.0.2",
+                  # spk_model="cam++", spk_model_revision="v2.0.2",
+                  )
 res = model.generate(input=f"{model.model_path}/example/asr_example.wav", 
-            batch_size=64, 
+            batch_size_s=300, 
             hotword='魔搭')
 print(res)
 ```

examples/industrial_data_pretraining/paraformer/infer_after_finetune.sh

@@ -0,0 +1,12 @@
+
+
+python funasr/bin/inference.py \
+--config-path="/Users/zhifu/funasr_github/test_local/funasr_cli_egs" \
+--config-name="config.yaml" \
+++init_param="/Users/zhifu/funasr_github/test_local/funasr_cli_egs/model.pt" \
++tokenizer_conf.token_list="/Users/zhifu/funasr_github/test_local/funasr_cli_egs/tokens.txt" \
++frontend_conf.cmvn_file="/Users/zhifu/funasr_github/test_local/funasr_cli_egs/am.mvn" \
++input="data/wav.scp" \
++output_dir="./outputs/debug" \
++device="cuda" \
+

examples/industrial_data_pretraining/scama/demo.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+from funasr import AutoModel
+
+chunk_size = [5, 10, 5] #[0, 10, 5] 600ms, [0, 8, 4] 480ms
+encoder_chunk_look_back = 0 #number of chunks to lookback for encoder self-attention
+decoder_chunk_look_back = 0 #number of encoder chunks to lookback for decoder cross-attention
+
+model = AutoModel(model="/Users/zhifu/Downloads/modelscope_models/speech_SCAMA_asr-zh-cn-16k-common-vocab8358-streaming", model_revision="v2.0.2")
+cache = {}
+res = model.generate(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav",
+            chunk_size=chunk_size,
+            encoder_chunk_look_back=encoder_chunk_look_back,
+            decoder_chunk_look_back=decoder_chunk_look_back,
+            )
+print(res)
+
+
+import soundfile
+import os
+
+wav_file = os.path.join(model.model_path, "example/asr_example.wav")
+speech, sample_rate = soundfile.read(wav_file)
+
+chunk_stride = chunk_size[1] * 960 # 600ms、480ms
+
+cache = {}
+total_chunk_num = int(len((speech)-1)/chunk_stride+1)
+for i in range(total_chunk_num):
+    speech_chunk = speech[i*chunk_stride:(i+1)*chunk_stride]
+    is_final = i == total_chunk_num - 1
+    res = model.generate(input=speech_chunk,
+                         cache=cache,
+                         is_final=is_final,
+                         chunk_size=chunk_size,
+                         encoder_chunk_look_back=encoder_chunk_look_back,
+                         decoder_chunk_look_back=decoder_chunk_look_back,
+                         )
+    print(res)

examples/industrial_data_pretraining/scama/infer.sh

@@ -0,0 +1,11 @@
+
+model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online"
+model_revision="v2.0.2"
+
+python funasr/bin/inference.py \
++model=${model} \
++model_revision=${model_revision} \
++input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav" \
++output_dir="./outputs/debug" \
++device="cpu" \
+