[Not for merge] Begin to support tinyalsa

CMakeLists.txt

@@ -42,6 +42,7 @@ option(SHERPA_NCNN_ENABLE_TEST "Whether to build tests" OFF)
 option(SHERPA_NCNN_ENABLE_C_API "Whether to build C API" ON)
 option(SHERPA_NCNN_ENABLE_GENERATE_INT8_SCALE_TABLE "Whether to generate-int8-scale-table" ON)
 option(SHERPA_NCNN_ENABLE_FFMPEG_EXAMPLES "Whether to enable ffmpeg-examples" OFF)
+option(SHERPA_NCNN_ENABLE_TINY_ALSA "Whether to enable tinyalsa" ON)
 
 if(DEFINED ANDROID_ABI)
   message(STATUS "Set SHERPA_NCNN_ENABLE_JNI to ON for Android")
@@ -63,6 +64,7 @@ message(STATUS "SHERPA_NCNN_ENABLE_TEST ${SHERPA_NCNN_ENABLE_TEST}")
 message(STATUS "SHERPA_NCNN_ENABLE_C_API ${SHERPA_NCNN_ENABLE_C_API}")
 message(STATUS "SHERPA_NCNN_ENABLE_GENERATE_INT8_SCALE_TABLE ${SHERPA_NCNN_ENABLE_GENERATE_INT8_SCALE_TABLE}")
 message(STATUS "SHERPA_NCNN_ENABLE_FFMPEG_EXAMPLES ${SHERPA_NCNN_ENABLE_FFMPEG_EXAMPLES}")
+message(STATUS "SHERPA_NCNN_ENABLE_TINY_ALSA ${SHERPA_NCNN_ENABLE_TINY_ALSA}")
 
 if(NOT CMAKE_BUILD_TYPE)
   message(STATUS "No CMAKE_BUILD_TYPE given, default to Release")
@@ -125,6 +127,10 @@ if(SHERPA_NCNN_ENABLE_PYTHON)
   include(pybind11)
 endif()
 
+if(SHERPA_NCNN_ENABLE_TINY_ALSA)
+  include(tiny-alsa)
+endif()
+
 add_subdirectory(sherpa-ncnn)
 
 if(SHERPA_NCNN_ENABLE_FFMPEG_EXAMPLES)

build-android-arm64-v8a.sh

@@ -48,7 +48,8 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake"
     -DCMAKE_BUILD_TYPE=Release \
     -DBUILD_SHARED_LIBS=ON \
     -DSHERPA_NCNN_ENABLE_PORTAUDIO=OFF \
-    -DSHERPA_NCNN_ENABLE_BINARY=OFF \
+    -DSHERPA_NCNN_ENABLE_BINARY=ON \
+    -DSHERPA_NCNN_ENABLE_TINY_ALSA=ON \
     -DSHERPA_NCNN_ENABLE_TEST=OFF \
     -DSHERPA_NCNN_ENABLE_C_API=OFF \
     -DSHERPA_NCNN_ENABLE_GENERATE_INT8_SCALE_TABLE=OFF \

cmake/tiny-alsa.cmake

@@ -0,0 +1,50 @@
+function(download_tiny_alsa)
+  include(FetchContent)
+
+  set(tiny_alsa_URL  "https://github.com/tinyalsa//tinyalsa/archive/4fbaeef03cd1cb216e0f356c0433ca70f8b9c464.zip")
+  set(tiny_alsa_URL2 "https://huggingface.co/csukuangfj/sherpa-ncnn-cmake-deps/resolve/main/tinyalsa-4fbaeef03cd1cb216e0f356c0433ca70f8b9c464.zip")
+  set(tiny_alsa_HASH "SHA256=13a0e55fef7fa114db843b2ce51bf406f1b6861aed4340984adb13f49bd8457c")
+
+  # If you don't have access to the Internet, please download it to your
+  # local drive and modify the following line according to your needs.
+  set(possible_file_locations
+    $ENV{HOME}/Downloads/tinyalsa-4fbaeef03cd1cb216e0f356c0433ca70f8b9c464.zip
+    $ENV{HOME}/asr/tinyalsa-4fbaeef03cd1cb216e0f356c0433ca70f8b9c464.zip
+    ${PROJECT_SOURCE_DIR}/tinyalsa-4fbaeef03cd1cb216e0f356c0433ca70f8b9c464.zip
+    ${PROJECT_BINARY_DIR}/tinyalsa-4fbaeef03cd1cb216e0f356c0433ca70f8b9c464.zip
+    /tmp/tinyalsa-4fbaeef03cd1cb216e0f356c0433ca70f8b9c464.zip
+  )
+
+  foreach(f IN LISTS possible_file_locations)
+    if(EXISTS ${f})
+      set(tiny_alsa_URL  "${f}")
+      file(TO_CMAKE_PATH "${tiny_alsa_URL}" tiny_alsa_URL)
+      set(tiny_alsa_URL2)
+      break()
+    endif()
+  endforeach()
+
+  FetchContent_Declare(tiny_alsa
+    URL
+      ${tiny_alsa_URL}
+      ${tiny_alsa_URL2}
+    URL_HASH          ${tiny_alsa_HASH}
+  )
+
+  set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS} CACHE BOOL "" FORCE)
+  set(TINYALSA_USES_PLUGINS OFF CACHE BOOL "" FORCE)
+  set(TINYALSA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
+  set(TINYALSA_BUILD_UTILS OFF CACHE BOOL "" FORCE)
+
+  FetchContent_GetProperties(tiny_alsa)
+  if(NOT tiny_alsa_POPULATED)
+    message(STATUS "Downloading tiny_alsa from ${tiny_alsa_URL}")
+    FetchContent_Populate(tiny_alsa)
+  endif()
+  message(STATUS "tiny_alsa is downloaded to ${tiny_alsa_SOURCE_DIR}")
+  message(STATUS "tiny_alsa's binary dir is ${tiny_alsa_BINARY_DIR}")
+
+  add_subdirectory(${tiny_alsa_SOURCE_DIR} ${tiny_alsa_BINARY_DIR})
+endfunction()
+
+download_tiny_alsa()

sherpa-ncnn/csrc/CMakeLists.txt

@@ -66,6 +66,16 @@ if(NOT SHERPA_NCNN_ENABLE_PYTHON)
       install(TARGETS sherpa-ncnn-microphone DESTINATION bin)
     endif()
 
+    if(SHERPA_NCNN_ENABLE_TINY_ALSA)
+      add_executable(sherpa-ncnn-tinyalsa
+        sherpa-ncnn-tinyalsa.cc
+        tinyalsa.cc
+      )
+      target_link_libraries(sherpa-ncnn-tinyalsa PRIVATE tinyalsa sherpa-ncnn-core)
+
+      install(TARGETS sherpa-ncnn-tinyalsa DESTINATION bin)
+    endif()
+
     set(hdrs
       features.h
       model.h

sherpa-ncnn/csrc/sherpa-ncnn-tinyalsa.cc

@@ -0,0 +1,172 @@
+/**
+ * Copyright (c)  2023  Xiaomi Corporation (authors: Fangjun Kuang)
+ *
+ * See LICENSE for clarification regarding multiple authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <algorithm>
+#include <cctype>  // std::tolower
+#include <cstdint>
+
+#include "sherpa-ncnn/csrc/tinyalsa.h"
+#include "sherpa-ncnn/csrc/display.h"
+#include "sherpa-ncnn/csrc/recognizer.h"
+
+bool stop = false;
+
+static void Handler(int sig) {
+  stop = true;
+  fprintf(stderr, "\nCaught Ctrl + C. Exiting...\n");
+};
+
+int main(int32_t argc, char *argv[]) {
+  if (argc < 9 || argc > 11) {
+    const char *usage = R"usage(
+Usage:
+  ./bin/sherpa-ncnn-alsa \
+    /path/to/tokens.txt \
+    /path/to/encoder.ncnn.param \
+    /path/to/encoder.ncnn.bin \
+    /path/to/decoder.ncnn.param \
+    /path/to/decoder.ncnn.bin \
+    /path/to/joiner.ncnn.param \
+    /path/to/joiner.ncnn.bin \
+    device_name \
+    [num_threads] [decode_method, can be greedy_search/modified_beam_search]
+
+Please refer to
+https://k2-fsa.github.io/sherpa/ncnn/pretrained_models/index.html
+for a list of pre-trained models to download.
+
+The device name specifies which microphone to use in case there are several
+on you system. You can use
+
+  arecord -l
+
+to find all available microphones on your computer. For instance, if it outputs
+
+**** List of CAPTURE Hardware Devices ****
+card 3: UACDemoV10 [UACDemoV1.0], device 0: USB Audio [USB Audio]
+  Subdevices: 1/1
+  Subdevice #0: subdevice #0
+
+and if you want to select card 3 and the device 0 on that card, please use:
+
+  hw:3,0
+
+as the device_name.
+)usage";
+
+    fprintf(stderr, "%s\n", usage);
+    fprintf(stderr, "argc, %d\n", argc);
+
+    return 0;
+  }
+
+  signal(SIGINT, Handler);
+
+  sherpa_ncnn::RecognizerConfig config;
+  config.model_config.tokens = argv[1];
+  config.model_config.encoder_param = argv[2];
+  config.model_config.encoder_bin = argv[3];
+  config.model_config.decoder_param = argv[4];
+  config.model_config.decoder_bin = argv[5];
+  config.model_config.joiner_param = argv[6];
+  config.model_config.joiner_bin = argv[7];
+
+  const char *device_name = argv[8];
+
+  int num_threads = 4;
+  if (argc >= 10 && atoi(argv[9]) > 0) {
+    num_threads = atoi(argv[9]);
+  }
+
+  config.model_config.encoder_opt.num_threads = num_threads;
+  config.model_config.decoder_opt.num_threads = num_threads;
+  config.model_config.joiner_opt.num_threads = num_threads;
+
+  sherpa_ncnn::DecoderConfig decoder_conf;
+  if (argc == 11) {
+    std::string method = argv[10];
+    if (method == "greedy_search" || method == "modified_beam_search") {
+      decoder_conf.method = method;
+    }
+  }
+
+  int32_t expected_sampling_rate = 16000;
+
+  config.enable_endpoint = true;
+
+  config.endpoint_config.rule1.min_trailing_silence = 2.4;
+  config.endpoint_config.rule2.min_trailing_silence = 1.2;
+  config.endpoint_config.rule3.min_utterance_length = 300;
+
+  config.feat_config.sampling_rate = expected_sampling_rate;
+  config.feat_config.feature_dim = 80;
+
+  fprintf(stderr, "%s\n", config.ToString().c_str());
+
+  sherpa_ncnn::Recognizer recognizer(config);
+
+  sherpa_ncnn::TinyAlsa tinyalsa(device_name);
+  fprintf(stderr, "Use recording device: %s\n", device_name);
+
+  if (tinyalsa.GetExpectedSampleRate() != expected_sampling_rate) {
+    fprintf(stderr, "sample rate: %d != %d\n", tinyalsa.GetExpectedSampleRate(),
+            expected_sampling_rate);
+    exit(-1);
+  }
+
+  auto s = recognizer.CreateStream();
+
+  int32_t chunk = 0.1 * tinyalsa.GetActualSampleRate();
+
+  std::string last_text;
+  int32_t segment_index = 0;
+  sherpa_ncnn::Display display;
+  while (!stop) {
+    const std::vector<float> samples = tinyalsa.Read(chunk);
+
+    s->AcceptWaveform(expected_sampling_rate, samples.data(), samples.size());
+    while (recognizer.IsReady(s.get())) {
+      recognizer.DecodeStream(s.get());
+    }
+
+    bool is_endpoint = recognizer.IsEndpoint(s.get());
+    auto text = recognizer.GetResult(s.get()).text;
+
+    if (!text.empty() && last_text != text) {
+      last_text = text;
+
+      std::transform(text.begin(), text.end(), text.begin(),
+                     [](auto c) { return std::tolower(c); });
+
+      display.Print(segment_index, text);
+    }
+
+    if (is_endpoint) {
+      if (!text.empty()) {
+        ++segment_index;
+      }
+
+      recognizer.Reset(s.get());
+    }
+  }
+
+  return 0;
+}

sherpa-ncnn/csrc/tinyalsa.cc

@@ -0,0 +1,88 @@
+/**
+ * Copyright (c)  2023  Xiaomi Corporation (authors: Fangjun Kuang)
+ *
+ * See LICENSE for clarification regarding multiple authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #ifdef SHERPA_NCNN_ENABLE_TINYALSA
+
+#include "sherpa-ncnn/csrc/tinyalsa.h"
+
+#include <algorithm>
+#include <string.h>
+
+#include "tinyalsa/asoundlib.h"
+
+namespace sherpa_ncnn {
+
+void ToFloat(const std::vector<int16_t> &in, int32_t num_channels,
+             std::vector<float> *out) {
+  out->resize(in.size() / num_channels);
+
+  int32_t n = in.size();
+  for (int32_t i = 0, k = 0; i < n; i += num_channels, ++k) {
+    (*out)[k] = in[i] / 32768.;
+  }
+}
+
+TinyAlsa::TinyAlsa(const char *device_name) {
+  unsigned int card = 0;
+  unsigned int device = 0;
+  int flags = PCM_IN;
+
+  struct pcm_config config;
+
+  memset(&config, 0, sizeof(config));
+  config.channels = 2;
+  config.rate = 48000;
+  config.format = PCM_FORMAT_S32_LE;
+  config.period_size = 1024;
+  config.period_count = 2;
+  config.start_threshold = 1024;
+  config.silence_threshold = 1024 * 2;
+  config.stop_threshold = 1024 * 2;
+
+  tinyalsa_pcm  = pcm_open(card, device, flags, &config);
+  if (tinyalsa_pcm == NULL) {
+      fprintf(stderr, "failed to allocate memory for PCM\n");
+  } else if (!pcm_is_ready(tinyalsa_pcm)) {
+      pcm_close(tinyalsa_pcm);
+      fprintf(stderr, "failed to open PCM\n");
+  }
+}
+
+TinyAlsa::~TinyAlsa() { pcm_close(tinyalsa_pcm); }
+
+const std::vector<float> &TinyAlsa::Read(int32_t num_samples) {
+  samples_.resize(num_samples * actual_channel_count_);
+
+  // count is in frames. Each frame contains actual_channel_count_ samples
+  int32_t count = pcm_readi(tinyalsa_pcm, samples_.data(), num_samples);
+
+  samples_.resize(count * actual_channel_count_);
+
+  ToFloat(samples_, actual_channel_count_, &samples1_);
+
+  if (!resampler_) {
+    return samples1_;
+  }
+
+  resampler_->Resample(samples1_.data(), samples_.size(), false, &samples2_);
+  return samples2_;
+}
+
+}  // namespace sherpa_ncnn
+
+// #endif