Support accent-insensitive search using SQLean unicode sqlite3 extension

.gitignore

@@ -15,8 +15,9 @@
 /.nyc_output/
 /ffmpeg*
 /ffprobe*
+/unicode*
 
 sw.*
 .DS_STORE
 .idea/*
-tailwind.compiled.css
+tailwind.compiled.css

build/debian/DEBIAN/preinst

@@ -2,7 +2,6 @@
 set -e
 set -o pipefail
 
-FFMPEG_INSTALL_DIR="/usr/lib/audiobookshelf-ffmpeg"
 DEFAULT_DATA_DIR="/usr/share/audiobookshelf"
 CONFIG_PATH="/etc/default/audiobookshelf"
 DEFAULT_PORT=13378
@@ -46,25 +45,6 @@ add_group() {
   fi
 }
 
-install_ffmpeg() {
-  echo "Starting FFMPEG Install"
-
-  WGET="wget https://johnvansickle.com/ffmpeg/builds/ffmpeg-git-amd64-static.tar.xz --output-document=ffmpeg-git-amd64-static.tar.xz"
-
-  if ! cd "$FFMPEG_INSTALL_DIR"; then
-    echo "Creating ffmpeg install dir at $FFMPEG_INSTALL_DIR"
-    mkdir "$FFMPEG_INSTALL_DIR"
-    chown -R 'audiobookshelf:audiobookshelf' "$FFMPEG_INSTALL_DIR"
-    cd "$FFMPEG_INSTALL_DIR"
-  fi
-
-  $WGET
-  tar xvf ffmpeg-git-amd64-static.tar.xz --strip-components=1 --no-same-owner
-  rm ffmpeg-git-amd64-static.tar.xz
-
-  echo "Good to go on Ffmpeg... hopefully"
-}
-
 setup_config() {
   if [ -f "$CONFIG_PATH" ]; then
     echo "Existing config found."
@@ -83,8 +63,6 @@ setup_config() {
 
     config_text="METADATA_PATH=$DEFAULT_DATA_DIR/metadata
 CONFIG_PATH=$DEFAULT_DATA_DIR/config
-FFMPEG_PATH=$FFMPEG_INSTALL_DIR/ffmpeg
-FFPROBE_PATH=$FFMPEG_INSTALL_DIR/ffprobe
 PORT=$DEFAULT_PORT
 HOST=$DEFAULT_HOST"
 
@@ -101,5 +79,3 @@ add_group 'audiobookshelf' ''
 add_user 'audiobookshelf' '' 'audiobookshelf' 'audiobookshelf user-daemon' '/bin/false'
 
 setup_config
-
-install_ffmpeg

server/Database.js

@@ -207,6 +207,7 @@ class Database {
 
     try {
       await this.sequelize.authenticate()
+      await this.loadExtensions([process.env.SQLEAN_UNICODE_PATH])
       Logger.info(`[Database] Db connection was successful`)
       return true
     } catch (error) {
@@ -215,6 +216,34 @@ class Database {
     }
   }
 
+  /**
+   *
+   * @param {string[]} extensions paths to extension binaries
+   */
+  async loadExtensions(extensions) {
+    // This is a hack to get the db connection for loading extensions.
+    // The proper way would be to use the 'afterConnect' hook, but that hook is never called for sqlite due to a bug in sequelize.
+    // See https://github.com/sequelize/sequelize/issues/12487
+    // This is not a public API and may break in the future.
+    const db = await this.sequelize.dialect.connectionManager.getConnection()
+    if (typeof db?.loadExtension !== 'function') throw new Error('Failed to get db connection for loading extensions')
+
+    for (const ext of extensions) {
+      Logger.info(`[Database] Loading extension ${ext}`)
+      await new Promise((resolve, reject) => {
+        db.loadExtension(ext, (err) => {
+          if (err) {
+            Logger.error(`[Database] Failed to load extension ${ext}`, err)
+            reject(err)
+            return
+          }
+          Logger.info(`[Database] Successfully loaded extension ${ext}`)
+          resolve()
+        })
+      })
+    }
+  }
+
   /**
    * Disconnect from db
    */
@@ -801,6 +830,39 @@ class Database {
       Logger.warn(`Removed ${badSessionsRemoved} sessions that were 3 seconds or less`)
     }
   }
+
+  /**
+   *
+   * @param {string} value
+   * @returns {string}
+   */
+  normalize(value) {
+    return `lower(unaccent(${value}))`
+  }
+
+  /**
+   *
+   * @param {string} query
+   * @returns {Promise<string>}
+   */
+  async getNormalizedQuery(query) {
+    const escapedQuery = this.sequelize.escape(query)
+    const normalizedQuery = this.normalize(escapedQuery)
+    const normalizedQueryResult = await this.sequelize.query(`SELECT ${normalizedQuery} as normalized_query`)
+    return normalizedQueryResult[0][0].normalized_query
+  }
+
+  /**
+   *
+   * @param {string} column
+   * @param {string} normalizedQuery
+   * @returns {string}
+   */
+  matchExpression(column, normalizedQuery) {
+    const normalizedPattern = this.sequelize.escape(`%${normalizedQuery}%`)
+    const normalizedColumn = this.normalize(column)
+    return `${normalizedColumn} LIKE ${normalizedPattern}`
+  }
 }
 
 module.exports = new Database()

server/Server.js

@@ -108,6 +108,8 @@ class Server {
 
     await this.playbackSessionManager.removeOrphanStreams()
 
+    await this.binaryManager.init()
+
     await Database.init(false)
 
     await Logger.logManager.init()
@@ -128,11 +130,6 @@ class Server {
     await this.cronManager.init(libraries)
     this.apiCacheManager.init()
 
-    // Download ffmpeg & ffprobe if not found (Currently only in use for Windows installs)
-    if (global.isWin || Logger.isDev) {
-      await this.binaryManager.init()
-    }
-
     if (Database.serverSettings.scannerDisableWatcher) {
       Logger.info(`[Server] Watcher is disabled`)
       this.watcher.disabled = true