Add new podcast scanner and remove old scanner

server/Server.js

@@ -35,6 +35,7 @@ const AudioMetadataMangaer = require('./managers/AudioMetadataManager')
 const RssFeedManager = require('./managers/RssFeedManager')
 const CronManager = require('./managers/CronManager')
 const TaskManager = require('./managers/TaskManager')
+const LibraryScanner = require('./scanner/LibraryScanner')
 
 class Server {
   constructor(SOURCE, PORT, HOST, UID, GID, CONFIG_PATH, METADATA_PATH, ROUTER_BASE_PATH) {
@@ -76,7 +77,7 @@ class Server {
     this.rssFeedManager = new RssFeedManager()
 
     this.scanner = new Scanner(this.coverManager, this.taskManager)
-    this.cronManager = new CronManager(this.scanner, this.podcastManager)
+    this.cronManager = new CronManager(this.podcastManager)
 
     // Routers
     this.apiRouter = new ApiRouter(this)
@@ -92,6 +93,10 @@ class Server {
     this.auth.authMiddleware(req, res, next)
   }
 
+  cancelLibraryScan(libraryId) {
+    LibraryScanner.setCancelLibraryScan(libraryId)
+  }
+
   /**
    * Initialize database, backups, logs, rss feeds, cron jobs & watcher
    * Cleanup stale/invalid data
@@ -122,7 +127,6 @@ class Server {
       this.watcher.disabled = true
     } else {
       this.watcher.initWatcher(libraries)
-      this.watcher.on('files', this.filesChanged.bind(this))
     }
   }
 
@@ -241,11 +245,6 @@ class Server {
     res.sendStatus(200)
   }
 
-  async filesChanged(fileUpdates) {
-    Logger.info('[Server]', fileUpdates.length, 'Files Changed')
-    await this.scanner.scanFilesChanged(fileUpdates)
-  }
-
   /**
    * Remove user media progress for items that no longer exist & remove seriesHideFrom that no longer exist
    */

server/SocketAuthority.js

@@ -95,7 +95,7 @@ class SocketAuthority {
       socket.on('auth', (token) => this.authenticateSocket(socket, token))
 
       // Scanning
-      socket.on('cancel_scan', this.cancelScan.bind(this))
+      socket.on('cancel_scan', (libraryId) => this.cancelScan(libraryId))
 
       // Logs
       socket.on('set_log_listener', (level) => Logger.addSocketListener(socket, level))
@@ -209,7 +209,7 @@ class SocketAuthority {
 
   cancelScan(id) {
     Logger.debug('[SocketAuthority] Cancel scan', id)
-    this.Server.scanner.setCancelLibraryScan(id)
+    this.Server.cancelLibraryScan(id)
   }
 }
 module.exports = new SocketAuthority()

server/Watcher.js

@@ -1,20 +1,29 @@
 const EventEmitter = require('events')
 const Watcher = require('./libs/watcher/watcher')
 const Logger = require('./Logger')
+const LibraryScanner = require('./scanner/LibraryScanner')
 
 const { filePathToPOSIX } = require('./utils/fileUtils')
 
+/**
+ * @typedef PendingFileUpdate
+ * @property {string} path
+ * @property {string} relPath
+ * @property {string} folderId
+ * @property {string} type
+ */
 class FolderWatcher extends EventEmitter {
   constructor() {
     super()
-    this.paths = [] // Not used
-    this.pendingFiles = [] // Not used
 
+    /** @type {{id:string, name:string, folders:import('./objects/Folder')[], paths:string[], watcher:Watcher[]}[]} */
     this.libraryWatchers = []
+    /** @type {PendingFileUpdate[]} */
     this.pendingFileUpdates = []
     this.pendingDelay = 4000
     this.pendingTimeout = null
 
+    /** @type {string[]} */
     this.ignoreDirs = []
     this.disabled = false
   }
@@ -29,11 +38,12 @@ class FolderWatcher extends EventEmitter {
       return
     }
     Logger.info(`[Watcher] Initializing watcher for "${library.name}".`)
-    var folderPaths = library.folderPaths
+
+    const folderPaths = library.folderPaths
     folderPaths.forEach((fp) => {
       Logger.debug(`[Watcher] Init watcher for library folder path "${fp}"`)
     })
-    var watcher = new Watcher(folderPaths, {
+    const watcher = new Watcher(folderPaths, {
       ignored: /(^|[\/\\])\../, // ignore dotfiles
       renameDetection: true,
       renameTimeout: 2000,
@@ -144,6 +154,12 @@ class FolderWatcher extends EventEmitter {
     this.addFileUpdate(libraryId, pathTo, 'renamed')
   }
 
+  /**
+   * File update detected from watcher
+   * @param {string} libraryId 
+   * @param {string} path 
+   * @param {string} type 
+   */
   addFileUpdate(libraryId, path, type) {
     path = filePathToPOSIX(path)
     if (this.pendingFilePaths.includes(path)) return
@@ -163,9 +179,10 @@ class FolderWatcher extends EventEmitter {
     }
     const folderFullPath = filePathToPOSIX(folder.fullPath)
 
-    var relPath = path.replace(folderFullPath, '')
+    const relPath = path.replace(folderFullPath, '')
 
-    var hasDotPath = relPath.split('/').find(p => p.startsWith('.'))
+    // Ignore files/folders starting with "."
+    const hasDotPath = relPath.split('/').find(p => p.startsWith('.'))
     if (hasDotPath) {
       Logger.debug(`[Watcher] Ignoring dot path "${relPath}" | Piece "${hasDotPath}"`)
       return
@@ -184,7 +201,8 @@ class FolderWatcher extends EventEmitter {
     // Notify server of update after "pendingDelay"
     clearTimeout(this.pendingTimeout)
     this.pendingTimeout = setTimeout(() => {
-      this.emit('files', this.pendingFileUpdates)
+      // this.emit('files', this.pendingFileUpdates)
+      LibraryScanner.scanFilesChanged(this.pendingFileUpdates)
       this.pendingFileUpdates = []
     }, this.pendingDelay)
   }

server/controllers/LibraryController.js

@@ -978,12 +978,8 @@ class LibraryController {
       forceRescan: req.query.force == 1
     }
     res.sendStatus(200)
-    if (req.library.mediaType === 'podcast') {
-      // TODO: New library scanner for podcasts
-      await this.scanner.scan(req.library, options)
-    } else {
-      await LibraryScanner.scan(req.library, options)
-    }
+
+    await LibraryScanner.scan(req.library, options)
 
     await Database.resetLibraryIssuesFilterData(req.library.id)
     Logger.info('[LibraryController] Scan complete')

server/controllers/LibraryItemController.js

@@ -445,7 +445,12 @@ class LibraryItemController {
       return res.sendStatus(400)
     }
 
-    const libraryItems = req.body.libraryItemIds.map(lid => Database.getLibraryItem(lid)).filter(li => li)
+    const libraryItems = await Database.libraryItemModel.findAll({
+      where: {
+        id: req.body.libraryItemIds
+      },
+      attributes: ['id', 'libraryId', 'isFile']
+    })
     if (!libraryItems?.length) {
       return res.sendStatus(400)
     }
@@ -457,7 +462,7 @@ class LibraryItemController {
       if (libraryItem.isFile) {
         Logger.warn(`[LibraryItemController] Re-scanning file library items not yet supported`)
       } else {
-        await this.scanner.scanLibraryItemByRequest(libraryItem)
+        await LibraryItemScanner.scanLibraryItem(libraryItem.id)
       }
     }
 
@@ -476,14 +481,7 @@ class LibraryItemController {
       return res.sendStatus(500)
     }
 
-    let result = 0
-    if (req.libraryItem.isPodcast) {
-      // TODO: New library item scanner for podcast
-      result = await this.scanner.scanLibraryItemByRequest(req.libraryItem)
-    } else {
-      result = await LibraryItemScanner.scanLibraryItem(req.libraryItem.id)
-    }
-
+    const result = await LibraryItemScanner.scanLibraryItem(req.libraryItem.id)
     await Database.resetLibraryIssuesFilterData(req.libraryItem.libraryId)
     res.json({
       result: Object.keys(ScanResult).find(key => ScanResult[key] == result)

server/managers/CronManager.js

@@ -2,10 +2,10 @@ const Sequelize = require('sequelize')
 const cron = require('../libs/nodeCron')
 const Logger = require('../Logger')
 const Database = require('../Database')
+const LibraryScanner = require('../scanner/LibraryScanner')
 
 class CronManager {
-  constructor(scanner, podcastManager) {
-    this.scanner = scanner
+  constructor(podcastManager) {
     this.podcastManager = podcastManager
 
     this.libraryScanCrons = []
@@ -39,7 +39,7 @@ class CronManager {
     Logger.debug(`[CronManager] Init library scan cron for ${library.name} on schedule ${library.settings.autoScanCronExpression}`)
     const libScanCron = cron.schedule(library.settings.autoScanCronExpression, () => {
       Logger.debug(`[CronManager] Library scan cron executing for ${library.name}`)
-      this.scanner.scan(library)
+      LibraryScanner.scan(library)
     })
     this.libraryScanCrons.push({
       libraryId: library.id,

server/models/PodcastEpisode.js

@@ -39,7 +39,7 @@ class PodcastEpisode extends Model {
     this.enclosureType
     /** @type {Date} */
     this.publishedAt
-    /** @type {Object} */
+    /** @type {import('./Book').AudioFileObject} */
     this.audioFile
     /** @type {ChapterObject[]} */
     this.chapters

server/objects/files/AudioFile.js

@@ -6,6 +6,7 @@ class AudioFile {
   constructor(data) {
     this.index = null
     this.ino = null
+    /** @type {FileMetadata} */
     this.metadata = null
     this.addedAt = null
     this.updatedAt = null
@@ -27,6 +28,7 @@ class AudioFile {
     this.embeddedCoverArt = null
 
     // Tags scraped from the audio file
+    /** @type {AudioMetaTags} */
     this.metaTags = null
 
     this.manuallyVerified = false

server/objects/mediaTypes/Podcast.js

@@ -4,10 +4,6 @@ const PodcastMetadata = require('../metadata/PodcastMetadata')
 const { areEquivalent, copyValue, cleanStringForSearch } = require('../../utils/index')
 const abmetadataGenerator = require('../../utils/generators/abmetadataGenerator')
 const { readTextFile, filePathToPOSIX } = require('../../utils/fileUtils')
-const { createNewSortInstance } = require('../../libs/fastSort')
-const naturalSort = createNewSortInstance({
-  comparer: new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare
-})
 
 class Podcast {
   constructor(podcast) {

server/scanner/BookScanner.js

@@ -360,7 +360,7 @@ class BookScanner {
    * @param {import('./LibraryItemScanData')} libraryItemData 
    * @param {import('../models/Library').LibrarySettingsObject} librarySettings
    * @param {import('./LibraryScan')} libraryScan 
-   * @returns {import('../models/LibraryItem')}
+   * @returns {Promise<import('../models/LibraryItem')>}
    */
   async scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan) {
     // Scan audio files found
@@ -439,6 +439,7 @@ class BookScanner {
     libraryItemObj.id = uuidv4() // Generate library item id ahead of time to use for saving extracted cover image
     libraryItemObj.isMissing = false
     libraryItemObj.isInvalid = false
+    libraryItemObj.extraData = {}
 
     // Set isSupplementary flag on ebook library files
     for (const libraryFile of libraryItemObj.libraryFiles) {
@@ -739,6 +740,8 @@ class BookScanner {
       bookMetadata.coverPath = coverMatch?.metadata.path || libraryItemData.imageLibraryFiles[0].metadata.path
     }
 
+    bookMetadata.titleIgnorePrefix = getTitleIgnorePrefix(bookMetadata.title)
+
     return bookMetadata
   }
 

server/scanner/LibraryItemScanner.js

@@ -8,6 +8,7 @@ const Database = require('../Database')
 const LibraryScan = require('./LibraryScan')
 const LibraryItemScanData = require('./LibraryItemScanData')
 const BookScanner = require('./BookScanner')
+const PodcastScanner = require('./PodcastScanner')
 const ScanLogger = require('./ScanLogger')
 const LibraryItem = require('../models/LibraryItem')
 const LibraryFile = require('../objects/files/LibraryFile')
@@ -158,13 +159,13 @@ class LibraryItemScanner {
    * @returns {Promise<LibraryItem>}
    */
   async rescanLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
+    let newLibraryItem = null
     if (existingLibraryItem.mediaType === 'book') {
-      const libraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
-      return libraryItem
+      newLibraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
     } else {
-      // TODO: Scan updated podcast
-      return null
+      newLibraryItem = await PodcastScanner.rescanExistingPodcastLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
     }
+    return newLibraryItem
   }
 
   /**
@@ -175,16 +176,34 @@ class LibraryItemScanner {
    * @returns {Promise<LibraryItem>}
    */
   async scanNewLibraryItem(libraryItemData, librarySettings, libraryScan) {
-    if (libraryScan.libraryMediaType === 'book') {
-      const newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan)
-      if (newLibraryItem) {
-        libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`)
-      }
-      return newLibraryItem
+    let newLibraryItem = null
+    if (libraryItemData.mediaType === 'book') {
+      newLibraryItem = await BookScanner.scanNewBookLibraryItem(libraryItemData, librarySettings, libraryScan)
     } else {
-      // TODO: Scan new podcast
-      return null
+      newLibraryItem = await PodcastScanner.scanNewPodcastLibraryItem(libraryItemData, librarySettings, libraryScan)
+    }
+    if (newLibraryItem) {
+      libraryScan.addLog(LogLevel.INFO, `Created new library item "${newLibraryItem.relPath}"`)
     }
+    return newLibraryItem
+  }
+
+  /**
+   * Scan library item folder coming from Watcher
+   * @param {string} libraryItemPath 
+   * @param {import('../models/Library')} library 
+   * @param {import('../models/LibraryFolder')} folder 
+   * @param {boolean} isSingleMediaItem 
+   * @returns {Promise<LibraryItem>} ScanResult
+   */
+  async scanPotentialNewLibraryItem(libraryItemPath, library, folder, isSingleMediaItem) {
+    const libraryItemScanData = await this.getLibraryItemScanData(libraryItemPath, library, folder, isSingleMediaItem)
+
+    const scanLogger = new ScanLogger()
+    scanLogger.verbose = true
+    scanLogger.setData('libraryItem', libraryItemScanData.relPath)
+
+    return this.scanNewLibraryItem(libraryItemScanData, library.settings, scanLogger)
   }
 }
 module.exports = new LibraryItemScanner()