Update match all books to load items from DB, remove library items lo…

…ading to memory on init

server/Database.js

@@ -15,9 +15,6 @@ class Database {
     this.isNew = false // New absdatabase.sqlite created
     this.hasRootUser = false // Used to show initialization page in web ui
 
-    // Temporarily using format of old DB
-    // TODO: below data should be loaded from the DB as needed
-    this.libraryItems = []
     this.settings = []
 
     // Cached library filter data
@@ -255,8 +252,6 @@ class Database {
       await dbMigration.migrate(this.models)
     }
 
-    const startTime = Date.now()
-
     const settingsData = await this.models.setting.getOldSettings()
     this.settings = settingsData.settings
     this.emailSettings = settingsData.emailSettings
@@ -272,16 +267,9 @@ class Database {
       await dbMigration.migrationPatch2(this)
     }
 
-    Logger.info(`[Database] Loading db data...`)
-
-    this.libraryItems = await this.models.libraryItem.loadAllLibraryItems()
-    Logger.info(`[Database] Loaded ${this.libraryItems.length} library items`)
-
     // Set if root user has been created
     this.hasRootUser = await this.models.user.getHasRootUser()
 
-    Logger.info(`[Database] Db data loaded in ${((Date.now() - startTime) / 1000).toFixed(2)}s`)
-
     if (packageJson.version !== this.serverSettings.version) {
       Logger.info(`[Database] Server upgrade detected from ${this.serverSettings.version} to ${packageJson.version}`)
       this.serverSettings.version = packageJson.version
@@ -380,20 +368,10 @@ class Database {
     return this.models.playlistMediaItem.bulkCreate(playlistMediaItems)
   }
 
-  getLibraryItem(libraryItemId) {
-    if (!this.sequelize || !libraryItemId) return false
-
-    // Temp support for old library item ids from mobile
-    if (libraryItemId.startsWith('li_')) return this.libraryItems.find(li => li.oldLibraryItemId === libraryItemId)
-
-    return this.libraryItems.find(li => li.id === libraryItemId)
-  }
-
   async createLibraryItem(oldLibraryItem) {
     if (!this.sequelize) return false
     await oldLibraryItem.saveMetadata()
     await this.models.libraryItem.fullCreateFromOld(oldLibraryItem)
-    this.libraryItems.push(oldLibraryItem)
   }
 
   async updateLibraryItem(oldLibraryItem) {
@@ -420,14 +398,12 @@ class Database {
     for (const oldLibraryItem of oldLibraryItems) {
       await oldLibraryItem.saveMetadata()
       await this.models.libraryItem.fullCreateFromOld(oldLibraryItem)
-      this.libraryItems.push(oldLibraryItem)
     }
   }
 
   async removeLibraryItem(libraryItemId) {
     if (!this.sequelize) return false
     await this.models.libraryItem.removeById(libraryItemId)
-    this.libraryItems = this.libraryItems.filter(li => li.id !== libraryItemId)
   }
 
   async createFeed(oldFeed) {

server/Server.js

@@ -76,7 +76,7 @@ class Server {
     this.audioMetadataManager = new AudioMetadataMangaer(this.taskManager)
     this.rssFeedManager = new RssFeedManager()
 
-    this.scanner = new Scanner(this.coverManager, this.taskManager)
+    this.scanner = new Scanner(this.coverManager)
     this.cronManager = new CronManager(this.podcastManager)
 
     // Routers

server/controllers/AuthorController.js

@@ -5,6 +5,7 @@ const { createNewSortInstance } = require('../libs/fastSort')
 const Logger = require('../Logger')
 const SocketAuthority = require('../SocketAuthority')
 const Database = require('../Database')
+const AuthorFinder = require('../finders/AuthorFinder')
 
 const { reqSupportsWebp } = require('../utils/index')
 
@@ -70,7 +71,7 @@ class AuthorController {
         await this.cacheManager.purgeImageCache(req.author.id) // Purge cache
         await this.coverManager.removeFile(req.author.imagePath)
       } else if (payload.imagePath.startsWith('http')) { // Check if image path is a url
-        const imageData = await this.authorFinder.saveAuthorImage(req.author.id, payload.imagePath)
+        const imageData = await AuthorFinder.saveAuthorImage(req.author.id, payload.imagePath)
         if (imageData) {
           if (req.author.imagePath) {
             await this.cacheManager.purgeImageCache(req.author.id) // Purge cache
@@ -168,9 +169,9 @@ class AuthorController {
     let authorData = null
     const region = req.body.region || 'us'
     if (req.body.asin) {
-      authorData = await this.authorFinder.findAuthorByASIN(req.body.asin, region)
+      authorData = await AuthorFinder.findAuthorByASIN(req.body.asin, region)
     } else {
-      authorData = await this.authorFinder.findAuthorByName(req.body.q, region)
+      authorData = await AuthorFinder.findAuthorByName(req.body.q, region)
     }
     if (!authorData) {
       return res.status(404).send('Author not found')
@@ -187,7 +188,7 @@ class AuthorController {
     if (authorData.image && (!req.author.imagePath || hasUpdates)) {
       this.cacheManager.purgeImageCache(req.author.id)
 
-      const imageData = await this.authorFinder.saveAuthorImage(req.author.id, authorData.image)
+      const imageData = await AuthorFinder.saveAuthorImage(req.author.id, authorData.image)
       if (imageData) {
         req.author.imagePath = imageData.path
         hasUpdates = true

server/controllers/EmailController.js

@@ -54,7 +54,7 @@ class EmailController {
   async sendEBookToDevice(req, res) {
     Logger.debug(`[EmailController] Send ebook to device request for libraryItemId=${req.body.libraryItemId}, deviceName=${req.body.deviceName}`)
 
-    const libraryItem = Database.getLibraryItem(req.body.libraryItemId)
+    const libraryItem = await Database.libraryItemModel.getOldById(req.body.libraryItemId)
     if (!libraryItem) {
       return res.status(404).send('Library item not found')
     }

server/controllers/LibraryItemController.js

@@ -411,7 +411,9 @@ class LibraryItemController {
       return res.sendStatus(400)
     }
 
-    const libraryItems = req.body.libraryItemIds.map(lid => Database.getLibraryItem(lid)).filter(li => li)
+    const libraryItems = await Database.libraryItemModel.getAllOldLibraryItems({
+      id: req.body.libraryItemIds
+    })
     if (!libraryItems?.length) {
       return res.sendStatus(400)
     }

server/controllers/MeController.js

@@ -193,7 +193,8 @@ class MeController {
         Logger.error(`[MeController] syncLocalMediaProgress invalid local media progress object`, localProgress)
         continue
       }
-      const libraryItem = Database.getLibraryItem(localProgress.libraryItemId)
+
+      const libraryItem = await Database.libraryItemModel.getOldById(localProgress.libraryItemId)
       if (!libraryItem) {
         Logger.error(`[MeController] syncLocalMediaProgress invalid local media progress object no library item`, localProgress)
         continue
@@ -245,13 +246,15 @@ class MeController {
   }
 
   // GET: api/me/items-in-progress
-  getAllLibraryItemsInProgress(req, res) {
+  async getAllLibraryItemsInProgress(req, res) {
     const limit = !isNaN(req.query.limit) ? Number(req.query.limit) || 25 : 25
 
     let itemsInProgress = []
+    // TODO: More efficient to do this in a single query
     for (const mediaProgress of req.user.mediaProgress) {
       if (!mediaProgress.isFinished && (mediaProgress.progress > 0 || mediaProgress.ebookProgress > 0)) {
-        const libraryItem = Database.getLibraryItem(mediaProgress.libraryItemId)
+
+        const libraryItem = await Database.libraryItemModel.getOldById(mediaProgress.libraryItemId)
         if (libraryItem) {
           if (mediaProgress.episodeId && libraryItem.mediaType === 'podcast') {
             const episode = libraryItem.media.episodes.find(ep => ep.id === mediaProgress.episodeId)

server/controllers/SearchController.js

@@ -1,4 +1,8 @@
 const Logger = require("../Logger")
+const BookFinder = require('../finders/BookFinder')
+const PodcastFinder = require('../finders/PodcastFinder')
+const AuthorFinder = require('../finders/AuthorFinder')
+const MusicFinder = require('../finders/MusicFinder')
 
 class SearchController {
   constructor() { }
@@ -7,7 +11,7 @@ class SearchController {
     const provider = req.query.provider || 'google'
     const title = req.query.title || ''
     const author = req.query.author || ''
-    const results = await this.bookFinder.search(provider, title, author)
+    const results = await BookFinder.search(provider, title, author)
     res.json(results)
   }
 
@@ -21,37 +25,37 @@ class SearchController {
     }
 
     let results = null
-    if (podcast) results = await this.podcastFinder.findCovers(query.title)
-    else results = await this.bookFinder.findCovers(query.provider || 'google', query.title, query.author || null)
+    if (podcast) results = await PodcastFinder.findCovers(query.title)
+    else results = await BookFinder.findCovers(query.provider || 'google', query.title, query.author || null)
     res.json({
       results
     })
   }
 
   async findPodcasts(req, res) {
     const term = req.query.term
-    const results = await this.podcastFinder.search(term)
+    const results = await PodcastFinder.search(term)
     res.json(results)
   }
 
   async findAuthor(req, res) {
     const query = req.query.q
-    const author = await this.authorFinder.findAuthorByName(query)
+    const author = await AuthorFinder.findAuthorByName(query)
     res.json(author)
   }
 
   async findChapters(req, res) {
     const asin = req.query.asin
     const region = (req.query.region || 'us').toLowerCase()
-    const chapterData = await this.bookFinder.findChapters(asin, region)
+    const chapterData = await BookFinder.findChapters(asin, region)
     if (!chapterData) {
       return res.json({ error: 'Chapters not found' })
     }
     res.json(chapterData)
   }
 
   async findMusicTrack(req, res) {
-    const tracks = await this.musicFinder.searchTrack(req.query || {})
+    const tracks = await MusicFinder.searchTrack(req.query || {})
     res.json({
       tracks
     })

server/controllers/SessionController.js

@@ -62,9 +62,9 @@ class SessionController {
     })
   }
 
-  getOpenSession(req, res) {
-    var libraryItem = Database.getLibraryItem(req.session.libraryItemId)
-    var sessionForClient = req.session.toJSONForClient(libraryItem)
+  async getOpenSession(req, res) {
+    const libraryItem = await Database.libraryItemModel.getOldById(req.session.libraryItemId)
+    const sessionForClient = req.session.toJSONForClient(libraryItem)
     res.json(sessionForClient)
   }
 

server/controllers/ToolsController.js

@@ -66,7 +66,7 @@ class ToolsController {
 
     const libraryItems = []
     for (const libraryItemId of libraryItemIds) {
-      const libraryItem = Database.getLibraryItem(libraryItemId)
+      const libraryItem = await Database.libraryItemModel.getOldById(libraryItemId)
       if (!libraryItem) {
         Logger.error(`[ToolsController] Batch embed metadata library item (${libraryItemId}) not found`)
         return res.sendStatus(404)

server/finders/AuthorFinder.js

@@ -8,8 +8,6 @@ const filePerms = require('../utils/filePerms')
 
 class AuthorFinder {
   constructor() {
-    this.AuthorPath = Path.join(global.MetadataPath, 'authors')
-
     this.audnexus = new Audnexus()
   }
 
@@ -37,7 +35,7 @@ class AuthorFinder {
   }
 
   async saveAuthorImage(authorId, url) {
-    var authorDir = this.AuthorPath
+    var authorDir = Path.join(global.MetadataPath, 'authors')
     var relAuthorDir = Path.posix.join('/metadata', 'authors')
 
     if (!await fs.pathExists(authorDir)) {
@@ -61,4 +59,4 @@ class AuthorFinder {
     }
   }
 }
-module.exports = AuthorFinder
+module.exports = new AuthorFinder()

server/finders/BookFinder.js

@@ -253,4 +253,4 @@ class BookFinder {
     return this.audnexus.getChaptersByASIN(asin, region)
   }
 }
-module.exports = BookFinder
+module.exports = new BookFinder()

server/finders/MusicFinder.js

@@ -9,4 +9,4 @@ class MusicFinder {
     return this.musicBrainz.searchTrack(options)
   }
 }
-module.exports = MusicFinder
+module.exports = new MusicFinder()

server/finders/PodcastFinder.js

@@ -22,4 +22,4 @@ class PodcastFinder {
     return results.map(r => r.cover).filter(r => r)
   }
 }
-module.exports = PodcastFinder
+module.exports = new PodcastFinder()

server/managers/PlaybackSessionManager.js

@@ -93,7 +93,7 @@ class PlaybackSessionManager {
   }
 
   async syncLocalSession(user, sessionJson, deviceInfo) {
-    const libraryItem = Database.getLibraryItem(sessionJson.libraryItemId)
+    const libraryItem = await Database.libraryItemModel.getOldById(sessionJson.libraryItemId)
     const episode = (sessionJson.episodeId && libraryItem && libraryItem.isPodcast) ? libraryItem.media.getEpisode(sessionJson.episodeId) : null
     if (!libraryItem || (libraryItem.isPodcast && !episode)) {
       Logger.error(`[PlaybackSessionManager] syncLocalSession: Media item not found for session "${sessionJson.displayTitle}" (${sessionJson.id})`)

server/managers/RssFeedManager.js

@@ -87,7 +87,7 @@ class RssFeedManager {
 
     // Check if feed needs to be updated
     if (feed.entityType === 'libraryItem') {
-      const libraryItem = Database.getLibraryItem(feed.entityId)
+      const libraryItem = await Database.libraryItemModel.getOldById(feed.entityId)
 
       let mostRecentlyUpdatedAt = libraryItem.updatedAt
       if (libraryItem.isPodcast) {

server/models/LibraryItem.js

@@ -118,12 +118,13 @@ class LibraryItem extends Model {
    * @param {number} limit
    * @returns {Promise<Model<LibraryItem>[]>} LibraryItem
    */
-  static getLibraryItemsIncrement(offset, limit) {
+  static getLibraryItemsIncrement(offset, limit, where = null) {
     return this.findAll({
       benchmark: true,
       logging: (sql, timeMs) => {
         console.log(`[Query] Elapsed ${timeMs}ms.`)
       },
+      where,
       include: [
         {
           model: this.sequelize.models.book,

server/routers/ApiRouter.js

@@ -29,11 +29,6 @@ const ToolsController = require('../controllers/ToolsController')
 const RSSFeedController = require('../controllers/RSSFeedController')
 const MiscController = require('../controllers/MiscController')
 
-const BookFinder = require('../finders/BookFinder')
-const AuthorFinder = require('../finders/AuthorFinder')
-const PodcastFinder = require('../finders/PodcastFinder')
-const MusicFinder = require('../finders/MusicFinder')
-
 const Author = require('../objects/entities/Author')
 const Series = require('../objects/entities/Series')
 
@@ -55,11 +50,6 @@ class ApiRouter {
     this.emailManager = Server.emailManager
     this.taskManager = Server.taskManager
 
-    this.bookFinder = new BookFinder()
-    this.authorFinder = new AuthorFinder()
-    this.podcastFinder = new PodcastFinder()
-    this.musicFinder = new MusicFinder()
-
     this.router = express()
     this.router.disable('x-powered-by')
     this.init()

server/scanner/BookScanner.js

@@ -16,6 +16,7 @@ const CoverManager = require('../managers/CoverManager')
 const LibraryFile = require('../objects/files/LibraryFile')
 const SocketAuthority = require('../SocketAuthority')
 const fsExtra = require("../libs/fsExtra")
+// const BookFinder = require('../finders/BookFinder')
 
 /**
  * Metadata for books pulled from files
@@ -1049,5 +1050,32 @@ class BookScanner {
       scanLogger.addLog(LogLevel.INFO, `Removed ${bookSeriesToRemove.length} series`)
     }
   }
+
+  // async searchForCover(libraryItem, libraryScan = null) {
+  //   const options = {
+  //     titleDistance: 2,
+  //     authorDistance: 2
+  //   }
+  //   const scannerCoverProvider = Database.serverSettings.scannerCoverProvider
+  //   const results = await BookFinder.findCovers(scannerCoverProvider, libraryItem.media.metadata.title, libraryItem.media.metadata.authorName, options)
+  //   if (results.length) {
+  //     if (libraryScan) libraryScan.addLog(LogLevel.DEBUG, `Found best cover for "${libraryItem.media.metadata.title}"`)
+  //     else Logger.debug(`[Scanner] Found best cover for "${libraryItem.media.metadata.title}"`)
+
+  //     // If the first cover result fails, attempt to download the second
+  //     for (let i = 0; i < results.length && i < 2; i++) {
+
+  //       // Downloads and updates the book cover
+  //       const result = await this.coverManager.downloadCoverFromUrl(libraryItem, results[i])
+
+  //       if (result.error) {
+  //         Logger.error(`[Scanner] Failed to download cover from url "${results[i]}" | Attempt ${i + 1}`, result.error)
+  //       } else {
+  //         return true
+  //       }
+  //     }
+  //   }
+  //   return false
+  // }
 }
 module.exports = new BookScanner()