Skip to content
This repository has been archived by the owner on Aug 27, 2023. It is now read-only.

Commit

Permalink
fix: improve db operations to avoid document conflicts
Browse files Browse the repository at this point in the history
  • Loading branch information
@jstarpl
jstarpl committed Mar 29, 2019
1 parent 8aaebf7 commit 2f752eb
Show file tree
Hide file tree
Showing 5 changed files with 56 additions and 51 deletions.
2 changes: 1 addition & 1 deletion package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
"cors": "^2.8.5",
"express": "^4.16.4",
"express-pouchdb": "^4.1.0",
"lodash": "^4.17.11",
"mkdirp-promise": "^5.0.1",
"moment": "^2.24.0",
"nconf": "^0.10.0",
Expand Down
43 changes: 19 additions & 24 deletions src/app.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ const cors = require('cors')
const PouchDB = require('pouchdb-node')
const util = require('util')
const path = require('path')
const _ = require('lodash')
const { generateInfo, generateThumb, generateAdvancedInfo, scanFile, lookForFile } = require('./scanner')
const { generatePreview } = require('./previews')
const recursiveReadDir = require('recursive-readdir')
Expand Down Expand Up @@ -169,7 +170,10 @@ module.exports = function ({ db, config, logger }) {
switch (name) {
case 'MEDIA INFO':
return generateInfo(config, doc)
.then(() => {
.then((mod) => {
return Promise.all([db.get(doc._id), mod])
}).then((arg) => {
const doc = _.merge(arg[0], arg[1])
return db.put(doc)
})
.then(() => {
Expand All @@ -179,13 +183,19 @@ module.exports = function ({ db, config, logger }) {
case 'METADATA':
return generateAdvancedInfo(config, doc)
.then((mediainfo) => {
doc.mediainfo = mediainfo
console.log(JSON.stringify(doc))
return Promise.all([db.get(doc._id), mediainfo])
}).then((arg) => {
const doc = arg[0]
doc.mediainfo = _.merge(doc.mediainfo, arg[1])
return db.put(doc)
})
case 'THUMBNAIL GENERATE':
return generateThumb(config, doc)
.then(() => {
.then((mod) => {
return Promise.all([db.get(doc._id), mod])
})
.then((arg) => {
const doc = _.merge(arg[0], arg[1])
return db.put(doc)
})
case 'PREVIEW GENERATE':
Expand Down Expand Up @@ -349,31 +359,16 @@ module.exports = function ({ db, config, logger }) {
* Start media scan of file
*/
let ongoingMediaMetadataScans = {}
app.post('/metadata/scanAsync/:fileName', wrap(async (req, res) => {
logger.info(`Looking for file "${req.params.fileName}"...`)
const stat = await lookForFile(req.params.fileName, config)

if (stat === false) {
res.send(`404 FILE NOT FOUND\r\n`)
return
}

const mediaId = req.params.fileName
.replace(/\.[^/.]+$/, '')
.replace(/\\+/g, '/')
.toUpperCase()

metaGenerate(res, mediaId, ongoingMediaMetadataScans, 'METADATA', stat)
app.post('/metadata/generateAsync/:id', wrap(async (req, res) => {
let mediaId = req.params.id.toUpperCase()
metaGenerate(res, mediaId, ongoingMediaMetadataScans, 'METADATA')
}))

/**
* Get status of a media scan
*/
app.get('/metadata/scanAsync/:fileName', wrap(async (req, res) => {
const mediaId = req.params.fileName
.replace(/\.[^/.]+$/, '')
.replace(/\\+/g, '/')
.toUpperCase()
app.get('/metadata/generateAsync/:id', wrap(async (req, res) => {
let mediaId = req.params.id.toUpperCase()
metaStatus(mediaId, 'METADATA', ongoingMediaMetadataScans, req, res)
}))

Expand Down
15 changes: 11 additions & 4 deletions src/previews.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ const util = require('util')
const mkdirp = require('mkdirp-promise')
const fs = require('fs')
const path = require('path')
const _ = require('lodash')
const { fileExists } = require('./util')
const { getManualMode } = require('./manual')
const { ProcessLimiter } = require('./processLimiter')
Expand Down Expand Up @@ -71,15 +72,21 @@ async function generatePreview (db, config, logger, mediaId) {
})

const previewStat = await statAsync(tmpPath)
doc.previewSize = previewStat.size
doc.previewTime = doc.mediaTime
doc.previewPath = destPath
const modifier = {}
modifier.previewSize = previewStat.size
modifier.previewTime = doc.mediaTime
modifier.previewPath = destPath

await renameAsync(tmpPath, destPath)

await db.put(doc)
let updateDoc = await db.get(mediaId)
updateDoc = _.merge(updateDoc, modifier)

db.put(updateDoc)

mediaLogger.info('Finished preview generation')

return modifier
} catch (err) {
logger.error({ name: 'generatePreview', err })
logger.error(err.stack)
Expand Down
46 changes: 24 additions & 22 deletions src/scanner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const { getId, fileExists } = require('./util')
const moment = require('moment')
const { getManualMode } = require('./manual')
const { crossPlatformKillProcessIfValid } = require('./processHandler')
const _ = require('lodash')
const statAsync = util.promisify(fs.stat)
const unlinkAsync = util.promisify(fs.unlink)
const readFileAsync = util.promisify(fs.readFile)
Expand Down Expand Up @@ -93,12 +94,10 @@ async function scanFile (db, config, logger, mediaPath, mediaId, mediaStat, gene
if (!mediaId || mediaStat.isDirectory()) {
return
}
if (!filesToScan[mediaId]) {
filesToScan[mediaId] = {
db, config, logger, mediaPath, mediaId, mediaStat, generateInfoWhenFound
}
filesToScan[mediaId] = {
db, config, logger, mediaPath, mediaId, mediaStat, generateInfoWhenFound
}
if (isCurrentlyScanning) {
if (!getManualMode() && isCurrentlyScanning) { // if MS is in manualMode, then
return
}
isCurrentlyScanning = true
Expand Down Expand Up @@ -144,7 +143,7 @@ async function scanFile (db, config, logger, mediaPath, mediaId, mediaStat, gene
mediaLogger.error({ err }, 'Thumbnail Failed')
})
])
} else if (getManualMode() && generateInfoWhenFound) {
} else if (getManualMode() && generateInfoWhenFound) { // Check if basic file probe should be run in manualMode
await generateInfo(config, doc).catch(err => {
mediaLogger.error({ err }, 'Info Failed')
})
Expand Down Expand Up @@ -195,23 +194,27 @@ async function generateThumb (config, doc) {
})
})

const modifier = {}

const thumbStat = await statAsync(tmpPath)
doc.thumbSize = thumbStat.size
doc.thumbTime = thumbStat.mtime.getTime()
doc.tinf = [
modifier.thumbSize = thumbStat.size
modifier.thumbTime = thumbStat.mtime.getTime()
modifier.tinf = [
`"${getId(config.paths.media, doc.mediaPath)}"`,
moment(doc.thumbTime).format('YYYYMMDDTHHmmss'),
// TODO (fix) Binary or base64 size?
doc.thumbSize
].join(' ') + '\r\n'

doc._attachments = {
modifier._attachments = {
'thumb.png': {
content_type: 'image/png',
data: (await readFileAsync(tmpPath))
}
}
await unlinkAsync(tmpPath)
_.merge(doc, modifier)
return modifier
}
let runningffprobeProcess = null
async function generateInfo (config, doc) {
Expand Down Expand Up @@ -245,15 +248,20 @@ async function generateInfo (config, doc) {
})
})

doc.cinf = generateCinf(config, doc, json)
const modifier = {}

modifier.cinf = generateCinf(config, doc, json)

if (config.metadata !== null) {
doc.mediainfo = await generateBasicMetadata(config, doc, json)
modifier.mediainfo = await generateBasicMetadata(config, doc, json)

if (!getManualMode()) {
doc.mediainfo = await generateAdvancedMetadata(config, doc)
_.merge(modifier.mediainfo, await generateAdvancedMetadata(config, doc))
}
}

_.merge(doc, modifier)
return modifier
}

function generateCinf (config, doc, json) {
Expand Down Expand Up @@ -672,27 +680,21 @@ async function generateAdvancedMetadata (config, doc) {
size: doc.mediaSize,
time: doc.mediaTime,

type: (doc.mediainfo || {}).type,
field_order: fieldOrder,
scenes: metadata.scenes,
freezes: metadata.freezes,
blacks: metadata.blacks,

streams: (doc.mediainfo || {}).streams,
format: (doc.mediainfo || {}).format
blacks: metadata.blacks
})
}

function fileAdded (mediaPath, mediaStat, db, config, logger) {
const mediaId = getId(config.paths.media, mediaPath)
if (getManualMode()) return Promise.resolve()
return scanFile(db, config, logger, mediaPath, mediaId, mediaStat)
return scanFile(db, config, logger, mediaPath, mediaId, mediaStat, false)
.catch(error => { logger.error(error) })
}
function fileChanged (mediaPath, mediaStat, db, config, logger) {
const mediaId = getId(config.paths.media, mediaPath)
if (getManualMode()) return Promise.resolve()
return scanFile(db, config, logger, mediaPath, mediaId, mediaStat)
return scanFile(db, config, logger, mediaPath, mediaId, mediaStat, false)
.catch(error => { logger.error(error) })
}
function fileUnlinked (mediaPath, mediaStat, db, config, logger) {
Expand Down

0 comments on commit 2f752eb

Please sign in to comment.