Skip to content

Commit

Permalink
Avoid useless calls to imagemagick (#4121)
Browse files Browse the repository at this point in the history
It happens that the stack creates several jobs that will process a tiny
thumbnail for the same image with imagemagick. We can avoid the useless
calls to imagemagick with a lock + looking at Swift if the thumbnail
exists.
  • Loading branch information
nono authored Sep 19, 2023
2 parents f2d60cd + b0965e4 commit 7bde4bf
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 7 deletions.
8 changes: 3 additions & 5 deletions model/vfs/vfsswift/thumbs_v2_v3.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,17 +103,15 @@ func (t *thumbsV2) CreateThumb(img *vfs.FileDoc, format string) (vfs.ThumbFiler,

func (t *thumbsV2) ThumbExists(img *vfs.FileDoc, format string) (bool, error) {
name := t.makeName(img.ID(), format)
infos, headers, err := t.c.Object(t.ctx, t.container, name)
_, headers, err := t.c.Object(t.ctx, t.container, name)
if errors.Is(err, swift.ObjectNotFound) {
return false, nil
}
if err != nil {
return false, err
}
if infos.Bytes == 0 {
return false, nil
}
if md5 := headers["file-md5"]; md5 != "" {
meta := headers.ObjectMetadata()
if md5 := meta["file-md5"]; md5 != "" {
var md5sum []byte
md5sum, err = hex.DecodeString(md5)
if err == nil && !bytes.Equal(md5sum, img.MD5Sum) {
Expand Down
32 changes: 30 additions & 2 deletions worker/thumbnail/thumbnail.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,18 @@ func Worker(ctx *job.WorkerContext) error {
if err := ctx.UnmarshalMessage(&msg); err != nil {
return err
}
log := ctx.Logger()

if msg.NoteImage != nil {
return resizeNoteImage(ctx, msg.NoteImage)
}
if msg.File != nil {
mutex := config.Lock().ReadWrite(ctx.Instance, "thumbnails/"+msg.File.ID())
if err := mutex.Lock(); err != nil {
return err
}
defer mutex.Unlock()
log.Debugf("%s %s", msg.File.ID(), msg.Format)
if _, ok := formats[msg.Format]; !ok {
return errors.New("invalid format")
}
Expand All @@ -90,8 +97,13 @@ func Worker(ctx *job.WorkerContext) error {
return nil
}

log := ctx.Logger()
mutex := config.Lock().ReadWrite(ctx.Instance, "thumbnails/"+img.Doc.ID())
if err := mutex.Lock(); err != nil {
return err
}
defer mutex.Unlock()
log.Debugf("%s %s", img.Verb, img.Doc.ID())

switch img.Verb {
case "CREATED":
return generateThumbnails(ctx, &img.Doc)
Expand Down Expand Up @@ -211,8 +223,16 @@ func generateSingleThumbnail(ctx *job.WorkerContext, img *vfs.FileDoc, format st
}

fs := ctx.Instance.ThumbsFS()
exists, err := fs.ThumbExists(img, format)
if err != nil {
return err
}
if exists {
return nil
}

var in io.Reader
in, err := ctx.Instance.VFS().OpenFile(img)
in, err = ctx.Instance.VFS().OpenFile(img)
if err != nil {
return err
}
Expand Down Expand Up @@ -268,6 +288,14 @@ func generateThumbnails(ctx *job.WorkerContext, img *vfs.FileDoc) error {
return err
}
}

exists, err := fs.ThumbExists(img, "tiny")
if err != nil {
return err
}
if exists {
return nil
}
_, err = recGenerateThumb(ctx, in, fs, img, "tiny", env, true)
return err
}
Expand Down

0 comments on commit 7bde4bf

Please sign in to comment.