WIP: WebP Server Go 0.13.0

.github/workflows/integration-test.yaml

@@ -37,7 +37,7 @@ jobs:
       - name: Send Requests to Server
         run: |
           cd pics
-          find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" http://localhost:3333/{}
+          find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" --silent http://localhost:3333/{}
 
       - name: Get container RAM stats
         run: |
@@ -71,7 +71,7 @@ jobs:
       - name: Send Requests to Server
         run: |
           cd pics
-          find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" http://localhost:3333/{}
+          find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" --silent http://localhost:3333/{}
 
       - name: Get container RAM stats
         run: |

config/config.go

@@ -43,26 +43,40 @@ const (
 )
 
 var (
-	ConfigPath         string
-	Jobs               int
-	DumpSystemd        bool
-	DumpConfig         bool
-	ShowVersion        bool
-	ProxyMode          bool
-	Prefetch           bool // Prefech in go-routine, with WebP Server Go launch normally
-	PrefetchForeground bool // Standalone prefetch, prefetch and exit
-	Config             = NewWebPConfig()
-	Version            = "0.12.3"
-	WriteLock          = cache.New(5*time.Minute, 10*time.Minute)
-	ConvertLock        = cache.New(5*time.Minute, 10*time.Minute)
-	LocalHostAlias     = "local"
-	RemoteCache        *cache.Cache
+	ConfigPath          string
+	Jobs                int
+	DumpSystemd         bool
+	DumpConfig          bool
+	ShowVersion         bool
+	ProxyMode           bool
+	Prefetch            bool // Prefech in go-routine, with WebP Server Go launch normally
+	PrefetchForeground  bool // Standalone prefetch, prefetch and exit
+	AllowNonImage       bool
+	Config              = NewWebPConfig()
+	Version             = "0.13.0"
+	WriteLock           = cache.New(5*time.Minute, 10*time.Minute)
+	ConvertLock         = cache.New(5*time.Minute, 10*time.Minute)
+	LocalHostAlias      = "local"
+	RemoteCache         *cache.Cache
+	DefaultAllowedTypes = []string{"jpg", "png", "jpeg", "bmp", "gif", "svg", "nef", "heic", "webp", "avif", "jxl"} // Default allowed image types
 )
 
+type ImageMeta struct {
+	Width      int    `json:"width"`
+	Height     int    `json:"height"`
+	Format     string `json:"format"`
+	Size       int    `json:"size"`
+	NumPages   int    `json:"num_pages"`
+	Blurhash   string `json:"blurhash"`
+	Colorspace string `json:"colorspace"`
+}
+
 type MetaFile struct {
 	Id       string `json:"id"`       // hash of below path️, also json file name id.webp
 	Path     string `json:"path"`     // local: path with width and height, proxy: full url
 	Checksum string `json:"checksum"` // hash of original file or hash(etag). Use this to identify changes
+
+	ImageMeta
 }
 
 type WebpConfig struct {
@@ -94,12 +108,15 @@ type WebpConfig struct {
 }
 
 func NewWebPConfig() *WebpConfig {
+	// Copy DefaultAllowedTypes to avoid modification
+	defaultAllowedTypes := make([]string, len(DefaultAllowedTypes))
+	copy(defaultAllowedTypes, DefaultAllowedTypes)
 	return &WebpConfig{
 		Host:          "0.0.0.0",
 		Port:          "3333",
 		ImgPath:       "./pics",
 		Quality:       80,
-		AllowedTypes:  []string{"jpg", "png", "jpeg", "bmp", "gif", "svg", "nef", "heic", "webp"},
+		AllowedTypes:  defaultAllowedTypes,
 		ConvertTypes:  []string{"webp"},
 		ImageMap:      map[string]string{},
 		ExhaustPath:   "./exhaust",

encoder/encoder.go

@@ -33,6 +33,14 @@ func init() {
 	intMinusOne.Set(-1)
 }
 
+func LoadImage(filename string) (*vips.ImageRef, error) {
+	img, err := vips.LoadImageFromFile(filename, &vips.ImportParams{
+		FailOnError: boolFalse,
+		NumPages:    intMinusOne,
+	})
+	return img, err
+}
+
 func ConvertFilter(rawPath, jxlPath, avifPath, webpPath string, extraParams config.ExtraParams, supportedFormats map[string]bool, c chan int) {
 	// Wait for the conversion to complete and return the converted image
 	retryDelay := 100 * time.Millisecond // Initial retry delay
@@ -122,30 +130,40 @@ func convertImage(rawPath, optimizedPath, imageType string, extraParams config.E
 	}
 
 	// Image is only opened here
-	img, err := vips.LoadImageFromFile(rawPath, &vips.ImportParams{
-		FailOnError: boolFalse,
-		NumPages:    intMinusOne,
-	})
-	if err != nil {
-		log.Warnf("Can't open source image: %v", err)
-		return err
-	}
+	img, err := LoadImage(rawPath)
 	defer img.Close()
 
 	// Pre-process image(auto rotate, resize, etc.)
 	err = preProcessImage(img, imageType, extraParams)
 	if err != nil {
 		log.Warnf("Can't pre-process source image: %v", err)
-		return err
 	}
 
+	// If image is already in the target format, just copy it
+	imageFormat := img.Format()
+
 	switch imageType {
 	case "webp":
-		err = webpEncoder(img, rawPath, optimizedPath)
+		if imageFormat == vips.ImageTypeWEBP {
+			log.Infof("Image is already in WebP format, copying %s to %s", rawPath, optimizedPath)
+			return helper.CopyFile(rawPath, optimizedPath)
+		} else {
+			err = webpEncoder(img, rawPath, optimizedPath)
+		}
 	case "avif":
-		err = avifEncoder(img, rawPath, optimizedPath)
+		if imageFormat == vips.ImageTypeAVIF {
+			log.Infof("Image is already in AVIF format, copying %s to %s", rawPath, optimizedPath)
+			return helper.CopyFile(rawPath, optimizedPath)
+		} else {
+			err = avifEncoder(img, rawPath, optimizedPath)
+		}
 	case "jxl":
-		err = jxlEncoder(img, rawPath, optimizedPath)
+		if imageFormat == vips.ImageTypeJXL {
+			log.Infof("Image is already in JXL format, copying %s to %s", rawPath, optimizedPath)
+			return helper.CopyFile(rawPath, optimizedPath)
+		} else {
+			err = jxlEncoder(img, rawPath, optimizedPath)
+		}
 	}
 
 	return err

encoder/prefetch.go

@@ -33,9 +33,18 @@ func PrefetchImages() {
 			if info.IsDir() {
 				return nil
 			}
-			if !helper.CheckAllowedType(picAbsPath) {
+			// Only convert files with image extensions, use smaller of config.DefaultAllowedTypes and config.Config.AllowedTypes
+			if helper.CheckAllowedExtension(picAbsPath) {
+				// File type is allowed by user, check if it is an image
+				if helper.CheckImageExtension(picAbsPath) {
+					// File is an image, continue
+				} else {
+					return nil
+				}
+			} else {
 				return nil
 			}
+
 			// RawImagePath string, ImgFilename string, reqURI string
 			metadata := helper.ReadMetadata(picAbsPath, "", config.LocalHostAlias)
 			avifAbsPath, webpAbsPath, jxlAbsPath := helper.GenOptimizedAbsPath(metadata, config.LocalHostAlias)

encoder/process.go

@@ -114,6 +114,7 @@ func ResizeItself(raw, dest string, extraParams config.ExtraParams) {
 
 	img, err := vips.LoadImageFromFile(raw, &vips.ImportParams{
 		FailOnError: boolFalse,
+		NumPages:    intMinusOne,
 	})
 	if err != nil {
 		log.Warnf("Could not load %s: %s", raw, err)
@@ -152,13 +153,18 @@ func preProcessImage(img *vips.ImageRef, imageType string, extraParams config.Ex
 		}
 	}
 
-	// Auto rotate
-	err := img.AutoRotate()
-	if err != nil {
-		return err
-	}
 	if config.Config.EnableExtraParams {
-		err = resizeImage(img, extraParams)
+		err := resizeImage(img, extraParams)
+		if err != nil {
+			return err
+		}
+	}
+	// Skip auto rotate for GIF/WebP
+	if img.Format() == vips.ImageTypeGIF || img.Format() == vips.ImageTypeWEBP {
+		return nil
+	} else {
+		// Auto rotate
+		err := img.AutoRotate()
 		if err != nil {
 			return err
 		}

go.mod

@@ -3,12 +3,14 @@ module webp_server_go
 go 1.23
 
 require (
+	github.com/buckket/go-blurhash v1.1.0
 	github.com/cespare/xxhash v1.1.0
 	github.com/davidbyttow/govips/v2 v2.15.0
 	github.com/gofiber/fiber/v2 v2.52.5
 	github.com/h2non/filetype v1.1.4-0.20230123234534-cfcd7d097bc4
 	github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c
 	github.com/jeremytorres/rawparser v1.0.2
+	github.com/mileusna/useragent v1.3.5
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/schollz/progressbar/v3 v3.17.1
 	github.com/sirupsen/logrus v1.9.3

go.sum

@@ -2,6 +2,8 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
+github.com/buckket/go-blurhash v1.1.0 h1:X5M6r0LIvwdvKiUtiNcRL2YlmOfMzYobI3VCKCZc9Do=
+github.com/buckket/go-blurhash v1.1.0/go.mod h1:aT2iqo5W9vu9GpyoLErKfTHwgODsZp3bQfXjXJUxNb8=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
@@ -31,6 +33,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mileusna/useragent v1.3.5 h1:SJM5NzBmh/hO+4LGeATKpaEX9+b4vcGg2qXGLiNGDws=
+github.com/mileusna/useragent v1.3.5/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=

handler/router.go

@@ -1,6 +1,7 @@
 package handler
 
 import (
+	"fmt"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -45,6 +46,8 @@ func Convert(c *fiber.Ctx) error {
 		proxyMode      = config.ProxyMode
 		mapMode        = false
 
+		meta = c.Query("meta") // Meta request
+
 		width, _     = strconv.Atoi(c.Query("width"))      // Extra Params
 		height, _    = strconv.Atoi(c.Query("height"))     // Extra Params
 		maxHeight, _ = strconv.Atoi(c.Query("max_height")) // Extra Params
@@ -59,14 +62,21 @@ func Convert(c *fiber.Ctx) error {
 
 	log.Debugf("Incoming connection from %s %s %s", c.IP(), reqHostname, reqURIwithQuery)
 
-	if !helper.CheckAllowedType(filename) {
+	if !helper.CheckAllowedExtension(filename) {
 		msg := "File extension not allowed! " + filename
 		log.Warn(msg)
 		c.Status(http.StatusBadRequest)
-		_ = c.Send([]byte(msg))
+		_ = c.SendString(msg)
 		return nil
 	}
 
+	// Check if the file extension is allowed and not with image extension
+	// In this case we will serve the file directly
+	if helper.CheckAllowedExtension(filename) && !helper.CheckImageExtension(filename) {
+		fmt.Println("File extension is allowed and not with image extension")
+		return c.SendFile(path.Join(config.Config.ImgPath, reqURI))
+	}
+
 	// Rewrite the target backend if a mapping rule matches the hostname
 	if hostMap, hostMapFound := config.Config.ImageMap[reqHost]; hostMapFound {
 		log.Debugf("Found host mapping %s -> %s", reqHostname, hostMap)
@@ -141,12 +151,26 @@ func Convert(c *fiber.Ctx) error {
 		}
 	}
 
+	// If meta request, return the metadata
+	if meta == "full" {
+		return c.JSON(fiber.Map{
+			"height":     metadata.ImageMeta.Height,
+			"width":      metadata.ImageMeta.Width,
+			"size":       metadata.ImageMeta.Size,
+			"format":     metadata.ImageMeta.Format,
+			"colorspace": metadata.ImageMeta.Colorspace,
+			"num_pages":  metadata.ImageMeta.NumPages,
+			"blurhash":   metadata.ImageMeta.Blurhash,
+		})
+	}
+
 	supportedFormats := helper.GuessSupportedFormat(reqHeader)
 	// resize itself and return if only raw(original format) is supported
 	if supportedFormats["raw"] == true &&
 		supportedFormats["webp"] == false &&
 		supportedFormats["avif"] == false &&
-		supportedFormats["jxl"] == false {
+		supportedFormats["jxl"] == false &&
+		supportedFormats["heic"] == false {
 		dest := path.Join(config.Config.ExhaustPath, targetHostName, metadata.Id)
 		if !helper.ImageExists(dest) {
 			encoder.ResizeItself(rawImageAbs, dest, extraParams)
@@ -178,6 +202,12 @@ func Convert(c *fiber.Ctx) error {
 	if supportedFormats["jxl"] {
 		availableFiles = append(availableFiles, jxlAbs)
 	}
+	// If raw format is not supported(e,g: heic), remove it from the list
+	// Because we shouldn't serve the heic format if it's not supported
+	if !supportedFormats["heic"] && helper.GetImageExtension(rawImageAbs) == "heic" {
+		// Remove the "raw" from the list
+		availableFiles = availableFiles[1:]
+	}
 
 	finalFilename := helper.FindSmallestFiles(availableFiles)
 	contentType := helper.GetFileContentType(finalFilename)

handler/router_test.go

@@ -32,7 +32,7 @@ func setupParam() {
 	// setup parameters here...
 	config.Config.ImgPath = "../pics"
 	config.Config.ExhaustPath = "../exhaust_test"
-	config.Config.AllowedTypes = []string{"jpg", "png", "jpeg", "bmp"}
+	config.Config.AllowedTypes = []string{"jpg", "png", "jpeg", "bmp", "heic"}
 	config.Config.MetadataPath = "../metadata"
 	config.Config.RemoteRawPath = "../remote-raw"
 	config.ProxyMode = false
@@ -128,6 +128,7 @@ func TestConvert(t *testing.T) {
 		"http://127.0.0.1:3333/dir1/inside.jpg":                 "image/webp",
 		"http://127.0.0.1:3333/%e5%a4%aa%e7%a5%9e%e5%95%a6.png": "image/webp",
 		"http://127.0.0.1:3333/太神啦.png":                         "image/webp",
+		"http://127.0.0.1:3333/sample3.heic":                    "image/webp", // webp because browser does not support heic
 	}
 
 	var testChromeAvifLink = map[string]string{
@@ -195,12 +196,33 @@ func TestConvertNotAllowed(t *testing.T) {
 	defer resp.Body.Close()
 	assert.Contains(t, string(data), "File extension not allowed")
 
+	// not allowed, but we have the file, this should return File extension not allowed
+	url = "http://127.0.0.1:3333/config.json"
+	resp, data = requestToServer(url, app, chromeUA, acceptWebP)
+	defer resp.Body.Close()
+	assert.Contains(t, string(data), "File extension not allowed")
+
 	// not allowed, random file
 	url = url + "hagdgd"
 	resp, data = requestToServer(url, app, chromeUA, acceptWebP)
 	defer resp.Body.Close()
 	assert.Contains(t, string(data), "File extension not allowed")
+}
+
+func TestConvertPassThrough(t *testing.T) {
+	setupParam()
+	config.Config.AllowedTypes = []string{"*"}
 
+	var app = fiber.New()
+	app.Get("/*", Convert)
+
+	// not allowed, but we have the file, this should return File extension not allowed
+	url := "http://127.0.0.1:3333/config.json"
+	resp, data := requestToServer(url, app, chromeUA, acceptWebP)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+	assert.Contains(t, string(data), "HOST")
 }
 
 func TestConvertProxyModeBad(t *testing.T) {