From 84eff135381132e40cd83d0d48181feb889a4db7 Mon Sep 17 00:00:00 2001 From: Akmal Date: Sat, 6 Jul 2024 19:22:26 +0700 Subject: [PATCH] Change grid cache to LRU - Change max size to max entries - Use freelru --- go.mod | 3 +- go.sum | 2 ++ handlers/grid.go | 3 +- handlers/scraper/cache.go | 35 ++++++++++++++++++++ main.go | 67 ++++----------------------------------- 5 files changed, 48 insertions(+), 62 deletions(-) diff --git a/go.mod b/go.mod index 6f7b7a2..977b98c 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,9 @@ require ( github.com/RyanCarrier/dijkstra/v2 v2.0.2 github.com/ansrivas/fiberprometheus/v2 v2.6.1 github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b + github.com/cespare/xxhash/v2 v2.3.0 github.com/cockroachdb/pebble v1.1.1 + github.com/elastic/go-freelru v0.13.0 github.com/gofiber/fiber/v2 v2.52.4 github.com/kelindar/binary v1.0.19 github.com/rs/zerolog v1.33.0 @@ -27,7 +29,6 @@ require ( github.com/andybalholm/brotli v1.1.0 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/errors v1.11.3 // indirect github.com/cockroachdb/fifo v0.0.0-20240616162244-4768e80dfb9a // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect diff --git a/go.sum b/go.sum index 5182903..ac8c3a0 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,8 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/go-freelru v0.13.0 h1:TKKY6yCfNNNky7Pj9xZAOEpBcdNgZJfihEftOb55omg= +github.com/elastic/go-freelru v0.13.0/go.mod h1:bSdWT4M0lW79K8QbX6XY2heQYSCqD7THoYf82pT/H3I= github.com/getsentry/sentry-go v0.28.1 h1:zzaSm/vHmGllRM6Tpx1492r0YDzauArdBfkJRtY6P5k= github.com/getsentry/sentry-go v0.28.1/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= diff --git a/handlers/grid.go b/handlers/grid.go index 060a429..c545303 100644 --- a/handlers/grid.go +++ b/handlers/grid.go @@ -131,7 +131,7 @@ func Grid() fiber.Handler { gridFname := filepath.Join("static", postID+".jpeg") // If already exists, return - if _, err := os.Stat(gridFname); err == nil { + if _, ok := scraper.LRU.Get(gridFname); ok { return c.SendFile(gridFname) } @@ -197,6 +197,7 @@ func Grid() fiber.Handler { if err := jpeg.Encode(f, grid, &jpeg.Options{Quality: 80}); err != nil { return err } + scraper.LRU.Add(gridFname, true) return c.SendFile(gridFname) } } diff --git a/handlers/scraper/cache.go b/handlers/scraper/cache.go index 15f8626..06e2c4c 100644 --- a/handlers/scraper/cache.go +++ b/handlers/scraper/cache.go @@ -1,10 +1,20 @@ package handlers import ( + "os" + + "github.com/cespare/xxhash/v2" "github.com/cockroachdb/pebble" + "github.com/elastic/go-freelru" + "github.com/rs/zerolog/log" ) var DB *pebble.DB +var LRU *freelru.SyncedLRU[string, bool] + +func hashStringXXHASH(s string) uint32 { + return uint32(xxhash.Sum64String(s)) +} func InitDB() { db, err := pebble.Open("database", &pebble.Options{}) @@ -14,3 +24,28 @@ func InitDB() { } DB = db } + +func InitLRU(maxEntries int) { + // Initialize LRU for grid caching + lru, err := freelru.NewSynced[string, bool](uint32(maxEntries), hashStringXXHASH) + if err != nil { + panic(err) + } + + lru.SetOnEvict(func(key string, value bool) { + os.Remove(key) + }) + + // Fill LRU with existing files + dir, err := os.ReadDir("static") + if err != nil { + log.Fatal().Err(err).Msg("Failed to read static folder") + } + for _, d := range dir { + if !d.IsDir() { + lru.Add("static/"+d.Name(), true) + } + } + + LRU = lru +} diff --git a/main.go b/main.go index c307834..177b0fb 100644 --- a/main.go +++ b/main.go @@ -7,9 +7,7 @@ import ( scraper "instafix/handlers/scraper" "instafix/utils" "instafix/views" - "io/fs" "os" - "path/filepath" "strconv" "strings" "time" @@ -24,32 +22,6 @@ import ( "github.com/valyala/bytebufferpool" ) -func byteSizeStrToInt(n string) (int64, error) { - sizeStr := strings.ToLower(n) - sizeStr = strings.TrimSpace(sizeStr) - - units := map[string]int64{ - "kb": 1024, - "mb": 1024 * 1024, - "gb": 1024 * 1024 * 1024, - "tb": 1024 * 1024 * 1024 * 1024, - } - - for unit, multiplier := range units { - if strings.HasSuffix(sizeStr, unit) { - sizeStr = strings.TrimSuffix(sizeStr, unit) - sizeStr = strings.TrimSpace(sizeStr) - - size, err := strconv.ParseInt(sizeStr, 10, 64) - if err != nil { - return -1, err - } - return size * multiplier, nil - } - } - return -1, nil -} - func init() { scraper.InitDB() @@ -59,7 +31,7 @@ func init() { func main() { listenAddr := flag.String("listen", "0.0.0.0:3000", "Address to listen on") - gridCacheSize := flag.String("grid-cache-size", "25GB", "Grid cache size") + gridCacheMaxFlag := flag.String("grid-cache-entries", "1024", "Maximum number of grid images to cache") remoteScraperAddr := flag.String("remote-scraper", "", "Remote scraper address") flag.Parse() @@ -90,18 +62,18 @@ func main() { zerolog.TimeFieldFormat = zerolog.TimeFormatUnix zerolog.SetGlobalLevel(zerolog.InfoLevel) - // Parse grid-cache-size - gridCacheSizeParsed, err := byteSizeStrToInt(*gridCacheSize) - if err != nil { - log.Fatal().Err(err).Msg("Failed to parse grid-cache-size") + // Initialize LRU + gridCacheMax, err := strconv.Atoi(*gridCacheMaxFlag) + if err != nil || gridCacheMax <= 0 { + log.Fatal().Err(err).Msg("Failed to parse grid-cache-entries or invalid value") } + scraper.InitLRU(gridCacheMax) // Evict cache every minute go func() { for { - time.Sleep(5 * time.Minute) - evictStatic(gridCacheSizeParsed) evictCache() + time.Sleep(5 * time.Minute) } }() @@ -113,8 +85,6 @@ func main() { return c.Send(viewsBuf.Bytes()) }) - // GET /p/Cw8X2wXPjiM - // GET /stories/fatfatpankocat/3226148677671954631/ app.Get("/p/:postID/", handlers.Embed()) app.Get("/tv/:postID", handlers.Embed()) app.Get("/reel/:postID", handlers.Embed()) @@ -135,29 +105,6 @@ func main() { } } -// Remove file in static folder until below threshold -func evictStatic(threshold int64) { - var dirSize int64 = 0 - readSize := func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if !d.IsDir() { - if dirSize > threshold { - err := os.Remove(path) - return err - } - info, err := d.Info() - if err != nil { - return err - } - dirSize += info.Size() - } - return nil - } - filepath.WalkDir("static", readSize) -} - // Remove cache from Pebble if already expired func evictCache() { iter, err := scraper.DB.NewIter(&pebble.IterOptions{LowerBound: []byte("exp-")})