From c7d019168a57e6c04a61bad34202882c23bcbc48 Mon Sep 17 00:00:00 2001 From: Vilsol Date: Mon, 9 Nov 2020 05:36:27 +0200 Subject: [PATCH] Optimizations, file watcher feature --- README.md | 50 +++++++----- cache/cache.go | 16 ++++ cache/readonly.go | 94 +++++++++++++++++++++ cache/readwrite.go | 154 +++++++++++++++++++++++++++++++++++ cache/shared.go | 120 +++++++++++++++++++++++++++ cmd/root.go | 6 +- go.mod | 6 +- go.sum | 16 ++-- server/cache.go | 163 ------------------------------------- server/direct.go | 23 ------ server/expiry.go | 41 ---------- server/webserver.go | 57 ++++++++----- server/webserver_test.go | 105 ++++++++++++++---------- {server => utils}/utils.go | 2 +- 14 files changed, 529 insertions(+), 324 deletions(-) create mode 100644 cache/cache.go create mode 100644 cache/readonly.go create mode 100644 cache/readwrite.go create mode 100644 cache/shared.go delete mode 100644 server/cache.go delete mode 100644 server/direct.go delete mode 100644 server/expiry.go rename {server => utils}/utils.go (95%) diff --git a/README.md b/README.md index bf25333..7066913 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,6 @@ Flags: --colors Force output with colors --expiry Use cache expiry --expiry-interval duration Interval between cache GC's (default 10m0s) - --expiry-memory int Max memory usage in MB (default 128) - --expiry-shards int Cache shard count (default 64) --expiry-time duration Lifetime of a cache entry (default 1h0m0s) -h, --help help for yeet --host string Hostname to bind the webserver (default "0.0.0.0") @@ -26,6 +24,7 @@ Flags: --paths strings Paths to serve on the webserver (default [./www]) --port int Port to run the webserver on (default 8080) --warmup Load all files into memory on startup + --watch Watch filesystem for changes Use "yeet [command] --help" for more information about a command. ``` @@ -38,28 +37,41 @@ docker run -v /path/to/data:/www -p 8080:8080 ghcr.io/vilsol/yeet:latest ## Benchmarks (GOMAXPROCS=1) -### Without cache expiry +### Baseline ``` -BenchmarkServerGet1ReqPerConn 8684911 1345 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet2ReqPerConn 11419969 1046 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10ReqPerConn 15521960 763 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10KReqPerConn 17596045 690 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet1ReqPerConn10KClients 8395502 1386 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet2ReqPerConn10KClients 11061466 1053 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet10ReqPerConn10KClients 15229926 771 ns/op 0 B/op 0 allocs/op -BenchmarkServerGet100ReqPerConn10KClients 16757032 699 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet1ReqPerConn 8512062 1379 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet2ReqPerConn 11406890 1057 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet10ReqPerConn 15189015 775 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet10KReqPerConn 17068996 698 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet1ReqPerConn10KClients 8310056 1409 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet2ReqPerConn10KClients 10608926 1058 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet10ReqPerConn10KClients 15363962 773 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet100ReqPerConn10KClients 16854955 707 ns/op 0 B/op 0 allocs/op ``` ### With cache expiry ``` -BenchmarkServerGet1ReqPerConnExpiry 2419442 4780 ns/op 6176 B/op 2 allocs/op -BenchmarkServerGet2ReqPerConnExpiry 2912630 4074 ns/op 6176 B/op 2 allocs/op -BenchmarkServerGet10ReqPerConnExpiry 3479211 3468 ns/op 6176 B/op 2 allocs/op -BenchmarkServerGet10KReqPerConnExpiry 4027263 2952 ns/op 6176 B/op 2 allocs/op -BenchmarkServerGet1ReqPerConn10KClientsExpiry 1645986 7244 ns/op 6411 B/op 2 allocs/op -BenchmarkServerGet2ReqPerConn10KClientsExpiry 1818912 7026 ns/op 6177 B/op 2 allocs/op -BenchmarkServerGet10ReqPerConn10KClientsExpiry 1960964 6217 ns/op 6177 B/op 2 allocs/op -BenchmarkServerGet100ReqPerConn10KClientsExpiry 2766102 4114 ns/op 6223 B/op 2 allocs/op +BenchmarkServerGet1ReqPerConnExpiry 8677137 1375 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet2ReqPerConnExpiry 11386528 1053 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet10ReqPerConnExpiry 15480867 773 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet10KReqPerConnExpiry 16949194 707 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet1ReqPerConn10KClientsExpiry 8515335 1388 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet2ReqPerConn10KClientsExpiry 11266317 1060 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet10ReqPerConn10KClientsExpiry 15184057 776 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet100ReqPerConn10KClientsExpiry 16339011 714 ns/op 0 B/op 0 allocs/op +``` + +### With file watching + +``` +BenchmarkServerGet1ReqPerConnWatch 8282121 1430 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet2ReqPerConnWatch 11046026 1080 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet10ReqPerConnWatch 15087688 791 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet10KReqPerConnWatch 16619936 733 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet1ReqPerConn10KClientsWatch 8312827 1460 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet2ReqPerConn10KClientsWatch 10938738 1097 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet10ReqPerConn10KClientsWatch 14515534 811 ns/op 0 B/op 0 allocs/op +BenchmarkServerGet100ReqPerConn10KClientsWatch 16010731 717 ns/op 0 B/op 0 allocs/op ``` \ No newline at end of file diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 0000000..c3fb4cd --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,16 @@ +package cache + +import "time" + +type CachedInstance struct { + RelativePath string + AbsolutePath string + Data []byte + ContentType string + LoadTime time.Time +} + +type Cache interface { + Index() (int64, error) + Get(path []byte) (string, []byte) +} diff --git a/cache/readonly.go b/cache/readonly.go new file mode 100644 index 0000000..c6c4611 --- /dev/null +++ b/cache/readonly.go @@ -0,0 +1,94 @@ +package cache + +import ( + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "time" +) + +type readOnlyInstance struct { + Instance *CachedInstance + Get func(instance *readOnlyInstance) (string, []byte) +} + +type readOnlyCache struct { + Cache + data map[string]*readOnlyInstance +} + +func NewReadOnlyCache() (*readOnlyCache, error) { + if viper.GetBool("watch") { + return nil, errors.New("read only cache doesn't support file watching") + } + + c := &readOnlyCache{ + data: make(map[string]*readOnlyInstance), + } + + if viper.GetBool("expiry") { + go func(c *readOnlyCache) { + ticker := time.NewTicker(viper.GetDuration("expiry.interval")) + defer ticker.Stop() + for range ticker.C { + cleanBefore := time.Now().Add(viper.GetDuration("expiry.time") * -1) + for key, instance := range c.data { + if instance.Instance.Data != nil && instance.Instance.LoadTime.Before(cleanBefore) { + instance.Get = loadReadOnly + instance.Instance.Data = nil + instance.Instance.ContentType = "" + log.Tracef("Evicted from cache: %s", key) + } + } + } + }(c) + } + + return c, nil +} + +func (c *readOnlyCache) Index() (int64, error) { + return index(func(absolutePath string, cleanedPath string) int64 { + instance := &readOnlyInstance{ + Instance: &CachedInstance{ + RelativePath: cleanedPath, + AbsolutePath: absolutePath, + }, + Get: loadReadOnly, + } + + c.data[cleanedPath] = instance + + if viper.GetBool("warmup") { + if viper.GetBool("expiry") { + panic("expiry not supported if warmup is enabled") + } + + instance.Get(instance) + return int64(len(instance.Instance.Data)) + } + + return 0 + }) +} + +func (c *readOnlyCache) Get(path []byte) (string, []byte) { + if instance, ok := c.data[string(path)]; ok { + return instance.Get(instance) + } + + return "", nil +} + +func loadReadOnly(instance *readOnlyInstance) (string, []byte) { + fileType, data := load(instance.Instance) + + instance.Instance.LoadTime = time.Now() + instance.Instance.Data = data + instance.Instance.ContentType = fileType + instance.Get = func(cache *readOnlyInstance) (string, []byte) { + return cache.Instance.ContentType, cache.Instance.Data + } + + return fileType, data +} diff --git a/cache/readwrite.go b/cache/readwrite.go new file mode 100644 index 0000000..a3d7fea --- /dev/null +++ b/cache/readwrite.go @@ -0,0 +1,154 @@ +package cache + +import ( + "github.com/cornelk/hashmap" + "github.com/fsnotify/fsnotify" + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "path/filepath" + "strings" + "time" +) + +type readWriteInstance struct { + Instance *CachedInstance + Get func(instance *readWriteInstance) (string, []byte) +} + +type readWriteCache struct { + Cache + data *hashmap.HashMap +} + +func NewReadWriteCache() (*readWriteCache, error) { + data := &hashmap.HashMap{} + + c := &readWriteCache{ + data: data, + } + + if viper.GetBool("expiry") { + go func(c *readWriteCache) { + ticker := time.NewTicker(viper.GetDuration("expiry.interval")) + defer ticker.Stop() + for range ticker.C { + cleanBefore := time.Now().Add(viper.GetDuration("expiry.time") * -1) + for keyVal := range c.data.Iter() { + instance := keyVal.Value.(*readWriteInstance) + if instance.Instance.Data != nil && instance.Instance.LoadTime.Before(cleanBefore) { + instance.Get = loadReadWrite + instance.Instance.Data = nil + instance.Instance.ContentType = "" + log.Tracef("Evicted from cache: %s", keyVal.Key) + } + } + } + }(c) + } + + return c, nil +} + +func (c *readWriteCache) Index() (int64, error) { + indexFunc := func(absolutePath string, cleanedPath string) int64 { + instance := &readWriteInstance{ + Instance: &CachedInstance{ + RelativePath: cleanedPath, + AbsolutePath: absolutePath, + }, + Get: loadReadWrite, + } + + c.data.Set(cleanedPath, instance) + + if viper.GetBool("warmup") { + if viper.GetBool("expiry") { + panic("expiry not supported if warmup is enabled") + } + + instance.Get(instance) + return int64(len(instance.Instance.Data)) + } + + return 0 + } + + totalCount, err := index(indexFunc) + + if viper.GetBool("watch") { + watcher, err := fsnotify.NewWatcher() + + if err != nil { + return 0, err + } + + go func(c *readWriteCache) { + for event := range watcher.Events { + dirPath := "" + for _, path := range viper.GetStringSlice("paths") { + if strings.HasPrefix(event.Name, filepath.Clean(path)) { + dirPath = path + } + } + + if dirPath == "" { + log.Warnf("Received update about an unknown path: %s", event.Name) + continue + } + + cleanPath := cleanPath(event.Name, dirPath) + + switch { + case event.Op&fsnotify.Rename == fsnotify.Rename: + fallthrough + case event.Op&fsnotify.Remove == fsnotify.Remove: + c.data.Del(cleanPath) + log.Tracef("File removed from disk and cache: %s", cleanPath) + case event.Op&fsnotify.Create == fsnotify.Create: + absPath, _ := filepath.Abs(event.Name) + indexFunc(absPath, cleanPath) + log.Tracef("File created on disk and added to cache: %s", cleanPath) + case event.Op&fsnotify.Write == fsnotify.Write: + instance, ok := c.data.GetStringKey(cleanPath) + if ok { + instance.(*readWriteInstance).Get = loadReadWrite + instance.(*readWriteInstance).Instance.Data = nil + instance.(*readWriteInstance).Instance.ContentType = "" + log.Tracef("File updated on disk and evicted from cache: %s", cleanPath) + } + } + } + }(c) + + for _, dirPath := range viper.GetStringSlice("paths") { + log.Debugf("Watching path: %s", dirPath) + err = watcher.Add(dirPath) + if err != nil { + return 0, err + } + } + } + + return totalCount, err +} + +func (c *readWriteCache) Get(path []byte) (string, []byte) { + if instance, ok := c.data.GetStringKey(string(path)); ok { + return instance.(*readWriteInstance).Get(instance.(*readWriteInstance)) + } + + return "", nil +} + +func loadReadWrite(instance *readWriteInstance) (string, []byte) { + fileType, data := load(instance.Instance) + + instance.Instance.LoadTime = time.Now() + instance.Instance.Data = data + instance.Instance.ContentType = fileType + instance.Get = func(cache *readWriteInstance) (string, []byte) { + return cache.Instance.ContentType, cache.Instance.Data + } + + return fileType, data +} diff --git a/cache/shared.go b/cache/shared.go new file mode 100644 index 0000000..7cb7be9 --- /dev/null +++ b/cache/shared.go @@ -0,0 +1,120 @@ +package cache + +import ( + "github.com/Vilsol/yeet/utils" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "io/ioutil" + "mime" + "net/http" + "os" + "path" + "path/filepath" + "strings" +) + +type IndexFunc = func(absolutePath string, cleanedPath string) int64 + +func load(instance *CachedInstance) (string, []byte) { + fileName := filepath.Base(instance.AbsolutePath) + fileType := mime.TypeByExtension(filepath.Ext(fileName)) + + data, err := ioutil.ReadFile(instance.AbsolutePath) + if err != nil { + log.Error(errors.Wrap(err, "error reading file")) + return "", nil + } + + if fileType == "" { + fileType = http.DetectContentType(data[:512]) + } + + log.Debugf("Loaded into cache: %s", instance.AbsolutePath) + + return fileType, data +} + +func index(f IndexFunc) (int64, error) { + totalSize := int64(0) + totalCount := int64(0) + for _, dirPath := range viper.GetStringSlice("paths") { + cleanPath := path.Clean(dirPath) + pathSize, pathCount, err := indexPath(cleanPath, f) + if err != nil { + return 0, errors.Wrap(err, "error indexing path "+cleanPath) + } + totalSize += pathSize + totalCount += pathCount + } + + if viper.GetBool("warmup") { + log.Infof("Indexed %d files with %s of memory usage", totalCount, utils.ByteCountToHuman(totalSize)) + } else { + log.Infof("Indexed %d files", totalCount) + } + + return totalSize, nil +} + +func indexPath(dirPath string, f IndexFunc) (int64, int64, error) { + totalSize := int64(0) + totalCount := int64(0) + + if err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + filePath := path + + if info.IsDir() { + indexFile := viper.GetString("index.file") + if indexFile != "" { + joined := filepath.Join(path, indexFile) + _, err := os.Stat(joined) + if err != nil && !os.IsNotExist(err) { + return err + } else if err != nil { + return nil + } + filePath = joined + } else { + return nil + } + } + + absPath, _ := filepath.Abs(filePath) + cleanedPath := cleanPath(path, dirPath) + totalSize += f(absPath, cleanedPath) + totalCount++ + + log.Tracef("Indexed: %s -> %s", cleanedPath, absPath) + + return nil + }); err != nil { + return 0, 0, err + } + + return totalSize, totalCount, nil +} + +func cleanPath(path string, dirPath string) string { + trimmed := strings.Trim(strings.ReplaceAll(filepath.Clean(dirPath), "\\", "/"), "/") + toRemove := len(strings.Split(trimmed, "/")) + + if trimmed == "." || trimmed == "" { + toRemove = 0 + } + + cleanedPath := strings.ReplaceAll(filepath.Clean(path), "\\", "/") + + // Remove the initial path + cleanedPath = strings.Join(strings.Split(cleanedPath, "/")[toRemove:], "/") + + if !strings.HasPrefix(cleanedPath, "/") { + cleanedPath = "/" + cleanedPath + } + + return cleanedPath +} diff --git a/cmd/root.go b/cmd/root.go index a9dff8a..ddb42da 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -58,12 +58,11 @@ func init() { rootCmd.PersistentFlags().StringSlice("paths", []string{"./www"}, "Paths to serve on the webserver") rootCmd.PersistentFlags().Bool("warmup", false, "Load all files into memory on startup") + rootCmd.PersistentFlags().Bool("watch", false, "Watch filesystem for changes") rootCmd.PersistentFlags().Bool("expiry", false, "Use cache expiry") rootCmd.PersistentFlags().Duration("expiry-time", time.Minute*60, "Lifetime of a cache entry") rootCmd.PersistentFlags().Duration("expiry-interval", time.Minute*10, "Interval between cache GC's") - rootCmd.PersistentFlags().Int("expiry-memory", 128, "Max memory usage in MB") - rootCmd.PersistentFlags().Int("expiry-shards", 64, "Cache shard count") rootCmd.PersistentFlags().String("index-file", "index.html", "The directory default index file") @@ -75,12 +74,11 @@ func init() { _ = viper.BindPFlag("paths", rootCmd.PersistentFlags().Lookup("paths")) _ = viper.BindPFlag("warmup", rootCmd.PersistentFlags().Lookup("warmup")) + _ = viper.BindPFlag("watch", rootCmd.PersistentFlags().Lookup("watch")) _ = viper.BindPFlag("expiry", rootCmd.PersistentFlags().Lookup("expiry")) _ = viper.BindPFlag("expiry.time", rootCmd.PersistentFlags().Lookup("expiry-time")) _ = viper.BindPFlag("expiry.interval", rootCmd.PersistentFlags().Lookup("expiry-interval")) - _ = viper.BindPFlag("expiry.memory", rootCmd.PersistentFlags().Lookup("expiry-memory")) - _ = viper.BindPFlag("expiry.shards", rootCmd.PersistentFlags().Lookup("expiry-shards")) _ = viper.BindPFlag("index.file", rootCmd.PersistentFlags().Lookup("index-file")) } diff --git a/go.mod b/go.mod index 56d4ab1..191e346 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,12 @@ module github.com/Vilsol/yeet go 1.15 require ( - github.com/VictoriaMetrics/fastcache v1.5.7 - github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 - github.com/davecgh/go-spew v1.1.1 + github.com/cornelk/hashmap v1.0.1 + github.com/fsnotify/fsnotify v1.4.7 github.com/pkg/errors v0.8.1 github.com/sirupsen/logrus v1.7.0 github.com/spf13/cobra v1.1.1 github.com/spf13/viper v1.7.1 github.com/valyala/fasthttp v1.16.0 + google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a ) diff --git a/go.sum b/go.sum index 9ef9722..95ac610 100644 --- a/go.sum +++ b/go.sum @@ -15,12 +15,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= -github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= -github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -32,18 +28,20 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cornelk/hashmap v1.0.1 h1:RXGcy29hEdLLV8T6aK4s+BAd4tq4+3Hq50N2GoG0uIg= +github.com/cornelk/hashmap v1.0.1/go.mod h1:8wbysTUDnwJGrPZ1Iwsou3m+An6sldFrJItjRhfegCw= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/siphash v1.1.0 h1:1Rs9eTUlZLPBEvV+2sTaM8O0NWn0ppbgqS7p11aWawI= +github.com/dchest/siphash v1.1.0/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -64,9 +62,8 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -246,6 +243,7 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -314,9 +312,11 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/server/cache.go b/server/cache.go deleted file mode 100644 index c507855..0000000 --- a/server/cache.go +++ /dev/null @@ -1,163 +0,0 @@ -package server - -import ( - "github.com/allegro/bigcache" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "github.com/spf13/viper" - "io/ioutil" - "mime" - "net/http" - "os" - "path" - "path/filepath" - "strings" -) - -type CachedInstance struct { - RelativePath string - AbsolutePath string - Data []byte - ContentType string - Get func(cache *CachedInstance) (string, []byte) - GetExpiry func(cache *CachedInstance, big *bigcache.BigCache) (string, []byte) -} - -func (cache *CachedInstance) Reset() { - cache.Get = LoadCache - cache.GetExpiry = LoadCacheExpiry -} - -func IndexCache(cache map[string]*CachedInstance) (int64, error) { - totalSize := int64(0) - for _, dirPath := range viper.GetStringSlice("paths") { - cleanPath := path.Clean(dirPath) - pathSize, err := indexPath(cache, cleanPath) - if err != nil { - return 0, errors.Wrap(err, "error indexing path "+cleanPath) - } - totalSize += pathSize - } - return totalSize, nil -} - -func indexPath(cache map[string]*CachedInstance, dirPath string) (int64, error) { - trimmed := strings.Trim(dirPath, "/") - toRemove := len(strings.Split(trimmed, "/")) - - if trimmed == "." || trimmed == "" { - toRemove = 0 - } - - totalSize := int64(0) - - if err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - filePath := path - - if info.IsDir() { - indexFile := viper.GetString("index.file") - if indexFile != "" { - joined := filepath.Join(path, indexFile) - _, err := os.Stat(joined) - if err != nil && !os.IsNotExist(err) { - return err - } else if err != nil { - return nil - } - filePath = joined - } else { - return nil - } - } - - absPath, _ := filepath.Abs(filePath) - - cleanedPath := strings.ReplaceAll(filepath.Clean(path), "\\", "/") - - // Remove the initial path - cleanedPath = strings.Join(strings.Split(cleanedPath, "/")[toRemove:], "/") - - if !strings.HasPrefix(cleanedPath, "/") { - cleanedPath = "/" + cleanedPath - } - - log.Tracef("Indexed: %s -> %s", cleanedPath, absPath) - - instance := &CachedInstance{ - RelativePath: cleanedPath, - AbsolutePath: absPath, - Get: LoadCache, - GetExpiry: LoadCacheExpiry, - } - - cache[cleanedPath] = instance - - if viper.GetBool("warmup") { - if viper.GetBool("expiry") { - panic("expiry not supported if warmup is enabled") - } - - instance.Get(instance) - totalSize += int64(len(instance.Data)) - } - - return nil - }); err != nil { - return 0, err - } - - return totalSize, nil -} - -// TODO Streaming -func load(instance *CachedInstance) (string, []byte) { - fileName := filepath.Base(instance.AbsolutePath) - fileType := mime.TypeByExtension(filepath.Ext(fileName)) - - data, err := ioutil.ReadFile(instance.AbsolutePath) - if err != nil { - log.Error(errors.Wrap(err, "error reading file")) - return "", nil - } - - if fileType == "" { - fileType = http.DetectContentType(data[:512]) - } - - log.Debugf("Loaded into cache: %s", instance.AbsolutePath) - - return fileType, data -} - -func LoadCache(instance *CachedInstance) (string, []byte) { - fileType, data := load(instance) - - instance.Data = data - instance.ContentType = fileType - instance.Get = func(cache *CachedInstance) (string, []byte) { - return cache.ContentType, cache.Data - } - - return fileType, data -} - -func LoadCacheExpiry(instance *CachedInstance, big *bigcache.BigCache) (string, []byte) { - fileType, data := load(instance) - - if err := big.Set(instance.RelativePath, data); err != nil { - log.Error(errors.Wrap(err, "error setting cache")) - return "", nil - } - - instance.ContentType = fileType - instance.GetExpiry = func(cache *CachedInstance, _ *bigcache.BigCache) (string, []byte) { - data, _ := big.Get(cache.RelativePath) - return cache.ContentType, data - } - - return fileType, data -} diff --git a/server/direct.go b/server/direct.go deleted file mode 100644 index a1f6959..0000000 --- a/server/direct.go +++ /dev/null @@ -1,23 +0,0 @@ -package server - -import ( - "github.com/valyala/fasthttp" -) - -type DirectWebserver struct { - Cache map[string]*CachedInstance -} - -func (h *DirectWebserver) HandleFastHTTP(ctx *fasthttp.RequestCtx) { - if data, ok := h.Cache[string(ctx.Path())]; ok { - ctx.Success(data.Get(data)) - } else { - ctx.SetStatusCode(404) - } -} - -func GetWebserver(cache map[string]*CachedInstance) Webserver { - return &DirectWebserver{ - Cache: cache, - } -} diff --git a/server/expiry.go b/server/expiry.go deleted file mode 100644 index 2cd196e..0000000 --- a/server/expiry.go +++ /dev/null @@ -1,41 +0,0 @@ -package server - -import ( - "github.com/allegro/bigcache" - "github.com/spf13/viper" - "github.com/valyala/fasthttp" -) - -type ExpiryWebserver struct { - Cache map[string]*CachedInstance - Expiry *bigcache.BigCache -} - -func (h *ExpiryWebserver) HandleFastHTTP(ctx *fasthttp.RequestCtx) { - if cache, ok := h.Cache[string(ctx.Path())]; ok { - ctx.Success(cache.GetExpiry(cache, h.Expiry)) - } else { - ctx.SetStatusCode(404) - } -} - -func GetExpiryWebserver(cache map[string]*CachedInstance) (*ExpiryWebserver, error) { - bigCache, err := bigcache.NewBigCache(bigcache.Config{ - Shards: viper.GetInt("expiry.shards"), - LifeWindow: viper.GetDuration("expiry.time"), - CleanWindow: viper.GetDuration("expiry.interval"), - HardMaxCacheSize: viper.GetInt("expiry.memory"), - OnRemove: func(key string, entry []byte) { - cache[key].Reset() - }, - }) - - if err != nil { - return nil, err - } - - return &ExpiryWebserver{ - Cache: cache, - Expiry: bigCache, - }, nil -} diff --git a/server/webserver.go b/server/webserver.go index 9b1c73a..70c5160 100644 --- a/server/webserver.go +++ b/server/webserver.go @@ -1,49 +1,64 @@ package server import ( + "github.com/Vilsol/yeet/cache" log "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/valyala/fasthttp" "strconv" ) -type Webserver interface { - HandleFastHTTP(ctx *fasthttp.RequestCtx) -} - func Run() error { - cache := make(map[string]*CachedInstance) - - totalSize, err := IndexCache(cache) + ws, err := GetWebserver() if err != nil { return err } - if viper.GetBool("warmup") { - log.Infof("Indexed %d files with %s of memory usage", len(cache), ByteCountToHuman(totalSize)) + address := viper.GetString("host") + ":" + strconv.Itoa(viper.GetInt("port")) + + log.Infof("Starting webserver on %s", address) + + if err := fasthttp.ListenAndServe(address, ws.HandleFastHTTP); err != nil { + return err + } + + return nil +} + +type Webserver struct { + Cache cache.Cache +} + +func (h *Webserver) HandleFastHTTP(ctx *fasthttp.RequestCtx) { + if fileType, b := h.Cache.Get(ctx.Path()); b != nil { + ctx.Success(fileType, b) } else { - log.Infof("Indexed %d files", len(cache)) + ctx.SetStatusCode(404) } +} - var ws Webserver - if viper.GetBool("expiry.enabled") { - ws, err = GetExpiryWebserver(cache) +func GetWebserver() (*Webserver, error) { + var c cache.Cache + var err error + + if viper.GetBool("watch") { + c, err = cache.NewReadWriteCache() } else { - ws = GetWebserver(cache) + c, err = cache.NewReadOnlyCache() } if err != nil { - return err + return nil, err } - address := viper.GetString("host") + ":" + strconv.Itoa(viper.GetInt("port")) - - log.Infof("Starting webserver on %s", address) + if _, err := c.Index(); err != nil { + return nil, err + } - if err := fasthttp.ListenAndServe(address, ws.HandleFastHTTP); err != nil { - return err + ws := &Webserver{ + Cache: c, } - return nil + return ws, nil } diff --git a/server/webserver_test.go b/server/webserver_test.go index 2d6fad4..f8b3af9 100644 --- a/server/webserver_test.go +++ b/server/webserver_test.go @@ -1,6 +1,7 @@ package server import ( + log "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/valyala/fasthttp" "io" @@ -14,48 +15,35 @@ import ( const getRequest = "GET /webserver_test.go HTTP/1.1\r\nHost: google.com\r\n\r\n" -var cache map[string]*CachedInstance - func init() { viper.Set("paths", []string{"."}) - - cache = make(map[string]*CachedInstance) - - if _, err := IndexCache(cache); err != nil { - panic(err) - } + log.SetLevel(log.FatalLevel) } -func benchmarkServerGet(b *testing.B, clientsCount, requestsPerConn int, expiry bool) { - var handler fasthttp.RequestHandler - +func benchmarkServerGet(b *testing.B, clientsCount, requestsPerConn int, expiry bool, watch bool) { if expiry { viper.Set("expiry", true) - viper.Set("expiry.shards", 64) viper.Set("expiry.time", time.Minute*10) viper.Set("expiry.interval", time.Minute*60) - viper.Set("expiry.memory", 128) - - ws, err := GetExpiryWebserver(cache) + } - if err != nil { - panic(err) - } + if watch { + viper.Set("watch", true) + } - handler = ws.HandleFastHTTP - } else { - ws := &DirectWebserver{ - Cache: cache, - } + ws, err := GetWebserver() - handler = ws.HandleFastHTTP + if err != nil { + panic(err) } s := &fasthttp.Server{ - Handler: handler, + Handler: ws.HandleFastHTTP, Concurrency: 16 * clientsCount, } + b.ReportAllocs() + b.ResetTimer() benchmarkServer(b, s, clientsCount, requestsPerConn, getRequest) } @@ -199,66 +187,101 @@ func newFakeListener(requestsCount, clientsCount, requestsPerConn int, request s return ln } +// Baseline func BenchmarkServerGet1ReqPerConn(b *testing.B) { - benchmarkServerGet(b, runtime.NumCPU(), 1, false) + benchmarkServerGet(b, runtime.NumCPU(), 1, false, false) } func BenchmarkServerGet2ReqPerConn(b *testing.B) { - benchmarkServerGet(b, runtime.NumCPU(), 2, false) + benchmarkServerGet(b, runtime.NumCPU(), 2, false, false) } func BenchmarkServerGet10ReqPerConn(b *testing.B) { - benchmarkServerGet(b, runtime.NumCPU(), 10, false) + benchmarkServerGet(b, runtime.NumCPU(), 10, false, false) } func BenchmarkServerGet10KReqPerConn(b *testing.B) { - benchmarkServerGet(b, runtime.NumCPU(), 10000, false) + benchmarkServerGet(b, runtime.NumCPU(), 10000, false, false) } func BenchmarkServerGet1ReqPerConn10KClients(b *testing.B) { - benchmarkServerGet(b, 10000, 1, false) + benchmarkServerGet(b, 10000, 1, false, false) } func BenchmarkServerGet2ReqPerConn10KClients(b *testing.B) { - benchmarkServerGet(b, 10000, 2, false) + benchmarkServerGet(b, 10000, 2, false, false) } func BenchmarkServerGet10ReqPerConn10KClients(b *testing.B) { - benchmarkServerGet(b, 10000, 10, false) + benchmarkServerGet(b, 10000, 10, false, false) } func BenchmarkServerGet100ReqPerConn10KClients(b *testing.B) { - benchmarkServerGet(b, 10000, 100, false) + benchmarkServerGet(b, 10000, 100, false, false) } +// With expiry func BenchmarkServerGet1ReqPerConnExpiry(b *testing.B) { - benchmarkServerGet(b, runtime.NumCPU(), 1, true) + benchmarkServerGet(b, runtime.NumCPU(), 1, true, false) } func BenchmarkServerGet2ReqPerConnExpiry(b *testing.B) { - benchmarkServerGet(b, runtime.NumCPU(), 2, true) + benchmarkServerGet(b, runtime.NumCPU(), 2, true, false) } func BenchmarkServerGet10ReqPerConnExpiry(b *testing.B) { - benchmarkServerGet(b, runtime.NumCPU(), 10, true) + benchmarkServerGet(b, runtime.NumCPU(), 10, true, false) } func BenchmarkServerGet10KReqPerConnExpiry(b *testing.B) { - benchmarkServerGet(b, runtime.NumCPU(), 10000, true) + benchmarkServerGet(b, runtime.NumCPU(), 10000, true, false) } func BenchmarkServerGet1ReqPerConn10KClientsExpiry(b *testing.B) { - benchmarkServerGet(b, 10000, 1, true) + benchmarkServerGet(b, 10000, 1, true, false) } func BenchmarkServerGet2ReqPerConn10KClientsExpiry(b *testing.B) { - benchmarkServerGet(b, 10000, 2, true) + benchmarkServerGet(b, 10000, 2, true, false) } func BenchmarkServerGet10ReqPerConn10KClientsExpiry(b *testing.B) { - benchmarkServerGet(b, 10000, 10, true) + benchmarkServerGet(b, 10000, 10, true, false) } func BenchmarkServerGet100ReqPerConn10KClientsExpiry(b *testing.B) { - benchmarkServerGet(b, 10000, 100, true) + benchmarkServerGet(b, 10000, 100, true, false) +} + +// With watch +func BenchmarkServerGet1ReqPerConnWatch(b *testing.B) { + benchmarkServerGet(b, runtime.NumCPU(), 1, false, true) +} + +func BenchmarkServerGet2ReqPerConnWatch(b *testing.B) { + benchmarkServerGet(b, runtime.NumCPU(), 2, false, true) +} + +func BenchmarkServerGet10ReqPerConnWatch(b *testing.B) { + benchmarkServerGet(b, runtime.NumCPU(), 10, false, true) +} + +func BenchmarkServerGet10KReqPerConnWatch(b *testing.B) { + benchmarkServerGet(b, runtime.NumCPU(), 10000, false, true) +} + +func BenchmarkServerGet1ReqPerConn10KClientsWatch(b *testing.B) { + benchmarkServerGet(b, 10000, 1, false, true) +} + +func BenchmarkServerGet2ReqPerConn10KClientsWatch(b *testing.B) { + benchmarkServerGet(b, 10000, 2, false, true) +} + +func BenchmarkServerGet10ReqPerConn10KClientsWatch(b *testing.B) { + benchmarkServerGet(b, 10000, 10, false, true) +} + +func BenchmarkServerGet100ReqPerConn10KClientsWatch(b *testing.B) { + benchmarkServerGet(b, 10000, 100, false, true) } diff --git a/server/utils.go b/utils/utils.go similarity index 95% rename from server/utils.go rename to utils/utils.go index d736289..a1a7060 100644 --- a/server/utils.go +++ b/utils/utils.go @@ -1,4 +1,4 @@ -package server +package utils import "fmt"