zstd + dict v0.2

Wikidepia · Aug 9, 2024 · f6c49d9 · f6c49d9
1 parent 0762485
commit f6c49d9
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 5 deletions.
diff --git a/go.mod b/go.mod
@@ -12,6 +12,7 @@ require (
 	github.com/elastic/go-freelru v0.13.0
 	github.com/go-chi/chi/v5 v5.1.0
 	github.com/kelindar/binary v1.0.19
+	github.com/klauspost/compress v1.17.9
 	github.com/tdewolff/parse/v2 v2.7.15
 	github.com/tidwall/gjson v1.17.1
 	go.etcd.io/bbolt v1.3.10

diff --git a/go.sum b/go.sum
@@ -16,6 +16,8 @@ github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
 github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/kelindar/binary v1.0.19 h1:DNyQCtKjkLhBh9pnP49OWREddLB0Mho+1U/AOt/Qzxw=
 github.com/kelindar/binary v1.0.19/go.mod h1:/twdz8gRLNMffx0U4UOgqm1LywPs6nd9YK2TX52MDh8=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=

diff --git a/handlers/scraper/data.go b/handlers/scraper/data.go
@@ -2,6 +2,7 @@ package handlers
 
 import (
 	"bytes"
+	_ "embed"
 	"errors"
 	"instafix/utils"
 	"io"
@@ -15,6 +16,7 @@ import (
 	"github.com/PuerkitoBio/goquery"
 	"github.com/PurpleSec/escape"
 	"github.com/kelindar/binary"
+	"github.com/klauspost/compress/zstd"
 	"github.com/tdewolff/parse/v2"
 	"github.com/tdewolff/parse/v2/js"
 	"github.com/tidwall/gjson"
@@ -23,16 +25,17 @@ import (
 	"golang.org/x/sync/singleflight"
 )
 
-var timeout = 10 * time.Second
-
 var (
+	timeout     = 10 * time.Second
 	ErrNotFound = errors.New("post not found")
 )
 
 var RemoteScraperAddr string
-
 var sflightScraper singleflight.Group
 
+//go:embed dictionary.bin
+var zstdDict []byte
+
 type Media struct {
 	TypeName string
 	URL      string
@@ -138,12 +141,21 @@ func (i *InstaData) ScrapeData() error {
 		if err != nil {
 			return err
 		}
+		req.Header.Set("Accept-Encoding", "zstd")
 		res, err := client.Do(req)
 		if res != nil && res.StatusCode == 200 {
 			defer res.Body.Close()
-			iDataGunzip, err := io.ReadAll(res.Body)
+			zstdReader, err := zstd.NewReader(nil, zstd.WithDecoderLowmem(true), zstd.WithDecoderDicts(zstdDict))
+			if err != nil {
+				return err
+			}
+			remoteData, err := io.ReadAll(res.Body)
 			if err == nil {
-				if err = binary.Unmarshal(iDataGunzip, i); err == nil {
+				remoteDecomp, err := zstdReader.DecodeAll(remoteData, nil)
+				if err != nil {
+					return err
+				}
+				if err = binary.Unmarshal(remoteDecomp, i); err == nil {
 					slog.Info("Data parsed from remote scraper", "postID", i.PostID)
 					return nil
 				}