Share remote zstd decoder

Wikidepia · Aug 10, 2024 · 7f84f0c · 7f84f0c
1 parent f9bc02a
commit 7f84f0c
Showing 1 changed file with 8 additions and 2 deletions.
diff --git a/handlers/scraper/data.go b/handlers/scraper/data.go
@@ -33,6 +33,7 @@ var (
 	transport         http.RoundTripper
 	transportNoProxy  *http.Transport
 	sflightScraper    singleflight.Group
+	remoteZSTDReader  *zstd.Decoder
 )
 
 //go:embed dictionary.bin
@@ -51,9 +52,15 @@ type InstaData struct {
 }
 
 func init() {
+	var err error
 	transport = gzhttp.Transport(http.DefaultTransport, gzhttp.TransportAlwaysDecompress(true))
 	transportNoProxy = http.DefaultTransport.(*http.Transport).Clone()
 	transportNoProxy.Proxy = nil // Skip any proxy
+
+	remoteZSTDReader, err = zstd.NewReader(nil, zstd.WithDecoderLowmem(true), zstd.WithDecoderDicts(zstdDict))
+	if err != nil {
+		panic(err)
+	}
 }
 
 func GetData(postID string) (*InstaData, error) {
@@ -152,13 +159,12 @@ func (i *InstaData) ScrapeData() error {
 		res, err := remoteClient.Do(req)
 		if res != nil && res.StatusCode == 200 {
 			defer res.Body.Close()
-			zstdReader, err := zstd.NewReader(nil, zstd.WithDecoderLowmem(true), zstd.WithDecoderDicts(zstdDict))
 			if err != nil {
 				return err
 			}
 			remoteData, err := io.ReadAll(res.Body)
 			if err == nil {
-				remoteDecomp, err := zstdReader.DecodeAll(remoteData, nil)
+				remoteDecomp, err := remoteZSTDReader.DecodeAll(remoteData, nil)
 				if err != nil {
 					return err
 				}