Skip to content

Commit

Permalink
Marshal data for every scrape type
Browse files Browse the repository at this point in the history
This fixes 7660c93 as replace is done after marshalling, move marshalling after replacement of cdn hostname
  • Loading branch information
Wikidepia committed Jun 20, 2024
1 parent 6aaa329 commit 9813d0b
Showing 1 changed file with 9 additions and 10 deletions.
19 changes: 9 additions & 10 deletions handlers/data/data.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,6 @@ func (i *InstaData) GetData(postID string) error {
}

// Scrape from remote scraper, if available
var bb []byte // marshaled data
if len(RemoteScraperAddr) > 0 {
req, res := fasthttp.AcquireRequest(), fasthttp.AcquireResponse()
defer func() {
Expand All @@ -79,8 +78,8 @@ func (i *InstaData) GetData(postID string) error {
req.Header.Set("Accept-Encoding", "gzip, deflate, br")
req.SetRequestURI(RemoteScraperAddr + "/scrape/" + postID)
if err := client.DoTimeout(req, res, timeout); err == nil && res.StatusCode() == fasthttp.StatusOK {
bb, _ = res.BodyGunzip()
if err := binary.Unmarshal(bb, i); err == nil {
iDataGunzip, _ := res.BodyGunzip()
if err := binary.Unmarshal(iDataGunzip, i); err == nil {
log.Info().Str("postID", postID).Msg("Data parsed from remote scraper")
}
}
Expand Down Expand Up @@ -132,17 +131,11 @@ func (i *InstaData) GetData(postID string) error {
URL: mediaURL.String(),
})
}

bb, err = binary.Marshal(i)
if err != nil {
log.Error().Str("postID", postID).Err(err).Msg("Failed to marshal data")
return err
}
}

// Replace all media urls cdn to scontent.cdninstagram.com
for n, media := range i.Medias {
u, err := url.Parse(string(media.URL))
u, err := url.Parse(media.URL)
if err != nil {
log.Error().Str("postID", postID).Err(err).Msg("Failed to parse media URL")
return err
Expand All @@ -151,6 +144,12 @@ func (i *InstaData) GetData(postID string) error {
i.Medias[n].URL = u.String()
}

bb, err := binary.Marshal(i)
if err != nil {
log.Error().Str("postID", postID).Err(err).Msg("Failed to marshal data")
return err
}

batch := DB.NewBatch()
// Write cache to DB
if err := batch.Set(utils.S2B(postID), bb, pebble.Sync); err != nil {
Expand Down

0 comments on commit 9813d0b

Please sign in to comment.