use data from redis to populate new db

embedg-server/db/postgres/transfer/transfer.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"database/sql"
 	"encoding/json"
+	"fmt"
 	"log"
 	"time"
 
 	"github.com/lib/pq"
 	"github.com/merlinfuchs/discordgo"
 	"github.com/merlinfuchs/embed-generator/embedg-server/actions"
 	"github.com/merlinfuchs/embed-generator/embedg-server/db/postgres"
+	"github.com/redis/go-redis/v9"
 	"github.com/spf13/viper"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -37,6 +39,10 @@ type OldUsers struct {
 func TransferDB() {
 	pg := postgres.NewPostgresStore()
 
+	redisDB := redis.NewClient(&redis.Options{
+		Addr: "localhost:6379",
+	})
+
 	mongoURL := viper.GetString("mongo.url")
 	if mongoURL == "" {
 		mongoURL = "mongodb://localhost:27017"
@@ -49,9 +55,11 @@ func TransferDB() {
 
 	coll := mongo.Database("embedg").Collection("messages")
 
+	var skip int64 = 0
+
 	cursor, err := coll.Find(context.TODO(), bson.M{
 		// "owner_id": "386861188891279362",
-	})
+	}, options.Find().SetBatchSize(100).SetSkip(skip).SetSort(bson.M{"updated_at": -1}))
 	if err != nil {
 		panic(err)
 	}
@@ -61,6 +69,7 @@ func TransferDB() {
 		panic(err)
 	}
 
+	i := skip
 	for cursor.Next(context.Background()) {
 		var result OldSavedMessage
 		if err := cursor.Decode(&result); err != nil {
@@ -73,23 +82,48 @@ func TransferDB() {
 				panic(err)
 			}
 
-			user, err := session.User(result.OwnerID)
+			raw, err := redisDB.Get(context.Background(), "users:"+result.OwnerID).Result()
 			if err != nil {
-				panic(err)
+				if err != redis.Nil {
+					panic(err)
+				}
+
+				user, err := session.User(result.OwnerID)
+				if err != nil {
+					panic(err)
+				}
+
+				fmt.Println("fetched user")
+
+				pg.Q.UpsertUser(context.TODO(), postgres.UpsertUserParams{
+					ID:            user.ID,
+					Name:          user.Username,
+					Discriminator: user.Discriminator,
+					Avatar:        sql.NullString{String: user.Avatar, Valid: user.Avatar != ""},
+				})
+			} else {
+				var user OldUsers
+				err = json.Unmarshal([]byte(raw), &user)
+				if err != nil {
+					panic(err)
+				}
+
+				fmt.Println("got user from redis")
+
+				pg.Q.UpsertUser(context.TODO(), postgres.UpsertUserParams{
+					ID:            user.ID,
+					Name:          user.Username,
+					Discriminator: user.Discriminator,
+					Avatar:        sql.NullString{String: user.Avatar, Valid: user.Avatar != ""},
+				})
 			}
-
-			pg.Q.UpsertUser(context.TODO(), postgres.UpsertUserParams{
-				ID:            user.ID,
-				Name:          user.Username,
-				Discriminator: user.Discriminator,
-				Avatar:        sql.NullString{String: user.Avatar, Valid: user.Avatar != ""},
-			})
 		}
 
 		var msgData actions.MessageWithActions
 		err = json.Unmarshal([]byte(result.PayloadJSON), &msgData)
 		if err != nil {
-			panic(err)
+			fmt.Println("Failed to unmarshal message", result.ID)
+			continue
 		}
 
 		if msgData.Content == "" && len(msgData.Components) == 0 && len(msgData.Embeds) == 0 {
@@ -125,5 +159,8 @@ func TransferDB() {
 			}
 			panic(err)
 		}
+
+		i++
+		fmt.Println(i)
 	}
 }

embedg-server/go.mod

@@ -23,6 +23,8 @@ require (
 
 require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
@@ -43,6 +45,7 @@ require (
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/redis/go-redis/v9 v9.0.4 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect
 	github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect

embedg-server/go.sum

@@ -183,9 +183,12 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
 github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
 github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
@@ -352,6 +355,8 @@ github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27
 github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
 github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dhui/dktest v0.3.10 h1:0frpeeoM9pHouHjhLeZDuDTJ0PqjDTrycaHaMmkJAo8=
 github.com/dhui/dktest v0.3.10/go.mod h1:h5Enh0nG3Qbo9WjNFRrwmKUaePEBhXMOygbz3Ww7Sz0=
@@ -1004,6 +1009,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/ravener/discord-oauth2 v0.0.0-20220615092331-f6a9839c223e h1:7t1Ur+etUw2hE1GgSNWJGNi/D8zBMqwwkGwTiE/mWTE=
 github.com/ravener/discord-oauth2 v0.0.0-20220615092331-f6a9839c223e/go.mod h1:P/mZMYLZ87lqRSECEWsOqywGrO1hlZkk9RTwEw35IP4=
+github.com/redis/go-redis/v9 v9.0.4 h1:FC82T+CHJ/Q/PdyLW++GeCO+Ol59Y4T7R4jbgjvktgc=
+github.com/redis/go-redis/v9 v9.0.4/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=