From 2f8ffb2e677de62b45f29457b8a10d374d402528 Mon Sep 17 00:00:00 2001 From: Jesse Geens Date: Mon, 3 Feb 2025 10:38:18 +0100 Subject: [PATCH 1/9] reference to previous ID --- share/model.go | 1 + 1 file changed, 1 insertion(+) diff --git a/share/model.go b/share/model.go index c509558..d3f9fbb 100644 --- a/share/model.go +++ b/share/model.go @@ -45,6 +45,7 @@ type ProtoShare struct { Permissions uint8 Orphan bool Expiration datatypes.NullTime + PreviousID uint `gorm:"uniqueIndex"` } type Share struct { From 7e4e7530b9abd98dcf8da8aa32180f25a1939ff2 Mon Sep 17 00:00:00 2001 From: Jesse Geens Date: Wed, 8 Jan 2025 10:00:50 +0100 Subject: [PATCH 2/9] Implementing schema migration tool --- cmd/migrator.go | 23 +++++ share/sql/migrate.go | 224 +++++++++++++++++++++++++++++++++++++++++++ share/sql/share.go | 12 +-- 3 files changed, 251 insertions(+), 8 deletions(-) create mode 100644 cmd/migrator.go create mode 100644 share/sql/migrate.go diff --git a/cmd/migrator.go b/cmd/migrator.go new file mode 100644 index 0000000..14d312f --- /dev/null +++ b/cmd/migrator.go @@ -0,0 +1,23 @@ +package main + +import ( + "flag" + "fmt" + + "github.com/cernbox/reva-plugins/share/sql" +) + +func main() { + username := flag.String("username", "cernbox_server", "Database username") + password := flag.String("password", "", "Database password") + host := flag.String("host", "dbod-cboxeos.cern.ch", "Database host") + port := flag.Int("port", 5504, "Database port") + name := flag.String("name", "cernboxngcopy", "Database name") + gatewaysvc := flag.String("gatewaysvc", "localhost:9142", "Gateway service location") + token := flag.String("token", "", "JWT token for gateway svc") + + flag.Parse() + + fmt.Printf("Connecting to %s@%s:%d\n", *username, *host, *port) + sql.RunMigration(*username, *password, *host, *name, *gatewaysvc, *token, *port) +} diff --git a/share/sql/migrate.go b/share/sql/migrate.go new file mode 100644 index 0000000..bfc7a5e --- /dev/null +++ b/share/sql/migrate.go @@ -0,0 +1,224 @@ +package sql + +import ( + "context" + "database/sql" + "fmt" + "os" + "time" + + model "github.com/cernbox/reva-plugins/share" + providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/pkg/errors" + "google.golang.org/grpc/metadata" + "gorm.io/datatypes" + "gorm.io/gorm" +) + +type Migrator struct { + NewDb *gorm.DB + OldDb *sql.DB + ShareMgr *shareMgr + LinkMgr *publicShareMgr +} + +type ShareOrLink struct { + IsShare bool + Share *model.Share + Link *model.PublicLink +} + +func RunMigration(username, password, host, name, gatewaysvc, token string, port int) { + config := map[string]interface{}{ + "engine": "mysql", + "db_username": username, + "db_password": password, + "db_host": host, + "db_port": port, + "db_name": name, + "gatewaysvc": gatewaysvc, + "dry_run": false, + } + tokenlessCtx, cancel := context.WithCancel(context.Background()) + ctx := appctx.ContextSetToken(tokenlessCtx, token) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, token) + defer cancel() + + shareManager, err := NewShareManager(ctx, config) + if err != nil { + fmt.Println("Failed to create shareManager: " + err.Error()) + os.Exit(1) + } + sharemgr := shareManager.(*shareMgr) + oldDb, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", username, password, host, port, name)) + if err != nil { + fmt.Println("Failed to create db: " + err.Error()) + os.Exit(1) + } + migrator := Migrator{ + OldDb: oldDb, + NewDb: sharemgr.db, + ShareMgr: sharemgr, + } + + ch := make(chan *ShareOrLink, 100) + go getAllShares(ctx, migrator, ch) + for share := range ch { + // TODO error handling + if share.IsShare { + fmt.Printf("Creating share %d\n", share.Share.ID) + migrator.NewDb.Create(&share.Share) + } else { + fmt.Printf("Creating share %d\n", share.Link.ID) + migrator.NewDb.Create(&share.Link) + } + } + +} + +func getAllShares(ctx context.Context, migrator Migrator, ch chan *ShareOrLink) { + // First we find out what the highest ID is + count, err := getCount(migrator) + if err != nil { + fmt.Println("Error getting highest id: " + err.Error()) + close(ch) + return + } + fmt.Printf("Migrating %d shares\n", count) + + query := "select id, coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, lower(coalesce(share_with, '')) as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, stime, permissions, share_type, orphan FROM oc_share order by id desc" // AND id=?" + params := []interface{}{} + + res, err := migrator.OldDb.Query(query, params...) + + if err != nil { + fmt.Printf("Fatal error: %s", err.Error()) + close(ch) + return + } + + for res.Next() { + var s OldShareEntry + res.Scan(&s.ID, &s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.STime, &s.Permissions, &s.ShareType, &s.Orphan) + newShare, err := oldShareToNewShare(ctx, migrator, s) + if err == nil { + ch <- newShare + } else { + fmt.Printf("Error occured for share %s: %s\n", s.ID, err.Error()) + } + } + + close(ch) +} + +type OldShareEntry struct { + ID int + UIDOwner string + UIDInitiator string + Prefix string + ItemSource string + ItemType string + ShareWith string + Token string + Expiration string + Permissions int + ShareType int + ShareName string + STime int + FileTarget string + State int + Quicklink bool + Description string + NotifyUploads bool + NotifyUploadsExtraRecipients sql.NullString + Orphan bool +} + +func oldShareToNewShare(ctx context.Context, migrator Migrator, s OldShareEntry) (*ShareOrLink, error) { + expirationDate, expirationError := time.Parse("2006-01-02 15:04:05", s.Expiration) + + protoShare := model.ProtoShare{ + Model: gorm.Model{ + ID: uint(s.ID), + CreatedAt: time.Unix(int64(s.STime), 0), + UpdatedAt: time.Unix(int64(s.STime), 0), + }, + UIDOwner: s.UIDOwner, + UIDInitiator: s.UIDInitiator, + Description: s.Description, + Permissions: uint8(s.Permissions), + Orphan: s.Orphan, // will be re-checked later + Expiration: datatypes.Null[time.Time]{ + V: expirationDate, + Valid: expirationError == nil, + }, + ItemType: model.ItemType(s.ItemType), + InitialPath: "", // set later + Inode: s.ItemSource, + Instance: s.Prefix, + } + + // Getting InitialPath + if !protoShare.Orphan { + path, err := migrator.ShareMgr.getPath(ctx, &providerv1beta1.ResourceId{ + StorageId: protoShare.Instance, + OpaqueId: protoShare.Inode, + }) + if err == nil { + protoShare.InitialPath = path + } else if errors.Is(err, errtypes.NotFound(protoShare.Inode)) { + protoShare.Orphan = true + } else { + // We do not set, because of a general error + fmt.Printf("An error occured while statting (%s, %s): %s\n", protoShare.Instance, protoShare.Inode, err.Error()) + } + } + + // ShareTypeUser = 0 + // ShareTypeGroup = 1 + // ShareTypePublicLink = 3 + // ShareTypeFederatedCloudShare = 6 + // ShareTypeSpaceMembership = 7 + if s.ShareType == 0 || s.ShareType == 1 { + return &ShareOrLink{ + IsShare: true, + Share: &model.Share{ + ProtoShare: protoShare, + ShareWith: s.ShareWith, + SharedWithIsGroup: s.ShareType == 1, + }, + }, nil + } else if s.ShareType == 3 { + notifyUploadsExtraRecipients := "" + if s.NotifyUploadsExtraRecipients.Valid { + notifyUploadsExtraRecipients = s.NotifyUploadsExtraRecipients.String + } + return &ShareOrLink{ + IsShare: false, + Link: &model.PublicLink{ + ProtoShare: protoShare, + Token: s.Token, + Quicklink: s.Quicklink, + NotifyUploads: s.NotifyUploads, + NotifyUploadsExtraRecipients: notifyUploadsExtraRecipients, + Password: s.ShareWith, + LinkName: s.ShareName, + }, + }, nil + } else { + return nil, errors.New("Invalid share type") + } +} + +func getCount(migrator Migrator) (int, error) { + res := 0 + query := "select count(*) from oc_share" + params := []interface{}{} + + if err := migrator.OldDb.QueryRow(query, params...).Scan(&res); err != nil { + return 0, err + } + return res, nil +} diff --git a/share/sql/share.go b/share/sql/share.go index d94eba7..5824db2 100644 --- a/share/sql/share.go +++ b/share/sql/share.go @@ -88,6 +88,10 @@ func NewShareManager(ctx context.Context, m map[string]interface{}) (revashare.M return mgr, nil } +func (m *mgr) getDb() *gorm.DB { + return m.db +} + func (m *shareMgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { user := appctx.ContextMustGetUser(ctx) @@ -101,15 +105,7 @@ func (m *shareMgr) Share(ctx context.Context, md *provider.ResourceInfo, g *coll // check if share already exists. key := &collaboration.ShareKey{ Owner: md.Owner, - ResourceId: md.Id, Grantee: g.Grantee, - } - _, err := m.getShareByKey(ctx, key, true) - // share already exists - // TODO stricter error checking - if err == nil { - return nil, errors.New(errtypes.AlreadyExists(key.String()).Error()) - } var shareWith string if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER { From c38847d615ce50a0664d814ef0c092c43c1068ea Mon Sep 17 00:00:00 2001 From: Jesse Geens Date: Thu, 9 Jan 2025 14:33:38 +0100 Subject: [PATCH 3/9] Created worker pool and migrate states as well --- cmd/migrator.go | 3 +- share/sql/migrate.go | 190 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 187 insertions(+), 6 deletions(-) diff --git a/cmd/migrator.go b/cmd/migrator.go index 14d312f..f7c1973 100644 --- a/cmd/migrator.go +++ b/cmd/migrator.go @@ -15,9 +15,10 @@ func main() { name := flag.String("name", "cernboxngcopy", "Database name") gatewaysvc := flag.String("gatewaysvc", "localhost:9142", "Gateway service location") token := flag.String("token", "", "JWT token for gateway svc") + dryRun := flag.Bool("dryrun", true, "Use dry run?") flag.Parse() fmt.Printf("Connecting to %s@%s:%d\n", *username, *host, *port) - sql.RunMigration(*username, *password, *host, *name, *gatewaysvc, *token, *port) + sql.RunMigration(*username, *password, *host, *name, *gatewaysvc, *token, *port, *dryRun) } diff --git a/share/sql/migrate.go b/share/sql/migrate.go index bfc7a5e..f8d980a 100644 --- a/share/sql/migrate.go +++ b/share/sql/migrate.go @@ -136,7 +136,187 @@ type OldShareEntry struct { Orphan bool } -func oldShareToNewShare(ctx context.Context, migrator Migrator, s OldShareEntry) (*ShareOrLink, error) { +type OldShareState struct { + id int + recipient string + state int +} + +const ( + bufferSize = 10 + numWorkers = 10 +) + +func RunMigration(username, password, host, name, gatewaysvc, token string, port int, dryRun bool) { + // Config + config := map[string]interface{}{ + "engine": "mysql", + "db_username": username, + "db_password": password, + "db_host": host, + "db_port": port, + "db_name": name, + "gatewaysvc": gatewaysvc, + "dry_run": dryRun, + } + // Authenticate to gateway service + tokenlessCtx, cancel := context.WithCancel(context.Background()) + ctx := appctx.ContextSetToken(tokenlessCtx, token) + ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, token) + defer cancel() + + // Set up migrator + shareManager, err := New(ctx, config) + if err != nil { + fmt.Println("Failed to create shareManager: " + err.Error()) + os.Exit(1) + } + sharemgr := shareManager.(*mgr) + oldDb, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", username, password, host, port, name)) + if err != nil { + fmt.Println("Failed to create db: " + err.Error()) + os.Exit(1) + } + migrator := Migrator{ + OldDb: oldDb, + NewDb: sharemgr.db, + ShareMgr: sharemgr, + } + + if dryRun { + migrator.NewDb = migrator.NewDb.Debug() + } + + migrateShares(ctx, migrator) + fmt.Println("---------------------------------") + migrateShareStatuses(ctx, migrator) + +} + +func migrateShares(ctx context.Context, migrator Migrator) { + // Check how many shares are to be migrated + count, err := getCount(migrator, "oc_share") + if err != nil { + fmt.Println("Error getting count: " + err.Error()) + return + } + fmt.Printf("Migrating %d shares\n", count) + + // Get all old shares + query := "select id, coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, lower(coalesce(share_with, '')) as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, stime, permissions, share_type, orphan FROM oc_share order by id desc" // AND id=?" + params := []interface{}{} + + res, err := migrator.OldDb.Query(query, params...) + + if err != nil { + fmt.Printf("Fatal error: %s", err.Error()) + os.Exit(1) + } + + // Create channel for workers + ch := make(chan *OldShareEntry, bufferSize) + defer close(ch) + + // Start all workers + for range numWorkers { + go workerShare(ctx, migrator, ch) + } + + for res.Next() { + var s OldShareEntry + res.Scan(&s.ID, &s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.STime, &s.Permissions, &s.ShareType, &s.Orphan) + if err == nil { + ch <- &s + } else { + fmt.Printf("Error occured for share %d: %s\n", s.ID, err.Error()) + } + } +} + +func migrateShareStatuses(ctx context.Context, migrator Migrator) { + // Check how many shares are to be migrated + count, err := getCount(migrator, "oc_share_status") + if err != nil { + fmt.Println("Error getting count: " + err.Error()) + return + } + fmt.Printf("Migrating %d share statuses\n", count) + + // Get all old shares + query := "select id, coalesce(recipient, '') as recipient, state FROM oc_share_status order by id desc" + params := []interface{}{} + + res, err := migrator.OldDb.Query(query, params...) + + if err != nil { + fmt.Printf("Fatal error: %s", err.Error()) + os.Exit(1) + } + + // Create channel for workers + ch := make(chan *OldShareState, bufferSize) + defer close(ch) + + // Start all workers + for range numWorkers { + go workerState(ctx, migrator, ch) + } + + for res.Next() { + var s OldShareState + res.Scan(&s.id, &s.recipient, &s.state) + if err == nil { + ch <- &s + } else { + fmt.Printf("Error occured for share status%d: %s\n", s.id, err.Error()) + } + } +} + +func workerShare(ctx context.Context, migrator Migrator, ch chan *OldShareEntry) { + for share := range ch { + handleSingleShare(ctx, migrator, share) + } +} + +func workerState(ctx context.Context, migrator Migrator, ch chan *OldShareState) { + for state := range ch { + handleSingleState(ctx, migrator, state) + } +} + +func handleSingleShare(ctx context.Context, migrator Migrator, s *OldShareEntry) { + share, err := oldShareToNewShare(ctx, migrator, s) + if err != nil { + return + } + // TODO error handling + if share.IsShare { + migrator.NewDb.Create(&share.Share) + } else { + migrator.NewDb.Create(&share.Link) + } +} + +func handleSingleState(ctx context.Context, migrator Migrator, s *OldShareState) { + // case collaboration.ShareState_SHARE_STATE_REJECTED: + // state = -1 + // case collaboration.ShareState_SHARE_STATE_ACCEPTED: + // state = 1 + + newShareState := &model.ShareState{ + ShareID: uint(s.id), + Model: gorm.Model{ + ID: uint(s.id), + }, + User: s.recipient, + Hidden: s.state == -1, // Hidden if REJECTED + Synced: true, // for now, we always sync? or not? TODO + } + migrator.NewDb.Create(&newShareState) +} + +func oldShareToNewShare(ctx context.Context, migrator Migrator, s *OldShareEntry) (*ShareOrLink, error) { expirationDate, expirationError := time.Parse("2006-01-02 15:04:05", s.Expiration) protoShare := model.ProtoShare{ @@ -147,7 +327,6 @@ func oldShareToNewShare(ctx context.Context, migrator Migrator, s OldShareEntry) }, UIDOwner: s.UIDOwner, UIDInitiator: s.UIDInitiator, - Description: s.Description, Permissions: uint8(s.Permissions), Orphan: s.Orphan, // will be re-checked later Expiration: datatypes.Null[time.Time]{ @@ -172,7 +351,7 @@ func oldShareToNewShare(ctx context.Context, migrator Migrator, s OldShareEntry) protoShare.Orphan = true } else { // We do not set, because of a general error - fmt.Printf("An error occured while statting (%s, %s): %s\n", protoShare.Instance, protoShare.Inode, err.Error()) + fmt.Printf("An error occured for share %d while statting (%s, %s): %s\n", s.ID, protoShare.Instance, protoShare.Inode, err.Error()) } } @@ -188,6 +367,7 @@ func oldShareToNewShare(ctx context.Context, migrator Migrator, s OldShareEntry) ProtoShare: protoShare, ShareWith: s.ShareWith, SharedWithIsGroup: s.ShareType == 1, + Description: s.Description, }, }, nil } else if s.ShareType == 3 { @@ -212,9 +392,9 @@ func oldShareToNewShare(ctx context.Context, migrator Migrator, s OldShareEntry) } } -func getCount(migrator Migrator) (int, error) { +func getCount(migrator Migrator, table string) (int, error) { res := 0 - query := "select count(*) from oc_share" + query := "select count(*) from " + table params := []interface{}{} if err := migrator.OldDb.QueryRow(query, params...).Scan(&res); err != nil { From 291cfd1f227d3a05e7a256e5a94f3e5891d9dacd Mon Sep 17 00:00:00 2001 From: Jesse Geens Date: Thu, 30 Jan 2025 14:04:08 +0100 Subject: [PATCH 4/9] Better error handling + enfore that token is passed --- cmd/migrator.go | 8 +++++++- share/sql/migrate.go | 23 +++++++++++++---------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/cmd/migrator.go b/cmd/migrator.go index f7c1973..420efa3 100644 --- a/cmd/migrator.go +++ b/cmd/migrator.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "os" "github.com/cernbox/reva-plugins/share/sql" ) @@ -12,13 +13,18 @@ func main() { password := flag.String("password", "", "Database password") host := flag.String("host", "dbod-cboxeos.cern.ch", "Database host") port := flag.Int("port", 5504, "Database port") - name := flag.String("name", "cernboxngcopy", "Database name") + name := flag.String("name", "test", "Database name") gatewaysvc := flag.String("gatewaysvc", "localhost:9142", "Gateway service location") token := flag.String("token", "", "JWT token for gateway svc") dryRun := flag.Bool("dryrun", true, "Use dry run?") flag.Parse() + if *token == "" { + fmt.Println("Please pass a reva token using `-token`") + os.Exit(22) + } + fmt.Printf("Connecting to %s@%s:%d\n", *username, *host, *port) sql.RunMigration(*username, *password, *host, *name, *gatewaysvc, *token, *port, *dryRun) } diff --git a/share/sql/migrate.go b/share/sql/migrate.go index f8d980a..30ae345 100644 --- a/share/sql/migrate.go +++ b/share/sql/migrate.go @@ -288,22 +288,22 @@ func workerState(ctx context.Context, migrator Migrator, ch chan *OldShareState) func handleSingleShare(ctx context.Context, migrator Migrator, s *OldShareEntry) { share, err := oldShareToNewShare(ctx, migrator, s) if err != nil { + fmt.Printf("An error occured while migrating share %ds: %s\n", s.ID, err.Error()) return } - // TODO error handling + var res *gorm.DB if share.IsShare { - migrator.NewDb.Create(&share.Share) + res = migrator.NewDb.Create(&share.Share) } else { - migrator.NewDb.Create(&share.Link) + res = migrator.NewDb.Create(&share.Link) + } + + if res.Error != nil { + fmt.Printf("An error occured while migrating share %ds: %s\n", s.ID, res.Error.Error()) } } func handleSingleState(ctx context.Context, migrator Migrator, s *OldShareState) { - // case collaboration.ShareState_SHARE_STATE_REJECTED: - // state = -1 - // case collaboration.ShareState_SHARE_STATE_ACCEPTED: - // state = 1 - newShareState := &model.ShareState{ ShareID: uint(s.id), Model: gorm.Model{ @@ -311,9 +311,12 @@ func handleSingleState(ctx context.Context, migrator Migrator, s *OldShareState) }, User: s.recipient, Hidden: s.state == -1, // Hidden if REJECTED - Synced: true, // for now, we always sync? or not? TODO + Synced: false, + } + res := migrator.NewDb.Create(&newShareState) + if res.Error != nil { + fmt.Printf("An error occured while migrating share state (%d, %s): %s\n", s.id, s.recipient, res.Error.Error()) } - migrator.NewDb.Create(&newShareState) } func oldShareToNewShare(ctx context.Context, migrator Migrator, s *OldShareEntry) (*ShareOrLink, error) { From 3986d3235e9283d95caeee203efb6fb1db2e4881 Mon Sep 17 00:00:00 2001 From: Jesse Geens Date: Thu, 30 Jan 2025 16:37:03 +0100 Subject: [PATCH 5/9] Use sync.WaitGroup --- share/model.go | 1 - share/sql/migrate.go | 38 +++++++++++++++++++++++++------------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/share/model.go b/share/model.go index d3f9fbb..c509558 100644 --- a/share/model.go +++ b/share/model.go @@ -45,7 +45,6 @@ type ProtoShare struct { Permissions uint8 Orphan bool Expiration datatypes.NullTime - PreviousID uint `gorm:"uniqueIndex"` } type Share struct { diff --git a/share/sql/migrate.go b/share/sql/migrate.go index 30ae345..76b3f6c 100644 --- a/share/sql/migrate.go +++ b/share/sql/migrate.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "os" + "sync" "time" model "github.com/cernbox/reva-plugins/share" @@ -144,7 +145,7 @@ type OldShareState struct { const ( bufferSize = 10 - numWorkers = 10 + numWorkers = 5 ) func RunMigration(username, password, host, name, gatewaysvc, token string, port int, dryRun bool) { @@ -159,6 +160,7 @@ func RunMigration(username, password, host, name, gatewaysvc, token string, port "gatewaysvc": gatewaysvc, "dry_run": dryRun, } + // Authenticate to gateway service tokenlessCtx, cancel := context.WithCancel(context.Background()) ctx := appctx.ContextSetToken(tokenlessCtx, token) @@ -187,6 +189,8 @@ func RunMigration(username, password, host, name, gatewaysvc, token string, port migrator.NewDb = migrator.NewDb.Debug() } + migrator.NewDb.AutoMigrate(&model.Share{}, &model.PublicLink{}, &model.ShareState{}) + migrateShares(ctx, migrator) fmt.Println("---------------------------------") migrateShareStatuses(ctx, migrator) @@ -215,11 +219,11 @@ func migrateShares(ctx context.Context, migrator Migrator) { // Create channel for workers ch := make(chan *OldShareEntry, bufferSize) - defer close(ch) + var wg sync.WaitGroup // Start all workers for range numWorkers { - go workerShare(ctx, migrator, ch) + go workerShare(ctx, migrator, ch, &wg) } for res.Next() { @@ -231,6 +235,9 @@ func migrateShares(ctx context.Context, migrator Migrator) { fmt.Printf("Error occured for share %d: %s\n", s.ID, err.Error()) } } + + close(ch) + wg.Wait() } func migrateShareStatuses(ctx context.Context, migrator Migrator) { @@ -255,11 +262,12 @@ func migrateShareStatuses(ctx context.Context, migrator Migrator) { // Create channel for workers ch := make(chan *OldShareState, bufferSize) - defer close(ch) + + var wg sync.WaitGroup // Start all workers for range numWorkers { - go workerState(ctx, migrator, ch) + go workerState(ctx, migrator, ch, &wg) } for res.Next() { @@ -271,18 +279,24 @@ func migrateShareStatuses(ctx context.Context, migrator Migrator) { fmt.Printf("Error occured for share status%d: %s\n", s.id, err.Error()) } } + close(ch) + wg.Wait() } -func workerShare(ctx context.Context, migrator Migrator, ch chan *OldShareEntry) { +func workerShare(ctx context.Context, migrator Migrator, ch chan *OldShareEntry, wg *sync.WaitGroup) { + wg.Add(1) for share := range ch { handleSingleShare(ctx, migrator, share) } + wg.Done() } -func workerState(ctx context.Context, migrator Migrator, ch chan *OldShareState) { +func workerState(ctx context.Context, migrator Migrator, ch chan *OldShareState, wg *sync.WaitGroup) { + wg.Add(1) for state := range ch { handleSingleState(ctx, migrator, state) } + wg.Done() } func handleSingleShare(ctx context.Context, migrator Migrator, s *OldShareEntry) { @@ -306,12 +320,9 @@ func handleSingleShare(ctx context.Context, migrator Migrator, s *OldShareEntry) func handleSingleState(ctx context.Context, migrator Migrator, s *OldShareState) { newShareState := &model.ShareState{ ShareID: uint(s.id), - Model: gorm.Model{ - ID: uint(s.id), - }, - User: s.recipient, - Hidden: s.state == -1, // Hidden if REJECTED - Synced: false, + User: s.recipient, + Hidden: s.state == -1, // Hidden if REJECTED + Synced: false, } res := migrator.NewDb.Create(&newShareState) if res.Error != nil { @@ -351,6 +362,7 @@ func oldShareToNewShare(ctx context.Context, migrator Migrator, s *OldShareEntry if err == nil { protoShare.InitialPath = path } else if errors.Is(err, errtypes.NotFound(protoShare.Inode)) { + fmt.Printf("Marked share %d as an orphan (%s, %s)\n", s.ID, protoShare.Instance, protoShare.Inode) protoShare.Orphan = true } else { // We do not set, because of a general error From c96dbccc62dd4c36e5c2ce20305202a07d041285 Mon Sep 17 00:00:00 2001 From: Jesse Geens Date: Thu, 13 Feb 2025 09:51:56 +0100 Subject: [PATCH 6/9] Fix bug where token, share_name, notify_uploads and expiration are missing --- share/sql/migrate.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/share/sql/migrate.go b/share/sql/migrate.go index 76b3f6c..5a9abc6 100644 --- a/share/sql/migrate.go +++ b/share/sql/migrate.go @@ -133,7 +133,7 @@ type OldShareEntry struct { Quicklink bool Description string NotifyUploads bool - NotifyUploadsExtraRecipients sql.NullString + NotifyUploadsExtraRecipients string Orphan bool } @@ -207,7 +207,7 @@ func migrateShares(ctx context.Context, migrator Migrator) { fmt.Printf("Migrating %d shares\n", count) // Get all old shares - query := "select id, coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, lower(coalesce(share_with, '')) as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, stime, permissions, share_type, orphan FROM oc_share order by id desc" // AND id=?" + query := "select id, coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, lower(coalesce(share_with, '')) as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(token, '') as token, expiration, stime, permissions, share_type, coalesce(share_name, '') as share_name, notify_uploads, coalesce(notify_uploads_extra_recipients, '') as notify_uploads_extra_recipients, orphan FROM oc_share order by id desc" // AND id=?" params := []interface{}{} res, err := migrator.OldDb.Query(query, params...) @@ -228,7 +228,7 @@ func migrateShares(ctx context.Context, migrator Migrator) { for res.Next() { var s OldShareEntry - res.Scan(&s.ID, &s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.STime, &s.Permissions, &s.ShareType, &s.Orphan) + res.Scan(&s.ID, &s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.Token, &s.Expiration, &s.STime, &s.Permissions, &s.ShareType, &s.ShareName, &s.NotifyUploads, &s.NotifyUploadsExtraRecipients, &s.Orphan) if err == nil { ch <- &s } else { @@ -386,10 +386,6 @@ func oldShareToNewShare(ctx context.Context, migrator Migrator, s *OldShareEntry }, }, nil } else if s.ShareType == 3 { - notifyUploadsExtraRecipients := "" - if s.NotifyUploadsExtraRecipients.Valid { - notifyUploadsExtraRecipients = s.NotifyUploadsExtraRecipients.String - } return &ShareOrLink{ IsShare: false, Link: &model.PublicLink{ @@ -397,7 +393,7 @@ func oldShareToNewShare(ctx context.Context, migrator Migrator, s *OldShareEntry Token: s.Token, Quicklink: s.Quicklink, NotifyUploads: s.NotifyUploads, - NotifyUploadsExtraRecipients: notifyUploadsExtraRecipients, + NotifyUploadsExtraRecipients: s.NotifyUploadsExtraRecipients, Password: s.ShareWith, LinkName: s.ShareName, }, From 8cb2428ed6025f58d2d56e58d2fd866a597c184a Mon Sep 17 00:00:00 2001 From: Jesse Geens Date: Thu, 20 Feb 2025 10:37:43 +0100 Subject: [PATCH 7/9] logline for debugging --- go.mod | 2 +- go.sum | 22 ++-------------------- share/sql/migrate.go | 28 ++++++++++++++++++---------- 3 files changed, 21 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index c991d06..e78af91 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,6 @@ require ( github.com/rs/zerolog v1.33.0 google.golang.org/genproto v0.0.0-20241209162323-e6fa225c2576 google.golang.org/grpc v1.69.4 - gorm.io/datatypes v1.2.4 gorm.io/driver/mysql v1.5.7 gorm.io/driver/sqlite v1.5.7 gorm.io/gorm v1.25.12 @@ -64,6 +63,7 @@ require ( golang.org/x/text v0.22.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d // indirect google.golang.org/protobuf v1.36.4 // indirect + gorm.io/datatypes v1.2.5 ) replace github.com/cs3org/reva => ../reva diff --git a/go.sum b/go.sum index 1401483..11cc700 100644 --- a/go.sum +++ b/go.sum @@ -63,10 +63,6 @@ github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1 github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= -github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= -github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s= @@ -84,14 +80,6 @@ github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= -github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= -github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -127,8 +115,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/microsoft/go-mssqldb v0.17.0 h1:Fto83dMZPnYv1Zwx5vHHxpNraeEaUlQ/hhHLgZiaenE= -github.com/microsoft/go-mssqldb v0.17.0/go.mod h1:OkoNGhGEs8EZqchVTtochlXruEhEOaO4S0d2sB5aeGQ= github.com/mileusna/useragent v1.3.5 h1:SJM5NzBmh/hO+4LGeATKpaEX9+b4vcGg2qXGLiNGDws= github.com/mileusna/useragent v1.3.5/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -291,16 +277,12 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/datatypes v1.2.4 h1:uZmGAcK/QZ0uyfCuVg0VQY1ZmV9h1fuG0tMwKByO1z4= -gorm.io/datatypes v1.2.4/go.mod h1:f4BsLcFAX67szSv8svwLRjklArSHAvHLeE3pXAS5DZI= +gorm.io/datatypes v1.2.5 h1:9UogU3jkydFVW1bIVVeoYsTpLRgwDVW3rHfJG6/Ek9I= +gorm.io/datatypes v1.2.5/go.mod h1:I5FUdlKpLb5PMqeMQhm30CQ6jXP8Rj89xkTeCSAaAD4= gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= -gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U= -gorm.io/driver/postgres v1.5.0/go.mod h1:FUZXzO+5Uqg5zzwzv4KK49R8lvGIyscBOqYrtI1Ce9A= gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I= gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= -gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0= -gorm.io/driver/sqlserver v1.4.1/go.mod h1:DJ4P+MeZbc5rvY58PnmN1Lnyvb5gw5NPzGshHDnJLig= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= diff --git a/share/sql/migrate.go b/share/sql/migrate.go index 5a9abc6..047c115 100644 --- a/share/sql/migrate.go +++ b/share/sql/migrate.go @@ -123,7 +123,7 @@ type OldShareEntry struct { ItemType string ShareWith string Token string - Expiration string + Expiration sql.NullTime Permissions int ShareType int ShareName string @@ -174,7 +174,7 @@ func RunMigration(username, password, host, name, gatewaysvc, token string, port os.Exit(1) } sharemgr := shareManager.(*mgr) - oldDb, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", username, password, host, port, name)) + oldDb, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", username, password, host, port, name)) if err != nil { fmt.Println("Failed to create db: " + err.Error()) os.Exit(1) @@ -331,21 +331,28 @@ func handleSingleState(ctx context.Context, migrator Migrator, s *OldShareState) } func oldShareToNewShare(ctx context.Context, migrator Migrator, s *OldShareEntry) (*ShareOrLink, error) { - expirationDate, expirationError := time.Parse("2006-01-02 15:04:05", s.Expiration) - + var createdAt, updatedAt time.Time + if s.STime != 0 { + createdAt = time.Unix(int64(s.STime), 0) + updatedAt = time.Unix(int64(s.STime), 0) + } else { + createdAt = time.Now() + updatedAt = time.Now() + fmt.Printf("WARN: STime not set for share %d\n", s.ID) + } protoShare := model.ProtoShare{ Model: gorm.Model{ ID: uint(s.ID), - CreatedAt: time.Unix(int64(s.STime), 0), - UpdatedAt: time.Unix(int64(s.STime), 0), + CreatedAt: createdAt, + UpdatedAt: updatedAt, }, UIDOwner: s.UIDOwner, UIDInitiator: s.UIDInitiator, Permissions: uint8(s.Permissions), Orphan: s.Orphan, // will be re-checked later - Expiration: datatypes.Null[time.Time]{ - V: expirationDate, - Valid: expirationError == nil, + Expiration: datatypes.NullTime{ + Valid: s.Expiration.Valid, + V: s.Expiration.Time, }, ItemType: model.ItemType(s.ItemType), InitialPath: "", // set later @@ -366,7 +373,7 @@ func oldShareToNewShare(ctx context.Context, migrator Migrator, s *OldShareEntry protoShare.Orphan = true } else { // We do not set, because of a general error - fmt.Printf("An error occured for share %d while statting (%s, %s): %s\n", s.ID, protoShare.Instance, protoShare.Inode, err.Error()) + // fmt.Printf("An error occured for share %d while statting (%s, %s): %s\n", s.ID, protoShare.Instance, protoShare.Inode, err.Error()) } } @@ -386,6 +393,7 @@ func oldShareToNewShare(ctx context.Context, migrator Migrator, s *OldShareEntry }, }, nil } else if s.ShareType == 3 { + return &ShareOrLink{ IsShare: false, Link: &model.PublicLink{ From c27f53e280bc68070d212cb904d0a59ec3b64fa7 Mon Sep 17 00:00:00 2001 From: Jesse Geens Date: Tue, 4 Mar 2025 13:38:12 +0100 Subject: [PATCH 8/9] Merge public_links into migrator --- share/sql/migrate.go | 103 ++++++------------------------------------- share/sql/share.go | 11 +++-- 2 files changed, 21 insertions(+), 93 deletions(-) diff --git a/share/sql/migrate.go b/share/sql/migrate.go index 047c115..d93f918 100644 --- a/share/sql/migrate.go +++ b/share/sql/migrate.go @@ -31,89 +31,6 @@ type ShareOrLink struct { Link *model.PublicLink } -func RunMigration(username, password, host, name, gatewaysvc, token string, port int) { - config := map[string]interface{}{ - "engine": "mysql", - "db_username": username, - "db_password": password, - "db_host": host, - "db_port": port, - "db_name": name, - "gatewaysvc": gatewaysvc, - "dry_run": false, - } - tokenlessCtx, cancel := context.WithCancel(context.Background()) - ctx := appctx.ContextSetToken(tokenlessCtx, token) - ctx = metadata.AppendToOutgoingContext(ctx, appctx.TokenHeader, token) - defer cancel() - - shareManager, err := NewShareManager(ctx, config) - if err != nil { - fmt.Println("Failed to create shareManager: " + err.Error()) - os.Exit(1) - } - sharemgr := shareManager.(*shareMgr) - oldDb, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", username, password, host, port, name)) - if err != nil { - fmt.Println("Failed to create db: " + err.Error()) - os.Exit(1) - } - migrator := Migrator{ - OldDb: oldDb, - NewDb: sharemgr.db, - ShareMgr: sharemgr, - } - - ch := make(chan *ShareOrLink, 100) - go getAllShares(ctx, migrator, ch) - for share := range ch { - // TODO error handling - if share.IsShare { - fmt.Printf("Creating share %d\n", share.Share.ID) - migrator.NewDb.Create(&share.Share) - } else { - fmt.Printf("Creating share %d\n", share.Link.ID) - migrator.NewDb.Create(&share.Link) - } - } - -} - -func getAllShares(ctx context.Context, migrator Migrator, ch chan *ShareOrLink) { - // First we find out what the highest ID is - count, err := getCount(migrator) - if err != nil { - fmt.Println("Error getting highest id: " + err.Error()) - close(ch) - return - } - fmt.Printf("Migrating %d shares\n", count) - - query := "select id, coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, lower(coalesce(share_with, '')) as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, stime, permissions, share_type, orphan FROM oc_share order by id desc" // AND id=?" - params := []interface{}{} - - res, err := migrator.OldDb.Query(query, params...) - - if err != nil { - fmt.Printf("Fatal error: %s", err.Error()) - close(ch) - return - } - - for res.Next() { - var s OldShareEntry - res.Scan(&s.ID, &s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.STime, &s.Permissions, &s.ShareType, &s.Orphan) - newShare, err := oldShareToNewShare(ctx, migrator, s) - if err == nil { - ch <- newShare - } else { - fmt.Printf("Error occured for share %s: %s\n", s.ID, err.Error()) - } - } - - close(ch) -} - type OldShareEntry struct { ID int UIDOwner string @@ -168,12 +85,20 @@ func RunMigration(username, password, host, name, gatewaysvc, token string, port defer cancel() // Set up migrator - shareManager, err := New(ctx, config) + shareManager, err := NewShareManager(ctx, config) if err != nil { fmt.Println("Failed to create shareManager: " + err.Error()) os.Exit(1) } - sharemgr := shareManager.(*mgr) + sharemgr := shareManager.(*shareMgr) + + linkManager, err := NewPublicShareManager(ctx, config) + if err != nil { + fmt.Println("Failed to create shareManager: " + err.Error()) + os.Exit(1) + } + linkmgr := linkManager.(*publicShareMgr) + oldDb, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", username, password, host, port, name)) if err != nil { fmt.Println("Failed to create db: " + err.Error()) @@ -183,6 +108,7 @@ func RunMigration(username, password, host, name, gatewaysvc, token string, port OldDb: oldDb, NewDb: sharemgr.db, ShareMgr: sharemgr, + LinkMgr: linkmgr, } if dryRun { @@ -371,10 +297,9 @@ func oldShareToNewShare(ctx context.Context, migrator Migrator, s *OldShareEntry } else if errors.Is(err, errtypes.NotFound(protoShare.Inode)) { fmt.Printf("Marked share %d as an orphan (%s, %s)\n", s.ID, protoShare.Instance, protoShare.Inode) protoShare.Orphan = true - } else { - // We do not set, because of a general error - // fmt.Printf("An error occured for share %d while statting (%s, %s): %s\n", s.ID, protoShare.Instance, protoShare.Inode, err.Error()) - } + } // else { + // We do not set, because of a general error + // } } // ShareTypeUser = 0 diff --git a/share/sql/share.go b/share/sql/share.go index 5824db2..a6b7fb6 100644 --- a/share/sql/share.go +++ b/share/sql/share.go @@ -88,10 +88,6 @@ func NewShareManager(ctx context.Context, m map[string]interface{}) (revashare.M return mgr, nil } -func (m *mgr) getDb() *gorm.DB { - return m.db -} - func (m *shareMgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { user := appctx.ContextMustGetUser(ctx) @@ -105,7 +101,14 @@ func (m *shareMgr) Share(ctx context.Context, md *provider.ResourceInfo, g *coll // check if share already exists. key := &collaboration.ShareKey{ Owner: md.Owner, + ResourceId: md.Id, Grantee: g.Grantee, + } + _, err := m.getShareByKey(ctx, key, true) + // share already exists + if err == nil { + return nil, errors.New(errtypes.AlreadyExists(key.String()).Error()) + } var shareWith string if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER { From d7ea2b467094a7506267b8ff373b7e01b7b0f345 Mon Sep 17 00:00:00 2001 From: Jesse Geens Date: Fri, 7 Mar 2025 09:31:38 +0100 Subject: [PATCH 9/9] Updated migrator to also create entry in share_ids --- share/sql/migrate.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/share/sql/migrate.go b/share/sql/migrate.go index d93f918..4590610 100644 --- a/share/sql/migrate.go +++ b/share/sql/migrate.go @@ -115,7 +115,7 @@ func RunMigration(username, password, host, name, gatewaysvc, token string, port migrator.NewDb = migrator.NewDb.Debug() } - migrator.NewDb.AutoMigrate(&model.Share{}, &model.PublicLink{}, &model.ShareState{}) + migrator.NewDb.AutoMigrate(&model.ShareID{}, &model.Share{}, &model.PublicLink{}, &model.ShareState{}) migrateShares(ctx, migrator) fmt.Println("---------------------------------") @@ -231,7 +231,16 @@ func handleSingleShare(ctx context.Context, migrator Migrator, s *OldShareEntry) fmt.Printf("An error occured while migrating share %ds: %s\n", s.ID, err.Error()) return } - var res *gorm.DB + + shareId := model.ShareID{ + ID: uint(s.ID), + } + res := migrator.NewDb.Create(&shareId) + if res.Error != nil { + fmt.Printf("An error occured while creating ID for share %ds: %s\n", s.ID, res.Error.Error()) + return + } + if share.IsShare { res = migrator.NewDb.Create(&share.Share) } else { @@ -266,9 +275,10 @@ func oldShareToNewShare(ctx context.Context, migrator Migrator, s *OldShareEntry updatedAt = time.Now() fmt.Printf("WARN: STime not set for share %d\n", s.ID) } + protoShare := model.ProtoShare{ - Model: gorm.Model{ - ID: uint(s.ID), + BaseModel: model.BaseModel{ + Id: uint(s.ID), CreatedAt: createdAt, UpdatedAt: updatedAt, },