From 9d6991d3e82faa48f572dca79f1282275e470758 Mon Sep 17 00:00:00 2001 From: kawasaki Date: Thu, 16 May 2024 00:27:53 +0900 Subject: [PATCH 1/2] chore(web, server): add migration for ga support --- .../migration/240515221450_copy_scene_id.go | 98 +++++++++++++++++++ .../240515221450_copy_scene_id_test.go | 97 ++++++++++++++++++ .../mongo/migration/migrations.go | 15 +-- .../infrastructure/mongo/mongodoc/project.go | 2 +- 4 files changed, 204 insertions(+), 8 deletions(-) create mode 100644 server/internal/infrastructure/mongo/migration/240515221450_copy_scene_id.go create mode 100644 server/internal/infrastructure/mongo/migration/240515221450_copy_scene_id_test.go diff --git a/server/internal/infrastructure/mongo/migration/240515221450_copy_scene_id.go b/server/internal/infrastructure/mongo/migration/240515221450_copy_scene_id.go new file mode 100644 index 0000000000..14fd988972 --- /dev/null +++ b/server/internal/infrastructure/mongo/migration/240515221450_copy_scene_id.go @@ -0,0 +1,98 @@ +package migration + +import ( + "context" + "fmt" + + "github.com/reearth/reearth/server/internal/infrastructure/mongo/mongodoc" + "github.com/reearth/reearth/server/pkg/id" + "github.com/reearth/reearthx/log" + "github.com/reearth/reearthx/mongox" + "go.mongodb.org/mongo-driver/bson" +) + +type seed struct { + projectId id.ProjectID + sceneId id.SceneID +} + +func CopySceneId(ctx context.Context, c DBClient) error { + col := c.WithCollection("scene") + + var seeds []seed + + col.Find(ctx, bson.M{}, &mongox.BatchConsumer{ + Size: 1000, + Callback: func(rows []bson.Raw) error { + log.Infofc(ctx, "log: migration: CopySceneId: hit scenes: %d\n", len(rows)) + + for _, row := range rows { + log.Debugfc(ctx, "log: migration: CopySceneId: row: %s\n", row) + var doc mongodoc.SceneDocument + if err := bson.Unmarshal(row, &doc); err != nil { + return err + } + + log.Debugfc(ctx, "log: migration: CopySceneId: doc: %s\n", doc) + + pid, err := id.ProjectIDFrom(doc.Project) + if err != nil { + log.Errorfc(ctx, "log: migration: CopySceneId: project id error: %s\n", err) + } + sid, err := id.SceneIDFrom(doc.ID) + if err != nil { + log.Errorfc(ctx, "log: migration: CopySceneId: scene id error: %s\n", err) + } + + seeds = append(seeds, seed{pid, sid}) + } + + return nil + }, + }) + + col = c.WithCollection("project") + ids := make([]string, 0, len(seeds)) + newRows := make([]interface{}, 0, len(seeds)) + + for _, s := range seeds { + err := col.Find(ctx, bson.M{"id": s.projectId.String()}, &mongox.BatchConsumer{ + Size: 1000, + Callback: func(rows []bson.Raw) error { + if len(rows) == 0 { + log.Debugfc(ctx, "log: migration: CopySceneId: project not found: %s\n", s.projectId) + return nil + } + + if len(rows) > 1 { + return fmt.Errorf("project found multiple: %s", s.projectId) + } + + log.Debugfc(ctx, "log: migration: CopySceneId: project found: %s\n", s.projectId) + + p := rows[0] + var doc mongodoc.ProjectDocument + if err := bson.Unmarshal(p, &doc); err != nil { + return err + } + + if doc.Scene != "" { + log.Debugfc(ctx, "log: migration: CopySceneId: project already has scene: %s\n", doc.Scene) + return nil + } + + doc.Scene = s.sceneId.String() + ids = append(ids, doc.ID) + newRows = append(newRows, doc) + return nil + }, + }) + + if err != nil { + log.Errorfc(ctx, "log: migration: CopySceneId: project find error: %s\n", err) + continue + } + } + + return col.SaveAll(ctx, ids, newRows) +} diff --git a/server/internal/infrastructure/mongo/migration/240515221450_copy_scene_id_test.go b/server/internal/infrastructure/mongo/migration/240515221450_copy_scene_id_test.go new file mode 100644 index 0000000000..5ff8c30684 --- /dev/null +++ b/server/internal/infrastructure/mongo/migration/240515221450_copy_scene_id_test.go @@ -0,0 +1,97 @@ +package migration + +import ( + "context" + "testing" + + "github.com/reearth/reearth/server/internal/infrastructure/mongo/mongodoc" + "github.com/reearth/reearth/server/pkg/id" + "github.com/reearth/reearthx/log" + "github.com/reearth/reearthx/mongox" + "github.com/reearth/reearthx/mongox/mongotest" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/bson" +) + +func init() { + mongotest.Env = "REEARTH_DB" +} + +func TestCopySceneId(t *testing.T) { + ctx := context.Background() + log.Infofc(ctx, "migration: CopySceneId test") + + db := mongotest.Connect(t)(t) + c := mongox.NewClientWithDatabase(db) + + // arrange + type seed struct { + projectId id.ProjectID + sceneId id.SceneID + } + seed1, seed2, seed3 := seed{ + projectId: id.NewProjectID(), + sceneId: id.NewSceneID(), + }, seed{ + projectId: id.NewProjectID(), + sceneId: id.NewSceneID(), + }, seed{ + projectId: id.NewProjectID(), + sceneId: id.NewSceneID(), + } + + _, _ = db.Collection("project").InsertMany(ctx, []any{ + mongodoc.ProjectDocument{ + ID: seed1.projectId.String(), + }, + mongodoc.ProjectDocument{ + ID: seed2.projectId.String(), + }, + mongodoc.ProjectDocument{ + ID: seed3.projectId.String(), + }, + }) + + _, _ = db.Collection("scene").InsertMany(ctx, []any{ + mongodoc.SceneDocument{ + ID: seed1.sceneId.String(), + Project: seed1.projectId.String(), + }, + mongodoc.SceneDocument{ + ID: seed2.sceneId.String(), + Project: seed2.projectId.String(), + }, + mongodoc.SceneDocument{ + ID: seed3.sceneId.String(), + Project: seed3.projectId.String(), + }, + }) + + // act + assert.NoError(t, CopySceneId(ctx, c)) + + // assert + var projects []mongodoc.ProjectDocument + assert.NoError(t, c.WithCollection("project").Find(ctx, bson.M{}, &mongox.BatchConsumer{ + Size: 1000, + Callback: func(rows []bson.Raw) error { + for _, row := range rows { + var doc mongodoc.ProjectDocument + if err := bson.Unmarshal(row, &doc); err != nil { + return err + } + projects = append(projects, doc) + } + return nil + }, + })) + assert.Len(t, projects, 3) + for _, s := range []seed{seed1, seed2, seed3} { + p, ok := lo.Find(projects, func(doc mongodoc.ProjectDocument) bool { + return s.projectId.String() == doc.ID + }) + assert.True(t, ok) + assert.Equal(t, s.sceneId.String(), p.Scene) + } +} diff --git a/server/internal/infrastructure/mongo/migration/migrations.go b/server/internal/infrastructure/mongo/migration/migrations.go index e6ccbc8e35..65daa14eab 100644 --- a/server/internal/infrastructure/mongo/migration/migrations.go +++ b/server/internal/infrastructure/mongo/migration/migrations.go @@ -10,11 +10,12 @@ import "github.com/reearth/reearthx/usecasex/migration" // If the migration takes too long, the deployment may fail in a serverless environment. // Set the batch size to as large a value as possible without using up the RAM of the deployment destination. var migrations = migration.Migrations[DBClient]{ - 201217132559: AddSceneWidgetId, - 201217193948: AddSceneDefaultTile, - 210310145844: RemovePreviewToken, - 210730175108: AddSceneAlignSystem, - 220214180713: SplitSchemaOfProperties, - 220309174648: AddSceneFieldToPropertySchema, - 221028204300: MoveTerrainProperties, + 201217132559: AddSceneWidgetId, + 201217193948: AddSceneDefaultTile, + 210310145844: RemovePreviewToken, + 210730175108: AddSceneAlignSystem, + 220214180713: SplitSchemaOfProperties, + 220309174648: AddSceneFieldToPropertySchema, + 221028204300: MoveTerrainProperties, + 240515221450: CopySceneId, } diff --git a/server/internal/infrastructure/mongo/mongodoc/project.go b/server/internal/infrastructure/mongo/mongodoc/project.go index 433555908e..01a003748b 100644 --- a/server/internal/infrastructure/mongo/mongodoc/project.go +++ b/server/internal/infrastructure/mongo/mongodoc/project.go @@ -33,7 +33,7 @@ type ProjectDocument struct { CoreSupport bool EnableGA bool TrackingID string - // Scene string + Scene string } type ProjectConsumer = Consumer[*ProjectDocument, *project.Project] From 634a084feb25aa4580d18bcf3c8fe0a28ff771c0 Mon Sep 17 00:00:00 2001 From: kawasaki Date: Wed, 22 May 2024 22:00:59 +0900 Subject: [PATCH 2/2] fix: return err --- .../mongo/migration/240515221450_copy_scene_id.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/internal/infrastructure/mongo/migration/240515221450_copy_scene_id.go b/server/internal/infrastructure/mongo/migration/240515221450_copy_scene_id.go index 14fd988972..ba00f0e637 100644 --- a/server/internal/infrastructure/mongo/migration/240515221450_copy_scene_id.go +++ b/server/internal/infrastructure/mongo/migration/240515221450_copy_scene_id.go @@ -21,7 +21,7 @@ func CopySceneId(ctx context.Context, c DBClient) error { var seeds []seed - col.Find(ctx, bson.M{}, &mongox.BatchConsumer{ + err := col.Find(ctx, bson.M{}, &mongox.BatchConsumer{ Size: 1000, Callback: func(rows []bson.Raw) error { log.Infofc(ctx, "log: migration: CopySceneId: hit scenes: %d\n", len(rows)) @@ -37,11 +37,11 @@ func CopySceneId(ctx context.Context, c DBClient) error { pid, err := id.ProjectIDFrom(doc.Project) if err != nil { - log.Errorfc(ctx, "log: migration: CopySceneId: project id error: %s\n", err) + return fmt.Errorf("migration: CopySceneId: project id error: %s\n", err) } sid, err := id.SceneIDFrom(doc.ID) if err != nil { - log.Errorfc(ctx, "log: migration: CopySceneId: scene id error: %s\n", err) + return fmt.Errorf("migration: CopySceneId: scene id error: %s\n", err) } seeds = append(seeds, seed{pid, sid}) @@ -51,6 +51,10 @@ func CopySceneId(ctx context.Context, c DBClient) error { }, }) + if err != nil { + return err + } + col = c.WithCollection("project") ids := make([]string, 0, len(seeds)) newRows := make([]interface{}, 0, len(seeds)) @@ -73,7 +77,7 @@ func CopySceneId(ctx context.Context, c DBClient) error { p := rows[0] var doc mongodoc.ProjectDocument if err := bson.Unmarshal(p, &doc); err != nil { - return err + return fmt.Errorf("project unmarshal error: %s\n", err) } if doc.Scene != "" { @@ -89,8 +93,7 @@ func CopySceneId(ctx context.Context, c DBClient) error { }) if err != nil { - log.Errorfc(ctx, "log: migration: CopySceneId: project find error: %s\n", err) - continue + return err } }