From 847db6c209452250ca7be673388def5fb146aac1 Mon Sep 17 00:00:00 2001 From: yk-eukarya <81808708+yk-eukarya@users.noreply.github.com> Date: Fri, 28 Jul 2023 09:15:46 +0300 Subject: [PATCH] feat(server): extension system (#584) --- server/.env.example | 4 + server/internal/app/config/config.go | 3 + server/internal/app/config/config_test.go | 5 + server/internal/app/repo.go | 2 +- .../internal/infrastructure/adapter/plugin.go | 2 +- .../infrastructure/mongo/container.go | 45 +++++++++ server/internal/usecase/interactor/scene.go | 27 +++++- server/internal/usecase/repo/container.go | 3 + server/pkg/plugin/manifest/parser.go | 40 ++++++++ server/pkg/plugin/manifest/parser_test.go | 94 +++++++++++++++++++ 10 files changed, 221 insertions(+), 4 deletions(-) diff --git a/server/.env.example b/server/.env.example index b3d91b5b33..660da16c9e 100644 --- a/server/.env.example +++ b/server/.env.example @@ -74,3 +74,7 @@ REEARTH_SES_NAME= # Storage Google GCS #REEARTH_GCS_BUCKETNAME=bucket_name #REEARTH_GCS_PUBLICATIONCACHECONTROL= + +# Extension plugin url as csv +# each path should contais `reearth.yml` file +REEARTH_EXT_PLUGIN=http://fileserve.local:8090/pluging-01,http://fileserve.local:8090/pluging-02 diff --git a/server/internal/app/config/config.go b/server/internal/app/config/config.go index ae032bfd23..c24b0320c8 100644 --- a/server/internal/app/config/config.go +++ b/server/internal/app/config/config.go @@ -60,6 +60,9 @@ type Config struct { Auth_TTL *int Auth_ClientID *string Auth_JWKSURI *string + + // system extensions + Ext_Plugin []string } func ReadConfig(debug bool) (*Config, error) { diff --git a/server/internal/app/config/config_test.go b/server/internal/app/config/config_test.go index 7d3845147b..9412d09050 100644 --- a/server/internal/app/config/config_test.go +++ b/server/internal/app/config/config_test.go @@ -49,6 +49,11 @@ func TestReadConfig(t *testing.T) { }, cfg.Auths()) assert.Equal(t, "foo", cfg.Auth_AUD) assert.Equal(t, map[string]string{}, cfg.Web) + + t.Setenv("REEARTH_EXT_PLUGIN", "https://hoge.com/myplugin,https://hoge.com/myplugin2") + cfg, err = ReadConfig(false) + assert.NoError(t, err) + assert.Equal(t, []string{"https://hoge.com/myplugin", "https://hoge.com/myplugin2"}, cfg.Ext_Plugin) } func Test_AddHTTPScheme(t *testing.T) { diff --git a/server/internal/app/repo.go b/server/internal/app/repo.go index 0614038234..846d3b2ff9 100644 --- a/server/internal/app/repo.go +++ b/server/internal/app/repo.go @@ -36,7 +36,7 @@ func initReposAndGateways(ctx context.Context, conf *config.Config, debug bool) log.Fatalf("mongo error: %+v\n", err) } - repos, err := mongorepo.New(ctx, client.Database("reearth"), mongox.IsTransactionAvailable(conf.DB)) + repos, err := mongorepo.NewWithExtensions(ctx, client.Database("reearth"), mongox.IsTransactionAvailable(conf.DB), conf.Ext_Plugin) if err != nil { log.Fatalf("Failed to init mongo: %+v\n", err) } diff --git a/server/internal/infrastructure/adapter/plugin.go b/server/internal/infrastructure/adapter/plugin.go index d750a47167..708c262bf7 100644 --- a/server/internal/infrastructure/adapter/plugin.go +++ b/server/internal/infrastructure/adapter/plugin.go @@ -10,7 +10,7 @@ import ( "github.com/reearth/reearthx/rerror" ) -// TODO: ここで幅優先探索していくアルゴリズムを書いてmongoからビルトインの検索ロジックを除去する +// TODO: Write a width-first search algorithm here to remove the built-in search logic from mongo type pluginRepo struct { readers []repo.Plugin writer repo.Plugin diff --git a/server/internal/infrastructure/mongo/container.go b/server/internal/infrastructure/mongo/container.go index 125498814f..918f4e869f 100644 --- a/server/internal/infrastructure/mongo/container.go +++ b/server/internal/infrastructure/mongo/container.go @@ -3,14 +3,21 @@ package mongo import ( "context" + "github.com/reearth/reearth/server/internal/infrastructure/adapter" + "github.com/reearth/reearth/server/internal/infrastructure/memory" "github.com/reearth/reearth/server/internal/infrastructure/mongo/migration" "github.com/reearth/reearth/server/internal/usecase/repo" + "github.com/reearth/reearth/server/pkg/id" + "github.com/reearth/reearth/server/pkg/plugin" + "github.com/reearth/reearth/server/pkg/plugin/manifest" + "github.com/reearth/reearth/server/pkg/property" "github.com/reearth/reearth/server/pkg/scene" "github.com/reearth/reearth/server/pkg/user" "github.com/reearth/reearthx/authserver" "github.com/reearth/reearthx/log" "github.com/reearth/reearthx/mongox" "github.com/reearth/reearthx/util" + "github.com/samber/lo" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" ) @@ -60,6 +67,44 @@ func New(ctx context.Context, db *mongo.Database, useTransaction bool) (*repo.Co return c, nil } +func NewWithExtensions(ctx context.Context, db *mongo.Database, useTransaction bool, src []string) (*repo.Container, error) { + c, err := New(ctx, db, useTransaction) + if err != nil { + return nil, err + } + if len(src) == 0 { + return c, nil + } + + ms, err := manifest.ParseFromUrlList(ctx, src) + if err != nil { + return nil, err + } + + ids := lo.Map(ms, func(m *manifest.Manifest, _ int) id.PluginID { + return m.Plugin.ID() + }) + + plugins := lo.Map(ms, func(m *manifest.Manifest, _ int) *plugin.Plugin { + return m.Plugin + }) + + propertySchemas := lo.FlatMap(ms, func(m *manifest.Manifest, _ int) []*property.Schema { + return m.ExtensionSchema + }) + + c.Extensions = ids + c.Plugin = adapter.NewPlugin( + []repo.Plugin{memory.NewPluginWith(plugins...), c.Plugin}, + c.Plugin, + ) + c.PropertySchema = adapter.NewPropertySchema( + []repo.PropertySchema{memory.NewPropertySchemaWith(propertySchemas...), c.PropertySchema}, + c.PropertySchema, + ) + return c, nil +} + func Init(r *repo.Container) error { if r == nil { return nil diff --git a/server/internal/usecase/interactor/scene.go b/server/internal/usecase/interactor/scene.go index fe6ac0744e..66ad5b4b10 100644 --- a/server/internal/usecase/interactor/scene.go +++ b/server/internal/usecase/interactor/scene.go @@ -17,6 +17,7 @@ import ( "github.com/reearth/reearth/server/pkg/visualizer" "github.com/reearth/reearthx/rerror" "github.com/reearth/reearthx/usecasex" + "github.com/samber/lo" ) type Scene struct { @@ -31,6 +32,7 @@ type Scene struct { transaction usecasex.Transaction file gateway.File pluginRegistry gateway.PluginRegistry + extensions []plugin.ID } func NewScene(r *repo.Container, g *gateway.Container) interfaces.Scene { @@ -45,6 +47,7 @@ func NewScene(r *repo.Container, g *gateway.Container) interfaces.Scene { transaction: r.Transaction, file: g.File, pluginRegistry: g.PluginRegistry, + extensions: r.Extensions, } } @@ -58,11 +61,25 @@ func (i *Scene) pluginCommon() *pluginCommon { } func (i *Scene) Fetch(ctx context.Context, ids []id.SceneID, operator *usecase.Operator) ([]*scene.Scene, error) { - return i.sceneRepo.FindByIDs(ctx, ids) + s, err := i.sceneRepo.FindByIDs(ctx, ids) + if err != nil { + return nil, err + } + + lo.ForEach(s, func(s *scene.Scene, _ int) { + injectExtensionsToScene(s, i.extensions) + }) + + return s, nil } func (i *Scene) FindByProject(ctx context.Context, id id.ProjectID, operator *usecase.Operator) (*scene.Scene, error) { - return i.sceneRepo.FindByProject(ctx, id) + s, err := i.sceneRepo.FindByProject(ctx, id) + if err != nil { + return nil, err + } + injectExtensionsToScene(s, i.extensions) + return s, nil } func (i *Scene) Create(ctx context.Context, pid id.ProjectID, operator *usecase.Operator) (_ *scene.Scene, err error) { @@ -539,3 +556,9 @@ func (i *Scene) RemoveCluster(ctx context.Context, sceneID id.SceneID, clusterID tx.Commit() return s, nil } + +func injectExtensionsToScene(s *scene.Scene, ext []plugin.ID) { + lo.ForEach(ext, func(p plugin.ID, _ int) { + s.Plugins().Add(scene.NewPlugin(p, nil)) + }) +} diff --git a/server/internal/usecase/repo/container.go b/server/internal/usecase/repo/container.go index cfb13e14ea..9011c1f20b 100644 --- a/server/internal/usecase/repo/container.go +++ b/server/internal/usecase/repo/container.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/reearth/reearth/server/internal/usecase" + "github.com/reearth/reearth/server/pkg/plugin" "github.com/reearth/reearth/server/pkg/scene" "github.com/reearth/reearth/server/pkg/user" "github.com/reearth/reearthx/authserver" @@ -33,6 +34,7 @@ type Container struct { User User Policy Policy Transaction usecasex.Transaction + Extensions []plugin.ID } func (c *Container) Filtered(workspace WorkspaceFilter, scene SceneFilter) *Container { @@ -58,6 +60,7 @@ func (c *Container) Filtered(workspace WorkspaceFilter, scene SceneFilter) *Cont Transaction: c.Transaction, User: c.User, Workspace: c.Workspace, + Extensions: c.Extensions, } } diff --git a/server/pkg/plugin/manifest/parser.go b/server/pkg/plugin/manifest/parser.go index 7ee63f73fe..39903b6b04 100644 --- a/server/pkg/plugin/manifest/parser.go +++ b/server/pkg/plugin/manifest/parser.go @@ -3,8 +3,11 @@ package manifest //go:generate go run github.com/idubinskiy/schematyper -o schema_gen.go --package manifest ../../../schemas/plugin_manifest.json import ( + "context" "errors" "io" + "net/http" + "net/url" "github.com/goccy/go-yaml" "github.com/reearth/reearth/server/pkg/plugin" @@ -12,6 +15,7 @@ import ( var ( ErrInvalidManifest error = errors.New("invalid manifest") + ErrFailedToFetchManifest error = errors.New("failed to fetch manifest") ErrFailedToParseManifest error = errors.New("failed to parse plugin manifest") ErrSystemManifest = errors.New("cannot build system manifest") ) @@ -56,3 +60,39 @@ func MustParseSystemFromBytes(source []byte, scene *plugin.SceneID, tl *Translat } return m } + +func ParseFromUrl(ctx context.Context, u *url.URL) (*Manifest, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.JoinPath("reearth.yml").String(), nil) + if err != nil { + return nil, err + } + + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer func() { + _ = res.Body.Close() + }() + if res.StatusCode != http.StatusOK { + return nil, ErrFailedToFetchManifest + } + + return Parse(res.Body, nil, nil) +} + +func ParseFromUrlList(ctx context.Context, src []string) ([]*Manifest, error) { + ms := make([]*Manifest, 0, len(src)) + for _, s := range src { + u, err := url.Parse(s) + if err != nil { + return nil, err + } + m, err := ParseFromUrl(ctx, u) + if err != nil { + return nil, err + } + ms = append(ms, m) + } + return ms, nil +} diff --git a/server/pkg/plugin/manifest/parser_test.go b/server/pkg/plugin/manifest/parser_test.go index 8595bd2ccf..d44f4d106f 100644 --- a/server/pkg/plugin/manifest/parser_test.go +++ b/server/pkg/plugin/manifest/parser_test.go @@ -1,14 +1,19 @@ package manifest import ( + "context" _ "embed" + "net/http" + "net/url" "strings" "testing" + "github.com/jarcoal/httpmock" "github.com/reearth/reearth/server/pkg/i18n" "github.com/reearth/reearth/server/pkg/plugin" "github.com/reearth/reearth/server/pkg/property" "github.com/reearth/reearth/server/pkg/visualizer" + "github.com/samber/lo" "github.com/stretchr/testify/assert" ) @@ -106,6 +111,95 @@ func TestParse(t *testing.T) { } +func TestParseFromUrl(t *testing.T) { + tests := []struct { + name string + input string + expected *Manifest + err error + }{ + { + name: "success create simple manifest", + input: minimum, + expected: minimumExpected, + err: nil, + }, + { + name: "success create manifest", + input: normal, + expected: normalExpected, + err: nil, + }, + { + name: "fail not valid JSON", + input: "", + expected: nil, + err: ErrFailedToParseManifest, + }, + { + name: "fail system manifest", + input: `{ + "system": true, + "id": "reearth", + "title": "bbb", + "version": "1.1.1" + }`, + expected: nil, + err: ErrSystemManifest, + }, + { + name: "fail system manifest", + input: `{ + "system": true, + "id": "reearth", + "title": "bbb", + "version": "1.1.1" + }`, + expected: nil, + err: ErrFailedToFetchManifest, + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + // t.Parallel() // in parallel test, httpmock.RegisterResponder is not working, MockTransport should be used + + httpmock.Activate() + defer httpmock.Deactivate() + if tc.err != ErrFailedToFetchManifest { + httpmock.RegisterResponder("GET", "https://example.com/myPlugin/reearth.yml", httpmock.NewBytesResponder(http.StatusOK, []byte(tc.input))) + } else { + httpmock.RegisterResponder("GET", "https://example.com/myPlugin/reearth.yml", httpmock.NewBytesResponder(http.StatusNotFound, nil)) + } + + m, err := ParseFromUrl(context.Background(), lo.Must(url.Parse("https://example.com/myPlugin"))) + if tc.err == nil { + if !assert.NoError(t, err) { + return + } + assert.Equal(t, tc.expected, m) + return + } + assert.ErrorIs(t, tc.err, err) + }) + } + +} +func TestParseFromUrlList(t *testing.T) { + + httpmock.Activate() + defer httpmock.Deactivate() + httpmock.RegisterResponder("GET", "https://example.com/myPlugin1/reearth.yml", httpmock.NewBytesResponder(http.StatusOK, []byte(minimum))) + httpmock.RegisterResponder("GET", "https://example.com/myPlugin2/reearth.yml", httpmock.NewBytesResponder(http.StatusOK, []byte(normal))) + + m, err := ParseFromUrlList(context.Background(), []string{"https://example.com/myPlugin1", "https://example.com/myPlugin2"}) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, []*Manifest{minimumExpected, normalExpected}, m) +} + func TestParseSystemFromBytes(t *testing.T) { tests := []struct { name, input string