diff --git a/frontend/cli/cmd_schema.go b/frontend/cli/cmd_schema.go index 0fc81ced35..ed0b96c06f 100644 --- a/frontend/cli/cmd_schema.go +++ b/frontend/cli/cmd_schema.go @@ -1,7 +1,8 @@ package main type schemaCmd struct { - Get getSchemaCmd `default:"" cmd:"" help:"Retrieve the cluster FTL schema."` - Diff schemaDiffCmd `cmd:"" help:"Print any schema differences between this cluster and another cluster. Returns an exit code of 1 if there are differences."` - Import schemaImportCmd `cmd:"" help:"Import messages to the FTL schema."` + Get getSchemaCmd `default:"" cmd:"" help:"Retrieve the cluster FTL schema."` + Diff schemaDiffCmd `cmd:"" help:"Print any schema differences between this cluster and another cluster. Returns an exit code of 1 if there are differences."` + Generate schemaGenerateCmd `cmd:"" help:"Stream the schema from the cluster and generate files from the template."` + Import schemaImportCmd `cmd:"" help:"Import messages to the FTL schema."` } diff --git a/frontend/cli/cmd_schema_generate.go b/frontend/cli/cmd_schema_generate.go new file mode 100644 index 0000000000..18b55427ca --- /dev/null +++ b/frontend/cli/cmd_schema_generate.go @@ -0,0 +1,186 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "connectrpc.com/connect" + "github.com/block/scaffolder" + "github.com/block/scaffolder/extensions/javascript" + "github.com/radovskyb/watcher" + "golang.org/x/exp/maps" + "golang.org/x/sync/errgroup" + + ftlv1 "github.com/block/ftl/backend/protos/xyz/block/ftl/v1" + "github.com/block/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" + "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/slices" + "github.com/block/ftl/internal/log" +) + +type schemaGenerateCmd struct { + Watch time.Duration `short:"w" help:"Watch template directory at this frequency and regenerate on change."` + Template string `arg:"" help:"Template directory to use." type:"existingdir"` + Dest string `arg:"" help:"Destination directory to write files to (will be erased)."` + ReconnectDelay time.Duration `help:"Delay before attempting to reconnect to FTL." default:"5s"` +} + +func (s *schemaGenerateCmd) Run(ctx context.Context, client ftlv1connect.SchemaServiceClient) error { + if s.Watch == 0 { + return s.oneOffGenerate(ctx, client) + } + return s.hotReload(ctx, client) +} + +func (s *schemaGenerateCmd) oneOffGenerate(ctx context.Context, schemaClient ftlv1connect.SchemaServiceClient) error { + response, err := schemaClient.GetSchema(ctx, connect.NewRequest(&ftlv1.GetSchemaRequest{})) + if err != nil { + return fmt.Errorf("failed to get schema: %w", err) + } + modules, err := slices.MapErr(response.Msg.Schema.Modules, schema.ModuleFromProto) + if err != nil { + return fmt.Errorf("invalid module schema: %w", err) + } + return s.regenerateModules(log.FromContext(ctx), modules) +} + +func (s *schemaGenerateCmd) hotReload(ctx context.Context, client ftlv1connect.SchemaServiceClient) error { + watch := watcher.New() + defer watch.Close() + + absTemplatePath, err := filepath.Abs(s.Template) + if err != nil { + return fmt.Errorf("failed to get absolute path for template: %w", err) + } + absDestPath, err := filepath.Abs(s.Dest) + if err != nil { + return fmt.Errorf("failed to get absolute path for destination: %w", err) + } + + if strings.HasPrefix(absDestPath, absTemplatePath) { + return fmt.Errorf("destination directory %s must not be inside the template directory %s", absDestPath, absTemplatePath) + } + + logger := log.FromContext(ctx) + logger.Debugf("Watching %s", s.Template) + + if err := watch.AddRecursive(s.Template); err != nil { + return fmt.Errorf("failed to watch template directory: %w", err) + } + + wg, ctx := errgroup.WithContext(ctx) + + moduleChange := make(chan []*schema.Module) + + wg.Go(func() error { + for { + stream, err := client.PullSchema(ctx, connect.NewRequest(&ftlv1.PullSchemaRequest{})) + if err != nil { + return fmt.Errorf("failed to pull schema: %w", err) + } + + modules := map[string]*schema.Module{} + regenerate := false + for stream.Receive() { + msg := stream.Msg() + switch msg.ChangeType { + case ftlv1.DeploymentChangeType_DEPLOYMENT_CHANGE_TYPE_ADDED, ftlv1.DeploymentChangeType_DEPLOYMENT_CHANGE_TYPE_CHANGED: + if msg.Schema == nil { + return fmt.Errorf("schema is nil for added/changed deployment %q", msg.GetDeploymentKey()) + } + module, err := schema.ModuleFromProto(msg.Schema) + if err != nil { + return fmt.Errorf("failed to convert proto to module: %w", err) + } + modules[module.Name] = module + + case ftlv1.DeploymentChangeType_DEPLOYMENT_CHANGE_TYPE_REMOVED: + if msg.Schema == nil { + return fmt.Errorf("schema is nil for removed deployment %q", msg.GetDeploymentKey()) + } + if msg.ModuleRemoved { + delete(modules, msg.Schema.Name) + } + default: + } + if !msg.More { + regenerate = true + } + if !regenerate { + continue + } + + moduleChange <- maps.Values(modules) + } + + stream.Close() + logger.Debugf("Stream disconnected, attempting to reconnect...") + time.Sleep(s.ReconnectDelay) + } + }) + + wg.Go(func() error { return watch.Start(s.Watch) }) + + var previousModules []*schema.Module + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled: %w", wg.Wait()) + + case event := <-watch.Event: + logger.Debugf("Template changed (%s), regenerating modules", event.Path) + if err := s.regenerateModules(logger, previousModules); err != nil { + return fmt.Errorf("failed to regenerate modules: %w", err) + } + + case modules := <-moduleChange: + previousModules = modules + if err := s.regenerateModules(logger, modules); err != nil { + return fmt.Errorf("failed to regenerate modules: %w", err) + } + } + } +} + +func (s *schemaGenerateCmd) regenerateModules(logger *log.Logger, modules []*schema.Module) error { + if err := os.RemoveAll(s.Dest); err != nil { + return fmt.Errorf("failed to remove destination directory: %w", err) + } + + for _, module := range modules { + if err := scaffolder.Scaffold(s.Template, s.Dest, module, + scaffolder.Extend(javascript.Extension("template.js", javascript.WithLogger(makeJSLoggerAdapter(logger)))), + ); err != nil { + return fmt.Errorf("failed to scaffold module %s: %w", module.Name, err) + } + } + logger.Debugf("Generated %d modules in %s", len(modules), s.Dest) + return nil +} + +func makeJSLoggerAdapter(logger *log.Logger) func(args ...any) { + return func(args ...any) { + strs := slices.Map(args, func(v any) string { return fmt.Sprintf("%v", v) }) + level := log.Debug + if prefix, ok := args[0].(string); ok { + switch prefix { + case "log:": + level = log.Info + case "debug:": + level = log.Debug + case "error:": + level = log.Error + case "warn:": + level = log.Warn + } + } + logger.Log(log.Entry{ + Level: level, + Message: strings.Join(strs[1:], " "), + }) + } +} diff --git a/go.mod b/go.mod index eb2793c48b..eb8806a7a6 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( github.com/opencontainers/image-spec v1.1.0 github.com/otiai10/copy v1.14.1 github.com/posener/complete v1.2.3 + github.com/radovskyb/watcher v1.0.7 github.com/rs/cors v1.11.1 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/swaggest/jsonschema-go v0.3.72 @@ -219,9 +220,12 @@ require ( github.com/chzyer/readline v1.5.1 github.com/danieljoos/wincred v1.2.2 // indirect github.com/dlclark/regexp2 v1.11.4 // indirect + github.com/dop251/goja v0.0.0-20241009100908-5f46f2705ca3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/hexops/gotextdiff v1.0.3 github.com/jackc/pgpassfile v1.0.0 // indirect diff --git a/go.sum b/go.sum index 875d5400ab..0233e81022 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,9 @@ github.com/IBM/sarama v1.44.0 h1:puNKqcScjSAgVLramjsuovZrS0nJZFVsrvuUymkWqhE= github.com/IBM/sarama v1.44.0/go.mod h1:MxQ9SvGfvKIorbk077Ff6DUnBlGpidiQOtU2vuBaxVw= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= @@ -188,6 +191,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dop251/goja v0.0.0-20241009100908-5f46f2705ca3 h1:MXsAuToxwsTn5BEEYm2DheqIiC4jWGmkEJ1uy+KFhvQ= +github.com/dop251/goja v0.0.0-20241009100908-5f46f2705ca3/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -243,6 +248,8 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= @@ -558,6 +565,8 @@ github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/puzpuzpuz/xsync/v3 v3.4.0 h1:DuVBAdXuGFHv8adVXjWWZ63pJq+NRXOWVXlKDBZ+mJ4= github.com/puzpuzpuz/xsync/v3 v3.4.0/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= +github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8tUE= +github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=