feat: add ftl schema get --protobuf (#502)

backend/schema/jsonschema.go

@@ -5,22 +5,15 @@ import (
 	"strings"
 
 	"github.com/alecthomas/errors"
-	js "github.com/swaggest/jsonschema-go"
+	"github.com/swaggest/jsonschema-go"
 )
 
 // DataToJSONSchema converts the schema for a Data object to a JSON Schema.
 //
 // It takes in the full schema in order to resolve and define references.
-func DataToJSONSchema(schema *Schema, dataRef DataRef) (*js.Schema, error) {
+func DataToJSONSchema(schema *Schema, dataRef DataRef) (*jsonschema.Schema, error) {
 	// Collect all data types.
-	dataTypes := map[DataRef]*Data{}
-	for _, module := range schema.Modules {
-		for _, decl := range module.Decls {
-			if data, ok := decl.(*Data); ok {
-				dataTypes[DataRef{Module: module.Name, Name: data.Name}] = data
-			}
-		}
-	}
+	dataTypes := schema.DataMap()
 
 	// Find the root data type.
 	rootData, ok := dataTypes[dataRef]
@@ -35,69 +28,69 @@ func DataToJSONSchema(schema *Schema, dataRef DataRef) (*js.Schema, error) {
 		return root, nil
 	}
 	// Resolve and encode all data types reachable from the root.
-	root.Definitions = map[string]js.SchemaOrBool{}
+	root.Definitions = map[string]jsonschema.SchemaOrBool{}
 	for dataRef := range dataRefs {
 		data, ok := dataTypes[dataRef]
 		if !ok {
 			return nil, errors.Errorf("unknown data type %s", dataRef)
 		}
-		root.Definitions[dataRef.String()] = js.SchemaOrBool{TypeObject: nodeToJSSchema(data, dataRef, dataRefs)}
+		root.Definitions[dataRef.String()] = jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(data, dataRef, dataRefs)}
 	}
 	return root, nil
 }
 
-func nodeToJSSchema(node Node, rootRef DataRef, dataRefs map[DataRef]bool) *js.Schema {
+func nodeToJSSchema(node Node, rootRef DataRef, dataRefs map[DataRef]bool) *jsonschema.Schema {
 	switch node := node.(type) {
 	case *Data:
-		st := js.Object
-		schema := &js.Schema{
+		st := jsonschema.Object
+		schema := &jsonschema.Schema{
 			Description:          jsComments(node.Comments),
-			Type:                 &js.Type{SimpleTypes: &st},
-			Properties:           map[string]js.SchemaOrBool{},
+			Type:                 &jsonschema.Type{SimpleTypes: &st},
+			Properties:           map[string]jsonschema.SchemaOrBool{},
 			AdditionalProperties: jsBool(false),
 		}
 		for _, field := range node.Fields {
 			jsField := nodeToJSSchema(field.Type, rootRef, dataRefs)
 			jsField.Description = jsComments(field.Comments)
-			schema.Properties[field.Name] = js.SchemaOrBool{TypeObject: jsField}
+			schema.Properties[field.Name] = jsonschema.SchemaOrBool{TypeObject: jsField}
 		}
 		return schema
 
 	case *Int:
-		st := js.Integer
-		return &js.Schema{Type: &js.Type{SimpleTypes: &st}}
+		st := jsonschema.Integer
+		return &jsonschema.Schema{Type: &jsonschema.Type{SimpleTypes: &st}}
 
 	case *Float:
-		st := js.Number
-		return &js.Schema{Type: &js.Type{SimpleTypes: &st}}
+		st := jsonschema.Number
+		return &jsonschema.Schema{Type: &jsonschema.Type{SimpleTypes: &st}}
 
 	case *String:
-		st := js.String
-		return &js.Schema{Type: &js.Type{SimpleTypes: &st}}
+		st := jsonschema.String
+		return &jsonschema.Schema{Type: &jsonschema.Type{SimpleTypes: &st}}
 
 	case *Bool:
-		st := js.Boolean
-		return &js.Schema{Type: &js.Type{SimpleTypes: &st}}
+		st := jsonschema.Boolean
+		return &jsonschema.Schema{Type: &jsonschema.Type{SimpleTypes: &st}}
 
 	case *Time:
-		st := js.String
+		st := jsonschema.String
 		dt := "date-time"
-		return &js.Schema{Type: &js.Type{SimpleTypes: &st}, Format: &dt}
+		return &jsonschema.Schema{Type: &jsonschema.Type{SimpleTypes: &st}, Format: &dt}
 
 	case *Array:
-		st := js.Array
-		return &js.Schema{
-			Type:  &js.Type{SimpleTypes: &st},
-			Items: &js.Items{SchemaOrBool: &js.SchemaOrBool{TypeObject: nodeToJSSchema(node.Element, rootRef, dataRefs)}},
+		st := jsonschema.Array
+		return &jsonschema.Schema{
+			Type:  &jsonschema.Type{SimpleTypes: &st},
+			Items: &jsonschema.Items{SchemaOrBool: &jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(node.Element, rootRef, dataRefs)}},
 		}
 
 	case *Map:
-		st := js.Object
+		st := jsonschema.Object
 		// JSON schema generic map of key type to value type
-		return &js.Schema{
-			Type:                 &js.Type{SimpleTypes: &st},
-			AdditionalProperties: &js.SchemaOrBool{TypeObject: nodeToJSSchema(node.Value, rootRef, dataRefs)},
-			PropertyNames:        &js.SchemaOrBool{TypeObject: nodeToJSSchema(node.Key, rootRef, dataRefs)},
+		return &jsonschema.Schema{
+			Type:                 &jsonschema.Type{SimpleTypes: &st},
+			AdditionalProperties: &jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(node.Value, rootRef, dataRefs)},
+			PropertyNames:        &jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(node.Key, rootRef, dataRefs)},
 		}
 
 	case *DataRef:
@@ -109,7 +102,7 @@ func nodeToJSSchema(node Node, rootRef DataRef, dataRefs map[DataRef]bool) *js.S
 
 		ref := fmt.Sprintf("#/definitions/%s", dataRef.String())
 		dataRefs[dataRef] = true
-		return &js.Schema{Ref: &ref}
+		return &jsonschema.Schema{Ref: &ref}
 
 	case Decl, *Field, Metadata, *MetadataCalls, *MetadataIngress, *Module, *Schema, Type, *Verb, *VerbRef:
 		panic(fmt.Sprintf("unsupported node type %T", node))
@@ -119,8 +112,8 @@ func nodeToJSSchema(node Node, rootRef DataRef, dataRefs map[DataRef]bool) *js.S
 	}
 }
 
-func jsBool(ok bool) *js.SchemaOrBool {
-	return &js.SchemaOrBool{TypeBoolean: &ok}
+func jsBool(ok bool) *jsonschema.SchemaOrBool {
+	return &jsonschema.SchemaOrBool{TypeBoolean: &ok}
 }
 
 func jsComments(comments []string) *string {

backend/schema/schema.go

@@ -173,7 +173,17 @@ type MetadataIngress struct {
 	Pos Position `json:"pos,omitempty" parser:"" protobuf:"1,optional"`
 
 	Method string `parser:"'ingress' @('GET' | 'POST')" json:"method,omitempty" protobuf:"2"`
-	Path   string `parser:"@('/' @Ident)+" json:"path,omitempty" protobuf:"3"`
+	Path   string `parser:"@('/' @('{' | '}' | Ident)+)+" json:"path,omitempty" protobuf:"3"`
+}
+
+func (m *MetadataIngress) Parameters() []string {
+	var params []string
+	for _, part := range strings.Split(m.Path, "/") {
+		if len(part) > 0 && part[0] == '{' && part[len(part)-1] == '}' {
+			params = append(params, part[1:len(part)-1])
+		}
+	}
+	return params
 }
 
 type Module struct {
@@ -226,6 +236,18 @@ type Schema struct {
 	Modules []*Module `parser:"@@*" json:"modules,omitempty" protobuf:"2"`
 }
 
+func (s *Schema) DataMap() map[DataRef]*Data {
+	dataTypes := map[DataRef]*Data{}
+	for _, module := range s.Modules {
+		for _, decl := range module.Decls {
+			if data, ok := decl.(*Data); ok {
+				dataTypes[DataRef{Module: module.Name, Name: data.Name}] = data
+			}
+		}
+	}
+	return dataTypes
+}
+
 // Upsert inserts or replaces a module.
 func (s *Schema) Upsert(module *Module) {
 	for i, m := range s.Modules {

cmd/ftl/cmd_schema.go

@@ -3,13 +3,16 @@ package main
 import (
 	"context"
 	"fmt"
+	"os"
 
 	"connectrpc.com/connect"
 	"github.com/alecthomas/errors"
+	"github.com/golang/protobuf/proto"
 
 	"github.com/TBD54566975/ftl/backend/schema"
 	ftlv1 "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1"
 	"github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1/ftlv1connect"
+	schemapb "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1/schema"
 )
 
 type schemaCmd struct {
@@ -24,13 +27,18 @@ func (c *schemaProtobufCmd) Run() error { //nolint:unparam
 	return nil
 }
 
-type getSchemaCmd struct{}
+type getSchemaCmd struct {
+	Protobuf bool `help:"Output the schema as binary protobuf."`
+}
 
 func (g *getSchemaCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceClient) error {
 	resp, err := client.PullSchema(ctx, connect.NewRequest(&ftlv1.PullSchemaRequest{}))
 	if err != nil {
 		return errors.WithStack(err)
 	}
+	if g.Protobuf {
+		return g.generateProto(resp)
+	}
 	for resp.Receive() {
 		msg := resp.Msg()
 		module, err := schema.ModuleFromProto(msg.Schema)
@@ -44,3 +52,23 @@ func (g *getSchemaCmd) Run(ctx context.Context, client ftlv1connect.ControllerSe
 	}
 	return errors.WithStack(resp.Err())
 }
+
+func (g *getSchemaCmd) generateProto(resp *connect.ServerStreamForClient[ftlv1.PullSchemaResponse]) error {
+	schema := &schemapb.Schema{}
+	for resp.Receive() {
+		msg := resp.Msg()
+		schema.Modules = append(schema.Modules, msg.Schema)
+		if !msg.More {
+			break
+		}
+	}
+	if err := resp.Err(); err != nil {
+		return errors.WithStack(err)
+	}
+	pb, err := proto.Marshal(schema)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	_, err = os.Stdout.Write(pb)
+	return errors.WithStack(err)
+}

cmd/ftl/cmd_serve.go

@@ -34,6 +34,7 @@ func (s *serveCmd) Run(ctx context.Context) error {
 	nextBind := s.Bind
 
 	for i := 0; i < s.Controllers; i++ {
+		i := i
 		controllerAddresses = append(controllerAddresses, nextBind)
 		config := controller.Config{
 			Bind: nextBind,
@@ -42,10 +43,12 @@ func (s *serveCmd) Run(ctx context.Context) error {
 			return errors.WithStack(err)
 		}
 
-		scope := fmt.Sprintf("controller-%d", i)
+		scope := fmt.Sprintf("controller%d", i)
 		controllerCtx := log.ContextWithLogger(ctx, logger.Scope(scope))
 
-		wg.Go(func() error { return controller.Start(controllerCtx, config) })
+		wg.Go(func() error {
+			return errors.Wrapf(controller.Start(controllerCtx, config), "controller%d failed", i)
+		})
 
 		var err error
 		nextBind, err = incrementPort(nextBind)
@@ -60,8 +63,8 @@ func (s *serveCmd) Run(ctx context.Context) error {
 	}
 
 	for i := 0; i < s.Runners; i++ {
+		i := i
 		controllerEndpoint := controllerAddresses[i%len(controllerAddresses)]
-		fmt.Printf("controllerEndpoint: %s runner: %s\n", controllerEndpoint, nextBind)
 		config := runner.Config{
 			Bind:               nextBind,
 			ControllerEndpoint: controllerEndpoint,
@@ -86,7 +89,9 @@ func (s *serveCmd) Run(ctx context.Context) error {
 
 		runnerCtx := log.ContextWithLogger(ctx, logger.Scope(name))
 
-		wg.Go(func() error { return runner.Start(runnerCtx, config) })
+		wg.Go(func() error {
+			return errors.Wrapf(runner.Start(runnerCtx, config), "runner%d failed", i)
+		})
 
 		nextBind, err = incrementPort(nextBind)
 		if err != nil {

examples/go.mod

@@ -26,8 +26,8 @@ require (
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/oklog/ulid/v2 v2.1.0 // indirect
 	github.com/rs/cors v1.9.0 // indirect
-	github.com/swaggest/jsonschema-go v0.3.59 // indirect
-	github.com/swaggest/refl v1.2.0 // indirect
+	github.com/swaggest/jsonschema-go v0.3.62 // indirect
+	github.com/swaggest/refl v1.3.0 // indirect
 	github.com/zalando/go-keyring v0.2.1 // indirect
 	go.opentelemetry.io/otel v1.19.0 // indirect
 	go.opentelemetry.io/otel/metric v1.19.0 // indirect

examples/online-boutique/go.mod

@@ -30,8 +30,8 @@ require (
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/oklog/ulid/v2 v2.1.0 // indirect
 	github.com/rs/cors v1.9.0 // indirect
-	github.com/swaggest/jsonschema-go v0.3.59 // indirect
-	github.com/swaggest/refl v1.2.0 // indirect
+	github.com/swaggest/jsonschema-go v0.3.62 // indirect
+	github.com/swaggest/refl v1.3.0 // indirect
 	github.com/zalando/go-keyring v0.2.1 // indirect
 	go.opentelemetry.io/otel v1.19.0 // indirect
 	go.opentelemetry.io/otel/metric v1.19.0 // indirect

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/otiai10/copy v1.12.0
 	github.com/radovskyb/watcher v1.0.7
 	github.com/rs/cors v1.9.0
-	github.com/swaggest/jsonschema-go v0.3.59
+	github.com/swaggest/jsonschema-go v0.3.62
 	github.com/titanous/json5 v1.0.0
 	github.com/zalando/go-keyring v0.2.1
 	go.opentelemetry.io/otel v1.19.0
@@ -45,7 +45,7 @@ require (
 	github.com/jackc/puddle/v2 v2.2.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/stretchr/objx v0.2.0 // indirect
-	github.com/swaggest/refl v1.2.0 // indirect
+	github.com/swaggest/refl v1.3.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect
 	golang.org/x/crypto v0.14.0 // indirect