Skip to content

Commit

Permalink
bump
Browse files Browse the repository at this point in the history
  • Loading branch information
@ArnaudBger
ArnaudBger committed Aug 30, 2024
1 parent a02e2b3 commit e07f20b
Show file tree
Hide file tree
Showing 12 changed files with 399 additions and 126 deletions.
6 changes: 3 additions & 3 deletions cmd/substreams/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,15 +281,15 @@ func runSubstreamsInitE(cmd *cobra.Command, args []string) error {
msgString = "⚠️ " + msg.Message.Markdown
}

fmt.Print(toMarkdown(msgString))
fmt.Print(ToMarkdown(msgString))

case *pbconvo.SystemOutput_ImageWithText_:
input := msg.ImageWithText
if input.ImgUrl != "" {
fmt.Println("View image here:", input.ImgUrl)
}
if input.Markdown != "" {
fmt.Println(toMarkdown(input.Markdown))
fmt.Println(ToMarkdown(input.Markdown))
}

case *pbconvo.SystemOutput_ListSelect_:
Expand Down Expand Up @@ -617,7 +617,7 @@ func saveDownloadFile(path string, overwriteForm *OverwriteForm, inputFile *pbco
return nil
}

func toMarkdown(input string) string {
func ToMarkdown(input string) string {
style := "light"
if lipgloss.HasDarkBackground() {
style = "dark"
Expand Down
21 changes: 4 additions & 17 deletions codegen/entity.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,15 @@ type EntityType interface {

type SQLEntityType struct {
Name string
Type SqlType
Type SQLType
}

func (e *SQLEntityType) ToEntityTypeOut() string {
switch e.Type {
case SqlDecimal, SqlText, SqlBytes, SqlInt, SqlBoolean:
return fmt.Sprintf("input.%s", textcase.SnakeCase(e.Name))
default:
panic("unsupported type")
}
return fmt.Sprintf("input.%s", textcase.SnakeCase(e.Name))
}

func (e *SQLEntityType) ToSQLType() string {
return fmt.Sprintf(`"%s": %s,`, e.Name, e.Type)
return fmt.Sprintf(`"%s" %s,`, e.Name, e.Type)
}

type SubgraphEntityType struct {
Expand All @@ -40,15 +35,7 @@ type SubgraphEntityType struct {

func (e *SubgraphEntityType) ToEntityTypeOut() string {
switch e.Type {
case SubgraphBytes:
return fmt.Sprintf("input.%s", e.Name)
case SubgraphString:
return fmt.Sprintf("input.%s", e.Name)
case SubgraphBoolean:
return fmt.Sprintf("input.%s", e.Name)
case SubgraphInt:
return fmt.Sprintf("input.%s", e.Name)
case SubgraphInt8:
case SubgraphBytes, SubgraphString, SubgraphBoolean, SubgraphInt, SubgraphInt8:
return fmt.Sprintf("input.%s", e.Name)
case SubgraphBigInt:
return fmt.Sprintf("BigInt.fromString(input.%s.toString())", e.Name)
Expand Down
271 changes: 190 additions & 81 deletions codegen/project.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,112 +28,168 @@ type Project struct {
Flavor string
}

func (p *Project) AddSubgraphEntityType(name string, ttype SubgraphType) {
p.EntityTypes = append(p.EntityTypes, &SubgraphEntityType{
Name: name,
Type: ttype,
})
}
func (p *Project) AddSubgraphEntityType(name string, field *descriptorpb.FieldDescriptorProto) {
subgraphtype := func() *SubgraphType {
switch field.GetType() {
case
descriptorpb.FieldDescriptorProto_TYPE_UINT64,
descriptorpb.FieldDescriptorProto_TYPE_FIXED64,
descriptorpb.FieldDescriptorProto_TYPE_FIXED32,
descriptorpb.FieldDescriptorProto_TYPE_UINT32,
descriptorpb.FieldDescriptorProto_TYPE_SFIXED32,
descriptorpb.FieldDescriptorProto_TYPE_SINT32,
descriptorpb.FieldDescriptorProto_TYPE_SINT64,
descriptorpb.FieldDescriptorProto_TYPE_INT64,
descriptorpb.FieldDescriptorProto_TYPE_SFIXED64:
return &SubgraphBigInt
case descriptorpb.FieldDescriptorProto_TYPE_FLOAT, descriptorpb.FieldDescriptorProto_TYPE_DOUBLE:
return &SubgraphBigDecimal
case descriptorpb.FieldDescriptorProto_TYPE_INT32:
return &SubgraphInt
case descriptorpb.FieldDescriptorProto_TYPE_BOOL:
return &SubgraphBoolean
case descriptorpb.FieldDescriptorProto_TYPE_STRING:
return &SubgraphString
case descriptorpb.FieldDescriptorProto_TYPE_MESSAGE,
descriptorpb.FieldDescriptorProto_TYPE_GROUP,
descriptorpb.FieldDescriptorProto_TYPE_ENUM:
// Let's not support the nested message and groups for now as it is more complex
// and would probably require foreign tables / subgraph entities to work
// not even sure this works as of today
fmt.Printf("skipping message, group and enum - not supported for the moment ")
return nil
case descriptorpb.FieldDescriptorProto_TYPE_BYTES:
return &SubgraphBytes
// if field.GetType() == descriptorpb.FieldDescriptorProto_TYPE_MESSAGE {
// if *field.Label == descriptorpb.FieldDescriptorProto_LABEL_REPEATED {
// splitMessagePath := strings.Split(typeName, ".")
// name splitMessagePath[len(splitMessagePath)-1]

func (p *Project) AddSQLEntityType(name string, ttype SqlType) {
p.EntityTypes = append(p.EntityTypes, &SQLEntityType{
Name: name,
Type: ttype,
})
}
// p.Entities = append(p.Entities, &Entity{
// // NameAsProtoField: textcase.CamelCase(field.GetName()),
// // NameAsEntity: "My" + name,
// Name: name,
// })

func NewProject(
name string,
spkgProjectName string,
network, manifestPath string,
module *pbsubstreams.Module,
outputDescriptor *descriptorpb.DescriptorProto,
protoTypeMapping map[string]*descriptorpb.DescriptorProto,
outputType OutputType,
flavor string,
) *Project {
return &Project{
Network: network,
Name: name,
Module: module,
OutputDescriptor: outputDescriptor,
EntityTypes: []EntityType{},
EntityInfo: EntityInfo{},
protoTypeMapping: protoTypeMapping,
SpkgProjectName: spkgProjectName,
ManifestPath: manifestPath,
OutputType: outputType,
Flavor: flavor,
}
}
// if p.protoTypeMapping[*field.TypeName] == nil {
// return fmt.Errorf("nested message type: %q not found", *field.TypeName)
// }

func (p *Project) BuildOutputEntity() error {
for _, field := range p.OutputDescriptor.Field {
if strings.ToLower(field.GetName()) == "id" {
p.EntityInfo.HasAnID = true
p.EntityInfo.IDFieldName = field.GetName()
// for _, nestedMessageField := range p.protoTypeMapping[*field.TypeName].Field {
// switch *nestedMessageField.Type {
// case descriptorpb.FieldDescriptorProto_TYPE_STRING, descriptorpb.FieldDescriptorProto_TYPE_INT64, descriptorpb.FieldDescriptorProto_TYPE_UINT64, descriptorpb.FieldDescriptorProto_TYPE_UINT32, descriptorpb.FieldDescriptorProto_TYPE_BYTES:
// p.Entity.ID = textcase.CamelCase(nestedMessageField.GetName())
// default:
// continue
// }
// }
// }
// }
default:
panic("field type not supported")
}
}()

name := textcase.CamelCase(field.GetName())
if subgraphtype != nil {
p.EntityTypes = append(p.EntityTypes, &SubgraphEntityType{name, *subgraphtype})
}
}

func (p *Project) AddPostgreSQL(name string, field *descriptorpb.FieldDescriptorProto) {
postgreSqltype := func() *SQLType {
switch field.GetType() {
case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE,
descriptorpb.FieldDescriptorProto_TYPE_FLOAT,
descriptorpb.FieldDescriptorProto_TYPE_INT64,
case
descriptorpb.FieldDescriptorProto_TYPE_UINT64,
descriptorpb.FieldDescriptorProto_TYPE_FIXED64,
descriptorpb.FieldDescriptorProto_TYPE_FIXED32,
descriptorpb.FieldDescriptorProto_TYPE_UINT32,
descriptorpb.FieldDescriptorProto_TYPE_SFIXED32,
descriptorpb.FieldDescriptorProto_TYPE_SFIXED64,
descriptorpb.FieldDescriptorProto_TYPE_SINT32,
descriptorpb.FieldDescriptorProto_TYPE_SINT64:
descriptorpb.FieldDescriptorProto_TYPE_SINT64,
descriptorpb.FieldDescriptorProto_TYPE_INT64,
descriptorpb.FieldDescriptorProto_TYPE_SFIXED64,
descriptorpb.FieldDescriptorProto_TYPE_INT32:
return &PostgresSqlInt
case descriptorpb.FieldDescriptorProto_TYPE_FLOAT, descriptorpb.FieldDescriptorProto_TYPE_DOUBLE:
return &PostgresSqlDecimal
case descriptorpb.FieldDescriptorProto_TYPE_BOOL:
return &PostgresSqlBoolean
case descriptorpb.FieldDescriptorProto_TYPE_STRING:
return &PostgresSqlText
case descriptorpb.FieldDescriptorProto_TYPE_MESSAGE,
descriptorpb.FieldDescriptorProto_TYPE_GROUP,
descriptorpb.FieldDescriptorProto_TYPE_ENUM:
// Let's not support the nested message and groups for now as it is more complex
// and would probably require foreign tables / subgraph entities to work
// not even sure this works as of today
fmt.Printf("skipping message, group and enum - not supported for the moment ")
return nil
case descriptorpb.FieldDescriptorProto_TYPE_BYTES:
return &PostgresSqlBytes
// if field.GetType() == descriptorpb.FieldDescriptorProto_TYPE_MESSAGE {
// if *field.Label == descriptorpb.FieldDescriptorProto_LABEL_REPEATED {
// splitMessagePath := strings.Split(typeName, ".")
// name splitMessagePath[len(splitMessagePath)-1]

if p.OutputType == Subgraph {
p.AddSubgraphEntityType(name, SubgraphBigInt)
}
// p.Entities = append(p.Entities, &Entity{
// // NameAsProtoField: textcase.CamelCase(field.GetName()),
// // NameAsEntity: "My" + name,
// Name: name,
// })

if p.OutputType == Sql {
p.AddSQLEntityType(name, SqlInt)
}
// if p.protoTypeMapping[*field.TypeName] == nil {
// return fmt.Errorf("nested message type: %q not found", *field.TypeName)
// }

case descriptorpb.FieldDescriptorProto_TYPE_INT32:
if p.OutputType == Subgraph {
p.AddSubgraphEntityType(name, SubgraphInt)
}
// for _, nestedMessageField := range p.protoTypeMapping[*field.TypeName].Field {
// switch *nestedMessageField.Type {
// case descriptorpb.FieldDescriptorProto_TYPE_STRING, descriptorpb.FieldDescriptorProto_TYPE_INT64, descriptorpb.FieldDescriptorProto_TYPE_UINT64, descriptorpb.FieldDescriptorProto_TYPE_UINT32, descriptorpb.FieldDescriptorProto_TYPE_BYTES:
// p.Entity.ID = textcase.CamelCase(nestedMessageField.GetName())
// default:
// continue
// }
// }
// }
// }
default:
fmt.Printf("unsupported type %s", field.GetType())
return nil
}
}()

case descriptorpb.FieldDescriptorProto_TYPE_BOOL:
if p.OutputType == Subgraph {
p.AddSubgraphEntityType(name, SubgraphBoolean)
}
if p.OutputType == Sql {
p.AddSQLEntityType(name, SqlBoolean)
}
if postgreSqltype != nil {
p.EntityTypes = append(p.EntityTypes, &SQLEntityType{name, *postgreSqltype})
}
}

func (p *Project) AddClickHouseSQLType(name string, field *descriptorpb.FieldDescriptorProto) {
clickhouseSqlType := func() *SQLType {
switch field.GetType() {
case descriptorpb.FieldDescriptorProto_TYPE_BOOL:
return &ClickhouseBoolean
case descriptorpb.FieldDescriptorProto_TYPE_STRING:
if p.OutputType == Subgraph {
p.AddSubgraphEntityType(name, SubgraphString)
}
if p.OutputType == Sql {
p.AddSQLEntityType(name, SqlText)
}
return &ClickhouseString
case descriptorpb.FieldDescriptorProto_TYPE_BYTES:
return &ClickhouseString
case descriptorpb.FieldDescriptorProto_TYPE_FLOAT, descriptorpb.FieldDescriptorProto_TYPE_DOUBLE:
return &ClickhouseDecimal
case descriptorpb.FieldDescriptorProto_TYPE_UINT32:
return &ClickhouseUInt32
case descriptorpb.FieldDescriptorProto_TYPE_SFIXED32, descriptorpb.FieldDescriptorProto_TYPE_INT32, descriptorpb.FieldDescriptorProto_TYPE_SINT32, descriptorpb.FieldDescriptorProto_TYPE_FIXED32:
return &ClickhouseInt32
case descriptorpb.FieldDescriptorProto_TYPE_UINT64:
return &ClickhouseUInt64
case descriptorpb.FieldDescriptorProto_TYPE_SFIXED64, descriptorpb.FieldDescriptorProto_TYPE_FIXED64, descriptorpb.FieldDescriptorProto_TYPE_INT64, descriptorpb.FieldDescriptorProto_TYPE_SINT64:
return &ClickhouseInt64

case descriptorpb.FieldDescriptorProto_TYPE_MESSAGE,
descriptorpb.FieldDescriptorProto_TYPE_GROUP,
descriptorpb.FieldDescriptorProto_TYPE_ENUM:
// Let's not support the nested message and groups for now as it is more complex
// and would probably require foreign tables / subgraph entities to work
// not even sure this works as of today
fmt.Println("skipping message, group and enum - not supported for the moment")

case descriptorpb.FieldDescriptorProto_TYPE_BYTES:
if p.OutputType == Subgraph {
p.AddSubgraphEntityType(name, SubgraphBytes)
}
if p.OutputType == Sql {
p.AddSQLEntityType(name, SqlText)
}
}

fmt.Printf("skipping message, group and enum - not supported for the moment ")
return nil
// if field.GetType() == descriptorpb.FieldDescriptorProto_TYPE_MESSAGE {
// if *field.Label == descriptorpb.FieldDescriptorProto_LABEL_REPEATED {
// splitMessagePath := strings.Split(typeName, ".")
Expand All @@ -159,6 +215,59 @@ func (p *Project) BuildOutputEntity() error {
// }
// }
// }
default:
fmt.Printf("unsupported type %s", field.GetType())
return nil
}
}()

if clickhouseSqlType != nil {
p.EntityTypes = append(p.EntityTypes, &SQLEntityType{name, *clickhouseSqlType})
}
}

func NewProject(
name string,
spkgProjectName string,
network, manifestPath string,
module *pbsubstreams.Module,
outputDescriptor *descriptorpb.DescriptorProto,
protoTypeMapping map[string]*descriptorpb.DescriptorProto,
outputType OutputType,
flavor string,
) *Project {
return &Project{
Network: network,
Name: name,
Module: module,
OutputDescriptor: outputDescriptor,
EntityTypes: []EntityType{},
EntityInfo: EntityInfo{},
protoTypeMapping: protoTypeMapping,
SpkgProjectName: spkgProjectName,
ManifestPath: manifestPath,
OutputType: outputType,
Flavor: flavor,
}
}

func (p *Project) BuildOutputEntity() error {
for _, field := range p.OutputDescriptor.Field {
if strings.ToLower(field.GetName()) == "id" {
p.EntityInfo.HasAnID = true
p.EntityInfo.IDFieldName = field.GetName()
}

var entityType EntityType
if p.OutputType == "Subgraph" {
p.AddSubgraphEntityType(field.GetName(), field)
p.EntityTypes = append(p.EntityTypes, entityType)
}

if p.OutputType == "sql" {

}

}
return nil
}
Expand Down
12 changes: 12 additions & 0 deletions codegen/sql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,5 +27,17 @@ func generateSQLEnv(cmd *cobra.Command, args []string) error {
return fmt.Errorf("building generate command: %w", err)
}

input := fmt.Sprintf(
"Your Substreams SQL Extension is now generated!\n\n" +
"**Now follow the next steps:**\n\n" +
"Open the sql directory:\n\n" +
"`cd sql`\n\n" +
"Build the substreams package:\n\n" +
"`substreams build`\n\n" +
"Sink data into your database:\n\n" +
"`substreams-sink-sql`\n\n",
)
fmt.Println(ToMarkdown(input))

return nil
}
Loading

0 comments on commit e07f20b

Please sign in to comment.