diff --git a/changelog.md b/changelog.md index a6093b4f98..b2f0f9c8bb 100644 --- a/changelog.md +++ b/changelog.md @@ -33,6 +33,7 @@ - [#3610](https://github.com/ignite/cli/pull/3610) Fix overflow issue of cosmos faucet in `pkg/cosmosfaucet/transfer.go` and `pkg/cosmosfaucet/cosmosfaucet.go` - [#3618](https://github.com/ignite/cli/pull/3618) Fix TS client generation import path issue - [#3631](https://github.com/ignite/cli/pull/3631) Fix unnecessary vue import in hooks/composables template +- [#3655](https://github.com/ignite/cli/pull/3655) Re-enable TS client generation - [#3661](https://github.com/ignite/cli/pull/3661) Change `pkg/cosmosanalysis` to find Cosmos SDK runtime app registered modules ## [`v0.27.0`](https://github.com/ignite/cli/releases/tag/v0.27.0) diff --git a/go.mod b/go.mod index 28bca70619..30e707dec6 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/ignite/cli -go 1.21 +go 1.21.1 + +toolchain go1.21.3 require ( cosmossdk.io/math v1.0.1 @@ -37,7 +39,8 @@ require ( github.com/hashicorp/go-hclog v1.2.0 github.com/hashicorp/go-plugin v1.4.9 github.com/iancoleman/strcase v0.2.0 - github.com/ignite/ignite-files/nodetime v0.0.1 + github.com/ignite/ignite-files/nodetime v0.0.2 + github.com/ignite/ignite-files/protoc v0.0.1 github.com/ignite/web v0.4.3 github.com/imdario/mergo v0.3.15 github.com/jpillora/chisel v1.8.1 diff --git a/go.sum b/go.sum index 85cc1b036d..2905353ab7 100644 --- a/go.sum +++ b/go.sum @@ -765,8 +765,10 @@ github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHL github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ignite/ignite-files/nodetime v0.0.1 h1:GwsUI+hYOwT5w0GayHSJlOVoDJDhg7RVMrl4oV7VEbc= -github.com/ignite/ignite-files/nodetime v0.0.1/go.mod h1:oMsEk/+FHcHKNiCedFRbLmSAnb6zs3n23KbedSb09LE= +github.com/ignite/ignite-files/nodetime v0.0.2 h1:9Aj0OEa7FWI22J/Zdq7M2JvsjgFLngZSm7vB4i9X4t4= +github.com/ignite/ignite-files/nodetime v0.0.2/go.mod h1:GKDsXdeazHyhSBPdVLp7mNIo/m9LmZ6/h8RmQ0/CoaM= +github.com/ignite/ignite-files/protoc v0.0.1 h1:wXxU1dzruUgSVl1diAuAOA+xv0NQKXJFsDWht2+tAP8= +github.com/ignite/ignite-files/protoc v0.0.1/go.mod h1:cVCHJbEHPIeKHMPk3ZoPS0Xw4XQfUc76BAMAPU9Fwjg= github.com/ignite/web v0.4.3 h1:LHucUEXttzCf5JmxO5/xLKVx1Qz1o124HqF4Nii5uWQ= github.com/ignite/web v0.4.3/go.mod h1:WZWBaBYF8RazN7dE462BLpvXDY8ScacxcJ07BKwX/jY= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= diff --git a/ignite/pkg/cosmosanalysis/app/app.go b/ignite/pkg/cosmosanalysis/app/app.go index 77b956aeea..ca82b60a28 100644 --- a/ignite/pkg/cosmosanalysis/app/app.go +++ b/ignite/pkg/cosmosanalysis/app/app.go @@ -423,7 +423,7 @@ func resolveCosmosPackagePath(chainRoot string) (string, error) { return "", err } - deps, err := gomodule.ResolveDependencies(modFile) + deps, err := gomodule.ResolveDependencies(modFile, false) if err != nil { return "", err } diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index 3efce98138..7c4a70500e 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -23,9 +23,9 @@ type ( // Buf represents the buf application structure. Buf struct { - path string - sdkCache string - cache *protoanalysis.Cache + path string + sdkProtoDir string + cache *protoanalysis.Cache } ) @@ -39,15 +39,20 @@ const ( // CMDGenerate generate command. CMDGenerate Command = "generate" + CMDExport Command = "export" ) var ( commands = map[Command]struct{}{ CMDGenerate: {}, + CMDExport: {}, } - // ErrInvalidCommand error invalid command name. + // ErrInvalidCommand indicates an invalid command name. ErrInvalidCommand = errors.New("invalid command name") + + // ErrProtoFilesNotFound indicates that no ".proto" files were found. + ErrProtoFilesNotFound = errors.New("no proto files found") ) // New creates a new Buf based on the installed binary. @@ -67,6 +72,51 @@ func (c Command) String() string { return string(c) } +// Export runs the buf Export command for the files in the proto directory. +func (b Buf) Export(ctx context.Context, protoDir, output string) error { + // Check if the proto directory is the Cosmos SDK one + if strings.Contains(protoDir, cosmosver.CosmosModulePath) { + if b.sdkProtoDir == "" { + // Copy Cosmos SDK proto path without the Buf workspace. + // This is done because the workspace contains a reference to + // a "orm/internal" proto folder that is not present by default + // in the SDK repository. + d, err := copySDKProtoDir(protoDir) + if err != nil { + return err + } + + b.sdkProtoDir = d + } + + // Split absolute path into an absolute prefix and a relative suffix + paths := strings.Split(protoDir, "/proto") + if len(paths) < 2 { + return fmt.Errorf("invalid Cosmos SDK mod path: %s", protoDir) + } + + // Use the SDK copy to resolve SDK proto files + protoDir = filepath.Join(b.sdkProtoDir, paths[1]) + } + specs, err := xos.FindFiles(protoDir, xos.ProtoFile) + if err != nil { + return err + } + if len(specs) == 0 { + return fmt.Errorf("%w: %s", ErrProtoFilesNotFound, protoDir) + } + flags := map[string]string{ + flagOutput: output, + } + + cmd, err := b.generateCommand(CMDExport, flags, protoDir) + if err != nil { + return err + } + + return b.runCommand(ctx, cmd...) +} + // Generate runs the buf Generate command for each file into the proto directory. func (b Buf) Generate( ctx context.Context, @@ -94,17 +144,17 @@ func (b Buf) Generate( // change the workspace copying the files to another folder and generate the // files. if strings.Contains(protoDir, cosmosver.CosmosModulePath) { - if b.sdkCache == "" { - b.sdkCache, err = prepareSDK(protoDir) + if b.sdkProtoDir == "" { + b.sdkProtoDir, err = copySDKProtoDir(protoDir) if err != nil { return err } } dirs := strings.Split(protoDir, "/proto/") if len(dirs) < 2 { - return fmt.Errorf("invalid cosmos sdk mod path: %s", dirs) + return fmt.Errorf("invalid Cosmos SDK mod path: %s", dirs) } - protoDir = filepath.Join(b.sdkCache, dirs[1]) + protoDir = filepath.Join(b.sdkProtoDir, dirs[1]) } pkgs, err := protoanalysis.Parse(ctx, b.cache, protoDir) @@ -118,14 +168,20 @@ func (b Buf) Generate( if _, ok := excluded[filepath.Base(file.Path)]; ok { continue } - cmd, err := b.generateCommand( - CMDGenerate, - flags, - file.Path, - ) + + specs, err := xos.FindFiles(protoDir, "proto") if err != nil { return err } + if len(specs) == 0 { + continue + } + + cmd, err := b.generateCommand(CMDGenerate, flags, file.Path) + if err != nil { + return err + } + g.Go(func() error { cmd := cmd return b.runCommand(ctx, cmd...) @@ -135,6 +191,14 @@ func (b Buf) Generate( return g.Wait() } +// Cleanup deletes temporary files and directories. +func (b Buf) Cleanup() error { + if b.sdkProtoDir != "" { + return os.RemoveAll(b.sdkProtoDir) + } + return nil +} + // runCommand run the buf CLI command. func (b Buf) runCommand(ctx context.Context, cmd ...string) error { execOpts := []exec.Option{ @@ -167,7 +231,7 @@ func (b Buf) generateCommand( return command, nil } -// findSDKProtoPath find the cosmos-sdk proto folder path. +// findSDKProtoPath finds the Cosmos SDK proto folder path. func findSDKProtoPath(protoDir string) (string, error) { paths := strings.Split(protoDir, "@") if len(paths) < 2 { @@ -177,9 +241,9 @@ func findSDKProtoPath(protoDir string) (string, error) { return fmt.Sprintf("%s@%s/proto", paths[0], version), nil } -// prepareSDK copy the cosmos sdk proto folder to a temporary directory -// so we can skip the buf workspace. -func prepareSDK(protoDir string) (string, error) { +// copySDKProtoDir copies the Cosmos SDK proto folder to a temporary directory. +// The temporary directory must be removed by the caller. +func copySDKProtoDir(protoDir string) (string, error) { tmpDir, err := os.MkdirTemp("", "proto-sdk") if err != nil { return "", err diff --git a/ignite/pkg/cosmosgen/cosmosgen.go b/ignite/pkg/cosmosgen/cosmosgen.go index fd1200aa4b..57eb8417b3 100644 --- a/ignite/pkg/cosmosgen/cosmosgen.go +++ b/ignite/pkg/cosmosgen/cosmosgen.go @@ -2,6 +2,7 @@ package cosmosgen import ( "context" + "os" "path/filepath" "strings" @@ -36,8 +37,6 @@ type generateOptions struct { specOut string } -// TODO add WithInstall. - // ModulePathFunc defines a function type that returns a path based on a Cosmos SDK module. type ModulePathFunc func(module.Module) string @@ -106,17 +105,27 @@ func IncludeDirs(dirs []string) Option { // generator generates code for sdk and sdk apps. type generator struct { - ctx context.Context - buf cosmosbuf.Buf - cacheStorage cache.Storage - appPath string - protoDir string - gomodPath string - o *generateOptions - sdkImport string - deps []gomodule.Version - appModules []module.Module - thirdModules map[string][]module.Module // app dependency-modules pair. + ctx context.Context + buf cosmosbuf.Buf + cacheStorage cache.Storage + appPath string + protoDir string + gomodPath string + opts *generateOptions + sdkImport string + deps []gomodule.Version + appModules []module.Module + appIncludes []string + thirdModules map[string][]module.Module + thirdModuleIncludes map[string][]string + tmpDirs []string +} + +func (g *generator) cleanup() { + // Remove temporary directories created during generation + for _, path := range g.tmpDirs { + _ = os.RemoveAll(path) + } } // Generate generates code from protoDir of an SDK app residing at appPath with given options. @@ -127,19 +136,24 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir return err } + defer b.Cleanup() + g := &generator{ - ctx: ctx, - buf: b, - appPath: appPath, - protoDir: protoDir, - gomodPath: gomodPath, - o: &generateOptions{}, - thirdModules: make(map[string][]module.Module), - cacheStorage: cacheStorage, + ctx: ctx, + buf: b, + appPath: appPath, + protoDir: protoDir, + gomodPath: gomodPath, + opts: &generateOptions{}, + thirdModules: make(map[string][]module.Module), + thirdModuleIncludes: make(map[string][]string), + cacheStorage: cacheStorage, } + defer g.cleanup() + for _, apply := range options { - apply(g.o) + apply(g.opts) } if err := g.setup(); err != nil { @@ -148,24 +162,30 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir // Go generation must run first so the types are created before other // generated code that requires sdk.Msg implementations to be defined - if g.o.isGoEnabled { + if g.opts.isGoEnabled { if err := g.generateGo(); err != nil { return err } } - if g.o.isPulsarEnabled { + if g.opts.isPulsarEnabled { if err := g.generatePulsar(); err != nil { return err } } - if g.o.jsOut != nil { + if g.opts.specOut != "" { + if err := g.generateOpenAPISpec(); err != nil { + return err + } + } + + if g.opts.jsOut != nil { if err := g.generateTS(); err != nil { return err } } - if g.o.vuexOut != nil { + if g.opts.vuexOut != nil { if err := g.generateVuex(); err != nil { return err } @@ -186,7 +206,7 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir } - if g.o.composablesRootPath != "" { + if g.opts.composablesRootPath != "" { if err := g.generateComposables("vue"); err != nil { return err } @@ -198,7 +218,7 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir return err } } - if g.o.hooksRootPath != "" { + if g.opts.hooksRootPath != "" { if err := g.generateComposables("react"); err != nil { return err } @@ -211,12 +231,6 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir } } - if g.o.specOut != "" { - if err := g.generateOpenAPISpec(); err != nil { - return err - } - } - return nil } diff --git a/ignite/pkg/cosmosgen/generate.go b/ignite/pkg/cosmosgen/generate.go index 1af5fdc71f..fd85995f57 100644 --- a/ignite/pkg/cosmosgen/generate.go +++ b/ignite/pkg/cosmosgen/generate.go @@ -2,6 +2,8 @@ package cosmosgen import ( "bytes" + "io/fs" + "os" "path/filepath" "strings" @@ -11,17 +13,26 @@ import ( "github.com/ignite/cli/ignite/pkg/cmdrunner" "github.com/ignite/cli/ignite/pkg/cmdrunner/step" "github.com/ignite/cli/ignite/pkg/cosmosanalysis/module" + "github.com/ignite/cli/ignite/pkg/cosmosbuf" "github.com/ignite/cli/ignite/pkg/cosmosver" "github.com/ignite/cli/ignite/pkg/gomodule" + "github.com/ignite/cli/ignite/pkg/xfilepath" ) const ( - moduleCacheNamespace = "generate.setup.module" + moduleCacheNamespace = "generate.setup.module" + includeProtoCacheNamespace = "generator.includes.proto" +) + +var protocGlobalInclude = xfilepath.List( + xfilepath.JoinFromHome(xfilepath.Path("local/include")), + xfilepath.JoinFromHome(xfilepath.Path(".local/include")), ) type ModulesInPath struct { - Path string - Modules []module.Module + Path string + Modules []module.Module + Includes []string } func (g *generator) setup() (err error) { @@ -59,7 +70,7 @@ func (g *generator) setup() (err error) { } // Read the dependencies defined in the `go.mod` file - g.deps, err = gomodule.ResolveDependencies(modFile) + g.deps, err = gomodule.ResolveDependencies(modFile, false) if err != nil { return err } @@ -69,7 +80,10 @@ func (g *generator) setup() (err error) { if err != nil { return err } - + g.appIncludes, err = g.resolveIncludes(g.appPath) + if err != nil { + return err + } // Go through the Go dependencies of the user's app within go.mod, some of them might be hosting Cosmos SDK modules // that could be in use by user's blockchain. // @@ -105,9 +119,19 @@ func (g *generator) setup() (err error) { return err } + var includes []string + if len(modules) > 0 { + // For versioning issues, we do dependency/includes resolution per module + includes, err = g.resolveIncludes(path) + if err != nil { + return err + } + } + modulesInPath = ModulesInPath{ - Path: path, - Modules: modules, + Path: path, + Modules: modules, + Includes: includes, } if err := moduleCache.Put(cacheKey, modulesInPath); err != nil { @@ -115,12 +139,108 @@ func (g *generator) setup() (err error) { } } - g.thirdModules[modulesInPath.Path] = append(g.thirdModules[modulesInPath.Path], modulesInPath.Modules...) + g.thirdModules[modulesInPath.Path] = append( + g.thirdModules[modulesInPath.Path], + modulesInPath.Modules..., + ) + + if modulesInPath.Includes != nil { + g.thirdModuleIncludes[modulesInPath.Path] = append( + g.thirdModuleIncludes[modulesInPath.Path], + modulesInPath.Includes..., + ) + } } return nil } +func (g *generator) getProtoIncludeFolders(modPath string) []string { + // Add default protoDir and default includeDirs + includePaths := []string{filepath.Join(modPath, g.protoDir)} + for _, dir := range g.opts.includeDirs { + includePaths = append(includePaths, filepath.Join(modPath, dir)) + } + return includePaths +} + +func (g *generator) findBufPath(modpath string) (string, error) { + var bufPath string + err := filepath.WalkDir(modpath, func(path string, _ fs.DirEntry, err error) error { + if err != nil { + return err + } + if filepath.Base(path) == "buf.yaml" { + bufPath = path + return filepath.SkipAll + } + return nil + }) + if err != nil { + return "", err + } + return bufPath, nil +} + +func (g *generator) generateBufIncludeFolder(modpath string) (string, error) { + protoPath, err := os.MkdirTemp("", "includeFolder") + if err != nil { + return "", err + } + + g.tmpDirs = append(g.tmpDirs, protoPath) + + err = g.buf.Export(g.ctx, modpath, protoPath) + if err != nil { + return "", err + } + return protoPath, nil +} + +func (g *generator) resolveIncludes(path string) (paths []string, err error) { + // Init paths with the global include paths for protoc + paths, err = protocGlobalInclude() + if err != nil { + return nil, err + } + + // Check that the app proto directory exists + protoPath := filepath.Join(path, g.protoDir) + fi, err := os.Stat(protoPath) + if err != nil && !os.IsNotExist(err) { + return nil, err + } else if !fi.IsDir() { + // Just return the global includes when a proto directory doesn't exist + return paths, nil + } + + // Add app's proto path to the list of proto paths + paths = append(paths, protoPath) + + // Check if a Buf config file is present + bufPath, err := g.findBufPath(protoPath) + if err != nil { + return nil, err + } + + // When a Buf config exists export all protos needed + // to build the modules to a temporary include folder. + if bufPath != "" { + includePath, err := g.generateBufIncludeFolder(protoPath) + if err != nil && !errors.Is(err, cosmosbuf.ErrProtoFilesNotFound) { + return nil, err + } + + // Use exported files only when the path contains ".proto" files + if includePath != "" { + return append(paths, includePath), nil + } + } + + // By default use the configured directories + return append(paths, g.getProtoIncludeFolders(path)...), nil +} + func (g *generator) discoverModules(path, protoDir string) ([]module.Module, error) { var filteredModules []module.Module diff --git a/ignite/pkg/cosmosgen/generate_composables.go b/ignite/pkg/cosmosgen/generate_composables.go index 75a15239a1..a2badb70d6 100644 --- a/ignite/pkg/cosmosgen/generate_composables.go +++ b/ignite/pkg/cosmosgen/generate_composables.go @@ -50,7 +50,7 @@ func (g *generator) updateComposableDependencies(frontendType string) error { } // Make sure the TS client path is absolute - tsClientPath, err := filepath.Abs(g.o.tsClientRootPath) + tsClientPath, err := filepath.Abs(g.opts.tsClientRootPath) if err != nil { return fmt.Errorf("failed to read the absolute typescript client path: %w", err) } @@ -131,9 +131,9 @@ func (g *composablesGenerator) generateComposableTemplates(p generatePayload) er func (g *composablesGenerator) generateComposableTemplate(m module.Module, p generatePayload) error { var outDir string if g.frontendType == "vue" { - outDir = g.g.o.composablesOut(m) + outDir = g.g.opts.composablesOut(m) } else { - outDir = g.g.o.hooksOut(m) + outDir = g.g.opts.hooksOut(m) } if err := os.MkdirAll(outDir, 0o766); err != nil { @@ -154,9 +154,9 @@ func (g *composablesGenerator) generateComposableTemplate(m module.Module, p gen func (g *composablesGenerator) generateRootTemplates(p generatePayload) error { var outDir string if g.frontendType == "vue" { - outDir = g.g.o.composablesRootPath + outDir = g.g.opts.composablesRootPath } else { - outDir = g.g.o.hooksRootPath + outDir = g.g.opts.hooksRootPath } if err := os.MkdirAll(outDir, 0o766); err != nil { return err diff --git a/ignite/pkg/cosmosgen/generate_openapi.go b/ignite/pkg/cosmosgen/generate_openapi.go index 2de1b082db..f4897d5472 100644 --- a/ignite/pkg/cosmosgen/generate_openapi.go +++ b/ignite/pkg/cosmosgen/generate_openapi.go @@ -25,6 +25,10 @@ func (g *generator) openAPITemplate() string { return filepath.Join(g.appPath, g.protoDir, "buf.gen.swagger.yaml") } +func (g *generator) openAPITemplateForSTA() string { + return filepath.Join(g.appPath, g.protoDir, "buf.gen.sta.yaml") +} + func (g *generator) generateOpenAPISpec() error { var ( specDirs []string @@ -54,7 +58,7 @@ func (g *generator) generateOpenAPISpec() error { return err } - checksumPaths := append([]string{m.Pkg.Path}, g.o.includeDirs...) + checksumPaths := append([]string{m.Pkg.Path}, g.opts.includeDirs...) checksum, err := dirchange.ChecksumFromPaths(src, checksumPaths...) if err != nil { return err @@ -70,7 +74,7 @@ func (g *generator) generateOpenAPISpec() error { if err := os.WriteFile(specPath, existingSpec, 0o644); err != nil { return err } - return conf.AddSpec(strcase.ToCamel(m.Pkg.Name), specPath) + return conf.AddSpec(strcase.ToCamel(m.Pkg.Name), specPath, true) } hasAnySpecChanged = true @@ -97,7 +101,7 @@ func (g *generator) generateOpenAPISpec() error { if err := specCache.Put(cacheKey, f); err != nil { return err } - if err := conf.AddSpec(strcase.ToCamel(m.Pkg.Name), spec); err != nil { + if err := conf.AddSpec(strcase.ToCamel(m.Pkg.Name), spec, true); err != nil { return err } } @@ -130,7 +134,7 @@ func (g *generator) generateOpenAPISpec() error { } } - out := g.o.specOut + out := g.opts.specOut if !hasAnySpecChanged { // In case the generated output has been changed @@ -159,3 +163,86 @@ func (g *generator) generateOpenAPISpec() error { return dirchange.SaveDirChecksum(specCache, out, g.appPath, out) } + +func (g *generator) generateModuleOpenAPISpec(m module.Module, out string) error { + var ( + specDirs []string + conf = swaggercombine.Config{ + Swagger: "2.0", + Info: swaggercombine.Info{ + Title: "HTTP API Console " + m.Pkg.Name, + }, + } + ) + + defer func() { + for _, dir := range specDirs { + os.RemoveAll(dir) + } + }() + + // gen generates a spec for a module where it's source code resides at src. + // and adds needed swaggercombine configure for it. + gen := func(m module.Module) (err error) { + dir, err := os.MkdirTemp("", "gen-openapi-module-spec") + if err != nil { + return err + } + + if err := g.buf.Generate( + g.ctx, + m.Pkg.Path, + dir, + g.openAPITemplateForSTA(), + "module.proto", + ); err != nil { + return err + } + + specs, err := xos.FindFiles(dir, xos.JSONFile) + if err != nil { + return err + } + + for _, spec := range specs { + if err != nil { + return err + } + if err := conf.AddSpec(strcase.ToCamel(m.Pkg.Name), spec, false); err != nil { + return err + } + } + specDirs = append(specDirs, dir) + + return nil + } + + // generate specs for each module and persist them in the file system + // after add their path and config to swaggercombine.Config so we can combine them + // into a single spec. + + add := func(modules []module.Module) error { + for _, m := range modules { + if err := gen(m); err != nil { + return err + } + } + return nil + } + + // protoc openapi generator acts weird on concurrent run, so do not use goroutines here. + if err := add([]module.Module{m}); err != nil { + return err + } + + sort.Slice(conf.APIs, func(a, b int) bool { return conf.APIs[a].ID < conf.APIs[b].ID }) + + // ensure out dir exists. + outDir := filepath.Dir(out) + if err := os.MkdirAll(outDir, 0o766); err != nil { + return err + } + + // combine specs into one and save to out. + return swaggercombine.Combine(g.ctx, conf, out) +} diff --git a/ignite/pkg/cosmosgen/generate_typescript.go b/ignite/pkg/cosmosgen/generate_typescript.go index fd22cf8dd1..05f988a75e 100644 --- a/ignite/pkg/cosmosgen/generate_typescript.go +++ b/ignite/pkg/cosmosgen/generate_typescript.go @@ -7,7 +7,6 @@ import ( "sort" "strings" - "github.com/iancoleman/strcase" "golang.org/x/sync/errgroup" "github.com/ignite/cli/ignite/pkg/cache" @@ -15,11 +14,14 @@ import ( "github.com/ignite/cli/ignite/pkg/dirchange" "github.com/ignite/cli/ignite/pkg/gomodulepath" "github.com/ignite/cli/ignite/pkg/nodetime/programs/sta" - swaggercombine "github.com/ignite/cli/ignite/pkg/nodetime/programs/swagger-combine" - "github.com/ignite/cli/ignite/pkg/xos" + tsproto "github.com/ignite/cli/ignite/pkg/nodetime/programs/ts-proto" + "github.com/ignite/cli/ignite/pkg/protoc" ) -var dirchangeCacheNamespace = "generate.typescript.dirchange" +var ( + dirchangeCacheNamespace = "generate.typescript.dirchange" + tsOut = []string{"--ts_proto_out=."} +) type tsGenerator struct { g *generator @@ -75,32 +77,41 @@ func (g *generator) generateTS() error { return tsg.generateRootTemplates(data) } -func (g *tsGenerator) tsTemplate() string { - return filepath.Join(g.g.appPath, g.g.protoDir, "buf.gen.ts.yaml") -} - func (g *tsGenerator) generateModuleTemplates() error { + protocCmd, cleanupProtoc, err := protoc.Command() + if err != nil { + return err + } + + defer cleanupProtoc() + + tsprotoPluginPath, cleanupPlugin, err := tsproto.BinaryPath() + if err != nil { + return err + } + + defer cleanupPlugin() + staCmd, cleanupSTA, err := sta.Command() if err != nil { return err } defer cleanupSTA() - gg := &errgroup.Group{} dirCache := cache.New[[]byte](g.g.cacheStorage, dirchangeCacheNamespace) - add := func(sourcePath string, modules []module.Module) { + add := func(sourcePath string, modules []module.Module, includes []string) { for _, m := range modules { m := m gg.Go(func() error { cacheKey := m.Pkg.Path - paths := append([]string{m.Pkg.Path, g.g.o.jsOut(m)}, g.g.o.includeDirs...) + paths := append([]string{m.Pkg.Path, g.g.opts.jsOut(m)}, g.g.opts.includeDirs...) // Always generate module templates by default unless cache is enabled, in which // case the module template is generated when one or more files were changed in // the module since the last generation. - if g.g.o.useCache { + if g.g.opts.useCache { changed, err := dirchange.HasDirChecksumChanged(dirCache, cacheKey, sourcePath, paths...) if err != nil { return err @@ -111,7 +122,7 @@ func (g *tsGenerator) generateModuleTemplates() error { } } - err = g.generateModuleTemplate(g.g.ctx, staCmd, sourcePath, m) + err = g.generateModuleTemplate(g.g.ctx, protocCmd, staCmd, tsprotoPluginPath, sourcePath, m, includes) if err != nil { return err } @@ -121,7 +132,7 @@ func (g *tsGenerator) generateModuleTemplates() error { } } - add(g.g.appPath, g.g.appModules) + add(g.g.appPath, g.g.appModules, g.g.appIncludes) // Always generate third party modules; This is required because not generating them might // lead to issues with the module registration in the root template. The root template must @@ -129,7 +140,8 @@ func (g *tsGenerator) generateModuleTemplates() error { // is available and not generated it would lead to the registration of a new not generated // 3rd party module. for sourcePath, modules := range g.g.thirdModules { - add(sourcePath, modules) + includes := g.g.thirdModuleIncludes[sourcePath] + add(sourcePath, modules, append(g.g.appIncludes, includes...)) } return gg.Wait() @@ -137,74 +149,50 @@ func (g *tsGenerator) generateModuleTemplates() error { func (g *tsGenerator) generateModuleTemplate( ctx context.Context, + protocCmd protoc.Cmd, staCmd sta.Cmd, + tsprotoPluginPath, appPath string, m module.Module, + includePaths []string, ) error { var ( - out = g.g.o.jsOut(m) + out = g.g.opts.jsOut(m) typesOut = filepath.Join(out, "types") - conf = swaggercombine.Config{ - Swagger: "2.0", - Info: swaggercombine.Info{ - Title: "HTTP API Console", - }, - } ) - if err := os.MkdirAll(typesOut, 0o766); err != nil { return err } // generate ts-proto types - if err := g.g.buf.Generate( + err := protoc.Generate( ctx, - m.Pkg.Path, typesOut, - g.tsTemplate(), - "module.proto", - ); err != nil { - return err - } - - // generate OpenAPI spec - tmp, err := os.MkdirTemp("", "gen-js-openapi-module-spec") + m.Pkg.Path, + includePaths, + tsOut, + protoc.Plugin(tsprotoPluginPath, "--ts_proto_opt=snakeToCamel=true", "--ts_proto_opt=esModuleInterop=true"), + protoc.Env("NODE_OPTIONS="), // unset nodejs options to avoid unexpected issues with vercel "pkg" + protoc.WithCommand(protocCmd), + ) if err != nil { return err } - defer os.RemoveAll(tmp) + specPath := filepath.Join(out, "api.swagger.yml") - if err := g.g.buf.Generate( - ctx, - m.Pkg.Path, - tmp, - g.g.openAPITemplate(), - "module.proto", - ); err != nil { - return err - } + err = g.g.generateModuleOpenAPISpec(m, specPath) - // combine all swagger files - specs, err := xos.FindFiles(tmp, xos.JSONFile) if err != nil { return err } + // generate the REST client from the OpenAPI spec - for _, spec := range specs { - if err := conf.AddSpec(strcase.ToCamel(m.Pkg.Name), spec); err != nil { - return err - } - } - - // combine specs into one and save to out. - srcSpec := filepath.Join(tmp, "apidocs.swagger.json") - if err := swaggercombine.Combine(ctx, conf, srcSpec); err != nil { - return err - } + var ( + srcSpec = specPath + outREST = filepath.Join(out, "rest.ts") + ) - // generate the REST client from the OpenAPI spec - outREST := filepath.Join(out, "rest.ts") if err := sta.Generate(ctx, outREST, srcSpec, sta.WithCommand(staCmd)); err != nil { return err } @@ -219,7 +207,7 @@ func (g *tsGenerator) generateModuleTemplate( } func (g *tsGenerator) generateRootTemplates(p generatePayload) error { - outDir := g.g.o.tsClientRootPath + outDir := g.g.opts.tsClientRootPath if err := os.MkdirAll(outDir, 0o766); err != nil { return err } diff --git a/ignite/pkg/cosmosgen/generate_vuex.go b/ignite/pkg/cosmosgen/generate_vuex.go index 35ff5ebf86..d54436df5a 100644 --- a/ignite/pkg/cosmosgen/generate_vuex.go +++ b/ignite/pkg/cosmosgen/generate_vuex.go @@ -50,7 +50,7 @@ func (g *generator) updateVueDependencies() error { } // Make sure the TS client path is absolute - tsClientPath, err := filepath.Abs(g.o.tsClientRootPath) + tsClientPath, err := filepath.Abs(g.opts.tsClientRootPath) if err != nil { return fmt.Errorf("failed to read the absolute typescript client path: %w", err) } @@ -93,7 +93,7 @@ func (g *generator) updateVueDependencies() error { func (g *generator) updateVuexDependencies() error { // Init the path to the "vuex" folders inside the app - vuexPackagesPath := filepath.Join(g.o.vuexRootPath, "package.json") + vuexPackagesPath := filepath.Join(g.opts.vuexRootPath, "package.json") if _, err := os.Stat(vuexPackagesPath); errors.Is(err, os.ErrNotExist) { return nil @@ -117,7 +117,7 @@ func (g *generator) updateVuexDependencies() error { } // Make sure the TS client path is absolute - tsClientPath, err := filepath.Abs(g.o.tsClientRootPath) + tsClientPath, err := filepath.Abs(g.opts.tsClientRootPath) if err != nil { return fmt.Errorf("failed to read the absolute typescript client path: %w", err) } @@ -126,7 +126,7 @@ func (g *generator) updateVuexDependencies() error { appModulePath := gomodulepath.ExtractAppPath(chainPath.RawPath) tsClientNS := strings.ReplaceAll(appModulePath, "/", "-") tsClientName := fmt.Sprintf("%s-client-ts", tsClientNS) - tsClientVuexRelPath, err := filepath.Rel(g.o.vuexRootPath, tsClientPath) + tsClientVuexRelPath, err := filepath.Rel(g.opts.vuexRootPath, tsClientPath) if err != nil { return err } @@ -197,7 +197,7 @@ func (g *vuexGenerator) generateVueTemplates(p generatePayload) error { } func (g *vuexGenerator) generateVueTemplate(m module.Module, p generatePayload) error { - outDir := g.g.o.vuexOut(m) + outDir := g.g.opts.vuexOut(m) if err := os.MkdirAll(outDir, 0o766); err != nil { return err } @@ -212,7 +212,7 @@ func (g *vuexGenerator) generateVueTemplate(m module.Module, p generatePayload) } func (g *vuexGenerator) generateRootTemplates(p generatePayload) error { - outDir := g.g.o.vuexRootPath + outDir := g.g.opts.vuexRootPath if err := os.MkdirAll(outDir, 0o766); err != nil { return err } diff --git a/ignite/pkg/gomodule/gomodule.go b/ignite/pkg/gomodule/gomodule.go index ef5c892328..1418079fcb 100644 --- a/ignite/pkg/gomodule/gomodule.go +++ b/ignite/pkg/gomodule/gomodule.go @@ -52,7 +52,7 @@ func FilterVersions(dependencies []module.Version, paths ...string) []module.Ver return filtered } -func ResolveDependencies(f *modfile.File) ([]module.Version, error) { +func ResolveDependencies(f *modfile.File, includeIndirect bool) ([]module.Version, error) { var versions []module.Version isReplacementAdded := func(rv module.Version) bool { @@ -68,7 +68,7 @@ func ResolveDependencies(f *modfile.File) ([]module.Version, error) { } for _, req := range f.Require { - if req.Indirect { + if req.Indirect && !includeIndirect { continue } if !isReplacementAdded(req.Mod) { diff --git a/ignite/pkg/nodetime/programs/swagger-combine/swagger-combine.go b/ignite/pkg/nodetime/programs/swagger-combine/swagger-combine.go index 77f1b24db1..a1ec01ede4 100644 --- a/ignite/pkg/nodetime/programs/swagger-combine/swagger-combine.go +++ b/ignite/pkg/nodetime/programs/swagger-combine/swagger-combine.go @@ -40,7 +40,7 @@ type OperationIDs struct { var opReg = regexp.MustCompile(`(?m)operationId.+?(\w+)`) // AddSpec adds a new OpenAPI spec to Config by path in the fs and unique id of spec. -func (c *Config) AddSpec(id, path string) error { +func (c *Config) AddSpec(id, path string, makeUnique bool) error { // make operationId fields unique. f, err := os.Open(path) if err != nil { @@ -58,7 +58,11 @@ func (c *Config) AddSpec(id, path string) error { for _, op := range ops { o := op[1] - rename[o] = id + o + if makeUnique { + rename[o] = id + o + } else { + rename[o] = o + } } // add api with replaced operation ids. diff --git a/ignite/pkg/nodetime/programs/ts-proto/tsproto.go b/ignite/pkg/nodetime/programs/ts-proto/tsproto.go new file mode 100644 index 0000000000..9b7944ab56 --- /dev/null +++ b/ignite/pkg/nodetime/programs/ts-proto/tsproto.go @@ -0,0 +1,65 @@ +// Package tsproto provides access to protoc-gen-ts_proto protoc plugin. +package tsproto + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/ignite/cli/ignite/pkg/nodetime" +) + +const ( + pluginName = "protoc-gen-ts_proto" + scriptTemplate = "#!/bin/bash\n%s $@\n" +) + +// BinaryPath returns the path to the binary of the ts-proto plugin, so it can be passed to +// protoc via --plugin option. +// +// protoc is very picky about binary names of its plugins. for ts-proto, binary name +// will be protoc-gen-ts_proto. +// see why: https://github.com/stephenh/ts-proto/blob/7f76c05/README.markdown#quickstart. +func BinaryPath() (path string, cleanup func(), err error) { + // Create binary for the TypeScript protobuf generator + command, cleanupBin, err := nodetime.Command(nodetime.CommandTSProto) + if err != nil { + return path, cleanup, err + } + + defer func() { + if err != nil { + cleanupBin() + } + }() + + // Create a random directory for the script that runs the TypeScript protobuf generator. + // This is required to avoid potential flaky integration tests caused by one concurrent + // test overwriting the generator script while it is being run in a separate test process. + tmpDir, err := os.MkdirTemp("", "ts_proto_plugin") + if err != nil { + return path, cleanup, err + } + + cleanupScriptDir := func() { os.RemoveAll(tmpDir) } + + defer func() { + if err != nil { + cleanupScriptDir() + } + }() + + cleanup = func() { + cleanupBin() + cleanupScriptDir() + } + + // Wrap the TypeScript protobuf generator in a script with a fixed name + // located in a random temporary directory. + script := fmt.Sprintf(scriptTemplate, strings.Join(command, " ")) + path = filepath.Join(tmpDir, pluginName) + err = os.WriteFile(path, []byte(script), 0o755) + + return path, cleanup, err +} diff --git a/ignite/pkg/protoc/data/include/cosmos/ics23/v1/proofs.proto b/ignite/pkg/protoc/data/include/cosmos/ics23/v1/proofs.proto deleted file mode 100644 index 75907395e1..0000000000 --- a/ignite/pkg/protoc/data/include/cosmos/ics23/v1/proofs.proto +++ /dev/null @@ -1,234 +0,0 @@ -syntax = "proto3"; - -package cosmos.ics23.v1; - -option go_package = "github.com/cosmos/ics23/go;ics23"; - -enum HashOp { - // NO_HASH is the default if no data passed. Note this is an illegal argument some places. - NO_HASH = 0; - SHA256 = 1; - SHA512 = 2; - KECCAK = 3; - RIPEMD160 = 4; - BITCOIN = 5; // ripemd160(sha256(x)) - SHA512_256 = 6; -} - -/** -LengthOp defines how to process the key and value of the LeafOp -to include length information. After encoding the length with the given -algorithm, the length will be prepended to the key and value bytes. -(Each one with it's own encoded length) -*/ -enum LengthOp { - // NO_PREFIX don't include any length info - NO_PREFIX = 0; - // VAR_PROTO uses protobuf (and go-amino) varint encoding of the length - VAR_PROTO = 1; - // VAR_RLP uses rlp int encoding of the length - VAR_RLP = 2; - // FIXED32_BIG uses big-endian encoding of the length as a 32 bit integer - FIXED32_BIG = 3; - // FIXED32_LITTLE uses little-endian encoding of the length as a 32 bit integer - FIXED32_LITTLE = 4; - // FIXED64_BIG uses big-endian encoding of the length as a 64 bit integer - FIXED64_BIG = 5; - // FIXED64_LITTLE uses little-endian encoding of the length as a 64 bit integer - FIXED64_LITTLE = 6; - // REQUIRE_32_BYTES is like NONE, but will fail if the input is not exactly 32 bytes (sha256 output) - REQUIRE_32_BYTES = 7; - // REQUIRE_64_BYTES is like NONE, but will fail if the input is not exactly 64 bytes (sha512 output) - REQUIRE_64_BYTES = 8; -} - -/** -ExistenceProof takes a key and a value and a set of steps to perform on it. -The result of peforming all these steps will provide a "root hash", which can -be compared to the value in a header. - -Since it is computationally infeasible to produce a hash collission for any of the used -cryptographic hash functions, if someone can provide a series of operations to transform -a given key and value into a root hash that matches some trusted root, these key and values -must be in the referenced merkle tree. - -The only possible issue is maliablity in LeafOp, such as providing extra prefix data, -which should be controlled by a spec. Eg. with lengthOp as NONE, - prefix = FOO, key = BAR, value = CHOICE -and - prefix = F, key = OOBAR, value = CHOICE -would produce the same value. - -With LengthOp this is tricker but not impossible. Which is why the "leafPrefixEqual" field -in the ProofSpec is valuable to prevent this mutability. And why all trees should -length-prefix the data before hashing it. -*/ -message ExistenceProof { - bytes key = 1; - bytes value = 2; - LeafOp leaf = 3; - repeated InnerOp path = 4; -} - -/* -NonExistenceProof takes a proof of two neighbors, one left of the desired key, -one right of the desired key. If both proofs are valid AND they are neighbors, -then there is no valid proof for the given key. -*/ -message NonExistenceProof { - bytes key = 1; // TODO: remove this as unnecessary??? we prove a range - ExistenceProof left = 2; - ExistenceProof right = 3; -} - -/* -CommitmentProof is either an ExistenceProof or a NonExistenceProof, or a Batch of such messages -*/ -message CommitmentProof { - oneof proof { - ExistenceProof exist = 1; - NonExistenceProof nonexist = 2; - BatchProof batch = 3; - CompressedBatchProof compressed = 4; - } -} - -/** -LeafOp represents the raw key-value data we wish to prove, and -must be flexible to represent the internal transformation from -the original key-value pairs into the basis hash, for many existing -merkle trees. - -key and value are passed in. So that the signature of this operation is: - leafOp(key, value) -> output - -To process this, first prehash the keys and values if needed (ANY means no hash in this case): - hkey = prehashKey(key) - hvalue = prehashValue(value) - -Then combine the bytes, and hash it - output = hash(prefix || length(hkey) || hkey || length(hvalue) || hvalue) -*/ -message LeafOp { - HashOp hash = 1; - HashOp prehash_key = 2; - HashOp prehash_value = 3; - LengthOp length = 4; - // prefix is a fixed bytes that may optionally be included at the beginning to differentiate - // a leaf node from an inner node. - bytes prefix = 5; -} - -/** -InnerOp represents a merkle-proof step that is not a leaf. -It represents concatenating two children and hashing them to provide the next result. - -The result of the previous step is passed in, so the signature of this op is: - innerOp(child) -> output - -The result of applying InnerOp should be: - output = op.hash(op.prefix || child || op.suffix) - - where the || operator is concatenation of binary data, -and child is the result of hashing all the tree below this step. - -Any special data, like prepending child with the length, or prepending the entire operation with -some value to differentiate from leaf nodes, should be included in prefix and suffix. -If either of prefix or suffix is empty, we just treat it as an empty string -*/ -message InnerOp { - HashOp hash = 1; - bytes prefix = 2; - bytes suffix = 3; -} - -/** -ProofSpec defines what the expected parameters are for a given proof type. -This can be stored in the client and used to validate any incoming proofs. - - verify(ProofSpec, Proof) -> Proof | Error - -As demonstrated in tests, if we don't fix the algorithm used to calculate the -LeafHash for a given tree, there are many possible key-value pairs that can -generate a given hash (by interpretting the preimage differently). -We need this for proper security, requires client knows a priori what -tree format server uses. But not in code, rather a configuration object. -*/ -message ProofSpec { - // any field in the ExistenceProof must be the same as in this spec. - // except Prefix, which is just the first bytes of prefix (spec can be longer) - LeafOp leaf_spec = 1; - InnerSpec inner_spec = 2; - // max_depth (if > 0) is the maximum number of InnerOps allowed (mainly for fixed-depth tries) - int32 max_depth = 3; - // min_depth (if > 0) is the minimum number of InnerOps allowed (mainly for fixed-depth tries) - int32 min_depth = 4; -} - -/* -InnerSpec contains all store-specific structure info to determine if two proofs from a -given store are neighbors. - -This enables: - - isLeftMost(spec: InnerSpec, op: InnerOp) - isRightMost(spec: InnerSpec, op: InnerOp) - isLeftNeighbor(spec: InnerSpec, left: InnerOp, right: InnerOp) -*/ -message InnerSpec { - // Child order is the ordering of the children node, must count from 0 - // iavl tree is [0, 1] (left then right) - // merk is [0, 2, 1] (left, right, here) - repeated int32 child_order = 1; - int32 child_size = 2; - int32 min_prefix_length = 3; - int32 max_prefix_length = 4; - // empty child is the prehash image that is used when one child is nil (eg. 20 bytes of 0) - bytes empty_child = 5; - // hash is the algorithm that must be used for each InnerOp - HashOp hash = 6; -} - -/* -BatchProof is a group of multiple proof types than can be compressed -*/ -message BatchProof { - repeated BatchEntry entries = 1; -} - -// Use BatchEntry not CommitmentProof, to avoid recursion -message BatchEntry { - oneof proof { - ExistenceProof exist = 1; - NonExistenceProof nonexist = 2; - } -} - -/****** all items here are compressed forms *******/ - -message CompressedBatchProof { - repeated CompressedBatchEntry entries = 1; - repeated InnerOp lookup_inners = 2; -} - -// Use BatchEntry not CommitmentProof, to avoid recursion -message CompressedBatchEntry { - oneof proof { - CompressedExistenceProof exist = 1; - CompressedNonExistenceProof nonexist = 2; - } -} - -message CompressedExistenceProof { - bytes key = 1; - bytes value = 2; - LeafOp leaf = 3; - // these are indexes into the lookup_inners table in CompressedBatchProof - repeated int32 path = 4; -} - -message CompressedNonExistenceProof { - bytes key = 1; // TODO: remove this as unnecessary??? we prove a range - CompressedExistenceProof left = 2; - CompressedExistenceProof right = 3; -} diff --git a/ignite/pkg/protoc/protoc.go b/ignite/pkg/protoc/protoc.go new file mode 100644 index 0000000000..c8f8c726be --- /dev/null +++ b/ignite/pkg/protoc/protoc.go @@ -0,0 +1,284 @@ +// Package protoc provides high level access to protoc command. +package protoc + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/ignite/ignite-files/protoc" + + "github.com/ignite/cli/ignite/pkg/cmdrunner/exec" + "github.com/ignite/cli/ignite/pkg/cmdrunner/step" + "github.com/ignite/cli/ignite/pkg/localfs" + "github.com/ignite/cli/ignite/pkg/protoanalysis" +) + +// Option configures Generate configs. +type Option func(*configs) + +// configs holds Generate configs. +type configs struct { + pluginPath string + isGeneratedDepsEnabled bool + pluginOptions []string + env []string + command Cmd +} + +// Plugin configures a plugin for code generation. +func Plugin(path string, options ...string) Option { + return func(c *configs) { + c.pluginPath = path + c.pluginOptions = options + } +} + +// GenerateDependencies enables code generation for the proto files that your protofile depends on. +// use this if your protoc plugin does not give you an option to enable the same feature. +func GenerateDependencies() Option { + return func(c *configs) { + c.isGeneratedDepsEnabled = true + } +} + +// Env assigns environment values during the code generation. +func Env(v ...string) Option { + return func(c *configs) { + c.env = v + } +} + +// WithCommand assigns a protoc command to use for code generation. +// This allows to use a single protoc binary in multiple code generation calls. +// Otherwise, `Generate` creates a new protoc binary each time it is called. +func WithCommand(command Cmd) Option { + return func(c *configs) { + c.command = command + } +} + +// Cmd contains the information necessary to execute the protoc command. +type Cmd struct { + command []string + includes []string +} + +// Command returns the strings to execute the `protoc` command. +func (c Cmd) Command() []string { + return c.command +} + +// Includes returns the proto files import paths. +func (c Cmd) Includes() []string { + return c.includes +} + +// Command sets the protoc binary up and returns the command needed to execute c. +func Command() (command Cmd, cleanup func(), err error) { + // Unpack binary content + gzr, err := gzip.NewReader(bytes.NewReader(protoc.Binary())) + if err != nil { + return Cmd{}, nil, err + } + + defer gzr.Close() + + // Unarchive binary content + tr := tar.NewReader(gzr) + if _, err := tr.Next(); err != nil { + return Cmd{}, nil, err + } + + binary, err := io.ReadAll(tr) + if err != nil { + return Cmd{}, nil, err + } + + path, cleanupProto, err := localfs.SaveBytesTemp(binary, "protoc", 0o755) + if err != nil { + return Cmd{}, nil, err + } + + include, cleanupInclude, err := localfs.SaveTemp(protoc.Include()) + if err != nil { + cleanupProto() + return Cmd{}, nil, err + } + + cleanup = func() { + cleanupProto() + cleanupInclude() + } + + command = Cmd{ + command: []string{path, "-I", include}, + includes: []string{include}, + } + + return command, cleanup, nil +} + +// Generate generates code into outDir from protoPath and its includePaths by using plugins provided with protocOuts. +func Generate(ctx context.Context, outDir, protoPath string, includePaths, protocOuts []string, options ...Option) error { + c := configs{} + + for _, o := range options { + o(&c) + } + + // init the string to run the protoc command and the proto files import path + command := c.command.Command() + includes := c.command.Includes() + + if command == nil { + cmd, cleanup, err := Command() + if err != nil { + return err + } + + defer cleanup() + + command = cmd.Command() + includes = cmd.Includes() + } + // See: https://github.com/ignite/cli/issues/3698 + command = append(command, "--experimental_allow_proto3_optional") + + // add plugin if set. + if c.pluginPath != "" { + command = append(command, "--plugin", c.pluginPath) + } + var existentIncludePaths []string + + // skip if a third party proto source actually doesn't exist on the filesystem. + for _, path := range includePaths { + if _, err := os.Stat(path); os.IsNotExist(err) { + continue + } + existentIncludePaths = append(existentIncludePaths, path) + } + + // append third party proto locations to the command. + for _, importPath := range existentIncludePaths { + command = append(command, "-I", importPath) + } + + // find out the list of proto files to generate code for and perform code generation. + files, err := discoverFiles(ctx, c, protoPath, append(includes, existentIncludePaths...), protoanalysis.NewCache()) + if err != nil { + return err + } + + // run command for each protocOuts. + for _, out := range protocOuts { + command := append(command, out) + command = append(command, files...) + command = append(command, c.pluginOptions...) + + execOpts := []exec.Option{ + exec.StepOption(step.Workdir(outDir)), + exec.IncludeStdLogsToError(), + } + if c.env != nil { + execOpts = append(execOpts, exec.StepOption(step.Env(c.env...))) + } + + if err := exec.Exec(ctx, command, execOpts...); err != nil { + return err + } + } + + return nil +} + +// discoverFiles discovers .proto files to do code generation for. .proto files of the app +// (everything under protoPath) will always be a part of the discovered files. +// +// when .proto files of the app depends on another proto package under includePaths (dependencies), those +// may need to be discovered as well. some protoc plugins already do this discovery internally but +// for the ones that don't, it needs to be handled here if GenerateDependencies() is enabled. +func discoverFiles(ctx context.Context, c configs, protoPath string, includePaths []string, cache *protoanalysis.Cache) ( + discovered []string, err error, +) { + packages, err := protoanalysis.Parse(ctx, cache, protoPath) + if err != nil { + return nil, err + } + + discovered = packages.Files().Paths() + + if !c.isGeneratedDepsEnabled { + return discovered, nil + } + + for _, file := range packages.Files() { + d, err := searchFile(file, protoPath, includePaths) + if err != nil { + return nil, err + } + discovered = append(discovered, d...) + } + + return discovered, nil +} + +func searchFile(file protoanalysis.File, protoPath string, includePaths []string) (discovered []string, err error) { + dir := filepath.Dir(file.Path) + + for _, dep := range file.Dependencies { + // try to locate imported .proto file relative to the this .proto file. + guessedPath := filepath.Join(dir, dep) + _, err := os.Stat(guessedPath) + if err == nil { + discovered = append(discovered, guessedPath) + continue + } + if !os.IsNotExist(err) { + return nil, err + } + + // otherwise, search by absolute path in includePaths. + var found bool + for _, included := range includePaths { + guessedPath := filepath.Join(included, dep) + _, err := os.Stat(guessedPath) + if err == nil { + // found the dependency. + // if it's under protoPath, it is already discovered so, skip it. + if !strings.HasPrefix(guessedPath, protoPath) { + discovered = append(discovered, guessedPath) + + // perform a complete search on this one to discover its dependencies as well. + depFile, err := protoanalysis.ParseFile(guessedPath) + if err != nil { + return nil, err + } + d, err := searchFile(depFile, protoPath, includePaths) + if err != nil { + return nil, err + } + discovered = append(discovered, d...) + } + + found = true + break + } + if !os.IsNotExist(err) { + return nil, err + } + } + + if !found { + return nil, fmt.Errorf("cannot locate dependency %q for %q", dep, file.Path) + } + } + + return discovered, nil +} diff --git a/ignite/pkg/xos/files.go b/ignite/pkg/xos/files.go index 11a37c7d87..60eb6e4a0e 100644 --- a/ignite/pkg/xos/files.go +++ b/ignite/pkg/xos/files.go @@ -7,7 +7,8 @@ import ( ) const ( - JSONFile = "json" + JSONFile = "json" + ProtoFile = "proto" ) func FindFiles(directory, extension string) ([]string, error) { diff --git a/ignite/services/chain/generate.go b/ignite/services/chain/generate.go index d3ed81a70f..64f5570b6e 100644 --- a/ignite/services/chain/generate.go +++ b/ignite/services/chain/generate.go @@ -51,6 +51,7 @@ func GeneratePulsar() GenerateTarget { // overriding the configured or default path. Path can be an empty string. func GenerateTSClient(path string, useCache bool) GenerateTarget { return func(o *generateOptions) { + o.isOpenAPIEnabled = true o.isTSClientEnabled = true o.tsClientPath = path o.useCache = useCache @@ -60,6 +61,7 @@ func GenerateTSClient(path string, useCache bool) GenerateTarget { // GenerateVuex enables generating proto based Typescript Client and Vuex Stores. func GenerateVuex(path string) GenerateTarget { return func(o *generateOptions) { + o.isOpenAPIEnabled = true o.isTSClientEnabled = true o.isVuexEnabled = true o.vuexPath = path @@ -69,6 +71,7 @@ func GenerateVuex(path string) GenerateTarget { // GenerateComposables enables generating proto based Typescript Client and Vue 3 composables. func GenerateComposables(path string) GenerateTarget { return func(o *generateOptions) { + o.isOpenAPIEnabled = true o.isTSClientEnabled = true o.isComposablesEnabled = true o.composablesPath = path @@ -78,6 +81,7 @@ func GenerateComposables(path string) GenerateTarget { // GenerateHooks enables generating proto based Typescript Client and React composables. func GenerateHooks(path string) GenerateTarget { return func(o *generateOptions) { + o.isOpenAPIEnabled = true o.isTSClientEnabled = true o.isHooksEnabled = true o.hooksPath = path @@ -101,6 +105,10 @@ func (c *Chain) generateFromConfig(ctx context.Context, cacheStorage cache.Stora // Additional code generation targets var targets []GenerateTarget + if conf.Client.OpenAPI.Path != "" { + targets = append(targets, GenerateOpenAPI()) + } + if generateClients { if p := conf.Client.Typescript.Path; p != "" { targets = append(targets, GenerateTSClient(p, true)) @@ -120,10 +128,6 @@ func (c *Chain) generateFromConfig(ctx context.Context, cacheStorage cache.Stora } } - if conf.Client.OpenAPI.Path != "" { - targets = append(targets, GenerateOpenAPI()) - } - // Generate proto based code for Go and optionally for any optional targets return c.Generate(ctx, cacheStorage, GenerateGo(), targets...) } @@ -169,6 +173,20 @@ func (c *Chain) Generate( updateConfig bool ) + if targetOptions.isOpenAPIEnabled { + openAPIPath = conf.Client.OpenAPI.Path + if openAPIPath == "" { + openAPIPath = chainconfig.DefaultOpenAPIPath + } + + // Non absolute OpenAPI paths must be treated as relative to the app directory + if !filepath.IsAbs(openAPIPath) { + openAPIPath = filepath.Join(c.app.Path, openAPIPath) + } + + options = append(options, cosmosgen.WithOpenAPIGeneration(openAPIPath)) + } + if targetOptions.isTSClientEnabled { tsClientPath = targetOptions.tsClientPath if tsClientPath == "" { @@ -270,20 +288,6 @@ func (c *Chain) Generate( ) } - if targetOptions.isOpenAPIEnabled { - openAPIPath = conf.Client.OpenAPI.Path - if openAPIPath == "" { - openAPIPath = chainconfig.DefaultOpenAPIPath - } - - // Non absolute OpenAPI paths must be treated as relative to the app directory - if !filepath.IsAbs(openAPIPath) { - openAPIPath = filepath.Join(c.app.Path, openAPIPath) - } - - options = append(options, cosmosgen.WithOpenAPIGeneration(openAPIPath)) - } - if err := cosmosgen.Generate( ctx, cacheStorage, diff --git a/ignite/services/scaffolder/init.go b/ignite/services/scaffolder/init.go index e1c25b9f43..0c0d990c91 100644 --- a/ignite/services/scaffolder/init.go +++ b/ignite/services/scaffolder/init.go @@ -46,19 +46,13 @@ func Init( path = filepath.Join(root, appFolder) // create the project - if err := generate( - ctx, - tracer, - pathInfo, - addressPrefix, - path, - noDefaultModule, - params, - ); err != nil { + err = generate(ctx, tracer, pathInfo, addressPrefix, path, noDefaultModule, params) + if err != nil { return "", err } - if err := finish(ctx, cacheStorage, path, pathInfo.RawPath); err != nil { + err = finish(ctx, cacheStorage, path, pathInfo.RawPath) + if err != nil { return "", err } diff --git a/ignite/services/scaffolder/scaffolder.go b/ignite/services/scaffolder/scaffolder.go index d26fae4492..6caa944113 100644 --- a/ignite/services/scaffolder/scaffolder.go +++ b/ignite/services/scaffolder/scaffolder.go @@ -64,7 +64,8 @@ func New(appPath string) (Scaffolder, error) { } func finish(ctx context.Context, cacheStorage cache.Storage, path, gomodPath string) error { - if err := protoc(ctx, cacheStorage, path, gomodPath); err != nil { + err := protoc(ctx, cacheStorage, path, gomodPath) + if err != nil { return err } diff --git a/ignite/templates/app/files/proto/buf.gen.sta.yaml b/ignite/templates/app/files/proto/buf.gen.sta.yaml new file mode 100644 index 0000000000..d8cea8b342 --- /dev/null +++ b/ignite/templates/app/files/proto/buf.gen.sta.yaml @@ -0,0 +1,14 @@ +# This file is auto-generated from Ignite. You can edit +# the file content but do not change the file name or path. +# +# buf.gen.sta.yaml +# +version: v1 +plugins: + - name: openapiv2 + out: . + opt: + - logtostderr=true + - openapi_naming_strategy=simple + - simple_operation_ids=false + - json_names_for_fields=false diff --git a/ignite/templates/app/files/proto/buf.lock b/ignite/templates/app/files/proto/buf.lock index 6cc63b7edc..77287f75ce 100644 --- a/ignite/templates/app/files/proto/buf.lock +++ b/ignite/templates/app/files/proto/buf.lock @@ -25,3 +25,7 @@ deps: owner: googleapis repository: googleapis commit: 75b4300737fb4efca0831636be94e517 + - remote: buf.build + owner: protocolbuffers + repository: wellknowntypes + commit: 44e83bc050a4497fa7b36b34d95ca156 diff --git a/ignite/templates/app/files/proto/buf.yaml b/ignite/templates/app/files/proto/buf.yaml index c75488a338..7a86adcfac 100644 --- a/ignite/templates/app/files/proto/buf.yaml +++ b/ignite/templates/app/files/proto/buf.yaml @@ -5,6 +5,7 @@ # version: v1 deps: + - buf.build/protocolbuffers/wellknowntypes - buf.build/cosmos/cosmos-sdk - buf.build/cosmos/cosmos-proto - buf.build/cosmos/gogo-proto