From f643ccd78b3c2e5be657b4befc83b69fb0cd5841 Mon Sep 17 00:00:00 2001 From: Clockwork Date: Wed, 25 Oct 2023 01:38:44 +0300 Subject: [PATCH] feat: re-enable TS codegen (#3655) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Re-introduce module discovery * feat: Look for indirect proto deps in go.mod * feat: Introduce indirect proto dependency discovery * feat: Add buf export command * feat: User buf-based openapi generation * fix: Revert to previous sta version & peg TS version * feat(wip): Update dependency resolution algorithm * feat: Clean up include resolution * chore: Add changelog entry * chore: Fix linting issues & typos * fix: cosmos-sdk buf issue * feat(pkg/protoc): change package to use protoc binary from files repo (#3657) * feat(pkg/protoc): change package to use protoc binary from files repo * chore(pkg/protoc): remove embedded protoc binary * fix: correct buf.gen.sta.yml file * fix: address review * fix: Remove logging * fix: ResolveDependencies call * fix: correct lint issues and merge `main` (#3690) * chore: remove redundant variable * chore(pkg/cosmosbuf): improve snippet redability * chore: remove redundant default value * chore: code formatting * feat(pkg): remove nodetime binaries (#3670) * - remote nodetime binaries - import ignite-files repo pkg - remote gen binaries scripts and pipeline - run gofmt * remove tollchain from go.mod * add changelog * Update changelog * ci: update Go version to `1.21` in GitHub workflows (#3687) --------- Co-authored-by: Pantani Co-authored-by: Jerónimo Albi * fix: change Go version in nightly release config to `1.21.2` (#3688) * fix: change Go version in nightly release config to `1.21.2` The release action requires a version that includes the patch version to be able to download the right tarball with the Go binary. See https://github.com/wangyoucao577/go-release-action/blob/2ac3035fa4c4feed6a8272ce278b0577b93cf8e5/setup-go.sh#L24 * chore: change go releaser to use the lates Go version Latest is the default but it's setted here to be explicit about that fact. See https://github.com/wangyoucao577/go-release-action#parameters * fix: remove explicit "latest" for Go version release nightly (#3689) Lates version is used by default but it shouldn't be specified explicitly because the value "latest" is not supported. * chore: merge `main` * test: fix broken integration test * fix: change scaffolder to skip protoc when no module is scaffolded There are no proto file when an app is scaffolded without module which makes buf export fail because buf workspace references a folder without proto files. * fix(pkg/cosmosbuf): add check for proto files before running Buf Changes previous commit because some files like the OpenAPI file should be generated even when there are no proto files. We should also allow generating code for standard dependencies like Comos SDK when the app doesn't have proto files. --------- Co-authored-by: Danilo Pantani * feat: Bump nodetime version * refactor: cleanup tmp folders & change third party includes (#3696) * fix: implement third party includes using previous semantics Use `thirdModuleIncludes` instead of defining a new type to store modules and includes to be consistent with `appIncludes` and to avoid changing existing code to work with a new type. * feat: add tmp dir cleanup support to cosmos generator * chore: restore Go version to `1.21` and remove toolchain Currently CI is giving an error because is not properly configured to support the new Go 1.21 features for the mod file and we don't have a consensus yet on how to configure the toolchain and Go versions. * chore: go mod tidy * ci: change integration tests to use Go version from mod file This is required to avoid downloading a newer Go toolchain which will fail because GOSUMDB is disabled to fix the timeout issues because of the repository size. * ci: disable GOTOOLCHAIN when running integration tests This is required because toolchain won't work when GOSUMDB is disabled. * ci: change integration tests to use the stable go version The stable Go version should be used to avoid keeping updating the workflow config when a new patch version is released. This is to make sure that Go doesn't try to download a new toolchain version. * fix: correct CI linting issues * ci: remove redundant GH workflow cache step Go setup action caches since `v4` * fix: disable go setup GH action cache Action for `golangci-lint` already have caching functionality so Go setup must be disabled to avoid caching issues. * chore: add thitd party includes only when available * fix: correct directory remove Co-authored-by: Danilo Pantani * fix: Address code review comments * chore: Address review comment * chore: remove commented function * Update ignite/pkg/cosmosbuf/buf.go Co-authored-by: Jerónimo Albi * chore: fix function name * Update ignite/pkg/cosmosgen/generate_openapi.go Co-authored-by: Danilo Pantani * chore: address review * chore: linting * chore: test --experimental_allow_proto3_optional flag --------- Co-authored-by: Jerónimo Albi Co-authored-by: Danilo Pantani Co-authored-by: jeronimoalbi --- changelog.md | 1 + go.mod | 7 +- go.sum | 6 +- ignite/pkg/cosmosanalysis/app/app.go | 2 +- ignite/pkg/cosmosbuf/buf.go | 98 ++++-- ignite/pkg/cosmosgen/cosmosgen.go | 82 ++--- ignite/pkg/cosmosgen/generate.go | 136 ++++++++- ignite/pkg/cosmosgen/generate_composables.go | 10 +- ignite/pkg/cosmosgen/generate_openapi.go | 95 +++++- ignite/pkg/cosmosgen/generate_typescript.go | 106 +++---- ignite/pkg/cosmosgen/generate_vuex.go | 12 +- ignite/pkg/gomodule/gomodule.go | 4 +- .../swagger-combine/swagger-combine.go | 8 +- .../pkg/nodetime/programs/ts-proto/tsproto.go | 65 ++++ .../data/include/cosmos/ics23/v1/proofs.proto | 234 --------------- ignite/pkg/protoc/protoc.go | 284 ++++++++++++++++++ ignite/pkg/xos/files.go | 3 +- ignite/services/chain/generate.go | 40 +-- ignite/services/scaffolder/init.go | 14 +- ignite/services/scaffolder/scaffolder.go | 3 +- .../app/files/proto/buf.gen.sta.yaml | 14 + ignite/templates/app/files/proto/buf.lock | 4 + ignite/templates/app/files/proto/buf.yaml | 1 + 23 files changed, 823 insertions(+), 406 deletions(-) create mode 100644 ignite/pkg/nodetime/programs/ts-proto/tsproto.go delete mode 100644 ignite/pkg/protoc/data/include/cosmos/ics23/v1/proofs.proto create mode 100644 ignite/pkg/protoc/protoc.go create mode 100644 ignite/templates/app/files/proto/buf.gen.sta.yaml diff --git a/changelog.md b/changelog.md index a6093b4f98..b2f0f9c8bb 100644 --- a/changelog.md +++ b/changelog.md @@ -33,6 +33,7 @@ - [#3610](https://github.com/ignite/cli/pull/3610) Fix overflow issue of cosmos faucet in `pkg/cosmosfaucet/transfer.go` and `pkg/cosmosfaucet/cosmosfaucet.go` - [#3618](https://github.com/ignite/cli/pull/3618) Fix TS client generation import path issue - [#3631](https://github.com/ignite/cli/pull/3631) Fix unnecessary vue import in hooks/composables template +- [#3655](https://github.com/ignite/cli/pull/3655) Re-enable TS client generation - [#3661](https://github.com/ignite/cli/pull/3661) Change `pkg/cosmosanalysis` to find Cosmos SDK runtime app registered modules ## [`v0.27.0`](https://github.com/ignite/cli/releases/tag/v0.27.0) diff --git a/go.mod b/go.mod index 28bca70619..30e707dec6 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/ignite/cli -go 1.21 +go 1.21.1 + +toolchain go1.21.3 require ( cosmossdk.io/math v1.0.1 @@ -37,7 +39,8 @@ require ( github.com/hashicorp/go-hclog v1.2.0 github.com/hashicorp/go-plugin v1.4.9 github.com/iancoleman/strcase v0.2.0 - github.com/ignite/ignite-files/nodetime v0.0.1 + github.com/ignite/ignite-files/nodetime v0.0.2 + github.com/ignite/ignite-files/protoc v0.0.1 github.com/ignite/web v0.4.3 github.com/imdario/mergo v0.3.15 github.com/jpillora/chisel v1.8.1 diff --git a/go.sum b/go.sum index 85cc1b036d..2905353ab7 100644 --- a/go.sum +++ b/go.sum @@ -765,8 +765,10 @@ github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHL github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ignite/ignite-files/nodetime v0.0.1 h1:GwsUI+hYOwT5w0GayHSJlOVoDJDhg7RVMrl4oV7VEbc= -github.com/ignite/ignite-files/nodetime v0.0.1/go.mod h1:oMsEk/+FHcHKNiCedFRbLmSAnb6zs3n23KbedSb09LE= +github.com/ignite/ignite-files/nodetime v0.0.2 h1:9Aj0OEa7FWI22J/Zdq7M2JvsjgFLngZSm7vB4i9X4t4= +github.com/ignite/ignite-files/nodetime v0.0.2/go.mod h1:GKDsXdeazHyhSBPdVLp7mNIo/m9LmZ6/h8RmQ0/CoaM= +github.com/ignite/ignite-files/protoc v0.0.1 h1:wXxU1dzruUgSVl1diAuAOA+xv0NQKXJFsDWht2+tAP8= +github.com/ignite/ignite-files/protoc v0.0.1/go.mod h1:cVCHJbEHPIeKHMPk3ZoPS0Xw4XQfUc76BAMAPU9Fwjg= github.com/ignite/web v0.4.3 h1:LHucUEXttzCf5JmxO5/xLKVx1Qz1o124HqF4Nii5uWQ= github.com/ignite/web v0.4.3/go.mod h1:WZWBaBYF8RazN7dE462BLpvXDY8ScacxcJ07BKwX/jY= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= diff --git a/ignite/pkg/cosmosanalysis/app/app.go b/ignite/pkg/cosmosanalysis/app/app.go index 77b956aeea..ca82b60a28 100644 --- a/ignite/pkg/cosmosanalysis/app/app.go +++ b/ignite/pkg/cosmosanalysis/app/app.go @@ -423,7 +423,7 @@ func resolveCosmosPackagePath(chainRoot string) (string, error) { return "", err } - deps, err := gomodule.ResolveDependencies(modFile) + deps, err := gomodule.ResolveDependencies(modFile, false) if err != nil { return "", err } diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index 3efce98138..7c4a70500e 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -23,9 +23,9 @@ type ( // Buf represents the buf application structure. Buf struct { - path string - sdkCache string - cache *protoanalysis.Cache + path string + sdkProtoDir string + cache *protoanalysis.Cache } ) @@ -39,15 +39,20 @@ const ( // CMDGenerate generate command. CMDGenerate Command = "generate" + CMDExport Command = "export" ) var ( commands = map[Command]struct{}{ CMDGenerate: {}, + CMDExport: {}, } - // ErrInvalidCommand error invalid command name. + // ErrInvalidCommand indicates an invalid command name. ErrInvalidCommand = errors.New("invalid command name") + + // ErrProtoFilesNotFound indicates that no ".proto" files were found. + ErrProtoFilesNotFound = errors.New("no proto files found") ) // New creates a new Buf based on the installed binary. @@ -67,6 +72,51 @@ func (c Command) String() string { return string(c) } +// Export runs the buf Export command for the files in the proto directory. +func (b Buf) Export(ctx context.Context, protoDir, output string) error { + // Check if the proto directory is the Cosmos SDK one + if strings.Contains(protoDir, cosmosver.CosmosModulePath) { + if b.sdkProtoDir == "" { + // Copy Cosmos SDK proto path without the Buf workspace. + // This is done because the workspace contains a reference to + // a "orm/internal" proto folder that is not present by default + // in the SDK repository. + d, err := copySDKProtoDir(protoDir) + if err != nil { + return err + } + + b.sdkProtoDir = d + } + + // Split absolute path into an absolute prefix and a relative suffix + paths := strings.Split(protoDir, "/proto") + if len(paths) < 2 { + return fmt.Errorf("invalid Cosmos SDK mod path: %s", protoDir) + } + + // Use the SDK copy to resolve SDK proto files + protoDir = filepath.Join(b.sdkProtoDir, paths[1]) + } + specs, err := xos.FindFiles(protoDir, xos.ProtoFile) + if err != nil { + return err + } + if len(specs) == 0 { + return fmt.Errorf("%w: %s", ErrProtoFilesNotFound, protoDir) + } + flags := map[string]string{ + flagOutput: output, + } + + cmd, err := b.generateCommand(CMDExport, flags, protoDir) + if err != nil { + return err + } + + return b.runCommand(ctx, cmd...) +} + // Generate runs the buf Generate command for each file into the proto directory. func (b Buf) Generate( ctx context.Context, @@ -94,17 +144,17 @@ func (b Buf) Generate( // change the workspace copying the files to another folder and generate the // files. if strings.Contains(protoDir, cosmosver.CosmosModulePath) { - if b.sdkCache == "" { - b.sdkCache, err = prepareSDK(protoDir) + if b.sdkProtoDir == "" { + b.sdkProtoDir, err = copySDKProtoDir(protoDir) if err != nil { return err } } dirs := strings.Split(protoDir, "/proto/") if len(dirs) < 2 { - return fmt.Errorf("invalid cosmos sdk mod path: %s", dirs) + return fmt.Errorf("invalid Cosmos SDK mod path: %s", dirs) } - protoDir = filepath.Join(b.sdkCache, dirs[1]) + protoDir = filepath.Join(b.sdkProtoDir, dirs[1]) } pkgs, err := protoanalysis.Parse(ctx, b.cache, protoDir) @@ -118,14 +168,20 @@ func (b Buf) Generate( if _, ok := excluded[filepath.Base(file.Path)]; ok { continue } - cmd, err := b.generateCommand( - CMDGenerate, - flags, - file.Path, - ) + + specs, err := xos.FindFiles(protoDir, "proto") if err != nil { return err } + if len(specs) == 0 { + continue + } + + cmd, err := b.generateCommand(CMDGenerate, flags, file.Path) + if err != nil { + return err + } + g.Go(func() error { cmd := cmd return b.runCommand(ctx, cmd...) @@ -135,6 +191,14 @@ func (b Buf) Generate( return g.Wait() } +// Cleanup deletes temporary files and directories. +func (b Buf) Cleanup() error { + if b.sdkProtoDir != "" { + return os.RemoveAll(b.sdkProtoDir) + } + return nil +} + // runCommand run the buf CLI command. func (b Buf) runCommand(ctx context.Context, cmd ...string) error { execOpts := []exec.Option{ @@ -167,7 +231,7 @@ func (b Buf) generateCommand( return command, nil } -// findSDKProtoPath find the cosmos-sdk proto folder path. +// findSDKProtoPath finds the Cosmos SDK proto folder path. func findSDKProtoPath(protoDir string) (string, error) { paths := strings.Split(protoDir, "@") if len(paths) < 2 { @@ -177,9 +241,9 @@ func findSDKProtoPath(protoDir string) (string, error) { return fmt.Sprintf("%s@%s/proto", paths[0], version), nil } -// prepareSDK copy the cosmos sdk proto folder to a temporary directory -// so we can skip the buf workspace. -func prepareSDK(protoDir string) (string, error) { +// copySDKProtoDir copies the Cosmos SDK proto folder to a temporary directory. +// The temporary directory must be removed by the caller. +func copySDKProtoDir(protoDir string) (string, error) { tmpDir, err := os.MkdirTemp("", "proto-sdk") if err != nil { return "", err diff --git a/ignite/pkg/cosmosgen/cosmosgen.go b/ignite/pkg/cosmosgen/cosmosgen.go index fd1200aa4b..57eb8417b3 100644 --- a/ignite/pkg/cosmosgen/cosmosgen.go +++ b/ignite/pkg/cosmosgen/cosmosgen.go @@ -2,6 +2,7 @@ package cosmosgen import ( "context" + "os" "path/filepath" "strings" @@ -36,8 +37,6 @@ type generateOptions struct { specOut string } -// TODO add WithInstall. - // ModulePathFunc defines a function type that returns a path based on a Cosmos SDK module. type ModulePathFunc func(module.Module) string @@ -106,17 +105,27 @@ func IncludeDirs(dirs []string) Option { // generator generates code for sdk and sdk apps. type generator struct { - ctx context.Context - buf cosmosbuf.Buf - cacheStorage cache.Storage - appPath string - protoDir string - gomodPath string - o *generateOptions - sdkImport string - deps []gomodule.Version - appModules []module.Module - thirdModules map[string][]module.Module // app dependency-modules pair. + ctx context.Context + buf cosmosbuf.Buf + cacheStorage cache.Storage + appPath string + protoDir string + gomodPath string + opts *generateOptions + sdkImport string + deps []gomodule.Version + appModules []module.Module + appIncludes []string + thirdModules map[string][]module.Module + thirdModuleIncludes map[string][]string + tmpDirs []string +} + +func (g *generator) cleanup() { + // Remove temporary directories created during generation + for _, path := range g.tmpDirs { + _ = os.RemoveAll(path) + } } // Generate generates code from protoDir of an SDK app residing at appPath with given options. @@ -127,19 +136,24 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir return err } + defer b.Cleanup() + g := &generator{ - ctx: ctx, - buf: b, - appPath: appPath, - protoDir: protoDir, - gomodPath: gomodPath, - o: &generateOptions{}, - thirdModules: make(map[string][]module.Module), - cacheStorage: cacheStorage, + ctx: ctx, + buf: b, + appPath: appPath, + protoDir: protoDir, + gomodPath: gomodPath, + opts: &generateOptions{}, + thirdModules: make(map[string][]module.Module), + thirdModuleIncludes: make(map[string][]string), + cacheStorage: cacheStorage, } + defer g.cleanup() + for _, apply := range options { - apply(g.o) + apply(g.opts) } if err := g.setup(); err != nil { @@ -148,24 +162,30 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir // Go generation must run first so the types are created before other // generated code that requires sdk.Msg implementations to be defined - if g.o.isGoEnabled { + if g.opts.isGoEnabled { if err := g.generateGo(); err != nil { return err } } - if g.o.isPulsarEnabled { + if g.opts.isPulsarEnabled { if err := g.generatePulsar(); err != nil { return err } } - if g.o.jsOut != nil { + if g.opts.specOut != "" { + if err := g.generateOpenAPISpec(); err != nil { + return err + } + } + + if g.opts.jsOut != nil { if err := g.generateTS(); err != nil { return err } } - if g.o.vuexOut != nil { + if g.opts.vuexOut != nil { if err := g.generateVuex(); err != nil { return err } @@ -186,7 +206,7 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir } - if g.o.composablesRootPath != "" { + if g.opts.composablesRootPath != "" { if err := g.generateComposables("vue"); err != nil { return err } @@ -198,7 +218,7 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir return err } } - if g.o.hooksRootPath != "" { + if g.opts.hooksRootPath != "" { if err := g.generateComposables("react"); err != nil { return err } @@ -211,12 +231,6 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir } } - if g.o.specOut != "" { - if err := g.generateOpenAPISpec(); err != nil { - return err - } - } - return nil } diff --git a/ignite/pkg/cosmosgen/generate.go b/ignite/pkg/cosmosgen/generate.go index 1af5fdc71f..fd85995f57 100644 --- a/ignite/pkg/cosmosgen/generate.go +++ b/ignite/pkg/cosmosgen/generate.go @@ -2,6 +2,8 @@ package cosmosgen import ( "bytes" + "io/fs" + "os" "path/filepath" "strings" @@ -11,17 +13,26 @@ import ( "github.com/ignite/cli/ignite/pkg/cmdrunner" "github.com/ignite/cli/ignite/pkg/cmdrunner/step" "github.com/ignite/cli/ignite/pkg/cosmosanalysis/module" + "github.com/ignite/cli/ignite/pkg/cosmosbuf" "github.com/ignite/cli/ignite/pkg/cosmosver" "github.com/ignite/cli/ignite/pkg/gomodule" + "github.com/ignite/cli/ignite/pkg/xfilepath" ) const ( - moduleCacheNamespace = "generate.setup.module" + moduleCacheNamespace = "generate.setup.module" + includeProtoCacheNamespace = "generator.includes.proto" +) + +var protocGlobalInclude = xfilepath.List( + xfilepath.JoinFromHome(xfilepath.Path("local/include")), + xfilepath.JoinFromHome(xfilepath.Path(".local/include")), ) type ModulesInPath struct { - Path string - Modules []module.Module + Path string + Modules []module.Module + Includes []string } func (g *generator) setup() (err error) { @@ -59,7 +70,7 @@ func (g *generator) setup() (err error) { } // Read the dependencies defined in the `go.mod` file - g.deps, err = gomodule.ResolveDependencies(modFile) + g.deps, err = gomodule.ResolveDependencies(modFile, false) if err != nil { return err } @@ -69,7 +80,10 @@ func (g *generator) setup() (err error) { if err != nil { return err } - + g.appIncludes, err = g.resolveIncludes(g.appPath) + if err != nil { + return err + } // Go through the Go dependencies of the user's app within go.mod, some of them might be hosting Cosmos SDK modules // that could be in use by user's blockchain. // @@ -105,9 +119,19 @@ func (g *generator) setup() (err error) { return err } + var includes []string + if len(modules) > 0 { + // For versioning issues, we do dependency/includes resolution per module + includes, err = g.resolveIncludes(path) + if err != nil { + return err + } + } + modulesInPath = ModulesInPath{ - Path: path, - Modules: modules, + Path: path, + Modules: modules, + Includes: includes, } if err := moduleCache.Put(cacheKey, modulesInPath); err != nil { @@ -115,12 +139,108 @@ func (g *generator) setup() (err error) { } } - g.thirdModules[modulesInPath.Path] = append(g.thirdModules[modulesInPath.Path], modulesInPath.Modules...) + g.thirdModules[modulesInPath.Path] = append( + g.thirdModules[modulesInPath.Path], + modulesInPath.Modules..., + ) + + if modulesInPath.Includes != nil { + g.thirdModuleIncludes[modulesInPath.Path] = append( + g.thirdModuleIncludes[modulesInPath.Path], + modulesInPath.Includes..., + ) + } } return nil } +func (g *generator) getProtoIncludeFolders(modPath string) []string { + // Add default protoDir and default includeDirs + includePaths := []string{filepath.Join(modPath, g.protoDir)} + for _, dir := range g.opts.includeDirs { + includePaths = append(includePaths, filepath.Join(modPath, dir)) + } + return includePaths +} + +func (g *generator) findBufPath(modpath string) (string, error) { + var bufPath string + err := filepath.WalkDir(modpath, func(path string, _ fs.DirEntry, err error) error { + if err != nil { + return err + } + if filepath.Base(path) == "buf.yaml" { + bufPath = path + return filepath.SkipAll + } + return nil + }) + if err != nil { + return "", err + } + return bufPath, nil +} + +func (g *generator) generateBufIncludeFolder(modpath string) (string, error) { + protoPath, err := os.MkdirTemp("", "includeFolder") + if err != nil { + return "", err + } + + g.tmpDirs = append(g.tmpDirs, protoPath) + + err = g.buf.Export(g.ctx, modpath, protoPath) + if err != nil { + return "", err + } + return protoPath, nil +} + +func (g *generator) resolveIncludes(path string) (paths []string, err error) { + // Init paths with the global include paths for protoc + paths, err = protocGlobalInclude() + if err != nil { + return nil, err + } + + // Check that the app proto directory exists + protoPath := filepath.Join(path, g.protoDir) + fi, err := os.Stat(protoPath) + if err != nil && !os.IsNotExist(err) { + return nil, err + } else if !fi.IsDir() { + // Just return the global includes when a proto directory doesn't exist + return paths, nil + } + + // Add app's proto path to the list of proto paths + paths = append(paths, protoPath) + + // Check if a Buf config file is present + bufPath, err := g.findBufPath(protoPath) + if err != nil { + return nil, err + } + + // When a Buf config exists export all protos needed + // to build the modules to a temporary include folder. + if bufPath != "" { + includePath, err := g.generateBufIncludeFolder(protoPath) + if err != nil && !errors.Is(err, cosmosbuf.ErrProtoFilesNotFound) { + return nil, err + } + + // Use exported files only when the path contains ".proto" files + if includePath != "" { + return append(paths, includePath), nil + } + } + + // By default use the configured directories + return append(paths, g.getProtoIncludeFolders(path)...), nil +} + func (g *generator) discoverModules(path, protoDir string) ([]module.Module, error) { var filteredModules []module.Module diff --git a/ignite/pkg/cosmosgen/generate_composables.go b/ignite/pkg/cosmosgen/generate_composables.go index 75a15239a1..a2badb70d6 100644 --- a/ignite/pkg/cosmosgen/generate_composables.go +++ b/ignite/pkg/cosmosgen/generate_composables.go @@ -50,7 +50,7 @@ func (g *generator) updateComposableDependencies(frontendType string) error { } // Make sure the TS client path is absolute - tsClientPath, err := filepath.Abs(g.o.tsClientRootPath) + tsClientPath, err := filepath.Abs(g.opts.tsClientRootPath) if err != nil { return fmt.Errorf("failed to read the absolute typescript client path: %w", err) } @@ -131,9 +131,9 @@ func (g *composablesGenerator) generateComposableTemplates(p generatePayload) er func (g *composablesGenerator) generateComposableTemplate(m module.Module, p generatePayload) error { var outDir string if g.frontendType == "vue" { - outDir = g.g.o.composablesOut(m) + outDir = g.g.opts.composablesOut(m) } else { - outDir = g.g.o.hooksOut(m) + outDir = g.g.opts.hooksOut(m) } if err := os.MkdirAll(outDir, 0o766); err != nil { @@ -154,9 +154,9 @@ func (g *composablesGenerator) generateComposableTemplate(m module.Module, p gen func (g *composablesGenerator) generateRootTemplates(p generatePayload) error { var outDir string if g.frontendType == "vue" { - outDir = g.g.o.composablesRootPath + outDir = g.g.opts.composablesRootPath } else { - outDir = g.g.o.hooksRootPath + outDir = g.g.opts.hooksRootPath } if err := os.MkdirAll(outDir, 0o766); err != nil { return err diff --git a/ignite/pkg/cosmosgen/generate_openapi.go b/ignite/pkg/cosmosgen/generate_openapi.go index 2de1b082db..f4897d5472 100644 --- a/ignite/pkg/cosmosgen/generate_openapi.go +++ b/ignite/pkg/cosmosgen/generate_openapi.go @@ -25,6 +25,10 @@ func (g *generator) openAPITemplate() string { return filepath.Join(g.appPath, g.protoDir, "buf.gen.swagger.yaml") } +func (g *generator) openAPITemplateForSTA() string { + return filepath.Join(g.appPath, g.protoDir, "buf.gen.sta.yaml") +} + func (g *generator) generateOpenAPISpec() error { var ( specDirs []string @@ -54,7 +58,7 @@ func (g *generator) generateOpenAPISpec() error { return err } - checksumPaths := append([]string{m.Pkg.Path}, g.o.includeDirs...) + checksumPaths := append([]string{m.Pkg.Path}, g.opts.includeDirs...) checksum, err := dirchange.ChecksumFromPaths(src, checksumPaths...) if err != nil { return err @@ -70,7 +74,7 @@ func (g *generator) generateOpenAPISpec() error { if err := os.WriteFile(specPath, existingSpec, 0o644); err != nil { return err } - return conf.AddSpec(strcase.ToCamel(m.Pkg.Name), specPath) + return conf.AddSpec(strcase.ToCamel(m.Pkg.Name), specPath, true) } hasAnySpecChanged = true @@ -97,7 +101,7 @@ func (g *generator) generateOpenAPISpec() error { if err := specCache.Put(cacheKey, f); err != nil { return err } - if err := conf.AddSpec(strcase.ToCamel(m.Pkg.Name), spec); err != nil { + if err := conf.AddSpec(strcase.ToCamel(m.Pkg.Name), spec, true); err != nil { return err } } @@ -130,7 +134,7 @@ func (g *generator) generateOpenAPISpec() error { } } - out := g.o.specOut + out := g.opts.specOut if !hasAnySpecChanged { // In case the generated output has been changed @@ -159,3 +163,86 @@ func (g *generator) generateOpenAPISpec() error { return dirchange.SaveDirChecksum(specCache, out, g.appPath, out) } + +func (g *generator) generateModuleOpenAPISpec(m module.Module, out string) error { + var ( + specDirs []string + conf = swaggercombine.Config{ + Swagger: "2.0", + Info: swaggercombine.Info{ + Title: "HTTP API Console " + m.Pkg.Name, + }, + } + ) + + defer func() { + for _, dir := range specDirs { + os.RemoveAll(dir) + } + }() + + // gen generates a spec for a module where it's source code resides at src. + // and adds needed swaggercombine configure for it. + gen := func(m module.Module) (err error) { + dir, err := os.MkdirTemp("", "gen-openapi-module-spec") + if err != nil { + return err + } + + if err := g.buf.Generate( + g.ctx, + m.Pkg.Path, + dir, + g.openAPITemplateForSTA(), + "module.proto", + ); err != nil { + return err + } + + specs, err := xos.FindFiles(dir, xos.JSONFile) + if err != nil { + return err + } + + for _, spec := range specs { + if err != nil { + return err + } + if err := conf.AddSpec(strcase.ToCamel(m.Pkg.Name), spec, false); err != nil { + return err + } + } + specDirs = append(specDirs, dir) + + return nil + } + + // generate specs for each module and persist them in the file system + // after add their path and config to swaggercombine.Config so we can combine them + // into a single spec. + + add := func(modules []module.Module) error { + for _, m := range modules { + if err := gen(m); err != nil { + return err + } + } + return nil + } + + // protoc openapi generator acts weird on concurrent run, so do not use goroutines here. + if err := add([]module.Module{m}); err != nil { + return err + } + + sort.Slice(conf.APIs, func(a, b int) bool { return conf.APIs[a].ID < conf.APIs[b].ID }) + + // ensure out dir exists. + outDir := filepath.Dir(out) + if err := os.MkdirAll(outDir, 0o766); err != nil { + return err + } + + // combine specs into one and save to out. + return swaggercombine.Combine(g.ctx, conf, out) +} diff --git a/ignite/pkg/cosmosgen/generate_typescript.go b/ignite/pkg/cosmosgen/generate_typescript.go index fd22cf8dd1..05f988a75e 100644 --- a/ignite/pkg/cosmosgen/generate_typescript.go +++ b/ignite/pkg/cosmosgen/generate_typescript.go @@ -7,7 +7,6 @@ import ( "sort" "strings" - "github.com/iancoleman/strcase" "golang.org/x/sync/errgroup" "github.com/ignite/cli/ignite/pkg/cache" @@ -15,11 +14,14 @@ import ( "github.com/ignite/cli/ignite/pkg/dirchange" "github.com/ignite/cli/ignite/pkg/gomodulepath" "github.com/ignite/cli/ignite/pkg/nodetime/programs/sta" - swaggercombine "github.com/ignite/cli/ignite/pkg/nodetime/programs/swagger-combine" - "github.com/ignite/cli/ignite/pkg/xos" + tsproto "github.com/ignite/cli/ignite/pkg/nodetime/programs/ts-proto" + "github.com/ignite/cli/ignite/pkg/protoc" ) -var dirchangeCacheNamespace = "generate.typescript.dirchange" +var ( + dirchangeCacheNamespace = "generate.typescript.dirchange" + tsOut = []string{"--ts_proto_out=."} +) type tsGenerator struct { g *generator @@ -75,32 +77,41 @@ func (g *generator) generateTS() error { return tsg.generateRootTemplates(data) } -func (g *tsGenerator) tsTemplate() string { - return filepath.Join(g.g.appPath, g.g.protoDir, "buf.gen.ts.yaml") -} - func (g *tsGenerator) generateModuleTemplates() error { + protocCmd, cleanupProtoc, err := protoc.Command() + if err != nil { + return err + } + + defer cleanupProtoc() + + tsprotoPluginPath, cleanupPlugin, err := tsproto.BinaryPath() + if err != nil { + return err + } + + defer cleanupPlugin() + staCmd, cleanupSTA, err := sta.Command() if err != nil { return err } defer cleanupSTA() - gg := &errgroup.Group{} dirCache := cache.New[[]byte](g.g.cacheStorage, dirchangeCacheNamespace) - add := func(sourcePath string, modules []module.Module) { + add := func(sourcePath string, modules []module.Module, includes []string) { for _, m := range modules { m := m gg.Go(func() error { cacheKey := m.Pkg.Path - paths := append([]string{m.Pkg.Path, g.g.o.jsOut(m)}, g.g.o.includeDirs...) + paths := append([]string{m.Pkg.Path, g.g.opts.jsOut(m)}, g.g.opts.includeDirs...) // Always generate module templates by default unless cache is enabled, in which // case the module template is generated when one or more files were changed in // the module since the last generation. - if g.g.o.useCache { + if g.g.opts.useCache { changed, err := dirchange.HasDirChecksumChanged(dirCache, cacheKey, sourcePath, paths...) if err != nil { return err @@ -111,7 +122,7 @@ func (g *tsGenerator) generateModuleTemplates() error { } } - err = g.generateModuleTemplate(g.g.ctx, staCmd, sourcePath, m) + err = g.generateModuleTemplate(g.g.ctx, protocCmd, staCmd, tsprotoPluginPath, sourcePath, m, includes) if err != nil { return err } @@ -121,7 +132,7 @@ func (g *tsGenerator) generateModuleTemplates() error { } } - add(g.g.appPath, g.g.appModules) + add(g.g.appPath, g.g.appModules, g.g.appIncludes) // Always generate third party modules; This is required because not generating them might // lead to issues with the module registration in the root template. The root template must @@ -129,7 +140,8 @@ func (g *tsGenerator) generateModuleTemplates() error { // is available and not generated it would lead to the registration of a new not generated // 3rd party module. for sourcePath, modules := range g.g.thirdModules { - add(sourcePath, modules) + includes := g.g.thirdModuleIncludes[sourcePath] + add(sourcePath, modules, append(g.g.appIncludes, includes...)) } return gg.Wait() @@ -137,74 +149,50 @@ func (g *tsGenerator) generateModuleTemplates() error { func (g *tsGenerator) generateModuleTemplate( ctx context.Context, + protocCmd protoc.Cmd, staCmd sta.Cmd, + tsprotoPluginPath, appPath string, m module.Module, + includePaths []string, ) error { var ( - out = g.g.o.jsOut(m) + out = g.g.opts.jsOut(m) typesOut = filepath.Join(out, "types") - conf = swaggercombine.Config{ - Swagger: "2.0", - Info: swaggercombine.Info{ - Title: "HTTP API Console", - }, - } ) - if err := os.MkdirAll(typesOut, 0o766); err != nil { return err } // generate ts-proto types - if err := g.g.buf.Generate( + err := protoc.Generate( ctx, - m.Pkg.Path, typesOut, - g.tsTemplate(), - "module.proto", - ); err != nil { - return err - } - - // generate OpenAPI spec - tmp, err := os.MkdirTemp("", "gen-js-openapi-module-spec") + m.Pkg.Path, + includePaths, + tsOut, + protoc.Plugin(tsprotoPluginPath, "--ts_proto_opt=snakeToCamel=true", "--ts_proto_opt=esModuleInterop=true"), + protoc.Env("NODE_OPTIONS="), // unset nodejs options to avoid unexpected issues with vercel "pkg" + protoc.WithCommand(protocCmd), + ) if err != nil { return err } - defer os.RemoveAll(tmp) + specPath := filepath.Join(out, "api.swagger.yml") - if err := g.g.buf.Generate( - ctx, - m.Pkg.Path, - tmp, - g.g.openAPITemplate(), - "module.proto", - ); err != nil { - return err - } + err = g.g.generateModuleOpenAPISpec(m, specPath) - // combine all swagger files - specs, err := xos.FindFiles(tmp, xos.JSONFile) if err != nil { return err } + // generate the REST client from the OpenAPI spec - for _, spec := range specs { - if err := conf.AddSpec(strcase.ToCamel(m.Pkg.Name), spec); err != nil { - return err - } - } - - // combine specs into one and save to out. - srcSpec := filepath.Join(tmp, "apidocs.swagger.json") - if err := swaggercombine.Combine(ctx, conf, srcSpec); err != nil { - return err - } + var ( + srcSpec = specPath + outREST = filepath.Join(out, "rest.ts") + ) - // generate the REST client from the OpenAPI spec - outREST := filepath.Join(out, "rest.ts") if err := sta.Generate(ctx, outREST, srcSpec, sta.WithCommand(staCmd)); err != nil { return err } @@ -219,7 +207,7 @@ func (g *tsGenerator) generateModuleTemplate( } func (g *tsGenerator) generateRootTemplates(p generatePayload) error { - outDir := g.g.o.tsClientRootPath + outDir := g.g.opts.tsClientRootPath if err := os.MkdirAll(outDir, 0o766); err != nil { return err } diff --git a/ignite/pkg/cosmosgen/generate_vuex.go b/ignite/pkg/cosmosgen/generate_vuex.go index 35ff5ebf86..d54436df5a 100644 --- a/ignite/pkg/cosmosgen/generate_vuex.go +++ b/ignite/pkg/cosmosgen/generate_vuex.go @@ -50,7 +50,7 @@ func (g *generator) updateVueDependencies() error { } // Make sure the TS client path is absolute - tsClientPath, err := filepath.Abs(g.o.tsClientRootPath) + tsClientPath, err := filepath.Abs(g.opts.tsClientRootPath) if err != nil { return fmt.Errorf("failed to read the absolute typescript client path: %w", err) } @@ -93,7 +93,7 @@ func (g *generator) updateVueDependencies() error { func (g *generator) updateVuexDependencies() error { // Init the path to the "vuex" folders inside the app - vuexPackagesPath := filepath.Join(g.o.vuexRootPath, "package.json") + vuexPackagesPath := filepath.Join(g.opts.vuexRootPath, "package.json") if _, err := os.Stat(vuexPackagesPath); errors.Is(err, os.ErrNotExist) { return nil @@ -117,7 +117,7 @@ func (g *generator) updateVuexDependencies() error { } // Make sure the TS client path is absolute - tsClientPath, err := filepath.Abs(g.o.tsClientRootPath) + tsClientPath, err := filepath.Abs(g.opts.tsClientRootPath) if err != nil { return fmt.Errorf("failed to read the absolute typescript client path: %w", err) } @@ -126,7 +126,7 @@ func (g *generator) updateVuexDependencies() error { appModulePath := gomodulepath.ExtractAppPath(chainPath.RawPath) tsClientNS := strings.ReplaceAll(appModulePath, "/", "-") tsClientName := fmt.Sprintf("%s-client-ts", tsClientNS) - tsClientVuexRelPath, err := filepath.Rel(g.o.vuexRootPath, tsClientPath) + tsClientVuexRelPath, err := filepath.Rel(g.opts.vuexRootPath, tsClientPath) if err != nil { return err } @@ -197,7 +197,7 @@ func (g *vuexGenerator) generateVueTemplates(p generatePayload) error { } func (g *vuexGenerator) generateVueTemplate(m module.Module, p generatePayload) error { - outDir := g.g.o.vuexOut(m) + outDir := g.g.opts.vuexOut(m) if err := os.MkdirAll(outDir, 0o766); err != nil { return err } @@ -212,7 +212,7 @@ func (g *vuexGenerator) generateVueTemplate(m module.Module, p generatePayload) } func (g *vuexGenerator) generateRootTemplates(p generatePayload) error { - outDir := g.g.o.vuexRootPath + outDir := g.g.opts.vuexRootPath if err := os.MkdirAll(outDir, 0o766); err != nil { return err } diff --git a/ignite/pkg/gomodule/gomodule.go b/ignite/pkg/gomodule/gomodule.go index ef5c892328..1418079fcb 100644 --- a/ignite/pkg/gomodule/gomodule.go +++ b/ignite/pkg/gomodule/gomodule.go @@ -52,7 +52,7 @@ func FilterVersions(dependencies []module.Version, paths ...string) []module.Ver return filtered } -func ResolveDependencies(f *modfile.File) ([]module.Version, error) { +func ResolveDependencies(f *modfile.File, includeIndirect bool) ([]module.Version, error) { var versions []module.Version isReplacementAdded := func(rv module.Version) bool { @@ -68,7 +68,7 @@ func ResolveDependencies(f *modfile.File) ([]module.Version, error) { } for _, req := range f.Require { - if req.Indirect { + if req.Indirect && !includeIndirect { continue } if !isReplacementAdded(req.Mod) { diff --git a/ignite/pkg/nodetime/programs/swagger-combine/swagger-combine.go b/ignite/pkg/nodetime/programs/swagger-combine/swagger-combine.go index 77f1b24db1..a1ec01ede4 100644 --- a/ignite/pkg/nodetime/programs/swagger-combine/swagger-combine.go +++ b/ignite/pkg/nodetime/programs/swagger-combine/swagger-combine.go @@ -40,7 +40,7 @@ type OperationIDs struct { var opReg = regexp.MustCompile(`(?m)operationId.+?(\w+)`) // AddSpec adds a new OpenAPI spec to Config by path in the fs and unique id of spec. -func (c *Config) AddSpec(id, path string) error { +func (c *Config) AddSpec(id, path string, makeUnique bool) error { // make operationId fields unique. f, err := os.Open(path) if err != nil { @@ -58,7 +58,11 @@ func (c *Config) AddSpec(id, path string) error { for _, op := range ops { o := op[1] - rename[o] = id + o + if makeUnique { + rename[o] = id + o + } else { + rename[o] = o + } } // add api with replaced operation ids. diff --git a/ignite/pkg/nodetime/programs/ts-proto/tsproto.go b/ignite/pkg/nodetime/programs/ts-proto/tsproto.go new file mode 100644 index 0000000000..9b7944ab56 --- /dev/null +++ b/ignite/pkg/nodetime/programs/ts-proto/tsproto.go @@ -0,0 +1,65 @@ +// Package tsproto provides access to protoc-gen-ts_proto protoc plugin. +package tsproto + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/ignite/cli/ignite/pkg/nodetime" +) + +const ( + pluginName = "protoc-gen-ts_proto" + scriptTemplate = "#!/bin/bash\n%s $@\n" +) + +// BinaryPath returns the path to the binary of the ts-proto plugin, so it can be passed to +// protoc via --plugin option. +// +// protoc is very picky about binary names of its plugins. for ts-proto, binary name +// will be protoc-gen-ts_proto. +// see why: https://github.com/stephenh/ts-proto/blob/7f76c05/README.markdown#quickstart. +func BinaryPath() (path string, cleanup func(), err error) { + // Create binary for the TypeScript protobuf generator + command, cleanupBin, err := nodetime.Command(nodetime.CommandTSProto) + if err != nil { + return path, cleanup, err + } + + defer func() { + if err != nil { + cleanupBin() + } + }() + + // Create a random directory for the script that runs the TypeScript protobuf generator. + // This is required to avoid potential flaky integration tests caused by one concurrent + // test overwriting the generator script while it is being run in a separate test process. + tmpDir, err := os.MkdirTemp("", "ts_proto_plugin") + if err != nil { + return path, cleanup, err + } + + cleanupScriptDir := func() { os.RemoveAll(tmpDir) } + + defer func() { + if err != nil { + cleanupScriptDir() + } + }() + + cleanup = func() { + cleanupBin() + cleanupScriptDir() + } + + // Wrap the TypeScript protobuf generator in a script with a fixed name + // located in a random temporary directory. + script := fmt.Sprintf(scriptTemplate, strings.Join(command, " ")) + path = filepath.Join(tmpDir, pluginName) + err = os.WriteFile(path, []byte(script), 0o755) + + return path, cleanup, err +} diff --git a/ignite/pkg/protoc/data/include/cosmos/ics23/v1/proofs.proto b/ignite/pkg/protoc/data/include/cosmos/ics23/v1/proofs.proto deleted file mode 100644 index 75907395e1..0000000000 --- a/ignite/pkg/protoc/data/include/cosmos/ics23/v1/proofs.proto +++ /dev/null @@ -1,234 +0,0 @@ -syntax = "proto3"; - -package cosmos.ics23.v1; - -option go_package = "github.com/cosmos/ics23/go;ics23"; - -enum HashOp { - // NO_HASH is the default if no data passed. Note this is an illegal argument some places. - NO_HASH = 0; - SHA256 = 1; - SHA512 = 2; - KECCAK = 3; - RIPEMD160 = 4; - BITCOIN = 5; // ripemd160(sha256(x)) - SHA512_256 = 6; -} - -/** -LengthOp defines how to process the key and value of the LeafOp -to include length information. After encoding the length with the given -algorithm, the length will be prepended to the key and value bytes. -(Each one with it's own encoded length) -*/ -enum LengthOp { - // NO_PREFIX don't include any length info - NO_PREFIX = 0; - // VAR_PROTO uses protobuf (and go-amino) varint encoding of the length - VAR_PROTO = 1; - // VAR_RLP uses rlp int encoding of the length - VAR_RLP = 2; - // FIXED32_BIG uses big-endian encoding of the length as a 32 bit integer - FIXED32_BIG = 3; - // FIXED32_LITTLE uses little-endian encoding of the length as a 32 bit integer - FIXED32_LITTLE = 4; - // FIXED64_BIG uses big-endian encoding of the length as a 64 bit integer - FIXED64_BIG = 5; - // FIXED64_LITTLE uses little-endian encoding of the length as a 64 bit integer - FIXED64_LITTLE = 6; - // REQUIRE_32_BYTES is like NONE, but will fail if the input is not exactly 32 bytes (sha256 output) - REQUIRE_32_BYTES = 7; - // REQUIRE_64_BYTES is like NONE, but will fail if the input is not exactly 64 bytes (sha512 output) - REQUIRE_64_BYTES = 8; -} - -/** -ExistenceProof takes a key and a value and a set of steps to perform on it. -The result of peforming all these steps will provide a "root hash", which can -be compared to the value in a header. - -Since it is computationally infeasible to produce a hash collission for any of the used -cryptographic hash functions, if someone can provide a series of operations to transform -a given key and value into a root hash that matches some trusted root, these key and values -must be in the referenced merkle tree. - -The only possible issue is maliablity in LeafOp, such as providing extra prefix data, -which should be controlled by a spec. Eg. with lengthOp as NONE, - prefix = FOO, key = BAR, value = CHOICE -and - prefix = F, key = OOBAR, value = CHOICE -would produce the same value. - -With LengthOp this is tricker but not impossible. Which is why the "leafPrefixEqual" field -in the ProofSpec is valuable to prevent this mutability. And why all trees should -length-prefix the data before hashing it. -*/ -message ExistenceProof { - bytes key = 1; - bytes value = 2; - LeafOp leaf = 3; - repeated InnerOp path = 4; -} - -/* -NonExistenceProof takes a proof of two neighbors, one left of the desired key, -one right of the desired key. If both proofs are valid AND they are neighbors, -then there is no valid proof for the given key. -*/ -message NonExistenceProof { - bytes key = 1; // TODO: remove this as unnecessary??? we prove a range - ExistenceProof left = 2; - ExistenceProof right = 3; -} - -/* -CommitmentProof is either an ExistenceProof or a NonExistenceProof, or a Batch of such messages -*/ -message CommitmentProof { - oneof proof { - ExistenceProof exist = 1; - NonExistenceProof nonexist = 2; - BatchProof batch = 3; - CompressedBatchProof compressed = 4; - } -} - -/** -LeafOp represents the raw key-value data we wish to prove, and -must be flexible to represent the internal transformation from -the original key-value pairs into the basis hash, for many existing -merkle trees. - -key and value are passed in. So that the signature of this operation is: - leafOp(key, value) -> output - -To process this, first prehash the keys and values if needed (ANY means no hash in this case): - hkey = prehashKey(key) - hvalue = prehashValue(value) - -Then combine the bytes, and hash it - output = hash(prefix || length(hkey) || hkey || length(hvalue) || hvalue) -*/ -message LeafOp { - HashOp hash = 1; - HashOp prehash_key = 2; - HashOp prehash_value = 3; - LengthOp length = 4; - // prefix is a fixed bytes that may optionally be included at the beginning to differentiate - // a leaf node from an inner node. - bytes prefix = 5; -} - -/** -InnerOp represents a merkle-proof step that is not a leaf. -It represents concatenating two children and hashing them to provide the next result. - -The result of the previous step is passed in, so the signature of this op is: - innerOp(child) -> output - -The result of applying InnerOp should be: - output = op.hash(op.prefix || child || op.suffix) - - where the || operator is concatenation of binary data, -and child is the result of hashing all the tree below this step. - -Any special data, like prepending child with the length, or prepending the entire operation with -some value to differentiate from leaf nodes, should be included in prefix and suffix. -If either of prefix or suffix is empty, we just treat it as an empty string -*/ -message InnerOp { - HashOp hash = 1; - bytes prefix = 2; - bytes suffix = 3; -} - -/** -ProofSpec defines what the expected parameters are for a given proof type. -This can be stored in the client and used to validate any incoming proofs. - - verify(ProofSpec, Proof) -> Proof | Error - -As demonstrated in tests, if we don't fix the algorithm used to calculate the -LeafHash for a given tree, there are many possible key-value pairs that can -generate a given hash (by interpretting the preimage differently). -We need this for proper security, requires client knows a priori what -tree format server uses. But not in code, rather a configuration object. -*/ -message ProofSpec { - // any field in the ExistenceProof must be the same as in this spec. - // except Prefix, which is just the first bytes of prefix (spec can be longer) - LeafOp leaf_spec = 1; - InnerSpec inner_spec = 2; - // max_depth (if > 0) is the maximum number of InnerOps allowed (mainly for fixed-depth tries) - int32 max_depth = 3; - // min_depth (if > 0) is the minimum number of InnerOps allowed (mainly for fixed-depth tries) - int32 min_depth = 4; -} - -/* -InnerSpec contains all store-specific structure info to determine if two proofs from a -given store are neighbors. - -This enables: - - isLeftMost(spec: InnerSpec, op: InnerOp) - isRightMost(spec: InnerSpec, op: InnerOp) - isLeftNeighbor(spec: InnerSpec, left: InnerOp, right: InnerOp) -*/ -message InnerSpec { - // Child order is the ordering of the children node, must count from 0 - // iavl tree is [0, 1] (left then right) - // merk is [0, 2, 1] (left, right, here) - repeated int32 child_order = 1; - int32 child_size = 2; - int32 min_prefix_length = 3; - int32 max_prefix_length = 4; - // empty child is the prehash image that is used when one child is nil (eg. 20 bytes of 0) - bytes empty_child = 5; - // hash is the algorithm that must be used for each InnerOp - HashOp hash = 6; -} - -/* -BatchProof is a group of multiple proof types than can be compressed -*/ -message BatchProof { - repeated BatchEntry entries = 1; -} - -// Use BatchEntry not CommitmentProof, to avoid recursion -message BatchEntry { - oneof proof { - ExistenceProof exist = 1; - NonExistenceProof nonexist = 2; - } -} - -/****** all items here are compressed forms *******/ - -message CompressedBatchProof { - repeated CompressedBatchEntry entries = 1; - repeated InnerOp lookup_inners = 2; -} - -// Use BatchEntry not CommitmentProof, to avoid recursion -message CompressedBatchEntry { - oneof proof { - CompressedExistenceProof exist = 1; - CompressedNonExistenceProof nonexist = 2; - } -} - -message CompressedExistenceProof { - bytes key = 1; - bytes value = 2; - LeafOp leaf = 3; - // these are indexes into the lookup_inners table in CompressedBatchProof - repeated int32 path = 4; -} - -message CompressedNonExistenceProof { - bytes key = 1; // TODO: remove this as unnecessary??? we prove a range - CompressedExistenceProof left = 2; - CompressedExistenceProof right = 3; -} diff --git a/ignite/pkg/protoc/protoc.go b/ignite/pkg/protoc/protoc.go new file mode 100644 index 0000000000..c8f8c726be --- /dev/null +++ b/ignite/pkg/protoc/protoc.go @@ -0,0 +1,284 @@ +// Package protoc provides high level access to protoc command. +package protoc + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/ignite/ignite-files/protoc" + + "github.com/ignite/cli/ignite/pkg/cmdrunner/exec" + "github.com/ignite/cli/ignite/pkg/cmdrunner/step" + "github.com/ignite/cli/ignite/pkg/localfs" + "github.com/ignite/cli/ignite/pkg/protoanalysis" +) + +// Option configures Generate configs. +type Option func(*configs) + +// configs holds Generate configs. +type configs struct { + pluginPath string + isGeneratedDepsEnabled bool + pluginOptions []string + env []string + command Cmd +} + +// Plugin configures a plugin for code generation. +func Plugin(path string, options ...string) Option { + return func(c *configs) { + c.pluginPath = path + c.pluginOptions = options + } +} + +// GenerateDependencies enables code generation for the proto files that your protofile depends on. +// use this if your protoc plugin does not give you an option to enable the same feature. +func GenerateDependencies() Option { + return func(c *configs) { + c.isGeneratedDepsEnabled = true + } +} + +// Env assigns environment values during the code generation. +func Env(v ...string) Option { + return func(c *configs) { + c.env = v + } +} + +// WithCommand assigns a protoc command to use for code generation. +// This allows to use a single protoc binary in multiple code generation calls. +// Otherwise, `Generate` creates a new protoc binary each time it is called. +func WithCommand(command Cmd) Option { + return func(c *configs) { + c.command = command + } +} + +// Cmd contains the information necessary to execute the protoc command. +type Cmd struct { + command []string + includes []string +} + +// Command returns the strings to execute the `protoc` command. +func (c Cmd) Command() []string { + return c.command +} + +// Includes returns the proto files import paths. +func (c Cmd) Includes() []string { + return c.includes +} + +// Command sets the protoc binary up and returns the command needed to execute c. +func Command() (command Cmd, cleanup func(), err error) { + // Unpack binary content + gzr, err := gzip.NewReader(bytes.NewReader(protoc.Binary())) + if err != nil { + return Cmd{}, nil, err + } + + defer gzr.Close() + + // Unarchive binary content + tr := tar.NewReader(gzr) + if _, err := tr.Next(); err != nil { + return Cmd{}, nil, err + } + + binary, err := io.ReadAll(tr) + if err != nil { + return Cmd{}, nil, err + } + + path, cleanupProto, err := localfs.SaveBytesTemp(binary, "protoc", 0o755) + if err != nil { + return Cmd{}, nil, err + } + + include, cleanupInclude, err := localfs.SaveTemp(protoc.Include()) + if err != nil { + cleanupProto() + return Cmd{}, nil, err + } + + cleanup = func() { + cleanupProto() + cleanupInclude() + } + + command = Cmd{ + command: []string{path, "-I", include}, + includes: []string{include}, + } + + return command, cleanup, nil +} + +// Generate generates code into outDir from protoPath and its includePaths by using plugins provided with protocOuts. +func Generate(ctx context.Context, outDir, protoPath string, includePaths, protocOuts []string, options ...Option) error { + c := configs{} + + for _, o := range options { + o(&c) + } + + // init the string to run the protoc command and the proto files import path + command := c.command.Command() + includes := c.command.Includes() + + if command == nil { + cmd, cleanup, err := Command() + if err != nil { + return err + } + + defer cleanup() + + command = cmd.Command() + includes = cmd.Includes() + } + // See: https://github.com/ignite/cli/issues/3698 + command = append(command, "--experimental_allow_proto3_optional") + + // add plugin if set. + if c.pluginPath != "" { + command = append(command, "--plugin", c.pluginPath) + } + var existentIncludePaths []string + + // skip if a third party proto source actually doesn't exist on the filesystem. + for _, path := range includePaths { + if _, err := os.Stat(path); os.IsNotExist(err) { + continue + } + existentIncludePaths = append(existentIncludePaths, path) + } + + // append third party proto locations to the command. + for _, importPath := range existentIncludePaths { + command = append(command, "-I", importPath) + } + + // find out the list of proto files to generate code for and perform code generation. + files, err := discoverFiles(ctx, c, protoPath, append(includes, existentIncludePaths...), protoanalysis.NewCache()) + if err != nil { + return err + } + + // run command for each protocOuts. + for _, out := range protocOuts { + command := append(command, out) + command = append(command, files...) + command = append(command, c.pluginOptions...) + + execOpts := []exec.Option{ + exec.StepOption(step.Workdir(outDir)), + exec.IncludeStdLogsToError(), + } + if c.env != nil { + execOpts = append(execOpts, exec.StepOption(step.Env(c.env...))) + } + + if err := exec.Exec(ctx, command, execOpts...); err != nil { + return err + } + } + + return nil +} + +// discoverFiles discovers .proto files to do code generation for. .proto files of the app +// (everything under protoPath) will always be a part of the discovered files. +// +// when .proto files of the app depends on another proto package under includePaths (dependencies), those +// may need to be discovered as well. some protoc plugins already do this discovery internally but +// for the ones that don't, it needs to be handled here if GenerateDependencies() is enabled. +func discoverFiles(ctx context.Context, c configs, protoPath string, includePaths []string, cache *protoanalysis.Cache) ( + discovered []string, err error, +) { + packages, err := protoanalysis.Parse(ctx, cache, protoPath) + if err != nil { + return nil, err + } + + discovered = packages.Files().Paths() + + if !c.isGeneratedDepsEnabled { + return discovered, nil + } + + for _, file := range packages.Files() { + d, err := searchFile(file, protoPath, includePaths) + if err != nil { + return nil, err + } + discovered = append(discovered, d...) + } + + return discovered, nil +} + +func searchFile(file protoanalysis.File, protoPath string, includePaths []string) (discovered []string, err error) { + dir := filepath.Dir(file.Path) + + for _, dep := range file.Dependencies { + // try to locate imported .proto file relative to the this .proto file. + guessedPath := filepath.Join(dir, dep) + _, err := os.Stat(guessedPath) + if err == nil { + discovered = append(discovered, guessedPath) + continue + } + if !os.IsNotExist(err) { + return nil, err + } + + // otherwise, search by absolute path in includePaths. + var found bool + for _, included := range includePaths { + guessedPath := filepath.Join(included, dep) + _, err := os.Stat(guessedPath) + if err == nil { + // found the dependency. + // if it's under protoPath, it is already discovered so, skip it. + if !strings.HasPrefix(guessedPath, protoPath) { + discovered = append(discovered, guessedPath) + + // perform a complete search on this one to discover its dependencies as well. + depFile, err := protoanalysis.ParseFile(guessedPath) + if err != nil { + return nil, err + } + d, err := searchFile(depFile, protoPath, includePaths) + if err != nil { + return nil, err + } + discovered = append(discovered, d...) + } + + found = true + break + } + if !os.IsNotExist(err) { + return nil, err + } + } + + if !found { + return nil, fmt.Errorf("cannot locate dependency %q for %q", dep, file.Path) + } + } + + return discovered, nil +} diff --git a/ignite/pkg/xos/files.go b/ignite/pkg/xos/files.go index 11a37c7d87..60eb6e4a0e 100644 --- a/ignite/pkg/xos/files.go +++ b/ignite/pkg/xos/files.go @@ -7,7 +7,8 @@ import ( ) const ( - JSONFile = "json" + JSONFile = "json" + ProtoFile = "proto" ) func FindFiles(directory, extension string) ([]string, error) { diff --git a/ignite/services/chain/generate.go b/ignite/services/chain/generate.go index d3ed81a70f..64f5570b6e 100644 --- a/ignite/services/chain/generate.go +++ b/ignite/services/chain/generate.go @@ -51,6 +51,7 @@ func GeneratePulsar() GenerateTarget { // overriding the configured or default path. Path can be an empty string. func GenerateTSClient(path string, useCache bool) GenerateTarget { return func(o *generateOptions) { + o.isOpenAPIEnabled = true o.isTSClientEnabled = true o.tsClientPath = path o.useCache = useCache @@ -60,6 +61,7 @@ func GenerateTSClient(path string, useCache bool) GenerateTarget { // GenerateVuex enables generating proto based Typescript Client and Vuex Stores. func GenerateVuex(path string) GenerateTarget { return func(o *generateOptions) { + o.isOpenAPIEnabled = true o.isTSClientEnabled = true o.isVuexEnabled = true o.vuexPath = path @@ -69,6 +71,7 @@ func GenerateVuex(path string) GenerateTarget { // GenerateComposables enables generating proto based Typescript Client and Vue 3 composables. func GenerateComposables(path string) GenerateTarget { return func(o *generateOptions) { + o.isOpenAPIEnabled = true o.isTSClientEnabled = true o.isComposablesEnabled = true o.composablesPath = path @@ -78,6 +81,7 @@ func GenerateComposables(path string) GenerateTarget { // GenerateHooks enables generating proto based Typescript Client and React composables. func GenerateHooks(path string) GenerateTarget { return func(o *generateOptions) { + o.isOpenAPIEnabled = true o.isTSClientEnabled = true o.isHooksEnabled = true o.hooksPath = path @@ -101,6 +105,10 @@ func (c *Chain) generateFromConfig(ctx context.Context, cacheStorage cache.Stora // Additional code generation targets var targets []GenerateTarget + if conf.Client.OpenAPI.Path != "" { + targets = append(targets, GenerateOpenAPI()) + } + if generateClients { if p := conf.Client.Typescript.Path; p != "" { targets = append(targets, GenerateTSClient(p, true)) @@ -120,10 +128,6 @@ func (c *Chain) generateFromConfig(ctx context.Context, cacheStorage cache.Stora } } - if conf.Client.OpenAPI.Path != "" { - targets = append(targets, GenerateOpenAPI()) - } - // Generate proto based code for Go and optionally for any optional targets return c.Generate(ctx, cacheStorage, GenerateGo(), targets...) } @@ -169,6 +173,20 @@ func (c *Chain) Generate( updateConfig bool ) + if targetOptions.isOpenAPIEnabled { + openAPIPath = conf.Client.OpenAPI.Path + if openAPIPath == "" { + openAPIPath = chainconfig.DefaultOpenAPIPath + } + + // Non absolute OpenAPI paths must be treated as relative to the app directory + if !filepath.IsAbs(openAPIPath) { + openAPIPath = filepath.Join(c.app.Path, openAPIPath) + } + + options = append(options, cosmosgen.WithOpenAPIGeneration(openAPIPath)) + } + if targetOptions.isTSClientEnabled { tsClientPath = targetOptions.tsClientPath if tsClientPath == "" { @@ -270,20 +288,6 @@ func (c *Chain) Generate( ) } - if targetOptions.isOpenAPIEnabled { - openAPIPath = conf.Client.OpenAPI.Path - if openAPIPath == "" { - openAPIPath = chainconfig.DefaultOpenAPIPath - } - - // Non absolute OpenAPI paths must be treated as relative to the app directory - if !filepath.IsAbs(openAPIPath) { - openAPIPath = filepath.Join(c.app.Path, openAPIPath) - } - - options = append(options, cosmosgen.WithOpenAPIGeneration(openAPIPath)) - } - if err := cosmosgen.Generate( ctx, cacheStorage, diff --git a/ignite/services/scaffolder/init.go b/ignite/services/scaffolder/init.go index e1c25b9f43..0c0d990c91 100644 --- a/ignite/services/scaffolder/init.go +++ b/ignite/services/scaffolder/init.go @@ -46,19 +46,13 @@ func Init( path = filepath.Join(root, appFolder) // create the project - if err := generate( - ctx, - tracer, - pathInfo, - addressPrefix, - path, - noDefaultModule, - params, - ); err != nil { + err = generate(ctx, tracer, pathInfo, addressPrefix, path, noDefaultModule, params) + if err != nil { return "", err } - if err := finish(ctx, cacheStorage, path, pathInfo.RawPath); err != nil { + err = finish(ctx, cacheStorage, path, pathInfo.RawPath) + if err != nil { return "", err } diff --git a/ignite/services/scaffolder/scaffolder.go b/ignite/services/scaffolder/scaffolder.go index d26fae4492..6caa944113 100644 --- a/ignite/services/scaffolder/scaffolder.go +++ b/ignite/services/scaffolder/scaffolder.go @@ -64,7 +64,8 @@ func New(appPath string) (Scaffolder, error) { } func finish(ctx context.Context, cacheStorage cache.Storage, path, gomodPath string) error { - if err := protoc(ctx, cacheStorage, path, gomodPath); err != nil { + err := protoc(ctx, cacheStorage, path, gomodPath) + if err != nil { return err } diff --git a/ignite/templates/app/files/proto/buf.gen.sta.yaml b/ignite/templates/app/files/proto/buf.gen.sta.yaml new file mode 100644 index 0000000000..d8cea8b342 --- /dev/null +++ b/ignite/templates/app/files/proto/buf.gen.sta.yaml @@ -0,0 +1,14 @@ +# This file is auto-generated from Ignite. You can edit +# the file content but do not change the file name or path. +# +# buf.gen.sta.yaml +# +version: v1 +plugins: + - name: openapiv2 + out: . + opt: + - logtostderr=true + - openapi_naming_strategy=simple + - simple_operation_ids=false + - json_names_for_fields=false diff --git a/ignite/templates/app/files/proto/buf.lock b/ignite/templates/app/files/proto/buf.lock index 6cc63b7edc..77287f75ce 100644 --- a/ignite/templates/app/files/proto/buf.lock +++ b/ignite/templates/app/files/proto/buf.lock @@ -25,3 +25,7 @@ deps: owner: googleapis repository: googleapis commit: 75b4300737fb4efca0831636be94e517 + - remote: buf.build + owner: protocolbuffers + repository: wellknowntypes + commit: 44e83bc050a4497fa7b36b34d95ca156 diff --git a/ignite/templates/app/files/proto/buf.yaml b/ignite/templates/app/files/proto/buf.yaml index c75488a338..7a86adcfac 100644 --- a/ignite/templates/app/files/proto/buf.yaml +++ b/ignite/templates/app/files/proto/buf.yaml @@ -5,6 +5,7 @@ # version: v1 deps: + - buf.build/protocolbuffers/wellknowntypes - buf.build/cosmos/cosmos-sdk - buf.build/cosmos/cosmos-proto - buf.build/cosmos/gogo-proto