feat: re-enable TS codegen (#3655)

* feat: Re-introduce module discovery * feat: Look for indirect proto deps in go.mod * feat: Introduce indirect proto dependency discovery * feat: Add buf export command * feat: User buf-based openapi generation * fix: Revert to previous sta version & peg TS version * feat(wip): Update dependency resolution algorithm * feat: Clean up include resolution * chore: Add changelog entry * chore: Fix linting issues & typos * fix: cosmos-sdk buf issue * feat(pkg/protoc): change package to use protoc binary from files repo (#3657) * feat(pkg/protoc): change package to use protoc binary from files repo * chore(pkg/protoc): remove embedded protoc binary * fix: correct buf.gen.sta.yml file * fix: address review * fix: Remove logging * fix: ResolveDependencies call * fix: correct lint issues and merge `main` (#3690) * chore: remove redundant variable * chore(pkg/cosmosbuf): improve snippet redability * chore: remove redundant default value * chore: code formatting * feat(pkg): remove nodetime binaries (#3670) * - remote nodetime binaries - import ignite-files repo pkg - remote gen binaries scripts and pipeline - run gofmt * remove tollchain from go.mod * add changelog * Update changelog * ci: update Go version to `1.21` in GitHub workflows (#3687) --------- Co-authored-by: Pantani <Pantani> Co-authored-by: Jerónimo Albi <[email protected]> * fix: change Go version in nightly release config to `1.21.2` (#3688) * fix: change Go version in nightly release config to `1.21.2` The release action requires a version that includes the patch version to be able to download the right tarball with the Go binary. See https://github.com/wangyoucao577/go-release-action/blob/2ac3035fa4c4feed6a8272ce278b0577b93cf8e5/setup-go.sh#L24 * chore: change go releaser to use the lates Go version Latest is the default but it's setted here to be explicit about that fact. See https://github.com/wangyoucao577/go-release-action#parameters * fix: remove explicit "latest" for Go version release nightly (#3689) Lates version is used by default but it shouldn't be specified explicitly because the value "latest" is not supported. * chore: merge `main` * test: fix broken integration test * fix: change scaffolder to skip protoc when no module is scaffolded There are no proto file when an app is scaffolded without module which makes buf export fail because buf workspace references a folder without proto files. * fix(pkg/cosmosbuf): add check for proto files before running Buf Changes previous commit because some files like the OpenAPI file should be generated even when there are no proto files. We should also allow generating code for standard dependencies like Comos SDK when the app doesn't have proto files. --------- Co-authored-by: Danilo Pantani <[email protected]> * feat: Bump nodetime version * refactor: cleanup tmp folders & change third party includes (#3696) * fix: implement third party includes using previous semantics Use `thirdModuleIncludes` instead of defining a new type to store modules and includes to be consistent with `appIncludes` and to avoid changing existing code to work with a new type. * feat: add tmp dir cleanup support to cosmos generator * chore: restore Go version to `1.21` and remove toolchain Currently CI is giving an error because is not properly configured to support the new Go 1.21 features for the mod file and we don't have a consensus yet on how to configure the toolchain and Go versions. * chore: go mod tidy * ci: change integration tests to use Go version from mod file This is required to avoid downloading a newer Go toolchain which will fail because GOSUMDB is disabled to fix the timeout issues because of the repository size. * ci: disable GOTOOLCHAIN when running integration tests This is required because toolchain won't work when GOSUMDB is disabled. * ci: change integration tests to use the stable go version The stable Go version should be used to avoid keeping updating the workflow config when a new patch version is released. This is to make sure that Go doesn't try to download a new toolchain version. * fix: correct CI linting issues * ci: remove redundant GH workflow cache step Go setup action caches since `v4` * fix: disable go setup GH action cache Action for `golangci-lint` already have caching functionality so Go setup must be disabled to avoid caching issues. * chore: add thitd party includes only when available * fix: correct directory remove Co-authored-by: Danilo Pantani <[email protected]> * fix: Address code review comments * chore: Address review comment * chore: remove commented function * Update ignite/pkg/cosmosbuf/buf.go Co-authored-by: Jerónimo Albi <[email protected]> * chore: fix function name * Update ignite/pkg/cosmosgen/generate_openapi.go Co-authored-by: Danilo Pantani <[email protected]> * chore: address review * chore: linting * chore: test --experimental_allow_proto3_optional flag --------- Co-authored-by: Jerónimo Albi <[email protected]> Co-authored-by: Danilo Pantani <[email protected]> Co-authored-by: jeronimoalbi <[email protected]>
ignite · Oct 24, 2023 · f643ccd · f643ccd
1 parent 50bf90d
commit f643ccd
Show file tree

Hide file tree

Showing 23 changed files with 823 additions and 406 deletions.
diff --git a/changelog.md b/changelog.md
@@ -33,6 +33,7 @@
 - [#3610](https://github.com/ignite/cli/pull/3610) Fix overflow issue of cosmos faucet in `pkg/cosmosfaucet/transfer.go` and `pkg/cosmosfaucet/cosmosfaucet.go`
 - [#3618](https://github.com/ignite/cli/pull/3618) Fix TS client generation import path issue
 - [#3631](https://github.com/ignite/cli/pull/3631) Fix unnecessary vue import in hooks/composables template
+- [#3655](https://github.com/ignite/cli/pull/3655) Re-enable TS client generation
 - [#3661](https://github.com/ignite/cli/pull/3661) Change `pkg/cosmosanalysis` to find Cosmos SDK runtime app registered modules
 
 ## [`v0.27.0`](https://github.com/ignite/cli/releases/tag/v0.27.0)

diff --git a/go.mod b/go.mod
@@ -1,6 +1,8 @@
 module github.com/ignite/cli
 
-go 1.21
+go 1.21.1
+
+toolchain go1.21.3
 
 require (
 	cosmossdk.io/math v1.0.1
@@ -37,7 +39,8 @@ require (
 	github.com/hashicorp/go-hclog v1.2.0
 	github.com/hashicorp/go-plugin v1.4.9
 	github.com/iancoleman/strcase v0.2.0
-	github.com/ignite/ignite-files/nodetime v0.0.1
+	github.com/ignite/ignite-files/nodetime v0.0.2
+	github.com/ignite/ignite-files/protoc v0.0.1
 	github.com/ignite/web v0.4.3
 	github.com/imdario/mergo v0.3.15
 	github.com/jpillora/chisel v1.8.1

diff --git a/go.sum b/go.sum
@@ -765,8 +765,10 @@ github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHL
 github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/ignite/ignite-files/nodetime v0.0.1 h1:GwsUI+hYOwT5w0GayHSJlOVoDJDhg7RVMrl4oV7VEbc=
-github.com/ignite/ignite-files/nodetime v0.0.1/go.mod h1:oMsEk/+FHcHKNiCedFRbLmSAnb6zs3n23KbedSb09LE=
+github.com/ignite/ignite-files/nodetime v0.0.2 h1:9Aj0OEa7FWI22J/Zdq7M2JvsjgFLngZSm7vB4i9X4t4=
+github.com/ignite/ignite-files/nodetime v0.0.2/go.mod h1:GKDsXdeazHyhSBPdVLp7mNIo/m9LmZ6/h8RmQ0/CoaM=
+github.com/ignite/ignite-files/protoc v0.0.1 h1:wXxU1dzruUgSVl1diAuAOA+xv0NQKXJFsDWht2+tAP8=
+github.com/ignite/ignite-files/protoc v0.0.1/go.mod h1:cVCHJbEHPIeKHMPk3ZoPS0Xw4XQfUc76BAMAPU9Fwjg=
 github.com/ignite/web v0.4.3 h1:LHucUEXttzCf5JmxO5/xLKVx1Qz1o124HqF4Nii5uWQ=
 github.com/ignite/web v0.4.3/go.mod h1:WZWBaBYF8RazN7dE462BLpvXDY8ScacxcJ07BKwX/jY=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=

diff --git a/ignite/pkg/cosmosanalysis/app/app.go b/ignite/pkg/cosmosanalysis/app/app.go
@@ -423,7 +423,7 @@ func resolveCosmosPackagePath(chainRoot string) (string, error) {
 		return "", err
 	}
 
-	deps, err := gomodule.ResolveDependencies(modFile)
+	deps, err := gomodule.ResolveDependencies(modFile, false)
 	if err != nil {
 		return "", err
 	}

diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go
@@ -23,9 +23,9 @@ type (
 
 	// Buf represents the buf application structure.
 	Buf struct {
-		path     string
-		sdkCache string
-		cache    *protoanalysis.Cache
+		path        string
+		sdkProtoDir string
+		cache       *protoanalysis.Cache
 	}
 )
 
@@ -39,15 +39,20 @@ const (
 
 	// CMDGenerate generate command.
 	CMDGenerate Command = "generate"
+	CMDExport   Command = "export"
 )
 
 var (
 	commands = map[Command]struct{}{
 		CMDGenerate: {},
+		CMDExport:   {},
 	}
 
-	// ErrInvalidCommand error invalid command name.
+	// ErrInvalidCommand indicates an invalid command name.
 	ErrInvalidCommand = errors.New("invalid command name")
+
+	// ErrProtoFilesNotFound indicates that no ".proto" files were found.
+	ErrProtoFilesNotFound = errors.New("no proto files found")
 )
 
 // New creates a new Buf based on the installed binary.
@@ -67,6 +72,51 @@ func (c Command) String() string {
 	return string(c)
 }
 
+// Export runs the buf Export command for the files in the proto directory.
+func (b Buf) Export(ctx context.Context, protoDir, output string) error {
+	// Check if the proto directory is the Cosmos SDK one
+	if strings.Contains(protoDir, cosmosver.CosmosModulePath) {
+		if b.sdkProtoDir == "" {
+			// Copy Cosmos SDK proto path without the Buf workspace.
+			// This is done because the workspace contains a reference to
+			// a "orm/internal" proto folder that is not present by default
+			// in the SDK repository.
+			d, err := copySDKProtoDir(protoDir)
+			if err != nil {
+				return err
+			}
+
+			b.sdkProtoDir = d
+		}
+
+		// Split absolute path into an absolute prefix and a relative suffix
+		paths := strings.Split(protoDir, "/proto")
+		if len(paths) < 2 {
+			return fmt.Errorf("invalid Cosmos SDK mod path: %s", protoDir)
+		}
+
+		// Use the SDK copy to resolve SDK proto files
+		protoDir = filepath.Join(b.sdkProtoDir, paths[1])
+	}
+	specs, err := xos.FindFiles(protoDir, xos.ProtoFile)
+	if err != nil {
+		return err
+	}
+	if len(specs) == 0 {
+		return fmt.Errorf("%w: %s", ErrProtoFilesNotFound, protoDir)
+	}
+	flags := map[string]string{
+		flagOutput: output,
+	}
+
+	cmd, err := b.generateCommand(CMDExport, flags, protoDir)
+	if err != nil {
+		return err
+	}
+
+	return b.runCommand(ctx, cmd...)
+}
+
 // Generate runs the buf Generate command for each file into the proto directory.
 func (b Buf) Generate(
 	ctx context.Context,
@@ -94,17 +144,17 @@ func (b Buf) Generate(
 	// change the workspace copying the files to another folder and generate the
 	// files.
 	if strings.Contains(protoDir, cosmosver.CosmosModulePath) {
-		if b.sdkCache == "" {
-			b.sdkCache, err = prepareSDK(protoDir)
+		if b.sdkProtoDir == "" {
+			b.sdkProtoDir, err = copySDKProtoDir(protoDir)
 			if err != nil {
 				return err
 			}
 		}
 		dirs := strings.Split(protoDir, "/proto/")
 		if len(dirs) < 2 {
-			return fmt.Errorf("invalid cosmos sdk mod path: %s", dirs)
+			return fmt.Errorf("invalid Cosmos SDK mod path: %s", dirs)
 		}
-		protoDir = filepath.Join(b.sdkCache, dirs[1])
+		protoDir = filepath.Join(b.sdkProtoDir, dirs[1])
 	}
 
 	pkgs, err := protoanalysis.Parse(ctx, b.cache, protoDir)
@@ -118,14 +168,20 @@ func (b Buf) Generate(
 			if _, ok := excluded[filepath.Base(file.Path)]; ok {
 				continue
 			}
-			cmd, err := b.generateCommand(
-				CMDGenerate,
-				flags,
-				file.Path,
-			)
+
+			specs, err := xos.FindFiles(protoDir, "proto")
 			if err != nil {
 				return err
 			}
+			if len(specs) == 0 {
+				continue
+			}
+
+			cmd, err := b.generateCommand(CMDGenerate, flags, file.Path)
+			if err != nil {
+				return err
+			}
+
 			g.Go(func() error {
 				cmd := cmd
 				return b.runCommand(ctx, cmd...)
@@ -135,6 +191,14 @@ func (b Buf) Generate(
 	return g.Wait()
 }
 
+// Cleanup deletes temporary files and directories.
+func (b Buf) Cleanup() error {
+	if b.sdkProtoDir != "" {
+		return os.RemoveAll(b.sdkProtoDir)
+	}
+	return nil
+}
+
 // runCommand run the buf CLI command.
 func (b Buf) runCommand(ctx context.Context, cmd ...string) error {
 	execOpts := []exec.Option{
@@ -167,7 +231,7 @@ func (b Buf) generateCommand(
 	return command, nil
 }
 
-// findSDKProtoPath find the cosmos-sdk proto folder path.
+// findSDKProtoPath finds the Cosmos SDK proto folder path.
 func findSDKProtoPath(protoDir string) (string, error) {
 	paths := strings.Split(protoDir, "@")
 	if len(paths) < 2 {
@@ -177,9 +241,9 @@ func findSDKProtoPath(protoDir string) (string, error) {
 	return fmt.Sprintf("%s@%s/proto", paths[0], version), nil
 }
 
-// prepareSDK copy the cosmos sdk proto folder to a temporary directory
-// so we can skip the buf workspace.
-func prepareSDK(protoDir string) (string, error) {
+// copySDKProtoDir copies the Cosmos SDK proto folder to a temporary directory.
+// The temporary directory must be removed by the caller.
+func copySDKProtoDir(protoDir string) (string, error) {
 	tmpDir, err := os.MkdirTemp("", "proto-sdk")
 	if err != nil {
 		return "", err

diff --git a/ignite/pkg/cosmosgen/cosmosgen.go b/ignite/pkg/cosmosgen/cosmosgen.go
@@ -2,6 +2,7 @@ package cosmosgen
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -36,8 +37,6 @@ type generateOptions struct {
 	specOut string
 }
 
-// TODO add WithInstall.
-
 // ModulePathFunc defines a function type that returns a path based on a Cosmos SDK module.
 type ModulePathFunc func(module.Module) string
 
@@ -106,17 +105,27 @@ func IncludeDirs(dirs []string) Option {
 
 // generator generates code for sdk and sdk apps.
 type generator struct {
-	ctx          context.Context
-	buf          cosmosbuf.Buf
-	cacheStorage cache.Storage
-	appPath      string
-	protoDir     string
-	gomodPath    string
-	o            *generateOptions
-	sdkImport    string
-	deps         []gomodule.Version
-	appModules   []module.Module
-	thirdModules map[string][]module.Module // app dependency-modules pair.
+	ctx                 context.Context
+	buf                 cosmosbuf.Buf
+	cacheStorage        cache.Storage
+	appPath             string
+	protoDir            string
+	gomodPath           string
+	opts                *generateOptions
+	sdkImport           string
+	deps                []gomodule.Version
+	appModules          []module.Module
+	appIncludes         []string
+	thirdModules        map[string][]module.Module
+	thirdModuleIncludes map[string][]string
+	tmpDirs             []string
+}
+
+func (g *generator) cleanup() {
+	// Remove temporary directories created during generation
+	for _, path := range g.tmpDirs {
+		_ = os.RemoveAll(path)
+	}
 }
 
 // Generate generates code from protoDir of an SDK app residing at appPath with given options.
@@ -127,19 +136,24 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir
 		return err
 	}
 
+	defer b.Cleanup()
+
 	g := &generator{
-		ctx:          ctx,
-		buf:          b,
-		appPath:      appPath,
-		protoDir:     protoDir,
-		gomodPath:    gomodPath,
-		o:            &generateOptions{},
-		thirdModules: make(map[string][]module.Module),
-		cacheStorage: cacheStorage,
+		ctx:                 ctx,
+		buf:                 b,
+		appPath:             appPath,
+		protoDir:            protoDir,
+		gomodPath:           gomodPath,
+		opts:                &generateOptions{},
+		thirdModules:        make(map[string][]module.Module),
+		thirdModuleIncludes: make(map[string][]string),
+		cacheStorage:        cacheStorage,
 	}
 
+	defer g.cleanup()
+
 	for _, apply := range options {
-		apply(g.o)
+		apply(g.opts)
 	}
 
 	if err := g.setup(); err != nil {
@@ -148,24 +162,30 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir
 
 	// Go generation must run first so the types are created before other
 	// generated code that requires sdk.Msg implementations to be defined
-	if g.o.isGoEnabled {
+	if g.opts.isGoEnabled {
 		if err := g.generateGo(); err != nil {
 			return err
 		}
 	}
-	if g.o.isPulsarEnabled {
+	if g.opts.isPulsarEnabled {
 		if err := g.generatePulsar(); err != nil {
 			return err
 		}
 	}
 
-	if g.o.jsOut != nil {
+	if g.opts.specOut != "" {
+		if err := g.generateOpenAPISpec(); err != nil {
+			return err
+		}
+	}
+
+	if g.opts.jsOut != nil {
 		if err := g.generateTS(); err != nil {
 			return err
 		}
 	}
 
-	if g.o.vuexOut != nil {
+	if g.opts.vuexOut != nil {
 		if err := g.generateVuex(); err != nil {
 			return err
 		}
@@ -186,7 +206,7 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir
 
 	}
 
-	if g.o.composablesRootPath != "" {
+	if g.opts.composablesRootPath != "" {
 		if err := g.generateComposables("vue"); err != nil {
 			return err
 		}
@@ -198,7 +218,7 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir
 			return err
 		}
 	}
-	if g.o.hooksRootPath != "" {
+	if g.opts.hooksRootPath != "" {
 		if err := g.generateComposables("react"); err != nil {
 			return err
 		}
@@ -211,12 +231,6 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir
 		}
 	}
 
-	if g.o.specOut != "" {
-		if err := g.generateOpenAPISpec(); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }