From 9663523662397e266f318bf7ed4c5de0ca106526 Mon Sep 17 00:00:00 2001 From: Pantani Date: Wed, 8 May 2024 18:59:17 +0200 Subject: [PATCH 01/20] draft improvements --- ignite/pkg/cosmosbuf/buf.go | 153 ++++++----------------- ignite/pkg/cosmosbuf/buf_test.go | 44 ++++--- ignite/pkg/cosmosgen/cosmosgen.go | 4 +- ignite/pkg/cosmosgen/generate_go.go | 39 +----- ignite/pkg/cosmosgen/generate_openapi.go | 25 +--- 5 files changed, 71 insertions(+), 194 deletions(-) diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index 94dedeb047..1513ab3bfc 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -3,14 +3,11 @@ package cosmosbuf import ( "context" "fmt" - "os" - "path/filepath" + "strconv" "strings" - "golang.org/x/sync/errgroup" - + "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cmdrunner/exec" - "github.com/ignite/cli/v29/ignite/pkg/cosmosver" "github.com/ignite/cli/v29/ignite/pkg/errors" "github.com/ignite/cli/v29/ignite/pkg/protoanalysis" "github.com/ignite/cli/v29/ignite/pkg/xexec" @@ -23,9 +20,9 @@ type ( // Buf represents the buf application structure. Buf struct { - path string - sdkProtoDir string - cache *protoanalysis.Cache + path string + protoCache *protoanalysis.Cache + storageCache cache.Cache[[]byte] } ) @@ -35,6 +32,7 @@ const ( flagOutput = "output" flagErrorFormat = "error-format" flagLogFormat = "log-format" + flagExcludePath = "exclude-path" flagOnly = "only" fmtJSON = "json" @@ -42,6 +40,8 @@ const ( CMDGenerate Command = "generate" CMDExport Command = "export" CMDMod Command = "mod" + + specCacheNamespace = "generate.buf" ) var ( @@ -59,14 +59,16 @@ var ( ) // New creates a new Buf based on the installed binary. -func New() (Buf, error) { +func New(cacheStorage cache.Storage) (Buf, error) { path, err := xexec.ResolveAbsPath(binaryName) if err != nil { return Buf{}, err } + return Buf{ - path: path, - cache: protoanalysis.NewCache(), + path: path, + storageCache: cache.New[[]byte](cacheStorage, specCacheNamespace), + protoCache: protoanalysis.NewCache(), }, nil } @@ -95,32 +97,6 @@ func (b Buf) Update(ctx context.Context, modDir string, dependencies ...string) // Export runs the buf Export command for the files in the proto directory. func (b Buf) Export(ctx context.Context, protoDir, output string) error { - // Check if the proto directory is the Cosmos SDK one - // TODO(@julienrbrt): this whole custom handling can be deleted - // after https://github.com/cosmos/cosmos-sdk/pull/18993 in v29. - if strings.Contains(protoDir, cosmosver.CosmosSDKRepoName) { - if b.sdkProtoDir == "" { - // Copy Cosmos SDK proto path without the Buf workspace. - // This is done because the workspace contains a reference to - // a "orm/internal" proto folder that is not present by default - // in the SDK repository. - d, err := copySDKProtoDir(protoDir) - if err != nil { - return err - } - - b.sdkProtoDir = d - } - - // Split absolute path into an absolute prefix and a relative suffix - paths := strings.Split(protoDir, "/proto") - if len(paths) < 2 { - return errors.Errorf("invalid Cosmos SDK mod path: %s", protoDir) - } - - // Use the SDK copy to resolve SDK proto files - protoDir = filepath.Join(b.sdkProtoDir, paths[1]) - } specs, err := xos.FindFiles(protoDir, xos.ProtoFile) if err != nil { return err @@ -146,77 +122,27 @@ func (b Buf) Generate( protoDir, output, template string, - excludeFilename ...string, + excluded ...string, ) (err error) { - var ( - excluded = make(map[string]struct{}) - flags = map[string]string{ - flagTemplate: template, - flagOutput: output, - flagErrorFormat: fmtJSON, - flagLogFormat: fmtJSON, - } - ) - for _, file := range excludeFilename { - excluded[file] = struct{}{} + protoFiles, err := xos.FindFiles(protoDir, xos.ProtoFile) + if err != nil || len(protoFiles) == 0 { + return err } - // TODO(@julienrbrt): this whole custom handling can be deleted - // after https://github.com/cosmos/cosmos-sdk/pull/18993 in v29. - if strings.Contains(protoDir, cosmosver.CosmosSDKRepoName) { - if b.sdkProtoDir == "" { - b.sdkProtoDir, err = copySDKProtoDir(protoDir) - if err != nil { - return err - } - } - dirs := strings.Split(protoDir, "/proto/") - if len(dirs) < 2 { - return errors.Errorf("invalid Cosmos SDK mod path: %s", dirs) - } - protoDir = filepath.Join(b.sdkProtoDir, dirs[1]) + flags := map[string]string{ + flagTemplate: template, + flagOutput: output, + flagErrorFormat: fmtJSON, + flagLogFormat: fmtJSON, + flagExcludePath: join(excluded...), } - pkgs, err := protoanalysis.Parse(ctx, b.cache, protoDir) + cmd, err := b.generateCommand(CMDGenerate, flags, protoDir) if err != nil { return err } - g, ctx := errgroup.WithContext(ctx) - for _, pkg := range pkgs { - for _, file := range pkg.Files { - if _, ok := excluded[filepath.Base(file.Path)]; ok { - continue - } - - specs, err := xos.FindFiles(protoDir, xos.ProtoFile) - if err != nil { - return err - } - if len(specs) == 0 { - continue - } - - cmd, err := b.generateCommand(CMDGenerate, flags, file.Path) - if err != nil { - return err - } - - g.Go(func() error { - cmd := cmd - return b.runCommand(ctx, cmd...) - }) - } - } - return g.Wait() -} - -// Cleanup deletes temporary files and directories. -func (b Buf) Cleanup() error { - if b.sdkProtoDir != "" { - return os.RemoveAll(b.sdkProtoDir) - } - return nil + return b.runCommand(ctx, cmd...) } // runCommand run the buf CLI command. @@ -251,24 +177,19 @@ func (b Buf) generateCommand( return command, nil } -// findSDKProtoPath finds the Cosmos SDK proto folder path. -func findSDKProtoPath(protoDir string) string { - paths := strings.Split(protoDir, "@") - if len(paths) < 2 { - return protoDir +func join(elems ...string) (value string) { + switch len(elems) { + case 0: + return "" + case 1: + return strconv.Quote(elems[0]) } - version := strings.Split(paths[1], "/")[0] - return fmt.Sprintf("%s@%s/proto", paths[0], version) -} -// copySDKProtoDir copies the Cosmos SDK proto folder to a temporary directory. -// The temporary directory must be removed by the caller. -func copySDKProtoDir(protoDir string) (string, error) { - tmpDir, err := os.MkdirTemp("", "proto-sdk") - if err != nil { - return "", err + var b strings.Builder + b.WriteString(strconv.Quote(elems[0])) + for _, s := range elems[1:] { + b.WriteString(",") + b.WriteString(strconv.Quote(s)) } - - srcPath := findSDKProtoPath(protoDir) - return tmpDir, xos.CopyFolder(srcPath, tmpDir) + return b.String() } diff --git a/ignite/pkg/cosmosbuf/buf_test.go b/ignite/pkg/cosmosbuf/buf_test.go index 68187bf165..9039c92214 100644 --- a/ignite/pkg/cosmosbuf/buf_test.go +++ b/ignite/pkg/cosmosbuf/buf_test.go @@ -6,42 +6,40 @@ import ( "github.com/stretchr/testify/require" ) -func TestFindSDKPath(t *testing.T) { - testCases := []struct { - name string - protoDir string - want string +func Test_join(t *testing.T) { + tests := []struct { + name string + values []string + want string }{ { - name: "full path", - protoDir: "/mod/github.com/cosmos/cosmos-sdk@v0.47.2/test/path/proto", - want: "/mod/github.com/cosmos/cosmos-sdk@v0.47.2/proto", + name: "empty", + values: []string{}, }, { - name: "simple path", - protoDir: "myproto@v1/test/proto/animo/sdk", - want: "myproto@v1/proto", + name: "single", + values: []string{"foo"}, + want: `"foo"`, }, { - name: "only version", - protoDir: "test/myproto@v1", - want: "test/myproto@v1/proto", + name: "two", + values: []string{"foo", "bar"}, + want: `"foo","bar"`, }, { - name: "semantic version", - protoDir: "test/myproto@v0.3.1/test/proto", - want: "test/myproto@v0.3.1/proto", + name: "three", + values: []string{"foo", "bar", "baz"}, + want: `"foo","bar","baz"`, }, { - name: "no version (local)", - protoDir: "test/myproto/test/proto", - want: "test/myproto/test/proto", + name: "repeated", + values: []string{"foo", "bar", "baz", "foo", "bar", "baz"}, + want: `"foo","bar","baz","foo","bar","baz"`, }, } - - for _, tt := range testCases { + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := findSDKProtoPath(tt.protoDir) + got := join(tt.values...) require.Equal(t, tt.want, got) }) } diff --git a/ignite/pkg/cosmosgen/cosmosgen.go b/ignite/pkg/cosmosgen/cosmosgen.go index 4ea01a58f1..520f4ab793 100644 --- a/ignite/pkg/cosmosgen/cosmosgen.go +++ b/ignite/pkg/cosmosgen/cosmosgen.go @@ -132,13 +132,11 @@ func (g *generator) cleanup() { // Generate generates code from protoDir of an SDK app residing at appPath with given options. // protoDir must be relative to the projectPath. func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir, gomodPath string, options ...Option) error { - b, err := cosmosbuf.New() + b, err := cosmosbuf.New(cacheStorage) if err != nil { return err } - defer b.Cleanup() - g := &generator{ buf: b, appPath: appPath, diff --git a/ignite/pkg/cosmosgen/generate_go.go b/ignite/pkg/cosmosgen/generate_go.go index c8049c3134..56673b691c 100644 --- a/ignite/pkg/cosmosgen/generate_go.go +++ b/ignite/pkg/cosmosgen/generate_go.go @@ -19,40 +19,14 @@ func (g *generator) pulsarTemplate() string { } func (g *generator) generateGoGo(ctx context.Context) error { - // create a temporary dir to locate generated code under which later only some of them will be moved to the - // app's source code. this also prevents having leftover files in the app's source code or its parent dir - when - // command executed directly there - in case of an interrupt. - tmp, err := os.MkdirTemp("", "") - if err != nil { - return err - } - defer os.RemoveAll(tmp) - - protoPath := filepath.Join(g.appPath, g.protoDir) - - // code generate for each module. - err = g.buf.Generate(ctx, protoPath, tmp, g.gogoTemplate(), "module.proto") - if err != nil { - return err - } - - // move generated code for the app under the relative locations in its source code. - generatedPath := filepath.Join(tmp, g.gomodPath) - - _, err = os.Stat(generatedPath) - if err == nil { - err = copy.Copy(generatedPath, g.appPath) - if err != nil { - return errors.Wrap(err, "cannot copy path") - } - } else if !os.IsNotExist(err) { - return err - } - - return nil + return g.generate(ctx, g.gogoTemplate(), "module.proto") } func (g *generator) generatePulsar(ctx context.Context) error { + return g.generate(ctx, g.pulsarTemplate()) +} + +func (g *generator) generate(ctx context.Context, template string, excluded ...string) error { // create a temporary dir to locate generated code under which later only some of them will be moved to the // app's source code. this also prevents having leftover files in the app's source code or its parent dir - when // command executed directly there - in case of an interrupt. @@ -65,8 +39,7 @@ func (g *generator) generatePulsar(ctx context.Context) error { protoPath := filepath.Join(g.appPath, g.protoDir) // code generate for each module. - err = g.buf.Generate(ctx, protoPath, tmp, g.pulsarTemplate()) - if err != nil { + if err = g.buf.Generate(ctx, protoPath, tmp, template, excluded...); err != nil { return err } diff --git a/ignite/pkg/cosmosgen/generate_openapi.go b/ignite/pkg/cosmosgen/generate_openapi.go index ebc029f6c3..f614614cf3 100644 --- a/ignite/pkg/cosmosgen/generate_openapi.go +++ b/ignite/pkg/cosmosgen/generate_openapi.go @@ -46,14 +46,14 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { // gen generates a spec for a module where it's source code resides at src. // and adds needed swaggercombine configure for it. - gen := func(src string, m module.Module) (err error) { + gen := func(protoPath string) (err error) { dir, err := os.MkdirTemp("", "gen-openapi-module-spec") if err != nil { return err } - checksumPaths := append([]string{m.Pkg.Path}, g.opts.includeDirs...) - checksum, err := dirchange.ChecksumFromPaths(src, checksumPaths...) + checksumPaths := append([]string{protoPath}, g.opts.includeDirs...) + checksum, err := dirchange.ChecksumFromPaths(protoPath, checksumPaths...) if err != nil { return err } @@ -72,7 +72,7 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { } hasAnySpecChanged = true - err = g.buf.Generate(ctx, m.Pkg.Path, dir, g.openAPITemplate(), "module.proto") + err = g.buf.Generate(ctx, protoPath, dir, g.openAPITemplate(), "module.proto") if err != nil { return err } @@ -99,26 +99,13 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { return nil } - // generate specs for each module and persist them in the file system - // after add their path and config to swaggercombine.Config so we can combine them - // into a single spec. - - add := func(src string, modules []module.Module) error { - for _, m := range modules { - if err := gen(src, m); err != nil { - return err - } - } - return nil - } - // protoc openapi generator acts weird on concurrent run, so do not use goroutines here. - if err := add(g.appPath, g.appModules); err != nil { + if err := gen(g.protoDir); err != nil { return err } for src, modules := range g.thirdModules { - if err := add(src, modules); err != nil { + if err := gen(src, modules); err != nil { return err } } From 0389eb2d0bf99e621aad3b0cd0e545bb7390d47f Mon Sep 17 00:00:00 2001 From: Pantani Date: Thu, 9 May 2024 02:59:44 +0200 Subject: [PATCH 02/20] use --path flag mutilple times --- ignite/pkg/cosmosbuf/buf.go | 49 ++++++++++-------------- ignite/pkg/cosmosbuf/buf_test.go | 46 ---------------------- ignite/pkg/cosmosgen/generate_openapi.go | 25 +++++++++--- ignite/pkg/xos/files_test.go | 10 ++++- 4 files changed, 48 insertions(+), 82 deletions(-) delete mode 100644 ignite/pkg/cosmosbuf/buf_test.go diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index 1513ab3bfc..301a53c8d5 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -3,13 +3,12 @@ package cosmosbuf import ( "context" "fmt" - "strconv" + "path/filepath" "strings" "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cmdrunner/exec" "github.com/ignite/cli/v29/ignite/pkg/errors" - "github.com/ignite/cli/v29/ignite/pkg/protoanalysis" "github.com/ignite/cli/v29/ignite/pkg/xexec" "github.com/ignite/cli/v29/ignite/pkg/xos" ) @@ -21,7 +20,6 @@ type ( // Buf represents the buf application structure. Buf struct { path string - protoCache *protoanalysis.Cache storageCache cache.Cache[[]byte] } ) @@ -32,7 +30,7 @@ const ( flagOutput = "output" flagErrorFormat = "error-format" flagLogFormat = "log-format" - flagExcludePath = "exclude-path" + flagPath = "path" flagOnly = "only" fmtJSON = "json" @@ -68,7 +66,6 @@ func New(cacheStorage cache.Storage) (Buf, error) { return Buf{ path: path, storageCache: cache.New[[]byte](cacheStorage, specCacheNamespace), - protoCache: protoanalysis.NewCache(), }, nil } @@ -87,7 +84,7 @@ func (b Buf) Update(ctx context.Context, modDir string, dependencies ...string) } } - cmd, err := b.generateCommand(CMDMod, flags, "update", modDir) + cmd, err := b.command(CMDMod, flags, "update", modDir) if err != nil { return err } @@ -108,7 +105,7 @@ func (b Buf) Export(ctx context.Context, protoDir, output string) error { flagOutput: output, } - cmd, err := b.generateCommand(CMDExport, flags, protoDir) + cmd, err := b.command(CMDExport, flags, protoDir) if err != nil { return err } @@ -122,26 +119,37 @@ func (b Buf) Generate( protoDir, output, template string, - excluded ...string, + excludedFile ...string, ) (err error) { protoFiles, err := xos.FindFiles(protoDir, xos.ProtoFile) if err != nil || len(protoFiles) == 0 { return err } + excluded := make(map[string]struct{}) + for _, file := range excludedFile { + excluded[file] = struct{}{} + } + flags := map[string]string{ flagTemplate: template, flagOutput: output, flagErrorFormat: fmtJSON, flagLogFormat: fmtJSON, - flagExcludePath: join(excluded...), } - cmd, err := b.generateCommand(CMDGenerate, flags, protoDir) + cmd, err := b.command(CMDGenerate, flags, protoDir) if err != nil { return err } + for _, path := range protoFiles { + if _, ok := excluded[filepath.Base(path)]; ok { + continue + } + cmd = append(cmd, fmt.Sprintf("--%s", flagPath), path) + } + return b.runCommand(ctx, cmd...) } @@ -153,8 +161,8 @@ func (b Buf) runCommand(ctx context.Context, cmd ...string) error { return exec.Exec(ctx, cmd, execOpts...) } -// generateCommand generate the buf CLI command. -func (b Buf) generateCommand( +// command generate the buf CLI command. +func (b Buf) command( c Command, flags map[string]string, args ...string, @@ -176,20 +184,3 @@ func (b Buf) generateCommand( } return command, nil } - -func join(elems ...string) (value string) { - switch len(elems) { - case 0: - return "" - case 1: - return strconv.Quote(elems[0]) - } - - var b strings.Builder - b.WriteString(strconv.Quote(elems[0])) - for _, s := range elems[1:] { - b.WriteString(",") - b.WriteString(strconv.Quote(s)) - } - return b.String() -} diff --git a/ignite/pkg/cosmosbuf/buf_test.go b/ignite/pkg/cosmosbuf/buf_test.go deleted file mode 100644 index 9039c92214..0000000000 --- a/ignite/pkg/cosmosbuf/buf_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package cosmosbuf - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func Test_join(t *testing.T) { - tests := []struct { - name string - values []string - want string - }{ - { - name: "empty", - values: []string{}, - }, - { - name: "single", - values: []string{"foo"}, - want: `"foo"`, - }, - { - name: "two", - values: []string{"foo", "bar"}, - want: `"foo","bar"`, - }, - { - name: "three", - values: []string{"foo", "bar", "baz"}, - want: `"foo","bar","baz"`, - }, - { - name: "repeated", - values: []string{"foo", "bar", "baz", "foo", "bar", "baz"}, - want: `"foo","bar","baz","foo","bar","baz"`, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := join(tt.values...) - require.Equal(t, tt.want, got) - }) - } -} diff --git a/ignite/pkg/cosmosgen/generate_openapi.go b/ignite/pkg/cosmosgen/generate_openapi.go index f614614cf3..ebc029f6c3 100644 --- a/ignite/pkg/cosmosgen/generate_openapi.go +++ b/ignite/pkg/cosmosgen/generate_openapi.go @@ -46,14 +46,14 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { // gen generates a spec for a module where it's source code resides at src. // and adds needed swaggercombine configure for it. - gen := func(protoPath string) (err error) { + gen := func(src string, m module.Module) (err error) { dir, err := os.MkdirTemp("", "gen-openapi-module-spec") if err != nil { return err } - checksumPaths := append([]string{protoPath}, g.opts.includeDirs...) - checksum, err := dirchange.ChecksumFromPaths(protoPath, checksumPaths...) + checksumPaths := append([]string{m.Pkg.Path}, g.opts.includeDirs...) + checksum, err := dirchange.ChecksumFromPaths(src, checksumPaths...) if err != nil { return err } @@ -72,7 +72,7 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { } hasAnySpecChanged = true - err = g.buf.Generate(ctx, protoPath, dir, g.openAPITemplate(), "module.proto") + err = g.buf.Generate(ctx, m.Pkg.Path, dir, g.openAPITemplate(), "module.proto") if err != nil { return err } @@ -99,13 +99,26 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { return nil } + // generate specs for each module and persist them in the file system + // after add their path and config to swaggercombine.Config so we can combine them + // into a single spec. + + add := func(src string, modules []module.Module) error { + for _, m := range modules { + if err := gen(src, m); err != nil { + return err + } + } + return nil + } + // protoc openapi generator acts weird on concurrent run, so do not use goroutines here. - if err := gen(g.protoDir); err != nil { + if err := add(g.appPath, g.appModules); err != nil { return err } for src, modules := range g.thirdModules { - if err := gen(src, modules); err != nil { + if err := add(src, modules); err != nil { return err } } diff --git a/ignite/pkg/xos/files_test.go b/ignite/pkg/xos/files_test.go index dec88c6fc1..daf45a7269 100644 --- a/ignite/pkg/xos/files_test.go +++ b/ignite/pkg/xos/files_test.go @@ -26,6 +26,13 @@ func TestFindFiles(t *testing.T) { want: []string{"file1.json", "file3.json", "file4.json"}, err: nil, }, + { + name: "test 3 json files with subfolder", + files: []string{"testdata/file1.json", "file2.txt", "foo/file3.json", "file4.json"}, + extension: "json", + want: []string{"testdata/file1.json", "foo/file3.json", "file4.json"}, + err: nil, + }, { name: "test 1 txt files", files: []string{"file1.json", "file2.txt", "file3.json", "file4.json"}, @@ -59,6 +66,7 @@ func TestFindFiles(t *testing.T) { for _, filename := range tt.files { filePath := filepath.Join(tempDir, filename) + require.NoError(t, os.MkdirAll(filepath.Dir(filePath), 0o755)) file, err := os.Create(filePath) require.NoError(t, err) require.NoError(t, file.Close()) @@ -76,7 +84,7 @@ func TestFindFiles(t *testing.T) { for i, filename := range tt.want { want[i] = filepath.Join(tempDir, filename) } - require.EqualValues(t, want, gotFiles) + require.ElementsMatch(t, want, gotFiles) }) } } From b86760eec5cf1e87589a800deabe5369e93139d2 Mon Sep 17 00:00:00 2001 From: Pantani Date: Thu, 9 May 2024 14:43:30 +0200 Subject: [PATCH 03/20] remove `Build.Proto.ThirdPartyPaths` from the config --- ignite/config/chain/base/config.go | 7 +-- ignite/pkg/cosmosgen/cosmosgen.go | 13 ++--- ignite/pkg/cosmosgen/generate.go | 7 +-- ignite/pkg/cosmosgen/generate_openapi.go | 38 +++++++-------- ignite/pkg/cosmosgen/generate_openapi_test.go | 47 +++++++++++++++++++ ignite/pkg/cosmosgen/generate_typescript.go | 2 +- ignite/services/chain/generate.go | 5 +- ignite/services/scaffolder/scaffolder.go | 1 - 8 files changed, 74 insertions(+), 46 deletions(-) create mode 100644 ignite/pkg/cosmosgen/generate_openapi_test.go diff --git a/ignite/config/chain/base/config.go b/ignite/config/chain/base/config.go index bf8d4ce4cd..544cedc1a2 100644 --- a/ignite/config/chain/base/config.go +++ b/ignite/config/chain/base/config.go @@ -31,10 +31,6 @@ type Build struct { type Proto struct { // Path is the relative path of where app's proto files are located at. Path string `yaml:"path"` - - // ThirdPartyPath is the relative path of where the third party proto files are - // located that used by the app. - ThirdPartyPaths []string `yaml:"third_party_paths"` } // Client configures code generation for clients. @@ -187,8 +183,7 @@ func DefaultConfig() Config { return Config{ Build: Build{ Proto: Proto{ - Path: defaults.ProtoDir, - ThirdPartyPaths: []string{"third_party/proto", "proto_vendor"}, + Path: defaults.ProtoDir, }, }, Faucet: Faucet{ diff --git a/ignite/pkg/cosmosgen/cosmosgen.go b/ignite/pkg/cosmosgen/cosmosgen.go index 520f4ab793..d6910c411a 100644 --- a/ignite/pkg/cosmosgen/cosmosgen.go +++ b/ignite/pkg/cosmosgen/cosmosgen.go @@ -17,7 +17,6 @@ import ( // generateOptions used to configure code generation. type generateOptions struct { - includeDirs []string useCache bool updateBufModule bool ev events.Bus @@ -80,14 +79,6 @@ func WithOpenAPIGeneration(out string) Option { } } -// IncludeDirs configures the third party proto dirs that used by app's proto. -// relative to the projectPath. -func IncludeDirs(dirs []string) Option { - return func(o *generateOptions) { - o.includeDirs = dirs - } -} - // UpdateBufModule enables Buf config proto dependencies update. // This option updates app's Buf config when proto packages or // Buf modules are found within the Go dependencies. @@ -122,6 +113,10 @@ type generator struct { tmpDirs []string } +func (g *generator) protoPath() string { + return filepath.Join(g.appPath, g.protoDir) +} + func (g *generator) cleanup() { // Remove temporary directories created during generation for _, path := range g.tmpDirs { diff --git a/ignite/pkg/cosmosgen/generate.go b/ignite/pkg/cosmosgen/generate.go index 81e1ec0543..fa4bfc7344 100644 --- a/ignite/pkg/cosmosgen/generate.go +++ b/ignite/pkg/cosmosgen/generate.go @@ -213,12 +213,7 @@ func (g *generator) setup(ctx context.Context) (err error) { } func (g *generator) getProtoIncludeFolders(modPath string) []string { - // Add default protoDir and default includeDirs - includePaths := []string{filepath.Join(modPath, g.protoDir)} - for _, dir := range g.opts.includeDirs { - includePaths = append(includePaths, filepath.Join(modPath, dir)) - } - return includePaths + return []string{filepath.Join(modPath, g.protoDir)} } func (g *generator) findBufPath(modpath string) (string, error) { diff --git a/ignite/pkg/cosmosgen/generate_openapi.go b/ignite/pkg/cosmosgen/generate_openapi.go index ebc029f6c3..00bff26e13 100644 --- a/ignite/pkg/cosmosgen/generate_openapi.go +++ b/ignite/pkg/cosmosgen/generate_openapi.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/iancoleman/strcase" @@ -46,14 +47,14 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { // gen generates a spec for a module where it's source code resides at src. // and adds needed swaggercombine configure for it. - gen := func(src string, m module.Module) (err error) { + gen := func(appPath, protoPath string) (err error) { + name := extractName(appPath) dir, err := os.MkdirTemp("", "gen-openapi-module-spec") if err != nil { return err } - checksumPaths := append([]string{m.Pkg.Path}, g.opts.includeDirs...) - checksum, err := dirchange.ChecksumFromPaths(src, checksumPaths...) + checksum, err := dirchange.ChecksumFromPaths(appPath, protoPath) if err != nil { return err } @@ -68,12 +69,11 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { if err := os.WriteFile(specPath, existingSpec, 0o644); err != nil { return err } - return conf.AddSpec(strcase.ToCamel(m.Pkg.Name), specPath, true) + return conf.AddSpec(name, specPath, true) } hasAnySpecChanged = true - err = g.buf.Generate(ctx, m.Pkg.Path, dir, g.openAPITemplate(), "module.proto") - if err != nil { + if err = g.buf.Generate(ctx, filepath.Join(appPath, g.protoDir), dir, g.openAPITemplate(), "module.proto"); err != nil { return err } @@ -90,7 +90,7 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { if err := specCache.Put(cacheKey, f); err != nil { return err } - if err := conf.AddSpec(strcase.ToCamel(m.Pkg.Name), spec, true); err != nil { + if err := conf.AddSpec(name, spec, true); err != nil { return err } } @@ -103,22 +103,13 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { // after add their path and config to swaggercombine.Config so we can combine them // into a single spec. - add := func(src string, modules []module.Module) error { - for _, m := range modules { - if err := gen(src, m); err != nil { - return err - } - } - return nil - } - // protoc openapi generator acts weird on concurrent run, so do not use goroutines here. - if err := add(g.appPath, g.appModules); err != nil { + if err := gen(g.appPath, g.protoDir); err != nil { return err } - for src, modules := range g.thirdModules { - if err := add(src, modules); err != nil { + for src := range g.thirdModules { + if err := gen(src, ""); err != nil { return err } } @@ -187,3 +178,12 @@ func (g *generator) generateModuleOpenAPISpec(ctx context.Context, m module.Modu // combine specs into one and save to out. return conf.Combine(out) } + +// extractName takes a full path and returns the name. +func extractName(path string) string { + // Extract the last part of the path + lastPart := filepath.Base(path) + // If there is a version suffix (e.g., @v0.50), remove it + name := strings.Split(lastPart, "@")[0] + return strcase.ToCamel(name) +} diff --git a/ignite/pkg/cosmosgen/generate_openapi_test.go b/ignite/pkg/cosmosgen/generate_openapi_test.go new file mode 100644 index 0000000000..4fa30f9dfe --- /dev/null +++ b/ignite/pkg/cosmosgen/generate_openapi_test.go @@ -0,0 +1,47 @@ +package cosmosgen + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_extractName(t *testing.T) { + tests := []struct { + name string + path string + want string + }{ + { + name: "test module path", + path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6", + want: "CosmosSdk", + }, + { + name: "test chain path", + path: "/Users/danilopantani/Desktop/go/src/github.com/ignite/venus", + want: "Venus", + }, + { + name: "test module path without version", + path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway", + want: "GrpcGateway", + }, + { + name: "test module path with broken version", + path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway@v1.$", + want: "GrpcGateway", + }, + { + name: "test module path", + path: "/Users/danilopantani/Desktop/go/pkg/mod/cosmossdk.io/x/evidence@v0.1.0", + want: "Evidence", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractName(tt.path) + require.Equal(t, tt.want, got) + }) + } +} diff --git a/ignite/pkg/cosmosgen/generate_typescript.go b/ignite/pkg/cosmosgen/generate_typescript.go index 66d5f65179..47508c80aa 100644 --- a/ignite/pkg/cosmosgen/generate_typescript.go +++ b/ignite/pkg/cosmosgen/generate_typescript.go @@ -106,7 +106,7 @@ func (g *tsGenerator) generateModuleTemplates(ctx context.Context) error { gg.Go(func() error { cacheKey := m.Pkg.Path - paths := append([]string{m.Pkg.Path, g.g.opts.jsOut(m)}, g.g.opts.includeDirs...) + paths := []string{m.Pkg.Path, g.g.opts.jsOut(m)} // Always generate module templates by default unless cache is enabled, in which // case the module template is generated when one or more files were changed in diff --git a/ignite/services/chain/generate.go b/ignite/services/chain/generate.go index 2def391c22..c36854450b 100644 --- a/ignite/services/chain/generate.go +++ b/ignite/services/chain/generate.go @@ -145,10 +145,7 @@ func (c *Chain) Generate( c.ev.Send("Building proto...", events.ProgressUpdate()) - options := []cosmosgen.Option{ - cosmosgen.CollectEvents(c.ev), - cosmosgen.IncludeDirs(conf.Build.Proto.ThirdPartyPaths), - } + options := []cosmosgen.Option{cosmosgen.CollectEvents(c.ev)} if targetOptions.isGoEnabled { options = append(options, cosmosgen.WithGoGeneration()) diff --git a/ignite/services/scaffolder/scaffolder.go b/ignite/services/scaffolder/scaffolder.go index f50e55a2b1..71dfeb0ef5 100644 --- a/ignite/services/scaffolder/scaffolder.go +++ b/ignite/services/scaffolder/scaffolder.go @@ -124,7 +124,6 @@ func protoc(ctx context.Context, cacheStorage cache.Storage, projectPath, protoD options := []cosmosgen.Option{ cosmosgen.UpdateBufModule(), cosmosgen.WithGoGeneration(), - cosmosgen.IncludeDirs(conf.Build.Proto.ThirdPartyPaths), } // Generate Typescript client code if it's enabled From 11c162edcd7736321e80dec301a0f235d2f05d34 Mon Sep 17 00:00:00 2001 From: Pantani Date: Sat, 11 May 2024 18:18:32 +0200 Subject: [PATCH 04/20] add logic for run buf by folder or by file --- ignite/config/chain/v1/config_test.go | 3 +- ignite/pkg/cosmosbuf/buf.go | 128 +++++++++++++++--- ignite/pkg/cosmosbuf/cache.go | 106 +++++++++++++++ ignite/pkg/cosmosgen/cosmosgen.go | 4 - ignite/pkg/cosmosgen/generate_go.go | 21 +-- ignite/pkg/cosmosgen/generate_openapi.go | 68 +++++++--- ignite/pkg/cosmosgen/generate_openapi_test.go | 31 +++-- ignite/pkg/xos/files.go | 16 ++- ignite/pkg/xos/files_test.go | 2 +- 9 files changed, 316 insertions(+), 63 deletions(-) create mode 100644 ignite/pkg/cosmosbuf/cache.go diff --git a/ignite/config/chain/v1/config_test.go b/ignite/config/chain/v1/config_test.go index cbf9b3058c..b97c65a64c 100644 --- a/ignite/config/chain/v1/config_test.go +++ b/ignite/config/chain/v1/config_test.go @@ -30,8 +30,7 @@ func TestConfigDecode(t *testing.T) { Build: base.Build{ Binary: "evmosd", Proto: base.Proto{ - Path: "proto", - ThirdPartyPaths: []string{"third_party/proto", "proto_vendor"}, + Path: "proto", }, }, Accounts: []base.Account{ diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index 301a53c8d5..3063227423 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -3,11 +3,15 @@ package cosmosbuf import ( "context" "fmt" + "os" "path/filepath" "strings" + "golang.org/x/sync/errgroup" + "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cmdrunner/exec" + "github.com/ignite/cli/v29/ignite/pkg/cosmosver" "github.com/ignite/cli/v29/ignite/pkg/errors" "github.com/ignite/cli/v29/ignite/pkg/xexec" "github.com/ignite/cli/v29/ignite/pkg/xos" @@ -20,8 +24,18 @@ type ( // Buf represents the buf application structure. Buf struct { path string + sdkProtoDir string storageCache cache.Cache[[]byte] } + + // genOptions used to configure code generation. + genOptions struct { + excluded map[string]struct{} + fileByFile bool + } + + // GenOption configures code generation. + GenOption func(*genOptions) ) const ( @@ -56,6 +70,22 @@ var ( ErrProtoFilesNotFound = errors.New("no proto files found") ) +func ExcludeFiles(files ...string) GenOption { + return func(o *genOptions) { + excluded := make(map[string]struct{}) + for _, file := range files { + excluded[file] = struct{}{} + } + o.excluded = excluded + } +} + +func FileByFile() GenOption { + return func(o *genOptions) { + o.fileByFile = true + } +} + // New creates a new Buf based on the installed binary. func New(cacheStorage cache.Storage) (Buf, error) { path, err := xexec.ResolveAbsPath(binaryName) @@ -94,7 +124,7 @@ func (b Buf) Update(ctx context.Context, modDir string, dependencies ...string) // Export runs the buf Export command for the files in the proto directory. func (b Buf) Export(ctx context.Context, protoDir, output string) error { - specs, err := xos.FindFiles(protoDir, xos.ProtoFile) + specs, err := xos.FindFilesExtension(protoDir, xos.ProtoFile) if err != nil { return err } @@ -116,19 +146,46 @@ func (b Buf) Export(ctx context.Context, protoDir, output string) error { // Generate runs the buf Generate command for each file into the proto directory. func (b Buf) Generate( ctx context.Context, - protoDir, + protoPath, output, template string, - excludedFile ...string, + options ...GenOption, ) (err error) { - protoFiles, err := xos.FindFiles(protoDir, xos.ProtoFile) + opts := genOptions{} + for _, apply := range options { + apply(&opts) + } + + // TODO(@julienrbrt): this whole custom handling can be deleted + // after https://github.com/cosmos/cosmos-sdk/pull/18993 in v29. + if strings.Contains(protoPath, cosmosver.CosmosSDKRepoName) { + if b.sdkProtoDir == "" { + b.sdkProtoDir, err = copySDKProtoDir(protoPath) + if err != nil { + return err + } + } + protoPath = b.sdkProtoDir + } + + // find all proto files into the path + protoFiles, err := xos.FindFilesExtension(protoPath, xos.ProtoFile) if err != nil || len(protoFiles) == 0 { return err } - excluded := make(map[string]struct{}) - for _, file := range excludedFile { - excluded[file] = struct{}{} + cached, err := b.getDirCache(protoPath, output, template) + if err != nil { + return err + } + + // remove excluded and cached files + for i, file := range protoFiles { + if _, ok := opts.excluded[filepath.Base(file)]; ok { + protoFiles = append(protoFiles[:i], protoFiles[i+1:]...) + } else if _, ok := cached[file]; ok { + protoFiles = append(protoFiles[:i], protoFiles[i+1:]...) + } } flags := map[string]string{ @@ -138,19 +195,36 @@ func (b Buf) Generate( flagLogFormat: fmtJSON, } - cmd, err := b.command(CMDGenerate, flags, protoDir) - if err != nil { - return err - } + if !opts.fileByFile { + cmd, err := b.command(CMDGenerate, flags, protoPath) + if err != nil { + return err + } + for _, file := range protoFiles { + cmd = append(cmd, fmt.Sprintf("--%s=%s", flagPath, file)) + } + if err := b.runCommand(ctx, cmd...); err != nil { + return err + } + } else { + g, ctx := errgroup.WithContext(ctx) + for _, file := range protoFiles { + cmd, err := b.command(CMDGenerate, flags, file) + if err != nil { + return err + } - for _, path := range protoFiles { - if _, ok := excluded[filepath.Base(path)]; ok { - continue + g.Go(func() error { + cmd := cmd + return b.runCommand(ctx, cmd...) + }) + } + if err := g.Wait(); err != nil { + return err } - cmd = append(cmd, fmt.Sprintf("--%s", flagPath), path) } - return b.runCommand(ctx, cmd...) + return b.saveDirCache(output, template) } // runCommand run the buf CLI command. @@ -184,3 +258,25 @@ func (b Buf) command( } return command, nil } + +// findSDKProtoPath finds the Cosmos SDK proto folder path. +func findSDKProtoPath(protoDir string) string { + paths := strings.Split(protoDir, "@") + if len(paths) < 2 { + return protoDir + } + version := strings.Split(paths[1], "/")[0] + return fmt.Sprintf("%s@%s/proto", paths[0], version) +} + +// copySDKProtoDir copies the Cosmos SDK proto folder to a temporary directory. +// The temporary directory must be removed by the caller. +func copySDKProtoDir(protoDir string) (string, error) { + tmpDir, err := os.MkdirTemp("", "proto-sdk") + if err != nil { + return "", err + } + + srcPath := findSDKProtoPath(protoDir) + return tmpDir, xos.CopyFolder(srcPath, tmpDir) +} diff --git a/ignite/pkg/cosmosbuf/cache.go b/ignite/pkg/cosmosbuf/cache.go new file mode 100644 index 0000000000..66ed294def --- /dev/null +++ b/ignite/pkg/cosmosbuf/cache.go @@ -0,0 +1,106 @@ +package cosmosbuf + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + + "github.com/ignite/cli/v29/ignite/pkg/cache" + "github.com/ignite/cli/v29/ignite/pkg/dirchange" + "github.com/ignite/cli/v29/ignite/pkg/errors" + "github.com/ignite/cli/v29/ignite/pkg/xos" +) + +func cacheKey(src, file, template string) (string, error) { + relPath, err := filepath.Rel(src, file) + if err != nil { + return "", err + } + + checksum, err := dirchange.ChecksumFromPaths(src, relPath) + if err != nil { + return "", err + } + + h := sha256.New() + if _, err := h.Write(checksum); err != nil { + return "", err + } + if _, err := h.Write([]byte(template)); err != nil { + return "", err + } + key := fmt.Sprintf("%x", h.Sum(nil)) + return key, nil +} + +func (b Buf) getFileCache(src, file, template, output string) (bool, error) { + key, err := cacheKey(src, file, template) + if err != nil { + return false, err + } + + existingFile, err := b.storageCache.Get(key) + if errors.Is(err, cache.ErrorNotFound) { + return false, nil + } else if err != nil { + return false, err + } + + relPath, err := filepath.Rel(src, file) + if err != nil { + return false, err + } + + filePath := filepath.Join(output, relPath) + if err := os.WriteFile(filePath, existingFile, 0o644); err != nil { + return false, err + } + return true, nil +} + +func (b Buf) getDirCache(src, output, template string) (map[string]struct{}, error) { + result := make(map[string]struct{}) + files, err := xos.FindFiles(src) + if err != nil { + return result, err + } + + for _, file := range files { + ok, err := b.getFileCache(src, file, template, output) + if err != nil { + return result, err + } + if ok { + result[file] = struct{}{} + } + } + return result, nil +} + +func (b Buf) saveFileCache(src, file, template string) error { + key, err := cacheKey(src, file, template) + if err != nil { + return err + } + + f, err := os.ReadFile(file) + if err != nil { + return err + } + return b.storageCache.Put(key, f) +} + +func (b Buf) saveDirCache(src, template string) error { + files, err := xos.FindFiles(src) + if err != nil { + return err + } + + for _, file := range files { + if err := b.saveFileCache(src, file, template); err != nil { + return err + } + } + return nil +} diff --git a/ignite/pkg/cosmosgen/cosmosgen.go b/ignite/pkg/cosmosgen/cosmosgen.go index d6910c411a..ae14ab952c 100644 --- a/ignite/pkg/cosmosgen/cosmosgen.go +++ b/ignite/pkg/cosmosgen/cosmosgen.go @@ -113,10 +113,6 @@ type generator struct { tmpDirs []string } -func (g *generator) protoPath() string { - return filepath.Join(g.appPath, g.protoDir) -} - func (g *generator) cleanup() { // Remove temporary directories created during generation for _, path := range g.tmpDirs { diff --git a/ignite/pkg/cosmosgen/generate_go.go b/ignite/pkg/cosmosgen/generate_go.go index 56673b691c..8c10facf98 100644 --- a/ignite/pkg/cosmosgen/generate_go.go +++ b/ignite/pkg/cosmosgen/generate_go.go @@ -7,6 +7,7 @@ import ( "github.com/otiai10/copy" + "github.com/ignite/cli/v29/ignite/pkg/cosmosbuf" "github.com/ignite/cli/v29/ignite/pkg/errors" ) @@ -18,15 +19,19 @@ func (g *generator) pulsarTemplate() string { return filepath.Join(g.appPath, g.protoDir, "buf.gen.pulsar.yaml") } +func (g *generator) protoPath() string { + return filepath.Join(g.appPath, g.protoDir) +} + func (g *generator) generateGoGo(ctx context.Context) error { - return g.generate(ctx, g.gogoTemplate(), "module.proto") + return g.generate(ctx, g.gogoTemplate(), g.gomodPath, "module.proto") } func (g *generator) generatePulsar(ctx context.Context) error { - return g.generate(ctx, g.pulsarTemplate()) + return g.generate(ctx, g.pulsarTemplate(), "") } -func (g *generator) generate(ctx context.Context, template string, excluded ...string) error { +func (g *generator) generate(ctx context.Context, template, fromPath string, excluded ...string) error { // create a temporary dir to locate generated code under which later only some of them will be moved to the // app's source code. this also prevents having leftover files in the app's source code or its parent dir - when // command executed directly there - in case of an interrupt. @@ -36,17 +41,15 @@ func (g *generator) generate(ctx context.Context, template string, excluded ...s } defer os.RemoveAll(tmp) - protoPath := filepath.Join(g.appPath, g.protoDir) - // code generate for each module. - if err = g.buf.Generate(ctx, protoPath, tmp, template, excluded...); err != nil { + if err := g.buf.Generate(ctx, g.protoPath(), tmp, template, cosmosbuf.ExcludeFiles(excluded...)); err != nil { return err } // move generated code for the app under the relative locations in its source code. - _, err = os.Stat(tmp) - if err == nil { - err = copy.Copy(tmp, g.appPath) + path := filepath.Join(tmp, fromPath) + if _, err := os.Stat(path); err == nil { + err = copy.Copy(path, g.appPath) if err != nil { return errors.Wrap(err, "cannot copy path") } diff --git a/ignite/pkg/cosmosgen/generate_openapi.go b/ignite/pkg/cosmosgen/generate_openapi.go index 00bff26e13..f58b9b0486 100644 --- a/ignite/pkg/cosmosgen/generate_openapi.go +++ b/ignite/pkg/cosmosgen/generate_openapi.go @@ -7,10 +7,12 @@ import ( "path/filepath" "strings" + "github.com/blang/semver/v4" "github.com/iancoleman/strcase" "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cosmosanalysis/module" + "github.com/ignite/cli/v29/ignite/pkg/cosmosbuf" "github.com/ignite/cli/v29/ignite/pkg/dirchange" "github.com/ignite/cli/v29/ignite/pkg/errors" swaggercombine "github.com/ignite/cli/v29/ignite/pkg/swagger-combine" @@ -47,14 +49,16 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { // gen generates a spec for a module where it's source code resides at src. // and adds needed swaggercombine configure for it. - gen := func(appPath, protoPath string) (err error) { - name := extractName(appPath) + gen := func(appPath, protoDir, name string) error { + name = strcase.ToCamel(name) + protoPath := filepath.Join(appPath, protoDir) + dir, err := os.MkdirTemp("", "gen-openapi-module-spec") if err != nil { return err } - checksum, err := dirchange.ChecksumFromPaths(appPath, protoPath) + checksum, err := dirchange.ChecksumFromPaths(appPath, protoDir) if err != nil { return err } @@ -73,11 +77,18 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { } hasAnySpecChanged = true - if err = g.buf.Generate(ctx, filepath.Join(appPath, g.protoDir), dir, g.openAPITemplate(), "module.proto"); err != nil { + if err = g.buf.Generate( + ctx, + protoPath, + dir, + g.openAPITemplate(), + cosmosbuf.ExcludeFiles("module.proto"), + cosmosbuf.FileByFile(), + ); err != nil { return err } - specs, err := xos.FindFiles(dir, xos.JSONFile) + specs, err := xos.FindFilesExtension(dir, xos.JSONFile) if err != nil { return err } @@ -104,12 +115,26 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { // into a single spec. // protoc openapi generator acts weird on concurrent run, so do not use goroutines here. - if err := gen(g.appPath, g.protoDir); err != nil { + if err := gen(g.appPath, g.protoDir, g.gomodPath); err != nil { return err } - for src := range g.thirdModules { - if err := gen(src, ""); err != nil { + doneMods := make(map[string]struct{}) + for _, modules := range g.thirdModules { + if len(modules) == 0 { + continue + } + var ( + m = modules[0] + path = extractRootModulePath(m.Pkg.Path) + ) + + if _, ok := doneMods[path]; ok { + continue + } + doneMods[path] = struct{}{} + + if err := gen(path, "", m.Name); err != nil { return err } } @@ -158,12 +183,12 @@ func (g *generator) generateModuleOpenAPISpec(ctx context.Context, m module.Modu return err } - err = g.buf.Generate(ctx, m.Pkg.Path, dir, g.openAPITemplateForSTA(), "module.proto") + err = g.buf.Generate(ctx, m.Pkg.Path, dir, g.openAPITemplateForSTA(), cosmosbuf.ExcludeFiles("module.proto")) if err != nil { return err } - specs, err := xos.FindFiles(dir, xos.JSONFile) + specs, err := xos.FindFilesExtension(dir, xos.JSONFile) if err != nil { return err } @@ -179,11 +204,20 @@ func (g *generator) generateModuleOpenAPISpec(ctx context.Context, m module.Modu return conf.Combine(out) } -// extractName takes a full path and returns the name. -func extractName(path string) string { - // Extract the last part of the path - lastPart := filepath.Base(path) - // If there is a version suffix (e.g., @v0.50), remove it - name := strings.Split(lastPart, "@")[0] - return strcase.ToCamel(name) +func extractRootModulePath(fullPath string) string { + var ( + segments = strings.Split(fullPath, "/") + modulePath = "/" + ) + + for _, segment := range segments { + modulePath = filepath.Join(modulePath, segment) + segmentName := strings.Split(segment, "@") + if len(segmentName) > 1 { + if _, err := semver.ParseTolerant(segmentName[1]); err == nil { + return modulePath + } + } + } + return fullPath } diff --git a/ignite/pkg/cosmosgen/generate_openapi_test.go b/ignite/pkg/cosmosgen/generate_openapi_test.go index 4fa30f9dfe..bc4cefb775 100644 --- a/ignite/pkg/cosmosgen/generate_openapi_test.go +++ b/ignite/pkg/cosmosgen/generate_openapi_test.go @@ -6,41 +6,46 @@ import ( "github.com/stretchr/testify/require" ) -func Test_extractName(t *testing.T) { +func Test_extractRootModulePath(t *testing.T) { tests := []struct { name string path string want string }{ { - name: "test module path", - path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6", - want: "CosmosSdk", + name: "test cosmos-sdk path", + path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6/proto/cosmos/distribution/v1beta1", + want: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6", + }, + { + name: "test ibc path", + path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/cosmos/ibc-go/v8@v8.2.0/proto/ibc/applications/interchain_accounts/controller/v1", + want: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/cosmos/ibc-go/v8@v8.2.0", }, { name: "test chain path", path: "/Users/danilopantani/Desktop/go/src/github.com/ignite/venus", - want: "Venus", + want: "/Users/danilopantani/Desktop/go/src/github.com/ignite/venus", }, { name: "test module path without version", - path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway", - want: "GrpcGateway", + path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/proto/applications", + want: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/proto/applications", }, { name: "test module path with broken version", - path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway@v1.$", - want: "GrpcGateway", + path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway@v1.$/controller", + want: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway@v1.$/controller", }, { - name: "test module path", - path: "/Users/danilopantani/Desktop/go/pkg/mod/cosmossdk.io/x/evidence@v0.1.0", - want: "Evidence", + name: "test module path with v2 version", + path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/v2@v2.19.1/proto/files", + want: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/v2@v2.19.1", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := extractName(tt.path) + got := extractRootModulePath(tt.path) require.Equal(t, tt.want, got) }) } diff --git a/ignite/pkg/xos/files.go b/ignite/pkg/xos/files.go index 60eb6e4a0e..0ef8ea661c 100644 --- a/ignite/pkg/xos/files.go +++ b/ignite/pkg/xos/files.go @@ -11,7 +11,21 @@ const ( ProtoFile = "proto" ) -func FindFiles(directory, extension string) ([]string, error) { +func FindFiles(directory string) ([]string, error) { + files := make([]string, 0) + return files, filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.IsDir() { + files = append(files, path) + } + return nil + }) +} + +func FindFilesExtension(directory, extension string) ([]string, error) { files := make([]string, 0) return files, filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { if err != nil { diff --git a/ignite/pkg/xos/files_test.go b/ignite/pkg/xos/files_test.go index daf45a7269..cbd5a34903 100644 --- a/ignite/pkg/xos/files_test.go +++ b/ignite/pkg/xos/files_test.go @@ -72,7 +72,7 @@ func TestFindFiles(t *testing.T) { require.NoError(t, file.Close()) } - gotFiles, err := xos.FindFiles(tempDir, tt.extension) + gotFiles, err := xos.FindFilesExtension(tempDir, tt.extension) if tt.err != nil { require.Error(t, err) require.ErrorIs(t, err, tt.err) From 46ce07142071234dacee045a4023c787b2180e44 Mon Sep 17 00:00:00 2001 From: Pantani Date: Mon, 13 May 2024 02:22:16 +0200 Subject: [PATCH 05/20] update cosmos-sdk --- ignite/pkg/cosmosbuf/buf.go | 36 ------------------- .../templates/app/files/{{protoDir}}/buf.lock | 6 ++-- 2 files changed, 3 insertions(+), 39 deletions(-) diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index 3063227423..3377c36713 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -3,7 +3,6 @@ package cosmosbuf import ( "context" "fmt" - "os" "path/filepath" "strings" @@ -11,7 +10,6 @@ import ( "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cmdrunner/exec" - "github.com/ignite/cli/v29/ignite/pkg/cosmosver" "github.com/ignite/cli/v29/ignite/pkg/errors" "github.com/ignite/cli/v29/ignite/pkg/xexec" "github.com/ignite/cli/v29/ignite/pkg/xos" @@ -156,18 +154,6 @@ func (b Buf) Generate( apply(&opts) } - // TODO(@julienrbrt): this whole custom handling can be deleted - // after https://github.com/cosmos/cosmos-sdk/pull/18993 in v29. - if strings.Contains(protoPath, cosmosver.CosmosSDKRepoName) { - if b.sdkProtoDir == "" { - b.sdkProtoDir, err = copySDKProtoDir(protoPath) - if err != nil { - return err - } - } - protoPath = b.sdkProtoDir - } - // find all proto files into the path protoFiles, err := xos.FindFilesExtension(protoPath, xos.ProtoFile) if err != nil || len(protoFiles) == 0 { @@ -258,25 +244,3 @@ func (b Buf) command( } return command, nil } - -// findSDKProtoPath finds the Cosmos SDK proto folder path. -func findSDKProtoPath(protoDir string) string { - paths := strings.Split(protoDir, "@") - if len(paths) < 2 { - return protoDir - } - version := strings.Split(paths[1], "/")[0] - return fmt.Sprintf("%s@%s/proto", paths[0], version) -} - -// copySDKProtoDir copies the Cosmos SDK proto folder to a temporary directory. -// The temporary directory must be removed by the caller. -func copySDKProtoDir(protoDir string) (string, error) { - tmpDir, err := os.MkdirTemp("", "proto-sdk") - if err != nil { - return "", err - } - - srcPath := findSDKProtoPath(protoDir) - return tmpDir, xos.CopyFolder(srcPath, tmpDir) -} diff --git a/ignite/templates/app/files/{{protoDir}}/buf.lock b/ignite/templates/app/files/{{protoDir}}/buf.lock index 77287f75ce..f998ad903d 100644 --- a/ignite/templates/app/files/{{protoDir}}/buf.lock +++ b/ignite/templates/app/files/{{protoDir}}/buf.lock @@ -12,11 +12,11 @@ deps: - remote: buf.build owner: cosmos repository: cosmos-sdk - commit: 954f7b05f38440fc8250134b15adec47 + commit: 05419252bcc241ea8023acf1ed4cadc5 - remote: buf.build owner: cosmos repository: gogo-proto - commit: 34d970b699f84aa382f3c29773a60836 + commit: 5e5b9fdd01804356895f8f79a6f1ddc1 - remote: buf.build owner: cosmos repository: ics23 @@ -24,7 +24,7 @@ deps: - remote: buf.build owner: googleapis repository: googleapis - commit: 75b4300737fb4efca0831636be94e517 + commit: cc916c31859748a68fd229a3c8d7a2e8 - remote: buf.build owner: protocolbuffers repository: wellknowntypes From 121fe26d55c11e8aa67e48a5c31a9501d0cab281 Mon Sep 17 00:00:00 2001 From: Pantani Date: Mon, 13 May 2024 13:57:06 +0200 Subject: [PATCH 06/20] fix exclude files for swagger --- ignite/pkg/cosmosbuf/buf.go | 47 ++++++++++++++---------- ignite/pkg/cosmosgen/generate_openapi.go | 13 ++++++- ignite/templates/typed/map/map.go | 4 +- 3 files changed, 41 insertions(+), 23 deletions(-) diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index 3377c36713..ca32832986 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -3,14 +3,15 @@ package cosmosbuf import ( "context" "fmt" - "path/filepath" "strings" + "github.com/gobwas/glob" "golang.org/x/sync/errgroup" "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cmdrunner/exec" "github.com/ignite/cli/v29/ignite/pkg/errors" + "github.com/ignite/cli/v29/ignite/pkg/protoanalysis" "github.com/ignite/cli/v29/ignite/pkg/xexec" "github.com/ignite/cli/v29/ignite/pkg/xos" ) @@ -21,14 +22,14 @@ type ( // Buf represents the buf application structure. Buf struct { - path string - sdkProtoDir string - storageCache cache.Cache[[]byte] + path string + storageCache cache.Cache[[]byte] + analysisCache *protoanalysis.Cache } // genOptions used to configure code generation. genOptions struct { - excluded map[string]struct{} + excluded []glob.Glob fileByFile bool } @@ -68,16 +69,16 @@ var ( ErrProtoFilesNotFound = errors.New("no proto files found") ) -func ExcludeFiles(files ...string) GenOption { +// ExcludeFiles exclude file names from the generate command using glob. +func ExcludeFiles(patterns ...string) GenOption { return func(o *genOptions) { - excluded := make(map[string]struct{}) - for _, file := range files { - excluded[file] = struct{}{} + for _, pattern := range patterns { + o.excluded = append(o.excluded, glob.MustCompile(pattern)) } - o.excluded = excluded } } +// FileByFile runs the generate command for each proto file. func FileByFile() GenOption { return func(o *genOptions) { o.fileByFile = true @@ -92,8 +93,9 @@ func New(cacheStorage cache.Storage) (Buf, error) { } return Buf{ - path: path, - storageCache: cache.New[[]byte](cacheStorage, specCacheNamespace), + path: path, + storageCache: cache.New[[]byte](cacheStorage, specCacheNamespace), + analysisCache: protoanalysis.NewCache(), }, nil } @@ -155,8 +157,8 @@ func (b Buf) Generate( } // find all proto files into the path - protoFiles, err := xos.FindFilesExtension(protoPath, xos.ProtoFile) - if err != nil || len(protoFiles) == 0 { + found, err := xos.FindFilesExtension(protoPath, xos.ProtoFile) + if err != nil || len(found) == 0 { return err } @@ -166,11 +168,18 @@ func (b Buf) Generate( } // remove excluded and cached files - for i, file := range protoFiles { - if _, ok := opts.excluded[filepath.Base(file)]; ok { - protoFiles = append(protoFiles[:i], protoFiles[i+1:]...) - } else if _, ok := cached[file]; ok { - protoFiles = append(protoFiles[:i], protoFiles[i+1:]...) + protoFiles := make([]string, 0) + for _, file := range found { + okExclude := false + for _, g := range opts.excluded { + if g.Match(file) { + okExclude = true + break + } + } + _, okCached := cached[file] + if !okExclude && !okCached { + protoFiles = append(protoFiles, file) } } diff --git a/ignite/pkg/cosmosgen/generate_openapi.go b/ignite/pkg/cosmosgen/generate_openapi.go index f58b9b0486..1ac0a11b64 100644 --- a/ignite/pkg/cosmosgen/generate_openapi.go +++ b/ignite/pkg/cosmosgen/generate_openapi.go @@ -82,10 +82,19 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { protoPath, dir, g.openAPITemplate(), - cosmosbuf.ExcludeFiles("module.proto"), + cosmosbuf.ExcludeFiles( + "*/module.proto", + "*/testutil/*", + "*/testdata/*", + "*/cosmos/orm/*", + "*/cosmos/reflection/*", + "*/cosmos/app/v1alpha1/*", + "*/cosmos/tx/config/v1/config.proto", + "*/cosmos/msg/textual/v1/textual.proto", + ), cosmosbuf.FileByFile(), ); err != nil { - return err + return errors.Wrapf(err, "failed to generate openapi spec %s, probally you need to exclude some proto files", protoPath) } specs, err := xos.FindFilesExtension(dir, xos.JSONFile) diff --git a/ignite/templates/typed/map/map.go b/ignite/templates/typed/map/map.go index cf50de9389..a0b2a72a84 100644 --- a/ignite/templates/typed/map/map.go +++ b/ignite/templates/typed/map/map.go @@ -117,7 +117,7 @@ func NewGenerator(replacer placeholder.Replacer, opts *typed.Options) (*genny.Ge return g, typed.Box(componentTemplate, opts, g) } -// keeperModify modifies the keeper to add a new collections map type +// keeperModify modifies the keeper to add a new collections map type. func keeperModify(replacer placeholder.Replacer, opts *typed.Options) genny.RunFn { return func(r *genny.Runner) error { path := filepath.Join(opts.AppPath, "x", opts.ModuleName, "keeper/keeper.go") @@ -729,7 +729,7 @@ func typesCodecModify(replacer placeholder.Replacer, opts *typed.Options) genny. } } -// TODO(@julienrbrt): extend support of dataTypeToCollectionKeyValue +// TODO(@julienrbrt): extend support of dataTypeToCollectionKeyValue. func dataTypeToCollectionKeyValue(f field.Field) string { var collectionKeyValue string switch f.DataType() { From 8a792a865621ffe3e8f65dbe9f58ff8accde4fee Mon Sep 17 00:00:00 2001 From: Pantani Date: Mon, 13 May 2024 19:17:42 +0200 Subject: [PATCH 07/20] fix buf cache --- ignite/internal/analytics/analytics.go | 9 +-- ignite/pkg/cosmosbuf/buf.go | 48 +++++++++------ ignite/pkg/cosmosbuf/cache.go | 84 +++++++------------------- 3 files changed, 57 insertions(+), 84 deletions(-) diff --git a/ignite/internal/analytics/analytics.go b/ignite/internal/analytics/analytics.go index 8ad12669b5..08f85b9302 100644 --- a/ignite/internal/analytics/analytics.go +++ b/ignite/internal/analytics/analytics.go @@ -12,6 +12,7 @@ import ( "github.com/manifoldco/promptui" "github.com/spf13/cobra" + "github.com/ignite/cli/v29/ignite/config" "github.com/ignite/cli/v29/ignite/pkg/gacli" "github.com/ignite/cli/v29/ignite/pkg/gitpod" "github.com/ignite/cli/v29/ignite/pkg/randstr" @@ -23,7 +24,6 @@ const ( envDoNotTrack = "DO_NOT_TRACK" envCI = "CI" envGitHubActions = "GITHUB_ACTIONS" - igniteDir = ".ignite" igniteAnonIdentity = "anon_identity.json" ) @@ -79,14 +79,15 @@ func checkDNT() (anonIdentity, error) { return anonIdentity{DoNotTrack: true}, nil } - home, err := os.UserHomeDir() + globalPath, err := config.DirPath() if err != nil { return anonIdentity{}, err } - if err := os.Mkdir(filepath.Join(home, igniteDir), 0o700); err != nil && !os.IsExist(err) { + if err := os.Mkdir(globalPath, 0o700); err != nil && !os.IsExist(err) { return anonIdentity{}, err } - identityPath := filepath.Join(home, igniteDir, igniteAnonIdentity) + + identityPath := filepath.Join(globalPath, igniteAnonIdentity) data, err := os.ReadFile(identityPath) if err != nil && !os.IsNotExist(err) { return anonIdentity{}, err diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index ca32832986..b287565e4c 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -3,15 +3,17 @@ package cosmosbuf import ( "context" "fmt" + "os" + "path/filepath" "strings" "github.com/gobwas/glob" "golang.org/x/sync/errgroup" + "github.com/ignite/cli/v29/ignite/config" "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cmdrunner/exec" "github.com/ignite/cli/v29/ignite/pkg/errors" - "github.com/ignite/cli/v29/ignite/pkg/protoanalysis" "github.com/ignite/cli/v29/ignite/pkg/xexec" "github.com/ignite/cli/v29/ignite/pkg/xos" ) @@ -22,9 +24,9 @@ type ( // Buf represents the buf application structure. Buf struct { - path string - storageCache cache.Cache[[]byte] - analysisCache *protoanalysis.Cache + path string + bufCachePath string + storageCache cache.Cache[string] } // genOptions used to configure code generation. @@ -92,10 +94,19 @@ func New(cacheStorage cache.Storage) (Buf, error) { return Buf{}, err } + globalPath, err := config.DirPath() + if err != nil { + return Buf{}, err + } + bufCachePath := filepath.Join(globalPath, "buf") + if err := os.Mkdir(bufCachePath, 0o700); err != nil && !os.IsExist(err) { + return Buf{}, err + } + return Buf{ - path: path, - storageCache: cache.New[[]byte](cacheStorage, specCacheNamespace), - analysisCache: protoanalysis.NewCache(), + path: path, + bufCachePath: bufCachePath, + storageCache: cache.New[string](cacheStorage, specCacheNamespace), }, nil } @@ -156,20 +167,21 @@ func (b Buf) Generate( apply(&opts) } - // find all proto files into the path - found, err := xos.FindFilesExtension(protoPath, xos.ProtoFile) - if err != nil || len(found) == 0 { + // find all proto files into the path. + foundFiles, err := xos.FindFilesExtension(protoPath, xos.ProtoFile) + if err != nil || len(foundFiles) == 0 { return err } - cached, err := b.getDirCache(protoPath, output, template) - if err != nil { + // check if already exist a cache for the template. + key, found, err := b.getCache(protoPath, template, output) + if err != nil || found { return err } - // remove excluded and cached files + // remove excluded and cached files. protoFiles := make([]string, 0) - for _, file := range found { + for _, file := range foundFiles { okExclude := false for _, g := range opts.excluded { if g.Match(file) { @@ -177,11 +189,13 @@ func (b Buf) Generate( break } } - _, okCached := cached[file] - if !okExclude && !okCached { + if !okExclude { protoFiles = append(protoFiles, file) } } + if len(protoFiles) == 0 { + return nil + } flags := map[string]string{ flagTemplate: template, @@ -219,7 +233,7 @@ func (b Buf) Generate( } } - return b.saveDirCache(output, template) + return b.saveCache(key, output) } // runCommand run the buf CLI command. diff --git a/ignite/pkg/cosmosbuf/cache.go b/ignite/pkg/cosmosbuf/cache.go index 66ed294def..9354d298ca 100644 --- a/ignite/pkg/cosmosbuf/cache.go +++ b/ignite/pkg/cosmosbuf/cache.go @@ -6,19 +6,15 @@ import ( "os" "path/filepath" + "github.com/otiai10/copy" + "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/dirchange" "github.com/ignite/cli/v29/ignite/pkg/errors" - "github.com/ignite/cli/v29/ignite/pkg/xos" ) -func cacheKey(src, file, template string) (string, error) { - relPath, err := filepath.Rel(src, file) - if err != nil { - return "", err - } - - checksum, err := dirchange.ChecksumFromPaths(src, relPath) +func cacheKey(src, template string) (string, error) { + checksum, err := dirchange.ChecksumFromPaths(src, "") if err != nil { return "", err } @@ -34,73 +30,35 @@ func cacheKey(src, file, template string) (string, error) { return key, nil } -func (b Buf) getFileCache(src, file, template, output string) (bool, error) { - key, err := cacheKey(src, file, template) +func (b Buf) getCache(src, template, output string) (string, bool, error) { + key, err := cacheKey(src, template) if err != nil { - return false, err + return key, false, err } - existingFile, err := b.storageCache.Get(key) + cachedPath, err := b.storageCache.Get(key) if errors.Is(err, cache.ErrorNotFound) { - return false, nil + return key, false, nil } else if err != nil { - return false, err + return key, false, err } - relPath, err := filepath.Rel(src, file) - if err != nil { - return false, err + if err := copy.Copy(cachedPath, output); err != nil { + return "", false, errors.Wrapf(err, "buf get cache cannot copy path %s to %s", cachedPath, output) } - - filePath := filepath.Join(output, relPath) - if err := os.WriteFile(filePath, existingFile, 0o644); err != nil { - return false, err - } - return true, nil + return key, true, nil } -func (b Buf) getDirCache(src, output, template string) (map[string]struct{}, error) { - result := make(map[string]struct{}) - files, err := xos.FindFiles(src) - if err != nil { - return result, err - } - - for _, file := range files { - ok, err := b.getFileCache(src, file, template, output) - if err != nil { - return result, err - } - if ok { - result[file] = struct{}{} - } - } - return result, nil -} - -func (b Buf) saveFileCache(src, file, template string) error { - key, err := cacheKey(src, file, template) - if err != nil { - return err - } - - f, err := os.ReadFile(file) - if err != nil { - return err - } - return b.storageCache.Put(key, f) -} - -func (b Buf) saveDirCache(src, template string) error { - files, err := xos.FindFiles(src) - if err != nil { +func (b Buf) saveCache(key, src string) error { + cachePath := filepath.Join(b.bufCachePath, key) + if err := os.Mkdir(cachePath, 0o700); os.IsExist(err) { + return nil + } else if err != nil { return err } - for _, file := range files { - if err := b.saveFileCache(src, file, template); err != nil { - return err - } + if err := copy.Copy(src, cachePath); err != nil { + return errors.Wrapf(err, "buf save cache cannot copy path %s to %s", src, cachePath) } - return nil + return b.storageCache.Put(key, cachePath) } From fab3e8e30f54423f88ab54cc394cb7294933ec81 Mon Sep 17 00:00:00 2001 From: Pantani Date: Mon, 13 May 2024 19:26:51 +0200 Subject: [PATCH 08/20] add changelog --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 5e8c430bb1..fa1c6aabeb 100644 --- a/changelog.md +++ b/changelog.md @@ -18,6 +18,7 @@ - [#4110](https://github.com/ignite/cli/pull/4110) Scaffold a consumer chain with `interchain-security` v5.0.0-rc0. - [#4111](https://github.com/ignite/cli/pull/4111) Remove vuex generation - [#4117](https://github.com/ignite/cli/pull/4117), [#4125](https://github.com/ignite/cli/pull/4125) Support relative path when installing local plugins +- [#4133](https://github.com/ignite/cli/pull/4133) Improve buf rate limit ### Changes From 23f30dacc697c2e36c640f40c16c6493bd0f9bb9 Mon Sep 17 00:00:00 2001 From: Pantani Date: Mon, 13 May 2024 19:35:26 +0200 Subject: [PATCH 09/20] update buf deps --- ignite/templates/app/files/{{protoDir}}/buf.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ignite/templates/app/files/{{protoDir}}/buf.lock b/ignite/templates/app/files/{{protoDir}}/buf.lock index f998ad903d..0a6a9564ce 100644 --- a/ignite/templates/app/files/{{protoDir}}/buf.lock +++ b/ignite/templates/app/files/{{protoDir}}/buf.lock @@ -8,7 +8,7 @@ deps: - remote: buf.build owner: cosmos repository: cosmos-proto - commit: 1935555c206d4afb9e94615dfd0fad31 + commit: 04467658e59e44bbb22fe568206e1f70 - remote: buf.build owner: cosmos repository: cosmos-sdk @@ -16,16 +16,16 @@ deps: - remote: buf.build owner: cosmos repository: gogo-proto - commit: 5e5b9fdd01804356895f8f79a6f1ddc1 + commit: 88ef6483f90f478fb938c37dde52ece3 - remote: buf.build owner: cosmos repository: ics23 - commit: 3c44d8daa8b44059ac744cd17d4a49d7 + commit: a9ee7c290ef34ee69d3f141b9b44dcee - remote: buf.build owner: googleapis repository: googleapis - commit: cc916c31859748a68fd229a3c8d7a2e8 + commit: 09703837a2ed48dbbbb3fdfbe6a84f5c - remote: buf.build owner: protocolbuffers repository: wellknowntypes - commit: 44e83bc050a4497fa7b36b34d95ca156 + commit: 3186086b2a8e44d9acdeeef2423c5de7 From 697c1bf9e3bb70b26b39a293defe40ed7700fa65 Mon Sep 17 00:00:00 2001 From: Pantani Date: Mon, 13 May 2024 19:44:37 +0200 Subject: [PATCH 10/20] fix unit test --- ignite/config/chain/v1/testdata/config.yaml | 3 --- ignite/config/chain/v1/testdata/config2.yaml | 3 --- 2 files changed, 6 deletions(-) diff --git a/ignite/config/chain/v1/testdata/config.yaml b/ignite/config/chain/v1/testdata/config.yaml index 6221affba7..efebd4a499 100644 --- a/ignite/config/chain/v1/testdata/config.yaml +++ b/ignite/config/chain/v1/testdata/config.yaml @@ -3,9 +3,6 @@ build: binary: evmosd proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: diff --git a/ignite/config/chain/v1/testdata/config2.yaml b/ignite/config/chain/v1/testdata/config2.yaml index 11afb27845..e611e287ff 100644 --- a/ignite/config/chain/v1/testdata/config2.yaml +++ b/ignite/config/chain/v1/testdata/config2.yaml @@ -3,9 +3,6 @@ build: binary: evmosd proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: From f0195eca333ee729f5a1c07e0066ba8c72011bab Mon Sep 17 00:00:00 2001 From: Pantani Date: Tue, 14 May 2024 01:46:24 +0200 Subject: [PATCH 11/20] fix wrong proto go path --- ignite/pkg/cosmosbuf/buf.go | 6 +++--- ignite/pkg/cosmosgen/cosmosgen.go | 8 ++++---- ignite/pkg/cosmosgen/generate_go.go | 2 +- ignite/pkg/cosmosgen/generate_openapi.go | 8 ++++---- ignite/templates/module/create/configs.go | 3 +-- ignite/templates/module/create/params.go | 2 +- 6 files changed, 14 insertions(+), 15 deletions(-) diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index b287565e4c..b6540c5eab 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -88,7 +88,7 @@ func FileByFile() GenOption { } // New creates a new Buf based on the installed binary. -func New(cacheStorage cache.Storage) (Buf, error) { +func New(cacheStorage cache.Storage, goModPath string) (Buf, error) { path, err := xexec.ResolveAbsPath(binaryName) if err != nil { return Buf{}, err @@ -98,8 +98,8 @@ func New(cacheStorage cache.Storage) (Buf, error) { if err != nil { return Buf{}, err } - bufCachePath := filepath.Join(globalPath, "buf") - if err := os.Mkdir(bufCachePath, 0o700); err != nil && !os.IsExist(err) { + bufCachePath := filepath.Join(globalPath, "buf", goModPath) + if err := os.MkdirAll(bufCachePath, 0o755); err != nil && !os.IsExist(err) { return Buf{}, err } diff --git a/ignite/pkg/cosmosgen/cosmosgen.go b/ignite/pkg/cosmosgen/cosmosgen.go index ae14ab952c..6235d736d9 100644 --- a/ignite/pkg/cosmosgen/cosmosgen.go +++ b/ignite/pkg/cosmosgen/cosmosgen.go @@ -101,7 +101,7 @@ type generator struct { cacheStorage cache.Storage appPath string protoDir string - gomodPath string + goModPath string opts *generateOptions sdkImport string sdkDir string @@ -122,8 +122,8 @@ func (g *generator) cleanup() { // Generate generates code from protoDir of an SDK app residing at appPath with given options. // protoDir must be relative to the projectPath. -func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir, gomodPath string, options ...Option) error { - b, err := cosmosbuf.New(cacheStorage) +func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir, goModPath string, options ...Option) error { + b, err := cosmosbuf.New(cacheStorage, goModPath) if err != nil { return err } @@ -132,7 +132,7 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir buf: b, appPath: appPath, protoDir: protoDir, - gomodPath: gomodPath, + goModPath: goModPath, opts: &generateOptions{}, thirdModules: make(map[string][]module.Module), thirdModuleIncludes: make(map[string]protoIncludes), diff --git a/ignite/pkg/cosmosgen/generate_go.go b/ignite/pkg/cosmosgen/generate_go.go index 8c10facf98..2aeb52f6c9 100644 --- a/ignite/pkg/cosmosgen/generate_go.go +++ b/ignite/pkg/cosmosgen/generate_go.go @@ -24,7 +24,7 @@ func (g *generator) protoPath() string { } func (g *generator) generateGoGo(ctx context.Context) error { - return g.generate(ctx, g.gogoTemplate(), g.gomodPath, "module.proto") + return g.generate(ctx, g.gogoTemplate(), g.goModPath, "*/module.proto") } func (g *generator) generatePulsar(ctx context.Context) error { diff --git a/ignite/pkg/cosmosgen/generate_openapi.go b/ignite/pkg/cosmosgen/generate_openapi.go index 1ac0a11b64..28bf99a0a9 100644 --- a/ignite/pkg/cosmosgen/generate_openapi.go +++ b/ignite/pkg/cosmosgen/generate_openapi.go @@ -35,7 +35,7 @@ func (g *generator) openAPITemplateForSTA() string { func (g *generator) generateOpenAPISpec(ctx context.Context) error { var ( specDirs []string - conf = swaggercombine.New("HTTP API Console", g.gomodPath) + conf = swaggercombine.New("HTTP API Console", g.goModPath) ) defer func() { for _, dir := range specDirs { @@ -124,7 +124,7 @@ func (g *generator) generateOpenAPISpec(ctx context.Context) error { // into a single spec. // protoc openapi generator acts weird on concurrent run, so do not use goroutines here. - if err := gen(g.appPath, g.protoDir, g.gomodPath); err != nil { + if err := gen(g.appPath, g.protoDir, g.goModPath); err != nil { return err } @@ -176,7 +176,7 @@ func (g *generator) generateModuleOpenAPISpec(ctx context.Context, m module.Modu var ( specDirs []string title = "HTTP API Console " + m.Pkg.Name - conf = swaggercombine.New(title, g.gomodPath) + conf = swaggercombine.New(title, g.goModPath) ) defer func() { for _, dir := range specDirs { @@ -192,7 +192,7 @@ func (g *generator) generateModuleOpenAPISpec(ctx context.Context, m module.Modu return err } - err = g.buf.Generate(ctx, m.Pkg.Path, dir, g.openAPITemplateForSTA(), cosmosbuf.ExcludeFiles("module.proto")) + err = g.buf.Generate(ctx, m.Pkg.Path, dir, g.openAPITemplateForSTA(), cosmosbuf.ExcludeFiles("*/module.proto")) if err != nil { return err } diff --git a/ignite/templates/module/create/configs.go b/ignite/templates/module/create/configs.go index 5d24de6aa5..10ccd2fa28 100644 --- a/ignite/templates/module/create/configs.go +++ b/ignite/templates/module/create/configs.go @@ -1,7 +1,6 @@ package modulecreate import ( - "fmt" "path/filepath" "github.com/gobuffalo/genny/v2" @@ -37,7 +36,7 @@ func configsProtoModify(opts ConfigsOptions) genny.RunFn { for _, paramField := range opts.Configs { _, err := protoutil.GetFieldByName(params, paramField.Name.LowerCamel) if err == nil { - return fmt.Errorf("duplicate field %s in %s", paramField.Name.LowerCamel, params.Name) + return errors.Errorf("duplicate field %s in %s", paramField.Name.LowerCamel, params.Name) } param := protoutil.NewField( diff --git a/ignite/templates/module/create/params.go b/ignite/templates/module/create/params.go index 265bf6d487..7e94a57b70 100644 --- a/ignite/templates/module/create/params.go +++ b/ignite/templates/module/create/params.go @@ -38,7 +38,7 @@ func paramsProtoModify(opts ParamsOptions) genny.RunFn { for _, paramField := range opts.Params { _, err := protoutil.GetFieldByName(params, paramField.Name.LowerCamel) if err == nil { - return fmt.Errorf("duplicate field %s in %s", paramField.Name.LowerCamel, params.Name) + return errors.Errorf("duplicate field %s in %s", paramField.Name.LowerCamel, params.Name) } param := protoutil.NewField( From c044a5b28f3a06f6033b693f36d28b949dd9b2ee Mon Sep 17 00:00:00 2001 From: Pantani Date: Tue, 14 May 2024 02:01:08 +0200 Subject: [PATCH 12/20] clear buf cache --- ignite/cmd/cmd.go | 4 ++++ ignite/pkg/cosmosbuf/buf.go | 5 ++--- ignite/pkg/cosmosbuf/cache.go | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/ignite/cmd/cmd.go b/ignite/cmd/cmd.go index da86ec4aa2..11ffe5de06 100644 --- a/ignite/cmd/cmd.go +++ b/ignite/cmd/cmd.go @@ -17,6 +17,7 @@ import ( "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cliui" uilog "github.com/ignite/cli/v29/ignite/pkg/cliui/log" + "github.com/ignite/cli/v29/ignite/pkg/cosmosbuf" "github.com/ignite/cli/v29/ignite/pkg/errors" "github.com/ignite/cli/v29/ignite/pkg/gitpod" "github.com/ignite/cli/v29/ignite/pkg/goenv" @@ -275,6 +276,9 @@ func newCache(cmd *cobra.Command) (cache.Storage, error) { if err := storage.Clear(); err != nil { return cache.Storage{}, err } + if err := cosmosbuf.ClearCache(); err != nil { + return cache.Storage{}, err + } } return storage, nil diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index b6540c5eab..a90f8b7dd4 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -10,7 +10,6 @@ import ( "github.com/gobwas/glob" "golang.org/x/sync/errgroup" - "github.com/ignite/cli/v29/ignite/config" "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cmdrunner/exec" "github.com/ignite/cli/v29/ignite/pkg/errors" @@ -94,11 +93,11 @@ func New(cacheStorage cache.Storage, goModPath string) (Buf, error) { return Buf{}, err } - globalPath, err := config.DirPath() + bufCachePath, err := cachePath() if err != nil { return Buf{}, err } - bufCachePath := filepath.Join(globalPath, "buf", goModPath) + bufCachePath = filepath.Join(bufCachePath, "buf", goModPath) if err := os.MkdirAll(bufCachePath, 0o755); err != nil && !os.IsExist(err) { return Buf{}, err } diff --git a/ignite/pkg/cosmosbuf/cache.go b/ignite/pkg/cosmosbuf/cache.go index 9354d298ca..d07f16fbb2 100644 --- a/ignite/pkg/cosmosbuf/cache.go +++ b/ignite/pkg/cosmosbuf/cache.go @@ -8,11 +8,28 @@ import ( "github.com/otiai10/copy" + "github.com/ignite/cli/v29/ignite/config" "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/dirchange" "github.com/ignite/cli/v29/ignite/pkg/errors" ) +func ClearCache() error { + path, err := cachePath() + if err != nil { + return err + } + return os.RemoveAll(path) +} + +func cachePath() (string, error) { + globalPath, err := config.DirPath() + if err != nil { + return "", err + } + return filepath.Join(globalPath, "buf"), nil +} + func cacheKey(src, template string) (string, error) { checksum, err := dirchange.ChecksumFromPaths(src, "") if err != nil { From 4acabf356eee13d8595fcf0da9e5c04d2fe80721 Mon Sep 17 00:00:00 2001 From: Pantani Date: Tue, 14 May 2024 02:09:28 +0200 Subject: [PATCH 13/20] fix doctor tests and remove `third_party_paths` from the docs --- docs/docs/02-guide/06-ibc.md | 6 ------ docs/docs/02-guide/07-interchange/02-init.md | 6 ------ docs/docs/08-references/02-config.md | 11 ----------- docs/versioned_docs/version-v0.25/kb/03-config.md | 1 - docs/versioned_docs/version-v0.25/kb/06-proto.md | 8 -------- integration/doctor/testdata/config-need-migrate.txt | 3 --- 6 files changed, 35 deletions(-) diff --git a/docs/docs/02-guide/06-ibc.md b/docs/docs/02-guide/06-ibc.md index faa01a1cae..abddcd9c14 100644 --- a/docs/docs/02-guide/06-ibc.md +++ b/docs/docs/02-guide/06-ibc.md @@ -432,9 +432,6 @@ version: 1 build: proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: @@ -463,9 +460,6 @@ version: 1 build: proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: diff --git a/docs/docs/02-guide/07-interchange/02-init.md b/docs/docs/02-guide/07-interchange/02-init.md index a8c96755f2..2f19beee72 100644 --- a/docs/docs/02-guide/07-interchange/02-init.md +++ b/docs/docs/02-guide/07-interchange/02-init.md @@ -129,9 +129,6 @@ version: 1 build: proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: @@ -164,9 +161,6 @@ version: 1 build: proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: diff --git a/docs/docs/08-references/02-config.md b/docs/docs/08-references/02-config.md index 34347b9565..d0228a4a18 100644 --- a/docs/docs/08-references/02-config.md +++ b/docs/docs/08-references/02-config.md @@ -203,17 +203,6 @@ build: path: "myproto" ``` -Ignite comes with required third-party proto out of the box. Ignite also looks -into `third_party/proto` and `proto_vendor` directories for extra proto files. -If your project keeps third-party proto files in a different directory, you -should tell Ignite about this: - -```yml -build: - proto: - third_party_paths: ["my_third_party/proto"] -``` - ## Faucet The faucet service sends tokens to addresses. diff --git a/docs/versioned_docs/version-v0.25/kb/03-config.md b/docs/versioned_docs/version-v0.25/kb/03-config.md index 278498af6b..8357afa66b 100644 --- a/docs/versioned_docs/version-v0.25/kb/03-config.md +++ b/docs/versioned_docs/version-v0.25/kb/03-config.md @@ -57,7 +57,6 @@ build: | Key | Required | Type | Description | | ----------------- | -------- | --------------- | ------------------------------------------------------------------------------------------ | | path | N | String | Path to protocol buffer files. Default: `"proto"`. | -| third_party_paths | N | List of Strings | Path to third-party protocol buffer files. Default: `["third_party/proto", "proto_vendor"]`. | ## client diff --git a/docs/versioned_docs/version-v0.25/kb/06-proto.md b/docs/versioned_docs/version-v0.25/kb/06-proto.md index ce2ecc6c2f..34b6018b93 100644 --- a/docs/versioned_docs/version-v0.25/kb/06-proto.md +++ b/docs/versioned_docs/version-v0.25/kb/06-proto.md @@ -20,11 +20,3 @@ Third-party proto files, including those of Cosmos SDK and Tendermint, are bundl ```protobuf import "cosmos/base/query/v1beta1/pagination.proto"; ``` - -You can also manually add third-party proto files. By default, Ignite CLI imports proto files from these directories: `third_party/proto` and `proto_vendor`. You can define third-party paths of the import directory in `config.yml`: - -```yaml -build: - proto: - third_party_paths: ["my_third_party_proto"] -``` diff --git a/integration/doctor/testdata/config-need-migrate.txt b/integration/doctor/testdata/config-need-migrate.txt index e29d184340..388673c6f8 100644 --- a/integration/doctor/testdata/config-need-migrate.txt +++ b/integration/doctor/testdata/config-need-migrate.txt @@ -35,9 +35,6 @@ version: 1 build: proto: path: proto - third_party_paths: - - third_party/proto - - proto_vendor accounts: - name: alice coins: From b647f095aa3fe08f8253c649698abb2116b492d3 Mon Sep 17 00:00:00 2001 From: Pantani Date: Wed, 15 May 2024 01:31:58 +0200 Subject: [PATCH 14/20] improve Test_extractRootModulePath test cases --- ignite/pkg/cosmosgen/generate_openapi_test.go | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/ignite/pkg/cosmosgen/generate_openapi_test.go b/ignite/pkg/cosmosgen/generate_openapi_test.go index bc4cefb775..eca07b518d 100644 --- a/ignite/pkg/cosmosgen/generate_openapi_test.go +++ b/ignite/pkg/cosmosgen/generate_openapi_test.go @@ -14,33 +14,38 @@ func Test_extractRootModulePath(t *testing.T) { }{ { name: "test cosmos-sdk path", - path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6/proto/cosmos/distribution/v1beta1", - want: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6", + path: "/Users/ignite/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6/proto/cosmos/distribution/v1beta1", + want: "/Users/ignite/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6", + }, + { + name: "test cosmos-sdk module proto path", + path: "/Users/ignite/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6/x/bank", + want: "/Users/ignite/Desktop/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.50.6", }, { name: "test ibc path", - path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/cosmos/ibc-go/v8@v8.2.0/proto/ibc/applications/interchain_accounts/controller/v1", - want: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/cosmos/ibc-go/v8@v8.2.0", + path: "/Users/ignite/Desktop/go/pkg/mod/github.com/cosmos/ibc-go/v8@v8.2.0/proto/ibc/applications/interchain_accounts/controller/v1", + want: "/Users/ignite/Desktop/go/pkg/mod/github.com/cosmos/ibc-go/v8@v8.2.0", }, { name: "test chain path", - path: "/Users/danilopantani/Desktop/go/src/github.com/ignite/venus", - want: "/Users/danilopantani/Desktop/go/src/github.com/ignite/venus", + path: "/Users/ignite/Desktop/go/src/github.com/ignite/venus", + want: "/Users/ignite/Desktop/go/src/github.com/ignite/venus", }, { name: "test module path without version", - path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/proto/applications", - want: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/proto/applications", + path: "/Users/ignite/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/proto/applications", + want: "/Users/ignite/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/proto/applications", }, { name: "test module path with broken version", - path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway@v1.$/controller", - want: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway@v1.$/controller", + path: "/Users/ignite/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway@v1.$/controller", + want: "/Users/ignite/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway@v1.$/controller", }, { name: "test module path with v2 version", - path: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/v2@v2.19.1/proto/files", - want: "/Users/danilopantani/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/v2@v2.19.1", + path: "/Users/ignite/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/v2@v2.19.1/proto/files", + want: "/Users/ignite/Desktop/go/pkg/mod/github.com/grpc-ecosystem/grpc-gateway/v2@v2.19.1", }, } for _, tt := range tests { From 22c04008f37f4b26afbadcbe08d746fe972cd726 Mon Sep 17 00:00:00 2001 From: Danilo Pantani Date: Thu, 16 May 2024 15:38:42 +0200 Subject: [PATCH 15/20] Update ignite/pkg/cosmosbuf/cache.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jerónimo Albi --- ignite/pkg/cosmosbuf/cache.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ignite/pkg/cosmosbuf/cache.go b/ignite/pkg/cosmosbuf/cache.go index d07f16fbb2..0684c1426b 100644 --- a/ignite/pkg/cosmosbuf/cache.go +++ b/ignite/pkg/cosmosbuf/cache.go @@ -43,8 +43,7 @@ func cacheKey(src, template string) (string, error) { if _, err := h.Write([]byte(template)); err != nil { return "", err } - key := fmt.Sprintf("%x", h.Sum(nil)) - return key, nil + return fmt.Sprintf("%x", h.Sum(nil)), nil } func (b Buf) getCache(src, template, output string) (string, bool, error) { From 9d9d8074c15d6705b28d4d02ae7647c1afadaa08 Mon Sep 17 00:00:00 2001 From: Danilo Pantani Date: Thu, 16 May 2024 15:39:39 +0200 Subject: [PATCH 16/20] fix changelog.md --- changelog.md | 1 - 1 file changed, 1 deletion(-) diff --git a/changelog.md b/changelog.md index 65aaa6bcb8..2f3c931e83 100644 --- a/changelog.md +++ b/changelog.md @@ -16,7 +16,6 @@ - [#4077](https://github.com/ignite/cli/pull/4077) Merge the swagger files manually instead use nodetime `swagger-combine` - [#4100](https://github.com/ignite/cli/pull/4100) Set the `proto-dir` flag only for the `scaffold chain` command and use the proto path from the config - [#4111](https://github.com/ignite/cli/pull/4111) Remove vuex generation -- [#4117](https://github.com/ignite/cli/pull/4117), [#4125](https://github.com/ignite/cli/pull/4125) Support relative path when installing local plugins - [#4133](https://github.com/ignite/cli/pull/4133) Improve buf rate limit - [#4131](https://github.com/ignite/cli/pull/4131) Support `bytes` as data type in the `scaffold` commands From a90951519180e331103a6094cf0be57b749bf36a Mon Sep 17 00:00:00 2001 From: Pantani Date: Thu, 16 May 2024 19:27:38 +0200 Subject: [PATCH 17/20] improve cache logic --- ignite/pkg/cosmosbuf/buf.go | 6 ++++-- ignite/pkg/cosmosbuf/cache.go | 14 ++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index a90f8b7dd4..676ebd2ee4 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -173,9 +173,11 @@ func (b Buf) Generate( } // check if already exist a cache for the template. - key, found, err := b.getCache(protoPath, template, output) - if err != nil || found { + key, err := b.copyCache(protoPath, template, output) + if err != nil && !errors.Is(err, ErrCacheNotFound) { return err + } else if err == nil { + return nil } // remove excluded and cached files. diff --git a/ignite/pkg/cosmosbuf/cache.go b/ignite/pkg/cosmosbuf/cache.go index 0684c1426b..4142ea8587 100644 --- a/ignite/pkg/cosmosbuf/cache.go +++ b/ignite/pkg/cosmosbuf/cache.go @@ -14,6 +14,8 @@ import ( "github.com/ignite/cli/v29/ignite/pkg/errors" ) +var ErrCacheNotFound = errors.New("cache not found") + func ClearCache() error { path, err := cachePath() if err != nil { @@ -46,23 +48,23 @@ func cacheKey(src, template string) (string, error) { return fmt.Sprintf("%x", h.Sum(nil)), nil } -func (b Buf) getCache(src, template, output string) (string, bool, error) { +func (b Buf) copyCache(src, template, output string) (string, error) { key, err := cacheKey(src, template) if err != nil { - return key, false, err + return key, err } cachedPath, err := b.storageCache.Get(key) if errors.Is(err, cache.ErrorNotFound) { - return key, false, nil + return key, ErrCacheNotFound } else if err != nil { - return key, false, err + return key, err } if err := copy.Copy(cachedPath, output); err != nil { - return "", false, errors.Wrapf(err, "buf get cache cannot copy path %s to %s", cachedPath, output) + return "", errors.Wrapf(err, "buf get cache cannot copy path %s to %s", cachedPath, output) } - return key, true, nil + return key, nil } func (b Buf) saveCache(key, src string) error { From d11942a69c4f57f32faa3cf8e59ea86a13d216e9 Mon Sep 17 00:00:00 2001 From: Pantani Date: Thu, 16 May 2024 20:51:07 +0200 Subject: [PATCH 18/20] create a dircache pkg --- ignite/cmd/cmd.go | 4 +- ignite/pkg/cosmosbuf/buf.go | 25 ++--- ignite/pkg/cosmosbuf/cache.go | 82 -------------- ignite/pkg/dircache/cache.go | 109 +++++++++++++++++++ ignite/pkg/dircache/cache_test.go | 87 +++++++++++++++ ignite/pkg/dircache/testdata/subdata/subfile | 1 + ignite/pkg/dircache/testdata/testfile | 1 + 7 files changed, 210 insertions(+), 99 deletions(-) delete mode 100644 ignite/pkg/cosmosbuf/cache.go create mode 100644 ignite/pkg/dircache/cache.go create mode 100644 ignite/pkg/dircache/cache_test.go create mode 100644 ignite/pkg/dircache/testdata/subdata/subfile create mode 100644 ignite/pkg/dircache/testdata/testfile diff --git a/ignite/cmd/cmd.go b/ignite/cmd/cmd.go index 11ffe5de06..034af6cb3a 100644 --- a/ignite/cmd/cmd.go +++ b/ignite/cmd/cmd.go @@ -17,7 +17,7 @@ import ( "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cliui" uilog "github.com/ignite/cli/v29/ignite/pkg/cliui/log" - "github.com/ignite/cli/v29/ignite/pkg/cosmosbuf" + "github.com/ignite/cli/v29/ignite/pkg/dircache" "github.com/ignite/cli/v29/ignite/pkg/errors" "github.com/ignite/cli/v29/ignite/pkg/gitpod" "github.com/ignite/cli/v29/ignite/pkg/goenv" @@ -276,7 +276,7 @@ func newCache(cmd *cobra.Command) (cache.Storage, error) { if err := storage.Clear(); err != nil { return cache.Storage{}, err } - if err := cosmosbuf.ClearCache(); err != nil { + if err := dircache.ClearCache(); err != nil { return cache.Storage{}, err } } diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index 676ebd2ee4..6970998a7d 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -3,7 +3,6 @@ package cosmosbuf import ( "context" "fmt" - "os" "path/filepath" "strings" @@ -12,6 +11,7 @@ import ( "github.com/ignite/cli/v29/ignite/pkg/cache" "github.com/ignite/cli/v29/ignite/pkg/cmdrunner/exec" + "github.com/ignite/cli/v29/ignite/pkg/dircache" "github.com/ignite/cli/v29/ignite/pkg/errors" "github.com/ignite/cli/v29/ignite/pkg/xexec" "github.com/ignite/cli/v29/ignite/pkg/xos" @@ -23,9 +23,8 @@ type ( // Buf represents the buf application structure. Buf struct { - path string - bufCachePath string - storageCache cache.Cache[string] + path string + cache dircache.Cache } // genOptions used to configure code generation. @@ -93,19 +92,15 @@ func New(cacheStorage cache.Storage, goModPath string) (Buf, error) { return Buf{}, err } - bufCachePath, err := cachePath() + bufCacheDir := filepath.Join("buf", goModPath) + c, err := dircache.New(cacheStorage, bufCacheDir, specCacheNamespace) if err != nil { return Buf{}, err } - bufCachePath = filepath.Join(bufCachePath, "buf", goModPath) - if err := os.MkdirAll(bufCachePath, 0o755); err != nil && !os.IsExist(err) { - return Buf{}, err - } return Buf{ - path: path, - bufCachePath: bufCachePath, - storageCache: cache.New[string](cacheStorage, specCacheNamespace), + path: path, + cache: c, }, nil } @@ -173,8 +168,8 @@ func (b Buf) Generate( } // check if already exist a cache for the template. - key, err := b.copyCache(protoPath, template, output) - if err != nil && !errors.Is(err, ErrCacheNotFound) { + key, err := b.cache.CopyCache(protoPath, output, template) + if err != nil && !errors.Is(err, dircache.ErrCacheNotFound) { return err } else if err == nil { return nil @@ -234,7 +229,7 @@ func (b Buf) Generate( } } - return b.saveCache(key, output) + return b.cache.SaveCache(output, key) } // runCommand run the buf CLI command. diff --git a/ignite/pkg/cosmosbuf/cache.go b/ignite/pkg/cosmosbuf/cache.go deleted file mode 100644 index 4142ea8587..0000000000 --- a/ignite/pkg/cosmosbuf/cache.go +++ /dev/null @@ -1,82 +0,0 @@ -package cosmosbuf - -import ( - "crypto/sha256" - "fmt" - "os" - "path/filepath" - - "github.com/otiai10/copy" - - "github.com/ignite/cli/v29/ignite/config" - "github.com/ignite/cli/v29/ignite/pkg/cache" - "github.com/ignite/cli/v29/ignite/pkg/dirchange" - "github.com/ignite/cli/v29/ignite/pkg/errors" -) - -var ErrCacheNotFound = errors.New("cache not found") - -func ClearCache() error { - path, err := cachePath() - if err != nil { - return err - } - return os.RemoveAll(path) -} - -func cachePath() (string, error) { - globalPath, err := config.DirPath() - if err != nil { - return "", err - } - return filepath.Join(globalPath, "buf"), nil -} - -func cacheKey(src, template string) (string, error) { - checksum, err := dirchange.ChecksumFromPaths(src, "") - if err != nil { - return "", err - } - - h := sha256.New() - if _, err := h.Write(checksum); err != nil { - return "", err - } - if _, err := h.Write([]byte(template)); err != nil { - return "", err - } - return fmt.Sprintf("%x", h.Sum(nil)), nil -} - -func (b Buf) copyCache(src, template, output string) (string, error) { - key, err := cacheKey(src, template) - if err != nil { - return key, err - } - - cachedPath, err := b.storageCache.Get(key) - if errors.Is(err, cache.ErrorNotFound) { - return key, ErrCacheNotFound - } else if err != nil { - return key, err - } - - if err := copy.Copy(cachedPath, output); err != nil { - return "", errors.Wrapf(err, "buf get cache cannot copy path %s to %s", cachedPath, output) - } - return key, nil -} - -func (b Buf) saveCache(key, src string) error { - cachePath := filepath.Join(b.bufCachePath, key) - if err := os.Mkdir(cachePath, 0o700); os.IsExist(err) { - return nil - } else if err != nil { - return err - } - - if err := copy.Copy(src, cachePath); err != nil { - return errors.Wrapf(err, "buf save cache cannot copy path %s to %s", src, cachePath) - } - return b.storageCache.Put(key, cachePath) -} diff --git a/ignite/pkg/dircache/cache.go b/ignite/pkg/dircache/cache.go new file mode 100644 index 0000000000..064b76fa4c --- /dev/null +++ b/ignite/pkg/dircache/cache.go @@ -0,0 +1,109 @@ +package dircache + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + + "github.com/otiai10/copy" + + "github.com/ignite/cli/v29/ignite/config" + "github.com/ignite/cli/v29/ignite/pkg/cache" + "github.com/ignite/cli/v29/ignite/pkg/dirchange" + "github.com/ignite/cli/v29/ignite/pkg/errors" +) + +var ErrCacheNotFound = errors.New("cache not found") + +type Cache struct { + path string + storageCache cache.Cache[string] +} + +// New creates a new Buf based on the installed binary. +func New(cacheStorage cache.Storage, dir, specNamespace string) (Cache, error) { + path, err := cachePath() + if err != nil { + return Cache{}, err + } + path = filepath.Join(path, dir) + if err := os.MkdirAll(path, 0o755); err != nil && !os.IsExist(err) { + return Cache{}, err + } + + return Cache{ + path: path, + storageCache: cache.New[string](cacheStorage, specNamespace), + }, nil +} + +// ClearCache remove the cache path. +func ClearCache() error { + path, err := cachePath() + if err != nil { + return err + } + return os.RemoveAll(path) +} + +// cachePath returns the cache path. +func cachePath() (string, error) { + globalPath, err := config.DirPath() + if err != nil { + return "", err + } + return filepath.Join(globalPath, "cache"), nil +} + +// cacheKey create the cache key. +func cacheKey(src string, keys ...string) (string, error) { + checksum, err := dirchange.ChecksumFromPaths(src, "") + if err != nil { + return "", err + } + + h := sha256.New() + if _, err := h.Write(checksum); err != nil { + return "", err + } + for _, key := range keys { + if _, err := h.Write([]byte(key)); err != nil { + return "", err + } + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +func (c Cache) CopyCache(src, output string, keys ...string) (string, error) { + key, err := cacheKey(src, keys...) + if err != nil { + return key, err + } + + cachedPath, err := c.storageCache.Get(key) + if errors.Is(err, cache.ErrorNotFound) { + return key, ErrCacheNotFound + } else if err != nil { + return key, err + } + + if err := copy.Copy(cachedPath, output); err != nil { + return "", errors.Wrapf(err, "get dir cache cannot copy path %s to %s", cachedPath, output) + } + return key, nil +} + +func (c Cache) SaveCache(src, key string) error { + path := filepath.Join(c.path, key) + if err := os.Mkdir(path, 0o700); os.IsExist(err) { + return nil + } else if err != nil { + return err + } + + if err := copy.Copy(src, path); err != nil { + return errors.Wrapf(err, "save dir cache cannot copy path %s to %s", src, path) + } + return c.storageCache.Put(key, path) +} diff --git a/ignite/pkg/dircache/cache_test.go b/ignite/pkg/dircache/cache_test.go new file mode 100644 index 0000000000..8ba7d24953 --- /dev/null +++ b/ignite/pkg/dircache/cache_test.go @@ -0,0 +1,87 @@ +package dircache + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/ignite/cli/v29/ignite/pkg/errors" +) + +func Test_cacheKey(t *testing.T) { + wd, err := os.Getwd() + require.NoError(t, err) + wd = filepath.Join(wd, "testdata") + + type args struct { + src string + keys []string + } + tests := []struct { + name string + args args + want string + err error + }{ + { + name: "no keys", + args: args{ + src: wd, + }, + want: "78f544d2184b8076ac527ba4728822de1a7fc77bf2d6a77e44d0193cb63ed26e", + }, + { + name: "one key", + args: args{ + src: wd, + keys: []string{"test"}, + }, + want: "5701099a1fcc67cd8b694295fbdecf537edcc8733bcc3adae0bdd7e65e28c8e5", + }, + { + name: "two keys", + args: args{ + src: wd, + keys: []string{"test1", "test2"}, + }, + want: "6299c9bd405a1c073fa711006f8aadf6420cf522ef446e36fc01586354726095", + }, + { + name: "duplicated keys", + args: args{ + src: wd, + keys: []string{"test", "test"}, + }, + want: "b9eb1b01931deccc44a354ab5aeb52337a465e5559069eb35b71ea0cbfe3c87f", + }, + { + name: "many keys", + args: args{ + src: wd, + keys: []string{"test1", "test2", "test3", "test4", "test5", "test6", "test6"}, + }, + want: "bbe74cfd33ba4d1244e8d0ea3e430081d06ed55be12c7772d345d3117a4dfc90", + }, + { + name: "invalid source", + args: args{ + src: "invalid_source", + }, + err: errors.New("no file in specified paths"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := cacheKey(tt.args.src, tt.args.keys...) + if tt.err != nil { + require.Error(t, err) + require.Equal(t, tt.err.Error(), err.Error()) + return + } + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} diff --git a/ignite/pkg/dircache/testdata/subdata/subfile b/ignite/pkg/dircache/testdata/subdata/subfile new file mode 100644 index 0000000000..41823eb5b1 --- /dev/null +++ b/ignite/pkg/dircache/testdata/subdata/subfile @@ -0,0 +1 @@ +subtest \ No newline at end of file diff --git a/ignite/pkg/dircache/testdata/testfile b/ignite/pkg/dircache/testdata/testfile new file mode 100644 index 0000000000..30d74d2584 --- /dev/null +++ b/ignite/pkg/dircache/testdata/testfile @@ -0,0 +1 @@ +test \ No newline at end of file From bd94345bde5e890c0e4507ed0d2d75b5541fe492 Mon Sep 17 00:00:00 2001 From: Pantani Date: Thu, 16 May 2024 20:57:31 +0200 Subject: [PATCH 19/20] add comments --- ignite/pkg/dircache/cache.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ignite/pkg/dircache/cache.go b/ignite/pkg/dircache/cache.go index 064b76fa4c..3e279d9c40 100644 --- a/ignite/pkg/dircache/cache.go +++ b/ignite/pkg/dircache/cache.go @@ -75,6 +75,7 @@ func cacheKey(src string, keys ...string) (string, error) { return fmt.Sprintf("%x", h.Sum(nil)), nil } +// CopyCache gets the cache folder based on the cache key from the storage and copies the folder to the output. func (c Cache) CopyCache(src, output string, keys ...string) (string, error) { key, err := cacheKey(src, keys...) if err != nil { @@ -94,6 +95,7 @@ func (c Cache) CopyCache(src, output string, keys ...string) (string, error) { return key, nil } +// SaveCache copies the source to the cache folder and saves the path into the storage based on the key. func (c Cache) SaveCache(src, key string) error { path := filepath.Join(c.path, key) if err := os.Mkdir(path, 0o700); os.IsExist(err) { From f6aeed72c6c60e993cc4b27343d94c4ea08f45d8 Mon Sep 17 00:00:00 2001 From: Pantani Date: Fri, 17 May 2024 18:39:32 +0200 Subject: [PATCH 20/20] rename methods `CopyCache` to `CopyTo` and `SaveCache` to `Save` --- ignite/pkg/cosmosbuf/buf.go | 4 ++-- ignite/pkg/dircache/cache.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index 6970998a7d..a0470532ea 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -168,7 +168,7 @@ func (b Buf) Generate( } // check if already exist a cache for the template. - key, err := b.cache.CopyCache(protoPath, output, template) + key, err := b.cache.CopyTo(protoPath, output, template) if err != nil && !errors.Is(err, dircache.ErrCacheNotFound) { return err } else if err == nil { @@ -229,7 +229,7 @@ func (b Buf) Generate( } } - return b.cache.SaveCache(output, key) + return b.cache.Save(output, key) } // runCommand run the buf CLI command. diff --git a/ignite/pkg/dircache/cache.go b/ignite/pkg/dircache/cache.go index 3e279d9c40..14efc36504 100644 --- a/ignite/pkg/dircache/cache.go +++ b/ignite/pkg/dircache/cache.go @@ -75,8 +75,8 @@ func cacheKey(src string, keys ...string) (string, error) { return fmt.Sprintf("%x", h.Sum(nil)), nil } -// CopyCache gets the cache folder based on the cache key from the storage and copies the folder to the output. -func (c Cache) CopyCache(src, output string, keys ...string) (string, error) { +// CopyTo gets the cache folder based on the cache key from the storage and copies the folder to the output. +func (c Cache) CopyTo(src, output string, keys ...string) (string, error) { key, err := cacheKey(src, keys...) if err != nil { return key, err @@ -95,8 +95,8 @@ func (c Cache) CopyCache(src, output string, keys ...string) (string, error) { return key, nil } -// SaveCache copies the source to the cache folder and saves the path into the storage based on the key. -func (c Cache) SaveCache(src, key string) error { +// Save copies the source to the cache folder and saves the path into the storage based on the key. +func (c Cache) Save(src, key string) error { path := filepath.Join(c.path, key) if err := os.Mkdir(path, 0o700); os.IsExist(err) { return nil