From 876d9a9de3e2f08292546be2222d665dd46a9d5c Mon Sep 17 00:00:00 2001 From: Eitol Date: Tue, 9 Jul 2024 20:03:17 -0400 Subject: [PATCH 1/9] Add functionality to export proto files Added a new function, `WriteProtoFiles` in `desc_source.go` which is used to generate .proto files. The process involves resolving symbols from the descriptor source and writing their definitions to a designated output directory. The corresponding flag `--proto-out` has been included in `grpcurl.go` to allow users to specify the directory path. --- README.md | 5 +++ cmd/grpcurl/grpcurl.go | 24 ++++++++++++ desc_source.go | 86 +++++++++++++++++++++++++++++++++++------- 3 files changed, 102 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 49960186..e2d4b331 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,11 @@ grpcurl -protoset my-protos.bin list # Using proto sources grpcurl -import-path ../protos -proto my-stuff.proto list + +# Export proto files +grpcurl -plaintext -proto-out "out_protos" "192.168.100.1:9200" describe Api.Service + + ``` The "list" verb also lets you see all methods in a particular service: diff --git a/cmd/grpcurl/grpcurl.go b/cmd/grpcurl/grpcurl.go index 70d079bc..73e31f41 100644 --- a/cmd/grpcurl/grpcurl.go +++ b/cmd/grpcurl/grpcurl.go @@ -151,6 +151,14 @@ var ( file if this option is given. When invoking an RPC and this option is given, the method being invoked and its transitive dependencies will be included in the output file.`)) + protoOut = flags.String("proto-out", "", prettify(` + The name of a directory where the generated .proto files will be written. + With the list and describe verbs, the listed or described elements and + their transitive dependencies will be written as .proto files in the + specified directory if this option is given. When invoking an RPC and + this option is given, the method being invoked and its transitive + dependencies will be included in the generated .proto files in the + output directory.`)) msgTemplate = flags.Bool("msg-template", false, prettify(` When describing messages, show a template of input data.`)) verbose = flags.Bool("v", false, prettify(` @@ -645,6 +653,9 @@ func main() { if err := writeProtoset(descSource, svcs...); err != nil { fail(err, "Failed to write protoset to %s", *protosetOut) } + if err := writeProtos(descSource, svcs...); err != nil { + fail(err, "Failed to write protos to %s", *protoOut) + } } else { methods, err := grpcurl.ListMethods(descSource, symbol) if err != nil { @@ -660,6 +671,9 @@ func main() { if err := writeProtoset(descSource, symbol); err != nil { fail(err, "Failed to write protoset to %s", *protosetOut) } + if err := writeProtos(descSource, symbol); err != nil { + fail(err, "Failed to write protos to %s", *protoOut) + } } } else if describe { @@ -764,6 +778,9 @@ func main() { if err := writeProtoset(descSource, symbols...); err != nil { fail(err, "Failed to write protoset to %s", *protosetOut) } + if err := writeProtos(descSource, symbol); err != nil { + fail(err, "Failed to write protos to %s", *protoOut) + } } else { // Invoke an RPC @@ -923,6 +940,13 @@ func writeProtoset(descSource grpcurl.DescriptorSource, symbols ...string) error return grpcurl.WriteProtoset(f, descSource, symbols...) } +func writeProtos(descSource grpcurl.DescriptorSource, symbols ...string) error { + if *protoOut == "" { + return nil + } + return grpcurl.WriteProtoFiles(*protoOut, descSource, symbols...) +} + type optionalBoolFlag struct { set, val bool } diff --git a/desc_source.go b/desc_source.go index d12fae0d..89a7ccc8 100644 --- a/desc_source.go +++ b/desc_source.go @@ -4,8 +4,10 @@ import ( "context" "errors" "fmt" + "github.com/jhump/protoreflect/desc/protoprint" "io" "os" + "path/filepath" "sync" "github.com/golang/protobuf/proto" //lint:ignore SA1019 we have to import this because it appears in exported API @@ -258,19 +260,9 @@ func reflectionSupport(err error) error { // given output. The output will include descriptors for all files in which the // symbols are defined as well as their transitive dependencies. func WriteProtoset(out io.Writer, descSource DescriptorSource, symbols ...string) error { - // compute set of file descriptors - filenames := make([]string, 0, len(symbols)) - fds := make(map[string]*desc.FileDescriptor, len(symbols)) - for _, sym := range symbols { - d, err := descSource.FindSymbol(sym) - if err != nil { - return fmt.Errorf("failed to find descriptor for %q: %v", sym, err) - } - fd := d.GetFile() - if _, ok := fds[fd.GetName()]; !ok { - fds[fd.GetName()] = fd - filenames = append(filenames, fd.GetName()) - } + filenames, fds, err := getFileDescriptors(symbols, descSource) + if err != nil { + return err } // now expand that to include transitive dependencies in topologically sorted // order (such that file always appears after its dependencies) @@ -302,3 +294,71 @@ func addFilesToSet(allFiles []*descriptorpb.FileDescriptorProto, expanded map[st } return append(allFiles, fd.AsFileDescriptorProto()) } + +// WriteProtoFiles will use the given descriptor source to resolve all the given +// symbols and write proto files with their definitions to the given output directory. +func WriteProtoFiles(outProtoDirPath string, descSource DescriptorSource, symbols ...string) error { + filenames, fds, err := getFileDescriptors(symbols, descSource) + if err != nil { + return err + } + // now expand that to include transitive dependencies in topologically sorted + // order (such that file always appears after its dependencies) + expandedFiles := make(map[string]struct{}, len(fds)) + allFilesSlice := make([]*desc.FileDescriptor, 0, len(fds)) + for _, filename := range filenames { + allFilesSlice = addFilesToFileDescriptorList(allFilesSlice, expandedFiles, fds[filename]) + } + pr := protoprint.Printer{} + // now we can serialize to files + for _, fd := range allFilesSlice { + fdFQName := fd.GetFullyQualifiedName() + dirPath := filepath.Dir(fdFQName) + outFilepath := filepath.Join(outProtoDirPath, dirPath) + if err := os.MkdirAll(outFilepath, 0755); err != nil { + return fmt.Errorf("failed to create directory %q: %v", outFilepath, err) + } + fileName := filepath.Base(fdFQName) + filePath := filepath.Join(outFilepath, fileName) + f, err := os.Create(filePath) + defer f.Close() + if err != nil { + return fmt.Errorf("failed to create file %q: %v", filePath, err) + } + if err := pr.PrintProtoFile(fd, f); err != nil { + return fmt.Errorf("failed to write file %q: %v", filePath, err) + } + } + return nil +} + +func getFileDescriptors(symbols []string, descSource DescriptorSource) ([]string, map[string]*desc.FileDescriptor, error) { + // compute set of file descriptors + filenames := make([]string, 0, len(symbols)) + fds := make(map[string]*desc.FileDescriptor, len(symbols)) + for _, sym := range symbols { + d, err := descSource.FindSymbol(sym) + if err != nil { + return nil, nil, fmt.Errorf("failed to find descriptor for %q: %v", sym, err) + } + fd := d.GetFile() + if _, ok := fds[fd.GetName()]; !ok { + fds[fd.GetName()] = fd + filenames = append(filenames, fd.GetName()) + } + } + return filenames, fds, nil +} + +func addFilesToFileDescriptorList(allFiles []*desc.FileDescriptor, expanded map[string]struct{}, fd *desc.FileDescriptor) []*desc.FileDescriptor { + if _, ok := expanded[fd.GetName()]; ok { + // already seen this one + return allFiles + } + expanded[fd.GetName()] = struct{}{} + // add all dependencies first + for _, dep := range fd.GetDependencies() { + allFiles = addFilesToFileDescriptorList(allFiles, expanded, dep) + } + return append(allFiles, fd) +} From 23bde38f6e3d9b8b2214feccccc6ece1997534a1 Mon Sep 17 00:00:00 2001 From: Eitol Date: Wed, 10 Jul 2024 09:05:04 -0400 Subject: [PATCH 2/9] Refactor file creation error handling The code for file creation and error handling in desc_source.go has been refactored. Previously, the file closure operation was executed irrespective of whether the file was created successfully or not. Now, the file will only be closed if it was successfully created, improving error handling. --- desc_source.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/desc_source.go b/desc_source.go index 89a7ccc8..4f917144 100644 --- a/desc_source.go +++ b/desc_source.go @@ -321,10 +321,13 @@ func WriteProtoFiles(outProtoDirPath string, descSource DescriptorSource, symbol fileName := filepath.Base(fdFQName) filePath := filepath.Join(outFilepath, fileName) f, err := os.Create(filePath) - defer f.Close() if err != nil { + if f != nil { + _ = f.Close() + } return fmt.Errorf("failed to create file %q: %v", filePath, err) } + _ = f.Close() if err := pr.PrintProtoFile(fd, f); err != nil { return fmt.Errorf("failed to write file %q: %v", filePath, err) } From 142da3b2f58309c68f580160abf46c106a5c1c62 Mon Sep 17 00:00:00 2001 From: Eitol Date: Wed, 10 Jul 2024 10:14:42 -0400 Subject: [PATCH 3/9] Update 'proto-out' command to 'proto-out-dir' The command for exporting proto files and setting the output directory has been updated from 'proto-out' to 'proto-out-dir'. This change has been made both in the README and the grpcurl go file. This makes the command name more descriptive, accurately reflecting its functionality. --- README.md | 2 +- cmd/grpcurl/grpcurl.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e2d4b331..5e6c2f7a 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ grpcurl -protoset my-protos.bin list grpcurl -import-path ../protos -proto my-stuff.proto list # Export proto files -grpcurl -plaintext -proto-out "out_protos" "192.168.100.1:9200" describe Api.Service +grpcurl -plaintext -proto-out-dir "out_protos" "192.168.100.1:9200" describe Api.Service ``` diff --git a/cmd/grpcurl/grpcurl.go b/cmd/grpcurl/grpcurl.go index 73e31f41..80f87bfc 100644 --- a/cmd/grpcurl/grpcurl.go +++ b/cmd/grpcurl/grpcurl.go @@ -151,7 +151,7 @@ var ( file if this option is given. When invoking an RPC and this option is given, the method being invoked and its transitive dependencies will be included in the output file.`)) - protoOut = flags.String("proto-out", "", prettify(` + protoOut = flags.String("proto-out-dir", "", prettify(` The name of a directory where the generated .proto files will be written. With the list and describe verbs, the listed or described elements and their transitive dependencies will be written as .proto files in the From 2cc97815223c90bd8de7e1e4a3ac7b9655da7a5c Mon Sep 17 00:00:00 2001 From: Eitol Date: Wed, 10 Jul 2024 11:05:03 -0400 Subject: [PATCH 4/9] fix import sort --- desc_source.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/desc_source.go b/desc_source.go index 4f917144..617839e6 100644 --- a/desc_source.go +++ b/desc_source.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/jhump/protoreflect/desc/protoprint" "io" "os" "path/filepath" @@ -13,6 +12,7 @@ import ( "github.com/golang/protobuf/proto" //lint:ignore SA1019 we have to import this because it appears in exported API "github.com/jhump/protoreflect/desc" "github.com/jhump/protoreflect/desc/protoparse" + "github.com/jhump/protoreflect/desc/protoprint" "github.com/jhump/protoreflect/dynamic" "github.com/jhump/protoreflect/grpcreflect" "google.golang.org/grpc/codes" From 414ffa3a2c19d155b07c04412188f8c7e347eac5 Mon Sep 17 00:00:00 2001 From: Eitol Date: Wed, 10 Jul 2024 11:11:34 -0400 Subject: [PATCH 5/9] Reorder file close operation in error handling The file close operation has been moved within the error handling of the 'PrintProtoFile' function. Previously, it was being executed before this function, now it's executed immediately after. Moreover, an additional close operation has been added after the function success ensuring the file is properly closed in all scenarios. --- desc_source.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/desc_source.go b/desc_source.go index 617839e6..fd40b27f 100644 --- a/desc_source.go +++ b/desc_source.go @@ -327,10 +327,11 @@ func WriteProtoFiles(outProtoDirPath string, descSource DescriptorSource, symbol } return fmt.Errorf("failed to create file %q: %v", filePath, err) } - _ = f.Close() if err := pr.PrintProtoFile(fd, f); err != nil { + _ = f.Close() return fmt.Errorf("failed to write file %q: %v", filePath, err) } + _ = f.Close() } return nil } From 9706903f6e090aeccb45fdb9748fa672dd75eca1 Mon Sep 17 00:00:00 2001 From: Eitol Date: Wed, 10 Jul 2024 11:15:35 -0400 Subject: [PATCH 6/9] Update grpcurl commands in README The grpcurl commands for exporting proto files and protoset files in the README are updated. The IP address has been changed to localhost and port number to '8787'. Also, the service name is adjusted to 'my.custom.server.Service'. Instructions for use of specific command options are added for enhanced clarity. --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5e6c2f7a..85e923b0 100644 --- a/README.md +++ b/README.md @@ -160,9 +160,11 @@ grpcurl -protoset my-protos.bin list # Using proto sources grpcurl -import-path ../protos -proto my-stuff.proto list -# Export proto files -grpcurl -plaintext -proto-out-dir "out_protos" "192.168.100.1:9200" describe Api.Service +# Export proto files (use -proto-out-dir to specify the output directory) +grpcurl -plaintext -proto-out-dir "out_protos" "localhost:8787" describe my.custom.server.Service +# Export protoset file (use -protoset-out to specify the output file) +grpcurl -plaintext -protoset-out "out.protoset" "localhost:8787" describe my.custom.server.Service ``` From 21c947e1073c9f93649eac7566824688fc070f14 Mon Sep 17 00:00:00 2001 From: Eitol Date: Wed, 10 Jul 2024 11:40:46 -0400 Subject: [PATCH 7/9] Refactor error handling in file creation The code responsible for error handling during file creation in the desc_source.go file has been streamlined. This modification simplifies the code by reducing unnecessary condition checks and redundant file closure action after an error has occurred. --- desc_source.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/desc_source.go b/desc_source.go index fd40b27f..b9c6a624 100644 --- a/desc_source.go +++ b/desc_source.go @@ -322,16 +322,14 @@ func WriteProtoFiles(outProtoDirPath string, descSource DescriptorSource, symbol filePath := filepath.Join(outFilepath, fileName) f, err := os.Create(filePath) if err != nil { - if f != nil { - _ = f.Close() - } return fmt.Errorf("failed to create file %q: %v", filePath, err) } - if err := pr.PrintProtoFile(fd, f); err != nil { + err = pr.PrintProtoFile(fd, f) + if err == nil { _ = f.Close() + } else { return fmt.Errorf("failed to write file %q: %v", filePath, err) } - _ = f.Close() } return nil } From 0bb9027957a40785d7ba8bf0eb3f74b828d7f653 Mon Sep 17 00:00:00 2001 From: Eitol Date: Wed, 10 Jul 2024 23:30:04 -0400 Subject: [PATCH 8/9] Refactor proto file writing into separate function The file writing process for protobuf files has been extracted into a new function called writeProtoFile(). This refactoring simplifies the main function. The code is cleaner and more manageable this way, improving maintainability and readability. --- desc_source.go | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/desc_source.go b/desc_source.go index b9c6a624..9c55fd91 100644 --- a/desc_source.go +++ b/desc_source.go @@ -305,35 +305,41 @@ func WriteProtoFiles(outProtoDirPath string, descSource DescriptorSource, symbol // now expand that to include transitive dependencies in topologically sorted // order (such that file always appears after its dependencies) expandedFiles := make(map[string]struct{}, len(fds)) - allFilesSlice := make([]*desc.FileDescriptor, 0, len(fds)) + allFileDescriptors := make([]*desc.FileDescriptor, 0, len(fds)) for _, filename := range filenames { - allFilesSlice = addFilesToFileDescriptorList(allFilesSlice, expandedFiles, fds[filename]) + allFileDescriptors = addFilesToFileDescriptorList(allFileDescriptors, expandedFiles, fds[filename]) } pr := protoprint.Printer{} // now we can serialize to files - for _, fd := range allFilesSlice { - fdFQName := fd.GetFullyQualifiedName() - dirPath := filepath.Dir(fdFQName) - outFilepath := filepath.Join(outProtoDirPath, dirPath) - if err := os.MkdirAll(outFilepath, 0755); err != nil { - return fmt.Errorf("failed to create directory %q: %v", outFilepath, err) - } - fileName := filepath.Base(fdFQName) - filePath := filepath.Join(outFilepath, fileName) - f, err := os.Create(filePath) - if err != nil { - return fmt.Errorf("failed to create file %q: %v", filePath, err) - } - err = pr.PrintProtoFile(fd, f) - if err == nil { - _ = f.Close() - } else { - return fmt.Errorf("failed to write file %q: %v", filePath, err) + for i := range allFileDescriptors { + if err := writeProtoFile(outProtoDirPath, allFileDescriptors[i], &pr); err != nil { + return err } } return nil } +func writeProtoFile(outProtoDirPath string, fd *desc.FileDescriptor, pr *protoprint.Printer) error { + fdFQName := fd.GetFullyQualifiedName() + dirPath := filepath.Dir(fdFQName) + outFilepath := filepath.Join(outProtoDirPath, dirPath) + if err := os.MkdirAll(outFilepath, 0755); err != nil { + return fmt.Errorf("failed to create directory %q: %v", outFilepath, err) + } + fileName := filepath.Base(fdFQName) + filePath := filepath.Join(outFilepath, fileName) + + f, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("failed to create proto file: %v", err) + } + defer f.Close() + if err := pr.PrintProtoFile(fd, f); err != nil { + return fmt.Errorf("failed to write proto file: %v", err) + } + return nil +} + func getFileDescriptors(symbols []string, descSource DescriptorSource) ([]string, map[string]*desc.FileDescriptor, error) { // compute set of file descriptors filenames := make([]string, 0, len(symbols)) From 9901b7affc5f4817b100b6ed1587b19d37b60734 Mon Sep 17 00:00:00 2001 From: Eitol Date: Fri, 12 Jul 2024 01:00:54 -0400 Subject: [PATCH 9/9] Refactor writeProtoFile function for better error handling Streamlined the writeProtoFile function in desc_source.go file. Simplified path calculations and improved error messages for file-creation functions, making it easier to trace the exact point of failure and enhance the debugging process. --- desc_source.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/desc_source.go b/desc_source.go index 9c55fd91..87f1403b 100644 --- a/desc_source.go +++ b/desc_source.go @@ -320,22 +320,19 @@ func WriteProtoFiles(outProtoDirPath string, descSource DescriptorSource, symbol } func writeProtoFile(outProtoDirPath string, fd *desc.FileDescriptor, pr *protoprint.Printer) error { - fdFQName := fd.GetFullyQualifiedName() - dirPath := filepath.Dir(fdFQName) - outFilepath := filepath.Join(outProtoDirPath, dirPath) - if err := os.MkdirAll(outFilepath, 0755); err != nil { - return fmt.Errorf("failed to create directory %q: %v", outFilepath, err) - } - fileName := filepath.Base(fdFQName) - filePath := filepath.Join(outFilepath, fileName) + outFile := filepath.Join(outProtoDirPath, fd.GetFullyQualifiedName()) + outDir := filepath.Dir(outFile) + if err := os.MkdirAll(outDir, 0777); err != nil { + return fmt.Errorf("failed to create directory %q: %w", outDir, err) + } - f, err := os.Create(filePath) + f, err := os.Create(outFile) if err != nil { - return fmt.Errorf("failed to create proto file: %v", err) + return fmt.Errorf("failed to create proto file %q: %w", outFile, err) } defer f.Close() if err := pr.PrintProtoFile(fd, f); err != nil { - return fmt.Errorf("failed to write proto file: %v", err) + return fmt.Errorf("failed to write proto file %q: %w", outFile, err) } return nil }