From 58a6494d5d5e175690bd7eea6690aa34127bcfac Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Tue, 14 Sep 2021 15:24:56 +0200 Subject: [PATCH 01/24] init --- .../process/extension/stdio_executable.go | 69 +++++++ .../process/extension/uds_executable.go | 106 ++++++++++ pkg/transport/process/pipeline.go | 82 ++++++++ pkg/transport/process/types.go | 16 ++ pkg/transport/process/util.go | 181 ++++++++++++++++++ 5 files changed, 454 insertions(+) create mode 100644 pkg/transport/process/extension/stdio_executable.go create mode 100644 pkg/transport/process/extension/uds_executable.go create mode 100644 pkg/transport/process/pipeline.go create mode 100644 pkg/transport/process/types.go create mode 100644 pkg/transport/process/util.go diff --git a/pkg/transport/process/extension/stdio_executable.go b/pkg/transport/process/extension/stdio_executable.go new file mode 100644 index 00000000..9f7e956c --- /dev/null +++ b/pkg/transport/process/extension/stdio_executable.go @@ -0,0 +1,69 @@ +package extension + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + + "github.com/gardener/component-cli/pkg/transport/process" +) + +type stdIOExecutable struct { + processor *exec.Cmd + stdin io.WriteCloser + stdout io.Reader +} + +// NewStdIOExecutable runs resource processor in the background. +// It communicates with this processor via stdin/stdout pipes. +func NewStdIOExecutable(ctx context.Context, bin string, args ...string) (process.ResourceStreamProcessor, error) { + cmd := exec.CommandContext(ctx, bin, args...) + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + cmd.Stderr = os.Stderr + + err = cmd.Start() + if err != nil { + return nil, fmt.Errorf("unable to start processor: %w", err) + } + + e := stdIOExecutable{ + processor: cmd, + stdin: stdin, + stdout: stdout, + } + + return &e, nil +} + +func (e *stdIOExecutable) Process(ctx context.Context, r io.Reader, w io.Writer) error { + _, err := io.Copy(e.stdin, r) + if err != nil { + return fmt.Errorf("unable to write input: %w", err) + } + + err = e.stdin.Close() + if err != nil { + return fmt.Errorf("unable to close input writer: %w", err) + } + + _, err = io.Copy(w, e.stdout) + if err != nil { + return fmt.Errorf("unable to read output: %w", err) + } + + err = e.processor.Wait() + if err != nil { + return fmt.Errorf("unable to stop processor: %w", err) + } + + return nil +} diff --git a/pkg/transport/process/extension/uds_executable.go b/pkg/transport/process/extension/uds_executable.go new file mode 100644 index 00000000..fa4c7cf9 --- /dev/null +++ b/pkg/transport/process/extension/uds_executable.go @@ -0,0 +1,106 @@ +package extension + +import ( + "context" + "fmt" + "io" + "net" + "os" + "os/exec" + "time" + + "github.com/gardener/component-cli/pkg/transport/process" + "github.com/gardener/component-cli/pkg/utils" +) + +const serverAddressFlag = "--addr" + +type udsExecutable struct { + processor *exec.Cmd + addr string + conn net.Conn +} + +// NewUDSExecutable runs a resource processor in the background. +// It communicates with this processor via Unix Domain Sockets. +func NewUDSExecutable(ctx context.Context, bin string, args ...string) (process.ResourceStreamProcessor, error) { + for _, arg := range args { + if arg == serverAddressFlag { + return nil, fmt.Errorf("the flag %s is not allowed to be set manually", serverAddressFlag) + } + } + + wd, err := os.Getwd() + if err != nil { + return nil, err + } + addr := fmt.Sprintf("%s/%s.sock", wd, utils.RandomString(8)) + args = append(args, "--addr", addr) + + cmd := exec.CommandContext(ctx, bin, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err = cmd.Start() + if err != nil { + return nil, fmt.Errorf("unable to start processor: %w", err) + } + + conn, err := tryConnect(addr) + if err != nil { + return nil, fmt.Errorf("unable to connect to processor: %w", err) + } + + e := udsExecutable{ + processor: cmd, + addr: addr, + conn: conn, + } + + return &e, nil +} + +func (e *udsExecutable) Process(ctx context.Context, r io.Reader, w io.Writer) error { + _, err := io.Copy(e.conn, r) + if err != nil { + return fmt.Errorf("unable to write input: %w", err) + } + + usock := e.conn.(*net.UnixConn) + err = usock.CloseWrite() + if err != nil { + return fmt.Errorf("unable to close input writer: %w", err) + } + + _, err = io.Copy(w, e.conn) + if err != nil { + return fmt.Errorf("unable to read output: %w", err) + } + + err = e.processor.Wait() + if err != nil { + return fmt.Errorf("unable to stop processor: %w", err) + } + + return nil +} + +func tryConnect(addr string) (net.Conn, error) { + const ( + maxRetries = 5 + sleeptime = 500 * time.Millisecond + ) + + var conn net.Conn + var err error + for i := 0; i <= maxRetries; i++ { + conn, err = net.Dial("unix", addr) + if err == nil { + break + } + + time.Sleep(sleeptime) + } + + return conn, err +} diff --git a/pkg/transport/process/pipeline.go b/pkg/transport/process/pipeline.go new file mode 100644 index 00000000..ad5b6864 --- /dev/null +++ b/pkg/transport/process/pipeline.go @@ -0,0 +1,82 @@ +package process + +import ( + "context" + "os" + + "archive/tar" + "fmt" + "io/ioutil" + + cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" +) + +type resourceProcessingPipelineImpl struct { + processors []ResourceStreamProcessor +} + +func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.ComponentDescriptor, res cdv2.Resource) (*cdv2.ComponentDescriptor, cdv2.Resource, error) { + infile, err := ioutil.TempFile("", "out") + if err != nil { + return nil, cdv2.Resource{}, fmt.Errorf("unable to create temporary infile: %w", err) + } + + err = WriteArchive(ctx, &cd, res, nil, tar.NewWriter(infile)) + if err != nil { + return nil, cdv2.Resource{}, fmt.Errorf("unable to write: %w", err) + } + + for _, proc := range p.processors { + outfile, err := p.process(ctx, infile, proc) + if err != nil { + return nil, cdv2.Resource{}, err + } + + infile = outfile + } + defer infile.Close() + + _, err = infile.Seek(0, 0) + if err != nil { + return nil, cdv2.Resource{}, err + } + + processedCD, processedRes, blobreader, err := ReadArchive(tar.NewReader(infile)) + if err != nil { + return nil, cdv2.Resource{}, fmt.Errorf("unable to read output data: %w", err) + } + defer blobreader.Close() + + return processedCD, processedRes, nil +} + +func (p *resourceProcessingPipelineImpl) process(ctx context.Context, infile *os.File, proc ResourceStreamProcessor) (*os.File, error) { + defer infile.Close() + + _, err := infile.Seek(0, 0) + if err != nil { + return nil, fmt.Errorf("unable to seek to beginning of input file: %w", err) + } + + outfile, err := ioutil.TempFile("", "out") + if err != nil { + return nil, fmt.Errorf("unable to create temporary outfile: %w", err) + } + + inreader := infile + outwriter := outfile + + err = proc.Process(ctx, inreader, outwriter) + if err != nil { + return nil, fmt.Errorf("unable to process resource: %w", err) + } + + return outfile, nil +} + +func NewResourceProcessingPipeline(procs ...ResourceStreamProcessor) (ResourceProcessingPipeline, error) { + p := resourceProcessingPipelineImpl{ + processors: procs, + } + return &p, nil +} diff --git a/pkg/transport/process/types.go b/pkg/transport/process/types.go new file mode 100644 index 00000000..4faa38b9 --- /dev/null +++ b/pkg/transport/process/types.go @@ -0,0 +1,16 @@ +package process + +import ( + "context" + "io" + + cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" +) + +type ResourceProcessingPipeline interface { + Process(context.Context, cdv2.ComponentDescriptor, cdv2.Resource) (*cdv2.ComponentDescriptor, cdv2.Resource, error) +} + +type ResourceStreamProcessor interface { + Process(context.Context, io.Reader, io.Writer) error +} diff --git a/pkg/transport/process/util.go b/pkg/transport/process/util.go new file mode 100644 index 00000000..e1984c40 --- /dev/null +++ b/pkg/transport/process/util.go @@ -0,0 +1,181 @@ +package process + +import ( + "archive/tar" + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "os" + "time" + + cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" + "sigs.k8s.io/yaml" +) + +const ( + ResourceFile = "resource.yaml" + ComponentDescriptorFile = "component-descriptor.yaml" + BlobFile = "blob" +) + +func WriteFile(name string, contentReader io.Reader, outArchive *tar.Writer) error { + tmpfile, err := ioutil.TempFile("", "tmp") + if err != nil { + return fmt.Errorf("unable to create tempfile: %w", err) + } + defer tmpfile.Close() + + _, err = io.Copy(tmpfile, contentReader) + if err != nil { + return fmt.Errorf("unable to write content to tempfile: %w", err) + } + + _, err = tmpfile.Seek(0, 0) + if err != nil { + return fmt.Errorf("unable to seek to beginning of tempfile: %w", err) + } + + fstat, err := tmpfile.Stat() + if err != nil { + return fmt.Errorf("unable to get file stats: %w", err) + } + + header := tar.Header{ + Name: name, + Size: fstat.Size(), + Mode: int64(fstat.Mode()), + ModTime: time.Now(), + } + + if err = outArchive.WriteHeader(&header); err != nil { + return fmt.Errorf("unable to write tar header: %w", err) + } + + _, err = io.Copy(outArchive, tmpfile) + if err != nil { + return fmt.Errorf("unable to write file to archive: %w", err) + } + + return nil +} + +func WriteArchive(ctx context.Context, cd *cdv2.ComponentDescriptor, res cdv2.Resource, blobReader io.Reader, outwriter *tar.Writer) error { + defer outwriter.Close() + + println("start writing data") + + marshaledCD, err := yaml.Marshal(cd) + if err != nil { + return fmt.Errorf("unable to marshal component descriptor: %w", err) + } + + println("writing component descriptor") + err = WriteFile(ComponentDescriptorFile, bytes.NewReader(marshaledCD), outwriter) + if err != nil { + return fmt.Errorf("unable to write component descriptor: %w", err) + } + + marshaledRes, err := yaml.Marshal(res) + if err != nil { + return fmt.Errorf("unable to marshal resource: %w", err) + } + + println("writing resource") + err = WriteFile(ResourceFile, bytes.NewReader(marshaledRes), outwriter) + if err != nil { + return fmt.Errorf("unable to write resource: %w", err) + } + + if blobReader != nil { + println("writing blob") + err = WriteFile(BlobFile, blobReader, outwriter) + if err != nil { + return fmt.Errorf("unable to write blob: %w", err) + } + } + + println("finished writing data") + + return nil +} + +func ReadArchive(r *tar.Reader) (*cdv2.ComponentDescriptor, cdv2.Resource, io.ReadCloser, error) { + var cd *cdv2.ComponentDescriptor + var res cdv2.Resource + var blobFile *os.File + + for { + header, err := r.Next() + if err != nil { + if err == io.EOF { + break + } + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read header: %w", err) + } + + switch header.Name { + case ResourceFile: + res, err = ParseResource(r) + if err != nil { + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", ResourceFile, err) + } + case ComponentDescriptorFile: + cd, err = ParseComponentDescriptor(r) + if err != nil { + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", ComponentDescriptorFile, err) + } + case BlobFile: + blobFile, err = ioutil.TempFile("", "") + if err != nil { + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to create tempfile: %w", err) + } + _, err = io.Copy(blobFile, r) + if err != nil { + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", BlobFile, err) + } + } + } + + if blobFile != nil { + _, err := blobFile.Seek(0, 0) + if err != nil { + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to seek to beginning of blobfile: %w", err) + } + } + + return cd, res, blobFile, nil +} + +func ParseResource(r *tar.Reader) (cdv2.Resource, error) { + buf := bytes.NewBuffer([]byte{}) + _, err := io.Copy(buf, r) + if err != nil { + return cdv2.Resource{}, fmt.Errorf("unable to read from stream: %w", err) + } + + var res cdv2.Resource + err = yaml.Unmarshal(buf.Bytes(), &res) + if err != nil { + return cdv2.Resource{}, fmt.Errorf("unable to unmarshal: %w", err) + } + + return res, nil +} + +func ParseComponentDescriptor(r *tar.Reader) (*cdv2.ComponentDescriptor, error) { + buf := bytes.NewBuffer([]byte{}) + _, err := io.Copy(buf, r) + if err != nil { + return nil, fmt.Errorf("unable to read from stream: %w", err) + } + + var cd cdv2.ComponentDescriptor + err = yaml.Unmarshal(buf.Bytes(), &cd) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal: %w", err) + } + + return &cd, nil +} From 2813a7d6fdbbf706e25b992028c49ff57a28325b Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Tue, 14 Sep 2021 17:02:44 +0200 Subject: [PATCH 02/24] refactoring --- .../process/extension/stdio_executable.go | 2 +- .../process/extension/uds_executable.go | 3 +- pkg/transport/process/pipeline.go | 11 +- pkg/transport/process/types.go | 10 ++ pkg/transport/process/util.go | 132 +++++++++--------- 5 files changed, 83 insertions(+), 75 deletions(-) diff --git a/pkg/transport/process/extension/stdio_executable.go b/pkg/transport/process/extension/stdio_executable.go index 9f7e956c..5fafcc8b 100644 --- a/pkg/transport/process/extension/stdio_executable.go +++ b/pkg/transport/process/extension/stdio_executable.go @@ -16,7 +16,7 @@ type stdIOExecutable struct { stdout io.Reader } -// NewStdIOExecutable runs resource processor in the background. +// NewStdIOExecutable runs resource processor extension executable in the background. // It communicates with this processor via stdin/stdout pipes. func NewStdIOExecutable(ctx context.Context, bin string, args ...string) (process.ResourceStreamProcessor, error) { cmd := exec.CommandContext(ctx, bin, args...) diff --git a/pkg/transport/process/extension/uds_executable.go b/pkg/transport/process/extension/uds_executable.go index fa4c7cf9..b5e4d2c4 100644 --- a/pkg/transport/process/extension/uds_executable.go +++ b/pkg/transport/process/extension/uds_executable.go @@ -21,7 +21,7 @@ type udsExecutable struct { conn net.Conn } -// NewUDSExecutable runs a resource processor in the background. +// NewUDSExecutable runs a resource processor extension executable in the background. // It communicates with this processor via Unix Domain Sockets. func NewUDSExecutable(ctx context.Context, bin string, args ...string) (process.ResourceStreamProcessor, error) { for _, arg := range args { @@ -77,6 +77,7 @@ func (e *udsExecutable) Process(ctx context.Context, r io.Reader, w io.Writer) e return fmt.Errorf("unable to read output: %w", err) } + // extension servers must implement ordinary shutdown (!) err = e.processor.Wait() if err != nil { return fmt.Errorf("unable to stop processor: %w", err) diff --git a/pkg/transport/process/pipeline.go b/pkg/transport/process/pipeline.go index ad5b6864..98de0227 100644 --- a/pkg/transport/process/pipeline.go +++ b/pkg/transport/process/pipeline.go @@ -21,7 +21,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co return nil, cdv2.Resource{}, fmt.Errorf("unable to create temporary infile: %w", err) } - err = WriteArchive(ctx, &cd, res, nil, tar.NewWriter(infile)) + err = WriteTARArchive(ctx, cd, res, nil, tar.NewWriter(infile)) if err != nil { return nil, cdv2.Resource{}, fmt.Errorf("unable to write: %w", err) } @@ -41,7 +41,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co return nil, cdv2.Resource{}, err } - processedCD, processedRes, blobreader, err := ReadArchive(tar.NewReader(infile)) + processedCD, processedRes, blobreader, err := ReadTARArchive(tar.NewReader(infile)) if err != nil { return nil, cdv2.Resource{}, fmt.Errorf("unable to read output data: %w", err) } @@ -74,9 +74,10 @@ func (p *resourceProcessingPipelineImpl) process(ctx context.Context, infile *os return outfile, nil } -func NewResourceProcessingPipeline(procs ...ResourceStreamProcessor) (ResourceProcessingPipeline, error) { +// NewResourceProcessingPipeline returns a new ResourceProcessingPipeline +func NewResourceProcessingPipeline(processors ...ResourceStreamProcessor) ResourceProcessingPipeline { p := resourceProcessingPipelineImpl{ - processors: procs, + processors: processors, } - return &p, nil + return &p } diff --git a/pkg/transport/process/types.go b/pkg/transport/process/types.go index 4faa38b9..d0b1a998 100644 --- a/pkg/transport/process/types.go +++ b/pkg/transport/process/types.go @@ -7,10 +7,20 @@ import ( cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" ) +// ResourceProcessingPipeline describes a chain of multiple processors for processing a resource. +// Each processor receives its input from the preceding processor and writes the output for the +// subsequent processor. To work correctly, a pipeline must consist of 1 downloader, 0..n processors, +// and 1..n uploaders. type ResourceProcessingPipeline interface { + // Process executes all processors for a resource. + // Returns the component descriptor and resource of the last processor. Process(context.Context, cdv2.ComponentDescriptor, cdv2.Resource) (*cdv2.ComponentDescriptor, cdv2.Resource, error) } +// ResourceStreamProcessor describes an individual processor for processing a resource. +// A processor can upload, modify, or download a resource. type ResourceStreamProcessor interface { + // Process executes the processor for a resource. Input and Output streams must be TAR + // archives which contain the component descriptor, resource, and resource blob. Process(context.Context, io.Reader, io.Writer) error } diff --git a/pkg/transport/process/util.go b/pkg/transport/process/util.go index e1984c40..a07f3e63 100644 --- a/pkg/transport/process/util.go +++ b/pkg/transport/process/util.go @@ -15,96 +15,92 @@ import ( ) const ( - ResourceFile = "resource.yaml" - ComponentDescriptorFile = "component-descriptor.yaml" - BlobFile = "blob" + componentDescriptorFile = "component-descriptor.yaml" + resourceFile = "resource.yaml" + resourceBlobFile = "resource-blob" ) -func WriteFile(name string, contentReader io.Reader, outArchive *tar.Writer) error { - tmpfile, err := ioutil.TempFile("", "tmp") - if err != nil { - return fmt.Errorf("unable to create tempfile: %w", err) - } - defer tmpfile.Close() +// WriteTARArchive writes the component descriptor, resource and resource blob to a TAR archive +func WriteTARArchive(ctx context.Context, cd cdv2.ComponentDescriptor, res cdv2.Resource, resourceBlobReader io.Reader, outArchive *tar.Writer) error { + defer outArchive.Close() - _, err = io.Copy(tmpfile, contentReader) + marshaledCD, err := yaml.Marshal(cd) if err != nil { - return fmt.Errorf("unable to write content to tempfile: %w", err) + return fmt.Errorf("unable to marshal component descriptor: %w", err) } - _, err = tmpfile.Seek(0, 0) + err = writeFileToTARArchive(componentDescriptorFile, bytes.NewReader(marshaledCD), outArchive) if err != nil { - return fmt.Errorf("unable to seek to beginning of tempfile: %w", err) + return fmt.Errorf("unable to write %s: %w", componentDescriptorFile, err) } - fstat, err := tmpfile.Stat() + marshaledRes, err := yaml.Marshal(res) if err != nil { - return fmt.Errorf("unable to get file stats: %w", err) - } - - header := tar.Header{ - Name: name, - Size: fstat.Size(), - Mode: int64(fstat.Mode()), - ModTime: time.Now(), + return fmt.Errorf("unable to marshal resource: %w", err) } - if err = outArchive.WriteHeader(&header); err != nil { - return fmt.Errorf("unable to write tar header: %w", err) + err = writeFileToTARArchive(resourceFile, bytes.NewReader(marshaledRes), outArchive) + if err != nil { + return fmt.Errorf("unable to write %s: %w", resourceFile, err) } - _, err = io.Copy(outArchive, tmpfile) - if err != nil { - return fmt.Errorf("unable to write file to archive: %w", err) + if resourceBlobReader != nil { + err = writeFileToTARArchive(resourceBlobFile, resourceBlobReader, outArchive) + if err != nil { + return fmt.Errorf("unable to write %s: %w", resourceBlobFile, err) + } } return nil } -func WriteArchive(ctx context.Context, cd *cdv2.ComponentDescriptor, res cdv2.Resource, blobReader io.Reader, outwriter *tar.Writer) error { - defer outwriter.Close() - - println("start writing data") - - marshaledCD, err := yaml.Marshal(cd) +func writeFileToTARArchive(filename string, contentReader io.Reader, outArchive *tar.Writer) error { + tempfile, err := ioutil.TempFile("", "") if err != nil { - return fmt.Errorf("unable to marshal component descriptor: %w", err) + return fmt.Errorf("unable to create tempfile: %w", err) } + defer tempfile.Close() - println("writing component descriptor") - err = WriteFile(ComponentDescriptorFile, bytes.NewReader(marshaledCD), outwriter) + _, err = io.Copy(tempfile, contentReader) if err != nil { - return fmt.Errorf("unable to write component descriptor: %w", err) + return fmt.Errorf("unable to write content to file: %w", err) } - marshaledRes, err := yaml.Marshal(res) + _, err = tempfile.Seek(0, 0) if err != nil { - return fmt.Errorf("unable to marshal resource: %w", err) + return fmt.Errorf("unable to seek to beginning of file: %w", err) } - println("writing resource") - err = WriteFile(ResourceFile, bytes.NewReader(marshaledRes), outwriter) + fstat, err := tempfile.Stat() if err != nil { - return fmt.Errorf("unable to write resource: %w", err) + return fmt.Errorf("unable to get file info: %w", err) } - if blobReader != nil { - println("writing blob") - err = WriteFile(BlobFile, blobReader, outwriter) - if err != nil { - return fmt.Errorf("unable to write blob: %w", err) - } + header := tar.Header{ + Name: filename, + Size: fstat.Size(), + Mode: int64(fstat.Mode()), + ModTime: time.Now(), } - println("finished writing data") + if err = outArchive.WriteHeader(&header); err != nil { + return fmt.Errorf("unable to write tar header: %w", err) + } + + _, err = io.Copy(outArchive, tempfile) + if err != nil { + return fmt.Errorf("unable to write file to tar archive: %w", err) + } return nil } -func ReadArchive(r *tar.Reader) (*cdv2.ComponentDescriptor, cdv2.Resource, io.ReadCloser, error) { +// ReadTARArchive reads the component descriptor, resource and resource blob from a TAR archive. +// The resource blob reader can be nil. If a non-nil value is returned, it must be closed by the caller. +func ReadTARArchive(r *tar.Reader) (*cdv2.ComponentDescriptor, cdv2.Resource, io.ReadCloser, error) { var cd *cdv2.ComponentDescriptor var res cdv2.Resource - var blobFile *os.File + var f *os.File for { header, err := r.Next() @@ -112,43 +108,43 @@ func ReadArchive(r *tar.Reader) (*cdv2.ComponentDescriptor, cdv2.Resource, io.Re if err == io.EOF { break } - return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read header: %w", err) + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read tar header: %w", err) } switch header.Name { - case ResourceFile: - res, err = ParseResource(r) + case resourceFile: + res, err = readResource(r) if err != nil { - return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", ResourceFile, err) + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", resourceFile, err) } - case ComponentDescriptorFile: - cd, err = ParseComponentDescriptor(r) + case componentDescriptorFile: + cd, err = readComponentDescriptor(r) if err != nil { - return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", ComponentDescriptorFile, err) + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", componentDescriptorFile, err) } - case BlobFile: - blobFile, err = ioutil.TempFile("", "") + case resourceBlobFile: + f, err = ioutil.TempFile("", "") if err != nil { return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to create tempfile: %w", err) } - _, err = io.Copy(blobFile, r) + _, err = io.Copy(f, r) if err != nil { - return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", BlobFile, err) + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", resourceBlobFile, err) } } } - if blobFile != nil { - _, err := blobFile.Seek(0, 0) + if f != nil { + _, err := f.Seek(0, 0) if err != nil { - return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to seek to beginning of blobfile: %w", err) + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to seek to beginning of file: %w", err) } } - return cd, res, blobFile, nil + return cd, res, f, nil } -func ParseResource(r *tar.Reader) (cdv2.Resource, error) { +func readResource(r *tar.Reader) (cdv2.Resource, error) { buf := bytes.NewBuffer([]byte{}) _, err := io.Copy(buf, r) if err != nil { @@ -164,7 +160,7 @@ func ParseResource(r *tar.Reader) (cdv2.Resource, error) { return res, nil } -func ParseComponentDescriptor(r *tar.Reader) (*cdv2.ComponentDescriptor, error) { +func readComponentDescriptor(r *tar.Reader) (*cdv2.ComponentDescriptor, error) { buf := bytes.NewBuffer([]byte{}) _, err := io.Copy(buf, r) if err != nil { From a5bdb52b4c14bb8b611085f19e2b9fede8074de8 Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Thu, 16 Sep 2021 09:58:06 +0200 Subject: [PATCH 03/24] review feedback --- .../process/extension/stdio_executable.go | 26 +++++----- .../process/extension/uds_executable.go | 6 +-- pkg/transport/process/pipeline.go | 15 +++--- pkg/transport/process/util.go | 47 +++++++------------ 4 files changed, 36 insertions(+), 58 deletions(-) diff --git a/pkg/transport/process/extension/stdio_executable.go b/pkg/transport/process/extension/stdio_executable.go index 5fafcc8b..f487e91c 100644 --- a/pkg/transport/process/extension/stdio_executable.go +++ b/pkg/transport/process/extension/stdio_executable.go @@ -16,10 +16,11 @@ type stdIOExecutable struct { stdout io.Reader } -// NewStdIOExecutable runs resource processor extension executable in the background. +// NewStdIOExecutable runs a resource processor extension executable in the background. // It communicates with this processor via stdin/stdout pipes. -func NewStdIOExecutable(ctx context.Context, bin string, args ...string) (process.ResourceStreamProcessor, error) { +func NewStdIOExecutable(ctx context.Context, bin string, args []string, env []string) (process.ResourceStreamProcessor, error) { cmd := exec.CommandContext(ctx, bin, args...) + cmd.Env = env stdin, err := cmd.StdinPipe() if err != nil { return nil, err @@ -30,11 +31,6 @@ func NewStdIOExecutable(ctx context.Context, bin string, args ...string) (proces } cmd.Stderr = os.Stderr - err = cmd.Start() - if err != nil { - return nil, fmt.Errorf("unable to start processor: %w", err) - } - e := stdIOExecutable{ processor: cmd, stdin: stdin, @@ -45,23 +41,23 @@ func NewStdIOExecutable(ctx context.Context, bin string, args ...string) (proces } func (e *stdIOExecutable) Process(ctx context.Context, r io.Reader, w io.Writer) error { - _, err := io.Copy(e.stdin, r) - if err != nil { + if err := e.processor.Start(); err != nil { + return fmt.Errorf("unable to start processor: %w", err) + } + + if _, err := io.Copy(e.stdin, r); err != nil { return fmt.Errorf("unable to write input: %w", err) } - err = e.stdin.Close() - if err != nil { + if err := e.stdin.Close(); err != nil { return fmt.Errorf("unable to close input writer: %w", err) } - _, err = io.Copy(w, e.stdout) - if err != nil { + if _, err := io.Copy(w, e.stdout); err != nil { return fmt.Errorf("unable to read output: %w", err) } - err = e.processor.Wait() - if err != nil { + if err := e.processor.Wait(); err != nil { return fmt.Errorf("unable to stop processor: %w", err) } diff --git a/pkg/transport/process/extension/uds_executable.go b/pkg/transport/process/extension/uds_executable.go index b5e4d2c4..99cc410c 100644 --- a/pkg/transport/process/extension/uds_executable.go +++ b/pkg/transport/process/extension/uds_executable.go @@ -23,7 +23,7 @@ type udsExecutable struct { // NewUDSExecutable runs a resource processor extension executable in the background. // It communicates with this processor via Unix Domain Sockets. -func NewUDSExecutable(ctx context.Context, bin string, args ...string) (process.ResourceStreamProcessor, error) { +func NewUDSExecutable(ctx context.Context, bin string, args []string, env []string) (process.ResourceStreamProcessor, error) { for _, arg := range args { if arg == serverAddressFlag { return nil, fmt.Errorf("the flag %s is not allowed to be set manually", serverAddressFlag) @@ -38,11 +38,11 @@ func NewUDSExecutable(ctx context.Context, bin string, args ...string) (process. args = append(args, "--addr", addr) cmd := exec.CommandContext(ctx, bin, args...) + cmd.Env = env cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - err = cmd.Start() - if err != nil { + if err := cmd.Start(); err != nil { return nil, fmt.Errorf("unable to start processor: %w", err) } diff --git a/pkg/transport/process/pipeline.go b/pkg/transport/process/pipeline.go index 98de0227..26d49df7 100644 --- a/pkg/transport/process/pipeline.go +++ b/pkg/transport/process/pipeline.go @@ -2,6 +2,7 @@ package process import ( "context" + "io" "os" "archive/tar" @@ -16,13 +17,12 @@ type resourceProcessingPipelineImpl struct { } func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.ComponentDescriptor, res cdv2.Resource) (*cdv2.ComponentDescriptor, cdv2.Resource, error) { - infile, err := ioutil.TempFile("", "out") + infile, err := ioutil.TempFile("", "") if err != nil { return nil, cdv2.Resource{}, fmt.Errorf("unable to create temporary infile: %w", err) } - err = WriteTARArchive(ctx, cd, res, nil, tar.NewWriter(infile)) - if err != nil { + if err := WriteTARArchive(ctx, cd, res, nil, tar.NewWriter(infile)); err != nil { return nil, cdv2.Resource{}, fmt.Errorf("unable to write: %w", err) } @@ -36,8 +36,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co } defer infile.Close() - _, err = infile.Seek(0, 0) - if err != nil { + if _, err := infile.Seek(0, io.SeekStart); err != nil { return nil, cdv2.Resource{}, err } @@ -53,8 +52,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co func (p *resourceProcessingPipelineImpl) process(ctx context.Context, infile *os.File, proc ResourceStreamProcessor) (*os.File, error) { defer infile.Close() - _, err := infile.Seek(0, 0) - if err != nil { + if _, err := infile.Seek(0, io.SeekStart); err != nil { return nil, fmt.Errorf("unable to seek to beginning of input file: %w", err) } @@ -66,8 +64,7 @@ func (p *resourceProcessingPipelineImpl) process(ctx context.Context, infile *os inreader := infile outwriter := outfile - err = proc.Process(ctx, inreader, outwriter) - if err != nil { + if err := proc.Process(ctx, inreader, outwriter); err != nil { return nil, fmt.Errorf("unable to process resource: %w", err) } diff --git a/pkg/transport/process/util.go b/pkg/transport/process/util.go index a07f3e63..37760486 100644 --- a/pkg/transport/process/util.go +++ b/pkg/transport/process/util.go @@ -29,8 +29,7 @@ func WriteTARArchive(ctx context.Context, cd cdv2.ComponentDescriptor, res cdv2. return fmt.Errorf("unable to marshal component descriptor: %w", err) } - err = writeFileToTARArchive(componentDescriptorFile, bytes.NewReader(marshaledCD), outArchive) - if err != nil { + if err := writeFileToTARArchive(componentDescriptorFile, bytes.NewReader(marshaledCD), outArchive); err != nil { return fmt.Errorf("unable to write %s: %w", componentDescriptorFile, err) } @@ -39,14 +38,12 @@ func WriteTARArchive(ctx context.Context, cd cdv2.ComponentDescriptor, res cdv2. return fmt.Errorf("unable to marshal resource: %w", err) } - err = writeFileToTARArchive(resourceFile, bytes.NewReader(marshaledRes), outArchive) - if err != nil { + if err := writeFileToTARArchive(resourceFile, bytes.NewReader(marshaledRes), outArchive); err != nil { return fmt.Errorf("unable to write %s: %w", resourceFile, err) } if resourceBlobReader != nil { - err = writeFileToTARArchive(resourceBlobFile, resourceBlobReader, outArchive) - if err != nil { + if err := writeFileToTARArchive(resourceBlobFile, resourceBlobReader, outArchive); err != nil { return fmt.Errorf("unable to write %s: %w", resourceBlobFile, err) } } @@ -61,13 +58,11 @@ func writeFileToTARArchive(filename string, contentReader io.Reader, outArchive } defer tempfile.Close() - _, err = io.Copy(tempfile, contentReader) - if err != nil { + if _, err := io.Copy(tempfile, contentReader); err != nil { return fmt.Errorf("unable to write content to file: %w", err) } - _, err = tempfile.Seek(0, 0) - if err != nil { + if _, err := tempfile.Seek(0, io.SeekStart); err != nil { return fmt.Errorf("unable to seek to beginning of file: %w", err) } @@ -83,12 +78,11 @@ func writeFileToTARArchive(filename string, contentReader io.Reader, outArchive ModTime: time.Now(), } - if err = outArchive.WriteHeader(&header); err != nil { + if err := outArchive.WriteHeader(&header); err != nil { return fmt.Errorf("unable to write tar header: %w", err) } - _, err = io.Copy(outArchive, tempfile) - if err != nil { + if _, err := io.Copy(outArchive, tempfile); err != nil { return fmt.Errorf("unable to write file to tar archive: %w", err) } @@ -113,30 +107,25 @@ func ReadTARArchive(r *tar.Reader) (*cdv2.ComponentDescriptor, cdv2.Resource, io switch header.Name { case resourceFile: - res, err = readResource(r) - if err != nil { + if res, err = readResource(r); err != nil { return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", resourceFile, err) } case componentDescriptorFile: - cd, err = readComponentDescriptor(r) - if err != nil { + if cd, err = readComponentDescriptor(r); err != nil { return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", componentDescriptorFile, err) } case resourceBlobFile: - f, err = ioutil.TempFile("", "") - if err != nil { + if f, err = ioutil.TempFile("", ""); err != nil { return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to create tempfile: %w", err) } - _, err = io.Copy(f, r) - if err != nil { + if _, err := io.Copy(f, r); err != nil { return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", resourceBlobFile, err) } } } if f != nil { - _, err := f.Seek(0, 0) - if err != nil { + if _, err := f.Seek(0, io.SeekStart); err != nil { return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to seek to beginning of file: %w", err) } } @@ -146,14 +135,12 @@ func ReadTARArchive(r *tar.Reader) (*cdv2.ComponentDescriptor, cdv2.Resource, io func readResource(r *tar.Reader) (cdv2.Resource, error) { buf := bytes.NewBuffer([]byte{}) - _, err := io.Copy(buf, r) - if err != nil { + if _, err := io.Copy(buf, r); err != nil { return cdv2.Resource{}, fmt.Errorf("unable to read from stream: %w", err) } var res cdv2.Resource - err = yaml.Unmarshal(buf.Bytes(), &res) - if err != nil { + if err := yaml.Unmarshal(buf.Bytes(), &res); err != nil { return cdv2.Resource{}, fmt.Errorf("unable to unmarshal: %w", err) } @@ -162,14 +149,12 @@ func readResource(r *tar.Reader) (cdv2.Resource, error) { func readComponentDescriptor(r *tar.Reader) (*cdv2.ComponentDescriptor, error) { buf := bytes.NewBuffer([]byte{}) - _, err := io.Copy(buf, r) - if err != nil { + if _, err := io.Copy(buf, r); err != nil { return nil, fmt.Errorf("unable to read from stream: %w", err) } var cd cdv2.ComponentDescriptor - err = yaml.Unmarshal(buf.Bytes(), &cd) - if err != nil { + if err := yaml.Unmarshal(buf.Bytes(), &cd); err != nil { return nil, fmt.Errorf("unable to unmarshal: %w", err) } From 44752389cb5e13a20ac9e66beb9fada325c3cd1b Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Thu, 16 Sep 2021 09:59:18 +0200 Subject: [PATCH 04/24] run formatter --- pkg/transport/process/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/transport/process/types.go b/pkg/transport/process/types.go index d0b1a998..8379a93a 100644 --- a/pkg/transport/process/types.go +++ b/pkg/transport/process/types.go @@ -20,7 +20,7 @@ type ResourceProcessingPipeline interface { // ResourceStreamProcessor describes an individual processor for processing a resource. // A processor can upload, modify, or download a resource. type ResourceStreamProcessor interface { - // Process executes the processor for a resource. Input and Output streams must be TAR + // Process executes the processor for a resource. Input and Output streams must be TAR // archives which contain the component descriptor, resource, and resource blob. Process(context.Context, io.Reader, io.Writer) error } From 9d7a1d32adf5176cef7419b7607c7854addcbcf5 Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Mon, 20 Sep 2021 09:38:32 +0200 Subject: [PATCH 05/24] renames extension pkg and adds tests --- hack/install-requirements.sh | 5 + .../extensions/extensions_suite_test.go | 106 ++++++++++++ .../stdio_executable.go | 5 +- .../uds_executable.go | 24 +-- pkg/transport/process/processors/test.go | 157 ++++++++++++++++++ 5 files changed, 286 insertions(+), 11 deletions(-) create mode 100644 pkg/transport/process/extensions/extensions_suite_test.go rename pkg/transport/process/{extension => extensions}/stdio_executable.go (90%) rename pkg/transport/process/{extension => extensions}/uds_executable.go (78%) create mode 100644 pkg/transport/process/processors/test.go diff --git a/hack/install-requirements.sh b/hack/install-requirements.sh index d219507b..19db5de2 100755 --- a/hack/install-requirements.sh +++ b/hack/install-requirements.sh @@ -36,3 +36,8 @@ $ export PATH=/usr/local/opt/gnu-tar/libexec/gnubin:\$PATH $ export PATH=/usr/local/opt/grep/libexec/gnubin:\$PATH EOM fi + + +echo "> Compile test processor binary" + +go build -o "${PROJECT_ROOT}/tmp/test/bin/processor" "${PROJECT_ROOT}/pkg/transport/process/processors" \ No newline at end of file diff --git a/pkg/transport/process/extensions/extensions_suite_test.go b/pkg/transport/process/extensions/extensions_suite_test.go new file mode 100644 index 00000000..f22cdd77 --- /dev/null +++ b/pkg/transport/process/extensions/extensions_suite_test.go @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package extensions_test + +import ( + "archive/tar" + "bytes" + "context" + "encoding/json" + "io" + "strings" + "testing" + + "github.com/gardener/component-cli/pkg/transport/process" + "github.com/gardener/component-cli/pkg/transport/process/extensions" + cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + defaultProcessorBinaryPath = "../../../../tmp/test/bin/processor" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "transport extensions Test Suite") +} + +var _ = Describe("transport extensions", func() { + + Context("stdio executable", func() { + It("should modify the processed resource correctly", func() { + args := []string{} + env := []string{} + processor, err := extensions.NewStdIOExecutable(context.TODO(), defaultProcessorBinaryPath, args, env) + Expect(err).ToNot(HaveOccurred()) + + testProcessor(processor) + }) + }) + + Context("uds executable", func() { + It("should modify the processed resource correctly", func() { + args := []string{} + env := []string{} + processor, err := extensions.NewUDSExecutable(context.TODO(), defaultProcessorBinaryPath, args, env) + Expect(err).ToNot(HaveOccurred()) + + testProcessor(processor) + }) + }) + +}) + +func testProcessor(processor process.ResourceStreamProcessor) { + const ( + processorName = "test-processor" + resourceData = "12345" + expectedResourceData = resourceData + "\n" + processorName + ) + + res := cdv2.Resource{ + IdentityObjectMeta: cdv2.IdentityObjectMeta{ + Name: "my-res", + Version: "v0.1.0", + Type: "ociImage", + }, + } + + l := cdv2.Label{ + Name: "processor-name", + Value: json.RawMessage(`"` + processorName + `"`), + } + expectedRes := res + expectedRes.Labels = append(expectedRes.Labels, l) + + cd := cdv2.ComponentDescriptor{ + ComponentSpec: cdv2.ComponentSpec{ + Resources: []cdv2.Resource{ + res, + }, + }, + } + + inputBuf := bytes.NewBuffer([]byte{}) + err := process.WriteTARArchive(cd, res, strings.NewReader(resourceData), tar.NewWriter(inputBuf)) + Expect(err).ToNot(HaveOccurred()) + + outputBuf := bytes.NewBuffer([]byte{}) + err = processor.Process(context.TODO(), inputBuf, outputBuf) + Expect(err).ToNot(HaveOccurred()) + + processedCD, processedRes, processedBlobReader, err := process.ReadTARArchive(tar.NewReader(outputBuf)) + Expect(err).ToNot(HaveOccurred()) + + Expect(*processedCD).To(Equal(cd)) + Expect(processedRes).To(Equal(expectedRes)) + + processedResourceDataBuf := bytes.NewBuffer([]byte{}) + _, err = io.Copy(processedResourceDataBuf, processedBlobReader) + Expect(err).ToNot(HaveOccurred()) + + Expect(processedResourceDataBuf.String()).To(Equal(expectedResourceData)) +} diff --git a/pkg/transport/process/extension/stdio_executable.go b/pkg/transport/process/extensions/stdio_executable.go similarity index 90% rename from pkg/transport/process/extension/stdio_executable.go rename to pkg/transport/process/extensions/stdio_executable.go index f487e91c..6596805e 100644 --- a/pkg/transport/process/extension/stdio_executable.go +++ b/pkg/transport/process/extensions/stdio_executable.go @@ -1,4 +1,7 @@ -package extension +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package extensions import ( "context" diff --git a/pkg/transport/process/extension/uds_executable.go b/pkg/transport/process/extensions/uds_executable.go similarity index 78% rename from pkg/transport/process/extension/uds_executable.go rename to pkg/transport/process/extensions/uds_executable.go index 99cc410c..6ac31004 100644 --- a/pkg/transport/process/extension/uds_executable.go +++ b/pkg/transport/process/extensions/uds_executable.go @@ -1,4 +1,7 @@ -package extension +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package extensions import ( "context" @@ -7,6 +10,7 @@ import ( "net" "os" "os/exec" + "syscall" "time" "github.com/gardener/component-cli/pkg/transport/process" @@ -61,26 +65,26 @@ func NewUDSExecutable(ctx context.Context, bin string, args []string, env []stri } func (e *udsExecutable) Process(ctx context.Context, r io.Reader, w io.Writer) error { - _, err := io.Copy(e.conn, r) - if err != nil { + if _, err := io.Copy(e.conn, r); err != nil { return fmt.Errorf("unable to write input: %w", err) } usock := e.conn.(*net.UnixConn) - err = usock.CloseWrite() - if err != nil { + if err := usock.CloseWrite(); err != nil { return fmt.Errorf("unable to close input writer: %w", err) } - _, err = io.Copy(w, e.conn) - if err != nil { + if _, err := io.Copy(w, e.conn); err != nil { return fmt.Errorf("unable to read output: %w", err) } + if err := e.processor.Process.Signal(syscall.SIGTERM); err != nil { + return fmt.Errorf("unable to send SIGTERM to processor: %w", err) + } + // extension servers must implement ordinary shutdown (!) - err = e.processor.Wait() - if err != nil { - return fmt.Errorf("unable to stop processor: %w", err) + if err := e.processor.Wait(); err != nil { + return fmt.Errorf("unable to wait for processor: %w", err) } return nil diff --git a/pkg/transport/process/processors/test.go b/pkg/transport/process/processors/test.go new file mode 100644 index 00000000..e7679067 --- /dev/null +++ b/pkg/transport/process/processors/test.go @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package main + +import ( + "archive/tar" + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "os" + "os/signal" + "strings" + "sync" + "syscall" + + "github.com/gardener/component-cli/pkg/transport/process" + cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" +) + +const processorName = "test-processor" + +type ProcessorHandlerFunc func(io.Reader, io.WriteCloser) + +type Server struct { + listener net.Listener + quit chan interface{} + wg sync.WaitGroup + handler ProcessorHandlerFunc +} + +func NewServer(addr string, h ProcessorHandlerFunc) (*Server, error) { + l, err := net.Listen("unix", addr) + if err != nil { + return nil, err + } + s := &Server{ + quit: make(chan interface{}), + listener: l, + handler: h, + } + return s, nil +} + +func (s *Server) Start() { + s.wg.Add(1) + go s.serve() +} + +func (s *Server) serve() { + defer s.wg.Done() + + for { + conn, err := s.listener.Accept() + if err != nil { + select { + case <-s.quit: + return + default: + log.Println("accept error", err) + } + } else { + s.wg.Add(1) + go func() { + defer s.wg.Done() + s.handler(conn, conn) + }() + } + } +} + +func (s *Server) Stop() { + close(s.quit) + if err := s.listener.Close(); err != nil { + println(err) + } + s.wg.Wait() +} + +func main() { + addr := flag.String("addr", "", "") + flag.Parse() + + if *addr == "" { + // if addr is not set, use stdin/stdout for communication + if err := ProcessorRoutine(os.Stdin, os.Stdout); err != nil { + log.Fatal(err) + } + return + } + + h := func(r io.Reader, w io.WriteCloser) { + ProcessorRoutine(r, w) + } + + srv, err := NewServer(*addr, h) + if err != nil { + log.Fatal(err) + } + + srv.Start() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + <-stop + + srv.Stop() +} + +func ProcessorRoutine(inputStream io.Reader, outputStream io.WriteCloser) error { + defer outputStream.Close() + + tmpfile, err := ioutil.TempFile("", "") + if err != nil { + return err + } + defer tmpfile.Close() + + if _, err := io.Copy(tmpfile, inputStream); err != nil { + return err + } + + if _, err := tmpfile.Seek(0, io.SeekStart); err != nil { + return err + } + + cd, res, resourceBlobReader, err := process.ReadTARArchive(tar.NewReader(tmpfile)) + if err != nil { + return err + } + if resourceBlobReader != nil { + defer resourceBlobReader.Close() + } + + buf := bytes.NewBuffer([]byte{}) + if _, err := io.Copy(buf, resourceBlobReader); err != nil { + return err + } + outputData := fmt.Sprintf("%s\n%s", buf.String(), processorName) + + l := cdv2.Label{ + Name: "processor-name", + Value: json.RawMessage(`"` + processorName + `"`), + } + res.Labels = append(res.Labels, l) + + if err := process.WriteTARArchive(*cd, res, strings.NewReader(outputData), tar.NewWriter(outputStream)); err != nil { + return err + } + + return nil +} From 38fd5ff9e8b17eab02a0398b520935cc81bf4cf3 Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Mon, 20 Sep 2021 09:38:52 +0200 Subject: [PATCH 06/24] adds license headers --- pkg/transport/process/pipeline.go | 5 ++++- pkg/transport/process/types.go | 3 +++ pkg/transport/process/util.go | 6 ++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/transport/process/pipeline.go b/pkg/transport/process/pipeline.go index 26d49df7..0ac32887 100644 --- a/pkg/transport/process/pipeline.go +++ b/pkg/transport/process/pipeline.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 package process import ( @@ -22,7 +25,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co return nil, cdv2.Resource{}, fmt.Errorf("unable to create temporary infile: %w", err) } - if err := WriteTARArchive(ctx, cd, res, nil, tar.NewWriter(infile)); err != nil { + if err := WriteTARArchive(cd, res, nil, tar.NewWriter(infile)); err != nil { return nil, cdv2.Resource{}, fmt.Errorf("unable to write: %w", err) } diff --git a/pkg/transport/process/types.go b/pkg/transport/process/types.go index 8379a93a..6e970285 100644 --- a/pkg/transport/process/types.go +++ b/pkg/transport/process/types.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 package process import ( diff --git a/pkg/transport/process/util.go b/pkg/transport/process/util.go index 37760486..85c91c81 100644 --- a/pkg/transport/process/util.go +++ b/pkg/transport/process/util.go @@ -1,9 +1,11 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 package process import ( "archive/tar" "bytes" - "context" "fmt" "io" "io/ioutil" @@ -21,7 +23,7 @@ const ( ) // WriteTARArchive writes the component descriptor, resource and resource blob to a TAR archive -func WriteTARArchive(ctx context.Context, cd cdv2.ComponentDescriptor, res cdv2.Resource, resourceBlobReader io.Reader, outArchive *tar.Writer) error { +func WriteTARArchive(cd cdv2.ComponentDescriptor, res cdv2.Resource, resourceBlobReader io.Reader, outArchive *tar.Writer) error { defer outArchive.Close() marshaledCD, err := yaml.Marshal(cd) From 8fe08355797df82788ab4c7d118075a6ad61dac0 Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Mon, 20 Sep 2021 13:52:11 +0200 Subject: [PATCH 07/24] refactoring + adds test --- .../extensions/extensions_suite_test.go | 4 +- pkg/transport/process/pipeline.go | 4 +- pkg/transport/process/process_suite_test.go | 16 +++++ pkg/transport/process/processors/test.go | 8 ++- pkg/transport/process/util.go | 62 +++++++++++-------- pkg/transport/process/util_test.go | 56 +++++++++++++++++ 6 files changed, 118 insertions(+), 32 deletions(-) create mode 100644 pkg/transport/process/process_suite_test.go create mode 100644 pkg/transport/process/util_test.go diff --git a/pkg/transport/process/extensions/extensions_suite_test.go b/pkg/transport/process/extensions/extensions_suite_test.go index f22cdd77..4f88035b 100644 --- a/pkg/transport/process/extensions/extensions_suite_test.go +++ b/pkg/transport/process/extensions/extensions_suite_test.go @@ -85,14 +85,14 @@ func testProcessor(processor process.ResourceStreamProcessor) { } inputBuf := bytes.NewBuffer([]byte{}) - err := process.WriteTARArchive(cd, res, strings.NewReader(resourceData), tar.NewWriter(inputBuf)) + err := process.WriteProcessorMessage(cd, res, strings.NewReader(resourceData), tar.NewWriter(inputBuf)) Expect(err).ToNot(HaveOccurred()) outputBuf := bytes.NewBuffer([]byte{}) err = processor.Process(context.TODO(), inputBuf, outputBuf) Expect(err).ToNot(HaveOccurred()) - processedCD, processedRes, processedBlobReader, err := process.ReadTARArchive(tar.NewReader(outputBuf)) + processedCD, processedRes, processedBlobReader, err := process.ReadProcessorMessage(tar.NewReader(outputBuf)) Expect(err).ToNot(HaveOccurred()) Expect(*processedCD).To(Equal(cd)) diff --git a/pkg/transport/process/pipeline.go b/pkg/transport/process/pipeline.go index 0ac32887..71b15819 100644 --- a/pkg/transport/process/pipeline.go +++ b/pkg/transport/process/pipeline.go @@ -25,7 +25,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co return nil, cdv2.Resource{}, fmt.Errorf("unable to create temporary infile: %w", err) } - if err := WriteTARArchive(cd, res, nil, tar.NewWriter(infile)); err != nil { + if err := WriteProcessorMessage(cd, res, nil, tar.NewWriter(infile)); err != nil { return nil, cdv2.Resource{}, fmt.Errorf("unable to write: %w", err) } @@ -43,7 +43,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co return nil, cdv2.Resource{}, err } - processedCD, processedRes, blobreader, err := ReadTARArchive(tar.NewReader(infile)) + processedCD, processedRes, blobreader, err := ReadProcessorMessage(tar.NewReader(infile)) if err != nil { return nil, cdv2.Resource{}, fmt.Errorf("unable to read output data: %w", err) } diff --git a/pkg/transport/process/process_suite_test.go b/pkg/transport/process/process_suite_test.go new file mode 100644 index 00000000..0f1b48d0 --- /dev/null +++ b/pkg/transport/process/process_suite_test.go @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package process_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Transport Process Test Suite") +} \ No newline at end of file diff --git a/pkg/transport/process/processors/test.go b/pkg/transport/process/processors/test.go index e7679067..35215aa9 100644 --- a/pkg/transport/process/processors/test.go +++ b/pkg/transport/process/processors/test.go @@ -95,7 +95,9 @@ func main() { } h := func(r io.Reader, w io.WriteCloser) { - ProcessorRoutine(r, w) + if err := ProcessorRoutine(r, w); err != nil { + log.Fatal(err) + } } srv, err := NewServer(*addr, h) @@ -129,7 +131,7 @@ func ProcessorRoutine(inputStream io.Reader, outputStream io.WriteCloser) error return err } - cd, res, resourceBlobReader, err := process.ReadTARArchive(tar.NewReader(tmpfile)) + cd, res, resourceBlobReader, err := process.ReadProcessorMessage(tar.NewReader(tmpfile)) if err != nil { return err } @@ -149,7 +151,7 @@ func ProcessorRoutine(inputStream io.Reader, outputStream io.WriteCloser) error } res.Labels = append(res.Labels, l) - if err := process.WriteTARArchive(*cd, res, strings.NewReader(outputData), tar.NewWriter(outputStream)); err != nil { + if err := process.WriteProcessorMessage(*cd, res, strings.NewReader(outputData), tar.NewWriter(outputStream)); err != nil { return err } diff --git a/pkg/transport/process/util.go b/pkg/transport/process/util.go index 85c91c81..09af0cec 100644 --- a/pkg/transport/process/util.go +++ b/pkg/transport/process/util.go @@ -17,22 +17,30 @@ import ( ) const ( - componentDescriptorFile = "component-descriptor.yaml" - resourceFile = "resource.yaml" - resourceBlobFile = "resource-blob" + // ComponentDescriptorFile is the filename of the component descriptor in a processor message tar archive + ComponentDescriptorFile = "component-descriptor.yaml" + + // ResourceFile is the filename of the resource in a processor message tar archive + ResourceFile = "resource.yaml" + + // ResourceBlobFile is the filename of the resource blob in a processor message tar archive + ResourceBlobFile = "resource-blob" ) -// WriteTARArchive writes the component descriptor, resource and resource blob to a TAR archive -func WriteTARArchive(cd cdv2.ComponentDescriptor, res cdv2.Resource, resourceBlobReader io.Reader, outArchive *tar.Writer) error { - defer outArchive.Close() +// WriteProcessorMessage writes a component descriptor, resource and resource blob as a processor +// message (tar archive with fixed filenames for component descriptor, resource, and resource blob) +// which can be consumed by processors. +func WriteProcessorMessage(cd cdv2.ComponentDescriptor, res cdv2.Resource, resourceBlobReader io.Reader, w io.Writer) error { + tw := tar.NewWriter(w) + defer tw.Close() marshaledCD, err := yaml.Marshal(cd) if err != nil { return fmt.Errorf("unable to marshal component descriptor: %w", err) } - if err := writeFileToTARArchive(componentDescriptorFile, bytes.NewReader(marshaledCD), outArchive); err != nil { - return fmt.Errorf("unable to write %s: %w", componentDescriptorFile, err) + if err := writeFileToTARArchive(ComponentDescriptorFile, bytes.NewReader(marshaledCD), tw); err != nil { + return fmt.Errorf("unable to write %s: %w", ComponentDescriptorFile, err) } marshaledRes, err := yaml.Marshal(res) @@ -40,13 +48,13 @@ func WriteTARArchive(cd cdv2.ComponentDescriptor, res cdv2.Resource, resourceBlo return fmt.Errorf("unable to marshal resource: %w", err) } - if err := writeFileToTARArchive(resourceFile, bytes.NewReader(marshaledRes), outArchive); err != nil { - return fmt.Errorf("unable to write %s: %w", resourceFile, err) + if err := writeFileToTARArchive(ResourceFile, bytes.NewReader(marshaledRes), tw); err != nil { + return fmt.Errorf("unable to write %s: %w", ResourceFile, err) } if resourceBlobReader != nil { - if err := writeFileToTARArchive(resourceBlobFile, resourceBlobReader, outArchive); err != nil { - return fmt.Errorf("unable to write %s: %w", resourceBlobFile, err) + if err := writeFileToTARArchive(ResourceBlobFile, resourceBlobReader, tw); err != nil { + return fmt.Errorf("unable to write %s: %w", ResourceBlobFile, err) } } @@ -91,15 +99,19 @@ func writeFileToTARArchive(filename string, contentReader io.Reader, outArchive return nil } -// ReadTARArchive reads the component descriptor, resource and resource blob from a TAR archive. -// The resource blob reader can be nil. If a non-nil value is returned, it must be closed by the caller. -func ReadTARArchive(r *tar.Reader) (*cdv2.ComponentDescriptor, cdv2.Resource, io.ReadCloser, error) { +// ReadProcessorMessage reads the component descriptor, resource and resource blob from a processor message +// (tar archive with fixed filenames for component descriptor, resource, and resource blob) which is +// produced by processors. The resource blob reader can be nil. If a non-nil value is returned, it must +// be closed by the caller. +func ReadProcessorMessage(r io.Reader) (*cdv2.ComponentDescriptor, cdv2.Resource, io.ReadCloser, error) { + tr := tar.NewReader(r) + var cd *cdv2.ComponentDescriptor var res cdv2.Resource var f *os.File for { - header, err := r.Next() + header, err := tr.Next() if err != nil { if err == io.EOF { break @@ -108,20 +120,20 @@ func ReadTARArchive(r *tar.Reader) (*cdv2.ComponentDescriptor, cdv2.Resource, io } switch header.Name { - case resourceFile: - if res, err = readResource(r); err != nil { - return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", resourceFile, err) + case ResourceFile: + if res, err = readResource(tr); err != nil { + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", ResourceFile, err) } - case componentDescriptorFile: - if cd, err = readComponentDescriptor(r); err != nil { - return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", componentDescriptorFile, err) + case ComponentDescriptorFile: + if cd, err = readComponentDescriptor(tr); err != nil { + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", ComponentDescriptorFile, err) } - case resourceBlobFile: + case ResourceBlobFile: if f, err = ioutil.TempFile("", ""); err != nil { return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to create tempfile: %w", err) } - if _, err := io.Copy(f, r); err != nil { - return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", resourceBlobFile, err) + if _, err := io.Copy(f, tr); err != nil { + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to read %s: %w", ResourceBlobFile, err) } } } diff --git a/pkg/transport/process/util_test.go b/pkg/transport/process/util_test.go new file mode 100644 index 00000000..b9b8dcda --- /dev/null +++ b/pkg/transport/process/util_test.go @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package process_test + +import ( + "bytes" + "io" + "strings" + + "github.com/gardener/component-cli/pkg/transport/process" + cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("utils", func() { + + Context("WriteProcessMessage & ReadProcessMessage", func() { + + It("should correctly write and read a process message", func() { + res := cdv2.Resource{ + IdentityObjectMeta: cdv2.IdentityObjectMeta{ + Name: "my-res", + Version: "v0.1.0", + Type: "ociImage", + }, + } + resourceData := "test-data" + + cd := cdv2.ComponentDescriptor{ + ComponentSpec: cdv2.ComponentSpec{ + Resources: []cdv2.Resource{ + res, + }, + }, + } + + processMsgBuf := bytes.NewBuffer([]byte{}) + err := process.WriteProcessorMessage(cd, res, strings.NewReader(resourceData), processMsgBuf) + Expect(err).ToNot(HaveOccurred()) + + actualCD, actualRes, resourceBlobReader, err := process.ReadProcessorMessage(processMsgBuf) + Expect(err).ToNot(HaveOccurred()) + + Expect(*actualCD).To(Equal(cd)) + Expect(actualRes).To(Equal(res)) + + resourceBlobBuf := bytes.NewBuffer([]byte{}) + _, err = io.Copy(resourceBlobBuf, resourceBlobReader) + Expect(err).ToNot(HaveOccurred()) + Expect(resourceBlobBuf.String()).To(Equal(resourceData)) + }) + + }) +}) From 5fbf6badf2aa83130e958fb701f2eb1a79ef35ab Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Tue, 21 Sep 2021 09:42:49 +0200 Subject: [PATCH 08/24] fix tests and adds check for test processor binary --- .../process/extensions/extensions_suite_test.go | 12 +++++++++--- pkg/transport/process/pipeline.go | 7 +++---- pkg/transport/process/processors/test.go | 5 ++--- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pkg/transport/process/extensions/extensions_suite_test.go b/pkg/transport/process/extensions/extensions_suite_test.go index 4f88035b..abd54dcd 100644 --- a/pkg/transport/process/extensions/extensions_suite_test.go +++ b/pkg/transport/process/extensions/extensions_suite_test.go @@ -4,11 +4,11 @@ package extensions_test import ( - "archive/tar" "bytes" "context" "encoding/json" "io" + "os" "strings" "testing" @@ -28,6 +28,12 @@ func TestConfig(t *testing.T) { RunSpecs(t, "transport extensions Test Suite") } +var _ = BeforeSuite(func() { + info, err := os.Stat(defaultProcessorBinaryPath) + Expect(err).ToNot(HaveOccurred()) + Expect(info.IsDir()).To(BeFalse()) +}, 5) + var _ = Describe("transport extensions", func() { Context("stdio executable", func() { @@ -85,14 +91,14 @@ func testProcessor(processor process.ResourceStreamProcessor) { } inputBuf := bytes.NewBuffer([]byte{}) - err := process.WriteProcessorMessage(cd, res, strings.NewReader(resourceData), tar.NewWriter(inputBuf)) + err := process.WriteProcessorMessage(cd, res, strings.NewReader(resourceData), inputBuf) Expect(err).ToNot(HaveOccurred()) outputBuf := bytes.NewBuffer([]byte{}) err = processor.Process(context.TODO(), inputBuf, outputBuf) Expect(err).ToNot(HaveOccurred()) - processedCD, processedRes, processedBlobReader, err := process.ReadProcessorMessage(tar.NewReader(outputBuf)) + processedCD, processedRes, processedBlobReader, err := process.ReadProcessorMessage(outputBuf) Expect(err).ToNot(HaveOccurred()) Expect(*processedCD).To(Equal(cd)) diff --git a/pkg/transport/process/pipeline.go b/pkg/transport/process/pipeline.go index 71b15819..77941189 100644 --- a/pkg/transport/process/pipeline.go +++ b/pkg/transport/process/pipeline.go @@ -8,7 +8,6 @@ import ( "io" "os" - "archive/tar" "fmt" "io/ioutil" @@ -25,7 +24,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co return nil, cdv2.Resource{}, fmt.Errorf("unable to create temporary infile: %w", err) } - if err := WriteProcessorMessage(cd, res, nil, tar.NewWriter(infile)); err != nil { + if err := WriteProcessorMessage(cd, res, nil, infile); err != nil { return nil, cdv2.Resource{}, fmt.Errorf("unable to write: %w", err) } @@ -43,7 +42,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co return nil, cdv2.Resource{}, err } - processedCD, processedRes, blobreader, err := ReadProcessorMessage(tar.NewReader(infile)) + processedCD, processedRes, blobreader, err := ReadProcessorMessage(infile) if err != nil { return nil, cdv2.Resource{}, fmt.Errorf("unable to read output data: %w", err) } @@ -59,7 +58,7 @@ func (p *resourceProcessingPipelineImpl) process(ctx context.Context, infile *os return nil, fmt.Errorf("unable to seek to beginning of input file: %w", err) } - outfile, err := ioutil.TempFile("", "out") + outfile, err := ioutil.TempFile("", "") if err != nil { return nil, fmt.Errorf("unable to create temporary outfile: %w", err) } diff --git a/pkg/transport/process/processors/test.go b/pkg/transport/process/processors/test.go index 35215aa9..13c0f7a8 100644 --- a/pkg/transport/process/processors/test.go +++ b/pkg/transport/process/processors/test.go @@ -4,7 +4,6 @@ package main import ( - "archive/tar" "bytes" "encoding/json" "flag" @@ -131,7 +130,7 @@ func ProcessorRoutine(inputStream io.Reader, outputStream io.WriteCloser) error return err } - cd, res, resourceBlobReader, err := process.ReadProcessorMessage(tar.NewReader(tmpfile)) + cd, res, resourceBlobReader, err := process.ReadProcessorMessage(tmpfile) if err != nil { return err } @@ -151,7 +150,7 @@ func ProcessorRoutine(inputStream io.Reader, outputStream io.WriteCloser) error } res.Labels = append(res.Labels, l) - if err := process.WriteProcessorMessage(*cd, res, strings.NewReader(outputData), tar.NewWriter(outputStream)); err != nil { + if err := process.WriteProcessorMessage(*cd, res, strings.NewReader(outputData), outputStream); err != nil { return err } From 94da006ea1c2368816b27c426c7a2643894273a6 Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Tue, 21 Sep 2021 09:43:51 +0200 Subject: [PATCH 09/24] formatting --- pkg/transport/process/extensions/extensions_suite_test.go | 5 +++-- pkg/transport/process/process_suite_test.go | 2 +- pkg/transport/process/processors/test.go | 3 ++- pkg/transport/process/util.go | 6 +++--- pkg/transport/process/util_test.go | 3 ++- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/transport/process/extensions/extensions_suite_test.go b/pkg/transport/process/extensions/extensions_suite_test.go index abd54dcd..fca9f68e 100644 --- a/pkg/transport/process/extensions/extensions_suite_test.go +++ b/pkg/transport/process/extensions/extensions_suite_test.go @@ -12,11 +12,12 @@ import ( "strings" "testing" - "github.com/gardener/component-cli/pkg/transport/process" - "github.com/gardener/component-cli/pkg/transport/process/extensions" cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "github.com/gardener/component-cli/pkg/transport/process" + "github.com/gardener/component-cli/pkg/transport/process/extensions" ) const ( diff --git a/pkg/transport/process/process_suite_test.go b/pkg/transport/process/process_suite_test.go index 0f1b48d0..b891cba1 100644 --- a/pkg/transport/process/process_suite_test.go +++ b/pkg/transport/process/process_suite_test.go @@ -13,4 +13,4 @@ import ( func TestConfig(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Transport Process Test Suite") -} \ No newline at end of file +} diff --git a/pkg/transport/process/processors/test.go b/pkg/transport/process/processors/test.go index 13c0f7a8..2ecc0b9c 100644 --- a/pkg/transport/process/processors/test.go +++ b/pkg/transport/process/processors/test.go @@ -18,8 +18,9 @@ import ( "sync" "syscall" - "github.com/gardener/component-cli/pkg/transport/process" cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" + + "github.com/gardener/component-cli/pkg/transport/process" ) const processorName = "test-processor" diff --git a/pkg/transport/process/util.go b/pkg/transport/process/util.go index 09af0cec..880b6a6b 100644 --- a/pkg/transport/process/util.go +++ b/pkg/transport/process/util.go @@ -28,7 +28,7 @@ const ( ) // WriteProcessorMessage writes a component descriptor, resource and resource blob as a processor -// message (tar archive with fixed filenames for component descriptor, resource, and resource blob) +// message (tar archive with fixed filenames for component descriptor, resource, and resource blob) // which can be consumed by processors. func WriteProcessorMessage(cd cdv2.ComponentDescriptor, res cdv2.Resource, resourceBlobReader io.Reader, w io.Writer) error { tw := tar.NewWriter(w) @@ -100,8 +100,8 @@ func writeFileToTARArchive(filename string, contentReader io.Reader, outArchive } // ReadProcessorMessage reads the component descriptor, resource and resource blob from a processor message -// (tar archive with fixed filenames for component descriptor, resource, and resource blob) which is -// produced by processors. The resource blob reader can be nil. If a non-nil value is returned, it must +// (tar archive with fixed filenames for component descriptor, resource, and resource blob) which is +// produced by processors. The resource blob reader can be nil. If a non-nil value is returned, it must // be closed by the caller. func ReadProcessorMessage(r io.Reader) (*cdv2.ComponentDescriptor, cdv2.Resource, io.ReadCloser, error) { tr := tar.NewReader(r) diff --git a/pkg/transport/process/util_test.go b/pkg/transport/process/util_test.go index b9b8dcda..b10b596f 100644 --- a/pkg/transport/process/util_test.go +++ b/pkg/transport/process/util_test.go @@ -8,10 +8,11 @@ import ( "io" "strings" - "github.com/gardener/component-cli/pkg/transport/process" cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "github.com/gardener/component-cli/pkg/transport/process" ) var _ = Describe("utils", func() { From 2c8504e4bfd5026ebc043a6cc0f0c16c5aea3b65 Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Tue, 21 Sep 2021 09:46:21 +0200 Subject: [PATCH 10/24] improves before suite check --- pkg/transport/process/extensions/extensions_suite_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/transport/process/extensions/extensions_suite_test.go b/pkg/transport/process/extensions/extensions_suite_test.go index fca9f68e..5d21c01c 100644 --- a/pkg/transport/process/extensions/extensions_suite_test.go +++ b/pkg/transport/process/extensions/extensions_suite_test.go @@ -30,9 +30,8 @@ func TestConfig(t *testing.T) { } var _ = BeforeSuite(func() { - info, err := os.Stat(defaultProcessorBinaryPath) - Expect(err).ToNot(HaveOccurred()) - Expect(info.IsDir()).To(BeFalse()) + _, err := os.Stat(defaultProcessorBinaryPath) + Expect(err).ToNot(HaveOccurred(), "test processor doesn't exists. pls run make install-requirements.") }, 5) var _ = Describe("transport extensions", func() { From a878e86c0911e7bcfc0f8a92d6dd6ef9a6abe354 Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Tue, 21 Sep 2021 16:05:11 +0200 Subject: [PATCH 11/24] refactoring + adds processor timeouts --- hack/install-requirements.sh | 5 +- .../extensions/extensions_suite_test.go | 66 +++++++++++++--- .../process/extensions/stdio_executable.go | 52 ++++++------ .../process/extensions/uds_executable.go | 63 ++++++++------- pkg/transport/process/pipeline.go | 6 ++ .../processors/{test.go => example/main.go} | 79 +++---------------- .../process/processors/sleep/main.go | 51 ++++++++++++ pkg/transport/process/processors/util.go | 68 ++++++++++++++++ pkg/transport/process/util_test.go | 2 +- 9 files changed, 257 insertions(+), 135 deletions(-) rename pkg/transport/process/processors/{test.go => example/main.go} (57%) create mode 100644 pkg/transport/process/processors/sleep/main.go create mode 100644 pkg/transport/process/processors/util.go diff --git a/hack/install-requirements.sh b/hack/install-requirements.sh index 19db5de2..0496b95d 100755 --- a/hack/install-requirements.sh +++ b/hack/install-requirements.sh @@ -38,6 +38,7 @@ EOM fi -echo "> Compile test processor binary" +echo "> Compile processor binaries for testing" -go build -o "${PROJECT_ROOT}/tmp/test/bin/processor" "${PROJECT_ROOT}/pkg/transport/process/processors" \ No newline at end of file +go build -o "${PROJECT_ROOT}/tmp/test/bin/example-processor" "${PROJECT_ROOT}/pkg/transport/process/processors/example" +go build -o "${PROJECT_ROOT}/tmp/test/bin/sleep-processor" "${PROJECT_ROOT}/pkg/transport/process/processors/sleep" \ No newline at end of file diff --git a/pkg/transport/process/extensions/extensions_suite_test.go b/pkg/transport/process/extensions/extensions_suite_test.go index 5d21c01c..c70458a3 100644 --- a/pkg/transport/process/extensions/extensions_suite_test.go +++ b/pkg/transport/process/extensions/extensions_suite_test.go @@ -7,10 +7,12 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "os" "strings" "testing" + "time" cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" . "github.com/onsi/ginkgo" @@ -21,7 +23,11 @@ import ( ) const ( - defaultProcessorBinaryPath = "../../../../tmp/test/bin/processor" + exampleProcessorBinaryPath = "../../../../tmp/test/bin/example-processor" + sleepProcessorBinaryPath = "../../../../tmp/test/bin/sleep-processor" + sleepTimeEnv = "SLEEP_TIME" + timeout = 2 * time.Second + sleepTime = 5 * time.Second ) func TestConfig(t *testing.T) { @@ -30,8 +36,11 @@ func TestConfig(t *testing.T) { } var _ = BeforeSuite(func() { - _, err := os.Stat(defaultProcessorBinaryPath) - Expect(err).ToNot(HaveOccurred(), "test processor doesn't exists. pls run make install-requirements.") + _, err := os.Stat(exampleProcessorBinaryPath) + Expect(err).ToNot(HaveOccurred(), exampleProcessorBinaryPath+" doesn't exists. pls run make install-requirements.") + + _, err = os.Stat(sleepProcessorBinaryPath) + Expect(err).ToNot(HaveOccurred(), sleepProcessorBinaryPath+" doesn't exists. pls run make install-requirements.") }, 5) var _ = Describe("transport extensions", func() { @@ -40,10 +49,21 @@ var _ = Describe("transport extensions", func() { It("should modify the processed resource correctly", func() { args := []string{} env := []string{} - processor, err := extensions.NewStdIOExecutable(context.TODO(), defaultProcessorBinaryPath, args, env) + processor, err := extensions.NewStdIOExecutable(exampleProcessorBinaryPath, args, env) Expect(err).ToNot(HaveOccurred()) - testProcessor(processor) + runExampleResourceTest(processor) + }) + + It("should exit with error when timeout is reached", func() { + args := []string{} + env := []string{ + fmt.Sprintf("%s=%s", sleepTimeEnv, sleepTime.String()), + } + processor, err := extensions.NewStdIOExecutable(sleepProcessorBinaryPath, args, env) + Expect(err).ToNot(HaveOccurred()) + + runTimeoutTest(processor) }) }) @@ -51,18 +71,46 @@ var _ = Describe("transport extensions", func() { It("should modify the processed resource correctly", func() { args := []string{} env := []string{} - processor, err := extensions.NewUDSExecutable(context.TODO(), defaultProcessorBinaryPath, args, env) + processor, err := extensions.NewUDSExecutable(exampleProcessorBinaryPath, args, env) Expect(err).ToNot(HaveOccurred()) - testProcessor(processor) + runExampleResourceTest(processor) + }) + + It("should raise an error when trying to set the server address env variable manually", func() { + args := []string{} + env := []string{ + extensions.ServerAddressEnv + "=/tmp/my-processor.sock", + } + _, err := extensions.NewUDSExecutable(exampleProcessorBinaryPath, args, env) + Expect(err).To(MatchError(fmt.Sprintf("the env variable %s is not allowed to be set manually", extensions.ServerAddressEnv))) + }) + + It("should exit with error when timeout is reached", func() { + args := []string{} + env := []string{ + fmt.Sprintf("%s=%s", sleepTimeEnv, sleepTime.String()), + } + processor, err := extensions.NewUDSExecutable(sleepProcessorBinaryPath, args, env) + Expect(err).ToNot(HaveOccurred()) + + runTimeoutTest(processor) }) }) }) -func testProcessor(processor process.ResourceStreamProcessor) { +func runTimeoutTest(processor process.ResourceStreamProcessor) { + ctx, cancelfunc := context.WithTimeout(context.TODO(), timeout) + defer cancelfunc() + + err := processor.Process(ctx, bytes.NewBuffer([]byte{}), bytes.NewBuffer([]byte{})) + Expect(err).To(MatchError("unable to wait for processor: signal: killed")) +} + +func runExampleResourceTest(processor process.ResourceStreamProcessor) { const ( - processorName = "test-processor" + processorName = "example-processor" resourceData = "12345" expectedResourceData = resourceData + "\n" + processorName ) diff --git a/pkg/transport/process/extensions/stdio_executable.go b/pkg/transport/process/extensions/stdio_executable.go index 6596805e..5ae2b17f 100644 --- a/pkg/transport/process/extensions/stdio_executable.go +++ b/pkg/transport/process/extensions/stdio_executable.go @@ -14,54 +14,54 @@ import ( ) type stdIOExecutable struct { - processor *exec.Cmd - stdin io.WriteCloser - stdout io.Reader + bin string + args []string + env []string } -// NewStdIOExecutable runs a resource processor extension executable in the background. -// It communicates with this processor via stdin/stdout pipes. -func NewStdIOExecutable(ctx context.Context, bin string, args []string, env []string) (process.ResourceStreamProcessor, error) { - cmd := exec.CommandContext(ctx, bin, args...) - cmd.Env = env +// NewStdIOExecutable returns a resource processor extension which runs an executable. +// in the background. It communicates with this processor via stdin/stdout pipes. +func NewStdIOExecutable(bin string, args []string, env []string) (process.ResourceStreamProcessor, error) { + e := stdIOExecutable{ + bin: bin, + args: args, + env: env, + } + + return &e, nil +} + +func (e *stdIOExecutable) Process(ctx context.Context, r io.Reader, w io.Writer) error { + cmd := exec.CommandContext(ctx, e.bin, e.args...) + cmd.Env = e.env stdin, err := cmd.StdinPipe() if err != nil { - return nil, err + return fmt.Errorf("unable to get stdin pipe: %w", err) } stdout, err := cmd.StdoutPipe() if err != nil { - return nil, err + return fmt.Errorf("unable to get stdout pipe: %w", err) } cmd.Stderr = os.Stderr - e := stdIOExecutable{ - processor: cmd, - stdin: stdin, - stdout: stdout, - } - - return &e, nil -} - -func (e *stdIOExecutable) Process(ctx context.Context, r io.Reader, w io.Writer) error { - if err := e.processor.Start(); err != nil { + if err := cmd.Start(); err != nil { return fmt.Errorf("unable to start processor: %w", err) } - if _, err := io.Copy(e.stdin, r); err != nil { + if _, err := io.Copy(stdin, r); err != nil { return fmt.Errorf("unable to write input: %w", err) } - if err := e.stdin.Close(); err != nil { + if err := stdin.Close(); err != nil { return fmt.Errorf("unable to close input writer: %w", err) } - if _, err := io.Copy(w, e.stdout); err != nil { + if _, err := io.Copy(w, stdout); err != nil { return fmt.Errorf("unable to read output: %w", err) } - if err := e.processor.Wait(); err != nil { - return fmt.Errorf("unable to stop processor: %w", err) + if err := cmd.Wait(); err != nil { + return fmt.Errorf("unable to wait for processor: %w", err) } return nil diff --git a/pkg/transport/process/extensions/uds_executable.go b/pkg/transport/process/extensions/uds_executable.go index 6ac31004..eb0dcd97 100644 --- a/pkg/transport/process/extensions/uds_executable.go +++ b/pkg/transport/process/extensions/uds_executable.go @@ -10,6 +10,7 @@ import ( "net" "os" "os/exec" + "strings" "syscall" "time" @@ -17,20 +18,23 @@ import ( "github.com/gardener/component-cli/pkg/utils" ) -const serverAddressFlag = "--addr" +// ServerAddressEnv is the environment variable key which is used for propagating the +// address under which a processor server should start to a processor binary. +const ServerAddressEnv = "SERVER_ADDRESS" type udsExecutable struct { - processor *exec.Cmd - addr string - conn net.Conn + bin string + args []string + env []string + addr string } // NewUDSExecutable runs a resource processor extension executable in the background. // It communicates with this processor via Unix Domain Sockets. -func NewUDSExecutable(ctx context.Context, bin string, args []string, env []string) (process.ResourceStreamProcessor, error) { - for _, arg := range args { - if arg == serverAddressFlag { - return nil, fmt.Errorf("the flag %s is not allowed to be set manually", serverAddressFlag) +func NewUDSExecutable(bin string, args []string, env []string) (process.ResourceStreamProcessor, error) { + for _, e := range env { + if strings.HasPrefix(e, ServerAddressEnv+"=") { + return nil, fmt.Errorf("the env variable %s is not allowed to be set manually", ServerAddressEnv) } } @@ -39,51 +43,52 @@ func NewUDSExecutable(ctx context.Context, bin string, args []string, env []stri return nil, err } addr := fmt.Sprintf("%s/%s.sock", wd, utils.RandomString(8)) - args = append(args, "--addr", addr) + env = append(env, fmt.Sprintf("%s=%s", ServerAddressEnv, addr)) - cmd := exec.CommandContext(ctx, bin, args...) - cmd.Env = env + e := udsExecutable{ + bin: bin, + args: args, + env: env, + addr: addr, + } + + return &e, nil +} + +func (e *udsExecutable) Process(ctx context.Context, r io.Reader, w io.Writer) error { + cmd := exec.CommandContext(ctx, e.bin, e.args...) + cmd.Env = e.env cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("unable to start processor: %w", err) + return fmt.Errorf("unable to start processor: %w", err) } - conn, err := tryConnect(addr) + conn, err := tryConnect(e.addr) if err != nil { - return nil, fmt.Errorf("unable to connect to processor: %w", err) + return fmt.Errorf("unable to connect to processor: %w", err) } - e := udsExecutable{ - processor: cmd, - addr: addr, - conn: conn, - } - - return &e, nil -} - -func (e *udsExecutable) Process(ctx context.Context, r io.Reader, w io.Writer) error { - if _, err := io.Copy(e.conn, r); err != nil { + if _, err := io.Copy(conn, r); err != nil { return fmt.Errorf("unable to write input: %w", err) } - usock := e.conn.(*net.UnixConn) + usock := conn.(*net.UnixConn) if err := usock.CloseWrite(); err != nil { return fmt.Errorf("unable to close input writer: %w", err) } - if _, err := io.Copy(w, e.conn); err != nil { + if _, err := io.Copy(w, conn); err != nil { return fmt.Errorf("unable to read output: %w", err) } - if err := e.processor.Process.Signal(syscall.SIGTERM); err != nil { + if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { return fmt.Errorf("unable to send SIGTERM to processor: %w", err) } // extension servers must implement ordinary shutdown (!) - if err := e.processor.Wait(); err != nil { + if err := cmd.Wait(); err != nil { return fmt.Errorf("unable to wait for processor: %w", err) } diff --git a/pkg/transport/process/pipeline.go b/pkg/transport/process/pipeline.go index 77941189..41dd01fe 100644 --- a/pkg/transport/process/pipeline.go +++ b/pkg/transport/process/pipeline.go @@ -7,6 +7,7 @@ import ( "context" "io" "os" + "time" "fmt" "io/ioutil" @@ -14,6 +15,8 @@ import ( cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" ) +const processorTimeout = 30 * time.Second + type resourceProcessingPipelineImpl struct { processors []ResourceStreamProcessor } @@ -66,6 +69,9 @@ func (p *resourceProcessingPipelineImpl) process(ctx context.Context, infile *os inreader := infile outwriter := outfile + ctx, cancelfunc := context.WithTimeout(ctx, processorTimeout) + defer cancelfunc() + if err := proc.Process(ctx, inreader, outwriter); err != nil { return nil, fmt.Errorf("unable to process resource: %w", err) } diff --git a/pkg/transport/process/processors/test.go b/pkg/transport/process/processors/example/main.go similarity index 57% rename from pkg/transport/process/processors/test.go rename to pkg/transport/process/processors/example/main.go index 2ecc0b9c..f5c66780 100644 --- a/pkg/transport/process/processors/test.go +++ b/pkg/transport/process/processors/example/main.go @@ -6,101 +6,44 @@ package main import ( "bytes" "encoding/json" - "flag" "fmt" "io" "io/ioutil" "log" - "net" "os" "os/signal" "strings" - "sync" "syscall" cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" "github.com/gardener/component-cli/pkg/transport/process" + "github.com/gardener/component-cli/pkg/transport/process/extensions" + "github.com/gardener/component-cli/pkg/transport/process/processors" ) -const processorName = "test-processor" - -type ProcessorHandlerFunc func(io.Reader, io.WriteCloser) - -type Server struct { - listener net.Listener - quit chan interface{} - wg sync.WaitGroup - handler ProcessorHandlerFunc -} - -func NewServer(addr string, h ProcessorHandlerFunc) (*Server, error) { - l, err := net.Listen("unix", addr) - if err != nil { - return nil, err - } - s := &Server{ - quit: make(chan interface{}), - listener: l, - handler: h, - } - return s, nil -} - -func (s *Server) Start() { - s.wg.Add(1) - go s.serve() -} - -func (s *Server) serve() { - defer s.wg.Done() - - for { - conn, err := s.listener.Accept() - if err != nil { - select { - case <-s.quit: - return - default: - log.Println("accept error", err) - } - } else { - s.wg.Add(1) - go func() { - defer s.wg.Done() - s.handler(conn, conn) - }() - } - } -} - -func (s *Server) Stop() { - close(s.quit) - if err := s.listener.Close(); err != nil { - println(err) - } - s.wg.Wait() -} +const processorName = "example-processor" +// a test processor which adds its name to the resource labels and the resource blob. +// the resource blob is expected to be plain text data. func main() { - addr := flag.String("addr", "", "") - flag.Parse() + addr := os.Getenv(extensions.ServerAddressEnv) - if *addr == "" { + if addr == "" { // if addr is not set, use stdin/stdout for communication - if err := ProcessorRoutine(os.Stdin, os.Stdout); err != nil { + if err := processorRoutine(os.Stdin, os.Stdout); err != nil { log.Fatal(err) } return } h := func(r io.Reader, w io.WriteCloser) { - if err := ProcessorRoutine(r, w); err != nil { + if err := processorRoutine(r, w); err != nil { log.Fatal(err) } } - srv, err := NewServer(*addr, h) + srv, err := processors.NewUDSServer(addr, h) if err != nil { log.Fatal(err) } @@ -114,7 +57,7 @@ func main() { srv.Stop() } -func ProcessorRoutine(inputStream io.Reader, outputStream io.WriteCloser) error { +func processorRoutine(inputStream io.Reader, outputStream io.WriteCloser) error { defer outputStream.Close() tmpfile, err := ioutil.TempFile("", "") diff --git a/pkg/transport/process/processors/sleep/main.go b/pkg/transport/process/processors/sleep/main.go new file mode 100644 index 00000000..477c8c0c --- /dev/null +++ b/pkg/transport/process/processors/sleep/main.go @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package main + +import ( + "io" + "log" + "os" + "os/signal" + "syscall" + "time" + + "github.com/gardener/component-cli/pkg/transport/process/extensions" + "github.com/gardener/component-cli/pkg/transport/process/processors" +) + +const sleepTimeEnv = "SLEEP_TIME" + +// a test processor which sleeps for a configurable duration and then exists with an error. +func main() { + sleepTime, err := time.ParseDuration(os.Getenv(sleepTimeEnv)) + if err != nil { + log.Fatal(err) + } + + addr := os.Getenv(extensions.ServerAddressEnv) + + if addr == "" { + time.Sleep(sleepTime) + log.Fatal("finished sleeping -> exit with error") + } + + h := func(r io.Reader, w io.WriteCloser) { + time.Sleep(sleepTime) + log.Fatal("finished sleeping -> exit with error") + } + + srv, err := processors.NewUDSServer(addr, h) + if err != nil { + log.Fatal(err) + } + + srv.Start() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + <-stop + + srv.Stop() +} diff --git a/pkg/transport/process/processors/util.go b/pkg/transport/process/processors/util.go new file mode 100644 index 00000000..8109721c --- /dev/null +++ b/pkg/transport/process/processors/util.go @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package processors + +import ( + "io" + "log" + "net" + "sync" +) + +type ProcessorHandlerFunc func(io.Reader, io.WriteCloser) + +type UDSServer struct { + listener net.Listener + quit chan interface{} + wg sync.WaitGroup + handler ProcessorHandlerFunc +} + +func NewUDSServer(addr string, h ProcessorHandlerFunc) (*UDSServer, error) { + l, err := net.Listen("unix", addr) + if err != nil { + return nil, err + } + s := &UDSServer{ + quit: make(chan interface{}), + listener: l, + handler: h, + } + return s, nil +} + +func (s *UDSServer) Start() { + s.wg.Add(1) + go s.serve() +} + +func (s *UDSServer) serve() { + defer s.wg.Done() + + for { + conn, err := s.listener.Accept() + if err != nil { + select { + case <-s.quit: + return + default: + log.Println("accept error", err) + } + } else { + s.wg.Add(1) + go func() { + defer s.wg.Done() + s.handler(conn, conn) + }() + } + } +} + +func (s *UDSServer) Stop() { + close(s.quit) + if err := s.listener.Close(); err != nil { + println(err) + } + s.wg.Wait() +} diff --git a/pkg/transport/process/util_test.go b/pkg/transport/process/util_test.go index b10b596f..91f5b569 100644 --- a/pkg/transport/process/util_test.go +++ b/pkg/transport/process/util_test.go @@ -15,7 +15,7 @@ import ( "github.com/gardener/component-cli/pkg/transport/process" ) -var _ = Describe("utils", func() { +var _ = Describe("util", func() { Context("WriteProcessMessage & ReadProcessMessage", func() { From acc06bedcf4594c0430b7c17e12fb0082f12e9bc Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Tue, 21 Sep 2021 16:22:09 +0200 Subject: [PATCH 12/24] refactoring + updates doc --- pkg/transport/process/extensions/extensions_suite_test.go | 3 ++- pkg/transport/process/types.go | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/transport/process/extensions/extensions_suite_test.go b/pkg/transport/process/extensions/extensions_suite_test.go index c70458a3..1b38dfff 100644 --- a/pkg/transport/process/extensions/extensions_suite_test.go +++ b/pkg/transport/process/extensions/extensions_suite_test.go @@ -26,7 +26,6 @@ const ( exampleProcessorBinaryPath = "../../../../tmp/test/bin/example-processor" sleepProcessorBinaryPath = "../../../../tmp/test/bin/sleep-processor" sleepTimeEnv = "SLEEP_TIME" - timeout = 2 * time.Second sleepTime = 5 * time.Second ) @@ -101,6 +100,8 @@ var _ = Describe("transport extensions", func() { }) func runTimeoutTest(processor process.ResourceStreamProcessor) { + const timeout = 2 * time.Second + ctx, cancelfunc := context.WithTimeout(context.TODO(), timeout) defer cancelfunc() diff --git a/pkg/transport/process/types.go b/pkg/transport/process/types.go index 6e970285..1004f940 100644 --- a/pkg/transport/process/types.go +++ b/pkg/transport/process/types.go @@ -23,7 +23,8 @@ type ResourceProcessingPipeline interface { // ResourceStreamProcessor describes an individual processor for processing a resource. // A processor can upload, modify, or download a resource. type ResourceStreamProcessor interface { - // Process executes the processor for a resource. Input and Output streams must be TAR - // archives which contain the component descriptor, resource, and resource blob. + // Process executes the processor for a resource. Input and Output streams must be + // compliant to a specific format ("processor message"). See also ./util.go for helper + // functions to read/write processor messages. Process(context.Context, io.Reader, io.Writer) error } From e41fd557fd78b8bc24246eae4dc52ab34fe1cfdf Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Wed, 22 Sep 2021 14:53:23 +0200 Subject: [PATCH 13/24] refactoring, adds pipeline tests, adds labelling processor --- pkg/transport/process/pipeline.go | 4 +- pkg/transport/process/pipeline_test.go | 63 +++++++++++++++++++ pkg/transport/process/processors/labelling.go | 44 +++++++++++++ .../process/processors/labelling_test.go | 4 ++ pkg/transport/process/types.go | 2 +- pkg/transport/process/util.go | 10 +-- pkg/transport/process/util_test.go | 4 +- 7 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 pkg/transport/process/pipeline_test.go create mode 100644 pkg/transport/process/processors/labelling.go create mode 100644 pkg/transport/process/processors/labelling_test.go diff --git a/pkg/transport/process/pipeline.go b/pkg/transport/process/pipeline.go index 41dd01fe..40f9ef35 100644 --- a/pkg/transport/process/pipeline.go +++ b/pkg/transport/process/pipeline.go @@ -49,7 +49,9 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co if err != nil { return nil, cdv2.Resource{}, fmt.Errorf("unable to read output data: %w", err) } - defer blobreader.Close() + if blobreader != nil { + defer blobreader.Close() + } return processedCD, processedRes, nil } diff --git a/pkg/transport/process/pipeline_test.go b/pkg/transport/process/pipeline_test.go new file mode 100644 index 00000000..c0a0f4c5 --- /dev/null +++ b/pkg/transport/process/pipeline_test.go @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package process_test + +import ( + "context" + "encoding/json" + + cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/gardener/component-cli/pkg/transport/process" + "github.com/gardener/component-cli/pkg/transport/process/processors" +) + +var _ = Describe("pipeline", func() { + + Context("Process", func() { + + It("should correctly process resource", func() { + res := cdv2.Resource{ + IdentityObjectMeta: cdv2.IdentityObjectMeta{ + Name: "my-res", + Version: "v0.1.0", + Type: "ociImage", + }, + } + + l1 := cdv2.Label{ + Name: "processor-0", + Value: json.RawMessage(`"true"`), + } + l2 := cdv2.Label{ + Name: "processor-1", + Value: json.RawMessage(`"true"`), + } + expectedRes := res + expectedRes.Labels = append(expectedRes.Labels, l1) + expectedRes.Labels = append(expectedRes.Labels, l2) + + cd := cdv2.ComponentDescriptor{ + ComponentSpec: cdv2.ComponentSpec{ + Resources: []cdv2.Resource{ + res, + }, + }, + } + + p1 := processors.NewLabellingProcessor(l1) + p2 := processors.NewLabellingProcessor(l2) + pipeline := process.NewResourceProcessingPipeline(p1, p2) + + actualCD, actualRes, err := pipeline.Process(context.TODO(), cd, res) + Expect(err).ToNot(HaveOccurred()) + + Expect(*actualCD).To(Equal(cd)) + Expect(actualRes).To(Equal(expectedRes)) + }) + + }) +}) diff --git a/pkg/transport/process/processors/labelling.go b/pkg/transport/process/processors/labelling.go new file mode 100644 index 00000000..7cc17e39 --- /dev/null +++ b/pkg/transport/process/processors/labelling.go @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier +package processors + +import ( + "context" + "fmt" + "io" + + cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" + + "github.com/gardener/component-cli/pkg/transport/process" +) + +type labellingProcessor struct { + labels cdv2.Labels +} + +// NewLabellingProcessor returns a processor that appends one or more labels to a resource +func NewLabellingProcessor(labels ...cdv2.Label) process.ResourceStreamProcessor { + obj := labellingProcessor{ + labels: labels, + } + return &obj +} + +func (p *labellingProcessor) Process(ctx context.Context, r io.Reader, w io.Writer) error { + cd, res, resBlobReader, err := process.ReadProcessorMessage(r) + if err != nil { + return fmt.Errorf("unable to read processor message: %w", err) + } + if resBlobReader != nil { + defer resBlobReader.Close() + } + + res.Labels = append(res.Labels, p.labels...) + + if err := process.WriteProcessorMessage(*cd, res, resBlobReader, w); err != nil { + return fmt.Errorf("unable to write processor message: %w", err) + } + + return nil +} diff --git a/pkg/transport/process/processors/labelling_test.go b/pkg/transport/process/processors/labelling_test.go new file mode 100644 index 00000000..556ce187 --- /dev/null +++ b/pkg/transport/process/processors/labelling_test.go @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier +package processors_test diff --git a/pkg/transport/process/types.go b/pkg/transport/process/types.go index 1004f940..9889d07e 100644 --- a/pkg/transport/process/types.go +++ b/pkg/transport/process/types.go @@ -24,7 +24,7 @@ type ResourceProcessingPipeline interface { // A processor can upload, modify, or download a resource. type ResourceStreamProcessor interface { // Process executes the processor for a resource. Input and Output streams must be - // compliant to a specific format ("processor message"). See also ./util.go for helper + // compliant to a specific format ("processor message"). See also ./util.go for helper // functions to read/write processor messages. Process(context.Context, io.Reader, io.Writer) error } diff --git a/pkg/transport/process/util.go b/pkg/transport/process/util.go index 880b6a6b..c4a2268f 100644 --- a/pkg/transport/process/util.go +++ b/pkg/transport/process/util.go @@ -138,10 +138,12 @@ func ReadProcessorMessage(r io.Reader) (*cdv2.ComponentDescriptor, cdv2.Resource } } - if f != nil { - if _, err := f.Seek(0, io.SeekStart); err != nil { - return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to seek to beginning of file: %w", err) - } + if f == nil { + return cd, res, nil, nil + } + + if _, err := f.Seek(0, io.SeekStart); err != nil { + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to seek to beginning of file: %w", err) } return cd, res, f, nil diff --git a/pkg/transport/process/util_test.go b/pkg/transport/process/util_test.go index 91f5b569..c668e5cb 100644 --- a/pkg/transport/process/util_test.go +++ b/pkg/transport/process/util_test.go @@ -17,9 +17,9 @@ import ( var _ = Describe("util", func() { - Context("WriteProcessMessage & ReadProcessMessage", func() { + Context("WriteProcessorMessage & ReadProcessorMessage", func() { - It("should correctly write and read a process message", func() { + It("should correctly write and read a processor message", func() { res := cdv2.Resource{ IdentityObjectMeta: cdv2.IdentityObjectMeta{ Name: "my-res", From e62058bd76e75efd54f339fc7ba82a98d85ba9ba Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Wed, 13 Oct 2021 14:57:35 +0200 Subject: [PATCH 14/24] refactoring and godoc --- .../process/extensions/uds_executable.go | 4 +- .../process/processors/example/main.go | 3 +- .../process/processors/sleep/main.go | 6 +- pkg/transport/process/processors/util.go | 68 ------------------- pkg/transport/process/util.go | 66 ++++++++++++++++++ 5 files changed, 72 insertions(+), 75 deletions(-) delete mode 100644 pkg/transport/process/processors/util.go diff --git a/pkg/transport/process/extensions/uds_executable.go b/pkg/transport/process/extensions/uds_executable.go index eb0dcd97..be078160 100644 --- a/pkg/transport/process/extensions/uds_executable.go +++ b/pkg/transport/process/extensions/uds_executable.go @@ -18,8 +18,8 @@ import ( "github.com/gardener/component-cli/pkg/utils" ) -// ServerAddressEnv is the environment variable key which is used for propagating the -// address under which a processor server should start to a processor binary. +// ServerAddressEnv is the environment variable key which is used to store the +// address under which a resource processor server should start. const ServerAddressEnv = "SERVER_ADDRESS" type udsExecutable struct { diff --git a/pkg/transport/process/processors/example/main.go b/pkg/transport/process/processors/example/main.go index f5c66780..cc581f3a 100644 --- a/pkg/transport/process/processors/example/main.go +++ b/pkg/transport/process/processors/example/main.go @@ -19,7 +19,6 @@ import ( "github.com/gardener/component-cli/pkg/transport/process" "github.com/gardener/component-cli/pkg/transport/process/extensions" - "github.com/gardener/component-cli/pkg/transport/process/processors" ) const processorName = "example-processor" @@ -43,7 +42,7 @@ func main() { } } - srv, err := processors.NewUDSServer(addr, h) + srv, err := process.NewUDSServer(addr, h) if err != nil { log.Fatal(err) } diff --git a/pkg/transport/process/processors/sleep/main.go b/pkg/transport/process/processors/sleep/main.go index 477c8c0c..ad8b4e28 100644 --- a/pkg/transport/process/processors/sleep/main.go +++ b/pkg/transport/process/processors/sleep/main.go @@ -11,13 +11,13 @@ import ( "syscall" "time" + "github.com/gardener/component-cli/pkg/transport/process" "github.com/gardener/component-cli/pkg/transport/process/extensions" - "github.com/gardener/component-cli/pkg/transport/process/processors" ) const sleepTimeEnv = "SLEEP_TIME" -// a test processor which sleeps for a configurable duration and then exists with an error. +// a test processor which sleeps for a configurable duration and then exits with an error. func main() { sleepTime, err := time.ParseDuration(os.Getenv(sleepTimeEnv)) if err != nil { @@ -36,7 +36,7 @@ func main() { log.Fatal("finished sleeping -> exit with error") } - srv, err := processors.NewUDSServer(addr, h) + srv, err := process.NewUDSServer(addr, h) if err != nil { log.Fatal(err) } diff --git a/pkg/transport/process/processors/util.go b/pkg/transport/process/processors/util.go deleted file mode 100644 index 8109721c..00000000 --- a/pkg/transport/process/processors/util.go +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. -// -// SPDX-License-Identifier: Apache-2.0 -package processors - -import ( - "io" - "log" - "net" - "sync" -) - -type ProcessorHandlerFunc func(io.Reader, io.WriteCloser) - -type UDSServer struct { - listener net.Listener - quit chan interface{} - wg sync.WaitGroup - handler ProcessorHandlerFunc -} - -func NewUDSServer(addr string, h ProcessorHandlerFunc) (*UDSServer, error) { - l, err := net.Listen("unix", addr) - if err != nil { - return nil, err - } - s := &UDSServer{ - quit: make(chan interface{}), - listener: l, - handler: h, - } - return s, nil -} - -func (s *UDSServer) Start() { - s.wg.Add(1) - go s.serve() -} - -func (s *UDSServer) serve() { - defer s.wg.Done() - - for { - conn, err := s.listener.Accept() - if err != nil { - select { - case <-s.quit: - return - default: - log.Println("accept error", err) - } - } else { - s.wg.Add(1) - go func() { - defer s.wg.Done() - s.handler(conn, conn) - }() - } - } -} - -func (s *UDSServer) Stop() { - close(s.quit) - if err := s.listener.Close(); err != nil { - println(err) - } - s.wg.Wait() -} diff --git a/pkg/transport/process/util.go b/pkg/transport/process/util.go index c4a2268f..5019d077 100644 --- a/pkg/transport/process/util.go +++ b/pkg/transport/process/util.go @@ -9,7 +9,10 @@ import ( "fmt" "io" "io/ioutil" + "log" + "net" "os" + "sync" "time" cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" @@ -176,3 +179,66 @@ func readComponentDescriptor(r *tar.Reader) (*cdv2.ComponentDescriptor, error) { return &cd, nil } + +// HandlerFunc defines the interface of a function that should be served by a UDS server +type HandlerFunc func(io.Reader, io.WriteCloser) + +// UDSServer implements a Unix Domain Socket server +type UDSServer struct { + listener net.Listener + quit chan interface{} + wg sync.WaitGroup + handler HandlerFunc +} + +// NewUDSServer returns a new UDS server. +// The parameters define the server address and the handler func it serves +func NewUDSServer(addr string, handler HandlerFunc) (*UDSServer, error) { + l, err := net.Listen("unix", addr) + if err != nil { + return nil, err + } + s := &UDSServer{ + quit: make(chan interface{}), + listener: l, + handler: handler, + } + return s, nil +} + +// Start starts the server goroutine +func (s *UDSServer) Start() { + s.wg.Add(1) + go s.serve() +} + +func (s *UDSServer) serve() { + defer s.wg.Done() + + for { + conn, err := s.listener.Accept() + if err != nil { + select { + case <-s.quit: + return + default: + log.Println("accept error", err) + } + } else { + s.wg.Add(1) + go func() { + defer s.wg.Done() + s.handler(conn, conn) + }() + } + } +} + +// Stop stops the server goroutine +func (s *UDSServer) Stop() { + close(s.quit) + if err := s.listener.Close(); err != nil { + println(err) + } + s.wg.Wait() +} From 5b1ab2b78bbafb1b819154526b750eda68ca5a1d Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Tue, 2 Nov 2021 16:31:52 +0100 Subject: [PATCH 15/24] moves function to utils package and adds tests --- pkg/transport/process/util.go | 47 ++-------------- pkg/transport/process/util_test.go | 1 + pkg/utils/utils.go | 52 ++++++++++++++++++ pkg/utils/utils_suite_test.go | 16 ++++++ pkg/utils/utils_test.go | 86 ++++++++++++++++++++++++++++++ 5 files changed, 160 insertions(+), 42 deletions(-) create mode 100644 pkg/utils/utils_suite_test.go create mode 100644 pkg/utils/utils_test.go diff --git a/pkg/transport/process/util.go b/pkg/transport/process/util.go index 5019d077..a991c89a 100644 --- a/pkg/transport/process/util.go +++ b/pkg/transport/process/util.go @@ -13,10 +13,11 @@ import ( "net" "os" "sync" - "time" cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" "sigs.k8s.io/yaml" + + "github.com/gardener/component-cli/pkg/utils" ) const ( @@ -42,7 +43,7 @@ func WriteProcessorMessage(cd cdv2.ComponentDescriptor, res cdv2.Resource, resou return fmt.Errorf("unable to marshal component descriptor: %w", err) } - if err := writeFileToTARArchive(ComponentDescriptorFile, bytes.NewReader(marshaledCD), tw); err != nil { + if err := utils.WriteFileToTARArchive(ComponentDescriptorFile, bytes.NewReader(marshaledCD), tw); err != nil { return fmt.Errorf("unable to write %s: %w", ComponentDescriptorFile, err) } @@ -51,12 +52,12 @@ func WriteProcessorMessage(cd cdv2.ComponentDescriptor, res cdv2.Resource, resou return fmt.Errorf("unable to marshal resource: %w", err) } - if err := writeFileToTARArchive(ResourceFile, bytes.NewReader(marshaledRes), tw); err != nil { + if err := utils.WriteFileToTARArchive(ResourceFile, bytes.NewReader(marshaledRes), tw); err != nil { return fmt.Errorf("unable to write %s: %w", ResourceFile, err) } if resourceBlobReader != nil { - if err := writeFileToTARArchive(ResourceBlobFile, resourceBlobReader, tw); err != nil { + if err := utils.WriteFileToTARArchive(ResourceBlobFile, resourceBlobReader, tw); err != nil { return fmt.Errorf("unable to write %s: %w", ResourceBlobFile, err) } } @@ -64,44 +65,6 @@ func WriteProcessorMessage(cd cdv2.ComponentDescriptor, res cdv2.Resource, resou return nil } -func writeFileToTARArchive(filename string, contentReader io.Reader, outArchive *tar.Writer) error { - tempfile, err := ioutil.TempFile("", "") - if err != nil { - return fmt.Errorf("unable to create tempfile: %w", err) - } - defer tempfile.Close() - - if _, err := io.Copy(tempfile, contentReader); err != nil { - return fmt.Errorf("unable to write content to file: %w", err) - } - - if _, err := tempfile.Seek(0, io.SeekStart); err != nil { - return fmt.Errorf("unable to seek to beginning of file: %w", err) - } - - fstat, err := tempfile.Stat() - if err != nil { - return fmt.Errorf("unable to get file info: %w", err) - } - - header := tar.Header{ - Name: filename, - Size: fstat.Size(), - Mode: int64(fstat.Mode()), - ModTime: time.Now(), - } - - if err := outArchive.WriteHeader(&header); err != nil { - return fmt.Errorf("unable to write tar header: %w", err) - } - - if _, err := io.Copy(outArchive, tempfile); err != nil { - return fmt.Errorf("unable to write file to tar archive: %w", err) - } - - return nil -} - // ReadProcessorMessage reads the component descriptor, resource and resource blob from a processor message // (tar archive with fixed filenames for component descriptor, resource, and resource blob) which is // produced by processors. The resource blob reader can be nil. If a non-nil value is returned, it must diff --git a/pkg/transport/process/util_test.go b/pkg/transport/process/util_test.go index c668e5cb..cd3c40b3 100644 --- a/pkg/transport/process/util_test.go +++ b/pkg/transport/process/util_test.go @@ -54,4 +54,5 @@ var _ = Describe("util", func() { }) }) + }) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index e32b2fee..e136a934 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -5,15 +5,20 @@ package utils import ( + "archive/tar" "bytes" "compress/gzip" "encoding/json" + "errors" "fmt" + "io" + "io/ioutil" "math/rand" "net/http" "os" "path/filepath" "strings" + "time" "github.com/mandelsoft/vfs/pkg/vfs" "github.com/spf13/cobra" @@ -168,3 +173,50 @@ func BytesString(bytes uint64, accuracy int) string { return fmt.Sprintf("%s %s", stringValue, unit) } + +// WriteFileToTARArchive writes a new file with name=filename and content=contentReader to archiveWriter +func WriteFileToTARArchive(filename string, contentReader io.Reader, archiveWriter *tar.Writer) error { + if filename == "" { + return errors.New("filename must not be empty") + } + + if contentReader == nil { + return errors.New("contentReader must not be nil") + } + + if archiveWriter == nil { + return errors.New("archiveWriter must not be nil") + } + + tempfile, err := ioutil.TempFile("", "") + if err != nil { + return fmt.Errorf("unable to create tempfile: %w", err) + } + defer tempfile.Close() + + fsize, err := io.Copy(tempfile, contentReader) + if err != nil { + return fmt.Errorf("unable to copy content to tempfile: %w", err) + } + + if _, err := tempfile.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("unable to seek to beginning of tempfile: %w", err) + } + + header := tar.Header{ + Name: filename, + Size: int64(fsize), + Mode: 0600, + ModTime: time.Now(), + } + + if err := archiveWriter.WriteHeader(&header); err != nil { + return fmt.Errorf("unable to write tar header: %w", err) + } + + if _, err := io.Copy(archiveWriter, tempfile); err != nil { + return fmt.Errorf("unable to write file to tar archive: %w", err) + } + + return nil +} diff --git a/pkg/utils/utils_suite_test.go b/pkg/utils/utils_suite_test.go new file mode 100644 index 00000000..03afdd59 --- /dev/null +++ b/pkg/utils/utils_suite_test.go @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package utils_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Utils Test Suite") +} diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go new file mode 100644 index 00000000..1fcc428f --- /dev/null +++ b/pkg/utils/utils_test.go @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package utils_test + +import ( + "archive/tar" + "bytes" + "io" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/gardener/component-cli/pkg/utils" +) + +var _ = Describe("utils", func() { + + Context("WriteFileToTARArchive", func() { + + It("should write file", func() { + fname := "testfile" + content := []byte("testcontent") + + archiveBuf := bytes.NewBuffer([]byte{}) + tw := tar.NewWriter(archiveBuf) + + Expect(utils.WriteFileToTARArchive(fname, bytes.NewReader(content), tw)).To(Succeed()) + Expect(tw.Close()).To(Succeed()) + + tr := tar.NewReader(archiveBuf) + fheader, err := tr.Next() + Expect(err).ToNot(HaveOccurred()) + Expect(fheader.Name).To(Equal(fname)) + + actualContentBuf := bytes.NewBuffer([]byte{}) + _, err = io.Copy(actualContentBuf, tr) + Expect(err).ToNot(HaveOccurred()) + Expect(actualContentBuf.Bytes()).To(Equal(content)) + + _, err = tr.Next() + Expect(err).To(Equal(io.EOF)) + }) + + It("should write empty file", func() { + fname := "testfile" + + archiveBuf := bytes.NewBuffer([]byte{}) + tw := tar.NewWriter(archiveBuf) + + Expect(utils.WriteFileToTARArchive(fname, bytes.NewReader([]byte{}), tw)).To(Succeed()) + Expect(tw.Close()).To(Succeed()) + + tr := tar.NewReader(archiveBuf) + fheader, err := tr.Next() + Expect(err).ToNot(HaveOccurred()) + Expect(fheader.Name).To(Equal(fname)) + + actualContentBuf := bytes.NewBuffer([]byte{}) + contentLenght, err := io.Copy(actualContentBuf, tr) + Expect(err).ToNot(HaveOccurred()) + Expect(contentLenght).To(Equal(int64(0))) + + _, err = tr.Next() + Expect(err).To(Equal(io.EOF)) + }) + + It("should return error if filename is empty", func() { + tw := tar.NewWriter(bytes.NewBuffer([]byte{})) + contentReader := bytes.NewReader([]byte{}) + Expect(utils.WriteFileToTARArchive("", contentReader, tw)).To(MatchError("filename must not be empty")) + }) + + It("should return error if contentReader is nil", func() { + tw := tar.NewWriter(bytes.NewBuffer([]byte{})) + Expect(utils.WriteFileToTARArchive("testfile", nil, tw)).To(MatchError("contentReader must not be nil")) + }) + + It("should return error if outArchive is nil", func() { + contentReader := bytes.NewReader([]byte{}) + Expect(utils.WriteFileToTARArchive("testfile", contentReader, nil)).To(MatchError("archiveWriter must not be nil")) + }) + + }) + +}) From e5fc684103e610b78ecd19b9fd98a56e8b95c81b Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Wed, 3 Nov 2021 08:58:49 +0100 Subject: [PATCH 16/24] renames labeling processor and adds test --- pkg/transport/process/pipeline_test.go | 4 +- .../process/processors/labelling_test.go | 4 - .../processors/processors_suite_test.go | 16 ++++ .../{labelling.go => resource_labeler.go} | 12 +-- .../processors/resource_labeler_test.go | 76 +++++++++++++++++++ 5 files changed, 100 insertions(+), 12 deletions(-) delete mode 100644 pkg/transport/process/processors/labelling_test.go create mode 100644 pkg/transport/process/processors/processors_suite_test.go rename pkg/transport/process/processors/{labelling.go => resource_labeler.go} (68%) create mode 100644 pkg/transport/process/processors/resource_labeler_test.go diff --git a/pkg/transport/process/pipeline_test.go b/pkg/transport/process/pipeline_test.go index c0a0f4c5..baaff70a 100644 --- a/pkg/transport/process/pipeline_test.go +++ b/pkg/transport/process/pipeline_test.go @@ -48,8 +48,8 @@ var _ = Describe("pipeline", func() { }, } - p1 := processors.NewLabellingProcessor(l1) - p2 := processors.NewLabellingProcessor(l2) + p1 := processors.NewResourceLabeler(l1) + p2 := processors.NewResourceLabeler(l2) pipeline := process.NewResourceProcessingPipeline(p1, p2) actualCD, actualRes, err := pipeline.Process(context.TODO(), cd, res) diff --git a/pkg/transport/process/processors/labelling_test.go b/pkg/transport/process/processors/labelling_test.go deleted file mode 100644 index 556ce187..00000000 --- a/pkg/transport/process/processors/labelling_test.go +++ /dev/null @@ -1,4 +0,0 @@ -// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. -// -// SPDX-License-Identifier -package processors_test diff --git a/pkg/transport/process/processors/processors_suite_test.go b/pkg/transport/process/processors/processors_suite_test.go new file mode 100644 index 00000000..b4add5bd --- /dev/null +++ b/pkg/transport/process/processors/processors_suite_test.go @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package processors_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Processors Test Suite") +} diff --git a/pkg/transport/process/processors/labelling.go b/pkg/transport/process/processors/resource_labeler.go similarity index 68% rename from pkg/transport/process/processors/labelling.go rename to pkg/transport/process/processors/resource_labeler.go index 7cc17e39..fba440cd 100644 --- a/pkg/transport/process/processors/labelling.go +++ b/pkg/transport/process/processors/resource_labeler.go @@ -1,6 +1,6 @@ // SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. // -// SPDX-License-Identifier +// SPDX-License-Identifier: Apache-2.0 package processors import ( @@ -13,19 +13,19 @@ import ( "github.com/gardener/component-cli/pkg/transport/process" ) -type labellingProcessor struct { +type resourceLabeler struct { labels cdv2.Labels } -// NewLabellingProcessor returns a processor that appends one or more labels to a resource -func NewLabellingProcessor(labels ...cdv2.Label) process.ResourceStreamProcessor { - obj := labellingProcessor{ +// NewResourceLabeler returns a processor that appends one or more labels to a resource +func NewResourceLabeler(labels ...cdv2.Label) process.ResourceStreamProcessor { + obj := resourceLabeler{ labels: labels, } return &obj } -func (p *labellingProcessor) Process(ctx context.Context, r io.Reader, w io.Writer) error { +func (p *resourceLabeler) Process(ctx context.Context, r io.Reader, w io.Writer) error { cd, res, resBlobReader, err := process.ReadProcessorMessage(r) if err != nil { return fmt.Errorf("unable to read processor message: %w", err) diff --git a/pkg/transport/process/processors/resource_labeler_test.go b/pkg/transport/process/processors/resource_labeler_test.go new file mode 100644 index 00000000..d1ea99eb --- /dev/null +++ b/pkg/transport/process/processors/resource_labeler_test.go @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package processors_test + +import ( + "bytes" + "context" + "encoding/json" + "io" + + "github.com/gardener/component-cli/pkg/transport/process" + "github.com/gardener/component-cli/pkg/transport/process/processors" + cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("resourceLabeler", func() { + + Context("Process", func() { + + It("should correctly add labels", func() { + res := cdv2.Resource{ + IdentityObjectMeta: cdv2.IdentityObjectMeta{ + Name: "my-res", + Version: "v0.1.0", + Type: "ociImage", + }, + } + + l1 := cdv2.Label{ + Name: "first-label", + Value: json.RawMessage(`"true"`), + } + l2 := cdv2.Label{ + Name: "second-label", + Value: json.RawMessage(`"true"`), + } + + resBytes := []byte("resource-blob") + + expectedRes := res + expectedRes.Labels = append(expectedRes.Labels, l1) + expectedRes.Labels = append(expectedRes.Labels, l2) + + cd := cdv2.ComponentDescriptor{ + ComponentSpec: cdv2.ComponentSpec{ + Resources: []cdv2.Resource{ + res, + }, + }, + } + + inBuf := bytes.NewBuffer([]byte{}) + Expect(process.WriteProcessorMessage(cd, res, bytes.NewReader(resBytes), inBuf)).To(Succeed()) + + outbuf := bytes.NewBuffer([]byte{}) + + p1 := processors.NewResourceLabeler(l1, l2) + Expect(p1.Process(context.TODO(), inBuf, outbuf)).To(Succeed()) + + actualCD, actualRes, actualResBlobReader, err := process.ReadProcessorMessage(outbuf) + Expect(err).ToNot(HaveOccurred()) + + Expect(*actualCD).To(Equal(cd)) + Expect(actualRes).To(Equal(expectedRes)) + + actualResBlobBuf := bytes.NewBuffer([]byte{}) + _, err =io.Copy(actualResBlobBuf, actualResBlobReader) + Expect(err).ToNot(HaveOccurred()) + Expect(actualResBlobBuf.Bytes()).To(Equal(resBytes)) + }) + + }) +}) From e1d99f4de978866230fe55685fa8bba45f0a8253 Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Wed, 3 Nov 2021 09:16:15 +0100 Subject: [PATCH 17/24] refactoring --- .../extensions/extensions_suite_test.go | 5 +- pkg/transport/process/pipeline.go | 6 +- .../process/processors/example/main.go | 8 +- .../process/processors/resource_labeler.go | 5 +- .../processors/resource_labeler_test.go | 11 +-- .../process/processors/sleep/main.go | 4 +- pkg/transport/process/utils/uds_server.go | 74 +++++++++++++++++++ pkg/transport/process/{ => utils}/util.go | 68 +---------------- .../process/{ => utils}/util_test.go | 8 +- 9 files changed, 101 insertions(+), 88 deletions(-) create mode 100644 pkg/transport/process/utils/uds_server.go rename pkg/transport/process/{ => utils}/util.go (77%) rename pkg/transport/process/{ => utils}/util_test.go (81%) diff --git a/pkg/transport/process/extensions/extensions_suite_test.go b/pkg/transport/process/extensions/extensions_suite_test.go index 1b38dfff..28df2cd5 100644 --- a/pkg/transport/process/extensions/extensions_suite_test.go +++ b/pkg/transport/process/extensions/extensions_suite_test.go @@ -20,6 +20,7 @@ import ( "github.com/gardener/component-cli/pkg/transport/process" "github.com/gardener/component-cli/pkg/transport/process/extensions" + "github.com/gardener/component-cli/pkg/transport/process/utils" ) const ( @@ -140,14 +141,14 @@ func runExampleResourceTest(processor process.ResourceStreamProcessor) { } inputBuf := bytes.NewBuffer([]byte{}) - err := process.WriteProcessorMessage(cd, res, strings.NewReader(resourceData), inputBuf) + err := utils.WriteProcessorMessage(cd, res, strings.NewReader(resourceData), inputBuf) Expect(err).ToNot(HaveOccurred()) outputBuf := bytes.NewBuffer([]byte{}) err = processor.Process(context.TODO(), inputBuf, outputBuf) Expect(err).ToNot(HaveOccurred()) - processedCD, processedRes, processedBlobReader, err := process.ReadProcessorMessage(outputBuf) + processedCD, processedRes, processedBlobReader, err := utils.ReadProcessorMessage(outputBuf) Expect(err).ToNot(HaveOccurred()) Expect(*processedCD).To(Equal(cd)) diff --git a/pkg/transport/process/pipeline.go b/pkg/transport/process/pipeline.go index 40f9ef35..fe9da4db 100644 --- a/pkg/transport/process/pipeline.go +++ b/pkg/transport/process/pipeline.go @@ -13,6 +13,8 @@ import ( "io/ioutil" cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" + + "github.com/gardener/component-cli/pkg/transport/process/utils" ) const processorTimeout = 30 * time.Second @@ -27,7 +29,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co return nil, cdv2.Resource{}, fmt.Errorf("unable to create temporary infile: %w", err) } - if err := WriteProcessorMessage(cd, res, nil, infile); err != nil { + if err := utils.WriteProcessorMessage(cd, res, nil, infile); err != nil { return nil, cdv2.Resource{}, fmt.Errorf("unable to write: %w", err) } @@ -45,7 +47,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co return nil, cdv2.Resource{}, err } - processedCD, processedRes, blobreader, err := ReadProcessorMessage(infile) + processedCD, processedRes, blobreader, err := utils.ReadProcessorMessage(infile) if err != nil { return nil, cdv2.Resource{}, fmt.Errorf("unable to read output data: %w", err) } diff --git a/pkg/transport/process/processors/example/main.go b/pkg/transport/process/processors/example/main.go index cc581f3a..2bb26490 100644 --- a/pkg/transport/process/processors/example/main.go +++ b/pkg/transport/process/processors/example/main.go @@ -17,8 +17,8 @@ import ( cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" - "github.com/gardener/component-cli/pkg/transport/process" "github.com/gardener/component-cli/pkg/transport/process/extensions" + "github.com/gardener/component-cli/pkg/transport/process/utils" ) const processorName = "example-processor" @@ -42,7 +42,7 @@ func main() { } } - srv, err := process.NewUDSServer(addr, h) + srv, err := utils.NewUDSServer(addr, h) if err != nil { log.Fatal(err) } @@ -73,7 +73,7 @@ func processorRoutine(inputStream io.Reader, outputStream io.WriteCloser) error return err } - cd, res, resourceBlobReader, err := process.ReadProcessorMessage(tmpfile) + cd, res, resourceBlobReader, err := utils.ReadProcessorMessage(tmpfile) if err != nil { return err } @@ -93,7 +93,7 @@ func processorRoutine(inputStream io.Reader, outputStream io.WriteCloser) error } res.Labels = append(res.Labels, l) - if err := process.WriteProcessorMessage(*cd, res, strings.NewReader(outputData), outputStream); err != nil { + if err := utils.WriteProcessorMessage(*cd, res, strings.NewReader(outputData), outputStream); err != nil { return err } diff --git a/pkg/transport/process/processors/resource_labeler.go b/pkg/transport/process/processors/resource_labeler.go index fba440cd..22c3d480 100644 --- a/pkg/transport/process/processors/resource_labeler.go +++ b/pkg/transport/process/processors/resource_labeler.go @@ -11,6 +11,7 @@ import ( cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" "github.com/gardener/component-cli/pkg/transport/process" + "github.com/gardener/component-cli/pkg/transport/process/utils" ) type resourceLabeler struct { @@ -26,7 +27,7 @@ func NewResourceLabeler(labels ...cdv2.Label) process.ResourceStreamProcessor { } func (p *resourceLabeler) Process(ctx context.Context, r io.Reader, w io.Writer) error { - cd, res, resBlobReader, err := process.ReadProcessorMessage(r) + cd, res, resBlobReader, err := utils.ReadProcessorMessage(r) if err != nil { return fmt.Errorf("unable to read processor message: %w", err) } @@ -36,7 +37,7 @@ func (p *resourceLabeler) Process(ctx context.Context, r io.Reader, w io.Writer) res.Labels = append(res.Labels, p.labels...) - if err := process.WriteProcessorMessage(*cd, res, resBlobReader, w); err != nil { + if err := utils.WriteProcessorMessage(*cd, res, resBlobReader, w); err != nil { return fmt.Errorf("unable to write processor message: %w", err) } diff --git a/pkg/transport/process/processors/resource_labeler_test.go b/pkg/transport/process/processors/resource_labeler_test.go index d1ea99eb..263d7a39 100644 --- a/pkg/transport/process/processors/resource_labeler_test.go +++ b/pkg/transport/process/processors/resource_labeler_test.go @@ -9,11 +9,12 @@ import ( "encoding/json" "io" - "github.com/gardener/component-cli/pkg/transport/process" - "github.com/gardener/component-cli/pkg/transport/process/processors" cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "github.com/gardener/component-cli/pkg/transport/process/processors" + "github.com/gardener/component-cli/pkg/transport/process/utils" ) var _ = Describe("resourceLabeler", func() { @@ -53,21 +54,21 @@ var _ = Describe("resourceLabeler", func() { } inBuf := bytes.NewBuffer([]byte{}) - Expect(process.WriteProcessorMessage(cd, res, bytes.NewReader(resBytes), inBuf)).To(Succeed()) + Expect(utils.WriteProcessorMessage(cd, res, bytes.NewReader(resBytes), inBuf)).To(Succeed()) outbuf := bytes.NewBuffer([]byte{}) p1 := processors.NewResourceLabeler(l1, l2) Expect(p1.Process(context.TODO(), inBuf, outbuf)).To(Succeed()) - actualCD, actualRes, actualResBlobReader, err := process.ReadProcessorMessage(outbuf) + actualCD, actualRes, actualResBlobReader, err := utils.ReadProcessorMessage(outbuf) Expect(err).ToNot(HaveOccurred()) Expect(*actualCD).To(Equal(cd)) Expect(actualRes).To(Equal(expectedRes)) actualResBlobBuf := bytes.NewBuffer([]byte{}) - _, err =io.Copy(actualResBlobBuf, actualResBlobReader) + _, err = io.Copy(actualResBlobBuf, actualResBlobReader) Expect(err).ToNot(HaveOccurred()) Expect(actualResBlobBuf.Bytes()).To(Equal(resBytes)) }) diff --git a/pkg/transport/process/processors/sleep/main.go b/pkg/transport/process/processors/sleep/main.go index ad8b4e28..7be914c7 100644 --- a/pkg/transport/process/processors/sleep/main.go +++ b/pkg/transport/process/processors/sleep/main.go @@ -11,8 +11,8 @@ import ( "syscall" "time" - "github.com/gardener/component-cli/pkg/transport/process" "github.com/gardener/component-cli/pkg/transport/process/extensions" + "github.com/gardener/component-cli/pkg/transport/process/utils" ) const sleepTimeEnv = "SLEEP_TIME" @@ -36,7 +36,7 @@ func main() { log.Fatal("finished sleeping -> exit with error") } - srv, err := process.NewUDSServer(addr, h) + srv, err := utils.NewUDSServer(addr, h) if err != nil { log.Fatal(err) } diff --git a/pkg/transport/process/utils/uds_server.go b/pkg/transport/process/utils/uds_server.go new file mode 100644 index 00000000..4f88c865 --- /dev/null +++ b/pkg/transport/process/utils/uds_server.go @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package utils + +import ( + "io" + "log" + "net" + "sync" +) + +// HandlerFunc defines the interface of a function that should be served by a UDS server +type HandlerFunc func(io.Reader, io.WriteCloser) + +// UDSServer implements a Unix Domain Socket server +type UDSServer struct { + listener net.Listener + quit chan interface{} + wg sync.WaitGroup + handler HandlerFunc +} + +// NewUDSServer returns a new UDS server. +// The parameters define the server address and the handler func it serves +func NewUDSServer(addr string, handler HandlerFunc) (*UDSServer, error) { + l, err := net.Listen("unix", addr) + if err != nil { + return nil, err + } + s := &UDSServer{ + quit: make(chan interface{}), + listener: l, + handler: handler, + } + return s, nil +} + +// Start starts the server goroutine +func (s *UDSServer) Start() { + s.wg.Add(1) + go s.serve() +} + +func (s *UDSServer) serve() { + defer s.wg.Done() + + for { + conn, err := s.listener.Accept() + if err != nil { + select { + case <-s.quit: + return + default: + log.Println("accept error", err) + } + } else { + s.wg.Add(1) + go func() { + defer s.wg.Done() + s.handler(conn, conn) + }() + } + } +} + +// Stop stops the server goroutine +func (s *UDSServer) Stop() { + close(s.quit) + if err := s.listener.Close(); err != nil { + println(err) + } + s.wg.Wait() +} diff --git a/pkg/transport/process/util.go b/pkg/transport/process/utils/util.go similarity index 77% rename from pkg/transport/process/util.go rename to pkg/transport/process/utils/util.go index a991c89a..2f52cb7b 100644 --- a/pkg/transport/process/util.go +++ b/pkg/transport/process/utils/util.go @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. // // SPDX-License-Identifier: Apache-2.0 -package process +package utils import ( "archive/tar" @@ -9,10 +9,7 @@ import ( "fmt" "io" "io/ioutil" - "log" - "net" "os" - "sync" cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" "sigs.k8s.io/yaml" @@ -142,66 +139,3 @@ func readComponentDescriptor(r *tar.Reader) (*cdv2.ComponentDescriptor, error) { return &cd, nil } - -// HandlerFunc defines the interface of a function that should be served by a UDS server -type HandlerFunc func(io.Reader, io.WriteCloser) - -// UDSServer implements a Unix Domain Socket server -type UDSServer struct { - listener net.Listener - quit chan interface{} - wg sync.WaitGroup - handler HandlerFunc -} - -// NewUDSServer returns a new UDS server. -// The parameters define the server address and the handler func it serves -func NewUDSServer(addr string, handler HandlerFunc) (*UDSServer, error) { - l, err := net.Listen("unix", addr) - if err != nil { - return nil, err - } - s := &UDSServer{ - quit: make(chan interface{}), - listener: l, - handler: handler, - } - return s, nil -} - -// Start starts the server goroutine -func (s *UDSServer) Start() { - s.wg.Add(1) - go s.serve() -} - -func (s *UDSServer) serve() { - defer s.wg.Done() - - for { - conn, err := s.listener.Accept() - if err != nil { - select { - case <-s.quit: - return - default: - log.Println("accept error", err) - } - } else { - s.wg.Add(1) - go func() { - defer s.wg.Done() - s.handler(conn, conn) - }() - } - } -} - -// Stop stops the server goroutine -func (s *UDSServer) Stop() { - close(s.quit) - if err := s.listener.Close(); err != nil { - println(err) - } - s.wg.Wait() -} diff --git a/pkg/transport/process/util_test.go b/pkg/transport/process/utils/util_test.go similarity index 81% rename from pkg/transport/process/util_test.go rename to pkg/transport/process/utils/util_test.go index cd3c40b3..a99292e9 100644 --- a/pkg/transport/process/util_test.go +++ b/pkg/transport/process/utils/util_test.go @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. // // SPDX-License-Identifier: Apache-2.0 -package process_test +package utils_test import ( "bytes" @@ -12,7 +12,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/gardener/component-cli/pkg/transport/process" + "github.com/gardener/component-cli/pkg/transport/process/utils" ) var _ = Describe("util", func() { @@ -38,10 +38,10 @@ var _ = Describe("util", func() { } processMsgBuf := bytes.NewBuffer([]byte{}) - err := process.WriteProcessorMessage(cd, res, strings.NewReader(resourceData), processMsgBuf) + err := utils.WriteProcessorMessage(cd, res, strings.NewReader(resourceData), processMsgBuf) Expect(err).ToNot(HaveOccurred()) - actualCD, actualRes, resourceBlobReader, err := process.ReadProcessorMessage(processMsgBuf) + actualCD, actualRes, resourceBlobReader, err := utils.ReadProcessorMessage(processMsgBuf) Expect(err).ToNot(HaveOccurred()) Expect(*actualCD).To(Equal(cd)) From 779cd8ce959ca0ff9e91bb4ca6d4666ff726c0cb Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Wed, 3 Nov 2021 09:48:53 +0100 Subject: [PATCH 18/24] refactoring --- .../utils/{util.go => processor_message.go} | 0 .../{util_test.go => processor_message_test.go} | 0 pkg/transport/process/utils/utils_suite_test.go | 16 ++++++++++++++++ 3 files changed, 16 insertions(+) rename pkg/transport/process/utils/{util.go => processor_message.go} (100%) rename pkg/transport/process/utils/{util_test.go => processor_message_test.go} (100%) create mode 100644 pkg/transport/process/utils/utils_suite_test.go diff --git a/pkg/transport/process/utils/util.go b/pkg/transport/process/utils/processor_message.go similarity index 100% rename from pkg/transport/process/utils/util.go rename to pkg/transport/process/utils/processor_message.go diff --git a/pkg/transport/process/utils/util_test.go b/pkg/transport/process/utils/processor_message_test.go similarity index 100% rename from pkg/transport/process/utils/util_test.go rename to pkg/transport/process/utils/processor_message_test.go diff --git a/pkg/transport/process/utils/utils_suite_test.go b/pkg/transport/process/utils/utils_suite_test.go new file mode 100644 index 00000000..03afdd59 --- /dev/null +++ b/pkg/transport/process/utils/utils_suite_test.go @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 +package utils_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Utils Test Suite") +} From 526e68fc8636d1a09484c494e4e978801ae6814b Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Wed, 3 Nov 2021 10:04:17 +0100 Subject: [PATCH 19/24] use map[string]string for passing env variables --- .../extensions/extensions_suite_test.go | 28 +++++++++++++------ .../process/extensions/stdio_executable.go | 9 ++++-- .../process/extensions/uds_executable.go | 18 ++++++------ 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/pkg/transport/process/extensions/extensions_suite_test.go b/pkg/transport/process/extensions/extensions_suite_test.go index 28df2cd5..9ea2ce7b 100644 --- a/pkg/transport/process/extensions/extensions_suite_test.go +++ b/pkg/transport/process/extensions/extensions_suite_test.go @@ -46,9 +46,15 @@ var _ = BeforeSuite(func() { var _ = Describe("transport extensions", func() { Context("stdio executable", func() { + It("should create processor successfully if env is nil", func() { + args := []string{} + _, err := extensions.NewStdIOExecutable(exampleProcessorBinaryPath, args, nil) + Expect(err).ToNot(HaveOccurred()) + }) + It("should modify the processed resource correctly", func() { args := []string{} - env := []string{} + env := map[string]string{} processor, err := extensions.NewStdIOExecutable(exampleProcessorBinaryPath, args, env) Expect(err).ToNot(HaveOccurred()) @@ -57,8 +63,8 @@ var _ = Describe("transport extensions", func() { It("should exit with error when timeout is reached", func() { args := []string{} - env := []string{ - fmt.Sprintf("%s=%s", sleepTimeEnv, sleepTime.String()), + env := map[string]string{ + sleepTimeEnv: sleepTime.String(), } processor, err := extensions.NewStdIOExecutable(sleepProcessorBinaryPath, args, env) Expect(err).ToNot(HaveOccurred()) @@ -68,9 +74,15 @@ var _ = Describe("transport extensions", func() { }) Context("uds executable", func() { + It("should create processor successfully if env is nil", func() { + args := []string{} + _, err := extensions.NewUDSExecutable(exampleProcessorBinaryPath, args, nil) + Expect(err).ToNot(HaveOccurred()) + }) + It("should modify the processed resource correctly", func() { args := []string{} - env := []string{} + env := map[string]string{} processor, err := extensions.NewUDSExecutable(exampleProcessorBinaryPath, args, env) Expect(err).ToNot(HaveOccurred()) @@ -79,8 +91,8 @@ var _ = Describe("transport extensions", func() { It("should raise an error when trying to set the server address env variable manually", func() { args := []string{} - env := []string{ - extensions.ServerAddressEnv + "=/tmp/my-processor.sock", + env := map[string]string{ + extensions.ServerAddressEnv: "/tmp/my-processor.sock", } _, err := extensions.NewUDSExecutable(exampleProcessorBinaryPath, args, env) Expect(err).To(MatchError(fmt.Sprintf("the env variable %s is not allowed to be set manually", extensions.ServerAddressEnv))) @@ -88,8 +100,8 @@ var _ = Describe("transport extensions", func() { It("should exit with error when timeout is reached", func() { args := []string{} - env := []string{ - fmt.Sprintf("%s=%s", sleepTimeEnv, sleepTime.String()), + env := map[string]string{ + sleepTimeEnv: sleepTime.String(), } processor, err := extensions.NewUDSExecutable(sleepProcessorBinaryPath, args, env) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/transport/process/extensions/stdio_executable.go b/pkg/transport/process/extensions/stdio_executable.go index 5ae2b17f..a85a8283 100644 --- a/pkg/transport/process/extensions/stdio_executable.go +++ b/pkg/transport/process/extensions/stdio_executable.go @@ -21,11 +21,16 @@ type stdIOExecutable struct { // NewStdIOExecutable returns a resource processor extension which runs an executable. // in the background. It communicates with this processor via stdin/stdout pipes. -func NewStdIOExecutable(bin string, args []string, env []string) (process.ResourceStreamProcessor, error) { +func NewStdIOExecutable(bin string, args []string, env map[string]string) (process.ResourceStreamProcessor, error) { + parsedEnv := []string{} + for k, v := range env { + parsedEnv = append(parsedEnv, fmt.Sprintf("%s=%s", k, v)) + } + e := stdIOExecutable{ bin: bin, args: args, - env: env, + env: parsedEnv, } return &e, nil diff --git a/pkg/transport/process/extensions/uds_executable.go b/pkg/transport/process/extensions/uds_executable.go index be078160..6c14148d 100644 --- a/pkg/transport/process/extensions/uds_executable.go +++ b/pkg/transport/process/extensions/uds_executable.go @@ -10,7 +10,6 @@ import ( "net" "os" "os/exec" - "strings" "syscall" "time" @@ -31,11 +30,14 @@ type udsExecutable struct { // NewUDSExecutable runs a resource processor extension executable in the background. // It communicates with this processor via Unix Domain Sockets. -func NewUDSExecutable(bin string, args []string, env []string) (process.ResourceStreamProcessor, error) { - for _, e := range env { - if strings.HasPrefix(e, ServerAddressEnv+"=") { - return nil, fmt.Errorf("the env variable %s is not allowed to be set manually", ServerAddressEnv) - } +func NewUDSExecutable(bin string, args []string, env map[string]string) (process.ResourceStreamProcessor, error) { + if _, ok := env[ServerAddressEnv]; ok { + return nil, fmt.Errorf("the env variable %s is not allowed to be set manually", ServerAddressEnv) + } + + parsedEnv := []string{} + for k, v := range env { + parsedEnv = append(parsedEnv, fmt.Sprintf("%s=%s", k, v)) } wd, err := os.Getwd() @@ -43,12 +45,12 @@ func NewUDSExecutable(bin string, args []string, env []string) (process.Resource return nil, err } addr := fmt.Sprintf("%s/%s.sock", wd, utils.RandomString(8)) - env = append(env, fmt.Sprintf("%s=%s", ServerAddressEnv, addr)) + parsedEnv = append(parsedEnv, fmt.Sprintf("%s=%s", ServerAddressEnv, addr)) e := udsExecutable{ bin: bin, args: args, - env: env, + env: parsedEnv, addr: addr, } From 9fbcdc7d2f4278a691a10ca0b4ff39d0e1c655e2 Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Wed, 3 Nov 2021 10:40:19 +0100 Subject: [PATCH 20/24] refactoring + changes doc --- pkg/transport/process/pipeline.go | 4 ++-- pkg/transport/process/types.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/transport/process/pipeline.go b/pkg/transport/process/pipeline.go index fe9da4db..77861dac 100644 --- a/pkg/transport/process/pipeline.go +++ b/pkg/transport/process/pipeline.go @@ -34,7 +34,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co } for _, proc := range p.processors { - outfile, err := p.process(ctx, infile, proc) + outfile, err := p.runProcessor(ctx, infile, proc) if err != nil { return nil, cdv2.Resource{}, err } @@ -58,7 +58,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co return processedCD, processedRes, nil } -func (p *resourceProcessingPipelineImpl) process(ctx context.Context, infile *os.File, proc ResourceStreamProcessor) (*os.File, error) { +func (p *resourceProcessingPipelineImpl) runProcessor(ctx context.Context, infile *os.File, proc ResourceStreamProcessor) (*os.File, error) { defer infile.Close() if _, err := infile.Seek(0, io.SeekStart); err != nil { diff --git a/pkg/transport/process/types.go b/pkg/transport/process/types.go index 9889d07e..d8b69eb3 100644 --- a/pkg/transport/process/types.go +++ b/pkg/transport/process/types.go @@ -24,7 +24,7 @@ type ResourceProcessingPipeline interface { // A processor can upload, modify, or download a resource. type ResourceStreamProcessor interface { // Process executes the processor for a resource. Input and Output streams must be - // compliant to a specific format ("processor message"). See also ./util.go for helper - // functions to read/write processor messages. + // compliant to a specific format ("processor message"). See also ./utils/processor_message.go + // which describes the format and provides helper functions to read/write processor messages. Process(context.Context, io.Reader, io.Writer) error } From a29600058b816717f8a93424aa98c3b691eb89b1 Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Mon, 29 Nov 2021 11:58:56 +0100 Subject: [PATCH 21/24] renames uds to unix domain socket --- .../extensions/extensions_suite_test.go | 10 +++++----- ...ble.go => unix_domain_socket_executable.go} | 10 +++++----- .../process/processors/example/main.go | 2 +- pkg/transport/process/processors/sleep/main.go | 2 +- ..._server.go => unix_domain_socket_server.go} | 18 +++++++++--------- 5 files changed, 21 insertions(+), 21 deletions(-) rename pkg/transport/process/extensions/{uds_executable.go => unix_domain_socket_executable.go} (86%) rename pkg/transport/process/utils/{uds_server.go => unix_domain_socket_server.go} (70%) diff --git a/pkg/transport/process/extensions/extensions_suite_test.go b/pkg/transport/process/extensions/extensions_suite_test.go index 9ea2ce7b..68f3cda4 100644 --- a/pkg/transport/process/extensions/extensions_suite_test.go +++ b/pkg/transport/process/extensions/extensions_suite_test.go @@ -73,17 +73,17 @@ var _ = Describe("transport extensions", func() { }) }) - Context("uds executable", func() { + Context("unix domain socket executable", func() { It("should create processor successfully if env is nil", func() { args := []string{} - _, err := extensions.NewUDSExecutable(exampleProcessorBinaryPath, args, nil) + _, err := extensions.NewUnixDomainSocketExecutable(exampleProcessorBinaryPath, args, nil) Expect(err).ToNot(HaveOccurred()) }) It("should modify the processed resource correctly", func() { args := []string{} env := map[string]string{} - processor, err := extensions.NewUDSExecutable(exampleProcessorBinaryPath, args, env) + processor, err := extensions.NewUnixDomainSocketExecutable(exampleProcessorBinaryPath, args, env) Expect(err).ToNot(HaveOccurred()) runExampleResourceTest(processor) @@ -94,7 +94,7 @@ var _ = Describe("transport extensions", func() { env := map[string]string{ extensions.ServerAddressEnv: "/tmp/my-processor.sock", } - _, err := extensions.NewUDSExecutable(exampleProcessorBinaryPath, args, env) + _, err := extensions.NewUnixDomainSocketExecutable(exampleProcessorBinaryPath, args, env) Expect(err).To(MatchError(fmt.Sprintf("the env variable %s is not allowed to be set manually", extensions.ServerAddressEnv))) }) @@ -103,7 +103,7 @@ var _ = Describe("transport extensions", func() { env := map[string]string{ sleepTimeEnv: sleepTime.String(), } - processor, err := extensions.NewUDSExecutable(sleepProcessorBinaryPath, args, env) + processor, err := extensions.NewUnixDomainSocketExecutable(sleepProcessorBinaryPath, args, env) Expect(err).ToNot(HaveOccurred()) runTimeoutTest(processor) diff --git a/pkg/transport/process/extensions/uds_executable.go b/pkg/transport/process/extensions/unix_domain_socket_executable.go similarity index 86% rename from pkg/transport/process/extensions/uds_executable.go rename to pkg/transport/process/extensions/unix_domain_socket_executable.go index 6c14148d..63446099 100644 --- a/pkg/transport/process/extensions/uds_executable.go +++ b/pkg/transport/process/extensions/unix_domain_socket_executable.go @@ -21,16 +21,16 @@ import ( // address under which a resource processor server should start. const ServerAddressEnv = "SERVER_ADDRESS" -type udsExecutable struct { +type unixDomainSocketExecutable struct { bin string args []string env []string addr string } -// NewUDSExecutable runs a resource processor extension executable in the background. +// NewUnixDomainSocketExecutable runs a resource processor extension executable in the background. // It communicates with this processor via Unix Domain Sockets. -func NewUDSExecutable(bin string, args []string, env map[string]string) (process.ResourceStreamProcessor, error) { +func NewUnixDomainSocketExecutable(bin string, args []string, env map[string]string) (process.ResourceStreamProcessor, error) { if _, ok := env[ServerAddressEnv]; ok { return nil, fmt.Errorf("the env variable %s is not allowed to be set manually", ServerAddressEnv) } @@ -47,7 +47,7 @@ func NewUDSExecutable(bin string, args []string, env map[string]string) (process addr := fmt.Sprintf("%s/%s.sock", wd, utils.RandomString(8)) parsedEnv = append(parsedEnv, fmt.Sprintf("%s=%s", ServerAddressEnv, addr)) - e := udsExecutable{ + e := unixDomainSocketExecutable{ bin: bin, args: args, env: parsedEnv, @@ -57,7 +57,7 @@ func NewUDSExecutable(bin string, args []string, env map[string]string) (process return &e, nil } -func (e *udsExecutable) Process(ctx context.Context, r io.Reader, w io.Writer) error { +func (e *unixDomainSocketExecutable) Process(ctx context.Context, r io.Reader, w io.Writer) error { cmd := exec.CommandContext(ctx, e.bin, e.args...) cmd.Env = e.env cmd.Stdout = os.Stdout diff --git a/pkg/transport/process/processors/example/main.go b/pkg/transport/process/processors/example/main.go index 2bb26490..de1c758f 100644 --- a/pkg/transport/process/processors/example/main.go +++ b/pkg/transport/process/processors/example/main.go @@ -42,7 +42,7 @@ func main() { } } - srv, err := utils.NewUDSServer(addr, h) + srv, err := utils.NewUnixDomainSocketServer(addr, h) if err != nil { log.Fatal(err) } diff --git a/pkg/transport/process/processors/sleep/main.go b/pkg/transport/process/processors/sleep/main.go index 7be914c7..0ed1a801 100644 --- a/pkg/transport/process/processors/sleep/main.go +++ b/pkg/transport/process/processors/sleep/main.go @@ -36,7 +36,7 @@ func main() { log.Fatal("finished sleeping -> exit with error") } - srv, err := utils.NewUDSServer(addr, h) + srv, err := utils.NewUnixDomainSocketServer(addr, h) if err != nil { log.Fatal(err) } diff --git a/pkg/transport/process/utils/uds_server.go b/pkg/transport/process/utils/unix_domain_socket_server.go similarity index 70% rename from pkg/transport/process/utils/uds_server.go rename to pkg/transport/process/utils/unix_domain_socket_server.go index 4f88c865..5d10edfe 100644 --- a/pkg/transport/process/utils/uds_server.go +++ b/pkg/transport/process/utils/unix_domain_socket_server.go @@ -10,25 +10,25 @@ import ( "sync" ) -// HandlerFunc defines the interface of a function that should be served by a UDS server +// HandlerFunc defines the interface of a function that should be served by a Unix Domain Socket server type HandlerFunc func(io.Reader, io.WriteCloser) -// UDSServer implements a Unix Domain Socket server -type UDSServer struct { +// UnixDomainSocketServer implements a Unix Domain Socket server +type UnixDomainSocketServer struct { listener net.Listener quit chan interface{} wg sync.WaitGroup handler HandlerFunc } -// NewUDSServer returns a new UDS server. +// NewUnixDomainSocketServer returns a new Unix Domain Socket server. // The parameters define the server address and the handler func it serves -func NewUDSServer(addr string, handler HandlerFunc) (*UDSServer, error) { +func NewUnixDomainSocketServer(addr string, handler HandlerFunc) (*UnixDomainSocketServer, error) { l, err := net.Listen("unix", addr) if err != nil { return nil, err } - s := &UDSServer{ + s := &UnixDomainSocketServer{ quit: make(chan interface{}), listener: l, handler: handler, @@ -37,12 +37,12 @@ func NewUDSServer(addr string, handler HandlerFunc) (*UDSServer, error) { } // Start starts the server goroutine -func (s *UDSServer) Start() { +func (s *UnixDomainSocketServer) Start() { s.wg.Add(1) go s.serve() } -func (s *UDSServer) serve() { +func (s *UnixDomainSocketServer) serve() { defer s.wg.Done() for { @@ -65,7 +65,7 @@ func (s *UDSServer) serve() { } // Stop stops the server goroutine -func (s *UDSServer) Stop() { +func (s *UnixDomainSocketServer) Stop() { close(s.quit) if err := s.listener.Close(); err != nil { println(err) From 9dc8e97d5a279a41321fd91765028779ce5091e2 Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Mon, 29 Nov 2021 16:11:22 +0100 Subject: [PATCH 22/24] review feedback --- .../process/extensions/extensions_suite_test.go | 4 ++-- .../process/extensions/stdio_executable.go | 4 ++-- .../extensions/unix_domain_socket_executable.go | 14 +++++++------- pkg/transport/process/pipeline.go | 2 +- pkg/transport/process/processors/example/main.go | 9 ++++++++- pkg/transport/process/processors/sleep/main.go | 2 +- pkg/transport/process/utils/processor_message.go | 2 +- 7 files changed, 22 insertions(+), 15 deletions(-) diff --git a/pkg/transport/process/extensions/extensions_suite_test.go b/pkg/transport/process/extensions/extensions_suite_test.go index 68f3cda4..0d904f4c 100644 --- a/pkg/transport/process/extensions/extensions_suite_test.go +++ b/pkg/transport/process/extensions/extensions_suite_test.go @@ -92,10 +92,10 @@ var _ = Describe("transport extensions", func() { It("should raise an error when trying to set the server address env variable manually", func() { args := []string{} env := map[string]string{ - extensions.ServerAddressEnv: "/tmp/my-processor.sock", + extensions.ProcessorServerAddressEnv: "/tmp/my-processor.sock", } _, err := extensions.NewUnixDomainSocketExecutable(exampleProcessorBinaryPath, args, env) - Expect(err).To(MatchError(fmt.Sprintf("the env variable %s is not allowed to be set manually", extensions.ServerAddressEnv))) + Expect(err).To(MatchError(fmt.Sprintf("the env variable %s is not allowed to be set manually", extensions.ProcessorServerAddressEnv))) }) It("should exit with error when timeout is reached", func() { diff --git a/pkg/transport/process/extensions/stdio_executable.go b/pkg/transport/process/extensions/stdio_executable.go index a85a8283..18ef96c2 100644 --- a/pkg/transport/process/extensions/stdio_executable.go +++ b/pkg/transport/process/extensions/stdio_executable.go @@ -19,8 +19,8 @@ type stdIOExecutable struct { env []string } -// NewStdIOExecutable returns a resource processor extension which runs an executable. -// in the background. It communicates with this processor via stdin/stdout pipes. +// NewStdIOExecutable returns a resource processor extension which runs an executable in the +// background when calling Process(). It communicates with this processor via stdin/stdout pipes. func NewStdIOExecutable(bin string, args []string, env map[string]string) (process.ResourceStreamProcessor, error) { parsedEnv := []string{} for k, v := range env { diff --git a/pkg/transport/process/extensions/unix_domain_socket_executable.go b/pkg/transport/process/extensions/unix_domain_socket_executable.go index 63446099..5a1241a4 100644 --- a/pkg/transport/process/extensions/unix_domain_socket_executable.go +++ b/pkg/transport/process/extensions/unix_domain_socket_executable.go @@ -17,9 +17,9 @@ import ( "github.com/gardener/component-cli/pkg/utils" ) -// ServerAddressEnv is the environment variable key which is used to store the +// ProcessorServerAddressEnv is the environment variable key which is used to store the // address under which a resource processor server should start. -const ServerAddressEnv = "SERVER_ADDRESS" +const ProcessorServerAddressEnv = "PROCESSOR_SERVER_ADDRESS" type unixDomainSocketExecutable struct { bin string @@ -28,11 +28,11 @@ type unixDomainSocketExecutable struct { addr string } -// NewUnixDomainSocketExecutable runs a resource processor extension executable in the background. -// It communicates with this processor via Unix Domain Sockets. +// NewUnixDomainSocketExecutable returns a resource processor extension which runs an executable in the +// background when calling Process(). It communicates with this processor via Unix Domain Sockets. func NewUnixDomainSocketExecutable(bin string, args []string, env map[string]string) (process.ResourceStreamProcessor, error) { - if _, ok := env[ServerAddressEnv]; ok { - return nil, fmt.Errorf("the env variable %s is not allowed to be set manually", ServerAddressEnv) + if _, ok := env[ProcessorServerAddressEnv]; ok { + return nil, fmt.Errorf("the env variable %s is not allowed to be set manually", ProcessorServerAddressEnv) } parsedEnv := []string{} @@ -45,7 +45,7 @@ func NewUnixDomainSocketExecutable(bin string, args []string, env map[string]str return nil, err } addr := fmt.Sprintf("%s/%s.sock", wd, utils.RandomString(8)) - parsedEnv = append(parsedEnv, fmt.Sprintf("%s=%s", ServerAddressEnv, addr)) + parsedEnv = append(parsedEnv, fmt.Sprintf("%s=%s", ProcessorServerAddressEnv, addr)) e := unixDomainSocketExecutable{ bin: bin, diff --git a/pkg/transport/process/pipeline.go b/pkg/transport/process/pipeline.go index 77861dac..98caafba 100644 --- a/pkg/transport/process/pipeline.go +++ b/pkg/transport/process/pipeline.go @@ -44,7 +44,7 @@ func (p *resourceProcessingPipelineImpl) Process(ctx context.Context, cd cdv2.Co defer infile.Close() if _, err := infile.Seek(0, io.SeekStart); err != nil { - return nil, cdv2.Resource{}, err + return nil, cdv2.Resource{}, fmt.Errorf("unable to seek to beginning of input file: %w", err) } processedCD, processedRes, blobreader, err := utils.ReadProcessorMessage(infile) diff --git a/pkg/transport/process/processors/example/main.go b/pkg/transport/process/processors/example/main.go index de1c758f..c2b0dcab 100644 --- a/pkg/transport/process/processors/example/main.go +++ b/pkg/transport/process/processors/example/main.go @@ -26,7 +26,8 @@ const processorName = "example-processor" // a test processor which adds its name to the resource labels and the resource blob. // the resource blob is expected to be plain text data. func main() { - addr := os.Getenv(extensions.ServerAddressEnv) + // read the address under which the unix domain socket server should start + addr := os.Getenv(extensions.ProcessorServerAddressEnv) if addr == "" { // if addr is not set, use stdin/stdout for communication @@ -35,6 +36,7 @@ func main() { } return } + // if addr is set, use unix domain sockets for communication h := func(r io.Reader, w io.WriteCloser) { if err := processorRoutine(r, w); err != nil { @@ -65,6 +67,7 @@ func processorRoutine(inputStream io.Reader, outputStream io.WriteCloser) error } defer tmpfile.Close() + // read the input stream if _, err := io.Copy(tmpfile, inputStream); err != nil { return err } @@ -73,6 +76,7 @@ func processorRoutine(inputStream io.Reader, outputStream io.WriteCloser) error return err } + // split up the input stream into component descriptor, resource, and resource blob cd, res, resourceBlobReader, err := utils.ReadProcessorMessage(tmpfile) if err != nil { return err @@ -81,18 +85,21 @@ func processorRoutine(inputStream io.Reader, outputStream io.WriteCloser) error defer resourceBlobReader.Close() } + // modify resource blob buf := bytes.NewBuffer([]byte{}) if _, err := io.Copy(buf, resourceBlobReader); err != nil { return err } outputData := fmt.Sprintf("%s\n%s", buf.String(), processorName) + // modify resource yaml l := cdv2.Label{ Name: "processor-name", Value: json.RawMessage(`"` + processorName + `"`), } res.Labels = append(res.Labels, l) + // write modified output to output stream if err := utils.WriteProcessorMessage(*cd, res, strings.NewReader(outputData), outputStream); err != nil { return err } diff --git a/pkg/transport/process/processors/sleep/main.go b/pkg/transport/process/processors/sleep/main.go index 0ed1a801..54aefa35 100644 --- a/pkg/transport/process/processors/sleep/main.go +++ b/pkg/transport/process/processors/sleep/main.go @@ -24,7 +24,7 @@ func main() { log.Fatal(err) } - addr := os.Getenv(extensions.ServerAddressEnv) + addr := os.Getenv(extensions.ProcessorServerAddressEnv) if addr == "" { time.Sleep(sleepTime) diff --git a/pkg/transport/process/utils/processor_message.go b/pkg/transport/process/utils/processor_message.go index 2f52cb7b..38500ee3 100644 --- a/pkg/transport/process/utils/processor_message.go +++ b/pkg/transport/process/utils/processor_message.go @@ -106,7 +106,7 @@ func ReadProcessorMessage(r io.Reader) (*cdv2.ComponentDescriptor, cdv2.Resource } if _, err := f.Seek(0, io.SeekStart); err != nil { - return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to seek to beginning of file: %w", err) + return nil, cdv2.Resource{}, nil, fmt.Errorf("unable to seek to beginning of resource blob file: %w", err) } return cd, res, f, nil From 6d0d6d2d53d0ec9d86035e38193db5cd25da64d8 Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Mon, 29 Nov 2021 16:59:41 +0100 Subject: [PATCH 23/24] remove unix domain socket file after processor finished --- .../process/extensions/unix_domain_socket_executable.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/transport/process/extensions/unix_domain_socket_executable.go b/pkg/transport/process/extensions/unix_domain_socket_executable.go index 5a1241a4..9e6f3864 100644 --- a/pkg/transport/process/extensions/unix_domain_socket_executable.go +++ b/pkg/transport/process/extensions/unix_domain_socket_executable.go @@ -94,6 +94,15 @@ func (e *unixDomainSocketExecutable) Process(ctx context.Context, r io.Reader, w return fmt.Errorf("unable to wait for processor: %w", err) } + // remove socket file if server hasn't already cleaned up + if _, err := os.Stat(e.addr); err == nil { + if err := os.Remove(e.addr); err != nil { + return fmt.Errorf("unable to remove %s: %w", e.addr, err) + } + } else if !os.IsNotExist(err) { + return fmt.Errorf("unable to get file stats for %s: %w", e.addr, err) + } + return nil } From ec0e058da25204100be9bdf10a342ae027f51090 Mon Sep 17 00:00:00 2001 From: Johannes Schicktanz Date: Tue, 30 Nov 2021 10:40:29 +0100 Subject: [PATCH 24/24] remove unnecessary copy step from example processor --- .../process/processors/example/main.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/pkg/transport/process/processors/example/main.go b/pkg/transport/process/processors/example/main.go index c2b0dcab..3b97ec70 100644 --- a/pkg/transport/process/processors/example/main.go +++ b/pkg/transport/process/processors/example/main.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "log" "os" "os/signal" @@ -61,23 +60,8 @@ func main() { func processorRoutine(inputStream io.Reader, outputStream io.WriteCloser) error { defer outputStream.Close() - tmpfile, err := ioutil.TempFile("", "") - if err != nil { - return err - } - defer tmpfile.Close() - - // read the input stream - if _, err := io.Copy(tmpfile, inputStream); err != nil { - return err - } - - if _, err := tmpfile.Seek(0, io.SeekStart); err != nil { - return err - } - // split up the input stream into component descriptor, resource, and resource blob - cd, res, resourceBlobReader, err := utils.ReadProcessorMessage(tmpfile) + cd, res, resourceBlobReader, err := utils.ReadProcessorMessage(inputStream) if err != nil { return err }