Implement optional gzip compression (#9)

close #2

---------

Co-authored-by: Jaime Pillora <[email protected]>
Co-authored-by: Patrick Schratz <[email protected]>
Co-authored-by: qwerty287 <[email protected]>

docs.md

@@ -81,7 +81,7 @@ pipeline:
       strip_prefix: public/
 ```
 
-Configure the plugin to exclude files from upload:
+Configure the plugin to exclude files from upload and compress:
 
 ```yml
 pipeline:
@@ -93,6 +93,7 @@ pipeline:
       target: /target/location
       exclude:
         - **/*.xml
+      compress: true
 ```
 
 Configure the plugin to connect to a Minio server:
@@ -149,3 +150,6 @@ path_style
 
 env_file
 : load env vars from file
+
+compress
+: prior to upload, compress files and use gzip content-encoding (false by default)

main.go

@@ -124,6 +124,11 @@ func main() {
 			Usage:   "source env file",
 			EnvVars: []string{"PLUGIN_ENV_FILE"},
 		},
+		&cli.BoolFlag{
+			Name:    "compress",
+			Usage:   "prior to upload, compress files and use gzip content-encoding",
+			EnvVars: []string{"PLUGIN_COMPRESS"},
+		},
 	}
 
 	if err := app.Run(os.Args); err != nil {
@@ -156,6 +161,7 @@ func run(c *cli.Context) error {
 		StorageClass:          c.String("storage-class"),
 		PathStyle:             c.Bool("path-style"),
 		DryRun:                c.Bool("dry-run"),
+		Compress:              c.Bool("compress"),
 	}
 
 	return plugin.Exec()

plugin.go

@@ -1,6 +1,9 @@
 package main
 
 import (
+	"compress/gzip"
+	"errors"
+	"io"
 	"mime"
 	"os"
 	"path/filepath"
@@ -89,6 +92,8 @@ type Plugin struct {
 	PathStyle bool
 	// Dry run without uploading/
 	DryRun bool
+	// Compress objects and upload with Content-Encoding: gzip
+	Compress bool
 }
 
 // Exec runs the plugin
@@ -210,6 +215,25 @@ func (p *Plugin) Exec() error {
 			putObjectInput.StorageClass = &(p.StorageClass)
 		}
 
+		// optionally compress
+		if p.Compress {
+			gr, err := gzip.NewReader(f)
+			if err != nil {
+				log.WithFields(log.Fields{
+					"error": err,
+					"file":  match,
+				}).Error("Problem gzipping file")
+				return err
+			}
+			// wrap with gzip
+			putObjectInput.Body = &gzipReadSeeker{rs: f, z: gr}
+			// set encoding
+			putObjectInput.ContentEncoding = aws.String("gzip")
+		} else {
+			putObjectInput.Body = f
+		}
+
+		// upload
 		_, err = client.PutObject(putObjectInput)
 
 		if err != nil {
@@ -228,6 +252,28 @@ func (p *Plugin) Exec() error {
 	return nil
 }
 
+// gzipReadSeeker implements Seek over gzip.Reader
+type gzipReadSeeker struct {
+	rs io.ReadSeeker
+	z  *gzip.Reader
+}
+
+func (grs *gzipReadSeeker) Read(p []byte) (n int, err error) {
+	return grs.z.Read(p)
+}
+
+func (grs *gzipReadSeeker) Seek(offset int64, whence int) (int64, error) {
+	// only zero (reset) seeks are supported, in this case, this should be fine
+	// since AWS will rarely seek mid-file
+	if offset == 0 && whence == 0 {
+		if err := grs.z.Reset(grs.rs); err != nil {
+			return 0, err
+		}
+		return grs.rs.Seek(0, 0)
+	}
+	return 0, errors.New("Non-zero seek not supported")
+}
+
 // matches is a helper function that returns a list of all files matching the
 // included Glob pattern, while excluding all files that matche the exclusion
 // Glob pattners.