Merge pull request #75 from krishicks/add-unpack-ability

Add unpack ability

README.md

@@ -82,7 +82,7 @@ Places the following files in the destination:
 
 #### Parameters
 
-*None.*
+* `unpack`: *Optional.* If true and the file is an archive (tar, gzipped tar, other gzipped file, or zip), unpack the file. Gzipped tarballs will be both ungzipped and untarred.
 
 
 ### `out`: Upload an object to the bucket.

in/archive.go

@@ -0,0 +1,76 @@
+package in
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+
+	"bitbucket.org/taruti/mimemagic"
+)
+
+var archiveMimetypes = []string{
+	"application/x-gzip",
+	"application/gzip",
+	"application/x-tar",
+	"application/zip",
+}
+
+func mimetype(r *bufio.Reader) (string, error) {
+	bs, err := r.Peek(512)
+	if err != nil && err != io.EOF {
+		return "", err
+	}
+
+	if len(bs) == 0 {
+		return "", errors.New("cannot determine mimetype from empty bytes")
+	}
+
+	return mimemagic.Match("", bs), nil
+}
+
+func archiveMimetype(filename string) string {
+	f, err := os.Open(filename)
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+
+	mime, err := mimetype(bufio.NewReader(f))
+	if err != nil {
+		return ""
+	}
+
+	for i := range archiveMimetypes {
+		if strings.HasPrefix(mime, archiveMimetypes[i]) {
+			return archiveMimetypes[i]
+		}
+	}
+
+	return ""
+}
+
+func inflate(mime, path, destination string) error {
+	var cmd *exec.Cmd
+
+	switch mime {
+	case "application/zip":
+		cmd = exec.Command("unzip", "-P", "", "-d", destination, path)
+		defer os.Remove(path)
+
+	case "application/x-tar":
+		cmd = exec.Command("tar", "xf", path, "-C", destination)
+		defer os.Remove(path)
+
+	case "application/gzip", "application/x-gzip":
+		cmd = exec.Command("gunzip", path)
+
+	default:
+		return fmt.Errorf("don't know how to extract %s", mime)
+	}
+
+	return cmd.Run()
+}

in/in_command.go

@@ -45,106 +45,86 @@ func (command *InCommand) Run(destinationDir string, request InRequest) (InRespo
 		return InResponse{}, errors.New(message)
 	}
 
-	err := command.createDirectory(destinationDir)
+	err := os.MkdirAll(destinationDir, 0755)
 	if err != nil {
 		return InResponse{}, err
 	}
 
-	if request.Source.Regexp != "" {
-		return command.inByRegex(destinationDir, request)
-	} else {
-		return command.inByVersionedFile(destinationDir, request)
-	}
-}
+	var remotePath string
+	var versionNumber string
+	var versionID string
 
-func (command *InCommand) inByRegex(destinationDir string, request InRequest) (InResponse, error) {
-	if request.Version.Path == "" {
-		return InResponse{}, ErrMissingPath
-	}
+	if request.Source.Regexp != "" {
+		if request.Version.Path == "" {
+			return InResponse{}, ErrMissingPath
+		}
 
-	remotePath := request.Version.Path
+		remotePath = request.Version.Path
 
-	extraction, ok := versions.Extract(remotePath, request.Source.Regexp)
-	if !ok {
-		return InResponse{}, fmt.Errorf("regex does not match provided version: %#v", request.Version)
+		extraction, ok := versions.Extract(remotePath, request.Source.Regexp)
+		if !ok {
+			return InResponse{}, fmt.Errorf("regex does not match provided version: %#v", request.Version)
+		}
 
-	}
-
-	err := command.writeVersionFile(extraction.VersionNumber, destinationDir)
-	if err != nil {
-		return InResponse{}, err
+		versionNumber = extraction.VersionNumber
+	} else {
+		remotePath = request.Source.VersionedFile
+		versionNumber = request.Version.VersionID
+		versionID = request.Version.VersionID
 	}
 
 	err = command.downloadFile(
 		request.Source.Bucket,
 		remotePath,
-		"",
+		versionID,
 		destinationDir,
 		path.Base(remotePath),
 	)
-	if err != nil {
-		return InResponse{}, err
-	}
 
-	url := command.urlProvider.GetURL(request, remotePath)
-	err = command.writeURLFile(
-		destinationDir,
-		url,
-	)
 	if err != nil {
 		return InResponse{}, err
 	}
 
-	return InResponse{
-		Version: s3resource.Version{
-			Path: remotePath,
-		},
-		Metadata: command.metadata(remotePath, request.Source.Private, url),
-	}, nil
-}
-
-func (command *InCommand) inByVersionedFile(destinationDir string, request InRequest) (InResponse, error) {
+	if request.Params.Unpack {
+		destinationPath := filepath.Join(destinationDir, path.Base(remotePath))
+		mime := archiveMimetype(destinationPath)
+		if mime == "" {
+			return InResponse{}, fmt.Errorf("not an archive: %s", destinationPath)
+		}
+
+		err = extractArchive(mime, destinationPath)
+		if err != nil {
+			return InResponse{}, err
+		}
+	}
 
-	err := command.writeVersionFile(request.Version.VersionID, destinationDir)
-	if err != nil {
+	url := command.urlProvider.GetURL(request, remotePath)
+	if err = command.writeURLFile(destinationDir, url); err != nil {
 		return InResponse{}, err
 	}
 
-	remotePath := request.Source.VersionedFile
-
-	err = command.downloadFile(
-		request.Source.Bucket,
-		remotePath,
-		request.Version.VersionID,
-		destinationDir,
-		path.Base(remotePath),
-	)
-
+	err = command.writeVersionFile(versionNumber, destinationDir)
 	if err != nil {
 		return InResponse{}, err
 	}
 
-	url := command.urlProvider.GetURL(request, remotePath)
-	err = command.writeURLFile(
-		destinationDir,
-		url,
-	)
+	metadata := command.metadata(remotePath, request.Source.Private, url)
 
-	if err != nil {
-		return InResponse{}, err
+	if versionID == "" {
+		return InResponse{
+			Version: s3resource.Version{
+				Path: remotePath,
+			},
+			Metadata: metadata,
+		}, nil
 	}
 
 	return InResponse{
 		Version: s3resource.Version{
-			VersionID: request.Version.VersionID,
+			VersionID: versionID,
 		},
-		Metadata: command.metadata(remotePath, request.Source.Private, url),
+		Metadata: metadata,
 	}, nil
-
-}
-
-func (command *InCommand) createDirectory(destDir string) error {
-	return os.MkdirAll(destDir, 0755)
 }
 
 func (command *InCommand) writeURLFile(destDir string, url string) error {
@@ -185,3 +165,34 @@ func (command *InCommand) metadata(remotePath string, private bool, url string)
 
 	return metadata
 }
+
+func extractArchive(mime, filename string) error {
+	destDir := filepath.Dir(filename)
+
+	err := inflate(mime, filename, destDir)
+	if err != nil {
+		return fmt.Errorf("failed to extract archive: %s", err)
+	}
+
+	if mime == "application/gzip" || mime == "application/x-gzip" {
+		fileInfos, err := ioutil.ReadDir(destDir)
+		if err != nil {
+			return fmt.Errorf("failed to read dir: %s", err)
+		}
+
+		if len(fileInfos) != 1 {
+			return fmt.Errorf("%d files found after gunzip; expected 1", len(fileInfos))
+		}
+
+		filename = filepath.Join(destDir, fileInfos[0].Name())
+		mime = archiveMimetype(filename)
+		if mime == "application/x-tar" {
+			err = inflate(mime, filename, destDir)
+			if err != nil {
+				return fmt.Errorf("failed to extract archive: %s", err)
+			}
+		}
+	}
+
+	return nil
+}