Merge pull request #99 from alphagov/initial_version_pr

Add ability to initialise non-existent resources
concourse · May 28, 2018 · 320096c · 320096c
2 parents a0ae177 + cb40258
commit 320096c
Show file tree

Hide file tree

Showing 7 changed files with 445 additions and 74 deletions.
diff --git a/README.md b/README.md
@@ -63,6 +63,22 @@ One of the following two options must be specified:
   without resorting to version numbers. This property is the path to the file
   in your S3 bucket.
 
+### Initial state
+
+If no resource versions exist you can set up this resource to emit an initial version with a specified content. This won't create a real resource in S3 but only create an initial version for Concourse. The resource file will be created as usual when you `get` a resource with an initial version.
+
+You can define one of the following two options:
+
+* `initial_path`: *Optional.* Must be used with the `regexp` option. You should set this to the file path containing the initial version which would match the given regexp. E.g. if `regexp` is `file/build-(.*).zip`, then `initial_path` might be `file/build-0.0.0.zip`. The resource version will be `0.0.0` in this case.
+
+* `initial_version`: *Optional.* Must be used with the `versioned_file` option. This will be the resource version.
+
+By default the resource file will be created with no content when `get` runs. You can set the content by using one of the following options:
+
+* `initial_content_text`: *Optional.* Initial content as a string.
+
+* `initial_content_binary`: *Optional.* You can pass binary content as a base64 encoded string.
+
 ## Behavior
 
 ### `check`: Extract versions from the bucket.
@@ -85,7 +101,7 @@ Places the following files in the destination:
 
 #### Parameters
 
-* `unpack`: *Optional.* If true and the file is an archive (tar, gzipped tar, other gzipped file, or zip), unpack the file. Gzipped tarballs will be both ungzipped and untarred.
+* `unpack`: *Optional.* If true and the file is an archive (tar, gzipped tar, other gzipped file, or zip), unpack the file. Gzipped tarballs will be both ungzipped and untarred. It is ignored when `get` is running on the initial version.
 
 
 ### `out`: Upload an object to the bucket.

diff --git a/check/command.go b/check/command.go
@@ -32,6 +32,13 @@ func (command *Command) Run(request Request) (Response, error) {
 func (command *Command) checkByRegex(request Request) Response {
 	extractions := versions.GetBucketFileVersions(command.s3client, request.Source)
 
+	if request.Source.InitialPath != "" {
+		extraction, ok := versions.Extract(request.Source.InitialPath, request.Source.Regexp)
+		if ok {
+			extractions = append([]versions.Extraction{extraction}, extractions...)
+		}
+	}
+
 	if len(extractions) == 0 {
 		return nil
 	}
@@ -53,6 +60,10 @@ func (command *Command) checkByVersionedFile(request Request) Response {
 		s3resource.Fatal("finding versions", err)
 	}
 
+	if request.Source.InitialVersion != "" {
+		bucketVersions = append(bucketVersions, request.Source.InitialVersion)
+	}
+
 	if len(bucketVersions) == 0 {
 		return response
 	}

diff --git a/check/command_test.go b/check/command_test.go
@@ -66,6 +66,24 @@ var _ = Describe("Check Command", func() {
 				))
 			})
 
+			Context("when the initial version is set", func() {
+				It("still returns the latest version", func() {
+					request.Version.Path = ""
+					request.Source.InitialPath = "files/abc-0.0.tgz"
+					request.Source.Regexp = "files/abc-(.*).tgz"
+
+					response, err := command.Run(request)
+					Ω(err).ShouldNot(HaveOccurred())
+
+					Ω(response).Should(HaveLen(1))
+					Ω(response).Should(ConsistOf(
+						s3resource.Version{
+							Path: "files/abc-3.53.tgz",
+						},
+					))
+				})
+			})
+
 			Context("when the regexp does not match anything", func() {
 				It("does not explode", func() {
 					request.Source.Regexp = "no-files/missing-(.*).tgz"
@@ -74,6 +92,24 @@ var _ = Describe("Check Command", func() {
 
 					Ω(response).Should(HaveLen(0))
 				})
+
+				Context("when the initial version is set", func() {
+					It("returns the initial version", func() {
+						request.Version.Path = ""
+						request.Source.InitialPath = "no-files/missing-0.0.tgz"
+						request.Source.Regexp = "no-files/missing-(.*).tgz"
+
+						response, err := command.Run(request)
+						Ω(err).ShouldNot(HaveOccurred())
+
+						Ω(response).Should(HaveLen(1))
+						Ω(response).Should(ConsistOf(
+							s3resource.Version{
+								Path: "no-files/missing-0.0.tgz",
+							},
+						))
+					})
+				})
 			})
 
 			Context("when the regex does not match the previous version", func() {
@@ -114,30 +150,66 @@ var _ = Describe("Check Command", func() {
 			})
 
 			Context("when using versioned file", func() {
-				BeforeEach(func() {
-					s3client.BucketFileVersionsReturns([]string{
-						"file-version-3",
-						"file-version-2",
-						"file-version-1",
-					}, nil)
+				Context("when there are existing versions", func() {
+					BeforeEach(func() {
+						s3client.BucketFileVersionsReturns([]string{
+							"file-version-3",
+							"file-version-2",
+							"file-version-1",
+						}, nil)
+					})
+
+					It("includes all versions from the previous one and the current one", func() {
+						request.Version.VersionID = "file-version-2"
+						request.Source.VersionedFile = "files/versioned-file"
+
+						response, err := command.Run(request)
+						Ω(err).ShouldNot(HaveOccurred())
+
+						Ω(response).Should(HaveLen(2))
+						Ω(response).Should(ConsistOf(
+							s3resource.Version{
+								VersionID: "file-version-2",
+							},
+							s3resource.Version{
+								VersionID: "file-version-3",
+							},
+						))
+					})
 				})
 
-				It("includes all versions from the previous one and the current one", func() {
-					request.Version.VersionID = "file-version-2"
-					request.Source.VersionedFile = "files/(.*).tgz"
-
-					response, err := command.Run(request)
-					Ω(err).ShouldNot(HaveOccurred())
-
-					Ω(response).Should(HaveLen(2))
-					Ω(response).Should(ConsistOf(
-						s3resource.Version{
-							VersionID: "file-version-2",
-						},
-						s3resource.Version{
-							VersionID: "file-version-3",
-						},
-					))
+				Context("when no version exists", func() {
+					BeforeEach(func() {
+						s3client.BucketFileVersionsReturns([]string{}, nil)
+					})
+
+					It("returns no versions", func() {
+						request.Version.VersionID = ""
+						request.Source.VersionedFile = "files/versioned-file"
+
+						response, err := command.Run(request)
+						Ω(err).ShouldNot(HaveOccurred())
+
+						Ω(response).Should(HaveLen(0))
+					})
+
+					Context("when the initial version is set", func() {
+						It("returns the initial version", func() {
+							request.Version.VersionID = ""
+							request.Source.VersionedFile = "files/versioned-file"
+							request.Source.InitialVersion = "file-version-0"
+
+							response, err := command.Run(request)
+							Ω(err).ShouldNot(HaveOccurred())
+
+							Ω(response).Should(HaveLen(1))
+							Ω(response).Should(ConsistOf(
+								s3resource.Version{
+									VersionID: "file-version-0",
+								},
+							))
+						})
+					})
 				})
 			})
 		})

diff --git a/in/command.go b/in/command.go
@@ -1,6 +1,7 @@
 package in
 
 import (
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -53,6 +54,8 @@ func (command *Command) Run(destinationDir string, request Request) (Response, e
 	var remotePath string
 	var versionNumber string
 	var versionID string
+	var url string
+	var isInitialVersion bool
 
 	if request.Source.Regexp != "" {
 		if request.Version.Path == "" {
@@ -67,40 +70,62 @@ func (command *Command) Run(destinationDir string, request Request) (Response, e
 		}
 
 		versionNumber = extraction.VersionNumber
+
+		isInitialVersion = request.Source.InitialPath != "" && request.Version.Path == request.Source.InitialPath
 	} else {
 		remotePath = request.Source.VersionedFile
 		versionNumber = request.Version.VersionID
 		versionID = request.Version.VersionID
-	}
-
-	err = command.downloadFile(
-		request.Source.Bucket,
-		remotePath,
-		versionID,
-		destinationDir,
-		path.Base(remotePath),
-	)
 
-	if err != nil {
-		return Response{}, err
+		isInitialVersion = request.Source.InitialVersion != "" && request.Version.VersionID == request.Source.InitialVersion
 	}
 
-	if request.Params.Unpack {
-		destinationPath := filepath.Join(destinationDir, path.Base(remotePath))
-		mime := archiveMimetype(destinationPath)
-		if mime == "" {
-			return Response{}, fmt.Errorf("not an archive: %s", destinationPath)
+	if isInitialVersion {
+		if request.Source.InitialContentText != "" || request.Source.InitialContentBinary == "" {
+			err = command.createInitialFile(destinationDir, path.Base(remotePath), []byte(request.Source.InitialContentText))
+			if err != nil {
+				return Response{}, err
+			}
 		}
-
-		err = extractArchive(mime, destinationPath)
+		if request.Source.InitialContentBinary != "" {
+			b, err := base64.StdEncoding.DecodeString(request.Source.InitialContentBinary)
+			if err != nil {
+				return Response{}, errors.New("failed to decode initial_content_binary, make sure it's base64 encoded")
+			}
+			err = command.createInitialFile(destinationDir, path.Base(remotePath), b)
+			if err != nil {
+				return Response{}, err
+			}
+		}
+	} else {
+		err = command.downloadFile(
+			request.Source.Bucket,
+			remotePath,
+			versionID,
+			destinationDir,
+			path.Base(remotePath),
+		)
 		if err != nil {
 			return Response{}, err
 		}
-	}
 
-	url := command.urlProvider.GetURL(request, remotePath)
-	if err = command.writeURLFile(destinationDir, url); err != nil {
-		return Response{}, err
+		if request.Params.Unpack {
+			destinationPath := filepath.Join(destinationDir, path.Base(remotePath))
+			mime := archiveMimetype(destinationPath)
+			if mime == "" {
+				return Response{}, fmt.Errorf("not an archive: %s", destinationPath)
+			}
+
+			err = extractArchive(mime, destinationPath)
+			if err != nil {
+				return Response{}, err
+			}
+		}
+
+		url = command.urlProvider.GetURL(request, remotePath)
+		if err = command.writeURLFile(destinationDir, url); err != nil {
+			return Response{}, err
+		}
 	}
 
 	err = command.writeVersionFile(versionNumber, destinationDir)
@@ -146,6 +171,10 @@ func (command *Command) downloadFile(bucketName string, remotePath string, versi
 	)
 }
 
+func (command *Command) createInitialFile(destDir string, destFile string, data []byte) error {
+	return ioutil.WriteFile(filepath.Join(destDir, destFile), []byte(data), 0644)
+}
+
 func (command *Command) metadata(remotePath string, private bool, url string) []s3resource.MetadataPair {
 	remoteFilename := filepath.Base(remotePath)
 
@@ -156,7 +185,7 @@ func (command *Command) metadata(remotePath string, private bool, url string) []
 		},
 	}
 
-	if !private {
+	if url != "" && !private {
 		metadata = append(metadata, s3resource.MetadataPair{
 			Name:  "url",
 			Value: url,