Adds --unbuffered flag to create-stack for really large stack images

commands/create_stack.go

@@ -20,6 +20,7 @@ type createStackFlags struct {
 	buildOutput string
 	runOutput   string
 	secrets     []string
+	unbuffered  bool
 }
 
 func createStack() *cobra.Command {
@@ -35,6 +36,7 @@ func createStack() *cobra.Command {
 	cmd.Flags().StringVar(&flags.buildOutput, "build-output", "", "path to output the build image OCI archive (required)")
 	cmd.Flags().StringVar(&flags.runOutput, "run-output", "", "path to output the run image OCI archive (required)")
 	cmd.Flags().StringSliceVar(&flags.secrets, "secret", nil, "secret to be passed to your Dockerfile")
+	cmd.Flags().BoolVar(&flags.unbuffered, "unbuffered", false, "do not buffer image contents into memory for fast access")
 
 	err := cmd.MarkFlagRequired("config")
 	if err != nil {
@@ -55,7 +57,7 @@ func createStack() *cobra.Command {
 }
 
 func createStackRun(flags createStackFlags) error {
-	definition, err := ihop.NewDefinitionFromFile(flags.config, flags.secrets...)
+	definition, err := ihop.NewDefinitionFromFile(flags.config, flags.unbuffered, flags.secrets...)
 	if err != nil {
 		return err
 	}

internal/ihop/client.go

@@ -51,6 +51,8 @@ type Image struct {
 	Labels map[string]string
 
 	Layers []Layer
+
+	unbuffered bool
 }
 
 // ToDaemonImage returns the GGCR v1.Image associated with this Image.
@@ -60,7 +62,12 @@ func (i Image) ToDaemonImage() (v1.Image, error) {
 		return nil, err
 	}
 
-	image, err := daemon.Image(ref)
+	option := daemon.WithBufferedOpener()
+	if i.unbuffered {
+		option = daemon.WithUnbufferedOpener()
+	}
+
+	image, err := daemon.Image(ref, option)
 	if err != nil {
 		return nil, err
 	}
@@ -277,7 +284,7 @@ func (c Client) Build(def DefinitionImage, platform string) (Image, error) {
 	}
 
 	// fetch and return a reference to the built image
-	return c.get(Image{Tag: tag})
+	return c.get(Image{Tag: tag, unbuffered: def.unbuffered})
 }
 
 // Update will apply any modifications made to the Image reference onto the
@@ -507,6 +514,7 @@ func (c Client) get(img Image) (Image, error) {
 		User:         file.Config.User,
 		OS:           file.OS,
 		Architecture: file.Architecture,
+		unbuffered:   img.unbuffered,
 	}, nil
 }
 

internal/ihop/definition.go

@@ -66,6 +66,8 @@ type DefinitionImage struct {
 
 	// UID is the cnb user id to be specified in the image.
 	UID int `toml:"uid"`
+
+	unbuffered bool
 }
 
 // DefinitionDeprecated defines the deprecated features of the stack.
@@ -91,7 +93,7 @@ func (i DefinitionImage) Arguments() []string {
 }
 
 // NewDefinitionFromFile parses the stack descriptor from a file location.
-func NewDefinitionFromFile(path string, secrets ...string) (Definition, error) {
+func NewDefinitionFromFile(path string, unbuffered bool, secrets ...string) (Definition, error) {
 	path, err := filepath.Abs(path)
 	if err != nil {
 		return Definition{}, err
@@ -175,6 +177,9 @@ func NewDefinitionFromFile(path string, secrets ...string) (Definition, error) {
 		definition.Run.Secrets = definition.Build.Secrets
 	}
 
+	definition.Build.unbuffered = unbuffered
+	definition.Run.unbuffered = unbuffered
+
 	return definition, nil
 }
 

internal/ihop/definition_test.go

@@ -63,7 +63,7 @@ platforms = ["some-stack-platform"]
 		})
 
 		it("creates a definition from a config file", func() {
-			definition, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"))
+			definition, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), false)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(definition).To(Equal(ihop.Definition{
 				ID:         "some-stack-id",
@@ -117,7 +117,7 @@ id = "some-stack-id"
 				})
 
 				it("sets the defaults", func() {
-					definition, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"))
+					definition, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), false)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(definition).To(Equal(ihop.Definition{
 						ID:        "some-stack-id",
@@ -157,7 +157,7 @@ id = "some-stack-id"
 				})
 
 				it("returns an error", func() {
-					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"))
+					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), false)
 					Expect(err).To(MatchError("failed to parse stack descriptor: 'id' is a required field"))
 				})
 			})
@@ -180,7 +180,7 @@ id = "some-stack-id"
 				})
 
 				it("returns an error", func() {
-					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"))
+					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), false)
 					Expect(err).To(MatchError("failed to parse stack descriptor: 'build.dockerfile' is a required field"))
 				})
 			})
@@ -203,7 +203,7 @@ id = "some-stack-id"
 				})
 
 				it("returns an error", func() {
-					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"))
+					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), false)
 					Expect(err).To(MatchError("failed to parse stack descriptor: 'build.uid' is a required field"))
 				})
 			})
@@ -227,7 +227,7 @@ id = "some-stack-id"
 				})
 
 				it("returns an error", func() {
-					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"))
+					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), false)
 					Expect(err).To(MatchError("failed to parse stack descriptor: 'build.gid' is a required field"))
 				})
 			})
@@ -250,7 +250,7 @@ id = "some-stack-id"
 				})
 
 				it("returns an error", func() {
-					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"))
+					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), false)
 					Expect(err).To(MatchError("failed to parse stack descriptor: 'run.dockerfile' is a required field"))
 				})
 			})
@@ -273,7 +273,7 @@ id = "some-stack-id"
 				})
 
 				it("returns an error", func() {
-					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"))
+					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), false)
 					Expect(err).To(MatchError("failed to parse stack descriptor: 'run.uid' is a required field"))
 				})
 			})
@@ -296,15 +296,15 @@ id = "some-stack-id"
 				})
 
 				it("returns an error", func() {
-					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"))
+					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), false)
 					Expect(err).To(MatchError("failed to parse stack descriptor: 'run.gid' is a required field"))
 				})
 			})
 		})
 
 		context("when secrets are given", func() {
 			it("includes them in the build/run image definitions", func() {
-				definition, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), "first-secret=first-value", "second-secret=second-value")
+				definition, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), false, "first-secret=first-value", "second-secret=second-value")
 				Expect(err).NotTo(HaveOccurred())
 				Expect(definition.Build.Secrets).To(Equal(map[string]string{
 					"first-secret":  "first-value",
@@ -320,7 +320,7 @@ id = "some-stack-id"
 		context("failure cases", func() {
 			context("when the file cannot be opened", func() {
 				it("returns an error", func() {
-					_, err := ihop.NewDefinitionFromFile("this file does not exist")
+					_, err := ihop.NewDefinitionFromFile("this file does not exist", false)
 					Expect(err).To(MatchError(ContainSubstring("no such file or directory")))
 				})
 			})
@@ -332,14 +332,14 @@ id = "some-stack-id"
 				})
 
 				it("returns an error", func() {
-					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"))
+					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), false)
 					Expect(err).To(MatchError(ContainSubstring("but got '%' instead")))
 				})
 			})
 
 			context("when a secret is malformed", func() {
 				it("returns an error", func() {
-					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), "this secret is malformed")
+					_, err := ihop.NewDefinitionFromFile(filepath.Join(dir, "stack.toml"), false, "this secret is malformed")
 					Expect(err).To(MatchError("malformed secret: \"this secret is malformed\" must be in the form \"key=value\""))
 				})
 			})