packer: pick protobuf/gob for serialisation (#13025)

As we're trying to move away from gob for serialising data over the
wire, this commit adds the capability for Packer to pick dynamically
between gob or protobuf for the serialisation format to communicate with
plugins.

As it stands, if all the plugins discovered are compatible with
protobuf, and we have not forced gob usage, protobuf will be the
serialisation format picked.

If any plugin is not compatible with protobuf, gob will be used for
communicating with all the plugins that will be used over the course of
a command.

command/build.go

@@ -102,6 +102,10 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
 		return ret
 	}
 
+	if packer.PackerUseProto {
+		log.Printf("[TRACE] Using protobuf for communication with plugins")
+	}
+
 	diags = packerStarter.Initialize(packer.InitializeOptions{})
 	ret = writeDiags(c.Ui, nil, diags)
 	if ret != 0 {

command/execute.go

@@ -5,8 +5,8 @@
 package command
 
 import (
+	"flag"
 	"fmt"
-	"log"
 	"regexp"
 	"strings"
 
@@ -75,18 +75,45 @@ var Datasources = map[string]packersdk.Datasource{
 
 var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner|datasource)-(.+)")
 
-func (c *ExecuteCommand) Run(args []string) int {
-	// This is an internal call (users should not call this directly) so we're
-	// not going to do much input validation. If there's a problem we'll often
-	// just crash. Error handling should be added to facilitate debugging.
-	log.Printf("args: %#v", args)
+type ExecuteArgs struct {
+	UseProtobuf bool
+	CommandType string
+}
+
+func (ea *ExecuteArgs) AddFlagSets(flags *flag.FlagSet) {
+	flags.BoolVar(&ea.UseProtobuf, "protobuf", false, "Use protobuf for serialising data over the wire instead of gob")
+}
+
+func (c *ExecuteCommand) ParseArgs(args []string) (*ExecuteArgs, int) {
+	var cfg ExecuteArgs
+	flags := c.Meta.FlagSet("")
+	flags.Usage = func() { c.Ui.Say(c.Help()) }
+	cfg.AddFlagSets(flags)
+	if err := flags.Parse(args); err != nil {
+		return &cfg, 1
+	}
+
+	args = flags.Args()
 	if len(args) != 1 {
-		c.Ui.Error(c.Help())
-		return 1
+		flags.Usage()
+		return &cfg, 1
 	}
+	cfg.CommandType = args[0]
+	return &cfg, 0
+}
 
+func (c *ExecuteCommand) Run(args []string) int {
+	cfg, ret := c.ParseArgs(args)
+	if ret != 0 {
+		return ret
+	}
+
+	return c.RunContext(cfg)
+}
+
+func (c *ExecuteCommand) RunContext(args *ExecuteArgs) int {
 	// Plugin will match something like "packer-builder-amazon-ebs"
-	parts := pluginRegexp.FindStringSubmatch(args[0])
+	parts := pluginRegexp.FindStringSubmatch(args.CommandType)
 	if len(parts) != 3 {
 		c.Ui.Error(c.Help())
 		return 1
@@ -100,6 +127,10 @@ func (c *ExecuteCommand) Run(args []string) int {
 		return 1
 	}
 
+	if args.UseProtobuf {
+		server.UseProto = true
+	}
+
 	switch pluginType {
 	case "builder":
 		builder, found := Builders[pluginName]
@@ -138,11 +169,15 @@ func (c *ExecuteCommand) Run(args []string) int {
 
 func (*ExecuteCommand) Help() string {
 	helpText := `
-Usage: packer execute PLUGIN
+Usage: packer execute [options] PLUGIN
 
   Runs an internally-compiled version of a plugin from the packer binary.
 
   NOTE: this is an internal command and you should not call it yourself.
+
+Options:
+
+  --protobuf: use protobuf for serialising data over-the-wire instead of gob.
 `
 
 	return strings.TrimSpace(helpText)

command/validate.go

@@ -5,6 +5,7 @@ package command
 
 import (
 	"context"
+	"log"
 	"strings"
 
 	"github.com/hashicorp/packer/packer"
@@ -76,6 +77,10 @@ func (c *ValidateCommand) RunContext(ctx context.Context, cla *ValidateArgs) int
 		return ret
 	}
 
+	if packer.PackerUseProto {
+		log.Printf("[TRACE] Using protobuf for communication with plugins")
+	}
+
 	diags = packerStarter.Initialize(packer.InitializeOptions{
 		SkipDatasourcesExecution: !cla.EvaluateDatasources,
 	})

config.go

@@ -16,10 +16,6 @@ import (
 	"github.com/hashicorp/packer/packer"
 )
 
-// PACKERSPACE is used to represent the spaces that separate args for a command
-// without being confused with spaces in the path to the command itself.
-const PACKERSPACE = "-PACKERSPACE-"
-
 type config struct {
 	DisableCheckpoint          bool              `json:"disable_checkpoint"`
 	DisableCheckpointSignature bool              `json:"disable_checkpoint_signature"`
@@ -109,43 +105,67 @@ func (c *config) discoverInternalComponents() error {
 	for builder := range command.Builders {
 		builder := builder
 		if !c.Plugins.Builders.Has(builder) {
-			bin := fmt.Sprintf("%s%sexecute%spacker-builder-%s",
-				packerPath, PACKERSPACE, PACKERSPACE, builder)
 			c.Plugins.Builders.Set(builder, func() (packersdk.Builder, error) {
-				return c.Plugins.Client(bin).Builder()
+				args := []string{"execute"}
+
+				if packer.PackerUseProto {
+					args = append(args, "--protobuf")
+				}
+
+				args = append(args, fmt.Sprintf("packer-builder-%s", builder))
+
+				return c.Plugins.Client(packerPath, args...).Builder()
 			})
 		}
 	}
 
 	for provisioner := range command.Provisioners {
 		provisioner := provisioner
 		if !c.Plugins.Provisioners.Has(provisioner) {
-			bin := fmt.Sprintf("%s%sexecute%spacker-provisioner-%s",
-				packerPath, PACKERSPACE, PACKERSPACE, provisioner)
 			c.Plugins.Provisioners.Set(provisioner, func() (packersdk.Provisioner, error) {
-				return c.Plugins.Client(bin).Provisioner()
+				args := []string{"execute"}
+
+				if packer.PackerUseProto {
+					args = append(args, "--protobuf")
+				}
+
+				args = append(args, fmt.Sprintf("packer-provisioner-%s", provisioner))
+
+				return c.Plugins.Client(packerPath, args...).Provisioner()
 			})
 		}
 	}
 
 	for postProcessor := range command.PostProcessors {
 		postProcessor := postProcessor
 		if !c.Plugins.PostProcessors.Has(postProcessor) {
-			bin := fmt.Sprintf("%s%sexecute%spacker-post-processor-%s",
-				packerPath, PACKERSPACE, PACKERSPACE, postProcessor)
 			c.Plugins.PostProcessors.Set(postProcessor, func() (packersdk.PostProcessor, error) {
-				return c.Plugins.Client(bin).PostProcessor()
+				args := []string{"execute"}
+
+				if packer.PackerUseProto {
+					args = append(args, "--protobuf")
+				}
+
+				args = append(args, fmt.Sprintf("packer-post-processor-%s", postProcessor))
+
+				return c.Plugins.Client(packerPath, args...).PostProcessor()
 			})
 		}
 	}
 
 	for dataSource := range command.Datasources {
 		dataSource := dataSource
 		if !c.Plugins.DataSources.Has(dataSource) {
-			bin := fmt.Sprintf("%s%sexecute%spacker-datasource-%s",
-				packerPath, PACKERSPACE, PACKERSPACE, dataSource)
 			c.Plugins.DataSources.Set(dataSource, func() (packersdk.Datasource, error) {
-				return c.Plugins.Client(bin).Datasource()
+				args := []string{"execute"}
+
+				if packer.PackerUseProto {
+					args = append(args, "--protobuf")
+				}
+
+				args = append(args, fmt.Sprintf("packer-datasource-%s", dataSource))
+
+				return c.Plugins.Client(packerPath, args...).Datasource()
 			})
 		}
 	}

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/hashicorp/hcl/v2 v2.19.1
 	github.com/hashicorp/hcp-sdk-go v0.105.0
 	github.com/hashicorp/packer-plugin-amazon v1.2.1
-	github.com/hashicorp/packer-plugin-sdk v0.5.4
+	github.com/hashicorp/packer-plugin-sdk v0.5.5-0.20240715190433-28e1cb00bc48
 	github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/klauspost/pgzip v1.2.5
@@ -84,7 +84,7 @@ require (
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
-	github.com/aws/aws-sdk-go v1.44.114 // indirect
+	github.com/aws/aws-sdk-go v1.45.6 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/bgentry/speakeasy v0.1.0 // indirect
 	github.com/bmatcuk/doublestar v1.1.5 // indirect
@@ -169,6 +169,8 @@ require (
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/ugorji/go/codec v1.2.6 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect

go.sum

@@ -64,8 +64,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.31.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/aws/aws-sdk-go v1.44.114 h1:plIkWc/RsHr3DXBj4MEw9sEW4CcL/e2ryokc+CKyq1I=
-github.com/aws/aws-sdk-go v1.44.114/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.45.6 h1:Y2isQQBZsnO15dzUQo9YQRThtHgrV200XCH05BRHVJI=
+github.com/aws/aws-sdk-go v1.45.6/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@@ -305,8 +305,8 @@ github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR
 github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0=
 github.com/hashicorp/packer-plugin-amazon v1.2.1 h1:0Xqr8KsTJJhIo0vvjqPYrVMgyVxNRuYH4DeB5m/WAtw=
 github.com/hashicorp/packer-plugin-amazon v1.2.1/go.mod h1:qlp0h5TWVGgcPzN9mSxPiEAwOUOW3XU/zep0pGd0ZsM=
-github.com/hashicorp/packer-plugin-sdk v0.5.4 h1:5Bl5DMEa//G4gBNcl842JopM9L4KSSsxpvB4W1lEwIA=
-github.com/hashicorp/packer-plugin-sdk v0.5.4/go.mod h1:ALm0ZIK3c/F4iOqPNi7xVuHTgrR5dxzOK+DhFN5DHj4=
+github.com/hashicorp/packer-plugin-sdk v0.5.5-0.20240715190433-28e1cb00bc48 h1:MxQWh/2TOwX7j+iAkDzvWV2+YlZokDr4zQfaEp7nnYM=
+github.com/hashicorp/packer-plugin-sdk v0.5.5-0.20240715190433-28e1cb00bc48/go.mod h1:ALm0ZIK3c/F4iOqPNi7xVuHTgrR5dxzOK+DhFN5DHj4=
 github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
 github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
 github.com/hashicorp/vault/api v1.14.0 h1:Ah3CFLixD5jmjusOgm8grfN9M0d+Y8fVR2SW0K6pJLU=
@@ -528,7 +528,9 @@ github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxW
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
 github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
+github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
@@ -604,8 +606,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
@@ -651,13 +653,13 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -667,6 +669,7 @@ golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=