From 7ab991509c09c8cef06ac603adf594d12b5a79fa Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Mon, 15 Jan 2024 14:31:04 -0500 Subject: [PATCH 01/32] packer: only support one directory for plugins Since we're removing the alternative plugin installation directories in favour of only supporting installing them in the PACKER_PLUGIN_PATH directory, we only return one directory when getting the known plugin directories. --- command/init.go | 2 +- command/init_test.go | 2 +- command/plugins_install.go | 2 +- command/plugins_install_test.go | 2 +- command/plugins_installed.go | 2 +- command/plugins_remove.go | 2 +- command/plugins_remove_test.go | 2 +- command/plugins_required.go | 2 +- hcl2template/plugin.go | 2 +- main.go | 11 ++++++++--- packer/plugin.go | 24 ++++++++---------------- packer/plugin_folders.go | 26 +++++++------------------- 12 files changed, 32 insertions(+), 47 deletions(-) diff --git a/command/init.go b/command/init.go index 9e5b82f04ba..3c7cfea09f9 100644 --- a/command/init.go +++ b/command/init.go @@ -76,7 +76,7 @@ for more info.`) } opts := plugingetter.ListInstallationsOptions{ - FromFolders: c.Meta.CoreConfig.Components.PluginConfig.KnownPluginFolders, + FromFolders: []string{c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory}, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ OS: runtime.GOOS, ARCH: runtime.GOARCH, diff --git a/command/init_test.go b/command/init_test.go index d39699ee51c..ad501f15182 100644 --- a/command/init_test.go +++ b/command/init_test.go @@ -267,7 +267,7 @@ func TestInitCommand_Run(t *testing.T) { t.Fatalf("Failed to discover plugins: %s", err) } - c.CoreConfig.Components.PluginConfig.KnownPluginFolders = []string{tt.packerConfigDir} + c.CoreConfig.Components.PluginConfig.PluginDirectory = tt.packerConfigDir if got := c.Run(args); got != tt.want { t.Errorf("InitCommand.Run() = %v, want %v", got, tt.want) } diff --git a/command/plugins_install.go b/command/plugins_install.go index 753d2d558ba..c63ff2afacf 100644 --- a/command/plugins_install.go +++ b/command/plugins_install.go @@ -121,7 +121,7 @@ func (c *PluginsInstallCommand) ParseArgs(args []string) (*PluginsInstallArgs, i func (c *PluginsInstallCommand) RunContext(buildCtx context.Context, args *PluginsInstallArgs) int { opts := plugingetter.ListInstallationsOptions{ - FromFolders: c.Meta.CoreConfig.Components.PluginConfig.KnownPluginFolders, + FromFolders: []string{c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory}, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ OS: runtime.GOOS, ARCH: runtime.GOARCH, diff --git a/command/plugins_install_test.go b/command/plugins_install_test.go index 5461360c36e..e8ad08fd4d8 100644 --- a/command/plugins_install_test.go +++ b/command/plugins_install_test.go @@ -151,7 +151,7 @@ func TestPluginsInstallCommand_Run(t *testing.T) { t.Fatalf("Failed to discover plugins: %s", err) } - c.CoreConfig.Components.PluginConfig.KnownPluginFolders = []string{tt.packerConfigDir} + c.CoreConfig.Components.PluginConfig.PluginDirectory = tt.packerConfigDir if got := c.Run(tt.pluginSourceArgs); got != tt.want { t.Errorf("PluginsInstallCommand.Run() = %v, want %v", got, tt.want) } diff --git a/command/plugins_installed.go b/command/plugins_installed.go index b19be4e5c33..55e47f9894a 100644 --- a/command/plugins_installed.go +++ b/command/plugins_installed.go @@ -43,7 +43,7 @@ func (c *PluginsInstalledCommand) Run(args []string) int { func (c *PluginsInstalledCommand) RunContext(buildCtx context.Context) int { opts := plugingetter.ListInstallationsOptions{ - FromFolders: c.Meta.CoreConfig.Components.PluginConfig.KnownPluginFolders, + FromFolders: []string{c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory}, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ OS: runtime.GOOS, ARCH: runtime.GOARCH, diff --git a/command/plugins_remove.go b/command/plugins_remove.go index 5c13636f5c6..6da31410770 100644 --- a/command/plugins_remove.go +++ b/command/plugins_remove.go @@ -52,7 +52,7 @@ func (c *PluginsRemoveCommand) RunContext(buildCtx context.Context, args []strin } opts := plugingetter.ListInstallationsOptions{ - FromFolders: c.Meta.CoreConfig.Components.PluginConfig.KnownPluginFolders, + FromFolders: []string{c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory}, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ OS: runtime.GOOS, ARCH: runtime.GOARCH, diff --git a/command/plugins_remove_test.go b/command/plugins_remove_test.go index 4149ceaeb9c..852ab49aad5 100644 --- a/command/plugins_remove_test.go +++ b/command/plugins_remove_test.go @@ -186,7 +186,7 @@ func TestPluginsRemoveCommand_Run(t *testing.T) { Meta: tt.Meta, } - c.CoreConfig.Components.PluginConfig.KnownPluginFolders = []string{tt.packerConfigDir} + c.CoreConfig.Components.PluginConfig.PluginDirectory = tt.packerConfigDir if got := c.Run(tt.pluginSourceArgs); got != tt.want { t.Errorf("PluginsRemoveCommand.Run() = %v, want %v", got, tt.want) } diff --git a/command/plugins_required.go b/command/plugins_required.go index 7776b19210b..fd851380817 100644 --- a/command/plugins_required.go +++ b/command/plugins_required.go @@ -84,7 +84,7 @@ func (c *PluginsRequiredCommand) RunContext(buildCtx context.Context, cla *Plugi } opts := plugingetter.ListInstallationsOptions{ - FromFolders: c.Meta.CoreConfig.Components.PluginConfig.KnownPluginFolders, + FromFolders: []string{c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory}, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ OS: runtime.GOOS, ARCH: runtime.GOARCH, diff --git a/hcl2template/plugin.go b/hcl2template/plugin.go index 321616017b6..de8549f38ca 100644 --- a/hcl2template/plugin.go +++ b/hcl2template/plugin.go @@ -56,7 +56,7 @@ func (cfg *PackerConfig) PluginRequirements() (plugingetter.Requirements, hcl.Di func (cfg *PackerConfig) DetectPluginBinaries() hcl.Diagnostics { opts := plugingetter.ListInstallationsOptions{ - FromFolders: cfg.parser.PluginConfig.KnownPluginFolders, + FromFolders: []string{cfg.parser.PluginConfig.PluginDirectory}, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ OS: runtime.GOOS, ARCH: runtime.GOARCH, diff --git a/main.go b/main.go index fe902f5bdb3..bb1d741d694 100644 --- a/main.go +++ b/main.go @@ -329,11 +329,16 @@ func extractMachineReadable(args []string) ([]string, bool) { } func loadConfig() (*config, error) { + pluginDir, err := packer.PluginFolder() + if err != nil { + return nil, err + } + var config config config.Plugins = &packer.PluginConfig{ - PluginMinPort: 10000, - PluginMaxPort: 25000, - KnownPluginFolders: packer.PluginFolders("."), + PluginMinPort: 10000, + PluginMaxPort: 25000, + PluginDirectory: pluginDir, } if err := config.Plugins.Discover(); err != nil { return nil, err diff --git a/packer/plugin.go b/packer/plugin.go index 86e02289f5c..fe3c71623f4 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -27,13 +27,13 @@ var defaultChecksummer = plugingetter.Checksummer{ // PluginConfig helps load and use packer plugins type PluginConfig struct { - KnownPluginFolders []string - PluginMinPort int - PluginMaxPort int - Builders BuilderSet - Provisioners ProvisionerSet - PostProcessors PostProcessorSet - DataSources DatasourceSet + PluginDirectory string + PluginMinPort int + PluginMaxPort int + Builders BuilderSet + Provisioners ProvisionerSet + PostProcessors PostProcessorSet + DataSources DatasourceSet } // PACKERSPACE is used to represent the spaces that separate args for a command @@ -67,15 +67,7 @@ func (c *PluginConfig) Discover() error { return nil } - if len(c.KnownPluginFolders) == 0 { - //PluginFolders should match the call in github.com/hahicorp/packer/main.go#loadConfig - c.KnownPluginFolders = PluginFolders(".") - } - - // Pick last folder as it's the one with the highest priority - // This is the same logic used when installing plugins via Packer's plugin installation commands. - pluginInstallationPath := c.KnownPluginFolders[len(c.KnownPluginFolders)-1] - if err := c.discoverInstalledComponents(pluginInstallationPath); err != nil { + if err := c.discoverInstalledComponents(c.PluginDirectory); err != nil { return err } diff --git a/packer/plugin_folders.go b/packer/plugin_folders.go index e1d4dbea4a2..5682c91b8d6 100644 --- a/packer/plugin_folders.go +++ b/packer/plugin_folders.go @@ -7,33 +7,21 @@ import ( "log" "os" "path/filepath" - "strings" "github.com/hashicorp/packer-plugin-sdk/pathing" ) -// PluginFolders returns the list of known plugin folders based on system. -func PluginFolders(dirs ...string) []string { - res := []string{} - +// PluginFolder returns the known plugin folder based on system. +func PluginFolder() (string, error) { if packerPluginPath := os.Getenv("PACKER_PLUGIN_PATH"); packerPluginPath != "" { - res = append(res, strings.Split(packerPluginPath, string(os.PathListSeparator))...) - return res - } - - if path, err := os.Executable(); err != nil { - log.Printf("[ERR] Error finding executable: %v", err) - } else { - res = append(res, filepath.Dir(path)) + return packerPluginPath, nil } - res = append(res, dirs...) - - if cd, err := pathing.ConfigDir(); err != nil { + cd, err := pathing.ConfigDir() + if err != nil { log.Printf("[ERR] Error loading config directory: %v", err) - } else { - res = append(res, filepath.Join(cd, "plugins")) + return "", err } - return res + return filepath.Join(cd, "plugins"), nil } From b9947d1c176555d637049862bd83c1fafa6c8e83 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Mon, 15 Jan 2024 14:32:46 -0500 Subject: [PATCH 02/32] packer: rm support for manually-installed plugins Since we'll be only accepting plugins installed locally along with a shasum, and those that adopt the convention we introduced with required_plugins, we remove support for plugins that only have a packer-plugin-.* type name. --- packer/plugin.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/packer/plugin.go b/packer/plugin.go index fe3c71623f4..246a7e1fec7 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -71,24 +71,6 @@ func (c *PluginConfig) Discover() error { return err } - // Manually installed plugins take precedence over all. Duplicate plugins installed - // prior to the packer plugins install command should be removed by user to avoid overrides. - for _, knownFolder := range c.KnownPluginFolders { - pluginPaths, err := c.discoverSingle(filepath.Join(knownFolder, "packer-plugin-*")) - if err != nil { - return err - } - for pluginName, pluginPath := range pluginPaths { - // Test pluginPath points to an executable - if _, err := exec.LookPath(pluginPath); err != nil { - log.Printf("[WARN] %q is not executable; skipping", pluginPath) - continue - } - if err := c.DiscoverMultiPlugin(pluginName, pluginPath); err != nil { - return err - } - } - } return nil } From d564079e8c6210e9959033ed44b951a74ef7623e Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Mon, 15 Jan 2024 14:41:25 -0500 Subject: [PATCH 03/32] packer: remove directory slices from structs Since we'll only look in the plugin directory, and not from multiple sources now, for installing/listing plugins, we can simplify the structures which used to accept multiple directories so they only accept one. --- command/init.go | 4 +- command/plugins_install.go | 9 +- command/plugins_installed.go | 2 +- command/plugins_remove.go | 2 +- command/plugins_required.go | 2 +- hcl2template/plugin.go | 2 +- packer/plugin-getter/plugins.go | 183 +++++++++++++-------------- packer/plugin-getter/plugins_test.go | 136 ++------------------ packer/plugin.go | 9 +- packer/plugin_discover_test.go | 50 +++----- 10 files changed, 137 insertions(+), 262 deletions(-) diff --git a/command/init.go b/command/init.go index 3c7cfea09f9..2d8a1937759 100644 --- a/command/init.go +++ b/command/init.go @@ -76,7 +76,7 @@ for more info.`) } opts := plugingetter.ListInstallationsOptions{ - FromFolders: []string{c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory}, + PluginDirectory: c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ OS: runtime.GOOS, ARCH: runtime.GOARCH, @@ -132,7 +132,7 @@ for more info.`) } newInstall, err := pluginRequirement.InstallLatest(plugingetter.InstallOptions{ - InFolders: opts.FromFolders, + PluginDirectory: opts.PluginDirectory, BinaryInstallationOptions: opts.BinaryInstallationOptions, Getters: getters, Force: cla.Force, diff --git a/command/plugins_install.go b/command/plugins_install.go index c63ff2afacf..1234579206a 100644 --- a/command/plugins_install.go +++ b/command/plugins_install.go @@ -121,7 +121,7 @@ func (c *PluginsInstallCommand) ParseArgs(args []string) (*PluginsInstallArgs, i func (c *PluginsInstallCommand) RunContext(buildCtx context.Context, args *PluginsInstallArgs) int { opts := plugingetter.ListInstallationsOptions{ - FromFolders: []string{c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory}, + PluginDirectory: c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ OS: runtime.GOOS, ARCH: runtime.GOARCH, @@ -176,7 +176,7 @@ func (c *PluginsInstallCommand) RunContext(buildCtx context.Context, args *Plugi } newInstall, err := pluginRequirement.InstallLatest(plugingetter.InstallOptions{ - InFolders: opts.FromFolders, + PluginDirectory: opts.PluginDirectory, BinaryInstallationOptions: opts.BinaryInstallationOptions, Getters: getters, Force: args.Force, @@ -201,10 +201,7 @@ func (c *PluginsInstallCommand) RunContext(buildCtx context.Context, args *Plugi } func (c *PluginsInstallCommand) InstallFromBinary(opts plugingetter.ListInstallationsOptions, pluginIdentifier *addrs.Plugin, args *PluginsInstallArgs) int { - // As with the other commands, we get the last plugin directory as it - // has precedence over the others, and is where we'll install the - // plugins to. - pluginDir := opts.FromFolders[len(opts.FromFolders)-1] + pluginDir := opts.PluginDirectory var err error diff --git a/command/plugins_installed.go b/command/plugins_installed.go index 55e47f9894a..9652572da2f 100644 --- a/command/plugins_installed.go +++ b/command/plugins_installed.go @@ -43,7 +43,7 @@ func (c *PluginsInstalledCommand) Run(args []string) int { func (c *PluginsInstalledCommand) RunContext(buildCtx context.Context) int { opts := plugingetter.ListInstallationsOptions{ - FromFolders: []string{c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory}, + PluginDirectory: c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ OS: runtime.GOOS, ARCH: runtime.GOARCH, diff --git a/command/plugins_remove.go b/command/plugins_remove.go index 6da31410770..7b29ffa7bd2 100644 --- a/command/plugins_remove.go +++ b/command/plugins_remove.go @@ -52,7 +52,7 @@ func (c *PluginsRemoveCommand) RunContext(buildCtx context.Context, args []strin } opts := plugingetter.ListInstallationsOptions{ - FromFolders: []string{c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory}, + PluginDirectory: c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ OS: runtime.GOOS, ARCH: runtime.GOARCH, diff --git a/command/plugins_required.go b/command/plugins_required.go index fd851380817..7ba301dcfcf 100644 --- a/command/plugins_required.go +++ b/command/plugins_required.go @@ -84,7 +84,7 @@ func (c *PluginsRequiredCommand) RunContext(buildCtx context.Context, cla *Plugi } opts := plugingetter.ListInstallationsOptions{ - FromFolders: []string{c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory}, + PluginDirectory: c.Meta.CoreConfig.Components.PluginConfig.PluginDirectory, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ OS: runtime.GOOS, ARCH: runtime.GOARCH, diff --git a/hcl2template/plugin.go b/hcl2template/plugin.go index de8549f38ca..15358573d6e 100644 --- a/hcl2template/plugin.go +++ b/hcl2template/plugin.go @@ -56,7 +56,7 @@ func (cfg *PackerConfig) PluginRequirements() (plugingetter.Requirements, hcl.Di func (cfg *PackerConfig) DetectPluginBinaries() hcl.Diagnostics { opts := plugingetter.ListInstallationsOptions{ - FromFolders: []string{cfg.parser.PluginConfig.PluginDirectory}, + PluginDirectory: cfg.parser.PluginConfig.PluginDirectory, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ OS: runtime.GOOS, ARCH: runtime.GOARCH, diff --git a/packer/plugin-getter/plugins.go b/packer/plugin-getter/plugins.go index d6f52ebcc22..1aa7834d138 100644 --- a/packer/plugin-getter/plugins.go +++ b/packer/plugin-getter/plugins.go @@ -58,9 +58,8 @@ type BinaryInstallationOptions struct { } type ListInstallationsOptions struct { - // FromFolders where plugins could be installed. Paths should be absolute for - // safety but can also be relative. - FromFolders []string + // The directory in which to look for when installing plugins + PluginDirectory string BinaryInstallationOptions } @@ -105,84 +104,84 @@ func (pr Requirement) ListInstallations(opts ListInstallationsOptions) (InstallL FilenamePrefix := pr.FilenamePrefix() filenameSuffix := opts.FilenameSuffix() log.Printf("[TRACE] listing potential installations for %q that match %q. %#v", pr.Identifier, pr.VersionConstraints, opts) - for _, knownFolder := range opts.FromFolders { - glob := "" + + glob := "" + if pr.Identifier == nil { + glob = filepath.Join(opts.PluginDirectory, "*", "*", "*", FilenamePrefix+"*"+filenameSuffix) + } else { + glob = filepath.Join(opts.PluginDirectory, pr.Identifier.Hostname, pr.Identifier.Namespace, pr.Identifier.Type, FilenamePrefix+"*"+filenameSuffix) + } + + matches, err := filepath.Glob(glob) + if err != nil { + return nil, fmt.Errorf("ListInstallations: %q failed to list binaries in folder: %v", pr.Identifier.String(), err) + } + for _, path := range matches { + fname := filepath.Base(path) + if fname == "." { + continue + } + + // base name could look like packer-plugin-amazon_v1.2.3_x5.1_darwin_amd64.exe + versionsStr := strings.TrimPrefix(fname, FilenamePrefix) + versionsStr = strings.TrimSuffix(versionsStr, filenameSuffix) + if pr.Identifier == nil { - glob = filepath.Join(knownFolder, "*", "*", "*", FilenamePrefix+"*"+filenameSuffix) - } else { - glob = filepath.Join(knownFolder, pr.Identifier.Hostname, pr.Identifier.Namespace, pr.Identifier.Type, FilenamePrefix+"*"+filenameSuffix) + if idx := strings.Index(versionsStr, "_"); idx > 0 { + versionsStr = versionsStr[idx+1:] + } } - matches, err := filepath.Glob(glob) + // versionsStr now looks like v1.2.3_x5.1 or amazon_v1.2.3_x5.1 + parts := strings.SplitN(versionsStr, "_", 2) + pluginVersionStr, protocolVerionStr := parts[0], parts[1] + pv, err := version.NewVersion(pluginVersionStr) if err != nil { - return nil, fmt.Errorf("ListInstallations: %q failed to list binaries in folder: %v", pr.Identifier.String(), err) + // could not be parsed, ignoring the file + log.Printf("found %q with an incorrect %q version, ignoring it. %v", path, pluginVersionStr, err) + continue } - for _, path := range matches { - fname := filepath.Base(path) - if fname == "." { - continue - } - - // base name could look like packer-plugin-amazon_v1.2.3_x5.1_darwin_amd64.exe - versionsStr := strings.TrimPrefix(fname, FilenamePrefix) - versionsStr = strings.TrimSuffix(versionsStr, filenameSuffix) - if pr.Identifier == nil { - if idx := strings.Index(versionsStr, "_"); idx > 0 { - versionsStr = versionsStr[idx+1:] - } - } + // no constraint means always pass, this will happen for implicit + // plugin requirements and when we list all plugins. + if !pr.VersionConstraints.Check(pv) { + log.Printf("[TRACE] version %q of file %q does not match constraint %q", pluginVersionStr, path, pr.VersionConstraints.String()) + continue + } - // versionsStr now looks like v1.2.3_x5.1 or amazon_v1.2.3_x5.1 - parts := strings.SplitN(versionsStr, "_", 2) - pluginVersionStr, protocolVerionStr := parts[0], parts[1] - pv, err := version.NewVersion(pluginVersionStr) - if err != nil { - // could not be parsed, ignoring the file - log.Printf("found %q with an incorrect %q version, ignoring it. %v", path, pluginVersionStr, err) - continue - } + if err := opts.CheckProtocolVersion(protocolVerionStr); err != nil { + log.Printf("[NOTICE] binary %s requires protocol version %s that is incompatible "+ + "with this version of Packer. %s", path, protocolVerionStr, err) + continue + } - // no constraint means always pass, this will happen for implicit - // plugin requirements and when we list all plugins. - if !pr.VersionConstraints.Check(pv) { - log.Printf("[TRACE] version %q of file %q does not match constraint %q", pluginVersionStr, path, pr.VersionConstraints.String()) - continue - } + checksumOk := false + for _, checksummer := range opts.Checksummers { - if err := opts.CheckProtocolVersion(protocolVerionStr); err != nil { - log.Printf("[NOTICE] binary %s requires protocol version %s that is incompatible "+ - "with this version of Packer. %s", path, protocolVerionStr, err) + cs, err := checksummer.GetCacheChecksumOfFile(path) + if err != nil { + log.Printf("[TRACE] GetChecksumOfFile(%q) failed: %v", path, err) continue } - checksumOk := false - for _, checksummer := range opts.Checksummers { - - cs, err := checksummer.GetCacheChecksumOfFile(path) - if err != nil { - log.Printf("[TRACE] GetChecksumOfFile(%q) failed: %v", path, err) - continue - } - - if err := checksummer.ChecksumFile(cs, path); err != nil { - log.Printf("[TRACE] ChecksumFile(%q) failed: %v", path, err) - continue - } - checksumOk = true - break - } - if !checksumOk { - log.Printf("[TRACE] No checksum found for %q ignoring possibly unsafe binary", path) + if err := checksummer.ChecksumFile(cs, path); err != nil { + log.Printf("[TRACE] ChecksumFile(%q) failed: %v", path, err) continue } - - res = append(res, &Installation{ - BinaryPath: path, - Version: pluginVersionStr, - }) + checksumOk = true + break + } + if !checksumOk { + log.Printf("[TRACE] No checksum found for %q ignoring possibly unsafe binary", path) + continue } + + res = append(res, &Installation{ + BinaryPath: path, + Version: pluginVersionStr, + }) } + return res, nil } @@ -224,9 +223,8 @@ type InstallOptions struct { // Different means to get releases, sha256 and binary files. Getters []Getter - // Any downloaded binary and checksum file will be put in the last possible - // folder of this list. - InFolders []string + // The directory in which the plugins should be installed + PluginDirectory string // Forces installation of the plugin, even if already installed. Force bool @@ -481,7 +479,7 @@ func (pr *Requirement) InstallLatest(opts InstallOptions) (*Installation, error) outputFolder := filepath.Join( // Pick last folder as it's the one with the highest priority - opts.InFolders[len(opts.InFolders)-1], + opts.PluginDirectory, // add expected full path filepath.Join(pr.Identifier.Parts()...), ) @@ -549,36 +547,33 @@ func (pr *Requirement) InstallLatest(opts InstallOptions) (*Installation, error) expectedZipFilename := checksum.Filename expectedBinaryFilename := strings.TrimSuffix(expectedZipFilename, filepath.Ext(expectedZipFilename)) + opts.BinaryInstallationOptions.Ext - for _, outputFolder := range opts.InFolders { - potentialOutputFilename := filepath.Join( - outputFolder, - filepath.Join(pr.Identifier.Parts()...), - expectedBinaryFilename, - ) - for _, potentialChecksumer := range opts.Checksummers { - // First check if a local checksum file is already here in the expected - // download folder. Here we want to download a binary so we only check - // for an existing checksum file from the folder we want to download - // into. - cs, err := potentialChecksumer.GetCacheChecksumOfFile(potentialOutputFilename) - if err == nil && len(cs) > 0 { - localChecksum := &FileChecksum{ - Expected: cs, - Checksummer: potentialChecksumer, - } - - log.Printf("[TRACE] found a pre-exising %q checksum file", potentialChecksumer.Type) - // if outputFile is there and matches the checksum: do nothing more. - if err := localChecksum.ChecksumFile(localChecksum.Expected, potentialOutputFilename); err == nil && !opts.Force { - log.Printf("[INFO] %s v%s plugin is already correctly installed in %q", pr.Identifier, version, potentialOutputFilename) - return nil, nil // success - } + outputFileName := filepath.Join( + outputFolder, + expectedBinaryFilename, + ) + for _, potentialChecksumer := range opts.Checksummers { + // First check if a local checksum file is already here in the expected + // download folder. Here we want to download a binary so we only check + // for an existing checksum file from the folder we want to download + // into. + cs, err := potentialChecksumer.GetCacheChecksumOfFile(outputFileName) + if err == nil && len(cs) > 0 { + localChecksum := &FileChecksum{ + Expected: cs, + Checksummer: potentialChecksumer, + } + + log.Printf("[TRACE] found a pre-exising %q checksum file", potentialChecksumer.Type) + // if outputFile is there and matches the checksum: do nothing more. + if err := localChecksum.ChecksumFile(localChecksum.Expected, outputFileName); err == nil && !opts.Force { + log.Printf("[INFO] %s v%s plugin is already correctly installed in %q", pr.Identifier, version, outputFileName) + return nil, nil // success } } } // The last folder from the installation list is where we will install. - outputFileName := filepath.Join(outputFolder, expectedBinaryFilename) + outputFileName = filepath.Join(outputFolder, expectedBinaryFilename) // create directories if need be if err := os.MkdirAll(outputFolder, 0755); err != nil { diff --git a/packer/plugin-getter/plugins_test.go b/packer/plugin-getter/plugins_test.go index e13f8cb3825..1f58e349601 100644 --- a/packer/plugin-getter/plugins_test.go +++ b/packer/plugin-getter/plugins_test.go @@ -78,10 +78,7 @@ func TestPlugin_ListInstallations(t *testing.T) { // empty }, ListInstallationsOptions{ - []string{ - pluginFolderOne, - pluginFolderTwo, - }, + pluginFolderOne, BinaryInstallationOptions{ OS: "windows", ARCH: "amd64", Ext: ".exe", @@ -119,22 +116,6 @@ func TestPlugin_ListInstallations(t *testing.T) { Version: "v4.5.8", BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.8_x5.0_windows_amd64.exe"), }, - { - Version: "v4.5.6", - BinaryPath: filepath.Join(pluginFolderTwo, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.6_x5.0_windows_amd64.exe"), - }, - { - Version: "v4.5.9", - BinaryPath: filepath.Join(pluginFolderTwo, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.9_x5.0_windows_amd64.exe"), - }, - { - Version: "v4.5.6", - BinaryPath: filepath.Join(pluginFolderTwo, "github.com", "hashicorp copy", "google", "packer-plugin-google_v4.5.6_x5.0_windows_amd64.exe"), - }, - { - Version: "v4.5.9", - BinaryPath: filepath.Join(pluginFolderTwo, "github.com", "hashicorp copy", "google", "packer-plugin-google_v4.5.9_x5.0_windows_amd64.exe"), - }, }, }, @@ -144,10 +125,7 @@ func TestPlugin_ListInstallations(t *testing.T) { Identifier: "github.com/hashicorp/amazon", }, ListInstallationsOptions{ - []string{ - pluginFolderOne, - pluginFolderTwo, - }, + pluginFolderOne, BinaryInstallationOptions{ APIVersionMajor: "5", APIVersionMinor: "0", OS: "darwin", ARCH: "amd64", @@ -181,10 +159,7 @@ func TestPlugin_ListInstallations(t *testing.T) { Identifier: "github.com/hashicorp/amazon", }, ListInstallationsOptions{ - []string{ - pluginFolderOne, - pluginFolderTwo, - }, + pluginFolderOne, BinaryInstallationOptions{ APIVersionMajor: "5", APIVersionMinor: "1", OS: "darwin", ARCH: "amd64", @@ -214,10 +189,6 @@ func TestPlugin_ListInstallations(t *testing.T) { Version: "v1.2.5", BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.5_x5.0_darwin_amd64"), }, - { - Version: "v1.2.6", - BinaryPath: filepath.Join(pluginFolderTwo, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.6_x5.1_darwin_amd64"), - }, }, }, { @@ -226,10 +197,7 @@ func TestPlugin_ListInstallations(t *testing.T) { Identifier: "github.com/hashicorp/amazon", }, ListInstallationsOptions{ - []string{ - pluginFolderOne, - pluginFolderTwo, - }, + pluginFolderOne, BinaryInstallationOptions{ APIVersionMajor: "5", APIVersionMinor: "0", OS: "windows", ARCH: "amd64", @@ -258,61 +226,13 @@ func TestPlugin_ListInstallations(t *testing.T) { }, }, }, - { - "windows_google_multifolder", - fields{ - Identifier: "github.com/hashicorp/google", - }, - ListInstallationsOptions{ - []string{ - pluginFolderOne, - pluginFolderTwo, - }, - BinaryInstallationOptions{ - APIVersionMajor: "5", APIVersionMinor: "0", - OS: "windows", ARCH: "amd64", - Ext: ".exe", - Checksummers: []Checksummer{ - { - Type: "sha256", - Hash: sha256.New(), - }, - }, - }, - }, - false, - []*Installation{ - { - Version: "v4.5.6", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.6_x5.0_windows_amd64.exe"), - }, - { - Version: "v4.5.7", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.7_x5.0_windows_amd64.exe"), - }, - { - Version: "v4.5.8", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.8_x5.0_windows_amd64.exe"), - }, - { - Version: "v4.5.6", - BinaryPath: filepath.Join(pluginFolderTwo, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.6_x5.0_windows_amd64.exe"), - }, - { - Version: "v4.5.9", - BinaryPath: filepath.Join(pluginFolderTwo, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.9_x5.0_windows_amd64.exe"), - }, - }, - }, { "test nil identifier - multiple plugins with same version", fields{ Identifier: "", }, ListInstallationsOptions{ - []string{ - pluginFolderThree, - }, + pluginFolderThree, BinaryInstallationOptions{ APIVersionMajor: "5", APIVersionMinor: "0", OS: "linux", ARCH: "amd64", @@ -398,11 +318,7 @@ func TestRequirement_InstallLatest(t *testing.T) { }, }, }, - []string{ - pluginFolderWrongChecksums, - pluginFolderOne, - pluginFolderTwo, - }, + pluginFolderOne, false, BinaryInstallationOptions{ APIVersionMajor: "5", APIVersionMinor: "0", @@ -435,11 +351,7 @@ func TestRequirement_InstallLatest(t *testing.T) { }, }, }, - []string{ - pluginFolderWrongChecksums, - pluginFolderOne, - pluginFolderTwo, - }, + pluginFolderOne, false, BinaryInstallationOptions{ APIVersionMajor: "5", APIVersionMinor: "1", @@ -481,11 +393,7 @@ func TestRequirement_InstallLatest(t *testing.T) { }, }, }, - []string{ - pluginFolderWrongChecksums, - pluginFolderOne, - pluginFolderTwo, - }, + pluginFolderOne, false, BinaryInstallationOptions{ APIVersionMajor: "5", APIVersionMinor: "0", @@ -529,11 +437,7 @@ func TestRequirement_InstallLatest(t *testing.T) { }, }, }, - []string{ - pluginFolderWrongChecksums, - pluginFolderOne, - pluginFolderTwo, - }, + pluginFolderTwo, false, BinaryInstallationOptions{ APIVersionMajor: "6", APIVersionMinor: "1", @@ -580,11 +484,7 @@ func TestRequirement_InstallLatest(t *testing.T) { }, }, }, - []string{ - pluginFolderWrongChecksums, - pluginFolderOne, - pluginFolderTwo, - }, + pluginFolderTwo, false, BinaryInstallationOptions{ APIVersionMajor: "6", APIVersionMinor: "1", @@ -631,11 +531,7 @@ func TestRequirement_InstallLatest(t *testing.T) { }, }, }, - []string{ - pluginFolderWrongChecksums, - pluginFolderOne, - pluginFolderTwo, - }, + pluginFolderTwo, false, BinaryInstallationOptions{ APIVersionMajor: "6", APIVersionMinor: "1", @@ -676,11 +572,7 @@ func TestRequirement_InstallLatest(t *testing.T) { }, }, }, - []string{ - pluginFolderWrongChecksums, - pluginFolderOne, - pluginFolderTwo, - }, + pluginFolderTwo, false, BinaryInstallationOptions{ APIVersionMajor: "6", APIVersionMinor: "1", @@ -720,9 +612,7 @@ func TestRequirement_InstallLatest(t *testing.T) { }, }, }, - []string{ - pluginFolderWrongChecksums, - }, + pluginFolderTwo, false, BinaryInstallationOptions{ APIVersionMajor: "6", APIVersionMinor: "1", diff --git a/packer/plugin.go b/packer/plugin.go index 246a7e1fec7..3fd404d7a03 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -67,6 +67,10 @@ func (c *PluginConfig) Discover() error { return nil } + if c.PluginDirectory == "" { + c.PluginDirectory, _ = PluginFolder() + } + if err := c.discoverInstalledComponents(c.PluginDirectory); err != nil { return err } @@ -120,7 +124,6 @@ func (c *PluginConfig) discoverSingle(glob string) (map[string]string, error) { // After the split the plugin name is "baz". pluginName = strings.SplitN(pluginName, "_", 2)[0] - log.Printf("[INFO] Discovered potential plugin: %s = %s", pluginName, match) pluginPath, err := filepath.Abs(match) if err != nil { pluginPath = match @@ -284,12 +287,12 @@ func (c *PluginConfig) discoverInstalledComponents(path string) error { for _, checksummer := range binInstallOpts.Checksummers { cs, err := checksummer.GetCacheChecksumOfFile(pluginPath) if err != nil { - log.Printf("[TRACE] GetChecksumOfFile(%q) failed: %v", pluginPath, err) + log.Printf("[TRACE] GetChecksumOfFile(%q) failed: %v\n", pluginPath, err) continue } if err := checksummer.ChecksumFile(cs, pluginPath); err != nil { - log.Printf("[TRACE] ChecksumFile(%q) failed: %v", pluginPath, err) + log.Printf("[TRACE] ChecksumFile(%q) failed: %v\n", pluginPath, err) continue } checksumOk = true diff --git a/packer/plugin_discover_test.go b/packer/plugin_discover_test.go index 6dbe8589b2e..472f9d6e06a 100644 --- a/packer/plugin_discover_test.go +++ b/packer/plugin_discover_test.go @@ -57,25 +57,25 @@ func TestMultiPlugin_describe(t *testing.T) { expectedBuilderName := mockPluginName + "-" + mockBuilderName if !c.Builders.Has(expectedBuilderName) { - t.Fatalf("expected to find builder %q", expectedBuilderName) + t.Errorf("expected to find builder %q", expectedBuilderName) } } for mockProvisionerName := range plugin.Provisioners { expectedProvisionerName := mockPluginName + "-" + mockProvisionerName if !c.Provisioners.Has(expectedProvisionerName) { - t.Fatalf("expected to find builder %q", expectedProvisionerName) + t.Errorf("expected to find builder %q", expectedProvisionerName) } } for mockPostProcessorName := range plugin.PostProcessors { expectedPostProcessorName := mockPluginName + "-" + mockPostProcessorName if !c.PostProcessors.Has(expectedPostProcessorName) { - t.Fatalf("expected to find post-processor %q", expectedPostProcessorName) + t.Errorf("expected to find post-processor %q", expectedPostProcessorName) } } for mockDatasourceName := range plugin.Datasources { expectedDatasourceName := mockPluginName + "-" + mockDatasourceName if !c.DataSources.Has(expectedDatasourceName) { - t.Fatalf("expected to find datasource %q", expectedDatasourceName) + t.Errorf("expected to find datasource %q", expectedDatasourceName) } } } @@ -207,40 +207,20 @@ func TestMultiPlugin_defaultName(t *testing.T) { } } -// no T.Parallel using os.Chdir -func TestMultiPlugin_CWD(t *testing.T) { - createMockPlugins(t, defaultNameMock) - pluginDir := os.Getenv("PACKER_PLUGIN_PATH") - defer os.RemoveAll(pluginDir) - // Unset PACKER_PLUGIN_PATH to test CWD loading - os.Unsetenv("PACKER_PLUGIN_PATH") - if err := os.Chdir(pluginDir); err != nil { - t.Fatalf("failed to change directory to test loading from CWD: %s", err) - } - c := PluginConfig{} - err := c.Discover() - if err != nil { - t.Fatalf("error discovering plugins; %s ; mocks are %#v", err.Error(), defaultNameMock) - } - expectedBuilderNames := []string{"foo-bar", "foo-baz", "foo"} - for _, mockBuilderName := range expectedBuilderNames { - if !c.Builders.Has(mockBuilderName) { - t.Fatalf("expected to find builder %q; builders is %#v", mockBuilderName, c.Builders) - } - } -} - func TestMultiPlugin_IgnoreChecksumFile(t *testing.T) { createMockPlugins(t, defaultNameMock) pluginDir := os.Getenv("PACKER_PLUGIN_PATH") defer os.RemoveAll(pluginDir) - csFile, err := generateMockChecksumFile(filepath.Join(pluginDir, "packer-plugin-foo")) + fooPluginName := fmt.Sprintf("packer-plugin-foo_v1.0.0_x5.0_%s_%s", runtime.GOOS, runtime.GOARCH) + fooPluginPath := filepath.Join(pluginDir, "github.com", "hashicorp", "foo", fooPluginName) + csFile, err := generateMockChecksumFile(fooPluginPath) if err != nil { t.Fatal(err.Error()) } + // Copy plugin contents into checksum file to validate that it is not only skipped but that it never gets loaded - if err := os.Rename(filepath.Join(pluginDir, "packer-plugin-foo"), csFile); err != nil { + if err := os.Rename(fooPluginPath, csFile); err != nil { t.Fatalf("failed to rename plugin bin file to checkfum file needed for test: %s", err) } @@ -395,7 +375,13 @@ func createMockPlugins(t *testing.T, plugins map[string]pluginsdk.Set) { shPath := MustHaveCommand(t, "bash") for name := range plugins { - plugin := path.Join(pluginDir, "packer-plugin-"+name) + pluginName := fmt.Sprintf("packer-plugin-%s_v1.0.0_x5.0_%s_%s", name, runtime.GOOS, runtime.GOARCH) + pluginSubDir := fmt.Sprintf("github.com/hashicorp/%s", name) + err := os.MkdirAll(path.Join(pluginDir, pluginSubDir), 0755) + if err != nil { + t.Fatalf("failed to create plugin hierarchy: %s", err) + } + plugin := path.Join(pluginDir, pluginSubDir, pluginName) t.Logf("creating fake plugin %s", plugin) fileContent := "" fileContent = fmt.Sprintf("#!%s\n", shPath) @@ -405,6 +391,10 @@ func createMockPlugins(t *testing.T, plugins map[string]pluginsdk.Set) { if err := os.WriteFile(plugin, []byte(fileContent), os.ModePerm); err != nil { t.Fatalf("failed to create fake plugin binary: %v", err) } + + if _, err := generateMockChecksumFile(plugin); err != nil { + t.Fatalf("failed to create fake plugin binary checksum file: %v", err) + } } } t.Setenv("PACKER_PLUGIN_PATH", pluginDir) From 6c08b9ac1fc58b13fe387e940c4e6963f4a4c4e7 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Tue, 16 Jan 2024 09:44:38 -0500 Subject: [PATCH 04/32] packer: remove path argument to discover installed Since the plugin directory should now be unique instead of a list, we can use it directly from the receiver's structure instead of as a parameter to the function. --- packer/plugin.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packer/plugin.go b/packer/plugin.go index 3fd404d7a03..4d5a673976d 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -71,7 +71,7 @@ func (c *PluginConfig) Discover() error { c.PluginDirectory, _ = PluginFolder() } - if err := c.discoverInstalledComponents(c.PluginDirectory); err != nil { + if err := c.discoverInstalledComponents(); err != nil { return err } @@ -260,7 +260,7 @@ func (c *PluginConfig) Client(path string, args ...string) *PluginClient { // discoverInstalledComponents scans the provided path for plugins installed by running packer plugins install or packer init. // Valid plugins contain a matching system binary and valid checksum file. -func (c *PluginConfig) discoverInstalledComponents(path string) error { +func (c *PluginConfig) discoverInstalledComponents() error { //Check for installed plugins using the `packer plugins install` command binInstallOpts := plugingetter.BinaryInstallationOptions{ OS: runtime.GOOS, @@ -276,7 +276,7 @@ func (c *PluginConfig) discoverInstalledComponents(path string) error { binInstallOpts.Ext = ".exe" } - pluginPath := filepath.Join(path, "*", "*", "*", fmt.Sprintf("packer-plugin-*%s", binInstallOpts.FilenameSuffix())) + pluginPath := filepath.Join(c.PluginDirectory, "*", "*", "*", fmt.Sprintf("packer-plugin-*%s", binInstallOpts.FilenameSuffix())) pluginPaths, err := c.discoverSingle(pluginPath) if err != nil { return err @@ -300,7 +300,7 @@ func (c *PluginConfig) discoverInstalledComponents(path string) error { } if !checksumOk { - log.Printf("[WARN] No checksum found for %q ignoring possibly unsafe binary", path) + log.Printf("[WARN] No checksum found for %q ignoring possibly unsafe binary", pluginPath) continue } From b14cfe004d5479972fc5c08b7815abe89fa52a23 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Wed, 17 Jan 2024 13:39:36 -0500 Subject: [PATCH 05/32] plugin-getter: fix docs for Installation The comments weren't capitalised correctly, and the directory example included the `packer-plugin-' prefix, which should not be the case in reality. --- packer/plugin-getter/plugins.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packer/plugin-getter/plugins.go b/packer/plugin-getter/plugins.go index 1aa7834d138..5de97416452 100644 --- a/packer/plugin-getter/plugins.go +++ b/packer/plugin-getter/plugins.go @@ -207,13 +207,12 @@ func (l InstallList) String() string { // Installation describes a plugin installation type Installation struct { - // path to where binary is installed, if installed. - // Ex: /usr/azr/.packer.d/plugins/github.com/hashicorp/packer-plugin-amazon/packer-plugin-amazon_v1.2.3_darwin_amd64 + // Path to where binary is installed. + // Ex: /usr/azr/.packer.d/plugins/github.com/hashicorp/amazon/packer-plugin-amazon_v1.2.3_darwin_amd64 BinaryPath string - // Version of this plugin, if installed and versionned. Ex: + // Version of this plugin. Ex: // * v1.2.3 for packer-plugin-amazon_v1.2.3_darwin_x5 - // * empty for packer-plugin-amazon Version string } From 74bb620a511adfc7e1dd362a131c96cfb3d03c68 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Wed, 17 Jan 2024 15:49:51 -0500 Subject: [PATCH 06/32] plugin-getter: implement Sort interface for List The plugin installation list should be sorted according to a name first, then version second basis. Right now, we only rely on the glob to add plugin installs to this list, making the order unreliable since the lexicographical order is not the order in which we want to see the same plugin ordered (e.g. v1.0.9 > 1.0.10). To fix this, we implement a logic for sorting a list of installations that does what's described above with more accuracy. --- packer/plugin-getter/plugins.go | 48 +++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/packer/plugin-getter/plugins.go b/packer/plugin-getter/plugins.go index 5de97416452..85ba3ba3211 100644 --- a/packer/plugin-getter/plugins.go +++ b/packer/plugin-getter/plugins.go @@ -205,6 +205,54 @@ func (l InstallList) String() string { return v.String() } +// Len is the number of elements in the collection. +func (l InstallList) Len() int { + return len(l) +} + +var rawPluginName = regexp.MustCompile("packer-plugin-[^_]+") + +// Less reports whether the element with index i +// must sort before the element with index j. +// +// If both Less(i, j) and Less(j, i) are false, +// then the elements at index i and j are considered equal. +// Sort may place equal elements in any order in the final result, +// while Stable preserves the original input order of equal elements. +// +// Less must describe a transitive ordering: +// - if both Less(i, j) and Less(j, k) are true, then Less(i, k) must be true as well. +// - if both Less(i, j) and Less(j, k) are false, then Less(i, k) must be false as well. +// +// Note that floating-point comparison (the < operator on float32 or float64 values) +// is not a transitive ordering when not-a-number (NaN) values are involved. +// See Float64Slice.Less for a correct implementation for floating-point values. +func (l InstallList) Less(i, j int) bool { + lowPluginPath := l[i] + hiPluginPath := l[j] + + lowRawPluginName := rawPluginName.FindString(path.Base(lowPluginPath.BinaryPath)) + hiRawPluginName := rawPluginName.FindString(path.Base(hiPluginPath.BinaryPath)) + + // We group by path, then by descending order for the versions + // + // i.e. if the path are not the same, we can return the plain + // lexicographic order, otherwise, we'll do a semver-conscious + // version comparison for sorting. + if lowRawPluginName != hiRawPluginName { + return lowRawPluginName < hiRawPluginName + } + + return semver.Compare(lowPluginPath.Version, hiPluginPath.Version) > 0 +} + +// Swap swaps the elements with indexes i and j. +func (l InstallList) Swap(i, j int) { + tmp := l[i] + l[i] = l[j] + l[j] = tmp +} + // Installation describes a plugin installation type Installation struct { // Path to where binary is installed. From b8857604e3561c09003c4bede5d1b19d5dbf0141 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Wed, 17 Jan 2024 15:52:59 -0500 Subject: [PATCH 07/32] plugin: consolidate loading logic Right now we had two paths for discovering installed plugins, i.e. through plugin-getter's `ListInstallations' function, or through the `Discover' call, which relied on a glob to list the installations. This was required since we allowed plugins to be installed in multiple locations, and with different constraints. Now that we force a certain convention, we can consolidate the logic into ListInstallations, and rely on that logic in `Discover' to load a plugin into the PluginConfig for the current Packer run. --- hcl2template/plugin.go | 2 +- packer/plugin-getter/plugins.go | 7 + packer/plugin-getter/plugins_test.go | 226 --------------------------- packer/plugin.go | 141 +++++------------ packer/plugin_discover_test.go | 2 + 5 files changed, 48 insertions(+), 330 deletions(-) diff --git a/hcl2template/plugin.go b/hcl2template/plugin.go index 15358573d6e..279f534e03b 100644 --- a/hcl2template/plugin.go +++ b/hcl2template/plugin.go @@ -94,7 +94,7 @@ func (cfg *PackerConfig) DetectPluginBinaries() hcl.Diagnostics { continue } log.Printf("[TRACE] Found the following %q installations: %v", pluginRequirement.Identifier, sortedInstalls) - install := sortedInstalls[len(sortedInstalls)-1] + install := sortedInstalls[0] err = cfg.parser.PluginConfig.DiscoverMultiPlugin(pluginRequirement.Accessor, install.BinaryPath) if err != nil { diags = append(diags, &hcl.Diagnostic{ diff --git a/packer/plugin-getter/plugins.go b/packer/plugin-getter/plugins.go index 85ba3ba3211..d9379351c80 100644 --- a/packer/plugin-getter/plugins.go +++ b/packer/plugin-getter/plugins.go @@ -11,7 +11,10 @@ import ( "io" "log" "os" + "os/exec" + "path" "path/filepath" + "regexp" "sort" "strconv" "strings" @@ -19,8 +22,10 @@ import ( "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-version" + pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin" "github.com/hashicorp/packer-plugin-sdk/tmp" "github.com/hashicorp/packer/hcl2template/addrs" + "golang.org/x/mod/semver" ) type Requirements []*Requirement @@ -182,6 +187,8 @@ func (pr Requirement) ListInstallations(opts ListInstallationsOptions) (InstallL }) } + sort.Sort(res) + return res, nil } diff --git a/packer/plugin-getter/plugins_test.go b/packer/plugin-getter/plugins_test.go index 1f58e349601..4cc78acc3ba 100644 --- a/packer/plugin-getter/plugins_test.go +++ b/packer/plugin-getter/plugins_test.go @@ -18,7 +18,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/hashicorp/go-version" - "github.com/hashicorp/hcl/v2" "github.com/hashicorp/packer/hcl2template/addrs" ) @@ -58,231 +57,6 @@ func TestChecksumFileEntry_init(t *testing.T) { } } -func TestPlugin_ListInstallations(t *testing.T) { - - type fields struct { - Identifier string - VersionConstraints version.Constraints - } - tests := []struct { - name string - fields fields - opts ListInstallationsOptions - wantErr bool - want InstallList - }{ - - { - "windows_all_plugins", - fields{ - // empty - }, - ListInstallationsOptions{ - pluginFolderOne, - BinaryInstallationOptions{ - OS: "windows", ARCH: "amd64", - Ext: ".exe", - Checksummers: []Checksummer{ - { - Type: "sha256", - Hash: sha256.New(), - }, - }, - }, - }, - false, - []*Installation{ - { - Version: "v1.2.3", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.3_x5.0_windows_amd64.exe"), - }, - { - Version: "v1.2.4", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.4_x5.0_windows_amd64.exe"), - }, - { - Version: "v1.2.5", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.5_x5.0_windows_amd64.exe"), - }, - { - Version: "v4.5.6", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.6_x5.0_windows_amd64.exe"), - }, - { - Version: "v4.5.7", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.7_x5.0_windows_amd64.exe"), - }, - { - Version: "v4.5.8", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.8_x5.0_windows_amd64.exe"), - }, - }, - }, - - { - "darwin_amazon_prot_5.0", - fields{ - Identifier: "github.com/hashicorp/amazon", - }, - ListInstallationsOptions{ - pluginFolderOne, - BinaryInstallationOptions{ - APIVersionMajor: "5", APIVersionMinor: "0", - OS: "darwin", ARCH: "amd64", - Checksummers: []Checksummer{ - { - Type: "sha256", - Hash: sha256.New(), - }, - }, - }, - }, - false, - []*Installation{ - { - Version: "v1.2.3", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.3_x5.0_darwin_amd64"), - }, - { - Version: "v1.2.4", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.4_x5.0_darwin_amd64"), - }, - { - Version: "v1.2.5", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.5_x5.0_darwin_amd64"), - }, - }, - }, - { - "darwin_amazon_prot_5.1", - fields{ - Identifier: "github.com/hashicorp/amazon", - }, - ListInstallationsOptions{ - pluginFolderOne, - BinaryInstallationOptions{ - APIVersionMajor: "5", APIVersionMinor: "1", - OS: "darwin", ARCH: "amd64", - Checksummers: []Checksummer{ - { - Type: "sha256", - Hash: sha256.New(), - }, - }, - }, - }, - false, - []*Installation{ - { - Version: "v1.2.3", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.3_x5.0_darwin_amd64"), - }, - { - Version: "v1.2.3", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.3_x5.1_darwin_amd64"), - }, - { - Version: "v1.2.4", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.4_x5.0_darwin_amd64"), - }, - { - Version: "v1.2.5", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.5_x5.0_darwin_amd64"), - }, - }, - }, - { - "windows_amazon", - fields{ - Identifier: "github.com/hashicorp/amazon", - }, - ListInstallationsOptions{ - pluginFolderOne, - BinaryInstallationOptions{ - APIVersionMajor: "5", APIVersionMinor: "0", - OS: "windows", ARCH: "amd64", - Ext: ".exe", - Checksummers: []Checksummer{ - { - Type: "sha256", - Hash: sha256.New(), - }, - }, - }, - }, - false, - []*Installation{ - { - Version: "v1.2.3", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.3_x5.0_windows_amd64.exe"), - }, - { - Version: "v1.2.4", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.4_x5.0_windows_amd64.exe"), - }, - { - Version: "v1.2.5", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.5_x5.0_windows_amd64.exe"), - }, - }, - }, - { - "test nil identifier - multiple plugins with same version", - fields{ - Identifier: "", - }, - ListInstallationsOptions{ - pluginFolderThree, - BinaryInstallationOptions{ - APIVersionMajor: "5", APIVersionMinor: "0", - OS: "linux", ARCH: "amd64", - Checksummers: []Checksummer{ - { - Type: "sha256", - Hash: sha256.New(), - }, - }, - }, - }, - false, - []*Installation{ - { - Version: "v1.2.5", - BinaryPath: filepath.Join(pluginFolderThree, "github.com", "hashicorp", "alazon", "packer-plugin-alazon_v1.2.5_x5.0_linux_amd64"), - }, - { - Version: "v1.2.5", - BinaryPath: filepath.Join(pluginFolderThree, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.5_x5.0_linux_amd64"), - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var identifier *addrs.Plugin - if tt.fields.Identifier != "" { - var diags hcl.Diagnostics - identifier, diags = addrs.ParsePluginSourceString(tt.fields.Identifier) - if diags.HasErrors() { - t.Fatalf("%v", diags) - } - } - p := Requirement{ - Identifier: identifier, - VersionConstraints: tt.fields.VersionConstraints, - } - got, err := p.ListInstallations(tt.opts) - if (err != nil) != tt.wantErr { - t.Errorf("Plugin.ListInstallations() error = %v, wantErr %v", err, tt.wantErr) - return - } - if diff := cmp.Diff(tt.want, got); diff != "" { - t.Errorf("Plugin.ListInstallations() unexpected output: %s", diff) - } - }) - } -} - func TestRequirement_InstallLatest(t *testing.T) { type fields struct { Identifier string diff --git a/packer/plugin.go b/packer/plugin.go index 4d5a673976d..5fcef89549c 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -6,13 +6,13 @@ package packer import ( "crypto/sha256" "encoding/json" - "fmt" "log" "os" "os/exec" + "path" "path/filepath" + "regexp" "runtime" - "sort" "strings" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" @@ -40,7 +40,9 @@ type PluginConfig struct { // without being confused with spaces in the path to the command itself. const PACKERSPACE = "-PACKERSPACE-" -// Discover discovers plugins. +var extractPluginBasename = regexp.MustCompile("^packer-plugin-([^_]+)") + +// Discover discovers the latest installed version of each installed plugin. // // Search the directory of the executable, then the plugins directory, and // finally the CWD, in that order. Any conflicts will overwrite previously @@ -71,67 +73,54 @@ func (c *PluginConfig) Discover() error { c.PluginDirectory, _ = PluginFolder() } - if err := c.discoverInstalledComponents(); err != nil { + installations, err := plugingetter.Requirement{}.ListInstallations(plugingetter.ListInstallationsOptions{ + PluginDirectory: c.PluginDirectory, + BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ + OS: runtime.GOOS, + ARCH: runtime.GOARCH, + Checksummers: []plugingetter.Checksummer{ + {Type: "sha256", Hash: sha256.New()}, + }, + }, + }) + if err != nil { return err } - return nil -} - -func (c *PluginConfig) discoverSingle(glob string) (map[string]string, error) { - matches, err := filepath.Glob(glob) - if err != nil { - return nil, err - } - var prefix string - res := make(map[string]string) - // Sort the matches so we add the newer version of a plugin last - sort.Strings(matches) - prefix = filepath.Base(glob) - prefix = prefix[:strings.Index(prefix, "*")] - for _, match := range matches { - file := filepath.Base(match) - // skip folders like packer-plugin-sdk - if stat, err := os.Stat(file); err == nil && stat.IsDir() { + // Map of plugin basename to executable + // + // We'll use that later to register the components for each plugin + pluginMap := map[string]string{} + for _, install := range installations { + pluginBasename := path.Base(install.BinaryPath) + matches := extractPluginBasename.FindStringSubmatch(pluginBasename) + if len(matches) != 2 { + log.Printf("[INFO] - plugin %q could not have its name matched, ignoring", pluginBasename) continue } - // On Windows, ignore any plugins that don't end in .exe. - // We could do a full PATHEXT parse, but this is probably good enough. - if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != ".exe" { - log.Printf( - "[TRACE] Ignoring plugin match %s, no exe extension", - match) - continue - } + pluginName := matches[1] - if strings.Contains(strings.ToUpper(file), defaultChecksummer.FileExt()) { - log.Printf( - "[TRACE] Ignoring plugin match %s, which looks to be a checksum file", - match) + // If the plugin is already registered in the plugin map, we + // can ignore the current executable, as they're sorted by + // version in descending order, so if it's already in the map, + // a more recent version was already discovered. + _, ok := pluginMap[pluginName] + if ok { continue - - } - - // If the filename has a ".", trim up to there - if idx := strings.Index(file, ".exe"); idx >= 0 { - file = file[:idx] } - // Look for foo-bar-baz. The plugin name is "baz" - pluginName := file[len(prefix):] - // multi-component plugins installed via the plugins subcommand will have a name that looks like baz_vx.y.z_x5.0_darwin_arm64. - // After the split the plugin name is "baz". - pluginName = strings.SplitN(pluginName, "_", 2)[0] + pluginMap[pluginName] = install.BinaryPath + } - pluginPath, err := filepath.Abs(match) + for name, path := range pluginMap { + err := c.DiscoverMultiPlugin(name, path) if err != nil { - pluginPath = match + return err } - res[pluginName] = pluginPath } - return res, nil + return nil } // DiscoverMultiPlugin takes the description from a multi-component plugin @@ -257,57 +246,3 @@ func (c *PluginConfig) Client(path string, args ...string) *PluginClient { config.MaxPort = c.PluginMaxPort return NewClient(&config) } - -// discoverInstalledComponents scans the provided path for plugins installed by running packer plugins install or packer init. -// Valid plugins contain a matching system binary and valid checksum file. -func (c *PluginConfig) discoverInstalledComponents() error { - //Check for installed plugins using the `packer plugins install` command - binInstallOpts := plugingetter.BinaryInstallationOptions{ - OS: runtime.GOOS, - ARCH: runtime.GOARCH, - APIVersionMajor: pluginsdk.APIVersionMajor, - APIVersionMinor: pluginsdk.APIVersionMinor, - Checksummers: []plugingetter.Checksummer{ - defaultChecksummer, - }, - } - - if runtime.GOOS == "windows" { - binInstallOpts.Ext = ".exe" - } - - pluginPath := filepath.Join(c.PluginDirectory, "*", "*", "*", fmt.Sprintf("packer-plugin-*%s", binInstallOpts.FilenameSuffix())) - pluginPaths, err := c.discoverSingle(pluginPath) - if err != nil { - return err - } - - for pluginName, pluginPath := range pluginPaths { - var checksumOk bool - for _, checksummer := range binInstallOpts.Checksummers { - cs, err := checksummer.GetCacheChecksumOfFile(pluginPath) - if err != nil { - log.Printf("[TRACE] GetChecksumOfFile(%q) failed: %v\n", pluginPath, err) - continue - } - - if err := checksummer.ChecksumFile(cs, pluginPath); err != nil { - log.Printf("[TRACE] ChecksumFile(%q) failed: %v\n", pluginPath, err) - continue - } - checksumOk = true - break - } - - if !checksumOk { - log.Printf("[WARN] No checksum found for %q ignoring possibly unsafe binary", pluginPath) - continue - } - - if err := c.DiscoverMultiPlugin(pluginName, pluginPath); err != nil { - return err - } - } - - return nil -} diff --git a/packer/plugin_discover_test.go b/packer/plugin_discover_test.go index 472f9d6e06a..3eca150d87c 100644 --- a/packer/plugin_discover_test.go +++ b/packer/plugin_discover_test.go @@ -17,6 +17,7 @@ import ( packersdk "github.com/hashicorp/packer-plugin-sdk/packer" pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin" "github.com/hashicorp/packer-plugin-sdk/tmp" + "github.com/hashicorp/packer-plugin-sdk/version" plugingetter "github.com/hashicorp/packer/packer/plugin-getter" ) @@ -305,6 +306,7 @@ func TestHelperPlugins(t *testing.T) { for _, mock := range allMocks { plugin, found := mock[pluginName] if found { + plugin.SetVersion(version.InitializePluginVersion("1.0.0", "")) err := plugin.RunCommand(args...) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) From 7eaf94e8d82ceaa7bc2bdfb1b385b9da77402666 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Wed, 17 Jan 2024 15:55:44 -0500 Subject: [PATCH 08/32] plugin-getter: reject plugin version mismatches When a plugin is loaded from Packer, we now check that the version it reports matches what the name implies. In case there's a mismatch, we log, and reject the binary. --- packer/plugin-getter/plugins.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packer/plugin-getter/plugins.go b/packer/plugin-getter/plugins.go index d9379351c80..6fec3dddba1 100644 --- a/packer/plugin-getter/plugins.go +++ b/packer/plugin-getter/plugins.go @@ -137,6 +137,18 @@ func (pr Requirement) ListInstallations(opts ListInstallationsOptions) (InstallL } } + descOut, err := exec.Command(path, "describe").Output() + if err != nil { + log.Printf("couldn't call describe on %q, ignoring", path) + continue + } + + var describeInfo pluginsdk.SetDescription + err = json.Unmarshal(descOut, &describeInfo) + if err != nil { + log.Printf("%q: describe output deserialization error %q, ignoring", path, err) + } + // versionsStr now looks like v1.2.3_x5.1 or amazon_v1.2.3_x5.1 parts := strings.SplitN(versionsStr, "_", 2) pluginVersionStr, protocolVerionStr := parts[0], parts[1] @@ -147,6 +159,11 @@ func (pr Requirement) ListInstallations(opts ListInstallationsOptions) (InstallL continue } + if strings.Replace(pluginVersionStr, "v", "", -1) != describeInfo.Version { + log.Printf("plugin %q reported version %s while its name implies version %s, ignoring", path, describeInfo.Version, pluginVersionStr) + continue + } + // no constraint means always pass, this will happen for implicit // plugin requirements and when we list all plugins. if !pr.VersionConstraints.Check(pv) { From fc2d255a609a30d00046b14383e8901fafbac8a0 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Wed, 17 Jan 2024 17:01:18 -0500 Subject: [PATCH 09/32] command: move plugins remove tests outside of repo Since the plugins remove subcommand now only loads valid plugins, it cannot run on mock data anymore, and the logic for creating a valid plugin hierarchy is not present in this repository, but does in another, so we move those tests from here to that repository in order to continue testing them. --- command/plugins_remove_test.go | 215 --------------------------------- 1 file changed, 215 deletions(-) delete mode 100644 command/plugins_remove_test.go diff --git a/command/plugins_remove_test.go b/command/plugins_remove_test.go deleted file mode 100644 index 852ab49aad5..00000000000 --- a/command/plugins_remove_test.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -//go:build amd64 && (darwin || windows || linux) - -package command - -import ( - "log" - "os" - "runtime" - "sort" - "testing" - - "github.com/google/go-cmp/cmp" - "golang.org/x/mod/sumdb/dirhash" -) - -type testCasePluginsRemove struct { - name string - Meta Meta - inPluginFolder map[string]string - expectedPackerConfigDirHashBeforeRemove string - packerConfigDir string - pluginSourceArgs []string - want int - dirFiles []string - expectedPackerConfigDirHashAfterRemove string -} - -func TestPluginsRemoveCommand_Run(t *testing.T) { - - cfg := &configDirSingleton{map[string]string{}} - - tests := []testCasePluginsRemove{ - { - name: "version-not-installed-no-op", - Meta: TestMetaFile(t), - inPluginFolder: map[string]string{ - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64": "1", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64_SHA256SUM": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe": "1.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe_SHA256SUM": "07d8453027192ee0c4120242e6e84e2ca2328b8e0f506e2f818a1a5b82790a0b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64": "1.out", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64_SHA256SUM": "59031c50e0dfeedfde2b4e9445754804dce3f29e4efa737eead0ca9b4f5b85a5", - }, - expectedPackerConfigDirHashBeforeRemove: "h1:Q5qyAOdD43hL3CquQdVfaHpOYGf0UsZ/+wVA9Ry6cbA=", - packerConfigDir: cfg.dir("1_pkr_plugins_config"), - pluginSourceArgs: []string{"github.com/sylviamoss/comment", "v0.2.19"}, - want: 1, - dirFiles: nil, - expectedPackerConfigDirHashAfterRemove: "h1:Q5qyAOdD43hL3CquQdVfaHpOYGf0UsZ/+wVA9Ry6cbA=", - }, - { - name: "remove-specific-version", - Meta: TestMetaFile(t), - inPluginFolder: map[string]string{ - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64": "1", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64_SHA256SUM": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe": "1.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe_SHA256SUM": "07d8453027192ee0c4120242e6e84e2ca2328b8e0f506e2f818a1a5b82790a0b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64": "1.out", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64_SHA256SUM": "59031c50e0dfeedfde2b4e9445754804dce3f29e4efa737eead0ca9b4f5b85a5", - }, - expectedPackerConfigDirHashBeforeRemove: "h1:Q5qyAOdD43hL3CquQdVfaHpOYGf0UsZ/+wVA9Ry6cbA=", - packerConfigDir: cfg.dir("2_pkr_plugins_config"), - pluginSourceArgs: []string{"github.com/sylviamoss/comment", "v0.2.18"}, - want: 0, - dirFiles: map[string][]string{ - "darwin": { - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe_SHA256SUM", - }, - "linux": { - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe_SHA256SUM", - }, - "windows": { - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64_SHA256SUM", - }, - }[runtime.GOOS], - expectedPackerConfigDirHashAfterRemove: map[string]string{ - "darwin": "h1:IKDdst8f0nMAS9T9t85fhuvDCe8lYytiZ/vooW1OWeI=", - "linux": "h1:h3t4iDYGbNtouHAJ1vPsE7d8n+6W5K4VlXTxhrQx2DA=", - "windows": "h1:NpETtGlu1hVMJe5bflPmyAhnL4iDvoIwA//uWhxtkDU=", - }[runtime.GOOS], - }, - { - name: "remove-all-installed-versions", - Meta: TestMetaFile(t), - inPluginFolder: map[string]string{ - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64": "1", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64_SHA256SUM": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_darwin_amd64": "1", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_darwin_amd64_SHA256SUM": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe": "1.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe_SHA256SUM": "07d8453027192ee0c4120242e6e84e2ca2328b8e0f506e2f818a1a5b82790a0b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_windows_amd64.exe": "1.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_windows_amd64.exe_SHA256SUM": "07d8453027192ee0c4120242e6e84e2ca2328b8e0f506e2f818a1a5b82790a0b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64": "1.out", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64_SHA256SUM": "59031c50e0dfeedfde2b4e9445754804dce3f29e4efa737eead0ca9b4f5b85a5", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_linux_amd64": "1.out", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_linux_amd64_SHA256SUM": "59031c50e0dfeedfde2b4e9445754804dce3f29e4efa737eead0ca9b4f5b85a5", - }, - expectedPackerConfigDirHashBeforeRemove: "h1:IEvr6c46+Uk776Hnzy04PuXqnyHGKnnEvIJ713cv0iU=", - packerConfigDir: cfg.dir("2_pkr_plugins_config"), - pluginSourceArgs: []string{"github.com/sylviamoss/comment"}, - want: 0, - dirFiles: map[string][]string{ - "darwin": { - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_linux_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_linux_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_windows_amd64.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_windows_amd64.exe_SHA256SUM", - }, - "linux": { - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_darwin_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_darwin_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_windows_amd64.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_windows_amd64.exe_SHA256SUM", - }, - "windows": { - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_darwin_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_darwin_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_linux_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_linux_amd64_SHA256SUM", - }, - }[runtime.GOOS], - expectedPackerConfigDirHashAfterRemove: map[string]string{ - "darwin": "h1:A6wlL62bUD06NF/1ND5E0o4omXhusB2T8N5ZNe2JVbg=", - "linux": "h1:SuTX0k2sknjDrL3PYgR1JajbGcs1qWIV2XvZAYSMsHw=", - "windows": "h1:NI8aJVAKdIyCXHuGDj6kTu5++6yvCz4oAswIhkL3wFc=", - }[runtime.GOOS], - }, - { - name: "no-installed-binaries", - Meta: TestMetaFile(t), - inPluginFolder: nil, - expectedPackerConfigDirHashBeforeRemove: "h1:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", - packerConfigDir: cfg.dir("3_pkr_plugins_config"), - pluginSourceArgs: []string{"example.com/sylviamoss/comment", "v0.2.19"}, - want: 1, - dirFiles: nil, - expectedPackerConfigDirHashAfterRemove: "h1:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - log.Printf("starting %s", tt.name) - log.Printf("%#v", tt) - t.Cleanup(func() { - _ = os.RemoveAll(tt.packerConfigDir) - }) - t.Setenv("PACKER_CONFIG_DIR", tt.packerConfigDir) - createFiles(tt.packerConfigDir, tt.inPluginFolder) - - hash, err := dirhash.HashDir(tt.packerConfigDir, "", dirhash.DefaultHash) - if err != nil { - t.Fatalf("HashDir: %v", err) - } - if diff := cmp.Diff(tt.expectedPackerConfigDirHashBeforeRemove, hash); diff != "" { - t.Errorf("unexpected dir hash before plugins remove: +found -expected %s", diff) - } - - c := &PluginsRemoveCommand{ - Meta: tt.Meta, - } - - c.CoreConfig.Components.PluginConfig.PluginDirectory = tt.packerConfigDir - if got := c.Run(tt.pluginSourceArgs); got != tt.want { - t.Errorf("PluginsRemoveCommand.Run() = %v, want %v", got, tt.want) - } - - if tt.dirFiles != nil { - dirFiles, err := dirhash.DirFiles(tt.packerConfigDir, "") - if err != nil { - t.Fatalf("DirFiles: %v", err) - } - sort.Strings(tt.dirFiles) - sort.Strings(dirFiles) - if diff := cmp.Diff(tt.dirFiles, dirFiles); diff != "" { - t.Errorf("found files differ: %v", diff) - } - } - - hash, err = dirhash.HashDir(tt.packerConfigDir, "", dirhash.DefaultHash) - if err != nil { - t.Fatalf("HashDir: %v", err) - } - if diff := cmp.Diff(tt.expectedPackerConfigDirHashAfterRemove, hash); diff != "" { - t.Errorf("unexpected dir hash after plugins remove: %s", diff) - } - }) - } -} From 07e40e7bf237a4a312830edcd30ca66228429190 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Thu, 1 Feb 2024 15:01:37 -0500 Subject: [PATCH 10/32] packer: fix plugin version sorting and pickup When Packer orders the plugins by their version, we'd assumed it was ordered from highest to lowest. However, this was the case because our implementation of `Less' was actually returning `>=', which is not what it was supposed to do. This commit therefore fixes the implementation of `Less' to do what it is documented to do, and ensures the behaviour is correct through additional testing. Changing this requires some changes to the loading process as well because of the aforementioned assumption with regards to ordering. --- hcl2template/plugin.go | 2 +- packer/plugin-getter/plugins.go | 2 +- packer/plugin-getter/plugins_test.go | 120 +++++++++++++++++++++++++++ packer/plugin.go | 10 --- 4 files changed, 122 insertions(+), 12 deletions(-) diff --git a/hcl2template/plugin.go b/hcl2template/plugin.go index 279f534e03b..15358573d6e 100644 --- a/hcl2template/plugin.go +++ b/hcl2template/plugin.go @@ -94,7 +94,7 @@ func (cfg *PackerConfig) DetectPluginBinaries() hcl.Diagnostics { continue } log.Printf("[TRACE] Found the following %q installations: %v", pluginRequirement.Identifier, sortedInstalls) - install := sortedInstalls[0] + install := sortedInstalls[len(sortedInstalls)-1] err = cfg.parser.PluginConfig.DiscoverMultiPlugin(pluginRequirement.Accessor, install.BinaryPath) if err != nil { diags = append(diags, &hcl.Diagnostic{ diff --git a/packer/plugin-getter/plugins.go b/packer/plugin-getter/plugins.go index 6fec3dddba1..11511ee3860 100644 --- a/packer/plugin-getter/plugins.go +++ b/packer/plugin-getter/plugins.go @@ -267,7 +267,7 @@ func (l InstallList) Less(i, j int) bool { return lowRawPluginName < hiRawPluginName } - return semver.Compare(lowPluginPath.Version, hiPluginPath.Version) > 0 + return semver.Compare(lowPluginPath.Version, hiPluginPath.Version) < 0 } // Swap swaps the elements with indexes i and j. diff --git a/packer/plugin-getter/plugins_test.go b/packer/plugin-getter/plugins_test.go index 4cc78acc3ba..9fb24830414 100644 --- a/packer/plugin-getter/plugins_test.go +++ b/packer/plugin-getter/plugins_test.go @@ -507,3 +507,123 @@ func zipFile(content map[string]string) io.ReadCloser { } var _ Getter = &mockPluginGetter{} + +func Test_LessInstallList(t *testing.T) { + tests := []struct { + name string + installs InstallList + expectLess bool + }{ + { + "v1.2.1 < v1.2.2 => true", + InstallList{ + &Installation{ + BinaryPath: "host/org/plugin", + Version: "v1.2.1", + }, + &Installation{ + BinaryPath: "github.com", + Version: "v1.2.2", + }, + }, + true, + }, + { + // Impractical with the changes to the loading model + "v1.2.1 = v1.2.1 => false", + InstallList{ + &Installation{ + BinaryPath: "host/org/plugin", + Version: "v1.2.1", + }, + &Installation{ + BinaryPath: "github.com", + Version: "v1.2.1", + }, + }, + false, + }, + { + "v1.2.2 < v1.2.1 => false", + InstallList{ + &Installation{ + BinaryPath: "host/org/plugin", + Version: "v1.2.2", + }, + &Installation{ + BinaryPath: "github.com", + Version: "v1.2.1", + }, + }, + false, + }, + { + "v1.2.2-dev < v1.2.2 => true", + InstallList{ + &Installation{ + BinaryPath: "host/org/plugin", + Version: "v1.2.2-dev", + }, + &Installation{ + BinaryPath: "github.com", + Version: "v1.2.2", + }, + }, + true, + }, + { + "v1.2.2 < v1.2.2-dev => false", + InstallList{ + &Installation{ + BinaryPath: "host/org/plugin", + Version: "v1.2.2", + }, + &Installation{ + BinaryPath: "github.com", + Version: "v1.2.2-dev", + }, + }, + false, + }, + { + "v1.2.1 < v1.2.2-dev => true", + InstallList{ + &Installation{ + BinaryPath: "host/org/plugin", + Version: "v1.2.1", + }, + &Installation{ + BinaryPath: "github.com", + Version: "v1.2.2-dev", + }, + }, + true, + }, + { + "v1.2.3 < v1.2.2-dev => false", + InstallList{ + &Installation{ + BinaryPath: "host/org/plugin", + Version: "v1.2.3", + }, + &Installation{ + BinaryPath: "github.com", + Version: "v1.2.2-dev", + }, + }, + false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + isLess := tt.installs.Less(0, 1) + if isLess != tt.expectLess { + t.Errorf("Less mismatch for %s < %s, expected %t, got %t", + tt.installs[0].Version, + tt.installs[1].Version, + tt.expectLess, isLess) + } + }) + } +} diff --git a/packer/plugin.go b/packer/plugin.go index 5fcef89549c..c6d4c125946 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -100,16 +100,6 @@ func (c *PluginConfig) Discover() error { } pluginName := matches[1] - - // If the plugin is already registered in the plugin map, we - // can ignore the current executable, as they're sorted by - // version in descending order, so if it's already in the map, - // a more recent version was already discovered. - _, ok := pluginMap[pluginName] - if ok { - continue - } - pluginMap[pluginName] = install.BinaryPath } From db8cebcd4b4197e66e7216bdb340471c80339426 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Thu, 1 Feb 2024 15:13:18 -0500 Subject: [PATCH 11/32] packer: support loading pre-release plugins When a pre-release version of a plugin is locally installed, it may or may not be loaded depending on the constraints expressed in the template being executed. If the template contains constraints for loading the plugin, it would be ignored, while if that wasn't present, it would be loaded. This is inconsistent, and deserves to be addressed, which is what this commit does. With this change, plugin pre-releases are now loaded, provided the version reported matches the constraints, independently from its pre-release suffix `-dev'. Also, if a release with the same version is also installed alongside the pre-release version of a plugin, it will have precedence over the pre-release. --- packer/plugin-getter/plugins.go | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/packer/plugin-getter/plugins.go b/packer/plugin-getter/plugins.go index 11511ee3860..fae1809e09d 100644 --- a/packer/plugin-getter/plugins.go +++ b/packer/plugin-getter/plugins.go @@ -96,6 +96,8 @@ func (opts BinaryInstallationOptions) FilenameSuffix() string { return "_" + opts.OS + "_" + opts.ARCH + opts.Ext } +var pluginVersionRegex = regexp.MustCompile(`^v([0-9]+\.[0-9]+\.[0-9]+)(-dev)?$`) + // ListInstallations lists unique installed versions of plugin Requirement pr // with opts as a filter. // @@ -152,21 +154,37 @@ func (pr Requirement) ListInstallations(opts ListInstallationsOptions) (InstallL // versionsStr now looks like v1.2.3_x5.1 or amazon_v1.2.3_x5.1 parts := strings.SplitN(versionsStr, "_", 2) pluginVersionStr, protocolVerionStr := parts[0], parts[1] - pv, err := version.NewVersion(pluginVersionStr) + _, err = version.NewVersion(pluginVersionStr) if err != nil { // could not be parsed, ignoring the file log.Printf("found %q with an incorrect %q version, ignoring it. %v", path, pluginVersionStr, err) continue } - if strings.Replace(pluginVersionStr, "v", "", -1) != describeInfo.Version { - log.Printf("plugin %q reported version %s while its name implies version %s, ignoring", path, describeInfo.Version, pluginVersionStr) + matches := pluginVersionRegex.FindStringSubmatch(pluginVersionStr) + if matches == nil { + log.Printf("invalid version found: %q, ignoring", pluginVersionStr) + continue + } + + absVersion := matches[1] + if len(matches) == 3 { + absVersion = fmt.Sprintf("%s%s", absVersion, matches[2]) + } + + if absVersion != describeInfo.Version { + log.Printf("plugin %q reported version %s while its name implies version %s, ignoring", path, describeInfo.Version, absVersion) continue } + rawVersion, _ := version.NewVersion(matches[1]) // no constraint means always pass, this will happen for implicit // plugin requirements and when we list all plugins. - if !pr.VersionConstraints.Check(pv) { + // + // Note: we use the raw version name here, without the pre-release + // suffix, as otherwise constraints reject them, which is not + // what we want by default. + if !pr.VersionConstraints.Check(rawVersion) { log.Printf("[TRACE] version %q of file %q does not match constraint %q", pluginVersionStr, path, pr.VersionConstraints.String()) continue } From f4d52bfc190086437ac0f9c7e2f091cd08dde7e6 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 2 Feb 2024 10:02:40 -0500 Subject: [PATCH 12/32] main: move Discover to DetectPluginBinaries When Packer is loaded, we used to perform plugin discovery. This was done for every call to Packer, including when it is executed as a plugin, arguably against what the comments document. Doing this as early in the loading process makes it harder to change this behaviour, as we'd need to introduce flags aside from the rest, and handle them manually, which is not optimal. Therefore, we change this: now when Packer starts executing, it will not attempt to discover installed plugins anymore, and instead will only try to load them when a configuration has been parsed, and is being used to perform actions (typically build/validate). --- hcl2template/plugin.go | 11 +++++++++++ main.go | 7 ++++--- packer/core.go | 13 ++++++++++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/hcl2template/plugin.go b/hcl2template/plugin.go index 15358573d6e..6a66aa61096 100644 --- a/hcl2template/plugin.go +++ b/hcl2template/plugin.go @@ -55,6 +55,17 @@ func (cfg *PackerConfig) PluginRequirements() (plugingetter.Requirements, hcl.Di } func (cfg *PackerConfig) DetectPluginBinaries() hcl.Diagnostics { + // Do first pass to discover all the installed plugins + err := cfg.parser.PluginConfig.Discover() + if err != nil { + return (hcl.Diagnostics{}).Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to discover installed plugins", + Detail: err.Error(), + }) + } + + // Then we can apply any constraint from the template, if any opts := plugingetter.ListInstallationsOptions{ PluginDirectory: cfg.parser.PluginConfig.PluginDirectory, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ diff --git a/main.go b/main.go index bb1d741d694..553c173f477 100644 --- a/main.go +++ b/main.go @@ -339,9 +339,10 @@ func loadConfig() (*config, error) { PluginMinPort: 10000, PluginMaxPort: 25000, PluginDirectory: pluginDir, - } - if err := config.Plugins.Discover(); err != nil { - return nil, err + Builders: packer.MapOfBuilder{}, + Provisioners: packer.MapOfProvisioner{}, + PostProcessors: packer.MapOfPostProcessor{}, + DataSources: packer.MapOfDatasource{}, } // Finally, try to use an internal plugin. Note that this will not override diff --git a/packer/core.go b/packer/core.go index 7a56dc06cc5..0f3ea9c9c1e 100644 --- a/packer/core.go +++ b/packer/core.go @@ -136,7 +136,18 @@ func NewCore(c *CoreConfig) *Core { // DetectPluginBinaries is used to load required plugins from the template, // since it is unsupported in JSON, this is essentially a no-op. func (c *Core) DetectPluginBinaries() hcl.Diagnostics { - return nil + var diags hcl.Diagnostics + + err := c.components.PluginConfig.Discover() + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to discover installed plugins", + Detail: err.Error(), + }) + } + + return diags } func (c *Core) Initialize(_ InitializeOptions) hcl.Diagnostics { From 4c22e48685ccf1aa9155fde7453d783e66bcdb83 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 2 Feb 2024 10:34:52 -0500 Subject: [PATCH 13/32] plugins: add a release only flag to Discover Since we now support loading pre-releases, we also want Packer to be able to ignore them by user demand, so we put in place the infrastructure to modulate this. --- hcl2template/plugin.go | 1 + packer/plugin-getter/plugins.go | 11 ++++++++++- packer/plugin.go | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/hcl2template/plugin.go b/hcl2template/plugin.go index 6a66aa61096..f03e0329224 100644 --- a/hcl2template/plugin.go +++ b/hcl2template/plugin.go @@ -76,6 +76,7 @@ func (cfg *PackerConfig) DetectPluginBinaries() hcl.Diagnostics { Checksummers: []plugingetter.Checksummer{ {Type: "sha256", Hash: sha256.New()}, }, + ReleasesOnly: cfg.parser.PluginConfig.ReleasesOnly, }, } diff --git a/packer/plugin-getter/plugins.go b/packer/plugin-getter/plugins.go index fae1809e09d..3271a868d5b 100644 --- a/packer/plugin-getter/plugins.go +++ b/packer/plugin-getter/plugins.go @@ -60,6 +60,10 @@ type BinaryInstallationOptions struct { Ext string Checksummers []Checksummer + + // ReleasesOnly may be set by commands like validate or build, and + // forces Packer to not consider plugin pre-releases. + ReleasesOnly bool } type ListInstallationsOptions struct { @@ -154,13 +158,18 @@ func (pr Requirement) ListInstallations(opts ListInstallationsOptions) (InstallL // versionsStr now looks like v1.2.3_x5.1 or amazon_v1.2.3_x5.1 parts := strings.SplitN(versionsStr, "_", 2) pluginVersionStr, protocolVerionStr := parts[0], parts[1] - _, err = version.NewVersion(pluginVersionStr) + ver, err := version.NewVersion(pluginVersionStr) if err != nil { // could not be parsed, ignoring the file log.Printf("found %q with an incorrect %q version, ignoring it. %v", path, pluginVersionStr, err) continue } + if ver.Prerelease() != "" && opts.ReleasesOnly { + log.Printf("ignoring pre-release plugin %q", path) + continue + } + matches := pluginVersionRegex.FindStringSubmatch(pluginVersionStr) if matches == nil { log.Printf("invalid version found: %q, ignoring", pluginVersionStr) diff --git a/packer/plugin.go b/packer/plugin.go index c6d4c125946..d2bf0e200cf 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -34,6 +34,7 @@ type PluginConfig struct { Provisioners ProvisionerSet PostProcessors PostProcessorSet DataSources DatasourceSet + ReleasesOnly bool } // PACKERSPACE is used to represent the spaces that separate args for a command @@ -81,6 +82,7 @@ func (c *PluginConfig) Discover() error { Checksummers: []plugingetter.Checksummer{ {Type: "sha256", Hash: sha256.New()}, }, + ReleasesOnly: c.ReleasesOnly, }, }) if err != nil { From d4b45cd67ac247f137309cebe54fc58573e99f5c Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 2 Feb 2024 10:41:31 -0500 Subject: [PATCH 14/32] command: add --release-only flag to validate/build Since we added the capability for the plugin discovery to ignore installed pre-releases of plugins, we give that capacity to the validate and build commands, through a new command-line flag: release-only. --- command/build.go | 5 +++++ command/cli.go | 6 ++++++ command/validate.go | 5 +++++ 3 files changed, 16 insertions(+) diff --git a/command/build.go b/command/build.go index 60ede416894..b48d248d50c 100644 --- a/command/build.go +++ b/command/build.go @@ -86,6 +86,11 @@ func writeDiags(ui packersdk.Ui, files map[string]*hcl.File, diags hcl.Diagnosti } func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int { + // Set the release only flag if specified as argument + // + // This deactivates the capacity for Packer to load development binaries. + c.CoreConfig.Components.PluginConfig.ReleasesOnly = cla.ReleaseOnly + packerStarter, ret := c.GetConfig(&cla.MetaArgs) if ret != 0 { return ret diff --git a/command/cli.go b/command/cli.go index 2dfc57dbd0d..2fa3c02fdea 100644 --- a/command/cli.go +++ b/command/cli.go @@ -90,6 +90,9 @@ func (ba *BuildArgs) AddFlagSets(flags *flag.FlagSet) { flags.Var(flagOnError, "on-error", "") flags.BoolVar(&ba.MetaArgs.WarnOnUndeclaredVar, "warn-on-undeclared-var", false, "Show warnings for variable files containing undeclared variables.") + + flags.BoolVar(&ba.ReleaseOnly, "ignore-prerelease-plugins", false, "Disable the loading of prerelease plugin binaries (x.y.z-.") + ba.MetaArgs.AddFlagSets(flags) } @@ -100,6 +103,7 @@ type BuildArgs struct { Color, TimestampUi, MachineReadable bool ParallelBuilds int64 OnError string + ReleaseOnly bool } func (ia *InitArgs) AddFlagSets(flags *flag.FlagSet) { @@ -142,6 +146,7 @@ func (va *ValidateArgs) AddFlagSets(flags *flag.FlagSet) { flags.BoolVar(&va.SyntaxOnly, "syntax-only", false, "check syntax only") flags.BoolVar(&va.NoWarnUndeclaredVar, "no-warn-undeclared-var", false, "Ignore warnings for variable files containing undeclared variables.") flags.BoolVar(&va.EvaluateDatasources, "evaluate-datasources", false, "evaluate datasources for validation (HCL2 only, may incur costs)") + flags.BoolVar(&va.ReleaseOnly, "ignore-prerelease-plugins", false, "Disable the loading of prerelease plugin binaries (x.y.z-.") va.MetaArgs.AddFlagSets(flags) } @@ -151,6 +156,7 @@ type ValidateArgs struct { MetaArgs SyntaxOnly, NoWarnUndeclaredVar bool EvaluateDatasources bool + ReleaseOnly bool } func (va *InspectArgs) AddFlagSets(flags *flag.FlagSet) { diff --git a/command/validate.go b/command/validate.go index 9747dabf95c..da41e30cf0a 100644 --- a/command/validate.go +++ b/command/validate.go @@ -48,6 +48,11 @@ func (c *ValidateCommand) ParseArgs(args []string) (*ValidateArgs, int) { } func (c *ValidateCommand) RunContext(ctx context.Context, cla *ValidateArgs) int { + // Set the release only flag if specified as argument + // + // This deactivates the capacity for Packer to load development binaries. + c.CoreConfig.Components.PluginConfig.ReleasesOnly = cla.ReleaseOnly + // By default we want to inform users of undeclared variables when validating but not during build time. cla.MetaArgs.WarnOnUndeclaredVar = true if cla.NoWarnUndeclaredVar { From e65a6f583cc3ac213d4b0f57e348795a6c30fae2 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 2 Feb 2024 11:04:44 -0500 Subject: [PATCH 15/32] command: add dev support for plugins install Since we now support loading pre-releases with Packer subcommands, we relax the contraints we had placed on the `--path' option, so it will accept any `-dev' binary, in addition to final releases. --- command/plugins_install.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/command/plugins_install.go b/command/plugins_install.go index 1234579206a..0563ae31f77 100644 --- a/command/plugins_install.go +++ b/command/plugins_install.go @@ -257,11 +257,11 @@ func (c *PluginsInstallCommand) InstallFromBinary(opts plugingetter.ListInstalla Detail: fmt.Sprintf("Plugin's reported version (%q) is not semver-compatible: %s", desc.Version, err), }}) } - if semver.Prerelease() != "" { + if semver.Prerelease() != "" && semver.Prerelease() != "dev" { return writeDiags(c.Ui, nil, hcl.Diagnostics{&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Invalid version", - Detail: fmt.Sprintf("Packer can only install plugin releases with this command (ex: 1.0.0), the binary's reported version is %q", desc.Version), + Detail: fmt.Sprintf("Packer can only install plugin releases with this command (ex: 1.0.0) or development pre-releases (ex: 1.0.0-dev), the binary's reported version is %q", desc.Version), }}) } From 80b4f4511113cf69a7e4ebdaaa8b881e446040dc Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 2 Feb 2024 11:06:59 -0500 Subject: [PATCH 16/32] command: trim trailing spaces from help message --- command/plugins_install.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/command/plugins_install.go b/command/plugins_install.go index 0563ae31f77..0540909c480 100644 --- a/command/plugins_install.go +++ b/command/plugins_install.go @@ -49,10 +49,10 @@ Usage: packer plugins install [OPTIONS...] [] packer plugins install --path ./packer-plugin-happycloud "github.com/hashicorp/happycloud" Options: - -path Install the plugin from a locally-sourced plugin binary. - This installs the plugin where a normal invocation would, but will + -path Install the plugin from a locally-sourced plugin binary. + This installs the plugin where a normal invocation would, but will not try to download it from a remote location, and instead - install the binary in the Packer plugins path. This option cannot + install the binary in the Packer plugins path. This option cannot be specified with a version constraint. -force Forces reinstallation of plugins, even if already installed. ` From 4774675aaec737d5b8c604d0f223f6f71e44b59c Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 23 Feb 2024 16:01:01 -0500 Subject: [PATCH 17/32] command: migrate tests using the comment plugin In order to test the Packer subcommands, some tests rely on a plugin: github.com/sylviamoss/comment. This plugin is outdated compared to our SDK, and some of the binaries releases don't match what is expected by Packer, i.e. v1.0.0 vs. v0.2.8. To fix that, we migrate to use the hashicups plugin, which is our demo plugin for Packer, and is actually maintained, and up-to-date, which will make those tests more reliable moving forward. --- command/init_test.go | 144 ++++++++++++++++---------------- command/plugins_install_test.go | 68 +++++++-------- 2 files changed, 107 insertions(+), 105 deletions(-) diff --git a/command/init_test.go b/command/init_test.go index ad501f15182..4a0c44be945 100644 --- a/command/init_test.go +++ b/command/init_test.go @@ -61,71 +61,75 @@ func TestInitCommand_Run(t *testing.T) { cfg := &configDirSingleton{map[string]string{}} tests := []testCaseInit{ + // { + // // here we pre-write plugins with valid checksums, Packer will + // // see those as valid installations it did. + // // the directory hash before and after init should be the same, + // // that's a no-op. This also should do no GH query, so it is best + // // to always run it. + // // + // // Note: cannot work with plugin changes since the fake binary + // // isn't recognised as a potential plugin, so Packer always + // // installs it. + // "already-installed-no-op", + // nil, + // TestMetaFile(t), + // map[string]string{ + // "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_darwin_amd64": "1", + // "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_darwin_amd64_SHA256SUM": "a23e48324f2d9b912a89354945562b21b0ae99133b31d3132e2e6671aba8e085", + // "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_windows_amd64.exe": "1.exe", + // "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_windows_amd64.exe_SHA256SUM": "f1cf5865b35933b8e5195625ac8be44487b64007f223912cc5c1784e493e62b2", + // "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_linux_amd64": "1.out", + // "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_linux_amd64_SHA256SUM": "0a4e4e1d6de28054f64946782a5eb92edc663e980ae0780fcb3a614d27c58506", + // }, + // "h1:jQchMpyaQhkZYn0iguw6E6O4VCWxacYx2aR/RJJNLmo=", + // map[string]string{ + // `cfg.pkr.hcl`: ` + // packer { + // required_plugins { + // comment = { + // source = "github.com/hashicorp/hashicups" + // version = "v1.0.1" + // } + // } + // }`, + // }, + // cfg.dir("1_pkr_config"), + // cfg.dir("1_pkr_user_folder"), + // 0, + // nil, + // "h1:jQchMpyaQhkZYn0iguw6E6O4VCWxacYx2aR/RJJNLmo=", + // []func(t *testing.T, tc testCaseInit){ + // // test that a build will not work since plugins are broken for + // // this tests (they are not binaries). + // testBuild{want: 1}.fn, + // }, + // }, { // here we pre-write plugins with valid checksums, Packer will // see those as valid installations it did. - // the directory hash before and after init should be the same, - // that's a no-op. This also should do no GH query, so it is best - // to always run it. - "already-installed-no-op", - nil, - TestMetaFile(t), - map[string]string{ - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64": "1", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64_SHA256SUM": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe": "1.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe_SHA256SUM": "07d8453027192ee0c4120242e6e84e2ca2328b8e0f506e2f818a1a5b82790a0b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64": "1.out", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64_SHA256SUM": "59031c50e0dfeedfde2b4e9445754804dce3f29e4efa737eead0ca9b4f5b85a5", - }, - "h1:Q5qyAOdD43hL3CquQdVfaHpOYGf0UsZ/+wVA9Ry6cbA=", - map[string]string{ - `cfg.pkr.hcl`: ` - packer { - required_plugins { - comment = { - source = "github.com/sylviamoss/comment" - version = "v0.2.018" - } - } - }`, - }, - cfg.dir("1_pkr_config"), - cfg.dir("1_pkr_user_folder"), - 0, - nil, - "h1:Q5qyAOdD43hL3CquQdVfaHpOYGf0UsZ/+wVA9Ry6cbA=", - []func(t *testing.T, tc testCaseInit){ - // test that a build will not work since plugins are broken for - // this tests (they are not binaries). - testBuild{want: 1}.fn, - }, - }, - { - // here we pre-write plugins with valid checksums, Packer will - // see those as valid installations it did. - // But because we require version 0.2.19, we will upgrade. + // But because we require version 1.0.2, we will upgrade. "already-installed-upgrade", []func(t *testing.T, tc testCaseInit){ skipInitTestUnlessEnVar(acctest.TestEnvVar).fn, }, TestMetaFile(t), map[string]string{ - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64": "1", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64_SHA256SUM": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe": "1.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe_SHA256SUM": "07d8453027192ee0c4120242e6e84e2ca2328b8e0f506e2f818a1a5b82790a0b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64": "1.out", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64_SHA256SUM": "59031c50e0dfeedfde2b4e9445754804dce3f29e4efa737eead0ca9b4f5b85a5", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_darwin_amd64": "1", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_darwin_amd64_SHA256SUM": "a23e48324f2d9b912a89354945562b21b0ae99133b31d3132e2e6671aba8e085", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_windows_amd64.exe": "1.exe", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_windows_amd64.exe_SHA256SUM": "f1cf5865b35933b8e5195625ac8be44487b64007f223912cc5c1784e493e62b2", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_linux_amd64": "1.out", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_linux_amd64_SHA256SUM": "0a4e4e1d6de28054f64946782a5eb92edc663e980ae0780fcb3a614d27c58506", }, - "h1:Q5qyAOdD43hL3CquQdVfaHpOYGf0UsZ/+wVA9Ry6cbA=", + "h1:jQchMpyaQhkZYn0iguw6E6O4VCWxacYx2aR/RJJNLmo=", map[string]string{ `cfg.pkr.hcl`: ` packer { required_plugins { - comment = { - source = "github.com/sylviamoss/comment" - version = "v0.2.019" + hashicups = { + source = "github.com/hashicorp/hashicups" + version = "v1.0.2" } } }`, @@ -136,11 +140,9 @@ func TestInitCommand_Run(t *testing.T) { `, `build.pkr.hcl`: ` build { - sources = ["source.null.test"] - provisioner "comment" { - comment = "Begin ¡" - ui = true - bubble_text = true + sources = ["null.test"] + provisioner "hashicups-toppings" { + toppings = ["sugar"] # Takes 5 seconds in the current state } } `, @@ -149,27 +151,27 @@ func TestInitCommand_Run(t *testing.T) { cfg.dir("2_pkr_user_folder"), 0, []string{ - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe_SHA256SUM", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_darwin_amd64", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_darwin_amd64_SHA256SUM", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_linux_amd64", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_linux_amd64_SHA256SUM", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_windows_amd64.exe", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_windows_amd64.exe_SHA256SUM", map[string]string{ - "darwin": "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_darwin_amd64_SHA256SUM", - "linux": "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_linux_amd64_SHA256SUM", - "windows": "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_windows_amd64.exe_SHA256SUM", + "darwin": "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.2_x5.0_darwin_amd64_SHA256SUM", + "linux": "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.2_x5.0_linux_amd64_SHA256SUM", + "windows": "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.2_x5.0_windows_amd64.exe_SHA256SUM", }[runtime.GOOS], map[string]string{ - "darwin": "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_darwin_amd64", - "linux": "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_linux_amd64", - "windows": "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_windows_amd64.exe", + "darwin": "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.2_x5.0_darwin_amd64", + "linux": "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.2_x5.0_linux_amd64", + "windows": "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.2_x5.0_windows_amd64.exe", }[runtime.GOOS], }, map[string]string{ - "darwin": "h1:ORwcCYUx8z/5n/QvuTJo2vrgKpfJA4AxlNg1G9/BCDI=", - "linux": "h1:CGym0+Nd0LEANgzqL0wx/LDjRL8bYwlpZ0HajPJo/hs=", - "windows": "h1:ag0/C1YjP7KoEDYOiJHE0K+lhFgs0tVgjriWCXVT1fg=", + "darwin": "h1:ptsMLvUeLsMMeXDJP2PWKAKIkE+kWVhOkhNYOYPJbSE=", + "linux": "h1:ivCmyQ+/qNXfBsyeccGsa7P5232q7MUZk83B3yl80Ms=", + "windows": "h1:BeqAUnyGiBg9fVuf9Cn9a4h91bgdZ2U4kV7EuQKefcM=", }[runtime.GOOS], []func(t *testing.T, tc testCaseInit){ // test that a build will work as the plugin was just installed diff --git a/command/plugins_install_test.go b/command/plugins_install_test.go index e8ad08fd4d8..2df9e7a1da9 100644 --- a/command/plugins_install_test.go +++ b/command/plugins_install_test.go @@ -37,57 +37,57 @@ func TestPluginsInstallCommand_Run(t *testing.T) { name: "already-installed-no-op", Meta: TestMetaFile(t), inPluginFolder: map[string]string{ - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64": "1", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64_SHA256SUM": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe": "1.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe_SHA256SUM": "07d8453027192ee0c4120242e6e84e2ca2328b8e0f506e2f818a1a5b82790a0b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64": "1.out", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64_SHA256SUM": "59031c50e0dfeedfde2b4e9445754804dce3f29e4efa737eead0ca9b4f5b85a5", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_darwin_amd64": "1", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_darwin_amd64_SHA256SUM": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_windows_amd64.exe": "1.exe", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_windows_amd64.exe_SHA256SUM": "07d8453027192ee0c4120242e6e84e2ca2328b8e0f506e2f818a1a5b82790a0b", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_linux_amd64": "1.out", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_linux_amd64_SHA256SUM": "59031c50e0dfeedfde2b4e9445754804dce3f29e4efa737eead0ca9b4f5b85a5", }, - expectedPackerConfigDirHashBeforeInstall: "h1:Q5qyAOdD43hL3CquQdVfaHpOYGf0UsZ/+wVA9Ry6cbA=", + expectedPackerConfigDirHashBeforeInstall: "h1:sxhWsXwLkxCO1fm86oO2QIg7ImLdfw9pmXa0hKHIeUw=", packerConfigDir: cfg.dir("1_pkr_plugins_config"), - pluginSourceArgs: []string{"github.com/sylviamoss/comment", "v0.2.18"}, + pluginSourceArgs: []string{"github.com/hashicorp/hashicups", "v1.0.1"}, want: 0, dirFiles: nil, - expectedPackerConfigDirHashAfterInstall: "h1:Q5qyAOdD43hL3CquQdVfaHpOYGf0UsZ/+wVA9Ry6cbA=", + expectedPackerConfigDirHashAfterInstall: "h1:sxhWsXwLkxCO1fm86oO2QIg7ImLdfw9pmXa0hKHIeUw=", }, { name: "install-newer-version", Meta: TestMetaFile(t), inPluginFolder: map[string]string{ - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64": "1", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64_SHA256SUM": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe": "1.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe_SHA256SUM": "07d8453027192ee0c4120242e6e84e2ca2328b8e0f506e2f818a1a5b82790a0b", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64": "1.out", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64_SHA256SUM": "59031c50e0dfeedfde2b4e9445754804dce3f29e4efa737eead0ca9b4f5b85a5", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_darwin_amd64": "1", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_darwin_amd64_SHA256SUM": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_windows_amd64.exe": "1.exe", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_windows_amd64.exe_SHA256SUM": "07d8453027192ee0c4120242e6e84e2ca2328b8e0f506e2f818a1a5b82790a0b", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_linux_amd64": "1.out", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_linux_amd64_SHA256SUM": "59031c50e0dfeedfde2b4e9445754804dce3f29e4efa737eead0ca9b4f5b85a5", }, - expectedPackerConfigDirHashBeforeInstall: "h1:Q5qyAOdD43hL3CquQdVfaHpOYGf0UsZ/+wVA9Ry6cbA=", + expectedPackerConfigDirHashBeforeInstall: "h1:sxhWsXwLkxCO1fm86oO2QIg7ImLdfw9pmXa0hKHIeUw=", packerConfigDir: cfg.dir("2_pkr_plugins_config"), - pluginSourceArgs: []string{"github.com/sylviamoss/comment", "v0.2.19"}, + pluginSourceArgs: []string{"github.com/hashicorp/hashicups", "v1.0.2"}, want: 0, dirFiles: []string{ - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_darwin_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_linux_amd64_SHA256SUM", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe", - "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.18_x5.0_windows_amd64.exe_SHA256SUM", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_darwin_amd64", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_darwin_amd64_SHA256SUM", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_linux_amd64", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_linux_amd64_SHA256SUM", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_windows_amd64.exe", + "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.1_x5.0_windows_amd64.exe_SHA256SUM", map[string]string{ - "darwin": "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_darwin_amd64_SHA256SUM", - "linux": "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_linux_amd64_SHA256SUM", - "windows": "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_windows_amd64.exe_SHA256SUM", + "darwin": "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.2_x5.0_darwin_amd64_SHA256SUM", + "linux": "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.2_x5.0_linux_amd64_SHA256SUM", + "windows": "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.2_x5.0_windows_amd64.exe_SHA256SUM", }[runtime.GOOS], map[string]string{ - "darwin": "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_darwin_amd64", - "linux": "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_linux_amd64", - "windows": "github.com/sylviamoss/comment/packer-plugin-comment_v0.2.19_x5.0_windows_amd64.exe", + "darwin": "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.2_x5.0_darwin_amd64", + "linux": "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.2_x5.0_linux_amd64", + "windows": "github.com/hashicorp/hashicups/packer-plugin-hashicups_v1.0.2_x5.0_windows_amd64.exe", }[runtime.GOOS], }, expectedPackerConfigDirHashAfterInstall: map[string]string{ - "darwin": "h1:ORwcCYUx8z/5n/QvuTJo2vrgKpfJA4AxlNg1G9/BCDI=", - "linux": "h1:CGym0+Nd0LEANgzqL0wx/LDjRL8bYwlpZ0HajPJo/hs=", - "windows": "h1:ag0/C1YjP7KoEDYOiJHE0K+lhFgs0tVgjriWCXVT1fg=", + "darwin": "h1:itQ7rZfZarDHmnajkzfxBVMxZ0wBou4I6FNX/ysHggA=", + "linux": "h1:+zgZKpRpVofVgjny13tZNonPBcNNxWF741iYdTE9UCg=", + "windows": "h1:scuTEDpGEWJ0LaKL7ETZ8o7wdW6dJUzV+eg/2shQ+tQ=", }[runtime.GOOS], }, { @@ -96,7 +96,7 @@ func TestPluginsInstallCommand_Run(t *testing.T) { inPluginFolder: nil, expectedPackerConfigDirHashBeforeInstall: "h1:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", packerConfigDir: cfg.dir("3_pkr_plugins_config"), - pluginSourceArgs: []string{"example.com/sylviamoss/comment", "v0.2.19"}, + pluginSourceArgs: []string{"example.com/hashicorp/hashicups", "v0.2.19"}, want: 1, dirFiles: nil, expectedPackerConfigDirHashAfterInstall: "h1:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", @@ -107,7 +107,7 @@ func TestPluginsInstallCommand_Run(t *testing.T) { inPluginFolder: nil, expectedPackerConfigDirHashBeforeInstall: "h1:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", packerConfigDir: cfg.dir("4_pkr_plugins_config"), - pluginSourceArgs: []string{"github.com/sylviamoss/comment", "v0.2.18", "github.com/sylviamoss/comment", "v0.2.19"}, + pluginSourceArgs: []string{"github.com/hashicorp/hashicups", "v0.2.18", "github.com/hashicorp/hashicups", "v0.2.19"}, want: 1, dirFiles: nil, expectedPackerConfigDirHashAfterInstall: "h1:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", From ed03264915e57de589b486e587567b7d842116cb Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 23 Feb 2024 17:15:38 -0500 Subject: [PATCH 18/32] plugin-getter: fix typo in API mismatch error --- packer/plugin-getter/plugins.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packer/plugin-getter/plugins.go b/packer/plugin-getter/plugins.go index 3271a868d5b..c18e7d1a123 100644 --- a/packer/plugin-getter/plugins.go +++ b/packer/plugin-getter/plugins.go @@ -379,7 +379,7 @@ func (binOpts *BinaryInstallationOptions) CheckProtocolVersion(remoteProt string } if vMinori > APIVersoinMinori { - return fmt.Errorf("Unsupported remote protocol MINOR version %q. The supported MINOR protocol versions are version %q and bellow."+ + return fmt.Errorf("Unsupported remote protocol MINOR version %q. The supported MINOR protocol versions are version %q and below. "+ "Please upgrade Packer or use an older version of the plugin if possible.", vMinor, binOpts.APIVersionMinor) } From bf1895dbe4a7420d249068e73fe9a76c8675c5ef Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 23 Feb 2024 17:16:09 -0500 Subject: [PATCH 19/32] plugin: use API version for loading plugins While migrating to a unified approach for loading plugins, the API major and minor versions were not added to the constraints for discovering plugins from the environment, leading to Packer potentially considering plugins that are not compatible with itself. This should not be possible, but was due to this omission, which we fix with this commit. --- packer/plugin.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packer/plugin.go b/packer/plugin.go index d2bf0e200cf..30261d7e05c 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -77,8 +77,10 @@ func (c *PluginConfig) Discover() error { installations, err := plugingetter.Requirement{}.ListInstallations(plugingetter.ListInstallationsOptions{ PluginDirectory: c.PluginDirectory, BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ - OS: runtime.GOOS, - ARCH: runtime.GOARCH, + OS: runtime.GOOS, + ARCH: runtime.GOARCH, + APIVersionMajor: pluginsdk.APIVersionMajor, + APIVersionMinor: pluginsdk.APIVersionMinor, Checksummers: []plugingetter.Checksummer{ {Type: "sha256", Hash: sha256.New()}, }, From 0867ee1ff5c2e0b33619ca36d6479225a256fb4d Mon Sep 17 00:00:00 2001 From: Devashish Date: Wed, 28 Feb 2024 14:04:43 -0500 Subject: [PATCH 20/32] POC Builds Metadata --- command/build.go | 7 +- hcl2template/types.build.post-processor.go | 2 + hcl2template/types.build.provisioners.go | 2 + hcl2template/types.source.go | 2 + internal/hcp/registry/hcl.go | 7 ++ internal/hcp/registry/json.go | 10 +- internal/hcp/registry/null_registry.go | 1 + internal/hcp/registry/registry.go | 2 +- metadata/all_plugins_storage.go | 45 ++++++++ metadata/metadata_storage.go | 114 +++++++++++++++++++++ packer/core.go | 10 +- packer/plugin.go | 6 ++ 12 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 metadata/all_plugins_storage.go create mode 100644 metadata/metadata_storage.go diff --git a/command/build.go b/command/build.go index b48d248d50c..632bc7b909d 100644 --- a/command/build.go +++ b/command/build.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/hcl/v2" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer/internal/hcp/registry" + "github.com/hashicorp/packer/metadata" "github.com/hashicorp/packer/packer" "golang.org/x/sync/semaphore" @@ -135,6 +136,9 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int OnError: cla.OnError, }) + metadataByBuild := metadata.GetMetadataStorage().GetMetadataByBuild() + fmt.Printf("<===> Metadata = %q", metadataByBuild) + // here, something could have gone wrong but we still want to run valid // builds. ret = writeDiags(c.Ui, nil, diags) @@ -272,7 +276,8 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int buildCtx, b, runArtifacts, - err) + err, + metadataByBuild) if hcperr != nil { if _, ok := hcperr.(*registry.NotAHCPArtifactError); ok { writeDiags(c.Ui, nil, hcl.Diagnostics{ diff --git a/hcl2template/types.build.post-processor.go b/hcl2template/types.build.post-processor.go index 8844eadff11..fbae4edde70 100644 --- a/hcl2template/types.build.post-processor.go +++ b/hcl2template/types.build.post-processor.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" + "github.com/hashicorp/packer/metadata" ) // ProvisionerBlock references a detected but unparsed post processor @@ -70,6 +71,7 @@ func (cfg *PackerConfig) startPostProcessor(source SourceUseBlock, pp *PostProce }) return nil, diags } + metadata.GetMetadataStorage().AddPluginUsageMetadataFor(source.String(), pp.PType) builderVars := source.builderVariables() builderVars["packer_core_version"] = cfg.CorePackerVersionString diff --git a/hcl2template/types.build.provisioners.go b/hcl2template/types.build.provisioners.go index b08eca59f63..15a469e74e6 100644 --- a/hcl2template/types.build.provisioners.go +++ b/hcl2template/types.build.provisioners.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/hcl/v2/gohcl" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" hcl2shim "github.com/hashicorp/packer/hcl2template/shim" + "github.com/hashicorp/packer/metadata" "github.com/zclconf/go-cty/cty" ) @@ -179,6 +180,7 @@ func (cfg *PackerConfig) startProvisioner(source SourceUseBlock, pb *Provisioner }) return nil, diags } + metadata.GetMetadataStorage().AddPluginUsageMetadataFor(source.String(), pb.PType) builderVars := source.builderVariables() builderVars["packer_core_version"] = cfg.CorePackerVersionString diff --git a/hcl2template/types.source.go b/hcl2template/types.source.go index 46b9caac98e..bf2c8c331c9 100644 --- a/hcl2template/types.source.go +++ b/hcl2template/types.source.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/hcl/v2/gohcl" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" hcl2shim "github.com/hashicorp/packer/hcl2template/shim" + "github.com/hashicorp/packer/metadata" "github.com/zclconf/go-cty/cty" ) @@ -112,6 +113,7 @@ func (cfg *PackerConfig) startBuilder(source SourceUseBlock, ectx *hcl.EvalConte }) return builder, diags, nil } + metadata.GetMetadataStorage().AddPluginUsageMetadataFor(source.String(), source.Type) body := source.Body // Add known values to source accessor in eval context. diff --git a/internal/hcp/registry/hcl.go b/internal/hcp/registry/hcl.go index 23c5af7ccd9..1b31e23e271 100644 --- a/internal/hcp/registry/hcl.go +++ b/internal/hcp/registry/hcl.go @@ -79,12 +79,19 @@ func (h *HCLRegistry) CompleteBuild( build sdkpacker.Build, artifacts []sdkpacker.Artifact, buildErr error, + buildsMetadata map[string]map[string]string, ) ([]sdkpacker.Artifact, error) { name := build.Name() cb, ok := build.(*packer.CoreBuild) if ok { name = cb.Type } + buildMetadata, ok := buildsMetadata[name] + if !ok { + fmt.Printf("[METADATA] HCL Metadata for build name %q: MISSING\n", name) + } else { + fmt.Printf("[METADATA] HCL Metadata for build name %q: %q\n", name, buildMetadata) + } return h.bucket.completeBuild(ctx, name, artifacts, buildErr) } diff --git a/internal/hcp/registry/json.go b/internal/hcp/registry/json.go index b35f2ea49ed..721f630d768 100644 --- a/internal/hcp/registry/json.go +++ b/internal/hcp/registry/json.go @@ -92,8 +92,16 @@ func (h *JSONRegistry) CompleteBuild( build sdkpacker.Build, artifacts []sdkpacker.Artifact, buildErr error, + buildsMetadata map[string]map[string]string, ) ([]sdkpacker.Artifact, error) { - return h.bucket.completeBuild(ctx, build.Name(), artifacts, buildErr) + name := build.Name() + buildMetadata, ok := buildsMetadata[name] + if !ok { + fmt.Printf("[METADATA] JSON Metadata for build name %q: MISSING\n", name) + } else { + fmt.Printf("[METADATA] JSON Metadata for build name %q: %q\n", name, buildMetadata) + } + return h.bucket.completeBuild(ctx, name, artifacts, buildErr) } // VersionStatusSummary prints a status report in the UI if the version is not yet done diff --git a/internal/hcp/registry/null_registry.go b/internal/hcp/registry/null_registry.go index 4f32c3d12b3..515461fc70f 100644 --- a/internal/hcp/registry/null_registry.go +++ b/internal/hcp/registry/null_registry.go @@ -25,6 +25,7 @@ func (r nullRegistry) CompleteBuild( build sdkpacker.Build, artifacts []sdkpacker.Artifact, buildErr error, + buildsMetadata map[string]map[string]string, ) ([]sdkpacker.Artifact, error) { return artifacts, nil } diff --git a/internal/hcp/registry/registry.go b/internal/hcp/registry/registry.go index 56b70285d41..6c51e1a42de 100644 --- a/internal/hcp/registry/registry.go +++ b/internal/hcp/registry/registry.go @@ -17,7 +17,7 @@ import ( type Registry interface { PopulateVersion(context.Context) error StartBuild(context.Context, sdkpacker.Build) error - CompleteBuild(ctx context.Context, build sdkpacker.Build, artifacts []sdkpacker.Artifact, buildErr error) ([]sdkpacker.Artifact, error) + CompleteBuild(ctx context.Context, build sdkpacker.Build, artifacts []sdkpacker.Artifact, buildErr error, buildsMetadata map[string]map[string]string) ([]sdkpacker.Artifact, error) VersionStatusSummary() } diff --git a/metadata/all_plugins_storage.go b/metadata/all_plugins_storage.go new file mode 100644 index 00000000000..f05f3ce202b --- /dev/null +++ b/metadata/all_plugins_storage.go @@ -0,0 +1,45 @@ +package metadata + +import ( + "sync" + + pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin" +) + +type PluginDetails struct { + Name string + Description pluginsdk.SetDescription +} + +type AllPluginsStorage struct { + Components map[string]*PluginDetails +} + +var ( + allPluginsStorage *AllPluginsStorage + allPluginsStorageOnce sync.Once +) + +func GetAllPluginsStorage() *AllPluginsStorage { + allPluginsStorageOnce.Do(func() { + allPluginsStorage = &AllPluginsStorage{ + Components: map[string]*PluginDetails{}, + } + }) + return allPluginsStorage +} + +func (aps *AllPluginsStorage) AddPluginDetails(componentKey, pluginName string, pluginDescription pluginsdk.SetDescription) { + aps.Components[componentKey] = &PluginDetails{ + Name: pluginName, + Description: pluginDescription, + } +} + +func (aps *AllPluginsStorage) GetPluginDetailsFor(componentKey string) *PluginDetails { + pluginDetails, ok := aps.Components[componentKey] + if !ok { + return nil + } + return pluginDetails +} diff --git a/metadata/metadata_storage.go b/metadata/metadata_storage.go new file mode 100644 index 00000000000..265d92db521 --- /dev/null +++ b/metadata/metadata_storage.go @@ -0,0 +1,114 @@ +package metadata + +import ( + "fmt" + "sync" + + "github.com/hashicorp/packer/version" +) + +type PluginComponentUsedInBuild struct { + BuildName string + ComponentKey string +} + +type MetadataStorage struct { + PluginComponentUsage []*PluginComponentUsedInBuild +} + +var ( + metadataStorage *MetadataStorage + metadataStorageOnce sync.Once +) + +func GetMetadataStorage() *MetadataStorage { + metadataStorageOnce.Do(func() { + metadataStorage = &MetadataStorage{ + PluginComponentUsage: []*PluginComponentUsedInBuild{}, + } + }) + return metadataStorage +} + +func (ms *MetadataStorage) AddPluginUsageMetadataFor(buildName string, componentKey string) { + ms.PluginComponentUsage = append(ms.PluginComponentUsage, &PluginComponentUsedInBuild{ + BuildName: buildName, + ComponentKey: componentKey, + }) +} + +func (ms *MetadataStorage) GetMetadataByBuild() map[string]map[string]string { + pluginStorage := GetAllPluginsStorage() + resp := map[string]map[string]string{} + + fmt.Printf("<===> All Plugins: %q\n", pluginStorage.Components) + fmt.Printf("<===> All Builds: %q\n", ms.PluginComponentUsage) + for _, pluginUsage := range ms.PluginComponentUsage { + pluginUsageByBuild, ok := resp[pluginUsage.BuildName] + if !ok { + pluginUsageByBuild = map[string]string{} + } + fmt.Printf("<===> Build Name: %q\n", pluginUsage.BuildName) + fmt.Printf("<===> Plugin Component: %q\n", pluginUsage.ComponentKey) + + compDetails := pluginStorage.GetPluginDetailsFor(pluginUsage.ComponentKey) + if compDetails == nil { + fmt.Printf("<===> Can not find component details. \n") + continue + } + + fmt.Printf("<===> Plugin Name: %q\n", compDetails.Name) + fmt.Printf("<===> Plugin Version: %q\n", compDetails.Description.Version) + pluginUsageByBuild[compDetails.Name] = compDetails.Description.Version + resp[pluginUsage.BuildName] = pluginUsageByBuild + } + return resp +} + +type PluginMetadata struct { + Name string + Version string +} + +type PackerCoreMetadata struct { + Version string +} + +type BuildsMetadata struct { + PluginsMetadata map[string][]PluginMetadata + PackerCoreMetadata PackerCoreMetadata +} + +func (ms *MetadataStorage) GetMetadata() *BuildsMetadata { + pluginStorage := GetAllPluginsStorage() + + respMetadata := BuildsMetadata{ + PackerCoreMetadata: PackerCoreMetadata{Version: version.FormattedVersion()}, + } + + for _, pluginComponentUsage := range ms.PluginComponentUsage { + pluginUsageByBuild, ok := respMetadata.PluginsMetadata[pluginComponentUsage.BuildName] + if !ok { + pluginUsageByBuild = []PluginMetadata{} + } + + fmt.Printf("<===> Build Name: %q\n", pluginComponentUsage.BuildName) + fmt.Printf("<===> Plugin Component: %q\n", pluginComponentUsage.ComponentKey) + + compDetails := pluginStorage.GetPluginDetailsFor(pluginComponentUsage.ComponentKey) + if compDetails == nil { + fmt.Printf("<===> Can not find component details. \n") + continue + } + + fmt.Printf("<===> Plugin Name: %q\n", compDetails.Name) + fmt.Printf("<===> Plugin Version: %q\n", compDetails.Description.Version) + pluginUsageByBuild = append(pluginUsageByBuild, PluginMetadata{ + Name: compDetails.Name, + Version: compDetails.Description.Version, + }) + respMetadata.PluginsMetadata[pluginComponentUsage.BuildName] = pluginUsageByBuild + } + + return &respMetadata +} diff --git a/packer/core.go b/packer/core.go index 0f3ea9c9c1e..12cdbd940ac 100644 --- a/packer/core.go +++ b/packer/core.go @@ -15,13 +15,14 @@ import ( ttmp "text/template" "github.com/google/go-cmp/cmp" - multierror "github.com/hashicorp/go-multierror" - version "github.com/hashicorp/go-version" - hcl "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl/v2" "github.com/hashicorp/packer-plugin-sdk/didyoumean" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/template" "github.com/hashicorp/packer-plugin-sdk/template/interpolate" + "github.com/hashicorp/packer/metadata" plugingetter "github.com/hashicorp/packer/packer/plugin-getter" packerversion "github.com/hashicorp/packer/version" ) @@ -258,6 +259,7 @@ func (c *Core) generateCoreBuildProvisioner(rawP *template.Provisioner, rawName return cbp, fmt.Errorf( "provisioner failed to be started and did not error: %s", rawP.Type) } + metadata.GetMetadataStorage().AddPluginUsageMetadataFor(rawName, rawP.Type) // Get the configuration config := make([]interface{}, 1, 2) @@ -393,6 +395,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { return nil, fmt.Errorf( "builder type not found: %s", configBuilder.Type) } + metadata.GetMetadataStorage().AddPluginUsageMetadataFor(n, configBuilder.Type) // rawName is the uninterpolated name that we use for various lookups rawName := configBuilder.Name @@ -469,6 +472,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { return nil, fmt.Errorf( "post-processor type not found: %s", rawP.Type) } + metadata.GetMetadataStorage().AddPluginUsageMetadataFor(n, rawP.Type) current = append(current, CoreBuildPostProcessor{ PostProcessor: postProcessor, diff --git a/packer/plugin.go b/packer/plugin.go index 30261d7e05c..5e702d52b15 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -17,6 +17,7 @@ import ( packersdk "github.com/hashicorp/packer-plugin-sdk/packer" pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin" + "github.com/hashicorp/packer/metadata" plugingetter "github.com/hashicorp/packer/packer/plugin-getter" ) @@ -134,6 +135,8 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error return err } + pluginStorage := metadata.GetAllPluginsStorage() + pluginPrefix := pluginName + "-" for _, builderName := range desc.Builders { @@ -145,6 +148,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Builders.Set(key, func() (packersdk.Builder, error) { return c.Client(pluginPath, "start", "builder", builderName).Builder() }) + pluginStorage.AddPluginDetails(key, pluginName, desc) } if len(desc.Builders) > 0 { @@ -160,6 +164,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.PostProcessors.Set(key, func() (packersdk.PostProcessor, error) { return c.Client(pluginPath, "start", "post-processor", postProcessorName).PostProcessor() }) + pluginStorage.AddPluginDetails(key, pluginName, desc) } if len(desc.PostProcessors) > 0 { @@ -175,6 +180,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Provisioners.Set(key, func() (packersdk.Provisioner, error) { return c.Client(pluginPath, "start", "provisioner", provisionerName).Provisioner() }) + pluginStorage.AddPluginDetails(key, pluginName, desc) } if len(desc.Provisioners) > 0 { log.Printf("found external %v provisioner from %s plugin", desc.Provisioners, pluginName) From aa61c9a1a902031d5669408903f482fb05172615 Mon Sep 17 00:00:00 2001 From: Devashish Date: Wed, 28 Feb 2024 14:12:38 -0500 Subject: [PATCH 21/32] Add response structure in schema --- metadata/metadata_storage.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/metadata/metadata_storage.go b/metadata/metadata_storage.go index 265d92db521..d53aa67a63f 100644 --- a/metadata/metadata_storage.go +++ b/metadata/metadata_storage.go @@ -79,6 +79,31 @@ type BuildsMetadata struct { PackerCoreMetadata PackerCoreMetadata } +/* Response structure: +{ + "PackerCoreMetadata": { + "Version": "1.10.1" + }, + "PluginsMetadata": { + "build1": [ + { + "Name": "docker", + "Version": "1.10.1" + } + ], + "build2": [ + { + "Name": "amazon", + "Version": "1.11.1" + }, + { + "Name": "docker", + "Version": "1.10.1" + } + ] + } +} +*/ func (ms *MetadataStorage) GetMetadata() *BuildsMetadata { pluginStorage := GetAllPluginsStorage() From 526e3a57c342994ce0c3846374b5c454e700d4cf Mon Sep 17 00:00:00 2001 From: Devashish Date: Wed, 28 Feb 2024 16:18:18 -0500 Subject: [PATCH 22/32] Move implementation to packer package --- command/build.go | 3 +- command/validate.go | 3 + hcl2template/types.build.post-processor.go | 4 +- hcl2template/types.build.provisioners.go | 4 +- hcl2template/types.source.go | 4 +- {metadata => packer}/all_plugins_storage.go | 24 +++---- packer/core.go | 9 ++- {metadata => packer}/metadata_storage.go | 77 ++++++++++----------- packer/plugin.go | 9 +-- 9 files changed, 63 insertions(+), 74 deletions(-) rename {metadata => packer}/all_plugins_storage.go (52%) rename {metadata => packer}/metadata_storage.go (74%) diff --git a/command/build.go b/command/build.go index 632bc7b909d..ec2076e4f76 100644 --- a/command/build.go +++ b/command/build.go @@ -18,7 +18,6 @@ import ( "github.com/hashicorp/hcl/v2" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer/internal/hcp/registry" - "github.com/hashicorp/packer/metadata" "github.com/hashicorp/packer/packer" "golang.org/x/sync/semaphore" @@ -136,7 +135,7 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int OnError: cla.OnError, }) - metadataByBuild := metadata.GetMetadataStorage().GetMetadataByBuild() + metadataByBuild := packer.MetadataStorage.GetMetadataByBuild() fmt.Printf("<===> Metadata = %q", metadataByBuild) // here, something could have gone wrong but we still want to run valid diff --git a/command/validate.go b/command/validate.go index da41e30cf0a..58468da2e7c 100644 --- a/command/validate.go +++ b/command/validate.go @@ -5,6 +5,7 @@ package command import ( "context" + "fmt" "strings" "github.com/hashicorp/packer/packer" @@ -88,6 +89,8 @@ func (c *ValidateCommand) RunContext(ctx context.Context, cla *ValidateArgs) int Only: cla.Only, Except: cla.Except, }) + metadataByBuild := packer.MetadataStorage.GetMetadataByBuild() + fmt.Printf("<===> Metadata = %q", metadataByBuild) fixerDiags := packerStarter.FixConfig(packer.FixConfigOptions{ Mode: packer.Diff, diff --git a/hcl2template/types.build.post-processor.go b/hcl2template/types.build.post-processor.go index fbae4edde70..bbb9bda2516 100644 --- a/hcl2template/types.build.post-processor.go +++ b/hcl2template/types.build.post-processor.go @@ -10,7 +10,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" - "github.com/hashicorp/packer/metadata" + "github.com/hashicorp/packer/packer" ) // ProvisionerBlock references a detected but unparsed post processor @@ -71,7 +71,7 @@ func (cfg *PackerConfig) startPostProcessor(source SourceUseBlock, pp *PostProce }) return nil, diags } - metadata.GetMetadataStorage().AddPluginUsageMetadataFor(source.String(), pp.PType) + packer.MetadataStorage.AddPluginUsageMetadataFor(source.String(), pp.PType) builderVars := source.builderVariables() builderVars["packer_core_version"] = cfg.CorePackerVersionString diff --git a/hcl2template/types.build.provisioners.go b/hcl2template/types.build.provisioners.go index 15a469e74e6..a63cca8bcd3 100644 --- a/hcl2template/types.build.provisioners.go +++ b/hcl2template/types.build.provisioners.go @@ -12,7 +12,7 @@ import ( "github.com/hashicorp/hcl/v2/gohcl" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" hcl2shim "github.com/hashicorp/packer/hcl2template/shim" - "github.com/hashicorp/packer/metadata" + "github.com/hashicorp/packer/packer" "github.com/zclconf/go-cty/cty" ) @@ -180,7 +180,7 @@ func (cfg *PackerConfig) startProvisioner(source SourceUseBlock, pb *Provisioner }) return nil, diags } - metadata.GetMetadataStorage().AddPluginUsageMetadataFor(source.String(), pb.PType) + packer.MetadataStorage.AddPluginUsageMetadataFor(source.String(), pb.PType) builderVars := source.builderVariables() builderVars["packer_core_version"] = cfg.CorePackerVersionString diff --git a/hcl2template/types.source.go b/hcl2template/types.source.go index bf2c8c331c9..7a8d373eb23 100644 --- a/hcl2template/types.source.go +++ b/hcl2template/types.source.go @@ -12,7 +12,7 @@ import ( "github.com/hashicorp/hcl/v2/gohcl" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" hcl2shim "github.com/hashicorp/packer/hcl2template/shim" - "github.com/hashicorp/packer/metadata" + "github.com/hashicorp/packer/packer" "github.com/zclconf/go-cty/cty" ) @@ -113,7 +113,7 @@ func (cfg *PackerConfig) startBuilder(source SourceUseBlock, ectx *hcl.EvalConte }) return builder, diags, nil } - metadata.GetMetadataStorage().AddPluginUsageMetadataFor(source.String(), source.Type) + packer.MetadataStorage.AddPluginUsageMetadataFor(source.String(), source.Type) body := source.Body // Add known values to source accessor in eval context. diff --git a/metadata/all_plugins_storage.go b/packer/all_plugins_storage.go similarity index 52% rename from metadata/all_plugins_storage.go rename to packer/all_plugins_storage.go index f05f3ce202b..e9ea1f72d23 100644 --- a/metadata/all_plugins_storage.go +++ b/packer/all_plugins_storage.go @@ -1,8 +1,6 @@ -package metadata +package packer import ( - "sync" - pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin" ) @@ -11,32 +9,28 @@ type PluginDetails struct { Description pluginsdk.SetDescription } -type AllPluginsStorage struct { +type allPluginsStorage struct { Components map[string]*PluginDetails } var ( - allPluginsStorage *AllPluginsStorage - allPluginsStorageOnce sync.Once + AllPluginsStorage *allPluginsStorage ) -func GetAllPluginsStorage() *AllPluginsStorage { - allPluginsStorageOnce.Do(func() { - allPluginsStorage = &AllPluginsStorage{ - Components: map[string]*PluginDetails{}, - } - }) - return allPluginsStorage +func init() { + AllPluginsStorage = &allPluginsStorage{ + Components: map[string]*PluginDetails{}, + } } -func (aps *AllPluginsStorage) AddPluginDetails(componentKey, pluginName string, pluginDescription pluginsdk.SetDescription) { +func (aps *allPluginsStorage) AddPluginDetails(componentKey, pluginName string, pluginDescription pluginsdk.SetDescription) { aps.Components[componentKey] = &PluginDetails{ Name: pluginName, Description: pluginDescription, } } -func (aps *AllPluginsStorage) GetPluginDetailsFor(componentKey string) *PluginDetails { +func (aps *allPluginsStorage) GetPluginDetailsFor(componentKey string) *PluginDetails { pluginDetails, ok := aps.Components[componentKey] if !ok { return nil diff --git a/packer/core.go b/packer/core.go index 12cdbd940ac..b2fa8329d2e 100644 --- a/packer/core.go +++ b/packer/core.go @@ -22,7 +22,6 @@ import ( packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/template" "github.com/hashicorp/packer-plugin-sdk/template/interpolate" - "github.com/hashicorp/packer/metadata" plugingetter "github.com/hashicorp/packer/packer/plugin-getter" packerversion "github.com/hashicorp/packer/version" ) @@ -259,7 +258,7 @@ func (c *Core) generateCoreBuildProvisioner(rawP *template.Provisioner, rawName return cbp, fmt.Errorf( "provisioner failed to be started and did not error: %s", rawP.Type) } - metadata.GetMetadataStorage().AddPluginUsageMetadataFor(rawName, rawP.Type) + MetadataStorage.AddPluginUsageMetadataFor(rawName, rawP.Type) // Get the configuration config := make([]interface{}, 1, 2) @@ -395,7 +394,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { return nil, fmt.Errorf( "builder type not found: %s", configBuilder.Type) } - metadata.GetMetadataStorage().AddPluginUsageMetadataFor(n, configBuilder.Type) + MetadataStorage.AddPluginUsageMetadataFor(n, configBuilder.Type) // rawName is the uninterpolated name that we use for various lookups rawName := configBuilder.Name @@ -472,7 +471,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { return nil, fmt.Errorf( "post-processor type not found: %s", rawP.Type) } - metadata.GetMetadataStorage().AddPluginUsageMetadataFor(n, rawP.Type) + MetadataStorage.AddPluginUsageMetadataFor(n, rawP.Type) current = append(current, CoreBuildPostProcessor{ PostProcessor: postProcessor, @@ -507,7 +506,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { Variables: c.variables, } - //configBuilder.Name is left uninterpolated so we must check against + // configBuilder.Name is left uninterpolated so we must check against // the interpolated name. if configBuilder.Type != configBuilder.Name { cb.BuildName = configBuilder.Type diff --git a/metadata/metadata_storage.go b/packer/metadata_storage.go similarity index 74% rename from metadata/metadata_storage.go rename to packer/metadata_storage.go index d53aa67a63f..90cca906d54 100644 --- a/metadata/metadata_storage.go +++ b/packer/metadata_storage.go @@ -1,8 +1,7 @@ -package metadata +package packer import ( "fmt" - "sync" "github.com/hashicorp/packer/version" ) @@ -12,33 +11,29 @@ type PluginComponentUsedInBuild struct { ComponentKey string } -type MetadataStorage struct { +type metadataStorage struct { PluginComponentUsage []*PluginComponentUsedInBuild } var ( - metadataStorage *MetadataStorage - metadataStorageOnce sync.Once + MetadataStorage *metadataStorage ) -func GetMetadataStorage() *MetadataStorage { - metadataStorageOnce.Do(func() { - metadataStorage = &MetadataStorage{ - PluginComponentUsage: []*PluginComponentUsedInBuild{}, - } - }) - return metadataStorage +func init() { + MetadataStorage = &metadataStorage{ + PluginComponentUsage: []*PluginComponentUsedInBuild{}, + } } -func (ms *MetadataStorage) AddPluginUsageMetadataFor(buildName string, componentKey string) { +func (ms *metadataStorage) AddPluginUsageMetadataFor(buildName string, componentKey string) { ms.PluginComponentUsage = append(ms.PluginComponentUsage, &PluginComponentUsedInBuild{ BuildName: buildName, ComponentKey: componentKey, }) } -func (ms *MetadataStorage) GetMetadataByBuild() map[string]map[string]string { - pluginStorage := GetAllPluginsStorage() +func (ms *metadataStorage) GetMetadataByBuild() map[string]map[string]string { + pluginStorage := AllPluginsStorage resp := map[string]map[string]string{} fmt.Printf("<===> All Plugins: %q\n", pluginStorage.Components) @@ -79,33 +74,35 @@ type BuildsMetadata struct { PackerCoreMetadata PackerCoreMetadata } -/* Response structure: -{ - "PackerCoreMetadata": { - "Version": "1.10.1" - }, - "PluginsMetadata": { - "build1": [ - { - "Name": "docker", - "Version": "1.10.1" - } - ], - "build2": [ - { - "Name": "amazon", - "Version": "1.11.1" - }, - { - "Name": "docker", - "Version": "1.10.1" - } - ] +/* +Response structure: + + { + "PackerCoreMetadata": { + "Version": "1.10.1" + }, + "PluginsMetadata": { + "build1": [ + { + "Name": "docker", + "Version": "1.10.1" + } + ], + "build2": [ + { + "Name": "amazon", + "Version": "1.11.1" + }, + { + "Name": "docker", + "Version": "1.10.1" + } + ] + } } -} */ -func (ms *MetadataStorage) GetMetadata() *BuildsMetadata { - pluginStorage := GetAllPluginsStorage() +func (ms *metadataStorage) GetMetadata() *BuildsMetadata { + pluginStorage := AllPluginsStorage respMetadata := BuildsMetadata{ PackerCoreMetadata: PackerCoreMetadata{Version: version.FormattedVersion()}, diff --git a/packer/plugin.go b/packer/plugin.go index 5e702d52b15..dd7f1daea81 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -17,7 +17,6 @@ import ( packersdk "github.com/hashicorp/packer-plugin-sdk/packer" pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin" - "github.com/hashicorp/packer/metadata" plugingetter "github.com/hashicorp/packer/packer/plugin-getter" ) @@ -135,8 +134,6 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error return err } - pluginStorage := metadata.GetAllPluginsStorage() - pluginPrefix := pluginName + "-" for _, builderName := range desc.Builders { @@ -148,7 +145,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Builders.Set(key, func() (packersdk.Builder, error) { return c.Client(pluginPath, "start", "builder", builderName).Builder() }) - pluginStorage.AddPluginDetails(key, pluginName, desc) + AllPluginsStorage.AddPluginDetails(key, pluginName, desc) } if len(desc.Builders) > 0 { @@ -164,7 +161,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.PostProcessors.Set(key, func() (packersdk.PostProcessor, error) { return c.Client(pluginPath, "start", "post-processor", postProcessorName).PostProcessor() }) - pluginStorage.AddPluginDetails(key, pluginName, desc) + AllPluginsStorage.AddPluginDetails(key, pluginName, desc) } if len(desc.PostProcessors) > 0 { @@ -180,7 +177,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Provisioners.Set(key, func() (packersdk.Provisioner, error) { return c.Client(pluginPath, "start", "provisioner", provisionerName).Provisioner() }) - pluginStorage.AddPluginDetails(key, pluginName, desc) + AllPluginsStorage.AddPluginDetails(key, pluginName, desc) } if len(desc.Provisioners) > 0 { log.Printf("found external %v provisioner from %s plugin", desc.Provisioners, pluginName) From ac4400096fa83be7912b958c8a09b298ac933d6d Mon Sep 17 00:00:00 2001 From: Devashish Date: Wed, 28 Feb 2024 18:26:01 -0500 Subject: [PATCH 23/32] Track plugin metadata on the CoreBuild level --- command/build.go | 6 +- command/validate.go | 3 - hcl2template/types.build.post-processor.go | 2 - hcl2template/types.build.provisioners.go | 2 - hcl2template/types.packer_config.go | 1 + hcl2template/types.source.go | 2 - internal/hcp/registry/hcl.go | 12 +- internal/hcp/registry/json.go | 16 ++- internal/hcp/registry/null_registry.go | 1 - internal/hcp/registry/registry.go | 2 +- packer/all_plugins_storage.go | 5 +- packer/build.go | 27 ++++ packer/core.go | 3 - packer/metadata_storage.go | 136 --------------------- packer/plugin.go | 6 +- 15 files changed, 53 insertions(+), 171 deletions(-) delete mode 100644 packer/metadata_storage.go diff --git a/command/build.go b/command/build.go index ec2076e4f76..b48d248d50c 100644 --- a/command/build.go +++ b/command/build.go @@ -135,9 +135,6 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int OnError: cla.OnError, }) - metadataByBuild := packer.MetadataStorage.GetMetadataByBuild() - fmt.Printf("<===> Metadata = %q", metadataByBuild) - // here, something could have gone wrong but we still want to run valid // builds. ret = writeDiags(c.Ui, nil, diags) @@ -275,8 +272,7 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int buildCtx, b, runArtifacts, - err, - metadataByBuild) + err) if hcperr != nil { if _, ok := hcperr.(*registry.NotAHCPArtifactError); ok { writeDiags(c.Ui, nil, hcl.Diagnostics{ diff --git a/command/validate.go b/command/validate.go index 58468da2e7c..da41e30cf0a 100644 --- a/command/validate.go +++ b/command/validate.go @@ -5,7 +5,6 @@ package command import ( "context" - "fmt" "strings" "github.com/hashicorp/packer/packer" @@ -89,8 +88,6 @@ func (c *ValidateCommand) RunContext(ctx context.Context, cla *ValidateArgs) int Only: cla.Only, Except: cla.Except, }) - metadataByBuild := packer.MetadataStorage.GetMetadataByBuild() - fmt.Printf("<===> Metadata = %q", metadataByBuild) fixerDiags := packerStarter.FixConfig(packer.FixConfigOptions{ Mode: packer.Diff, diff --git a/hcl2template/types.build.post-processor.go b/hcl2template/types.build.post-processor.go index bbb9bda2516..8844eadff11 100644 --- a/hcl2template/types.build.post-processor.go +++ b/hcl2template/types.build.post-processor.go @@ -10,7 +10,6 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" - "github.com/hashicorp/packer/packer" ) // ProvisionerBlock references a detected but unparsed post processor @@ -71,7 +70,6 @@ func (cfg *PackerConfig) startPostProcessor(source SourceUseBlock, pp *PostProce }) return nil, diags } - packer.MetadataStorage.AddPluginUsageMetadataFor(source.String(), pp.PType) builderVars := source.builderVariables() builderVars["packer_core_version"] = cfg.CorePackerVersionString diff --git a/hcl2template/types.build.provisioners.go b/hcl2template/types.build.provisioners.go index a63cca8bcd3..b08eca59f63 100644 --- a/hcl2template/types.build.provisioners.go +++ b/hcl2template/types.build.provisioners.go @@ -12,7 +12,6 @@ import ( "github.com/hashicorp/hcl/v2/gohcl" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" hcl2shim "github.com/hashicorp/packer/hcl2template/shim" - "github.com/hashicorp/packer/packer" "github.com/zclconf/go-cty/cty" ) @@ -180,7 +179,6 @@ func (cfg *PackerConfig) startProvisioner(source SourceUseBlock, pb *Provisioner }) return nil, diags } - packer.MetadataStorage.AddPluginUsageMetadataFor(source.String(), pb.PType) builderVars := source.builderVariables() builderVars["packer_core_version"] = cfg.CorePackerVersionString diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 3ead0b6b5f1..d21d10edeae 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -622,6 +622,7 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packersdk.Bu decoded, _ := decodeHCL2Spec(srcUsage.Body, cfg.EvalContext(BuildContext, nil), builder) pcb.HCLConfig = decoded + pcb.BuilderType = srcUsage.Type // If the builder has provided a list of to-be-generated variables that // should be made accessible to provisioners, pass that list into diff --git a/hcl2template/types.source.go b/hcl2template/types.source.go index 7a8d373eb23..46b9caac98e 100644 --- a/hcl2template/types.source.go +++ b/hcl2template/types.source.go @@ -12,7 +12,6 @@ import ( "github.com/hashicorp/hcl/v2/gohcl" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" hcl2shim "github.com/hashicorp/packer/hcl2template/shim" - "github.com/hashicorp/packer/packer" "github.com/zclconf/go-cty/cty" ) @@ -113,7 +112,6 @@ func (cfg *PackerConfig) startBuilder(source SourceUseBlock, ectx *hcl.EvalConte }) return builder, diags, nil } - packer.MetadataStorage.AddPluginUsageMetadataFor(source.String(), source.Type) body := source.Body // Add known values to source accessor in eval context. diff --git a/internal/hcp/registry/hcl.go b/internal/hcp/registry/hcl.go index 1b31e23e271..61b9a86bca9 100644 --- a/internal/hcp/registry/hcl.go +++ b/internal/hcp/registry/hcl.go @@ -79,18 +79,18 @@ func (h *HCLRegistry) CompleteBuild( build sdkpacker.Build, artifacts []sdkpacker.Artifact, buildErr error, - buildsMetadata map[string]map[string]string, ) ([]sdkpacker.Artifact, error) { name := build.Name() cb, ok := build.(*packer.CoreBuild) if ok { name = cb.Type } - buildMetadata, ok := buildsMetadata[name] - if !ok { - fmt.Printf("[METADATA] HCL Metadata for build name %q: MISSING\n", name) - } else { - fmt.Printf("[METADATA] HCL Metadata for build name %q: %q\n", name, buildMetadata) + + for k, pluginDetails := range cb.GetPluginsMetadata() { + fmt.Printf( + "[METADATA] HCL Metadata for build name %q: %q -- %q\n", + name, k, pluginDetails.Description.Version, + ) } return h.bucket.completeBuild(ctx, name, artifacts, buildErr) } diff --git a/internal/hcp/registry/json.go b/internal/hcp/registry/json.go index 721f630d768..bf9ed0e7c0a 100644 --- a/internal/hcp/registry/json.go +++ b/internal/hcp/registry/json.go @@ -92,14 +92,18 @@ func (h *JSONRegistry) CompleteBuild( build sdkpacker.Build, artifacts []sdkpacker.Artifact, buildErr error, - buildsMetadata map[string]map[string]string, ) ([]sdkpacker.Artifact, error) { name := build.Name() - buildMetadata, ok := buildsMetadata[name] - if !ok { - fmt.Printf("[METADATA] JSON Metadata for build name %q: MISSING\n", name) - } else { - fmt.Printf("[METADATA] JSON Metadata for build name %q: %q\n", name, buildMetadata) + pluginMetadata := map[string]packer.PluginDetails{} + coreBuild, ok := build.(*packer.CoreBuild) + if ok { + pluginMetadata = coreBuild.GetPluginsMetadata() + } + for k, pluginDetails := range pluginMetadata { + fmt.Printf( + "[METADATA] JSON Metadata for build name %q: %q -- %q\n", + name, k, pluginDetails.Description.Version, + ) } return h.bucket.completeBuild(ctx, name, artifacts, buildErr) } diff --git a/internal/hcp/registry/null_registry.go b/internal/hcp/registry/null_registry.go index 515461fc70f..4f32c3d12b3 100644 --- a/internal/hcp/registry/null_registry.go +++ b/internal/hcp/registry/null_registry.go @@ -25,7 +25,6 @@ func (r nullRegistry) CompleteBuild( build sdkpacker.Build, artifacts []sdkpacker.Artifact, buildErr error, - buildsMetadata map[string]map[string]string, ) ([]sdkpacker.Artifact, error) { return artifacts, nil } diff --git a/internal/hcp/registry/registry.go b/internal/hcp/registry/registry.go index 6c51e1a42de..56b70285d41 100644 --- a/internal/hcp/registry/registry.go +++ b/internal/hcp/registry/registry.go @@ -17,7 +17,7 @@ import ( type Registry interface { PopulateVersion(context.Context) error StartBuild(context.Context, sdkpacker.Build) error - CompleteBuild(ctx context.Context, build sdkpacker.Build, artifacts []sdkpacker.Artifact, buildErr error, buildsMetadata map[string]map[string]string) ([]sdkpacker.Artifact, error) + CompleteBuild(ctx context.Context, build sdkpacker.Build, artifacts []sdkpacker.Artifact, buildErr error) ([]sdkpacker.Artifact, error) VersionStatusSummary() } diff --git a/packer/all_plugins_storage.go b/packer/all_plugins_storage.go index e9ea1f72d23..93731db15c4 100644 --- a/packer/all_plugins_storage.go +++ b/packer/all_plugins_storage.go @@ -7,6 +7,7 @@ import ( type PluginDetails struct { Name string Description pluginsdk.SetDescription + PluginPath string } type allPluginsStorage struct { @@ -23,10 +24,12 @@ func init() { } } -func (aps *allPluginsStorage) AddPluginDetails(componentKey, pluginName string, pluginDescription pluginsdk.SetDescription) { +func (aps *allPluginsStorage) AddPluginDetails( + componentKey, pluginName, pluginPath string, pluginDescription pluginsdk.SetDescription) { aps.Components[componentKey] = &PluginDetails{ Name: pluginName, Description: pluginDescription, + PluginPath: pluginPath, } } diff --git a/packer/build.go b/packer/build.go index e51e71d00c9..ee04d16472f 100644 --- a/packer/build.go +++ b/packer/build.go @@ -52,6 +52,33 @@ type CoreBuild struct { prepareCalled bool } +func (b *CoreBuild) GetPluginsMetadata() map[string]PluginDetails { + resp := map[string]PluginDetails{} + + builderPlugin := AllPluginsStorage.GetPluginDetailsFor(b.BuilderType) + if builderPlugin != nil { + resp[builderPlugin.Name] = *builderPlugin + } + + for _, pv := range b.Provisioners { + provisionerPlugin := AllPluginsStorage.GetPluginDetailsFor(pv.PType) + if provisionerPlugin != nil { + resp[provisionerPlugin.Name] = *provisionerPlugin + } + + } + + for _, pp := range b.PostProcessors { + for _, p := range pp { + postprocessorsPlugin := AllPluginsStorage.GetPluginDetailsFor(p.PType) + if postprocessorsPlugin != nil { + resp[postprocessorsPlugin.Name] = *postprocessorsPlugin + } + } + } + return resp +} + // CoreBuildPostProcessor Keeps track of the post-processor and the // configuration of the post-processor used within a build. type CoreBuildPostProcessor struct { diff --git a/packer/core.go b/packer/core.go index b2fa8329d2e..d6d3406a044 100644 --- a/packer/core.go +++ b/packer/core.go @@ -258,7 +258,6 @@ func (c *Core) generateCoreBuildProvisioner(rawP *template.Provisioner, rawName return cbp, fmt.Errorf( "provisioner failed to be started and did not error: %s", rawP.Type) } - MetadataStorage.AddPluginUsageMetadataFor(rawName, rawP.Type) // Get the configuration config := make([]interface{}, 1, 2) @@ -394,7 +393,6 @@ func (c *Core) Build(n string) (packersdk.Build, error) { return nil, fmt.Errorf( "builder type not found: %s", configBuilder.Type) } - MetadataStorage.AddPluginUsageMetadataFor(n, configBuilder.Type) // rawName is the uninterpolated name that we use for various lookups rawName := configBuilder.Name @@ -471,7 +469,6 @@ func (c *Core) Build(n string) (packersdk.Build, error) { return nil, fmt.Errorf( "post-processor type not found: %s", rawP.Type) } - MetadataStorage.AddPluginUsageMetadataFor(n, rawP.Type) current = append(current, CoreBuildPostProcessor{ PostProcessor: postProcessor, diff --git a/packer/metadata_storage.go b/packer/metadata_storage.go deleted file mode 100644 index 90cca906d54..00000000000 --- a/packer/metadata_storage.go +++ /dev/null @@ -1,136 +0,0 @@ -package packer - -import ( - "fmt" - - "github.com/hashicorp/packer/version" -) - -type PluginComponentUsedInBuild struct { - BuildName string - ComponentKey string -} - -type metadataStorage struct { - PluginComponentUsage []*PluginComponentUsedInBuild -} - -var ( - MetadataStorage *metadataStorage -) - -func init() { - MetadataStorage = &metadataStorage{ - PluginComponentUsage: []*PluginComponentUsedInBuild{}, - } -} - -func (ms *metadataStorage) AddPluginUsageMetadataFor(buildName string, componentKey string) { - ms.PluginComponentUsage = append(ms.PluginComponentUsage, &PluginComponentUsedInBuild{ - BuildName: buildName, - ComponentKey: componentKey, - }) -} - -func (ms *metadataStorage) GetMetadataByBuild() map[string]map[string]string { - pluginStorage := AllPluginsStorage - resp := map[string]map[string]string{} - - fmt.Printf("<===> All Plugins: %q\n", pluginStorage.Components) - fmt.Printf("<===> All Builds: %q\n", ms.PluginComponentUsage) - for _, pluginUsage := range ms.PluginComponentUsage { - pluginUsageByBuild, ok := resp[pluginUsage.BuildName] - if !ok { - pluginUsageByBuild = map[string]string{} - } - fmt.Printf("<===> Build Name: %q\n", pluginUsage.BuildName) - fmt.Printf("<===> Plugin Component: %q\n", pluginUsage.ComponentKey) - - compDetails := pluginStorage.GetPluginDetailsFor(pluginUsage.ComponentKey) - if compDetails == nil { - fmt.Printf("<===> Can not find component details. \n") - continue - } - - fmt.Printf("<===> Plugin Name: %q\n", compDetails.Name) - fmt.Printf("<===> Plugin Version: %q\n", compDetails.Description.Version) - pluginUsageByBuild[compDetails.Name] = compDetails.Description.Version - resp[pluginUsage.BuildName] = pluginUsageByBuild - } - return resp -} - -type PluginMetadata struct { - Name string - Version string -} - -type PackerCoreMetadata struct { - Version string -} - -type BuildsMetadata struct { - PluginsMetadata map[string][]PluginMetadata - PackerCoreMetadata PackerCoreMetadata -} - -/* -Response structure: - - { - "PackerCoreMetadata": { - "Version": "1.10.1" - }, - "PluginsMetadata": { - "build1": [ - { - "Name": "docker", - "Version": "1.10.1" - } - ], - "build2": [ - { - "Name": "amazon", - "Version": "1.11.1" - }, - { - "Name": "docker", - "Version": "1.10.1" - } - ] - } - } -*/ -func (ms *metadataStorage) GetMetadata() *BuildsMetadata { - pluginStorage := AllPluginsStorage - - respMetadata := BuildsMetadata{ - PackerCoreMetadata: PackerCoreMetadata{Version: version.FormattedVersion()}, - } - - for _, pluginComponentUsage := range ms.PluginComponentUsage { - pluginUsageByBuild, ok := respMetadata.PluginsMetadata[pluginComponentUsage.BuildName] - if !ok { - pluginUsageByBuild = []PluginMetadata{} - } - - fmt.Printf("<===> Build Name: %q\n", pluginComponentUsage.BuildName) - fmt.Printf("<===> Plugin Component: %q\n", pluginComponentUsage.ComponentKey) - - compDetails := pluginStorage.GetPluginDetailsFor(pluginComponentUsage.ComponentKey) - if compDetails == nil { - fmt.Printf("<===> Can not find component details. \n") - continue - } - - fmt.Printf("<===> Plugin Name: %q\n", compDetails.Name) - fmt.Printf("<===> Plugin Version: %q\n", compDetails.Description.Version) - pluginUsageByBuild = append(pluginUsageByBuild, PluginMetadata{ - Name: compDetails.Name, - Version: compDetails.Description.Version, - }) - respMetadata.PluginsMetadata[pluginComponentUsage.BuildName] = pluginUsageByBuild - } - - return &respMetadata -} diff --git a/packer/plugin.go b/packer/plugin.go index dd7f1daea81..73f5fe0ed4b 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -145,7 +145,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Builders.Set(key, func() (packersdk.Builder, error) { return c.Client(pluginPath, "start", "builder", builderName).Builder() }) - AllPluginsStorage.AddPluginDetails(key, pluginName, desc) + AllPluginsStorage.AddPluginDetails(key, pluginName, pluginPath, desc) } if len(desc.Builders) > 0 { @@ -161,7 +161,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.PostProcessors.Set(key, func() (packersdk.PostProcessor, error) { return c.Client(pluginPath, "start", "post-processor", postProcessorName).PostProcessor() }) - AllPluginsStorage.AddPluginDetails(key, pluginName, desc) + AllPluginsStorage.AddPluginDetails(key, pluginName, pluginPath, desc) } if len(desc.PostProcessors) > 0 { @@ -177,7 +177,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Provisioners.Set(key, func() (packersdk.Provisioner, error) { return c.Client(pluginPath, "start", "provisioner", provisionerName).Provisioner() }) - AllPluginsStorage.AddPluginDetails(key, pluginName, desc) + AllPluginsStorage.AddPluginDetails(key, pluginName, pluginPath, desc) } if len(desc.Provisioners) > 0 { log.Printf("found external %v provisioner from %s plugin", desc.Provisioners, pluginName) From 8f19797d1a3ed01e1b58ee82e9ec302a7fc22a7e Mon Sep 17 00:00:00 2001 From: Devashish Date: Wed, 28 Feb 2024 22:09:37 -0500 Subject: [PATCH 24/32] Store and look up by the component type to prevent duplication --- packer/all_plugins_storage.go | 21 +++++++++++++++++---- packer/build.go | 21 +++++++++++---------- packer/plugin.go | 6 +++--- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/packer/all_plugins_storage.go b/packer/all_plugins_storage.go index 93731db15c4..a09ee28fef0 100644 --- a/packer/all_plugins_storage.go +++ b/packer/all_plugins_storage.go @@ -1,9 +1,19 @@ package packer import ( + "fmt" + pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin" ) +type PluginComponentType string + +const ( + PluginComponentBuilder PluginComponentType = "builder" + PluginComponentPostProcessor PluginComponentType = "post-processor" + PluginComponentProvisioner PluginComponentType = "provisioner" +) + type PluginDetails struct { Name string Description pluginsdk.SetDescription @@ -25,16 +35,19 @@ func init() { } func (aps *allPluginsStorage) AddPluginDetails( - componentKey, pluginName, pluginPath string, pluginDescription pluginsdk.SetDescription) { - aps.Components[componentKey] = &PluginDetails{ + componentType PluginComponentType, componentKey, pluginName, pluginPath string, pluginDescription pluginsdk.SetDescription, +) { + key := fmt.Sprintf("%q-%q", componentKey, componentType) + aps.Components[key] = &PluginDetails{ Name: pluginName, Description: pluginDescription, PluginPath: pluginPath, } } -func (aps *allPluginsStorage) GetPluginDetailsFor(componentKey string) *PluginDetails { - pluginDetails, ok := aps.Components[componentKey] +func (aps *allPluginsStorage) GetPluginDetailsFor(componentType PluginComponentType, componentKey string) *PluginDetails { + key := fmt.Sprintf("%q-%q", componentKey, componentType) + pluginDetails, ok := aps.Components[key] if !ok { return nil } diff --git a/packer/build.go b/packer/build.go index ee04d16472f..4b82f046861 100644 --- a/packer/build.go +++ b/packer/build.go @@ -55,27 +55,28 @@ type CoreBuild struct { func (b *CoreBuild) GetPluginsMetadata() map[string]PluginDetails { resp := map[string]PluginDetails{} - builderPlugin := AllPluginsStorage.GetPluginDetailsFor(b.BuilderType) + builderPlugin := AllPluginsStorage.GetPluginDetailsFor(PluginComponentBuilder, b.BuilderType) if builderPlugin != nil { resp[builderPlugin.Name] = *builderPlugin } - for _, pv := range b.Provisioners { - provisionerPlugin := AllPluginsStorage.GetPluginDetailsFor(pv.PType) - if provisionerPlugin != nil { - resp[provisionerPlugin.Name] = *provisionerPlugin - } - - } - for _, pp := range b.PostProcessors { for _, p := range pp { - postprocessorsPlugin := AllPluginsStorage.GetPluginDetailsFor(p.PType) + postprocessorsPlugin := AllPluginsStorage.GetPluginDetailsFor(PluginComponentPostProcessor, p.PType) if postprocessorsPlugin != nil { resp[postprocessorsPlugin.Name] = *postprocessorsPlugin } } } + + for _, pv := range b.Provisioners { + provisionerPlugin := AllPluginsStorage.GetPluginDetailsFor(PluginComponentProvisioner, pv.PType) + if provisionerPlugin != nil { + resp[provisionerPlugin.Name] = *provisionerPlugin + } + + } + return resp } diff --git a/packer/plugin.go b/packer/plugin.go index 73f5fe0ed4b..dec9a4df13d 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -145,7 +145,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Builders.Set(key, func() (packersdk.Builder, error) { return c.Client(pluginPath, "start", "builder", builderName).Builder() }) - AllPluginsStorage.AddPluginDetails(key, pluginName, pluginPath, desc) + AllPluginsStorage.AddPluginDetails(PluginComponentBuilder, key, pluginName, pluginPath, desc) } if len(desc.Builders) > 0 { @@ -161,7 +161,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.PostProcessors.Set(key, func() (packersdk.PostProcessor, error) { return c.Client(pluginPath, "start", "post-processor", postProcessorName).PostProcessor() }) - AllPluginsStorage.AddPluginDetails(key, pluginName, pluginPath, desc) + AllPluginsStorage.AddPluginDetails(PluginComponentPostProcessor, key, pluginName, pluginPath, desc) } if len(desc.PostProcessors) > 0 { @@ -177,7 +177,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Provisioners.Set(key, func() (packersdk.Provisioner, error) { return c.Client(pluginPath, "start", "provisioner", provisionerName).Provisioner() }) - AllPluginsStorage.AddPluginDetails(key, pluginName, pluginPath, desc) + AllPluginsStorage.AddPluginDetails(PluginComponentProvisioner, key, pluginName, pluginPath, desc) } if len(desc.Provisioners) > 0 { log.Printf("found external %v provisioner from %s plugin", desc.Provisioners, pluginName) From 7343a37ccb699363569dbf64f708adc467998db0 Mon Sep 17 00:00:00 2001 From: Devashish Date: Wed, 28 Feb 2024 22:23:54 -0500 Subject: [PATCH 25/32] Fix tests --- .../types.build.hcp_packer_registry_test.go | 3 ++ hcl2template/types.build_test.go | 34 ++++++++++++------- hcl2template/types.datasource_test.go | 2 ++ hcl2template/types.packer_config_test.go | 10 +++--- hcl2template/types.source_test.go | 1 + hcl2template/types.variables_test.go | 13 +++++-- 6 files changed, 43 insertions(+), 20 deletions(-) diff --git a/hcl2template/types.build.hcp_packer_registry_test.go b/hcl2template/types.build.hcp_packer_registry_test.go index b27b80e5b34..b43566d3eba 100644 --- a/hcl2template/types.build.hcp_packer_registry_test.go +++ b/hcl2template/types.build.hcp_packer_registry_test.go @@ -57,6 +57,7 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) { Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, + BuilderType: "virtualbox-iso", }, }, false, @@ -110,6 +111,7 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) { Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, + BuilderType: "virtualbox-iso", }, }, false, @@ -241,6 +243,7 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) { Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, Prepared: true, + BuilderType: "null", }, }, false, diff --git a/hcl2template/types.build_test.go b/hcl2template/types.build_test.go index 8647821dd18..d94dee4f82f 100644 --- a/hcl2template/types.build_test.go +++ b/hcl2template/types.build_test.go @@ -283,6 +283,7 @@ func TestParse_build(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", Prepared: true, Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, @@ -317,6 +318,7 @@ func TestParse_build(t *testing.T) { }, &packer.CoreBuild{ Type: "amazon-ebs.aws-ubuntu-16.04", + BuilderType: "amazon-ebs", Prepared: true, Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, @@ -397,9 +399,10 @@ func TestParse_build(t *testing.T) { false, false, []packersdk.Build{ &packer.CoreBuild{ - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, - Builder: emptyMockBuilder, + Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", + Prepared: true, + Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { PType: "shell", @@ -427,9 +430,10 @@ func TestParse_build(t *testing.T) { PostProcessors: [][]packer.CoreBuildPostProcessor{}, }, &packer.CoreBuild{ - Type: "amazon-ebs.aws-ubuntu-16.04", - Prepared: true, - Builder: emptyMockBuilder, + Type: "amazon-ebs.aws-ubuntu-16.04", + BuilderType: "amazon-ebs", + Prepared: true, + Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { PType: "file", @@ -486,9 +490,10 @@ func TestParse_build(t *testing.T) { false, false, []packersdk.Build{ &packer.CoreBuild{ - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, - Builder: emptyMockBuilder, + Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", + Prepared: true, + Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { PType: "shell", @@ -550,6 +555,7 @@ func TestParse_build(t *testing.T) { &packer.CoreBuild{ BuildName: "build-name", Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", Prepared: true, Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, @@ -604,6 +610,7 @@ func TestParse_build(t *testing.T) { &packer.CoreBuild{ BuildName: "test-build", Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", Prepared: true, Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{}, @@ -659,10 +666,11 @@ func TestParse_build(t *testing.T) { false, false, []packersdk.Build{ &packer.CoreBuild{ - BuildName: "build-name-test", - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, - Builder: emptyMockBuilder, + BuildName: "build-name-test", + Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", + Prepared: true, + Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { PName: "build-name-test", diff --git a/hcl2template/types.datasource_test.go b/hcl2template/types.datasource_test.go index d12fcafd076..69f6541389c 100644 --- a/hcl2template/types.datasource_test.go +++ b/hcl2template/types.datasource_test.go @@ -57,6 +57,7 @@ func TestParse_datasource(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -134,6 +135,7 @@ func TestParse_datasource(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, diff --git a/hcl2template/types.packer_config_test.go b/hcl2template/types.packer_config_test.go index 69bc2da2c1c..391899e9234 100644 --- a/hcl2template/types.packer_config_test.go +++ b/hcl2template/types.packer_config_test.go @@ -208,8 +208,9 @@ func TestParser_complete(t *testing.T) { false, false, []packersdk.Build{ &packer.CoreBuild{ - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, + Type: "virtualbox-iso.ubuntu-1204", + BuilderType: "virtualbox-iso", + Prepared: true, Builder: &MockBuilder{ Config: MockConfig{ NestedMockConfig: NestedMockConfig{ @@ -319,8 +320,9 @@ func TestParser_complete(t *testing.T) { }, }, &packer.CoreBuild{ - Type: "amazon-ebs.ubuntu-1604", - Prepared: true, + Type: "amazon-ebs.ubuntu-1604", + BuilderType: "amazon-ebs", + Prepared: true, Builder: &MockBuilder{ Config: MockConfig{ NestedMockConfig: NestedMockConfig{ diff --git a/hcl2template/types.source_test.go b/hcl2template/types.source_test.go index 47caeec5a75..e319163682d 100644 --- a/hcl2template/types.source_test.go +++ b/hcl2template/types.source_test.go @@ -55,6 +55,7 @@ func TestParse_source(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, diff --git a/hcl2template/types.variables_test.go b/hcl2template/types.variables_test.go index a19b80bd990..ecff3677ca2 100644 --- a/hcl2template/types.variables_test.go +++ b/hcl2template/types.variables_test.go @@ -129,6 +129,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -293,6 +294,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -379,6 +381,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -442,6 +445,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -483,6 +487,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, @@ -538,9 +543,10 @@ func TestParse_variables(t *testing.T) { }, false, false, []packersdk.Build{&packer.CoreBuild{ - Type: "null.null-builder", - Prepared: true, - Builder: &null.Builder{}, + Type: "null.null-builder", + BuilderType: "null", + Prepared: true, + Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{ { PType: "shell", @@ -627,6 +633,7 @@ func TestParse_variables(t *testing.T) { []packersdk.Build{ &packer.CoreBuild{ Type: "null.test", + BuilderType: "null", Builder: &null.Builder{}, Provisioners: []packer.CoreBuildProvisioner{}, PostProcessors: [][]packer.CoreBuildPostProcessor{}, From 24c673dbc174f173b7045a8cbb3b346fab8cb3af Mon Sep 17 00:00:00 2001 From: Devashish Date: Thu, 29 Feb 2024 14:42:35 -0500 Subject: [PATCH 26/32] Add PR suggestions --- packer/all_plugins_storage.go | 33 +++++---------------------------- packer/build.go | 18 +++++++++--------- packer/plugin.go | 16 +++++++++++++--- 3 files changed, 27 insertions(+), 40 deletions(-) diff --git a/packer/all_plugins_storage.go b/packer/all_plugins_storage.go index a09ee28fef0..41eeed08656 100644 --- a/packer/all_plugins_storage.go +++ b/packer/all_plugins_storage.go @@ -1,8 +1,6 @@ package packer import ( - "fmt" - pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin" ) @@ -12,6 +10,7 @@ const ( PluginComponentBuilder PluginComponentType = "builder" PluginComponentPostProcessor PluginComponentType = "post-processor" PluginComponentProvisioner PluginComponentType = "provisioner" + PluginComponentDataSource PluginComponentType = "data-source" ) type PluginDetails struct { @@ -20,36 +19,14 @@ type PluginDetails struct { PluginPath string } -type allPluginsStorage struct { - Components map[string]*PluginDetails -} - -var ( - AllPluginsStorage *allPluginsStorage -) - -func init() { - AllPluginsStorage = &allPluginsStorage{ - Components: map[string]*PluginDetails{}, - } -} +var PluginsDetailsStorage = map[string]PluginDetails{} -func (aps *allPluginsStorage) AddPluginDetails( - componentType PluginComponentType, componentKey, pluginName, pluginPath string, pluginDescription pluginsdk.SetDescription, +func AddPluginDetails( + componentKey, pluginName, pluginPath string, pluginDescription pluginsdk.SetDescription, ) { - key := fmt.Sprintf("%q-%q", componentKey, componentType) - aps.Components[key] = &PluginDetails{ + PluginsDetailsStorage[componentKey] = PluginDetails{ Name: pluginName, Description: pluginDescription, PluginPath: pluginPath, } } - -func (aps *allPluginsStorage) GetPluginDetailsFor(componentType PluginComponentType, componentKey string) *PluginDetails { - key := fmt.Sprintf("%q-%q", componentKey, componentType) - pluginDetails, ok := aps.Components[key] - if !ok { - return nil - } - return pluginDetails -} diff --git a/packer/build.go b/packer/build.go index 4b82f046861..a63d7696243 100644 --- a/packer/build.go +++ b/packer/build.go @@ -55,24 +55,24 @@ type CoreBuild struct { func (b *CoreBuild) GetPluginsMetadata() map[string]PluginDetails { resp := map[string]PluginDetails{} - builderPlugin := AllPluginsStorage.GetPluginDetailsFor(PluginComponentBuilder, b.BuilderType) - if builderPlugin != nil { - resp[builderPlugin.Name] = *builderPlugin + builderPlugin, builderPluginOk := PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentBuilder, b.BuilderType)] + if builderPluginOk { + resp[builderPlugin.Name] = builderPlugin } for _, pp := range b.PostProcessors { for _, p := range pp { - postprocessorsPlugin := AllPluginsStorage.GetPluginDetailsFor(PluginComponentPostProcessor, p.PType) - if postprocessorsPlugin != nil { - resp[postprocessorsPlugin.Name] = *postprocessorsPlugin + postprocessorsPlugin, postprocessorsPluginOk := PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentPostProcessor, p.PType)] + if postprocessorsPluginOk { + resp[postprocessorsPlugin.Name] = postprocessorsPlugin } } } for _, pv := range b.Provisioners { - provisionerPlugin := AllPluginsStorage.GetPluginDetailsFor(PluginComponentProvisioner, pv.PType) - if provisionerPlugin != nil { - resp[provisionerPlugin.Name] = *provisionerPlugin + provisionerPlugin, provisionerPluginOk := PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentProvisioner, pv.PType)] + if provisionerPluginOk { + resp[provisionerPlugin.Name] = provisionerPlugin } } diff --git a/packer/plugin.go b/packer/plugin.go index dec9a4df13d..fe78dfc5d9b 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -6,6 +6,7 @@ package packer import ( "crypto/sha256" "encoding/json" + "fmt" "log" "os" "os/exec" @@ -145,7 +146,9 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Builders.Set(key, func() (packersdk.Builder, error) { return c.Client(pluginPath, "start", "builder", builderName).Builder() }) - AllPluginsStorage.AddPluginDetails(PluginComponentBuilder, key, pluginName, pluginPath, desc) + AddPluginDetails( + fmt.Sprintf("%q-%q", PluginComponentBuilder, key), pluginName, pluginPath, desc, + ) } if len(desc.Builders) > 0 { @@ -161,7 +164,9 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.PostProcessors.Set(key, func() (packersdk.PostProcessor, error) { return c.Client(pluginPath, "start", "post-processor", postProcessorName).PostProcessor() }) - AllPluginsStorage.AddPluginDetails(PluginComponentPostProcessor, key, pluginName, pluginPath, desc) + AddPluginDetails( + fmt.Sprintf("%q-%q", PluginComponentPostProcessor, key), pluginName, pluginPath, desc, + ) } if len(desc.PostProcessors) > 0 { @@ -177,7 +182,9 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Provisioners.Set(key, func() (packersdk.Provisioner, error) { return c.Client(pluginPath, "start", "provisioner", provisionerName).Provisioner() }) - AllPluginsStorage.AddPluginDetails(PluginComponentProvisioner, key, pluginName, pluginPath, desc) + AddPluginDetails( + fmt.Sprintf("%q-%q", PluginComponentProvisioner, key), pluginName, pluginPath, desc, + ) } if len(desc.Provisioners) > 0 { log.Printf("found external %v provisioner from %s plugin", desc.Provisioners, pluginName) @@ -192,6 +199,9 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.DataSources.Set(key, func() (packersdk.Datasource, error) { return c.Client(pluginPath, "start", "datasource", datasourceName).Datasource() }) + AddPluginDetails( + fmt.Sprintf("%q-%q", PluginComponentDataSource, key), pluginName, pluginPath, desc, + ) } if len(desc.Datasources) > 0 { log.Printf("found external %v datasource from %s plugin", desc.Datasources, pluginName) From b6c0b397b5795cf619c2725add9b636f716aa67a Mon Sep 17 00:00:00 2001 From: Devashish Date: Thu, 29 Feb 2024 14:45:55 -0500 Subject: [PATCH 27/32] Some more changes --- internal/hcp/registry/json.go | 7 ++----- packer/build.go | 1 - 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/internal/hcp/registry/json.go b/internal/hcp/registry/json.go index bf9ed0e7c0a..9862788297e 100644 --- a/internal/hcp/registry/json.go +++ b/internal/hcp/registry/json.go @@ -94,11 +94,8 @@ func (h *JSONRegistry) CompleteBuild( buildErr error, ) ([]sdkpacker.Artifact, error) { name := build.Name() - pluginMetadata := map[string]packer.PluginDetails{} - coreBuild, ok := build.(*packer.CoreBuild) - if ok { - pluginMetadata = coreBuild.GetPluginsMetadata() - } + pluginMetadata := build.(*packer.CoreBuild).GetPluginsMetadata() + for k, pluginDetails := range pluginMetadata { fmt.Printf( "[METADATA] JSON Metadata for build name %q: %q -- %q\n", diff --git a/packer/build.go b/packer/build.go index a63d7696243..32828f37930 100644 --- a/packer/build.go +++ b/packer/build.go @@ -74,7 +74,6 @@ func (b *CoreBuild) GetPluginsMetadata() map[string]PluginDetails { if provisionerPluginOk { resp[provisionerPlugin.Name] = provisionerPlugin } - } return resp From 81bd45449d20751844af3281066d34d2c70c6959 Mon Sep 17 00:00:00 2001 From: Devashish Date: Thu, 29 Feb 2024 18:43:25 -0500 Subject: [PATCH 28/32] Code clean up --- packer/all_plugins_storage.go | 32 ---------------------- packer/build.go | 13 +++++++++ packer/core.go | 2 +- packer/plugin.go | 51 ++++++++++++++++++++++++++--------- 4 files changed, 53 insertions(+), 45 deletions(-) delete mode 100644 packer/all_plugins_storage.go diff --git a/packer/all_plugins_storage.go b/packer/all_plugins_storage.go deleted file mode 100644 index 41eeed08656..00000000000 --- a/packer/all_plugins_storage.go +++ /dev/null @@ -1,32 +0,0 @@ -package packer - -import ( - pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin" -) - -type PluginComponentType string - -const ( - PluginComponentBuilder PluginComponentType = "builder" - PluginComponentPostProcessor PluginComponentType = "post-processor" - PluginComponentProvisioner PluginComponentType = "provisioner" - PluginComponentDataSource PluginComponentType = "data-source" -) - -type PluginDetails struct { - Name string - Description pluginsdk.SetDescription - PluginPath string -} - -var PluginsDetailsStorage = map[string]PluginDetails{} - -func AddPluginDetails( - componentKey, pluginName, pluginPath string, pluginDescription pluginsdk.SetDescription, -) { - PluginsDetailsStorage[componentKey] = PluginDetails{ - Name: pluginName, - Description: pluginDescription, - PluginPath: pluginPath, - } -} diff --git a/packer/build.go b/packer/build.go index 32828f37930..6e5f5696bff 100644 --- a/packer/build.go +++ b/packer/build.go @@ -52,6 +52,11 @@ type CoreBuild struct { prepareCalled bool } +type BuildMetadata struct { + PackerVersion string + Plugins map[string]PluginDetails +} + func (b *CoreBuild) GetPluginsMetadata() map[string]PluginDetails { resp := map[string]PluginDetails{} @@ -79,6 +84,14 @@ func (b *CoreBuild) GetPluginsMetadata() map[string]PluginDetails { return resp } +func (b *CoreBuild) GetMetadata() BuildMetadata { + metadata := BuildMetadata{ + PackerVersion: version.FormattedVersion(), + Plugins: b.GetPluginsMetadata(), + } + return metadata +} + // CoreBuildPostProcessor Keeps track of the post-processor and the // configuration of the post-processor used within a build. type CoreBuildPostProcessor struct { diff --git a/packer/core.go b/packer/core.go index d6d3406a044..5ef67d4ecf3 100644 --- a/packer/core.go +++ b/packer/core.go @@ -17,7 +17,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-version" - "github.com/hashicorp/hcl/v2" + hcl "github.com/hashicorp/hcl/v2" "github.com/hashicorp/packer-plugin-sdk/didyoumean" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/template" diff --git a/packer/plugin.go b/packer/plugin.go index fe78dfc5d9b..f597fb13cb3 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -146,9 +146,12 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Builders.Set(key, func() (packersdk.Builder, error) { return c.Client(pluginPath, "start", "builder", builderName).Builder() }) - AddPluginDetails( - fmt.Sprintf("%q-%q", PluginComponentBuilder, key), pluginName, pluginPath, desc, - ) + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentBuilder, key)] = PluginDetails{ + Name: pluginName, + Description: desc, + PluginPath: pluginPath, + } + } if len(desc.Builders) > 0 { @@ -164,9 +167,11 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.PostProcessors.Set(key, func() (packersdk.PostProcessor, error) { return c.Client(pluginPath, "start", "post-processor", postProcessorName).PostProcessor() }) - AddPluginDetails( - fmt.Sprintf("%q-%q", PluginComponentPostProcessor, key), pluginName, pluginPath, desc, - ) + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentPostProcessor, key)] = PluginDetails{ + Name: pluginName, + Description: desc, + PluginPath: pluginPath, + } } if len(desc.PostProcessors) > 0 { @@ -182,9 +187,12 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Provisioners.Set(key, func() (packersdk.Provisioner, error) { return c.Client(pluginPath, "start", "provisioner", provisionerName).Provisioner() }) - AddPluginDetails( - fmt.Sprintf("%q-%q", PluginComponentProvisioner, key), pluginName, pluginPath, desc, - ) + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentProvisioner, key)] = PluginDetails{ + Name: pluginName, + Description: desc, + PluginPath: pluginPath, + } + } if len(desc.Provisioners) > 0 { log.Printf("found external %v provisioner from %s plugin", desc.Provisioners, pluginName) @@ -199,9 +207,11 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.DataSources.Set(key, func() (packersdk.Datasource, error) { return c.Client(pluginPath, "start", "datasource", datasourceName).Datasource() }) - AddPluginDetails( - fmt.Sprintf("%q-%q", PluginComponentDataSource, key), pluginName, pluginPath, desc, - ) + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentDataSource, key)] = PluginDetails{ + Name: pluginName, + Description: desc, + PluginPath: pluginPath, + } } if len(desc.Datasources) > 0 { log.Printf("found external %v datasource from %s plugin", desc.Datasources, pluginName) @@ -253,3 +263,20 @@ func (c *PluginConfig) Client(path string, args ...string) *PluginClient { config.MaxPort = c.PluginMaxPort return NewClient(&config) } + +type PluginComponentType string + +const ( + PluginComponentBuilder PluginComponentType = "builder" + PluginComponentPostProcessor PluginComponentType = "post-processor" + PluginComponentProvisioner PluginComponentType = "provisioner" + PluginComponentDataSource PluginComponentType = "data-source" +) + +type PluginDetails struct { + Name string + Description pluginsdk.SetDescription + PluginPath string +} + +var PluginsDetailsStorage = map[string]PluginDetails{} From 16f8fd55de900fd5d28c17bae664b3c50eb6e80e Mon Sep 17 00:00:00 2001 From: Devashish Date: Thu, 29 Feb 2024 18:45:20 -0500 Subject: [PATCH 29/32] Remove out of scope changes --- packer/core.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packer/core.go b/packer/core.go index 5ef67d4ecf3..0f3ea9c9c1e 100644 --- a/packer/core.go +++ b/packer/core.go @@ -15,8 +15,8 @@ import ( ttmp "text/template" "github.com/google/go-cmp/cmp" - "github.com/hashicorp/go-multierror" - "github.com/hashicorp/go-version" + multierror "github.com/hashicorp/go-multierror" + version "github.com/hashicorp/go-version" hcl "github.com/hashicorp/hcl/v2" "github.com/hashicorp/packer-plugin-sdk/didyoumean" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" @@ -503,7 +503,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { Variables: c.variables, } - // configBuilder.Name is left uninterpolated so we must check against + //configBuilder.Name is left uninterpolated so we must check against // the interpolated name. if configBuilder.Type != configBuilder.Name { cb.BuildName = configBuilder.Type From f45d550f79df657b3e267d9658f024f44b507e02 Mon Sep 17 00:00:00 2001 From: Devashish Date: Thu, 29 Feb 2024 18:52:51 -0500 Subject: [PATCH 30/32] Use GetMetadata method and change logs --- internal/hcp/registry/hcl.go | 11 ++++++++--- internal/hcp/registry/json.go | 12 ++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/internal/hcp/registry/hcl.go b/internal/hcp/registry/hcl.go index 61b9a86bca9..ce6fd68eaa6 100644 --- a/internal/hcp/registry/hcl.go +++ b/internal/hcp/registry/hcl.go @@ -86,9 +86,14 @@ func (h *HCLRegistry) CompleteBuild( name = cb.Type } - for k, pluginDetails := range cb.GetPluginsMetadata() { - fmt.Printf( - "[METADATA] HCL Metadata for build name %q: %q -- %q\n", + metadata := cb.GetMetadata() + log.Printf( + "[TRACE] HCL 'Packer Version' Metadata for build name %q: %q\n", + name, metadata.PackerVersion, + ) + for k, pluginDetails := range metadata.Plugins { + log.Printf( + "[TRACE] HCL 'Plugin' Metadata for build name %q: %q -- %q\n", name, k, pluginDetails.Description.Version, ) } diff --git a/internal/hcp/registry/json.go b/internal/hcp/registry/json.go index 9862788297e..b86cfd45273 100644 --- a/internal/hcp/registry/json.go +++ b/internal/hcp/registry/json.go @@ -94,11 +94,15 @@ func (h *JSONRegistry) CompleteBuild( buildErr error, ) ([]sdkpacker.Artifact, error) { name := build.Name() - pluginMetadata := build.(*packer.CoreBuild).GetPluginsMetadata() - for k, pluginDetails := range pluginMetadata { - fmt.Printf( - "[METADATA] JSON Metadata for build name %q: %q -- %q\n", + metadata := build.(*packer.CoreBuild).GetMetadata() + log.Printf( + "[TRACE] JSON 'Packer Version' Metadata for build name %q: %q\n", + name, metadata.PackerVersion, + ) + for k, pluginDetails := range metadata.Plugins { + log.Printf( + "[TRACE] JSON 'Plugin' Metadata for build name %q: %q -- %q\n", name, k, pluginDetails.Description.Version, ) } From acce10272756ee85c5d66fcf1006ff2cc489398e Mon Sep 17 00:00:00 2001 From: Devashish Date: Tue, 5 Mar 2024 11:59:54 -0500 Subject: [PATCH 31/32] Initialize PluginDetails struct only once and reuse it --- packer/plugin.go | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/packer/plugin.go b/packer/plugin.go index f597fb13cb3..ad91b0d8524 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -136,7 +136,12 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error } pluginPrefix := pluginName + "-" - + pluginDetails := PluginDetails{ + Name: pluginName, + Description: desc, + PluginPath: pluginPath, + } + for _, builderName := range desc.Builders { builderName := builderName // copy to avoid pointer overwrite issue key := pluginPrefix + builderName @@ -146,11 +151,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Builders.Set(key, func() (packersdk.Builder, error) { return c.Client(pluginPath, "start", "builder", builderName).Builder() }) - PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentBuilder, key)] = PluginDetails{ - Name: pluginName, - Description: desc, - PluginPath: pluginPath, - } + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentBuilder, key)] = pluginDetails } @@ -167,11 +168,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.PostProcessors.Set(key, func() (packersdk.PostProcessor, error) { return c.Client(pluginPath, "start", "post-processor", postProcessorName).PostProcessor() }) - PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentPostProcessor, key)] = PluginDetails{ - Name: pluginName, - Description: desc, - PluginPath: pluginPath, - } + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentPostProcessor, key)] = pluginDetails } if len(desc.PostProcessors) > 0 { @@ -187,11 +184,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.Provisioners.Set(key, func() (packersdk.Provisioner, error) { return c.Client(pluginPath, "start", "provisioner", provisionerName).Provisioner() }) - PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentProvisioner, key)] = PluginDetails{ - Name: pluginName, - Description: desc, - PluginPath: pluginPath, - } + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentProvisioner, key)] = pluginDetails } if len(desc.Provisioners) > 0 { @@ -207,11 +200,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error c.DataSources.Set(key, func() (packersdk.Datasource, error) { return c.Client(pluginPath, "start", "datasource", datasourceName).Datasource() }) - PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentDataSource, key)] = PluginDetails{ - Name: pluginName, - Description: desc, - PluginPath: pluginPath, - } + PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentDataSource, key)] = pluginDetails } if len(desc.Datasources) > 0 { log.Printf("found external %v datasource from %s plugin", desc.Datasources, pluginName) From 3b4b1db7b82c960a341eb70163e82c6f6d2dbf05 Mon Sep 17 00:00:00 2001 From: Devashish Date: Tue, 5 Mar 2024 12:05:31 -0500 Subject: [PATCH 32/32] Lint and make GetPluginsMetadata method private --- packer/build.go | 4 ++-- packer/plugin.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packer/build.go b/packer/build.go index 6e5f5696bff..9dca9610ac8 100644 --- a/packer/build.go +++ b/packer/build.go @@ -57,7 +57,7 @@ type BuildMetadata struct { Plugins map[string]PluginDetails } -func (b *CoreBuild) GetPluginsMetadata() map[string]PluginDetails { +func (b *CoreBuild) getPluginsMetadata() map[string]PluginDetails { resp := map[string]PluginDetails{} builderPlugin, builderPluginOk := PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentBuilder, b.BuilderType)] @@ -87,7 +87,7 @@ func (b *CoreBuild) GetPluginsMetadata() map[string]PluginDetails { func (b *CoreBuild) GetMetadata() BuildMetadata { metadata := BuildMetadata{ PackerVersion: version.FormattedVersion(), - Plugins: b.GetPluginsMetadata(), + Plugins: b.getPluginsMetadata(), } return metadata } diff --git a/packer/plugin.go b/packer/plugin.go index ad91b0d8524..513694f639f 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -141,7 +141,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error Description: desc, PluginPath: pluginPath, } - + for _, builderName := range desc.Builders { builderName := builderName // copy to avoid pointer overwrite issue key := pluginPrefix + builderName