feat(builder-indexes): cli flag to build Images map

cmd/root.go

@@ -144,7 +144,7 @@ func NewRootCmd() *RootCmd {
 	flags.StringVar(&rootOpts.ModuleDeviceName, "moduledevicename", rootOpts.ModuleDeviceName, "kernel module device name (the default is falco, so the device will be under /dev/falco*)")
 	flags.StringVar(&rootOpts.ModuleDriverName, "moduledrivername", rootOpts.ModuleDriverName, "kernel module driver name, i.e. the name you see when you check installed modules via lsmod")
 	flags.StringVar(&rootOpts.BuilderImage, "builderimage", rootOpts.BuilderImage, "docker image to be used to build the kernel module and eBPF probe. If not provided, an automatically selected image will be used.")
-	flags.StringSliceVar(&rootOpts.BuilderRepos, "builderrepo", rootOpts.BuilderRepos, "list of docker repositories in descending priority order, used to search for builder images. Default falcosecurity/driverkit will always be enforced as lowest priority repo. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit")
+	flags.StringSliceVar(&rootOpts.BuilderRepos, "builderrepo", rootOpts.BuilderRepos, "list of docker repositories or file (absolute path) containing builder images index with the format '<image>,<target>,<gcc-version>[,<gcc-version>,...]', in descending priority order. Used to search for builder images. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit --builderrepo /path/to/my/index.txt. Index file line exmaple: yourorg/driverkit-builder;any;4.9.0;5.0.0;6.0.0;8.0.0")
 	flags.StringVar(&rootOpts.GCCVersion, "gccversion", rootOpts.GCCVersion, "enforce a specific gcc version for the build")
 
 	flags.StringSliceVar(&rootOpts.KernelUrls, "kernelurls", nil, "list of kernel header urls (e.g. --kernelurls <URL1> --kernelurls <URL2> --kernelurls \"<URL3>,<URL4>\")")

cmd/root_options.go

@@ -2,13 +2,13 @@ package cmd
 
 import (
 	"fmt"
-
 	"github.com/creasty/defaults"
 	"github.com/falcosecurity/driverkit/pkg/driverbuilder/builder"
 	"github.com/falcosecurity/driverkit/pkg/kernelrelease"
 	"github.com/falcosecurity/driverkit/validate"
 	"github.com/go-playground/validator/v10"
 	logger "github.com/sirupsen/logrus"
+	"strings"
 )
 
 // OutputOptions wraps the two drivers that driverkit builds.
@@ -33,7 +33,7 @@ type RootOptions struct {
 	Target           string   `validate:"required,target" name:"target"`
 	KernelConfigData string   `validate:"omitempty,base64" name:"kernel config data"` // fixme > tag "name" does not seem to work when used at struct level, but works when used at inner level
 	BuilderImage     string   `validate:"omitempty,imagename" name:"builder image"`
-	BuilderRepos     []string `validate:"omitempty" name:"docker repositories to look for builder images"`
+	BuilderRepos     []string `validate:"omitempty" name:"docker repositories to look for builder images or absolute path pointing to a file container builder image index"`
 	GCCVersion       string   `validate:"omitempty,semvertolerant" name:"gcc version"`
 	KernelUrls       []string `name:"kernel header urls"`
 	Repo             RepoOptions
@@ -132,11 +132,21 @@ func (ro *RootOptions) toBuild() *builder.Build {
 		KernelUrls:       ro.KernelUrls,
 		RepoOrg:          ro.Repo.Org,
 		RepoName:         ro.Repo.Name,
+		Images:           make(builder.ImagesMap),
+	}
+
+	if len(build.BuilderRepos) == 0 {
+		build.BuilderRepos = append(build.BuilderRepos, "docker.io/falcosecurity/driverkit")
 	}
 
-	// Always append falcosecurity repo; Note: this is a prio first slice
-	// therefore, default falcosecurity repo has lowest prio.
-	build.BuilderRepos = append(build.BuilderRepos, "docker.io/falcosecurity/driverkit")
+	// loop over BuilderRepos to constuct the list ImagesListers based on the value of the builderRepo, if it's a local path, add FileImagesLister, otherwise add RepoImagesLister
+	for _, builderRepo := range build.BuilderRepos {
+		if strings.HasPrefix(builderRepo, "/") {
+			build.ImagesListers = append(build.ImagesListers, &builder.FileImagesLister{FilePath: builderRepo})
+		} else {
+			build.ImagesListers = append(build.ImagesListers, builder.NewRepoImagesLister(builderRepo, build))
+		}
+	}
 
 	// attempt the build in case it comes from an invalid config
 	kr := build.KernelReleaseFromBuildConfig()

cmd/testdata/templates/flags.txt

@@ -1,7 +1,7 @@
 Flags:
       --architecture string       target architecture for the built driver, one of {{ .Architectures }} (default "{{ .CurrentArch }}")
       --builderimage string       docker image to be used to build the kernel module and eBPF probe. If not provided, an automatically selected image will be used.
-      --builderrepo strings       list of docker repositories in descending priority order, used to search for builder images. Default falcosecurity/driverkit will always be enforced as lowest priority repo. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit
+      --builderrepo strings       list of docker repositories or file (absolute path) containing builder images index with the format '<image>,<target>,<gcc-version>[,<gcc-version>,...]', in descending priority order. Used to search for builder images. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit --builderrepo /path/to/my/index.txt. Index file line exmaple: yourorg/driverkit-builder;any;4.9.0;5.0.0;6.0.0;8.0.0
   -c, --config string             config file path (default $HOME/.driverkit.yaml if exists)
       --driverversion string      driver version as a git commit hash or as a git tag (default "master")
       --dryrun                    do not actually perform the action

docs/driverkit.md

@@ -11,7 +11,7 @@ driverkit
 ```
       --architecture string       target architecture for the built driver, one of [amd64,arm64] (default "amd64")
       --builderimage string       docker image to be used to build the kernel module and eBPF probe. If not provided, an automatically selected image will be used.
-      --builderrepo strings       list of docker repositories in descending priority order, used to search for builder images. Default falcosecurity/driverkit will always be enforced as lowest priority repo. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit
+      --builderrepo strings       list of docker repositories or file (absolute path) containing builder images index with the format '<image>,<target>,<gcc-version>[,<gcc-version>,...]', in descending priority order. Used to search for builder images. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit --builderrepo /path/to/my/index.txt. Index file line exmaple: yourorg/driverkit-builder;any;4.9.0;5.0.0;6.0.0;8.0.0
   -c, --config string             config file path (default $HOME/.driverkit.yaml if exists)
       --driverversion string      driver version as a git commit hash or as a git tag (default "master")
       --dryrun                    do not actually perform the action

docs/driverkit_docker.md

@@ -11,7 +11,7 @@ driverkit docker [flags]
 ```
       --architecture string       target architecture for the built driver, one of [amd64,arm64] (default "amd64")
       --builderimage string       docker image to be used to build the kernel module and eBPF probe. If not provided, an automatically selected image will be used.
-      --builderrepo strings       list of docker repositories in descending priority order, used to search for builder images. Default falcosecurity/driverkit will always be enforced as lowest priority repo. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit
+      --builderrepo strings       list of docker repositories or file (absolute path) containing builder images index with the format '<image>,<target>,<gcc-version>[,<gcc-version>,...]', in descending priority order. Used to search for builder images. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit --builderrepo /path/to/my/index.txt. Index file line exmaple: yourorg/driverkit-builder;any;4.9.0;5.0.0;6.0.0;8.0.0
   -c, --config string             config file path (default $HOME/.driverkit.yaml if exists)
       --driverversion string      driver version as a git commit hash or as a git tag (default "master")
       --dryrun                    do not actually perform the action

docs/driverkit_images.md

@@ -11,7 +11,7 @@ driverkit images [flags]
 ```
       --architecture string       target architecture for the built driver, one of [amd64,arm64] (default "amd64")
       --builderimage string       docker image to be used to build the kernel module and eBPF probe. If not provided, an automatically selected image will be used.
-      --builderrepo strings       list of docker repositories in descending priority order, used to search for builder images. Default falcosecurity/driverkit will always be enforced as lowest priority repo. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit
+      --builderrepo strings       list of docker repositories or file (absolute path) containing builder images index with the format '<image>,<target>,<gcc-version>[,<gcc-version>,...]', in descending priority order. Used to search for builder images. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit --builderrepo /path/to/my/index.txt. Index file line exmaple: yourorg/driverkit-builder;any;4.9.0;5.0.0;6.0.0;8.0.0
   -c, --config string             config file path (default $HOME/.driverkit.yaml if exists)
       --driverversion string      driver version as a git commit hash or as a git tag (default "master")
       --dryrun                    do not actually perform the action

docs/driverkit_kubernetes-in-cluster.md

@@ -11,7 +11,7 @@ driverkit kubernetes-in-cluster [flags]
 ```
       --architecture string        target architecture for the built driver, one of [amd64,arm64] (default "amd64")
       --builderimage string        docker image to be used to build the kernel module and eBPF probe. If not provided, an automatically selected image will be used.
-      --builderrepo strings        list of docker repositories in descending priority order, used to search for builder images. Default falcosecurity/driverkit will always be enforced as lowest priority repo. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit
+      --builderrepo strings        list of docker repositories or file (absolute path) containing builder images index with the format '<image>,<target>,<gcc-version>[,<gcc-version>,...]', in descending priority order. Used to search for builder images. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit --builderrepo /path/to/my/index.txt. Index file line exmaple: yourorg/driverkit-builder;any;4.9.0;5.0.0;6.0.0;8.0.0
   -c, --config string              config file path (default $HOME/.driverkit.yaml if exists)
       --driverversion string       driver version as a git commit hash or as a git tag (default "master")
       --dryrun                     do not actually perform the action

docs/driverkit_kubernetes.md

@@ -14,7 +14,7 @@ driverkit kubernetes [flags]
       --as-group stringArray           group to impersonate for the operation, this flag can be repeated to specify multiple groups
       --as-uid string                  uID to impersonate for the operation
       --builderimage string            docker image to be used to build the kernel module and eBPF probe. If not provided, an automatically selected image will be used.
-      --builderrepo strings            list of docker repositories in descending priority order, used to search for builder images. Default falcosecurity/driverkit will always be enforced as lowest priority repo. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit
+      --builderrepo strings            list of docker repositories or file (absolute path) containing builder images index with the format '<image>,<target>,<gcc-version>[,<gcc-version>,...]', in descending priority order. Used to search for builder images. eg: --builderrepo myorg/driverkit --builderrepo falcosecurity/driverkit --builderrepo /path/to/my/index.txt. Index file line exmaple: yourorg/driverkit-builder;any;4.9.0;5.0.0;6.0.0;8.0.0
       --cache-dir string               default cache directory (default "$HOME/.kube/cache")
       --certificate-authority string   path to a cert file for the certificate authority
       --client-certificate string      path to a client certificate file for TLS

pkg/driverbuilder/builder/build.go

@@ -19,6 +19,7 @@ type Build struct {
 	ModuleDeviceName string
 	BuilderImage     string
 	BuilderRepos     []string
+	ImagesListers    []ImagesLister
 	KernelUrls       []string
 	GCCVersion       string
 	RepoOrg          string

pkg/driverbuilder/builder/image.go

@@ -1,6 +1,7 @@
 package builder
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"github.com/blang/semver"
@@ -9,7 +10,9 @@ import (
 	"github.com/falcosecurity/driverkit/pkg/kernelrelease"
 	logger "github.com/sirupsen/logrus"
 	"log"
+	"os"
 	"regexp"
+	"strconv"
 	"strings"
 )
 
@@ -19,6 +22,19 @@ type Image struct {
 	Name       string
 }
 
+type ImagesLister interface {
+	LoadImages() []Image
+}
+
+type FileImagesLister struct {
+	file     *os.File
+	FilePath string
+}
+
+type RepoImagesLister struct {
+	repo string
+}
+
 type ImageKey string
 
 func (i *Image) toKey() ImageKey {
@@ -27,6 +43,8 @@ func (i *Image) toKey() ImageKey {
 
 type ImagesMap map[ImageKey]Image
 
+var repoRegs = make([]*regexp.Regexp, 0, 2)
+
 func (im ImagesMap) findImage(target Type, gccVers semver.Version) (Image, bool) {
 	targetImage := Image{
 		Target:     target,
@@ -45,78 +63,123 @@ func (im ImagesMap) findImage(target Type, gccVers semver.Version) (Image, bool)
 	return Image{}, false
 }
 
-func (b *Build) LoadImages() {
+func (f *FileImagesLister) LoadImages() []Image {
+	// loop over lines in file to print them
+	file, err := os.Open(f.FilePath)
+	if err != nil {
+		logger.WithError(err).WithField("FilePath", f.FilePath).Fatal("error opening builder repo file")
+	}
+	scanner := bufio.NewScanner(file)
+	var res []Image
+	for scanner.Scan() {
+		infos := strings.Split(scanner.Text(), ",")
+		if len(infos) < 3 {
+			logger.WithField("FilePath", f.FilePath).WithField("line", scanner.Text()).Fatal("Invalid image list file: expected at least 3 fields (name,target,gcc_version) but got " + strconv.Itoa(len(infos)) + ".")
+		}
+		name := infos[0]
+		target := Type(infos[1])
+		gccVersions := infos[2:]
+		for _, gcc := range gccVersions {
+			buildImage := Image{
+				Name:       name,
+				Target:     target,
+				GCCVersion: mustParseTolerant(gcc),
+			}
+			res = append(res, buildImage)
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		logger.WithField("file", file.Name()).WithError(err).Fatal()
+	}
+	err = file.Close()
+	if err != nil {
+		logger.WithField("file", file.Name()).WithError(err).Fatal()
+	}
+	return res
+}
+
+func NewRepoImagesLister(repo string, build *Build) *RepoImagesLister {
+	if len(repoRegs) == 0 {
+		// Create the proper regexes to load "any" and target-specific images for requested arch
+		arch := kernelrelease.Architecture(build.Architecture).ToNonDeb()
+		targetFmt := fmt.Sprintf("driverkit-builder-%s-%s(?P<gccVers>(_gcc[0-9]+.[0-9]+.[0-9]+)+)$", build.TargetType.String(), arch)
+		repoRegs = append(repoRegs, regexp.MustCompile(targetFmt))
+		genericFmt := fmt.Sprintf("driverkit-builder-any-%s(?P<gccVers>(_gcc[0-9]+.[0-9]+.[0-9]+)+)$", arch)
+		repoRegs = append(repoRegs, regexp.MustCompile(genericFmt))
+	}
+	return &RepoImagesLister{repo: repo}
+}
+
+func (repo *RepoImagesLister) LoadImages() []Image {
 	cli, err := client.NewClientWithOpts(client.FromEnv)
 	if err != nil {
 		log.Fatal(err)
 	}
+	imgs, err := cli.ImageSearch(context.Background(), repo.repo, types.ImageSearchOptions{Limit: 100})
+	if err != nil {
+		logger.WithField("Repository", repo.repo).WithError(err).Warnf("Skipping repo")
+		return []Image{}
+	}
+	var res []Image
+	for _, img := range imgs {
+		for regIdx, reg := range repoRegs {
+			match := reg.FindStringSubmatch(img.Name)
+			if len(match) == 0 {
+				continue
+			}
 
-	// Create the proper regexes to load "any" and target-specific images for requested arch
-	arch := kernelrelease.Architecture(b.Architecture).ToNonDeb()
-	regs := make([]*regexp.Regexp, 0)
-	targetFmt := fmt.Sprintf("driverkit-builder-%s-%s(?P<gccVers>(_gcc[0-9]+.[0-9]+.[0-9]+)+)$", b.TargetType.String(), arch)
-	regs = append(regs, regexp.MustCompile(targetFmt))
-	genericFmt := fmt.Sprintf("driverkit-builder-any-%s(?P<gccVers>(_gcc[0-9]+.[0-9]+.[0-9]+)+)$", arch)
-	regs = append(regs, regexp.MustCompile(genericFmt))
-
-	b.Images = make(ImagesMap)
-	for _, repo := range b.BuilderRepos {
-		imgs, err := cli.ImageSearch(context.Background(), repo, types.ImageSearchOptions{Limit: 100})
-		if err != nil {
-			logger.Warnf("Skipping repo %s: %s\n", repo, err.Error())
-			continue
-		}
-		for _, img := range imgs {
-			for regIdx, reg := range regs {
-				match := reg.FindStringSubmatch(img.Name)
-				if len(match) == 0 {
-					continue
-				}
-
-				var gccVers []string
-				for i, name := range reg.SubexpNames() {
-					if i > 0 && i <= len(match) {
-						switch name {
-						case "gccVers":
-							gccVers = strings.Split(match[i], "_gcc")
-							gccVers = gccVers[1:] // remove initial whitespace
-						}
+			var gccVers []string
+			for i, name := range reg.SubexpNames() {
+				if i > 0 && i <= len(match) {
+					switch name {
+					case "gccVers":
+						gccVers = strings.Split(match[i], "_gcc")
+						gccVers = gccVers[1:] // remove initial whitespace
 					}
 				}
+			}
 
-				if len(gccVers) == 0 {
-					logger.Debug("Malformed image name: ", img.Name, len(match))
-					continue
-				}
+			if len(gccVers) == 0 {
+				logger.Debug("Malformed image name: ", img.Name, len(match))
+				continue
+			}
 
-				// Note: we store "any" target images as "any",
-				// instead of adding them to the target,
-				// because we always prefer specific target images,
-				// and we cannot guarantee here that any subsequent docker repos
-				// does not provide a target-specific image that offers same gcc version
-				for _, gccVer := range gccVers {
-					// If user set a fixed gcc version, only load images that provide it.
-					if b.GCCVersion != "" && b.GCCVersion != gccVer {
-						continue
-					}
-					buildImage := Image{
-						GCCVersion: mustParseTolerant(gccVer),
-						Name:       img.Name,
-					}
-					if regIdx == 0 {
-						buildImage.Target = b.TargetType
-					} else {
-						buildImage.Target = Type("any")
-					}
-					// Skip if key already exists: we have a descending prio list of docker repos!
-					if _, ok := b.Images[buildImage.toKey()]; !ok {
-						b.Images[buildImage.toKey()] = buildImage
-					}
+			// Note: we store "any" target images as "any",
+			// instead of adding them to the target,
+			// because we always prefer specific target images,
+			// and we cannot guarantee here that any subsequent docker repos
+			// does not provide a target-specific image that offers same gcc version
+			for _, gccVer := range gccVers {
+				// If user set a fixed gcc version, only load images that provide it.
+				buildImage := Image{
+					GCCVersion: mustParseTolerant(gccVer),
+					Name:       img.Name,
+				}
+				if regIdx == 0 {
+					buildImage.Target = Type("target-placeholder")
+				} else {
+					buildImage.Target = Type("any")
 				}
+				res = append(res, buildImage)
 			}
 		}
 	}
-	if len(b.Images) == 0 {
-		log.Fatal("Could not load any builder image. Leaving.")
+	return res
+}
+
+func (b *Build) LoadImages() {
+	for _, imagesLister := range b.ImagesListers {
+		for _, image := range imagesLister.LoadImages() {
+			if b.GCCVersion != "" && b.GCCVersion != image.GCCVersion.String() {
+				continue
+			}
+			if image.Target == "target-placeholder" {
+				image.Target = b.TargetType
+			}
+			// Skip if key already exists: we have a descending prio list of docker repos!
+			if _, ok := b.Images[image.toKey()]; !ok {
+				b.Images[image.toKey()] = image
+			}
+		}
 	}
 }