feat: support platform override

Support setting `platform` for dependencies and also allow overriding
`platform` for a specific package.

Signed-off-by: Noel Georgi <[email protected]>

Dockerfile

@@ -1,8 +1,8 @@
-# syntax = docker/dockerfile-upstream:1.6.0-labs
+# syntax = docker/dockerfile-upstream:1.7.0-labs
 
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-03-05T05:52:53Z by kres latest.
+# Generated on 2024-03-06T10:32:46Z by kres latest.
 
 ARG TOOLCHAIN
 

Makefile

@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-03-05T05:52:53Z by kres latest.
+# Generated on 2024-03-06T10:32:46Z by kres latest.
 
 # common variables
 
@@ -14,15 +14,15 @@ WITH_RACE ?= false
 REGISTRY ?= ghcr.io
 USERNAME ?= siderolabs
 REGISTRY_AND_USERNAME ?= $(REGISTRY)/$(USERNAME)
-PROTOBUF_GO_VERSION ?= 1.32.0
+PROTOBUF_GO_VERSION ?= 1.33.0
 GRPC_GO_VERSION ?= 1.3.0
 GRPC_GATEWAY_VERSION ?= 2.19.1
 VTPROTOBUF_VERSION ?= 0.6.0
 DEEPCOPY_VERSION ?= v0.5.6
 GOLANGCILINT_VERSION ?= v1.56.2
 GOFUMPT_VERSION ?= v0.6.0
-GO_VERSION ?= 1.22.0
-GOIMPORTS_VERSION ?= v0.18.0
+GO_VERSION ?= 1.22.1
+GOIMPORTS_VERSION ?= v0.19.0
 GO_BUILDFLAGS ?=
 GO_LDFLAGS ?=
 CGO_ENABLED ?= 0

README.md

@@ -227,6 +227,8 @@ Additionally, hermetic text functions from [Sprig](http://masterminds.github.io/
 On the root level, following properties are available:
 
 - `name` (*str*, *required*): name of the package, also used to reference this package from other packages as dependency.
+- `platform` (*str*, *optional*): platform override for the build.
+  If not set, defaults to the platform of the build.
 - `variant` (*str*, *optional*): variant of the base image of the build.
   Two variants are available:
   - `alpine`: Alpine Linux 3.16 image with `bash` package pre-installed
@@ -270,6 +272,8 @@ Properties:
   Contents of the stage are poured into the build at the location specified with `to:` parameter.
 - `image` (*str*, *external dependency*): reference to the registry container image this package depends on.
   Contents of the image are poured into the build at the location specified with `to:` parameter.
+- `platform` (*str*, *optional*): platform to pull the image for.
+  If not set, defaults to the platform of the build.
 - `runtime` (*bool*, *optional*): if set, marks dependency as runtime.
   This means that when this package is pulled in into the build, all the runtime dependencies are pulled in automatically as well.
   This also applies to transitive runtime dependencies.

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/alessio/shellescape v1.4.2
-	github.com/containerd/containerd v1.7.11
+	github.com/containerd/containerd v1.7.13
 	github.com/emicklei/dot v1.6.1
 	github.com/google/go-github/v60 v60.0.0
 	github.com/hashicorp/go-multierror v1.1.1

go.sum

@@ -12,8 +12,8 @@ github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7
 github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
 github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0=
 github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
-github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw=
-github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE=
+github.com/containerd/containerd v1.7.13 h1:wPYKIeGMN8vaggSKuV1X0wZulpMz4CrgEsZdaCyB6Is=
+github.com/containerd/containerd v1.7.13/go.mod h1:zT3up6yTRfEUa6+GsITYIJNgSVL9NQ4x4h1RPzk0Wu4=
 github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM=
 github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=

internal/pkg/convert/graph.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/siderolabs/bldr/internal/pkg/constants"
 	"github.com/siderolabs/bldr/internal/pkg/environment"
+	"github.com/siderolabs/bldr/internal/pkg/platform"
 	"github.com/siderolabs/bldr/internal/pkg/solver"
 	"github.com/siderolabs/bldr/internal/pkg/types/v1alpha2"
 )
@@ -29,19 +30,24 @@ type GraphLLB struct {
 	LocalContext llb.State
 
 	baseImageProcessor llbProcessor
-	cache              map[*solver.PackageNode]llb.State
+	cache              map[cacheKey]llb.State
 
 	commonRunOptions []llb.RunOption
 }
 
+type cacheKey struct {
+	*solver.PackageNode
+	Platform string
+}
+
 type llbProcessor func(llb.State) llb.State
 
 // NewGraphLLB creates new GraphLLB and initializes shared images.
 func NewGraphLLB(graph *solver.PackageGraph, options *environment.Options) *GraphLLB {
 	result := &GraphLLB{
 		PackageGraph: graph,
 		Options:      options,
-		cache:        make(map[*solver.PackageNode]llb.State),
+		cache:        make(map[cacheKey]llb.State),
 	}
 
 	if options.ProxyEnv != nil {
@@ -86,9 +92,12 @@ func (graph *GraphLLB) buildBaseImages() {
 		return addEnv(addPkg(root))
 	}
 
+	platform, _ := platform.ToV1Platform(graph.Root.Pkg.Platform, graph.Options.TargetPlatform.String()) //nolint:errcheck
+
 	graph.BaseImages[v1alpha2.Alpine] = graph.baseImageProcessor(llb.Image(
 		constants.DefaultBaseImage,
 		llb.WithCustomName(graph.Options.CommonPrefix+"base"),
+		llb.Platform(platform),
 	).Run(
 		append(graph.commonRunOptions,
 			llb.Shlex("apk --no-cache --update add bash"),
@@ -105,9 +114,12 @@ func (graph *GraphLLB) buildBaseImages() {
 }
 
 func (graph *GraphLLB) buildChecksummer() {
+	platform, _ := platform.ToV1Platform(graph.Root.Pkg.Platform, graph.Options.TargetPlatform.String()) //nolint:errcheck
+
 	graph.Checksummer = llb.Image(
 		constants.DefaultBaseImage,
 		llb.WithCustomName(graph.Options.CommonPrefix+"cksum"),
+		llb.Platform(platform),
 	).Run(
 		append(graph.commonRunOptions,
 			llb.Shlex("apk --no-cache --update add coreutils"),
@@ -132,7 +144,7 @@ func (graph *GraphLLB) buildLocalContext() {
 
 // Build converts package graph to LLB.
 func (graph *GraphLLB) Build() (llb.State, error) {
-	return NewNodeLLB(graph.Root, graph).Build()
+	return NewNodeLLB(graph.Root, graph, graph.Root.Pkg.Platform).Build()
 }
 
 // Marshal returns marshaled LLB.

internal/pkg/convert/node.go

@@ -12,11 +12,11 @@ import (
 
 	"github.com/moby/buildkit/client/llb"
 	"github.com/opencontainers/go-digest"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/siderolabs/gen/xslices"
 
 	"github.com/siderolabs/bldr/internal/pkg/constants"
 	"github.com/siderolabs/bldr/internal/pkg/environment"
+	"github.com/siderolabs/bldr/internal/pkg/platform"
 	"github.com/siderolabs/bldr/internal/pkg/solver"
 	"github.com/siderolabs/bldr/internal/pkg/types/v1alpha2"
 )
@@ -55,17 +55,24 @@ func defaultCopyOptions(options *environment.Options, reproducible bool) *llb.Co
 type NodeLLB struct {
 	*solver.PackageNode
 
-	Graph  *GraphLLB
-	Prefix string
+	Graph    *GraphLLB
+	Prefix   string
+	Platform string
 }
 
 // NewNodeLLB wraps PackageNode for LLB conversion.
-func NewNodeLLB(node *solver.PackageNode, graph *GraphLLB) *NodeLLB {
+func NewNodeLLB(node *solver.PackageNode, graph *GraphLLB, platformOverride string) *NodeLLB {
+	// set default platform if not set
+	if platformOverride == "" {
+		platformOverride = graph.Options.TargetPlatform.String()
+	}
+
 	return &NodeLLB{
 		PackageNode: node,
 
-		Graph:  graph,
-		Prefix: graph.Options.CommonPrefix + node.Name + ":",
+		Graph:    graph,
+		Prefix:   graph.Options.CommonPrefix + node.Name + ":",
+		Platform: platformOverride,
 	}
 }
 
@@ -96,26 +103,9 @@ func (node *NodeLLB) context(root llb.State) llb.State {
 	)
 }
 
-func (node *NodeLLB) convertPlatform(platform string) (v1.Platform, error) {
-	switch platform {
-	case "linux/amd64":
-		return v1.Platform{
-			OS:           "linux",
-			Architecture: "amd64",
-		}, nil
-	case "linux/arm64":
-		return v1.Platform{
-			OS:           "linux",
-			Architecture: "arm64",
-		}, nil
-	default:
-		return v1.Platform{}, fmt.Errorf("unknown platform %q", platform)
-	}
-}
-
 func (node *NodeLLB) convertDependency(dep solver.PackageDependency) (depState llb.State, srcName string, err error) {
 	if dep.IsInternal() {
-		depState, err = NewNodeLLB(dep.Node, node.Graph).Build()
+		depState, err = NewNodeLLB(dep.Node, node.Graph, node.Pkg.Platform).Build()
 		if err != nil {
 			return llb.Scratch(), "", err
 		}
@@ -126,7 +116,7 @@ func (node *NodeLLB) convertDependency(dep solver.PackageDependency) (depState l
 		srcName = dep.Image
 
 		if dep.Platform != "" {
-			platform, err := node.convertPlatform(dep.Platform)
+			platform, err := platform.ToV1Platform(dep.Platform, "")
 			if err != nil {
 				return llb.Scratch(), "", err
 			}
@@ -160,11 +150,18 @@ func (node *NodeLLB) dependencies(root llb.State) (llb.State, error) {
 	stages := []llb.State{root}
 
 	for _, dep := range deps {
-		if _, alreadyProcessed := seen[dep.ID()]; alreadyProcessed {
+		depID := dep.ID() + node.Platform
+
+		// set dep platform to node platform if not set
+		if dep.Platform == "" {
+			dep.Platform = node.Platform
+		}
+
+		if _, alreadyProcessed := seen[depID]; alreadyProcessed {
 			continue
 		}
 
-		seen[dep.ID()] = struct{}{}
+		seen[depID] = struct{}{}
 
 		depState, srcName, err := node.convertDependency(dep)
 		if err != nil {
@@ -328,7 +325,12 @@ func (node *NodeLLB) finalize(root llb.State) llb.State {
 
 // Build converts PackageNode to buildkit LLB.
 func (node *NodeLLB) Build() (llb.State, error) {
-	if state, ok := node.Graph.cache[node.PackageNode]; ok {
+	cacheSt := cacheKey{
+		PackageNode: node.PackageNode,
+		Platform:    node.Platform,
+	}
+
+	if state, ok := node.Graph.cache[cacheSt]; ok {
 		return state, nil
 	}
 
@@ -348,7 +350,7 @@ func (node *NodeLLB) Build() (llb.State, error) {
 
 	root = node.finalize(root)
 
-	node.Graph.cache[node.PackageNode] = root
+	node.Graph.cache[cacheSt] = root
 
 	return root, nil
 }

internal/pkg/integration/testdata/arch-amd64/final/pkg.yaml

@@ -1,5 +1,12 @@
 ---
 name: final
+dependencies:
+  # this arm64 variant of the image  only contains the u-boot binary,
+  # so this is a good test to ensure that we can copy the binary from
+  # the arm64 image to the x86_64 image
+  - image: ghcr.io/siderolabs/u-boot:v1.7.0-alpha.0-31-gcb39126
+    platform: linux/arm64
+    from: /rpi_generic
 steps:
 - test:
     - test "${BUILD:-x}" = "x86_64-linux-musl"

internal/pkg/integration/testdata/platform-override/Pkgfile

@@ -0,0 +1,3 @@
+# syntax = SHEBANG
+
+format: v1alpha2

internal/pkg/integration/testdata/platform-override/final/pkg.yaml

@@ -0,0 +1,9 @@
+---
+name: final
+platform: linux/arm64
+steps:
+- test:
+    - test `uname -m` = "aarch64"
+finalize:
+  - from: /
+    to: /

internal/pkg/integration/testdata/platform-override/test.yaml

@@ -0,0 +1,17 @@
+---
+run:
+  - name: docker-amd64
+    runner: docker
+    platform: linux/amd64
+    target: final
+    expect: success
+  - name: llb-amd64
+    runner: llb
+    platform: linux/amd64
+    target: final
+    expect: success
+  - name: buildkit-amd64
+    runner: buildkit
+    platform: linux/amd64
+    target: final
+    expect: success

internal/pkg/platform/platform.go

@@ -0,0 +1,34 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package platform provides a function to convert a platform string to a v1.Platform.
+package platform
+
+import (
+	"fmt"
+
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// ToV1Platform converts a platform string to a v1.Platform.
+func ToV1Platform(platform, targetPlatform string) (v1.Platform, error) {
+	if platform == "" {
+		platform = targetPlatform
+	}
+
+	switch platform {
+	case "linux/amd64":
+		return v1.Platform{
+			OS:           "linux",
+			Architecture: "amd64",
+		}, nil
+	case "linux/arm64":
+		return v1.Platform{
+			OS:           "linux",
+			Architecture: "arm64",
+		}, nil
+	default:
+		return v1.Platform{}, fmt.Errorf("unknown platform %q", platform)
+	}
+}

internal/pkg/types/v1alpha2/pkg.go

@@ -24,6 +24,7 @@ type Pkg struct {
 	Shell        Shell           `yaml:"shell,omitempty"`
 	BaseDir      string          `yaml:"-"`
 	FileName     string          `yaml:"-"`
+	Platform     string          `yaml:"platform,omitempty"`
 	Install      Install         `yaml:"install,omitempty"`
 	Dependencies Dependencies    `yaml:"dependencies,omitempty"`
 	Steps        Steps           `yaml:"steps,omitempty"`