From 65e6a18efe15490a02e55b6e9e753185a4ab224f Mon Sep 17 00:00:00 2001 From: Noel Georgi Date: Tue, 5 Mar 2024 19:26:32 +0530 Subject: [PATCH] feat: support platform override Support setting `platform` for dependencies and also allow overriding `platform` for a specific package. Signed-off-by: Noel Georgi --- Dockerfile | 4 +- Makefile | 8 +-- README.md | 4 ++ go.mod | 2 +- go.sum | 4 +- internal/pkg/convert/graph.go | 18 +++++- internal/pkg/convert/node.go | 60 ++++++++++--------- .../testdata/arch-amd64/final/pkg.yaml | 7 +++ .../testdata/platform-override/Pkgfile | 3 + .../testdata/platform-override/final/pkg.yaml | 9 +++ .../testdata/platform-override/test.yaml | 17 ++++++ internal/pkg/platform/platform.go | 34 +++++++++++ internal/pkg/types/v1alpha2/pkg.go | 1 + 13 files changed, 130 insertions(+), 41 deletions(-) create mode 100644 internal/pkg/integration/testdata/platform-override/Pkgfile create mode 100644 internal/pkg/integration/testdata/platform-override/final/pkg.yaml create mode 100644 internal/pkg/integration/testdata/platform-override/test.yaml create mode 100644 internal/pkg/platform/platform.go diff --git a/Dockerfile b/Dockerfile index 9309898..1b14131 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -# syntax = docker/dockerfile-upstream:1.6.0-labs +# syntax = docker/dockerfile-upstream:1.7.0-labs # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-03-05T05:52:53Z by kres latest. +# Generated on 2024-03-06T10:32:46Z by kres latest. ARG TOOLCHAIN diff --git a/Makefile b/Makefile index 8c5b05f..f57382a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-03-05T05:52:53Z by kres latest. +# Generated on 2024-03-06T10:32:46Z by kres latest. # common variables @@ -14,15 +14,15 @@ WITH_RACE ?= false REGISTRY ?= ghcr.io USERNAME ?= siderolabs REGISTRY_AND_USERNAME ?= $(REGISTRY)/$(USERNAME) -PROTOBUF_GO_VERSION ?= 1.32.0 +PROTOBUF_GO_VERSION ?= 1.33.0 GRPC_GO_VERSION ?= 1.3.0 GRPC_GATEWAY_VERSION ?= 2.19.1 VTPROTOBUF_VERSION ?= 0.6.0 DEEPCOPY_VERSION ?= v0.5.6 GOLANGCILINT_VERSION ?= v1.56.2 GOFUMPT_VERSION ?= v0.6.0 -GO_VERSION ?= 1.22.0 -GOIMPORTS_VERSION ?= v0.18.0 +GO_VERSION ?= 1.22.1 +GOIMPORTS_VERSION ?= v0.19.0 GO_BUILDFLAGS ?= GO_LDFLAGS ?= CGO_ENABLED ?= 0 diff --git a/README.md b/README.md index 5a45ed2..1b62019 100644 --- a/README.md +++ b/README.md @@ -227,6 +227,8 @@ Additionally, hermetic text functions from [Sprig](http://masterminds.github.io/ On the root level, following properties are available: - `name` (*str*, *required*): name of the package, also used to reference this package from other packages as dependency. +- `platform` (*str*, *optional*): platform override for the build. + If not set, defaults to the platform of the build. - `variant` (*str*, *optional*): variant of the base image of the build. Two variants are available: - `alpine`: Alpine Linux 3.16 image with `bash` package pre-installed @@ -270,6 +272,8 @@ Properties: Contents of the stage are poured into the build at the location specified with `to:` parameter. - `image` (*str*, *external dependency*): reference to the registry container image this package depends on. Contents of the image are poured into the build at the location specified with `to:` parameter. +- `platform` (*str*, *optional*): platform to pull the image for. + If not set, defaults to the platform of the build. - `runtime` (*bool*, *optional*): if set, marks dependency as runtime. This means that when this package is pulled in into the build, all the runtime dependencies are pulled in automatically as well. This also applies to transitive runtime dependencies. diff --git a/go.mod b/go.mod index 2b7508f..aa5d04b 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Masterminds/semver v1.5.0 github.com/Masterminds/sprig/v3 v3.2.3 github.com/alessio/shellescape v1.4.2 - github.com/containerd/containerd v1.7.11 + github.com/containerd/containerd v1.7.13 github.com/emicklei/dot v1.6.1 github.com/google/go-github/v60 v60.0.0 github.com/hashicorp/go-multierror v1.1.1 diff --git a/go.sum b/go.sum index 8690bd2..451609a 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7 github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= -github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw= -github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE= +github.com/containerd/containerd v1.7.13 h1:wPYKIeGMN8vaggSKuV1X0wZulpMz4CrgEsZdaCyB6Is= +github.com/containerd/containerd v1.7.13/go.mod h1:zT3up6yTRfEUa6+GsITYIJNgSVL9NQ4x4h1RPzk0Wu4= github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= diff --git a/internal/pkg/convert/graph.go b/internal/pkg/convert/graph.go index bc2bb16..ce225e1 100644 --- a/internal/pkg/convert/graph.go +++ b/internal/pkg/convert/graph.go @@ -12,6 +12,7 @@ import ( "github.com/siderolabs/bldr/internal/pkg/constants" "github.com/siderolabs/bldr/internal/pkg/environment" + "github.com/siderolabs/bldr/internal/pkg/platform" "github.com/siderolabs/bldr/internal/pkg/solver" "github.com/siderolabs/bldr/internal/pkg/types/v1alpha2" ) @@ -29,11 +30,16 @@ type GraphLLB struct { LocalContext llb.State baseImageProcessor llbProcessor - cache map[*solver.PackageNode]llb.State + cache map[cacheKey]llb.State commonRunOptions []llb.RunOption } +type cacheKey struct { + *solver.PackageNode + Platform string +} + type llbProcessor func(llb.State) llb.State // NewGraphLLB creates new GraphLLB and initializes shared images. @@ -41,7 +47,7 @@ func NewGraphLLB(graph *solver.PackageGraph, options *environment.Options) *Grap result := &GraphLLB{ PackageGraph: graph, Options: options, - cache: make(map[*solver.PackageNode]llb.State), + cache: make(map[cacheKey]llb.State), } if options.ProxyEnv != nil { @@ -86,9 +92,12 @@ func (graph *GraphLLB) buildBaseImages() { return addEnv(addPkg(root)) } + platform, _ := platform.ToV1Platform(graph.Root.Pkg.Platform, graph.Options.TargetPlatform.String()) //nolint:errcheck + graph.BaseImages[v1alpha2.Alpine] = graph.baseImageProcessor(llb.Image( constants.DefaultBaseImage, llb.WithCustomName(graph.Options.CommonPrefix+"base"), + llb.Platform(platform), ).Run( append(graph.commonRunOptions, llb.Shlex("apk --no-cache --update add bash"), @@ -105,9 +114,12 @@ func (graph *GraphLLB) buildBaseImages() { } func (graph *GraphLLB) buildChecksummer() { + platform, _ := platform.ToV1Platform(graph.Root.Pkg.Platform, graph.Options.TargetPlatform.String()) //nolint:errcheck + graph.Checksummer = llb.Image( constants.DefaultBaseImage, llb.WithCustomName(graph.Options.CommonPrefix+"cksum"), + llb.Platform(platform), ).Run( append(graph.commonRunOptions, llb.Shlex("apk --no-cache --update add coreutils"), @@ -132,7 +144,7 @@ func (graph *GraphLLB) buildLocalContext() { // Build converts package graph to LLB. func (graph *GraphLLB) Build() (llb.State, error) { - return NewNodeLLB(graph.Root, graph).Build() + return NewNodeLLB(graph.Root, graph, graph.Root.Pkg.Platform).Build() } // Marshal returns marshaled LLB. diff --git a/internal/pkg/convert/node.go b/internal/pkg/convert/node.go index 6779666..d533d1d 100644 --- a/internal/pkg/convert/node.go +++ b/internal/pkg/convert/node.go @@ -12,11 +12,11 @@ import ( "github.com/moby/buildkit/client/llb" "github.com/opencontainers/go-digest" - v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/siderolabs/gen/xslices" "github.com/siderolabs/bldr/internal/pkg/constants" "github.com/siderolabs/bldr/internal/pkg/environment" + "github.com/siderolabs/bldr/internal/pkg/platform" "github.com/siderolabs/bldr/internal/pkg/solver" "github.com/siderolabs/bldr/internal/pkg/types/v1alpha2" ) @@ -55,17 +55,24 @@ func defaultCopyOptions(options *environment.Options, reproducible bool) *llb.Co type NodeLLB struct { *solver.PackageNode - Graph *GraphLLB - Prefix string + Graph *GraphLLB + Prefix string + Platform string } // NewNodeLLB wraps PackageNode for LLB conversion. -func NewNodeLLB(node *solver.PackageNode, graph *GraphLLB) *NodeLLB { +func NewNodeLLB(node *solver.PackageNode, graph *GraphLLB, platformOverride string) *NodeLLB { + // set default platform if not set + if platformOverride == "" { + platformOverride = graph.Options.TargetPlatform.String() + } + return &NodeLLB{ PackageNode: node, - Graph: graph, - Prefix: graph.Options.CommonPrefix + node.Name + ":", + Graph: graph, + Prefix: graph.Options.CommonPrefix + node.Name + ":", + Platform: platformOverride, } } @@ -96,26 +103,9 @@ func (node *NodeLLB) context(root llb.State) llb.State { ) } -func (node *NodeLLB) convertPlatform(platform string) (v1.Platform, error) { - switch platform { - case "linux/amd64": - return v1.Platform{ - OS: "linux", - Architecture: "amd64", - }, nil - case "linux/arm64": - return v1.Platform{ - OS: "linux", - Architecture: "arm64", - }, nil - default: - return v1.Platform{}, fmt.Errorf("unknown platform %q", platform) - } -} - func (node *NodeLLB) convertDependency(dep solver.PackageDependency) (depState llb.State, srcName string, err error) { if dep.IsInternal() { - depState, err = NewNodeLLB(dep.Node, node.Graph).Build() + depState, err = NewNodeLLB(dep.Node, node.Graph, node.Pkg.Platform).Build() if err != nil { return llb.Scratch(), "", err } @@ -126,7 +116,7 @@ func (node *NodeLLB) convertDependency(dep solver.PackageDependency) (depState l srcName = dep.Image if dep.Platform != "" { - platform, err := node.convertPlatform(dep.Platform) + platform, err := platform.ToV1Platform(dep.Platform, "") if err != nil { return llb.Scratch(), "", err } @@ -160,11 +150,18 @@ func (node *NodeLLB) dependencies(root llb.State) (llb.State, error) { stages := []llb.State{root} for _, dep := range deps { - if _, alreadyProcessed := seen[dep.ID()]; alreadyProcessed { + depID := dep.ID() + node.Platform + + // set dep platform to node platform if not set + if dep.Platform == "" { + dep.Platform = node.Platform + } + + if _, alreadyProcessed := seen[depID]; alreadyProcessed { continue } - seen[dep.ID()] = struct{}{} + seen[depID] = struct{}{} depState, srcName, err := node.convertDependency(dep) if err != nil { @@ -328,7 +325,12 @@ func (node *NodeLLB) finalize(root llb.State) llb.State { // Build converts PackageNode to buildkit LLB. func (node *NodeLLB) Build() (llb.State, error) { - if state, ok := node.Graph.cache[node.PackageNode]; ok { + cacheSt := cacheKey{ + PackageNode: node.PackageNode, + Platform: node.Platform, + } + + if state, ok := node.Graph.cache[cacheSt]; ok { return state, nil } @@ -348,7 +350,7 @@ func (node *NodeLLB) Build() (llb.State, error) { root = node.finalize(root) - node.Graph.cache[node.PackageNode] = root + node.Graph.cache[cacheSt] = root return root, nil } diff --git a/internal/pkg/integration/testdata/arch-amd64/final/pkg.yaml b/internal/pkg/integration/testdata/arch-amd64/final/pkg.yaml index e6876a8..9ac3629 100644 --- a/internal/pkg/integration/testdata/arch-amd64/final/pkg.yaml +++ b/internal/pkg/integration/testdata/arch-amd64/final/pkg.yaml @@ -1,5 +1,12 @@ --- name: final +dependencies: + # this arm64 variant of the image only contains the u-boot binary, + # so this is a good test to ensure that we can copy the binary from + # the arm64 image to the x86_64 image + - image: ghcr.io/siderolabs/u-boot:v1.7.0-alpha.0-31-gcb39126 + platform: linux/arm64 + from: /rpi_generic steps: - test: - test "${BUILD:-x}" = "x86_64-linux-musl" diff --git a/internal/pkg/integration/testdata/platform-override/Pkgfile b/internal/pkg/integration/testdata/platform-override/Pkgfile new file mode 100644 index 0000000..dcbe35e --- /dev/null +++ b/internal/pkg/integration/testdata/platform-override/Pkgfile @@ -0,0 +1,3 @@ +# syntax = SHEBANG + +format: v1alpha2 diff --git a/internal/pkg/integration/testdata/platform-override/final/pkg.yaml b/internal/pkg/integration/testdata/platform-override/final/pkg.yaml new file mode 100644 index 0000000..6dd5bd9 --- /dev/null +++ b/internal/pkg/integration/testdata/platform-override/final/pkg.yaml @@ -0,0 +1,9 @@ +--- +name: final +platform: linux/arm64 +steps: +- test: + - test `uname -m` = "aarch64" +finalize: + - from: / + to: / diff --git a/internal/pkg/integration/testdata/platform-override/test.yaml b/internal/pkg/integration/testdata/platform-override/test.yaml new file mode 100644 index 0000000..739dacd --- /dev/null +++ b/internal/pkg/integration/testdata/platform-override/test.yaml @@ -0,0 +1,17 @@ +--- +run: + - name: docker-amd64 + runner: docker + platform: linux/amd64 + target: final + expect: success + - name: llb-amd64 + runner: llb + platform: linux/amd64 + target: final + expect: success + - name: buildkit-amd64 + runner: buildkit + platform: linux/amd64 + target: final + expect: success diff --git a/internal/pkg/platform/platform.go b/internal/pkg/platform/platform.go new file mode 100644 index 0000000..a45e4e2 --- /dev/null +++ b/internal/pkg/platform/platform.go @@ -0,0 +1,34 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package platform provides a function to convert a platform string to a v1.Platform. +package platform + +import ( + "fmt" + + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +// ToV1Platform converts a platform string to a v1.Platform. +func ToV1Platform(platform, targetPlatform string) (v1.Platform, error) { + if platform == "" { + platform = targetPlatform + } + + switch platform { + case "linux/amd64": + return v1.Platform{ + OS: "linux", + Architecture: "amd64", + }, nil + case "linux/arm64": + return v1.Platform{ + OS: "linux", + Architecture: "arm64", + }, nil + default: + return v1.Platform{}, fmt.Errorf("unknown platform %q", platform) + } +} diff --git a/internal/pkg/types/v1alpha2/pkg.go b/internal/pkg/types/v1alpha2/pkg.go index 5736131..01fe531 100644 --- a/internal/pkg/types/v1alpha2/pkg.go +++ b/internal/pkg/types/v1alpha2/pkg.go @@ -24,6 +24,7 @@ type Pkg struct { Shell Shell `yaml:"shell,omitempty"` BaseDir string `yaml:"-"` FileName string `yaml:"-"` + Platform string `yaml:"platform,omitempty"` Install Install `yaml:"install,omitempty"` Dependencies Dependencies `yaml:"dependencies,omitempty"` Steps Steps `yaml:"steps,omitempty"`