From 03f3c2cc8a0bc8c236fca2b6bfc819300b7c7d81 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 7 Jun 2024 14:31:14 +0200 Subject: [PATCH 1/5] Add NRI plugin for CDI device injection Signed-off-by: Evan Lezar --- cmd/plugins/cdi-device-injector/Dockerfile | 24 +++ .../cdi-device-injector.go | 183 +++++++++++++++++ cmd/plugins/cdi-device-injector/cdi-device.go | 186 ++++++++++++++++++ go.mod | 12 +- go.sum | 29 +++ 5 files changed, 429 insertions(+), 5 deletions(-) create mode 100644 cmd/plugins/cdi-device-injector/Dockerfile create mode 100644 cmd/plugins/cdi-device-injector/cdi-device-injector.go create mode 100644 cmd/plugins/cdi-device-injector/cdi-device.go diff --git a/cmd/plugins/cdi-device-injector/Dockerfile b/cmd/plugins/cdi-device-injector/Dockerfile new file mode 100644 index 000000000..219196986 --- /dev/null +++ b/cmd/plugins/cdi-device-injector/Dockerfile @@ -0,0 +1,24 @@ +ARG GO_VERSION=1.22 + +FROM golang:${GO_VERSION}-bullseye as builder + +ARG IMAGE_VERSION +ARG BUILD_VERSION +ARG BUILD_BUILDID +WORKDIR /go/builder + +# Fetch go dependencies in a separate layer for caching +COPY go.mod go.sum ./ +RUN go mod download + +# Build the nri-cdi-device-injector plugin. +COPY . . + +RUN make clean +RUN make IMAGE_VERSION=${IMAGE_VERSION} BUILD_VERSION=${BUILD_VERSION} BUILD_BUILDID=${BUILD_BUILDID} PLUGINS=nri-cdi-device-injector build-plugins-static + +FROM gcr.io/distroless/static + +COPY --from=builder /go/builder/build/bin/nri-cdi-device-injector /bin/nri-cdi-device-injector + +ENTRYPOINT ["/bin/nri-cdi-device-injector", "-idx", "40"] diff --git a/cmd/plugins/cdi-device-injector/cdi-device-injector.go b/cmd/plugins/cdi-device-injector/cdi-device-injector.go new file mode 100644 index 000000000..07834b4c2 --- /dev/null +++ b/cmd/plugins/cdi-device-injector/cdi-device-injector.go @@ -0,0 +1,183 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "strings" + + "github.com/sirupsen/logrus" + "sigs.k8s.io/yaml" + "tags.cncf.io/container-device-interface/pkg/cdi" + "tags.cncf.io/container-device-interface/pkg/parser" + + "github.com/containerd/nri/pkg/api" + "github.com/containerd/nri/pkg/stub" +) + +const ( + cdiDeviceKey = "cdi.nri.io" +) + +var ( + log *logrus.Logger + verbose bool +) + +// our injector plugin +type plugin struct { + stub stub.Stub + cdiCache *cdiCache +} + +// CreateContainer handles container creation requests. +func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, container *api.Container) (_ *api.ContainerAdjustment, _ []*api.ContainerUpdate, err error) { + defer func() { + if err != nil { + log.Error(err) + } + }() + name := containerName(pod, container) + + if verbose { + dump("CreateContainer", "pod", pod, "container", container) + } else { + log.Infof("CreateContainer %s", name) + } + + cdiDevices, err := parseCdiDevices(pod.Annotations, container.Name) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse CDI Device annotations: %w", err) + } + + if len(cdiDevices) == 0 { + return nil, nil, nil + } + + adjust := &api.ContainerAdjustment{} + if _, err := p.cdiCache.InjectDevices(adjust, cdiDevices...); err != nil { + return nil, nil, fmt.Errorf("CDI device injection failed: %w", err) + } + + return adjust, nil, nil +} + +func parseCdiDevices(annotations map[string]string, ctr string) ([]string, error) { + var errs error + var cdiDevices []string + + for _, key := range []string{ + cdiDeviceKey + "/container." + ctr, + cdiDeviceKey + "/pod", + cdiDeviceKey, + } { + if value, ok := annotations[key]; ok { + for _, device := range strings.Split(value, ",") { + if !parser.IsQualifiedName(device) { + errs = errors.Join(errs, fmt.Errorf("invalid CDI device name %v", device)) + continue + } + cdiDevices = append(cdiDevices, device) + } + } + } + return cdiDevices, errs +} + +// Construct a container name for log messages. +func containerName(pod *api.PodSandbox, container *api.Container) string { + if pod != nil { + return pod.Namespace + "/" + pod.Name + "/" + container.Name + } + return container.Name +} + +// Dump one or more objects, with an optional global prefix and per-object tags. +func dump(args ...interface{}) { + var ( + prefix string + idx int + ) + + if len(args)&0x1 == 1 { + prefix = args[0].(string) + idx++ + } + + for ; idx < len(args)-1; idx += 2 { + tag, obj := args[idx], args[idx+1] + msg, err := yaml.Marshal(obj) + if err != nil { + log.Infof("%s: %s: failed to dump object: %v", prefix, tag, err) + continue + } + + if prefix != "" { + log.Infof("%s: %s:", prefix, tag) + for _, line := range strings.Split(strings.TrimSpace(string(msg)), "\n") { + log.Infof("%s: %s", prefix, line) + } + } else { + log.Infof("%s:", tag) + for _, line := range strings.Split(strings.TrimSpace(string(msg)), "\n") { + log.Infof(" %s", line) + } + } + } +} + +func main() { + var ( + pluginName string + pluginIdx string + opts []stub.Option + err error + ) + + log = logrus.StandardLogger() + log.SetFormatter(&logrus.TextFormatter{ + PadLevelText: true, + }) + + flag.StringVar(&pluginName, "name", "", "plugin name to register to NRI") + flag.StringVar(&pluginIdx, "idx", "", "plugin index to register to NRI") + flag.BoolVar(&verbose, "verbose", false, "enable (more) verbose logging") + flag.Parse() + + if pluginName != "" { + opts = append(opts, stub.WithPluginName(pluginName)) + } + if pluginIdx != "" { + opts = append(opts, stub.WithPluginIdx(pluginIdx)) + } + + p := &plugin{ + cdiCache: &cdiCache{ + // TODO: We should allow this to be configured + Cache: cdi.GetDefaultCache(), + }, + } + if p.stub, err = stub.New(p, opts...); err != nil { + log.Fatalf("failed to create plugin stub: %v", err) + } + + err = p.stub.Run(context.Background()) + if err != nil { + log.Fatalf("plugin exited with error %v", err) + } +} diff --git a/cmd/plugins/cdi-device-injector/cdi-device.go b/cmd/plugins/cdi-device-injector/cdi-device.go new file mode 100644 index 000000000..cfad6231d --- /dev/null +++ b/cmd/plugins/cdi-device-injector/cdi-device.go @@ -0,0 +1,186 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "strings" + + "github.com/containerd/nri/pkg/api" + oci "github.com/opencontainers/runtime-spec/specs-go" + "tags.cncf.io/container-device-interface/pkg/cdi" +) + +type cdiCache struct { + *cdi.Cache +} + +// InjectDevices applies the specified CDI devices to the NRI container adjustments. +// This implementation first applies the edits to an empty OCI Spec and then +// converts these edits to NRI adjustments. +func (c *cdiCache) InjectDevices(adjustments *api.ContainerAdjustment, devices ...string) ([]string, error) { + ociSpec := &oci.Spec{} + + unresolved, err := c.Cache.InjectDevices(ociSpec, devices...) + if err != nil { + return unresolved, err + } + + for _, ociEnv := range ociSpec.Process.Env { + parts := strings.SplitN(ociEnv, "=", 2) + var value string + if len(parts) > 1 { + value = parts[1] + } + adjustments.AddEnv(parts[0], value) + } + + for _, oci := range ociSpec.Linux.Devices { + device := (ociLinuxDevice)(oci).toNRI() + adjustments.AddDevice(device) + } + + for _, oci := range ociSpec.Linux.Resources.Devices { + deviceCgroup := (ociLinuxDeviceCgroup)(oci).toNRI() + adjustments.Linux.Resources.Devices = append(adjustments.Linux.Resources.Devices, deviceCgroup) + } + + for _, oci := range ociSpec.Mounts { + mount := (ociMount)(oci).toNRI() + adjustments.AddMount(mount) + } + + if oci := ociSpec.Hooks; oci != nil { + hooks := (*ociHooks)(oci).toNRI() + adjustments.AddHooks(hooks) + } + + // TODO: Handle IntelRdt fields + // TODO: Handle additional GID fields + return nil, nil +} + +type ociLinuxDevice oci.LinuxDevice + +func (o ociLinuxDevice) toNRI() *api.LinuxDevice { + var filemode *api.OptionalFileMode + if mode := o.FileMode; mode != nil { + filemode = &api.OptionalFileMode{ + Value: (uint32)(*mode), + } + } + var uid *api.OptionalUInt32 + if u := o.UID; u != nil { + uid = &api.OptionalUInt32{ + Value: *u, + } + } + var gid *api.OptionalUInt32 + if g := o.UID; g != nil { + gid = &api.OptionalUInt32{ + Value: *g, + } + } + + return &api.LinuxDevice{ + Path: o.Path, + Type: o.Type, + Major: o.Major, + Minor: o.Minor, + FileMode: filemode, + Uid: uid, + Gid: gid, + } +} + +type ociLinuxDeviceCgroup oci.LinuxDeviceCgroup + +func (o ociLinuxDeviceCgroup) toNRI() *api.LinuxDeviceCgroup { + var major *api.OptionalInt64 + if m := o.Major; m != nil { + major = &api.OptionalInt64{ + Value: *m, + } + } + var minor *api.OptionalInt64 + if m := o.Minor; m != nil { + minor = &api.OptionalInt64{ + Value: *m, + } + } + + return &api.LinuxDeviceCgroup{ + Allow: o.Allow, + Type: o.Type, + Major: major, + Minor: minor, + Access: o.Access, + } +} + +type ociMount oci.Mount + +func (o ociMount) toNRI() *api.Mount { + return &api.Mount{ + Destination: o.Destination, + Type: o.Type, + Source: o.Source, + Options: o.Options, + // TODO: We don't handle the following fields: + // UIDMappings []LinuxIDMapping `json:"uidMappings,omitempty" platform:"linux"` + // GIDMappings []LinuxIDMapping `json:"gidMappings,omitempty" platform:"linux"` + } +} + +type ociHooks oci.Hooks + +func (o *ociHooks) toNRI() *api.Hooks { + hooks := &api.Hooks{} + for _, h := range o.Prestart { + hooks.Prestart = append(hooks.Prestart, (ociHook)(h).toNRI()) + } + for _, h := range o.CreateRuntime { + hooks.Prestart = append(hooks.CreateRuntime, (ociHook)(h).toNRI()) + } + for _, h := range o.CreateContainer { + hooks.Prestart = append(hooks.CreateContainer, (ociHook)(h).toNRI()) + } + for _, h := range o.StartContainer { + hooks.Prestart = append(hooks.StartContainer, (ociHook)(h).toNRI()) + } + for _, h := range o.Poststart { + hooks.Prestart = append(hooks.Poststart, (ociHook)(h).toNRI()) + } + for _, h := range o.Poststop { + hooks.Prestart = append(hooks.Poststop, (ociHook)(h).toNRI()) + } + return hooks +} + +type ociHook oci.Hook + +func (o ociHook) toNRI() *api.Hook { + var timeout *api.OptionalInt + if t := o.Timeout; t != nil { + timeout = &api.OptionalInt{ + Value: (int64)(*t), + } + } + return &api.Hook{ + Path: o.Path, + Args: o.Args, + Env: o.Env, + Timeout: timeout, + } +} diff --git a/go.mod b/go.mod index 6f4124bb0..948f8c306 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.1.2 github.com/onsi/ginkgo/v2 v2.16.0 github.com/onsi/gomega v1.30.0 + github.com/opencontainers/runtime-spec v1.1.0 github.com/pelletier/go-toml/v2 v2.1.0 github.com/prometheus/client_golang v1.17.0 github.com/prometheus/client_model v0.5.0 @@ -35,6 +36,7 @@ require ( k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 sigs.k8s.io/controller-runtime v0.16.2 sigs.k8s.io/yaml v1.3.0 + tags.cncf.io/container-device-interface v0.7.2 ) require ( @@ -70,15 +72,17 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/opencontainers/runtime-spec v1.1.0 // indirect + github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/prometheus/statsd_exporter v0.24.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect go.opentelemetry.io/otel/metric v1.19.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect + golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/term v0.18.0 // indirect @@ -96,9 +100,7 @@ require ( k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + tags.cncf.io/container-device-interface/specs-go v0.7.0 // indirect ) -replace ( - github.com/containers/nri-plugins/pkg/topology v0.0.0 => ./pkg/topology - github.com/opencontainers/runtime-tools => github.com/opencontainers/runtime-tools v0.0.0-20221026201742-946c877fa809 -) +replace github.com/containers/nri-plugins/pkg/topology v0.0.0 => ./pkg/topology diff --git a/go.sum b/go.sum index 07001d870..6bbc68484 100644 --- a/go.sum +++ b/go.sum @@ -620,6 +620,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -847,6 +849,10 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4Zs github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 h1:RtRsiaGvWxcwd8y3BiRZxsylPT8hLWZ5SPcfI+3IDNk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0/go.mod h1:TzP6duP4Py2pHLVPPQp42aoYI92+PCrVotyR5e8Vqlk= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= @@ -904,6 +910,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -911,6 +918,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -919,8 +927,13 @@ github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0= +github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI= +github.com/opencontainers/selinux v1.9.1 h1:b4VPEF3O5JLZgdTDBmGepaaIbAo0GqoF6EBRq5f/g3Y= +github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= @@ -1009,6 +1022,15 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1115,6 +1137,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1238,6 +1262,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1794,3 +1819,7 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kF sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +tags.cncf.io/container-device-interface v0.7.2 h1:MLqGnWfOr1wB7m08ieI4YJ3IoLKKozEnnNYBtacDPQU= +tags.cncf.io/container-device-interface v0.7.2/go.mod h1:Xb1PvXv2BhfNb3tla4r9JL129ck1Lxv9KuU6eVOfKto= +tags.cncf.io/container-device-interface/specs-go v0.7.0 h1:w/maMGVeLP6TIQJVYT5pbqTi8SCw/iHZ+n4ignuGHqg= +tags.cncf.io/container-device-interface/specs-go v0.7.0/go.mod h1:hMAwAbMZyBLdmYqWgYcKH0F/yctNpV3P35f+/088A80= From ec2f159ba4550a21e9bf769ef1e9b27f6e374b1c Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 27 Jun 2024 16:37:23 +0200 Subject: [PATCH 2/5] Add basic README explaining the intent around CDI injection Signed-off-by: Evan Lezar --- cmd/plugins/cdi-device-injector/README.md | 96 +++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 cmd/plugins/cdi-device-injector/README.md diff --git a/cmd/plugins/cdi-device-injector/README.md b/cmd/plugins/cdi-device-injector/README.md new file mode 100644 index 000000000..4163e7df6 --- /dev/null +++ b/cmd/plugins/cdi-device-injector/README.md @@ -0,0 +1,96 @@ +# CDI Device Injector using NRI + +The purpose of this NRI plugin is to provide a controlled mechansim for injecting +CDI devices into containers. This can be separated into two aspects: + +1. Requesting devices +2. Controlling access to devices + + +## Requesting devices + +For reqesting devices, we use pod annotations to indicate which devices should +be made available to a particular container or containers in a pod. Here a +pre-defined annotation prefix `cdi.nri.io/` is used for these annotations. +Devices are requested for a specific container by including +`container.{{ .ContainerName }}` as a suffix in the annotation key. For a +request targeting ALL containers in a pod `cdi.nri.io/pod` is used as the pod +annotaion key. + +In either case, the corresponding annotation value represents a comma-separated +list of fully-qualified CDI device names. + +As examples, consider the following pod annotation: +```yaml +apiVersion: v1 +kind: Pod +metadata: + namespace: management + name: nri-injection-example + annotations: + cdi.nri.io/container.first-ctr: "example.com/class=device0,example.com/class=device1" +spec: + containers: + - name: first-ctr + image: ubuntu + - name: second-ctr + image: ubuntu +``` + +This will trigger the injection of the `example.com/class=device0` and +`example.com/class=device1` devices into the `first-ctr` container, but not into +the `second-ctr` container. + +When the annotations are updated as follows: +```yaml +apiVersion: v1 +kind: Pod +metadata: + namespace: management + name: nri-injection-example + annotations: + cdi.nri.io/pod: "example.com/class=device0,example.com/class=device1" +spec: + containers: + - name: first-ctr + image: ubuntu + - name: second-ctr + image: ubuntu +``` + +the same `example.com/class=device0` and `example.com/class=device1` devices +will be injected into all (`first-ctr` and `second-ctr`) containers in the pod. + +## Controlling Access + +In order to control access to specific CDI devices, we make use of namespace +annotations. Here, the same `cdi.nri.io/` prefix is used to identify an +annotation for controlling the injection of CDI devices using NRI. The +pre-defined annotation key `cdi.nri.io/allow` is used to explicitly allow access +to CDI devices. + +The value field is interpreted as a filename glob to allow for wildcard matches. + +For example: +* `*` will allow any CDI device to be injected +* `example.com/*` will allow any CDI device with the explicit `example.com` to be + injected. +* `example.com/class=*` will allow any CDI devices from vendor `example.com` and + class `class` to be injected. +* `example.com/class=device0` will only allow the specified CDI device to be + injected. + +Consider the following example namespace (which was also referenced in the +pod examples above): + +```yaml +apiVersion: v1 +kind: Namespace +metadata: + name: management + annotations: + cdi.nri.io/allow: "*" +``` + +This allows the injection of any CDI devices into containers belonging to pods +in this namespace. From 2206edc91e97e6b5e3fb316f48243a70adbd7d8f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 27 Jun 2024 16:38:06 +0200 Subject: [PATCH 3/5] Add logic to filter out CDI devices based on a config option This change adds an allowed-cdi-device-pattern command line option to the CDI device injector to allow CDI device names to be filtered. Signed-off-by: Evan Lezar --- .../cdi-device-injector.go | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/cmd/plugins/cdi-device-injector/cdi-device-injector.go b/cmd/plugins/cdi-device-injector/cdi-device-injector.go index 07834b4c2..53e6c6372 100644 --- a/cmd/plugins/cdi-device-injector/cdi-device-injector.go +++ b/cmd/plugins/cdi-device-injector/cdi-device-injector.go @@ -19,6 +19,7 @@ import ( "errors" "flag" "fmt" + "path/filepath" "strings" "github.com/sirupsen/logrus" @@ -41,12 +42,13 @@ var ( // our injector plugin type plugin struct { - stub stub.Stub - cdiCache *cdiCache + stub stub.Stub + allowedCDIDevicePattern string + cdiCache *cdiCache } // CreateContainer handles container creation requests. -func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, container *api.Container) (_ *api.ContainerAdjustment, _ []*api.ContainerUpdate, err error) { +func (p *plugin) CreateContainer(ctx context.Context, pod *api.PodSandbox, container *api.Container) (_ *api.ContainerAdjustment, _ []*api.ContainerUpdate, err error) { defer func() { if err != nil { log.Error(err) @@ -60,6 +62,10 @@ func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, contain log.Infof("CreateContainer %s", name) } + if p.allowedCDIDevicePattern == "" { + return nil, nil, nil + } + cdiDevices, err := parseCdiDevices(pod.Annotations, container.Name) if err != nil { return nil, nil, fmt.Errorf("failed to parse CDI Device annotations: %w", err) @@ -69,8 +75,17 @@ func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, contain return nil, nil, nil } + var allowedCDIDevices []string + for _, cdiDevice := range cdiDevices { + match, _ := filepath.Match(p.allowedCDIDevicePattern, cdiDevice) + if !match { + continue + } + allowedCDIDevices = append(allowedCDIDevices, cdiDevice) + } + adjust := &api.ContainerAdjustment{} - if _, err := p.cdiCache.InjectDevices(adjust, cdiDevices...); err != nil { + if _, err := p.cdiCache.InjectDevices(adjust, allowedCDIDevices...); err != nil { return nil, nil, fmt.Errorf("CDI device injection failed: %w", err) } @@ -143,10 +158,11 @@ func dump(args ...interface{}) { func main() { var ( - pluginName string - pluginIdx string - opts []stub.Option - err error + pluginName string + pluginIdx string + allowedCDIDevicePattern string + opts []stub.Option + err error ) log = logrus.StandardLogger() @@ -156,6 +172,7 @@ func main() { flag.StringVar(&pluginName, "name", "", "plugin name to register to NRI") flag.StringVar(&pluginIdx, "idx", "", "plugin index to register to NRI") + flag.StringVar(&allowedCDIDevicePattern, "allowed-cdi-device-pattern", "*", "glob pattern for allowed CDI device names") flag.BoolVar(&verbose, "verbose", false, "enable (more) verbose logging") flag.Parse() @@ -167,6 +184,7 @@ func main() { } p := &plugin{ + allowedCDIDevicePattern: allowedCDIDevicePattern, cdiCache: &cdiCache{ // TODO: We should allow this to be configured Cache: cdi.GetDefaultCache(), From 5efa8def1979f09f147089c5beaa0c33c4dd039c Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 9 Aug 2024 16:53:49 +0200 Subject: [PATCH 4/5] TOFIX: Use CDIDevices API field Signed-off-by: Evan Lezar --- .../cdi-device-injector.go | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/cmd/plugins/cdi-device-injector/cdi-device-injector.go b/cmd/plugins/cdi-device-injector/cdi-device-injector.go index 53e6c6372..34bc9dc77 100644 --- a/cmd/plugins/cdi-device-injector/cdi-device-injector.go +++ b/cmd/plugins/cdi-device-injector/cdi-device-injector.go @@ -66,27 +66,29 @@ func (p *plugin) CreateContainer(ctx context.Context, pod *api.PodSandbox, conta return nil, nil, nil } - cdiDevices, err := parseCdiDevices(pod.Annotations, container.Name) + cdiDeviceNames, err := parseCdiDevices(pod.Annotations, container.Name) if err != nil { return nil, nil, fmt.Errorf("failed to parse CDI Device annotations: %w", err) } - if len(cdiDevices) == 0 { + if len(cdiDeviceNames) == 0 { return nil, nil, nil } - var allowedCDIDevices []string - for _, cdiDevice := range cdiDevices { - match, _ := filepath.Match(p.allowedCDIDevicePattern, cdiDevice) + var cdiDevices []*api.CDIDevice + for _, cdiDeviceName := range cdiDeviceNames { + match, _ := filepath.Match(p.allowedCDIDevicePattern, cdiDeviceName) if !match { continue } - allowedCDIDevices = append(allowedCDIDevices, cdiDevice) + + cdiDevices = append(cdiDevices, &api.CDIDevice{ + Name: cdiDeviceName, + }) } - adjust := &api.ContainerAdjustment{} - if _, err := p.cdiCache.InjectDevices(adjust, allowedCDIDevices...); err != nil { - return nil, nil, fmt.Errorf("CDI device injection failed: %w", err) + adjust := &api.ContainerAdjustment{ + CDIDevices: cdiDevices, } return adjust, nil, nil From 709089b45db2c13f8a698eb5cab5c79091547567 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 9 Aug 2024 16:54:02 +0200 Subject: [PATCH 5/5] TOFIX: module replacement rule Signed-off-by: Evan Lezar --- go.mod | 4 +++- go.sum | 6 ++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 948f8c306..753396b20 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 github.com/containerd/nri v0.6.0 github.com/containerd/otelttrpc v0.0.0-20240305015340-ea5083fda723 - github.com/containerd/ttrpc v1.2.3-0.20231030150553-baadfd8e7956 + github.com/containerd/ttrpc v1.2.3 github.com/containers/nri-plugins/pkg/topology v0.0.0 github.com/coreos/go-systemd/v22 v22.5.0 github.com/fsnotify/fsnotify v1.6.0 @@ -104,3 +104,5 @@ require ( ) replace github.com/containers/nri-plugins/pkg/topology v0.0.0 => ./pkg/topology + +replace github.com/containerd/nri => ../nri diff --git a/go.sum b/go.sum index 6bbc68484..77884b384 100644 --- a/go.sum +++ b/go.sum @@ -651,13 +651,11 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/containerd/nri v0.6.0 h1:hdztxwL0gCS1CrCa9bvD1SoJiFN4jBuRQhplCvCPMj8= -github.com/containerd/nri v0.6.0/go.mod h1:F7OZfO4QTPqw5r87aq+syZJwiVvRYLIlHZiZDBV1W3A= github.com/containerd/otelttrpc v0.0.0-20240305015340-ea5083fda723 h1:swk9KxrmARZjSMrHc1Lzb39XhcDwAhYpqkBhinCFLCQ= github.com/containerd/otelttrpc v0.0.0-20240305015340-ea5083fda723/go.mod h1:ZKzztepTSz/LKtbUSzfBNVwgqBEPABVZV9PQF/l53+Q= github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= -github.com/containerd/ttrpc v1.2.3-0.20231030150553-baadfd8e7956 h1:BQwXCrKPRdDQvTYfiDatp36FIH/EF7JTBOZU+EPIKWY= -github.com/containerd/ttrpc v1.2.3-0.20231030150553-baadfd8e7956/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM= +github.com/containerd/ttrpc v1.2.3 h1:4jlhbXIGvijRtNC8F/5CpuJZ7yKOBFGFOOXg1bkISz0= +github.com/containerd/ttrpc v1.2.3/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=