Skip to content

Commit

Permalink
cache: Check allowed and denied mounts/devices for hints
Browse files Browse the repository at this point in the history 
Check if user has specified any allowed or denied mount or device
paths when generating hints. The code first checks the deny path
list and then the allowed one.

The annotations could look like this:

  annotations:
    allow.topologyhints.resource-policy.nri.io/pod: |+
      type: prefix
      paths:
        - /foo/bar/whitelisted-path1
        - /xy-zy/another-whitelisted-path1
    allow.topologyhints.resource-policy.nri.io/container.pod0c0: |+
      type: glob
      paths:
        - /whitelisted-path*2
        - /xy-zy/another-whitelisted-path2
    deny.topologyhints.resource-policy.nri.io: |+
      type: prefix
      paths:
        - /foo/bar/blacklisted-path3
    deny.topologyhints.resource-policy.nri.io/pod: |+
      type: glob
      paths:
        - /blacklisted-path*4
    deny.topologyhints.resource-policy.nri.io/container.pod0c1: |+
      type: prefix
      paths:
        - /foo/bar/blacklisted-path5
        - /xy-zy/another-blacklisted-path5

Signed-off-by: Jukka Rissanen <[email protected]>
  • Loading branch information
@jukkar @klihub
jukkar authored and klihub committed Jun 20, 2023
1 parent a69df1b commit b2c2fb4
Show file tree
Hide file tree
Showing 2 changed files with 188 additions and 4 deletions.
43 changes: 43 additions & 0 deletions docs/resource-policy/policy/topology-aware.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -430,3 +430,46 @@ metadata:
prefer-reserved-cpus.resource-policy.nri.io/pod: "true"
prefer-reserved-cpus.resource-policy.nri.io/container.special: "false"
```

## Allowing or denying mount/device paths via annotations

User is able mark certain pods and containers to have allowed or denied
paths for mounts or devices. What this means is that when the system
is generating topology hints, it will consult this allowed / denied path
list to determine what hints are created. The deny path list is checked
first and then the allowed path list. This means that you can deny all
the mount/device paths and then allow only the needed ones for example.
User can either set the path with "prefix" (this is the default) or with
"glob" type. The "prefix" type means that the prefix of the mount/device
path is checked for matches. The "glob" type means that user is able to
put wildcards to the matched paths.

For example:

```yaml
metadata:
annotations:
allow.topologyhints.resource-policy.nri.io/pod: |+
type: prefix
paths:
- /foo/bar/whitelisted-path1
- /xy-zy/another-whitelisted-path1
allow.topologyhints.resource-policy.nri.io/container.pod0c0: |+
type: glob
paths:
- /whitelisted-path*2
- /xy-zy/another-whitelisted-path2
deny.topologyhints.resource-policy.nri.io: |+
type: prefix
paths:
- /foo/bar/blacklisted-path3
deny.topologyhints.resource-policy.nri.io/pod: |+
type: glob
paths:
- /blacklisted-path*4
deny.topologyhints.resource-policy.nri.io/container.pod0c1: |+
type: prefix
paths:
- /foo/bar/blacklisted-path5
- /xy-zy/another-blacklisted-path5
```
149 changes: 145 additions & 4 deletions pkg/resmgr/cache/container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package cache
import (
"encoding/json"
"fmt"
"path/filepath"
"regexp"
"sort"
"strconv"
Expand All @@ -30,8 +31,60 @@ import (
nri "github.com/containerd/nri/pkg/api"
v1 "k8s.io/api/core/v1"
resapi "k8s.io/apimachinery/pkg/api/resource"
"sigs.k8s.io/yaml"
)

type MatchType int

const (
PrefixMatch MatchType = iota
GlobMatch
)

type PathList struct {
Type MatchType `yaml:"type"`
Paths []string `yaml:"paths"`
}

func (t *MatchType) UnmarshalJSON(data []byte) error {
switch string(data) {
case "\"prefix\"":
*t = PrefixMatch
case "\"glob\"":
*t = GlobMatch
default:
return fmt.Errorf("invalid MatchType %s", string(data))
}

return nil
}

func (c *container) getAllowDenyPathList(typeStr string) (*PathList, bool, error) {
var hints string
var v PathList
var ok bool

if hints, ok = c.GetEffectiveAnnotation(typeStr + "." + TopologyHintsKey); !ok {
log.Debug("Cannot get %s hints for %s", typeStr, c.GetName())
return nil, false, nil
}

if err := yaml.Unmarshal([]byte(hints), &v); err != nil {
log.Debug("Error (%v) when trying to parse \"%s\"", err, hints)
return nil, false, err
}

return &v, true, nil
}

func (c *container) getAllowPathList() (*PathList, bool, error) {
return c.getAllowDenyPathList("allow")
}

func (c *container) getDenyPathList() (*PathList, bool, error) {
return c.getAllowDenyPathList("deny")
}

// Create and initialize a cached container.
func (cch *cache) createContainer(nriCtr *nri.Container) (*container, error) {
podID := nriCtr.GetPodSandboxId()
Expand All @@ -58,6 +111,62 @@ func (cch *cache) createContainer(nriCtr *nri.Container) (*container, error) {
return c, nil
}

func checkAllowedAndDeniedPaths(hostPath string, allowPathList, denyPathList *PathList) bool {
var denied bool

// Currently we first check deny list, and then allow list
if denyPathList != nil {
for _, path := range denyPathList.Paths {
var matched bool
var err error

if denyPathList.Type == GlobMatch {
matched, err = filepath.Match(path, hostPath)
} else {
// Note that match requires pattern to match all of name, not just a substring.
matched = strings.HasPrefix(hostPath, path)
}

if err != nil {
log.Error("Malformed pattern \"%s\"", matched)
return false
}

if matched {
log.Debug("Deny match, removing %s from hints", path)
denied = true
break
}
}
}

if allowPathList != nil {
for _, path := range allowPathList.Paths {
var matched bool
var err error

if allowPathList.Type == GlobMatch {
matched, err = filepath.Match(path, hostPath)
} else {
// Note that match requires pattern to match all of name, not just a substring.
matched = strings.HasPrefix(hostPath, path)
}

if err != nil {
log.Error("Malformed pattern \"%s\"", matched)
return denied
}

if matched {
log.Debug("Allow match, adding %s to hints", path)
return false
}
}
}

return denied
}

func (c *container) generateTopologyHints() {
var (
mountHints = true
Expand Down Expand Up @@ -88,10 +197,25 @@ func (c *container) generateTopologyHints() {
}
}

allowPathList, ok, err := c.getAllowPathList()
if ok {
// Ignore any errors as that indicates that there were no hints specified
if err == nil {
log.Debug("Allow hints %v", allowPathList)
}
}

denyPathList, ok, err := c.getDenyPathList()
if ok {
if err == nil {
log.Debug("Deny hints %v", denyPathList)
}
}

if mountHints {
for _, m := range c.Ctr.GetMounts() {
readOnly := isReadOnlyMount(m)
if hints := getTopologyHintsForMount(m.Destination, m.Source, readOnly); len(hints) > 0 {
if hints := getTopologyHintsForMount(m.Destination, m.Source, readOnly, allowPathList, denyPathList); len(hints) > 0 {
c.TopologyHints = topology.MergeTopologyHints(c.TopologyHints, hints)
}
}
Expand All @@ -102,7 +226,7 @@ func (c *container) generateTopologyHints() {
if deviceHints {
for _, d := range c.Ctr.GetLinux().GetDevices() {
if !isReadOnlyDevice(c.Ctr.GetLinux().GetResources().GetDevices(), d) {
if hints := getTopologyHintsForDevice(d.Type, d.Major, d.Minor); len(hints) > 0 {
if hints := getTopologyHintsForDevice(d.Type, d.Major, d.Minor, allowPathList, denyPathList); len(hints) > 0 {
c.TopologyHints = topology.MergeTopologyHints(c.TopologyHints, hints)
}
}
Expand Down Expand Up @@ -530,7 +654,7 @@ var (
}
)

func getTopologyHintsForMount(hostPath, containerPath string, readOnly bool) topology.Hints {
func getTopologyHintsForMount(hostPath, containerPath string, readOnly bool, allowPathList, denyPathList *PathList) topology.Hints {

if readOnly {
// if device or path is read-only, assume it as non-important for now
Expand All @@ -555,7 +679,19 @@ func getTopologyHintsForMount(hostPath, containerPath string, readOnly bool) top
}
}

// First check the hostPath before resolving to device path
if denied := checkAllowedAndDeniedPaths(hostPath, allowPathList, denyPathList); denied {
// Ignoring hints for this path
return topology.Hints{}
}

if devPath, err := topology.FindSysFsDevice(hostPath); err == nil {
// Check against the resolved device path
if denied := checkAllowedAndDeniedPaths(devPath, allowPathList, denyPathList); denied {
// Ignoring hints for this path
return topology.Hints{}
}

// errors are ignored
if hints, err := topology.NewTopologyHints(devPath); err == nil && len(hints) > 0 {
return hints
Expand All @@ -565,10 +701,15 @@ func getTopologyHintsForMount(hostPath, containerPath string, readOnly bool) top
return topology.Hints{}
}

func getTopologyHintsForDevice(devType string, major, minor int64) topology.Hints {
func getTopologyHintsForDevice(devType string, major, minor int64, allowPathList, denyPathList *PathList) topology.Hints {
log.Debug("getting topology hints for device <%s %d,%d>", devType, major, minor)

if devPath, err := topology.FindGivenSysFsDevice(devType, major, minor); err == nil {
if denied := checkAllowedAndDeniedPaths(devPath, allowPathList, denyPathList); denied {
// Ignoring hints for this device
return topology.Hints{}
}

// errors are ignored
if hints, err := topology.NewTopologyHints(devPath); err == nil && len(hints) > 0 {
return hints
Expand Down

0 comments on commit b2c2fb4

Please sign in to comment.