Skip to content

Commit

Permalink
feat(yaml): Allow remapping raw YAML files
Browse files Browse the repository at this point in the history 
By default reimage processes K8S manifests. If `--input=yaml` is passed
on the CLI, reimage processes files as raw YAML. The actual fields to
be remapped must be identified using the `-rules-config` file. Any rules
using a kind of "Raw" will be used (these are ignored by regular k8s
processing.

```yaml
- kind: Raw
  imageJSONP:
  - '$.images.*'
```
will rename all the images pointed to by identifiers under the `images`
key.

```yaml
---
images:
  myImage: "alpine:latest"
  yourImage: "docker:dind"
```

Signed-off-by: Tristan Colgate-McFarlane <[email protected]>
  • Loading branch information
@tcolgate
tcolgate committed Feb 6, 2024
1 parent abedf2d commit a7e5789
Show file tree
Hide file tree
Showing 3 changed files with 156 additions and 45 deletions.
26 changes: 20 additions & 6 deletions cmd/reimage/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,13 @@ import (
"k8s.io/apimachinery/pkg/util/yaml"
)

type inputFn func(io.Writer, io.Reader, reimage.Updater) error

type app struct {
Version bool
MappingsOnly bool
Input string
inputFn inputFn
Ignore string
ignore *regexp.Regexp
RenameIgnore string
Expand Down Expand Up @@ -82,7 +86,8 @@ func setup() (*app, error) {
flag.BoolVar(&a.DryRun, "dryrun", false, "only log actions")
flag.BoolVar(&a.Debug, "debug", false, "enable debug logging")

flag.StringVar(&a.RulesConfigFile, "rules-config", "", "yaml definition of kind/image-path mappings")
flag.StringVar(&a.Input, "input", "k8s", "type of input, (k8s or yaml)")
flag.StringVar(&a.RulesConfigFile, "rules-config", "", "yaml definition of kind/image-path mappings, (kind: raw for raw yaml input rules)")

flag.BoolVar(&a.MappingsOnly, "mappings-only", false, "skip yaml processing, run copying, checks and attestations on all images in the static mappings")

Expand Down Expand Up @@ -187,6 +192,15 @@ func setup() (*app, error) {
return &a, fmt.Errorf("could not parse trivy command, %w", err)
}

switch a.Input {
case "k8s":
a.inputFn = reimage.ProcessK8s
case "yaml":
a.inputFn = reimage.ProcessRawYAML
default:
return &a, fmt.Errorf("invalid input type, should be k8s or yaml")
}

return &a, nil
}

Expand Down Expand Up @@ -592,13 +606,13 @@ func main() {

if !app.MappingsOnly {
s := &reimage.RenameUpdater{
Ignore: app.ignore,
Remapper: rm,
UnstructuredImagesFinder: app.imagFinder,
ForceDigests: app.RenameForceToDigest,
Ignore: app.ignore,
Remapper: rm,
ImagesFinder: app.imagFinder,
ForceDigests: app.RenameForceToDigest,
}

err = reimage.Process(os.Stdout, os.Stdin, s)
err = app.inputFn(os.Stdout, os.Stdin, s)
if err != nil {
app.log.Error(fmt.Errorf("failed processing input, %w", err).Error())
os.Exit(1)
Expand Down
162 changes: 129 additions & 33 deletions reimage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/google/go-containerregistry/pkg/crane"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
yamlv3 "gopkg.in/yaml.v3"

appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
Expand Down Expand Up @@ -372,12 +373,18 @@ func (t *EnsureRemapper) ReMap(h *History) error {
return nil
}

// ErrSkip if this is returned by ReMap then MultiRemapper will
// ignore this image and skip further processing
var ErrSkip = errors.New("skip further processing")

// IgnoreRemapper will return ErrSkip for any image name that
// natches the Ignore regexp
type IgnoreRemapper struct {
Ignore *regexp.Regexp
}

// ReMap will return ErrSkip for any image name that
// natches the Ignore regexp
func (t *IgnoreRemapper) ReMap(h *History) error {
name := h.Latest().Name()
if t.Ignore != nil && t.Ignore.MatchString(name) {
Expand Down Expand Up @@ -445,17 +452,18 @@ func (r *RecorderRemapper) Mappings() (map[string]QualifiedImage, error) {
// found. Calling the Set method on the map values will replace the discovered
// image name with a replacement.
type ImagesFinder interface {
FindImages(obj *unstructured.Unstructured) (map[string]ImageSetters, error)
FindImages(obj any) (map[string]ImageSetters, error)
FindK8sImages(obj *unstructured.Unstructured) (map[string]ImageSetters, error)
}

// RenameUpdater applies the Remapper to all images found in object passed to Update.
// For Objects of unknown types the UnstructuredImagesFinder is used.
// TODO(tcm): rename this thinger.
type RenameUpdater struct {
Ignore *regexp.Regexp // Completely ignore images strings matching this regexp
UnstructuredImagesFinder ImagesFinder
Remapper Remapper
ForceDigests bool
Ignore *regexp.Regexp // Completely ignore images strings matching this regexp
ImagesFinder ImagesFinder
Remapper Remapper
ForceDigests bool
}

func (s *RenameUpdater) remapImageString(img string) (string, error) {
Expand Down Expand Up @@ -517,9 +525,53 @@ func (s *RenameUpdater) processPodSpec(spec *corev1.PodSpec) error {
return nil
}

func (s *RenameUpdater) processUnstructured(obj *unstructured.Unstructured) error {
matches, err := s.ImagesFinder.FindK8sImages(obj)
if err != nil {
return err
}
for img, setters := range matches {
newImg, err := s.remapImageString(img)
if err != nil {
return err
}

setters.Set(newImg)
}
return nil
}

func (s *RenameUpdater) processRaw(obj any) error {
matches, err := s.ImagesFinder.FindImages(obj)
if err != nil {
return err
}
for img, setters := range matches {
newImg, err := s.remapImageString(img)
if err != nil {
return err
}

setters.Set(newImg)
}
return nil
}

// RawYAML is intended to wrap objects that are return from raw YAML unmarshaling
// the Update method of RenameUpdater will process these by looking for images
// using FindImages (rather than FindK8sImages). By default this will be any
// rules that were compiled with "Kind: Raw"
type RawYAML struct {
Object any
}

// Update applies the Remapper to all found images in the object
func (s *RenameUpdater) Update(obj any) error {
switch t := obj.(type) {
case RawYAML:
return s.processRaw(t.Object)
case *RawYAML:
return s.processRaw(t.Object)
case *corev1.Pod:
return s.processPodSpec(&t.Spec)
case *corev1.PodList:
Expand Down Expand Up @@ -591,18 +643,7 @@ func (s *RenameUpdater) Update(obj any) error {
t.Items[i] = p
}
case *unstructured.Unstructured:
matches, err := s.UnstructuredImagesFinder.FindImages(t)
if err != nil {
return err
}
for img, setters := range matches {
newImg, err := s.remapImageString(img)
if err != nil {
return err
}

setters.Set(newImg)
}
return s.processUnstructured(t)
case *runtime.Unknown:
return fmt.Errorf("cannot process unknown resource type")
default:
Expand All @@ -617,9 +658,9 @@ type Updater interface {
Update(obj any) error
}

// Process runs the Updater for each kubernetes resource found in the file.
// ProcessK8s runs the Updater for each kubernetes resource found in the file.
// Unknown field are converted to
func Process(w io.Writer, r io.Reader, u Updater) error {
func ProcessK8s(w io.Writer, r io.Reader, u Updater) error {
yr := yaml.NewYAMLReader(bufio.NewReader(r))

decoder := scheme.Codecs.UniversalDeserializer()
Expand Down Expand Up @@ -655,15 +696,56 @@ func Process(w io.Writer, r io.Reader, u Updater) error {

err = u.Update(obj)
if err != nil {
gvk := obj.GetObjectKind().GroupVersionKind()
return fmt.Errorf("error updating input %#v, %w", gvk, err)
return fmt.Errorf("error updating input %w,", err)
}

pr.PrintObj(obj, w)
}
return nil
}

// ProcessRawYAML runs the Updater for each YAML document
func ProcessRawYAML(w io.Writer, r io.Reader, u Updater) error {
yr := yaml.NewYAMLReader(bufio.NewReader(r))

count := 0
for {
doc, err := yr.Read()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return err
}

var obj any
err = yamlv3.Unmarshal(doc, &obj)
if err != nil {
return fmt.Errorf("could not read YAML input[%d], %#v,", count, err)
}

err = u.Update(&RawYAML{Object: obj})
if err != nil {
return fmt.Errorf("error updating input[%d], %w,", count, err)
}

if count != 0 {
fmt.Fprintln(w, "---")
}

err = func() error {
enc := yamlv3.NewEncoder(w)
defer enc.Close()
return enc.Encode(obj)
}()
if err != nil {
return fmt.Errorf("error encoding output[%d], %w,", count, err)
}
count++
}
return nil
}

type jsonPathFunc func(src interface{}) ([]interface{}, error)

// JSONImageFinderConfig describes the settings for finding
Expand Down Expand Up @@ -697,12 +779,11 @@ func (ss ImageSetters) Set(img string) {
s(img)
}
}

func (jm jsonImageFinder) FindImages(obj *unstructured.Unstructured) (map[string]ImageSetters, error) {
func (jm jsonImageFinder) FindImages(obj any) (map[string]ImageSetters, error) {
res := map[string]ImageSetters{}

for _, jpf := range jm.imageJSONPFns {
vs, err := jpf((map[string]interface{})(obj.Object))
vs, err := jpf(obj)
if err != nil {
return nil, fmt.Errorf("jsonpath function failed, got %w", err)
}
Expand All @@ -721,29 +802,44 @@ func (jm jsonImageFinder) FindImages(obj *unstructured.Unstructured) (map[string
return res, nil
}

func (jm jsonImageFinder) FindK8sImages(obj *unstructured.Unstructured) (map[string]ImageSetters, error) {
return jm.FindImages((map[string]interface{})(obj.Object))
}

type jsonImageFinders []*jsonImageFinder

func (jms jsonImageFinders) FindImages(obj *unstructured.Unstructured) (map[string]ImageSetters, error) {
func (jms jsonImageFinders) FindImages(obj any) (map[string]ImageSetters, error) {
for i := range jms {
if jms[i].matches(obj) {
if jms[i].kind == nil {
return jms[i].FindImages(obj)
}
}
return nil, nil
}

func (jms jsonImageFinders) FindK8sImages(obj *unstructured.Unstructured) (map[string]ImageSetters, error) {
for i := range jms {
if jms[i].kind != nil && jms[i].matches(obj) {
return jms[i].FindK8sImages(obj)
}
}
return nil, nil
}

func compileJSONImageFinder(cfg JSONImageFinderConfig) (*jsonImageFinder, error) {
var err error
jm := jsonImageFinder{}

jm.kind, err = regexp.Compile(cfg.Kind)
if err != nil {
return nil, fmt.Errorf("failed to compile Kind regexp, %v", err)
}
if cfg.Kind != "Raw" {
jm.kind, err = regexp.Compile(cfg.Kind)
if err != nil {
return nil, fmt.Errorf("failed to compile Kind regexp, %v", err)
}

jm.apiVersion, err = regexp.Compile(cfg.APIVersion)
if err != nil {
return nil, fmt.Errorf("failed to compile APIVersion regexp, %v", err)
jm.apiVersion, err = regexp.Compile(cfg.APIVersion)
if err != nil {
return nil, fmt.Errorf("failed to compile APIVersion regexp, %v", err)
}
}

config := jsonpath.Config{}
Expand Down
13 changes: 7 additions & 6 deletions reimage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func TestProcess_invalid_yaml(t *testing.T) {
`
out := bytes.NewBuffer([]byte{})
tu := &testUpdater{}
err := Process(out, bytes.NewBufferString(in), tu)
err := ProcessK8s(out, bytes.NewBufferString(in), tu)
if err == nil {
t.Fatalf("expected invalid json to faili")
}
Expand All @@ -93,7 +93,7 @@ func TestProcess_empty_yamls(t *testing.T) {
`
out := bytes.NewBuffer([]byte{})
tu := &testUpdater{}
err := Process(out, bytes.NewBufferString(in), tu)
err := ProcessK8s(out, bytes.NewBufferString(in), tu)
if err != nil {
t.Fatalf("empty yaml blobs should parse")
}
Expand All @@ -107,7 +107,7 @@ noKind: nonehere
`
out := bytes.NewBuffer([]byte{})
tu := &testUpdater{}
err := Process(out, bytes.NewBufferString(in), tu)
err := ProcessK8s(out, bytes.NewBufferString(in), tu)
if err != nil {
t.Fatalf("non-kube yaml is passed on")
}
Expand All @@ -129,7 +129,7 @@ spec:

out := bytes.NewBuffer([]byte{})
tu := &testUpdater{}
err := Process(out, bytes.NewBufferString(in), tu)
err := ProcessK8s(out, bytes.NewBufferString(in), tu)
if err != nil {
t.Fatalf("non-kube yaml is passed on")
}
Expand Down Expand Up @@ -165,10 +165,11 @@ spec:
err: te,
}

err := Process(out, bytes.NewBufferString(in), tu)
err := ProcessK8s(out, bytes.NewBufferString(in), tu)
if err == nil {
t.Fatalf("expected an error")
}
t.Logf("err: %T %v", err, err)
if !errors.Is(err, te) {
t.Fatalf("expected a testError")
}
Expand Down Expand Up @@ -293,7 +294,7 @@ func TestCompileJSONImageFinders(t *testing.T) {
}

obj := &unstructured.Unstructured{Object: tt.dataIn}
ms, err := mtchr.FindImages(obj)
ms, err := mtchr.FindK8sImages(obj)
if err != nil {
t.Fatalf("master errored, %v", err)
}
Expand Down

0 comments on commit a7e5789

Please sign in to comment.