From a7e578907139c3f89939149053d65dd4d528edef Mon Sep 17 00:00:00 2001 From: Tristan Colgate-McFarlane Date: Tue, 6 Feb 2024 09:27:58 +0000 Subject: [PATCH] feat(yaml): Allow remapping raw YAML files By default reimage processes K8S manifests. If `--input=yaml` is passed on the CLI, reimage processes files as raw YAML. The actual fields to be remapped must be identified using the `-rules-config` file. Any rules using a kind of "Raw" will be used (these are ignored by regular k8s processing. ```yaml - kind: Raw imageJSONP: - '$.images.*' ``` will rename all the images pointed to by identifiers under the `images` key. ```yaml --- images: myImage: "alpine:latest" yourImage: "docker:dind" ``` Signed-off-by: Tristan Colgate-McFarlane --- cmd/reimage/main.go | 26 +++++-- reimage.go | 162 +++++++++++++++++++++++++++++++++++--------- reimage_test.go | 13 ++-- 3 files changed, 156 insertions(+), 45 deletions(-) diff --git a/cmd/reimage/main.go b/cmd/reimage/main.go index 9785655..4aa5194 100644 --- a/cmd/reimage/main.go +++ b/cmd/reimage/main.go @@ -32,9 +32,13 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" ) +type inputFn func(io.Writer, io.Reader, reimage.Updater) error + type app struct { Version bool MappingsOnly bool + Input string + inputFn inputFn Ignore string ignore *regexp.Regexp RenameIgnore string @@ -82,7 +86,8 @@ func setup() (*app, error) { flag.BoolVar(&a.DryRun, "dryrun", false, "only log actions") flag.BoolVar(&a.Debug, "debug", false, "enable debug logging") - flag.StringVar(&a.RulesConfigFile, "rules-config", "", "yaml definition of kind/image-path mappings") + flag.StringVar(&a.Input, "input", "k8s", "type of input, (k8s or yaml)") + flag.StringVar(&a.RulesConfigFile, "rules-config", "", "yaml definition of kind/image-path mappings, (kind: raw for raw yaml input rules)") flag.BoolVar(&a.MappingsOnly, "mappings-only", false, "skip yaml processing, run copying, checks and attestations on all images in the static mappings") @@ -187,6 +192,15 @@ func setup() (*app, error) { return &a, fmt.Errorf("could not parse trivy command, %w", err) } + switch a.Input { + case "k8s": + a.inputFn = reimage.ProcessK8s + case "yaml": + a.inputFn = reimage.ProcessRawYAML + default: + return &a, fmt.Errorf("invalid input type, should be k8s or yaml") + } + return &a, nil } @@ -592,13 +606,13 @@ func main() { if !app.MappingsOnly { s := &reimage.RenameUpdater{ - Ignore: app.ignore, - Remapper: rm, - UnstructuredImagesFinder: app.imagFinder, - ForceDigests: app.RenameForceToDigest, + Ignore: app.ignore, + Remapper: rm, + ImagesFinder: app.imagFinder, + ForceDigests: app.RenameForceToDigest, } - err = reimage.Process(os.Stdout, os.Stdin, s) + err = app.inputFn(os.Stdout, os.Stdin, s) if err != nil { app.log.Error(fmt.Errorf("failed processing input, %w", err).Error()) os.Exit(1) diff --git a/reimage.go b/reimage.go index 1b2b08d..b9de73b 100644 --- a/reimage.go +++ b/reimage.go @@ -23,6 +23,7 @@ import ( "github.com/google/go-containerregistry/pkg/crane" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote/transport" + yamlv3 "gopkg.in/yaml.v3" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" @@ -372,12 +373,18 @@ func (t *EnsureRemapper) ReMap(h *History) error { return nil } +// ErrSkip if this is returned by ReMap then MultiRemapper will +// ignore this image and skip further processing var ErrSkip = errors.New("skip further processing") +// IgnoreRemapper will return ErrSkip for any image name that +// natches the Ignore regexp type IgnoreRemapper struct { Ignore *regexp.Regexp } +// ReMap will return ErrSkip for any image name that +// natches the Ignore regexp func (t *IgnoreRemapper) ReMap(h *History) error { name := h.Latest().Name() if t.Ignore != nil && t.Ignore.MatchString(name) { @@ -445,17 +452,18 @@ func (r *RecorderRemapper) Mappings() (map[string]QualifiedImage, error) { // found. Calling the Set method on the map values will replace the discovered // image name with a replacement. type ImagesFinder interface { - FindImages(obj *unstructured.Unstructured) (map[string]ImageSetters, error) + FindImages(obj any) (map[string]ImageSetters, error) + FindK8sImages(obj *unstructured.Unstructured) (map[string]ImageSetters, error) } // RenameUpdater applies the Remapper to all images found in object passed to Update. // For Objects of unknown types the UnstructuredImagesFinder is used. // TODO(tcm): rename this thinger. type RenameUpdater struct { - Ignore *regexp.Regexp // Completely ignore images strings matching this regexp - UnstructuredImagesFinder ImagesFinder - Remapper Remapper - ForceDigests bool + Ignore *regexp.Regexp // Completely ignore images strings matching this regexp + ImagesFinder ImagesFinder + Remapper Remapper + ForceDigests bool } func (s *RenameUpdater) remapImageString(img string) (string, error) { @@ -517,9 +525,53 @@ func (s *RenameUpdater) processPodSpec(spec *corev1.PodSpec) error { return nil } +func (s *RenameUpdater) processUnstructured(obj *unstructured.Unstructured) error { + matches, err := s.ImagesFinder.FindK8sImages(obj) + if err != nil { + return err + } + for img, setters := range matches { + newImg, err := s.remapImageString(img) + if err != nil { + return err + } + + setters.Set(newImg) + } + return nil +} + +func (s *RenameUpdater) processRaw(obj any) error { + matches, err := s.ImagesFinder.FindImages(obj) + if err != nil { + return err + } + for img, setters := range matches { + newImg, err := s.remapImageString(img) + if err != nil { + return err + } + + setters.Set(newImg) + } + return nil +} + +// RawYAML is intended to wrap objects that are return from raw YAML unmarshaling +// the Update method of RenameUpdater will process these by looking for images +// using FindImages (rather than FindK8sImages). By default this will be any +// rules that were compiled with "Kind: Raw" +type RawYAML struct { + Object any +} + // Update applies the Remapper to all found images in the object func (s *RenameUpdater) Update(obj any) error { switch t := obj.(type) { + case RawYAML: + return s.processRaw(t.Object) + case *RawYAML: + return s.processRaw(t.Object) case *corev1.Pod: return s.processPodSpec(&t.Spec) case *corev1.PodList: @@ -591,18 +643,7 @@ func (s *RenameUpdater) Update(obj any) error { t.Items[i] = p } case *unstructured.Unstructured: - matches, err := s.UnstructuredImagesFinder.FindImages(t) - if err != nil { - return err - } - for img, setters := range matches { - newImg, err := s.remapImageString(img) - if err != nil { - return err - } - - setters.Set(newImg) - } + return s.processUnstructured(t) case *runtime.Unknown: return fmt.Errorf("cannot process unknown resource type") default: @@ -617,9 +658,9 @@ type Updater interface { Update(obj any) error } -// Process runs the Updater for each kubernetes resource found in the file. +// ProcessK8s runs the Updater for each kubernetes resource found in the file. // Unknown field are converted to -func Process(w io.Writer, r io.Reader, u Updater) error { +func ProcessK8s(w io.Writer, r io.Reader, u Updater) error { yr := yaml.NewYAMLReader(bufio.NewReader(r)) decoder := scheme.Codecs.UniversalDeserializer() @@ -655,8 +696,7 @@ func Process(w io.Writer, r io.Reader, u Updater) error { err = u.Update(obj) if err != nil { - gvk := obj.GetObjectKind().GroupVersionKind() - return fmt.Errorf("error updating input %#v, %w", gvk, err) + return fmt.Errorf("error updating input %w,", err) } pr.PrintObj(obj, w) @@ -664,6 +704,48 @@ func Process(w io.Writer, r io.Reader, u Updater) error { return nil } +// ProcessRawYAML runs the Updater for each YAML document +func ProcessRawYAML(w io.Writer, r io.Reader, u Updater) error { + yr := yaml.NewYAMLReader(bufio.NewReader(r)) + + count := 0 + for { + doc, err := yr.Read() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + + var obj any + err = yamlv3.Unmarshal(doc, &obj) + if err != nil { + return fmt.Errorf("could not read YAML input[%d], %#v,", count, err) + } + + err = u.Update(&RawYAML{Object: obj}) + if err != nil { + return fmt.Errorf("error updating input[%d], %w,", count, err) + } + + if count != 0 { + fmt.Fprintln(w, "---") + } + + err = func() error { + enc := yamlv3.NewEncoder(w) + defer enc.Close() + return enc.Encode(obj) + }() + if err != nil { + return fmt.Errorf("error encoding output[%d], %w,", count, err) + } + count++ + } + return nil +} + type jsonPathFunc func(src interface{}) ([]interface{}, error) // JSONImageFinderConfig describes the settings for finding @@ -697,12 +779,11 @@ func (ss ImageSetters) Set(img string) { s(img) } } - -func (jm jsonImageFinder) FindImages(obj *unstructured.Unstructured) (map[string]ImageSetters, error) { +func (jm jsonImageFinder) FindImages(obj any) (map[string]ImageSetters, error) { res := map[string]ImageSetters{} for _, jpf := range jm.imageJSONPFns { - vs, err := jpf((map[string]interface{})(obj.Object)) + vs, err := jpf(obj) if err != nil { return nil, fmt.Errorf("jsonpath function failed, got %w", err) } @@ -721,29 +802,44 @@ func (jm jsonImageFinder) FindImages(obj *unstructured.Unstructured) (map[string return res, nil } +func (jm jsonImageFinder) FindK8sImages(obj *unstructured.Unstructured) (map[string]ImageSetters, error) { + return jm.FindImages((map[string]interface{})(obj.Object)) +} + type jsonImageFinders []*jsonImageFinder -func (jms jsonImageFinders) FindImages(obj *unstructured.Unstructured) (map[string]ImageSetters, error) { +func (jms jsonImageFinders) FindImages(obj any) (map[string]ImageSetters, error) { for i := range jms { - if jms[i].matches(obj) { + if jms[i].kind == nil { return jms[i].FindImages(obj) } } return nil, nil } +func (jms jsonImageFinders) FindK8sImages(obj *unstructured.Unstructured) (map[string]ImageSetters, error) { + for i := range jms { + if jms[i].kind != nil && jms[i].matches(obj) { + return jms[i].FindK8sImages(obj) + } + } + return nil, nil +} + func compileJSONImageFinder(cfg JSONImageFinderConfig) (*jsonImageFinder, error) { var err error jm := jsonImageFinder{} - jm.kind, err = regexp.Compile(cfg.Kind) - if err != nil { - return nil, fmt.Errorf("failed to compile Kind regexp, %v", err) - } + if cfg.Kind != "Raw" { + jm.kind, err = regexp.Compile(cfg.Kind) + if err != nil { + return nil, fmt.Errorf("failed to compile Kind regexp, %v", err) + } - jm.apiVersion, err = regexp.Compile(cfg.APIVersion) - if err != nil { - return nil, fmt.Errorf("failed to compile APIVersion regexp, %v", err) + jm.apiVersion, err = regexp.Compile(cfg.APIVersion) + if err != nil { + return nil, fmt.Errorf("failed to compile APIVersion regexp, %v", err) + } } config := jsonpath.Config{} diff --git a/reimage_test.go b/reimage_test.go index 54b2ce0..31af500 100644 --- a/reimage_test.go +++ b/reimage_test.go @@ -78,7 +78,7 @@ func TestProcess_invalid_yaml(t *testing.T) { ` out := bytes.NewBuffer([]byte{}) tu := &testUpdater{} - err := Process(out, bytes.NewBufferString(in), tu) + err := ProcessK8s(out, bytes.NewBufferString(in), tu) if err == nil { t.Fatalf("expected invalid json to faili") } @@ -93,7 +93,7 @@ func TestProcess_empty_yamls(t *testing.T) { ` out := bytes.NewBuffer([]byte{}) tu := &testUpdater{} - err := Process(out, bytes.NewBufferString(in), tu) + err := ProcessK8s(out, bytes.NewBufferString(in), tu) if err != nil { t.Fatalf("empty yaml blobs should parse") } @@ -107,7 +107,7 @@ noKind: nonehere ` out := bytes.NewBuffer([]byte{}) tu := &testUpdater{} - err := Process(out, bytes.NewBufferString(in), tu) + err := ProcessK8s(out, bytes.NewBufferString(in), tu) if err != nil { t.Fatalf("non-kube yaml is passed on") } @@ -129,7 +129,7 @@ spec: out := bytes.NewBuffer([]byte{}) tu := &testUpdater{} - err := Process(out, bytes.NewBufferString(in), tu) + err := ProcessK8s(out, bytes.NewBufferString(in), tu) if err != nil { t.Fatalf("non-kube yaml is passed on") } @@ -165,10 +165,11 @@ spec: err: te, } - err := Process(out, bytes.NewBufferString(in), tu) + err := ProcessK8s(out, bytes.NewBufferString(in), tu) if err == nil { t.Fatalf("expected an error") } + t.Logf("err: %T %v", err, err) if !errors.Is(err, te) { t.Fatalf("expected a testError") } @@ -293,7 +294,7 @@ func TestCompileJSONImageFinders(t *testing.T) { } obj := &unstructured.Unstructured{Object: tt.dataIn} - ms, err := mtchr.FindImages(obj) + ms, err := mtchr.FindK8sImages(obj) if err != nil { t.Fatalf("master errored, %v", err) }