Merge pull request #1883 from lcarva/EC-756

Add ec.oci.image_files rego function

acceptance/examples/oci_image_files.rego

@@ -0,0 +1,24 @@
+package files
+
+import rego.v1
+
+
+# METADATA
+# custom:
+#   short_name: match
+deny contains result if {
+    files := ec.oci.image_files(input.image.ref, ["manifests"])
+    not matches(files)
+    result := {
+        "code": "files.match",
+        "msg": json.marshal(files)
+    }
+}
+
+matches(files) if {
+    files["manifests/some.crd.yaml"].kind == "CustomResourceDefinition"
+    files["manifests/some.crd.yaml"].spec.names.singular == "memcached"
+
+    files["manifests/some.clusterserviceversion.yaml"].kind == "ClusterServiceVersion"
+    files["manifests/some.clusterserviceversion.yaml"].spec.displayName == "Memcached Operator"
+}

docs/modules/ROOT/pages/ec_oci_image_files.adoc

@@ -0,0 +1,21 @@
+= ec.oci.image_files
+
+Fetch structured files (YAML or JSON) from within an image.
+
+== Usage
+
+  files = ec.oci.image_files(ref: string, paths: array<string>)
+
+== Parameters
+
+* `ref` (`string`): OCI image reference
+* `paths` (`array<string>`): the list of paths
+
+== Return
+
+`files` (`object`): object representing the extracted files
+
+The object contains dynamic attributes.
+The attributes are of `string` type and represent the full path of the file within the image.
+The values are of `any` type and hold the file contents.
+

docs/modules/ROOT/pages/rego_builtins.adoc

@@ -10,6 +10,8 @@ information.
 |===
 |xref:ec_oci_blob.adoc[ec.oci.blob]
 |Fetch a blob from an OCI registry.
+|xref:ec_oci_image_files.adoc[ec.oci.image_files]
+|Fetch structured files (YAML or JSON) from within an image.
 |xref:ec_oci_image_manifest.adoc[ec.oci.image_manifest]
 |Fetch an Image Manifest from an OCI registry.
 |xref:ec_purl_is_valid.adoc[ec.purl.is_valid]

docs/modules/ROOT/partials/rego_nav.adoc

@@ -1,5 +1,6 @@
 * xref:rego_builtins.adoc[Rego Reference]
 ** xref:ec_oci_blob.adoc[ec.oci.blob]
+** xref:ec_oci_image_files.adoc[ec.oci.image_files]
 ** xref:ec_oci_image_manifest.adoc[ec.oci.image_manifest]
 ** xref:ec_purl_is_valid.adoc[ec.purl.is_valid]
 ** xref:ec_purl_parse.adoc[ec.purl.parse]

features/__snapshots__/validate_image.snap

@@ -5083,3 +5083,80 @@ Error: success criteria not met
   "effective-time": "${TIMESTAMP}"
 }
 ---
+
+[fetch OCI image files:stdout - 1]
+{
+  "success": true,
+  "components": [
+    {
+      "name": "Unnamed",
+      "containerImage": "${REGISTRY}/acceptance/oci-image-files@sha256:${REGISTRY_acceptance/oci-image-files:latest_DIGEST}",
+      "source": {},
+      "successes": [
+        {
+          "msg": "Pass",
+          "metadata": {
+            "code": "builtin.attestation.signature_check"
+          }
+        },
+        {
+          "msg": "Pass",
+          "metadata": {
+            "code": "builtin.attestation.syntax_check"
+          }
+        },
+        {
+          "msg": "Pass",
+          "metadata": {
+            "code": "builtin.image.signature_check"
+          }
+        },
+        {
+          "msg": "Pass",
+          "metadata": {
+            "code": "files.match"
+          }
+        }
+      ],
+      "success": true,
+      "signatures": [
+        {
+          "keyid": "",
+          "sig": "${IMAGE_SIGNATURE_acceptance/oci-image-files}"
+        }
+      ],
+      "attestations": [
+        {
+          "type": "https://in-toto.io/Statement/v0.1",
+          "predicateType": "https://slsa.dev/provenance/v0.2",
+          "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2",
+          "signatures": [
+            {
+              "keyid": "",
+              "sig": "${ATTESTATION_SIGNATURE_acceptance/oci-image-files}"
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "key": "${known_PUBLIC_KEY_JSON}",
+  "policy": {
+    "sources": [
+      {
+        "policy": [
+          "git::https://${GITHOST}/git/oci-image-files-policy"
+        ]
+      }
+    ],
+    "rekorUrl": "${REKOR}",
+    "publicKey": "${known_PUBLIC_KEY}"
+  },
+  "ec-version": "${EC_VERSION}",
+  "effective-time": "${TIMESTAMP}"
+}
+---
+
+[fetch OCI image files:stderr - 1]
+
+---

features/validate_image.feature

@@ -1018,6 +1018,33 @@ Feature: evaluate enterprise contract
     Then the exit status should be 0
     Then the output should match the snapshot
 
+  Scenario: fetch OCI image files
+    Given a key pair named "known"
+    Given an image named "acceptance/oci-image-files" containing a layer with:
+      | manifests/some.crd.yaml                   | examples/some.crd.yaml                   |
+      | manifests/some.clusterserviceversion.yaml | examples/some.clusterserviceversion.yaml |
+    Given a valid image signature of "acceptance/oci-image-files" image signed by the "known" key
+    Given a valid Rekor entry for image signature of "acceptance/oci-image-files"
+    Given a valid attestation of "acceptance/oci-image-files" signed by the "known" key
+    Given a valid Rekor entry for attestation of "acceptance/oci-image-files"
+    Given a git repository named "oci-image-files-policy" with
+      | main.rego | examples/oci_image_files.rego |
+    Given policy configuration named "ec-policy" with specification
+      """
+      {
+        "sources": [
+          {
+            "policy": [
+              "git::https://${GITHOST}/git/oci-image-files-policy"
+            ]
+          }
+        ]
+      }
+      """
+    When ec command is run with "validate image --image ${REGISTRY}/acceptance/oci-image-files --policy acceptance/ec-policy --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes"
+    Then the exit status should be 0
+    Then the output should match the snapshot
+
   Scenario: tracing and debug logging
     Given a key pair named "trace_debug"
       And an image named "acceptance/trace-debug"

internal/documentation/asciidoc/rego/rego.tmpl

@@ -57,6 +57,16 @@
 `{{ .Decl.NamedResult.Name }}` (`{{ if $isObject }}object{{ else }}{{ .Decl.NamedResult.Type }}{{ end }}`): {{.Decl.NamedResult.Descr}}
 {{- if $isObject }}
 
+{{ if gt (len .Decl.NamedResult.Type.StaticProperties) 0 -}}
 The object contains the following attributes:
 {{ template "properties" (params (.Decl.NamedResult.Type.StaticProperties) 1) -}}
+{{ end }}
+
+{{- if .Decl.NamedResult.Type.DynamicProperties -}}
+{{- with .Decl.NamedResult.Type.DynamicProperties -}}
+The object contains dynamic attributes.
+The attributes are of `{{ .Key.Type }}` type and represent {{ .Key.Descr }}.
+The values are of `{{ .Value.Type }}` type and hold {{ .Value.Descr }}.
+{{- end }}
+{{ end }}
 {{- end }}

internal/evaluation_target/application_snapshot_image/application_snapshot_image.go

@@ -132,7 +132,11 @@ func (a *ApplicationSnapshotImage) FetchParentImageConfig(ctx context.Context) e
 
 func (a *ApplicationSnapshotImage) FetchImageFiles(ctx context.Context) error {
 	var err error
-	a.files, err = files.ImageFiles(ctx, a.reference)
+	extractors := []files.Extractor{
+		files.OLMManifest{},
+		files.RedHatManifest{},
+	}
+	a.files, err = files.ImageFiles(ctx, a.reference, extractors)
 	return err
 }
 

internal/fetchers/oci/files/files.go

@@ -33,28 +33,23 @@ import (
 	"github.com/enterprise-contract/ec-cli/internal/utils/oci"
 )
 
-type extractor interface {
-	matcher(v1.Image) (matcher, error)
+type Extractor interface {
+	Matcher(v1.Image) (Matcher, error)
 }
 
-type matcher func(*tar.Header) bool
-
-var supported = []extractor{
-	olmManifest{},
-	redHatManifest{},
-}
+type Matcher func(*tar.Header) bool
 
 var supportedExtensions = []string{".yaml", ".yml", ".json"}
 
-func ImageFiles(ctx context.Context, ref name.Reference) (map[string]json.RawMessage, error) {
+func ImageFiles(ctx context.Context, ref name.Reference, extractors []Extractor) (map[string]json.RawMessage, error) {
 	img, err := oci.NewClient(ctx).Image(ref)
 	if err != nil {
 		return nil, err
 	}
 
-	matchers := make([]matcher, 0, len(supported))
-	for _, f := range supported {
-		if m, err := f.matcher(img); err != nil {
+	matchers := make([]Matcher, 0, len(extractors))
+	for _, f := range extractors {
+		if m, err := f.Matcher(img); err != nil {
 			return nil, err
 		} else if m != nil {
 			matchers = append(matchers, m)
@@ -109,19 +104,19 @@ func ImageFiles(ctx context.Context, ref name.Reference) (map[string]json.RawMes
 	return files, nil
 }
 
-type pathMatcher struct {
-	path string
+type PathMatcher struct {
+	Path string
 }
 
-func (f *pathMatcher) match(header *tar.Header) bool {
+func (f *PathMatcher) Match(header *tar.Header) bool {
 	if header == nil {
 		return false
 	}
 
 	name := header.Name
 
 	// we're only interested in files in `<path>/*`
-	if !strings.EqualFold(path.Dir(name), path.Clean(f.path)) {
+	if !strings.EqualFold(path.Dir(name), path.Clean(f.Path)) {
 		return false
 	}
 
@@ -135,3 +130,14 @@ func (f *pathMatcher) match(header *tar.Header) bool {
 
 	return false
 }
+
+type PathExtractor struct {
+	Path string
+}
+
+func (p PathExtractor) Matcher(img v1.Image) (Matcher, error) {
+	if img == nil {
+		return nil, nil
+	}
+	return (&PathMatcher{Path: p.Path}).Match, nil
+}

internal/fetchers/oci/files/files_test.go

@@ -60,7 +60,8 @@ func TestImageManifests(t *testing.T) {
 
 	ctx := oci.WithClient(context.Background(), &client)
 
-	files, err := ImageFiles(ctx, ref)
+	extractors := []Extractor{OLMManifest{}, RedHatManifest{}}
+	files, err := ImageFiles(ctx, ref, extractors)
 
 	assert.NoError(t, err)
 
@@ -85,18 +86,48 @@ func TestDoesntFetchLayersForUnsupported(t *testing.T) {
 
 	ctx := oci.WithClient(context.Background(), &client)
 
-	files, err := ImageFiles(ctx, ref)
+	extractors := []Extractor{OLMManifest{}, RedHatManifest{}}
+	files, err := ImageFiles(ctx, ref, extractors)
 
 	assert.NoError(t, err)
 	assert.Nil(t, files)
 
 	client.AssertNotCalled(t, "Layers")
 }
 
+func TestCustomExtractor(t *testing.T) {
+	ref := name.MustParseReference("registry.io/repository/image:tag")
+
+	image, err := crane.Image(map[string][]byte{
+		"autoexec.bat":              []byte(`@ECHO OFF`),
+		"manifests/a.json":          []byte(`{"a":1}`),
+		"manifests/b.yaml":          []byte(`b: 2`),
+		"manifests/c.xml":           []byte(`<?xml version="1.0" encoding="UTF-8"?>`),
+		"manifests/unreadable.yaml": []byte(`***`),
+		"manifests/unreadable.json": []byte(`***`),
+	})
+	require.NoError(t, err)
+
+	client := fake.FakeClient{}
+	client.On("Image", ref).Return(image, nil)
+
+	ctx := oci.WithClient(context.Background(), &client)
+
+	extractors := []Extractor{PathExtractor{Path: "manifests"}}
+	files, err := ImageFiles(ctx, ref, extractors)
+
+	assert.NoError(t, err)
+
+	assert.Equal(t, map[string]json.RawMessage{
+		"manifests/a.json": []byte(`{"a":1}`),
+		"manifests/b.yaml": []byte(`{"b":2}`),
+	}, files)
+}
+
 func TestShouldFilter(t *testing.T) {
 	cases := []struct {
 		name     string
-		matcher  pathMatcher
+		matcher  PathMatcher
 		header   *tar.Header
 		decision bool
 	}{
@@ -105,36 +136,36 @@ func TestShouldFilter(t *testing.T) {
 		{name: "zero header", header: &tar.Header{}},
 		{
 			name:    "unrelated file",
-			matcher: pathMatcher{"path"},
+			matcher: PathMatcher{"path"},
 			header: &tar.Header{
 				Name: "autoexec.bat",
 			},
 		},
 		{
 			name:    "not in considered path",
-			matcher: pathMatcher{"one/"},
+			matcher: PathMatcher{"one/"},
 			header: &tar.Header{
 				Name: "else/manifest.json",
 			},
 		},
 		{
 			name:    "unsupported extension",
-			matcher: pathMatcher{"manifests/"},
+			matcher: PathMatcher{"manifests/"},
 			header: &tar.Header{
 				Name: "manifests/autoexec.bat",
 			},
 		},
 		{
 			name:    "happy day",
-			matcher: pathMatcher{"manifests/"},
+			matcher: PathMatcher{"manifests/"},
 			header: &tar.Header{
 				Name: "manifests/something.json",
 			},
 			decision: true,
 		},
 		{
 			name:    "happy day - no trailing slash",
-			matcher: pathMatcher{"manifests"},
+			matcher: PathMatcher{"manifests"},
 			header: &tar.Header{
 				Name: "manifests/something.json",
 			},
@@ -144,7 +175,7 @@ func TestShouldFilter(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			decision := c.matcher.match(c.header)
+			decision := c.matcher.Match(c.header)
 			assert.Equal(t, c.decision, decision)
 		})
 	}