Simple macro benchmark

Implementation of a simple macro benchmark executing `ec validate image`
with the fixed state and no external dependencies.

For this the OCI distribution registry is run in a container with the
data from the `benchmark/simple/data.tar.gz` that contains the copy of
all images/blobs that the command and the policy rules currently access
and the git repository.

The benchmark outputs in the golang standard benchmark format that can
be utilized with tools in the golang benchmarking ecosystem.

Due to the global `downloadCache` variable in
`internal/policy/source/source.go` the benchmark cannot be run in
parallel. So only invoking the benchmark once is currently possible.

Reference: https://issues.redhat.com/browse/EC-968

.gitleaks.toml

@@ -0,0 +1,7 @@
+[allowlist]
+  description = "Project allowlist"
+
+  paths = [
+    '''benchmark\/internal\/registry\/fake_quay.key$''',
+    '''benchmark\/simple\/data.tar.gz$''',
+  ]

Makefile

@@ -144,6 +144,13 @@ feature_%: ## Run acceptance tests for a single feature file, e.g. make feature_
 scenario_%: build ## Run acceptance tests for a single scenario, e.g. make scenario_inline_policy
 	@cd acceptance && go test -test.run 'TestFeatures/$*'
 
+benchmark_%:
+	@cd benchmark/$*
+	@go run .
+
+.PHONY: benchmark
+benchmark: benchmark_simple ## Run benchmarks
+
 .PHONY: ci
 ci: test lint-fix acceptance ## Run the usual required CI tasks
 

benchmark/README.md

@@ -0,0 +1,12 @@
+# Benchmarks of ec CLI
+
+Benchmarks within this directory use the [golang
+benchmarking](golang.org/x/benchmarks/) package and output in the [standard
+benchmark
+format](https://go.googlesource.com/proposal/+/master/design/14313-benchmark-format.md).
+
+Each benchmark is built as a standalone executable with no external dependency
+other than any data that is contained within it. Benchmarks are run from within
+the directory they're defined in, simply by running `go run .`, additional
+arguments can be passed in, for example `-benchnum 10` to run the benchmark 10
+times.

benchmark/internal/registry/fake_quay.cer

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDOjCCAiKgAwIBAgIUOc/vG5o8pNnIBCIYsrYJ01jkmfEwDQYJKoZIhvcNAQEL
+BQAwFzEVMBMGA1UEAwwMYmVuY2htYXJrLmVjMB4XDTI0MTIwOTE1MzIyMVoXDTM0
+MTIwNzE1MzIyMVowFzEVMBMGA1UEAwwMYmVuY2htYXJrLmVjMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnVIvUGHA2XFScaRcN8vxNiuDShN4SNuCWcdC
+cbzvCbA7yu4v2LJBqsbDIs5zWcr9kb15vg0Cg3dCHztnsJCmFIwp6lKc3l60eD4+
+Qke2YnD3cUB/P4A7jMUC2kEjjlqdnoLw6IIwt7D6JZeaZhDpPfGyf3llxoVdT5+J
+dfBtn6U4WnYqqTg7vccoe0RTXkotLAUMyshd00QaGKIOZ5HdzDBXHUVszTU+Vidf
+9xyxndDVlxnE70zWv/jeZLhgnkxMRzoi0q2hq25TeugQ+yrGKVD4UBB6CaGWgAIN
+Sw0iTFmYd+0uBVpTYF8KLO2jc/bNqFGJn2t/+P88XqfHTCZVfwIDAQABo34wfDA5
+BgNVHREEMjAwggdxdWF5LmlvghpyZWdpc3RyeS5hY2Nlc3MucmVkaGF0LmNvbYIJ
+bG9jYWxob3N0MAsGA1UdDwQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNV
+HQ4EFgQUP3lNVxeEEpMjoKQ2wPk6hhLsXOwwDQYJKoZIhvcNAQELBQADggEBAFKp
+MpYwkR5vb73JsRATYr6H/A5bRAjVg+NoMklOJ5lu5PtwomD0rlD8OIJeXuT57Bsy
+PAukG7gXlHk0zJ3cExxsys+jscsGKi2+gKEWvOYj06Z5xug7GPZyoA9r6MOKclZJ
+VXCxpQfkl5pGDQiIhtpvuADt8oBogx4mwDhP16WBz/5BkKIpsLn7jGZOUYaSrjG3
+OLf5AArhHzs0rRRcaCjbybuah8nQMMqG7Q6bz7jF4szCeIL7ndWro2U5kgVVMrXS
+5zbEklvW5OXFO/UREiY+5YBAu6gCr0APFLAQi5qfAhKpbQkjZ28unuB/3aQlT0kR
+wHo3oDNZDNaIwBoP/54=
+-----END CERTIFICATE-----

benchmark/internal/registry/fake_quay.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCdUi9QYcDZcVJx
+pFw3y/E2K4NKE3hI24JZx0JxvO8JsDvK7i/YskGqxsMiznNZyv2RvXm+DQKDd0If
+O2ewkKYUjCnqUpzeXrR4Pj5CR7ZicPdxQH8/gDuMxQLaQSOOWp2egvDogjC3sPol
+l5pmEOk98bJ/eWXGhV1Pn4l18G2fpThadiqpODu9xyh7RFNeSi0sBQzKyF3TRBoY
+og5nkd3MMFcdRWzNNT5WJ1/3HLGd0NWXGcTvTNa/+N5kuGCeTExHOiLSraGrblN6
+6BD7KsYpUPhQEHoJoZaAAg1LDSJMWZh37S4FWlNgXwos7aNz9s2oUYmfa3/4/zxe
+p8dMJlV/AgMBAAECggEASM1Xh+M002tVs7FT255NKbxJLWaFK3IpiFJltSyxkdI0
+WVyQV79mFBmXA+x2BP+jCGaeiAyNNTljKADdox/NO4UBgKZqYZ2B9oMXXKTDChss
+5wPp3wEAjcw93bJ3OdFFT61YvsBAu0DzAaJ+mO0Kgtfuun77Ujs1SzuQ6Tsx/8ZD
+STr8q7DcMAd6afHFFk5NZDqyFPgzmEMUWRFy7BH/vLLEgdRi7I7UbTYhjpP7uyjF
+/0HLSjBQJr58554CyL11OcVOwqktKZYKMVsCNWg6l4uhYxg0ChyJgO+PuUhPqh1i
+eIybcaDUVQ0LG/gBYzDlnw8yNXPO4+HX+MdIXafBAQKBgQDZx6m2rsI3L4M0ow89
+T6L8jMrFF6bUp2PBVPOYvobk4de3x7ma4WwK212l869A2u536+8BlBaiZN0yRSmI
+XXX0oL8JfqTO7d5efa9zHdkEN+t7iwVFWsYe74jjVR/oOyW7ZJ07gP/kZudbm8uk
+DnGzEGsjTYo7K6dEcEiXXmZiDwKBgQC47j8iR7v6a0E7EzEcM6FIpAV8tlpj0wUt
+pXKHPRdMQaOi39YyNWCGGake7bRCGz2ni1zqSVqVIcm6yvzlS/o3lLMTUYlUxtqt
+0cGrTJXkyhwkyoAJg0vwU43BJkbUNsX0y5revv/T4TaVTYK4dv2RTv8Jbdp6sT7i
+B6I7tFmFkQKBgQDQShe24rRchbPOrzoPINC5DYuOMA3fC+3r5e1KJMftt1dTrdGG
+IZ3tUFvkMgpnVD2KMvyYHOP9E80/nEiZ5RHBE9FDKn5Eb9sjssAQsPZ0A2vU8GTt
+LWGaCu14yFzJB3ESJqv/UmTsNcOHqZ1+XlY+tjBSRoI0D49edKnpJF913wKBgQCn
+WSHUuamIIYr0FJf6d1ZaT+OPSc1eTFWBSxjq/QOaREP6XiiNYvQoJgx/KohW0iPm
+/Bxm/15zWhIdcReNwEV8CppKvxxRlWnKpehmRXXXhnYVbRKUiTYtEs9SnHq4C6Dz
+t1Q8ggcsC7/DOFM07qjj1+K++6QcJ01KabIL6VahoQKBgGNCgw4+0291OA8nwjXl
+HNu9WBzJVHsAL9RDvrL6c4wcUWmhJTUcKN/TnndYN5SdvrOzZ2RjysS8KZoLnHgI
+c41uBNLvGtE9RPuzFe7U3zt9t80d4gcjy92tBru6xszPIgmKsUe8Hrl2EWbJvCET
+/BbfoyzzqH3f+vYm25GFSIy5
+-----END PRIVATE KEY-----

benchmark/internal/registry/registry.go

@@ -0,0 +1,167 @@
+// Copyright The Enterprise Contract Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//type Closer func()
+// SPDX-License-Identifier: Apache-2.0
+
+package registry
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	_ "embed"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/enterprise-contract/ec-cli/benchmark/internal/suite"
+	"github.com/smarty/cproxy/v2"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/modules/registry"
+	"github.com/testcontainers/testcontainers-go/wait"
+)
+
+//go:embed fake_quay.cer
+var certificate []byte
+
+//go:embed fake_quay.key
+var key []byte
+
+type registryCloser struct {
+	tempDir   string
+	container *registry.RegistryContainer
+	proxy     *httptest.Server
+}
+
+func (r *registryCloser) Close() {
+	if r == nil {
+		return
+	}
+
+	if r.tempDir != "" {
+		_ = os.RemoveAll(r.tempDir)
+	}
+
+	if r.container != nil {
+		_ = r.container.Terminate(context.Background())
+	}
+
+	if r.proxy != nil {
+		r.proxy.Close()
+	}
+}
+
+type registryProxy struct {
+	registry string
+}
+
+func (f *registryProxy) IsAuthorized(_ http.ResponseWriter, req *http.Request) bool {
+	req.RequestURI = fmt.Sprintf("https://%s/", f.registry)
+	req.URL, _ = url.Parse(req.RequestURI)
+	req.Host = f.registry
+
+	return true
+}
+
+func Launch(data string) (suite.Closer, error) {
+	ctx := context.Background()
+
+	env := testcontainers.WithEnv(map[string]string{
+		"REGISTRY_HTTP_TLS_CERTIFICATE": "/tls/fake_quay.cer",
+		"REGISTRY_HTTP_TLS_KEY":         "/tls/fake_quay.key",
+	})
+
+	dir, err := os.MkdirTemp("", "ec-benchmark-tls-*")
+	if err != nil {
+		return nil, err
+	}
+	closer := &registryCloser{dir, nil, nil}
+
+	if err := os.Setenv("https_proxy", "http://localhost:3128"); err != nil {
+		return nil, err
+	}
+
+	if err := os.Chmod(dir, 0755); err != nil {
+		return closer.Close, err
+	}
+
+	certPath := path.Join(dir, "fake_quay.cer")
+	if err := os.WriteFile(certPath, certificate, 0644); err != nil {
+		return closer.Close, err
+	}
+
+	if err := os.Setenv("SSL_CERT_FILE", certPath); err != nil {
+		return closer.Close, err
+	}
+
+	if err := os.WriteFile(path.Join(dir, "fake_quay.key"), key, 0644); err != nil {
+		return closer.Close, err
+	}
+
+	rp := registryProxy{}
+	proxy := cproxy.New(cproxy.Options.Filter(&rp), cproxy.Options.Logger(log.Default()))
+	proxyServer := httptest.NewUnstartedServer(proxy)
+	proxyServer.Listener, err = net.Listen("tcp", "127.0.0.1:3128")
+	if err != nil {
+		return closer.Close, err
+	}
+	proxyServer.Config.ErrorLog = log.Default()
+	proxyServer.Start()
+	closer.proxy = proxyServer
+
+	tlsMount := testcontainers.WithHostConfigModifier(func(hostConfig *container.HostConfig) {
+		hostConfig.Binds = append(hostConfig.Binds, fmt.Sprintf("%s:/tls:ro,Z", dir))
+	})
+
+	roots := x509.NewCertPool()
+	roots.AppendCertsFromPEM(certificate)
+	waitStrategy := testcontainers.WithWaitStrategy(wait.ForHTTP("/").
+		WithPort("5000/tcp").
+		WithTLS(true, &tls.Config{
+			RootCAs: roots,
+		}).
+		WithStartupTimeout(10 * time.Second))
+
+	opts := []testcontainers.ContainerCustomizer{
+		registry.WithData(data),
+		env,
+		tlsMount,
+		waitStrategy,
+	}
+
+	if false {
+		opts = append(opts,
+			testcontainers.WithConfigModifier(func(config *container.Config) {
+				config.AttachStdout = true
+			}),
+			testcontainers.WithLogger(log.Default()))
+	}
+
+	r, err := registry.Run(ctx, "registry:2.8.3", opts...)
+	if err != nil {
+		return closer.Close, err
+	}
+	closer.container = r
+
+	rp.registry = r.RegistryName
+
+	return closer.Close, nil
+}

benchmark/internal/suite/closer.go

@@ -0,0 +1,19 @@
+// Copyright The Enterprise Contract Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suite
+
+type Closer func()

benchmark/internal/suite/suite.go

@@ -0,0 +1,32 @@
+// Copyright The Enterprise Contract Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package suite
+
+import (
+	"io"
+
+	"github.com/enterprise-contract/ec-cli/cmd"
+	"github.com/enterprise-contract/ec-cli/cmd/root"
+)
+
+func Execute(args []string) error {
+	c := root.NewRootCmd()
+	cmd.AddCommandsTo(c)
+	c.SetArgs(args)
+	c.SetOutput(io.Discard)
+	return c.Execute()
+}

benchmark/internal/untar/untar.go

@@ -0,0 +1,91 @@
+// Copyright The Enterprise Contract Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package untar
+
+import (
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"math"
+	"os"
+	"path"
+
+	"github.com/google/safearchive/tar"
+)
+
+func UnTar(a string) (string, error) {
+	dir, err := os.MkdirTemp("", "ec-benchmark")
+	if err != nil {
+		return "", err
+	}
+
+	archive, err := os.Open(a)
+	if err != nil {
+		return "", err
+	}
+	defer archive.Close()
+
+	gz, err := gzip.NewReader(archive)
+	if err != nil {
+		return "", err
+	}
+	defer gz.Close()
+
+	t := tar.NewReader(gz)
+	for {
+		hdr, err := t.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+
+		dst := path.Join(dir, path.Clean(hdr.Name))
+		if hdr.Mode < 0 || hdr.Mode > math.MaxUint32 {
+			panic(fmt.Sprintf("weird tar header mode: %d", hdr.Mode))
+		}
+		mode := fs.FileMode(hdr.Mode)
+
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			err = os.MkdirAll(dst, mode)
+		case tar.TypeLink:
+			err = os.Symlink(path.Join(dir, path.Clean(hdr.Linkname)), dst)
+		case tar.TypeReg:
+			var f io.WriteCloser
+			f, err = os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, mode)
+			if err != nil {
+				break
+			}
+
+			_, err = io.Copy(f, t)
+			if err != nil && !errors.Is(err, io.EOF) {
+				break
+			}
+			err = f.Close()
+		}
+
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return dir, nil
+}