Merge branch 'main' into rpi-zero-arch-fix

Signed-off-by: Łukasz Stolcman <[email protected]>

.cirrus.yml

@@ -29,14 +29,8 @@ env:
     # test changes made there, in a PR in this repository.
     SKOPEO_PR:
 
-    ####
-    #### Cache-image names to test with (double-quotes around names are critical)
-    ####
-    FEDORA_NAME: "fedora-38"
-    DEBIAN_NAME: "debian-13"
-
     # Google-cloud VM Images
-    IMAGE_SUFFIX: "c20230807t144831z-f38f37d13"
+    IMAGE_SUFFIX: "c20231208t193858z-f39f38d13"
     FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
     DEBIAN_CACHE_IMAGE_NAME: "debian-${IMAGE_SUFFIX}"
 

.codespellrc

@@ -0,0 +1,6 @@
+# See https://github.com/codespell-project/codespell#using-a-config-file
+[codespell]
+skip = .git,*.pdf,*.svg,.codespellrc,go.sum,system_registries_v2_test.go,Makefile,build,buildah,buildah.spec,imgtype,copy,AUTHORS,bin,vendor,CHANGELOG.md,changelog.txt,seccomp.json,.cirrus.yml,*.xz,*.gz,*.tar,*.tgz,*ico,*.png,*.1,*.5,*.orig,*.rej,*.gpg
+check-hidden = true
+ignore-regex = \b(isT|BU|this/doesnt:match)\b
+ignore-words-list = te,pathc

.github/renovate.json5

@@ -36,7 +36,7 @@
    ****** Global/general configuration options *****
    *************************************************/
 
-  // Re-use predefined sets of configuration options to DRY
+  // Reuse predefined sets of configuration options to DRY
   "extends": [
     // https://github.com/containers/automation/blob/main/renovate/defaults.json5
     "github>containers/automation//renovate/defaults.json5"

.github/workflows/codespell.yml

@@ -0,0 +1,23 @@
+---
+name: Codespell
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  codespell:
+    name: Check for spelling errors
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Codespell
+        # uses configuration within .codespellrc file
+        uses: codespell-project/actions-codespell@v2

Makefile

@@ -24,6 +24,18 @@ GOMD2MAN ?= $(shell command -v go-md2man || echo '$(GOBIN)/go-md2man')
 MANPAGES_MD = $(wildcard docs/*.5.md)
 MANPAGES ?= $(MANPAGES_MD:%.md=%)
 
+ifeq ($(shell uname -s),FreeBSD)
+CONTAINERSCONFDIR ?= /usr/local/etc/containers
+else
+CONTAINERSCONFDIR ?= /etc/containers
+endif
+REGISTRIESDDIR ?= ${CONTAINERSCONFDIR}/registries.d
+
+# N/B: This value is managed by Renovate, manual changes are
+# possible, as long as they don't disturb the formatting
+# (i.e. DO NOT ADD A 'v' prefix!)
+GOLANGCI_LINT_VERSION := 1.55.2
+
 export PATH := $(PATH):${GOBIN}
 
 all: tools test validate .gitvalidation
@@ -41,12 +53,16 @@ install-docs: docs
 	install -m 644 docs/*.5 ${MANINSTALLDIR}/man5/
 
 install: install-docs
+	install -d -m 755 ${DESTDIR}${CONTAINERSCONFDIR}
+	install -m 644 default-policy.json ${DESTDIR}${CONTAINERSCONFDIR}/policy.json
+	install -d -m 755 ${DESTDIR}${REGISTRIESDDIR}
+	install -m 644 default.yaml ${DESTDIR}${REGISTRIESDDIR}/default.yaml
 
 cross:
 	GOOS=windows $(MAKE) build BUILDTAGS="$(BUILDTAGS) $(BUILD_TAGS_WINDOWS_CROSS)"
 	GOOS=darwin $(MAKE) build BUILDTAGS="$(BUILDTAGS) $(BUILD_TAGS_DARWIN_CROSS)"
 
-tools: .install.gitvalidation .install.golangci-lint .install.golint
+tools: .install.gitvalidation .install.golangci-lint
 
 .install.gitvalidation:
 	if [ ! -x "$(GOBIN)/git-validation" ]; then \
@@ -55,13 +71,7 @@ tools: .install.gitvalidation .install.golangci-lint .install.golint
 
 .install.golangci-lint:
 	if [ ! -x "$(GOBIN)/golangci-lint" ]; then \
-		curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(GOBIN) v1.51.0; \
-	fi
-
-.install.golint:
-	# Note, golint is only needed for Skopeo's tests.
-	if [ ! -x "$(GOBIN)/golint" ]; then \
-		GO111MODULE="off" go get -u $(BUILDFLAGS) golang.org/x/lint/golint; \
+		curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(GOBIN) v$(GOLANGCI_LINT_VERSION) ; \
 	fi
 
 clean:
@@ -90,4 +100,4 @@ vendor-in-container:
 	podman run --privileged --rm --env HOME=/root -v `pwd`:/src -w /src golang go mod tidy
 
 codespell:
-	codespell -S Makefile,build,buildah,buildah.spec,imgtype,copy,AUTHORS,bin,vendor,.git,go.sum,CHANGELOG.md,changelog.txt,seccomp.json,.cirrus.yml,"*.xz,*.gz,*.tar,*.tgz,*ico,*.png,*.1,*.5,*.orig,*.rej" -L keypair,flate,uint,iff,od,ERRO -w
+	codespell -w

README.md

@@ -63,6 +63,8 @@ the primary downside is that creating new signatures with the Golang-only implem
 - `containers_image_ostree`: Import `ostree:` transport in `github.com/containers/image/transports/alltransports`. This builds the library requiring the `libostree` development libraries. Otherwise a stub which reports that the transport is not supported gets used. The `github.com/containers/image/ostree` package is completely disabled
 and impossible to import when this build tag is not in use.
 - `containers_image_storage_stub`: Don’t import the `containers-storage:` transport in `github.com/containers/image/transports/alltransports`, to decrease the amount of required dependencies.  Use a stub which reports that the transport is not supported instead.
+- `containers_image_fulcio_stub`: Don't import sigstore/fulcio code, all fulcio operations will return an error code
+- `containers_image_rekor_stub`: Don't import sigstore/reckor code, all rekor operations will return an error code
 
 ## [Contributing](CONTRIBUTING.md)
 

copy/compression.go

@@ -284,10 +284,24 @@ func (d *bpCompressionStepData) recordValidatedDigestData(c *copier, uploadedInf
 		}
 	}
 	if d.uploadedCompressorName != "" && d.uploadedCompressorName != internalblobinfocache.UnknownCompression {
-		c.blobInfoCache.RecordDigestCompressorName(uploadedInfo.Digest, d.uploadedCompressorName)
+		if d.uploadedCompressorName != compressiontypes.ZstdChunkedAlgorithmName {
+			// HACK: Don’t record zstd:chunked algorithms.
+			// There is already a similar hack in internal/imagedestination/impl/helpers.BlobMatchesRequiredCompression,
+			// and that one prevents reusing zstd:chunked blobs, so recording the algorithm here would be mostly harmless.
+			//
+			// We skip that here anyway to work around the inability of blobPipelineDetectCompressionStep to differentiate
+			// between zstd and zstd:chunked; so we could, in varying situations over time, call RecordDigestCompressorName
+			// with the same digest and both ZstdAlgorithmName and ZstdChunkedAlgorithmName , which causes warnings about
+			// inconsistent data to be logged.
+			c.blobInfoCache.RecordDigestCompressorName(uploadedInfo.Digest, d.uploadedCompressorName)
+		}
 	}
-	if srcInfo.Digest != "" && d.srcCompressorName != "" && d.srcCompressorName != internalblobinfocache.UnknownCompression {
-		c.blobInfoCache.RecordDigestCompressorName(srcInfo.Digest, d.srcCompressorName)
+	if srcInfo.Digest != "" && srcInfo.Digest != uploadedInfo.Digest &&
+		d.srcCompressorName != "" && d.srcCompressorName != internalblobinfocache.UnknownCompression {
+		if d.srcCompressorName != compressiontypes.ZstdChunkedAlgorithmName {
+			// HACK: Don’t record zstd:chunked algorithms, see above.
+			c.blobInfoCache.RecordDigestCompressorName(srcInfo.Digest, d.srcCompressorName)
+		}
 	}
 	return nil
 }

copy/copy.go

@@ -242,11 +242,13 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
 
 		unparsedToplevel: image.UnparsedInstance(rawSource, nil),
 		// FIXME? The cache is used for sources and destinations equally, but we only have a SourceCtx and DestinationCtx.
-		// For now, use DestinationCtx (because blob reuse changes the behavior of the destination side more); eventually
-		// we might want to add a separate CommonCtx — or would that be too confusing?
+		// For now, use DestinationCtx (because blob reuse changes the behavior of the destination side more).
+		// Conceptually the cache settings should be in copy.Options instead.
 		blobInfoCache: internalblobinfocache.FromBlobInfoCache(blobinfocache.DefaultCache(options.DestinationCtx)),
 	}
 	defer c.close()
+	c.blobInfoCache.Open()
+	defer c.blobInfoCache.Close()
 
 	// Set the concurrentBlobCopiesSemaphore if we can copy layers in parallel.
 	if dest.HasThreadSafePutBlob() && rawSource.HasThreadSafeGetBlob() {

copy/encryption.go

@@ -70,7 +70,7 @@ func (d *bpDecryptionStepData) updateCryptoOperation(operation *types.LayerCrypt
 	}
 }
 
-// bpdData contains data that the copy pipeline needs about the encryption step.
+// bpEncryptionStepData contains data that the copy pipeline needs about the encryption step.
 type bpEncryptionStepData struct {
 	encrypting bool // We are actually encrypting the stream
 	finalizer  ocicrypt.EncryptLayerFinalizer

copy/manifest.go

@@ -6,8 +6,10 @@ import (
 	"fmt"
 	"strings"
 
+	internalManifest "github.com/containers/image/v5/internal/manifest"
 	"github.com/containers/image/v5/internal/set"
 	"github.com/containers/image/v5/manifest"
+	compressiontypes "github.com/containers/image/v5/pkg/compression/types"
 	"github.com/containers/image/v5/types"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sirupsen/logrus"
@@ -19,8 +21,8 @@ import (
 // Include v2s1 signed but not v2s1 unsigned, because docker/distribution requires a signature even if the unsigned MIME type is used.
 var preferredManifestMIMETypes = []string{manifest.DockerV2Schema2MediaType, manifest.DockerV2Schema1SignedMediaType}
 
-// ociEncryptionMIMETypes lists manifest MIME types that are known to support OCI encryption.
-var ociEncryptionMIMETypes = []string{v1.MediaTypeImageManifest}
+// allManifestMIMETypes lists all possible manifest MIME types.
+var allManifestMIMETypes = []string{v1.MediaTypeImageManifest, manifest.DockerV2Schema2MediaType, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema1MediaType}
 
 // orderedSet is a list of strings (MIME types or platform descriptors in our case), with each string appearing at most once.
 type orderedSet struct {
@@ -51,9 +53,10 @@ type determineManifestConversionInputs struct {
 
 	destSupportedManifestMIMETypes []string // MIME types supported by the destination, per types.ImageDestination.SupportedManifestMIMETypes()
 
-	forceManifestMIMEType      string // User’s choice of forced manifest MIME type
-	requiresOCIEncryption      bool   // Restrict to manifest formats that can support OCI encryption
-	cannotModifyManifestReason string // The reason the manifest cannot be modified, or an empty string if it can
+	forceManifestMIMEType      string                      // User’s choice of forced manifest MIME type
+	requestedCompressionFormat *compressiontypes.Algorithm // Compression algorithm to use, if the user _explictily_ requested one.
+	requiresOCIEncryption      bool                        // Restrict to manifest formats that can support OCI encryption
+	cannotModifyManifestReason string                      // The reason the manifest cannot be modified, or an empty string if it can
 }
 
 // manifestConversionPlan contains the decisions made by determineManifestConversion.
@@ -80,41 +83,74 @@ func determineManifestConversion(in determineManifestConversionInputs) (manifest
 		destSupportedManifestMIMETypes = []string{in.forceManifestMIMEType}
 	}
 
+	restrictiveCompressionRequired := in.requestedCompressionFormat != nil && !internalManifest.CompressionAlgorithmIsUniversallySupported(*in.requestedCompressionFormat)
 	if len(destSupportedManifestMIMETypes) == 0 {
-		if !in.requiresOCIEncryption || manifest.MIMETypeSupportsEncryption(srcType) {
+		if (!in.requiresOCIEncryption || manifest.MIMETypeSupportsEncryption(srcType)) &&
+			(!restrictiveCompressionRequired || internalManifest.MIMETypeSupportsCompressionAlgorithm(srcType, *in.requestedCompressionFormat)) {
 			return manifestConversionPlan{ // Anything goes; just use the original as is, do not try any conversions.
 				preferredMIMEType:       srcType,
 				otherMIMETypeCandidates: []string{},
 			}, nil
 		}
-		destSupportedManifestMIMETypes = ociEncryptionMIMETypes
+		destSupportedManifestMIMETypes = allManifestMIMETypes
 	}
 	supportedByDest := set.New[string]()
 	for _, t := range destSupportedManifestMIMETypes {
-		if !in.requiresOCIEncryption || manifest.MIMETypeSupportsEncryption(t) {
-			supportedByDest.Add(t)
+		if in.requiresOCIEncryption && !manifest.MIMETypeSupportsEncryption(t) {
+			continue
 		}
+		if restrictiveCompressionRequired && !internalManifest.MIMETypeSupportsCompressionAlgorithm(t, *in.requestedCompressionFormat) {
+			continue
+		}
+		supportedByDest.Add(t)
 	}
 	if supportedByDest.Empty() {
-		if len(destSupportedManifestMIMETypes) == 0 { // Coverage: This should never happen, empty values were replaced by ociEncryptionMIMETypes
+		if len(destSupportedManifestMIMETypes) == 0 { // Coverage: This should never happen, empty values were replaced by allManifestMIMETypes
 			return manifestConversionPlan{}, errors.New("internal error: destSupportedManifestMIMETypes is empty")
 		}
-		// We know, and have verified, that destSupportedManifestMIMETypes is not empty, so encryption must have been involved.
-		if !in.requiresOCIEncryption { // Coverage: This should never happen, destSupportedManifestMIMETypes was not empty, so we should have filtered for encryption.
-			return manifestConversionPlan{}, errors.New("internal error: supportedByDest is empty but destSupportedManifestMIMETypes is not, and not encrypting")
-		}
+		// We know, and have verified, that destSupportedManifestMIMETypes is not empty, so some filtering of supported MIME types must have been involved.
+
 		// destSupportedManifestMIMETypes has three possible origins:
 		if in.forceManifestMIMEType != "" { // 1. forceManifestType specified
-			return manifestConversionPlan{}, fmt.Errorf("encryption required together with format %s, which does not support encryption",
-				in.forceManifestMIMEType)
+			switch {
+			case in.requiresOCIEncryption && restrictiveCompressionRequired:
+				return manifestConversionPlan{}, fmt.Errorf("compression using %s, and encryption, required together with format %s, which does not support both",
+					in.requestedCompressionFormat.Name(), in.forceManifestMIMEType)
+			case in.requiresOCIEncryption:
+				return manifestConversionPlan{}, fmt.Errorf("encryption required together with format %s, which does not support encryption",
+					in.forceManifestMIMEType)
+			case restrictiveCompressionRequired:
+				return manifestConversionPlan{}, fmt.Errorf("compression using %s required together with format %s, which does not support it",
+					in.requestedCompressionFormat.Name(), in.forceManifestMIMEType)
+			default:
+				return manifestConversionPlan{}, errors.New("internal error: forceManifestMIMEType was rejected for an unknown reason")
+			}
+		}
+		if len(in.destSupportedManifestMIMETypes) == 0 { // 2. destination accepts anything and we have chosen allManifestTypes
+			if !restrictiveCompressionRequired {
+				// Coverage: This should never happen.
+				// If we have not rejected for encryption reasons, we must have rejected due to encryption, but
+				// allManifestTypes includes OCI, which supports encryption.
+				return manifestConversionPlan{}, errors.New("internal error: in.destSupportedManifestMIMETypes is empty but supportedByDest is empty as well")
+			}
+			// This can legitimately happen when the user asks for completely unsupported formats like Bzip2 or Xz.
+			return manifestConversionPlan{}, fmt.Errorf("compression using %s required, but none of the known manifest formats support it", in.requestedCompressionFormat.Name())
 		}
-		if len(in.destSupportedManifestMIMETypes) == 0 { // 2. destination accepts anything and we have chosen ociEncryptionMIMETypes
-			// Coverage: This should never happen, ociEncryptionMIMETypes all support encryption
-			return manifestConversionPlan{}, errors.New("internal error: in.destSupportedManifestMIMETypes is empty but supportedByDest is empty as well")
+		// 3. destination accepts a restricted list of mime types
+		destMIMEList := strings.Join(destSupportedManifestMIMETypes, ", ")
+		switch {
+		case in.requiresOCIEncryption && restrictiveCompressionRequired:
+			return manifestConversionPlan{}, fmt.Errorf("compression using %s, and encryption, required but the destination only supports MIME types [%s], none of which support both",
+				in.requestedCompressionFormat.Name(), destMIMEList)
+		case in.requiresOCIEncryption:
+			return manifestConversionPlan{}, fmt.Errorf("encryption required but the destination only supports MIME types [%s], none of which support encryption",
+				destMIMEList)
+		case restrictiveCompressionRequired:
+			return manifestConversionPlan{}, fmt.Errorf("compression using %s required but the destination only supports MIME types [%s], none of which support it",
+				in.requestedCompressionFormat.Name(), destMIMEList)
+		default: // Coverage: This should never happen, we only filter for in.requiresOCIEncryption || restrictiveCompressionRequired
+			return manifestConversionPlan{}, errors.New("internal error: supportedByDest is empty but destSupportedManifestMIMETypes is not, and we are neither encrypting nor requiring a restrictive compression algorithm")
 		}
-		// 3. destination does not support encryption.
-		return manifestConversionPlan{}, fmt.Errorf("encryption required but the destination only supports MIME types [%s], none of which support encryption",
-			strings.Join(destSupportedManifestMIMETypes, ", "))
 	}
 
 	// destSupportedManifestMIMETypes is a static guess; a particular registry may still only support a subset of the types.
@@ -156,7 +192,7 @@ func determineManifestConversion(in determineManifestConversionInputs) (manifest
 	}
 
 	logrus.Debugf("Manifest has MIME type %s, ordered candidate list [%s]", srcType, strings.Join(prioritizedTypes.list, ", "))
-	if len(prioritizedTypes.list) == 0 { // Coverage: destSupportedManifestMIMETypes and supportedByDest, which is a subset, is not empty (or we would have exited  above), so this should never happen.
+	if len(prioritizedTypes.list) == 0 { // Coverage: destSupportedManifestMIMETypes and supportedByDest, which is a subset, is not empty (or we would have exited above), so this should never happen.
 		return manifestConversionPlan{}, errors.New("Internal error: no candidate MIME types")
 	}
 	res := manifestConversionPlan{