diff --git a/copy/encryption.go b/copy/encryption.go
index 01d1ab79cb..555f25e33d 100644
--- a/copy/encryption.go
+++ b/copy/encryption.go
@@ -34,30 +34,28 @@ type bpDecryptionStepData struct {
 // srcInfo is only used for error messages.
 // Returns data for other steps; the caller should eventually use updateCryptoOperation.
 func (ic *imageCopier) blobPipelineDecryptionStep(stream *sourceStream, srcInfo types.BlobInfo) (*bpDecryptionStepData, error) {
-	if isOciEncrypted(stream.info.MediaType) && ic.c.ociDecryptConfig != nil {
-		if ic.cannotModifyManifestReason != "" {
-			return nil, fmt.Errorf("layer %s should be decrypted, but we can’t modify the manifest: %s", srcInfo.Digest, ic.cannotModifyManifestReason)
-		}
-		desc := imgspecv1.Descriptor{
-			Annotations: stream.info.Annotations,
-		}
-		reader, decryptedDigest, err := ocicrypt.DecryptLayer(ic.c.ociDecryptConfig, stream.reader, desc, false)
-		if err != nil {
-			return nil, fmt.Errorf("decrypting layer %s: %w", srcInfo.Digest, err)
-		}
-
-		stream.reader = reader
-		stream.info.Digest = decryptedDigest
-		stream.info.Size = -1
-		maps.DeleteFunc(stream.info.Annotations, func(k string, _ string) bool {
-			return strings.HasPrefix(k, "org.opencontainers.image.enc")
-		})
+	if !isOciEncrypted(stream.info.MediaType) || ic.c.ociDecryptConfig == nil {
 		return &bpDecryptionStepData{
-			decrypting: true,
+			decrypting: false,
 		}, nil
 	}
+
+	desc := imgspecv1.Descriptor{
+		Annotations: stream.info.Annotations,
+	}
+	reader, decryptedDigest, err := ocicrypt.DecryptLayer(ic.c.ociDecryptConfig, stream.reader, desc, false)
+	if err != nil {
+		return nil, fmt.Errorf("decrypting layer %s: %w", srcInfo.Digest, err)
+	}
+
+	stream.reader = reader
+	stream.info.Digest = decryptedDigest
+	stream.info.Size = -1
+	maps.DeleteFunc(stream.info.Annotations, func(k string, _ string) bool {
+		return strings.HasPrefix(k, "org.opencontainers.image.enc")
+	})
 	return &bpDecryptionStepData{
-		decrypting: false,
+		decrypting: true,
 	}, nil
 }
 
@@ -79,36 +77,33 @@ type bpEncryptionStepData struct {
 // Returns data for other steps; the caller should eventually call updateCryptoOperationAndAnnotations.
 func (ic *imageCopier) blobPipelineEncryptionStep(stream *sourceStream, toEncrypt bool, srcInfo types.BlobInfo,
 	decryptionStep *bpDecryptionStepData) (*bpEncryptionStepData, error) {
-	if toEncrypt && !isOciEncrypted(srcInfo.MediaType) && ic.c.ociEncryptConfig != nil {
-		if ic.cannotModifyManifestReason != "" {
-			return nil, fmt.Errorf("layer %s should be encrypted, but we can’t modify the manifest: %s", srcInfo.Digest, ic.cannotModifyManifestReason)
-		}
-
-		var annotations map[string]string
-		if !decryptionStep.decrypting {
-			annotations = srcInfo.Annotations
-		}
-		desc := imgspecv1.Descriptor{
-			MediaType:   srcInfo.MediaType,
-			Digest:      srcInfo.Digest,
-			Size:        srcInfo.Size,
-			Annotations: annotations,
-		}
-		reader, finalizer, err := ocicrypt.EncryptLayer(ic.c.ociEncryptConfig, stream.reader, desc)
-		if err != nil {
-			return nil, fmt.Errorf("encrypting blob %s: %w", srcInfo.Digest, err)
-		}
-
-		stream.reader = reader
-		stream.info.Digest = ""
-		stream.info.Size = -1
+	if !toEncrypt || isOciEncrypted(srcInfo.MediaType) || ic.c.ociEncryptConfig == nil {
 		return &bpEncryptionStepData{
-			encrypting: true,
-			finalizer:  finalizer,
+			encrypting: false,
 		}, nil
 	}
+
+	var annotations map[string]string
+	if !decryptionStep.decrypting {
+		annotations = srcInfo.Annotations
+	}
+	desc := imgspecv1.Descriptor{
+		MediaType:   srcInfo.MediaType,
+		Digest:      srcInfo.Digest,
+		Size:        srcInfo.Size,
+		Annotations: annotations,
+	}
+	reader, finalizer, err := ocicrypt.EncryptLayer(ic.c.ociEncryptConfig, stream.reader, desc)
+	if err != nil {
+		return nil, fmt.Errorf("encrypting blob %s: %w", srcInfo.Digest, err)
+	}
+
+	stream.reader = reader
+	stream.info.Digest = ""
+	stream.info.Size = -1
 	return &bpEncryptionStepData{
-		encrypting: false,
+		encrypting: true,
+		finalizer:  finalizer,
 	}, nil
 }