From 6817504cee3e0c2a6dfef9c47903f351b7a4783f Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Wed, 25 Sep 2024 22:52:50 -0300 Subject: [PATCH 01/19] feat(copy): support mounting existing descriptors from other repositories Signed-off-by: Leonardo Chaia --- src/OrasProject.Oras/Content/MemoryStore.cs | 17 ++- src/OrasProject.Oras/Extensions.cs | 104 ++++++++++++++-- src/OrasProject.Oras/Registry/IMounter.cs | 24 ++++ src/OrasProject.Oras/Registry/IRepository.cs | 2 +- .../Registry/Remote/BlobStore.cs | 116 +++++++++++++++--- .../Registry/Remote/Repository.cs | 18 +++ tests/OrasProject.Oras.Tests/CopyTest.cs | 92 ++++++++++++++ 7 files changed, 341 insertions(+), 32 deletions(-) create mode 100644 src/OrasProject.Oras/Registry/IMounter.cs diff --git a/src/OrasProject.Oras/Content/MemoryStore.cs b/src/OrasProject.Oras/Content/MemoryStore.cs index 9a9f0f0..528765c 100644 --- a/src/OrasProject.Oras/Content/MemoryStore.cs +++ b/src/OrasProject.Oras/Content/MemoryStore.cs @@ -11,16 +11,19 @@ // See the License for the specific language governing permissions and // limitations under the License. +using System; using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using OrasProject.Oras.Registry; namespace OrasProject.Oras.Content; -public class MemoryStore : ITarget, IPredecessorFindable +public class MemoryStore : ITarget, IPredecessorFindable, IMounter { private readonly MemoryStorage _storage = new(); private readonly MemoryTagStore _tagResolver = new(); @@ -94,4 +97,16 @@ public async Task TagAsync(Descriptor descriptor, string reference, Cancellation /// public async Task> GetPredecessorsAsync(Descriptor node, CancellationToken cancellationToken = default) => await _graph.GetPredecessorsAsync(node, cancellationToken).ConfigureAwait(false); + + public async Task MountAsync(Descriptor descriptor, string contentReference, Func>? getContents, CancellationToken cancellationToken) + { + var taggedDescriptor = await _tagResolver.ResolveAsync(contentReference, cancellationToken).ConfigureAwait(false); + var successors = await _storage.GetSuccessorsAsync(taggedDescriptor, cancellationToken); + + if (descriptor != taggedDescriptor && !successors.Contains(descriptor)) + { + await _storage.PushAsync(descriptor, await getContents(cancellationToken), cancellationToken).ConfigureAwait(false); + await _graph.IndexAsync(_storage, descriptor, cancellationToken).ConfigureAwait(false); + } + } } diff --git a/src/OrasProject.Oras/Extensions.cs b/src/OrasProject.Oras/Extensions.cs index 6b048ac..595948b 100644 --- a/src/OrasProject.Oras/Extensions.cs +++ b/src/OrasProject.Oras/Extensions.cs @@ -11,14 +11,47 @@ // See the License for the specific language governing permissions and // limitations under the License. -using OrasProject.Oras.Oci; using System; +using System.IO; using System.Threading; using System.Threading.Tasks; +using OrasProject.Oras.Oci; +using OrasProject.Oras.Registry; using static OrasProject.Oras.Content.Extensions; namespace OrasProject.Oras; +public struct CopyOptions +{ + // public int Concurrency { get; set; } + + public event Action OnPreCopy; + public event Action OnPostCopy; + public event Action OnCopySkipped; + public event Action OnMounted; + + public Func MountFrom { get; set; } + + internal void PreCopy(Descriptor descriptor) + { + OnPreCopy?.Invoke(descriptor); + } + + internal void PostCopy(Descriptor descriptor) + { + OnPostCopy?.Invoke(descriptor); + } + + internal void CopySkipped(Descriptor descriptor) + { + OnCopySkipped?.Invoke(descriptor); + } + + internal void Mounted(Descriptor descriptor, string sourceRepository) + { + OnMounted?.Invoke(descriptor, sourceRepository); + } +} public static class Extensions { @@ -36,38 +69,89 @@ public static class Extensions /// /// /// - public static async Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CancellationToken cancellationToken = default) + public static async Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CancellationToken cancellationToken = default, CopyOptions? copyOptions = default) { if (string.IsNullOrEmpty(dstRef)) { dstRef = srcRef; } var root = await src.ResolveAsync(srcRef, cancellationToken).ConfigureAwait(false); - await src.CopyGraphAsync(dst, root, cancellationToken).ConfigureAwait(false); + await src.CopyGraphAsync(dst, root, cancellationToken, copyOptions).ConfigureAwait(false); await dst.TagAsync(root, dstRef, cancellationToken).ConfigureAwait(false); return root; } - public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CancellationToken cancellationToken) + public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CancellationToken cancellationToken, CopyOptions? copyOptions = default) { // check if node exists in target if (await dst.ExistsAsync(node, cancellationToken).ConfigureAwait(false)) { + copyOptions?.CopySkipped(node); return; } // retrieve successors var successors = await src.GetSuccessorsAsync(node, cancellationToken).ConfigureAwait(false); - // obtain data stream - var dataStream = await src.FetchAsync(node, cancellationToken).ConfigureAwait(false); + // check if the node has successors - if (successors != null) + foreach (var childNode in successors) + { + await src.CopyGraphAsync(dst, childNode, cancellationToken, copyOptions).ConfigureAwait(false); + } + + var sourceRepositories = copyOptions?.MountFrom(node) ?? []; + if (dst is IMounter mounter && sourceRepositories.Length > 0) { - foreach (var childNode in successors) + for (var i = 0; i < sourceRepositories.Length; i++) { - await src.CopyGraphAsync(dst, childNode, cancellationToken).ConfigureAwait(false); + var sourceRepository = sourceRepositories[i]; + var mountFailed = false; + + async Task GetContents(CancellationToken token) + { + // the invocation of getContent indicates that mounting has failed + mountFailed = true; + + if (i < sourceRepositories.Length - 1) + { + // If this is not the last one, skip this source and try next one + // We want to return an error that we will test for from mounter.Mount() + throw new SkipSourceException(); + } + + // this is the last iteration so we need to actually get the content and do the copy + // but first call the PreCopy function + copyOptions?.PreCopy(node); + return await src.FetchAsync(node, token).ConfigureAwait(false); + } + + try + { + await mounter.MountAsync(node, sourceRepository, GetContents, cancellationToken).ConfigureAwait(false); + } + catch (SkipSourceException) + { + } + + if (!mountFailed) + { + copyOptions?.Mounted(node, sourceRepository); + return; + } } } - await dst.PushAsync(node, dataStream, cancellationToken).ConfigureAwait(false); + else + { + // alternatively we just copy it + copyOptions?.PreCopy(node); + var dataStream = await src.FetchAsync(node, cancellationToken).ConfigureAwait(false); + await dst.PushAsync(node, dataStream, cancellationToken).ConfigureAwait(false); + } + + // we copied it + copyOptions?.PostCopy(node); } + + private class SkipSourceException : Exception {} } + diff --git a/src/OrasProject.Oras/Registry/IMounter.cs b/src/OrasProject.Oras/Registry/IMounter.cs new file mode 100644 index 0000000..8c645dc --- /dev/null +++ b/src/OrasProject.Oras/Registry/IMounter.cs @@ -0,0 +1,24 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using OrasProject.Oras.Oci; + +namespace OrasProject.Oras.Registry; + +/// +/// Mounter allows cross-repository blob mounts. +/// +public interface IMounter +{ + /// + /// Mount makes the blob with the given descriptor in fromRepo + /// available in the repository signified by the receiver. + /// + /// + /// + /// + /// + /// + Task MountAsync(Descriptor descriptor, string contentReference, Func>? getContents, CancellationToken cancellationToken); +} diff --git a/src/OrasProject.Oras/Registry/IRepository.cs b/src/OrasProject.Oras/Registry/IRepository.cs index b163e2f..41682c5 100644 --- a/src/OrasProject.Oras/Registry/IRepository.cs +++ b/src/OrasProject.Oras/Registry/IRepository.cs @@ -27,7 +27,7 @@ namespace OrasProject.Oras.Registry; /// Furthermore, this interface also provides the ability to enforce the /// separation of the blob and the manifests CASs. /// -public interface IRepository : ITarget, IReferenceFetchable, IReferencePushable, IDeletable, ITagListable +public interface IRepository : ITarget, IReferenceFetchable, IReferencePushable, IDeletable, ITagListable, IMounter { /// /// Blobs provides access to the blob CAS only, which contains config blobs,layers, and other generic blobs. diff --git a/src/OrasProject.Oras/Registry/Remote/BlobStore.cs b/src/OrasProject.Oras/Registry/Remote/BlobStore.cs index 52b0783..791acb7 100644 --- a/src/OrasProject.Oras/Registry/Remote/BlobStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/BlobStore.cs @@ -25,7 +25,7 @@ namespace OrasProject.Oras.Registry.Remote; -public class BlobStore(Repository repository) : IBlobStore +public class BlobStore(Repository repository) : IBlobStore, IMounter { public Repository Repository { get; init; } = repository; @@ -148,25 +148,7 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok url = location.IsAbsoluteUri ? location : new Uri(url, location); } - // monolithic upload - // add digest key to query string with expected digest value - var req = new HttpRequestMessage(HttpMethod.Put, new UriBuilder(url) - { - Query = $"{url.Query}&digest={HttpUtility.UrlEncode(expected.Digest)}" - }.Uri); - req.Content = new StreamContent(content); - req.Content.Headers.ContentLength = expected.Size; - - // the expected media type is ignored as in the API doc. - req.Content.Headers.ContentType = new MediaTypeHeaderValue(MediaTypeNames.Application.Octet); - - using (var response = await Repository.Options.HttpClient.SendAsync(req, cancellationToken).ConfigureAwait(false)) - { - if (response.StatusCode != HttpStatusCode.Created) - { - throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); - } - } + await InternalPushAsync(url, expected, content, cancellationToken); } /// @@ -198,4 +180,98 @@ public async Task ResolveAsync(string reference, CancellationToken c /// public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) => await Repository.DeleteAsync(target, false, cancellationToken).ConfigureAwait(false); + + /// + /// Mounts the given descriptor from contentReference into the blob store. + /// + /// + /// + /// + /// + /// + /// + public async Task MountAsync(Descriptor descriptor, string contentReference, + Func>? getContents, CancellationToken cancellationToken) + { + var url = new UriFactory(Repository.Options).BuildRepositoryBlobUpload(); + var mountReq = new HttpRequestMessage(HttpMethod.Post, new UriBuilder(url) + { + Query = + $"{url.Query}&mount={HttpUtility.UrlEncode(descriptor.Digest)}&from={HttpUtility.UrlEncode(contentReference)}" + }.Uri); + + using (var response = await Repository.Options.HttpClient.SendAsync(mountReq, cancellationToken) + .ConfigureAwait(false)) + { + switch (response.StatusCode) + { + case HttpStatusCode.Created: + // 201, layer has been mounted + return; + case HttpStatusCode.Accepted: + { + // 202, mounting failed. upload session has begun + var location = response.Headers.Location ?? + throw new HttpRequestException("missing location header"); + url = location.IsAbsoluteUri ? location : new Uri(url, location); + break; + } + default: + throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); + } + } + + // From the [spec]: + // + // "If a registry does not support cross-repository mounting + // or is unable to mount the requested blob, + // it SHOULD return a 202. + // This indicates that the upload session has begun + // and that the client MAY proceed with the upload." + // + // So we need to get the content from somewhere in order to + // push it. If the caller has provided a getContent function, we + // can use that, otherwise pull the content from the source repository. + // + // [spec]: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#mounting-a-blob-from-another-repository + + Stream contents; + if (getContents != null) + { + contents = await getContents(cancellationToken).ConfigureAwait(false); + } + else + { + var referenceOptions = repository.Options with + { + Reference = Reference.Parse(contentReference), + }; + contents = await new Repository(referenceOptions).FetchAsync(descriptor, cancellationToken); + } + + await InternalPushAsync(url, descriptor, contents, cancellationToken).ConfigureAwait(false); + } + + private async Task InternalPushAsync(Uri url, Descriptor descriptor, Stream content, + CancellationToken cancellationToken) + { + // monolithic upload + // add digest key to query string with descriptor digest value + var req = new HttpRequestMessage(HttpMethod.Put, new UriBuilder(url) + { + Query = $"{url.Query}&digest={HttpUtility.UrlEncode(descriptor.Digest)}" + }.Uri); + req.Content = new StreamContent(content); + req.Content.Headers.ContentLength = descriptor.Size; + + // the descriptor media type is ignored as in the API doc. + req.Content.Headers.ContentType = new MediaTypeHeaderValue(MediaTypeNames.Application.Octet); + + using var response = + await Repository.Options.HttpClient.SendAsync(req, cancellationToken).ConfigureAwait(false); + if (response.StatusCode != HttpStatusCode.Created) + { + throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); + } + } } diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 62d73bc..49d9328 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -331,4 +331,22 @@ internal Reference ParseReferenceFromContentReference(string reference) /// /// private IBlobStore BlobStore(Descriptor desc) => IsManifest(desc) ? Manifests : Blobs; + + /// + /// Mount makes the blob with the given digest in fromRepo + /// available in the repository signified by the receiver. + /// + /// This avoids the need to pull content down from fromRepo only to push it to r. + /// + /// If the registry does not implement mounting, getContent will be used to get the + /// content to push. If getContent is null, the content will be pulled from the source + /// repository. + /// + /// + /// + /// + /// + /// + public Task MountAsync(Descriptor descriptor, string contentReference, Func>? getContents, CancellationToken cancellationToken) + => ((IMounter)Blobs).MountAsync(descriptor,contentReference, getContents, cancellationToken); } diff --git a/tests/OrasProject.Oras.Tests/CopyTest.cs b/tests/OrasProject.Oras.Tests/CopyTest.cs index 4f26873..3959b62 100644 --- a/tests/OrasProject.Oras.Tests/CopyTest.cs +++ b/tests/OrasProject.Oras.Tests/CopyTest.cs @@ -142,4 +142,96 @@ public async Task CanCopyBetweenMemoryTargets() } } + + [Fact] + public async Task CanCopyBetweenMemoryTargetsMountingFromDestination() + { + var sourceTarget = new MemoryStore(); + var cancellationToken = new CancellationToken(); + var blobs = new List(); + var descs = new List(); + var appendBlob = (string mediaType, byte[] blob) => + { + blobs.Add(blob); + var desc = new Descriptor + { + MediaType = mediaType, + Digest = Digest.ComputeSHA256(blob), + Size = blob.Length + }; + descs.Add(desc); + }; + var generateManifest = (Descriptor config, List layers) => + { + var manifest = new Manifest + { + Config = config, + Layers = layers + }; + var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); + appendBlob(MediaType.ImageManifest, manifestBytes); + }; + var getBytes = (string data) => Encoding.UTF8.GetBytes(data); + appendBlob(MediaType.ImageConfig, getBytes("config")); // blob 0 + appendBlob(MediaType.ImageLayer, getBytes("foo")); // blob 1 + appendBlob(MediaType.ImageLayer, getBytes("bar")); // blob 2 + generateManifest(descs[0], descs.GetRange(1, 2)); // blob 3 + + appendBlob(MediaType.ImageConfig, getBytes("config2")); // blob 4 + appendBlob(MediaType.ImageLayer, getBytes("bar2")); // blob 5 + generateManifest(descs[4], [descs[1], descs[5]]); // blob 6 + + for (var i = 0; i < blobs.Count; i++) + { + await sourceTarget.PushAsync(descs[i], new MemoryStream(blobs[i]), cancellationToken); + } + + var root = descs[3]; + var reference = "foobar"; + await sourceTarget.TagAsync(root, reference, cancellationToken); + + var root2 = descs[6]; + var reference2 = "other/foobar"; + await sourceTarget.TagAsync(root2, reference2, cancellationToken); + + var destinationTarget = new MemoryStore(); + var gotDesc = await sourceTarget.CopyAsync(reference, destinationTarget, "", cancellationToken); + Assert.Equal(gotDesc, root); + Assert.Equal(await destinationTarget.ResolveAsync(reference, cancellationToken), root); + + for (var i = 0; i < 3; i++) + { + Assert.True(await destinationTarget.ExistsAsync(descs[i], cancellationToken)); + var fetchContent = await destinationTarget.FetchAsync(descs[i], cancellationToken); + var memoryStream = new MemoryStream(); + await fetchContent.CopyToAsync(memoryStream, cancellationToken); + var bytes = memoryStream.ToArray(); + Assert.Equal(blobs[i], bytes); + } + + var copyOpts = new CopyOptions() + { + MountFrom = d => [reference] + }; + var mounted = false; + copyOpts.OnMounted += (d, s) => + { + mounted = true; + }; + var gotDesc2 = await sourceTarget.CopyAsync(reference2, destinationTarget, reference2, cancellationToken, copyOpts); + + Assert.Equal(gotDesc2, root2); + Assert.Equal(await destinationTarget.ResolveAsync(reference2, cancellationToken), root2); + Assert.True(mounted); + + for (var i = 4; i < descs.Count; i++) + { + Assert.True(await destinationTarget.ExistsAsync(descs[i], cancellationToken)); + var fetchContent = await destinationTarget.FetchAsync(descs[i], cancellationToken); + var memoryStream = new MemoryStream(); + await fetchContent.CopyToAsync(memoryStream, cancellationToken); + var bytes = memoryStream.ToArray(); + Assert.Equal(blobs[i], bytes); + } + } } From b453fbcda61651c4136dcc83591cec06ed2081e4 Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Wed, 25 Sep 2024 23:02:06 -0300 Subject: [PATCH 02/19] fix: standard header Signed-off-by: Leonardo Chaia --- src/OrasProject.Oras/Registry/IMounter.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/OrasProject.Oras/Registry/IMounter.cs b/src/OrasProject.Oras/Registry/IMounter.cs index 8c645dc..10cd3f4 100644 --- a/src/OrasProject.Oras/Registry/IMounter.cs +++ b/src/OrasProject.Oras/Registry/IMounter.cs @@ -1,3 +1,16 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + using System; using System.IO; using System.Threading; From 6a7a1e9f3df76b362bc2b10cc534188e59b86cbe Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Wed, 2 Oct 2024 22:14:19 -0300 Subject: [PATCH 03/19] chore: adds repository tests Signed-off-by: Leonardo Chaia --- .../Remote/RepositoryTest.cs | 312 +++++++++++++++++- 1 file changed, 310 insertions(+), 2 deletions(-) diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 84df80a..737ee0e 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -330,6 +330,310 @@ public async Task Repository_PushAsync() await repo.PushAsync(indexDesc, new MemoryStream(index), cancellationToken); Assert.Equal(index, gotIndex); } + + /// + /// Repository_MountAsync tests the MountAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_MountAsync() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var gotMount = 0; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/") + { + var queries = HttpUtility.ParseQueryString(req.RequestUri.Query); + if (queries["mount"] != blobDesc.Digest) + { + resp.StatusCode = HttpStatusCode.InternalServerError; + return resp; + } + if (queries["from"] != "test") + { + resp.StatusCode = HttpStatusCode.InternalServerError; + return resp; + } + gotMount++; + resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + resp.StatusCode = HttpStatusCode.Created; + return resp; + } + + resp.StatusCode = HttpStatusCode.InternalServerError; + return resp; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test2"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + await repo.MountAsync(blobDesc, "test", null, cancellationToken); + Assert.Equal(1, gotMount); + } + + /// + /// Repository_MountAsync_Fallback tests the MountAsync method of the Repository when the server doesn't support mount query parameters. + /// + /// + [Fact] + public async Task Repository_MountAsync_Fallback() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + string sequence = ""; + byte[] gotBlob = Array.Empty(); + var uuid = "4fd53bc9-565d-4527-ab80-3e051ac4880c"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/") + { + resp.Headers.Location = new Uri("/v2/test2/blobs/uploads/" + uuid, UriKind.Relative); + resp.StatusCode = HttpStatusCode.Accepted; + sequence += "post "; + return resp; + } + if (req.Method == HttpMethod.Get && req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) + { + resp.Content.Headers.Add("Content-Type", "application/octet-stream"); + resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + resp.Content = new ByteArrayContent(blob); + resp.StatusCode = HttpStatusCode.OK; + sequence += "get "; + return resp; + } + if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/" + uuid) + { + if (req.Content?.Headers.GetValues("Content-Type").FirstOrDefault() != "application/octet-stream") + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + gotBlob = req.Content!.ReadAsByteArrayAsync(cancellationToken).Result; + resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + resp.StatusCode = HttpStatusCode.Created; + sequence += "put "; + return resp; + } + + resp.StatusCode = HttpStatusCode.Forbidden; + return resp; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test2"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + // getContent is null + sequence = ""; + await repo.MountAsync(blobDesc, "localhost:5000/test", null, cancellationToken); + Assert.Equal(blob, gotBlob); + Assert.Equal("post get put ", sequence); + + // getContent is non-null + sequence = ""; + await repo.MountAsync(blobDesc, "localhost:5000/test", _ => Task.FromResult(new MemoryStream(blob)), cancellationToken); + Assert.Equal(blob, gotBlob); + Assert.Equal("post put ", sequence); + } + + /// + /// Repository_MountAsync_Error tests the error handling of the MountAsync method of the Repository. + /// + /// + [Fact] + public async Task Repository_MountAsync_Error() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/") + { + resp.StatusCode = HttpStatusCode.BadRequest; + resp.Content = new StringContent(@"{ ""errors"": [ { ""code"": ""NAME_UNKNOWN"", ""message"": ""some error"" } ] }"); + return resp; + } + + resp.StatusCode = HttpStatusCode.InternalServerError; + return resp; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + var ex = await Assert.ThrowsAsync(async () => + { + await repo.MountAsync(blobDesc, "foo", null, cancellationToken); + }); + + Assert.NotNull(ex.Errors); + Assert.Single(ex.Errors); + Assert.Equal("NAME_UNKNOWN", ex.Errors[0].Code); + Assert.Equal("some error", ex.Errors[0].Message); + } + + /// + /// Repository_MountAsync_Fallback_GetContent tests the case where the server doesn't recognize mount query parameters, + /// falling back to the regular push flow, using the getContent function parameter. + /// + /// + [Fact] + public async Task Repository_MountAsync_Fallback_GetContent() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + string sequence = ""; + byte[] gotBlob = Array.Empty(); + var uuid = "4fd53bc9-565d-4527-ab80-3e051ac4880c"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + + if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/") + { + resp.Headers.Location = new Uri("/v2/test2/blobs/uploads/" + uuid, UriKind.Relative); + resp.StatusCode = HttpStatusCode.Accepted; + sequence += "post "; + return resp; + } + if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/" + uuid) + { + if (req.Content?.Headers.GetValues("Content-Type").FirstOrDefault() != "application/octet-stream") + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + gotBlob = req.Content!.ReadAsByteArrayAsync(cancellationToken).Result; + resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + resp.StatusCode = HttpStatusCode.Created; + sequence += "put "; + return resp; + } + + resp.StatusCode = HttpStatusCode.Forbidden; + return resp; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test2"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + await repo.MountAsync(blobDesc, "test", _ => Task.FromResult(new MemoryStream(blob)), cancellationToken); + + // Assert that the blob was pushed correctly + Assert.Equal(blob, gotBlob); + // Assert that the request sequence matches the expected behavior + Assert.Equal("post put ", sequence); + } + + /// + /// Repository_MountAsync_Fallback_GetContentError tests the case where the server doesn't recognize mount query parameters, + /// falling back to the regular push flow, but the caller wants to avoid the pull/push pattern, so an error is returned from getContent. + /// + /// + [Fact] + public async Task Repository_MountAsync_Fallback_GetContentError() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + string sequence = ""; + var uuid = "4fd53bc9-565d-4527-ab80-3e051ac4880c"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + + if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/") + { + resp.Headers.Location = new Uri("/v2/test2/blobs/uploads/" + uuid, UriKind.Relative); + resp.StatusCode = HttpStatusCode.Accepted; + sequence += "post "; + return resp; + } + + resp.StatusCode = HttpStatusCode.Forbidden; + return resp; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test2"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + var testErr = new Exception("test error"); + var ex = await Assert.ThrowsAsync(async () => + { + await repo.MountAsync(blobDesc, "test", _ => throw testErr, cancellationToken); + }); + + Assert.Equal(testErr, ex); + Assert.Equal("post ", sequence); + } /// /// Repository_ExistsAsync tests the ExistsAsync method of the Repository @@ -2243,14 +2547,17 @@ public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigest if (method == HttpMethod.Get) { resp.Content = new ByteArrayContent(_theAmazingBanClan); - resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Content.Headers.Add("Content-Type", + new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } else { - resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Content.Headers.Add("Content-Type", + new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } + resp.RequestMessage = new HttpRequestMessage() { Method = method @@ -2273,6 +2580,7 @@ public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigest } } + if (errExpected && !err) { throw new Exception($"[Manifest.{method}] {testName}; expected error for request, but got none"); From 88c6793aa9b3e27ca4d244a744224e9fa01a63fe Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Thu, 17 Oct 2024 15:38:20 -0300 Subject: [PATCH 04/19] fix: copy test Signed-off-by: Leonardo Chaia --- src/OrasProject.Oras/Content/MemoryStore.cs | 17 +-- tests/OrasProject.Oras.Tests/CopyTest.cs | 157 ++++++++++++++------ 2 files changed, 113 insertions(+), 61 deletions(-) diff --git a/src/OrasProject.Oras/Content/MemoryStore.cs b/src/OrasProject.Oras/Content/MemoryStore.cs index 528765c..9a9f0f0 100644 --- a/src/OrasProject.Oras/Content/MemoryStore.cs +++ b/src/OrasProject.Oras/Content/MemoryStore.cs @@ -11,19 +11,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Threading; using System.Threading.Tasks; -using OrasProject.Oras.Registry; namespace OrasProject.Oras.Content; -public class MemoryStore : ITarget, IPredecessorFindable, IMounter +public class MemoryStore : ITarget, IPredecessorFindable { private readonly MemoryStorage _storage = new(); private readonly MemoryTagStore _tagResolver = new(); @@ -97,16 +94,4 @@ public async Task TagAsync(Descriptor descriptor, string reference, Cancellation /// public async Task> GetPredecessorsAsync(Descriptor node, CancellationToken cancellationToken = default) => await _graph.GetPredecessorsAsync(node, cancellationToken).ConfigureAwait(false); - - public async Task MountAsync(Descriptor descriptor, string contentReference, Func>? getContents, CancellationToken cancellationToken) - { - var taggedDescriptor = await _tagResolver.ResolveAsync(contentReference, cancellationToken).ConfigureAwait(false); - var successors = await _storage.GetSuccessorsAsync(taggedDescriptor, cancellationToken); - - if (descriptor != taggedDescriptor && !successors.Contains(descriptor)) - { - await _storage.PushAsync(descriptor, await getContents(cancellationToken), cancellationToken).ConfigureAwait(false); - await _graph.IndexAsync(_storage, descriptor, cancellationToken).ConfigureAwait(false); - } - } } diff --git a/tests/OrasProject.Oras.Tests/CopyTest.cs b/tests/OrasProject.Oras.Tests/CopyTest.cs index 3959b62..04ed235 100644 --- a/tests/OrasProject.Oras.Tests/CopyTest.cs +++ b/tests/OrasProject.Oras.Tests/CopyTest.cs @@ -15,7 +15,9 @@ using OrasProject.Oras.Oci; using System.Text; using System.Text.Json; +using OrasProject.Oras.Registry; using Xunit; +using Index = OrasProject.Oras.Oci.Index; namespace OrasProject.Oras.Tests; @@ -144,12 +146,14 @@ public async Task CanCopyBetweenMemoryTargets() } [Fact] - public async Task CanCopyBetweenMemoryTargetsMountingFromDestination() + public async Task CanMountFromSourceRepository() { var sourceTarget = new MemoryStore(); var cancellationToken = new CancellationToken(); var blobs = new List(); var descs = new List(); + + // Utility function to append blobs var appendBlob = (string mediaType, byte[] blob) => { blobs.Add(blob); @@ -161,6 +165,7 @@ public async Task CanCopyBetweenMemoryTargetsMountingFromDestination() }; descs.Add(desc); }; + var generateManifest = (Descriptor config, List layers) => { var manifest = new Manifest @@ -171,15 +176,26 @@ public async Task CanCopyBetweenMemoryTargetsMountingFromDestination() var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); appendBlob(MediaType.ImageManifest, manifestBytes); }; - var getBytes = (string data) => Encoding.UTF8.GetBytes(data); - appendBlob(MediaType.ImageConfig, getBytes("config")); // blob 0 - appendBlob(MediaType.ImageLayer, getBytes("foo")); // blob 1 - appendBlob(MediaType.ImageLayer, getBytes("bar")); // blob 2 - generateManifest(descs[0], descs.GetRange(1, 2)); // blob 3 - appendBlob(MediaType.ImageConfig, getBytes("config2")); // blob 4 - appendBlob(MediaType.ImageLayer, getBytes("bar2")); // blob 5 - generateManifest(descs[4], [descs[1], descs[5]]); // blob 6 + var generateIndex = (Descriptor config, List manifests) => + { + var manifest = new Index() + { + Manifests = manifests + }; + var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); + appendBlob(MediaType.ImageIndex, manifestBytes); + }; + + // Generate blobs and manifest + var getBytes = (string data) => Encoding.UTF8.GetBytes(data); + appendBlob(MediaType.ImageConfig, getBytes("config")); // Blob 0 + appendBlob(MediaType.ImageLayer, getBytes("foo")); // Blob 1 + appendBlob(MediaType.ImageLayer, getBytes("bar")); // Blob 2 + generateManifest(descs[0], descs[1..3]); // Blob 3 + appendBlob(MediaType.ImageLayer, getBytes("hello")); // Blob 4 + generateManifest(descs[0], [descs[4]]); // Blob 5 + generateIndex(descs[3], [descs[5]]); // Blob 6 for (var i = 0; i < blobs.Count; i++) { @@ -187,51 +203,102 @@ public async Task CanCopyBetweenMemoryTargetsMountingFromDestination() } var root = descs[3]; - var reference = "foobar"; - await sourceTarget.TagAsync(root, reference, cancellationToken); + var destinationTarget = new CountingStore(new MemoryStore()); - var root2 = descs[6]; - var reference2 = "other/foobar"; - await sourceTarget.TagAsync(root2, reference2, cancellationToken); - - var destinationTarget = new MemoryStore(); - var gotDesc = await sourceTarget.CopyAsync(reference, destinationTarget, "", cancellationToken); - Assert.Equal(gotDesc, root); - Assert.Equal(await destinationTarget.ResolveAsync(reference, cancellationToken), root); + var numMount = new AtomicCounter(); + destinationTarget.OnMount = async (descriptor,contentReference, getContents, cancellationToken) => + { + numMount.Increment(); + if (contentReference != "source") + { + throw new Exception($"fromRepo = {contentReference}, want source"); + } + + var fetchedContent = await sourceTarget.FetchAsync(descriptor, cancellationToken); + await destinationTarget.PushAsync(descriptor, fetchedContent, cancellationToken); // Bypass counters + }; - for (var i = 0; i < 3; i++) + var copyOptions = new CopyOptions { - Assert.True(await destinationTarget.ExistsAsync(descs[i], cancellationToken)); - var fetchContent = await destinationTarget.FetchAsync(descs[i], cancellationToken); - var memoryStream = new MemoryStream(); - await fetchContent.CopyToAsync(memoryStream, cancellationToken); - var bytes = memoryStream.ToArray(); - Assert.Equal(blobs[i], bytes); + MountFrom = (desc) => new[] { "source" }, + }; + copyOptions.OnPreCopy += (desc) => destinationTarget.PreCopyCounter.Increment(); + copyOptions.OnPostCopy += (desc) => destinationTarget.PostCopyCounter.Increment(); + copyOptions.OnMounted += (desc,reference) => destinationTarget.OnMountedCounter.Increment(); + + // Perform the CopyGraph operation + await sourceTarget.CopyGraphAsync(destinationTarget, root, cancellationToken, copyOptions); + + // Verify the expected counts + Assert.Equal(4, numMount.Value); + Assert.Equal(4, destinationTarget.OnMountedCounter.Value); + Assert.Equal(4, destinationTarget.MountFromCounter.Value); + } + + // Custom class for tracking operation counts + public class CountingStore : ITarget, IMounter + { + public MemoryStore Store { get; } + public AtomicCounter ExistsCounter { get; } = new AtomicCounter(); + public AtomicCounter FetchCounter { get; } = new AtomicCounter(); + public AtomicCounter PushCounter { get; } = new AtomicCounter(); + public AtomicCounter OnMountedCounter { get; } = new AtomicCounter(); + public AtomicCounter PreCopyCounter { get; } = new AtomicCounter(); + public AtomicCounter PostCopyCounter { get; } = new AtomicCounter(); + public AtomicCounter MountFromCounter { get; } = new AtomicCounter(); + + public Func>?, CancellationToken, Task> OnMount { get; set; } + + public CountingStore(MemoryStore store) + { + Store = store; } - var copyOpts = new CopyOptions() + public async Task ExistsAsync(Descriptor desc, CancellationToken cancellationToken) { - MountFrom = d => [reference] - }; - var mounted = false; - copyOpts.OnMounted += (d, s) => + ExistsCounter.Increment(); + return await Store.ExistsAsync(desc, cancellationToken); + } + + public async Task FetchAsync(Descriptor desc, CancellationToken cancellationToken) { - mounted = true; - }; - var gotDesc2 = await sourceTarget.CopyAsync(reference2, destinationTarget, reference2, cancellationToken, copyOpts); + FetchCounter.Increment(); + return await Store.FetchAsync(desc, cancellationToken); + } - Assert.Equal(gotDesc2, root2); - Assert.Equal(await destinationTarget.ResolveAsync(reference2, cancellationToken), root2); - Assert.True(mounted); - - for (var i = 4; i < descs.Count; i++) + public async Task PushAsync(Descriptor desc, Stream content, CancellationToken cancellationToken) { - Assert.True(await destinationTarget.ExistsAsync(descs[i], cancellationToken)); - var fetchContent = await destinationTarget.FetchAsync(descs[i], cancellationToken); - var memoryStream = new MemoryStream(); - await fetchContent.CopyToAsync(memoryStream, cancellationToken); - var bytes = memoryStream.ToArray(); - Assert.Equal(blobs[i], bytes); + PushCounter.Increment(); + await Store.PushAsync(desc, content, cancellationToken); + } + + public async Task TagAsync(Descriptor desc, string reference, CancellationToken cancellationToken) + { + await Store.TagAsync(desc, reference, cancellationToken); + } + + public Task ResolveAsync(string reference, CancellationToken cancellationToken) + { + return Store.ResolveAsync(reference, cancellationToken); + } + + public async Task MountAsync(Descriptor descriptor, string contentReference, Func>? getContents, CancellationToken cancellationToken) + { + MountFromCounter.Increment(); + await OnMount.Invoke(descriptor,contentReference, getContents, cancellationToken); + } + } + + // Utility class for thread-safe counter increments + public class AtomicCounter + { + private long _value; + + public long Value => Interlocked.Read(ref _value); + + public void Increment() + { + Interlocked.Increment(ref _value); } } } From dbe3509f379138f23e69c4dc6d03ed4ca3f352c7 Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Mon, 11 Nov 2024 07:32:50 -0300 Subject: [PATCH 05/19] chore: removes mounting support to be implemented in separate PR Signed-off-by: Leonardo Chaia --- src/OrasProject.Oras/CopyOptions.cs | 39 +++ src/OrasProject.Oras/Extensions.cs | 85 +---- src/OrasProject.Oras/Registry/IMounter.cs | 37 --- src/OrasProject.Oras/Registry/IRepository.cs | 2 +- .../Registry/Remote/BlobStore.cs | 116 ++----- .../Registry/Remote/Repository.cs | 18 - tests/OrasProject.Oras.Tests/CopyTest.cs | 157 --------- .../Remote/RepositoryTest.cs | 312 +----------------- 8 files changed, 66 insertions(+), 700 deletions(-) create mode 100644 src/OrasProject.Oras/CopyOptions.cs delete mode 100644 src/OrasProject.Oras/Registry/IMounter.cs diff --git a/src/OrasProject.Oras/CopyOptions.cs b/src/OrasProject.Oras/CopyOptions.cs new file mode 100644 index 0000000..7d77ea2 --- /dev/null +++ b/src/OrasProject.Oras/CopyOptions.cs @@ -0,0 +1,39 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using OrasProject.Oras.Oci; + +namespace OrasProject.Oras; + +public struct CopyOptions +{ + public event Action OnPreCopy; + public event Action OnPostCopy; + public event Action OnCopySkipped; + + internal void PreCopy(Descriptor descriptor) + { + OnPreCopy?.Invoke(descriptor); + } + + internal void PostCopy(Descriptor descriptor) + { + OnPostCopy?.Invoke(descriptor); + } + + internal void CopySkipped(Descriptor descriptor) + { + OnCopySkipped?.Invoke(descriptor); + } +} diff --git a/src/OrasProject.Oras/Extensions.cs b/src/OrasProject.Oras/Extensions.cs index 595948b..8c566c4 100644 --- a/src/OrasProject.Oras/Extensions.cs +++ b/src/OrasProject.Oras/Extensions.cs @@ -21,37 +21,6 @@ namespace OrasProject.Oras; -public struct CopyOptions -{ - // public int Concurrency { get; set; } - - public event Action OnPreCopy; - public event Action OnPostCopy; - public event Action OnCopySkipped; - public event Action OnMounted; - - public Func MountFrom { get; set; } - - internal void PreCopy(Descriptor descriptor) - { - OnPreCopy?.Invoke(descriptor); - } - - internal void PostCopy(Descriptor descriptor) - { - OnPostCopy?.Invoke(descriptor); - } - - internal void CopySkipped(Descriptor descriptor) - { - OnCopySkipped?.Invoke(descriptor); - } - - internal void Mounted(Descriptor descriptor, string sourceRepository) - { - OnMounted?.Invoke(descriptor, sourceRepository); - } -} public static class Extensions { @@ -99,59 +68,13 @@ public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descripto await src.CopyGraphAsync(dst, childNode, cancellationToken, copyOptions).ConfigureAwait(false); } - var sourceRepositories = copyOptions?.MountFrom(node) ?? []; - if (dst is IMounter mounter && sourceRepositories.Length > 0) - { - for (var i = 0; i < sourceRepositories.Length; i++) - { - var sourceRepository = sourceRepositories[i]; - var mountFailed = false; - - async Task GetContents(CancellationToken token) - { - // the invocation of getContent indicates that mounting has failed - mountFailed = true; - - if (i < sourceRepositories.Length - 1) - { - // If this is not the last one, skip this source and try next one - // We want to return an error that we will test for from mounter.Mount() - throw new SkipSourceException(); - } - - // this is the last iteration so we need to actually get the content and do the copy - // but first call the PreCopy function - copyOptions?.PreCopy(node); - return await src.FetchAsync(node, token).ConfigureAwait(false); - } - - try - { - await mounter.MountAsync(node, sourceRepository, GetContents, cancellationToken).ConfigureAwait(false); - } - catch (SkipSourceException) - { - } - - if (!mountFailed) - { - copyOptions?.Mounted(node, sourceRepository); - return; - } - } - } - else - { - // alternatively we just copy it - copyOptions?.PreCopy(node); - var dataStream = await src.FetchAsync(node, cancellationToken).ConfigureAwait(false); - await dst.PushAsync(node, dataStream, cancellationToken).ConfigureAwait(false); - } + // perform the copy + copyOptions?.PreCopy(node); + var dataStream = await src.FetchAsync(node, cancellationToken).ConfigureAwait(false); + await dst.PushAsync(node, dataStream, cancellationToken).ConfigureAwait(false); // we copied it copyOptions?.PostCopy(node); } - - private class SkipSourceException : Exception {} } diff --git a/src/OrasProject.Oras/Registry/IMounter.cs b/src/OrasProject.Oras/Registry/IMounter.cs deleted file mode 100644 index 10cd3f4..0000000 --- a/src/OrasProject.Oras/Registry/IMounter.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright The ORAS Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using OrasProject.Oras.Oci; - -namespace OrasProject.Oras.Registry; - -/// -/// Mounter allows cross-repository blob mounts. -/// -public interface IMounter -{ - /// - /// Mount makes the blob with the given descriptor in fromRepo - /// available in the repository signified by the receiver. - /// - /// - /// - /// - /// - /// - Task MountAsync(Descriptor descriptor, string contentReference, Func>? getContents, CancellationToken cancellationToken); -} diff --git a/src/OrasProject.Oras/Registry/IRepository.cs b/src/OrasProject.Oras/Registry/IRepository.cs index 41682c5..b163e2f 100644 --- a/src/OrasProject.Oras/Registry/IRepository.cs +++ b/src/OrasProject.Oras/Registry/IRepository.cs @@ -27,7 +27,7 @@ namespace OrasProject.Oras.Registry; /// Furthermore, this interface also provides the ability to enforce the /// separation of the blob and the manifests CASs. /// -public interface IRepository : ITarget, IReferenceFetchable, IReferencePushable, IDeletable, ITagListable, IMounter +public interface IRepository : ITarget, IReferenceFetchable, IReferencePushable, IDeletable, ITagListable { /// /// Blobs provides access to the blob CAS only, which contains config blobs,layers, and other generic blobs. diff --git a/src/OrasProject.Oras/Registry/Remote/BlobStore.cs b/src/OrasProject.Oras/Registry/Remote/BlobStore.cs index 791acb7..52b0783 100644 --- a/src/OrasProject.Oras/Registry/Remote/BlobStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/BlobStore.cs @@ -25,7 +25,7 @@ namespace OrasProject.Oras.Registry.Remote; -public class BlobStore(Repository repository) : IBlobStore, IMounter +public class BlobStore(Repository repository) : IBlobStore { public Repository Repository { get; init; } = repository; @@ -148,7 +148,25 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok url = location.IsAbsoluteUri ? location : new Uri(url, location); } - await InternalPushAsync(url, expected, content, cancellationToken); + // monolithic upload + // add digest key to query string with expected digest value + var req = new HttpRequestMessage(HttpMethod.Put, new UriBuilder(url) + { + Query = $"{url.Query}&digest={HttpUtility.UrlEncode(expected.Digest)}" + }.Uri); + req.Content = new StreamContent(content); + req.Content.Headers.ContentLength = expected.Size; + + // the expected media type is ignored as in the API doc. + req.Content.Headers.ContentType = new MediaTypeHeaderValue(MediaTypeNames.Application.Octet); + + using (var response = await Repository.Options.HttpClient.SendAsync(req, cancellationToken).ConfigureAwait(false)) + { + if (response.StatusCode != HttpStatusCode.Created) + { + throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); + } + } } /// @@ -180,98 +198,4 @@ public async Task ResolveAsync(string reference, CancellationToken c /// public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) => await Repository.DeleteAsync(target, false, cancellationToken).ConfigureAwait(false); - - /// - /// Mounts the given descriptor from contentReference into the blob store. - /// - /// - /// - /// - /// - /// - /// - public async Task MountAsync(Descriptor descriptor, string contentReference, - Func>? getContents, CancellationToken cancellationToken) - { - var url = new UriFactory(Repository.Options).BuildRepositoryBlobUpload(); - var mountReq = new HttpRequestMessage(HttpMethod.Post, new UriBuilder(url) - { - Query = - $"{url.Query}&mount={HttpUtility.UrlEncode(descriptor.Digest)}&from={HttpUtility.UrlEncode(contentReference)}" - }.Uri); - - using (var response = await Repository.Options.HttpClient.SendAsync(mountReq, cancellationToken) - .ConfigureAwait(false)) - { - switch (response.StatusCode) - { - case HttpStatusCode.Created: - // 201, layer has been mounted - return; - case HttpStatusCode.Accepted: - { - // 202, mounting failed. upload session has begun - var location = response.Headers.Location ?? - throw new HttpRequestException("missing location header"); - url = location.IsAbsoluteUri ? location : new Uri(url, location); - break; - } - default: - throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); - } - } - - // From the [spec]: - // - // "If a registry does not support cross-repository mounting - // or is unable to mount the requested blob, - // it SHOULD return a 202. - // This indicates that the upload session has begun - // and that the client MAY proceed with the upload." - // - // So we need to get the content from somewhere in order to - // push it. If the caller has provided a getContent function, we - // can use that, otherwise pull the content from the source repository. - // - // [spec]: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#mounting-a-blob-from-another-repository - - Stream contents; - if (getContents != null) - { - contents = await getContents(cancellationToken).ConfigureAwait(false); - } - else - { - var referenceOptions = repository.Options with - { - Reference = Reference.Parse(contentReference), - }; - contents = await new Repository(referenceOptions).FetchAsync(descriptor, cancellationToken); - } - - await InternalPushAsync(url, descriptor, contents, cancellationToken).ConfigureAwait(false); - } - - private async Task InternalPushAsync(Uri url, Descriptor descriptor, Stream content, - CancellationToken cancellationToken) - { - // monolithic upload - // add digest key to query string with descriptor digest value - var req = new HttpRequestMessage(HttpMethod.Put, new UriBuilder(url) - { - Query = $"{url.Query}&digest={HttpUtility.UrlEncode(descriptor.Digest)}" - }.Uri); - req.Content = new StreamContent(content); - req.Content.Headers.ContentLength = descriptor.Size; - - // the descriptor media type is ignored as in the API doc. - req.Content.Headers.ContentType = new MediaTypeHeaderValue(MediaTypeNames.Application.Octet); - - using var response = - await Repository.Options.HttpClient.SendAsync(req, cancellationToken).ConfigureAwait(false); - if (response.StatusCode != HttpStatusCode.Created) - { - throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); - } - } } diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 49d9328..62d73bc 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -331,22 +331,4 @@ internal Reference ParseReferenceFromContentReference(string reference) /// /// private IBlobStore BlobStore(Descriptor desc) => IsManifest(desc) ? Manifests : Blobs; - - /// - /// Mount makes the blob with the given digest in fromRepo - /// available in the repository signified by the receiver. - /// - /// This avoids the need to pull content down from fromRepo only to push it to r. - /// - /// If the registry does not implement mounting, getContent will be used to get the - /// content to push. If getContent is null, the content will be pulled from the source - /// repository. - /// - /// - /// - /// - /// - /// - public Task MountAsync(Descriptor descriptor, string contentReference, Func>? getContents, CancellationToken cancellationToken) - => ((IMounter)Blobs).MountAsync(descriptor,contentReference, getContents, cancellationToken); } diff --git a/tests/OrasProject.Oras.Tests/CopyTest.cs b/tests/OrasProject.Oras.Tests/CopyTest.cs index 04ed235..4523bed 100644 --- a/tests/OrasProject.Oras.Tests/CopyTest.cs +++ b/tests/OrasProject.Oras.Tests/CopyTest.cs @@ -144,161 +144,4 @@ public async Task CanCopyBetweenMemoryTargets() } } - - [Fact] - public async Task CanMountFromSourceRepository() - { - var sourceTarget = new MemoryStore(); - var cancellationToken = new CancellationToken(); - var blobs = new List(); - var descs = new List(); - - // Utility function to append blobs - var appendBlob = (string mediaType, byte[] blob) => - { - blobs.Add(blob); - var desc = new Descriptor - { - MediaType = mediaType, - Digest = Digest.ComputeSHA256(blob), - Size = blob.Length - }; - descs.Add(desc); - }; - - var generateManifest = (Descriptor config, List layers) => - { - var manifest = new Manifest - { - Config = config, - Layers = layers - }; - var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); - appendBlob(MediaType.ImageManifest, manifestBytes); - }; - - var generateIndex = (Descriptor config, List manifests) => - { - var manifest = new Index() - { - Manifests = manifests - }; - var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); - appendBlob(MediaType.ImageIndex, manifestBytes); - }; - - // Generate blobs and manifest - var getBytes = (string data) => Encoding.UTF8.GetBytes(data); - appendBlob(MediaType.ImageConfig, getBytes("config")); // Blob 0 - appendBlob(MediaType.ImageLayer, getBytes("foo")); // Blob 1 - appendBlob(MediaType.ImageLayer, getBytes("bar")); // Blob 2 - generateManifest(descs[0], descs[1..3]); // Blob 3 - appendBlob(MediaType.ImageLayer, getBytes("hello")); // Blob 4 - generateManifest(descs[0], [descs[4]]); // Blob 5 - generateIndex(descs[3], [descs[5]]); // Blob 6 - - for (var i = 0; i < blobs.Count; i++) - { - await sourceTarget.PushAsync(descs[i], new MemoryStream(blobs[i]), cancellationToken); - } - - var root = descs[3]; - var destinationTarget = new CountingStore(new MemoryStore()); - - var numMount = new AtomicCounter(); - destinationTarget.OnMount = async (descriptor,contentReference, getContents, cancellationToken) => - { - numMount.Increment(); - if (contentReference != "source") - { - throw new Exception($"fromRepo = {contentReference}, want source"); - } - - var fetchedContent = await sourceTarget.FetchAsync(descriptor, cancellationToken); - await destinationTarget.PushAsync(descriptor, fetchedContent, cancellationToken); // Bypass counters - }; - - var copyOptions = new CopyOptions - { - MountFrom = (desc) => new[] { "source" }, - }; - copyOptions.OnPreCopy += (desc) => destinationTarget.PreCopyCounter.Increment(); - copyOptions.OnPostCopy += (desc) => destinationTarget.PostCopyCounter.Increment(); - copyOptions.OnMounted += (desc,reference) => destinationTarget.OnMountedCounter.Increment(); - - // Perform the CopyGraph operation - await sourceTarget.CopyGraphAsync(destinationTarget, root, cancellationToken, copyOptions); - - // Verify the expected counts - Assert.Equal(4, numMount.Value); - Assert.Equal(4, destinationTarget.OnMountedCounter.Value); - Assert.Equal(4, destinationTarget.MountFromCounter.Value); - } - - // Custom class for tracking operation counts - public class CountingStore : ITarget, IMounter - { - public MemoryStore Store { get; } - public AtomicCounter ExistsCounter { get; } = new AtomicCounter(); - public AtomicCounter FetchCounter { get; } = new AtomicCounter(); - public AtomicCounter PushCounter { get; } = new AtomicCounter(); - public AtomicCounter OnMountedCounter { get; } = new AtomicCounter(); - public AtomicCounter PreCopyCounter { get; } = new AtomicCounter(); - public AtomicCounter PostCopyCounter { get; } = new AtomicCounter(); - public AtomicCounter MountFromCounter { get; } = new AtomicCounter(); - - public Func>?, CancellationToken, Task> OnMount { get; set; } - - public CountingStore(MemoryStore store) - { - Store = store; - } - - public async Task ExistsAsync(Descriptor desc, CancellationToken cancellationToken) - { - ExistsCounter.Increment(); - return await Store.ExistsAsync(desc, cancellationToken); - } - - public async Task FetchAsync(Descriptor desc, CancellationToken cancellationToken) - { - FetchCounter.Increment(); - return await Store.FetchAsync(desc, cancellationToken); - } - - public async Task PushAsync(Descriptor desc, Stream content, CancellationToken cancellationToken) - { - PushCounter.Increment(); - await Store.PushAsync(desc, content, cancellationToken); - } - - public async Task TagAsync(Descriptor desc, string reference, CancellationToken cancellationToken) - { - await Store.TagAsync(desc, reference, cancellationToken); - } - - public Task ResolveAsync(string reference, CancellationToken cancellationToken) - { - return Store.ResolveAsync(reference, cancellationToken); - } - - public async Task MountAsync(Descriptor descriptor, string contentReference, Func>? getContents, CancellationToken cancellationToken) - { - MountFromCounter.Increment(); - await OnMount.Invoke(descriptor,contentReference, getContents, cancellationToken); - } - } - - // Utility class for thread-safe counter increments - public class AtomicCounter - { - private long _value; - - public long Value => Interlocked.Read(ref _value); - - public void Increment() - { - Interlocked.Increment(ref _value); - } - } } diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 737ee0e..84df80a 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -330,310 +330,6 @@ public async Task Repository_PushAsync() await repo.PushAsync(indexDesc, new MemoryStream(index), cancellationToken); Assert.Equal(index, gotIndex); } - - /// - /// Repository_MountAsync tests the MountAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_MountAsync() - { - var blob = @"hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var gotMount = 0; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var resp = new HttpResponseMessage(); - resp.RequestMessage = req; - if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/") - { - var queries = HttpUtility.ParseQueryString(req.RequestUri.Query); - if (queries["mount"] != blobDesc.Digest) - { - resp.StatusCode = HttpStatusCode.InternalServerError; - return resp; - } - if (queries["from"] != "test") - { - resp.StatusCode = HttpStatusCode.InternalServerError; - return resp; - } - gotMount++; - resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - resp.StatusCode = HttpStatusCode.Created; - return resp; - } - - resp.StatusCode = HttpStatusCode.InternalServerError; - return resp; - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test2"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - - await repo.MountAsync(blobDesc, "test", null, cancellationToken); - Assert.Equal(1, gotMount); - } - - /// - /// Repository_MountAsync_Fallback tests the MountAsync method of the Repository when the server doesn't support mount query parameters. - /// - /// - [Fact] - public async Task Repository_MountAsync_Fallback() - { - var blob = @"hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - string sequence = ""; - byte[] gotBlob = Array.Empty(); - var uuid = "4fd53bc9-565d-4527-ab80-3e051ac4880c"; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var resp = new HttpResponseMessage(); - resp.RequestMessage = req; - if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/") - { - resp.Headers.Location = new Uri("/v2/test2/blobs/uploads/" + uuid, UriKind.Relative); - resp.StatusCode = HttpStatusCode.Accepted; - sequence += "post "; - return resp; - } - if (req.Method == HttpMethod.Get && req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) - { - resp.Content.Headers.Add("Content-Type", "application/octet-stream"); - resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - resp.Content = new ByteArrayContent(blob); - resp.StatusCode = HttpStatusCode.OK; - sequence += "get "; - return resp; - } - if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/" + uuid) - { - if (req.Content?.Headers.GetValues("Content-Type").FirstOrDefault() != "application/octet-stream") - { - resp.StatusCode = HttpStatusCode.BadRequest; - return resp; - } - if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) - { - resp.StatusCode = HttpStatusCode.BadRequest; - return resp; - } - gotBlob = req.Content!.ReadAsByteArrayAsync(cancellationToken).Result; - resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - resp.StatusCode = HttpStatusCode.Created; - sequence += "put "; - return resp; - } - - resp.StatusCode = HttpStatusCode.Forbidden; - return resp; - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test2"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - - // getContent is null - sequence = ""; - await repo.MountAsync(blobDesc, "localhost:5000/test", null, cancellationToken); - Assert.Equal(blob, gotBlob); - Assert.Equal("post get put ", sequence); - - // getContent is non-null - sequence = ""; - await repo.MountAsync(blobDesc, "localhost:5000/test", _ => Task.FromResult(new MemoryStream(blob)), cancellationToken); - Assert.Equal(blob, gotBlob); - Assert.Equal("post put ", sequence); - } - - /// - /// Repository_MountAsync_Error tests the error handling of the MountAsync method of the Repository. - /// - /// - [Fact] - public async Task Repository_MountAsync_Error() - { - var blob = @"hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var resp = new HttpResponseMessage(); - resp.RequestMessage = req; - if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/") - { - resp.StatusCode = HttpStatusCode.BadRequest; - resp.Content = new StringContent(@"{ ""errors"": [ { ""code"": ""NAME_UNKNOWN"", ""message"": ""some error"" } ] }"); - return resp; - } - - resp.StatusCode = HttpStatusCode.InternalServerError; - return resp; - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - - var ex = await Assert.ThrowsAsync(async () => - { - await repo.MountAsync(blobDesc, "foo", null, cancellationToken); - }); - - Assert.NotNull(ex.Errors); - Assert.Single(ex.Errors); - Assert.Equal("NAME_UNKNOWN", ex.Errors[0].Code); - Assert.Equal("some error", ex.Errors[0].Message); - } - - /// - /// Repository_MountAsync_Fallback_GetContent tests the case where the server doesn't recognize mount query parameters, - /// falling back to the regular push flow, using the getContent function parameter. - /// - /// - [Fact] - public async Task Repository_MountAsync_Fallback_GetContent() - { - var blob = @"hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - string sequence = ""; - byte[] gotBlob = Array.Empty(); - var uuid = "4fd53bc9-565d-4527-ab80-3e051ac4880c"; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var resp = new HttpResponseMessage(); - resp.RequestMessage = req; - - if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/") - { - resp.Headers.Location = new Uri("/v2/test2/blobs/uploads/" + uuid, UriKind.Relative); - resp.StatusCode = HttpStatusCode.Accepted; - sequence += "post "; - return resp; - } - if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/" + uuid) - { - if (req.Content?.Headers.GetValues("Content-Type").FirstOrDefault() != "application/octet-stream") - { - resp.StatusCode = HttpStatusCode.BadRequest; - return resp; - } - if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) - { - resp.StatusCode = HttpStatusCode.BadRequest; - return resp; - } - gotBlob = req.Content!.ReadAsByteArrayAsync(cancellationToken).Result; - resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - resp.StatusCode = HttpStatusCode.Created; - sequence += "put "; - return resp; - } - - resp.StatusCode = HttpStatusCode.Forbidden; - return resp; - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test2"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - - await repo.MountAsync(blobDesc, "test", _ => Task.FromResult(new MemoryStream(blob)), cancellationToken); - - // Assert that the blob was pushed correctly - Assert.Equal(blob, gotBlob); - // Assert that the request sequence matches the expected behavior - Assert.Equal("post put ", sequence); - } - - /// - /// Repository_MountAsync_Fallback_GetContentError tests the case where the server doesn't recognize mount query parameters, - /// falling back to the regular push flow, but the caller wants to avoid the pull/push pattern, so an error is returned from getContent. - /// - /// - [Fact] - public async Task Repository_MountAsync_Fallback_GetContentError() - { - var blob = @"hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - string sequence = ""; - var uuid = "4fd53bc9-565d-4527-ab80-3e051ac4880c"; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var resp = new HttpResponseMessage(); - resp.RequestMessage = req; - - if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test2/blobs/uploads/") - { - resp.Headers.Location = new Uri("/v2/test2/blobs/uploads/" + uuid, UriKind.Relative); - resp.StatusCode = HttpStatusCode.Accepted; - sequence += "post "; - return resp; - } - - resp.StatusCode = HttpStatusCode.Forbidden; - return resp; - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test2"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - - var testErr = new Exception("test error"); - var ex = await Assert.ThrowsAsync(async () => - { - await repo.MountAsync(blobDesc, "test", _ => throw testErr, cancellationToken); - }); - - Assert.Equal(testErr, ex); - Assert.Equal("post ", sequence); - } /// /// Repository_ExistsAsync tests the ExistsAsync method of the Repository @@ -2547,17 +2243,14 @@ public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigest if (method == HttpMethod.Get) { resp.Content = new ByteArrayContent(_theAmazingBanClan); - resp.Content.Headers.Add("Content-Type", - new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } else { - resp.Content.Headers.Add("Content-Type", - new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } - resp.RequestMessage = new HttpRequestMessage() { Method = method @@ -2580,7 +2273,6 @@ public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigest } } - if (errExpected && !err) { throw new Exception($"[Manifest.{method}] {testName}; expected error for request, but got none"); From 571820c0f844645d6b0cb26523a16da4aaa3e7cd Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Mon, 11 Nov 2024 07:38:50 -0300 Subject: [PATCH 06/19] refactor: renames class and events as per review Signed-off-by: Leonardo Chaia --- .../{CopyOptions.cs => CopyGraphOptions.cs} | 22 ++++++++++--------- src/OrasProject.Oras/Extensions.cs | 14 +++++------- 2 files changed, 18 insertions(+), 18 deletions(-) rename src/OrasProject.Oras/{CopyOptions.cs => CopyGraphOptions.cs} (60%) diff --git a/src/OrasProject.Oras/CopyOptions.cs b/src/OrasProject.Oras/CopyGraphOptions.cs similarity index 60% rename from src/OrasProject.Oras/CopyOptions.cs rename to src/OrasProject.Oras/CopyGraphOptions.cs index 7d77ea2..5cb334f 100644 --- a/src/OrasProject.Oras/CopyOptions.cs +++ b/src/OrasProject.Oras/CopyGraphOptions.cs @@ -16,24 +16,26 @@ namespace OrasProject.Oras; -public struct CopyOptions +public struct CopyGraphOptions { - public event Action OnPreCopy; - public event Action OnPostCopy; - public event Action OnCopySkipped; + public event Action PreCopy; - internal void PreCopy(Descriptor descriptor) + public event Action PostCopy; + + public event Action CopySkipped; + + internal void OnPreCopy(Descriptor descriptor) { - OnPreCopy?.Invoke(descriptor); + PreCopy?.Invoke(descriptor); } - internal void PostCopy(Descriptor descriptor) + internal void OnPostCopy(Descriptor descriptor) { - OnPostCopy?.Invoke(descriptor); + PostCopy?.Invoke(descriptor); } - internal void CopySkipped(Descriptor descriptor) + internal void OnCopySkipped(Descriptor descriptor) { - OnCopySkipped?.Invoke(descriptor); + CopySkipped?.Invoke(descriptor); } } diff --git a/src/OrasProject.Oras/Extensions.cs b/src/OrasProject.Oras/Extensions.cs index 8c566c4..c5dfe29 100644 --- a/src/OrasProject.Oras/Extensions.cs +++ b/src/OrasProject.Oras/Extensions.cs @@ -12,18 +12,15 @@ // limitations under the License. using System; -using System.IO; using System.Threading; using System.Threading.Tasks; using OrasProject.Oras.Oci; -using OrasProject.Oras.Registry; using static OrasProject.Oras.Content.Extensions; namespace OrasProject.Oras; public static class Extensions { - /// /// Copy copies a rooted directed acyclic graph (DAG) with the tagged root node /// in the source Target to the destination Target. @@ -36,9 +33,10 @@ public static class Extensions /// /// /// + /// /// /// - public static async Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CancellationToken cancellationToken = default, CopyOptions? copyOptions = default) + public static async Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CancellationToken cancellationToken = default, CopyGraphOptions? copyOptions = default) { if (string.IsNullOrEmpty(dstRef)) { @@ -50,12 +48,12 @@ public static async Task CopyAsync(this ITarget src, string srcRef, return root; } - public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CancellationToken cancellationToken, CopyOptions? copyOptions = default) + public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CancellationToken cancellationToken, CopyGraphOptions? copyOptions = default) { // check if node exists in target if (await dst.ExistsAsync(node, cancellationToken).ConfigureAwait(false)) { - copyOptions?.CopySkipped(node); + copyOptions?.OnCopySkipped(node); return; } @@ -69,12 +67,12 @@ public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descripto } // perform the copy - copyOptions?.PreCopy(node); + copyOptions?.OnPreCopy(node); var dataStream = await src.FetchAsync(node, cancellationToken).ConfigureAwait(false); await dst.PushAsync(node, dataStream, cancellationToken).ConfigureAwait(false); // we copied it - copyOptions?.PostCopy(node); + copyOptions?.OnPostCopy(node); } } From 138120e6a4fd33612ebba01db4e414007de1eff7 Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Mon, 11 Nov 2024 08:30:26 -0300 Subject: [PATCH 07/19] chore: adds copy tests Signed-off-by: Leonardo Chaia --- tests/OrasProject.Oras.Tests/CopyTest.cs | 143 ++++++++++++++++++++++- 1 file changed, 142 insertions(+), 1 deletion(-) diff --git a/tests/OrasProject.Oras.Tests/CopyTest.cs b/tests/OrasProject.Oras.Tests/CopyTest.cs index 4523bed..b0de347 100644 --- a/tests/OrasProject.Oras.Tests/CopyTest.cs +++ b/tests/OrasProject.Oras.Tests/CopyTest.cs @@ -15,7 +15,6 @@ using OrasProject.Oras.Oci; using System.Text; using System.Text.Json; -using OrasProject.Oras.Registry; using Xunit; using Index = OrasProject.Oras.Oci.Index; @@ -144,4 +143,146 @@ public async Task CanCopyBetweenMemoryTargets() } } + + [Fact] + public async Task TestCopyGraph_FullCopy() + { + var src = new MemoryStore(); + var dst = new MemoryStore(); + + // Generate test content + var blobs = new List(); + var descs = new List(); + + void AppendBlob(string mediaType, byte[] blob) + { + blobs.Add(blob); + var desc = new Descriptor + { + MediaType = mediaType, + Digest = Digest.ComputeSHA256(blob), + Size = blob.Length + }; + descs.Add(desc); + } + + void GenerateManifest(Descriptor config, params Descriptor[] layers) + { + var manifest = new Manifest + { + Config = config, + Layers = layers.ToList() + }; + var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); + AppendBlob(MediaType.ImageManifest, manifestBytes); + } + + void GenerateIndex(params Descriptor[] manifests) + { + var index = new Index + { + Manifests = manifests.ToList() + }; + var indexBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(index)); + AppendBlob(MediaType.ImageIndex, indexBytes); + } + + // Append blobs and generate manifests and indices + var getBytes = (string data) => Encoding.UTF8.GetBytes(data); + AppendBlob(MediaType.ImageConfig, getBytes("config")); // Blob 0 + AppendBlob(MediaType.ImageLayer, getBytes("foo")); // Blob 1 + AppendBlob(MediaType.ImageLayer, getBytes("bar")); // Blob 2 + AppendBlob(MediaType.ImageLayer, getBytes("hello")); // Blob 3 + GenerateManifest(descs[0], descs[1], descs[2]); // Blob 4 + GenerateManifest(descs[0], descs[3]); // Blob 5 + GenerateManifest(descs[0], descs[1], descs[2], descs[3]); // Blob 6 + GenerateIndex(descs[4], descs[5]); // Blob 7 + GenerateIndex(descs[6]); // Blob 8 + GenerateIndex(); // Blob 9 + GenerateIndex(descs[7], descs[8], descs[9]); // Blob 10 + + var root = descs[^1]; // The last descriptor as the root + + // Push blobs to the source memory store + for (int i = 0; i < blobs.Count; i++) + { + await src.PushAsync(descs[i], new MemoryStream(blobs[i]), CancellationToken.None); + } + + // Set up tracking storage wrappers for verification + var srcTracker = new StorageTracker(src); + var dstTracker = new StorageTracker(dst); + + // Perform the copy graph operation + var copyOptions = new CopyGraphOptions(); + await srcTracker.CopyGraphAsync(dstTracker, root, CancellationToken.None, copyOptions); + + // Verify contents in the destination + foreach (var (desc, blob) in descs.Zip(blobs, Tuple.Create)) + { + Assert.True(await dst.ExistsAsync(desc, CancellationToken.None), $"Blob {desc.Digest} should exist in destination."); + var fetchedContent = await dst.FetchAsync(desc, CancellationToken.None); + using var memoryStream = new MemoryStream(); + await fetchedContent.CopyToAsync(memoryStream); + Assert.Equal(blob, memoryStream.ToArray()); + } + + // Verify API counts + // REMARKS: FetchCount should equal to blobs.Count + // but since there's no caching implemented, it is not + Assert.Equal(18, srcTracker.FetchCount); + Assert.Equal(0, srcTracker.PushCount); + Assert.Equal(0, srcTracker.ExistsCount); + Assert.Equal(0, dstTracker.FetchCount); + Assert.Equal(blobs.Count, dstTracker.PushCount); + + // REMARKS: ExistsCount should equal to blobs.Count + // but since there's no caching implemented, it is not + Assert.Equal(16, dstTracker.ExistsCount); + } + + private class StorageTracker : ITarget + { + private readonly ITarget _storage; + + public int FetchCount { get; private set; } + public int PushCount { get; private set; } + public int ExistsCount { get; private set; } + + public IList Fetched { get; } = []; + + public StorageTracker(ITarget storage) + { + _storage = storage; + } + + public async Task ExistsAsync(Descriptor desc, CancellationToken cancellationToken) + { + ExistsCount++; + return await _storage.ExistsAsync(desc, cancellationToken); + } + + public async Task FetchAsync(Descriptor desc, CancellationToken cancellationToken) + { + FetchCount++; + Fetched.Add(desc.Digest); + return await _storage.FetchAsync(desc, cancellationToken); + } + + public async Task PushAsync(Descriptor desc, Stream content, CancellationToken cancellationToken) + { + PushCount++; + await _storage.PushAsync(desc, content, cancellationToken); + } + + public async Task TagAsync(Descriptor desc, string reference, CancellationToken cancellationToken) + { + await _storage.TagAsync(desc, reference, cancellationToken); + } + + public Task ResolveAsync(string reference, CancellationToken cancellationToken) + { + return _storage.ResolveAsync(reference, cancellationToken); + } + } } From 77e8079423a5804947f494a84bd0c4dd8dfd87e8 Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Mon, 11 Nov 2024 09:13:41 -0300 Subject: [PATCH 08/19] chore: adds overload to prevent breaking change Signed-off-by: Leonardo Chaia --- src/OrasProject.Oras/Extensions.cs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/OrasProject.Oras/Extensions.cs b/src/OrasProject.Oras/Extensions.cs index c5dfe29..e75e068 100644 --- a/src/OrasProject.Oras/Extensions.cs +++ b/src/OrasProject.Oras/Extensions.cs @@ -21,6 +21,26 @@ namespace OrasProject.Oras; public static class Extensions { + /// + /// Copy copies a rooted directed acyclic graph (DAG) with the tagged root node + /// in the source Target to the destination Target. + /// The destination reference will be the same as the source reference if the + /// destination reference is left blank. + /// Returns the descriptor of the root node on successful copy. + /// + /// + /// + /// + /// + /// + /// + /// + public static Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, + CancellationToken cancellationToken = default) + { + return src.CopyAsync(srcRef, dst, dstRef, copyOptions: new CopyGraphOptions(), cancellationToken); + } + /// /// Copy copies a rooted directed acyclic graph (DAG) with the tagged root node /// in the source Target to the destination Target. @@ -36,7 +56,7 @@ public static class Extensions /// /// /// - public static async Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CancellationToken cancellationToken = default, CopyGraphOptions? copyOptions = default) + public static async Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CopyGraphOptions? copyOptions = default, CancellationToken cancellationToken = default) { if (string.IsNullOrEmpty(dstRef)) { From 06fd44265bc2f9489e7bfbc319ad729427d1390f Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Mon, 11 Nov 2024 09:16:21 -0300 Subject: [PATCH 09/19] chore: introduces overloads to keep previous signature on CopyGraphOptions Signed-off-by: Leonardo Chaia --- src/OrasProject.Oras/Extensions.cs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/OrasProject.Oras/Extensions.cs b/src/OrasProject.Oras/Extensions.cs index e75e068..4637868 100644 --- a/src/OrasProject.Oras/Extensions.cs +++ b/src/OrasProject.Oras/Extensions.cs @@ -38,7 +38,7 @@ public static class Extensions public static Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CancellationToken cancellationToken = default) { - return src.CopyAsync(srcRef, dst, dstRef, copyOptions: new CopyGraphOptions(), cancellationToken); + return src.CopyAsync(srcRef, dst, dstRef, new CopyGraphOptions(), cancellationToken); } /// @@ -52,8 +52,8 @@ public static Task CopyAsync(this ITarget src, string srcRef, ITarge /// /// /// - /// /// + /// /// /// public static async Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CopyGraphOptions? copyOptions = default, CancellationToken cancellationToken = default) @@ -63,12 +63,17 @@ public static async Task CopyAsync(this ITarget src, string srcRef, dstRef = srcRef; } var root = await src.ResolveAsync(srcRef, cancellationToken).ConfigureAwait(false); - await src.CopyGraphAsync(dst, root, cancellationToken, copyOptions).ConfigureAwait(false); + await src.CopyGraphAsync(dst, root, copyOptions, cancellationToken).ConfigureAwait(false); await dst.TagAsync(root, dstRef, cancellationToken).ConfigureAwait(false); return root; } - public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CancellationToken cancellationToken, CopyGraphOptions? copyOptions = default) + public static Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CancellationToken cancellationToken = default) + { + return src.CopyGraphAsync(dst, node, new CopyGraphOptions(), cancellationToken); + } + + public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CopyGraphOptions? copyOptions = default, CancellationToken cancellationToken = default) { // check if node exists in target if (await dst.ExistsAsync(node, cancellationToken).ConfigureAwait(false)) @@ -83,7 +88,7 @@ public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descripto // check if the node has successors foreach (var childNode in successors) { - await src.CopyGraphAsync(dst, childNode, cancellationToken, copyOptions).ConfigureAwait(false); + await src.CopyGraphAsync(dst, childNode, copyOptions, cancellationToken).ConfigureAwait(false); } // perform the copy From 5787c0437141c1f9bc5d33dbc8bc72998156c476 Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Tue, 12 Nov 2024 09:16:46 -0300 Subject: [PATCH 10/19] fix: copy signature Signed-off-by: Leonardo Chaia --- tests/OrasProject.Oras.Tests/CopyTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/OrasProject.Oras.Tests/CopyTest.cs b/tests/OrasProject.Oras.Tests/CopyTest.cs index b0de347..3ae2ab1 100644 --- a/tests/OrasProject.Oras.Tests/CopyTest.cs +++ b/tests/OrasProject.Oras.Tests/CopyTest.cs @@ -215,7 +215,7 @@ void GenerateIndex(params Descriptor[] manifests) // Perform the copy graph operation var copyOptions = new CopyGraphOptions(); - await srcTracker.CopyGraphAsync(dstTracker, root, CancellationToken.None, copyOptions); + await srcTracker.CopyGraphAsync(dstTracker, root, copyOptions, CancellationToken.None); // Verify contents in the destination foreach (var (desc, blob) in descs.Zip(blobs, Tuple.Create)) From 77ba179406f836ee2317052e0a88eefae7d191b4 Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Wed, 13 Nov 2024 12:52:40 -0300 Subject: [PATCH 11/19] refactor: introduces CopyOptions Signed-off-by: Leonardo Chaia --- src/OrasProject.Oras/CopyOptions.cs | 6 ++++++ src/OrasProject.Oras/Extensions.cs | 16 ++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 src/OrasProject.Oras/CopyOptions.cs diff --git a/src/OrasProject.Oras/CopyOptions.cs b/src/OrasProject.Oras/CopyOptions.cs new file mode 100644 index 0000000..6a1c1f1 --- /dev/null +++ b/src/OrasProject.Oras/CopyOptions.cs @@ -0,0 +1,6 @@ +namespace OrasProject.Oras; + +public struct CopyOptions +{ + public CopyGraphOptions CopyGraphOptions; +} diff --git a/src/OrasProject.Oras/Extensions.cs b/src/OrasProject.Oras/Extensions.cs index 4637868..e01a519 100644 --- a/src/OrasProject.Oras/Extensions.cs +++ b/src/OrasProject.Oras/Extensions.cs @@ -38,7 +38,7 @@ public static class Extensions public static Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CancellationToken cancellationToken = default) { - return src.CopyAsync(srcRef, dst, dstRef, new CopyGraphOptions(), cancellationToken); + return src.CopyAsync(srcRef, dst, dstRef, new CopyOptions(), cancellationToken); } /// @@ -56,14 +56,14 @@ public static Task CopyAsync(this ITarget src, string srcRef, ITarge /// /// /// - public static async Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CopyGraphOptions? copyOptions = default, CancellationToken cancellationToken = default) + public static async Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CopyOptions? copyOptions = default, CancellationToken cancellationToken = default) { if (string.IsNullOrEmpty(dstRef)) { dstRef = srcRef; } var root = await src.ResolveAsync(srcRef, cancellationToken).ConfigureAwait(false); - await src.CopyGraphAsync(dst, root, copyOptions, cancellationToken).ConfigureAwait(false); + await src.CopyGraphAsync(dst, root, copyOptions?.CopyGraphOptions, cancellationToken).ConfigureAwait(false); await dst.TagAsync(root, dstRef, cancellationToken).ConfigureAwait(false); return root; } @@ -73,12 +73,12 @@ public static Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node return src.CopyGraphAsync(dst, node, new CopyGraphOptions(), cancellationToken); } - public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CopyGraphOptions? copyOptions = default, CancellationToken cancellationToken = default) + public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CopyGraphOptions? copyGraphOptions = default, CancellationToken cancellationToken = default) { // check if node exists in target if (await dst.ExistsAsync(node, cancellationToken).ConfigureAwait(false)) { - copyOptions?.OnCopySkipped(node); + copyGraphOptions?.OnCopySkipped(node); return; } @@ -88,16 +88,16 @@ public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descripto // check if the node has successors foreach (var childNode in successors) { - await src.CopyGraphAsync(dst, childNode, copyOptions, cancellationToken).ConfigureAwait(false); + await src.CopyGraphAsync(dst, childNode, copyGraphOptions, cancellationToken).ConfigureAwait(false); } // perform the copy - copyOptions?.OnPreCopy(node); + copyGraphOptions?.OnPreCopy(node); var dataStream = await src.FetchAsync(node, cancellationToken).ConfigureAwait(false); await dst.PushAsync(node, dataStream, cancellationToken).ConfigureAwait(false); // we copied it - copyOptions?.OnPostCopy(node); + copyGraphOptions?.OnPostCopy(node); } } From 6ef7e74befe07eafb3262a8ba257f942fbf9cd99 Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Wed, 13 Nov 2024 12:55:01 -0300 Subject: [PATCH 12/19] chore: adds license Signed-off-by: Leonardo Chaia --- src/OrasProject.Oras/CopyOptions.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/OrasProject.Oras/CopyOptions.cs b/src/OrasProject.Oras/CopyOptions.cs index 6a1c1f1..84a9a38 100644 --- a/src/OrasProject.Oras/CopyOptions.cs +++ b/src/OrasProject.Oras/CopyOptions.cs @@ -1,3 +1,16 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + namespace OrasProject.Oras; public struct CopyOptions From a12ff787cb63735bcb2b1eb0d15ffce3f3ca5c50 Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Wed, 13 Nov 2024 13:36:07 -0300 Subject: [PATCH 13/19] chore: adds more tests from oras-go Signed-off-by: Leonardo Chaia --- tests/OrasProject.Oras.Tests/CopyTest.cs | 89 ++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/tests/OrasProject.Oras.Tests/CopyTest.cs b/tests/OrasProject.Oras.Tests/CopyTest.cs index 3ae2ab1..b86bf82 100644 --- a/tests/OrasProject.Oras.Tests/CopyTest.cs +++ b/tests/OrasProject.Oras.Tests/CopyTest.cs @@ -144,6 +144,95 @@ public async Task CanCopyBetweenMemoryTargets() } } + [Fact] + public async Task TestCopyExistedRoot() + { + var src = new MemoryStore(); + var dst = new MemoryStore(); + + // Generate test content + var blobs = new List(); + var descs = new List(); + + void AppendBlob(string mediaType, byte[] blob) + { + blobs.Add(blob); + var desc = new Descriptor + { + MediaType = mediaType, + Digest = Digest.ComputeSHA256(blob), + Size = blob.Length + }; + descs.Add(desc); + } + + void GenerateManifest(Descriptor config, params Descriptor[] layers) + { + var manifest = new Manifest + { + Config = config, + Layers = layers.ToList() + }; + var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); + AppendBlob(MediaType.ImageManifest, manifestBytes); + } + + // Add blobs and generate manifest + AppendBlob(MediaType.ImageConfig, Encoding.UTF8.GetBytes("config")); // Blob 0 + AppendBlob(MediaType.ImageLayer, Encoding.UTF8.GetBytes("foo")); // Blob 1 + AppendBlob(MediaType.ImageLayer, Encoding.UTF8.GetBytes("bar")); // Blob 2 + GenerateManifest(descs[0], descs[1], descs[2]); // Blob 3 + + // Push blobs to the source store + foreach (var (blob, desc) in blobs.Zip(descs)) + { + await src.PushAsync(desc, new MemoryStream(blob), CancellationToken.None); + } + + var root = descs[3]; + var refTag = "foobar"; + var newTag = "newtag"; + + // Tag root node in source + await src.TagAsync(root, refTag, CancellationToken.None); + + // Prepare copy options with OnCopySkipped + var skippedCount = 0; + var copyOptions = new CopyOptions + { + CopyGraphOptions = new CopyGraphOptions() + { + } + }; + copyOptions.CopyGraphOptions.CopySkipped += d => skippedCount++; + + // Copy with the source tag + var gotDesc = await src.CopyAsync(refTag, dst, "", copyOptions, CancellationToken.None); + Assert.Equal(root, gotDesc); + + // Copy with a new tag + gotDesc = await src.CopyAsync(refTag, dst, newTag, copyOptions, CancellationToken.None); + Assert.Equal(root, gotDesc); + + // Verify contents in the destination + foreach (var desc in descs) + { + var exists = await dst.ExistsAsync(desc, CancellationToken.None); + Assert.True(exists, $"Destination should contain descriptor {desc.Digest}"); + } + + // Verify the source tag in destination + gotDesc = await dst.ResolveAsync(refTag, CancellationToken.None); + Assert.Equal(root, gotDesc); + + // Verify the new tag in destination + gotDesc = await dst.ResolveAsync(newTag, CancellationToken.None); + Assert.Equal(root, gotDesc); + + // Verify the OnCopySkipped invocation count + Assert.Equal(1, skippedCount); + } + [Fact] public async Task TestCopyGraph_FullCopy() { From a7a1baf3470b9fac3e1f4817d99e6add575c36c8 Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Wed, 13 Nov 2024 13:46:20 -0300 Subject: [PATCH 14/19] chore: adds tests from oras-go Signed-off-by: Leonardo Chaia --- tests/OrasProject.Oras.Tests/CopyTest.cs | 108 +++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tests/OrasProject.Oras.Tests/CopyTest.cs b/tests/OrasProject.Oras.Tests/CopyTest.cs index b86bf82..d6d4d2d 100644 --- a/tests/OrasProject.Oras.Tests/CopyTest.cs +++ b/tests/OrasProject.Oras.Tests/CopyTest.cs @@ -330,6 +330,114 @@ void GenerateIndex(params Descriptor[] manifests) Assert.Equal(16, dstTracker.ExistsCount); } + [Fact] + public async Task TestCopyWithOptions() + { + var src = new MemoryStore(); + + // Generate test content + var blobs = new List(); + var descs = new List(); + + void AppendBlob(string mediaType, byte[] blob) + { + blobs.Add(blob); + descs.Add(new Descriptor + { + MediaType = mediaType, + Digest = Digest.ComputeSHA256(blob), + Size = blob.Length + }); + } + + void AppendManifest(string arc, string os, string mediaType, byte[] blob) + { + blobs.Add(blob); + descs.Add(new Descriptor + { + MediaType = mediaType, + Digest = Digest.ComputeSHA256(blob), + Size = blob.Length, + Platform = new Platform + { + Architecture = arc, + OS = os + } + }); + } + + void GenerateManifest(string arc, string os, Descriptor config, params Descriptor[] layers) + { + var manifest = new Manifest + { + Config = config, + Layers = layers.ToList() + }; + var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); + AppendManifest(arc, os, MediaType.ImageManifest, manifestBytes); + } + + void GenerateIndex(params Descriptor[] manifests) + { + var index = new Index + { + Manifests = manifests.ToList() + }; + var indexBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(index)); + AppendBlob(MediaType.ImageIndex, indexBytes); + } + + // Create blobs and manifests + AppendBlob(MediaType.ImageConfig, Encoding.UTF8.GetBytes("config")); // Blob 0 + AppendBlob(MediaType.ImageLayer, Encoding.UTF8.GetBytes("foo")); // Blob 1 + AppendBlob(MediaType.ImageLayer, Encoding.UTF8.GetBytes("bar")); // Blob 2 + GenerateManifest("test-arc-1", "test-os-1", descs[0], descs[1], descs[2]); // Blob 3 + AppendBlob(MediaType.ImageLayer, Encoding.UTF8.GetBytes("hello")); // Blob 4 + GenerateManifest("test-arc-2", "test-os-2", descs[0], descs[4]); // Blob 5 + GenerateIndex(descs[3], descs[5]); // Blob 6 + + var cancellationToken = CancellationToken.None; + + // Push blobs to source store + for (int i = 0; i < blobs.Count; i++) + { + await src.PushAsync(descs[i], new MemoryStream(blobs[i]), cancellationToken); + } + + var root = descs[6]; + var refTag = "foobar"; + await src.TagAsync(root, refTag, cancellationToken); + + // Test copy with platform filter and hooks + var dst = new MemoryStore(); + var preCopyCount = 0; + var postCopyCount = 0; + var copyOptions = new CopyOptions + { + CopyGraphOptions = new CopyGraphOptions + { + } + }; + + copyOptions.CopyGraphOptions.PreCopy += d => preCopyCount++; + copyOptions.CopyGraphOptions.PostCopy += d => postCopyCount++; + + var expectedDesc = descs[6]; + var gotDesc = await src.CopyAsync(refTag, dst, "", copyOptions, cancellationToken); + Assert.Equal(expectedDesc, gotDesc); + + // Verify platform-specific contents + var expectedDescs = new[] { descs[0], descs[4], descs[5] }; + foreach (var desc in expectedDescs) + { + Assert.True(await dst.ExistsAsync(desc, cancellationToken)); + } + + // Verify API counts + Assert.Equal(7, preCopyCount); + Assert.Equal(7, postCopyCount); + } + private class StorageTracker : ITarget { private readonly ITarget _storage; From 493a8d92fe15df540c6a9f10958fa770fca5b12f Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Mon, 25 Nov 2024 07:59:02 -0300 Subject: [PATCH 15/19] doc: adds comments to new option structs Signed-off-by: Leonardo Chaia --- src/OrasProject.Oras/CopyGraphOptions.cs | 13 +++++++++++++ src/OrasProject.Oras/CopyOptions.cs | 3 +++ 2 files changed, 16 insertions(+) diff --git a/src/OrasProject.Oras/CopyGraphOptions.cs b/src/OrasProject.Oras/CopyGraphOptions.cs index 5cb334f..2804a54 100644 --- a/src/OrasProject.Oras/CopyGraphOptions.cs +++ b/src/OrasProject.Oras/CopyGraphOptions.cs @@ -16,12 +16,25 @@ namespace OrasProject.Oras; +/// +/// CopyGraphOptions contains parameters for +/// public struct CopyGraphOptions { + /// + /// PreCopy handles the current descriptor before it is copied. + /// public event Action PreCopy; + /// + /// PostCopy handles the current descriptor after it is copied. + /// public event Action PostCopy; + /// + /// OnCopySkipped will be called when the sub-DAG rooted by the current node + /// is skipped. + /// public event Action CopySkipped; internal void OnPreCopy(Descriptor descriptor) diff --git a/src/OrasProject.Oras/CopyOptions.cs b/src/OrasProject.Oras/CopyOptions.cs index 84a9a38..a7b7b2c 100644 --- a/src/OrasProject.Oras/CopyOptions.cs +++ b/src/OrasProject.Oras/CopyOptions.cs @@ -13,6 +13,9 @@ namespace OrasProject.Oras; +/// +/// CopyOptions contains parameters for +/// public struct CopyOptions { public CopyGraphOptions CopyGraphOptions; From 9e406553fbcb83304d491bedd4b8a4f8b3178d22 Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Mon, 25 Nov 2024 08:05:48 -0300 Subject: [PATCH 16/19] refactor: makes copy events async Signed-off-by: Leonardo Chaia --- src/OrasProject.Oras/CopyGraphOptions.cs | 21 +++++++++++---------- src/OrasProject.Oras/Extensions.cs | 12 ++++++------ tests/OrasProject.Oras.Tests/CopyTest.cs | 22 +++++++++++++++++----- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/OrasProject.Oras/CopyGraphOptions.cs b/src/OrasProject.Oras/CopyGraphOptions.cs index 2804a54..ea07247 100644 --- a/src/OrasProject.Oras/CopyGraphOptions.cs +++ b/src/OrasProject.Oras/CopyGraphOptions.cs @@ -12,6 +12,7 @@ // limitations under the License. using System; +using System.Threading.Tasks; using OrasProject.Oras.Oci; namespace OrasProject.Oras; @@ -24,31 +25,31 @@ public struct CopyGraphOptions /// /// PreCopy handles the current descriptor before it is copied. /// - public event Action PreCopy; + public event Func PreCopy; /// /// PostCopy handles the current descriptor after it is copied. /// - public event Action PostCopy; + public event Func PostCopy; /// - /// OnCopySkipped will be called when the sub-DAG rooted by the current node + /// CopySkipped will be called when the sub-DAG rooted by the current node /// is skipped. /// - public event Action CopySkipped; + public event Func CopySkipped; - internal void OnPreCopy(Descriptor descriptor) + internal Task OnPreCopyAsync(Descriptor descriptor) { - PreCopy?.Invoke(descriptor); + return PreCopy?.Invoke(descriptor) ?? Task.CompletedTask; } - internal void OnPostCopy(Descriptor descriptor) + internal Task OnPostCopyAsync(Descriptor descriptor) { - PostCopy?.Invoke(descriptor); + return PostCopy?.Invoke(descriptor) ?? Task.CompletedTask; } - internal void OnCopySkipped(Descriptor descriptor) + internal Task OnCopySkippedAsync(Descriptor descriptor) { - CopySkipped?.Invoke(descriptor); + return CopySkipped?.Invoke(descriptor) ?? Task.CompletedTask; } } diff --git a/src/OrasProject.Oras/Extensions.cs b/src/OrasProject.Oras/Extensions.cs index e01a519..4dad917 100644 --- a/src/OrasProject.Oras/Extensions.cs +++ b/src/OrasProject.Oras/Extensions.cs @@ -56,14 +56,14 @@ public static Task CopyAsync(this ITarget src, string srcRef, ITarge /// /// /// - public static async Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CopyOptions? copyOptions = default, CancellationToken cancellationToken = default) + public static async Task CopyAsync(this ITarget src, string srcRef, ITarget dst, string dstRef, CopyOptions copyOptions = default, CancellationToken cancellationToken = default) { if (string.IsNullOrEmpty(dstRef)) { dstRef = srcRef; } var root = await src.ResolveAsync(srcRef, cancellationToken).ConfigureAwait(false); - await src.CopyGraphAsync(dst, root, copyOptions?.CopyGraphOptions, cancellationToken).ConfigureAwait(false); + await src.CopyGraphAsync(dst, root, copyOptions.CopyGraphOptions, cancellationToken).ConfigureAwait(false); await dst.TagAsync(root, dstRef, cancellationToken).ConfigureAwait(false); return root; } @@ -73,12 +73,12 @@ public static Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node return src.CopyGraphAsync(dst, node, new CopyGraphOptions(), cancellationToken); } - public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CopyGraphOptions? copyGraphOptions = default, CancellationToken cancellationToken = default) + public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descriptor node, CopyGraphOptions copyGraphOptions = default, CancellationToken cancellationToken = default) { // check if node exists in target if (await dst.ExistsAsync(node, cancellationToken).ConfigureAwait(false)) { - copyGraphOptions?.OnCopySkipped(node); + await copyGraphOptions.OnCopySkippedAsync(node); return; } @@ -92,12 +92,12 @@ public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descripto } // perform the copy - copyGraphOptions?.OnPreCopy(node); + await copyGraphOptions.OnPreCopyAsync(node); var dataStream = await src.FetchAsync(node, cancellationToken).ConfigureAwait(false); await dst.PushAsync(node, dataStream, cancellationToken).ConfigureAwait(false); // we copied it - copyGraphOptions?.OnPostCopy(node); + await copyGraphOptions.OnPostCopyAsync(node); } } diff --git a/tests/OrasProject.Oras.Tests/CopyTest.cs b/tests/OrasProject.Oras.Tests/CopyTest.cs index d6d4d2d..5b2d9a2 100644 --- a/tests/OrasProject.Oras.Tests/CopyTest.cs +++ b/tests/OrasProject.Oras.Tests/CopyTest.cs @@ -196,7 +196,7 @@ void GenerateManifest(Descriptor config, params Descriptor[] layers) // Tag root node in source await src.TagAsync(root, refTag, CancellationToken.None); - // Prepare copy options with OnCopySkipped + // Prepare copy options with OnCopySkippedAsync var skippedCount = 0; var copyOptions = new CopyOptions { @@ -204,7 +204,11 @@ void GenerateManifest(Descriptor config, params Descriptor[] layers) { } }; - copyOptions.CopyGraphOptions.CopySkipped += d => skippedCount++; + copyOptions.CopyGraphOptions.CopySkipped += d => + { + skippedCount++; + return Task.CompletedTask; + }; // Copy with the source tag var gotDesc = await src.CopyAsync(refTag, dst, "", copyOptions, CancellationToken.None); @@ -229,7 +233,7 @@ void GenerateManifest(Descriptor config, params Descriptor[] layers) gotDesc = await dst.ResolveAsync(newTag, CancellationToken.None); Assert.Equal(root, gotDesc); - // Verify the OnCopySkipped invocation count + // Verify the OnCopySkippedAsync invocation count Assert.Equal(1, skippedCount); } @@ -419,8 +423,16 @@ void GenerateIndex(params Descriptor[] manifests) } }; - copyOptions.CopyGraphOptions.PreCopy += d => preCopyCount++; - copyOptions.CopyGraphOptions.PostCopy += d => postCopyCount++; + copyOptions.CopyGraphOptions.PreCopy += d => + { + preCopyCount++; + return Task.CompletedTask; + }; + copyOptions.CopyGraphOptions.PostCopy += d => + { + postCopyCount++; + return Task.CompletedTask; + }; var expectedDesc = descs[6]; var gotDesc = await src.CopyAsync(refTag, dst, "", copyOptions, cancellationToken); From 68220bd8b74ea32352f61da825c0d85779c21d87 Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Tue, 26 Nov 2024 18:35:26 -0300 Subject: [PATCH 17/19] feat: introduces `InvokeAsync` extension method to support asynchronous event handlers. Signed-off-by: Leonardo Chaia --- .../AsyncInvocationExtensions.cs | 39 +++++++++++++++++++ src/OrasProject.Oras/CopyGraphOptions.cs | 10 ++--- src/OrasProject.Oras/Extensions.cs | 7 ++-- 3 files changed, 47 insertions(+), 9 deletions(-) create mode 100644 src/OrasProject.Oras/AsyncInvocationExtensions.cs diff --git a/src/OrasProject.Oras/AsyncInvocationExtensions.cs b/src/OrasProject.Oras/AsyncInvocationExtensions.cs new file mode 100644 index 0000000..968e598 --- /dev/null +++ b/src/OrasProject.Oras/AsyncInvocationExtensions.cs @@ -0,0 +1,39 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Threading.Tasks; + +namespace OrasProject.Oras; + +internal static class AsyncInvocationExtensions +{ + /// + /// Sequentially invokes an event that returns a . + /// Each event listener is executed in sequence, and it's returning + /// task awaited before executing the next one. + /// + /// + /// + /// + internal static async Task InvokeAsync( + this Func? eventDelegate, TEventArgs args) + { + if (eventDelegate == null) return; + + foreach (var handler in eventDelegate.GetInvocationList()) + { + await ((Task?)handler.DynamicInvoke(args) ?? Task.CompletedTask); + } + } +} diff --git a/src/OrasProject.Oras/CopyGraphOptions.cs b/src/OrasProject.Oras/CopyGraphOptions.cs index ea07247..c368c6c 100644 --- a/src/OrasProject.Oras/CopyGraphOptions.cs +++ b/src/OrasProject.Oras/CopyGraphOptions.cs @@ -25,27 +25,27 @@ public struct CopyGraphOptions /// /// PreCopy handles the current descriptor before it is copied. /// - public event Func PreCopy; + public event Func? PreCopy; /// /// PostCopy handles the current descriptor after it is copied. /// - public event Func PostCopy; + public event Func? PostCopy; /// /// CopySkipped will be called when the sub-DAG rooted by the current node /// is skipped. /// - public event Func CopySkipped; + public event Func? CopySkipped; internal Task OnPreCopyAsync(Descriptor descriptor) { - return PreCopy?.Invoke(descriptor) ?? Task.CompletedTask; + return PreCopy?.InvokeAsync(descriptor) ?? Task.CompletedTask; } internal Task OnPostCopyAsync(Descriptor descriptor) { - return PostCopy?.Invoke(descriptor) ?? Task.CompletedTask; + return PostCopy?.InvokeAsync(descriptor) ?? Task.CompletedTask; } internal Task OnCopySkippedAsync(Descriptor descriptor) diff --git a/src/OrasProject.Oras/Extensions.cs b/src/OrasProject.Oras/Extensions.cs index 4dad917..c3256a1 100644 --- a/src/OrasProject.Oras/Extensions.cs +++ b/src/OrasProject.Oras/Extensions.cs @@ -78,7 +78,7 @@ public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descripto // check if node exists in target if (await dst.ExistsAsync(node, cancellationToken).ConfigureAwait(false)) { - await copyGraphOptions.OnCopySkippedAsync(node); + await copyGraphOptions.OnCopySkippedAsync(node).ConfigureAwait(false); return; } @@ -92,12 +92,11 @@ public static async Task CopyGraphAsync(this ITarget src, ITarget dst, Descripto } // perform the copy - await copyGraphOptions.OnPreCopyAsync(node); + await copyGraphOptions.OnPreCopyAsync(node).ConfigureAwait(false); var dataStream = await src.FetchAsync(node, cancellationToken).ConfigureAwait(false); await dst.PushAsync(node, dataStream, cancellationToken).ConfigureAwait(false); // we copied it - await copyGraphOptions.OnPostCopyAsync(node); + await copyGraphOptions.OnPostCopyAsync(node).ConfigureAwait(false); } } - From e35fc7e61441ef75b4f9687bc0acf94136443036 Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Fri, 29 Nov 2024 19:49:14 -0300 Subject: [PATCH 18/19] refactor: delegate InvokeAsync to execute handlers in parallel Signed-off-by: Leonardo Chaia --- .../AsyncInvocationExtensions.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/OrasProject.Oras/AsyncInvocationExtensions.cs b/src/OrasProject.Oras/AsyncInvocationExtensions.cs index 968e598..371a0d9 100644 --- a/src/OrasProject.Oras/AsyncInvocationExtensions.cs +++ b/src/OrasProject.Oras/AsyncInvocationExtensions.cs @@ -12,6 +12,7 @@ // limitations under the License. using System; +using System.Linq; using System.Threading.Tasks; namespace OrasProject.Oras; @@ -19,21 +20,20 @@ namespace OrasProject.Oras; internal static class AsyncInvocationExtensions { /// - /// Sequentially invokes an event that returns a . - /// Each event listener is executed in sequence, and it's returning - /// task awaited before executing the next one. + /// Asynchronously invokes all handlers from an event that returns a . + /// All handlers are executed in parallel /// /// /// /// - internal static async Task InvokeAsync( + internal static Task InvokeAsync( this Func? eventDelegate, TEventArgs args) { - if (eventDelegate == null) return; + if (eventDelegate == null) return Task.CompletedTask; - foreach (var handler in eventDelegate.GetInvocationList()) - { - await ((Task?)handler.DynamicInvoke(args) ?? Task.CompletedTask); - } + var tasks = eventDelegate.GetInvocationList() + .Select(d => (Task?)d.DynamicInvoke(args) ?? Task.CompletedTask); + + return Task.WhenAll(tasks); } } From 827f62b00262ee9edabf3320f5ee80fc71602fbb Mon Sep 17 00:00:00 2001 From: Leonardo Chaia Date: Fri, 29 Nov 2024 19:49:32 -0300 Subject: [PATCH 19/19] refactor: Copy events to include sync and async variants. Signed-off-by: Leonardo Chaia --- src/OrasProject.Oras/CopyGraphOptions.cs | 31 +++++++++++++++++++----- tests/OrasProject.Oras.Tests/CopyTest.cs | 24 +++++++++++++----- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/OrasProject.Oras/CopyGraphOptions.cs b/src/OrasProject.Oras/CopyGraphOptions.cs index c368c6c..8aeac87 100644 --- a/src/OrasProject.Oras/CopyGraphOptions.cs +++ b/src/OrasProject.Oras/CopyGraphOptions.cs @@ -22,34 +22,53 @@ namespace OrasProject.Oras; /// public struct CopyGraphOptions { + /// + /// PreCopyAsync handles the current descriptor before it is copied. + /// + public event Func? PreCopyAsync; + /// /// PreCopy handles the current descriptor before it is copied. /// - public event Func? PreCopy; + public event Action? PreCopy; + + /// + /// PostCopyAsync handles the current descriptor after it is copied. + /// + public event Func? PostCopyAsync; /// /// PostCopy handles the current descriptor after it is copied. /// - public event Func? PostCopy; + public event Action? PostCopy; + + /// + /// CopySkippedAsync will be called when the sub-DAG rooted by the current node + /// is skipped. + /// + public event Func? CopySkippedAsync; /// /// CopySkipped will be called when the sub-DAG rooted by the current node /// is skipped. /// - public event Func? CopySkipped; + public event Action? CopySkipped; internal Task OnPreCopyAsync(Descriptor descriptor) { - return PreCopy?.InvokeAsync(descriptor) ?? Task.CompletedTask; + PreCopy?.Invoke(descriptor); + return PreCopyAsync?.InvokeAsync(descriptor) ?? Task.CompletedTask; } internal Task OnPostCopyAsync(Descriptor descriptor) { - return PostCopy?.InvokeAsync(descriptor) ?? Task.CompletedTask; + PostCopy?.Invoke(descriptor); + return PostCopyAsync?.InvokeAsync(descriptor) ?? Task.CompletedTask; } internal Task OnCopySkippedAsync(Descriptor descriptor) { - return CopySkipped?.Invoke(descriptor) ?? Task.CompletedTask; + CopySkipped?.Invoke(descriptor); + return CopySkippedAsync?.Invoke(descriptor) ?? Task.CompletedTask; } } diff --git a/tests/OrasProject.Oras.Tests/CopyTest.cs b/tests/OrasProject.Oras.Tests/CopyTest.cs index 5b2d9a2..c27f45b 100644 --- a/tests/OrasProject.Oras.Tests/CopyTest.cs +++ b/tests/OrasProject.Oras.Tests/CopyTest.cs @@ -198,15 +198,19 @@ void GenerateManifest(Descriptor config, params Descriptor[] layers) // Prepare copy options with OnCopySkippedAsync var skippedCount = 0; + var skippedAsyncCount = 0; var copyOptions = new CopyOptions { CopyGraphOptions = new CopyGraphOptions() { } }; - copyOptions.CopyGraphOptions.CopySkipped += d => + + copyOptions.CopyGraphOptions.CopySkipped += _ => skippedCount++; + + copyOptions.CopyGraphOptions.CopySkippedAsync += _ => { - skippedCount++; + skippedAsyncCount++; return Task.CompletedTask; }; @@ -235,6 +239,7 @@ void GenerateManifest(Descriptor config, params Descriptor[] layers) // Verify the OnCopySkippedAsync invocation count Assert.Equal(1, skippedCount); + Assert.Equal(1, skippedAsyncCount); } [Fact] @@ -416,6 +421,8 @@ void GenerateIndex(params Descriptor[] manifests) var dst = new MemoryStore(); var preCopyCount = 0; var postCopyCount = 0; + var preCopyAsyncCount = 0; + var postCopyAsyncCount = 0; var copyOptions = new CopyOptions { CopyGraphOptions = new CopyGraphOptions @@ -423,14 +430,17 @@ void GenerateIndex(params Descriptor[] manifests) } }; - copyOptions.CopyGraphOptions.PreCopy += d => + copyOptions.CopyGraphOptions.PreCopy += _ => preCopyCount++; + copyOptions.CopyGraphOptions.PreCopyAsync += d => { - preCopyCount++; + preCopyAsyncCount++; return Task.CompletedTask; }; - copyOptions.CopyGraphOptions.PostCopy += d => + + copyOptions.CopyGraphOptions.PostCopy += _ => postCopyCount++; + copyOptions.CopyGraphOptions.PostCopyAsync += d => { - postCopyCount++; + postCopyAsyncCount++; return Task.CompletedTask; }; @@ -447,7 +457,9 @@ void GenerateIndex(params Descriptor[] manifests) // Verify API counts Assert.Equal(7, preCopyCount); + Assert.Equal(7, preCopyAsyncCount); Assert.Equal(7, postCopyCount); + Assert.Equal(7, postCopyAsyncCount); } private class StorageTracker : ITarget