From 9beade5e44d80fa318d0551af29c1b8b4a22aa89 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Fri, 10 Oct 2025 16:38:40 -0400 Subject: [PATCH 1/6] Add Progress listeners for initiated, complete, and failed for simple upload update docs and function namse update comments add tests/update comments remove response from progressargs add missing properties update test Refactor test rearrange test update tests add commented out tests comments comments update comments fix spelling use interlocked read and fix build dev config update comment fix internals --- .../433a9a6d-b8ea-4676-b763-70711e8288e2.json | 12 + .../Transfer/Internal/SimpleUploadCommand.cs | 44 ++- .../_async/SimpleUploadCommand.async.cs | 13 +- .../Transfer/TransferUtilityUploadRequest.cs | 280 ++++++++++++++++++ .../IntegrationTests/TransferUtilityTests.cs | 188 ++++++++++++ 5 files changed, 534 insertions(+), 3 deletions(-) create mode 100644 generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json diff --git a/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json b/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json new file mode 100644 index 000000000000..e61f09349a79 --- /dev/null +++ b/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json @@ -0,0 +1,12 @@ +{ + "services": [ + { + "serviceName": "S3", + "type": "patch", + "changeLogMessages": [ + "Added progress tracking events to simple upload", + "Added PutObjectResponse to TransferUtilityUploadResponse mapping" + ] + } + ] +} diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs index 95a15611d2f5..d42a04692c46 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs @@ -41,6 +41,7 @@ internal partial class SimpleUploadCommand : BaseCommand IAmazonS3 _s3Client; TransferUtilityConfig _config; TransferUtilityUploadRequest _fileTransporterRequest; + long _totalTransferredBytes; internal SimpleUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config, TransferUtilityUploadRequest fileTransporterRequest) { @@ -103,9 +104,48 @@ private PutObjectRequest ConstructRequest() private void PutObjectProgressEventCallback(object sender, UploadProgressArgs e) { - var progressArgs = new UploadProgressArgs(e.IncrementTransferred, e.TransferredBytes, e.TotalBytes, - e.CompensationForRetry, _fileTransporterRequest.FilePath); + // Keep track of the total transferred bytes so that we can also return this value in case of failure + long transferredBytes = Interlocked.Add(ref _totalTransferredBytes, e.IncrementTransferred - e.CompensationForRetry); + + var progressArgs = new UploadProgressArgs(e.IncrementTransferred, transferredBytes, e.TotalBytes, + e.CompensationForRetry, _fileTransporterRequest.FilePath, _fileTransporterRequest); this._fileTransporterRequest.OnRaiseProgressEvent(progressArgs); } + + private void FireTransferInitiatedEvent() + { + var initiatedArgs = new UploadInitiatedEventArgs( + request: _fileTransporterRequest, + filePath: _fileTransporterRequest.FilePath, + totalBytes: _fileTransporterRequest.ContentLength + ); + + _fileTransporterRequest.OnRaiseTransferInitiatedEvent(initiatedArgs); + } + + private void FireTransferCompletedEvent(TransferUtilityUploadResponse response) + { + var completedArgs = new UploadCompletedEventArgs( + request: _fileTransporterRequest, + response: response, + filePath: _fileTransporterRequest.FilePath, + transferredBytes: Interlocked.Read(ref _totalTransferredBytes), + totalBytes: _fileTransporterRequest.ContentLength + ); + + _fileTransporterRequest.OnRaiseTransferCompletedEvent(completedArgs); + } + + private void FireTransferFailedEvent() + { + var failedArgs = new UploadFailedEventArgs( + request: _fileTransporterRequest, + filePath: _fileTransporterRequest.FilePath, + transferredBytes: Interlocked.Read(ref _totalTransferredBytes), + totalBytes: _fileTransporterRequest.ContentLength + ); + + _fileTransporterRequest.OnRaiseTransferFailedEvent(failedArgs); + } } } diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/_async/SimpleUploadCommand.async.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/_async/SimpleUploadCommand.async.cs index e4c94d65044f..51680eaaba09 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/_async/SimpleUploadCommand.async.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/_async/SimpleUploadCommand.async.cs @@ -38,9 +38,20 @@ await this.AsyncThrottler.WaitAsync(cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); } + FireTransferInitiatedEvent(); + var putRequest = ConstructRequest(); - await _s3Client.PutObjectAsync(putRequest, cancellationToken) + var response = await _s3Client.PutObjectAsync(putRequest, cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); + + var mappedResponse = ResponseMapper.MapPutObjectResponse(response); + + FireTransferCompletedEvent(mappedResponse); + } + catch (Exception) + { + FireTransferFailedEvent(); + throw; } finally { diff --git a/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadRequest.cs b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadRequest.cs index 868fcf697dd8..ff753b6efb13 100644 --- a/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadRequest.cs +++ b/sdk/src/Services/S3/Custom/Transfer/TransferUtilityUploadRequest.cs @@ -25,6 +25,7 @@ using System.IO; using System.Text; +using Amazon.Runtime; using Amazon.Runtime.Internal; using Amazon.S3.Model; using Amazon.Util; @@ -411,6 +412,132 @@ public List TagSet /// public event EventHandler UploadProgressEvent; + /// + /// The event for UploadInitiatedEvent notifications. All + /// subscribers will be notified when a transfer operation + /// starts. + /// + /// The UploadInitiatedEvent is fired exactly once when + /// a transfer operation begins. The delegates attached to the event + /// will be passed information about the upload request and + /// total file size, but no progress information. + /// + /// + /// + /// Subscribe to this event if you want to receive + /// UploadInitiatedEvent notifications. Here is how:
+ /// 1. Define a method with a signature similar to this one: + /// + /// private void uploadStarted(object sender, UploadInitiatedEventArgs args) + /// { + /// Console.WriteLine($"Upload started: {args.FilePath}"); + /// Console.WriteLine($"Total size: {args.TotalBytes} bytes"); + /// Console.WriteLine($"Bucket: {args.Request.BucketName}"); + /// Console.WriteLine($"Key: {args.Request.Key}"); + /// } + /// + /// 2. Add this method to the UploadInitiatedEvent delegate's invocation list + /// + /// TransferUtilityUploadRequest request = new TransferUtilityUploadRequest(); + /// request.UploadInitiatedEvent += uploadStarted; + /// + /// + public event EventHandler UploadInitiatedEvent; + + /// + /// The event for UploadCompletedEvent notifications. All + /// subscribers will be notified when a transfer operation + /// completes successfully. + /// + /// The UploadCompletedEvent is fired exactly once when + /// a transfer operation completes successfully. The delegates attached to the event + /// will be passed information about the completed upload including + /// the final response from S3 with ETag, VersionId, and other metadata. + /// + /// + /// + /// Subscribe to this event if you want to receive + /// UploadCompletedEvent notifications. Here is how:
+ /// 1. Define a method with a signature similar to this one: + /// + /// private void uploadCompleted(object sender, UploadCompletedEventArgs args) + /// { + /// Console.WriteLine($"Upload completed: {args.FilePath}"); + /// Console.WriteLine($"Transferred: {args.TransferredBytes} bytes"); + /// Console.WriteLine($"ETag: {args.Response.ETag}"); + /// Console.WriteLine($"S3 Key: {args.Response.Key}"); + /// Console.WriteLine($"Version ID: {args.Response.VersionId}"); + /// } + /// + /// 2. Add this method to the UploadCompletedEvent delegate's invocation list + /// + /// TransferUtilityUploadRequest request = new TransferUtilityUploadRequest(); + /// request.UploadCompletedEvent += uploadCompleted; + /// + /// + public event EventHandler UploadCompletedEvent; + + /// + /// The event for UploadFailedEvent notifications. All + /// subscribers will be notified when a transfer operation + /// fails. + /// + /// The UploadFailedEvent is fired exactly once when + /// a transfer operation fails. The delegates attached to the event + /// will be passed information about the failed upload including + /// partial progress information, but no response data since the upload failed. + /// + /// + /// + /// Subscribe to this event if you want to receive + /// UploadFailedEvent notifications. Here is how:
+ /// 1. Define a method with a signature similar to this one: + /// + /// private void uploadFailed(object sender, UploadFailedEventArgs args) + /// { + /// Console.WriteLine($"Upload failed: {args.FilePath}"); + /// Console.WriteLine($"Partial progress: {args.TransferredBytes} / {args.TotalBytes} bytes"); + /// var percent = (double)args.TransferredBytes / args.TotalBytes * 100; + /// Console.WriteLine($"Completion: {percent:F1}%"); + /// Console.WriteLine($"Bucket: {args.Request.BucketName}"); + /// Console.WriteLine($"Key: {args.Request.Key}"); + /// } + /// + /// 2. Add this method to the UploadFailedEvent delegate's invocation list + /// + /// TransferUtilityUploadRequest request = new TransferUtilityUploadRequest(); + /// request.UploadFailedEvent += uploadFailed; + /// + /// + public event EventHandler UploadFailedEvent; + + /// + /// Causes the UploadInitiatedEvent event to be fired. + /// + /// UploadInitiatedEventArgs args + internal void OnRaiseTransferInitiatedEvent(UploadInitiatedEventArgs args) + { + AWSSDKUtils.InvokeInBackground(UploadInitiatedEvent, args, this); + } + + /// + /// Causes the UploadCompletedEvent event to be fired. + /// + /// UploadCompletedEventArgs args + internal void OnRaiseTransferCompletedEvent(UploadCompletedEventArgs args) + { + AWSSDKUtils.InvokeInBackground(UploadCompletedEvent, args, this); + } + + /// + /// Causes the UploadFailedEvent event to be fired. + /// + /// UploadFailedEventArgs args + internal void OnRaiseTransferFailedEvent(UploadFailedEventArgs args) + { + AWSSDKUtils.InvokeInBackground(UploadFailedEvent, args, this); + } + /// /// Causes the UploadProgressEvent event to be fired. @@ -835,11 +962,164 @@ internal UploadProgressArgs(long incrementTransferred, long transferred, long to this.CompensationForRetry = compensationForRetry; } + /// + /// Constructor for upload progress with request + /// + /// The how many bytes were transferred since last event. + /// The number of bytes transferred + /// The total number of bytes to be transferred + /// A compensation for any upstream aggregators if this event to correct their totalTransferred count, + /// in case the underlying request is retried. + /// The file being uploaded + /// The original TransferUtilityUploadRequest created by the user + internal UploadProgressArgs(long incrementTransferred, long transferred, long total, long compensationForRetry, string filePath, TransferUtilityUploadRequest request) + : base(incrementTransferred, transferred, total) + { + this.FilePath = filePath; + this.CompensationForRetry = compensationForRetry; + this.Request = request; + } + /// /// Gets the FilePath. /// public string FilePath { get; private set; } internal long CompensationForRetry { get; set; } + + /// + /// The original TransferUtilityUploadRequest created by the user. + /// + public TransferUtilityUploadRequest Request { get; internal set; } + } + + /// + /// Encapsulates the information needed when a transfer operation is initiated. + /// Provides access to the original request and total file size without any progress information. + /// + public class UploadInitiatedEventArgs : EventArgs + { + /// + /// Initializes a new instance of the UploadInitiatedEventArgs class. + /// + /// The original TransferUtilityUploadRequest created by the user + /// The file being uploaded + /// The total number of bytes to be transferred + internal UploadInitiatedEventArgs(TransferUtilityUploadRequest request, string filePath, long totalBytes) + { + Request = request; + FilePath = filePath; + TotalBytes = totalBytes; + } + + /// + /// The original TransferUtilityUploadRequest created by the user. + /// Contains all the upload parameters and configuration. + /// + public TransferUtilityUploadRequest Request { get; private set; } + + /// + /// Gets the file path being uploaded. + /// + public string FilePath { get; private set; } + + /// + /// Gets the total number of bytes to be transferred. + /// + public long TotalBytes { get; private set; } + } + + /// + /// Encapsulates the information needed when a transfer operation completes successfully. + /// Provides access to the original request, final response, and completion details. + /// + public class UploadCompletedEventArgs : EventArgs + { + /// + /// Initializes a new instance of the UploadCompletedEventArgs class. + /// + /// The original TransferUtilityUploadRequest created by the user + /// The unified response from Transfer Utility + /// The file that was uploaded + /// The total number of bytes transferred + /// The total number of bytes that were transferred + internal UploadCompletedEventArgs(TransferUtilityUploadRequest request, TransferUtilityUploadResponse response, string filePath, long transferredBytes, long totalBytes) + { + Request = request; + Response = response; + FilePath = filePath; + TransferredBytes = transferredBytes; + TotalBytes = totalBytes; + } + + /// + /// The original TransferUtilityUploadRequest created by the user. + /// Contains all the upload parameters and configuration. + /// + public TransferUtilityUploadRequest Request { get; private set; } + + /// + /// The unified response from Transfer Utility after successful upload completion. + /// Contains mapped fields from either PutObjectResponse (simple uploads) or CompleteMultipartUploadResponse (multipart uploads). + /// + public TransferUtilityUploadResponse Response { get; private set; } + + /// + /// Gets the file path that was uploaded. + /// + public string FilePath { get; private set; } + + /// + /// Gets the total number of bytes that were successfully transferred. + /// + public long TransferredBytes { get; private set; } + + /// + /// Gets the total number of bytes that were transferred (should equal TransferredBytes for successful uploads). + /// + public long TotalBytes { get; private set; } + } + + /// + /// Encapsulates the information needed when a transfer operation fails. + /// Provides access to the original request and partial progress information. + /// + public class UploadFailedEventArgs : EventArgs + { + /// + /// Initializes a new instance of the UploadFailedEventArgs class. + /// + /// The original TransferUtilityUploadRequest created by the user + /// The file that was being uploaded + /// The number of bytes transferred before failure + /// The total number of bytes that should have been transferred + internal UploadFailedEventArgs(TransferUtilityUploadRequest request, string filePath, long transferredBytes, long totalBytes) + { + Request = request; + FilePath = filePath; + TransferredBytes = transferredBytes; + TotalBytes = totalBytes; + } + + /// + /// The original TransferUtilityUploadRequest created by the user. + /// Contains all the upload parameters and configuration. + /// + public TransferUtilityUploadRequest Request { get; private set; } + + /// + /// Gets the file path that was being uploaded. + /// + public string FilePath { get; private set; } + + /// + /// Gets the number of bytes that were transferred before the failure occurred. + /// + public long TransferredBytes { get; private set; } + + /// + /// Gets the total number of bytes that should have been transferred. + /// + public long TotalBytes { get; private set; } } } diff --git a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs index abc767a0bdd8..47c560715963 100644 --- a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs +++ b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs @@ -105,6 +105,113 @@ public void SimpleUploadProgressTest() progressValidator.AssertOnCompletion(); } + [TestMethod] + [TestCategory("S3")] + public void SimpleUploadInitiatedEventTest() + { + var fileName = UtilityMethods.GenerateName(@"SimpleUploadTest\InitiatedEvent"); + var eventValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName)); + Assert.IsTrue(args.TotalBytes > 0); + Assert.AreEqual(10 * MEG_SIZE, args.TotalBytes); + } + }; + UploadWithLifecycleEvents(fileName, 10 * MEG_SIZE, eventValidator, null, null); + eventValidator.AssertEventFired(); + } + + [TestMethod] + [TestCategory("S3")] + public void SimpleUploadCompletedEventTest() + { + var fileName = UtilityMethods.GenerateName(@"SimpleUploadTest\CompletedEvent"); + var eventValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.IsNotNull(args.Response); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName)); + Assert.AreEqual(args.TransferredBytes, args.TotalBytes); + Assert.AreEqual(10 * MEG_SIZE, args.TotalBytes); + Assert.IsTrue(!string.IsNullOrEmpty(args.Response.ETag)); + } + }; + UploadWithLifecycleEvents(fileName, 10 * MEG_SIZE, null, eventValidator, null); + eventValidator.AssertEventFired(); + } + + [TestMethod] + [TestCategory("S3")] + public void SimpleUploadFailedEventTest() + { + var fileName = UtilityMethods.GenerateName(@"SimpleUploadTest\FailedEvent"); + var eventValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName)); + Assert.IsTrue(args.TotalBytes > 0); + Assert.AreEqual(5 * MEG_SIZE, args.TotalBytes); + // For failed uploads, transferred bytes should be less than or equal to total bytes + Assert.IsTrue(args.TransferredBytes <= args.TotalBytes); + } + }; + + // Use invalid bucket name to force failure + var invalidBucketName = "invalid-bucket-name-" + Guid.NewGuid().ToString(); + + try + { + UploadWithLifecycleEventsAndBucket(fileName, 5 * MEG_SIZE, invalidBucketName, null, null, eventValidator); + Assert.Fail("Expected an exception to be thrown for invalid bucket"); + } + catch (AmazonS3Exception) + { + // Expected exception - the failed event should have been fired + eventValidator.AssertEventFired(); + } + } + + [TestMethod] + [TestCategory("S3")] + public void SimpleUploadCompleteLifecycleTest() + { + var fileName = UtilityMethods.GenerateName(@"SimpleUploadTest\CompleteLifecycle"); + + var initiatedValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName)); + Assert.AreEqual(8 * MEG_SIZE, args.TotalBytes); + } + }; + + var completedValidator = new TransferLifecycleEventValidator + { + Validate = (args) => + { + Assert.IsNotNull(args.Request); + Assert.IsNotNull(args.Response); + Assert.AreEqual(args.FilePath, Path.Combine(BasePath, fileName)); + Assert.AreEqual(args.TransferredBytes, args.TotalBytes); + Assert.AreEqual(8 * MEG_SIZE, args.TotalBytes); + } + }; + + UploadWithLifecycleEvents(fileName, 8 * MEG_SIZE, initiatedValidator, completedValidator, null); + + initiatedValidator.AssertEventFired(); + completedValidator.AssertEventFired(); + } + [TestMethod] [TestCategory("S3")] public void SimpleUpload() @@ -1374,6 +1481,87 @@ public void OnProgressEvent(object sender, T progress) } } } + + class TransferLifecycleEventValidator + { + public Action Validate { get; set; } + public bool EventFired { get; private set; } + public Exception EventException { get; private set; } + + public void OnEventFired(object sender, T eventArgs) + { + try + { + EventFired = true; + Console.WriteLine("Lifecycle Event Fired: {0}", typeof(T).Name); + Validate?.Invoke(eventArgs); + } + catch (Exception ex) + { + EventException = ex; + Console.WriteLine("Exception caught in lifecycle event: {0}", ex.Message); + throw; + } + } + + public void AssertEventFired() + { + if (EventException != null) + throw EventException; + + // Add some time for the background thread to finish before checking + for (int retries = 1; retries < 5 && !EventFired; retries++) + { + Thread.Sleep(1000 * retries); + } + Assert.IsTrue(EventFired, $"{typeof(T).Name} event was not fired"); + } + } + + void UploadWithLifecycleEvents(string fileName, long size, + TransferLifecycleEventValidator initiatedValidator, + TransferLifecycleEventValidator completedValidator, + TransferLifecycleEventValidator failedValidator) + { + UploadWithLifecycleEventsAndBucket(fileName, size, bucketName, initiatedValidator, completedValidator, failedValidator); + } + + void UploadWithLifecycleEventsAndBucket(string fileName, long size, string targetBucketName, + TransferLifecycleEventValidator initiatedValidator, + TransferLifecycleEventValidator completedValidator, + TransferLifecycleEventValidator failedValidator) + { + var key = fileName; + var path = Path.Combine(BasePath, fileName); + UtilityMethods.GenerateFile(path, size); + + var config = new TransferUtilityConfig(); + var transferUtility = new TransferUtility(Client, config); + var request = new TransferUtilityUploadRequest + { + BucketName = targetBucketName, + FilePath = path, + Key = key, + ContentType = octetStreamContentType + }; + + if (initiatedValidator != null) + { + request.UploadInitiatedEvent += initiatedValidator.OnEventFired; + } + + if (completedValidator != null) + { + request.UploadCompletedEvent += completedValidator.OnEventFired; + } + + if (failedValidator != null) + { + request.UploadFailedEvent += failedValidator.OnEventFired; + } + + transferUtility.Upload(request); + } private class UnseekableStream : MemoryStream { private readonly bool _setZeroLengthStream; From a5de17c6b6b4df381212e235a577161b964ecd53 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Tue, 14 Oct 2025 12:31:50 -0400 Subject: [PATCH 2/6] update dev config --- .../.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json b/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json index e61f09349a79..b925cee26b66 100644 --- a/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json +++ b/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json @@ -4,8 +4,7 @@ "serviceName": "S3", "type": "patch", "changeLogMessages": [ - "Added progress tracking events to simple upload", - "Added PutObjectResponse to TransferUtilityUploadResponse mapping" + "Added progress tracking events to simple upload" ] } ] From 810baa65d5d71a83938b41a436e150dd56c6ce8c Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Fri, 17 Oct 2025 12:39:11 -0400 Subject: [PATCH 3/6] cache content length --- .../Transfer/Internal/SimpleUploadCommand.cs | 13 +++-- .../IntegrationTests/TransferUtilityTests.cs | 52 +++++++++---------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs index d42a04692c46..d7383fbe9554 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs @@ -42,12 +42,17 @@ internal partial class SimpleUploadCommand : BaseCommand TransferUtilityConfig _config; TransferUtilityUploadRequest _fileTransporterRequest; long _totalTransferredBytes; + private readonly long _cachedContentLength; internal SimpleUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config, TransferUtilityUploadRequest fileTransporterRequest) { this._s3Client = s3Client; this._config = config; this._fileTransporterRequest = fileTransporterRequest; + + // Cache content length immediately while stream is accessible to avoid ObjectDisposedException in failure scenarios + this._cachedContentLength = this._fileTransporterRequest.ContentLength; + var fileName = fileTransporterRequest.FilePath; } @@ -107,7 +112,7 @@ private void PutObjectProgressEventCallback(object sender, UploadProgressArgs e) // Keep track of the total transferred bytes so that we can also return this value in case of failure long transferredBytes = Interlocked.Add(ref _totalTransferredBytes, e.IncrementTransferred - e.CompensationForRetry); - var progressArgs = new UploadProgressArgs(e.IncrementTransferred, transferredBytes, e.TotalBytes, + var progressArgs = new UploadProgressArgs(e.IncrementTransferred, transferredBytes, _cachedContentLength, e.CompensationForRetry, _fileTransporterRequest.FilePath, _fileTransporterRequest); this._fileTransporterRequest.OnRaiseProgressEvent(progressArgs); } @@ -117,7 +122,7 @@ private void FireTransferInitiatedEvent() var initiatedArgs = new UploadInitiatedEventArgs( request: _fileTransporterRequest, filePath: _fileTransporterRequest.FilePath, - totalBytes: _fileTransporterRequest.ContentLength + totalBytes: _cachedContentLength ); _fileTransporterRequest.OnRaiseTransferInitiatedEvent(initiatedArgs); @@ -130,7 +135,7 @@ private void FireTransferCompletedEvent(TransferUtilityUploadResponse response) response: response, filePath: _fileTransporterRequest.FilePath, transferredBytes: Interlocked.Read(ref _totalTransferredBytes), - totalBytes: _fileTransporterRequest.ContentLength + totalBytes: _cachedContentLength ); _fileTransporterRequest.OnRaiseTransferCompletedEvent(completedArgs); @@ -142,7 +147,7 @@ private void FireTransferFailedEvent() request: _fileTransporterRequest, filePath: _fileTransporterRequest.FilePath, transferredBytes: Interlocked.Read(ref _totalTransferredBytes), - totalBytes: _fileTransporterRequest.ContentLength + totalBytes: _cachedContentLength ); _fileTransporterRequest.OnRaiseTransferFailedEvent(failedArgs); diff --git a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs index 47c560715963..3280bc13b241 100644 --- a/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs +++ b/sdk/test/Services/S3/IntegrationTests/TransferUtilityTests.cs @@ -30,6 +30,8 @@ public class TransferUtilityTests : TestBase private static string fullPath; private const string testContent = "This is the content body!"; private const string testFile = "PutObjectFile.txt"; + private static string testFilePath; + private const string testKey = "SimpleUploadProgressTotalBytesTestFile.txt"; [ClassInitialize()] public static void ClassInitialize(TestContext a) @@ -66,6 +68,7 @@ public static void ClassInitialize(TestContext a) fullPath = Path.GetFullPath(testFile); File.WriteAllText(fullPath, testContent); + testFilePath = fullPath; // Use the same file for the TotalBytes test } [ClassCleanup] @@ -444,41 +447,36 @@ public void UploadUnSeekableStreamFileSizeEqualToPartBufferSize() } [TestMethod] - [TestCategory("S3")] - public void UploadUnseekableStreamFileSizeBetweenMinPartSizeAndPartBufferSize() + public void SimpleUploadProgressTotalBytesTest() { - var client = Client; - var fileName = UtilityMethods.GenerateName(@"SimpleUploadTest\BetweenMinPartSizeAndPartBufferSize"); - var path = Path.Combine(BasePath, fileName); - // there was a bug where the transfer utility was uploading 13MB file - // when the file size was between 5MB and (5MB + 8192). 8192 is the s3Client.Config.BufferSize - var fileSize = 5 * MEG_SIZE + 1; - - UtilityMethods.GenerateFile(path, fileSize); - //take the generated file and turn it into an unseekable stream - - var stream = GenerateUnseekableStreamFromFile(path); - using (var tu = new Amazon.S3.Transfer.TransferUtility(client)) + var transferConfig = new TransferUtilityConfig() { - tu.Upload(stream, bucketName, fileName); + MinSizeBeforePartUpload = 20 * MEG_SIZE, + }; - var metadata = Client.GetObjectMetadata(new GetObjectMetadataRequest + var progressValidator = new TransferProgressValidator + { + Validate = (progress) => { - BucketName = bucketName, - Key = fileName - }); - Assert.AreEqual(fileSize, metadata.ContentLength); + Assert.IsTrue(progress.TotalBytes > 0, "TotalBytes should be greater than 0"); + Assert.AreEqual(testContent.Length, progress.TotalBytes, "TotalBytes should equal file length"); + } + }; - //Download the file and validate content of downloaded file is equal. - var downloadPath = path + ".download"; - var downloadRequest = new TransferUtilityDownloadRequest + using (var fileTransferUtility = new TransferUtility(Client, transferConfig)) + { + var request = new TransferUtilityUploadRequest() { BucketName = bucketName, - Key = fileName, - FilePath = downloadPath + FilePath = testFilePath, + Key = testKey }; - tu.Download(downloadRequest); - UtilityMethods.CompareFiles(path, downloadPath); + + request.UploadProgressEvent += progressValidator.OnProgressEvent; + + fileTransferUtility.Upload(request); + + progressValidator.AssertOnCompletion(); } } From 5355e7e073b4c46c2e92b79c35fb0290c23ce91b Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Fri, 17 Oct 2025 12:42:47 -0400 Subject: [PATCH 4/6] Change S3 service type from patch to minor --- generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json b/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json index b925cee26b66..3f782fc50a20 100644 --- a/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json +++ b/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e2.json @@ -2,7 +2,7 @@ "services": [ { "serviceName": "S3", - "type": "patch", + "type": "minor", "changeLogMessages": [ "Added progress tracking events to simple upload" ] From 3f70b05f133d5c34223c0a3833bc95be63c8da28 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Fri, 17 Oct 2025 12:46:23 -0400 Subject: [PATCH 5/6] rename variable --- .../Custom/Transfer/Internal/SimpleUploadCommand.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs index d7383fbe9554..072d0ff3a6d1 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/SimpleUploadCommand.cs @@ -42,7 +42,7 @@ internal partial class SimpleUploadCommand : BaseCommand TransferUtilityConfig _config; TransferUtilityUploadRequest _fileTransporterRequest; long _totalTransferredBytes; - private readonly long _cachedContentLength; + private readonly long _contentLength; internal SimpleUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config, TransferUtilityUploadRequest fileTransporterRequest) { @@ -51,7 +51,7 @@ internal SimpleUploadCommand(IAmazonS3 s3Client, TransferUtilityConfig config, T this._fileTransporterRequest = fileTransporterRequest; // Cache content length immediately while stream is accessible to avoid ObjectDisposedException in failure scenarios - this._cachedContentLength = this._fileTransporterRequest.ContentLength; + this._contentLength = this._fileTransporterRequest.ContentLength; var fileName = fileTransporterRequest.FilePath; } @@ -112,7 +112,7 @@ private void PutObjectProgressEventCallback(object sender, UploadProgressArgs e) // Keep track of the total transferred bytes so that we can also return this value in case of failure long transferredBytes = Interlocked.Add(ref _totalTransferredBytes, e.IncrementTransferred - e.CompensationForRetry); - var progressArgs = new UploadProgressArgs(e.IncrementTransferred, transferredBytes, _cachedContentLength, + var progressArgs = new UploadProgressArgs(e.IncrementTransferred, transferredBytes, _contentLength, e.CompensationForRetry, _fileTransporterRequest.FilePath, _fileTransporterRequest); this._fileTransporterRequest.OnRaiseProgressEvent(progressArgs); } @@ -122,7 +122,7 @@ private void FireTransferInitiatedEvent() var initiatedArgs = new UploadInitiatedEventArgs( request: _fileTransporterRequest, filePath: _fileTransporterRequest.FilePath, - totalBytes: _cachedContentLength + totalBytes: _contentLength ); _fileTransporterRequest.OnRaiseTransferInitiatedEvent(initiatedArgs); @@ -135,7 +135,7 @@ private void FireTransferCompletedEvent(TransferUtilityUploadResponse response) response: response, filePath: _fileTransporterRequest.FilePath, transferredBytes: Interlocked.Read(ref _totalTransferredBytes), - totalBytes: _cachedContentLength + totalBytes: _contentLength ); _fileTransporterRequest.OnRaiseTransferCompletedEvent(completedArgs); @@ -147,7 +147,7 @@ private void FireTransferFailedEvent() request: _fileTransporterRequest, filePath: _fileTransporterRequest.FilePath, transferredBytes: Interlocked.Read(ref _totalTransferredBytes), - totalBytes: _cachedContentLength + totalBytes: _contentLength ); _fileTransporterRequest.OnRaiseTransferFailedEvent(failedArgs); From 4f32bc13b2ab619ef430aa4aecde075f42f49523 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Tue, 14 Oct 2025 12:50:10 -0400 Subject: [PATCH 6/6] add completeuploadresponse mapping to turesponse object --- .../433a9a6d-b8ea-4676-b763-70711e8288e6.json | 11 ++ .../Transfer/Internal/ResponseMapper.cs | 61 ++++++++++ .../UnitTests/Custom/ResponseMapperTests.cs | 111 ++++++++++++++++++ 3 files changed, 183 insertions(+) create mode 100644 generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e6.json diff --git a/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e6.json b/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e6.json new file mode 100644 index 000000000000..5d67e3a8b858 --- /dev/null +++ b/generator/.DevConfigs/433a9a6d-b8ea-4676-b763-70711e8288e6.json @@ -0,0 +1,11 @@ +{ + "services": [ + { + "serviceName": "S3", + "type": "patch", + "changeLogMessages": [ + "Added CompleteMultipartUploadResponse to TransferUtilityUploadResponse mapping" + ] + } + ] +} diff --git a/sdk/src/Services/S3/Custom/Transfer/Internal/ResponseMapper.cs b/sdk/src/Services/S3/Custom/Transfer/Internal/ResponseMapper.cs index d130aee20bff..7e8505ecbf69 100644 --- a/sdk/src/Services/S3/Custom/Transfer/Internal/ResponseMapper.cs +++ b/sdk/src/Services/S3/Custom/Transfer/Internal/ResponseMapper.cs @@ -99,6 +99,67 @@ internal static TransferUtilityUploadResponse MapPutObjectResponse(PutObjectResp return response; } + + /// + /// Maps a CompleteMultipartUploadResponse to TransferUtilityUploadResponse. + /// Uses the field mappings defined in mapping.json "Conversion" -> "CompleteMultipartResponse" -> "UploadResponse". + /// + /// The CompleteMultipartUploadResponse to map from + /// A new TransferUtilityUploadResponse with mapped fields + internal static TransferUtilityUploadResponse MapCompleteMultipartUploadResponse(CompleteMultipartUploadResponse source) + { + if (source == null) + return null; + + var response = new TransferUtilityUploadResponse(); + + // Map all fields as defined in mapping.json "Conversion" -> "CompleteMultipartResponse" -> "UploadResponse" + if (source.IsSetBucketKeyEnabled()) + response.BucketKeyEnabled = source.BucketKeyEnabled.GetValueOrDefault(); + + if (source.IsSetChecksumCRC32()) + response.ChecksumCRC32 = source.ChecksumCRC32; + + if (source.IsSetChecksumCRC32C()) + response.ChecksumCRC32C = source.ChecksumCRC32C; + + if (source.IsSetChecksumCRC64NVME()) + response.ChecksumCRC64NVME = source.ChecksumCRC64NVME; + + if (source.IsSetChecksumSHA1()) + response.ChecksumSHA1 = source.ChecksumSHA1; + + if (source.IsSetChecksumSHA256()) + response.ChecksumSHA256 = source.ChecksumSHA256; + + if (source.ChecksumType != null) + response.ChecksumType = source.ChecksumType; + + if (source.IsSetETag()) + response.ETag = source.ETag; + + if (source.Expiration != null) + response.Expiration = source.Expiration; + + if (source.IsSetRequestCharged()) + response.RequestCharged = source.RequestCharged; + + if (source.ServerSideEncryptionMethod != null) + response.ServerSideEncryptionMethod = source.ServerSideEncryptionMethod; + + if (source.IsSetServerSideEncryptionKeyManagementServiceKeyId()) + response.ServerSideEncryptionKeyManagementServiceKeyId = source.ServerSideEncryptionKeyManagementServiceKeyId; + + if (source.IsSetVersionId()) + response.VersionId = source.VersionId; + + // Copy response metadata + response.ResponseMetadata = source.ResponseMetadata; + response.ContentLength = source.ContentLength; + response.HttpStatusCode = source.HttpStatusCode; + + return response; + } } } diff --git a/sdk/test/Services/S3/UnitTests/Custom/ResponseMapperTests.cs b/sdk/test/Services/S3/UnitTests/Custom/ResponseMapperTests.cs index 0b57959f2b5a..01f251f4c415 100644 --- a/sdk/test/Services/S3/UnitTests/Custom/ResponseMapperTests.cs +++ b/sdk/test/Services/S3/UnitTests/Custom/ResponseMapperTests.cs @@ -254,6 +254,117 @@ public void ValidateTransferUtilityUploadResponseDefinitionCompleteness() "TransferUtilityUploadResponse"); } + [TestMethod] + [TestCategory("S3")] + public void MapCompleteMultipartUploadResponse_AllMappedProperties_WorkCorrectly() + { + // Get the expected mappings from JSON + var completeMultipartMappings = _mappingJson.RootElement + .GetProperty("Conversion") + .GetProperty("CompleteMultipartResponse") + .GetProperty("UploadResponse") + .EnumerateArray() + .Select(prop => prop.GetString()) + .ToList(); + + // Create source object with dynamically generated test data + var sourceResponse = new CompleteMultipartUploadResponse(); + var sourceType = typeof(CompleteMultipartUploadResponse); + var testDataValues = new Dictionary(); + + // Generate test data for each mapped property + foreach (var propertyName in completeMultipartMappings) + { + // Resolve alias to actual property name + var resolvedPropertyName = ResolvePropertyName(propertyName); + var sourceProperty = sourceType.GetProperty(resolvedPropertyName); + if (sourceProperty?.CanWrite == true) + { + var testValue = GenerateTestValue(sourceProperty.PropertyType, propertyName); + sourceProperty.SetValue(sourceResponse, testValue); + testDataValues[propertyName] = testValue; + } + } + + // Add inherited properties for comprehensive testing + sourceResponse.HttpStatusCode = HttpStatusCode.OK; + sourceResponse.ContentLength = 2048; + + // Map the response + var mappedResponse = ResponseMapper.MapCompleteMultipartUploadResponse(sourceResponse); + Assert.IsNotNull(mappedResponse, "Mapped response should not be null"); + + // Verify all mapped properties using reflection + var targetType = typeof(TransferUtilityUploadResponse); + var failedAssertions = new List(); + + foreach (var propertyName in completeMultipartMappings) + { + // Resolve alias to actual property name for reflection lookups + var resolvedPropertyName = ResolvePropertyName(propertyName); + var sourceProperty = sourceType.GetProperty(resolvedPropertyName); + var targetProperty = targetType.GetProperty(resolvedPropertyName); + + if (sourceProperty == null) + { + failedAssertions.Add($"Source property '{propertyName}' (resolved to: {resolvedPropertyName}) not found in CompleteMultipartUploadResponse"); + continue; + } + + if (targetProperty == null) + { + failedAssertions.Add($"Target property '{propertyName}' (resolved to: {resolvedPropertyName}) not found in TransferUtilityUploadResponse"); + continue; + } + + var sourceValue = sourceProperty.GetValue(sourceResponse); + var targetValue = targetProperty.GetValue(mappedResponse); + + // Special handling for complex object comparisons + if (!AreValuesEqual(sourceValue, targetValue)) + { + failedAssertions.Add($"{propertyName}: Expected '{sourceValue ?? "null"}', got '{targetValue ?? "null"}'"); + } + } + + // Test inherited properties + Assert.AreEqual(sourceResponse.HttpStatusCode, mappedResponse.HttpStatusCode, "HttpStatusCode should match"); + Assert.AreEqual(sourceResponse.ContentLength, mappedResponse.ContentLength, "ContentLength should match"); + + // Report any failures + if (failedAssertions.Any()) + { + Assert.Fail($"Property mapping failures:\n{string.Join("\n", failedAssertions)}"); + } + } + + [TestMethod] + [TestCategory("S3")] + public void MapCompleteMultipartUploadResponse_NullValues_HandledCorrectly() + { + // Test null handling scenarios + var testCases = new[] + { + // Test null Expiration + new CompleteMultipartUploadResponse { Expiration = null }, + + // Test null enum conversions + new CompleteMultipartUploadResponse { ChecksumType = null, RequestCharged = null, ServerSideEncryption = null } + }; + + foreach (var testCase in testCases) + { + var mapped = ResponseMapper.MapCompleteMultipartUploadResponse(testCase); + Assert.IsNotNull(mapped, "Response should always be mappable"); + + // Test null handling + if (testCase.Expiration == null) + { + Assert.IsNull(mapped.Expiration, "Null Expiration should map to null"); + } + } + } + [TestMethod] [TestCategory("S3")] public void ValidateCompleteMultipartUploadResponseConversionCompleteness()