From 3a38f1e68c7cc9374b76e0dd66f99c49b90fbab7 Mon Sep 17 00:00:00 2001 From: ebozduman Date: Fri, 20 Oct 2023 22:58:23 -0700 Subject: [PATCH] Lifecycle expiration date, WithEndpoint, WithSSL issues cause functional tests to fail (#844) --- Minio.Functional.Tests/FunctionalTest.cs | 83 ++++++++++--------- Minio/ApiEndpoints/BucketOperations.cs | 11 ++- Minio/DataModel/ILM/Duration.cs | 9 +- Minio/DataModel/ILM/LifecycleConfiguration.cs | 4 +- Minio/DataModel/Result/ResponseResult.cs | 2 +- Minio/Helper/OperationsHelper.cs | 3 +- Minio/MinioClientExtensions.cs | 23 +++-- Minio/RequestExtensions.cs | 48 +++++++++-- 8 files changed, 109 insertions(+), 74 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index 2497bd4a9..0024f3512 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -590,10 +590,9 @@ internal static async Task TearDown(IMinioClient minio, string bucketName) new GetObjectLockConfigurationArgs() .WithBucket(bucketName); ObjectLockConfiguration lockConfig = null; - VersioningConfiguration versioningConfig = null; try { - versioningConfig = await minio.GetVersioningAsync(new GetVersioningArgs() + var versioningConfig = await minio.GetVersioningAsync(new GetVersioningArgs() .WithBucket(bucketName)).ConfigureAwait(false); if (versioningConfig is not null && (versioningConfig.Status.Contains("Enabled", StringComparison.Ordinal) || @@ -633,7 +632,7 @@ internal static async Task TearDown(IMinioClient minio, string bucketName) exceptionList.Add, () => { }); - await Task.Delay(4500).ConfigureAwait(false); + await Task.Delay(20000).ConfigureAwait(false); if (lockConfig?.ObjectLockEnabled.Equals(ObjectLockConfiguration.LockEnabled, StringComparison.OrdinalIgnoreCase) == true) { @@ -806,8 +805,8 @@ internal static async Task PutGetStatEncryptedObject_Test2(IMinioClient minio) aesEncryption.KeySize = 256; aesEncryption.GenerateKey(); var ssec = new SSEC(aesEncryption.Key); - - using (var filestream = rsg.GenerateStreamFromSeed(6 * MB)) + Stream filestream; + await using ((filestream = rsg.GenerateStreamFromSeed(6 * MB)).ConfigureAwait(false)) { var file_write_size = filestream.Length; @@ -827,16 +826,18 @@ internal static async Task PutGetStatEncryptedObject_Test2(IMinioClient minio) .WithServerSideEncryption(ssec) .WithCallbackStream(async (stream, cancellationToken) => { - using var fileStream = File.Create(tempFileName); - - await stream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); - await fileStream.DisposeAsync().ConfigureAwait(false); + Stream fileStream; + await using ((fileStream = File.Create(tempFileName)).ConfigureAwait(false)) + { + await stream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); + await fileStream.DisposeAsync().ConfigureAwait(false); - var writtenInfo = new FileInfo(tempFileName); - file_read_size = writtenInfo.Length; + var writtenInfo = new FileInfo(tempFileName); + file_read_size = writtenInfo.Length; - Assert.AreEqual(file_write_size, file_read_size); - File.Delete(tempFileName); + Assert.AreEqual(file_write_size, file_read_size); + File.Delete(tempFileName); + } }); var statObjectArgs = new StatObjectArgs() .WithBucket(bucketName) @@ -849,24 +850,26 @@ internal static async Task PutGetStatEncryptedObject_Test2(IMinioClient minio) new MintLogger("PutGetStatEncryptedObject_Test2", putObjectSignature, "Tests whether Put/Get/Stat multipart upload with encryption passes", TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + File.Delete(tempFileName); + await TearDown(minio, bucketName).ConfigureAwait(false); } catch (NotImplementedException ex) { new MintLogger("PutGetStatEncryptedObject_Test2", putObjectSignature, "Tests whether Put/Get/Stat multipart upload with encryption passes", TestStatus.NA, DateTime.Now - startTime, "", ex.Message, ex.ToString(), args).Log(); + File.Delete(tempFileName); + await TearDown(minio, bucketName).ConfigureAwait(false); + throw; } catch (Exception ex) { new MintLogger("PutGetStatEncryptedObject_Test2", putObjectSignature, "Tests whether Put/Get/Stat multipart upload with encryption passes", TestStatus.FAIL, DateTime.Now - startTime, "", ex.Message, ex.ToString(), args).Log(); - throw; - } - finally - { File.Delete(tempFileName); await TearDown(minio, bucketName).ConfigureAwait(false); + throw; } } @@ -2702,7 +2705,7 @@ internal static async Task ListenBucketNotificationsAsync_Test1(IMinioClient min Exception ex = new UnexpectedMinioException(err.Message); if (string.Equals(err.Code, "NotImplemented", StringComparison.OrdinalIgnoreCase)) ex = new NotImplementedException(err.Message); - + await TearDown(minio, bucketName).ConfigureAwait(false); throw ex; } @@ -2731,6 +2734,7 @@ internal static async Task ListenBucketNotificationsAsync_Test1(IMinioClient min listenBucketNotificationsSignature, "Tests whether ListenBucketNotifications passes for small object", TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + await TearDown(minio, bucketName).ConfigureAwait(false); } catch (NotImplementedException ex) { @@ -2739,6 +2743,7 @@ internal static async Task ListenBucketNotificationsAsync_Test1(IMinioClient min "Tests whether ListenBucketNotifications passes for small object", TestStatus.NA, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); + await TearDown(minio, bucketName).ConfigureAwait(false); } catch (Exception ex) { @@ -2770,13 +2775,10 @@ static bool isAWS(string endPoint) "Tests whether ListenBucketNotifications passes for small object", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); + await TearDown(minio, bucketName).ConfigureAwait(false); throw; } } - finally - { - await TearDown(minio, bucketName).ConfigureAwait(false); - } } internal static async Task ListenBucketNotificationsAsync_Test2(IMinioClient minio) @@ -4873,18 +4875,20 @@ internal static async Task GetObject_3_OffsetLength_Tests(IMinioClient minio) new MintLogger(testName, getObjectSignature, "Tests whether GetObject returns all the data", TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); + + if (File.Exists(tempFileName)) File.Delete(tempFileName); + if (File.Exists(tempSource)) File.Delete(tempSource); + await TearDown(minio, bucketName).ConfigureAwait(false); } catch (Exception ex) { new MintLogger(testName, getObjectSignature, "Tests whether GetObject returns all the data", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); - throw; - } - finally - { + if (File.Exists(tempFileName)) File.Delete(tempFileName); if (File.Exists(tempSource)) File.Delete(tempSource); await TearDown(minio, bucketName).ConfigureAwait(false); + throw; } } } @@ -5389,7 +5393,7 @@ internal static async Task ListObjects_Test(IMinioClient minio, string bucketNam () => { }); } - await Task.Delay(5000).ConfigureAwait(false); + await Task.Delay(40000).ConfigureAwait(false); Assert.AreEqual(numObjects, count); } @@ -6046,21 +6050,23 @@ internal static async Task BucketLifecycleAsync_Test1(IMinioClient minio) catch (Exception ex) { await TearDown(minio, bucketName).ConfigureAwait(false); - new MintLogger(nameof(BucketLifecycleAsync_Test1), setBucketLifecycleSignature, + new MintLogger(nameof(BucketLifecycleAsync_Test1) + ".0", setBucketLifecycleSignature, "Tests whether SetBucketLifecycleAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); throw; } var rules = new List(); - var exp = new Expiration(DateTime.Now.AddYears(1)); - var compareDate = new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, 0, 0, 0); - var expInDays = (compareDate.AddYears(1) - compareDate).TotalDays; + var baseDate = DateTime.Now; + var expDate = baseDate.AddYears(1); + var exp = new Expiration(expDate); - var rule1 = new LifecycleRule(null, "txt", exp, null, - new RuleFilter(null, "txt/", null), - null, null, LifecycleRule.LifecycleRuleStatusEnabled - ); + var calcDateTime = DateTime.Parse(exp.ExpiryDate, null, DateTimeStyles.RoundtripKind); + var expInDays = (calcDateTime.Date - baseDate.ToUniversalTime().Date).TotalDays; + + var rule1 = new LifecycleRule(null, "txt", exp, + null, new RuleFilter(null, "txt/", null), null, + null, LifecycleRule.LifecycleRuleStatusEnabled); rules.Add(rule1); var lfc = new LifecycleConfiguration(rules); try @@ -6082,7 +6088,6 @@ internal static async Task BucketLifecycleAsync_Test1(IMinioClient minio) } catch (Exception ex) { - await TearDown(minio, bucketName).ConfigureAwait(false); new MintLogger(nameof(BucketLifecycleAsync_Test1) + ".1", setBucketLifecycleSignature, "Tests whether SetBucketLifecycleAsync passes", TestStatus.FAIL, DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log(); @@ -6098,8 +6103,10 @@ internal static async Task BucketLifecycleAsync_Test1(IMinioClient minio) Assert.IsNotNull(lfcObj.Rules); Assert.IsTrue(lfcObj.Rules.Count > 0); Assert.AreEqual(lfcObj.Rules.Count, lfc.Rules.Count); - var lfcDate = DateTime.Parse(lfcObj.Rules[0].Expiration.Date, null, DateTimeStyles.RoundtripKind); - Assert.AreEqual(Math.Floor((lfcDate - compareDate).TotalDays), expInDays); + var lfcDate = DateTime.Parse(lfcObj.Rules[0].Expiration.ExpiryDate, null, DateTimeStyles.RoundtripKind); + var lfcExpInDays = (lfcDate.Date - baseDate.ToUniversalTime().Date).TotalDays; + Assert.AreEqual(lfcExpInDays, expInDays); + new MintLogger(nameof(BucketLifecycleAsync_Test1) + ".2", getBucketLifecycleSignature, "Tests whether GetBucketLifecycleAsync passes", TestStatus.PASS, DateTime.Now - startTime, args: args) diff --git a/Minio/ApiEndpoints/BucketOperations.cs b/Minio/ApiEndpoints/BucketOperations.cs index ba91c2614..8756a5968 100644 --- a/Minio/ApiEndpoints/BucketOperations.cs +++ b/Minio/ApiEndpoints/BucketOperations.cs @@ -76,13 +76,16 @@ public async Task BucketExistsAsync(BucketExistsArgs args, CancellationTok var requestMessageBuilder = await this.CreateRequest(args).ConfigureAwait(false); using var response = await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, - cancellationToken: cancellationToken) - .ConfigureAwait(false); + cancellationToken: cancellationToken).ConfigureAwait(false); + if (response.Exception is not null && + response.Exception.GetType() == typeof(BucketNotFoundException)) + return false; } catch (InternalClientException ice) { - if ((ice.ServerResponse is not null && HttpStatusCode.NotFound.Equals(ice.ServerResponse.StatusCode)) - || ice.ServerResponse is null) + if ((ice.ServerResponse is not null && + HttpStatusCode.NotFound.Equals(ice.ServerResponse.StatusCode)) || + ice.ServerResponse is null) return false; } catch (Exception ex) diff --git a/Minio/DataModel/ILM/Duration.cs b/Minio/DataModel/ILM/Duration.cs index 87afe0870..f76a60e02 100644 --- a/Minio/DataModel/ILM/Duration.cs +++ b/Minio/DataModel/ILM/Duration.cs @@ -15,7 +15,6 @@ */ using System.Xml.Serialization; -using Minio.Helper; namespace Minio.DataModel.ILM; @@ -24,14 +23,14 @@ public abstract class Duration { protected Duration() { - Date = null; + ExpiryDate = null; Days = null; } protected Duration(DateTime date) { - date = new DateTime(date.Year, date.Month, date.Day, 0, 0, 0); - Date = Utils.To8601String(date); + ExpiryDate = date.ToUniversalTime().Date.ToString("o") + ?? date.AddDays(1).AddSeconds(-1).ToUniversalTime().Date.ToString("o"); } protected Duration(double days) @@ -40,7 +39,7 @@ protected Duration(double days) } [XmlElement(ElementName = "Date", IsNullable = true)] - public string Date { get; set; } + public string ExpiryDate { get; set; } [XmlElement(ElementName = "Days", IsNullable = true)] public double? Days { get; set; } diff --git a/Minio/DataModel/ILM/LifecycleConfiguration.cs b/Minio/DataModel/ILM/LifecycleConfiguration.cs index 63abac013..ccbce688b 100644 --- a/Minio/DataModel/ILM/LifecycleConfiguration.cs +++ b/Minio/DataModel/ILM/LifecycleConfiguration.cs @@ -39,7 +39,7 @@ public LifecycleConfiguration() public LifecycleConfiguration(IList rules) { - if (rules is null || rules.Count <= 0) + if (rules is null || rules.Count == 0) throw new ArgumentNullException(nameof(rules), "Rules object cannot be empty. A finite set of Lifecycle Rules are needed for LifecycleConfiguration."); @@ -73,7 +73,7 @@ public string MarshalXML() catch (Exception ex) { Console.WriteLine(ex.ToString()); - // throw ex; + throw; } finally { diff --git a/Minio/DataModel/Result/ResponseResult.cs b/Minio/DataModel/Result/ResponseResult.cs index 4464f3206..0f0791d99 100644 --- a/Minio/DataModel/Result/ResponseResult.cs +++ b/Minio/DataModel/Result/ResponseResult.cs @@ -40,7 +40,7 @@ public ResponseResult(HttpRequestMessage request, Exception exception) Exception = exception; } - private Exception Exception { get; } + public Exception Exception { get; set; } public HttpRequestMessage Request { get; } public HttpResponseMessage Response { get; } diff --git a/Minio/Helper/OperationsHelper.cs b/Minio/Helper/OperationsHelper.cs index 544539546..81ac99369 100644 --- a/Minio/Helper/OperationsHelper.cs +++ b/Minio/Helper/OperationsHelper.cs @@ -110,8 +110,7 @@ private async Task GetObjectStreamAsync(GetObjectArgs args, CancellationToken ca var requestMessageBuilder = await this.CreateRequest(args).ConfigureAwait(false); using var response = await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, - cancellationToken: cancellationToken) - .ConfigureAwait(false); + cancellationToken: cancellationToken).ConfigureAwait(false); } /// diff --git a/Minio/MinioClientExtensions.cs b/Minio/MinioClientExtensions.cs index d6a1a28bc..b6e0f45c3 100644 --- a/Minio/MinioClientExtensions.cs +++ b/Minio/MinioClientExtensions.cs @@ -50,7 +50,7 @@ public static IMinioClient WithEndpoint(this IMinioClient minioClient, string en { if (minioClient is null) throw new ArgumentNullException(nameof(minioClient)); - minioClient.Config.BaseUrl = endpoint; + minioClient.Config.Endpoint = endpoint; minioClient.SetBaseURL(GetBaseUrl(endpoint)); return minioClient; } @@ -63,7 +63,10 @@ public static IMinioClient WithEndpoint(this IMinioClient minioClient, string en throw new ArgumentException( string.Format(CultureInfo.InvariantCulture, "Port {0} is not a number between 1 and 65535", port), nameof(port)); - return minioClient.WithEndpoint(endpoint + ":" + port); + endpoint = endpoint + ":" + port.ToString(CultureInfo.InvariantCulture); + minioClient.Config.Endpoint = endpoint; + minioClient.SetBaseURL(GetBaseUrl(endpoint)); + return minioClient; } public static IMinioClient WithEndpoint(this IMinioClient minioClient, Uri url) @@ -71,8 +74,10 @@ public static IMinioClient WithEndpoint(this IMinioClient minioClient, Uri url) if (minioClient is null) throw new ArgumentNullException(nameof(minioClient)); if (url is null) throw new ArgumentNullException(nameof(url)); + minioClient.SetBaseURL(url); + minioClient.Config.Endpoint = url.AbsoluteUri; - return minioClient.WithEndpoint(url.AbsoluteUri); + return minioClient; } public static IMinioClient WithRegion(this IMinioClient minioClient, string region) @@ -92,7 +97,7 @@ public static IMinioClient WithRegion(this IMinioClient minioClient) { if (minioClient is null) throw new ArgumentNullException(nameof(minioClient)); // Set region to its default value if empty or null - minioClient.Config.Region = "us-east-1"; + minioClient.Config.Region ??= "us-east-1"; return minioClient; } @@ -120,15 +125,7 @@ public static IMinioClient WithSessionToken(this IMinioClient minioClient, strin public static IMinioClient WithSSL(this IMinioClient minioClient, bool secure = true) { if (minioClient is null) throw new ArgumentNullException(nameof(minioClient)); - - if (secure) - { - minioClient.Config.Secure = true; - if (string.IsNullOrEmpty(minioClient.Config.BaseUrl)) - return minioClient; - //var secureUrl = RequestUtil.MakeTargetURL(minioClient.BaseUrl, minioClient.Secure); - } - + minioClient.Config.Secure = secure; return minioClient; } diff --git a/Minio/RequestExtensions.cs b/Minio/RequestExtensions.cs index ee9f1f89d..827d0d44b 100644 --- a/Minio/RequestExtensions.cs +++ b/Minio/RequestExtensions.cs @@ -1,4 +1,5 @@ using System.Diagnostics.CodeAnalysis; +using System.Net; using Minio.Credentials; using Minio.DataModel; using Minio.DataModel.Args; @@ -47,18 +48,29 @@ internal static Task ExecuteTaskAsync(this IMinioClient minioCli bool isSts = false, CancellationToken cancellationToken = default) { - if (minioClient.Config.RequestTimeout > 0) + Task responseResult; + try + { + if (minioClient.Config.RequestTimeout > 0) + { + using var internalTokenSource = + new CancellationTokenSource(new TimeSpan(0, 0, 0, 0, minioClient.Config.RequestTimeout)); + using var timeoutTokenSource = + CancellationTokenSource.CreateLinkedTokenSource(internalTokenSource.Token, cancellationToken); + cancellationToken = timeoutTokenSource.Token; + } + + responseResult = minioClient.ExecuteWithRetry( + async () => await minioClient.ExecuteTaskCoreAsync(errorHandlers, requestMessageBuilder, + isSts, cancellationToken).ConfigureAwait(false)); + } + catch (Exception ex) { - using var internalTokenSource = - new CancellationTokenSource(new TimeSpan(0, 0, 0, 0, minioClient.Config.RequestTimeout)); - using var timeoutTokenSource = - CancellationTokenSource.CreateLinkedTokenSource(internalTokenSource.Token, cancellationToken); - cancellationToken = timeoutTokenSource.Token; + Console.WriteLine($"\n\n *** ExecuteTaskAsync::Threw an exception => {ex.Message}"); + throw; } - return minioClient.ExecuteWithRetry( - () => minioClient.ExecuteTaskCoreAsync(errorHandlers, requestMessageBuilder, - isSts, cancellationToken)); + return responseResult; } private static async Task ExecuteTaskCoreAsync(this IMinioClient minioClient, @@ -101,6 +113,24 @@ await requestMessageBuilder.FunctionResponseWriter(responseResult.ContentStream, responseResult = new ResponseResult(request, e); } + if (responseResult.StatusCode == HttpStatusCode.NotFound) + { + if (request.Method == HttpMethod.Head) + { + Exception ex = new BucketNotFoundException(); + responseResult.Exception = ex; + return responseResult; + } + + if (request.RequestUri.ToString().Contains("lock", StringComparison.OrdinalIgnoreCase) && + request.Method == HttpMethod.Get) + { + Exception ex = new MissingObjectLockConfigurationException(); + responseResult.Exception = ex; + return responseResult; + } + } + minioClient.HandleIfErrorResponse(responseResult, errorHandlers, startTime); return responseResult; }