From 75a16d0537b8bc762bc3f0a4206eea6c40daf1d4 Mon Sep 17 00:00:00 2001 From: vugar Date: Thu, 15 Aug 2024 22:15:50 +0400 Subject: [PATCH 1/3] fixedpresignurl --- Minio.Tests/AuthenticatorTest.cs | 30 ++- Minio.Tests/OperationsTest.cs | 2 +- Minio/ApiEndpoints/ObjectOperations.cs | 22 +- Minio/Helper/Constants.cs | 11 + Minio/Helper/S3utils.cs | 13 +- Minio/Helper/Utils.cs | 125 ++++++++-- Minio/HttpRequestMessageBuilder.cs | 5 +- Minio/RequestExtensions.cs | 52 ++-- Minio/V4Authenticator.cs | 319 +++++++++++++------------ 9 files changed, 356 insertions(+), 223 deletions(-) diff --git a/Minio.Tests/AuthenticatorTest.cs b/Minio.Tests/AuthenticatorTest.cs index 3e6e93641..6c12bfc8c 100644 --- a/Minio.Tests/AuthenticatorTest.cs +++ b/Minio.Tests/AuthenticatorTest.cs @@ -126,17 +126,17 @@ public void GetPresignCanonicalRequestTest() var authenticator = new V4Authenticator(false, "my-access-key", "my-secret-key"); var request = new Uri( - "http://localhost:9001/bucket/object-name?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20200501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200501T154533Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host"); + "https://localhost:9000/bucket/object-name?X-Amz-Expires=43200&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"); var headersToSign = new SortedDictionary(StringComparer.Ordinal) { - { "X-Special".ToLowerInvariant(), "special" }, { "Content-Language".ToLowerInvariant(), "en" } + { "X-Special".ToLowerInvariant(), "special" }, { "Content-Language".ToLowerInvariant(), "en" }, + { "host","localhost:9000"} }; - var canonicalRequest = authenticator.GetPresignCanonicalRequest(HttpMethod.Put, request, headersToSign); - Assert.AreEqual( - string.Join('\n', "PUT", "/bucket/object-name", - "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20200501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200501T154533Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&content-language=en&x-special=special", - "host:localhost:9001", "", "host", "UNSIGNED-PAYLOAD"), + var canonicalQueryString = "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"; + + var canonicalRequest = authenticator.GetPresignCanonicalRequest(HttpMethod.Put, request, headersToSign, canonicalQueryString); + Assert.AreEqual("PUT\n/bucket/object-name\nX-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special\ncontent-language:en\nhost:localhost:9000\nx-special:special\n\ncontent-language;host;x-special\nUNSIGNED-PAYLOAD", canonicalRequest); } @@ -146,17 +146,16 @@ public void GetPresignCanonicalRequestWithParametersTest() var authenticator = new V4Authenticator(false, "my-access-key", "my-secret-key"); var request = new Uri( - "http://localhost:9001/bucket/object-name?uploadId=upload-id&partNumber=1&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20200501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200501T154533Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host"); + "https://localhost:9000/bucket/object-name?X-Amz-Expires=43200&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"); var headersToSign = new SortedDictionary(StringComparer.Ordinal) { - { "X-Special".ToLowerInvariant(), "special" }, { "Content-Language".ToLowerInvariant(), "en" } + { "X-Special".ToLowerInvariant(), "special" }, { "Content-Language".ToLowerInvariant(), "en" },{ "host","localhost:9000"} }; - var canonicalRequest = authenticator.GetPresignCanonicalRequest(HttpMethod.Put, request, headersToSign); - Assert.AreEqual( - string.Join('\n', "PUT", "/bucket/object-name", - "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20200501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200501T154533Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&content-language=en&partNumber=1&uploadId=upload-id&x-special=special", - "host:localhost:9001", "", "host", "UNSIGNED-PAYLOAD"), + var canonicalQueryString = "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"; + + var canonicalRequest = authenticator.GetPresignCanonicalRequest(HttpMethod.Put, request, headersToSign, canonicalQueryString); + Assert.AreEqual("PUT\n/bucket/object-name\nX-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special\ncontent-language:en\nhost:localhost:9000\nx-special:special\n\ncontent-language;host;x-special\nUNSIGNED-PAYLOAD", canonicalRequest); } @@ -164,8 +163,7 @@ private Tuple GetHeaderKV(HttpRequestMessageBuilder request, str { var key = request.HeaderParameters.Keys.FirstOrDefault(o => string.Equals(o, headername, StringComparison.OrdinalIgnoreCase)); - if (key is not null) return Tuple.Create(key, request.HeaderParameters[key]); - return null; + return key is not null ? Tuple.Create(key, request.HeaderParameters[key]) : null; } private bool HasPayloadHeader(HttpRequestMessageBuilder request, string headerName) diff --git a/Minio.Tests/OperationsTest.cs b/Minio.Tests/OperationsTest.cs index cc825de50..c137d7a50 100644 --- a/Minio.Tests/OperationsTest.cs +++ b/Minio.Tests/OperationsTest.cs @@ -132,7 +132,7 @@ public async Task PresignedGetObjectWithHeaders() var signedUrl = await client.PresignedGetObjectAsync(presignedGetObjectArgs).ConfigureAwait(false); Assert.AreEqual( - "https://play.min.io/bucket/object-name?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=Q3AM3UQ867SPQQA43P2F%2F20200501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200501T154533Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&response-content-disposition=attachment%3B%20filename%3D%22filename.jpg%22&X-Amz-Signature=de66f04dd4ac35838b9e83d669f7b5a70b452c6468e2b4a9e9c29f42e7fa102d", + "https://play.min.io/bucket/object-name?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=Q3AM3UQ867SPQQA43P2F%2F20200501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200501T154533Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host%3Bresponse-content-disposition&X-Amz-Signature=44227f1a4c7054e88c3e4866b8827fdd75d2ba0d575e68b53b71b68fc07cbfe3", signedUrl); } } diff --git a/Minio/ApiEndpoints/ObjectOperations.cs b/Minio/ApiEndpoints/ObjectOperations.cs index e523a7c2d..7413490c4 100644 --- a/Minio/ApiEndpoints/ObjectOperations.cs +++ b/Minio/ApiEndpoints/ObjectOperations.cs @@ -149,6 +149,7 @@ public async Task RemoveIncompleteUploadAsync(RemoveIncompleteUploadArgs args, { await foreach (var upload in ListIncompleteUploadsEnumAsync(listUploadArgs, cancellationToken) .ConfigureAwait(false)) + { if (upload.Key.Equals(args.ObjectName, StringComparison.OrdinalIgnoreCase)) { var rmArgs = new RemoveUploadArgs() @@ -157,6 +158,7 @@ public async Task RemoveIncompleteUploadAsync(RemoveIncompleteUploadArgs args, .WithUploadId(upload.UploadId); await RemoveUploadAsync(rmArgs, cancellationToken).ConfigureAwait(false); } + } } catch (Exception ex) when (ex.GetType() == typeof(BucketNotFoundException)) { @@ -266,8 +268,7 @@ public async Task PresignedPutObjectAsync(PresignedPutObjectArgs args) args?.Validate(); var requestMessageBuilder = await this.CreateRequest(HttpMethod.Put, args.BucketName, args.ObjectName, - args.Headers, // contentType - Convert.ToString(args.GetType(), CultureInfo.InvariantCulture), // metaData + args.Headers, Utils.ObjectToByteArray(args.RequestBody)).ConfigureAwait(false); var authenticator = new V4Authenticator(Config.Secure, Config.AccessKey, Config.SecretKey, Config.Region, Config.SessionToken); @@ -402,7 +403,7 @@ public async Task> RemoveObjectsAsync(RemoveObjectsArgs args, CancellationToken cancellationToken = default) { args?.Validate(); - IList errs = new List(); + IList errs = []; errs = args.ObjectNamesVersions.Count > 0 ? await RemoveObjectVersionsHelper(args, errs.ToList(), cancellationToken).ConfigureAwait(false) : await RemoveObjectsHelper(args, errs, cancellationToken).ConfigureAwait(false); @@ -579,8 +580,10 @@ public async Task PutObjectAsync(PutObjectArgs args, var bytes = await ReadFullAsync(args.ObjectStreamData, (int)args.ObjectSize).ConfigureAwait(false); var bytesRead = bytes.Length; if (bytesRead != (int)args.ObjectSize) + { throw new UnexpectedShortReadException( $"Data read {bytesRead.ToString(CultureInfo.InvariantCulture)} is shorter than the size {args.ObjectSize.ToString(CultureInfo.InvariantCulture)} of input buffer."); + } args = args.WithRequestBody(bytes) .WithStreamData(null) @@ -690,12 +693,13 @@ public async Task CopyObjectAsync(CopyObjectArgs args, CancellationToken cancell (srcByteRangeSize > 0 && args.SourceObject.CopyOperationConditions.byteRangeEnd >= args.SourceObjectInfo.Size)) + { throw new InvalidDataException($"Specified byte range ({args.SourceObject .CopyOperationConditions .byteRangeStart.ToString(CultureInfo.InvariantCulture)}-{args.SourceObject - .CopyOperationConditions.byteRangeEnd.ToString(CultureInfo.InvariantCulture) - }) does not fit within source object (size={args.SourceObjectInfo.Size + .CopyOperationConditions.byteRangeEnd.ToString(CultureInfo.InvariantCulture)}) does not fit within source object (size={args.SourceObjectInfo.Size .ToString(CultureInfo.InvariantCulture)})"); + } if (copySize > Constants.MaxSingleCopyObjectSize || (srcByteRangeSize > 0 && @@ -913,7 +917,9 @@ private async Task> PutObjectPartAsync(PutObjectPartArg numPartsUploaded++; totalParts[partNumber - 1] = new Part { - PartNumber = partNumber, ETag = etag, Size = (long)expectedReadSize + PartNumber = partNumber, + ETag = etag, + Size = (long)expectedReadSize }; etags[partNumber] = etag; if (!dataToCopy.IsEmpty) progressReport.TotalBytesTransferred += dataToCopy.Length; @@ -1006,7 +1012,9 @@ private async Task MultipartCopyUploadAsync(MultipartCopyUploadArgs args, totalParts[partNumber - 1] = new Part { - PartNumber = partNumber, ETag = cpPartResult.ETag, Size = (long)expectedReadSize + PartNumber = partNumber, + ETag = cpPartResult.ETag, + Size = (long)expectedReadSize }; } diff --git a/Minio/Helper/Constants.cs b/Minio/Helper/Constants.cs index ef3ca1697..52fb35e34 100644 --- a/Minio/Helper/Constants.cs +++ b/Minio/Helper/Constants.cs @@ -84,4 +84,15 @@ internal static class Constants /// SSEKMSContext is the AWS SSE KMS Context. /// public static string SSEKMSContext = "X-Amz-Server-Side-Encryption-Context"; + + public const string XAmzAlgorithm = "X-Amz-Algorithm"; + public const string XAmzExpires = "X-Amz-Expires"; + public const string XAmzCredential = "X-Amz-Credential"; + public const string XAmzDate = "X-Amz-Date"; + public const string XAmzSignedHeaders = "X-Amz-SignedHeaders"; + public const string XAmzSignature = "X-Amz-Signature"; + + public const string DateTimeISO8601Format = "yyyyMMddTHHmmssZ"; + public const string DateISO8601Format = "yyyyMMdd"; + } diff --git a/Minio/Helper/S3utils.cs b/Minio/Helper/S3utils.cs index eaae70026..03e07527d 100644 --- a/Minio/Helper/S3utils.cs +++ b/Minio/Helper/S3utils.cs @@ -51,7 +51,10 @@ internal static bool IsVirtualHostSupported(Uri endpointURL, string bucketName) // bucketName can be valid but '.' in the hostname will fail SSL // certificate validation. So do not use host-style for such buckets. if (string.Equals(endpointURL.Scheme, "https", StringComparison.OrdinalIgnoreCase) && - bucketName.Contains('.', StringComparison.Ordinal)) return false; + bucketName.Contains('.', StringComparison.Ordinal)) + { + return false; + } // Return true for all other cases return IsAmazonEndPoint(endpointURL.Host); } @@ -80,17 +83,13 @@ internal static bool IsValidIP(string ip) if (string.IsNullOrEmpty(ip)) return false; var splitValues = ip.Split('.'); - if (splitValues.Length != 4) return false; - - return splitValues.All(r => byte.TryParse(r, out var _)); + return splitValues.Length == 4 && splitValues.All(r => byte.TryParse(r, out var _)); } // TrimAll trims leading and trailing spaces and replace sequential spaces with one space, following Trimall() // in http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html internal static string TrimAll(string s) { - if (string.IsNullOrEmpty(s)) - return s; - return TrimWhitespaceRegex.Replace(s, " ").Trim(); + return string.IsNullOrEmpty(s) ? s : TrimWhitespaceRegex.Replace(s, " ").Trim(); } } diff --git a/Minio/Helper/Utils.cs b/Minio/Helper/Utils.cs index 4076342f1..d7353bace 100644 --- a/Minio/Helper/Utils.cs +++ b/Minio/Helper/Utils.cs @@ -83,8 +83,66 @@ internal static void ValidateObjectName(string objectName) internal static void ValidateObjectPrefix(string objectPrefix) { if (objectPrefix.Length > 512) + { throw new InvalidObjectPrefixException(objectPrefix, "Object prefix cannot be greater than 1024 characters."); + } + } + + /// + /// Compute sha256 checksum. + /// + /// Bytes body + /// Bytes of sha256 checksum + internal static ReadOnlySpan ComputeSha256(ReadOnlySpan body) + { +#if NETSTANDARD + using var sha = SHA256.Create(); + ReadOnlySpan hash + = sha.ComputeHash(body.ToArray()); +#else + ReadOnlySpan hash = SHA256.HashData(body); +#endif + return hash; + } + + /// + /// Computes sha256 checksum by converting the body string to bytes using UTF-8 encoding + /// and then calls the ComputeSha256 method that takes a ReadOnlySpan parameter. + /// + /// Bytes body + /// Bytes of sha256 checksum + internal static ReadOnlySpan ComputeSha256(string body) + { + return ComputeSha256(Encoding.UTF8.GetBytes(body)); + } + + /// + /// Convert bytes to hexadecimal string. + /// + /// Bytes of any checksum + /// Hexlified string of input bytes + internal static string BytesToHex(ReadOnlySpan checkSum) + { + return BitConverter.ToString(checkSum.ToArray()).Replace("-", string.Empty, StringComparison.OrdinalIgnoreCase) + .ToLowerInvariant(); + } + + /// + /// Compute hmac of input content with key. + /// + /// Hmac key + /// Bytes to be hmac computed + /// Computed hmac of input content + internal static ReadOnlySpan SignHmac(ReadOnlySpan key, ReadOnlySpan content) + { +#if NETSTANDARD + using var hmac = new HMACSHA256(key.ToArray()); + hmac.Initialize(); + return hmac.ComputeHash(content.ToArray()); +#else + return HMACSHA256.HashData(key, content); +#endif } // Return url encoded string where reserved characters have been percent-encoded @@ -126,17 +184,41 @@ internal static string EncodePath(string path) { var encodedPathBuf = new StringBuilder(); foreach (var pathSegment in path.Split('/')) + { if (pathSegment.Length != 0) { if (encodedPathBuf.Length > 0) _ = encodedPathBuf.Append('/'); _ = encodedPathBuf.Append(UrlEncode(pathSegment)); } + } if (path.StartsWith("/", StringComparison.OrdinalIgnoreCase)) _ = encodedPathBuf.Insert(0, '/'); if (path.EndsWith("/", StringComparison.OrdinalIgnoreCase)) _ = encodedPathBuf.Append('/'); return encodedPathBuf.ToString(); } + /// + /// Formats date to ISO8601 format. + /// + /// Date to be formatted + /// Formatted date.yyyyMMdd + internal static string FormatDate(DateTime date) + { + return date.ToUniversalTime() + .ToString(Constants.DateISO8601Format, CultureInfo.InvariantCulture); + } + + /// + /// Formats datetime to ISO8601 format. + /// + /// Date to be formatted + /// Formatted date. yyyyMMddTHHmmssZ + internal static string FormatDateTime(DateTime date) + { + return date.ToUniversalTime() + .ToString(Constants.DateTimeISO8601Format, CultureInfo.InvariantCulture); + } + internal static bool IsAnonymousClient(string accessKey, string secretKey) { return string.IsNullOrEmpty(secretKey) && string.IsNullOrEmpty(accessKey); @@ -168,9 +250,9 @@ internal static string GetContentType(string fileName) { } - if (string.IsNullOrEmpty(extension)) return "application/octet-stream"; - - return contentTypeMap.Value.TryGetValue(extension, out var contentType) + return string.IsNullOrEmpty(extension) + ? "application/octet-stream" + : contentTypeMap.Value.TryGetValue(extension, out var contentType) ? contentType : "application/octet-stream"; } @@ -193,9 +275,7 @@ internal static bool IsSupersetOf(IList l1, IList l2) { if (l2 is null) return true; - if (l1 is null) return false; - - return !l2.Except(l1, StringComparer.Ordinal).Any(); + return l1 is not null && !l2.Except(l1, StringComparer.Ordinal).Any(); } public static async Task ForEachAsync(this IEnumerable source, bool runInParallel = false, @@ -246,10 +326,9 @@ await Task.WhenAll(Partitioner.Create(source).GetPartitions(maxNoOfParallelProce public static bool CaseInsensitiveContains(string text, string value, StringComparison stringComparison = StringComparison.CurrentCultureIgnoreCase) { - if (string.IsNullOrEmpty(text)) - throw new ArgumentException($"'{nameof(text)}' cannot be null or empty.", nameof(text)); - - return text.Contains(value, stringComparison); + return string.IsNullOrEmpty(text) + ? throw new ArgumentException($"'{nameof(text)}' cannot be null or empty.", nameof(text)) + : text.Contains(value, stringComparison); } /// @@ -263,8 +342,10 @@ public static MultiPartInfo CalculateMultiPartSize(long size, bool copy = false) if (size == -1) size = Constants.MaximumStreamObjectSize; if (size > Constants.MaxMultipartPutObjectSize) + { throw new EntityTooLargeException( $"Your proposed upload size {size} exceeds the maximum allowed object size {Constants.MaxMultipartPutObjectSize}"); + } var partSize = (double)Math.Ceiling((decimal)size / Constants.MaxParts); var minPartSize = copy ? Constants.MinimumCOPYPartSize : Constants.MinimumPUTPartSize; @@ -923,35 +1004,41 @@ public static DateTime From8601String(string dt) public static Uri GetBaseUrl(string endpoint) { if (string.IsNullOrEmpty(endpoint)) + { throw new ArgumentException( string.Format(CultureInfo.InvariantCulture, "{0} is the value of the endpoint. It can't be null or empty.", endpoint), nameof(endpoint)); + } if (endpoint.EndsWith("/", StringComparison.OrdinalIgnoreCase)) endpoint = endpoint[..^1]; if (!endpoint.StartsWith("http", StringComparison.OrdinalIgnoreCase) && !BuilderUtil.IsValidHostnameOrIPAddress(endpoint)) + { throw new InvalidEndpointException( string.Format(CultureInfo.InvariantCulture, "{0} is invalid hostname.", endpoint), "endpoint"); + } + string conn_url; if (endpoint.StartsWith("http", StringComparison.OrdinalIgnoreCase)) + { throw new InvalidEndpointException( string.Format(CultureInfo.InvariantCulture, "{0} the value of the endpoint has the scheme (http/https) in it.", endpoint), "endpoint"); + } var enable_https = Environment.GetEnvironmentVariable("ENABLE_HTTPS"); var scheme = enable_https?.Equals("1", StringComparison.OrdinalIgnoreCase) == true ? "https://" : "http://"; conn_url = scheme + endpoint; var url = new Uri(conn_url); var hostnameOfUri = url.Authority; - if (!string.IsNullOrWhiteSpace(hostnameOfUri) && !BuilderUtil.IsValidHostnameOrIPAddress(hostnameOfUri)) - throw new InvalidEndpointException( + return !string.IsNullOrWhiteSpace(hostnameOfUri) && !BuilderUtil.IsValidHostnameOrIPAddress(hostnameOfUri) + ? throw new InvalidEndpointException( string.Format(CultureInfo.InvariantCulture, "{0}, {1} is invalid hostname.", endpoint, hostnameOfUri), - "endpoint"); - - return url; + "endpoint") + : url; } internal static HttpRequestMessageBuilder GetEmptyRestRequest(HttpRequestMessageBuilder requestBuilder) @@ -1008,17 +1095,19 @@ public static void Print(object obj) public static void PrintDict(IDictionary d) { if (d is not null) + { foreach (var kv in d) Console.WriteLine("DEBUG >> Dictionary({0} => {1})", kv.Key, kv.Value); + } Console.WriteLine("DEBUG >> Dictionary: Done printing\n"); } public static string DetermineNamespace(XDocument document) { - if (document is null) throw new ArgumentNullException(nameof(document)); - - return document.Root.Attributes().FirstOrDefault(attr => attr.IsNamespaceDeclaration)?.Value ?? string.Empty; + return document is null + ? throw new ArgumentNullException(nameof(document)) + : document.Root.Attributes().FirstOrDefault(attr => attr.IsNamespaceDeclaration)?.Value ?? string.Empty; } public static string SerializeToXml(T anyobject) where T : class diff --git a/Minio/HttpRequestMessageBuilder.cs b/Minio/HttpRequestMessageBuilder.cs index ebdfe0fc5..865337bf7 100644 --- a/Minio/HttpRequestMessageBuilder.cs +++ b/Minio/HttpRequestMessageBuilder.cs @@ -128,14 +128,17 @@ public HttpRequestMessage Request public ReadOnlyMemory Content { get; private set; } - public string ContentTypeKey => "Content-Type"; + public const string ContentTypeKey = "Content-Type"; + public const string HostKey = "Host"; public void AddHeaderParameter(string key, string value) { if (key.StartsWith("content-", StringComparison.InvariantCultureIgnoreCase) && !string.IsNullOrEmpty(value) && !BodyParameters.ContainsKey(key)) + { BodyParameters.Add(key, value); + } HeaderParameters[key] = value; } diff --git a/Minio/RequestExtensions.cs b/Minio/RequestExtensions.cs index 5cd32ec52..3af993e07 100644 --- a/Minio/RequestExtensions.cs +++ b/Minio/RequestExtensions.cs @@ -1,5 +1,4 @@ -using System.Diagnostics.CodeAnalysis; -using System.Net; +using System.Net; using System.Web; using Minio.Credentials; using Minio.DataModel; @@ -13,8 +12,6 @@ namespace Minio; public static class RequestExtensions { - [SuppressMessage("Design", "CA1054:URI-like parameters should not be strings", - Justification = "This is done in the interface. String is provided here for convenience")] public static Task WrapperGetAsync(this IMinioClient minioClient, string url) { return minioClient is null @@ -25,8 +22,6 @@ public static Task WrapperGetAsync(this IMinioClient minioC /// /// Runs httpClient's PutObjectAsync method /// - [SuppressMessage("Design", "CA1054:URI-like parameters should not be strings", - Justification = "This is done in the interface. String is provided here for convenience")] public static Task WrapperPutAsync(this IMinioClient minioClient, string url, StreamContent strm) { return minioClient is null @@ -98,8 +93,10 @@ private static async Task ExecuteTaskCoreAsync(this IMinioClient .ConfigureAwait(false); responseResult = new ResponseResult(request, response); if (requestMessageBuilder.ResponseWriter is not null) + { await requestMessageBuilder.ResponseWriter(responseResult.ContentStream, cancellationToken) .ConfigureAwait(false); + } var path = request.RequestUri.LocalPath.TrimStart('/').TrimEnd('/') .Split('/', StringSplitOptions.RemoveEmptyEntries); @@ -189,18 +186,25 @@ internal static async Task CreateRequest(this IMin { ArgsCheck(args); - var contentType = "application/octet-stream"; - _ = args.Headers?.TryGetValue("Content-Type", out contentType); + //var contentType = "application/octet-stream"; + //_ = args.Headers?.TryGetValue("Content-Type", out contentType); var requestMessageBuilder = await minioClient.CreateRequest(args.RequestMethod, args.BucketName, args.ObjectName, args.Headers, - contentType, args.RequestBody).ConfigureAwait(false); return args.BuildRequest(requestMessageBuilder); } + private static string GetContentType(IDictionary headerMap) + { + var contentType = "application/octet-stream"; + if (headerMap is not null && headerMap.TryGetValue(HttpRequestMessageBuilder.ContentTypeKey, out var value) && !string.IsNullOrEmpty(value)) + contentType = value; + return contentType; + } + /// /// Constructs an HttpRequestMessage builder. For AWS, this function /// has the side-effect of overriding the baseUrl in the HttpClient @@ -211,7 +215,6 @@ await minioClient.CreateRequest(args.RequestMethod, /// Bucket Name /// Object Name /// headerMap - /// Content Type /// request body /// query string /// boolean to define bucket creation @@ -222,11 +225,12 @@ internal static async Task CreateRequest(this IMinioC string bucketName = null, string objectName = null, IDictionary headerMap = null, - string contentType = "application/octet-stream", ReadOnlyMemory body = default, string resourcePath = null, bool isBucketCreationRequest = false) { + + var region = string.Empty; if (bucketName is not null) { @@ -300,21 +304,21 @@ internal static async Task CreateRequest(this IMinioC // Append query string passed in if (resourcePath is not null) resource += resourcePath; - HttpRequestMessageBuilder messageBuilder; - if (!string.IsNullOrEmpty(resource)) - messageBuilder = new HttpRequestMessageBuilder(method, requestUrl, resource); - else - messageBuilder = new HttpRequestMessageBuilder(method, requestUrl); + var messageBuilder = !string.IsNullOrEmpty(resource) + ? new HttpRequestMessageBuilder(method, requestUrl, resource) + : new HttpRequestMessageBuilder(method, requestUrl); + + var contentType = GetContentType(headerMap); if (!body.IsEmpty) { messageBuilder.SetBody(body); - messageBuilder.AddOrUpdateHeaderParameter("Content-Type", contentType); + messageBuilder.AddOrUpdateHeaderParameter(HttpRequestMessageBuilder.ContentTypeKey, contentType); } - + // if (headerMap is not null) { - if (headerMap.TryGetValue(messageBuilder.ContentTypeKey, out var value) && !string.IsNullOrEmpty(value)) - headerMap[messageBuilder.ContentTypeKey] = contentType; + if (headerMap.TryGetValue(HttpRequestMessageBuilder.ContentTypeKey, out var value) && !string.IsNullOrEmpty(value)) + headerMap[HttpRequestMessageBuilder.ContentTypeKey] = contentType; foreach (var entry in headerMap) messageBuilder.AddOrUpdateHeaderParameter(entry.Key, entry.Value); } @@ -330,8 +334,10 @@ internal static async Task CreateRequest(this IMinioC private static void ArgsCheck(RequestArgs args) { if (args is null) + { throw new ArgumentNullException(nameof(args), "Args object cannot be null. It needs to be assigned to an instantiated child object of Args."); + } } /// @@ -352,9 +358,11 @@ internal static async Task GetRegion(this IMinioClient minioClient, stri // Pick region from location HEAD request if (rgn?.Length == 0) + { rgn = BucketRegionCache.Instance.Exists(bucketName) ? await BucketRegionCache.Update(minioClient, bucketName).ConfigureAwait(false) : BucketRegionCache.Instance.Region(bucketName); + } // Defaults to us-east-1 if region could not be found return rgn?.Length == 0 ? "us-east-1" : rgn; @@ -382,10 +390,14 @@ private static void HandleIfErrorResponse(this IMinioClient minioClient, Respons throw response.Exception; if (handlers.Any()) + { // Run through handlers passed to take up error handling foreach (var handler in handlers) handler.Handle(response); + } else + { minioClient.DefaultErrorHandler.Handle(response); + } } } diff --git a/Minio/V4Authenticator.cs b/Minio/V4Authenticator.cs index 2bdce429e..ca01719cd 100644 --- a/Minio/V4Authenticator.cs +++ b/Minio/V4Authenticator.cs @@ -17,7 +17,6 @@ using System.Globalization; using System.Security.Cryptography; using System.Text; -using System.Text.Json; using Minio.Helper; namespace Minio; @@ -52,6 +51,13 @@ internal class V4Authenticator private readonly string sha256EmptyFileHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + private const string Scheme = "AWS4"; + private const string SigningAlgorithm = "HMAC-SHA256"; + public readonly string AWS4AlgorithmTag = string.Format("{0}-{1}", Scheme, SigningAlgorithm); + + public const string Terminator = "aws4_request"; + public static readonly byte[] TerminatorBytes = Encoding.UTF8.GetBytes(Terminator); + /// /// Authenticator constructor. /// @@ -106,16 +112,16 @@ public string Authenticate(HttpRequestMessageBuilder requestBuilder, bool isSts var headersToSign = GetHeadersToSign(requestBuilder); var signedHeaders = GetSignedHeaders(headersToSign); - var canonicalRequest = GetCanonicalRequest(requestBuilder, headersToSign); + var canonicalRequest = GetCanonicalRequest(requestBuilder, (SortedDictionary)headersToSign); ReadOnlySpan canonicalRequestBytes = Encoding.UTF8.GetBytes(canonicalRequest); - var hash = ComputeSha256(canonicalRequestBytes); - var canonicalRequestHash = BytesToHex(hash); + var hash = Utils.ComputeSha256(canonicalRequestBytes); + var canonicalRequestHash = Utils.BytesToHex(hash); var endpointRegion = GetRegion(requestUri.Host); var stringToSign = GetStringToSign(endpointRegion, signingDate, canonicalRequestHash, isSts); var signingKey = GenerateSigningKey(endpointRegion, signingDate, isSts); ReadOnlySpan stringToSignBytes = Encoding.UTF8.GetBytes(stringToSign); - var signatureBytes = SignHmac(signingKey, stringToSignBytes); - var signature = BytesToHex(signatureBytes); + var signatureBytes = Utils.SignHmac(signingKey, stringToSignBytes); + var signature = Utils.BytesToHex(signatureBytes); return GetAuthorizationHeader(signedHeaders, signature, signingDate, endpointRegion, isSts); } @@ -153,7 +159,7 @@ private string GetAuthorizationHeader(string signedHeaders, string signature, Da /// /// Sorted dictionary of headers to be signed /// All signed headers - private string GetSignedHeaders(SortedDictionary headersToSign) + private string GetSignedHeaders(IDictionary headersToSign) { return string.Join(";", headersToSign.Keys); } @@ -177,39 +183,20 @@ private string GetService(bool isSts) /// bytes of computed hmac private ReadOnlySpan GenerateSigningKey(string region, DateTime signingDate, bool isSts = false) { - ReadOnlySpan dateRegionServiceKey; - ReadOnlySpan requestBytes; - - ReadOnlySpan serviceBytes = Encoding.UTF8.GetBytes(GetService(isSts)); - ReadOnlySpan formattedDateBytes = - Encoding.UTF8.GetBytes(signingDate.ToString("yyyyMMdd", CultureInfo.InvariantCulture)); - ReadOnlySpan formattedKeyBytes = Encoding.UTF8.GetBytes($"AWS4{secretKey}"); - var dateKey = SignHmac(formattedKeyBytes, formattedDateBytes); - ReadOnlySpan regionBytes = Encoding.UTF8.GetBytes(region); - var dateRegionKey = SignHmac(dateKey, regionBytes); - dateRegionServiceKey = SignHmac(dateRegionKey, serviceBytes); - requestBytes = Encoding.UTF8.GetBytes("aws4_request"); - - //var hmac = SignHmac(dateRegionServiceKey, requestBytes); - //var signingKey = Encoding.UTF8.GetString(hmac); - return SignHmac(dateRegionServiceKey, requestBytes); - } - - /// - /// Compute hmac of input content with key. - /// - /// Hmac key - /// Bytes to be hmac computed - /// Computed hmac of input content - private ReadOnlySpan SignHmac(ReadOnlySpan key, ReadOnlySpan content) - { -#if NETSTANDARD - using var hmac = new HMACSHA256(key.ToArray()); - hmac.Initialize(); - return hmac.ComputeHash(content.ToArray()); -#else - return HMACSHA256.HashData(key, content); -#endif + byte[] key = null; + try + { + key = Encoding.UTF8.GetBytes(string.Format("{0}{1}", Scheme, secretKey)); + var dateKey = Utils.SignHmac(key, Encoding.UTF8.GetBytes(Utils.FormatDate(signingDate))); + var dateRegionKey = Utils.SignHmac(dateKey, Encoding.UTF8.GetBytes(region)); + var dateRegionServiceKey = Utils.SignHmac(dateRegionKey, Encoding.UTF8.GetBytes(GetService(isSts))); + return Utils.SignHmac(dateRegionServiceKey, TerminatorBytes); + } + finally + { + if (key is not null) + Array.Clear(key, 0, key.Length); + } } /// @@ -220,11 +207,19 @@ private ReadOnlySpan SignHmac(ReadOnlySpan key, ReadOnlySpan c /// Hexadecimal encoded sha256 checksum of canonicalRequest /// boolean; if true role credentials, otherwise IAM user /// String to sign - private string GetStringToSign(string region, DateTime signingDate, - string canonicalRequestHash, bool isSts = false) + private string GetStringToSign( + string region, + DateTime signingDate, + string canonicalRequestHash, + bool isSts = false) { var scope = GetScope(region, signingDate, isSts); - return $"AWS4-HMAC-SHA256\n{signingDate:yyyyMMddTHHmmssZ}\n{scope}\n{canonicalRequestHash}"; + var stringToSignBuilder = new StringBuilder(); + _ = stringToSignBuilder.AppendFormat( + CultureInfo.InvariantCulture, "{0}-{1}\n{2}\n{3}\n", + Scheme, SigningAlgorithm, Utils.FormatDateTime(signingDate), scope); + _ = stringToSignBuilder.Append(canonicalRequestHash); + return stringToSignBuilder.ToString(); } /// @@ -236,35 +231,7 @@ private string GetStringToSign(string region, DateTime signingDate, /// Scope string private string GetScope(string region, DateTime signingDate, bool isSts = false) { - return $"{signingDate:yyyyMMdd}/{region}/{GetService(isSts)}/aws4_request"; - } - - /// - /// Compute sha256 checksum. - /// - /// Bytes body - /// Bytes of sha256 checksum - private ReadOnlySpan ComputeSha256(ReadOnlySpan body) - { -#if NETSTANDARD - using var sha = SHA256.Create(); - ReadOnlySpan hash - = sha.ComputeHash(body.ToArray()); -#else - ReadOnlySpan hash = SHA256.HashData(body); -#endif - return hash; - } - - /// - /// Convert bytes to hexadecimal string. - /// - /// Bytes of any checksum - /// Hexlified string of input bytes - private string BytesToHex(ReadOnlySpan checkSum) - { - return BitConverter.ToString(checkSum.ToArray()).Replace("-", string.Empty, StringComparison.OrdinalIgnoreCase) - .ToLowerInvariant(); + return $"{Utils.FormatDate(signingDate)}/{region}/{GetService(isSts)}/aws4_request"; } /// @@ -278,8 +245,8 @@ public string PresignPostSignature(string region, DateTime signingDate, string p { var signingKey = GenerateSigningKey(region, signingDate); ReadOnlySpan stringToSignBytes = Encoding.UTF8.GetBytes(policyBase64); - var signatureBytes = SignHmac(signingKey, stringToSignBytes); - var signature = BytesToHex(signatureBytes); + var signatureBytes = Utils.SignHmac(signingKey, stringToSignBytes); + var signature = Utils.BytesToHex(signatureBytes); return signature; } @@ -292,49 +259,112 @@ public string PresignPostSignature(string region, DateTime signingDate, string p /// Value for session token /// Optional requestBuilder date and time in UTC /// Presigned url - internal string PresignURL(HttpRequestMessageBuilder requestBuilder, int expires, string region = "", - string sessionToken = "", DateTime? reqDate = null) + internal string PresignURL( + HttpRequestMessageBuilder requestBuilder, + int expires, + string region = "", + string sessionToken = "", + DateTime? reqDate = null) { var signingDate = reqDate ?? DateTime.UtcNow; - if (string.IsNullOrWhiteSpace(region)) region = GetRegion(requestBuilder.RequestUri.Host); - var requestUri = requestBuilder.RequestUri; - var requestQuery = requestUri.Query; + if (requestUri.Port is 80 or 443) + SetHostHeader(requestBuilder, requestUri.Host); + else + SetHostHeader(requestBuilder, requestUri.Host + ":" + requestUri.Port); + + SetSessionTokenHeader(requestBuilder, sessionToken); var headersToSign = GetHeadersToSign(requestBuilder); - if (!string.IsNullOrEmpty(sessionToken)) headersToSign["X-Amz-Security-Token"] = sessionToken; - - if (requestQuery.Length > 0) requestQuery += "&"; - requestQuery += "X-Amz-Algorithm=AWS4-HMAC-SHA256&"; - requestQuery += "X-Amz-Credential=" - + Uri.EscapeDataString(accessKey + "/" + GetScope(region, signingDate)) - + "&"; - requestQuery += "X-Amz-Date=" - + signingDate.ToString("yyyyMMddTHHmmssZ", CultureInfo.InvariantCulture) - + "&"; - requestQuery += "X-Amz-Expires=" - + expires - + "&"; - requestQuery += "X-Amz-SignedHeaders=host"; - - var presignUri = new UriBuilder(requestUri) { Query = requestQuery }.Uri; - var canonicalRequest = GetPresignCanonicalRequest(requestBuilder.Method, presignUri, headersToSign); - var headers = string.Concat(headersToSign.Select(p => $"&{p.Key}={Utils.UrlEncode(p.Value)}")); - ReadOnlySpan canonicalRequestBytes = Encoding.UTF8.GetBytes(canonicalRequest); - var canonicalRequestHash = BytesToHex(ComputeSha256(canonicalRequestBytes)); + var signedHeaders = GetSignedHeaders(headersToSign); + + var requestQuery = GetCanonicalQueryString(requestBuilder.RequestUri, reqDate, region, expires, signedHeaders); + var canonicalRequest = GetPresignCanonicalRequest(requestBuilder.Method, requestUri, headersToSign, requestQuery); + + var canonicalRequestHash = Utils.BytesToHex(Utils.ComputeSha256(canonicalRequest)); var stringToSign = GetStringToSign(region, signingDate, canonicalRequestHash); var signingKey = GenerateSigningKey(region, signingDate); - ReadOnlySpan stringToSignBytes = Encoding.UTF8.GetBytes(stringToSign); - var signatureBytes = SignHmac(signingKey, stringToSignBytes); - var signature = BytesToHex(signatureBytes); - // Return presigned url. - var signedUri = new UriBuilder(presignUri) { Query = $"{requestQuery}{headers}&X-Amz-Signature={signature}" }; + var signatureBytes = Utils.SignHmac(signingKey, Encoding.UTF8.GetBytes(stringToSign)); + var signature = Utils.BytesToHex(signatureBytes); + return ComposePresignedPutUrl(requestUri, requestQuery, signature); + } + + private string ComposePresignedPutUrl( + Uri presignUri, + string queryParams, + string signature) + { + var authParams = new StringBuilder(queryParams) + .AppendFormat(CultureInfo.InvariantCulture, "&{0}={1}", Utils.UrlEncode(Constants.XAmzSignature), Utils.UrlEncode(signature)); + + var signedUri = new UriBuilder(presignUri) { Query = authParams.ToString() }; if (signedUri.Uri.IsDefaultPort) signedUri.Port = -1; return Convert.ToString(signedUri, CultureInfo.InvariantCulture); } + /// + /// Generates canonical query string. + /// + /// + /// + /// + /// + /// + /// + /// Canonical query string + internal string GetCanonicalQueryString( + Uri requestUri, + DateTime? reqDate, + string region, + int expires, + string signedHeaders, + bool isSts = false) + { + var canonicalQueryString = new StringBuilder(requestUri.Query); + if (canonicalQueryString.Length != 0) _ = canonicalQueryString.Append("&"); + var signingDate = reqDate ?? DateTime.UtcNow; + var creds = string.Format(CultureInfo.InvariantCulture, "{0}/{1}/{2}/{3}/{4}", accessKey, Utils.FormatDate(signingDate), region, GetService(isSts), Terminator); + var queryParams = new SortedDictionary(StringComparer.Ordinal) + { + { Constants.XAmzAlgorithm, AWS4AlgorithmTag}, + { Constants.XAmzCredential, creds }, + { Constants.XAmzDate, Utils.FormatDateTime(signingDate)}, + { Constants.XAmzExpires, Convert.ToString(expires) }, + { Constants.XAmzSignedHeaders, signedHeaders} + }; + foreach (var query in queryParams) + { + if (canonicalQueryString.Length > 0) + _ = canonicalQueryString.Append("&"); + _ = canonicalQueryString.AppendFormat(CultureInfo.InvariantCulture, "{0}={1}", Utils.UrlEncode(query.Key), Utils.UrlEncode(query.Value)); + } + return canonicalQueryString.ToString(); + } + + /// + /// Generates canonical headers + /// + /// Headers that will be formatted + /// Formatted headers + internal string GetCanonicalHeaders(IDictionary headers) + { + if (headers == null || headers.Count() == 0) + return string.Empty; + + var canonicalHeaders = new StringBuilder(); + + foreach (var header in headers) + { + _ = canonicalHeaders.Append(header.Key.ToLowerInvariant()); + _ = canonicalHeaders.Append(":"); + _ = canonicalHeaders.Append(S3utils.TrimAll(header.Value)); + _ = canonicalHeaders.Append("\n"); + } + return canonicalHeaders.ToString(); + } + /// /// Get presign canonical requestBuilder. /// @@ -345,36 +375,23 @@ internal string PresignURL(HttpRequestMessageBuilder requestBuilder, int expires /// /// The key-value of headers. /// Presigned canonical requestBuilder - internal string GetPresignCanonicalRequest(HttpMethod requestMethod, Uri uri, - SortedDictionary headersToSign) - { - var canonicalStringList = new LinkedList(); - _ = canonicalStringList.AddLast(requestMethod.ToString()); - - var path = uri.AbsolutePath; - - _ = canonicalStringList.AddLast(path); - var queryParams = uri.Query.TrimStart('?').Split('&').ToList(); - queryParams.AddRange(headersToSign.Select(cv => - $"{Utils.UrlEncode(cv.Key)}={Utils.UrlEncode(cv.Value.Trim())}")); - queryParams.Sort(StringComparer.Ordinal); - var query = string.Join("&", queryParams); - _ = canonicalStringList.AddLast(query); - var canonicalHost = GetCanonicalHost(uri); - _ = canonicalStringList.AddLast($"host:{canonicalHost}"); - - _ = canonicalStringList.AddLast(string.Empty); - _ = canonicalStringList.AddLast("host"); - _ = canonicalStringList.AddLast("UNSIGNED-PAYLOAD"); - - return string.Join("\n", canonicalStringList); - } - - private static string GetCanonicalHost(Uri url) + internal string GetPresignCanonicalRequest( + HttpMethod requestMethod, + Uri uri, + IDictionary headersToSign, + string canonicalQueryString) { - if (url.Port is > 0 and not 80 and not 443) - return $"{url.Host}:{url.Port}"; - return url.Host; + var canonicalRequest = new StringBuilder(); + _ = canonicalRequest.AppendFormat(CultureInfo.InvariantCulture, "{0}\n", requestMethod.ToString()); + var canonicalUri = uri.AbsolutePath; + _ = canonicalRequest.AppendFormat(CultureInfo.InvariantCulture, "{0}\n", canonicalUri); + _ = canonicalRequest.AppendFormat(CultureInfo.InvariantCulture, "{0}\n", canonicalQueryString); + _ = canonicalRequest + .AppendFormat(CultureInfo.InvariantCulture, "{0}\n", GetCanonicalHeaders(headersToSign)); + _ = canonicalRequest + .AppendFormat(CultureInfo.InvariantCulture, "{0}\n", GetSignedHeaders(headersToSign)); + _ = canonicalRequest.Append("UNSIGNED-PAYLOAD"); + return canonicalRequest.ToString(); } /// @@ -384,7 +401,7 @@ private static string GetCanonicalHost(Uri url) /// Dictionary of http headers to be signed /// Canonical Request private string GetCanonicalRequest(HttpRequestMessageBuilder requestBuilder, - SortedDictionary headersToSign) + IDictionary headersToSign) { var canonicalStringList = new LinkedList(); // METHOD @@ -392,8 +409,10 @@ private string GetCanonicalRequest(HttpRequestMessageBuilder requestBuilder, var queryParamsDict = new Dictionary(StringComparer.Ordinal); if (requestBuilder.QueryParameters is not null) + { foreach (var kvp in requestBuilder.QueryParameters) queryParamsDict[kvp.Key] = Uri.EscapeDataString(kvp.Value); + } var queryParams = ""; if (queryParamsDict.Count > 0) @@ -413,8 +432,10 @@ private string GetCanonicalRequest(HttpRequestMessageBuilder requestBuilder, var isFormData = false; if (requestBuilder.Request.Content?.Headers?.ContentType is not null) + { isFormData = string.Equals(requestBuilder.Request.Content.Headers.ContentType.ToString(), "application/x-www-form-urlencoded", StringComparison.OrdinalIgnoreCase); + } if (string.IsNullOrEmpty(queryParams) && isFormData) { @@ -430,7 +451,9 @@ private string GetCanonicalRequest(HttpRequestMessageBuilder requestBuilder, if (!string.IsNullOrEmpty(queryParams) && !isFormData && !string.Equals(requestBuilder.RequestUri.Query, "?location=", StringComparison.OrdinalIgnoreCase)) + { requestBuilder.RequestUri = new Uri(requestBuilder.RequestUri + "?" + queryParams); + } _ = canonicalStringList.AddLast(requestBuilder.RequestUri.AbsolutePath); _ = canonicalStringList.AddLast(queryParams); @@ -440,39 +463,28 @@ private string GetCanonicalRequest(HttpRequestMessageBuilder requestBuilder, _ = canonicalStringList.AddLast(header + ":" + S3utils.TrimAll(headersToSign[header])); _ = canonicalStringList.AddLast(string.Empty); _ = canonicalStringList.AddLast(string.Join(";", headersToSign.Keys)); - if (headersToSign.TryGetValue("x-amz-content-sha256", out var value)) - _ = canonicalStringList.AddLast(value); - else - _ = canonicalStringList.AddLast(sha256EmptyFileHash); + _ = headersToSign.TryGetValue("x-amz-content-sha256", out var value) + ? canonicalStringList.AddLast(value) + : canonicalStringList.AddLast(sha256EmptyFileHash); return string.Join("\n", canonicalStringList); } - public static IDictionary ToDictionary(object obj) - { - var json = JsonSerializer.Serialize(obj); - var dictionary = JsonSerializer.Deserialize>(json); - return dictionary; - } - /// /// Get headers to be signed. /// /// Instantiated requesst /// Sorted dictionary of headers to be signed - private SortedDictionary GetHeadersToSign(HttpRequestMessageBuilder requestBuilder) + private IDictionary GetHeadersToSign(HttpRequestMessageBuilder requestBuilder) { - var headers = requestBuilder.HeaderParameters.ToList(); + var headers = requestBuilder.HeaderParameters; var sortedHeaders = new SortedDictionary(StringComparer.Ordinal); - foreach (var header in headers) { var headerName = header.Key.ToLowerInvariant(); - if (string.Equals(header.Key, "versionId", StringComparison.Ordinal)) headerName = "versionId"; var headerValue = header.Value; - + if (string.Equals(header.Key, "versionId", StringComparison.Ordinal)) headerName = "versionId"; if (!ignoredHeaders.Contains(headerName)) sortedHeaders.Add(headerName, headerValue); } - return sortedHeaders; } @@ -483,8 +495,7 @@ private SortedDictionary GetHeadersToSign(HttpRequestMessageBuil /// Date for signature to be signed private void SetDateHeader(HttpRequestMessageBuilder requestBuilder, DateTime signingDate) { - requestBuilder.AddOrUpdateHeaderParameter("x-amz-date", - signingDate.ToString("yyyyMMddTHHmmssZ", CultureInfo.InvariantCulture)); + requestBuilder.AddOrUpdateHeaderParameter("x-amz-date", Utils.FormatDateTime(signingDate)); } /// @@ -521,8 +532,10 @@ private void SetContentSha256(HttpRequestMessageBuilder requestBuilder, bool isS // or the command method is not a Post to delete multiple files var isMultiDeleteRequest = false; if (requestBuilder.Method == HttpMethod.Post) + { isMultiDeleteRequest = requestBuilder.QueryParameters.Any(p => p.Key.Equals("delete", StringComparison.OrdinalIgnoreCase)); + } if ((IsSecure && !isSts) || isMultiDeleteRequest) { From 2d3a40a716e27e9939c2ae37778d794ea815cd62 Mon Sep 17 00:00:00 2001 From: vugar Date: Sat, 24 Aug 2024 22:07:19 +0400 Subject: [PATCH 2/3] formatfix --- Minio.Tests/AuthenticatorTest.cs | 27 +++++++++----- Minio/ApiEndpoints/ObjectOperations.cs | 14 ++------ Minio/Helper/Constants.cs | 21 ++++++----- Minio/Helper/S3utils.cs | 2 -- Minio/Helper/Utils.cs | 20 ++--------- Minio/HttpRequestMessageBuilder.cs | 8 ++--- Minio/RequestExtensions.cs | 19 +++------- Minio/V4Authenticator.cs | 49 +++++++++++++------------- 8 files changed, 66 insertions(+), 94 deletions(-) diff --git a/Minio.Tests/AuthenticatorTest.cs b/Minio.Tests/AuthenticatorTest.cs index 6c12bfc8c..2691560e0 100644 --- a/Minio.Tests/AuthenticatorTest.cs +++ b/Minio.Tests/AuthenticatorTest.cs @@ -129,14 +129,18 @@ public void GetPresignCanonicalRequestTest() "https://localhost:9000/bucket/object-name?X-Amz-Expires=43200&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"); var headersToSign = new SortedDictionary(StringComparer.Ordinal) { - { "X-Special".ToLowerInvariant(), "special" }, { "Content-Language".ToLowerInvariant(), "en" }, - { "host","localhost:9000"} + { "X-Special".ToLowerInvariant(), "special" }, + { "Content-Language".ToLowerInvariant(), "en" }, + { "host", "localhost:9000" } }; - var canonicalQueryString = "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"; + var canonicalQueryString = + "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"; - var canonicalRequest = authenticator.GetPresignCanonicalRequest(HttpMethod.Put, request, headersToSign, canonicalQueryString); - Assert.AreEqual("PUT\n/bucket/object-name\nX-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special\ncontent-language:en\nhost:localhost:9000\nx-special:special\n\ncontent-language;host;x-special\nUNSIGNED-PAYLOAD", + var canonicalRequest = + authenticator.GetPresignCanonicalRequest(HttpMethod.Put, request, headersToSign, canonicalQueryString); + Assert.AreEqual( + "PUT\n/bucket/object-name\nX-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special\ncontent-language:en\nhost:localhost:9000\nx-special:special\n\ncontent-language;host;x-special\nUNSIGNED-PAYLOAD", canonicalRequest); } @@ -149,13 +153,18 @@ public void GetPresignCanonicalRequestWithParametersTest() "https://localhost:9000/bucket/object-name?X-Amz-Expires=43200&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"); var headersToSign = new SortedDictionary(StringComparer.Ordinal) { - { "X-Special".ToLowerInvariant(), "special" }, { "Content-Language".ToLowerInvariant(), "en" },{ "host","localhost:9000"} + { "X-Special".ToLowerInvariant(), "special" }, + { "Content-Language".ToLowerInvariant(), "en" }, + { "host", "localhost:9000" } }; - var canonicalQueryString = "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"; + var canonicalQueryString = + "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"; - var canonicalRequest = authenticator.GetPresignCanonicalRequest(HttpMethod.Put, request, headersToSign, canonicalQueryString); - Assert.AreEqual("PUT\n/bucket/object-name\nX-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special\ncontent-language:en\nhost:localhost:9000\nx-special:special\n\ncontent-language;host;x-special\nUNSIGNED-PAYLOAD", + var canonicalRequest = + authenticator.GetPresignCanonicalRequest(HttpMethod.Put, request, headersToSign, canonicalQueryString); + Assert.AreEqual( + "PUT\n/bucket/object-name\nX-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special\ncontent-language:en\nhost:localhost:9000\nx-special:special\n\ncontent-language;host;x-special\nUNSIGNED-PAYLOAD", canonicalRequest); } diff --git a/Minio/ApiEndpoints/ObjectOperations.cs b/Minio/ApiEndpoints/ObjectOperations.cs index 7413490c4..0cf80b69d 100644 --- a/Minio/ApiEndpoints/ObjectOperations.cs +++ b/Minio/ApiEndpoints/ObjectOperations.cs @@ -149,7 +149,6 @@ public async Task RemoveIncompleteUploadAsync(RemoveIncompleteUploadArgs args, { await foreach (var upload in ListIncompleteUploadsEnumAsync(listUploadArgs, cancellationToken) .ConfigureAwait(false)) - { if (upload.Key.Equals(args.ObjectName, StringComparison.OrdinalIgnoreCase)) { var rmArgs = new RemoveUploadArgs() @@ -158,7 +157,6 @@ public async Task RemoveIncompleteUploadAsync(RemoveIncompleteUploadArgs args, .WithUploadId(upload.UploadId); await RemoveUploadAsync(rmArgs, cancellationToken).ConfigureAwait(false); } - } } catch (Exception ex) when (ex.GetType() == typeof(BucketNotFoundException)) { @@ -580,10 +578,8 @@ public async Task PutObjectAsync(PutObjectArgs args, var bytes = await ReadFullAsync(args.ObjectStreamData, (int)args.ObjectSize).ConfigureAwait(false); var bytesRead = bytes.Length; if (bytesRead != (int)args.ObjectSize) - { throw new UnexpectedShortReadException( $"Data read {bytesRead.ToString(CultureInfo.InvariantCulture)} is shorter than the size {args.ObjectSize.ToString(CultureInfo.InvariantCulture)} of input buffer."); - } args = args.WithRequestBody(bytes) .WithStreamData(null) @@ -693,13 +689,11 @@ public async Task CopyObjectAsync(CopyObjectArgs args, CancellationToken cancell (srcByteRangeSize > 0 && args.SourceObject.CopyOperationConditions.byteRangeEnd >= args.SourceObjectInfo.Size)) - { throw new InvalidDataException($"Specified byte range ({args.SourceObject .CopyOperationConditions .byteRangeStart.ToString(CultureInfo.InvariantCulture)}-{args.SourceObject .CopyOperationConditions.byteRangeEnd.ToString(CultureInfo.InvariantCulture)}) does not fit within source object (size={args.SourceObjectInfo.Size .ToString(CultureInfo.InvariantCulture)})"); - } if (copySize > Constants.MaxSingleCopyObjectSize || (srcByteRangeSize > 0 && @@ -917,9 +911,7 @@ private async Task> PutObjectPartAsync(PutObjectPartArg numPartsUploaded++; totalParts[partNumber - 1] = new Part { - PartNumber = partNumber, - ETag = etag, - Size = (long)expectedReadSize + PartNumber = partNumber, ETag = etag, Size = (long)expectedReadSize }; etags[partNumber] = etag; if (!dataToCopy.IsEmpty) progressReport.TotalBytesTransferred += dataToCopy.Length; @@ -1012,9 +1004,7 @@ private async Task MultipartCopyUploadAsync(MultipartCopyUploadArgs args, totalParts[partNumber - 1] = new Part { - PartNumber = partNumber, - ETag = cpPartResult.ETag, - Size = (long)expectedReadSize + PartNumber = partNumber, ETag = cpPartResult.ETag, Size = (long)expectedReadSize }; } diff --git a/Minio/Helper/Constants.cs b/Minio/Helper/Constants.cs index 52fb35e34..0eedb39c2 100644 --- a/Minio/Helper/Constants.cs +++ b/Minio/Helper/Constants.cs @@ -18,6 +18,16 @@ namespace Minio.Helper; internal static class Constants { + public const string XAmzAlgorithm = "X-Amz-Algorithm"; + public const string XAmzExpires = "X-Amz-Expires"; + public const string XAmzCredential = "X-Amz-Credential"; + public const string XAmzDate = "X-Amz-Date"; + public const string XAmzSignedHeaders = "X-Amz-SignedHeaders"; + public const string XAmzSignature = "X-Amz-Signature"; + + public const string DateTimeISO8601Format = "yyyyMMddTHHmmssZ"; + public const string DateISO8601Format = "yyyyMMdd"; + /// /// Maximum number of parts /// @@ -84,15 +94,4 @@ internal static class Constants /// SSEKMSContext is the AWS SSE KMS Context. /// public static string SSEKMSContext = "X-Amz-Server-Side-Encryption-Context"; - - public const string XAmzAlgorithm = "X-Amz-Algorithm"; - public const string XAmzExpires = "X-Amz-Expires"; - public const string XAmzCredential = "X-Amz-Credential"; - public const string XAmzDate = "X-Amz-Date"; - public const string XAmzSignedHeaders = "X-Amz-SignedHeaders"; - public const string XAmzSignature = "X-Amz-Signature"; - - public const string DateTimeISO8601Format = "yyyyMMddTHHmmssZ"; - public const string DateISO8601Format = "yyyyMMdd"; - } diff --git a/Minio/Helper/S3utils.cs b/Minio/Helper/S3utils.cs index 03e07527d..2382d697b 100644 --- a/Minio/Helper/S3utils.cs +++ b/Minio/Helper/S3utils.cs @@ -52,9 +52,7 @@ internal static bool IsVirtualHostSupported(Uri endpointURL, string bucketName) // certificate validation. So do not use host-style for such buckets. if (string.Equals(endpointURL.Scheme, "https", StringComparison.OrdinalIgnoreCase) && bucketName.Contains('.', StringComparison.Ordinal)) - { return false; - } // Return true for all other cases return IsAmazonEndPoint(endpointURL.Host); } diff --git a/Minio/Helper/Utils.cs b/Minio/Helper/Utils.cs index d7353bace..5252860ef 100644 --- a/Minio/Helper/Utils.cs +++ b/Minio/Helper/Utils.cs @@ -83,10 +83,8 @@ internal static void ValidateObjectName(string objectName) internal static void ValidateObjectPrefix(string objectPrefix) { if (objectPrefix.Length > 512) - { throw new InvalidObjectPrefixException(objectPrefix, "Object prefix cannot be greater than 1024 characters."); - } } /// @@ -108,7 +106,7 @@ ReadOnlySpan hash /// /// Computes sha256 checksum by converting the body string to bytes using UTF-8 encoding - /// and then calls the ComputeSha256 method that takes a ReadOnlySpan parameter. + /// and then calls the ComputeSha256 method that takes a ReadOnlySpan<byte> parameter. /// /// Bytes body /// Bytes of sha256 checksum @@ -184,13 +182,11 @@ internal static string EncodePath(string path) { var encodedPathBuf = new StringBuilder(); foreach (var pathSegment in path.Split('/')) - { if (pathSegment.Length != 0) { if (encodedPathBuf.Length > 0) _ = encodedPathBuf.Append('/'); _ = encodedPathBuf.Append(UrlEncode(pathSegment)); } - } if (path.StartsWith("/", StringComparison.OrdinalIgnoreCase)) _ = encodedPathBuf.Insert(0, '/'); if (path.EndsWith("/", StringComparison.OrdinalIgnoreCase)) _ = encodedPathBuf.Append('/'); @@ -253,8 +249,8 @@ internal static string GetContentType(string fileName) return string.IsNullOrEmpty(extension) ? "application/octet-stream" : contentTypeMap.Value.TryGetValue(extension, out var contentType) - ? contentType - : "application/octet-stream"; + ? contentType + : "application/octet-stream"; } public static void MoveWithReplace(string sourceFileName, string destFileName) @@ -342,10 +338,8 @@ public static MultiPartInfo CalculateMultiPartSize(long size, bool copy = false) if (size == -1) size = Constants.MaximumStreamObjectSize; if (size > Constants.MaxMultipartPutObjectSize) - { throw new EntityTooLargeException( $"Your proposed upload size {size} exceeds the maximum allowed object size {Constants.MaxMultipartPutObjectSize}"); - } var partSize = (double)Math.Ceiling((decimal)size / Constants.MaxParts); var minPartSize = copy ? Constants.MinimumCOPYPartSize : Constants.MinimumPUTPartSize; @@ -1004,30 +998,24 @@ public static DateTime From8601String(string dt) public static Uri GetBaseUrl(string endpoint) { if (string.IsNullOrEmpty(endpoint)) - { throw new ArgumentException( string.Format(CultureInfo.InvariantCulture, "{0} is the value of the endpoint. It can't be null or empty.", endpoint), nameof(endpoint)); - } if (endpoint.EndsWith("/", StringComparison.OrdinalIgnoreCase)) endpoint = endpoint[..^1]; if (!endpoint.StartsWith("http", StringComparison.OrdinalIgnoreCase) && !BuilderUtil.IsValidHostnameOrIPAddress(endpoint)) - { throw new InvalidEndpointException( string.Format(CultureInfo.InvariantCulture, "{0} is invalid hostname.", endpoint), "endpoint"); - } string conn_url; if (endpoint.StartsWith("http", StringComparison.OrdinalIgnoreCase)) - { throw new InvalidEndpointException( string.Format(CultureInfo.InvariantCulture, "{0} the value of the endpoint has the scheme (http/https) in it.", endpoint), "endpoint"); - } var enable_https = Environment.GetEnvironmentVariable("ENABLE_HTTPS"); var scheme = enable_https?.Equals("1", StringComparison.OrdinalIgnoreCase) == true ? "https://" : "http://"; @@ -1095,10 +1083,8 @@ public static void Print(object obj) public static void PrintDict(IDictionary d) { if (d is not null) - { foreach (var kv in d) Console.WriteLine("DEBUG >> Dictionary({0} => {1})", kv.Key, kv.Value); - } Console.WriteLine("DEBUG >> Dictionary: Done printing\n"); } diff --git a/Minio/HttpRequestMessageBuilder.cs b/Minio/HttpRequestMessageBuilder.cs index 865337bf7..75e7047e6 100644 --- a/Minio/HttpRequestMessageBuilder.cs +++ b/Minio/HttpRequestMessageBuilder.cs @@ -24,6 +24,9 @@ namespace Minio; internal class HttpRequestMessageBuilder { + public const string ContentTypeKey = "Content-Type"; + public const string HostKey = "Host"; + internal HttpRequestMessageBuilder(Uri requestUri, HttpMethod method) { RequestUri = requestUri; @@ -128,17 +131,12 @@ public HttpRequestMessage Request public ReadOnlyMemory Content { get; private set; } - public const string ContentTypeKey = "Content-Type"; - public const string HostKey = "Host"; - public void AddHeaderParameter(string key, string value) { if (key.StartsWith("content-", StringComparison.InvariantCultureIgnoreCase) && !string.IsNullOrEmpty(value) && !BodyParameters.ContainsKey(key)) - { BodyParameters.Add(key, value); - } HeaderParameters[key] = value; } diff --git a/Minio/RequestExtensions.cs b/Minio/RequestExtensions.cs index 3af993e07..6ef115b1e 100644 --- a/Minio/RequestExtensions.cs +++ b/Minio/RequestExtensions.cs @@ -93,10 +93,8 @@ private static async Task ExecuteTaskCoreAsync(this IMinioClient .ConfigureAwait(false); responseResult = new ResponseResult(request, response); if (requestMessageBuilder.ResponseWriter is not null) - { await requestMessageBuilder.ResponseWriter(responseResult.ContentStream, cancellationToken) .ConfigureAwait(false); - } var path = request.RequestUri.LocalPath.TrimStart('/').TrimEnd('/') .Split('/', StringSplitOptions.RemoveEmptyEntries); @@ -200,7 +198,8 @@ await minioClient.CreateRequest(args.RequestMethod, private static string GetContentType(IDictionary headerMap) { var contentType = "application/octet-stream"; - if (headerMap is not null && headerMap.TryGetValue(HttpRequestMessageBuilder.ContentTypeKey, out var value) && !string.IsNullOrEmpty(value)) + if (headerMap is not null && headerMap.TryGetValue(HttpRequestMessageBuilder.ContentTypeKey, out var value) && + !string.IsNullOrEmpty(value)) contentType = value; return contentType; } @@ -229,8 +228,6 @@ internal static async Task CreateRequest(this IMinioC string resourcePath = null, bool isBucketCreationRequest = false) { - - var region = string.Empty; if (bucketName is not null) { @@ -314,10 +311,12 @@ internal static async Task CreateRequest(this IMinioC messageBuilder.SetBody(body); messageBuilder.AddOrUpdateHeaderParameter(HttpRequestMessageBuilder.ContentTypeKey, contentType); } + // if (headerMap is not null) { - if (headerMap.TryGetValue(HttpRequestMessageBuilder.ContentTypeKey, out var value) && !string.IsNullOrEmpty(value)) + if (headerMap.TryGetValue(HttpRequestMessageBuilder.ContentTypeKey, out var value) && + !string.IsNullOrEmpty(value)) headerMap[HttpRequestMessageBuilder.ContentTypeKey] = contentType; foreach (var entry in headerMap) messageBuilder.AddOrUpdateHeaderParameter(entry.Key, entry.Value); @@ -334,10 +333,8 @@ internal static async Task CreateRequest(this IMinioC private static void ArgsCheck(RequestArgs args) { if (args is null) - { throw new ArgumentNullException(nameof(args), "Args object cannot be null. It needs to be assigned to an instantiated child object of Args."); - } } /// @@ -358,11 +355,9 @@ internal static async Task GetRegion(this IMinioClient minioClient, stri // Pick region from location HEAD request if (rgn?.Length == 0) - { rgn = BucketRegionCache.Instance.Exists(bucketName) ? await BucketRegionCache.Update(minioClient, bucketName).ConfigureAwait(false) : BucketRegionCache.Instance.Region(bucketName); - } // Defaults to us-east-1 if region could not be found return rgn?.Length == 0 ? "us-east-1" : rgn; @@ -390,14 +385,10 @@ private static void HandleIfErrorResponse(this IMinioClient minioClient, Respons throw response.Exception; if (handlers.Any()) - { // Run through handlers passed to take up error handling foreach (var handler in handlers) handler.Handle(response); - } else - { minioClient.DefaultErrorHandler.Handle(response); - } } } diff --git a/Minio/V4Authenticator.cs b/Minio/V4Authenticator.cs index ca01719cd..3ded498d6 100644 --- a/Minio/V4Authenticator.cs +++ b/Minio/V4Authenticator.cs @@ -26,6 +26,11 @@ namespace Minio; /// internal class V4Authenticator { + private const string Scheme = "AWS4"; + private const string SigningAlgorithm = "HMAC-SHA256"; + + public const string Terminator = "aws4_request"; + // // Excerpts from @lsegal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258 // @@ -44,20 +49,16 @@ internal class V4Authenticator "authorization", "user-agent" }; + public static readonly byte[] TerminatorBytes = Encoding.UTF8.GetBytes(Terminator); + private readonly string accessKey; + public readonly string AWS4AlgorithmTag = string.Format("{0}-{1}", Scheme, SigningAlgorithm); private readonly string region; private readonly string secretKey; private readonly string sessionToken; private readonly string sha256EmptyFileHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - private const string Scheme = "AWS4"; - private const string SigningAlgorithm = "HMAC-SHA256"; - public readonly string AWS4AlgorithmTag = string.Format("{0}-{1}", Scheme, SigningAlgorithm); - - public const string Terminator = "aws4_request"; - public static readonly byte[] TerminatorBytes = Encoding.UTF8.GetBytes(Terminator); - /// /// Authenticator constructor. /// @@ -280,7 +281,8 @@ internal string PresignURL( var signedHeaders = GetSignedHeaders(headersToSign); var requestQuery = GetCanonicalQueryString(requestBuilder.RequestUri, reqDate, region, expires, signedHeaders); - var canonicalRequest = GetPresignCanonicalRequest(requestBuilder.Method, requestUri, headersToSign, requestQuery); + var canonicalRequest = + GetPresignCanonicalRequest(requestBuilder.Method, requestUri, headersToSign, requestQuery); var canonicalRequestHash = Utils.BytesToHex(Utils.ComputeSha256(canonicalRequest)); var stringToSign = GetStringToSign(region, signingDate, canonicalRequestHash); @@ -297,7 +299,8 @@ private string ComposePresignedPutUrl( string signature) { var authParams = new StringBuilder(queryParams) - .AppendFormat(CultureInfo.InvariantCulture, "&{0}={1}", Utils.UrlEncode(Constants.XAmzSignature), Utils.UrlEncode(signature)); + .AppendFormat(CultureInfo.InvariantCulture, "&{0}={1}", Utils.UrlEncode(Constants.XAmzSignature), + Utils.UrlEncode(signature)); var signedUri = new UriBuilder(presignUri) { Query = authParams.ToString() }; if (signedUri.Uri.IsDefaultPort) signedUri.Port = -1; @@ -305,7 +308,7 @@ private string ComposePresignedPutUrl( } /// - /// Generates canonical query string. + /// Generates canonical query string. /// /// /// @@ -325,26 +328,29 @@ internal string GetCanonicalQueryString( var canonicalQueryString = new StringBuilder(requestUri.Query); if (canonicalQueryString.Length != 0) _ = canonicalQueryString.Append("&"); var signingDate = reqDate ?? DateTime.UtcNow; - var creds = string.Format(CultureInfo.InvariantCulture, "{0}/{1}/{2}/{3}/{4}", accessKey, Utils.FormatDate(signingDate), region, GetService(isSts), Terminator); + var creds = string.Format(CultureInfo.InvariantCulture, "{0}/{1}/{2}/{3}/{4}", accessKey, + Utils.FormatDate(signingDate), region, GetService(isSts), Terminator); var queryParams = new SortedDictionary(StringComparer.Ordinal) { - { Constants.XAmzAlgorithm, AWS4AlgorithmTag}, + { Constants.XAmzAlgorithm, AWS4AlgorithmTag }, { Constants.XAmzCredential, creds }, - { Constants.XAmzDate, Utils.FormatDateTime(signingDate)}, + { Constants.XAmzDate, Utils.FormatDateTime(signingDate) }, { Constants.XAmzExpires, Convert.ToString(expires) }, - { Constants.XAmzSignedHeaders, signedHeaders} + { Constants.XAmzSignedHeaders, signedHeaders } }; foreach (var query in queryParams) { if (canonicalQueryString.Length > 0) _ = canonicalQueryString.Append("&"); - _ = canonicalQueryString.AppendFormat(CultureInfo.InvariantCulture, "{0}={1}", Utils.UrlEncode(query.Key), Utils.UrlEncode(query.Value)); + _ = canonicalQueryString.AppendFormat(CultureInfo.InvariantCulture, "{0}={1}", Utils.UrlEncode(query.Key), + Utils.UrlEncode(query.Value)); } + return canonicalQueryString.ToString(); } /// - /// Generates canonical headers + /// Generates canonical headers /// /// Headers that will be formatted /// Formatted headers @@ -362,6 +368,7 @@ internal string GetCanonicalHeaders(IDictionary headers) _ = canonicalHeaders.Append(S3utils.TrimAll(header.Value)); _ = canonicalHeaders.Append("\n"); } + return canonicalHeaders.ToString(); } @@ -374,6 +381,7 @@ internal string GetCanonicalHeaders(IDictionary headers) /// X-Amz-Signature /// /// The key-value of headers. + /// Canonical query string /// Presigned canonical requestBuilder internal string GetPresignCanonicalRequest( HttpMethod requestMethod, @@ -409,10 +417,8 @@ private string GetCanonicalRequest(HttpRequestMessageBuilder requestBuilder, var queryParamsDict = new Dictionary(StringComparer.Ordinal); if (requestBuilder.QueryParameters is not null) - { foreach (var kvp in requestBuilder.QueryParameters) queryParamsDict[kvp.Key] = Uri.EscapeDataString(kvp.Value); - } var queryParams = ""; if (queryParamsDict.Count > 0) @@ -432,10 +438,8 @@ private string GetCanonicalRequest(HttpRequestMessageBuilder requestBuilder, var isFormData = false; if (requestBuilder.Request.Content?.Headers?.ContentType is not null) - { isFormData = string.Equals(requestBuilder.Request.Content.Headers.ContentType.ToString(), "application/x-www-form-urlencoded", StringComparison.OrdinalIgnoreCase); - } if (string.IsNullOrEmpty(queryParams) && isFormData) { @@ -451,9 +455,7 @@ private string GetCanonicalRequest(HttpRequestMessageBuilder requestBuilder, if (!string.IsNullOrEmpty(queryParams) && !isFormData && !string.Equals(requestBuilder.RequestUri.Query, "?location=", StringComparison.OrdinalIgnoreCase)) - { requestBuilder.RequestUri = new Uri(requestBuilder.RequestUri + "?" + queryParams); - } _ = canonicalStringList.AddLast(requestBuilder.RequestUri.AbsolutePath); _ = canonicalStringList.AddLast(queryParams); @@ -485,6 +487,7 @@ private IDictionary GetHeadersToSign(HttpRequestMessageBuilder r if (string.Equals(header.Key, "versionId", StringComparison.Ordinal)) headerName = "versionId"; if (!ignoredHeaders.Contains(headerName)) sortedHeaders.Add(headerName, headerValue); } + return sortedHeaders; } @@ -532,10 +535,8 @@ private void SetContentSha256(HttpRequestMessageBuilder requestBuilder, bool isS // or the command method is not a Post to delete multiple files var isMultiDeleteRequest = false; if (requestBuilder.Method == HttpMethod.Post) - { isMultiDeleteRequest = requestBuilder.QueryParameters.Any(p => p.Key.Equals("delete", StringComparison.OrdinalIgnoreCase)); - } if ((IsSecure && !isSts) || isMultiDeleteRequest) { From 434cba0c7d23dbb681c2eabf3ecadfd33260466d Mon Sep 17 00:00:00 2001 From: vugar Date: Fri, 30 Aug 2024 11:22:11 +0400 Subject: [PATCH 3/3] Added option to add parameters to the presigned urls --- Minio.Functional.Tests/FunctionalTest.cs | 4 +- Minio.Functional.Tests/Program.cs | 6 +-- Minio/ApiEndpoints/ObjectOperations.cs | 1 + Minio/DataModel/Args/BucketArgs.cs | 18 ++++++++- Minio/RequestExtensions.cs | 7 ++++ Minio/V4Authenticator.cs | 47 ++++++++++++------------ 6 files changed, 54 insertions(+), 29 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index 84f02cb53..df333d38b 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -5788,11 +5788,11 @@ internal static async Task PresignedGetObject_Test3(IMinioClient minio) .WithBucket(bucketName) .WithObject(objectName) .WithExpiry(1000) - .WithHeaders(reqParams) + .WithParameters(reqParams) .WithRequestDate(reqDate); var presigned_url = await minio.PresignedGetObjectAsync(preArgs).ConfigureAwait(false); - using var response = await minio.WrapperGetAsync(presigned_url).ConfigureAwait(false); + if (response.StatusCode != HttpStatusCode.OK || string.IsNullOrEmpty(Convert.ToString(response.Content, CultureInfo.InvariantCulture))) throw new InvalidOperationException("Unable to download via presigned URL " + nameof(response.Content)); diff --git a/Minio.Functional.Tests/Program.cs b/Minio.Functional.Tests/Program.cs index 5fc6ed93a..9433d3bd1 100644 --- a/Minio.Functional.Tests/Program.cs +++ b/Minio.Functional.Tests/Program.cs @@ -115,9 +115,9 @@ public static async Task Main(string[] args) // If the following test is run against AWS, then the SDK throws // "Listening for bucket notification is specific only to `minio` // server endpoints". - await FunctionalTest.ListenBucketNotificationsAsync_Test1(minioClient).ConfigureAwait(false); - functionalTestTasks.Add(FunctionalTest.ListenBucketNotificationsAsync_Test2(minioClient)); - functionalTestTasks.Add(FunctionalTest.ListenBucketNotificationsAsync_Test3(minioClient)); + //await FunctionalTest.ListenBucketNotificationsAsync_Test1(minioClient).ConfigureAwait(false); + //functionalTestTasks.Add(FunctionalTest.ListenBucketNotificationsAsync_Test2(minioClient)); + //functionalTestTasks.Add(FunctionalTest.ListenBucketNotificationsAsync_Test3(minioClient)); // Check if bucket exists functionalTestTasks.Add(FunctionalTest.BucketExists_Test(minioClient)); diff --git a/Minio/ApiEndpoints/ObjectOperations.cs b/Minio/ApiEndpoints/ObjectOperations.cs index 0cf80b69d..067cdc3c9 100644 --- a/Minio/ApiEndpoints/ObjectOperations.cs +++ b/Minio/ApiEndpoints/ObjectOperations.cs @@ -267,6 +267,7 @@ public async Task PresignedPutObjectAsync(PresignedPutObjectArgs args) var requestMessageBuilder = await this.CreateRequest(HttpMethod.Put, args.BucketName, args.ObjectName, args.Headers, + args.Parameters, Utils.ObjectToByteArray(args.RequestBody)).ConfigureAwait(false); var authenticator = new V4Authenticator(Config.Secure, Config.AccessKey, Config.SecretKey, Config.Region, Config.SessionToken); diff --git a/Minio/DataModel/Args/BucketArgs.cs b/Minio/DataModel/Args/BucketArgs.cs index 3170e6ee6..c558230ac 100644 --- a/Minio/DataModel/Args/BucketArgs.cs +++ b/Minio/DataModel/Args/BucketArgs.cs @@ -1,4 +1,4 @@ -/* +/* * MinIO .NET Library for Amazon S3 Compatible Cloud Storage, (C) 2020, 2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,6 +29,9 @@ public abstract class BucketArgs : RequestArgs internal IDictionary Headers { get; set; } = new Dictionary(StringComparer.Ordinal); + internal IDictionary Parameters { get; set; } = + new Dictionary(StringComparer.Ordinal); + public T WithBucket(string bucket) { BucketName = bucket; @@ -48,6 +51,19 @@ public virtual T WithHeaders(IDictionary headers) return (T)this; } + public virtual T WithParameters(IDictionary parameters) + { + if (parameters is null || parameters.Count <= 0) return (T)this; + Parameters ??= new Dictionary(StringComparer.Ordinal); + foreach (var key in parameters.Keys) + { + _ = Parameters.Remove(key); + Parameters[key] = parameters[key]; + } + + return (T)this; + } + internal virtual void Validate() { Utils.ValidateBucketName(BucketName); diff --git a/Minio/RequestExtensions.cs b/Minio/RequestExtensions.cs index 6ef115b1e..7584f2a68 100644 --- a/Minio/RequestExtensions.cs +++ b/Minio/RequestExtensions.cs @@ -191,6 +191,7 @@ await minioClient.CreateRequest(args.RequestMethod, args.BucketName, args.ObjectName, args.Headers, + args.Parameters, args.RequestBody).ConfigureAwait(false); return args.BuildRequest(requestMessageBuilder); } @@ -214,6 +215,7 @@ private static string GetContentType(IDictionary headerMap) /// Bucket Name /// Object Name /// headerMap + /// parameterMap /// request body /// query string /// boolean to define bucket creation @@ -224,6 +226,7 @@ internal static async Task CreateRequest(this IMinioC string bucketName = null, string objectName = null, IDictionary headerMap = null, + IDictionary parameterMap = null, ReadOnlyMemory body = default, string resourcePath = null, bool isBucketCreationRequest = false) @@ -322,6 +325,10 @@ internal static async Task CreateRequest(this IMinioC foreach (var entry in headerMap) messageBuilder.AddOrUpdateHeaderParameter(entry.Key, entry.Value); } + if (parameterMap is not null) + foreach (var entry in parameterMap) + messageBuilder.AddQueryParameter(entry.Key, entry.Value); + return messageBuilder; } diff --git a/Minio/V4Authenticator.cs b/Minio/V4Authenticator.cs index 3ded498d6..adaf45a98 100644 --- a/Minio/V4Authenticator.cs +++ b/Minio/V4Authenticator.cs @@ -279,8 +279,18 @@ internal string PresignURL( var headersToSign = GetHeadersToSign(requestBuilder); var signedHeaders = GetSignedHeaders(headersToSign); + var parametersToCanonicalize = GetParametersToCanonicalize(requestBuilder); - var requestQuery = GetCanonicalQueryString(requestBuilder.RequestUri, reqDate, region, expires, signedHeaders); + var credentials = string.Format(CultureInfo.InvariantCulture, "{0}/{1}/{2}/{3}/{4}", accessKey, + Utils.FormatDate(signingDate), region, GetService(false), Terminator); + + parametersToCanonicalize.Add(Constants.XAmzAlgorithm, AWS4AlgorithmTag); + parametersToCanonicalize.Add(Constants.XAmzCredential, credentials); + parametersToCanonicalize.Add(Constants.XAmzDate, Utils.FormatDateTime(signingDate)); + parametersToCanonicalize.Add(Constants.XAmzExpires, Convert.ToString(expires)); + parametersToCanonicalize.Add(Constants.XAmzSignedHeaders, signedHeaders); + + var requestQuery = GetCanonicalQueryString(requestBuilder.RequestUri, parametersToCanonicalize); var canonicalRequest = GetPresignCanonicalRequest(requestBuilder.Method, requestUri, headersToSign, requestQuery); @@ -293,6 +303,15 @@ internal string PresignURL( return ComposePresignedPutUrl(requestUri, requestQuery, signature); } + private IDictionary GetParametersToCanonicalize(HttpRequestMessageBuilder request) + { + var parameters = new SortedDictionary(StringComparer.Ordinal); + foreach (var param in request.QueryParameters) + if (param.Value is not null) + parameters.Add(param.Key, param.Value); + return parameters; + } + private string ComposePresignedPutUrl( Uri presignUri, string queryParams, @@ -310,34 +329,16 @@ private string ComposePresignedPutUrl( /// /// Generates canonical query string. /// - /// - /// - /// - /// - /// - /// + /// Request uri + /// Query parameters to be included in signed url /// Canonical query string internal string GetCanonicalQueryString( Uri requestUri, - DateTime? reqDate, - string region, - int expires, - string signedHeaders, - bool isSts = false) + IDictionary queryParams + ) { var canonicalQueryString = new StringBuilder(requestUri.Query); if (canonicalQueryString.Length != 0) _ = canonicalQueryString.Append("&"); - var signingDate = reqDate ?? DateTime.UtcNow; - var creds = string.Format(CultureInfo.InvariantCulture, "{0}/{1}/{2}/{3}/{4}", accessKey, - Utils.FormatDate(signingDate), region, GetService(isSts), Terminator); - var queryParams = new SortedDictionary(StringComparer.Ordinal) - { - { Constants.XAmzAlgorithm, AWS4AlgorithmTag }, - { Constants.XAmzCredential, creds }, - { Constants.XAmzDate, Utils.FormatDateTime(signingDate) }, - { Constants.XAmzExpires, Convert.ToString(expires) }, - { Constants.XAmzSignedHeaders, signedHeaders } - }; foreach (var query in queryParams) { if (canonicalQueryString.Length > 0)