From 3d7b10a23cf73a0612b091d52b7a8cf253f8bb75 Mon Sep 17 00:00:00 2001 From: Damien Rider Date: Tue, 18 May 2021 11:01:09 +1000 Subject: [PATCH 1/5] Update config to support configuring file ACLs --- README.md | 2 ++ .../BucketFormsFileSystemComposer.cs | 7 ++++++- .../BucketMediaFileSystemComposer.cs | 7 ++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3089813..ef802f1 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Add the following keys to `~/Web.config` + @@ -39,6 +40,7 @@ Add the following keys to `~/Web.config` | `MediaPrefix` | Sometimes | N/A | The prefix for any media files being added to S3. Essentially a root directory name. Required when using `Umbraco.Storage.S3.Media` | | `FormsPrefix` | Sometimes | N/A | The prefix for any Umbraco Forms data files being added to S3. Essentially a root directory name. Required when using `Umbraco.Storage.S3.Forms` | | `BucketName` | Yes | N/A | The name of your S3 bucket. | +| `FileACL` | Yes | N/A | The ACL to apply to S3 items. | | `BucketHostname` | Sometimes | N/A | The hostname for your bucket (e.g. `test-s3-bucket.s3.eu-west-2.amazonaws.com`). Required when `DisableVirtualPathProvider` is set to `true` | | `DisableVirtualPathProvider` | No | `false` | Setting this to `true` will disable the VPP functionality. See below for more info. | diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/BucketFormsFileSystemComposer.cs b/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/BucketFormsFileSystemComposer.cs index e75b494..85339af 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/BucketFormsFileSystemComposer.cs +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/BucketFormsFileSystemComposer.cs @@ -7,6 +7,7 @@ using Umbraco.Core.Logging; using Umbraco.Forms.Core.Components; using Umbraco.Forms.Data.FileSystem; +using Umbraco.Storage.S3.Extensions; using Umbraco.Storage.S3.Services; namespace Umbraco.Storage.S3.Forms @@ -47,6 +48,7 @@ private BucketFileSystemConfig CreateConfiguration() var bucketHostName = ConfigurationManager.AppSettings[$"{AppSettingsKey}:BucketHostname"]; var bucketPrefix = ConfigurationManager.AppSettings[$"{AppSettingsKey}:FormsPrefix"]; var region = ConfigurationManager.AppSettings[$"{AppSettingsKey}:Region"]; + var fileACL = ConfigurationManager.AppSettings[$"{AppSettingsKey}:FileACL"]; bool.TryParse(ConfigurationManager.AppSettings[$"{AppSettingsKey}:DisableVirtualPathProvider"], out var disableVirtualPathProvider); if (string.IsNullOrEmpty(bucketName)) @@ -61,13 +63,16 @@ private BucketFileSystemConfig CreateConfiguration() if (disableVirtualPathProvider && string.IsNullOrEmpty(bucketHostName)) throw new ArgumentNullOrEmptyException("BucketHostname", $"The AWS S3 Bucket File System (Forms) is missing the value '{AppSettingsKey}:BucketHostname' from AppSettings"); + if(string.IsNullOrEmpty(fileACL)) + throw new ArgumentNullOrEmptyException("FileACL", $"The AWS S3 Bucket File System (Forms) is missing the value '{AppSettingsKey}:FileACL' from AppSettings"); + return new BucketFileSystemConfig { BucketName = bucketName, BucketHostName = bucketHostName, BucketPrefix = bucketPrefix.Trim(Delimiters), Region = region, - CannedACL = new S3CannedACL("public-read"), + CannedACL = AclExtensions.ParseCannedAcl(fileACL), ServerSideEncryptionMethod = "", DisableVirtualPathProvider = disableVirtualPathProvider }; diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/BucketMediaFileSystemComposer.cs b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/BucketMediaFileSystemComposer.cs index 1885b8f..5b8df88 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/BucketMediaFileSystemComposer.cs +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/BucketMediaFileSystemComposer.cs @@ -4,6 +4,7 @@ using Umbraco.Core.Composing; using Umbraco.Core.Exceptions; using Umbraco.Core.Logging; +using Umbraco.Storage.S3.Extensions; using Umbraco.Storage.S3.Services; namespace Umbraco.Storage.S3.Media @@ -45,6 +46,7 @@ private BucketFileSystemConfig CreateConfiguration() var bucketHostName = ConfigurationManager.AppSettings[$"{AppSettingsKey}:BucketHostname"]; var bucketPrefix = ConfigurationManager.AppSettings[$"{AppSettingsKey}:MediaPrefix"]; var region = ConfigurationManager.AppSettings[$"{AppSettingsKey}:Region"]; + var fileACL = ConfigurationManager.AppSettings[$"{AppSettingsKey}:FileACL"]; bool.TryParse(ConfigurationManager.AppSettings[$"{AppSettingsKey}:DisableVirtualPathProvider"], out var disableVirtualPathProvider); if (string.IsNullOrEmpty(bucketName)) @@ -59,13 +61,16 @@ private BucketFileSystemConfig CreateConfiguration() if (disableVirtualPathProvider && string.IsNullOrEmpty(bucketHostName)) throw new ArgumentNullOrEmptyException("BucketHostname", $"The AWS S3 Bucket File System (Media) is missing the value '{AppSettingsKey}:BucketHostname' from AppSettings"); + if (string.IsNullOrEmpty(fileACL)) + throw new ArgumentNullOrEmptyException("FileACL", $"The AWS S3 Bucket File System (Forms) is missing the value '{AppSettingsKey}:FileACL' from AppSettings"); + return new BucketFileSystemConfig { BucketName = bucketName, BucketHostName = bucketHostName, BucketPrefix = bucketPrefix.Trim(Delimiters), Region = region, - CannedACL = new S3CannedACL("public-read"), + CannedACL = AclExtensions.ParseCannedAcl(fileACL), ServerSideEncryptionMethod = "", DisableVirtualPathProvider = disableVirtualPathProvider }; From 66ca97aea6869b5897d47433b8c2157d4f8ca641 Mon Sep 17 00:00:00 2001 From: Damien Rider Date: Tue, 18 May 2021 11:21:53 +1000 Subject: [PATCH 2/5] Update nuspecs for Citadelgroup --- .../Umbraco.Storage.S3.Forms.nuspec | 12 ++++++------ .../Umbraco.Storage.S3.Media.nuspec | 12 ++++++------ .../Umbraco.Storage.S3.Core.nuspec | 14 +++++++------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/Umbraco.Storage.S3.Forms.nuspec b/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/Umbraco.Storage.S3.Forms.nuspec index 44cd14d..e5c7407 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/Umbraco.Storage.S3.Forms.nuspec +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/Umbraco.Storage.S3.Forms.nuspec @@ -1,18 +1,18 @@ - Our.Umbraco.FileSystemProviders.S3.Forms - 8.1.1 + CitadelGroup.Umbraco.FileSystemProviders.S3.Forms + 8.2.0 $author$ $author$ - https://github.com/DannerrQ/Umbraco-S3-Provider - https://github.com/DannerrQ/Umbraco-S3-Provider + https://github.com/kingdamo/Umbraco-S3-Provider + https://github.com/kingdamo/Umbraco-S3-Provider false AWS S3 Filesystem provider for Umbraco Forms - Copyright 2020 + Copyright 2021 Umbraco Aws S3 Provider Forms - + \ No newline at end of file diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.nuspec b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.nuspec index ed49c87..69a0d70 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.nuspec +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.nuspec @@ -1,18 +1,18 @@ - Our.Umbraco.FileSystemProviders.S3.Media - 8.1.1 + CitadelGroup.Umbraco.FileSystemProviders.S3.Media + 8.2.0 $author$ $author$ - https://github.com/DannerrQ/Umbraco-S3-Provider - https://github.com/DannerrQ/Umbraco-S3-Provider + https://github.com/kingdamo/Umbraco-S3-Provider + https://github.com/kingdamo/Umbraco-S3-Provider false AWS S3 Filesystem provider for Umbraco media items - Copyright 2020 + Copyright 2021 Umbraco Aws S3 Provider Media - + \ No newline at end of file diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.Core.nuspec b/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.Core.nuspec index c22929a..2e260f9 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.Core.nuspec +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.Core.nuspec @@ -1,17 +1,17 @@ - Our.Umbraco.FileSystemProviders.S3.Core - 8.1.0 + CitadelGroup.Umbraco.FileSystemProviders.S3.Core + 8.2.0 $title$ - Elijah Glover, Neil Gaietto, Danny Quarton - Elijah Glover, Neil Gaietto, Danny Quarton - https://github.com/DannerrQ/Umbraco-S3-Provider - https://github.com/DannerrQ/Umbraco-S3-Provider + Elijah Glover, Neil Gaietto, Danny Quarton, Damien Rider + Elijah Glover, Neil Gaietto, Danny Quarton, Damien Rider + https://github.com/kingdamo/Umbraco-S3-Provider + https://github.com/kingdamo/Umbraco-S3-Provider false Base package for AWS S3 Umbraco providers. Required by Umbraco.Storage.S3.Forms and Umbraco.Storage.S3.Media - Copyright 2020 + Copyright 2021 Umbraco Aws S3 Provider From 281caf12f9aec5a9b18a530cb1a27b7dd922fde3 Mon Sep 17 00:00:00 2001 From: Damien Rider Date: Tue, 27 Jul 2021 12:42:37 +1000 Subject: [PATCH 3/5] Update to support caching via settings --- .../BucketFormsFileSystemComposer.cs | 52 ++++++++++++------- .../Umbraco.Storage.S3.Forms.csproj | 6 +-- .../Umbraco.Storage.S3.Forms.nuspec | 4 +- .../Umbraco.Storage.S3.Forms/packages.config | 4 +- .../BucketMediaFileSystemComposer.cs | 49 +++++++++++------ .../FileSystemVirtualFile.cs | 28 +++++++++- .../FileSystemVirtualPathProvider.cs | 2 +- .../Umbraco.Storage.S3.Media.csproj | 6 +-- .../Umbraco.Storage.S3.Media.nuspec | 4 +- .../Umbraco.Storage.S3.Media/packages.config | 4 +- .../Umbraco.Storage.S3.Tests.csproj | 6 +-- .../Umbraco.Storage.S3.Tests/packages.config | 4 +- .../Umbraco.Storage.S3/BucketFileSystem.cs | 8 +++ .../BucketFileSystemConfig.cs | 4 ++ .../Umbraco.Storage.S3.Core.nuspec | 6 +-- .../Umbraco.Storage.S3.csproj | 6 +-- .../Umbraco.Storage.S3/packages.config | 4 +- 17 files changed, 133 insertions(+), 64 deletions(-) diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/BucketFormsFileSystemComposer.cs b/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/BucketFormsFileSystemComposer.cs index 85339af..e3df248 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/BucketFormsFileSystemComposer.cs +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/BucketFormsFileSystemComposer.cs @@ -1,4 +1,5 @@ -using System.Configuration; +using System; +using System.Configuration; using Amazon.S3; using Umbraco.Core; using Umbraco.Core.Composing; @@ -24,21 +25,26 @@ public void Compose(Composition composition) { var bucketName = ConfigurationManager.AppSettings[$"{AppSettingsKey}:BucketName"]; - if (bucketName != null) - { - var config = CreateConfiguration(); + + if (bucketName == null) return; + + var config = CreateConfiguration(); + + composition.RegisterUnique(config); + composition.Register(new DefaultMimeTypeResolver()); + if (config.CacheEnabled) + composition.Register(new FileSystemCacheProvider(TimeSpan.FromMinutes(config.CacheMinutes), "~/App_Data/S3Cache/Forms/")); + else + composition.Register(null); - composition.RegisterUnique(config); - composition.Register(new DefaultMimeTypeResolver()); - composition.RegisterUniqueFor(f => new BucketFileSystem( - config: config, - mimeTypeResolver: f.GetInstance(), - fileCacheProvider: null, - logger: f.GetInstance(), - s3Client: new AmazonS3Client(Amazon.RegionEndpoint.GetBySystemName(config.Region)) - )); - } + composition.RegisterUniqueFor(f => new BucketFileSystem( + config: config, + mimeTypeResolver: f.GetInstance(), + fileCacheProvider: f.GetInstance(), + logger: f.GetInstance(), + s3Client: new AmazonS3Client(Amazon.RegionEndpoint.GetBySystemName(config.Region)) + )); } @@ -46,16 +52,19 @@ private BucketFileSystemConfig CreateConfiguration() { var bucketName = ConfigurationManager.AppSettings[$"{AppSettingsKey}:BucketName"]; var bucketHostName = ConfigurationManager.AppSettings[$"{AppSettingsKey}:BucketHostname"]; - var bucketPrefix = ConfigurationManager.AppSettings[$"{AppSettingsKey}:FormsPrefix"]; + var bucketPrefix = ConfigurationManager.AppSettings[$"{AppSettingsKey}:MediaPrefix"]; var region = ConfigurationManager.AppSettings[$"{AppSettingsKey}:Region"]; var fileACL = ConfigurationManager.AppSettings[$"{AppSettingsKey}:FileACL"]; + var cacheMinutes = ConfigurationManager.AppSettings[$"{AppSettingsKey}:CacheMinutes"]; + bool.TryParse(ConfigurationManager.AppSettings[$"{AppSettingsKey}:DisableVirtualPathProvider"], out var disableVirtualPathProvider); + bool.TryParse(ConfigurationManager.AppSettings[$"{AppSettingsKey}:CacheEnabled"], out var cacheEnabled); if (string.IsNullOrEmpty(bucketName)) throw new ArgumentNullOrEmptyException("BucketName", $"The AWS S3 Bucket File System (Forms) is missing the value '{AppSettingsKey}:BucketName' from AppSettings"); if (string.IsNullOrEmpty(bucketPrefix)) - throw new ArgumentNullOrEmptyException("BucketPrefix", $"The AWS S3 Bucket File System (Forms) is missing the value '{AppSettingsKey}:FormsPrefix' from AppSettings"); + throw new ArgumentNullOrEmptyException("BucketPrefix", $"The AWS S3 Bucket File System (Forms) is missing the value '{AppSettingsKey}:MediaPrefix' from AppSettings"); if (string.IsNullOrEmpty(region)) throw new ArgumentNullOrEmptyException("Region", $"The AWS S3 Bucket File System (Forms) is missing the value '{AppSettingsKey}:Region' from AppSettings"); @@ -63,9 +72,14 @@ private BucketFileSystemConfig CreateConfiguration() if (disableVirtualPathProvider && string.IsNullOrEmpty(bucketHostName)) throw new ArgumentNullOrEmptyException("BucketHostname", $"The AWS S3 Bucket File System (Forms) is missing the value '{AppSettingsKey}:BucketHostname' from AppSettings"); - if(string.IsNullOrEmpty(fileACL)) + if (string.IsNullOrEmpty(fileACL)) throw new ArgumentNullOrEmptyException("FileACL", $"The AWS S3 Bucket File System (Forms) is missing the value '{AppSettingsKey}:FileACL' from AppSettings"); + if (string.IsNullOrEmpty(cacheMinutes)) + throw new ArgumentNullOrEmptyException("CacheMinutes", $"The AWS S3 Bucket File System (Forms) is missing the value '{AppSettingsKey}:CacheMinutes' from AppSettings"); + if (!int.TryParse(cacheMinutes, out var minutesToCache)) + throw new ArgumentOutOfRangeException("CacheMinutes", $"The AWS S3 Bucket File System (Forms) value '{AppSettingsKey}:CacheMinutes' is not a valid integer"); + return new BucketFileSystemConfig { BucketName = bucketName, @@ -74,7 +88,9 @@ private BucketFileSystemConfig CreateConfiguration() Region = region, CannedACL = AclExtensions.ParseCannedAcl(fileACL), ServerSideEncryptionMethod = "", - DisableVirtualPathProvider = disableVirtualPathProvider + DisableVirtualPathProvider = disableVirtualPathProvider, + CacheEnabled = cacheEnabled, + CacheMinutes = minutesToCache }; } } diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/Umbraco.Storage.S3.Forms.csproj b/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/Umbraco.Storage.S3.Forms.csproj index 3bda97f..c2ec2fb 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/Umbraco.Storage.S3.Forms.csproj +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/Umbraco.Storage.S3.Forms.csproj @@ -34,10 +34,10 @@ - ..\packages\AWSSDK.Core.3.3.103.29\lib\net45\AWSSDK.Core.dll + ..\packages\AWSSDK.Core.3.7.0.44\lib\net45\AWSSDK.Core.dll - ..\packages\AWSSDK.S3.3.3.104.17\lib\net45\AWSSDK.S3.dll + ..\packages\AWSSDK.S3.3.7.1.14\lib\net45\AWSSDK.S3.dll ..\packages\EPPlus.4.5.3.2\lib\net40\EPPlus.dll @@ -181,7 +181,7 @@ - + diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/Umbraco.Storage.S3.Forms.nuspec b/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/Umbraco.Storage.S3.Forms.nuspec index e5c7407..83f6499 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/Umbraco.Storage.S3.Forms.nuspec +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/Umbraco.Storage.S3.Forms.nuspec @@ -2,7 +2,7 @@ CitadelGroup.Umbraco.FileSystemProviders.S3.Forms - 8.2.0 + 8.4.0 $author$ $author$ https://github.com/kingdamo/Umbraco-S3-Provider @@ -12,7 +12,7 @@ Copyright 2021 Umbraco Aws S3 Provider Forms - + \ No newline at end of file diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/packages.config b/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/packages.config index 5ae847c..68d28eb 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/packages.config +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Forms/packages.config @@ -1,7 +1,7 @@  - - + + diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/BucketMediaFileSystemComposer.cs b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/BucketMediaFileSystemComposer.cs index 5b8df88..5572726 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/BucketMediaFileSystemComposer.cs +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/BucketMediaFileSystemComposer.cs @@ -1,4 +1,5 @@ -using System.Configuration; +using System; +using System.Configuration; using Amazon.S3; using Umbraco.Core; using Umbraco.Core.Composing; @@ -19,24 +20,28 @@ public void Compose(Composition composition) { var bucketName = ConfigurationManager.AppSettings[$"{AppSettingsKey}:BucketName"]; - if (bucketName != null) - { - var config = CreateConfiguration(); + + if (bucketName == null) return; + + var config = CreateConfiguration(); - composition.RegisterUnique(config); - composition.Register(new DefaultMimeTypeResolver()); + composition.RegisterUnique(config); + composition.Register(new DefaultMimeTypeResolver()); - composition.SetMediaFileSystem((f) => new BucketFileSystem( - config: config, - mimeTypeResolver: f.GetInstance(), - fileCacheProvider: null, - logger: f.GetInstance(), - s3Client: new AmazonS3Client(Amazon.RegionEndpoint.GetBySystemName(config.Region)) - )); + if(config.CacheEnabled) + composition.Register(new FileSystemCacheProvider(TimeSpan.FromMinutes(config.CacheMinutes),"~/App_Data/S3Cache/Media/")); + else + composition.Register(null); - composition.Components().Append(); + composition.SetMediaFileSystem((f) => new BucketFileSystem( + config: config, + mimeTypeResolver: f.GetInstance(), + fileCacheProvider: f.GetInstance(), + logger: f.GetInstance(), + s3Client: new AmazonS3Client(Amazon.RegionEndpoint.GetBySystemName(config.Region)) + )); - } + composition.Components().Append(); } @@ -47,7 +52,10 @@ private BucketFileSystemConfig CreateConfiguration() var bucketPrefix = ConfigurationManager.AppSettings[$"{AppSettingsKey}:MediaPrefix"]; var region = ConfigurationManager.AppSettings[$"{AppSettingsKey}:Region"]; var fileACL = ConfigurationManager.AppSettings[$"{AppSettingsKey}:FileACL"]; + var cacheMinutes = ConfigurationManager.AppSettings[$"{AppSettingsKey}:CacheMinutes"]; + bool.TryParse(ConfigurationManager.AppSettings[$"{AppSettingsKey}:DisableVirtualPathProvider"], out var disableVirtualPathProvider); + bool.TryParse(ConfigurationManager.AppSettings[$"{AppSettingsKey}:CacheEnabled"], out var cacheEnabled); if (string.IsNullOrEmpty(bucketName)) throw new ArgumentNullOrEmptyException("BucketName", $"The AWS S3 Bucket File System (Media) is missing the value '{AppSettingsKey}:BucketName' from AppSettings"); @@ -62,7 +70,12 @@ private BucketFileSystemConfig CreateConfiguration() throw new ArgumentNullOrEmptyException("BucketHostname", $"The AWS S3 Bucket File System (Media) is missing the value '{AppSettingsKey}:BucketHostname' from AppSettings"); if (string.IsNullOrEmpty(fileACL)) - throw new ArgumentNullOrEmptyException("FileACL", $"The AWS S3 Bucket File System (Forms) is missing the value '{AppSettingsKey}:FileACL' from AppSettings"); + throw new ArgumentNullOrEmptyException("FileACL", $"The AWS S3 Bucket File System (Media) is missing the value '{AppSettingsKey}:FileACL' from AppSettings"); + + if (string.IsNullOrEmpty(cacheMinutes)) + throw new ArgumentNullOrEmptyException("CacheMinutes", $"The AWS S3 Bucket File System (Media) is missing the value '{AppSettingsKey}:CacheMinutes' from AppSettings"); + if(!int.TryParse(cacheMinutes, out var minutesToCache)) + throw new ArgumentOutOfRangeException("CacheMinutes", $"The AWS S3 Bucket File System (Media) value '{AppSettingsKey}:CacheMinutes' is not a valid integer"); return new BucketFileSystemConfig { @@ -72,7 +85,9 @@ private BucketFileSystemConfig CreateConfiguration() Region = region, CannedACL = AclExtensions.ParseCannedAcl(fileACL), ServerSideEncryptionMethod = "", - DisableVirtualPathProvider = disableVirtualPathProvider + DisableVirtualPathProvider = disableVirtualPathProvider, + CacheEnabled = cacheEnabled, + CacheMinutes = minutesToCache }; } } diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/FileSystemVirtualFile.cs b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/FileSystemVirtualFile.cs index 93b4ba8..1778183 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/FileSystemVirtualFile.cs +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/FileSystemVirtualFile.cs @@ -1,22 +1,31 @@ using System; +using System.Globalization; using System.IO; +using System.Web; using System.Web.Hosting; namespace Umbraco.Storage.S3.Media { internal class FileSystemVirtualFile : VirtualFile { + private readonly DateTimeOffset _lastModified; private readonly Func _stream; - public FileSystemVirtualFile(string virtualPath, Func stream) : base(virtualPath) + public FileSystemVirtualFile(string virtualPath, DateTimeOffset lastModified, Func stream) : base(virtualPath) { if (stream == null) throw new ArgumentNullException("stream"); + _lastModified = lastModified; _stream = stream; } public override Stream Open() { + HttpContext.Current.Response.Cache.SetCacheability(HttpCacheability.Public); + HttpContext.Current.Response.Cache.SetRevalidation(HttpCacheRevalidation.AllCaches); + HttpContext.Current.Response.Cache.SetMaxAge(TimeSpan.FromDays(7)); + HttpContext.Current.Response.Cache.SetExpires(DateTime.Now.AddDays(7)); + HttpContext.Current.Response.Cache.SetETag(GenerateETag(_lastModified.DateTime, DateTime.Now)); return _stream(); } @@ -24,5 +33,22 @@ public override bool IsDirectory { get { return false; } } + + private static string GenerateETag(DateTime lastModified, DateTime now) + { + // Get 64-bit FILETIME stamp + long lastModFileTime = lastModified.ToFileTime(); + long nowFileTime = now.ToFileTime(); + string hexFileTime = lastModFileTime.ToString("X8", CultureInfo.InvariantCulture); + + // Do what IIS does to determine if this is a weak ETag. + // Compare the last modified time to now and if the difference is + // less than or equal to 3 seconds, then it is weak + if ((nowFileTime - lastModFileTime) <= 30000000) + { + return "W/\"" + hexFileTime + "\""; + } + return "\"" + hexFileTime + "\""; + } } } diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/FileSystemVirtualPathProvider.cs b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/FileSystemVirtualPathProvider.cs index c4de0c3..1ab2e6b 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/FileSystemVirtualPathProvider.cs +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/FileSystemVirtualPathProvider.cs @@ -50,7 +50,7 @@ public override VirtualFile GetFile(string virtualPath) return base.GetFile(virtualPath); var fileSystemPath = RemovePathPrefix(path); - return new FileSystemVirtualFile(virtualPath, () => _fileSystem.Value.OpenFile(fileSystemPath)); + return new FileSystemVirtualFile(virtualPath, _fileSystem.Value.GetLastModified(virtualPath), () => _fileSystem.Value.OpenFile(fileSystemPath)); } private string RemovePathPrefix(string virtualPath) diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.csproj b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.csproj index 7abd831..a3a8d54 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.csproj +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.csproj @@ -57,10 +57,10 @@ - ..\packages\AWSSDK.Core.3.3.103.29\lib\net45\AWSSDK.Core.dll + ..\packages\AWSSDK.Core.3.7.0.44\lib\net45\AWSSDK.Core.dll - ..\packages\AWSSDK.S3.3.3.104.17\lib\net45\AWSSDK.S3.dll + ..\packages\AWSSDK.S3.3.7.1.14\lib\net45\AWSSDK.S3.dll ..\packages\LightInject.5.4.0\lib\net46\LightInject.dll @@ -168,7 +168,7 @@ - + diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.nuspec b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.nuspec index 69a0d70..bb10c9f 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.nuspec +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.nuspec @@ -2,7 +2,7 @@ CitadelGroup.Umbraco.FileSystemProviders.S3.Media - 8.2.0 + 8.4.0 $author$ $author$ https://github.com/kingdamo/Umbraco-S3-Provider @@ -12,7 +12,7 @@ Copyright 2021 Umbraco Aws S3 Provider Media - + \ No newline at end of file diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/packages.config b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/packages.config index 115f076..5413f15 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/packages.config +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/packages.config @@ -1,7 +1,7 @@  - - + + diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Tests/Umbraco.Storage.S3.Tests.csproj b/Umbraco.Storage.S3/Umbraco.Storage.S3.Tests/Umbraco.Storage.S3.Tests.csproj index 9f7d30a..fcc656f 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Tests/Umbraco.Storage.S3.Tests.csproj +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Tests/Umbraco.Storage.S3.Tests.csproj @@ -46,10 +46,10 @@ ..\packages\AutoMapper.8.0.0\lib\net461\AutoMapper.dll - ..\packages\AWSSDK.Core.3.3.103.29\lib\net45\AWSSDK.Core.dll + ..\packages\AWSSDK.Core.3.7.0.44\lib\net45\AWSSDK.Core.dll - ..\packages\AWSSDK.S3.3.3.104.17\lib\net45\AWSSDK.S3.dll + ..\packages\AWSSDK.S3.3.7.1.14\lib\net45\AWSSDK.S3.dll ..\packages\Castle.Core.4.3.1\lib\net45\Castle.Core.dll @@ -206,7 +206,7 @@ - + diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Tests/packages.config b/Umbraco.Storage.S3/Umbraco.Storage.S3.Tests/packages.config index 0b974ab..06c5512 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Tests/packages.config +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Tests/packages.config @@ -1,8 +1,8 @@  - - + + diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3/BucketFileSystem.cs b/Umbraco.Storage.S3/Umbraco.Storage.S3/BucketFileSystem.cs index c07b5a2..cd09bb1 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3/BucketFileSystem.cs +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3/BucketFileSystem.cs @@ -240,9 +240,14 @@ public virtual Stream OpenFile(string path) { var persistedStream = FileCacheProvider.Resolve(path); if (persistedStream != null) + { + Logger.Info(typeof(BucketFileSystem), "Loading {path} from cache", path); return persistedStream; + } } + Logger.Info(typeof(BucketFileSystem), "Loading {path} from S3", path); + var request = new GetObjectRequest { BucketName = Config.BucketName, @@ -260,7 +265,10 @@ public virtual Stream OpenFile(string path) stream.Seek(0, SeekOrigin.Begin); if (FileCacheProvider != null) + { + Logger.Info(typeof(BucketFileSystem), "Persisting {path} to cache", path); FileCacheProvider.Persist(path, stream); + } return stream; } diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3/BucketFileSystemConfig.cs b/Umbraco.Storage.S3/Umbraco.Storage.S3/BucketFileSystemConfig.cs index a4fc655..43a3a5e 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3/BucketFileSystemConfig.cs +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3/BucketFileSystemConfig.cs @@ -16,6 +16,10 @@ public class BucketFileSystemConfig public ServerSideEncryptionMethod ServerSideEncryptionMethod { get; set; } + public bool CacheEnabled { get; set; } + + public int CacheMinutes { get; set; } + public bool DisableVirtualPathProvider { get; set; } } } diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.Core.nuspec b/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.Core.nuspec index 2e260f9..851614d 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.Core.nuspec +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.Core.nuspec @@ -2,7 +2,7 @@ CitadelGroup.Umbraco.FileSystemProviders.S3.Core - 8.2.0 + 8.4.0 $title$ Elijah Glover, Neil Gaietto, Danny Quarton, Damien Rider Elijah Glover, Neil Gaietto, Danny Quarton, Damien Rider @@ -14,8 +14,8 @@ Copyright 2021 Umbraco Aws S3 Provider - - + + diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.csproj b/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.csproj index fcf2211..1a5d43f 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.csproj +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.csproj @@ -43,10 +43,10 @@ - ..\packages\AWSSDK.Core.3.3.103.29\lib\net45\AWSSDK.Core.dll + ..\packages\AWSSDK.Core.3.7.0.44\lib\net45\AWSSDK.Core.dll - ..\packages\AWSSDK.S3.3.3.104.17\lib\net45\AWSSDK.S3.dll + ..\packages\AWSSDK.S3.3.7.1.14\lib\net45\AWSSDK.S3.dll ..\packages\LightInject.5.4.0\lib\net46\LightInject.dll @@ -175,7 +175,7 @@ - + diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3/packages.config b/Umbraco.Storage.S3/Umbraco.Storage.S3/packages.config index 115f076..5413f15 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3/packages.config +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3/packages.config @@ -1,7 +1,7 @@  - - + + From fe82dd29da6f66f81cb07eb274d99f5e94d4aac6 Mon Sep 17 00:00:00 2001 From: Damien Rider Date: Tue, 27 Jul 2021 14:41:38 +1000 Subject: [PATCH 4/5] Swap logging to verbose --- Umbraco.Storage.S3/Umbraco.Storage.S3/BucketFileSystem.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3/BucketFileSystem.cs b/Umbraco.Storage.S3/Umbraco.Storage.S3/BucketFileSystem.cs index cd09bb1..0eb02ba 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3/BucketFileSystem.cs +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3/BucketFileSystem.cs @@ -241,12 +241,12 @@ public virtual Stream OpenFile(string path) var persistedStream = FileCacheProvider.Resolve(path); if (persistedStream != null) { - Logger.Info(typeof(BucketFileSystem), "Loading {path} from cache", path); + Logger.Verbose(typeof(BucketFileSystem), "Loading {path} from cache", path); return persistedStream; } } - Logger.Info(typeof(BucketFileSystem), "Loading {path} from S3", path); + Logger.Verbose(typeof(BucketFileSystem), "Loading {path} from S3", path); var request = new GetObjectRequest { @@ -266,7 +266,7 @@ public virtual Stream OpenFile(string path) if (FileCacheProvider != null) { - Logger.Info(typeof(BucketFileSystem), "Persisting {path} to cache", path); + Logger.Verbose(typeof(BucketFileSystem), "Persisting {path} to cache", path); FileCacheProvider.Persist(path, stream); } From 81eefd1e668c6ba3d2aa5c64311f5e762652d4d4 Mon Sep 17 00:00:00 2001 From: Damien Rider Date: Thu, 10 Feb 2022 09:21:21 +1000 Subject: [PATCH 5/5] Update nuspec --- .../Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.nuspec | 4 ++-- .../Umbraco.Storage.S3/Umbraco.Storage.S3.Core.nuspec | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.nuspec b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.nuspec index bb10c9f..a6deb2b 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.nuspec +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3.Media/Umbraco.Storage.S3.Media.nuspec @@ -2,7 +2,7 @@ CitadelGroup.Umbraco.FileSystemProviders.S3.Media - 8.4.0 + 8.4.1 $author$ $author$ https://github.com/kingdamo/Umbraco-S3-Provider @@ -12,7 +12,7 @@ Copyright 2021 Umbraco Aws S3 Provider Media - + \ No newline at end of file diff --git a/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.Core.nuspec b/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.Core.nuspec index 851614d..067d679 100644 --- a/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.Core.nuspec +++ b/Umbraco.Storage.S3/Umbraco.Storage.S3/Umbraco.Storage.S3.Core.nuspec @@ -2,7 +2,7 @@ CitadelGroup.Umbraco.FileSystemProviders.S3.Core - 8.4.0 + 8.4.1 $title$ Elijah Glover, Neil Gaietto, Danny Quarton, Damien Rider Elijah Glover, Neil Gaietto, Danny Quarton, Damien Rider