From 0c36335b7e2f68933f387bcc8a542809c3fbb97f Mon Sep 17 00:00:00 2001 From: Marc Duiker Date: Sun, 24 Mar 2019 23:32:18 +0100 Subject: [PATCH] Added code to sanitize Partition- and Rowkeys. Refactored to use builders for LatestReleases/Publications. --- .../Models/LatestReleasesTests.cs | 52 +++++++++++++++---- .../Storage/KeyFormatterTests.cs | 25 +++++++++ .../Activities/PostUpdate.cs | 7 ++- .../Builders/LatestObjectsBuilder.cs | 22 ++++++++ .../Builders/PublicationFunctionBuilder.cs | 14 +++++ .../Builders/ReleaseFunctionBuilder.cs | 13 +++++ .../Models/Publications/LatestPublications.cs | 26 ++++------ .../Models/Publications/Publication.cs | 5 +- .../RepositoryReleases/LatestReleases.cs | 21 +++----- .../RepositoryReleases/RepositoryRelease.cs | 5 +- .../PublicationUpdateOrchestration.cs | 15 +++++- .../ReleaseUpdateOrchestration.cs | 27 +++++++--- .../Storage/KeyFormatter.cs | 18 +++++++ 13 files changed, 193 insertions(+), 57 deletions(-) create mode 100644 src/AzureFunctionsUpdates.UnitTests/Storage/KeyFormatterTests.cs create mode 100644 src/AzureFunctionsUpdates/Builders/LatestObjectsBuilder.cs create mode 100644 src/AzureFunctionsUpdates/Builders/PublicationFunctionBuilder.cs create mode 100644 src/AzureFunctionsUpdates/Builders/ReleaseFunctionBuilder.cs create mode 100644 src/AzureFunctionsUpdates/Storage/KeyFormatter.cs diff --git a/src/AzureFunctionsUpdates.UnitTests/Models/LatestReleasesTests.cs b/src/AzureFunctionsUpdates.UnitTests/Models/LatestReleasesTests.cs index 9afb593..35539c8 100644 --- a/src/AzureFunctionsUpdates.UnitTests/Models/LatestReleasesTests.cs +++ b/src/AzureFunctionsUpdates.UnitTests/Models/LatestReleasesTests.cs @@ -1,7 +1,7 @@ -using AzureFunctionsUpdates.Models; -using AzureFunctionsUpdates.UnitTests.TestObjectBuilders; +using AzureFunctionsUpdates.UnitTests.TestObjectBuilders; using FluentAssertions; using System; +using AzureFunctionsUpdates.Builders; using AzureFunctionsUpdates.Models.RepositoryReleases; using Xunit; @@ -17,9 +17,14 @@ public void GivenHistoryReleaseIsNullRelease_WhenIsNewAndShouldBeStoredIsCalled_ var repoConfig = RepositoryConfigurationBuilder.BuildOne(repoName); var releasesFromGitHub = RepositoryReleaseBuilder.BuildListContainingOneWithReleaseId(repoName, 1); var releasesFromHistory = RepositoryReleaseBuilder.BuildListContainingOneNullRelease(repoName); - + var releaseMatchFunction = ReleaseFunctionBuilder.BuildForMatchingRepositoryName(); + // Act - var latestReleases = new LatestReleases(repoConfig, releasesFromGitHub, releasesFromHistory); + var latestReleases = LatestObjectsBuilder.Build( + repoConfig, + releasesFromGitHub, + releasesFromHistory, + releaseMatchFunction); // Assert latestReleases.IsNewAndShouldBeStored.Should().BeTrue("because no release was found in history data."); @@ -34,9 +39,14 @@ public void GivenHistoryReleaseIsReleaseWithMatchingReleaseId_WhenIsNewAndShould var repoConfig = RepositoryConfigurationBuilder.BuildOne(repoName); var releasesFromGitHub = RepositoryReleaseBuilder.BuildListContainingOneWithReleaseId(repoName, releaseId); var releasesFromHistory = RepositoryReleaseBuilder.BuildListContainingOneWithReleaseId(repoName, releaseId); - + var releaseMatchFunction = ReleaseFunctionBuilder.BuildForMatchingRepositoryName(); + // Act - var latestReleases = new LatestReleases(repoConfig, releasesFromGitHub, releasesFromHistory); + var latestReleases = LatestObjectsBuilder.Build( + repoConfig, + releasesFromGitHub, + releasesFromHistory, + releaseMatchFunction); // Assert latestReleases.IsNewAndShouldBeStored.Should().BeFalse("because the releaseIds are equal"); @@ -50,9 +60,14 @@ public void GivenHistoryReleaseIsNullReleaseAndGitHubReleaseIsNullRelease_WhenIs var repoConfig = RepositoryConfigurationBuilder.BuildOne(repoName); var releasesFromGitHub = RepositoryReleaseBuilder.BuildListContainingOneNullRelease(repoName); var releasesFromHistory = RepositoryReleaseBuilder.BuildListContainingOneNullRelease(repoName); - + var releaseMatchFunction = ReleaseFunctionBuilder.BuildForMatchingRepositoryName(); + // Act - var latestReleases = new LatestReleases(repoConfig, releasesFromGitHub, releasesFromHistory); + var latestReleases = LatestObjectsBuilder.Build( + repoConfig, + releasesFromGitHub, + releasesFromHistory, + releaseMatchFunction); // Assert latestReleases.IsNewAndShouldBeStored.Should().BeFalse("because there is no result from GitHub"); @@ -68,9 +83,14 @@ public void GivenHistoryReleaseIsReleaseWithNonMatchingReleaseId_WhenIsNewAndSho var repoConfig = RepositoryConfigurationBuilder.BuildOne(repoName); var releasesFromGitHub = RepositoryReleaseBuilder.BuildListContainingOneWithReleaseId(repoName, releaseIdGithub); var releasesFromHistory = RepositoryReleaseBuilder.BuildListContainingOneWithReleaseId(repoName, releaseIdHistory); + var releaseMatchFunction = ReleaseFunctionBuilder.BuildForMatchingRepositoryName(); // Act - var latestReleases = new LatestReleases(repoConfig, releasesFromGitHub, releasesFromHistory); + var latestReleases = LatestObjectsBuilder.Build( + repoConfig, + releasesFromGitHub, + releasesFromHistory, + releaseMatchFunction); // Assert latestReleases.IsNewAndShouldBeStored.Should().BeTrue("because the releaseIds are not equal"); @@ -87,9 +107,14 @@ public void GivenHistoryReleaseIsNullReleaseAndGitHubReleaseIsWithinTimeWindow_W var repoConfig = RepositoryConfigurationBuilder.BuildOne(repoName); var releasesFromGitHub = RepositoryReleaseBuilder.BuildListContainingOneWithReleaseIdAndDate(repoName, releaseIdGithub, gitHubReleaseDate); var releasesFromHistory = RepositoryReleaseBuilder.BuildListContainingOneNullRelease(repoName); + var releaseMatchFunction = ReleaseFunctionBuilder.BuildForMatchingRepositoryName(); // Act - var latestReleases = new LatestReleases(repoConfig, releasesFromGitHub, releasesFromHistory); + var latestReleases = LatestObjectsBuilder.Build( + repoConfig, + releasesFromGitHub, + releasesFromHistory, + releaseMatchFunction); // Assert latestReleases.IsNewAndShouldBeStored.Should().BeTrue("because the release is not in history yet."); @@ -107,9 +132,14 @@ public void GivenHistoryReleaseIsNullReleaseAndGitHubReleaseIsOutsideTimeWindow_ var repoConfig = RepositoryConfigurationBuilder.BuildOne(repoName); var releasesFromGitHub = RepositoryReleaseBuilder.BuildListContainingOneWithReleaseIdAndDate(repoName, releaseIdGithub, gitHubReleaseDate); var releasesFromHistory = RepositoryReleaseBuilder.BuildListContainingOneNullRelease(repoName); + var releaseMatchFunction = ReleaseFunctionBuilder.BuildForMatchingRepositoryName(); // Act - var latestReleases = new LatestReleases(repoConfig, releasesFromGitHub, releasesFromHistory); + var latestReleases = LatestObjectsBuilder.Build( + repoConfig, + releasesFromGitHub, + releasesFromHistory, + releaseMatchFunction); // Assert latestReleases.IsNewAndShouldBeStored.Should().BeTrue("because the release is not in history yet."); diff --git a/src/AzureFunctionsUpdates.UnitTests/Storage/KeyFormatterTests.cs b/src/AzureFunctionsUpdates.UnitTests/Storage/KeyFormatterTests.cs new file mode 100644 index 0000000..ddfc35d --- /dev/null +++ b/src/AzureFunctionsUpdates.UnitTests/Storage/KeyFormatterTests.cs @@ -0,0 +1,25 @@ +using AzureFunctionsUpdates.Storage; +using FluentAssertions; +using Xunit; + +namespace AzureFunctionsUpdates.UnitTests.Storage +{ + public class KeyFormatterTests + { + [Theory] + [InlineData("abcd123", "abcd123")] + [InlineData("abcd-123", "abcd-123")] + [InlineData("abcd?123", "abcd-123")] + [InlineData("abcd#123#", "abcd-123-")] + [InlineData("abcd/123", "abcd-123")] + [InlineData("abcd\\1?2#3", "abcd-1-2-3")] + public void SanitizeKeyTests(string oldKey, string newKey) + { + // Act + var result = KeyFormatter.SanitizeKey(oldKey); + + // Assert + result.Should().Be(newKey); + } + } +} \ No newline at end of file diff --git a/src/AzureFunctionsUpdates/Activities/PostUpdate.cs b/src/AzureFunctionsUpdates/Activities/PostUpdate.cs index bc03983..f2c8c5a 100644 --- a/src/AzureFunctionsUpdates/Activities/PostUpdate.cs +++ b/src/AzureFunctionsUpdates/Activities/PostUpdate.cs @@ -23,10 +23,9 @@ public void Run( var creds = new TwitterCredentials(consumerApiKey, consumerApiSecret, accessToken, accessTokenSecret); - var tweet = Auth.ExecuteOperationWithCredentials(creds, () => - { - return Tweet.PublishTweet(message.Content); - }); + var tweet = Auth.ExecuteOperationWithCredentials(creds, () => Tweet.PublishTweet(message.Content)); + + logger.LogInformation($"Finished {nameof(PostUpdate)} with tweet: {tweet.Url}."); } } } diff --git a/src/AzureFunctionsUpdates/Builders/LatestObjectsBuilder.cs b/src/AzureFunctionsUpdates/Builders/LatestObjectsBuilder.cs new file mode 100644 index 0000000..4850822 --- /dev/null +++ b/src/AzureFunctionsUpdates/Builders/LatestObjectsBuilder.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AzureFunctionsUpdates.Models.RepositoryReleases; + +namespace AzureFunctionsUpdates.Builders +{ + public static class LatestObjectsBuilder + { + public static TLatest Build( + TConfiguration configuration, + IEnumerable newObjects, + IEnumerable historicalObjects, + Func matchObject) where TLatest : new() + { + var latestNew = newObjects.First(obj => matchObject(configuration, obj)); + var latestHistorical = historicalObjects.First(obj => matchObject(configuration, obj)); + + return (TLatest)Activator.CreateInstance(typeof(TLatest), latestNew, latestHistorical); + } + } +} \ No newline at end of file diff --git a/src/AzureFunctionsUpdates/Builders/PublicationFunctionBuilder.cs b/src/AzureFunctionsUpdates/Builders/PublicationFunctionBuilder.cs new file mode 100644 index 0000000..1ba8269 --- /dev/null +++ b/src/AzureFunctionsUpdates/Builders/PublicationFunctionBuilder.cs @@ -0,0 +1,14 @@ +using System; +using AzureFunctionsUpdates.Models.Publications; +using AzureFunctionsUpdates.Models.RepositoryReleases; + +namespace AzureFunctionsUpdates.Builders +{ + public static class PublicationFunctionBuilder + { + public static Func BuildForMatchingPublicationSource() + { + return (config, publication) => publication.PublicationSourceName.Equals(config.PublicationSourceName, StringComparison.InvariantCultureIgnoreCase); + } + } +} \ No newline at end of file diff --git a/src/AzureFunctionsUpdates/Builders/ReleaseFunctionBuilder.cs b/src/AzureFunctionsUpdates/Builders/ReleaseFunctionBuilder.cs new file mode 100644 index 0000000..001e35d --- /dev/null +++ b/src/AzureFunctionsUpdates/Builders/ReleaseFunctionBuilder.cs @@ -0,0 +1,13 @@ +using System; +using AzureFunctionsUpdates.Models.RepositoryReleases; + +namespace AzureFunctionsUpdates.Builders +{ + public static class ReleaseFunctionBuilder + { + public static Func BuildForMatchingRepositoryName() + { + return (config, release) => release.RepositoryName.Equals(config.RepositoryName, StringComparison.InvariantCultureIgnoreCase); + } + } +} \ No newline at end of file diff --git a/src/AzureFunctionsUpdates/Models/Publications/LatestPublications.cs b/src/AzureFunctionsUpdates/Models/Publications/LatestPublications.cs index 450f62c..07c0fd7 100644 --- a/src/AzureFunctionsUpdates/Models/Publications/LatestPublications.cs +++ b/src/AzureFunctionsUpdates/Models/Publications/LatestPublications.cs @@ -6,25 +6,21 @@ namespace AzureFunctionsUpdates.Models.Publications { public class LatestPublications { - private readonly IReadOnlyList _latestPublicationsFromWeb; - private readonly IReadOnlyList _latestPublicationsFromHistory; - private readonly PublicationConfiguration _publicationConfiguration; - + public LatestPublications() + {} + public LatestPublications( - PublicationConfiguration publicationConfiguration, - IReadOnlyList latestReleasesFromGitHub, - IReadOnlyList latestReleasesFromHistory) + + Publication fromWeb, + Publication fromHistory) { - this._publicationConfiguration = publicationConfiguration; - _latestPublicationsFromWeb = latestReleasesFromGitHub; - _latestPublicationsFromHistory = latestReleasesFromHistory; - } - - + FromWeb = fromWeb; + FromHistory = fromHistory; + } - public Publication FromWeb => _latestPublicationsFromWeb.First(publication => publication.PublicationSourceName.Equals(_publicationConfiguration.PublicationSourceName, StringComparison.InvariantCultureIgnoreCase)); + public Publication FromWeb { get; } - public Publication FromHistory => _latestPublicationsFromHistory.First(publication => publication.PublicationSourceName.Equals(_publicationConfiguration.PublicationSourceName, StringComparison.InvariantCultureIgnoreCase)); + public Publication FromHistory { get; } public bool IsNewAndShouldBeStored { diff --git a/src/AzureFunctionsUpdates/Models/Publications/Publication.cs b/src/AzureFunctionsUpdates/Models/Publications/Publication.cs index 2b8b53a..df15622 100644 --- a/src/AzureFunctionsUpdates/Models/Publications/Publication.cs +++ b/src/AzureFunctionsUpdates/Models/Publications/Publication.cs @@ -1,5 +1,6 @@ using Microsoft.WindowsAzure.Storage.Table; using System; +using AzureFunctionsUpdates.Storage; namespace AzureFunctionsUpdates.Models.Publications { @@ -17,9 +18,9 @@ public Publication( string url, string hashTags) { - PartitionKey = publicationSourceName; + PartitionKey = KeyFormatter.SanitizeKey(publicationSourceName); PublicationSourceName = publicationSourceName; - RowKey = $"{id}-{publicationDate}"; + RowKey = KeyFormatter.SanitizeKey($"{id}-{publicationDate.ToUnixTimeSeconds()}"); Id = id; PublicationDate = publicationDate; Title = title; diff --git a/src/AzureFunctionsUpdates/Models/RepositoryReleases/LatestReleases.cs b/src/AzureFunctionsUpdates/Models/RepositoryReleases/LatestReleases.cs index e96da5c..53eb8bb 100644 --- a/src/AzureFunctionsUpdates/Models/RepositoryReleases/LatestReleases.cs +++ b/src/AzureFunctionsUpdates/Models/RepositoryReleases/LatestReleases.cs @@ -6,23 +6,20 @@ namespace AzureFunctionsUpdates.Models.RepositoryReleases { public class LatestReleases { - private readonly IReadOnlyList _latestReleasesFromGitHub; - private readonly IReadOnlyList _latestReleasesFromHistory; - private readonly RepositoryConfiguration _repository; + public LatestReleases() + {} public LatestReleases( - RepositoryConfiguration repository, - IReadOnlyList latestReleasesFromGitHub, - IReadOnlyList latestReleasesFromHistory) + RepositoryRelease latestReleaseFromGitHub, + RepositoryRelease latestReleaseFromHistory) { - _repository = repository; - _latestReleasesFromGitHub = latestReleasesFromGitHub; - _latestReleasesFromHistory = latestReleasesFromHistory; + FromGitHub = latestReleaseFromGitHub; + FromHistory = latestReleaseFromHistory; } - public RepositoryRelease FromGitHub => _latestReleasesFromGitHub.First(release => release.RepositoryName.Equals(_repository.RepositoryName, StringComparison.InvariantCultureIgnoreCase)); + public RepositoryRelease FromGitHub { get; } - public RepositoryRelease FromHistory => _latestReleasesFromHistory.First(release => release.RepositoryName.Equals(_repository.RepositoryName, StringComparison.InvariantCultureIgnoreCase)); + public RepositoryRelease FromHistory { get; } public bool IsNewAndShouldBeStored { @@ -53,8 +50,6 @@ bool IsWithinTimeWindow() return DateTimeOffset.UtcNow.Subtract(FromGitHub.ReleaseCreatedAt).Days < MaximumNumberOfDaysToPostAboutNewlyFoundRelease; } } - - } public const int MaximumNumberOfDaysToPostAboutNewlyFoundRelease = 2; diff --git a/src/AzureFunctionsUpdates/Models/RepositoryReleases/RepositoryRelease.cs b/src/AzureFunctionsUpdates/Models/RepositoryReleases/RepositoryRelease.cs index c16a54d..b615695 100644 --- a/src/AzureFunctionsUpdates/Models/RepositoryReleases/RepositoryRelease.cs +++ b/src/AzureFunctionsUpdates/Models/RepositoryReleases/RepositoryRelease.cs @@ -1,4 +1,5 @@ using System; +using AzureFunctionsUpdates.Storage; using Microsoft.WindowsAzure.Storage.Table; namespace AzureFunctionsUpdates.Models.RepositoryReleases @@ -19,8 +20,8 @@ public RepositoryRelease( string hashTags ) { - PartitionKey = repositoryName; - RowKey = $"{releaseId.ToString()}-{tagName}"; + PartitionKey = KeyFormatter.SanitizeKey(repositoryName); + RowKey = KeyFormatter.SanitizeKey($"{releaseId.ToString()}-{tagName}"); ReleaseId = releaseId; RepositoryName = repositoryName; diff --git a/src/AzureFunctionsUpdates/Orchestrations/PublicationUpdateOrchestration.cs b/src/AzureFunctionsUpdates/Orchestrations/PublicationUpdateOrchestration.cs index 8506c3f..79434af 100644 --- a/src/AzureFunctionsUpdates/Orchestrations/PublicationUpdateOrchestration.cs +++ b/src/AzureFunctionsUpdates/Orchestrations/PublicationUpdateOrchestration.cs @@ -51,12 +51,23 @@ public async Task Run( var latestFromWeb = await Task.WhenAll(getLatestPublicationsFromWebTasks); var latestFromHistory = await Task.WhenAll(getLatestPublicationsFromHistoryTasks); + var publicationMatchFunction = PublicationFunctionBuilder.BuildForMatchingPublicationSource(); foreach (var publicationConfiguration in publicationConfigurations) { - var latestPublications = new LatestPublications(publicationConfiguration, latestFromWeb, latestFromHistory); + var latestPublications = LatestObjectsBuilder.Build( + publicationConfiguration, + latestFromWeb, + latestFromHistory, + publicationMatchFunction); + + logger.LogInformation($"Publication: {publicationConfiguration.PublicationSourceName} " + + $"ID: {latestPublications.FromWeb.Id}," + + $"IsNewAndShouldBeStored: {latestPublications.IsNewAndShouldBeStored}, " + + $"IsNewAndShouldBePosted: {latestPublications.IsNewAndShouldBePosted}."); + if (latestPublications.IsNewAndShouldBeStored) - { + { var isSaveSuccessful = await context.CallActivityWithRetryAsync( nameof(SaveLatestPublication), GetDefaultRetryOptions(), diff --git a/src/AzureFunctionsUpdates/Orchestrations/ReleaseUpdateOrchestration.cs b/src/AzureFunctionsUpdates/Orchestrations/ReleaseUpdateOrchestration.cs index c1524c0..5f896db 100644 --- a/src/AzureFunctionsUpdates/Orchestrations/ReleaseUpdateOrchestration.cs +++ b/src/AzureFunctionsUpdates/Orchestrations/ReleaseUpdateOrchestration.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using AzureFunctionsUpdates.Activities.RepositoryReleases; using AzureFunctionsUpdates.Models.RepositoryReleases; +using AzureFunctionsUpdates.Storage; namespace AzureFunctionsUpdates.Orchestrations { @@ -22,38 +23,48 @@ public async Task Run( logger.LogInformation($"Started {nameof(ReleaseUpdateOrchestration)}."); // Read repo links from storage table - var repositories = await context.CallActivityWithRetryAsync>( + var repositoryConfigurations = await context.CallActivityWithRetryAsync>( functionName: nameof(GetRepositoryConfigurations), retryOptions: GetDefaultRetryOptions(), input: null); - if (repositories.Any()) + if (repositoryConfigurations.Any()) { var getLatestReleaseFromGitHubTasks = new List>(); var getLatestReleasesFromHistoryTasks = new List>(); // Fan out over the repos - foreach (var repo in repositories) + foreach (var repositoryConfiguration in repositoryConfigurations) { // Get most recent release from GitHub getLatestReleaseFromGitHubTasks.Add(context.CallActivityWithRetryAsync( nameof(GetLatestReleaseFromGitHub), GetDefaultRetryOptions(), - repo)); + repositoryConfiguration)); // Get most recent known releases from history getLatestReleasesFromHistoryTasks.Add(context.CallActivityWithRetryAsync( nameof(GetLatestReleaseFromHistory), GetDefaultRetryOptions(), - repo)); + repositoryConfiguration)); } var latestFromGitHub = await Task.WhenAll(getLatestReleaseFromGitHubTasks); var latestFromHistory = await Task.WhenAll(getLatestReleasesFromHistoryTasks); - - foreach (var repo in repositories) + var releaseMatchFunction = ReleaseFunctionBuilder.BuildForMatchingRepositoryName(); + + foreach (var repositoryConfiguration in repositoryConfigurations) { - var latestReleases = new LatestReleases(repo, latestFromGitHub, latestFromHistory); + var latestReleases = LatestObjectsBuilder.Build( + repositoryConfiguration, + latestFromGitHub, + latestFromHistory, + releaseMatchFunction); + logger.LogInformation($"Repository: {repositoryConfiguration.RepositoryName} " + + $"Tag: {latestReleases.FromGitHub.TagName}," + + $"IsNewAndShouldBeStored: {latestReleases.IsNewAndShouldBeStored}, " + + $"IsNewAndShouldBePosted: {latestReleases.IsNewAndShouldBePosted}."); + if (latestReleases.IsNewAndShouldBeStored) { var isSaveSuccessful = await context.CallActivityWithRetryAsync( diff --git a/src/AzureFunctionsUpdates/Storage/KeyFormatter.cs b/src/AzureFunctionsUpdates/Storage/KeyFormatter.cs new file mode 100644 index 0000000..9a0cac8 --- /dev/null +++ b/src/AzureFunctionsUpdates/Storage/KeyFormatter.cs @@ -0,0 +1,18 @@ +namespace AzureFunctionsUpdates.Storage +{ + public static class KeyFormatter + { + public static string SanitizeKey(string key) + { + var illegalChars = new[] {'/', '\\', '#', '?'}; + const char replacementChar = '-'; + + foreach (var illegalChar in illegalChars) + { + key = key.Replace(illegalChar, replacementChar); + } + + return key; + } + } +} \ No newline at end of file