diff --git a/docs/developers/decisions/backend/0019-updating-mods.md b/docs/developers/decisions/backend/0019-updating-mods.md index f145874808..22fbc53a01 100644 --- a/docs/developers/decisions/backend/0019-updating-mods.md +++ b/docs/developers/decisions/backend/0019-updating-mods.md @@ -9,28 +9,198 @@ A corresponding research document (original design) can be found on a [separate !!! tip "First read the [Problem Statement] in the [Research Document]" The requested approach (from business) has been to maximize the use of the V2 API, -as opposed to programming against the legacy V1 API. +as opposed to programming against the legacy V1 API wherever possible. -To achieve this, we will [NOT use the `file_updates` array from V1 API's Querying Mod Files][querying-mod-files]; -instead choosing to opt to wait until backend decides their future plans with -respect to 'Mods 2.0' project, and how mod updates will be handled in V2 API in the future. +!!! info "We will wait for the 'Mods 2.0' project if possible" + + Including how `Mod Updates` are to be handled in the future. + +To promote an iterative development and speed up production, this feature will be developed in +'phases'. Each phase will deliver a Minimum Viable Product (MVP) for a given feature set, then +once all initial functionality is completed, various aspects will be refined. + +### Step 1: Updating Mod Page Info For now, we will: - [1. Determine Updated Mod Pages], to update our local cache. - [2. Multi Query Pages], for update mod pages with a 'cache miss'. -## Displaying Mod Updates +### Step 2: Mapping Old Files to New Files + +!!! info "Once we have the updated mod pages, we can determine the updates available for files" + +- [Use the 'fuzzy' search strategy](#fuzzy-search-strategy) to match files. (Phase 0) + +***Once all other work around updates (UI, etc.)*** is complete, with a working prototype, we will +improve on this by doing the following (Phase 1): + +- [Use the `file_updates` array from V1 API's Querying Mod Files][querying-mod-files] + - Or an equivalent V2 API, if available. + +## Fuzzy Search Strategy + +!!! info "This is a 'cheap' strategy to try detect file updates in the presence of [missing update links][querying-mod-files]" + +Basically we match uploads by file name; trying to match mods together. + +Mods file names can be generally divided into the following classes: + +- Consistent-ish naming, with version in mod name (e.g. [Skyrim 202X](#reference-example-skyrim-202x)). +- Constant naming e.g. [USSEP](#reference-example-ussep) +- File Names with Substituted Characters e.g. [SkyUI](#reference-example-skyui) +- File Versions with non-semver suffixes e.g. [A Quality World Map](#reference-example-a-quality-world-map) +- File Names with File Extensions [Maestros of Synth](#reference-example-maestros-of-synth) + +The strategy is the following: + +1. Try to parse version strings into variants that may appear in file name: + - Make it semver compliant (e.g. `5.2SE` -> `5.2`) + - With stripped version prefix (e.g. `v5.2` -> `5.2`) + - With substituted underscores (e.g. `5.2` -> `5_2`) +2. Strip each of the above strings from the file name (loop, case insensitive). +3. Strip known file extensions from end (e.g. `Maestros of Synth.zip` -> `Maestros of Synth`) +4. Substitute underscores with spaces (e.g. `Ava_Complexions` -> `Ava Complexions`) + - In case mod author mixed spaces and underscores between mod authors. +5. Remove runs of spaces. +6. Match old file to current file by what remains of file name (case insensitive). + - Where file is newer and version string is newer (if possible to compare). + +### Reference Example: [Skyrim 202X] + +!!! info "Example with consistent naming for full releases, but incosistent naming for non-full releases." + +v9 Release: + +``` +-Skyrim 202X 9.0 - Architecture PART 1 +-Skyrim 202X 9.0 - Landscape PART 2 +-Skyrim 202X 9.0 - Other PART 3 +``` + +v10 Release: + +``` +-Skyrim 202X 10.0 - Architecture PART 1 +-Skyrim 202X 10.0 - Landscape PART 2 +-Skyrim 202X 10.0 - Other PART 3 +``` + +v10 Release (2): + +``` +-Skyrim 202X 10.0.1 - Architecture PART 1 +-Skyrim 202X 10.0.1 - Landscape PART 2 +-Skyrim 202X 10.0.1 - Other PART 3 +``` + +v10 Patches + +``` +-Skyrim 202X 10.1 Update +-Skyrim 202X 10.2 Update +-Skyrim 202X 10.3 Update +-Skyrim 202X 10.4 Update Solstheim [<= Does not contain files from other update packages] +``` +!!! note "Skyrim 202X 10.4 Update Solstheim does is meant to be installed over `10.3`" + + Rather than standalone. Edge cases like this is why we prefer to avoid false positives. + +In this case, our strategy extracts the strings: + +``` +-Skyrim 202X - Architecture PART 1 +-Skyrim 202X - Landscape PART 2 +-Skyrim 202X - Other PART 3 +``` + +Meaning files can be matched without correct `file_updates` mappings, even in presence of multiple +'parts' with same version number. + +### Reference Example: [USSEP] + +!!! info "Single mod, all same name, but version is provided in file name." + +All versions: + +``` +Unofficial Skyrim Special Edition Patch +``` + +In this case our logic still sees + +``` +Unofficial Skyrim Special Edition Patch +``` + +As the file names are all the same, it's an automatic match by file version. + +### Reference Example: [SkyUI] + +!!! info "Example case where part of the file name was substituted" + +``` +SkyUI_5_2_SE +SkyUI_5_1_SE +``` + +e.g. Spaces, dots, were substituted with underscores. More common with old/older files, rather than +recent ones. + +In this case the name becomes + +``` +SkyUI SE +SkyUI SE +``` + +We did the substitution of version to underscores and stripped it from the input. + +Notably the version string here is `5.2SE`, so we also need to filter out characters in version +strings. + +### Reference Example: [A Quality World Map] + +!!! info "Example where there are multiple variants of a mod, with differing version strings" + +``` +9.0 A Quality World Map - Paper +9.0 A Quality World Map - Vivid with Flat Roads +9.0 A Quality World Map - Vivid with Stone Roads +``` + +With the following version strings: + +``` +9.0P +9.0VF +9.0 +``` + +In this case, you want to filter out the 'P' and 'VF' in the version strings, turning our +file names to: + +``` +A Quality World Map - Paper +A Quality World Map - Vivid with Flat Roads +A Quality World Map - Vivid with Stone Roads +``` + +Then we can compare versions. -!!! info "We display all files on a given mod page that are more recent (file upload time) than the user's file." +### Reference Example: [Maestros of Synth] -Although uncommon this may include: +``` +Maestros of Synth.zip +Maestros of Synth +``` -- Files for other mods on same mod page. -- Older files (if uploaded out of order). +In this case, we strip file names: -We will for now rely on *users' common sense* to identify whether a file is an -update to a previous file or not. Until site decides on future plans. +``` +Maestros of Synth +Maestros of Synth +``` [Problem Statement]: ../../misc/research/00-update-implementation-research.md#problem-statement [1. Determine Updated Mod Pages]: ../../misc/research/00-update-implementation-research.md#1-determine-updated-mod-pages @@ -38,3 +208,8 @@ update to a previous file or not. Until site decides on future plans. [querying-mod-files]: ../../misc/research/00-update-implementation-research.md#2-querying-mod-files [Research Document]: ../../misc/research/00-update-implementation-research.md [research-doc]: ../../misc/research/00-update-implementation-research.md +[Skyrim 202X]: https://www.nexusmods.com/skyrimspecialedition/mods/2347?tab=files +[USSEP]: https://www.nexusmods.com/skyrimspecialedition/mods/266?tab=files +[SkyUI]: https://www.nexusmods.com/skyrimspecialedition/mods/12604?tab=files +[A Quality World Map]: https://www.nexusmods.com/skyrimspecialedition/mods/5804?tab=files +[Maestros of Synth]: https://www.nexusmods.com/cyberpunk2077/mods/3776?tab=files \ No newline at end of file diff --git a/src/Networking/NexusMods.Networking.ModUpdates/FuzzySearch.cs b/src/Networking/NexusMods.Networking.ModUpdates/FuzzySearch.cs new file mode 100644 index 0000000000..f88f8efe78 --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/FuzzySearch.cs @@ -0,0 +1,127 @@ +using System.Text.RegularExpressions; +namespace NexusMods.Networking.ModUpdates; + +/// +/// This class has all the code around the 'fuzzy search' feature documented in +/// `0019-updating-mods.md` page of the wiki. +/// +public static class FuzzySearch +{ + private static readonly string[] KnownFileExtensions = + [ + ".rar", ".zip", ".7z", ".exe", ".omod", ".nx", + ]; + + /// + /// Normalizes a filename by: + /// 1. Stripping all possible version number variants + /// 2. Replacing underscores with spaces + /// 3. Stripping file extensions + /// 4. Converting to lowercase for case-insensitive comparison + /// + /// The filename to normalize + /// The version to strip from the filename + /// The normalized filename + public static string NormalizeFileName(string? fileName, string? version) + { + if (string.IsNullOrWhiteSpace(fileName)) + return string.Empty; + + var result = fileName; + + // Strip all possible version number variants + if (!string.IsNullOrWhiteSpace(version)) + { + // Sort by length, to ensure we don't match a substring containing another string + var versionVariants = GetVersionPermutations(version).OrderByDescending(x => x.Length); + foreach (var variant in versionVariants) + result = result.Replace(variant, string.Empty, StringComparison.OrdinalIgnoreCase); + } + + // Replace underscores with spaces + result = result.Replace('_', ' '); + + // Strip extensions + result = StripFileExtension(result); + + // Normalize whitespace (trim and reduce multiple spaces to single) + result = string.Join(" ", result.Split([' '], + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + + // Convert to lowercase for case-insensitive comparison + return result.ToLowerInvariant(); + } + + /// + /// Given an existing version string, return all possible variations that + /// a user may place inside the file name on a mod page. + /// + /// Refer to tests of this for examples. + /// + /// The original version string attached + public static string[] GetVersionPermutations(string version) + { + if (string.IsNullOrWhiteSpace(version)) + return []; + + var results = new HashSet(8); + void AddIfValid(string value) + { + if (!string.IsNullOrWhiteSpace(value)) + results.Add(value); + } + + void AddVariants(string input) + { + AddIfValid(input); + AddIfValid(input.Replace(".", "_")); + } + + // Add original version and its variants + AddVariants(version); + + // Remove version suffix. + var withoutTextSuffix = version; + for (var x = version.Length - 1; x >= 0; x--) + { + if (!char.IsNumber(version[x])) continue; + withoutTextSuffix = version.Substring(0, x + 1); + break; + } + + if (withoutTextSuffix != version) + AddVariants(withoutTextSuffix); + + // Handle Single letter version prefixes variants ('v' for version, 'a' for 'alpha', 'b' for 'beta', etc.) + if (version.Length > 0 && char.IsLetter(version[0])) + { + var withoutPrefix = version.Substring(1); + AddVariants(withoutPrefix); + + // Also add variants without both prefix and suffix + if (withoutTextSuffix != version) + AddVariants(withoutPrefix.Substring(0, withoutTextSuffix.Length - 1)); + } + + return results.ToArray(); + } + + /// + /// Strips known file extensions from the end of a filename + /// + /// The filename to process + /// Filename with any known extension removed + public static string StripFileExtension(string fileName) + { + if (string.IsNullOrWhiteSpace(fileName)) + return fileName; + + foreach (var ext in KnownFileExtensions) + { + if (fileName.EndsWith(ext, StringComparison.OrdinalIgnoreCase)) + return fileName[..^ext.Length]; + } + + return fileName; + } +} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/Mixins/ModFileMetadataMixin.cs b/src/Networking/NexusMods.Networking.ModUpdates/Mixins/ModFileMetadataMixin.cs new file mode 100644 index 0000000000..904aad706d --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/Mixins/ModFileMetadataMixin.cs @@ -0,0 +1,34 @@ +using NexusMods.Abstractions.NexusModsLibrary; +using NexusMods.Networking.ModUpdates.Traits; +namespace NexusMods.Networking.ModUpdates.Mixins; + +/// +/// A mixin for mod file metadata that allows you to integrate mod file results +/// from the Nexus API. +/// +public class ModFileMetadataMixin : IAmAModFile +{ + /// + public string Name => Metadata.Name; + + /// + public string Version => Metadata.Version; + + /// + public DateTimeOffset UploadedAt => Metadata.UploadedAt; + + /// + public IEnumerable OtherFilesInSameModPage => + Metadata.ModPage.Files.Select(f => new ModFileMetadataMixin(f)); + + /// + public NexusModsFileMetadata.ReadOnly Metadata { get; } + + /// + public ModFileMetadataMixin(NexusModsFileMetadata.ReadOnly metadata) => Metadata = metadata; + + /// + /// Returns a normalized name and version. + /// + public string GetNormalizedFileName() => FuzzySearch.NormalizeFileName(Name, Version); +} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/Traits/IAmAModFile.cs b/src/Networking/NexusMods.Networking.ModUpdates/Traits/IAmAModFile.cs new file mode 100644 index 0000000000..ffc078e429 --- /dev/null +++ b/src/Networking/NexusMods.Networking.ModUpdates/Traits/IAmAModFile.cs @@ -0,0 +1,31 @@ +namespace NexusMods.Networking.ModUpdates.Traits; + +/// +/// Specifies the minimum required interface to represent a mod file within a mod page. +/// This API provides the minimum required +/// +public interface IAmAModFile +{ + /// + /// The name of the mod file. (File Name) + /// + public string Name { get; } + + /// + /// The version of the mod file. (File Version) + /// + /// + /// The Nexus mod site allows arbitrary input here, so this is a string. + /// + public string Version { get; } + + /// + /// When the file was uploaded to Nexus Mods + /// + public DateTimeOffset UploadedAt { get; } + + /// + /// Returns all other files the same mod page. + /// + public IEnumerable OtherFilesInSameModPage { get; } +} diff --git a/src/Networking/NexusMods.Networking.ModUpdates/Traits/ICanGetLastUpdatedTimestamp.cs b/src/Networking/NexusMods.Networking.ModUpdates/Traits/ICanGetLastUpdatedTimestamp.cs deleted file mode 100644 index 37fc2c1737..0000000000 --- a/src/Networking/NexusMods.Networking.ModUpdates/Traits/ICanGetLastUpdatedTimestamp.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace NexusMods.Networking.ModUpdates.Traits; - -/// -/// This interface marks an item which has the time it was last updated. -/// -public interface ICanGetLastUpdatedTimestamp -{ - /// - /// Retrieves the time the item was last updated. - /// - public DateTime GetLastUpdatedDate(); -} diff --git a/src/Networking/NexusMods.Networking.NexusWebApi/RunUpdateCheck.cs b/src/Networking/NexusMods.Networking.NexusWebApi/RunUpdateCheck.cs index 98e0e2e5ec..bdaa9861fe 100644 --- a/src/Networking/NexusMods.Networking.NexusWebApi/RunUpdateCheck.cs +++ b/src/Networking/NexusMods.Networking.NexusWebApi/RunUpdateCheck.cs @@ -6,6 +6,7 @@ using NexusMods.MnemonicDB.Abstractions; using NexusMods.Networking.ModUpdates; using NexusMods.Networking.ModUpdates.Mixins; +using NexusMods.Networking.ModUpdates.Traits; using NexusMods.Networking.NexusWebApi.Extensions; using StrawberryShake; namespace NexusMods.Networking.NexusWebApi; @@ -104,8 +105,36 @@ private static async Task UpdateModPage(IDb db, ITransaction tx, INexusGraphQLCl /// /// Returns all files which have a 'newer' date than the current one. /// - public static IEnumerable GetNewerFilesForExistingFile(NexusModsFileMetadata.ReadOnly file) + public static IEnumerable GetNewerFilesForExistingFile(NexusModsFileMetadata.ReadOnly file) + => GetNewerFilesForExistingFile(new ModFileMetadataMixin(file)).Select(x => ((ModFileMetadataMixin)x).Metadata); + + /// + /// Returns all files which have a 'newer' date than the current one, + /// using fuzzy name matching to handle variations in file naming. + /// + public static IEnumerable GetNewerFilesForExistingFile(IAmAModFile file) { - return file.ModPage.Files.Where(x => x.UploadedAt > file.UploadedAt); - } + // Get the normalized name for the current file + var normalizedName = FuzzySearch.NormalizeFileName(file.Name, file.Version); + + // Get all other files from the same mod page + return file.OtherFilesInSameModPage + .Where(otherFile => + // Must be uploaded later + // Note(sewer): + // + // In future we might check version too, but this may + // be a bit unreliable in cases where versions have + // a suffix and non-suffix variants. So these items would + // need to be put into a different group/pool to match from. + // + // Because, when mapped to SemVer, the suffix would be + // interpreted as a pre-release, and an item without a suffix + // may be assumed as a more recent version. + otherFile.UploadedAt > file.UploadedAt && + // Must have matching normalized name + FuzzySearch.NormalizeFileName(otherFile.Name, otherFile.Version) + .Equals(normalizedName, StringComparison.OrdinalIgnoreCase) + ); + } } diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/FuzzySearchTests.cs b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/FuzzySearchTests.cs new file mode 100644 index 0000000000..7f70d29607 --- /dev/null +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/FuzzySearchTests.cs @@ -0,0 +1,99 @@ +using FluentAssertions; +namespace NexusMods.Networking.ModUpdates.Tests; + +/// +/// Tests based around fuzzy searching of updates for files based on filenames. +/// +public class FuzzySearchTests +{ + [Theory] + // Reference Example: SkyUI (underscores and version in middle) + [InlineData("SkyUI_5_2_SE", "5.2SE", "skyui se")] + // Reference Example: Skyrim 202X (version after name) + [InlineData("Skyrim 202X 9.0 - Architecture PART 1", "9.0", "skyrim 202x - architecture part 1")] + // Reference Example: Quality World Map (file version with suffix) + [InlineData("9.0 A Quality World Map - Paper", "9.0P", "a quality world map - paper")] + // Reference Example: Maestros of Synth (extension stripping) + [InlineData("Maestros of Synth.zip", "", "maestros of synth")] + // USSEP style with underscores and extension + [InlineData("Unofficial_Skyrim_Special_Edition_Patch.rar", "", "unofficial skyrim special edition patch")] + // Multiple spaces normalization + [InlineData("Mod Name with multiple spaces", "", "mod name with multiple spaces")] + // Version with non-semver beta suffix + [InlineData("Mod 1.2beta.zip", "1.2beta", "mod")] + // Version with non-semver suffix and underscores + [InlineData("Cool_Mod_1_2alpha.7z", "1.2alpha", "cool mod")] + // (Sewer) Edge cases we don't currently handle: + // - Version with space between number and suffix in name + // - e.g. 'Cool Mod 1.2 alpha.7z' + '1.2alpha' will not eliminate '1.2 alpha', because version doesn't have a space in name. + public void NormalizeFileName_ShouldHandleVariousCases(string fileName, string version, string expected) + { + // Act + var result = FuzzySearch.NormalizeFileName(fileName, version); + + // Assert + result.Should().Be(expected); + } + + [Theory] + [InlineData(null, "1.0", "")] // Null input + [InlineData("test", null, "test")] // Null Version + [InlineData("", "1.0", "")] // Empty string + [InlineData(" ", "1.0", "")] // Whitespace only + public void NormalizeFileName_ShouldHandleEmptyInput(string? fileName, string version, string expected) + { + // Act + var result = FuzzySearch.NormalizeFileName(fileName, version); + + // Assert + result.Should().Be(expected); + } + + [Theory] + // Suffix + [InlineData("5.2SE", new[] { "5.2SE", "5.2", "5_2SE", "5_2" })] + [InlineData("9.0P", new[] { "9.0P", "9.0", "9_0P", "9_0" })] + [InlineData("9.0VF", new[] { "9.0VF", "9.0", "9_0VF", "9_0" })] + // Prefix + [InlineData("v5.2", new[] { "v5.2", "5.2", "v5_2", "5_2" })] + // Standard + [InlineData("10.0.1", new[] { "10.0.1", "10_0_1" })] + // Invalid + [InlineData("", new string[] { })] + [InlineData(null, new string[] { })] + // Prefix and Suffix + [InlineData("v1alpha", new[] { "v1alpha", "1alpha", "v1", "1" })] + [InlineData("v5.2SE", new[] { "v5.2SE", "5.2SE", "v5.2", "5.2", "v5_2SE", "5_2SE", "v5_2", "5_2" })] + [InlineData("v1.0beta", new[] { "v1.0beta", "1.0beta", "v1.0", "1.0", "v1_0beta", "1_0beta", "v1_0", "1_0" })] + // SemVer Prereleases + [InlineData("1.0-alpha", new[] { "1.0-alpha", "1.0", "1_0-alpha", "1_0" })] + [InlineData("v1.0-alpha", new[] { "v1.0-alpha", "1.0-alpha", "v1.0", "1.0", "v1_0-alpha", "1_0-alpha", "v1_0", "1_0" })] + public void GetVersionPermutations_ShouldReturnAllPossibleVersionFormats(string input, string[] expected) + { + // Act + var result = FuzzySearch.GetVersionPermutations(input); + + // Assert + result.Should().BeEquivalentTo(expected); + } + + [Theory] + [InlineData("Maestros of Synth.zip", "Maestros of Synth")] + [InlineData("Maestros of Synth.ZIP", "Maestros of Synth")] + [InlineData("Maestros of Synth.7z", "Maestros of Synth")] + [InlineData("Maestros of Synth.omod", "Maestros of Synth")] + [InlineData("Maestros of Synth", "Maestros of Synth")] + [InlineData("Something.with.dots.zip", "Something.with.dots")] + [InlineData("Maestros of Synth.nx", "Maestros of Synth")] // maybe one day + [InlineData(null, null)] + [InlineData("", "")] + [InlineData(" ", " ")] + public void StripFileExtensions_ShouldHandleAllowedFileExtension(string input, string expected) + { + // Act + var result = FuzzySearch.StripFileExtension(input); + + // Assert + result.Should().Be(expected); + } +} diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/MockModFile.cs b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/MockModFile.cs new file mode 100644 index 0000000000..e0ff77dd59 --- /dev/null +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/MockModFile.cs @@ -0,0 +1,52 @@ +using NexusMods.Networking.ModUpdates.Traits; + +/// +/// Represents a mock mod page for testing +/// +public class MockModPage +{ + private readonly List _files = new(); + public IReadOnlyList Files => _files; + + /// + /// Adds a new file to this mod page + /// + public MockModFile AddFile(string name, string version, DateTimeOffset uploadedAt) + { + var file = new MockModFile(name, version, uploadedAt, this); + _files.Add(file); + return file; + } + + /// + /// Creates a new mod page with the specified files + /// + public static MockModPage Create(Action configure) + { + var page = new MockModPage(); + configure(page); + return page; + } +} + +/// +/// A mock implementation of IAmAModFile for testing purposes +/// +public class MockModFile : IAmAModFile +{ + public string Name { get; } + public string Version { get; } + public DateTimeOffset UploadedAt { get; } + public IEnumerable OtherFilesInSameModPage => + _modPage.Files.Where(f => f != this); + + private readonly MockModPage _modPage; + + internal MockModFile(string name, string version, DateTimeOffset uploadedAt, MockModPage modPage) + { + Name = name; + Version = version; + UploadedAt = uploadedAt; + _modPage = modPage; + } +} diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestItem.cs b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestModFeedItem.cs similarity index 78% rename from tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestItem.cs rename to tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestModFeedItem.cs index 3317d01ef8..f9c96b0475 100644 --- a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestItem.cs +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/Helpers/TestModFeedItem.cs @@ -3,7 +3,7 @@ namespace NexusMods.Networking.ModUpdates.Tests.Helpers; // Helper class to simulate updateable items -public class TestItem : IModFeedItem +public class TestModFeedItem : IModFeedItem { public DateTime LastUpdated { get; set; } public UidForMod Uid { get; set; } @@ -12,9 +12,9 @@ public class TestItem : IModFeedItem public UidForMod GetModPageId() => Uid; // Helper method to create a test item - public static TestItem Create(uint gameId, uint modId, DateTime lastUpdated) + public static TestModFeedItem Create(uint gameId, uint modId, DateTime lastUpdated) { - return new TestItem + return new TestModFeedItem { Uid = new UidForMod { GameId = GameId.From(gameId), ModId = ModId.From(modId) }, LastUpdated = lastUpdated, diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/MultiFeedCacheUpdaterTests.cs b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/MultiFeedCacheUpdaterTests.cs index 63a8b7749b..bc4fbdae42 100644 --- a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/MultiFeedCacheUpdaterTests.cs +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/MultiFeedCacheUpdaterTests.cs @@ -1,6 +1,6 @@ using FluentAssertions; using NexusMods.Networking.ModUpdates.Tests.Helpers; -using static NexusMods.Networking.ModUpdates.Tests.Helpers.TestItem; +using static NexusMods.Networking.ModUpdates.Tests.Helpers.TestModFeedItem; namespace NexusMods.Networking.ModUpdates.Tests; @@ -10,7 +10,7 @@ public class MultiFeedCacheUpdaterTests public void Constructor_WithEmptyItems_ShouldNotThrow() { // Arrange & Act - Action act = () => new MultiFeedCacheUpdater([], TimeSpan.FromDays(30)); + Action act = () => new MultiFeedCacheUpdater([], TimeSpan.FromDays(30)); // Assert act.Should().NotThrow(); @@ -33,7 +33,7 @@ public void Constructor_ShouldSetOldItemsToNeedUpdate() }; // Act - var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); + var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); var result = updater.BuildFlattened(); // Assert @@ -61,7 +61,7 @@ public void Update_ShouldMarkMissingItemsAsUndetermined() Create(2, 2, now.AddDays(-8)), }; - var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); + var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); var updateItems = new[] { @@ -99,7 +99,7 @@ public void Update_ShouldMarkItemsAsUpToDateOrNeedingUpdateAcrossMultipleFeeds() Create(2, 2, now.AddDays(-7)), }; - var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); + var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); var updateItems = new[] { Create(1, 1, now.AddDays(-8)), // Newer, needs update @@ -140,7 +140,7 @@ public void Update_HavingExtraItemsInUpdatePayloadHasNoSideEffects() Create(2, 2, now.AddDays(-7)), }; - var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); + var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); var updateItems = new[] { @@ -186,7 +186,7 @@ public void Update_ShouldHandleUpdatesFromDifferentFeeds() Create(3, 1, now.AddDays(-15)), }; - var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); + var updater = new MultiFeedCacheUpdater(items, TimeSpan.FromDays(30)); var updateItems = new[] { diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/PerFeedCacheUpdaterTests.cs b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/PerFeedCacheUpdaterTests.cs index 7abcd9ad4a..414f54c41a 100644 --- a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/PerFeedCacheUpdaterTests.cs +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/PerFeedCacheUpdaterTests.cs @@ -1,7 +1,7 @@ using System.Diagnostics; using FluentAssertions; using NexusMods.Networking.ModUpdates.Tests.Helpers; -using static NexusMods.Networking.ModUpdates.Tests.Helpers.TestItem; +using static NexusMods.Networking.ModUpdates.Tests.Helpers.TestModFeedItem; namespace NexusMods.Networking.ModUpdates.Tests; @@ -11,7 +11,7 @@ public class PerFeedCacheUpdaterTests public void Constructor_WithEmptyItems_ShouldNotThrow() { // Arrange & Act - Action act = () => new PerFeedCacheUpdater([], TimeSpan.FromDays(30)); + Action act = () => new PerFeedCacheUpdater([], TimeSpan.FromDays(30)); // Assert act.Should().NotThrow(); @@ -31,7 +31,7 @@ public void Constructor_WithItemsFromDifferentGames_ShouldThrowArgumentException // Act // ReSharper disable once HeapView.ObjectAllocation.Evident // ReSharper disable once ObjectCreationAsStatement - Action act = () => new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); + Action act = () => new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); // Assert act.Should().Throw(); @@ -53,7 +53,7 @@ public void Constructor_ShouldSetOldItemsToNeedUpdate() }; // Act - var updater = new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); + var updater = new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); var result = updater.Build(); // Assert @@ -82,7 +82,7 @@ public void Update_ShouldMarkMissingItemsAsUndetermined() Create(1, 3, now.AddDays(-15)), }; - var updater = new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); + var updater = new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); var updateItems = new[] { @@ -118,7 +118,7 @@ public void Update_ShouldMarkItemsAsUpToDateOrNeedingUpdate() Create(1, 2, now.AddDays(-5)), }; - var updater = new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); + var updater = new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); var updateItems = new[] { Create(1, 1, now.AddDays(-8)), // Newer, needs update @@ -154,7 +154,7 @@ public void Update_HavingExtraItemsInUpdatePayloadHasNoSideEffects() Create(1, 3, now.AddDays(-15)), }; - var updater = new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); + var updater = new PerFeedCacheUpdater(items, TimeSpan.FromDays(30)); var updateItems = new[] { diff --git a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/RunUpdateCheckTests.cs b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/RunUpdateCheckTests.cs index ba64b28477..bc802d5cd6 100644 --- a/tests/Networking/NexusMods.Networking.ModUpdates.Tests/RunUpdateCheckTests.cs +++ b/tests/Networking/NexusMods.Networking.ModUpdates.Tests/RunUpdateCheckTests.cs @@ -36,8 +36,9 @@ public async Task UpdatingModPageMetadata_ViaWebApi_ShouldWork() var loadout = await CreateLoadout(); // Install a version of CET into the loadout. + // Name: 'CET 1.23.0' var modId = ModId.From(107u); // CET - var fileId = FileId.From(18963u); // 1.18.1 + var fileId = FileId.From(39639u); // 1.18.1 await using var tempFile = TemporaryFileManager.CreateFile(); var downloadJob = await _nexusModsLibrary.CreateDownloadJob( @@ -71,6 +72,170 @@ public async Task UpdatingModPageMetadata_ViaWebApi_ShouldWork() // Get the collection of newer mods, there should at least be 43 at time of // collection. We're not filtering out archived, so this number can never change var newerMods = RunUpdateCheck.GetNewerFilesForExistingFile(Connection.Db, outOfDateFileUid); - newerMods.Should().HaveCountGreaterThan(42); + newerMods.Should().HaveCountGreaterThan(7); + // Note: >=7 at time of writing. + // this is just a sanity test, there's a separate test against mocked + // data for the file picking logic + } +} + +/// +/// Tests specific to selecting newer files for an existing files. +/// This logic is part of but placed in separate +/// class for easier segregation. +/// +public class GetNewerFilesSelectorTests +{ + private static DateTimeOffset BaseTime => new(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + + [Fact] + public void GetNewerFilesForExistingFile_ShouldReturnEmpty_WhenNoOtherFiles() + { + var modPage = MockModPage.Create(page => + { + page.AddFile("TestMod.zip", "1.0", BaseTime); + }); + + var result = RunUpdateCheck.GetNewerFilesForExistingFile(modPage.Files[0]); + + result.Should().BeEmpty(); + } + + [Fact] + public void GetNewerFilesForExistingFile_ShouldReturnEmpty_WhenOnlyOlderFiles() + { + var modPage = MockModPage.Create(page => + { + page.AddFile("OldFile 1.0", "1.0", BaseTime.AddDays(-1)); + page.AddFile("OldFile 2.0", "2.0", BaseTime); + }); + + var result = RunUpdateCheck.GetNewerFilesForExistingFile(modPage.Files[1]); + + result.Should().BeEmpty(); + } + + [Fact] + public void GetNewerFilesForExistingFile_ShouldReturnNewerFiles_WhenMatchingNamesExist() + { + var modPage = MockModPage.Create(page => + { + page.AddFile("TestMod.zip", "1.0", BaseTime); + page.AddFile("TestMod.zip", "2.0", BaseTime.AddDays(1)); + page.AddFile("TestMod.zip", "3.0", BaseTime.AddDays(2)); + }); + + var result = RunUpdateCheck.GetNewerFilesForExistingFile(modPage.Files[0]).ToArray(); + + result.Should().HaveCount(2); + result.Select(f => f.Version).Should().BeEquivalentTo("2.0", "3.0"); + } + + [Fact] + public void GetNewerFilesForExistingFile_ShouldIgnoreDifferentNames() + { + var modPage = MockModPage.Create(page => + { + page.AddFile("TestMod.zip", "1.0", BaseTime); + page.AddFile("DifferentMod.zip", "2.0", BaseTime.AddDays(1)); + page.AddFile("TestMod.zip", "3.0", BaseTime.AddDays(2)); + }); + + var result = RunUpdateCheck.GetNewerFilesForExistingFile(modPage.Files[0]).ToArray(); + + result.Should().HaveCount(1); + result.Single().Version.Should().Be("3.0"); + } + + [Fact] + public void GetNewerFilesForExistingFile_ShouldHandleVersionsInNames() + { + var modPage = MockModPage.Create(page => + { + page.AddFile("TestMod 1.0.zip", "1.0", BaseTime); + page.AddFile("TestMod 2.0.zip", "2.0", BaseTime.AddDays(1)); + page.AddFile("TestMod 3.0.zip", "3.0", BaseTime.AddDays(2)); + }); + + var result = RunUpdateCheck.GetNewerFilesForExistingFile(modPage.Files[0]).ToArray(); + + result.Should().HaveCount(2); + result.Select(f => f.Version).Should().BeEquivalentTo("2.0", "3.0"); + } + + [Fact] + public void GetNewerFilesForExistingFile_ShouldHandleDifferentExtensions() + { + var modPage = MockModPage.Create(page => + { + page.AddFile("TestMod.zip", "1.0", BaseTime); + page.AddFile("TestMod.7z", "2.0", BaseTime.AddDays(1)); + page.AddFile("TestMod.rar", "3.0", BaseTime.AddDays(2)); + }); + + var result = RunUpdateCheck.GetNewerFilesForExistingFile(modPage.Files[0]).ToArray(); + + result.Should().HaveCount(2); + result.Select(f => f.Version).Should().BeEquivalentTo("2.0", "3.0"); + } + + [Fact] + public void GetNewerFilesForExistingFile_ShouldHandleUnderscoresAndSpaces() + { + var modPage = MockModPage.Create(page => + { + page.AddFile("Test_Mod.zip", "1.0", BaseTime); + page.AddFile("Test Mod.zip", "2.0", BaseTime.AddDays(1)); + page.AddFile("Test_Mod.zip", "3.0", BaseTime.AddDays(2)); + }); + + var result = RunUpdateCheck.GetNewerFilesForExistingFile(modPage.Files[0]).ToArray(); + + result.Should().HaveCount(2); + result.Select(f => f.Version).Should().BeEquivalentTo("2.0", "3.0"); + } + + [Fact] + public void GetNewerFilesForExistingFile_ShouldHandleVersionPrefixes() + { + var modPage = MockModPage.Create(page => + { + page.AddFile("TestMod.zip", "v1.0", BaseTime); + page.AddFile("TestMod.zip", "v2.0", BaseTime.AddDays(1)); + page.AddFile("TestMod.zip", "2.0", BaseTime.AddDays(2)); // No prefix + }); + + var result = RunUpdateCheck.GetNewerFilesForExistingFile(modPage.Files[0]).ToArray(); + + // Note(sewer): This tests that our code can match with either. + // The fact it returns both 'v2.0' and '2.0' here is not assumed to be an error, + // as it is unlikely a mod author would publish a mod ***with the same name*** + // and ***version*** with and without a prefix. + result.Should().HaveCount(2); + result.Select(f => f.Version).Should().BeEquivalentTo("v2.0", "2.0"); + } + + [Fact] + public void GetNewerFilesForExistingFile_ShouldHandleComplexExample() + { + var modPage = MockModPage.Create(page => + { + // Different parts with same version + page.AddFile("Skyrim 202X 9.0 - Architecture PART 1", "9.0", BaseTime); + page.AddFile("Skyrim 202X 9.0 - Landscape PART 2", "9.0", BaseTime); + + // Updates to Part 1 + page.AddFile("Skyrim 202X 10.0 - Architecture PART 1", "10.0", BaseTime.AddDays(1)); + page.AddFile("Skyrim_202X_10.0.1 - Architecture PART 1.zip", "10.0.1", BaseTime.AddDays(2)); + + // Updates to Part 2 (should not be matched) + page.AddFile("Skyrim 202X 10.0 - Landscape PART 2", "10.0", BaseTime.AddDays(1)); + }); + + var oldArchitecturePart = modPage.Files[0]; + var result = RunUpdateCheck.GetNewerFilesForExistingFile(oldArchitecturePart).ToArray(); + + result.Should().HaveCount(2); + result.Select(f => f.Version).Should().BeEquivalentTo("10.0", "10.0.1"); } }