From 130265c9486851b1c4edd5554e618975c7a93083 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 13:45:57 +0000 Subject: [PATCH 1/2] Bump the nexusmods group with 2 updates Bumps the nexusmods group with 2 updates: [NexusMods.Paths](https://github.com/Nexus-Mods/NexusMods.Paths) and [NexusMods.Paths.TestingHelpers](https://github.com/Nexus-Mods/NexusMods.Paths). Updates `NexusMods.Paths` from 0.1.8 to 0.2.0 - [Release notes](https://github.com/Nexus-Mods/NexusMods.Paths/releases) - [Commits](https://github.com/Nexus-Mods/NexusMods.Paths/compare/v0.1.8...v0.2) Updates `NexusMods.Paths.TestingHelpers` from 0.1.8 to 0.2.0 - [Release notes](https://github.com/Nexus-Mods/NexusMods.Paths/releases) - [Commits](https://github.com/Nexus-Mods/NexusMods.Paths/compare/v0.1.8...v0.2) --- updated-dependencies: - dependency-name: NexusMods.Paths dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nexusmods - dependency-name: NexusMods.Paths.TestingHelpers dependency-type: direct:production update-type: version-update:semver-minor dependency-group: nexusmods ... Signed-off-by: dependabot[bot] --- Directory.Packages.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 5df8f52d92..520b92aa47 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -2,9 +2,9 @@ - + - + From dc63e9862588232c52d0addf9f000ed75abb5811 Mon Sep 17 00:00:00 2001 From: sewer56 Date: Mon, 27 Nov 2023 17:00:06 +0000 Subject: [PATCH 2/2] Fix this PR with Imports & Dead Code Removal --- .../Benchmarks/ChangeCase.cs | 65 ---------- .../ChangeCaseVsOrdinalIgnoreCase.cs | 54 -------- .../Benchmarks/VectorisedStringHash.cs | 122 ------------------ .../Models/SMAPIVersion.cs | 1 + .../ChunkedStreams/ChunkedStream.cs | 13 +- .../ChunkedStreams/LightweightLRUCache.cs | 3 + .../Loadouts/Cursors/ModCursor.cs | 1 + .../Loadouts/DiskStateRegistry.cs | 1 + src/NexusMods.DataModel/NxFileStore.cs | 34 +++-- src/NexusMods.DataModel/ZipFileStore.cs | 1 + 10 files changed, 36 insertions(+), 259 deletions(-) delete mode 100644 benchmarks/NexusMods.Benchmarks/Benchmarks/ChangeCase.cs delete mode 100644 benchmarks/NexusMods.Benchmarks/Benchmarks/ChangeCaseVsOrdinalIgnoreCase.cs delete mode 100644 benchmarks/NexusMods.Benchmarks/Benchmarks/VectorisedStringHash.cs diff --git a/benchmarks/NexusMods.Benchmarks/Benchmarks/ChangeCase.cs b/benchmarks/NexusMods.Benchmarks/Benchmarks/ChangeCase.cs deleted file mode 100644 index fc8fb3ad45..0000000000 --- a/benchmarks/NexusMods.Benchmarks/Benchmarks/ChangeCase.cs +++ /dev/null @@ -1,65 +0,0 @@ -using System.Runtime.CompilerServices; -using BenchmarkDotNet.Attributes; -using NexusMods.Benchmarks.Interfaces; -using NexusMods.Paths.HighPerformance.Backports.System.Globalization; - -namespace NexusMods.Benchmarks.Benchmarks; - -[SkipLocalsInit] -[MemoryDiagnoser] -[BenchmarkInfo("Change Case", "Tests the performance of changing the case on a string.")] -public class ChangeCase : IBenchmark -{ - [Params(4, 16, 64, 256, 1024)] - // ReSharper disable once UnusedAutoPropertyAccessor.Global - public int StringLength { get; set; } - - private static Random _random = new(); - private string _value = null!; - private string _valueEmoji = null!; - - [GlobalSetup] - public void Setup() - { - _value = RandomStringUpper(StringLength); - _valueEmoji = RandomStringUpperWithEmoji(StringLength); - } - - [Benchmark] - public void Runtime_Current_NonAscii() - { - Span buffer = stackalloc char[_valueEmoji.Length]; - MemoryExtensions.ToLowerInvariant(_valueEmoji, buffer); - } - - [Benchmark(Baseline = true)] - public void Runtime_Current() - { - Span buffer = stackalloc char[_value.Length]; - MemoryExtensions.ToLowerInvariant(_value, buffer); - } - - [Benchmark] - public void NET8_Backport() - { - Span buffer = stackalloc char[_value.Length]; - TextInfo.ChangeCase(_value, buffer); - } - - [Benchmark] - public void NET8_Backport_NonAscii() - { - Span buffer = stackalloc char[_valueEmoji.Length]; - TextInfo.ChangeCase(_valueEmoji, buffer); - } - - private static string RandomString(int length, string charSet) - { - return new string(Enumerable.Repeat(charSet, length) - .Select(s => s[_random.Next(s.Length)]).ToArray()); - } - - private static string RandomStringUpper(int length) => RandomString(length, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); - - private static string RandomStringUpperWithEmoji(int length) => RandomString(length, "ABCDEFGHIJKLMNOPQRSTUVWXYZ⚠️🚦🔺🏒😕🏞🖌🖕🌷☠⛩🍸👳🍠🚦📟💦🚏🌥🏪🌖😱"); -} diff --git a/benchmarks/NexusMods.Benchmarks/Benchmarks/ChangeCaseVsOrdinalIgnoreCase.cs b/benchmarks/NexusMods.Benchmarks/Benchmarks/ChangeCaseVsOrdinalIgnoreCase.cs deleted file mode 100644 index 0cac14b9f9..0000000000 --- a/benchmarks/NexusMods.Benchmarks/Benchmarks/ChangeCaseVsOrdinalIgnoreCase.cs +++ /dev/null @@ -1,54 +0,0 @@ -using BenchmarkDotNet.Attributes; -using NexusMods.Benchmarks.Interfaces; -using NexusMods.Paths.HighPerformance.Backports.System.Globalization; - -namespace NexusMods.Benchmarks.Benchmarks; - -[MemoryDiagnoser] -[BenchmarkInfo("Change Case + Ordinal vs OrdinalIgnoreCase", "Compares Change Case + Ordinal vs OrdinalIgnoreCase")] -public class ChangeCaseVsOrdinalIgnoreCase : IBenchmark -{ - [Params(4, 16, 64, 256, 1024)] - // ReSharper disable once UnusedAutoPropertyAccessor.Global - public int StringLength { get; set; } - - private string _value1 = null!; - private string _value2 = null!; - - [GlobalSetup] - public void Setup() - { - _value1 = RandomStringUpper(StringLength); - _value2 = _value1 + "_"; - } - - [Benchmark(Baseline = true)] - public bool WithOrdinalIgnoreCase() - { - return string.Equals(_value1, _value2, StringComparison.OrdinalIgnoreCase); - } - - [Benchmark] - public bool WithChangeCaseOrdinal() - { - var span1 = _value1.Length > 512 - ? GC.AllocateUninitializedArray(_value1.Length) - : stackalloc char[_value1.Length]; - TextInfo.ChangeCase(_value1, span1); - - var span2 = _value1.Length > 512 - ? GC.AllocateUninitializedArray(_value2.Length) - : stackalloc char[_value2.Length]; - TextInfo.ChangeCase(_value2, span2); - - return span1.SequenceEqual(span2); - } - - private static string RandomString(int length, string charSet) - { - return new string(Enumerable.Repeat(charSet, length) - .Select(s => s[Random.Shared.Next(s.Length)]).ToArray()); - } - - private static string RandomStringUpper(int length) => RandomString(length, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); -} diff --git a/benchmarks/NexusMods.Benchmarks/Benchmarks/VectorisedStringHash.cs b/benchmarks/NexusMods.Benchmarks/Benchmarks/VectorisedStringHash.cs deleted file mode 100644 index 3b08626a40..0000000000 --- a/benchmarks/NexusMods.Benchmarks/Benchmarks/VectorisedStringHash.cs +++ /dev/null @@ -1,122 +0,0 @@ -using System.Numerics; -using BenchmarkDotNet.Attributes; -using NexusMods.Benchmarks.Interfaces; -using NexusMods.Paths.Extensions; -// ReSharper disable RedundantAssignment - -namespace NexusMods.Benchmarks.Benchmarks; - -[BenchmarkInfo("String Hash", "Tests speed of string hashing algorithm.")] -public class VectorisedStringHash : IBenchmark -{ - private const int ItemCount = 10000; - - [Params(12, 64, 96, 128, 256, 1024)] - // ReSharper disable once UnusedAutoPropertyAccessor.Global - public int CharacterCount { get; set; } - - public string[] Input { get; set; } = null!; - - [GlobalSetup] - public void Setup() - { - Input = new string[ItemCount]; - - for (var x = 0; x < ItemCount; x++) - Input[x] = RandomString(CharacterCount); - } - - [Benchmark] - public int Custom() - { - var result = 0; - var maxLen = Input.Length / 4; - // unroll - for (var x = 0; x < maxLen; x += 4) - { - result = StringExtensions.GetNonRandomizedHashCode32(Input.DangerousGetReferenceAt(x)); - result = StringExtensions.GetNonRandomizedHashCode32(Input.DangerousGetReferenceAt(x + 1)); - result = StringExtensions.GetNonRandomizedHashCode32(Input.DangerousGetReferenceAt(x + 2)); - result = StringExtensions.GetNonRandomizedHashCode32(Input.DangerousGetReferenceAt(x + 3)); - } - - return result; - } - - [Benchmark] - public int Runtime_NonRandom_NET702() - { - var result = 0; - var maxLen = Input.Length / 4; - // unroll - for (var x = 0; x < maxLen; x += 4) - { - result = Runtime_70_Impl(Input.DangerousGetReferenceAt(x)); - result = Runtime_70_Impl(Input.DangerousGetReferenceAt(x + 1)); - result = Runtime_70_Impl(Input.DangerousGetReferenceAt(x + 2)); - result = Runtime_70_Impl(Input.DangerousGetReferenceAt(x + 3)); - } - - return result; - } - - [Benchmark] - public int Runtime_Current() - { - var result = 0; - var maxLen = Input.Length / 4; - // unroll - for (var x = 0; x < maxLen; x += 4) - { - result = Input.DangerousGetReferenceAt(x).GetHashCode(); - result = Input.DangerousGetReferenceAt(x + 1).GetHashCode(); - result = Input.DangerousGetReferenceAt(x + 2).GetHashCode(); - result = Input.DangerousGetReferenceAt(x + 3).GetHashCode(); - } - - return result; - } - - /// - /// Implementation from .NET 7.0 - /// - /// - /// - private unsafe int Runtime_70_Impl(ReadOnlySpan text) - { - fixed (char* src = &text.GetPinnableReference()) - { - // Asserts here for alignment etc. are no longer valid as we are operating on a slice, so memory alignment is not guaranteed. - uint hash1 = (5381 << 16) + 5381; - var hash2 = hash1; - - var ptr = (uint*)src; - var length = text.Length; - - while (length > 2) - { - length -= 4; - // Where length is 4n-1 (e.g. 3,7,11,15,19) this additionally consumes the null terminator - hash1 = (BitOperations.RotateLeft(hash1, 5) + hash1) ^ ptr[0]; - hash2 = (BitOperations.RotateLeft(hash2, 5) + hash2) ^ ptr[1]; - ptr += 2; - } - - if (length > 0) - { - // Where length is 4n-3 (e.g. 1,5,9,13,17) this additionally consumes the null terminator - hash2 = (BitOperations.RotateLeft(hash2, 5) + hash2) ^ ptr[0]; - } - - return (int)(hash1 + (hash2 * 1566083941)); - } - } - - private static Random _random = new Random(); - private static string RandomString(int length) - { - const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - return new string(Enumerable.Repeat(chars, length) - .Select(s => s[_random.Next(s.Length)]).ToArray()); - } -} diff --git a/src/Games/NexusMods.Games.StardewValley/Models/SMAPIVersion.cs b/src/Games/NexusMods.Games.StardewValley/Models/SMAPIVersion.cs index 76392343bc..5712c1899b 100644 --- a/src/Games/NexusMods.Games.StardewValley/Models/SMAPIVersion.cs +++ b/src/Games/NexusMods.Games.StardewValley/Models/SMAPIVersion.cs @@ -5,6 +5,7 @@ using System.Text.Json.Serialization; using JetBrains.Annotations; using NexusMods.Paths.Extensions; +using Reloaded.Memory.Extensions; namespace NexusMods.Games.StardewValley.Models; diff --git a/src/NexusMods.DataModel/ChunkedStreams/ChunkedStream.cs b/src/NexusMods.DataModel/ChunkedStreams/ChunkedStream.cs index c0b0d20d84..1e3bdce5c8 100644 --- a/src/NexusMods.DataModel/ChunkedStreams/ChunkedStream.cs +++ b/src/NexusMods.DataModel/ChunkedStreams/ChunkedStream.cs @@ -1,5 +1,6 @@ using System.Buffers; using NexusMods.Paths.Extensions; +using Reloaded.Memory.Extensions; namespace NexusMods.DataModel.ChunkedStreams; @@ -28,9 +29,7 @@ public ChunkedStream(T source, int capacity = 16) } /// - public override void Flush() - { - } + public override void Flush() { } /// @@ -55,6 +54,7 @@ public override int Read(byte[] buffer, int offset, int count) toRead = Math.Min(toRead, (int)lastChunkExtraSize); } } + chunk.Slice((int)chunkOffset, toRead) .Span .CopyTo(buffer.AsSpan(offset, toRead)); @@ -64,7 +64,8 @@ public override int Read(byte[] buffer, int offset, int count) /// - public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = new CancellationToken()) + public override async ValueTask ReadAsync(Memory buffer, + CancellationToken cancellationToken = new CancellationToken()) { if (_position >= _source.Size.Value) { @@ -87,12 +88,12 @@ public override int Read(byte[] buffer, int offset, int count) toRead = Math.Min(toRead, (int)lastChunkExtraSize); } } + chunk.Slice((int)chunkOffset, toRead) .Span .CopyTo(buffer.Span.SliceFast(0, toRead)); _position += (ulong)toRead; return toRead; - } private async ValueTask> GetChunkAsync(ulong index, CancellationToken token) @@ -101,6 +102,7 @@ private async ValueTask> GetChunkAsync(ulong index, CancellationTok { return memory!.Memory; } + var memoryOwner = _pool.Rent((int)_source.ChunkSize.Value); await _source.ReadChunkAsync(memoryOwner.Memory, index, token); _cache.Add(index, memoryOwner); @@ -113,6 +115,7 @@ private Memory GetChunk(ulong index) { return memory!.Memory; } + var memoryOwner = _pool.Rent((int)_source.ChunkSize.Value); _source.ReadChunk(memoryOwner.Memory.Span, index); _cache.Add(index, memoryOwner); diff --git a/src/NexusMods.DataModel/ChunkedStreams/LightweightLRUCache.cs b/src/NexusMods.DataModel/ChunkedStreams/LightweightLRUCache.cs index f5eac03c4b..86aa5d8586 100644 --- a/src/NexusMods.DataModel/ChunkedStreams/LightweightLRUCache.cs +++ b/src/NexusMods.DataModel/ChunkedStreams/LightweightLRUCache.cs @@ -1,4 +1,5 @@ using NexusMods.Paths.Extensions; +using Reloaded.Memory.Extensions; namespace NexusMods.DataModel.ChunkedStreams; @@ -45,6 +46,7 @@ public bool TryGet(TK key, out TV? value) Touch(index); return true; } + value = default; return false; } @@ -78,6 +80,7 @@ private readonly int Find(TK key) { if (_keys[i].Equals(key)) return i; } + return -1; } diff --git a/src/NexusMods.DataModel/Loadouts/Cursors/ModCursor.cs b/src/NexusMods.DataModel/Loadouts/Cursors/ModCursor.cs index 068846f8cc..ec0cc7fcee 100644 --- a/src/NexusMods.DataModel/Loadouts/Cursors/ModCursor.cs +++ b/src/NexusMods.DataModel/Loadouts/Cursors/ModCursor.cs @@ -1,5 +1,6 @@ using NexusMods.DataModel.Interprocess; using NexusMods.Paths.Extensions; +using Reloaded.Memory.Extensions; namespace NexusMods.DataModel.Loadouts.Cursors; diff --git a/src/NexusMods.DataModel/Loadouts/DiskStateRegistry.cs b/src/NexusMods.DataModel/Loadouts/DiskStateRegistry.cs index 69c75ea63e..82fafa5882 100644 --- a/src/NexusMods.DataModel/Loadouts/DiskStateRegistry.cs +++ b/src/NexusMods.DataModel/Loadouts/DiskStateRegistry.cs @@ -4,6 +4,7 @@ using NexusMods.DataModel.Abstractions; using NexusMods.DataModel.LoadoutSynchronizer; using NexusMods.Paths.Extensions; +using Reloaded.Memory.Extensions; namespace NexusMods.DataModel.Loadouts; diff --git a/src/NexusMods.DataModel/NxFileStore.cs b/src/NexusMods.DataModel/NxFileStore.cs index d755a7923e..7629979e73 100644 --- a/src/NexusMods.DataModel/NxFileStore.cs +++ b/src/NexusMods.DataModel/NxFileStore.cs @@ -18,6 +18,7 @@ using NexusMods.Paths; using NexusMods.Paths.Extensions; using NexusMods.Paths.Utilities; +using Reloaded.Memory.Extensions; namespace NexusMods.DataModel; @@ -47,7 +48,6 @@ public NxFileStore(ILogger logger, IDataStore store, IDataModelSett _logger = logger; _store = store; - } /// @@ -77,7 +77,8 @@ public async Task BackupFiles(IEnumerable backups, Cancellati var id = guid.ToString(); var outputPath = _archiveLocations.First().Combine(id).AppendExtension(KnownExtensions.Tmp); - await using (var outputStream = outputPath.Create()){ + await using (var outputStream = outputPath.Create()) + { builder.WithOutput(outputStream); builder.Build(); } @@ -132,7 +133,7 @@ public async Task ExtractFiles(IEnumerable<(Hash Src, AbsolutePath Dest)> files, { var grouped = files.Distinct() .Select(input => TryGetLocation(input.Src, out var archivePath, out var fileEntry) - ? (true, Hash:input.Src, ArchivePath:archivePath, FileEntry:fileEntry, input.Dest) + ? (true, Hash: input.Src, ArchivePath: archivePath, FileEntry: fileEntry, input.Dest) : default) .Where(x => x.Item1) .ToLookup(l => l.ArchivePath, l => (l.Hash, l.FileEntry, l.Dest)); @@ -150,7 +151,8 @@ public async Task ExtractFiles(IEnumerable<(Hash Src, AbsolutePath Dest)> files, var toExtract = group .Select(entry => - (IOutputDataProvider)new OutputFileProvider(entry.Dest.Parent.GetFullPath(), entry.Dest.FileName, entry.FileEntry)) + (IOutputDataProvider)new OutputFileProvider(entry.Dest.Parent.GetFullPath(), entry.Dest.FileName, + entry.FileEntry)) .ToArray(); try @@ -177,7 +179,7 @@ public Task> ExtractFiles(IEnumerable files, Can var grouped = files.Distinct() .Select(hash => TryGetLocation(hash, out var archivePath, out var fileEntry) - ? (true, Hash:hash, ArchivePath:archivePath, FileEntry:fileEntry) + ? (true, Hash: hash, ArchivePath: archivePath, FileEntry: fileEntry) : default) .Where(x => x.Item1) .ToLookup(l => l.ArchivePath, l => (l.Hash, l.FileEntry)); @@ -195,7 +197,8 @@ public Task> ExtractFiles(IEnumerable files, Can var provider = new FromStreamProvider(file); var unpacker = new NxUnpacker(provider); - var infos = group.Select(entry => (entry.Hash, new OutputArrayProvider("", entry.FileEntry), entry.FileEntry.DecompressedSize)).ToList(); + var infos = group.Select(entry => (entry.Hash, new OutputArrayProvider("", entry.FileEntry), + entry.FileEntry.DecompressedSize)).ToList(); unpacker.ExtractFiles(infos.Select(o => (IOutputDataProvider)o.Item2).ToArray(), settings); foreach (var (hash, output, size) in infos) @@ -220,7 +223,8 @@ public Task GetFileStream(Hash hash, CancellationToken token = default) var provider = new FromStreamProvider(file); var header = HeaderParser.ParseHeader(provider); - return Task.FromResult(new ChunkedStream(new ChunkedArchiveStream(entry, header, file))); + return Task.FromResult( + new ChunkedStream(new ChunkedArchiveStream(entry, header, file))); } private class ChunkedArchiveStream : IChunkedStreamSource @@ -246,7 +250,8 @@ public ChunkedArchiveStream(FileEntry entry, ParsedHeader header, Stream stream) public async Task ReadChunkAsync(Memory buffer, ulong localIndex, CancellationToken token = default) { - var extractable = PreProcessBlock(localIndex, out var blockIndex, out var compressedBlockSize, out var offset); + var extractable = + PreProcessBlock(localIndex, out var blockIndex, out var compressedBlockSize, out var offset); _stream.Position = offset; using var compressedBlock = MemoryPool.Shared.Rent(compressedBlockSize); await _stream.ReadExactlyAsync(compressedBlock.Memory[..compressedBlockSize], token); @@ -255,7 +260,8 @@ public async Task ReadChunkAsync(Memory buffer, ulong localIndex, Cancella public void ReadChunk(Span buffer, ulong localIndex) { - var extractable = PreProcessBlock(localIndex, out var blockIndex, out var compressedBlockSize, out var offset); + var extractable = + PreProcessBlock(localIndex, out var blockIndex, out var compressedBlockSize, out var offset); _stream.Position = offset; using var compressedBlock = MemoryPool.Shared.Rent(compressedBlockSize); _stream.ReadExactly(compressedBlock.Memory.Span[..compressedBlockSize]); @@ -271,7 +277,8 @@ public void ReadChunk(Span buffer, ulong localIndex) /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private ExtractableBlock PreProcessBlock(ulong localIndex, out int blockIndex, out int compressedBlockSize, out long offset) + private ExtractableBlock PreProcessBlock(ulong localIndex, out int blockIndex, out int compressedBlockSize, + out long offset) { var extractable = _blocks[(int)localIndex]; blockIndex = extractable.BlockIndex; @@ -289,7 +296,8 @@ private ExtractableBlock PreProcessBlock(ulong localIndex, out int blockIndex, o /// /// /// - private unsafe void ProcessBlock(Span buffer, int blockIndex, ExtractableBlock extractable, Span compressedBlock, + private unsafe void ProcessBlock(Span buffer, int blockIndex, ExtractableBlock extractable, + Span compressedBlock, int blockSize) { var chunkSize = _header.Header.ChunkSizeBytes; @@ -364,7 +372,8 @@ private void MakeBlocks(int chunkSize) var block = new ExtractableBlock { BlockIndex = blockIndex, - DecompressSize = _entry.DecompressedBlockOffset + (int)Math.Min(remainingDecompSize, (ulong)chunkSize) + DecompressSize = _entry.DecompressedBlockOffset + + (int)Math.Min(remainingDecompSize, (ulong)chunkSize) }; _blocks.Add(block); @@ -411,7 +420,6 @@ private unsafe bool TryGetLocation(Hash hash, out AbsolutePath archivePath, out return true; } } - } archivePath = default; diff --git a/src/NexusMods.DataModel/ZipFileStore.cs b/src/NexusMods.DataModel/ZipFileStore.cs index 0337d5cc07..5d10b7a78e 100644 --- a/src/NexusMods.DataModel/ZipFileStore.cs +++ b/src/NexusMods.DataModel/ZipFileStore.cs @@ -9,6 +9,7 @@ using NexusMods.Paths; using NexusMods.Paths.Extensions; using NexusMods.Paths.Utilities; +using Reloaded.Memory.Extensions; namespace NexusMods.DataModel;