Skip to content

Commit

Permalink
Switch to xxHash3 (#2191)
Browse files Browse the repository at this point in the history
* Switch over to xxHash3

* Fix most of the tests

* Update NX to fix tests

* fix some broken tests

* Remove duplicate package version definition

* Add global.json file, maybe this fixes the build issue?

* Upgrade the hashing library version

* Fix the "hint" crap Rider put into the project files

* Add final missing package reference
halgari authored Oct 23, 2024
1 parent ecb5633 commit 5d1cfec
Showing 110 changed files with 721 additions and 712 deletions.
6 changes: 3 additions & 3 deletions Directory.Packages.props
Original file line number Diff line number Diff line change
@@ -11,13 +11,14 @@
<PackageVersion Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageVersion Include="Nerdbank.FullDuplexStream" Version="1.1.12" />
<PackageVersion Include="Nerdbank.Streams" Version="2.11.74" />
<PackageVersion Include="NexusMods.Hashing.xxHash3" Version="3.0.2" />
<PackageVersion Include="NexusMods.Hashing.xxHash3.Paths" Version="3.0.2" />
<PackageVersion Include="NexusMods.Paths" Version="0.10.0" />
<PackageVersion Include="NexusMods.MnemonicDB.Abstractions" Version="0.9.91" />
<PackageVersion Include="NexusMods.MnemonicDB" Version="0.9.91" />
<PackageVersion Include="NexusMods.Hashing.xxHash64" Version="2.0.2" />
<PackageVersion Include="NexusMods.Paths.Extensions.Nx" Version="0.10.0" />
<PackageVersion Include="NexusMods.Paths.TestingHelpers" Version="0.10.0" />
<PackageVersion Include="NexusMods.Archives.Nx" Version="0.5.0" />
<PackageVersion Include="NexusMods.Archives.Nx" Version="0.6.1" />
<PackageVersion Include="NexusMods.Telemetry.OpenTelemetry" Version="1.0.0" />
<PackageVersion Include="FomodInstaller.Interface" Version="1.2.0" />
<PackageVersion Include="FomodInstaller.Scripting.XmlScript" Version="1.0.0" />
@@ -132,7 +133,6 @@
<PackageVersion Include="OneOf" Version="3.0.271" />
<PackageVersion Include="ReactiveUI.Fody" Version="19.5.41" />
<PackageVersion Include="Sewer56.BitStream" Version="1.3.0" />
<PackageVersion Include="Spectre.Console" Version="0.48.0" />
<PackageVersion Include="Splat.Microsoft.Extensions.Logging" Version="15.2.22" />
<PackageVersion Include="TransparentValueObjects" Version="1.0.1" />
</ItemGroup>
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
using NexusMods.Abstractions.IO;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.Paths;

namespace NexusMods.Benchmarks.Benchmarks.Loadouts.Harness;
7 changes: 7 additions & 0 deletions global.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"sdk": {
"version": "8.0.0",
"rollForward": "latestMajor",
"allowPrerelease": true
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.Paths;

namespace NexusMods.Abstractions.Collections.Types;
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@
<!-- NuGet Package Shared Details -->
<Import Project="$([MSBuild]::GetPathOfFileAbove('NuGet.Build.props', '$(MSBuildThisFileDirectory)../'))" />
<ItemGroup>
<PackageReference Include="NexusMods.Hashing.xxHash3" />
<PackageReference Include="NexusMods.MnemonicDB.SourceGenerator" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
<PackageReference Include="NexusMods.MnemonicDB.Abstractions" />

Original file line number Diff line number Diff line change
@@ -3,9 +3,9 @@
<Import Project="$([MSBuild]::GetPathOfFileAbove('NuGet.Build.props', '$(MSBuildThisFileDirectory)../'))" />

<ItemGroup>
<PackageReference Include="NexusMods.Hashing.xxHash3" />
<PackageReference Include="TransparentValueObjects" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
<PackageReference Include="OneOf" />
<PackageReference Include="NexusMods.Hashing.xxHash64" />
</ItemGroup>

<ItemGroup>
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using OneOf;

namespace NexusMods.Abstractions.GuidedInstallers;
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.Paths;
using Size = NexusMods.Paths.Size;

Original file line number Diff line number Diff line change
@@ -4,8 +4,8 @@
<ItemGroup>
<!-- Todo: We need to get rid of 'optional' to remove this heavy dependency. -->
<PackageReference Include="DynamicData" />
<PackageReference Include="NexusMods.Hashing.xxHash3" />
<PackageReference Include="TransparentValueObjects" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
<PackageReference Include="NexusMods.Hashing.xxHash64" />
<PackageReference Include="NexusMods.Paths" />
</ItemGroup>
<ItemGroup>
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.Paths;

namespace NexusMods.Abstractions.IO;
3 changes: 1 addition & 2 deletions src/Abstractions/NexusMods.Abstractions.IO/IFileStore.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using System.Buffers;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.Paths;

namespace NexusMods.Abstractions.IO;
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@

<ItemGroup>
<!-- TODO: This library needs stabilized -->
<PackageReference Include="NexusMods.Hashing.xxHash3" />
<PackageReference Include="NexusMods.Paths" />
<PackageReference Include="NexusMods.Hashing.xxHash64" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@

<ItemGroup>
<ProjectReference Include="..\NexusMods.Abstractions.MnemonicDB.Attributes\NexusMods.Abstractions.MnemonicDB.Attributes.csproj" />
<PackageReference Include="NexusMods.Hashing.xxHash3" />
<PackageReference Include="NexusMods.MnemonicDB.Abstractions" />
<PackageReference Include="TransparentValueObjects" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
<PackageReference Include="NexusMods.MnemonicDB.SourceGenerator" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
Original file line number Diff line number Diff line change
@@ -14,8 +14,8 @@
using NexusMods.Abstractions.Loadouts.Extensions;
using NexusMods.Abstractions.Loadouts.Synchronizers.Rules;
using NexusMods.Extensions.BCL;
using NexusMods.Extensions.Hashing;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.Hashing.xxHash3.Paths;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.MnemonicDB.Abstractions.DatomIterators;
using NexusMods.MnemonicDB.Abstractions.IndexSegments;
@@ -895,7 +895,7 @@ await Parallel.ForEachAsync(location.Value.EnumerateFiles(), async (file, token)
// If the files don't match, update the entry
if (fileInfo.LastWriteTimeUtc > entry.LastModified || fileInfo.Size != entry.Size)
{
var newHash = await file.XxHash64Async();
var newHash = await file.XxHash3Async();
tx.Add(entry.Id, DiskStateEntry.Size, fileInfo.Size);
tx.Add(entry.Id, DiskStateEntry.Hash, newHash);
tx.Add(entry.Id, DiskStateEntry.LastModified, fileInfo.LastWriteTimeUtc);
@@ -905,7 +905,7 @@ await Parallel.ForEachAsync(location.Value.EnumerateFiles(), async (file, token)
else
{
// No previous entry found, so create a new one
var newHash = await file.XxHash64Async(token: token);
var newHash = await file.XxHash3Async(token: token);
_ = new DiskStateEntry.New(tx, tx.TempId(DiskStateEntry.EntryPartition))
{
Path = gamePath.ToGamePathParentTuple(metadata.Id),
@@ -960,7 +960,7 @@ await Parallel.ForEachAsync(location.Value.EnumerateFiles(), async (file, token)
return;
}

var newHash = await file.XxHash64Async(token: token);
var newHash = await file.XxHash3Async(token: token);
_ = new DiskStateEntry.New(tx, tx.TempId(DiskStateEntry.EntryPartition))
{
Path = gamePath.ToGamePathParentTuple(metaDataId),
Original file line number Diff line number Diff line change
@@ -7,6 +7,8 @@
<ProjectReference Include="..\NexusMods.Abstractions.GC\NexusMods.Abstractions.GC.csproj" />
<ProjectReference Include="..\NexusMods.Abstractions.Jobs\NexusMods.Abstractions.Jobs.csproj" />
<ProjectReference Include="..\NexusMods.Abstractions.Loadouts\NexusMods.Abstractions.Loadouts.csproj" />
<PackageReference Include="NexusMods.Hashing.xxHash3" />
<PackageReference Include="NexusMods.Hashing.xxHash3.Paths" />
<PackageReference Include="NexusMods.MnemonicDB.SourceGenerator" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
</ItemGroup>
<ItemGroup>
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
using DynamicData.Kernel;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;

namespace NexusMods.Abstractions.Loadouts.Synchronizers.Rules;

Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
using NexusMods.Abstractions.DiskState;
using NexusMods.Abstractions.GameLocators;
using NexusMods.Abstractions.Loadouts.Synchronizers.Rules;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.Paths;

Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
using NexusMods.Abstractions.GameLocators;
using NexusMods.Abstractions.Loadouts;
using NexusMods.Abstractions.MnemonicDB.Attributes;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.MnemonicDB.Abstractions.Attributes;
using NexusMods.MnemonicDB.Abstractions.Models;
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
using NexusMods.Abstractions.GameLocators;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.Paths;

namespace NexusMods.Abstractions.Loadouts.Synchronizers;
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
using NexusMods.Abstractions.GameLocators;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.Paths;

namespace NexusMods.Abstractions.Loadouts.Files;
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
using NexusMods.Abstractions.GameLocators;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.Paths;

Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
using JetBrains.Annotations;
using NexusMods.Abstractions.GameLocators;
using NexusMods.Abstractions.MnemonicDB.Attributes;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.MnemonicDB.Abstractions.Models;
using NexusMods.Paths;
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.MnemonicDB.Abstractions.Attributes;
using NexusMods.MnemonicDB.Abstractions.ElementComparers;
Original file line number Diff line number Diff line change
@@ -3,12 +3,10 @@
<Import Project="$([MSBuild]::GetPathOfFileAbove('NuGet.Build.props', '$(MSBuildThisFileDirectory)../'))" />

<ItemGroup>
<PackageReference Include="NexusMods.Hashing.xxHash64" />
<PackageReference Include="NexusMods.MnemonicDB.Abstractions" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\Extensions\NexusMods.Extensions.Hashing\NexusMods.Extensions.Hashing.csproj" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="NexusMods.Hashing.xxHash3" />
<PackageReference Include="NexusMods.MnemonicDB.Abstractions"/>
<PackageReference Include="NexusMods.MnemonicDB.SourceGenerator" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
</ItemGroup>
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
using DynamicData.Kernel;
using JetBrains.Annotations;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.MnemonicDB.Abstractions.Attributes;

Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
using JetBrains.Annotations;
using NexusMods.Abstractions.IO;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;

namespace NexusMods.Abstractions.Resources.IO;

Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
using JetBrains.Annotations;
using NexusMods.Abstractions.IO;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;

namespace NexusMods.Abstractions.Resources.IO;

Original file line number Diff line number Diff line change
@@ -7,5 +7,4 @@
<ProjectReference Include="..\NexusMods.Abstractions.IO\NexusMods.Abstractions.IO.csproj" />
<ProjectReference Include="..\NexusMods.Abstractions.Resources\NexusMods.Abstractions.Resources.csproj" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
using JetBrains.Annotations;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.Paths;

namespace NexusMods.Abstractions.Resources.IO;
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System.IO.Hashing;
using System.IO.MemoryMappedFiles;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.Paths;

namespace NexusMods.Extensions.Hashing;
@@ -18,6 +19,8 @@ public static async Task<Hash> XxHash64Async(this AbsolutePath input, Cancellati
return await inputStream.HashingCopyAsync(Stream.Null, token, static _ => Task.CompletedTask);
}

private static readonly Hash HashOfEmptyFile = Hash.From(XxHash3.HashToUInt64(ReadOnlySpan<byte>.Empty));

/// <summary>
/// Calculates the xxHash64 of a file by memory mapping it.
/// </summary>
@@ -26,12 +29,12 @@ public static Hash XxHash64MemoryMapped(this AbsolutePath input)
try
{
using var mmf = input.FileSystem.CreateMemoryMappedFile(input, FileMode.Open, MemoryMappedFileAccess.Read, 0);
var hashValue = XxHash64Algorithm.HashBytes(mmf.AsSpan());
var hashValue = XxHash3.HashToUInt64(mmf.AsSpan());
return Hash.From(hashValue);
}
catch (ArgumentException)
{
return Hash.From(XxHash64Algorithm.HashOfEmptyFile);
return HashOfEmptyFile;
}
}
}
Original file line number Diff line number Diff line change
@@ -3,7 +3,8 @@
<Import Project="$([MSBuild]::GetPathOfFileAbove('NuGet.Build.props', '$(MSBuildThisFileDirectory)../'))" />

<ItemGroup>
<PackageReference Include="NexusMods.Hashing.xxHash64" />
<PackageReference Include="NexusMods.Hashing.xxHash3" />
<PackageReference Include="NexusMods.Paths" />
<PackageReference Include="System.IO.Hashing" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;

namespace NexusMods.Extensions.Hashing;

@@ -18,7 +18,7 @@ public static async Task<Hash> HashingCopyAsync(this Stream inputStream, Stream
/// <summary>
/// Helper method to calculate the hash of a given stream.
/// </summary>
public static async Task<Hash> XxHash64Async(this Stream inputStream, CancellationToken token)
public static async Task<Hash> XxHash3Async(this Stream inputStream, CancellationToken token)
{
return await inputStream.HashingCopyAsync(Stream.Null, token, static _ => Task.CompletedTask);
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using NexusMods.Hashing.xxHash64;
using System.IO.Hashing;
using Reloaded.Memory.Extensions;

namespace NexusMods.Extensions.Hashing;
@@ -22,7 +22,7 @@ public static class StringExtensions
/// </remarks>
public static ulong GetStableHash(this ReadOnlySpan<char> input)
{
return XxHash64Algorithm.HashBytes(input.CastFast<char, byte>());
return XxHash3.HashToUInt64(input.CastFast<char, byte>());
}

}
}
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@
using NexusMods.Abstractions.Resources;
using NexusMods.Abstractions.Telemetry;
using NexusMods.Games.Larian.BaldursGate3.Utils.LsxXmlParsing;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using Polly;

namespace NexusMods.Games.Larian.BaldursGate3.Emitters;
2 changes: 1 addition & 1 deletion src/Games/NexusMods.Games.Larian/BaldursGate3/Pipelines.cs
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@
using NexusMods.Abstractions.Resources.IO;
using NexusMods.Games.Larian.BaldursGate3.Utils.LsxXmlParsing;
using NexusMods.Games.Larian.BaldursGate3.Utils.PakParsing;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using Polly;

namespace NexusMods.Games.Larian.BaldursGate3;
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@
using NexusMods.Abstractions.Library.Models;
using NexusMods.Abstractions.Loadouts;
using NexusMods.Games.RedEngine.Cyberpunk2077.Models;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.Paths;
using NexusMods.Paths.Extensions;
2 changes: 1 addition & 1 deletion src/Games/NexusMods.Games.StardewValley/Pipelines.cs
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@
using NexusMods.Abstractions.Resources.Caching;
using NexusMods.Abstractions.Resources.IO;
using NexusMods.Games.StardewValley.Models;
using NexusMods.Hashing.xxHash64;
using NexusMods.Hashing.xxHash3;
using SMAPIManifest = StardewModdingAPI.Toolkit.Serialization.Models.Manifest;

namespace NexusMods.Games.StardewValley;
Loading

0 comments on commit 5d1cfec

Please sign in to comment.