From 0f60e44e161bea1b8a89b9c00b1622d9501b909d Mon Sep 17 00:00:00 2001 From: Alexandre Mutel Date: Fri, 4 Nov 2022 08:00:53 +0100 Subject: [PATCH 1/3] Optimize XxHash3 on ARM platform --- .../src/System/IO/Hashing/XxHash3.cs | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs index 4e821618d36e3e..7a2af6ebf11f66 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs @@ -896,12 +896,24 @@ private static Vector128 Accumulate128(Vector128 accVec, byte* sou Vector128 sourceKey = sourceVec ^ secret; // TODO: Figure out how to unwind this shuffle and just use Vector128.Multiply - Vector128 sourceKeyLow = Vector128.Shuffle(sourceKey, Vector128.Create(1u, 0, 3, 0)); Vector128 sourceSwap = Vector128.Shuffle(sourceVec, Vector128.Create(2u, 3, 0, 1)); Vector128 sum = accVec + sourceSwap.AsUInt64(); - Vector128 product = Sse2.IsSupported ? - Sse2.Multiply(sourceKey, sourceKeyLow) : - (sourceKey & Vector128.Create(~0u, 0u, ~0u, 0u)).AsUInt64() * (sourceKeyLow & Vector128.Create(~0u, 0u, ~0u, 0u)).AsUInt64(); + + Vector128 product; + // TODO: Workaround for ARM as `*` is software emulated. + if (System.Runtime.Intrinsics.Arm.AdvSimd.IsSupported) + { + Vector64 sourceKeyLow = Vector128.Shuffle(sourceKey, Vector128.Create(0u, 2, 0, 0)).GetLower(); + Vector64 sourceKeyHigh = Vector128.Shuffle(sourceKey, Vector128.Create(1u, 3, 0, 0)).GetLower(); + product = System.Runtime.Intrinsics.Arm.AdvSimd.MultiplyWideningLower(sourceKeyLow, sourceKeyHigh); + } + else + { + Vector128 sourceKeyLow = Vector128.Shuffle(sourceKey, Vector128.Create(1u, 0, 3, 0)); + product = Sse2.IsSupported + ? Sse2.Multiply(sourceKey, sourceKeyLow) + : (sourceKey & Vector128.Create(~0u, 0u, ~0u, 0u)).AsUInt64() * (sourceKeyLow & Vector128.Create(~0u, 0u, ~0u, 0u)).AsUInt64(); + } accVec = product + sum; return accVec; From 6b07c0985fc3871c6d274abc79f7bc7fc77fcfa5 Mon Sep 17 00:00:00 2001 From: Alexandre Mutel Date: Fri, 4 Nov 2022 21:21:32 +0100 Subject: [PATCH 2/3] Extract code to MultiplyWideningLower --- .../src/System/IO/Hashing/XxHash3.cs | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs index 7a2af6ebf11f66..a700337efb2248 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs @@ -10,6 +10,7 @@ using System.Runtime.InteropServices; #if NET7_0_OR_GREATER using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; #endif @@ -899,24 +900,26 @@ private static Vector128 Accumulate128(Vector128 accVec, byte* sou Vector128 sourceSwap = Vector128.Shuffle(sourceVec, Vector128.Create(2u, 3, 0, 1)); Vector128 sum = accVec + sourceSwap.AsUInt64(); - Vector128 product; - // TODO: Workaround for ARM as `*` is software emulated. - if (System.Runtime.Intrinsics.Arm.AdvSimd.IsSupported) + Vector128 product = MultiplyWideningLower(sourceKey); + accVec = product + sum; + return accVec; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 MultiplyWideningLower(Vector128 source) + { + if (AdvSimd.IsSupported) { - Vector64 sourceKeyLow = Vector128.Shuffle(sourceKey, Vector128.Create(0u, 2, 0, 0)).GetLower(); - Vector64 sourceKeyHigh = Vector128.Shuffle(sourceKey, Vector128.Create(1u, 3, 0, 0)).GetLower(); - product = System.Runtime.Intrinsics.Arm.AdvSimd.MultiplyWideningLower(sourceKeyLow, sourceKeyHigh); + Vector64 sourceLow = Vector128.Shuffle(source, Vector128.Create(0u, 2, 0, 0)).GetLower(); + Vector64 sourceHigh = Vector128.Shuffle(source, Vector128.Create(1u, 3, 0, 0)).GetLower(); + return AdvSimd.MultiplyWideningLower(sourceLow, sourceHigh); } else { - Vector128 sourceKeyLow = Vector128.Shuffle(sourceKey, Vector128.Create(1u, 0, 3, 0)); - product = Sse2.IsSupported - ? Sse2.Multiply(sourceKey, sourceKeyLow) - : (sourceKey & Vector128.Create(~0u, 0u, ~0u, 0u)).AsUInt64() * (sourceKeyLow & Vector128.Create(~0u, 0u, ~0u, 0u)).AsUInt64(); + Vector128 sourceLow = Vector128.Shuffle(source, Vector128.Create(1u, 0, 3, 0)); + return Sse2.IsSupported ? Sse2.Multiply(source, sourceLow) : + (source & Vector128.Create(~0u, 0u, ~0u, 0u)).AsUInt64() * (sourceLow & Vector128.Create(~0u, 0u, ~0u, 0u)).AsUInt64(); } - - accVec = product + sum; - return accVec; } #endif From 1a310ccfd1f4d7e3eb254230ce24eecb60fbbdfe Mon Sep 17 00:00:00 2001 From: Alexandre Mutel Date: Fri, 4 Nov 2022 23:19:43 +0100 Subject: [PATCH 3/3] Update src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs Co-authored-by: Stephen Toub --- .../System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs index a700337efb2248..17087d87631036 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash3.cs @@ -917,7 +917,8 @@ private static Vector128 MultiplyWideningLower(Vector128 source) else { Vector128 sourceLow = Vector128.Shuffle(source, Vector128.Create(1u, 0, 3, 0)); - return Sse2.IsSupported ? Sse2.Multiply(source, sourceLow) : + return Sse2.IsSupported ? + Sse2.Multiply(source, sourceLow) : (source & Vector128.Create(~0u, 0u, ~0u, 0u)).AsUInt64() * (sourceLow & Vector128.Create(~0u, 0u, ~0u, 0u)).AsUInt64(); } }