From 682e3070a15257e81d37effded323638567f5f68 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Thu, 4 May 2023 01:35:03 +0200 Subject: [PATCH] Improve IndexOfAnyValues for needles with 0 --- .../SearchValues/IndexOfAnyAsciiSearcher.cs | 85 ++++++++++--------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs index 56e7bd741c0b25..45c5849388ca91 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs @@ -813,33 +813,10 @@ private static Vector128 IndexOfAnyLookup(Vector where TNegator : struct, INegator where TOptimizations : struct, IOptimizations { - // Pack two vectors of characters into bytes. While the type is Vector128, these are really UInt16 characters. - // X86 and WASM: Downcast every character using saturation. - // - Values <= 32767 result in min(value, 255). - // - Values > 32767 result in 0. Because of this we must do more work to handle needles that contain 0. - // ARM64: Do narrowing saturation over unsigned values. - // - All values result in min(value, 255) - Vector128 source = - Sse2.IsSupported ? Sse2.PackUnsignedSaturate(source0, source1) : - AdvSimd.IsSupported ? AdvSimd.ExtractNarrowingSaturateUpper(AdvSimd.ExtractNarrowingSaturateLower(source0.AsUInt16()), source1.AsUInt16()) : - PackedSimd.ConvertNarrowingUnsignedSaturate(source0, source1); + Vector128 source = TOptimizations.PackSources(source0.AsUInt16(), source1.AsUInt16()); Vector128 result = IndexOfAnyLookupCore(source, bitmapLookup); - // On X86 and WASM, the packing/narrowing above resulted in values becoming 0 for inputs above 32767. - // Any value above 32767 would therefore match against 0. If 0 is present in the needle, we must clear the false positives. - // We can correct the result by clearing any bits that matched with a non-ascii source character. - if (TOptimizations.NeedleContainsZero) - { - Debug.Assert(Sse2.IsSupported || PackedSimd.IsSupported); - Vector128 ascii0 = Vector128.LessThan(source0.AsUInt16(), Vector128.Create((ushort)128)).AsInt16(); - Vector128 ascii1 = Vector128.LessThan(source1.AsUInt16(), Vector128.Create((ushort)128)).AsInt16(); - Vector128 ascii = Sse2.IsSupported - ? Sse2.PackSignedSaturate(ascii0, ascii1).AsByte() - : PackedSimd.ConvertNarrowingSignedSaturate(ascii0, ascii1).AsByte(); - result &= ascii; - } - return TNegator.NegateIfNeeded(result); } @@ -870,23 +847,14 @@ private static Vector128 IndexOfAnyLookupCore(Vector128 source, Vect return result; } - [BypassReadyToRun] [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector256 IndexOfAnyLookup(Vector256 source0, Vector256 source1, Vector256 bitmapLookup) where TNegator : struct, INegator where TOptimizations : struct, IOptimizations { - // See comments in IndexOfAnyLookup(Vector128) above for more details. - Vector256 source = Avx2.PackUnsignedSaturate(source0, source1); - Vector256 result = IndexOfAnyLookupCore(source, bitmapLookup); + Vector256 source = TOptimizations.PackSources(source0.AsUInt16(), source1.AsUInt16()); - if (TOptimizations.NeedleContainsZero) - { - Vector256 ascii0 = Vector256.LessThan(source0.AsUInt16(), Vector256.Create((ushort)128)).AsInt16(); - Vector256 ascii1 = Vector256.LessThan(source1.AsUInt16(), Vector256.Create((ushort)128)).AsInt16(); - Vector256 ascii = Avx2.PackSignedSaturate(ascii0, ascii1).AsByte(); - result &= ascii; - } + Vector256 result = IndexOfAnyLookupCore(source, bitmapLookup); return TNegator.NegateIfNeeded(result); } @@ -1115,17 +1083,58 @@ internal interface INegator internal interface IOptimizations { - static abstract bool NeedleContainsZero { get; } + // Pack two vectors of characters into bytes. + // X86 and WASM when the needle does not contain 0: Downcast every character using saturation. + // - Values <= 32767 result in min(value, 255). + // - Values > 32767 result in 0. Because of this we must do more work to handle needles that contain 0. + // Otherwise: Do narrowing saturation over unsigned values. + // - All values result in min(value, 255) + static abstract Vector128 PackSources(Vector128 lower, Vector128 upper); + static abstract Vector256 PackSources(Vector256 lower, Vector256 upper); } internal readonly struct Ssse3AndWasmHandleZeroInNeedle : IOptimizations { - public static bool NeedleContainsZero => true; + // Replace with Vector128.NarrowWithSaturation once https://github.com/dotnet/runtime/issues/75724 is implemented. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 PackSources(Vector128 lower, Vector128 upper) + { + Vector128 lowerMin = Vector128.Min(lower, Vector128.Create((ushort)255)).AsInt16(); + Vector128 upperMin = Vector128.Min(upper, Vector128.Create((ushort)255)).AsInt16(); + + return Sse2.IsSupported + ? Sse2.PackUnsignedSaturate(lowerMin, upperMin) + : PackedSimd.ConvertNarrowingUnsignedSaturate(lowerMin, upperMin); + } + + // Replace with Vector256.NarrowWithSaturation once https://github.com/dotnet/runtime/issues/75724 is implemented. + [BypassReadyToRun] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 PackSources(Vector256 lower, Vector256 upper) + { + return Avx2.PackUnsignedSaturate( + Vector256.Min(lower, Vector256.Create((ushort)255)).AsInt16(), + Vector256.Min(upper, Vector256.Create((ushort)255)).AsInt16()); + } } internal readonly struct Default : IOptimizations { - public static bool NeedleContainsZero => false; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 PackSources(Vector128 lower, Vector128 upper) + { + return + Sse2.IsSupported ? Sse2.PackUnsignedSaturate(lower.AsInt16(), upper.AsInt16()) : + AdvSimd.IsSupported ? AdvSimd.ExtractNarrowingSaturateUpper(AdvSimd.ExtractNarrowingSaturateLower(lower), upper) : + PackedSimd.ConvertNarrowingUnsignedSaturate(lower.AsInt16(), upper.AsInt16()); + } + + [BypassReadyToRun] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 PackSources(Vector256 lower, Vector256 upper) + { + return Avx2.PackUnsignedSaturate(lower.AsInt16(), upper.AsInt16()); + } } } }