Faster IndexOfAny for IgnoreCase Ascii letters (#96588)

* Add packed IgnoreCase IndexOfAny variants * Emit SearchValues for ASCII sets of 4/5 values in RegexCompliler * Remove Any3CharPackedIgnoreCase * Update asserts * Attribute order * Fix build * More comments * Tweak ContainsCore
dotnet · Feb 17, 2024 · 432397d · 432397d
1 parent 483fd0c
commit 432397d
Show file tree

Hide file tree

Showing 22 changed files with 600 additions and 368 deletions.
diff --git a/src/libraries/System.Memory/tests/Span/SearchValues.cs b/src/libraries/System.Memory/tests/Span/SearchValues.cs
@@ -44,6 +44,10 @@ public static IEnumerable<object[]> Values_MemberData()
                 "aaa",
                 "aaaa",
                 "aaaaa",
+                "Aa",
+                "AaBb",
+                "AaBbCc",
+                "[]{}",
                 "\uFFF0",
                 "\uFFF0\uFFF2",
                 "\uFFF0\uFFF2\uFFF4",

diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems
@@ -423,14 +423,16 @@
     <Compile Include="$(MSBuildThisFileDirectory)System\IFormatProvider.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\IFormattable.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Index.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any1CharPackedSearchValues.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any1CharPackedIgnoreCaseSearchValues.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any2CharPackedIgnoreCaseSearchValues.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any3CharPackedSearchValues.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any2CharPackedSearchValues.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any1SearchValues.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any2SearchValues.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any3SearchValues.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\BitVector256.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\ProbabilisticWithAsciiCharSearchValues.cs" />
-    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\SingleCharSearchValues.cs" />
-    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\SingleByteSearchValues.cs" />
-    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any2ByteSearchValues.cs" />
-    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any2CharSearchValues.cs" />
-    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any3ByteSearchValues.cs" />
-    <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any3CharSearchValues.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any4SearchValues.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\Any5SearchValues.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\SearchValues\AsciiByteSearchValues.cs" />

diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs
@@ -397,7 +397,7 @@ internal static int IndexOfOrdinalIgnoreCase(ReadOnlySpan<char> source, ReadOnly
                 // Do a quick search for the first element of "value".
                 int relativeIndex = isLetter ?
                     PackedSpanHelpers.PackedIndexOfIsSupported
-                        ? PackedSpanHelpers.IndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceMinusValueTailLength)
+                        ? PackedSpanHelpers.IndexOfAnyIgnoreCase(ref Unsafe.Add(ref searchSpace, offset), valueCharL, searchSpaceMinusValueTailLength)
                         : SpanHelpers.IndexOfAnyChar(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceMinusValueTailLength) :
                     SpanHelpers.IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceMinusValueTailLength);
                 if (relativeIndex < 0)

diff --git a/...es/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs b/...es/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs
@@ -0,0 +1,52 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics.X86;
+
+namespace System.Buffers
+{
+    internal sealed class Any1CharPackedIgnoreCaseSearchValues : SearchValues<char>
+    {
+        // While this most commonly applies to ASCII letters, it also works for other values that differ by 0x20 (e.g. "[{" => "{").
+        // _lowerCase is therefore not necessarily a lower case ASCII letter, but just the higher value (the one with the 0x20 bit set).
+        private readonly char _lowerCase, _upperCase;
+        private readonly uint _lowerCaseUint;
+
+        public Any1CharPackedIgnoreCaseSearchValues(char value)
+        {
+            Debug.Assert((value | 0x20) == value);
+
+            _lowerCase = value;
+            _upperCase = (char)(value & ~0x20);
+            _lowerCaseUint = value;
+        }
+
+        internal override char[] GetValues() =>
+            [_upperCase, _lowerCase];
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override bool ContainsCore(char value) =>
+            (uint)(value | 0x20) == _lowerCaseUint;
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        [CompExactlyDependsOn(typeof(Sse2))]
+        internal override int IndexOfAny(ReadOnlySpan<char> span) =>
+            PackedSpanHelpers.IndexOfAnyIgnoreCase(ref MemoryMarshal.GetReference(span), _lowerCase, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        [CompExactlyDependsOn(typeof(Sse2))]
+        internal override int IndexOfAnyExcept(ReadOnlySpan<char> span) =>
+            PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref MemoryMarshal.GetReference(span), _lowerCase, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override int LastIndexOfAny(ReadOnlySpan<char> span) =>
+            span.LastIndexOfAny(_lowerCase, _upperCase);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override int LastIndexOfAnyExcept(ReadOnlySpan<char> span) =>
+            span.LastIndexOfAnyExcept(_lowerCase, _upperCase);
+    }
+}
diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedSearchValues.cs
@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics.X86;
+
+namespace System.Buffers
+{
+    internal sealed class Any1CharPackedSearchValues : SearchValues<char>
+    {
+        private readonly char _e0;
+
+        public Any1CharPackedSearchValues(char value) =>
+            _e0 = value;
+
+        internal override char[] GetValues() =>
+            [_e0];
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override bool ContainsCore(char value) =>
+            value == _e0;
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        [CompExactlyDependsOn(typeof(Sse2))]
+        internal override int IndexOfAny(ReadOnlySpan<char> span) =>
+            PackedSpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        [CompExactlyDependsOn(typeof(Sse2))]
+        internal override int IndexOfAnyExcept(ReadOnlySpan<char> span) =>
+            PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override int LastIndexOfAny(ReadOnlySpan<char> span) =>
+            span.LastIndexOf(_e0);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override int LastIndexOfAnyExcept(ReadOnlySpan<char> span) =>
+            span.LastIndexOfAnyExcept(_e0);
+    }
+}
diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1SearchValues.cs
@@ -0,0 +1,52 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using System.Numerics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+#pragma warning disable 8500 // address of managed types
+
+namespace System.Buffers
+{
+    internal sealed class Any1SearchValues<T, TImpl> : SearchValues<T>
+        where T : struct, IEquatable<T>
+        where TImpl : struct, INumber<TImpl>
+    {
+        private readonly TImpl _e0;
+
+        public Any1SearchValues(ReadOnlySpan<TImpl> values)
+        {
+            Debug.Assert(Unsafe.SizeOf<T>() == Unsafe.SizeOf<TImpl>());
+            Debug.Assert(values.Length == 1);
+            _e0 = values[0];
+        }
+
+        internal override unsafe T[] GetValues()
+        {
+            TImpl e0 = _e0;
+            return new[] { *(T*)&e0 };
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override unsafe bool ContainsCore(T value) =>
+            *(TImpl*)&value == _e0;
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override int IndexOfAny(ReadOnlySpan<T> span) =>
+            SpanHelpers.NonPackedIndexOfValueType<TImpl, SpanHelpers.DontNegate<TImpl>>(ref Unsafe.As<T, TImpl>(ref MemoryMarshal.GetReference(span)), _e0, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override int IndexOfAnyExcept(ReadOnlySpan<T> span) =>
+            SpanHelpers.NonPackedIndexOfValueType<TImpl, SpanHelpers.Negate<TImpl>>(ref Unsafe.As<T, TImpl>(ref MemoryMarshal.GetReference(span)), _e0, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override int LastIndexOfAny(ReadOnlySpan<T> span) =>
+            SpanHelpers.LastIndexOfValueType(ref Unsafe.As<T, TImpl>(ref MemoryMarshal.GetReference(span)), _e0, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override int LastIndexOfAnyExcept(ReadOnlySpan<T> span) =>
+            SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As<T, TImpl>(ref MemoryMarshal.GetReference(span)), _e0, span.Length);
+    }
+}
diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2ByteSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2ByteSearchValues.cs
diff --git a/...es/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs b/...es/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs
@@ -0,0 +1,67 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics.Arm;
+using System.Runtime.Intrinsics.Wasm;
+using System.Runtime.Intrinsics.X86;
+
+namespace System.Buffers
+{
+    internal sealed class Any2CharPackedIgnoreCaseSearchValues : SearchValues<char>
+    {
+        // While this most commonly applies to ASCII letters, it also works for other values that differ by 0x20 (e.g. "[]{}" => "{}").
+        // _e0 and _e1 are therefore not necessarily lower case ASCII letters, but just the higher values (the ones with the 0x20 bit set).
+        private readonly char _e0, _e1;
+        private readonly uint _uint0, _uint1;
+        private IndexOfAnyAsciiSearcher.AsciiState _state;
+
+        public Any2CharPackedIgnoreCaseSearchValues(char value0, char value1)
+        {
+            Debug.Assert((value0 | 0x20) == value0 && char.IsAscii(value0));
+            Debug.Assert((value1 | 0x20) == value1 && char.IsAscii(value1));
+
+            (_e0, _e1) = (value0, value1);
+            (_uint0, _uint1) = (value0, value1);
+            IndexOfAnyAsciiSearcher.ComputeAsciiState([(char)(_e0 & ~0x20), _e0, (char)(_e1 & ~0x20), _e1], out _state);
+        }
+
+        internal override char[] GetValues() =>
+            [(char)(_e0 & ~0x20), _e0, (char)(_e1 & ~0x20), _e1];
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override bool ContainsCore(char value)
+        {
+            uint lowerCase = (uint)(value | 0x20);
+            return lowerCase == _uint0 || lowerCase == _uint1;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        [CompExactlyDependsOn(typeof(Sse2))]
+        internal override int IndexOfAny(ReadOnlySpan<char> span) =>
+            PackedSpanHelpers.IndexOfAnyIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        [CompExactlyDependsOn(typeof(Sse2))]
+        internal override int IndexOfAnyExcept(ReadOnlySpan<char> span) =>
+            PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
+        internal override int LastIndexOfAny(ReadOnlySpan<char> span) =>
+            IndexOfAnyAsciiSearcher.LastIndexOfAny<IndexOfAnyAsciiSearcher.DontNegate, IndexOfAnyAsciiSearcher.Default>(
+                ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        [CompExactlyDependsOn(typeof(Ssse3))]
+        [CompExactlyDependsOn(typeof(AdvSimd))]
+        [CompExactlyDependsOn(typeof(PackedSimd))]
+        internal override int LastIndexOfAnyExcept(ReadOnlySpan<char> span) =>
+            IndexOfAnyAsciiSearcher.LastIndexOfAny<IndexOfAnyAsciiSearcher.Negate, IndexOfAnyAsciiSearcher.Default>(
+                ref Unsafe.As<char, short>(ref MemoryMarshal.GetReference(span)), span.Length, ref _state);
+    }
+}
diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedSearchValues.cs
@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics.X86;
+
+namespace System.Buffers
+{
+    internal sealed class Any2CharPackedSearchValues : SearchValues<char>
+    {
+        private readonly char _e0, _e1;
+
+        public Any2CharPackedSearchValues(char value0, char value1) =>
+            (_e0, _e1) = (value0, value1);
+
+        internal override char[] GetValues() =>
+            [_e0, _e1];
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override bool ContainsCore(char value) =>
+            value == _e0 || value == _e1;
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        [CompExactlyDependsOn(typeof(Sse2))]
+        internal override int IndexOfAny(ReadOnlySpan<char> span) =>
+            PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        [CompExactlyDependsOn(typeof(Sse2))]
+        internal override int IndexOfAnyExcept(ReadOnlySpan<char> span) =>
+            PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override int LastIndexOfAny(ReadOnlySpan<char> span) =>
+            span.LastIndexOfAny(_e0, _e1);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal override int LastIndexOfAnyExcept(ReadOnlySpan<char> span) =>
+            span.LastIndexOfAnyExcept(_e0, _e1);
+    }
+}
diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharSearchValues.cs