diff --git a/benchmark/Benchmark.cs b/benchmark/Benchmark.cs index 0900059..3f2a303 100644 --- a/benchmark/Benchmark.cs +++ b/benchmark/Benchmark.cs @@ -259,7 +259,13 @@ public unsafe void SIMDUtf8ValidationRealDataSse() { if (allLinesUtf8 != null) { - RunUtf8ValidationBenchmark(allLinesUtf8, SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + RunUtf8ValidationBenchmark(allLinesUtf8, (byte* pInputBuffer, int inputLength) => + { + int dummyUtf16CodeUnitCountAdjustment, dummyScalarCountAdjustment; + // Call the method with additional out parameters within the lambda. + // You must handle these additional out parameters inside the lambda, as they cannot be passed back through the delegate. + return SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse(pInputBuffer, inputLength, out dummyUtf16CodeUnitCountAdjustment, out dummyScalarCountAdjustment); + }); } } diff --git a/src/UTF8.cs b/src/UTF8.cs index 0a96ece..e38cb34 100644 --- a/src/UTF8.cs +++ b/src/UTF8.cs @@ -10,6 +10,7 @@ namespace SimdUnicode public static class UTF8 { + // Returns &inputBuffer[inputLength] if the input buffer is valid. /// /// Given an input buffer of byte length , @@ -35,11 +36,10 @@ public static class UTF8 { return GetPointerToFirstInvalidByteAvx512(pInputBuffer, inputLength); }*/ - // if (Ssse3.IsSupported) - // { - // return GetPointerToFirstInvalidByteSse(pInputBuffer, inputLength); - // } - // return GetPointerToFirstInvalidByteScalar(pInputBuffer, inputLength); + if (Ssse3.IsSupported) + { + return GetPointerToFirstInvalidByteSse(pInputBuffer, inputLength,out Utf16CodeUnitCountAdjustment, out ScalarCodeUnitCountAdjustment); + } return GetPointerToFirstInvalidByteScalar(pInputBuffer, inputLength, out Utf16CodeUnitCountAdjustment, out ScalarCodeUnitCountAdjustment); @@ -471,15 +471,13 @@ private unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust return (utfadjust, scalaradjust); } - public unsafe static byte* GetPointerToFirstInvalidByteSse(byte* pInputBuffer, int inputLength) + public unsafe static byte* GetPointerToFirstInvalidByteSse(byte* pInputBuffer, int inputLength, out int utf16CodeUnitCountAdjustment, out int scalarCountAdjustment) { - int processedLength = 0; - int TempUtf16CodeUnitCountAdjustment = 0; - int TempScalarCountAdjustment = 0; - if (pInputBuffer == null || inputLength <= 0) { + utf16CodeUnitCountAdjustment = 0; + scalarCountAdjustment = 0; return pInputBuffer; } if (inputLength > 128) @@ -503,16 +501,15 @@ private unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust if (processedLength + 16 < inputLength) { - // We still have work to do! Vector128 prevInputBlock = Vector128.Zero; Vector128 maxValue = Vector128.Create( 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0b11110000 - 1, 0b11100000 - 1, 0b11000000 - 1); - Vector128 prevIncomplete = Sse2.SubtractSaturate(prevInputBlock, maxValue); - + Vector128 prevIncomplete = Sse3.SubtractSaturate(prevInputBlock, maxValue); - Vector128 shuf1 = Vector128.Create(TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + Vector128 shuf1 = Vector128.Create( + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, TOO_SHORT | OVERLONG_2, @@ -520,7 +517,8 @@ private unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust TOO_SHORT | OVERLONG_3 | SURROGATE, TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4); - Vector128 shuf2 = Vector128.Create(CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + Vector128 shuf2 = Vector128.Create( + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, CARRY | OVERLONG_2, CARRY, CARRY, @@ -536,7 +534,8 @@ private unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, CARRY | TOO_LARGE | TOO_LARGE_1000, CARRY | TOO_LARGE | TOO_LARGE_1000); - Vector128 shuf3 = Vector128.Create(TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + Vector128 shuf3 = Vector128.Create( + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, @@ -548,24 +547,71 @@ private unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust Vector128 fourthByte = Vector128.Create((byte)(0b11110000u - 0x80)); Vector128 v0f = Vector128.Create((byte)0x0F); Vector128 v80 = Vector128.Create((byte)0x80); + /**** + * So we want to count the number of 4-byte sequences, + * the number of 4-byte sequences, 3-byte sequences, and + * the number of 2-byte sequences. + * We can do it indirectly. We know how many bytes in total + * we have (length). Let us assume that the length covers + * only complete sequences (we need to adjust otherwise). + * We have that + * length = 4 * n4 + 3 * n3 + 2 * n2 + n1 + * where n1 is the number of 1-byte sequences (ASCII), + * n2 is the number of 2-byte sequences, n3 is the number + * of 3-byte sequences, and n4 is the number of 4-byte sequences. + * + * Let ncon be the number of continuation bytes, then we have + * length = n4 + n3 + n2 + ncon + n1 + * + * We can solve for n2 and n3 in terms of the other variables: + * n3 = n1 - 2 * n4 + 2 * ncon - length + * n2 = -2 * n1 + n4 - 4 * ncon + 2 * length + * Thus we only need to count the number of continuation bytes, + * the number of ASCII bytes and the number of 4-byte sequences. + */ + //////////// + // The *block* here is what begins at processedLength and ends + // at processedLength/16*16 or when an error occurs. + /////////// + int start_point = processedLength; + + // The block goes from processedLength to processedLength/16*16. + int asciibytes = 0; // number of ascii bytes in the block (could also be called n1) + int contbytes = 0; // number of continuation bytes in the block + int n4 = 0; // number of 4-byte sequences that start in this block for (; processedLength + 16 <= inputLength; processedLength += 16) { - Vector128 currentBlock = Sse2.LoadVector128(pInputBuffer + processedLength); - - int mask = Sse2.MoveMask(currentBlock); + Vector128 currentBlock = Avx.LoadVector128(pInputBuffer + processedLength); + int mask = Sse42.MoveMask(currentBlock); if (mask == 0) { // We have an ASCII block, no need to process it, but // we need to check if the previous block was incomplete. - if (Sse2.MoveMask(prevIncomplete) != 0) + // + + if (!Sse41.TestZ(prevIncomplete, prevIncomplete)) { - return SimdUnicode.UTF8.RewindAndValidateWithErrors(processedLength, pInputBuffer + processedLength, inputLength - processedLength, ref TempUtf16CodeUnitCountAdjustment, ref TempScalarCountAdjustment); + int off = processedLength >= 3 ? processedLength - 3 : processedLength; + byte* invalidBytePointer = SimdUnicode.UTF8.SimpleRewindAndValidateWithErrors(16 - 3, pInputBuffer + processedLength - 3, inputLength - processedLength + 3); + // So the code is correct up to invalidBytePointer + if (invalidBytePointer < pInputBuffer + processedLength) + { + removeCounters(invalidBytePointer, pInputBuffer + processedLength, ref asciibytes, ref n4, ref contbytes); + } + else + { + addCounters(pInputBuffer + processedLength, invalidBytePointer, ref asciibytes, ref n4, ref contbytes); + } + int totalbyteasciierror = processedLength - start_point; + (utf16CodeUnitCountAdjustment, scalarCountAdjustment) = CalculateN2N3FinalSIMDAdjustments(asciibytes, n4, contbytes, totalbyteasciierror); + return invalidBytePointer; } prevIncomplete = Vector128.Zero; } - else + else // Contains non-ASCII characters, we need to do non-trivial processing { + // Use SubtractSaturate to effectively compare if bytes in block are greater than markers. // Contains non-ASCII characters, we need to do non-trivial processing Vector128 prev1 = Ssse3.AlignRight(currentBlock, prevInputBlock, (byte)(16 - 1)); Vector128 byte_1_high = Ssse3.Shuffle(shuf1, Sse2.ShiftRightLogical(prev1.AsUInt16(), 4).AsByte() & v0f); @@ -575,54 +621,93 @@ private unsafe static (int utfadjust, int scalaradjust) calculateErrorPathadjust Vector128 prev2 = Ssse3.AlignRight(currentBlock, prevInputBlock, (byte)(16 - 2)); Vector128 prev3 = Ssse3.AlignRight(currentBlock, prevInputBlock, (byte)(16 - 3)); prevInputBlock = currentBlock; + Vector128 isThirdByte = Sse2.SubtractSaturate(prev2, thirdByte); Vector128 isFourthByte = Sse2.SubtractSaturate(prev3, fourthByte); Vector128 must23 = Sse2.Or(isThirdByte, isFourthByte); Vector128 must23As80 = Sse2.And(must23, v80); Vector128 error = Sse2.Xor(must23As80, sc); - if (Sse2.MoveMask(error) != 0) + + if (!Sse42.TestZ(error, error)) { - return SimdUnicode.UTF8.RewindAndValidateWithErrors(processedLength, pInputBuffer + processedLength, inputLength - processedLength, ref TempUtf16CodeUnitCountAdjustment, ref TempScalarCountAdjustment); + + byte* invalidBytePointer; + if (processedLength == 0) + { + invalidBytePointer = SimdUnicode.UTF8.SimpleRewindAndValidateWithErrors(0, pInputBuffer + processedLength, inputLength - processedLength); + } + else + { + invalidBytePointer = SimdUnicode.UTF8.SimpleRewindAndValidateWithErrors(processedLength - 3, pInputBuffer + processedLength - 3, inputLength - processedLength + 3); + } + if (invalidBytePointer < pInputBuffer + processedLength) + { + removeCounters(invalidBytePointer, pInputBuffer + processedLength, ref asciibytes, ref n4, ref contbytes); + } + else + { + addCounters(pInputBuffer + processedLength, invalidBytePointer, ref asciibytes, ref n4, ref contbytes); + } + int total_bytes_processed = (int)(invalidBytePointer - (pInputBuffer + start_point)); + (utf16CodeUnitCountAdjustment, scalarCountAdjustment) = CalculateN2N3FinalSIMDAdjustments(asciibytes, n4, contbytes, total_bytes_processed); + return invalidBytePointer; } - prevIncomplete = Sse2.SubtractSaturate(currentBlock, maxValue); + + prevIncomplete = Sse3.SubtractSaturate(currentBlock, maxValue); + + contbytes += (int)Popcnt.PopCount((uint)Sse42.MoveMask(byte_2_high)); + // We use two instructions (SubtractSaturate and MoveMask) to update n4, with one arithmetic operation. + n4 += (int)Popcnt.PopCount((uint)Sse42.MoveMask(Sse42.SubtractSaturate(currentBlock, fourthByte))); } + + // important: we just update asciibytes if there was no error. + // We count the number of ascii bytes in the block using just some simple arithmetic + // and no expensive operation: + asciibytes += (int)(16 - Popcnt.PopCount((uint)mask)); } - } - } - // We have processed all the blocks using SIMD, we need to process the remaining bytes. - // Process the remaining bytes with the scalar function - if (processedLength < inputLength) - { - // We need to possibly backtrack to the start of the last code point - // worst possible case is 4 bytes, where we need to backtrack 3 bytes - // 11110xxxx 10xxxxxx 10xxxxxx 10xxxxxx <== we might be pointing at the last byte - if (processedLength > 0 && (sbyte)pInputBuffer[processedLength] <= -65) - { - processedLength -= 1; - if (processedLength > 0 && (sbyte)pInputBuffer[processedLength] <= -65) + + // We may still have an error. + if (processedLength < inputLength || !Sse42.TestZ(prevIncomplete, prevIncomplete)) { - processedLength -= 1; - if (processedLength > 0 && (sbyte)pInputBuffer[processedLength] <= -65) + byte* invalidBytePointer; + if (processedLength == 0) + { + invalidBytePointer = SimdUnicode.UTF8.SimpleRewindAndValidateWithErrors(0, pInputBuffer + processedLength, inputLength - processedLength); + } + else + { + invalidBytePointer = SimdUnicode.UTF8.SimpleRewindAndValidateWithErrors(processedLength - 3, pInputBuffer + processedLength - 3, inputLength - processedLength + 3); + + } + if (invalidBytePointer != pInputBuffer + inputLength) + { + if (invalidBytePointer < pInputBuffer + processedLength) + { + removeCounters(invalidBytePointer, pInputBuffer + processedLength, ref asciibytes, ref n4, ref contbytes); + } + else + { + addCounters(pInputBuffer + processedLength, invalidBytePointer, ref asciibytes, ref n4, ref contbytes); + } + int total_bytes_processed = (int)(invalidBytePointer - (pInputBuffer + start_point)); + (utf16CodeUnitCountAdjustment, scalarCountAdjustment) = CalculateN2N3FinalSIMDAdjustments(asciibytes, n4, contbytes, total_bytes_processed); + return invalidBytePointer; + } + else { - processedLength -= 1; + addCounters(pInputBuffer + processedLength, invalidBytePointer, ref asciibytes, ref n4, ref contbytes); } } - } - int TailScalarCodeUnitCountAdjustment = 0; - int TailUtf16CodeUnitCountAdjustment = 0; - byte* invalidBytePointer = SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar(pInputBuffer + processedLength, inputLength - processedLength, out TailUtf16CodeUnitCountAdjustment, out TailScalarCodeUnitCountAdjustment); - if (invalidBytePointer != pInputBuffer + inputLength) - { - // An invalid byte was found by the scalar function - return invalidBytePointer; + int final_total_bytes_processed = inputLength - start_point; + (utf16CodeUnitCountAdjustment, scalarCountAdjustment) = CalculateN2N3FinalSIMDAdjustments(asciibytes, n4, contbytes, final_total_bytes_processed); + return pInputBuffer + inputLength; } } - - return pInputBuffer + inputLength; + return GetPointerToFirstInvalidByteScalar(pInputBuffer + processedLength, inputLength - processedLength, out utf16CodeUnitCountAdjustment, out scalarCountAdjustment); } - +// public unsafe static byte* GetPointerToFirstInvalidByteAvx2(byte* pInputBuffer, int inputLength, out int utf16CodeUnitCountAdjustment, out int scalarCountAdjustment) { int processedLength = 0; diff --git a/test/UTF8ValidationTests.cs b/test/UTF8ValidationTests.cs index eef06e1..03cc0f6 100644 --- a/test/UTF8ValidationTests.cs +++ b/test/UTF8ValidationTests.cs @@ -132,10 +132,18 @@ public void simpleGoodSequencesArm64() simpleGoodSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void simpleGoodSequencesSse() + { + simpleGoodSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + private void BadSequences(Utf8ValidationFunction utf8ValidationDelegate) { string[] badSequences = { "\xC3\x28", + "\x80", "\xA0\xA1", "\xE2\x28\xA1", "\xE2\x82\x28", @@ -187,6 +195,13 @@ public void BadSequencesScalar() BadSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void BadSequencesSse() + { + BadSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void BadSequencesAvx2() @@ -201,12 +216,7 @@ public void BadSequencesArm64() BadSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); } - // this was in the C++ code - private void Node48995Test(Utf8ValidationFunction utf8ValidationDelegate) - { - byte[] bad = new byte[] { 0x80 }; - Assert.False(ValidateUtf8(bad, utf8ValidationDelegate)); - } + private void NoError(Utf8ValidationFunction utf8ValidationDelegate) { @@ -239,6 +249,13 @@ public void NoErrorScalar() NoError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void NoErrorSse() + { + NoError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void NoErrorAvx2() @@ -292,6 +309,13 @@ public void NoErrorSpecificByteCountScalar() NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void NoErrorSpecificByteCountSse() + { + NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void NoErrorSpecificByteCountAvx2() @@ -351,6 +375,13 @@ public void NoErrorIncompleteThenASCIIScalar() NoErrorIncompleteThenASCII(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void NoErrorIncompleteThenASCIISse() + { + NoErrorIncompleteThenASCII(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void NoErrorIncompleteThenASCIIAvx2() @@ -405,6 +436,13 @@ public void NoErrorIncompleteAt256VectorScalar() NoErrorIncompleteAt256Vector(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void NoErrorIncompleteAt256VectorSse() + { + NoErrorIncompleteAt256Vector(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void NoErrorIncompleteAt256VectorAvx2() @@ -461,6 +499,13 @@ public void BadHeaderBitsScalar() BadHeaderBits(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void BadHeaderBitsSse() + { + BadHeaderBits(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void BadHeaderBitsAvx2() @@ -516,6 +561,13 @@ public void TooShortErrorScalar() TooShortError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void TooShortErrorSse() + { + TooShortError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void TooShortErrorAvx2() @@ -571,6 +623,13 @@ public void TooLongErrorScalar() TooLongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void TooLongErrorSse() + { + TooLongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void TooLongErrorAvx2() @@ -635,6 +694,13 @@ public void OverlongErrorScalar() OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void OverlongErrorSse() + { + OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + [Trait("Category", "arm64")] [FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)] public void OverlongErrorArm64() @@ -700,6 +766,13 @@ public void TooShortErrorAtEndScalar() TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void TooShortErrorAtEndSse() + { + TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] @@ -743,6 +816,13 @@ public void Invalid0xf50xffScalar() Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void Invalid0xf50xffSse() + { + Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] @@ -858,6 +938,13 @@ public void TooLargeErrorScalar() TooLargeError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void TooLargeErrorSse() + { + TooLargeError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void TooLargeErrorAvx() @@ -903,6 +990,14 @@ public void AsciiPlusContinuationAtEndErrorScalar() AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void AsciiPlusContinuationAtEndErrorSse() + { + AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + + [Trait("Category", "arm64")] [FactOnSystemRequirementAttribute(TestSystemRequirements.Arm64)] public void AsciiPlusContinuationAtEndErrorArm64() @@ -959,6 +1054,14 @@ public void SurrogateErrorTestScalar() SurrogateErrorTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void SurrogateErrorTestSse() + { + SurrogateErrorTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void SurrogateErrorTestAvx2() @@ -1032,6 +1135,14 @@ public void BruteForceTestScalar() BruteForceTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteScalar); } + + [Trait("Category", "sse")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Sse)] + public void BruteForceTestSse() + { + BruteForceTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteSse); + } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void BruteForceTestAvx2()