Skip to content

Commit

Permalink
fix: complete correction
Browse files Browse the repository at this point in the history
  • Loading branch information
@lemire
lemire committed May 23, 2024
1 parent 900472e commit 58d67cc
Showing 1 changed file with 34 additions and 46 deletions.
80 changes: 34 additions & 46 deletions test/UTF8ValidationTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ public unsafe class Utf8SIMDValidationTests
{


private const int NumTrials = 1000;
private const int NumTrials = 100;
private static readonly RandomUtf8 generator = new RandomUtf8(1234, 1, 1, 1, 1);
private static readonly Random rand = new Random();

// int[] outputLengths = { 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1088, 1152, 1216, 1280, 1344, 1408, 1472, 1536, 1600, 1664, 1728, 1792, 1856, 1920, 1984, 2048, 2112, 2176, 2240, 2304, 2368, 2432, 2496, 2560, 2624, 2688, 2752, 2816, 2880, 2944, 3008, 3072, 3136, 3200, 3264, 3328, 3392, 3456, 3520, 3584, 3648, 3712, 3776, 3840, 3904, 3968, 4032, 4096, 4160, 4224, 4288, 4352, 4416, 4480, 4544, 4608, 4672, 4736, 4800, 4864, 4928, 4992, 5056, 5120, 5184, 5248, 5312, 5376, 5440, 5504, 5568, 5632, 5696, 5760, 5824, 5888, 5952, 6016, 6080, 6144, 6208, 6272, 6336, 6400, 6464, 6528, 6592, 6656, 6720, 6784, 6848, 6912, 6976, 7040, 7104, 7168, 7232, 7296, 7360, 7424, 7488, 7552, 7616, 7680, 7744, 7808, 7872, 7936, 8000, 8064, 8128, 8192, 8256, 8320, 8384, 8448, 8512, 8576, 8640, 8704, 8768, 8832, 8896, 8960, 9024, 9088, 9152, 9216, 9280, 9344, 9408, 9472, 9536, 9600, 9664, 9728, 9792, 9856, 9920, 9984, 10000 };
static int[] outputLengths = { 128, 256,345, 512,968, 1024, 1000 };
static int[] outputLengths = { 128, 345, 1000 };

[Flags]
public enum TestSystemRequirements
Expand Down Expand Up @@ -76,7 +76,7 @@ public TestIfCondition(Func<bool> condition, string skipReason)



public void simpleGoodSequences(Utf8ValidationDelegate utf8ValidationDelegate)
private void simpleGoodSequences(Utf8ValidationDelegate utf8ValidationDelegate)
{
string[] goodSequences = {
"a",
Expand Down Expand Up @@ -144,7 +144,7 @@ public void simpleGoodSequencesAVX()
}


public void BadSequences(Utf8ValidationDelegate utf8ValidationDelegate)
private void BadSequences(Utf8ValidationDelegate utf8ValidationDelegate)
{
string[] badSequences = {
"\xC3\x28",
Expand Down Expand Up @@ -231,13 +231,13 @@ public void BadSequencesAVX()
}

// this was in the C++ code
public void Node48995Test(Utf8ValidationDelegate utf8ValidationDelegate)
private void Node48995Test(Utf8ValidationDelegate utf8ValidationDelegate)
{
byte[] bad = new byte[] { 0x80 };
Assert.False(ValidateUtf8(bad,utf8ValidationDelegate));
}

public void NoError(Utf8ValidationDelegate utf8ValidationDelegate)
private void NoError(Utf8ValidationDelegate utf8ValidationDelegate)
{
foreach (int outputLength in outputLengths)
{
Expand Down Expand Up @@ -299,7 +299,7 @@ public void NoErrorAVX()
NoError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}

public void NoErrorSpecificByteCount(Utf8ValidationDelegate utf8ValidationDelegate)
private void NoErrorSpecificByteCount(Utf8ValidationDelegate utf8ValidationDelegate)
{
RunTestForByteLength(1,utf8ValidationDelegate);
RunTestForByteLength(2,utf8ValidationDelegate);
Expand Down Expand Up @@ -369,7 +369,7 @@ public void NoErrorSpecificByteCountAVX()
NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}

public void NoErrorIncompleteThenASCII(Utf8ValidationDelegate utf8ValidationDelegate)
private void NoErrorIncompleteThenASCII(Utf8ValidationDelegate utf8ValidationDelegate)
{
foreach (int outputLength in outputLengths){
for (int trial = 0; trial < NumTrials; trial++)
Expand Down Expand Up @@ -438,20 +438,18 @@ public void NoErrorIncompleteThenASCIIScalar()
// NoErrorIncompleteThenASCII(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
// }

[Fact]
[Trait("Category", "avx")]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void NoErrorIncompleteThenASCIIAVX()
{
NoErrorIncompleteThenASCII(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}



public void NoErrorIncompleteAt256Vector(Utf8ValidationDelegate utf8ValidationDelegate)
private void NoErrorIncompleteAt256Vector(Utf8ValidationDelegate utf8ValidationDelegate)
{
// foreach (int outputLength in outputLengths)
foreach (int outputLength in outputLengths)
{
int outputLength = 256;
for (int trial = 0; trial < NumTrials; trial++)
{

Expand Down Expand Up @@ -515,14 +513,13 @@ public void NoErrorIncompleteAt256VectorScalar()
// NoErrorIncompleteAt256Vector(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
// }

[Fact]
[Trait("Category", "avx")]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void NoErrorIncompleteAt256VectorAVX()
{
NoErrorIncompleteAt256Vector(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}

public void BadHeaderBits(Utf8ValidationDelegate utf8ValidationDelegate)
private void BadHeaderBits(Utf8ValidationDelegate utf8ValidationDelegate)
{
foreach (int outputLength in outputLengths)
{
Expand Down Expand Up @@ -590,14 +587,13 @@ public void BadHeaderBitsScalar()
// NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
// }

[Fact]
[Trait("Category", "avx")]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void BadHeaderBitsAVX()
{
BadHeaderBits(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}

public void TooShortError(Utf8ValidationDelegate utf8ValidationDelegate)
private void TooShortError(Utf8ValidationDelegate utf8ValidationDelegate)
{
foreach (int outputLength in outputLengths)
{
Expand Down Expand Up @@ -663,14 +659,13 @@ public void TooShortErrorScalar()
// TooShortError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
// }

[Fact]
[Trait("Category", "avx")]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void TooShortErrorAVX()
{
TooShortError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}

public void TooLongError(Utf8ValidationDelegate utf8ValidationDelegate)
private void TooLongError(Utf8ValidationDelegate utf8ValidationDelegate)
{

foreach (int outputLength in outputLengths)
Expand Down Expand Up @@ -736,14 +731,13 @@ public void TooLongErrorScalar()
// TooLongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
// }

[Fact]
[Trait("Category", "avx")]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void TooLongErrorAVX()
{
TooLongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}

public void OverlongError(Utf8ValidationDelegate utf8ValidationDelegate)
private void OverlongError(Utf8ValidationDelegate utf8ValidationDelegate)
{
for (int trial = 0; trial < NumTrials; trial++)
{
Expand Down Expand Up @@ -818,15 +812,14 @@ public void OverlongErrorScalar()
// OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
// }

[Fact]
[Trait("Category", "avx")]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void OverlongErrorAVX()
{
OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}


public void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)
private void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate)
{
for (int trial = 0; trial < NumTrials; trial++)
{
Expand Down Expand Up @@ -912,15 +905,15 @@ public void TooShortErrorAtEndAVX()
TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}

[Fact]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void TooShortErrorAtEndAvx2()
{
TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}


//corresponds to condition 5.4.1 in the paper
public void Invalid0xf50xff(Utf8ValidationDelegate utf8ValidationDelegate)
private void Invalid0xf50xff(Utf8ValidationDelegate utf8ValidationDelegate)
{

var invalidBytes = Enumerable.Range(0xF5, 0x100 - 0xF5).Select(i => (byte)i).ToArray(); // 0xF5 to 0xFF
Expand Down Expand Up @@ -972,14 +965,13 @@ public void Invalid0xf50xffScalar()
// Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
// }

[Fact]
[Trait("Category", "avx")]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void Invalid0xf50xffAVX()
{
Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}

[Fact]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void Invalid0xf50xffAvx2()
{
Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
Expand Down Expand Up @@ -1049,7 +1041,7 @@ static void PrintHexAndBinary(byte[] bytes, int highlightIndex = -1)
}


public void TooLargeError(Utf8ValidationDelegate utf8ValidationDelegate)
private void TooLargeError(Utf8ValidationDelegate utf8ValidationDelegate)
{
foreach (int outputLength in outputLengths)
{
Expand Down Expand Up @@ -1106,15 +1098,14 @@ public void TooLargeErrorScalar()
// TooLargeError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
// }

[Fact]
[Trait("Category", "avx")]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void TooLargeErrorAvx()
{
TooLargeError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}


public void AsciiPlusContinuationAtEndError(Utf8ValidationDelegate utf8ValidationDelegate)
private void AsciiPlusContinuationAtEndError(Utf8ValidationDelegate utf8ValidationDelegate)
{
foreach (int outputLength in outputLengths)
{
Expand Down Expand Up @@ -1169,20 +1160,19 @@ public void AsciiPlusContinuationAtEndErrorScalar()
// AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
// }

[Fact]
[Trait("Category", "avx")]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void AsciiPlusContinuationAtEndErrorAVX()
{
AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}

[Fact]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void AsciiPlusContinuationAtEndErrorAvx2()
{
AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}

public void SurrogateErrorTest(Utf8ValidationDelegate utf8ValidationDelegate)
private void SurrogateErrorTest(Utf8ValidationDelegate utf8ValidationDelegate)
{
foreach (int outputLength in outputLengths)
{
Expand Down Expand Up @@ -1249,15 +1239,14 @@ public void SurrogateErrorTestScalar()
// SurrogateErrorTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
// }

[Fact]
[Trait("Category", "avx")]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void SurrogateErrorTestAVX()
{
SurrogateErrorTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
}


public void BruteForceTest(Utf8ValidationDelegate utf8ValidationDelegate)
private void BruteForceTest(Utf8ValidationDelegate utf8ValidationDelegate)
{
foreach (int outputLength in outputLengths)
{
Expand Down Expand Up @@ -1336,8 +1325,7 @@ public void BruteForceTestScalar()
// BruteForceTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64);
// }

[Fact]
[Trait("Category", "avx")]
[FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)]
public void BruteForceTestAVX()
{
BruteForceTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2);
Expand Down

0 comments on commit 58d67cc

Please sign in to comment.