Lucene.Net.TestFramework: Improved error handling and test reporting (#…

…1092)

* Lucene.Net.Util.LuceneTestCase.SetUpFixture::OneTimeSetUpWrapper(): Removed debug code.

* Lucene.Net.Util.LuceneTestCase (OneTimeSetUp() + OneTimeTearDown()): Fixed error reporting, since NUnit swallows exceptions in these events. This will write them to stderr by default and allows enabling failing the tests if an exception occurs in LuceneTestCase.OneTimeSetUp() which will make them appear in the Test Output window in Visual Studio. The failures can be enabled by setting the tests:failontestfixtureonetimesetuperror system property to true.

* azure-pipelines.yml: Added FailOnTestFixtureOneTimeSetUpError environment variable (defaulting to 'true') to set the system property of the same name.

* Lucene.Net.Search.RandomSimilarityProvider: Use J2N.Collections.Generic.Dictionary<TKey, TValue>, which will format itself. Marked fields private instead of internal.

* PERFORMANCE: Moved logging random test settings and system properties from SetUp() to TearDown() and only include them in the test message upon failure.

* Lucene.Net.Support.ExceptionHandling.TestExceptionExtensions: Removed messages from calls to Assert.Pass(), since this writes to the output and bloats the size of the test logs with no benefit.

* publish-test-results.yml + run-tests-on-os.yml: Changed build to fail when maximumAllowedFailures is 0 and there are errors in the stderr file.

* publish-test-results.yml + run-tests-on-os.yml: Changed build to fail when maximumAllowedFailures is 0 and there are errors in the stdout in the TestResults.trx file.

* publish-test-results.yml: Check for [ERROR] in RunInfo rather than StdOut

.build/azure-templates/publish-test-results.yml

@@ -87,6 +87,13 @@ steps:
                             Write-Host "##vso[task.setvariable variable=CrashedRuns;]$crashedRuns"
                             $crashed = $true
                         }
+                        if ($innerXml -and ($innerXml.Contains('[ERROR]'))) {
+                            Write-Host "##vso[task.setvariable variable=StdOutFailure;]true"
+                            # Report all of the test projects that had stdout failures
+                            $stdOutFailureRuns = "$env:STDOUTFAILURERUNS,$testProjectName".TrimStart(',')
+                            Write-Host "##vso[task.setvariable variable=StdOutFailureRuns;]$stdOutFailureRuns"
+                            $crashed = $true
+                        }
                     }
                 }
                 if ($reader.NodeType -eq [System.Xml.XmlNodeType]::EndElement -and $reader.Name -eq 'RunInfos') {
@@ -102,6 +109,34 @@ steps:
     Write-Host "##vso[task.setvariable variable=TestResultsFileExists;]$testResultsFileExists"
   displayName: 'Parse Test Results File'
 
+- pwsh: |
+    $testProjectName = "${{ parameters.testProjectName }}"
+    $testStdErrFileName = "$(Build.ArtifactStagingDirectory)/${{ parameters.testResultsArtifactName }}/${{ parameters.osName }}/${{ parameters.framework }}/${{ parameters.vsTestPlatform }}/$testProjectName/dotnet-test-error.log"
+    $testStdErrFileExists = Test-Path $testStdErrFileName
+    if ($testStdErrFileExists) {
+        $fileLength = (Get-Item $testStdErrFileName).Length
+        if ($fileLength -gt 0) {
+            $stream = [System.IO.StreamReader]::new($testStdErrFileName)
+            try {
+                while (-not $stream.EndOfStream) {
+                    $line = $stream.ReadLine()
+                    if ($line -match "Test Run Failed" -or $line -match "\[ERROR\]") {
+                        Write-Host "##vso[task.setvariable variable=StdErrFailure;]true"
+                        # Report all of the test projects that had stderr failures
+                        $stdErrFailureRuns = "$env:STDERRFAILURERUNS,$testProjectName".TrimStart(',')
+                        Write-Host "##vso[task.setvariable variable=StdErrFailureRuns;]$stdErrFailureRuns"
+                        break # No need to continue reading after detecting a failure
+                    }
+                }
+            } finally {
+                $stream.Dispose()
+            }
+        }
+    } else {
+        Write-Host "WARNING: File not found: $testStdErrFileName"
+    }
+  displayName: 'Parse StdErr File'
+
 - task: PublishTestResults@2
   displayName: 'Publish Test Results ${{ parameters.testProjectName }},${{ parameters.framework }},${{ parameters.vsTestPlatform }}'
   inputs:

.build/azure-templates/run-tests-on-os.yml

@@ -310,6 +310,16 @@ steps:
         Write-Host "##vso[task.logissue type=error;]Test run failed due to too many failed tests. Maximum failures allowed: $maximumAllowedFailures, total failures: $($env:TOTALFAILURES)."
         $failed = $true
     }
+    if ([int]$maximumAllowedFailures -eq 0 -and $env:STDERRFAILURE -eq 'true') {
+        $runsExpanded = "$($env:STDERRFAILURERUNS)" -replace ',',"`r`n"
+        Write-Host "##vso[task.logissue type=error;]StdErr file(s) indicate test failures. Review the testresults artifacts for details. (click here to view the projects that failed)`r`n$runsExpanded"
+        $failed = $true
+    }
+    if ([int]$maximumAllowedFailures -eq 0 -and $env:STDOUTFAILURE -eq 'true') {
+        $runsExpanded = "$($env:STDOUTFAILURERUNS)" -replace ',',"`r`n"
+        Write-Host "##vso[task.logissue type=error;]StdOut file(s) indicate test failures. Review the testresults artifacts for details. (click here to view the projects that failed)`r`n$runsExpanded"
+        $failed = $true
+    }
     if ($failed) {
         Write-Host "##vso[task.complete result=Failed;]"
     }

azure-pipelines.yml

@@ -49,6 +49,7 @@ name: 'vNext$(rev:.r)' # Format for build number (will be overridden)
 # Verbose: 'false' (Optional - set to true for verbose logging output)
 # Multiplier: '1' (Optional - the number of iterations to multiply applicable tests by)
 # DisplayFullName: 'true' (Optional - set to 'false' to display only the test name instead of the full name with class and method)
+# FailOnTestFixtureOneTimeSetUpError: 'true' (Optional - set to 'false' to allow tests to pass if the test fixture (class) has a OneTimeSetUp failure.)
 
 # RunX86Tests: 'false' (Optional - set to 'true' to enable x86 tests)
 
@@ -162,6 +163,7 @@ stages:
         $directory = if ($env:Directory -eq $null) { 'random' } else { $env:Directory }
         $verbose = if ($env:Verbose -eq 'true') { 'true' } else { 'false' }
         $multiplier = if ($env:Multiplier -eq $null) { '1' } else { $env:Multiplier }
+        $failOnTestFixtureOneTimeSetUpError = if ($env.FailOnTestFixtureOneTimeSetUpError -eq 'false') { 'false' } else { 'true' }
         $fileText = "{`n`t" +
             """assert"": ""$assert"",`n`t" +
             """tests"": {`n`t`t" +
@@ -174,7 +176,8 @@ stages:
                 """postingsformat"": ""$postingsFormat"",`n`t`t" +
                 """directory"": ""$directory"",`n`t`t" +
                 """verbose"": ""$verbose"",`n`t`t" +
-                """multiplier"": ""$multiplier""`n`t" +
+                """multiplier"": ""$multiplier"",`n`t`t" +
+                """failontestfixtureonetimesetuperror"": ""$failOnTestFixtureOneTimeSetUpError""`n`t" +
             "}`n" +
         "}"
         Out-File -filePath "$(Build.ArtifactStagingDirectory)/$(TestSettingsFileName)" -encoding UTF8 -inputObject $fileText

src/Lucene.Net.TestFramework/Search/RandomSimilarityProvider.cs

@@ -36,12 +36,12 @@ namespace Lucene.Net.Search
     /// </summary>
     public class RandomSimilarityProvider : PerFieldSimilarityWrapper
     {
-        internal readonly DefaultSimilarity defaultSim = new DefaultSimilarity();
-        internal readonly IList<Similarity> knownSims;
-        internal IDictionary<string, Similarity> previousMappings = new Dictionary<string, Similarity>();
-        internal readonly int perFieldSeed;
-        internal readonly int coordType; // 0 = no coord, 1 = coord, 2 = crazy coord
-        internal readonly bool shouldQueryNorm;
+        private readonly DefaultSimilarity defaultSim = new DefaultSimilarity();
+        private readonly IList<Similarity> knownSims;
+        private readonly JCG.Dictionary<string, Similarity> previousMappings = new JCG.Dictionary<string, Similarity>();
+        private readonly int perFieldSeed;
+        private readonly int coordType; // 0 = no coord, 1 = coord, 2 = crazy coord
+        private readonly bool shouldQueryNorm;
 
         public RandomSimilarityProvider(Random random)
         {
@@ -162,7 +162,7 @@ public override string ToString()
                 else
                     sb.Append("crazy");
                 sb.Append("): ");
-                sb.AppendFormat(J2N.Text.StringFormatter.InvariantCulture, "{0}", previousMappings);
+                sb.Append(previousMappings);
                 return sb.ToString();
             }
             finally
@@ -171,4 +171,4 @@ public override string ToString()
             }
         }
     }
-}
+}

src/Lucene.Net.TestFramework/Support/Util/LuceneTestCase.SetUpFixture.cs

@@ -70,7 +70,6 @@ private static LuceneTestFrameworkInitializer Initialize(Test setUpFixture, Test
             [OneTimeSetUp]
             public void OneTimeSetUpWrapper()
             {
-                //var fixture = TestExecutionContext.CurrentContext.CurrentTest;
                 if (stackCount.GetAndIncrement() == 0)
                 {
                     // Set up for assembly

src/Lucene.Net.TestFramework/Support/Util/TestExtensions.cs

@@ -0,0 +1,90 @@
+using NUnit.Framework.Internal;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+#nullable enable
+
+namespace Lucene.Net.Util
+{
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one or more
+     * contributor license agreements.  See the NOTICE file distributed with
+     * this work for additional information regarding copyright ownership.
+     * The ASF licenses this file to You under the Apache License, Version 2.0
+     * (the "License"); you may not use this file except in compliance with
+     * the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+
+    /// <summary>
+    /// Extensions to <see cref="Test"/>.
+    /// </summary>
+    internal static class TestExtensions
+    {
+        /// <summary>
+        /// Mark the test and all descendents as Invalid (not runnable) specifying a reason and an exception.
+        /// </summary>
+        /// <param name="test">This <see cref="Test"/>.</param>
+        /// <param name="reason">The reason the test is not runnable</param>
+        /// <exception cref="ArgumentNullException"><paramref name="test"/> or <paramref name="reason"/> is <c>null</c>.</exception>
+        public static void MakeAllInvalid(this Test test, string reason)
+            => MakeAllInvalidInternal(test, null, reason);
+
+        /// <summary>
+        /// Mark the test and all descendents as Invalid (not runnable) specifying a reason and an exception.
+        /// </summary>
+        /// <param name="test">This <see cref="Test"/>.</param>
+        /// <param name="exception">The exception that was the cause.</param>
+        /// <param name="reason">The reason the test is not runnable</param>
+        /// <exception cref="ArgumentNullException"><paramref name="test"/>, <paramref name="exception"/> or <paramref name="reason"/> is <c>null</c>.</exception>
+        public static void MakeAllInvalid(this Test test, Exception exception, string reason)
+        {
+            if (exception is null)
+                throw new ArgumentNullException(nameof(exception));
+            MakeAllInvalidInternal(test, exception, reason);
+        }
+
+        private static void MakeAllInvalidInternal(this Test test, Exception? exception, string reason)
+        {
+            if (test is null)
+                throw new ArgumentNullException(nameof(test));
+            if (reason is null)
+                throw new ArgumentNullException(nameof(reason));
+
+            if (exception is null)
+                test.MakeInvalid(reason);
+            else
+                test.MakeInvalid(exception, reason);
+
+            if (test.HasChildren)
+            {
+                var stack = new Stack<Test>(test.Tests.OfType<Test>());
+
+                while (stack.Count > 0)
+                {
+                    var currentTest = stack.Pop();
+                    if (exception is null)
+                        currentTest.MakeInvalid(reason);
+                    else
+                        currentTest.MakeInvalid(exception, reason);
+
+                    // Add children to the stack if they exist
+                    if (currentTest.HasChildren)
+                    {
+                        foreach (var child in currentTest.Tests.OfType<Test>())
+                        {
+                            stack.Push(child);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}