DXC Tests: Filter out unsupported test, add verbose option and improv…

…e output (#5537)

Improve output of DXC test runs:
- Add `-lit-verbose` option to `hctbuild` to setup lit test runs with
verbose output (`hcttest` will list test names as they run)
- Hide tests that are never supported (there was over 10K of unsupported
tests found by LIT discovery)
- Show warnings about non-essential tools only in LIT debug mode
- Skip over file copying and non-LIT part of `hcttest.cmd` if all
requested tests have already been run
- CMD tests have been move under clang's lit tests; keep the `hcttest
cmd` option for running cmd tests separately

Number of unsupported tests was reduced from 10452 to 1 if spir-v is
enabled, 39 if not. The number of tests that run did not change.

test/Analysis/Delinearization/lit.local.cfg

@@ -1 +1,2 @@
-config.suffixes = ['.ll']
+# HLSL Change - Hide these lit suites.
+#config.suffixes = ['.ll']

test/Analysis/Lint/lit.local.cfg

@@ -1 +1,2 @@
-config.suffixes = ['.ll']
+# HLSL Change - Hide these lit suites.
+#config.suffixes = ['.ll']

test/Bitcode/lit.local.cfg

@@ -1,4 +1,5 @@
-config.unsupported = True # HLSL Change - Disable lit suites.
+config.unsupported = True # HLSL Change - Disable and hide these lit suites.
+config.suffixes = []
 
 # Failing Tests (3):
 #     LLVM :: Bitcode/highLevelStructure.3.2.ll

test/MC/Markup/lit.local.cfg

@@ -1,2 +1,2 @@
-config.suffixes = ['.mc']
-
+# HLSL Change - Hide these lit suites.
+#config.suffixes = ['.mc']

test/MC/lit.local.cfg

@@ -1 +1,2 @@
-config.unsupported = True # HLSL Change - Disable lit suites.
+config.unsupported = True # HLSL Change - Disable and hide these lit suites.
+config.suffixes = []

test/Object/lit.local.cfg

@@ -1,2 +1,3 @@
-config.unsupported = True # HLSL Change - Disable lit suites.
-config.suffixes = ['.test', '.ll', '.s', '.yaml']
+config.unsupported = True # HLSL Change - Disable and hide these lit suites.
+#config.suffixes = ['.test', '.ll', '.s', '.yaml']
+config.suffixes = []

test/Other/lit.local.cfg

@@ -1,5 +1,5 @@
-config.unsupported = True # HLSL Change - Disable lit suites.
-
+config.unsupported = True # HLSL Change - Disable and hide these lit suites.
+config.suffixes = []
 # Failing Tests (8):
 #     LLVM :: Other/2007-04-24-eliminate-mostly-empty-blocks.ll
 #     LLVM :: Other/2010-05-06-Printer.ll

test/Transforms/EarlyCSE/AArch64/lit.local.cfg

@@ -1,4 +1,5 @@
-config.suffixes = ['.ll']
+# HLSL Change - Hide these lit suites.
+#config.suffixes = ['.ll']
 
 targets = set(config.root.targets_to_build.split())
 if not 'AArch64' in targets:

test/Transforms/SimplifyCFG/AArch64/lit.local.cfg

@@ -1,4 +1,5 @@
-config.suffixes = ['.ll']
+# HLSL Change - Hide these lit suites.
+#config.suffixes = ['.ll']
 
 targets = set(config.root.targets_to_build.split())
 if not 'AArch64' in targets:

test/lit.cfg

@@ -41,7 +41,10 @@ config.test_format = lit.formats.ShTest(execute_external)
 
 # suffixes: A list of file extensions to treat as test files. This is overriden
 # by individual lit.local.cfg files in the test subdirectories.
-config.suffixes = ['.ll', '.c', '.cxx', '.test', '.txt', '.s']
+# HLSL Change Start - use just a subset of LLVM tests
+config.suffixes = ['.txt', '.td', '.test']
+#config.suffixes = ['.ll', '.c', '.cxx', '.test', '.txt', '.s']
+# HLSL Change End - use just a subset of LLVM tests
 
 # excludes: A list of directories to exclude from the testsuite. The 'Inputs'
 # subdirectories contain auxiliary inputs for various tests in their parent

test/tools/lit.local.cfg

@@ -0,0 +1,2 @@
+# HLSL Change - Hide these lit suites (except llvm-lit).
+config.suffixes = []

test/tools/llvm-cov/lit.local.cfg

@@ -1,2 +1,3 @@
-config.unsupported = True # HLSL Change - Disable lit suites.
-config.suffixes = ['.test', '.m', '.cpp', '.c']
+config.unsupported = True # HLSL Change - Disable and hide these lit suites.
+config.suffixes = []
+#config.suffixes = ['.test', '.m', '.cpp', '.c']

test/tools/llvm-lit/lit.local.cfg

@@ -0,0 +1,2 @@
+# HLSL Change - Enable LIT tool tests
+config.suffixes = ['.c']

test/tools/llvm-profdata/lit.local.cfg

@@ -1,2 +1,3 @@
-config.unsupported = True # HLSL Change - Disable lit suites.
-config.suffixes.add('.proftext')
+config.unsupported = True # HLSL Change - Disable and hide these lit suites.
+config.suffixes = []
+#config.suffixes.add('.proftext')

tools/clang/test/CodeGenHLSL/lit.local.cfg

@@ -1 +1,3 @@
-config.unsupported = True # HLSL Change - Disable lit suites.
+# HLSL Change - Hide files in CodeGenHLSL directory - they are tested by existing
+# TAEF tests (CompilerTest, DxilContainerTest, LinkerTest and  ValidationTest)
+config.suffixes = []

tools/clang/test/CodeGenSPIRV/lit.local.cfg

@@ -1 +1,2 @@
-config.unsupported = True # HLSL Change - Disable lit suites.
+config.unsupported = True # HLSL Change - Disable and hide these lit suites.
+config.suffixes = []

tools/clang/test/CodeGenSPIRV_Lit/unsupported/lit.local.cfg

@@ -1 +1,2 @@
 config.unsupported = True # Keep tests which not enabled before.
+config.suffixes = []

tools/clang/test/DXILValidation/lit.local.cfg

@@ -1 +1,3 @@
-config.unsupported = True # HLSL Change - Disable lit suites.
+# HLSL Change - Hide files in DxilValidation directory - they are tested by existing
+# TAEF tests (DxilValidation)
+config.suffixes = []

tools/clang/test/Frontend/lit.local.cfg

@@ -1,2 +1,3 @@
-config.unsupported = True # HLSL Change - Disable lit suites.
-config.suffixes = ['.c', '.cpp', '.m', '.mm', '.ll']
+# HLSL Change - Hide these lit suites.
+config.suffixes = []
+#config.suffixes = ['.c', '.cpp', '.m', '.mm', '.ll']

tools/clang/test/HLSL/lit.local.cfg

@@ -1,6 +1,3 @@
-import os
-
-config.unsupported = True # HLSL Change - Disable lit suites.
-
-# Add .hlsl as a test extension.
-config.suffixes.add('.hlsl')
+# HLSL Change - Hide files in HLSL directory - they are tested by existing
+# TAEF tests (RewriterTest, VerifierTest)
+config.suffixes = []

tools/clang/test/HLSLDisabled/lit.local.cfg

@@ -1 +1,2 @@
-config.unsupported = True # HLSL Change - Disable lit suites.
+# HLSL Change - Hide files in HLSLDisabled
+config.suffixes = []

tools/clang/test/HLSLFileCheck/lit.local.cfg

@@ -1 +1,3 @@
-config.unsupported = True # HLSL Change - Disable lit suites.
+# HLSL Change - Hide files in CodeGenHLSL directory - they are tested by existing
+# TAEF tests (CompilerTest, DxilContainerTest, LinkerTest)
+config.suffixes = []

tools/clang/test/Index/lit.local.cfg

@@ -1 +1,2 @@
-config.unsupported = True # HLSL Change - Disable lit suites.
+# HLSL Change - Hide these lit suites.
+config.suffixes = []

tools/clang/test/Index/skip-parsed-bodies/lit.local.cfg

@@ -1 +1,3 @@
-config.suffixes = ['.json']
+# HLSL Change - Hide these lit suites.
+config.suffixes = []
+#config.suffixes = ['.json']

tools/clang/test/lit.cfg

@@ -44,7 +44,10 @@ else:
 config.test_format = lit.formats.ShTest(execute_external)
 
 # suffixes: A list of file extensions to treat as test files.
-config.suffixes = ['.c', '.cpp', '.m', '.mm', '.cu', '.ll', '.cl', '.s', '.S', '.modulemap', '.hlsl', '.test']
+# HLSL Change start - look for just .hlsl, .ll and .test files
+config.suffixes = ['.ll', '.hlsl', '.test']
+#config.suffixes = ['.c', '.cpp', '.m', '.mm', '.cu', '.ll', '.cl', '.s', '.S', '.modulemap', '.hlsl', '.test']
+# HLSL Change end - look for just .hlsl, .ll and .test files
 
 # excludes: A list of directories to exclude from the testsuite. The 'Inputs'
 # subdirectories contain auxiliary inputs for various tests in their parent
@@ -356,7 +359,8 @@ for pattern in [r"\bFileCheck\b",
     tool_path = lit.util.which(tool_name, tool_dirs)
     if not tool_path:
         # Warn, but still provide a substitution.
-        lit_config.note('Did not find ' + tool_name + ' in ' + tool_dirs)
+        if lit_config.debug:
+            lit_config.note('Did not find ' + tool_name + ' in ' + tool_dirs)
         tool_path = clang_tools_dir + '/' + tool_name
     config.substitutions.append((pattern, tool_pipe + tool_path))
 

utils/hct/hctbuild.cmd

@@ -45,6 +45,7 @@ set SHOW_CMAKE_LOG=0
 set WINSDK_MIN_VERSION=10.0.17763.0
 set INSTALL_DIR=
 set DEFAULT_EXEC_ADAPTER=-DTAEF_EXEC_ADAPTER=
+set LIT_ARGS=
 
 :parse_args
 if "%1"=="" (
@@ -190,10 +191,14 @@ if "%1"=="-show-cmake-log" (
   shift /1 & goto :parse_args
 )
 if "%1"=="-lit-xml-output-path" (
-  set "CMAKE_OPTS=%CMAKE_OPTS% -DLLVM_LIT_ARGS=--xunit-xml-output=%~2"
+  set "LIT_ARGS=%LIT_ARGS% --xunit-xml-output=%~2"
   shift /1
   shift /1 & goto :parse_args
 )
+if "%1"=="-lit-verbose" (
+  set "LIT_ARGS=%LIT_ARGS% -v --no-progress-bar"
+  shift /1 & goto :parse_args
+)
 if "%1"=="-default-adapter" (
   set DEFAULT_EXEC_ADAPTER=-DTAEF_EXEC_ADAPTER=%~2
   shift /1
@@ -334,6 +339,10 @@ set CMAKE_OPTS=%CMAKE_OPTS% -DCLANG_CL:BOOL=OFF
 set CMAKE_OPTS=%CMAKE_OPTS% -DCMAKE_SYSTEM_VERSION=%DXC_CMAKE_SYSTEM_VERSION%
 set CMAKE_OPTS=%CMAKE_OPTS% -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR%
 
+if "%LIT_ARGS%" NEQ "" (
+  set CMAKE_OPTS=%CMAKE_OPTS% -DLLVM_LIT_ARGS="%LIT_ARGS%"
+)
+
 rem Setup taef exec adapter.
 set CMAKE_OPTS=%CMAKE_OPTS% %DEFAULT_EXEC_ADAPTER%
 

utils/hct/hcttest.cmd

@@ -231,9 +231,9 @@ rem Win32 to x86, no changes for other platforms
 set TEST_ARCH=%BUILD_ARCH:Win32=x86%
 
 rem By default, run all clang tests and execution tests and dxilconv tests
+rem Cmd tests are already included in the clang test suite.
 if "%TEST_ALL%"=="1" (
   set TEST_CLANG=1
-  set TEST_CMD=1
   set TEST_EXEC=1
   set TEST_EXTRAS=1
   set TEST_DXILCONV=1
@@ -299,6 +299,8 @@ if "%TEST_MANUAL_FILE_CHECK%"=="1" (
   )
 )
 
+echo Running HLSL tests for %BUILD_ARCH%...
+
 if "%TEST_USE_LIT%"=="1" (
   rem LIT does not separate spirv tests from other clang hlsl tests.
   if "%TEST_SPIRV%"=="1" (
@@ -308,9 +310,7 @@ if "%TEST_USE_LIT%"=="1" (
     rem check all except exec.
     cmake --build %HLSL_BLD_DIR% --config %BUILD_CONFIG% --target check-all
     set RES_CLANG=!ERRORLEVEL!
-    set RES_DXILCONV=%RES_CLANG%
-    set RES_EXEC=%RES_CLANG%
-    set RES_CMD=%RES_CLANG%
+    set RES_DXILCONV=!RES_CLANG!
   ) else (
     if "%TEST_DXILCONV%"=="1" (
       cmake --build %HLSL_BLD_DIR% --config %BUILD_CONFIG% --target check-dxilconv
@@ -326,7 +326,7 @@ if "%TEST_USE_LIT%"=="1" (
     )
     if "!TEST_EXEC!"=="1" (
       if defined EXEC_ADAPTER (
-        py %HLSL_SRC_DIR%/utils/lit/lit.py -sv --no-progress-bar --param build_mode=%BUILD_CONFIG% --param clang_site_config=%HLSL_BLD_DIR%/tools/clang/test/lit.site.cfg --param clang_taef_exec_site_config=%HLSL_BLD_DIR%/tools/clang/test/taef_exec/lit.site.cfg %EXEC_ADAPTER% %HLSL_SRC_DIR%/tools/clang/test/taef_exec
+        py %HLSL_SRC_DIR%/utils/lit/lit.py -v --no-progress-bar --param build_mode=%BUILD_CONFIG% --param clang_site_config=%HLSL_BLD_DIR%/tools/clang/test/lit.site.cfg --param clang_taef_exec_site_config=%HLSL_BLD_DIR%/tools/clang/test/taef_exec/lit.site.cfg %EXEC_ADAPTER% %HLSL_SRC_DIR%/tools/clang/test/taef_exec
       ) else (
         cmake --build %HLSL_BLD_DIR% --config %BUILD_CONFIG% --target check-clang-taef-exec
 	  )
@@ -338,6 +338,11 @@ if "%TEST_USE_LIT%"=="1" (
   set TEST_SPIRV=0
   set TEST_EXEC=0
   set TEST_CMD=0
+
+  rem No other tests to run - skip copying and move on to report the results
+  if not exist "%HCT_EXTRAS%\hcttest-extras.cmd" (
+    goto :report_results
+  )
 )
 
 if not exist %TEST_DIR% (mkdir %TEST_DIR%)
@@ -386,8 +391,6 @@ if "%TEST_SPIRV%"=="1" (
 )
 rem End SPIRV change
 
-echo Running HLSL tests for %BUILD_ARCH%...
-
 if "%BUILD_ARCH_DIR%"=="ARM64" (
     rem ARM64 TAEF has an issue when running ARM64X tests with /parallel flag
     set PARALLEL_OPTION=
@@ -473,6 +476,7 @@ if "%TEST_MANUAL_FILE_CHECK%"=="1" (
   set RES_EXEC=!ERRORLEVEL!
 )
 
+:report_results
 echo.
 echo ==================================
 echo Unit test results: