Port typecheck_one_test.sh to batch and fix check_differences.bat (…

…#349)

This ports over the changes made in #335 to Windows, and it should also
finally fix false positives regarding `check_differences.bat`.

check_differences/check_differences.bat

@@ -1,5 +1,4 @@
 @echo off
-
 rem On Windows, instead of using the diff command for SpeciminTestExecutor, we can use this
 rem command instead since diff is not included by default. This script outputs 0 if no differences
 rem are found, 1 if a difference in file structure or file content (all whitespace removed) is found
@@ -9,18 +8,18 @@ goto :MAIN
 rem Function to remove all whitespace from a file
 :RemoveWhitespace
     setlocal enabledelayedexpansion
-        set "content="
-        rem Simply calling (%~1) is not enough; in edge cases, empty lines containing delimiters (i.e. ;)
-        rem may be removed. Therefore, we need findstr to add line numbers so no line is empty.
-        for /F "tokens=* delims=" %%i in ('findstr /n "^" %~1') do (
-            set line=%%i
-            set line=!line:*:=!
-            if "!line!" NEQ "" set line=!line: =!
-            set content=!content!!line!
-        )
-        echo !content! >> %~2
+    set "content="
+    rem Simply calling (%~1) is not enough; in edge cases, empty lines containing delimiters (i.e. ;)
+    rem may be removed. Therefore, we need findstr to add line numbers so no line is empty.
+    for /F "tokens=* delims=" %%i in ('findstr /n "^" "%~1"') do (
+        set "line=%%i"
+        set "line=!line:*:=!"
+        if "!line!" NEQ "" set "line=!line: =!"
+        set "content=!content!!line!"
+    )
+    echo !content! > "%~2"
     endlocal
-exit /b 0
+    exit /b 0
 
 :MAIN
 
@@ -35,85 +34,63 @@ if "%2"=="" (
     exit /b 1
 )
 
-set CURRENT_DIRECTORY=%CD%
-
-cd %1
-set DIRECTORY_1=%CD%
-
-cd %CURRENT_DIRECTORY%
+set "CURRENT_DIRECTORY=%cd%"
 
-cd %2
-set DIRECTORY_2=%CD%
+rem use /D to handle drive differences (C:\ vs. D:\)
+cd /D "%1" || exit /b 1
+set "DIRECTORY_1=%cd%"
 
-cd %CURRENT_DIRECTORY%
+cd /D "%CURRENT_DIRECTORY%"
 
-set DIRECTORY_1_COPY=%DIRECTORY_1%
-set DIRECTORY_2_COPY=%DIRECTORY_2%
-
-set DIR_1_LENGTH=0
-set DIR_2_LENGTH=0
-
-rem Get the length of directory 1 so we can later convert absolute to relative paths
-:dir_1_length_loop
-if defined DIRECTORY_1_COPY (
-    set "DIRECTORY_1_COPY=!DIRECTORY_1_COPY:~1!"
-    set /A "DIR_1_LENGTH+=1"
-    goto :dir_1_length_loop
-)
-
-rem Get the length of directory 2 so we can later convert absolute to relative paths
-:dir_2_length_loop
-if defined DIRECTORY_2_COPY (
-    set "DIRECTORY_2_COPY=!DIRECTORY_2_COPY:~1!"
-    set /A "DIR_2_LENGTH+=1"
-    goto :dir_2_length_loop
-)
+cd /D "%2" || exit /b 1
+set "DIRECTORY_2=%cd%"
 
-rem Account for trailing \ (absent when using CD)
-set /A DIR_1_LENGTH+=1
-set /A DIR_2_LENGTH+=1
+echo %DIRECTORY_1%
+echo %DIRECTORY_2%
 
+cd /D "%DIRECTORY_1%"
 set DIR_1_STRUCTURE=
-for /F "delims=" %%i in ('dir "%1" /A-D /S /B') do (
+for /r %%i in (*) do (
     rem Convert absolute to relative path
-    set ABSOLUTE_PATH=%%i
-    set RELATIVE_PATH=!ABSOLUTE_PATH:~%DIR_1_LENGTH%!
+    set "ABSOLUTE_PATH=%%i"
+    set "RELATIVE_PATH=!ABSOLUTE_PATH:%DIRECTORY_1%\=!"
     rem Add each file path to the DIR_1_STRUCTURE list of files
-    set DIR_1_STRUCTURE=!DIR_1_STRUCTURE!;!RELATIVE_PATH!
+    set "DIR_1_STRUCTURE=!DIR_1_STRUCTURE!;!RELATIVE_PATH!"
 )
 
+cd /D "%DIRECTORY_2%"
 set DIR_2_STRUCTURE=
-for /F "delims=" %%i in ('dir "%2" /A-D /S /B') do (
+for /r %%i in (*) do (
     rem Convert absolute to relative path
-    set ABSOLUTE_PATH=%%i
-    set RELATIVE_PATH=!ABSOLUTE_PATH:~%DIR_2_LENGTH%!
+    set "ABSOLUTE_PATH=%%i"
+    set "RELATIVE_PATH=!ABSOLUTE_PATH:%DIRECTORY_2%\=!"
     rem Add each file path to the DIR_2_STRUCTURE list of files
-    set DIR_2_STRUCTURE=!DIR_2_STRUCTURE!;!RELATIVE_PATH!
+    set "DIR_2_STRUCTURE=!DIR_2_STRUCTURE!;!RELATIVE_PATH!"
 )
 
 if "!DIR_1_STRUCTURE!" NEQ "!DIR_2_STRUCTURE!" (
-    echo Folder structures are different
+    echo %DIRECTORY_1% and %DIRECTORY_2% have different directory structures
     endlocal
     exit /b 1
 )
 
 rem At this point, we have guaranteed that the files are the same, so we only
 rem need to iterate through one of the lists.
-for %%f in (%DIR_1_STRUCTURE%) do (
-    set FILE_1=!DIRECTORY_1!\%%f
-    set FILE_2=!DIRECTORY_2!\%%f
-    call :RemoveWhitespace !FILE_1! !FILE_1!.tmp
-    call :RemoveWhitespace !FILE_2! !FILE_2!.tmp
+for %%f in (!DIR_1_STRUCTURE!) do (
+    set "FILE_1=!DIRECTORY_1!\%%f"
+    set "FILE_2=!DIRECTORY_2!\%%f"
+    call :RemoveWhitespace "!FILE_1!" "!FILE_1!.tmp"
+    call :RemoveWhitespace "!FILE_2!" "!FILE_2!.tmp"
 
     set fail=0
     rem set fail=1 will only be called if fc exits with exit code 1 (differences)
-    fc !FILE_1!.tmp !FILE_2!.tmp > nul || set fail=1
+    fc "!FILE_1!.tmp" "!FILE_2!.tmp" > nul || set fail=1
 
-    del !FILE_1!.tmp
-    del !FILE_2!.tmp
+    del "!FILE_1!.tmp"
+    del "!FILE_2!.tmp"
     if !fail!==1 (
-        echo !FILE_1! and !FILE_2! are different
-        goto differences_found
+        echo "!FILE_1!" and "!FILE_2!" are different
+        goto :differences_found
     )
 )
 

check_differences/test/test_runner.bat

@@ -34,7 +34,7 @@ endlocal
             if "%%t" NEQ "base" (
                 @echo off
                 rem Redirect to nul because we don't want any output messages
-                call "../../check_differences.bat" "base" "%%t" > nul
+                call "../../check_differences.bat" base %%t > nul
                 set exitcode=!errorlevel!
                 rem "%%t" represents the folder name (i.e. base, 0, 1) since we have changed the directory
                 rem to the test case directory

src/test/java/org/checkerframework/specimin/SpeciminTestExecutor.java

@@ -66,11 +66,12 @@ public static void runTest(
 
     // Construct the list of arguments.
     List<String> speciminArgs = new ArrayList<>();
+
     speciminArgs.add("--outputDirectory");
     speciminArgs.add(outputDir.toAbsolutePath().toString());
     speciminArgs.add("--root");
     speciminArgs.add(
-        Path.of("src/test/resources/" + testName + "/input/").toAbsolutePath().toString() + "/");
+      Path.of("src/test/resources/" + testName + "/input/").toAbsolutePath().toString() + "/");
     for (String targetFile : targetFiles) {
       speciminArgs.add("--targetFile");
       speciminArgs.add(targetFile);
@@ -94,9 +95,9 @@ public static void runTest(
     // Run specimin on target
     SpeciminRunner.main(speciminArgs.toArray(new String[0]));
 
+    boolean isWindows = Ascii.toLowerCase(System.getProperty("os.name")).startsWith("windows");
     // Diff the files to ensure that specimin's output is what we expect
     ProcessBuilder builder = new ProcessBuilder();
-    boolean isWindows = Ascii.toLowerCase(System.getProperty("os.name")).startsWith("windows");
     if (isWindows) {
       builder.command(
           "check_differences/check_differences.bat",

typecheck_one_test.bat

@@ -0,0 +1,41 @@
+@echo off
+
+rem This script runs javac on a single expected test output. If the test output
+rem is compilable, it exits with code 0. If not, it exits with code 1. In all other
+rem cases (e.g., if a cd command fails), it exits with code 2. It includes
+rem logic for skipping test cases that don't need to compile, etc. It should be
+rem run in the directory src/test/resources.
+
+setlocal enabledelayedexpansion
+
+set testcase=%1
+
+if "%testcase%"=="shared" exit /b 0
+rem https://bugs.openjdk.org/browse/JDK-8319461 wasn't actually fixed (this test is based on that bug)
+if "%testcase%"=="superinterfaceextends" exit /b 0
+rem incomplete handling of method references: https://github.com/njit-jerse/specimin/issues/291
+rem this test exists to check that no crash occurs, not that Specimin produces the correct output
+if "%testcase%"=="methodref2" exit /b 0
+rem this test will not compile right now; this is a TODO in UnsolvedSymbolVisitor#lookupTypeArgumentFQN
+if "%testcase%"=="methodreturnfullyqualifiedgeneric" exit /b 0
+
+cd "%testcase%\expected\" || exit /b 2
+
+set JAVA_FILES=
+for /r %%F in (*.java) do (
+  set "JAVA_FILES=!JAVA_FILES! %%F"
+)
+
+javac -classpath "../../shared/checker-qual-3.42.0.jar" !JAVA_FILES!
+if errorlevel 1 (
+  echo Running javac on %testcase% resulted in one or more errors, which are printed above.
+  set returnval=1
+)
+
+rem clean up
+for /r %%F in (*.class) do (
+  del "%%F"
+)
+endlocal
+
+exit /b 0

typecheck_test_outputs.bat

@@ -10,46 +10,24 @@ rem should produce independently-compilable programs.
 setlocal enabledelayedexpansion
 set returnval=0
 
+set "current_directory=%cd%"
+
 cd src\test\resources || exit /b 1
 
 for /d %%t in (*) do (
-  set continue=0
-  if "%%t"=="shared" set continue=1
-  rem https://bugs.openjdk.org/browse/JDK-8319461 wasn't actually fixed (this test is based on that bug)
-  if "%%t"=="superinterfaceextends" set continue=1
-  rem incomplete handling of method references: https://github.com/njit-jerse/specimin/issues/291
-  rem this test exists to check that no crash occurs, not that Specimin produces the correct output
-  if "%%t"=="methodref2" set continue=1
-  rem this test will not compile right now; this is a TODO in UnsolvedSymbolVisitor#lookupTypeArgumentFQN
-  if "%%t"=="methodreturnfullyqualifiedgeneric" set continue=1
-  if !continue!==0 (
-    cd "%%t/expected/" || exit 1
-    rem javac relies on word splitting
-    rem shellcheck disable=SC2046
-    set JAVA_FILES=
-    for /r %%F in (*.java) do (
-      set "JAVA_FILES=!JAVA_FILES! %%F"
-    )
-    javac -classpath "../../shared/checker-qual-3.42.0.jar" !JAVA_FILES!
-    if errorlevel 1 (
-        echo Running javac on %%F resulted in one or more errors, which are printed above.
-        set returnval=2
-    )
-    cd ../.. || exit 1
-  )
+  call %current_directory%\typecheck_one_test.bat %%t
+  set test_retval=!errorlevel!
+  rem update overall return value
+  if !test_retval! NEQ 0 set returnval=1
 )
 
 if !returnval!==0 (
   echo All expected test outputs compiled successfully.
 ) else (
-  if !returnval!==2 (
+  if !returnval!==1 (
     echo Some expected test outputs do not compile successfully. See the above error output for details.
   )
 )
 
-for /r %%F in (*.class) do (
-    del "%%F"
-)
-
-exit /b !returnval!
-endlocal
+endlocal
+exit /b !returnval!