diff --git a/check_differences/check_differences.bat b/check_differences/check_differences.bat index b43085b2..cbd0d2e8 100644 --- a/check_differences/check_differences.bat +++ b/check_differences/check_differences.bat @@ -1,5 +1,4 @@ @echo off - rem On Windows, instead of using the diff command for SpeciminTestExecutor, we can use this rem command instead since diff is not included by default. This script outputs 0 if no differences rem are found, 1 if a difference in file structure or file content (all whitespace removed) is found @@ -9,18 +8,18 @@ goto :MAIN rem Function to remove all whitespace from a file :RemoveWhitespace setlocal enabledelayedexpansion - set "content=" - rem Simply calling (%~1) is not enough; in edge cases, empty lines containing delimiters (i.e. ;) - rem may be removed. Therefore, we need findstr to add line numbers so no line is empty. - for /F "tokens=* delims=" %%i in ('findstr /n "^" %~1') do ( - set line=%%i - set line=!line:*:=! - if "!line!" NEQ "" set line=!line: =! - set content=!content!!line! - ) - echo !content! >> %~2 + set "content=" + rem Simply calling (%~1) is not enough; in edge cases, empty lines containing delimiters (i.e. ;) + rem may be removed. Therefore, we need findstr to add line numbers so no line is empty. + for /F "tokens=* delims=" %%i in ('findstr /n "^" "%~1"') do ( + set "line=%%i" + set "line=!line:*:=!" + if "!line!" NEQ "" set "line=!line: =!" + set "content=!content!!line!" + ) + echo !content! > "%~2" endlocal -exit /b 0 + exit /b 0 :MAIN @@ -35,85 +34,63 @@ if "%2"=="" ( exit /b 1 ) -set CURRENT_DIRECTORY=%CD% - -cd %1 -set DIRECTORY_1=%CD% - -cd %CURRENT_DIRECTORY% +set "CURRENT_DIRECTORY=%cd%" -cd %2 -set DIRECTORY_2=%CD% +rem use /D to handle drive differences (C:\ vs. D:\) +cd /D "%1" || exit /b 1 +set "DIRECTORY_1=%cd%" -cd %CURRENT_DIRECTORY% +cd /D "%CURRENT_DIRECTORY%" -set DIRECTORY_1_COPY=%DIRECTORY_1% -set DIRECTORY_2_COPY=%DIRECTORY_2% - -set DIR_1_LENGTH=0 -set DIR_2_LENGTH=0 - -rem Get the length of directory 1 so we can later convert absolute to relative paths -:dir_1_length_loop -if defined DIRECTORY_1_COPY ( - set "DIRECTORY_1_COPY=!DIRECTORY_1_COPY:~1!" - set /A "DIR_1_LENGTH+=1" - goto :dir_1_length_loop -) - -rem Get the length of directory 2 so we can later convert absolute to relative paths -:dir_2_length_loop -if defined DIRECTORY_2_COPY ( - set "DIRECTORY_2_COPY=!DIRECTORY_2_COPY:~1!" - set /A "DIR_2_LENGTH+=1" - goto :dir_2_length_loop -) +cd /D "%2" || exit /b 1 +set "DIRECTORY_2=%cd%" -rem Account for trailing \ (absent when using CD) -set /A DIR_1_LENGTH+=1 -set /A DIR_2_LENGTH+=1 +echo %DIRECTORY_1% +echo %DIRECTORY_2% +cd /D "%DIRECTORY_1%" set DIR_1_STRUCTURE= -for /F "delims=" %%i in ('dir "%1" /A-D /S /B') do ( +for /r %%i in (*) do ( rem Convert absolute to relative path - set ABSOLUTE_PATH=%%i - set RELATIVE_PATH=!ABSOLUTE_PATH:~%DIR_1_LENGTH%! + set "ABSOLUTE_PATH=%%i" + set "RELATIVE_PATH=!ABSOLUTE_PATH:%DIRECTORY_1%\=!" rem Add each file path to the DIR_1_STRUCTURE list of files - set DIR_1_STRUCTURE=!DIR_1_STRUCTURE!;!RELATIVE_PATH! + set "DIR_1_STRUCTURE=!DIR_1_STRUCTURE!;!RELATIVE_PATH!" ) +cd /D "%DIRECTORY_2%" set DIR_2_STRUCTURE= -for /F "delims=" %%i in ('dir "%2" /A-D /S /B') do ( +for /r %%i in (*) do ( rem Convert absolute to relative path - set ABSOLUTE_PATH=%%i - set RELATIVE_PATH=!ABSOLUTE_PATH:~%DIR_2_LENGTH%! + set "ABSOLUTE_PATH=%%i" + set "RELATIVE_PATH=!ABSOLUTE_PATH:%DIRECTORY_2%\=!" rem Add each file path to the DIR_2_STRUCTURE list of files - set DIR_2_STRUCTURE=!DIR_2_STRUCTURE!;!RELATIVE_PATH! + set "DIR_2_STRUCTURE=!DIR_2_STRUCTURE!;!RELATIVE_PATH!" ) if "!DIR_1_STRUCTURE!" NEQ "!DIR_2_STRUCTURE!" ( - echo Folder structures are different + echo %DIRECTORY_1% and %DIRECTORY_2% have different directory structures endlocal exit /b 1 ) rem At this point, we have guaranteed that the files are the same, so we only rem need to iterate through one of the lists. -for %%f in (%DIR_1_STRUCTURE%) do ( - set FILE_1=!DIRECTORY_1!\%%f - set FILE_2=!DIRECTORY_2!\%%f - call :RemoveWhitespace !FILE_1! !FILE_1!.tmp - call :RemoveWhitespace !FILE_2! !FILE_2!.tmp +for %%f in (!DIR_1_STRUCTURE!) do ( + set "FILE_1=!DIRECTORY_1!\%%f" + set "FILE_2=!DIRECTORY_2!\%%f" + call :RemoveWhitespace "!FILE_1!" "!FILE_1!.tmp" + call :RemoveWhitespace "!FILE_2!" "!FILE_2!.tmp" set fail=0 rem set fail=1 will only be called if fc exits with exit code 1 (differences) - fc !FILE_1!.tmp !FILE_2!.tmp > nul || set fail=1 + fc "!FILE_1!.tmp" "!FILE_2!.tmp" > nul || set fail=1 - del !FILE_1!.tmp - del !FILE_2!.tmp + del "!FILE_1!.tmp" + del "!FILE_2!.tmp" if !fail!==1 ( - echo !FILE_1! and !FILE_2! are different - goto differences_found + echo "!FILE_1!" and "!FILE_2!" are different + goto :differences_found ) ) diff --git a/check_differences/test/test_runner.bat b/check_differences/test/test_runner.bat index 5f1eddd6..dad155f6 100644 --- a/check_differences/test/test_runner.bat +++ b/check_differences/test/test_runner.bat @@ -34,7 +34,7 @@ endlocal if "%%t" NEQ "base" ( @echo off rem Redirect to nul because we don't want any output messages - call "../../check_differences.bat" "base" "%%t" > nul + call "../../check_differences.bat" base %%t > nul set exitcode=!errorlevel! rem "%%t" represents the folder name (i.e. base, 0, 1) since we have changed the directory rem to the test case directory diff --git a/src/test/java/org/checkerframework/specimin/SpeciminTestExecutor.java b/src/test/java/org/checkerframework/specimin/SpeciminTestExecutor.java index 4c8a0cb9..2fa89c0c 100644 --- a/src/test/java/org/checkerframework/specimin/SpeciminTestExecutor.java +++ b/src/test/java/org/checkerframework/specimin/SpeciminTestExecutor.java @@ -66,11 +66,12 @@ public static void runTest( // Construct the list of arguments. List speciminArgs = new ArrayList<>(); + speciminArgs.add("--outputDirectory"); speciminArgs.add(outputDir.toAbsolutePath().toString()); speciminArgs.add("--root"); speciminArgs.add( - Path.of("src/test/resources/" + testName + "/input/").toAbsolutePath().toString() + "/"); + Path.of("src/test/resources/" + testName + "/input/").toAbsolutePath().toString() + "/"); for (String targetFile : targetFiles) { speciminArgs.add("--targetFile"); speciminArgs.add(targetFile); @@ -94,9 +95,9 @@ public static void runTest( // Run specimin on target SpeciminRunner.main(speciminArgs.toArray(new String[0])); + boolean isWindows = Ascii.toLowerCase(System.getProperty("os.name")).startsWith("windows"); // Diff the files to ensure that specimin's output is what we expect ProcessBuilder builder = new ProcessBuilder(); - boolean isWindows = Ascii.toLowerCase(System.getProperty("os.name")).startsWith("windows"); if (isWindows) { builder.command( "check_differences/check_differences.bat", diff --git a/typecheck_one_test.bat b/typecheck_one_test.bat new file mode 100644 index 00000000..8346beb2 --- /dev/null +++ b/typecheck_one_test.bat @@ -0,0 +1,41 @@ +@echo off + +rem This script runs javac on a single expected test output. If the test output +rem is compilable, it exits with code 0. If not, it exits with code 1. In all other +rem cases (e.g., if a cd command fails), it exits with code 2. It includes +rem logic for skipping test cases that don't need to compile, etc. It should be +rem run in the directory src/test/resources. + +setlocal enabledelayedexpansion + +set testcase=%1 + +if "%testcase%"=="shared" exit /b 0 +rem https://bugs.openjdk.org/browse/JDK-8319461 wasn't actually fixed (this test is based on that bug) +if "%testcase%"=="superinterfaceextends" exit /b 0 +rem incomplete handling of method references: https://github.com/njit-jerse/specimin/issues/291 +rem this test exists to check that no crash occurs, not that Specimin produces the correct output +if "%testcase%"=="methodref2" exit /b 0 +rem this test will not compile right now; this is a TODO in UnsolvedSymbolVisitor#lookupTypeArgumentFQN +if "%testcase%"=="methodreturnfullyqualifiedgeneric" exit /b 0 + +cd "%testcase%\expected\" || exit /b 2 + +set JAVA_FILES= +for /r %%F in (*.java) do ( + set "JAVA_FILES=!JAVA_FILES! %%F" +) + +javac -classpath "../../shared/checker-qual-3.42.0.jar" !JAVA_FILES! +if errorlevel 1 ( + echo Running javac on %testcase% resulted in one or more errors, which are printed above. + set returnval=1 +) + +rem clean up +for /r %%F in (*.class) do ( + del "%%F" +) +endlocal + +exit /b 0 \ No newline at end of file diff --git a/typecheck_test_outputs.bat b/typecheck_test_outputs.bat index 9efe890a..426bc1e8 100644 --- a/typecheck_test_outputs.bat +++ b/typecheck_test_outputs.bat @@ -10,46 +10,24 @@ rem should produce independently-compilable programs. setlocal enabledelayedexpansion set returnval=0 +set "current_directory=%cd%" + cd src\test\resources || exit /b 1 for /d %%t in (*) do ( - set continue=0 - if "%%t"=="shared" set continue=1 - rem https://bugs.openjdk.org/browse/JDK-8319461 wasn't actually fixed (this test is based on that bug) - if "%%t"=="superinterfaceextends" set continue=1 - rem incomplete handling of method references: https://github.com/njit-jerse/specimin/issues/291 - rem this test exists to check that no crash occurs, not that Specimin produces the correct output - if "%%t"=="methodref2" set continue=1 - rem this test will not compile right now; this is a TODO in UnsolvedSymbolVisitor#lookupTypeArgumentFQN - if "%%t"=="methodreturnfullyqualifiedgeneric" set continue=1 - if !continue!==0 ( - cd "%%t/expected/" || exit 1 - rem javac relies on word splitting - rem shellcheck disable=SC2046 - set JAVA_FILES= - for /r %%F in (*.java) do ( - set "JAVA_FILES=!JAVA_FILES! %%F" - ) - javac -classpath "../../shared/checker-qual-3.42.0.jar" !JAVA_FILES! - if errorlevel 1 ( - echo Running javac on %%F resulted in one or more errors, which are printed above. - set returnval=2 - ) - cd ../.. || exit 1 - ) + call %current_directory%\typecheck_one_test.bat %%t + set test_retval=!errorlevel! + rem update overall return value + if !test_retval! NEQ 0 set returnval=1 ) if !returnval!==0 ( echo All expected test outputs compiled successfully. ) else ( - if !returnval!==2 ( + if !returnval!==1 ( echo Some expected test outputs do not compile successfully. See the above error output for details. ) ) -for /r %%F in (*.class) do ( - del "%%F" -) - -exit /b !returnval! -endlocal \ No newline at end of file +endlocal +exit /b !returnval! \ No newline at end of file