diff --git a/src/test/make/lift-directories.mk b/src/test/make/lift-directories.mk index 1c1b8ee44..eaa1434d1 100644 --- a/src/test/make/lift-directories.mk +++ b/src/test/make/lift-directories.mk @@ -27,7 +27,8 @@ READELF ?= aarch64-linux-gnu-readelf BASIL=$(GIT_ROOT)/target/scala-3.3.1/wptool-boogie-assembly-0.0.1.jar -# paths below are relative to lift.mk's compilation_variant directory +# paths below are relative to lift.mk's compilation_variant directory. +# note, wildcard is relative to test_case directory. C_SOURCE ?=$(addprefix ../,$(wildcard *.c)) SPEC ?=$(addprefix ../,$(wildcard *.spec)) EXTRA_SPEC ?=$(addprefix ../,$(wildcard *.bpl)) @@ -44,9 +45,24 @@ TARGETS := all verify repro-stash repro-check md5sum-check md5sum-update clean c $(TARGETS): $(ENABLED_COMPILERS) +# an empty test case directory may be present for a number of reasons (e.g. git branch switching). +# ignore such directories to avoid more cryptic errors later. +ifeq ($(C_SOURCE),) + +# more validity checks +ifneq ($(SPEC)$(EXTRA_SPEC),) + $(error invalid test case: "$(realpath .)" has .bpl or .spec files but no C file) +endif + +$(ENABLED_COMPILERS): + @echo 'note: skipping directory "$(realpath .)" with no C files...' + +else + $(ENABLED_COMPILERS): mkdir -p $@/ - # - continue if fails $(MAKE) -C $(realpath .)/$@ -f $(MAKE_DIR)/$@.mk $(MAKECMDGOALS) +endif + diff --git a/src/test/make/lift.mk b/src/test/make/lift.mk index f59db2376..a828b6f42 100644 --- a/src/test/make/lift.mk +++ b/src/test/make/lift.mk @@ -37,10 +37,10 @@ ifeq ($(USE_DOCKER), 1) # $(DOCKER_CMD) hash > docker-hash-new # diff --color -u docker-hash docker-hash-new # if this fails, make sure your docker image is up-to-date. # rm docker-hash-new - cd $(BASE_DIR) && md5sum -c $(realpath md5sums) # using docker; checking compiler output hashes. + cd $(BASE_DIR) && md5sum -c $(realpath .)/md5sums # using docker; checking compiler output hashes. else echo "not running within docker; skipping docker image validation." - cd $(BASE_DIR) && md5sum -c $(realpath md5sums) + cd $(BASE_DIR) && md5sum -c $(realpath .)/md5sums endif # paths in md5sum are relative to src/test, to allow for collation into a big md5sums file @@ -66,7 +66,7 @@ $(BASIL): # don't re-lift only if binary is missing .SECONDARY: a.out a.out: $(C_SOURCE) - $(CC) $(CFLAGS) $(C_SOURCE) + $(CC) $(CFLAGS) '$(C_SOURCE)' .PHONY=all recompile verify repro-stash repro-check md5sum-check md5sum-update clean cleanlift cleanall cleanbin cleantest cleangts cleanrepro json gts verify: $(NAME)_bap.bpl $(NAME)_gtirb.bpl diff --git a/src/test/readme.md b/src/test/readme.md index 9f3b1b956..47207cca6 100644 --- a/src/test/readme.md +++ b/src/test/readme.md @@ -11,11 +11,12 @@ The compiled binaries and lifter outputs are not kept inside this repository. To get started, you can download pre-compiled copies of these files: ```bash cd src/test +make clean -j4 # `make extract` will refuse to overwrite existing files make extract ``` This should be enough to run the SystemTests through mill. -Note that `make extract` will not overwrite any existing output files. -Use `make clean -j4` beforehand if needed. +Note that `make extract` will not overwrite any existing output files, +so `make clean` is suggested beforehand. For much more detail about the lifting process, including how to add or edit test cases, keep reading. @@ -116,7 +117,7 @@ Now, running make commands should use compilers from Docker. ```bash make md5sum-check -j6 ``` - Alternatively, you can run `md5sum -c compiled.md5sums` + Alternatively, you can run `md5sum -c compiled.md5sum` to check all files in one batch (this will be faster, but the make command is more flexible). @@ -130,13 +131,15 @@ Now, running make commands should use compilers from Docker. correct/basic_function_call_caller/clang_O2/a.out: FAILED md5sum: WARNING: 1 computed checksum did NOT match ``` - If the mismatch is in the md5sum of a compilation output, this likely means the compiler is not being deterministic; this is a bug and should be reported. See the _Troubleshooting_ section below for steps to debug the issue and inspect differences between compilation runs. + If you see other errors, such as "file not found", it is possible that there + are test directories without md5sums, or some other invalid state. + 5. If the md5sum-check succeeds, you are good to go! You should repeat these steps if the files have been updated by someone else, e.g., to use a more recent version of BAP or ASLp. @@ -189,15 +192,25 @@ You can use these steps to do so. 6. In the src/tests directory, run `make compiled.md5sum` to update the combined md5sums file. 7. Create the tarball of generated files with `make compiled.tar.bz2`. Take note of the md5sum line at the bottom. -8. Upload compiled.tar.gz to a publicly-accessible file host and take note of the URL. +8. Upload compiled.tar.bz2 to a publicly-accessible file host and take note of the URL. ```bash curl -Freqtype=fileupload -FfileToUpload=@compiled.tar.bz2 https://catbox.moe/user/api.php ``` 9. Update compiled.url.txt with the new URL and the new md5sum. -10. Optionally, check your new hashes are valid and reproducible with `make clean -j4 && make md5sum-check -j4`. -11. Optionally, check the uploaded tarball is valid with `make clean -j4 && make extract`. +10. Optional but recommended, check your new hashes are valid and reproducible with `make clean -j4 && make md5sum-check -j4`. +11. Highly recommended, check the uploaded tarball is valid with + ```bash + make clean -j4 && make extract && GCC=/nowhere CLANG=/nowhere make md5sum-check -j6 + ``` 12. Commit and PR your changes. + +**Be careful!** If you use `make extract` after editing test cases +but before updating md5sums, the Makefile will assume the extracted +files (from the old test cases) are still valid. +Be sure to make your changes after extracting, or clean the folders +of the edited test cases. + For consistency, you *must* use the Docker environment when making changes to the hash files with md5sum-update. The Makefile should make sure that Docker is active.