handle empty directories and missing md5sum better

src/test/make/lift-directories.mk

@@ -27,7 +27,8 @@ READELF ?= aarch64-linux-gnu-readelf
 
 BASIL=$(GIT_ROOT)/target/scala-3.3.1/wptool-boogie-assembly-0.0.1.jar
 
-# paths below are relative to lift.mk's compilation_variant directory
+# paths below are relative to lift.mk's compilation_variant directory.
+# note, wildcard is relative to test_case directory.
 C_SOURCE ?=$(addprefix ../,$(wildcard *.c))
 SPEC ?=$(addprefix ../,$(wildcard *.spec))
 EXTRA_SPEC ?=$(addprefix ../,$(wildcard *.bpl))
@@ -44,9 +45,24 @@ TARGETS := all verify repro-stash repro-check md5sum-check md5sum-update clean c
 
 $(TARGETS): $(ENABLED_COMPILERS)
 
+# an empty test case directory may be present for a number of reasons (e.g. git branch switching).
+# ignore such directories to avoid more cryptic errors later.
+ifeq ($(C_SOURCE),)
+
+# more validity checks
+ifneq ($(SPEC)$(EXTRA_SPEC),)
+	$(error invalid test case: "$(realpath .)" has .bpl or .spec files but no C file)
+endif
+
+$(ENABLED_COMPILERS):
+	@echo 'note: skipping directory "$(realpath .)" with no C files...'
+
+else
+
 $(ENABLED_COMPILERS):
 	mkdir -p $@/
-	# - continue if fails
 	$(MAKE) -C $(realpath .)/$@ -f $(MAKE_DIR)/$@.mk $(MAKECMDGOALS)
 
+endif
+
 

src/test/make/lift.mk

@@ -37,10 +37,10 @@ ifeq ($(USE_DOCKER), 1)
 	# $(DOCKER_CMD) hash > docker-hash-new
 	# diff --color -u docker-hash docker-hash-new  # if this fails, make sure your docker image is up-to-date.
 	# rm docker-hash-new
-	cd $(BASE_DIR) && md5sum -c $(realpath md5sums)  # using docker; checking compiler output hashes.
+	cd $(BASE_DIR) && md5sum -c $(realpath .)/md5sums  # using docker; checking compiler output hashes.
 else
 	echo "not running within docker; skipping docker image validation."
-	cd $(BASE_DIR) && md5sum -c $(realpath md5sums)
+	cd $(BASE_DIR) && md5sum -c $(realpath .)/md5sums
 endif
 
 # paths in md5sum are relative to src/test, to allow for collation into a big md5sums file
@@ -66,7 +66,7 @@ $(BASIL):
 # don't re-lift only if binary is missing 
 .SECONDARY: a.out
 a.out: $(C_SOURCE)
-	$(CC) $(CFLAGS) $(C_SOURCE)
+	$(CC) $(CFLAGS) '$(C_SOURCE)'
 
 .PHONY=all recompile verify repro-stash repro-check md5sum-check md5sum-update clean cleanlift cleanall cleanbin cleantest cleangts cleanrepro json gts
 verify: $(NAME)_bap.bpl $(NAME)_gtirb.bpl

src/test/readme.md

@@ -11,11 +11,12 @@ The compiled binaries and lifter outputs are not kept inside this repository.
 To get started, you can download pre-compiled copies of these files:
 ```bash
 cd src/test
+make clean -j4  # `make extract` will refuse to overwrite existing files
 make extract
 ```
 This should be enough to run the SystemTests through mill.
-Note that `make extract` will not overwrite any existing output files.
-Use `make clean -j4` beforehand if needed.
+Note that `make extract` will not overwrite any existing output files,
+so `make clean` is suggested beforehand.
 
 For much more detail about the lifting process, including how to add or edit
 test cases, keep reading.
@@ -116,7 +117,7 @@ Now, running make commands should use compilers from Docker.
    ```bash
    make md5sum-check -j6
    ```
-   Alternatively, you can run `md5sum -c compiled.md5sums`
+   Alternatively, you can run `md5sum -c compiled.md5sum`
    to check all files in one batch
    (this will be faster, but the make command is more flexible).
 
@@ -130,13 +131,15 @@ Now, running make commands should use compilers from Docker.
    correct/basic_function_call_caller/clang_O2/a.out: FAILED
    md5sum: WARNING: 1 computed checksum did NOT match
    ```
-
    If the mismatch is in the md5sum of a compilation output, this likely means the
    compiler is not being deterministic;
    this is a bug and should be reported.
    See the _Troubleshooting_ section below for steps to debug the issue and inspect differences between
    compilation runs.
 
+   If you see other errors, such as "file not found", it is possible that there
+   are test directories without md5sums, or some other invalid state.
+
 5. If the md5sum-check succeeds, you are good to go!
    You should repeat these steps if the files have been updated by someone else,
    e.g., to use a more recent version of BAP or ASLp.
@@ -189,15 +192,25 @@ You can use these steps to do so.
 6. In the src/tests directory, run `make compiled.md5sum` to update the combined md5sums file.
 7. Create the tarball of generated files with `make compiled.tar.bz2`.
    Take note of the md5sum line at the bottom.
-8. Upload compiled.tar.gz to a publicly-accessible file host and take note of the URL.
+8. Upload compiled.tar.bz2 to a publicly-accessible file host and take note of the URL.
    ```bash
    curl -Freqtype=fileupload [email protected] https://catbox.moe/user/api.php
    ```
 9. Update compiled.url.txt with the new URL and the new md5sum.
-10. Optionally, check your new hashes are valid and reproducible with `make clean -j4 && make md5sum-check -j4`.
-11. Optionally, check the uploaded tarball is valid with `make clean -j4 && make extract`.
+10. Optional but recommended, check your new hashes are valid and reproducible with `make clean -j4 && make md5sum-check -j4`.
+11. Highly recommended, check the uploaded tarball is valid with
+   ```bash
+   make clean -j4 && make extract && GCC=/nowhere CLANG=/nowhere make md5sum-check -j6
+   ```
 12. Commit and PR your changes.
 
+
+**Be careful!** If you use `make extract` after editing test cases
+but before updating md5sums, the Makefile will assume the extracted
+files (from the old test cases) are still valid.
+Be sure to make your changes after extracting, or clean the folders
+of the edited test cases.
+
 For consistency, you *must* use the Docker environment when
 making changes to the hash files with md5sum-update.
 The Makefile should make sure that Docker is active.