From 8f490ccd330aa1702ee881c41b817ea32599599a Mon Sep 17 00:00:00 2001 From: Maryla Date: Tue, 21 Nov 2023 12:20:44 +0100 Subject: [PATCH 1/2] Improve instructions and output for golden tests. --- .github/workflows/ci-linux-golden-tests.yml | 6 ++- tests/golden_test_common.sh | 44 +++++++++++++++++---- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-linux-golden-tests.yml b/.github/workflows/ci-linux-golden-tests.yml index 8e0e7e944d..f21fcdb365 100644 --- a/.github/workflows/ci-linux-golden-tests.yml +++ b/.github/workflows/ci-linux-golden-tests.yml @@ -98,5 +98,7 @@ jobs: if: failure() run: > echo "If the tests fail, download the test results from the Artifacts list at https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} or with: - gh run download --dir /tmp/golden-tests-output --repo ${{ github.repository }} --name golden-tests-output ${{ github.run_id }} - Then look at the instructions in the unzipped file or in /tmp/golden-tests-output if you used the command line." + gh run download --dir /tmp/golden-tests-output-${{ github.run_id }} --repo ${{ github.repository }} --name golden-tests-output ${{ github.run_id }} + Update all files with: + cp /tmp/golden-tests-output-${{ github.run_id }}/*/*.xml ~/git/libavif/tests/data/goldens + Or look at detailed instructions in the README.txt file in the test ouputs." diff --git a/tests/golden_test_common.sh b/tests/golden_test_common.sh index 920da2891e..02c23e3d58 100755 --- a/tests/golden_test_common.sh +++ b/tests/golden_test_common.sh @@ -42,6 +42,12 @@ GOLDEN_DIR="${TESTDATA_DIR}/goldens" AVIFENC="${ENCODER_DIR}/avifenc" MP4BOX="${MP4BOX_DIR}/MP4Box" +# Colors for pretty formatting. +RED='\033[0;31m' +GREEN='\033[0;32m' +BLUE='\033[0;34m' +NO_COLOR='\033[0m' + if [[ ! -x "$AVIFENC" ]]; then echo "'$AVIFENC' does not exist or is not executable" exit 1 @@ -92,7 +98,15 @@ redact_xml() { diff_with_goldens() { echo "===" + diff_command="" + diff_command_with_full_paths="" + num_files=0 + num_passed=0 + files_with_diffs="" + files_missing_golden="" + for f in $(find . -name '*.avif'); do + num_files=$((num_files + 1)) echo "Testing $f" xml="$(basename "$f").xml" # Remove leading ./ for prettier paths. # Dump the file structure as XML @@ -106,15 +120,25 @@ diff_with_goldens() { diff -u "$golden" "$xml" > "$xml.diff" || golden_differs=true if $golden_differs; then has_errors=true + files_with_diffs="${files_with_diffs}$(basename "$f") " echo "FAILED: Differences found for file $xml, see details in $OUTPUT_DIR/$xml.diff" >&2 echo "Expected file: $golden" >&2 echo "Actual file: $OUTPUT_DIR/$xml" >&2 cp "$golden" "$xml.golden" # Copy golden to output directory for easier debugging. + + # "diff --color" is the default value if DIFFTOOL is not set. + printf -v diff_command "${diff_command}\${DIFFTOOL:-diff --color} $xml.golden $xml\n" + printf -v diff_command_with_full_paths "${diff_command_with_full_paths} \${DIFFTOOL:-diff --color} $OUTPUT_DIR/$xml.golden $OUTPUT_DIR/$xml\n" else + num_passed=$((num_passed + 1)) + # Remove files related to tests that passed to reduce visual noise in the output dir. + rm "$f" + rm "$xml" rm "$xml.diff" # Delete empty diff files. echo "Passed" fi else + files_missing_golden="${files_missing_golden}$(basename "$f") " echo "FAILED: Missing golden file $golden" >&2 echo "Actual file: $OUTPUT_DIR/$xml" >&2 has_errors=true @@ -137,20 +161,24 @@ popd > /dev/null if $has_errors; then cat >&2 << EOF -Test failed. +$(echo -e "${RED}")Golden test failed. Num passed: ${num_passed}/${num_files}$(echo -e "${NO_COLOR}") +Files that caused diffs: ${files_with_diffs} +Files with missing golden: ${files_missing_golden} # IF RUNNING ON GITHUB - Check the workflow step called "How to fix failing tests" for instructions on how to debug/fix this test. + Check below at the workflow step below called "How to fix failing tests" for instructions on how to debug/fix this test. # IF RUNNING LOCALLY - Look at the .xml.diff files in $OUTPUT_DIR - - If the diffs are expected, update the golden files with: - cp "$OUTPUT_DIR"/*.xml "$GOLDEN_DIR" + View diffs with: (set DIFFTOOL to your favorite tool if needed) +$(echo -e "${BLUE}")$diff_command_with_full_paths$(echo -e "${NO_COLOR}") + Update golden files with: + $(echo -e "${BLUE}")cp $OUTPUT_DIR/*.xml "$GOLDEN_DIR"$(echo -e "${NO_COLOR}") EOF cat > "$OUTPUT_DIR/README.txt" << EOF -To debug test failures, look at the .xml.diff files. +To debug test failures, look at the .xml.diff files or run (from this directory): + +$diff_command If the diffs are expected, update the golden files by copying the .xml files, e.g.: cp *.xml $HOME/git/libavif/tests/data/goldens @@ -159,6 +187,8 @@ You can also debug the test locally by setting AVIF_ENABLE_GOLDEN_TESTS=ON in cm EOF exit 1 +else + echo -e "${GREEN}Golden tests passed (${num_passed} tests)${NO_COLOR}" fi exit 0 From 7b360a927da026cefed2193728f49c6330816b98 Mon Sep 17 00:00:00 2001 From: Maryla Date: Wed, 22 Nov 2023 11:09:49 +0100 Subject: [PATCH 2/2] Minor text fixes. --- tests/golden_test_common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/golden_test_common.sh b/tests/golden_test_common.sh index 02c23e3d58..91023a5c2d 100755 --- a/tests/golden_test_common.sh +++ b/tests/golden_test_common.sh @@ -128,6 +128,7 @@ diff_with_goldens() { # "diff --color" is the default value if DIFFTOOL is not set. printf -v diff_command "${diff_command}\${DIFFTOOL:-diff --color} $xml.golden $xml\n" + # Same as diff_command but with absolute paths and an two space indent. printf -v diff_command_with_full_paths "${diff_command_with_full_paths} \${DIFFTOOL:-diff --color} $OUTPUT_DIR/$xml.golden $OUTPUT_DIR/$xml\n" else num_passed=$((num_passed + 1)) @@ -166,7 +167,7 @@ Files that caused diffs: ${files_with_diffs} Files with missing golden: ${files_missing_golden} # IF RUNNING ON GITHUB - Check below at the workflow step below called "How to fix failing tests" for instructions on how to debug/fix this test. + Check the workflow step below called "How to fix failing tests" for instructions on how to debug/fix this test. # IF RUNNING LOCALLY View diffs with: (set DIFFTOOL to your favorite tool if needed)