diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 3ce6e8be27b..87c97ff2e70 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -204,6 +204,7 @@ jobs: ## Compare test failures VS reference have_new_failures="" REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log' + ROOT_REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite-root.log' REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' @@ -223,68 +224,87 @@ jobs: rm -f ${COMMENT_LOG} touch ${COMMENT_LOG} - if test -f "${REF_LOG_FILE}"; then - echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" - REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort) - NEW_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort) - REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort) - NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort) - for LINE in ${REF_FAILING} - do - if ! grep -Fxq ${LINE}<<<"${NEW_FAILING}"; then - if ! grep ${LINE} ${IGNORE_INTERMITTENT} + compare_tests() { + local new_log_file=$1 + local ref_log_file=$2 + local test_type=$3 # "standard" or "root" + + if test -f "${ref_log_file}"; then + echo "Reference ${test_type} test log SHA1/ID: $(sha1sum -- "${ref_log_file}")" + REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) + NEW_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) + REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) + NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) + + # Compare failing and error tests + for LINE in ${NEW_FAILING} + do + if ! grep -Fxq ${LINE}<<<"${REF_FAILING}" then - MSG="Congrats! The gnu test ${LINE} is no longer failing!" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - else - MSG="Skipping an intermittent issue ${LINE}" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - echo "" + if ! grep ${LINE} ${IGNORE_INTERMITTENT} + then + MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?" + echo "::error ::$MSG" + echo $MSG >> ${COMMENT_LOG} + have_new_failures="true" + else + MSG="Skip an intermittent issue ${LINE}" + echo "::warning ::$MSG" + echo $MSG >> ${COMMENT_LOG} + echo "" + fi fi - fi - done - for LINE in ${NEW_FAILING} - do - if ! grep -Fxq ${LINE}<<<"${REF_FAILING}" - then - if ! grep ${LINE} ${IGNORE_INTERMITTENT} + done + + for LINE in ${REF_FAILING} + do + if ! grep -Fxq ${LINE}<<<"${NEW_FAILING}" + then + if ! grep ${LINE} ${IGNORE_INTERMITTENT} + then + MSG="Congrats! The gnu test ${LINE} is no longer failing!" + echo "::warning ::$MSG" + echo $MSG >> ${COMMENT_LOG} + else + MSG="Skipping an intermittent issue ${LINE}" + echo "::warning ::$MSG" + echo $MSG >> ${COMMENT_LOG} + echo "" + fi + fi + done + + for LINE in ${NEW_ERROR} + do + if ! grep -Fxq ${LINE}<<<"${REF_ERROR}" then - MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" + MSG="GNU test error: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?" echo "::error ::$MSG" echo $MSG >> ${COMMENT_LOG} have_new_failures="true" - else - MSG="Skip an intermittent issue ${LINE}" + fi + done + + for LINE in ${REF_ERROR} + do + if ! grep -Fxq ${LINE}<<<"${NEW_ERROR}" + then + MSG="Congrats! The gnu test ${LINE} is no longer ERROR!" echo "::warning ::$MSG" echo $MSG >> ${COMMENT_LOG} - echo "" fi - fi - done - for LINE in ${REF_ERROR} - do - if ! grep -Fxq ${LINE}<<<"${NEW_ERROR}"; then - MSG="Congrats! The gnu test ${LINE} is no longer ERROR!" - echo "::warning ::$MSG" - echo $MSG >> ${COMMENT_LOG} - fi - done - for LINE in ${NEW_ERROR} - do - if ! grep -Fxq ${LINE}<<<"${REF_ERROR}" - then - MSG="GNU test error: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" - echo "::error ::$MSG" - echo $MSG >> ${COMMENT_LOG} - have_new_failures="true" - fi - done + done + else + echo "::warning ::Skipping ${test_type} test failure comparison; no prior reference test logs are available." + fi + } + + # Compare standard tests + compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' "${REF_LOG_FILE}" "standard" + + # Compare root tests + compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite-root.log' "${ROOT_REF_LOG_FILE}" "root" - else - echo "::warning ::Skipping test failure comparison; no prior reference test logs are available." - fi if test -n "${have_new_failures}" ; then exit -1 ; fi - name: Upload comparison log (for GnuComment workflow) if: success() || failure() # run regardless of prior step success/failure