Adjust GitHub actions CI for microarchitectural tests

Signed-off-by: Maciej Kurc <[email protected]>

.github/scripts/convert_coverage_data.sh

@@ -4,46 +4,41 @@ SELF_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
 . ${SELF_DIR}/common.inc.sh
 
 convert_coverage_data(){
-    # This function uses verilator_coverage module to convert a coverage .dat file:
-    #   ${DAT_DIR}/coverage.dat
-    # into an .info file:
-    #   ${RESULTS_DIR}/${FILE_PREFIX}_${COVERAGE}.info
+    # This function uses verilator_coverage module to convert a coverage .dat
+    # file(s) into an .info file(s) for further processing.
     # Args:
-    #   COVERAGE : type of coverage
-    #   DAT_DIR: path to dir containing coverage.dat file
-    #   RESULTS_DIR: path to dir, where .info file will be placed
-    #   FILE_PREFIX: prefix used in the name of coverage_.info
-    check_args_count $# 4
-    COVERAGE=$1
-    DAT_DIR=$2
-    RESULTS_DIR=$3
-    FILE_PREFIX=$4
-    echo -e "${COLOR_WHITE}======= convert_coverage_data =======${COLOR_CLEAR}"
-    echo -e "${COLOR_WHITE}COVERAGE    = ${COVERAGE}"
-    echo -e "${COLOR_WHITE}DAT_DIR     = ${DAT_DIR}"
-    echo -e "${COLOR_WHITE}RESULTS_DIR = ${RESULTS_DIR}"
-    echo -e "${COLOR_WHITE}FILE_PREFIX = ${FILE_PREFIX}"
-    echo -e "${COLOR_WHITE}========== ${COVERAGE} coverage ==========${COLOR_CLEAR}"
+    #   DAT_DIR: path to dir containing coverage.dat file(s)
+    DAT_DIR="${1:-results_verification}"
+    echo -e "${COLOR_WHITE}======= Parse arguments =======${COLOR_CLEAR}"
+    echo -e "${COLOR_WHITE}DAT_DIR = ${DAT_DIR}"
+    echo -e "${COLOR_WHITE}===============================${COLOR_CLEAR}"
 
     # Function body
-    if ! [ -f "${DAT_DIR}/coverage.dat" ]; then
-        echo -e "${COLOR_WHITE}coverage.dat not found in dir=${DAT_DIR} ${COLOR_RED}FAIL${COLOR_CLEAR}"
+    FILES=`find ${DAT_DIR} -name "coverage*.dat"`
+    if [ -z "$FILES" ]; then
+        echo -e "${COLOR_RED}ERROR: No coverage data files were found${COLOR_CLEAR}"
+        echo -e "${COLOR_RED}ERROR: Searched directory: `realpath ${DAT_DIR}`${COLOR_CLEAR}"
+        echo -e "${COLOR_RED}ERROR: convert_coverage_data ended with errors${COLOR_CLEAR}"
         exit -1
     else
-        mkdir -p ${RESULTS_DIR}
-        cp ${DAT_DIR}/coverage.dat ${RESULTS_DIR}/${FILE_PREFIX}_${COVERAGE}.dat
-        verilator_coverage --write-info ${RESULTS_DIR}/${FILE_PREFIX}_${COVERAGE}.info ${RESULTS_DIR}/${FILE_PREFIX}_${COVERAGE}.dat
-        echo -e "${COLOR_WHITE}Conversion: ${DAT_DIR}/coverage.dat -> ${RESULTS_DIR}/${FILE_PREFIX}_${COVERAGE}.info ${COLOR_GREEN}SUCCEEDED${COLOR_CLEAR}"
+        for dat_file in ${FILES}; do
+            info_file=`basename -s .dat ${dat_file}`.info
+            info_realpath=`realpath \`dirname ${dat_file}\``
+            info_file=${info_realpath}/${info_file}
+            verilator_coverage --write-info ${info_file} ${dat_file}
+            echo -e "${COLOR_WHITE}Conversion: ${dat_file} -> ${info_file} ${COLOR_GREEN}SUCCEEDED${COLOR_CLEAR}"
+        done
     fi
 }
 
 # Example usage
-# RESULTS_DIR="results"
-# COVERAGE="branch"
-# DAT_DIR="."
-# FILE_PREFIX="coverage_test"
+# DAT_DIR="results_verification"
 #
-# convert_coverage_data $COVERAGE $DAT_DIR $RESULTS_DIR $FILE_PREFIX
+# convert_coverage_data $DAT_DIR
+
+echo -e "${COLOR_WHITE}========== convert_coverage_data ==============${COLOR_CLEAR}"
 
-check_args_count $# 4
 convert_coverage_data "$@"
+
+echo -e "${COLOR_WHITE}convert_coverage_data ${COLOR_GREEN}SUCCEEDED${COLOR_CLEAR}"
+echo -e "${COLOR_WHITE}========== convert_coverage_data ==============${COLOR_CLEAR}"

.github/workflows/test-regression.yml

@@ -86,17 +86,15 @@ jobs:
 
       - name: Prepare coverage data
         run: |
-          pushd ${{ github.workspace }}
-            mkdir -p coverage_${{ matrix.test }}
-            mv ${TEST_PATH}/coverage.dat coverage_${{ matrix.test }}/
-            echo "Prepared coverage data"
-            .github/scripts/convert_coverage_data.sh ${{ matrix.coverage }} ${{ github.workspace }}/coverage_${{ matrix.test }} ${{ github.workspace }}/results coverage_${{ matrix.test }}
-            echo "convert_coverage_data.sh exited with RET_CODE = "$?
-          popd
+          .github/scripts/convert_coverage_data.sh ${TEST_PATH}/
+          echo "convert_coverage_data.sh exited with RET_CODE = "$?
+          mkdir -p results
+          mv ${TEST_PATH}/coverage.info \
+             results/coverage_${{ matrix.test }}_${{ matrix.coverage }}.info
 
       - name: Pack artifacts
         if: always()
         uses: actions/upload-artifact@v3
         with:
           name: regression_tests_coverage_data
-          path: ./results/*.info
+          path: results/*.info

.github/workflows/test-riscof.yml

@@ -141,11 +141,10 @@ jobs:
       - name: Prepare coverage data
         run: |
           export PATH=/opt/verilator/bin:$PATH
-          .github/scripts/convert_coverage_data.sh \
-            ${{ matrix.coverage }} \
-            riscof/coverage \
-            riscof/coverage \
-            coverage_riscof
+          .github/scripts/convert_coverage_data.sh riscof/coverage/
+          echo "convert_coverage_data.sh exited with RET_CODE = "$?
+          mv riscof/coverage/coverage.info \
+             riscof/coverage/coverage_riscof_${{ matrix.coverage }}.info
 
       - name: Pack artifacts
         if: always()

.github/workflows/test-riscv-dv.yml

@@ -246,14 +246,11 @@ jobs:
 
       - name: Prepare coverage data
         run: |
-          mkdir -p coverage_riscv-dv_${{ matrix.test }}
-          mv ${RV_ROOT}/tools/riscv-dv/work/coverage.dat coverage_riscv-dv_${{ matrix.test }}/
-          echo "Prepared coverage data"
-          .github/scripts/convert_coverage_data.sh \
-              ${{ matrix.coverage }} \
-              coverage_riscv-dv_${{ matrix.test }} \
-              results coverage_riscv-dv_${{ matrix.test }}
+          .github/scripts/convert_coverage_data.sh ${RV_ROOT}/tools/riscv-dv/work/
           echo "convert_coverage_data.sh exited with RET_CODE = "$?
+          mkdir -p results
+          mv ${RV_ROOT}/tools/riscv-dv/work/coverage.info \
+             results/coverage_riscv-dv_${{ matrix.test }}_${{ matrix.coverage }}.info
 
       - name: Pack artifacts
         if: always()

.github/workflows/test-verification.yml

@@ -12,8 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        test: ["test_pyuvm"]
-        coverage: ["branch", "toggle"] #TODO: add functional coverage
+        test: ["block/pic", "block/pic_gw"]
     env:
       CCACHE_DIR: "/opt/verification/.cache/"
       VERILATOR_VERSION: v5.010
@@ -41,7 +40,6 @@ jobs:
           echo "cache_test_restore_key=$cache_test_restore_key" | tee -a "$GITHUB_ENV"
           echo "cache_test_key=$cache_test_key" | tee -a "$GITHUB_ENV"
 
-
       - name: Restore verilator cache
         id: cache-verilator-restore
         uses: actions/cache/restore@v3
@@ -85,42 +83,57 @@ jobs:
           echo "RV_ROOT=$RV_ROOT" >> $GITHUB_ENV
           PYTHONUNBUFFERED=1
           echo "PYTHONUNBUFFERED=$PYTHONUNBUFFERED" >> $GITHUB_ENV
-          TEST_PATH=$RV_ROOT/verification/top/${{ matrix.test }}
+
+          TEST_TYPE=`echo ${{ matrix.test }} | cut -d'/' -f1`
+          TEST_NAME=`echo ${{ matrix.test }} | cut -d'/' -f2`
+          TEST_PATH=$RV_ROOT/verification/${TEST_TYPE}
+
+          echo "TEST_TYPE=$TEST_TYPE" >> $GITHUB_ENV
+          echo "TEST_NAME=$TEST_NAME" >> $GITHUB_ENV
           echo "TEST_PATH=$TEST_PATH" >> $GITHUB_ENV
 
+          pip3 install meson nox
+          pip3 install -r ${RV_ROOT}/verification/${TEST_TYPE}/requirements.txt
+
       - name: Run ${{ matrix.test }}
         run: |
-          pip3 install meson
-          pip3 install -r $RV_ROOT/verification/top/requirements.txt
-          PYTEST_STYLE_SRC_DIR=${{ github.workspace }}/.github/scripts/pytest/
-          PYTEST_CSS=${PYTEST_STYLE_SRC_DIR}/css/styles.css
-          HTML_FILE=${{ matrix.test }}_${{ matrix.COVERAGE }}.html
           pushd ${TEST_PATH}
-            python -m pytest ${{ matrix.test }}.py -sv --coverage=${{ matrix.COVERAGE }} --html=$HTML_FILE --md=$GITHUB_STEP_SUMMARY --css=$PYTEST_CSS
-            bash ${PYTEST_STYLE_SRC_DIR}/style_pytest_report.sh ${PYTEST_STYLE_SRC_DIR} ${TEST_PATH} ${HTML_FILE}
+            nox -s ${TEST_NAME}_verify
           popd
 
       - name: Prepare pytest-html data
         run: |
           pushd ${{ github.workspace }}
-            WEBPAGE_DIR=webpage_${{ matrix.test }}_${{ matrix.COVERAGE }}
-            mkdir -p $WEBPAGE_DIR
-            mv ${TEST_PATH}/${{ matrix.test }}_${{ matrix.COVERAGE }}.html $WEBPAGE_DIR
-            mv ${TEST_PATH}/assets $WEBPAGE_DIR
-            JS_SCRIPT_DIR=${{ github.workspace }}/.github/scripts/pytest/script
-            mv $JS_SCRIPT_DIR $WEBPAGE_DIR
+            for COVERAGE in branch toggle all; do
+              WEBPAGE_DIR=webpage_${TEST_NAME}_${COVERAGE}
+              mkdir -p $WEBPAGE_DIR
+              # TODO: This is a mockup empty page. Ultimately we want it to contain
+              # a report based on the XUnit XML files generated by cocotb.
+              echo "<!DOCTYPE html><html><body></body></html>" >${WEBPAGE_DIR}/${TEST_NAME}_${COVERAGE}.html
+            done
           popd
 
       - name: Prepare coverage data
         run: |
           pushd ${{ github.workspace }}
-            mkdir -p coverage_${{ matrix.test }}
-            mv ${TEST_PATH}/coverage.dat coverage_${{ matrix.test }}/
-            .github/scripts/convert_coverage_data.sh ${{ matrix.COVERAGE }} ${{ github.workspace }}/coverage_${{ matrix.test }} ${{ github.workspace }}/results coverage_${{ matrix.test }}
-            echo "convert_coverage_data.sh exited with RET_CODE = "$?
+            mkdir -p results/coverage_${TEST_NAME}
+            for COVERAGE in branch toggle all; do
+                FILES=`find ${TEST_PATH}/${TEST_NAME}/ -name "coverage_test_*_${COVERAGE}.dat"`
+                if ! [ -z "${FILES}" ]; then
+                    for DAT_FILE in ${FILES}; do
+                        INFO_FILE=`basename ${DAT_FILE} .dat`
+                        INFO_FILE=${INFO_FILE/coverage_test_/}
+                        INFO_FILE=${INFO_FILE/_${COVERAGE}/}
+                        INFO_FILE=${{ github.workspace }}/results/coverage_${TEST_NAME}_${COVERAGE}.info
+                        echo "Converting '${DAT_FILE}' to '${INFO_FILE}'"
+                        verilator_coverage --write-info ${INFO_FILE} ${DAT_FILE}
+                    done
+                fi
+            done
           popd
 
       - name: Upload pytest-html artifacts
+
         if: always()
         uses: actions/upload-artifact@v3
         with: