Added Xfail and Mayfail attributes to city runner

Support attribute Xfail and Mayfail for city runner. These can be set
as additional elements in the csv file.

Xfail means we expect it to fail and will give a failing return code
if it unexpectedly pass, otherwise it does not contribute to an overall
fail.

Mayfail means it sometimes fails and will not contribute to a failing
return code.

Unlike some of the other attributes, these cannot be set with additional
csv files, as it was felt this leads to overcomplexity. Instead it is
expected that some form of csv combination script could be used alongside
this.

.github/actions/run_opencl_cts/action.yml

@@ -34,11 +34,16 @@ runs:
         fi
         echo QEMU SETTING: $QEMU_SETTING
         set -x
-        echo > expect_fail.csv
+        # Build override file, all is done first, then the target specific.
+        # The last file can overwrite previous overrides.        
+        for csv in .github/opencl_cts/override_all.csv .github/opencl_cts/override_${{ inputs.target }}.csv
+        do
+           [ -f $csv ] && cat $csv >> override.csv
+        done > override.csv
+        echo override file:
+        cat override.csv
+
         # $CTS_FILTER ignores certain test, so is treated differently to temporary fails.
-        [ -f .github/opencl_cts/expect_fail_all.csv ] && cat .github/opencl_cts/expect_fail_all.csv >> expect_fail.csv
-        [ -f .github/opencl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/opencl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv
-        cat expect_fail.csv "$GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER" > disable.csv 
         # Note: use 'eval' built-in to handle quoting/escaping/splitting reqs
         RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \
           --color=always --timeout $CTS_TIMEOUT \
@@ -49,23 +54,6 @@ runs:
           -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
           -e CL_PLATFORM_INDEX=0 \
           -s $GITHUB_WORKSPACE/test_conformance/$CTS_CSV_FILE \
-          -i disable.csv"
+          -i $GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER \
+          -o override.csv"
         eval $RUN_CITIES
-
-    - name: Run expected failed opencl cts
-      shell: bash
-      env:
-        CTS_TIMEOUT: 18:00:00
-      run: |
-        echo "Running OpenCL CTS tests with CTS file $CTS_CSV_FILE with filter $CTS_FILTER"
-        set -x
-        RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \
-          --color=always --timeout $CTS_TIMEOUT \
-          $QEMU_SETTING \
-          -b $GITHUB_WORKSPACE/test_conformance \
-          -L $GITHUB_WORKSPACE/install_icd/lib \
-          -e CLC_EXECUTABLE=$GITHUB_WORKSPACE/install_ock/bin/clc \
-          -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
-          -e CL_PLATFORM_INDEX=0 \
-          -s expect_fail.csv"
-        eval $RUN_CITIES || echo failed as expected

.github/actions/run_sycl_cts/action.yml

@@ -67,12 +67,18 @@ runs:
         export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib
         export ONEAPI_DEVICE_SELECTOR=opencl:0
         export CTS_CSV_FILE=$GITHUB_WORKSPACE/.github/scripts/sycl-cts.csv
-        echo > expect_fail.csv
+        echo > override.csv
         # $CTS_FILTER ignores certain test, so is treated differently to temporary fails.
-        [ -f .github/sycl_cts/expect_fail_all.csv ] && cat .github/sycl_cts/expect_fail_all.csv >> expect_fail.csv
-        [ -f .github/sycl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/sycl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv
-        cp expect_fail.csv disable.csv
-        [ -f  "$SYCL_CTS_FILTER" ] && cat "$SYCL_CTS_FILTER" >> disable.csv 
+
+        # Build override file, all is done first, then the target specific. The last file can overwrite prevous overrides.
+        for csv in .github/sycl_cts/override_all.csv ..github/sycl_cts/override_${{ inputs.target }}.csv
+        do
+           [ -f $csv ] && cat $csv
+        done > override.csv
+
+        echo override file:
+        cat override.csv
+
         python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \
           --color=always \
           --timeout $SYCL_CTS_TIMEOUT \
@@ -85,30 +91,11 @@ runs:
           -l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \
           -r SYCL-CTS/cts.xml \
           -v \
-          -i disable.csv || exitcode=$?
+          -o override.csv \
+          $SYCL_CTS_FILTER
+
         export OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so
         $GITHUB_WORKSPACE/.github/scripts/create_sycl_cts_test_lists.sh $PREPEND_PATH SYCL-CTS $CTS_CSV_FILE csv.txt cts_all.txt
         # output a diff of the generated list csv.txt and cts_all.txt
         diff csv.txt cts_all.txt || echo "WARNING - Missing some tests from sycl cts file based on test_all --list-tests - see > above"
         exit $exitcode
-
-    - name: run sycl cts expected fails
-      shell: bash
-      env:
-        PREPEND_PATH: ''  # TODO: have qemu as input and set up this
-        SYCL_CTS_TIMEOUT: '02:00:00'
-      run: |
-        echo running sycl cts
-        export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib
-        export ONEAPI_DEVICE_SELECTOR=opencl:0
-        python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \
-          --color=always \
-          --timeout $SYCL_CTS_TIMEOUT \
-          $PREPEND_PATH \
-          -p sycl_cts \
-          -b SYCL-CTS/bin \
-          -L SYCL-CTS/lib \
-          -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
-          -l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \
-          -r SYCL-CTS/cts.xml \
-          -s expect_fail.csv || echo failed as expected

.github/opencl_cts/override_all.csv

@@ -0,0 +1 @@
+API,api/test_api,Xfail

.github/opencl_cts/override_host_riscv64_linux.csv

@@ -0,0 +1 @@
+Math,math_brute_force/test_bruteforce -w,Xfail

.github/sycl_cts/override_all.csv

@@ -0,0 +1,3 @@
+SYCL_CTS,test_math_builtin_api "math_builtin_float_base_*",Xfail
+SYCL_CTS,test_math_builtin_api "math_builtin_float_double_*",Xfail
+SYCL_CTS,test_event "event::wait does not report asynchronous errors",Mayfail

doc/scripts/city_runner.rst

@@ -612,6 +612,16 @@ Other Options
   test list. Tests marked as ``Ignored`` are not run and not counted.
   This is only supported for the CTS and GTest profiles at the moment.
 
+``--override-source``, ``-o``
+  A csv file containing a list of tests which will override tests in the test
+  csv provided by `-s`. These will override if they match the first two values
+  of an entry in the csv file. This is useful for updating the attribute or pool
+  values (the 3rd and 4th optional values). For example if we have an
+  expected fail, we can add an `Xfail` element.
+
+  Note this works in order so if there are more than one line in the override
+  matching the test csv file it will choose the last one.
+
 CSV File Format
 ---------------
 
@@ -654,6 +664,16 @@ Unimplemented
   associated with tests which have been left unimplemented by the CTS, or test
   things that aren't in CL 1.x.
 
+Xfail
+  Tests marked with the ``Xfail`` attribute are run and expected to fail. Fails
+  are counted in the pass rate and showed up in number of expected fails. It will only
+  count towards a failing exit code if it unexpectedly passes.
+Mayfail
+  Tests marked with the ``Mayfail`` attribute are run and are allowed to fail.
+  This should usually be reserved for those tests that intermittently fail.
+  Fails are counted in the pass rate and show up in may fails. It will
+  not count towards a failing exit code.
+
 .. _pools:
 
 Pools

scripts/testing/city_runner/profile.py

@@ -121,6 +121,11 @@ def add_options(self, parser):
             "--ignored-source",
             default="",
             help="File containing a list of ignored tests to skip.")
+        parser.add_argument(
+            "-o",
+            "--override-source",
+            default="",
+            help="File containing a list of tests which if match the first 2 columns will override, in order.")
         parser.add_argument(
             "-l", "--log-file", type=str, help="File to log test output to")
         parser.add_argument(
@@ -225,16 +230,18 @@ def create_run(self, test, worker_state=None):
         """ Create a new test run from a test description.  """
         raise NotImplementedError()
 
-    def load_tests(self, csv_paths, disabled_path, ignored_path):
+    def load_tests(self, csv_paths, disabled_path, ignored_path, override_path):
         """ Create the list of tests to run from a CSV. """
         if not csv_paths or any(not csv_path or not os.path.exists(csv_path) for csv_path in csv_paths):
             raise Exception("Test list file not specified or does not exist")
         if disabled_path and not os.path.exists(disabled_path):
             raise Exception("Disabled test list file does not exist")
         if ignored_path and not os.path.exists(ignored_path):
             raise Exception("Ignored test list file does not exist")
+        if override_path and not os.path.exists(override_path):
+            raise Exception("Override test list file does not exist")            
         tests = (TestList
-                 .from_file(csv_paths, disabled_path, ignored_path, self.args.test_prefix)
+                 .from_file(csv_paths, disabled_path, ignored_path, override_path, self.args.test_prefix)
                  .filter(self.args.patterns))
         return tests
 

scripts/testing/city_runner/profiles/basic.py

@@ -100,14 +100,17 @@ def build_environment_vars(self):
 
         return env
 
-    def load_tests(self, csv_paths, disabled_path, ignored_path):
+    def load_tests(self, csv_paths, disabled_path, ignored_path, override_path):
         """ Find the list of tests from CSV. """
         if disabled_path:
             print("Warning: disabled list not supported for basic profile")
 
         if ignored_path:
             print("Warning: ignored list not supported for basic profile")
 
+        if override_path:
+            print("Warning: override list not supported for basic profile")
+
         parsed_tests = []
         # Load tests from CSV if any were provided
         if csv_paths:

scripts/testing/city_runner/profiles/gtest.py

@@ -195,10 +195,12 @@ def parse_gtest_csv(self, csv_file):
 
         return test_names
 
-    def load_tests(self, csv_paths, disabled_path, ignored_path):
+    def load_tests(self, csv_paths, disabled_path, ignored_path, override_path):
         """ Find the list of tests from CSV or fallback to gtest binary. """
         if disabled_path:
             print("Warning: disabled list not supported for gtest profile")
+        if override_path:
+            print("Warning: override list not supported for gtest profile")            
         executable = TestExecutable(self.args.binary_name,
                                     self.args.binary_name)
 

scripts/testing/city_runner/profiles/tensorflow.py

@@ -118,7 +118,7 @@ def parse_options(self, argv):
 
         return args
 
-    def load_tests(self, csv_paths, disabled_path, ignored_path):
+    def load_tests(self, csv_paths, disabled_path, ignored_path, override_path):
         """ 
         Find the list of tests from CSV or fallback to Tensorflow binary.
         """
@@ -128,6 +128,9 @@ def load_tests(self, csv_paths, disabled_path, ignored_path):
         if ignored_path:
             print("Warning: ignored list not supported for tensorflow profile")
 
+        if override_path:
+            print("Warning: override list not supported for tensorflow profile")
+
         # Find path to Tensorflow executable
         Tensorflow_exe_path = os.path.abspath(self.args.binary_path)
         Tensorflow_dir = os.path.dirname(Tensorflow_exe_path)

scripts/testing/city_runner/runner.py

@@ -121,9 +121,11 @@ def execute(self):
         test_source = self.args.test_source  # CSV
         disabled_source = self.args.disabled_source # Disabled CSV
         ignored_source = self.args.ignored_source # Ignored CSV
+        override_source = self.args.override_source # Override CSV        
         self.tests = self.profile.load_tests(test_source,
                                              disabled_source,
-                                             ignored_source)
+                                             ignored_source,
+                                             override_source)
         if self.args.repeat > 1:
             self.tests *= self.args.repeat
         self.num_tests = len(self.tests)
@@ -260,8 +262,11 @@ def process_results(self):
         # Return the city runner exit code.
         if self.aborted:
             return 130
-        if self.results.num_fails and not self.args.relaxed:
-            return 1
+        if not self.args.relaxed:
+            if self.results.num_fails > 0:
+                return 1
+            if self.results.num_xfail_unexpectedly_passed > 0:
+                return 1
         return 0
 
     def process_output(self, run):