Fixes and additions to test exclusion matrices functionality. (#1345)

Fixes and additions to test exclusion matrices.

Changes:
- Fix import path of exclusion matrices.
- Fix wrong locations of docstrings.
- Remove deprecated fixtures.
- Add missing marker to parametrize forgotten custom case.

docs/development/ADRs/0015-Test_Exclusion_Matrices.md

@@ -43,7 +43,7 @@ The test-exclusion matrix is a dictionary, where `key` is the backend name and e
 `(<marker[str]>, <skip_definition[SKIP,XFAIL]>, <skip_message(format keys: 'marker', 'backend')>)`
 
 The backend string, used both as dictionary key and as string formatter in the skip message, is retrieved
-by calling `tests.next_tests.get_processor_id()`, which returns the so-called processor name.
+by calling `next_tests.get_processor_id()`, which returns the so-called processor name.
 The following backend processors are defined:
 
 ```python

tests/next_tests/__init__.py

@@ -12,6 +12,11 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+from . import exclusion_matrices
+
+
+__all__ = ["exclusion_matrices", "get_processor_id"]
+
 
 def get_processor_id(processor):
     if hasattr(processor, "__module__") and hasattr(processor, "__name__"):

tests/next_tests/exclusion_matrices.py

@@ -14,23 +14,18 @@
 import pytest
 
 
-"""
-Contains definition of test-exclusion matrices, see ADR 15.
-"""
+"""Contains definition of test-exclusion matrices, see ADR 15."""
 
 # Skip definitions
 XFAIL = pytest.xfail
 SKIP = pytest.skip
 
-# Skip messages (available format keys: 'marker', 'backend')
-UNSUPPORTED_MESSAGE = "'{marker}' tests not supported by '{backend}' backend"
-BINDINGS_UNSUPPORTED_MESSAGE = "'{marker}' not supported by '{backend}' bindings"
-
 # Processor ids as returned by next_tests.get_processor_id()
 DACE = "dace_iterator.run_dace_iterator"
 GTFN_CPU = "otf_compile_executor.run_gtfn"
 GTFN_CPU_IMPERATIVE = "otf_compile_executor.run_gtfn_imperative"
 GTFN_CPU_WITH_TEMPORARIES = "otf_compile_executor.run_gtfn_with_temporaries"
+GTFN_FORMAT_SOURCECODE = "gtfn.format_sourcecode"
 
 # Test markers
 REQUIRES_ATLAS = "requires_atlas"
@@ -46,25 +41,31 @@
 USES_REDUCTION_OVER_LIFT_EXPRESSIONS = "uses_reduction_over_lift_expressions"
 USES_SCAN_IN_FIELD_OPERATOR = "uses_scan_in_field_operator"
 USES_SPARSE_FIELDS = "uses_sparse_fields"
+USES_REDUCTION_WITH_ONLY_SPARSE_FIELDS = "uses_reduction_with_only_sparse_fields"
 USES_STRIDED_NEIGHBOR_OFFSET = "uses_strided_neighbor_offset"
 USES_TUPLE_ARGS = "uses_tuple_args"
 USES_TUPLE_RETURNS = "uses_tuple_returns"
 USES_ZERO_DIMENSIONAL_FIELDS = "uses_zero_dimensional_fields"
 
+# Skip messages (available format keys: 'marker', 'backend')
+UNSUPPORTED_MESSAGE = "'{marker}' tests not supported by '{backend}' backend"
+BINDINGS_UNSUPPORTED_MESSAGE = "'{marker}' not supported by '{backend}' bindings"
+REDUCTION_WITH_ONLY_SPARSE_FIELDS_MESSAGE = (
+    "We cannot unroll a reduction on a sparse field only (not clear if it is legal ITIR)"
+)
 # Common list of feature markers to skip
 GTFN_SKIP_TEST_LIST = [
     (REQUIRES_ATLAS, XFAIL, BINDINGS_UNSUPPORTED_MESSAGE),
     (USES_APPLIED_SHIFTS, XFAIL, UNSUPPORTED_MESSAGE),
     (USES_IF_STMTS, XFAIL, UNSUPPORTED_MESSAGE),
     (USES_NEGATIVE_MODULO, XFAIL, UNSUPPORTED_MESSAGE),
+    (USES_REDUCTION_WITH_ONLY_SPARSE_FIELDS, XFAIL, REDUCTION_WITH_ONLY_SPARSE_FIELDS_MESSAGE),
     (USES_SCAN_IN_FIELD_OPERATOR, XFAIL, UNSUPPORTED_MESSAGE),
     (USES_STRIDED_NEIGHBOR_OFFSET, XFAIL, BINDINGS_UNSUPPORTED_MESSAGE),
 ]
 
-"""
-Skip matrix, contains for each backend processor a list of tuples with following fields:
-(<test_marker>, <skip_definition, <skip_message>)
-"""
+#: Skip matrix, contains for each backend processor a list of tuples with following fields:
+#: (<test_marker>, <skip_definition, <skip_message>)
 BACKEND_SKIP_TEST_MATRIX = {
     DACE: GTFN_SKIP_TEST_LIST
     + [
@@ -86,4 +87,7 @@
     + [
         (USES_DYNAMIC_OFFSETS, XFAIL, UNSUPPORTED_MESSAGE),
     ],
+    GTFN_FORMAT_SOURCECODE: [
+        (USES_REDUCTION_WITH_ONLY_SPARSE_FIELDS, XFAIL, REDUCTION_WITH_ONLY_SPARSE_FIELDS_MESSAGE),
+    ],
 }

tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py

@@ -23,7 +23,6 @@
 from gt4py.next.ffront import decorator
 from gt4py.next.iterator import embedded, ir as itir
 from gt4py.next.program_processors.runners import gtfn_cpu, roundtrip
-from tests.next_tests import exclusion_matrices
 
 
 try:
@@ -58,11 +57,18 @@ def no_backend(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> Non
     ids=lambda p: next_tests.get_processor_id(p),
 )
 def fieldview_backend(request):
+    """
+    Fixture creating field-view operator backend on-demand for tests.
+
+    Notes:
+        Check ADR 15 for details on the test-exclusion matrices.
+    """
     backend = request.param
     backend_id = next_tests.get_processor_id(backend)
 
-    """See ADR 15."""
-    for marker, skip_mark, msg in exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get(backend_id, []):
+    for marker, skip_mark, msg in next_tests.exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get(
+        backend_id, []
+    ):
         if request.node.get_closest_marker(marker):
             skip_mark(msg.format(marker=marker, backend=backend_id))
 

tests/next_tests/integration_tests/multi_feature_tests/iterator_tests/test_with_toy_connectivity.py

@@ -50,12 +50,7 @@
     v2e_arr,
     v2v_arr,
 )
-from next_tests.unit_tests.conftest import (
-    lift_mode,
-    program_processor,
-    program_processor_no_gtfn_exec,
-    run_processor,
-)
+from next_tests.unit_tests.conftest import lift_mode, program_processor, run_processor
 
 
 def edge_index_field():  # TODO replace by gtx.index_field once supported in bindings
@@ -138,8 +133,8 @@ def map_make_const_list(in_edges):
 
 
 @pytest.mark.uses_constant_fields
-def test_map_make_const_list(program_processor_no_gtfn_exec, lift_mode):
-    program_processor, validate = program_processor_no_gtfn_exec
+def test_map_make_const_list(program_processor, lift_mode):
+    program_processor, validate = program_processor
     inp = edge_index_field()
     out = gtx.np_as_located_field(Vertex)(np.zeros([9], inp.dtype))
     ref = 2 * np.sum(v2e_arr, axis=1)
@@ -462,12 +457,9 @@ def sum_(a, b):
 
 
 @pytest.mark.uses_sparse_fields
-def test_sparse_shifted_stencil_reduce(program_processor_no_gtfn_exec, lift_mode):
-    program_processor, validate = program_processor_no_gtfn_exec
-    if program_processor == gtfn.format_sourcecode:
-        pytest.xfail("We cannot unroll a reduction on a sparse field only.")
-        # With our current understanding, this iterator IR program is illegal, however we might want to fix it and therefore keep the test for now.
-
+@pytest.mark.uses_reduction_with_only_sparse_fields
+def test_sparse_shifted_stencil_reduce(program_processor, lift_mode):
+    program_processor, validate = program_processor
     if lift_mode != transforms.LiftMode.FORCE_INLINE:
         pytest.xfail("shifted input arguments not supported for lift_mode != LiftMode.FORCE_INLINE")
 

tests/next_tests/unit_tests/conftest.py

@@ -24,7 +24,6 @@
 from gt4py.next.program_processors import processor_interface as ppi
 from gt4py.next.program_processors.formatters import gtfn, lisp, type_check
 from gt4py.next.program_processors.runners import double_roundtrip, gtfn_cpu, roundtrip
-from tests.next_tests import exclusion_matrices
 
 
 try:
@@ -88,28 +87,24 @@ def pretty_format_and_check(root: itir.FencilDefinition, *args, **kwargs) -> str
     ids=lambda p: next_tests.get_processor_id(p[0]),
 )
 def program_processor(request):
+    """
+    Fixture creating program processors on-demand for tests.
+
+    Notes:
+        Check ADR 15 for details on the test-exclusion matrices.
+    """
     backend, _ = request.param
     backend_id = next_tests.get_processor_id(backend)
 
-    """See ADR 15."""
-    for marker, skip_mark, msg in exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get(backend_id, []):
+    for marker, skip_mark, msg in next_tests.exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get(
+        backend_id, []
+    ):
         if request.node.get_closest_marker(marker):
             skip_mark(msg.format(marker=marker, backend=backend_id))
 
     return request.param
 
 
-@pytest.fixture
-def program_processor_no_gtfn_exec(program_processor):
-    if (
-        program_processor[0] == gtfn_cpu.run_gtfn
-        or program_processor[0] == gtfn_cpu.run_gtfn_imperative
-        or program_processor[0] == gtfn_cpu.run_gtfn_with_temporaries
-    ):
-        pytest.xfail("gtfn backend not yet supported.")
-    return program_processor
-
-
 def run_processor(
     program: runtime.FendefDispatcher,
     processor: ppi.ProgramExecutor | ppi.ProgramFormatter,