test[next]: check for DaCe dependency in test execution (#1336)

Expanding the pytest fixture for unit tests with markers to exclude tests based on feature support in the selected backend. In addition, a check is added to the DaCe backend so that tests are skipped if dace module is not installed. This is required for Spack build of icon4py, which uses the base installation of gt4py, where dace module is optional.

docs/development/ADRs/0015-Test_Exclusion_Matrices.md

@@ -0,0 +1,80 @@
+---
+tags: []
+---
+
+# Test-Exclusion Matrices
+
+- **Status**: valid
+- **Authors**: Edoardo Paone (@edopao), Enrique G. Paredes (@egparedes)
+- **Created**: 2023-09-21
+- **Updated**: 2023-09-21
+
+In the context of Field View testing, lacking support for specific ITIR features while a certain backend
+is being developed, we decided to use `pytest` fixtures to exclude unsupported tests.
+
+## Context
+
+It should be possible to run Field View tests on different backends. However, specific tests could be unsupported
+on a certain backend, or the backend implementation could be only partially ready.
+Therefore, we need a mechanism to specify the features required by each test and selectively enable
+the supported backends, while keeping the test code clean.
+
+## Decision
+
+It was decided to apply fixtures and markers from `pytest` module. The fixture is the same used to execute the test
+on different backends (`fieldview_backend` and `program_processor`), but it is extended with a check on the available feature markers.
+If a test is annotated with a feature marker, the fixture will check if this feature is supported on the selected backend.
+If no marker is specified, the test is supposed to run on all backends.
+
+In the example below, `test_offset_field` requires the backend to support dynamic offsets in the translation from ITIR:
+
+```python
+@pytest.mark.uses_dynamic_offsets
+def test_offset_field(cartesian_case):
+```
+
+In order to selectively enable the backends, the dictionary `next_tests.exclusion_matrices.BACKEND_SKIP_TEST_MATRIX`
+lists for each backend the features that are not supported.
+The fixture will check if the annotated feature is present in the exclusion-matrix for the selected backend.
+If so, the exclusion matrix will also specify the action `pytest` should take (e.g. `SKIP` or `XFAIL`).
+
+The test-exclusion matrix is a dictionary, where `key` is the backend name and each entry is a tuple with the following fields:
+
+`(<marker[str]>, <skip_definition[SKIP,XFAIL]>, <skip_message(format keys: 'marker', 'backend')>)`
+
+The backend string, used both as dictionary key and as string formatter in the skip message, is retrieved
+by calling `tests.next_tests.get_processor_id()`, which returns the so-called processor name.
+The following backend processors are defined:
+
+```python
+DACE = "dace_iterator.run_dace_iterator"
+GTFN_CPU = "otf_compile_executor.run_gtfn"
+GTFN_CPU_IMPERATIVE = "otf_compile_executor.run_gtfn_imperative"
+GTFN_CPU_WITH_TEMPORARIES = "otf_compile_executor.run_gtfn_with_temporaries"
+```
+
+Following the previous example, the GTFN backend with temporaries does not support yet dynamic offsets in ITIR:
+
+```python
+BACKEND_SKIP_TEST_MATRIX = {
+    GTFN_CPU_WITH_TEMPORARIES: [
+        ("uses_dynamic_offsets", pytest.XFAIL, "'{marker}' tests not supported by '{backend}' backend"),
+    ]
+}
+```
+
+## Consequences
+
+Positive outcomes of this decision:
+
+- The solution provides a central place to specify test exclusion.
+- The test code remains clean from if-statements for backend exclusion.
+- The exclusion matrix gives an overview of the feature-readiness of different backends.
+
+Negative outcomes:
+
+- There is not (yet) any code-style check to enforce this solution, so code reviews should be aware of the ADR.
+
+## References <!-- optional -->
+
+- [pytest - Using markers to pass data to fixtures](https://docs.pytest.org/en/6.2.x/fixture.html#using-markers-to-pass-data-to-fixtures)

docs/development/ADRs/Index.md

@@ -51,9 +51,9 @@ _None_
 - [0011 - On The Fly Compilation](0011-On_The_Fly_Compilation.md)
 - [0012 - GridTools C++ OTF](0011-_GridTools_Cpp_OTF.md)
 
-### Miscellanea
+### Testing
 
-_None_
+- [0015 - Exclusion Matrices](0015-Test_Exclusion_Matrices.md)
 
 ### Superseded
 

pyproject.toml

@@ -326,9 +326,25 @@ module = 'gt4py.next.iterator.runtime'
 
 [tool.pytest.ini_options]
 markers = [
-  'requires_atlas',  # mark tests that require 'atlas4py' bindings package
-  'requires_dace',  # mark tests that require 'dace' package
-  'requires_gpu:'  # mark tests that require a NVidia GPU (assume 'cupy' and 'cudatoolkit' are installed)
+  'requires_atlas: tests that require `atlas4py` bindings package',
+  'requires_dace: tests that require `dace` package',
+  'requires_gpu: tests that require a NVidia GPU (`cupy` and `cudatoolkit` are required)',
+  'uses_applied_shifts: tests that require backend support for applied-shifts',
+  'uses_can_deref: tests that require backend support for can_deref',
+  'uses_constant_fields: tests that require backend support for constant fields',
+  'uses_dynamic_offsets: tests that require backend support for dynamic offsets',
+  'uses_if_stmts: tests that require backend support for if-statements',
+  'uses_index_fields: tests that require backend support for index fields',
+  'uses_lift_expressions: tests that require backend support for lift expressions',
+  'uses_negative_modulo: tests that require backend support for modulo on negative numbers',
+  'uses_origin: tests that require backend support for domain origin',
+  'uses_reduction_over_lift_expressions: tests that require backend support for reduction over lift expressions',
+  'uses_scan_in_field_operator: tests that require backend support for scan in field operator',
+  'uses_sparse_fields: tests that require backend support for sparse fields',
+  'uses_strided_neighbor_offset: tests that require backend support for strided neighbor offset',
+  'uses_tuple_args: tests that require backend support for tuple arguments',
+  'uses_tuple_returns: tests that require backend support for tuple results',
+  'uses_zero_dimensional_fields: tests that require backend support for zero-dimensional fields'
 ]
 norecursedirs = ['dist', 'build', 'cpp_backend_tests/build*', '_local/*', '.*']
 testpaths = 'tests'

tests/next_tests/exclusion_matrices.py

@@ -0,0 +1,89 @@
+# GT4Py - GridTools Framework
+#
+# Copyright (c) 2014-2023, ETH Zurich
+# All rights reserved.
+#
+# This file is part of the GT4Py project and the GridTools framework.
+# GT4Py is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or any later
+# version. See the LICENSE.txt file at the top-level directory of this
+# distribution for a copy of the license or check <https://www.gnu.org/licenses/>.
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+import pytest
+
+
+"""
+Contains definition of test-exclusion matrices, see ADR 15.
+"""
+
+# Skip definitions
+XFAIL = pytest.xfail
+SKIP = pytest.skip
+
+# Skip messages (available format keys: 'marker', 'backend')
+UNSUPPORTED_MESSAGE = "'{marker}' tests not supported by '{backend}' backend"
+BINDINGS_UNSUPPORTED_MESSAGE = "'{marker}' not supported by '{backend}' bindings"
+
+# Processor ids as returned by next_tests.get_processor_id()
+DACE = "dace_iterator.run_dace_iterator"
+GTFN_CPU = "otf_compile_executor.run_gtfn"
+GTFN_CPU_IMPERATIVE = "otf_compile_executor.run_gtfn_imperative"
+GTFN_CPU_WITH_TEMPORARIES = "otf_compile_executor.run_gtfn_with_temporaries"
+
+# Test markers
+REQUIRES_ATLAS = "requires_atlas"
+USES_APPLIED_SHIFTS = "uses_applied_shifts"
+USES_CAN_DEREF = "uses_can_deref"
+USES_CONSTANT_FIELDS = "uses_constant_fields"
+USES_DYNAMIC_OFFSETS = "uses_dynamic_offsets"
+USES_IF_STMTS = "uses_if_stmts"
+USES_INDEX_FIELDS = "uses_index_fields"
+USES_LIFT_EXPRESSIONS = "uses_lift_expressions"
+USES_NEGATIVE_MODULO = "uses_negative_modulo"
+USES_ORIGIN = "uses_origin"
+USES_REDUCTION_OVER_LIFT_EXPRESSIONS = "uses_reduction_over_lift_expressions"
+USES_SCAN_IN_FIELD_OPERATOR = "uses_scan_in_field_operator"
+USES_SPARSE_FIELDS = "uses_sparse_fields"
+USES_STRIDED_NEIGHBOR_OFFSET = "uses_strided_neighbor_offset"
+USES_TUPLE_ARGS = "uses_tuple_args"
+USES_TUPLE_RETURNS = "uses_tuple_returns"
+USES_ZERO_DIMENSIONAL_FIELDS = "uses_zero_dimensional_fields"
+
+# Common list of feature markers to skip
+GTFN_SKIP_TEST_LIST = [
+    (REQUIRES_ATLAS, XFAIL, BINDINGS_UNSUPPORTED_MESSAGE),
+    (USES_APPLIED_SHIFTS, XFAIL, UNSUPPORTED_MESSAGE),
+    (USES_IF_STMTS, XFAIL, UNSUPPORTED_MESSAGE),
+    (USES_NEGATIVE_MODULO, XFAIL, UNSUPPORTED_MESSAGE),
+    (USES_SCAN_IN_FIELD_OPERATOR, XFAIL, UNSUPPORTED_MESSAGE),
+    (USES_STRIDED_NEIGHBOR_OFFSET, XFAIL, BINDINGS_UNSUPPORTED_MESSAGE),
+]
+
+"""
+Skip matrix, contains for each backend processor a list of tuples with following fields:
+(<test_marker>, <skip_definition, <skip_message>)
+"""
+BACKEND_SKIP_TEST_MATRIX = {
+    DACE: GTFN_SKIP_TEST_LIST
+    + [
+        (USES_CAN_DEREF, XFAIL, UNSUPPORTED_MESSAGE),
+        (USES_CONSTANT_FIELDS, XFAIL, UNSUPPORTED_MESSAGE),
+        (USES_DYNAMIC_OFFSETS, XFAIL, UNSUPPORTED_MESSAGE),
+        (USES_INDEX_FIELDS, XFAIL, UNSUPPORTED_MESSAGE),
+        (USES_LIFT_EXPRESSIONS, XFAIL, UNSUPPORTED_MESSAGE),
+        (USES_ORIGIN, XFAIL, UNSUPPORTED_MESSAGE),
+        (USES_REDUCTION_OVER_LIFT_EXPRESSIONS, XFAIL, UNSUPPORTED_MESSAGE),
+        (USES_SPARSE_FIELDS, XFAIL, UNSUPPORTED_MESSAGE),
+        (USES_TUPLE_ARGS, XFAIL, UNSUPPORTED_MESSAGE),
+        (USES_TUPLE_RETURNS, XFAIL, UNSUPPORTED_MESSAGE),
+        (USES_ZERO_DIMENSIONAL_FIELDS, XFAIL, UNSUPPORTED_MESSAGE),
+    ],
+    GTFN_CPU: GTFN_SKIP_TEST_LIST,
+    GTFN_CPU_IMPERATIVE: GTFN_SKIP_TEST_LIST,
+    GTFN_CPU_WITH_TEMPORARIES: GTFN_SKIP_TEST_LIST
+    + [
+        (USES_DYNAMIC_OFFSETS, XFAIL, UNSUPPORTED_MESSAGE),
+    ],
+}

tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py

@@ -22,7 +22,17 @@
 import gt4py.next as gtx
 from gt4py.next.ffront import decorator
 from gt4py.next.iterator import embedded, ir as itir
-from gt4py.next.program_processors.runners import dace_iterator, gtfn_cpu, roundtrip
+from gt4py.next.program_processors.runners import gtfn_cpu, roundtrip
+from tests.next_tests import exclusion_matrices
+
+
+try:
+    from gt4py.next.program_processors.runners import dace_iterator
+except ModuleNotFoundError as e:
+    if "dace" in str(e):
+        dace_iterator = None
+    else:
+        raise e
 
 import next_tests
 
@@ -32,20 +42,33 @@ def no_backend(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> Non
     raise ValueError("No backend selected! Backend selection is mandatory in tests.")
 
 
+OPTIONAL_PROCESSORS = []
+if dace_iterator:
+    OPTIONAL_PROCESSORS.append(dace_iterator.run_dace_iterator)
+
+
 @pytest.fixture(
     params=[
         roundtrip.executor,
         gtfn_cpu.run_gtfn,
         gtfn_cpu.run_gtfn_imperative,
         gtfn_cpu.run_gtfn_with_temporaries,
-        dace_iterator.run_dace_iterator,
-    ],
+    ]
+    + OPTIONAL_PROCESSORS,
     ids=lambda p: next_tests.get_processor_id(p),
 )
 def fieldview_backend(request):
+    backend = request.param
+    backend_id = next_tests.get_processor_id(backend)
+
+    """See ADR 15."""
+    for marker, skip_mark, msg in exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get(backend_id, []):
+        if request.node.get_closest_marker(marker):
+            skip_mark(msg.format(marker=marker, backend=backend_id))
+
     backup_backend = decorator.DEFAULT_BACKEND
     decorator.DEFAULT_BACKEND = no_backend
-    yield request.param
+    yield backend
     decorator.DEFAULT_BACKEND = backup_backend
 
 

tests/next_tests/integration_tests/feature_tests/ffront_tests/test_arg_call_interface.py

@@ -24,7 +24,7 @@
 from gt4py.next.errors.exceptions import TypeError_
 from gt4py.next.ffront.decorator import field_operator, program, scan_operator
 from gt4py.next.ffront.fbuiltins import broadcast, int32, int64
-from gt4py.next.program_processors.runners import dace_iterator, gtfn_cpu
+from gt4py.next.program_processors.runners import gtfn_cpu
 
 from next_tests.integration_tests import cases
 from next_tests.integration_tests.cases import (
@@ -169,15 +169,8 @@ def testee(
     )
 
 
+@pytest.mark.uses_scan_in_field_operator
 def test_call_scan_operator_from_field_operator(cartesian_case):
-    if cartesian_case.backend in [
-        gtfn_cpu.run_gtfn,
-        gtfn_cpu.run_gtfn_imperative,
-        gtfn_cpu.run_gtfn_with_temporaries,
-        dace_iterator.run_dace_iterator,
-    ]:
-        pytest.xfail("Calling scan from field operator not fully supported.")
-
     @scan_operator(axis=KDim, forward=True, init=0.0)
     def testee_scan(state: float, x: float, y: float) -> float:
         return state + x + 2.0 * y