From 869ae6006bc1d4d1d69d0a0ef5db658a2ea38a2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20Gonz=C3=A1lez=20Paredes?= Date: Wed, 4 Oct 2023 15:57:35 +0200 Subject: [PATCH] Fixes and additions to test exclusion matrices functionality. (#1345) Fixes and additions to test exclusion matrices. Changes: - Fix import path of exclusion matrices. - Fix wrong locations of docstrings. - Remove deprecated fixtures. - Add missing marker to parametrize forgotten custom case. --- .../ADRs/0015-Test_Exclusion_Matrices.md | 2 +- tests/next_tests/__init__.py | 5 ++++ tests/next_tests/exclusion_matrices.py | 26 +++++++++++-------- .../ffront_tests/ffront_test_utils.py | 12 ++++++--- .../test_with_toy_connectivity.py | 20 +++++--------- tests/next_tests/unit_tests/conftest.py | 23 +++++++--------- 6 files changed, 45 insertions(+), 43 deletions(-) diff --git a/docs/development/ADRs/0015-Test_Exclusion_Matrices.md b/docs/development/ADRs/0015-Test_Exclusion_Matrices.md index 920504db9a..6c6a043560 100644 --- a/docs/development/ADRs/0015-Test_Exclusion_Matrices.md +++ b/docs/development/ADRs/0015-Test_Exclusion_Matrices.md @@ -43,7 +43,7 @@ The test-exclusion matrix is a dictionary, where `key` is the backend name and e `(, , )` The backend string, used both as dictionary key and as string formatter in the skip message, is retrieved -by calling `tests.next_tests.get_processor_id()`, which returns the so-called processor name. +by calling `next_tests.get_processor_id()`, which returns the so-called processor name. The following backend processors are defined: ```python diff --git a/tests/next_tests/__init__.py b/tests/next_tests/__init__.py index bd9b968948..54bc4d9c69 100644 --- a/tests/next_tests/__init__.py +++ b/tests/next_tests/__init__.py @@ -12,6 +12,11 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +from . import exclusion_matrices + + +__all__ = ["exclusion_matrices", "get_processor_id"] + def get_processor_id(processor): if hasattr(processor, "__module__") and hasattr(processor, "__name__"): diff --git a/tests/next_tests/exclusion_matrices.py b/tests/next_tests/exclusion_matrices.py index d0a44080ad..27ccb29095 100644 --- a/tests/next_tests/exclusion_matrices.py +++ b/tests/next_tests/exclusion_matrices.py @@ -14,23 +14,18 @@ import pytest -""" -Contains definition of test-exclusion matrices, see ADR 15. -""" +"""Contains definition of test-exclusion matrices, see ADR 15.""" # Skip definitions XFAIL = pytest.xfail SKIP = pytest.skip -# Skip messages (available format keys: 'marker', 'backend') -UNSUPPORTED_MESSAGE = "'{marker}' tests not supported by '{backend}' backend" -BINDINGS_UNSUPPORTED_MESSAGE = "'{marker}' not supported by '{backend}' bindings" - # Processor ids as returned by next_tests.get_processor_id() DACE = "dace_iterator.run_dace_iterator" GTFN_CPU = "otf_compile_executor.run_gtfn" GTFN_CPU_IMPERATIVE = "otf_compile_executor.run_gtfn_imperative" GTFN_CPU_WITH_TEMPORARIES = "otf_compile_executor.run_gtfn_with_temporaries" +GTFN_FORMAT_SOURCECODE = "gtfn.format_sourcecode" # Test markers REQUIRES_ATLAS = "requires_atlas" @@ -46,25 +41,31 @@ USES_REDUCTION_OVER_LIFT_EXPRESSIONS = "uses_reduction_over_lift_expressions" USES_SCAN_IN_FIELD_OPERATOR = "uses_scan_in_field_operator" USES_SPARSE_FIELDS = "uses_sparse_fields" +USES_REDUCTION_WITH_ONLY_SPARSE_FIELDS = "uses_reduction_with_only_sparse_fields" USES_STRIDED_NEIGHBOR_OFFSET = "uses_strided_neighbor_offset" USES_TUPLE_ARGS = "uses_tuple_args" USES_TUPLE_RETURNS = "uses_tuple_returns" USES_ZERO_DIMENSIONAL_FIELDS = "uses_zero_dimensional_fields" +# Skip messages (available format keys: 'marker', 'backend') +UNSUPPORTED_MESSAGE = "'{marker}' tests not supported by '{backend}' backend" +BINDINGS_UNSUPPORTED_MESSAGE = "'{marker}' not supported by '{backend}' bindings" +REDUCTION_WITH_ONLY_SPARSE_FIELDS_MESSAGE = ( + "We cannot unroll a reduction on a sparse field only (not clear if it is legal ITIR)" +) # Common list of feature markers to skip GTFN_SKIP_TEST_LIST = [ (REQUIRES_ATLAS, XFAIL, BINDINGS_UNSUPPORTED_MESSAGE), (USES_APPLIED_SHIFTS, XFAIL, UNSUPPORTED_MESSAGE), (USES_IF_STMTS, XFAIL, UNSUPPORTED_MESSAGE), (USES_NEGATIVE_MODULO, XFAIL, UNSUPPORTED_MESSAGE), + (USES_REDUCTION_WITH_ONLY_SPARSE_FIELDS, XFAIL, REDUCTION_WITH_ONLY_SPARSE_FIELDS_MESSAGE), (USES_SCAN_IN_FIELD_OPERATOR, XFAIL, UNSUPPORTED_MESSAGE), (USES_STRIDED_NEIGHBOR_OFFSET, XFAIL, BINDINGS_UNSUPPORTED_MESSAGE), ] -""" -Skip matrix, contains for each backend processor a list of tuples with following fields: -(, ) -""" +#: Skip matrix, contains for each backend processor a list of tuples with following fields: +#: (, ) BACKEND_SKIP_TEST_MATRIX = { DACE: GTFN_SKIP_TEST_LIST + [ @@ -86,4 +87,7 @@ + [ (USES_DYNAMIC_OFFSETS, XFAIL, UNSUPPORTED_MESSAGE), ], + GTFN_FORMAT_SOURCECODE: [ + (USES_REDUCTION_WITH_ONLY_SPARSE_FIELDS, XFAIL, REDUCTION_WITH_ONLY_SPARSE_FIELDS_MESSAGE), + ], } diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py index d3863f5a28..383716484e 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py @@ -23,7 +23,6 @@ from gt4py.next.ffront import decorator from gt4py.next.iterator import embedded, ir as itir from gt4py.next.program_processors.runners import gtfn_cpu, roundtrip -from tests.next_tests import exclusion_matrices try: @@ -58,11 +57,18 @@ def no_backend(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> Non ids=lambda p: next_tests.get_processor_id(p), ) def fieldview_backend(request): + """ + Fixture creating field-view operator backend on-demand for tests. + + Notes: + Check ADR 15 for details on the test-exclusion matrices. + """ backend = request.param backend_id = next_tests.get_processor_id(backend) - """See ADR 15.""" - for marker, skip_mark, msg in exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get(backend_id, []): + for marker, skip_mark, msg in next_tests.exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get( + backend_id, [] + ): if request.node.get_closest_marker(marker): skip_mark(msg.format(marker=marker, backend=backend_id)) diff --git a/tests/next_tests/integration_tests/multi_feature_tests/iterator_tests/test_with_toy_connectivity.py b/tests/next_tests/integration_tests/multi_feature_tests/iterator_tests/test_with_toy_connectivity.py index 27c9f6d124..92b93ddb63 100644 --- a/tests/next_tests/integration_tests/multi_feature_tests/iterator_tests/test_with_toy_connectivity.py +++ b/tests/next_tests/integration_tests/multi_feature_tests/iterator_tests/test_with_toy_connectivity.py @@ -50,12 +50,7 @@ v2e_arr, v2v_arr, ) -from next_tests.unit_tests.conftest import ( - lift_mode, - program_processor, - program_processor_no_gtfn_exec, - run_processor, -) +from next_tests.unit_tests.conftest import lift_mode, program_processor, run_processor def edge_index_field(): # TODO replace by gtx.index_field once supported in bindings @@ -138,8 +133,8 @@ def map_make_const_list(in_edges): @pytest.mark.uses_constant_fields -def test_map_make_const_list(program_processor_no_gtfn_exec, lift_mode): - program_processor, validate = program_processor_no_gtfn_exec +def test_map_make_const_list(program_processor, lift_mode): + program_processor, validate = program_processor inp = edge_index_field() out = gtx.np_as_located_field(Vertex)(np.zeros([9], inp.dtype)) ref = 2 * np.sum(v2e_arr, axis=1) @@ -462,12 +457,9 @@ def sum_(a, b): @pytest.mark.uses_sparse_fields -def test_sparse_shifted_stencil_reduce(program_processor_no_gtfn_exec, lift_mode): - program_processor, validate = program_processor_no_gtfn_exec - if program_processor == gtfn.format_sourcecode: - pytest.xfail("We cannot unroll a reduction on a sparse field only.") - # With our current understanding, this iterator IR program is illegal, however we might want to fix it and therefore keep the test for now. - +@pytest.mark.uses_reduction_with_only_sparse_fields +def test_sparse_shifted_stencil_reduce(program_processor, lift_mode): + program_processor, validate = program_processor if lift_mode != transforms.LiftMode.FORCE_INLINE: pytest.xfail("shifted input arguments not supported for lift_mode != LiftMode.FORCE_INLINE") diff --git a/tests/next_tests/unit_tests/conftest.py b/tests/next_tests/unit_tests/conftest.py index 04c34dfaab..7a62778be1 100644 --- a/tests/next_tests/unit_tests/conftest.py +++ b/tests/next_tests/unit_tests/conftest.py @@ -24,7 +24,6 @@ from gt4py.next.program_processors import processor_interface as ppi from gt4py.next.program_processors.formatters import gtfn, lisp, type_check from gt4py.next.program_processors.runners import double_roundtrip, gtfn_cpu, roundtrip -from tests.next_tests import exclusion_matrices try: @@ -88,28 +87,24 @@ def pretty_format_and_check(root: itir.FencilDefinition, *args, **kwargs) -> str ids=lambda p: next_tests.get_processor_id(p[0]), ) def program_processor(request): + """ + Fixture creating program processors on-demand for tests. + + Notes: + Check ADR 15 for details on the test-exclusion matrices. + """ backend, _ = request.param backend_id = next_tests.get_processor_id(backend) - """See ADR 15.""" - for marker, skip_mark, msg in exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get(backend_id, []): + for marker, skip_mark, msg in next_tests.exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get( + backend_id, [] + ): if request.node.get_closest_marker(marker): skip_mark(msg.format(marker=marker, backend=backend_id)) return request.param -@pytest.fixture -def program_processor_no_gtfn_exec(program_processor): - if ( - program_processor[0] == gtfn_cpu.run_gtfn - or program_processor[0] == gtfn_cpu.run_gtfn_imperative - or program_processor[0] == gtfn_cpu.run_gtfn_with_temporaries - ): - pytest.xfail("gtfn backend not yet supported.") - return program_processor - - def run_processor( program: runtime.FendefDispatcher, processor: ppi.ProgramExecutor | ppi.ProgramFormatter,