Warn on potential data races

dace/config_schema.yml

@@ -904,6 +904,7 @@ required:
             be called before every compiled SDFG's generated code is invoked. Used
             for functionality such as low-level profiling.
 
+
     #############################################
     # Experimental features
 
@@ -919,6 +920,12 @@ required:
                 description: >
                     Check for undefined symbols in memlets during SDFG validation.
 
+            check_race_conditions:
+                type: bool
+                default: false
+                title: Check race conditions
+                description: Check for probable race conditions during validation.
+
     #############################################
     # Features for unit testing
 

dace/sdfg/validation.py

@@ -1,12 +1,14 @@
 # Copyright 2019-2021 ETH Zurich and the DaCe authors. All rights reserved.
 """ Exception classes and methods for validation of SDFGs. """
+from collections import defaultdict
 import copy
 from dace.dtypes import DebugInfo
 import os
 from typing import TYPE_CHECKING, Dict, List, Set
 import warnings
 from dace import dtypes, subsets
 from dace import symbolic
+import networkx as nx
 
 if TYPE_CHECKING:
     import dace
@@ -790,6 +792,32 @@ def validate_state(state: 'dace.sdfg.SDFGState',
                     continue
                 raise error
 
+    if Config.get_bool("experimental.check_race_conditions"):
+        node_labels = []
+        write_accesses = defaultdict(list)
+        read_accesses = defaultdict(list)
+        for node in state.data_nodes():
+            node_labels.append(node.label)
+            write_accesses[node.label].extend([{"subset": e.data.dst_subset, "node": node, "wcr": e.data.wcr} for e in state.in_edges(node)])
+            read_accesses[node.label].extend([{"subset": e.data.src_subset, "node": node} for e in state.out_edges(node)])
+
+        for node_label in node_labels:
+            writes = write_accesses[node_label]
+            reads = read_accesses[node_label]
+            # check write-write data races
+            for i in range(len(writes)):
+                for j in range(i+1, len(writes)):
+                    if (writes[i]["node"] == writes[j]["node"] or not nx.has_path(state.nx, writes[i]["node"], writes[j]["node"])) and \
+                        writes[i]["wcr"] is None and writes[j]["wcr"] is None and \
+                        subsets.intersects(writes[i]["subset"], writes[j]["subset"]) is True:
+                        warnings.warn(f'Memlet range overlap while writing to "{node}" in state "{state.label}"')
+            # check read-write data races
+            for write in writes:
+                for read in reads:
+                    if not nx.has_path(state.nx, read["node"], write["node"]) and \
+                        subsets.intersects(write["subset"], read["subset"]) is True:
+                        warnings.warn(f'Memlet range overlap while writing to "{node}" in state "{state.label}"')
+
     ########################################
 
 

tests/warn_on_potential_data_race_test.py

@@ -0,0 +1,317 @@
+# Copyright 2019-2024 ETH Zurich and the DaCe authors. All rights reserved.
+
+import warnings
+import dace
+import pytest
+
+
+def test_memlet_range_not_overlap_ranges():
+    sdfg = dace.SDFG('memlet_range_not_overlap_ranges')
+    state = sdfg.add_state()
+    N = dace.symbol("N", dtype=dace.int32)
+    sdfg.add_array("A", (N//2,), dace.int32)
+    A = state.add_access("A")
+    sdfg.add_array("B", (N,), dace.int32)
+    B = state.add_access("B")
+    state.add_mapped_tasklet(
+        name="first_tasklet",
+        code="b = a + 10",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="k")},
+        map_ranges={"k": "0:N//2"},
+        external_edges=True,
+        input_nodes={"A": A},
+        output_nodes={"B": B}
+    )
+    state.add_mapped_tasklet(
+        name="second_tasklet",
+        code="b = a - 20",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="k+N//2")},
+        map_ranges={"k": "0:N//2"},
+        external_edges=True,
+        input_nodes={"A": A},
+        output_nodes={"B": B}
+    )
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        with dace.config.set_temporary("experimental.check_race_conditions", value=True):
+            sdfg.validate()
+
+
+def test_memlet_range_write_write_overlap_ranges():
+    sdfg = dace.SDFG('memlet_range_overlap_ranges')
+    state = sdfg.add_state()
+    N = dace.symbol("N", dtype=dace.int32)
+    sdfg.add_array("A", (N,), dace.int32)
+    A = state.add_access("A")
+    sdfg.add_array("B", (N,), dace.int32)
+    B = state.add_access("B")
+    state.add_mapped_tasklet(
+        name="first_tasklet",
+        code="b = a + 10",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="k")},
+        map_ranges={"k": "0:N"},
+        external_edges=True,
+        input_nodes={"A": A},
+        output_nodes={"B": B}
+    )
+    state.add_mapped_tasklet(
+        name="second_tasklet",
+        code="b = a - 20",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="k")},
+        map_ranges={"k": "0:N"},
+        external_edges=True,
+        input_nodes={"A": A},
+        output_nodes={"B": B}
+    )
+
+    with pytest.warns(UserWarning):
+        with dace.config.set_temporary("experimental.check_race_conditions", value=True):
+            sdfg.validate()
+
+def test_memlet_range_write_read_overlap_ranges():
+    sdfg = dace.SDFG('memlet_range_write_read_overlap_ranges')
+    state = sdfg.add_state()
+    N = dace.symbol("N", dtype=dace.int32)
+    sdfg.add_array("A", (N,), dace.int32)
+    A_read = state.add_read("A")
+    A_write = state.add_write("A")
+    sdfg.add_array("B", (N,), dace.int32)
+    B = state.add_access("B")
+    sdfg.add_array("C", (N,), dace.int32)
+    C = state.add_access("C")
+    state.add_mapped_tasklet(
+        name="first_tasklet",
+        code="b = a + 10",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="k")},
+        map_ranges={"k": "0:N"},
+        external_edges=True,
+        input_nodes={"A": A_read},
+        output_nodes={"B": B}
+    )
+    state.add_mapped_tasklet(
+        name="second_tasklet",
+        code="a = c - 20",
+        inputs={"c": dace.Memlet(data="C", subset="k")},
+        outputs={"a": dace.Memlet(data="A", subset="k")},
+        map_ranges={"k": "0:N"},
+        external_edges=True,
+        input_nodes={"C": C},
+        output_nodes={"A": A_write}
+    )
+
+    with pytest.warns(UserWarning):
+        with dace.config.set_temporary('experimental', 'check_race_conditions', value=True):
+            sdfg.validate()
+
+def test_memlet_overlap_ranges_two_access_nodes():
+    sdfg = dace.SDFG('memlet_range_write_read_overlap_ranges')
+    state = sdfg.add_state()
+    N = dace.symbol("N", dtype=dace.int32)
+    sdfg.add_array("A", (N,), dace.int32)
+    A1 = state.add_access("A")
+    A2 = state.add_access("A")
+    sdfg.add_array("B", (N,), dace.int32)
+    B1 = state.add_access("B")
+    B2 = state.add_access("B")
+
+    state.add_mapped_tasklet(
+        name="first_tasklet",
+        code="b = a + 10",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="k")},
+        map_ranges={"k": "0:N"},
+        external_edges=True,
+        input_nodes={"A": A1},
+        output_nodes={"B": B1}
+    )
+    state.add_mapped_tasklet(
+        name="second_tasklet",
+        code="b = a - 20",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="k")},
+        map_ranges={"k": "0:N"},
+        external_edges=True,
+        input_nodes={"A": A2},
+        output_nodes={"B": B2}
+    )
+
+    with pytest.warns(UserWarning):
+        with dace.config.set_temporary('experimental', 'check_race_conditions', value=True):
+            sdfg.validate()
+
+def test_memlet_overlap_symbolic_ranges():
+    sdfg = dace.SDFG('memlet_overlap_symbolic_ranges')
+    state = sdfg.add_state()
+    N = dace.symbol("N", dtype=dace.int32)
+    sdfg.add_array("A", (2*N,), dace.int32)
+    A = state.add_access("A")
+    sdfg.add_array("B", (2*N,), dace.int32)
+    B = state.add_access("B")
+
+    state.add_mapped_tasklet(
+        name="first_tasklet",
+        code="b = a + 10",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="k")},
+        map_ranges={"k": "0:N"},
+        external_edges=True,
+        input_nodes={"A": A},
+        output_nodes={"B": B}
+    )
+    state.add_mapped_tasklet(
+        name="second_tasklet",
+        code="b = a - 20",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="k")},
+        map_ranges={"k": "0:2*N"},
+        external_edges=True,
+        input_nodes={"A": A},
+        output_nodes={"B": B}
+    )
+
+    with pytest.warns(UserWarning):
+        with dace.config.set_temporary('experimental', 'check_race_conditions', value=True):
+            sdfg.validate()
+
+def test_constant_memlet_overlap():
+    sdfg = dace.SDFG('constant_memlet_overlap')
+    state = sdfg.add_state()
+    sdfg.add_array("A", (12,), dace.int32)
+    A = state.add_access("A")
+    sdfg.add_array("B", (12,), dace.int32)
+    B = state.add_access("B")
+
+    state.add_mapped_tasklet(
+        name="first_tasklet",
+        code="b = a + 10",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="k")},
+        map_ranges={"k": "3:10"},
+        external_edges=True,
+        input_nodes={"A": A},
+        output_nodes={"B": B}
+    )
+    state.add_mapped_tasklet(
+        name="second_tasklet",
+        code="b = a - 20",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="k")},
+        map_ranges={"k": "6:12"},
+        external_edges=True,
+        input_nodes={"A": A},
+        output_nodes={"B": B}
+    )
+
+    with pytest.warns(UserWarning):
+        with dace.config.set_temporary('experimental', 'check_race_conditions', value=True):
+            sdfg.validate()
+
+def test_constant_memlet_almost_overlap():
+    sdfg = dace.SDFG('constant_memlet_almost_overlap')
+    state = sdfg.add_state()
+    sdfg.add_array("A", (20,), dace.int32)
+    A = state.add_access("A")
+    sdfg.add_array("B", (20,), dace.int32)
+    B = state.add_access("B")
+
+    state.add_mapped_tasklet(
+        name="first_tasklet",
+        code="b = a + 10",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="k")},
+        map_ranges={"k": "3:10"},
+        external_edges=True,
+        input_nodes={"A": A},
+        output_nodes={"B": B}
+    )
+    state.add_mapped_tasklet(
+        name="second_tasklet",
+        code="b = a - 20",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="k")},
+        map_ranges={"k": "10:20"},
+        external_edges=True,
+        input_nodes={"A": A},
+        output_nodes={"B": B}
+    )
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        with dace.config.set_temporary('experimental', 'check_race_conditions', value=True):
+            sdfg.validate()
+
+def test_elementwise_map():
+    sdfg = dace.SDFG('elementwise_map')
+    state = sdfg.add_state()
+    sdfg.add_array("A", (20,), dace.int32)
+    A_read = state.add_read("A")
+    A_write = state.add_write("A")
+
+    state.add_mapped_tasklet(
+        name="first_tasklet",
+        code="aa = a + 10",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"aa": dace.Memlet(data="A", subset="k")},
+        map_ranges={"k": "0:20"},
+        external_edges=True,
+        input_nodes={"A": A_read},
+        output_nodes={"A": A_write}
+    )
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        with dace.config.set_temporary('experimental', 'check_race_conditions', value=True):
+            sdfg.validate()
+
+def test_memlet_overlap_with_wcr():
+    sdfg = dace.SDFG('memlet_overlap_with_wcr')
+    state = sdfg.add_state()
+    sdfg.add_array("A", (20,), dace.int32)
+    sdfg.add_array("B", (1,), dace.int32)
+    A = state.add_read("A")
+    B = state.add_write("B")
+
+    state.add_mapped_tasklet(
+        name="first_reduction",
+        code="b = a",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="0", wcr="lambda old, new: old + new")},
+        map_ranges={"k": "0:20"},
+        external_edges=True,
+        input_nodes={"A": A},
+        output_nodes={"B": B}
+    )
+
+    state.add_mapped_tasklet(
+        name="second_reduction",
+        code="b = a",
+        inputs={"a": dace.Memlet(data="A", subset="k")},
+        outputs={"b": dace.Memlet(data="B", subset="0", wcr="lambda old, new: old + new")},
+        map_ranges={"k": "0:20"},
+        external_edges=True,
+        input_nodes={"A": A},
+        output_nodes={"B": B}
+    )
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        with dace.config.set_temporary('experimental', 'check_race_conditions', value=True):
+            sdfg.validate()
+
+
+if __name__ == '__main__':
+    test_memlet_range_not_overlap_ranges()
+    test_memlet_range_write_write_overlap_ranges()
+    test_memlet_range_write_read_overlap_ranges()
+    test_memlet_overlap_ranges_two_access_nodes()
+    test_memlet_overlap_symbolic_ranges()
+    test_constant_memlet_overlap()
+    test_constant_memlet_almost_overlap()
+    test_elementwise_map()
+    test_memlet_overlap_with_wcr()