Merge remote-tracking branch 'gt4py/main' into updated_dacefieldview_…

…transformations

.devcontainer/.vscode/launch.json

@@ -0,0 +1,24 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Python: Current File (just my code)",
+      "type": "python",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "justMyCode": true
+    },
+    {
+      "name": "Python: Current File (all)",
+      "type": "python",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "justMyCode": false
+    }
+  ]
+}

.devcontainer/Dockerfile

@@ -0,0 +1,5 @@
+FROM mcr.microsoft.com/devcontainers/python:1-3.10-bookworm
+RUN apt-get update \
+    && export DEBIAN_FRONTEND=noninteractive && apt-get install -y libboost-dev \
+    && apt-get clean && rm -rf /var/cache/apt/* && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/*
+RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="/bin" sh

.devcontainer/devcontainer.json

@@ -0,0 +1,49 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+  "name": "Python 3",
+  // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+  "build": {
+    "dockerfile": "Dockerfile"
+  },
+
+  // Features to add to the dev container. More info: https://containers.dev/features.
+  // "features": {},
+
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  // "forwardPorts": [],
+
+  // Use 'postCreateCommand' to run commands after the container is created.
+  "postCreateCommand": "bash .devcontainer/setup.sh",
+
+  "containerEnv": {
+    "PRE_COMMIT_HOME": "/workspaces/gt4py/.caches/pre-commit"
+  },
+
+  // Configure tool-specific properties.
+  "customizations": {
+    // Configure properties specific to VS Code.
+    "vscode": {
+      // Set *default* container specific settings.json values on container create.
+      "settings": {
+        "python.formatting.provider": "ruff",
+        "python.testing.pytestEnabled": true,
+        "python.defaultInterpreterPath": "/workspaces/gt4py/.venv/bin/python",
+        "files.insertFinalNewline": true,
+        "python.terminal.activateEnvironment": true,
+        "cmake.ignoreCMakeListsMissing": true
+      },
+      "extensions": [
+        "charliermarsh.ruff",
+        "donjayamanne.githistory",
+        "github.vscode-github-actions",
+        "lextudio.restructuredtext",
+        "ms-python.python",
+        "ms-vsliveshare.vsliveshare",
+        "swyddfa.esbonio"
+      ]
+    }
+  }
+  // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+  // "remoteUser": "root"
+}

.devcontainer/setup.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+ln -sfn /workspaces/gt4py/.devcontainer/.vscode /workspaces/gt4py/.vscode
+uv venv .venv
+source .venv/bin/activate
+uv pip install -r requirements-dev.txt
+uv pip install -e .
+uv pip install -i https://test.pypi.org/simple/ atlas4py
+pre-commit install --install-hooks
+deactivate

.github/pull_request_template.md

@@ -15,7 +15,7 @@ Delete this comment and add a proper description of the changes contained in thi
         - test: Adding missing tests or correcting existing tests
 
     <scope>: cartesian | eve | next | storage
-    # ONLY if changes are limited to a specific subsytem
+    # ONLY if changes are limited to a specific subsystem
 
 - PR Description:
 
@@ -27,7 +27,7 @@ Delete this comment and add a proper description of the changes contained in thi
 ## Requirements
 
 - [ ] All fixes and/or new features come with corresponding tests.
-- [ ] Important design decisions have been documented in the approriate ADR inside the [docs/development/ADRs/](docs/development/ADRs/Index.md) folder.
+- [ ] Important design decisions have been documented in the appropriate ADR inside the [docs/development/ADRs/](docs/development/ADRs/Index.md) folder.
 
 If this PR contains code authored by new contributors please make sure:
 

.gitignore

@@ -159,5 +159,5 @@ venv.bak/
 
 ### Others ###
 .obsidian
-
 coverage.json
+.caches

.gitpod/.vscode/launch.json

@@ -6,19 +6,28 @@
   "configurations": [
     {
       "name": "Python: Current File (just my code)",
-      "type": "python",
+      "type": "debugpy",
       "request": "launch",
       "program": "${file}",
       "console": "integratedTerminal",
       "justMyCode": true
     },
     {
       "name": "Python: Current File (all)",
-      "type": "python",
+      "type": "debugpy",
       "request": "launch",
       "program": "${file}",
       "console": "integratedTerminal",
       "justMyCode": false
+    },
+    {
+      "name": "Python: Debug Tests",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "purpose": ["debug-test"],
+      "console": "integratedTerminal",
+      "justMyCode": true
     }
   ]
 }

.pre-commit-config.yaml

@@ -102,6 +102,7 @@ repos:
     - devtools==0.12.2
     - diskcache==5.6.3
     - factory-boy==3.3.1
+    - filelock==3.16.1
     - frozendict==2.4.6
     - gridtools-cpp==2.3.8
     - importlib-resources==6.4.5

constraints.txt

@@ -49,7 +49,7 @@ executing==2.1.0          # via devtools, stack-data
 factory-boy==3.3.1        # via gt4py (pyproject.toml), pytest-factoryboy
 faker==33.0.0             # via factory-boy
 fastjsonschema==2.20.0    # via nbformat
-filelock==3.16.1          # via tox, virtualenv
+filelock==3.16.1          # via gt4py (pyproject.toml), tox, virtualenv
 fonttools==4.55.0         # via matplotlib
 fparser==0.1.4            # via dace
 frozendict==2.4.6         # via gt4py (pyproject.toml)
@@ -113,8 +113,8 @@ psutil==6.1.0             # via -r requirements-dev.in, ipykernel, pytest-xdist
 ptyprocess==0.7.0         # via pexpect
 pure-eval==0.2.3          # via stack-data
 pybind11==2.13.6          # via gt4py (pyproject.toml)
-pydantic==2.9.2           # via bump-my-version, pydantic-settings
-pydantic-core==2.23.4     # via pydantic
+pydantic==2.10.0          # via bump-my-version, pydantic-settings
+pydantic-core==2.27.0     # via pydantic
 pydantic-settings==2.6.1  # via bump-my-version
 pydot==3.0.2              # via tach
 pygments==2.18.0          # via -r requirements-dev.in, devtools, ipython, nbmake, rich, sphinx
@@ -159,7 +159,7 @@ stack-data==0.6.3         # via ipython
 stdlib-list==0.10.0       # via tach
 sympy==1.13.3             # via dace
 tabulate==0.9.0           # via gt4py (pyproject.toml)
-tach==0.14.3              # via -r requirements-dev.in
+tach==0.14.4              # via -r requirements-dev.in
 tomli==2.1.0 ; python_version < "3.11"  # via -r requirements-dev.in, black, build, coverage, jupytext, mypy, pip-tools, pyproject-api, pytest, setuptools-scm, tach, tox
 tomli-w==1.0.0            # via tach
 tomlkit==0.13.2           # via bump-my-version

docs/development/ADRs/0008-Mapping_Domain_to_Cpp-Backend.md

@@ -20,7 +20,7 @@ The Python embedded execution for Iterator IR keeps track of the current locatio
 
 ### Python side
 
-On the Python side, we label dimensions of fields with the location type, e.g. `Edge` or `Vertex`. The domain uses `named_ranges` that uses the same location types to express _where_ to iterate, e.g. `named_range(Vertex, range(0, 100))` is an iteration over the `Vertex` dimension, no order in the domain is required. Additionally, the `Connectivity` (aka `NeighborTableOffsetProvider` in the current implementation) describes the mapping between location types.
+On the Python side, we label dimensions of fields with the location type, e.g. `Edge` or `Vertex`. The domain uses `named_ranges` that uses the same location types to express _where_ to iterate, e.g. `named_range(Vertex, range(0, 100))` is an iteration over the `Vertex` dimension, no order in the domain is required. Additionally, the `Connectivity` describes the mapping between location types.
 
 ### C++ side
 

docs/development/ADRs/0019-Connectivities.md

@@ -0,0 +1,55 @@
+---
+tags: []
+---
+
+# [Connectivities]
+
+- **Status**: valid
+- **Authors**: Hannes Vogt (@havogt)
+- **Created**: 2024-11-08
+- **Updated**: 2024-11-08
+
+The representation of Connectivities (neighbor tables, `NeighborTableOffsetProvider`) and their identifier (offset tag, `FieldOffset`, etc.) was extended and modified based on the needs of different parts of the toolchain. Here we outline the ideas for consolidating the different closely-related concepts.
+
+## History
+
+In the early days of Iterator IR (ITIR), an `offset` was a literal in the IR. Its meaning was only provided at execution time by a mapping from `offset` tag to an entity that we labelled `OffsetProvider`. We had mainly 2 kinds of `OffsetProvider`: a `Dimension` representing a Cartesian shift and a `NeighborTableOffsetProvider` for unstructured shifts. Since the type of `offset` needs to be known for compilation (strided for Cartesian, lookup-table for unstructured), this prevents a clean interface for ahead-of-time compilation.
+For the frontend type-checking we later introduce a `FieldOffset` which contained type information of the mapped dimensions.
+For (field-view) embedded we introduced a `ConnectivityField` (now `Connectivity`) which could be generated from the OffsetProvider information.
+
+These different concepts had overlap but were not 1-to-1 replacements.
+
+## Decision
+
+We update and introduce the following concepts
+
+### Conceptual definitions
+
+**Connectivity** is a mapping from index (or product of indices) to index. It covers 1-to-1 mappings, e.g. Cartesian shifts, NeighborTables (2D mappings) and dynamic Cartesian shifts.
+
+**NeighborConnectivity** is a 2D mapping of the N neighbors of a Location A to a Location B.
+
+**NeighborTable** is a _NeighborConnectivity_ backed by a buffer.
+
+**ConnectivityType**, **NeighborConnectivityType** contains all information that is needed for compilation.
+
+### Full definitions
+
+See `next.common` module
+
+Note: Currently, the compiled backends supports only `NeighborConnectivity`s that are `NeighborTable`s. We do not yet encode this in the type and postpone discussion to the point where we support alternative implementations (e.g. `StridedNeighborConnectivity`).
+
+## Which parts of the toolchain use which concept?
+
+### Embedded
+
+Embedded execution of field-view supports any kind of `Connectivity`.
+Embedded execution of iterator (local) view supports only `NeighborConnectivity`s.
+
+### IR transformations and compiled backends
+
+All transformations and code-generation should use `ConnectivityType`, not the `Connectivity` which contains the runtime mapping.
+
+Note, currently the `global_tmps` pass uses runtime information, therefore this is not strictly enforced.
+
+The only supported `Connectivity`s in compiled backends (currently) are `NeighborTable`s.

docs/user/next/QuickstartGuide.md

@@ -155,8 +155,6 @@ This section approaches the pseudo-laplacian by introducing the required APIs pr
 - [Using reductions on connected mesh elements](#Using-reductions-on-connected-mesh-elements)
 - [Implementing the actual pseudo-laplacian](#Implementing-the-pseudo-laplacian)
 
-+++
-
 #### Defining the mesh and its connectivities
 
 The examples related to unstructured meshes use the mesh below. The edges (in blue) and the cells (in red) are numbered with zero-based indices.
@@ -237,7 +235,7 @@ E2C = gtx.FieldOffset("E2C", source=CellDim, target=(EdgeDim,E2CDim))
 Note that the field offset does not contain the actual connectivity table, that's provided through an _offset provider_:
 
 ```{code-cell} ipython3
-E2C_offset_provider = gtx.NeighborTableOffsetProvider(edge_to_cell_table, EdgeDim, CellDim, 2)
+E2C_offset_provider = gtx.as_connectivity([EdgeDim, E2CDim], codomain=CellDim, data=edge_to_cell_table, skip_value=-1)
 ```
 
 The field operator `nearest_cell_to_edge` below shows an example of applying this transform. There is a little twist though: the subscript in `E2C[0]` means that only the value of the first connected cell is taken, the second (if exists) is ignored.
@@ -385,7 +383,7 @@ As explained in the section outline, the pseudo-laplacian needs the cell-to-edge
 C2EDim = gtx.Dimension("C2E", kind=gtx.DimensionKind.LOCAL)
 C2E = gtx.FieldOffset("C2E", source=EdgeDim, target=(CellDim, C2EDim))
 
-C2E_offset_provider = gtx.NeighborTableOffsetProvider(cell_to_edge_table, CellDim, EdgeDim, 3)
+C2E_offset_provider = gtx.as_connectivity([CellDim, C2EDim], codomain=EdgeDim, data=cell_to_edge_table, skip_value=-1)
 ```
 
 **Weights of edge differences:**

docs/user/next/workshop/exercises/2_divergence_exercise.ipynb

@@ -81,7 +81,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "id": "5dbd2f62",
    "metadata": {},
    "outputs": [],
@@ -113,7 +113,7 @@
     "        edge_orientation.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    c2e_connectivity = gtx.NeighborTableOffsetProvider(c2e_table, C, E, 3, has_skip_values=False)\n",
+    "    c2e_connectivity = gtx.as_connectivity([C, C2EDim], codomain=E, data=c2e_table)\n",
     "\n",
     "    divergence_gt4py = gtx.zeros(cell_domain, allocator=backend)\n",
     "\n",

docs/user/next/workshop/exercises/2_divergence_exercise_solution.ipynb

@@ -86,7 +86,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "id": "5dbd2f62",
    "metadata": {},
    "outputs": [],
@@ -118,7 +118,7 @@
     "        edge_orientation.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    c2e_connectivity = gtx.NeighborTableOffsetProvider(c2e_table, C, E, 3, has_skip_values=False)\n",
+    "    c2e_connectivity = gtx.as_connectivity([C, C2EDim], codomain=E, data=c2e_table)\n",
     "\n",
     "    divergence_gt4py = gtx.zeros(cell_domain, allocator=backend)\n",
     "\n",

docs/user/next/workshop/exercises/3_gradient_exercise.ipynb

@@ -80,7 +80,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "id": "84b02762",
    "metadata": {},
    "outputs": [],
@@ -110,7 +110,7 @@
     "        edge_orientation.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    c2e_connectivity = gtx.NeighborTableOffsetProvider(c2e_table, C, E, 3, has_skip_values=False)\n",
+    "    c2e_connectivity = gtx.as_connectivity([C, C2EDim], codomain=E, data=c2e_table)\n",
     "\n",
     "    gradient_gt4py_x = gtx.zeros(cell_domain, allocator=backend)\n",
     "    gradient_gt4py_y = gtx.zeros(cell_domain, allocator=backend)\n",

docs/user/next/workshop/exercises/3_gradient_exercise_solution.ipynb

@@ -93,7 +93,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "84b02762",
    "metadata": {},
    "outputs": [],
@@ -123,7 +123,7 @@
     "        edge_orientation.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    c2e_connectivity = gtx.NeighborTableOffsetProvider(c2e_table, C, E, 3, has_skip_values=False)\n",
+    "    c2e_connectivity = gtx.as_connectivity([C, C2EDim], codomain=E, data=c2e_table)\n",
     "\n",
     "    gradient_gt4py_x = gtx.zeros(cell_domain, allocator=backend)\n",
     "    gradient_gt4py_y = gtx.zeros(cell_domain, allocator=backend)\n",

docs/user/next/workshop/exercises/4_curl_exercise.ipynb

@@ -102,7 +102,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "5b6ffc9e",
    "metadata": {},
    "outputs": [],
@@ -134,7 +134,7 @@
     "        edge_orientation.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    v2e_connectivity = gtx.NeighborTableOffsetProvider(v2e_table, V, E, 6, has_skip_values=False)\n",
+    "    v2e_connectivity = gtx.as_connectivity([V, V2EDim], codomain=E, data=v2e_table)\n",
     "\n",
     "    curl_gt4py = gtx.zeros(vertex_domain, allocator=backend)\n",
     "\n",

docs/user/next/workshop/exercises/4_curl_exercise_solution.ipynb

@@ -107,7 +107,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "5b6ffc9e",
    "metadata": {},
    "outputs": [],
@@ -139,7 +139,7 @@
     "        edge_orientation.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    v2e_connectivity = gtx.NeighborTableOffsetProvider(v2e_table, V, E, 6, has_skip_values=False)\n",
+    "    v2e_connectivity = gtx.as_connectivity([V, V2EDim], codomain=E, data=v2e_table)\n",
     "\n",
     "    curl_gt4py = gtx.zeros(vertex_domain, allocator=backend)\n",
     "\n",