Merge origin/main

.devcontainer/.vscode/launch.json

@@ -0,0 +1,24 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Python: Current File (just my code)",
+      "type": "python",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "justMyCode": true
+    },
+    {
+      "name": "Python: Current File (all)",
+      "type": "python",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "justMyCode": false
+    }
+  ]
+}

.devcontainer/Dockerfile

@@ -0,0 +1,5 @@
+FROM mcr.microsoft.com/devcontainers/python:1-3.10-bookworm
+RUN apt-get update \
+    && export DEBIAN_FRONTEND=noninteractive && apt-get install -y libboost-dev \
+    && apt-get clean && rm -rf /var/cache/apt/* && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/*
+RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="/bin" sh

.devcontainer/devcontainer.json

@@ -0,0 +1,49 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+  "name": "Python 3",
+  // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+  "build": {
+    "dockerfile": "Dockerfile"
+  },
+
+  // Features to add to the dev container. More info: https://containers.dev/features.
+  // "features": {},
+
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  // "forwardPorts": [],
+
+  // Use 'postCreateCommand' to run commands after the container is created.
+  "postCreateCommand": "bash .devcontainer/setup.sh",
+
+  "containerEnv": {
+    "PRE_COMMIT_HOME": "/workspaces/gt4py/.caches/pre-commit"
+  },
+
+  // Configure tool-specific properties.
+  "customizations": {
+    // Configure properties specific to VS Code.
+    "vscode": {
+      // Set *default* container specific settings.json values on container create.
+      "settings": {
+        "python.formatting.provider": "ruff",
+        "python.testing.pytestEnabled": true,
+        "python.defaultInterpreterPath": "/workspaces/gt4py/.venv/bin/python",
+        "files.insertFinalNewline": true,
+        "python.terminal.activateEnvironment": true,
+        "cmake.ignoreCMakeListsMissing": true
+      },
+      "extensions": [
+        "charliermarsh.ruff",
+        "donjayamanne.githistory",
+        "github.vscode-github-actions",
+        "lextudio.restructuredtext",
+        "ms-python.python",
+        "ms-vsliveshare.vsliveshare",
+        "swyddfa.esbonio"
+      ]
+    }
+  }
+  // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+  // "remoteUser": "root"
+}

.devcontainer/setup.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+ln -sfn /workspaces/gt4py/.devcontainer/.vscode /workspaces/gt4py/.vscode
+uv venv .venv
+source .venv/bin/activate
+uv pip install -r requirements-dev.txt
+uv pip install -e .
+uv pip install -i https://test.pypi.org/simple/ atlas4py
+pre-commit install --install-hooks
+deactivate

.gitignore

@@ -159,5 +159,5 @@ venv.bak/
 
 ### Others ###
 .obsidian
-
 coverage.json
+.caches

.gitpod/.vscode/launch.json

@@ -6,19 +6,28 @@
   "configurations": [
     {
       "name": "Python: Current File (just my code)",
-      "type": "python",
+      "type": "debugpy",
       "request": "launch",
       "program": "${file}",
       "console": "integratedTerminal",
       "justMyCode": true
     },
     {
       "name": "Python: Current File (all)",
-      "type": "python",
+      "type": "debugpy",
       "request": "launch",
       "program": "${file}",
       "console": "integratedTerminal",
       "justMyCode": false
+    },
+    {
+      "name": "Python: Debug Tests",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "purpose": ["debug-test"],
+      "console": "integratedTerminal",
+      "justMyCode": true
     }
   ]
 }

docs/development/ADRs/0008-Mapping_Domain_to_Cpp-Backend.md

@@ -20,7 +20,7 @@ The Python embedded execution for Iterator IR keeps track of the current locatio
 
 ### Python side
 
-On the Python side, we label dimensions of fields with the location type, e.g. `Edge` or `Vertex`. The domain uses `named_ranges` that uses the same location types to express _where_ to iterate, e.g. `named_range(Vertex, range(0, 100))` is an iteration over the `Vertex` dimension, no order in the domain is required. Additionally, the `Connectivity` (aka `NeighborTableOffsetProvider` in the current implementation) describes the mapping between location types.
+On the Python side, we label dimensions of fields with the location type, e.g. `Edge` or `Vertex`. The domain uses `named_ranges` that uses the same location types to express _where_ to iterate, e.g. `named_range(Vertex, range(0, 100))` is an iteration over the `Vertex` dimension, no order in the domain is required. Additionally, the `Connectivity` describes the mapping between location types.
 
 ### C++ side
 

docs/development/ADRs/0019-Connectivities.md

@@ -0,0 +1,55 @@
+---
+tags: []
+---
+
+# [Connectivities]
+
+- **Status**: valid
+- **Authors**: Hannes Vogt (@havogt)
+- **Created**: 2024-11-08
+- **Updated**: 2024-11-08
+
+The representation of Connectivities (neighbor tables, `NeighborTableOffsetProvider`) and their identifier (offset tag, `FieldOffset`, etc.) was extended and modified based on the needs of different parts of the toolchain. Here we outline the ideas for consolidating the different closely-related concepts.
+
+## History
+
+In the early days of Iterator IR (ITIR), an `offset` was a literal in the IR. Its meaning was only provided at execution time by a mapping from `offset` tag to an entity that we labelled `OffsetProvider`. We had mainly 2 kinds of `OffsetProvider`: a `Dimension` representing a Cartesian shift and a `NeighborTableOffsetProvider` for unstructured shifts. Since the type of `offset` needs to be known for compilation (strided for Cartesian, lookup-table for unstructured), this prevents a clean interface for ahead-of-time compilation.
+For the frontend type-checking we later introduce a `FieldOffset` which contained type information of the mapped dimensions.
+For (field-view) embedded we introduced a `ConnectivityField` (now `Connectivity`) which could be generated from the OffsetProvider information.
+
+These different concepts had overlap but were not 1-to-1 replacements.
+
+## Decision
+
+We update and introduce the following concepts
+
+### Conceptual definitions
+
+**Connectivity** is a mapping from index (or product of indices) to index. It covers 1-to-1 mappings, e.g. Cartesian shifts, NeighborTables (2D mappings) and dynamic Cartesian shifts.
+
+**NeighborConnectivity** is a 2D mapping of the N neighbors of a Location A to a Location B.
+
+**NeighborTable** is a _NeighborConnectivity_ backed by a buffer.
+
+**ConnectivityType**, **NeighborConnectivityType** contains all information that is needed for compilation.
+
+### Full definitions
+
+See `next.common` module
+
+Note: Currently, the compiled backends supports only `NeighborConnectivity`s that are `NeighborTable`s. We do not yet encode this in the type and postpone discussion to the point where we support alternative implementations (e.g. `StridedNeighborConnectivity`).
+
+## Which parts of the toolchain use which concept?
+
+### Embedded
+
+Embedded execution of field-view supports any kind of `Connectivity`.
+Embedded execution of iterator (local) view supports only `NeighborConnectivity`s.
+
+### IR transformations and compiled backends
+
+All transformations and code-generation should use `ConnectivityType`, not the `Connectivity` which contains the runtime mapping.
+
+Note, currently the `global_tmps` pass uses runtime information, therefore this is not strictly enforced.
+
+The only supported `Connectivity`s in compiled backends (currently) are `NeighborTable`s.

docs/user/next/QuickstartGuide.md

@@ -155,8 +155,6 @@ This section approaches the pseudo-laplacian by introducing the required APIs pr
 - [Using reductions on connected mesh elements](#Using-reductions-on-connected-mesh-elements)
 - [Implementing the actual pseudo-laplacian](#Implementing-the-pseudo-laplacian)
 
-+++
-
 #### Defining the mesh and its connectivities
 
 The examples related to unstructured meshes use the mesh below. The edges (in blue) and the cells (in red) are numbered with zero-based indices.
@@ -237,7 +235,7 @@ E2C = gtx.FieldOffset("E2C", source=CellDim, target=(EdgeDim,E2CDim))
 Note that the field offset does not contain the actual connectivity table, that's provided through an _offset provider_:
 
 ```{code-cell} ipython3
-E2C_offset_provider = gtx.NeighborTableOffsetProvider(edge_to_cell_table, EdgeDim, CellDim, 2)
+E2C_offset_provider = gtx.as_connectivity([EdgeDim, E2CDim], codomain=CellDim, data=edge_to_cell_table, skip_value=-1)
 ```
 
 The field operator `nearest_cell_to_edge` below shows an example of applying this transform. There is a little twist though: the subscript in `E2C[0]` means that only the value of the first connected cell is taken, the second (if exists) is ignored.
@@ -385,7 +383,7 @@ As explained in the section outline, the pseudo-laplacian needs the cell-to-edge
 C2EDim = gtx.Dimension("C2E", kind=gtx.DimensionKind.LOCAL)
 C2E = gtx.FieldOffset("C2E", source=EdgeDim, target=(CellDim, C2EDim))
 
-C2E_offset_provider = gtx.NeighborTableOffsetProvider(cell_to_edge_table, CellDim, EdgeDim, 3)
+C2E_offset_provider = gtx.as_connectivity([CellDim, C2EDim], codomain=EdgeDim, data=cell_to_edge_table, skip_value=-1)
 ```
 
 **Weights of edge differences:**

docs/user/next/workshop/exercises/2_divergence_exercise.ipynb

@@ -81,7 +81,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "id": "5dbd2f62",
    "metadata": {},
    "outputs": [],
@@ -113,7 +113,7 @@
     "        edge_orientation.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    c2e_connectivity = gtx.NeighborTableOffsetProvider(c2e_table, C, E, 3, has_skip_values=False)\n",
+    "    c2e_connectivity = gtx.as_connectivity([C, C2EDim], codomain=E, data=c2e_table)\n",
     "\n",
     "    divergence_gt4py = gtx.zeros(cell_domain, allocator=backend)\n",
     "\n",

docs/user/next/workshop/exercises/2_divergence_exercise_solution.ipynb

@@ -86,7 +86,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "id": "5dbd2f62",
    "metadata": {},
    "outputs": [],
@@ -118,7 +118,7 @@
     "        edge_orientation.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    c2e_connectivity = gtx.NeighborTableOffsetProvider(c2e_table, C, E, 3, has_skip_values=False)\n",
+    "    c2e_connectivity = gtx.as_connectivity([C, C2EDim], codomain=E, data=c2e_table)\n",
     "\n",
     "    divergence_gt4py = gtx.zeros(cell_domain, allocator=backend)\n",
     "\n",

docs/user/next/workshop/exercises/3_gradient_exercise.ipynb

@@ -80,7 +80,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "id": "84b02762",
    "metadata": {},
    "outputs": [],
@@ -110,7 +110,7 @@
     "        edge_orientation.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    c2e_connectivity = gtx.NeighborTableOffsetProvider(c2e_table, C, E, 3, has_skip_values=False)\n",
+    "    c2e_connectivity = gtx.as_connectivity([C, C2EDim], codomain=E, data=c2e_table)\n",
     "\n",
     "    gradient_gt4py_x = gtx.zeros(cell_domain, allocator=backend)\n",
     "    gradient_gt4py_y = gtx.zeros(cell_domain, allocator=backend)\n",

docs/user/next/workshop/exercises/3_gradient_exercise_solution.ipynb

@@ -93,7 +93,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "84b02762",
    "metadata": {},
    "outputs": [],
@@ -123,7 +123,7 @@
     "        edge_orientation.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    c2e_connectivity = gtx.NeighborTableOffsetProvider(c2e_table, C, E, 3, has_skip_values=False)\n",
+    "    c2e_connectivity = gtx.as_connectivity([C, C2EDim], codomain=E, data=c2e_table)\n",
     "\n",
     "    gradient_gt4py_x = gtx.zeros(cell_domain, allocator=backend)\n",
     "    gradient_gt4py_y = gtx.zeros(cell_domain, allocator=backend)\n",

docs/user/next/workshop/exercises/4_curl_exercise.ipynb

@@ -102,7 +102,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "5b6ffc9e",
    "metadata": {},
    "outputs": [],
@@ -134,7 +134,7 @@
     "        edge_orientation.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    v2e_connectivity = gtx.NeighborTableOffsetProvider(v2e_table, V, E, 6, has_skip_values=False)\n",
+    "    v2e_connectivity = gtx.as_connectivity([V, V2EDim], codomain=E, data=v2e_table)\n",
     "\n",
     "    curl_gt4py = gtx.zeros(vertex_domain, allocator=backend)\n",
     "\n",

docs/user/next/workshop/exercises/4_curl_exercise_solution.ipynb

@@ -107,7 +107,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "5b6ffc9e",
    "metadata": {},
    "outputs": [],
@@ -139,7 +139,7 @@
     "        edge_orientation.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    v2e_connectivity = gtx.NeighborTableOffsetProvider(v2e_table, V, E, 6, has_skip_values=False)\n",
+    "    v2e_connectivity = gtx.as_connectivity([V, V2EDim], codomain=E, data=v2e_table)\n",
     "\n",
     "    curl_gt4py = gtx.zeros(vertex_domain, allocator=backend)\n",
     "\n",

docs/user/next/workshop/exercises/5_vector_laplace_exercise.ipynb

@@ -228,7 +228,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "id": "f9cfc097",
    "metadata": {},
    "outputs": [],
@@ -272,10 +272,10 @@
     "        edge_orientation_cell.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    c2e_connectivity = gtx.NeighborTableOffsetProvider(c2e_table, C, E, 3, has_skip_values=False)\n",
-    "    v2e_connectivity = gtx.NeighborTableOffsetProvider(v2e_table, V, E, 6, has_skip_values=False)\n",
-    "    e2v_connectivity = gtx.NeighborTableOffsetProvider(e2v_table, E, V, 2, has_skip_values=False)\n",
-    "    e2c_connectivity = gtx.NeighborTableOffsetProvider(e2c_table, E, C, 2, has_skip_values=False)\n",
+    "    c2e_connectivity = gtx.as_connectivity([C, C2EDim], codomain=E, data=c2e_table)\n",
+    "    v2e_connectivity = gtx.as_connectivity([V, V2EDim], codomain=E, data=v2e_table)\n",
+    "    e2v_connectivity = gtx.as_connectivity([E, E2VDim], codomain=V, data=e2v_table)\n",
+    "    e2c_connectivity = gtx.as_connectivity([E, E2CDim], codomain=C, data=e2c_table)\n",
     "\n",
     "    laplacian_gt4py = gtx.zeros(edge_domain, allocator=backend)\n",
     "\n",

docs/user/next/workshop/exercises/5_vector_laplace_exercise_solution.ipynb

@@ -249,7 +249,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "id": "f9cfc097",
    "metadata": {},
    "outputs": [],
@@ -293,10 +293,10 @@
     "        edge_orientation_cell.asnumpy(),\n",
     "    )\n",
     "\n",
-    "    c2e_connectivity = gtx.NeighborTableOffsetProvider(c2e_table, C, E, 3, has_skip_values=False)\n",
-    "    v2e_connectivity = gtx.NeighborTableOffsetProvider(v2e_table, V, E, 6, has_skip_values=False)\n",
-    "    e2v_connectivity = gtx.NeighborTableOffsetProvider(e2v_table, E, V, 2, has_skip_values=False)\n",
-    "    e2c_connectivity = gtx.NeighborTableOffsetProvider(e2c_table, E, C, 2, has_skip_values=False)\n",
+    "    c2e_connectivity = gtx.as_connectivity([C, C2EDim], codomain=E, data=c2e_table)\n",
+    "    v2e_connectivity = gtx.as_connectivity([V, V2EDim], codomain=E, data=v2e_table)\n",
+    "    e2v_connectivity = gtx.as_connectivity([E, E2VDim], codomain=V, data=e2v_table)\n",
+    "    e2c_connectivity = gtx.as_connectivity([E, E2CDim], codomain=C, data=e2c_table)\n",
     "\n",
     "    laplacian_gt4py = gtx.zeros(edge_domain, allocator=backend)\n",
     "\n",

docs/user/next/workshop/exercises/8_diffusion_exercise_solution.ipynb

@@ -118,7 +118,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 127,
+   "execution_count": null,
    "id": "f9cfc097",
    "metadata": {},
    "outputs": [],
@@ -156,10 +156,8 @@
     "        dt,\n",
     "    )\n",
     "\n",
-    "    e2c2v_connectivity = gtx.NeighborTableOffsetProvider(\n",
-    "        e2c2v_table, E, V, 4, has_skip_values=False\n",
-    "    )\n",
-    "    v2e_connectivity = gtx.NeighborTableOffsetProvider(v2e_table, V, E, 6, has_skip_values=False)\n",
+    "    e2c2v_connectivity = gtx.as_connectivity([E, E2C2VDim], codomain=V, data=e2c2v_table)\n",
+    "    v2e_connectivity = gtx.as_connectivity([V, V2EDim], codomain=E, data=v2e_table)\n",
     "\n",
     "    diffusion_step(\n",
     "        u,\n",