Add icon4pytools (#206)

Introduces the icon4pytools package. This package contains all icon4py (blue line) related tooling and CLIs such as: - liskov - icon4pygen - f2ser Co-authored-by: Enrique G. Paredes <[email protected]>
C2SM · May 31, 2023 · d935147 · d935147
1 parent 69248f1
commit d935147
Show file tree

Hide file tree

Showing 248 changed files with 2,111 additions and 1,363 deletions.
diff --git a/.flake8 b/.flake8
@@ -42,4 +42,4 @@ rst-roles =
     py:obj, obj,
 
 per-file-ignores =
-    pyutils/src/icon4py/icon4pygen/icochainsize.py:E800
+    tools/src/icon4pytools/icon4pygen/icochainsize.py:E800
diff --git a/.github/workflows/qa.yml → .github/workflows/icon4py-qa.yml b/.github/workflows/qa.yml → .github/workflows/icon4py-qa.yml
@@ -1,9 +1,11 @@
-name: Quality Assurance
+name: ICON4Py Quality Assurance
 
 on:
     push:
         branches:
             - main
+        paths-ignore:
+            - "tools/**"
     pull_request:
         branches:
             - main
@@ -29,4 +31,4 @@ jobs:
             - name: Run checks
               run: |
                   python -m pip list
-                  pre-commit run --all-files
+                  pre-commit run --config .pre-commit-config.yaml --all-files
diff --git a/.github/workflows/tox.yml → .github/workflows/icon4py-tox.yml b/.github/workflows/tox.yml → .github/workflows/icon4py-tox.yml
@@ -1,9 +1,11 @@
-name: Tox
+name: ICON4Py Tox
 
 on:
     push:
         branches:
             - main
+        paths-ignore:
+            - "tools/**"
     pull_request:
         branches:
             - main

diff --git a/.github/workflows/icon4pytools-qa.yml b/.github/workflows/icon4pytools-qa.yml
@@ -0,0 +1,30 @@
+name: ICON4PyTools Quality Assurance
+
+on:
+    push:
+        branches:
+            - main
+        paths:
+            - "tools/**"
+            - "base-requirements*"
+    pull_request:
+        branches:
+            - main
+
+jobs:
+    pre-commit:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v2
+            - name: Set up Python
+              uses: actions/setup-python@v3
+              with:
+                  python-version: "3.10"
+            - name: Install dependencies
+              run: |
+                  python -m pip install --upgrade pip setuptools wheel
+                  python -m pip install -r ./requirements-dev.txt
+            - name: Run checks
+              run: |
+                  python -m pip list
+                  pre-commit run --config tools/.pre-commit-config.yaml --all-files
diff --git a/.github/workflows/icon4pytools-tox.yml b/.github/workflows/icon4pytools-tox.yml
@@ -0,0 +1,52 @@
+name: ICON4PyTools Tox
+
+on:
+    push:
+        branches:
+            - main
+        paths:
+            - "tools/**"
+            - "base-requirements*"
+    pull_request:
+        branches:
+            - main
+
+jobs:
+    tox-tests:
+        runs-on: ubuntu-latest
+        strategy:
+            matrix:
+                python-version: ["3.10"]
+        defaults:
+            run:
+                working-directory: tools
+
+        steps:
+            - uses: actions/checkout@v2
+
+            - name: Install boost
+              run: |
+                  sudo apt-get update
+                  sudo apt-get install libboost-all-dev
+
+            - name: Set up Python ${{ matrix.python-version }}
+              uses: actions/setup-python@v2
+              with:
+                  python-version: ${{ matrix.python-version }}
+
+            - name: Install python dependencies
+              run: |
+                  python -m pip install --upgrade pip setuptools wheel
+
+            - name: Test with tox
+              run: |
+                  pyversion_no_dot="${{ matrix.python-version }}"
+                  pyversion_no_dot="${pyversion_no_dot/./}"
+                  pip install tox clang-format
+                  tox -r -e py${pyversion_no_dot}
+
+            - name: Archive code coverage results
+              uses: actions/upload-artifact@v2
+              with:
+                  name: code-coverage-report
+                  path: _reports/coverage_html/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -5,6 +5,7 @@ default_language_version:
     # Remove frozen version once we migrated away from tsa
     node: 17.9.1
 minimum_pre_commit_version: 2.20.0
+exclude: "tools/.*"
 
 repos:
     - repo: meta
@@ -35,9 +36,6 @@ repos:
           - id: destroyed-symlinks
           # - id: fix-encoding-pragma
           #   args: [--remove]
-          - id: name-tests-test
-            args: [--pytest-test-first]
-            exclude: ^liskov/tests/samples
 
     - repo: https://gitlab.com/bmares/check-json5
       rev: v1.0.0
@@ -117,16 +115,14 @@ repos:
                 - flake8-docstrings
                 - flake8-eradicate
                 - flake8-mutable
-                # Disabled for now due to random false positives
-                # - flake8-rst-docstrings
                 - pygments
 
     - repo: local
       hooks:
           - id: mypy
             name: mypy static type checker
             entry: |
-                mypy --install-types --non-interactive -p icon4py.common -p icon4py.icon4pygen -p icon4py.testutils
+                mypy --install-types --non-interactive -p icon4py.common
             language: system
             types_or: [python, pyi]
             always_run: true

diff --git a/README.md b/README.md
@@ -4,11 +4,11 @@
 
 ## Description
 
-ICON4Py contains Python (GT4Py) implementations of ICON (inspired) components for weather and climate models.
+ICON4Py contains Python (GT4Py) implementations of ICON (inspired) components for weather and climate models as well as `icon4pytools`, a package containing CLIs and utilties needed for the integration of ICON4Py code into the ICON Fortran model.
 
 ## Project structure
 
-Each directory contains Python packages of ICON components or utility packages that are deployable on its own. As these packages are not available from a package repository (yet), location of dependencies within this repository have to be provided explicitly, e.g. by installing the dependencies first. See [Installation instructions](#installation-instructions).
+Each directory contains Python packages of ICON components or utility packages that are deployable on their own. As these packages are not available from a package repository (yet), location of dependencies within this repository have to be provided explicitly, e.g. by installing the dependencies first. See [Installation instructions](#installation-instructions).
 
 ## Installation instructions
 
@@ -70,8 +70,11 @@ source .venv/bin/activate
 pip install --upgrade wheel
 
 # Install a specific ICON4Py subpackage and its dependencies
-cd _SUBPACKAGE_  # where _SUBPACKAGE_ in atm_dyn_iconam | common | icon4pygen | testutils | ...
+cd _SUBPACKAGE_  # where _SUBPACKAGE_ in atm_dyn_iconam | tools | ...
 pip install -r requirements-dev.txt
+
+# or in the case of there being a pyproject.toml file
+pip install .
 ```
 
 ## Development instructions

diff --git a/...py/liskov/codegen/integration/__init__.py → atm_dyn_iconam/__init__.py b/...py/liskov/codegen/integration/__init__.py → atm_dyn_iconam/__init__.py
diff --git a/atm_dyn_iconam/src/icon4py/atm_dyn_iconam/apply_nabla2_and_nabla4_to_vn.py b/atm_dyn_iconam/src/icon4py/atm_dyn_iconam/apply_nabla2_and_nabla4_to_vn.py
@@ -10,7 +10,6 @@
 # distribution for a copy of the license or check <https://www.gnu.org/licenses/>.
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
-
 from gt4py.next.ffront.decorator import field_operator, program
 from gt4py.next.ffront.fbuiltins import Field, maximum
 

diff --git a/atm_dyn_iconam/src/icon4py/atm_dyn_iconam/apply_nabla2_to_vn_in_lateral_boundary.py b/atm_dyn_iconam/src/icon4py/atm_dyn_iconam/apply_nabla2_to_vn_in_lateral_boundary.py
@@ -10,7 +10,6 @@
 # distribution for a copy of the license or check <https://www.gnu.org/licenses/>.
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
-
 from gt4py.next.ffront.decorator import field_operator, program
 from gt4py.next.ffront.fbuiltins import Field
 

diff --git a/atm_dyn_iconam/src/icon4py/atm_dyn_iconam/mo_solve_nonhydro_stencil_02.py b/atm_dyn_iconam/src/icon4py/atm_dyn_iconam/mo_solve_nonhydro_stencil_02.py
@@ -10,7 +10,6 @@
 # distribution for a copy of the license or check <https://www.gnu.org/licenses/>.
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
-
 from gt4py.next.ffront.decorator import field_operator, program
 from gt4py.next.ffront.fbuiltins import Field
 

diff --git a/.../liskov/codegen/serialisation/__init__.py → atm_dyn_iconam/tests/__init__.py b/.../liskov/codegen/serialisation/__init__.py → atm_dyn_iconam/tests/__init__.py
diff --git a/atm_dyn_iconam/tests/test_apply_nabla2_and_nabla4_global_to_vn.py b/atm_dyn_iconam/tests/test_apply_nabla2_and_nabla4_global_to_vn.py
@@ -17,8 +17,9 @@
     apply_nabla2_and_nabla4_global_to_vn,
 )
 from icon4py.common.dimension import EdgeDim, KDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import random_field
+
+from .test_utils.helpers import random_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def apply_nabla2_and_nabla4_global_to_vn_numpy(

diff --git a/atm_dyn_iconam/tests/test_apply_nabla2_and_nabla4_to_vn.py b/atm_dyn_iconam/tests/test_apply_nabla2_and_nabla4_to_vn.py
@@ -17,8 +17,9 @@
     apply_nabla2_and_nabla4_to_vn,
 )
 from icon4py.common.dimension import EdgeDim, KDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import random_field
+
+from .test_utils.helpers import random_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def apply_nabla2_and_nabla4_to_vn_numpy(

diff --git a/atm_dyn_iconam/tests/test_apply_nabla2_to_vn_in_lateral_boundary.py b/atm_dyn_iconam/tests/test_apply_nabla2_to_vn_in_lateral_boundary.py
@@ -17,8 +17,9 @@
     apply_nabla2_to_vn_in_lateral_boundary,
 )
 from icon4py.common.dimension import EdgeDim, KDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import random_field
+
+from .test_utils.helpers import random_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def apply_nabla2_to_vn_in_lateral_boundary_numpy(

diff --git a/atm_dyn_iconam/tests/test_apply_nabla2_to_w.py b/atm_dyn_iconam/tests/test_apply_nabla2_to_w.py
@@ -15,8 +15,9 @@
 
 from icon4py.atm_dyn_iconam.apply_nabla2_to_w import apply_nabla2_to_w
 from icon4py.common.dimension import C2E2CODim, CellDim, KDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import random_field
+
+from .test_utils.helpers import random_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def mo_apply_nabla2_to_w(

diff --git a/atm_dyn_iconam/tests/test_apply_nabla2_to_w_in_upper_damping_layer.py b/atm_dyn_iconam/tests/test_apply_nabla2_to_w_in_upper_damping_layer.py
@@ -17,8 +17,9 @@
     apply_nabla2_to_w_in_upper_damping_layer,
 )
 from icon4py.common.dimension import CellDim, KDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import random_field
+
+from .test_utils.helpers import random_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def apply_nabla2_to_w_in_upper_damping_layer_numpy(

diff --git a/atm_dyn_iconam/tests/test_calculate_diagnostics_for_turbulence.py b/atm_dyn_iconam/tests/test_calculate_diagnostics_for_turbulence.py
@@ -17,8 +17,9 @@
     calculate_diagnostics_for_turbulence,
 )
 from icon4py.common.dimension import CellDim, KDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import random_field, zero_field
+
+from .test_utils.helpers import random_field, zero_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def calculate_diagnostics_for_turbulence_numpy(

diff --git a/atm_dyn_iconam/tests/test_calculate_horizontal_gradients_for_turbulence.py b/atm_dyn_iconam/tests/test_calculate_horizontal_gradients_for_turbulence.py
@@ -17,8 +17,9 @@
     calculate_horizontal_gradients_for_turbulence,
 )
 from icon4py.common.dimension import C2E2CODim, CellDim, KDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import random_field, zero_field
+
+from .test_utils.helpers import random_field, zero_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def calculate_horizontal_gradients_for_turbulence_numpy(

diff --git a/atm_dyn_iconam/tests/test_calculate_nabla2_and_smag_coefficients_for_vn.py b/atm_dyn_iconam/tests/test_calculate_nabla2_and_smag_coefficients_for_vn.py
@@ -18,8 +18,9 @@
     calculate_nabla2_and_smag_coefficients_for_vn,
 )
 from icon4py.common.dimension import E2C2VDim, ECVDim, EdgeDim, KDim, VertexDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import as_1D_sparse_field, random_field, zero_field
+
+from .test_utils.helpers import as_1D_sparse_field, random_field, zero_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def calculate_nabla2_and_smag_coefficients_for_vn_numpy(

diff --git a/atm_dyn_iconam/tests/test_calculate_nabla2_for_w.py b/atm_dyn_iconam/tests/test_calculate_nabla2_for_w.py
@@ -15,8 +15,9 @@
 
 from icon4py.atm_dyn_iconam.calculate_nabla2_for_w import calculate_nabla2_for_w
 from icon4py.common.dimension import C2E2CODim, CellDim, KDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import random_field, zero_field
+
+from .test_utils.helpers import random_field, zero_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def calculate_nabla2_for_w_numpy(

diff --git a/atm_dyn_iconam/tests/test_calculate_nabla2_for_z.py b/atm_dyn_iconam/tests/test_calculate_nabla2_for_z.py
@@ -15,8 +15,9 @@
 
 from icon4py.atm_dyn_iconam.calculate_nabla2_for_z import calculate_nabla2_for_z
 from icon4py.common.dimension import CellDim, EdgeDim, KDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import random_field
+
+from .test_utils.helpers import random_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def calculate_nabla2_for_z_numpy(

diff --git a/atm_dyn_iconam/tests/test_calculate_nabla2_of_theta.py b/atm_dyn_iconam/tests/test_calculate_nabla2_of_theta.py
@@ -18,8 +18,9 @@
     calculate_nabla2_of_theta,
 )
 from icon4py.common.dimension import C2EDim, CEDim, CellDim, EdgeDim, KDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import as_1D_sparse_field, random_field, zero_field
+
+from .test_utils.helpers import as_1D_sparse_field, random_field, zero_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def calculate_nabla2_of_theta_numpy(

diff --git a/atm_dyn_iconam/tests/test_calculate_nabla4.py b/atm_dyn_iconam/tests/test_calculate_nabla4.py
@@ -16,8 +16,9 @@
 
 from icon4py.atm_dyn_iconam.calculate_nabla4 import calculate_nabla4
 from icon4py.common.dimension import E2C2VDim, ECVDim, EdgeDim, KDim, VertexDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import as_1D_sparse_field, random_field, zero_field
+
+from .test_utils.helpers import as_1D_sparse_field, random_field, zero_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def calculate_nabla4_numpy(

diff --git a/atm_dyn_iconam/tests/test_enhance_diffusion_coefficient_for_grid_point_cold_pools.py b/atm_dyn_iconam/tests/test_enhance_diffusion_coefficient_for_grid_point_cold_pools.py
@@ -17,8 +17,9 @@
     enhance_diffusion_coefficient_for_grid_point_cold_pools,
 )
 from icon4py.common.dimension import CellDim, EdgeDim, KDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import random_field
+
+from .test_utils.helpers import random_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def enhance_diffusion_coefficient_for_grid_point_cold_pools_numpy(

diff --git a/atm_dyn_iconam/tests/test_mo_advection_traj_btraj_compute_o1_dsl.py b/atm_dyn_iconam/tests/test_mo_advection_traj_btraj_compute_o1_dsl.py
@@ -19,12 +19,9 @@
     mo_advection_traj_btraj_compute_o1_dsl,
 )
 from icon4py.common.dimension import E2CDim, ECDim, EdgeDim, KDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import (
-    as_1D_sparse_field,
-    constant_field,
-    random_field,
-)
+
+from .test_utils.helpers import as_1D_sparse_field, constant_field, random_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def mo_advection_traj_btraj_compute_o1_dsl_numpy(

diff --git a/atm_dyn_iconam/tests/test_mo_icon_interpolation_scalar_cells2verts_scalar_ri_dsl.py b/atm_dyn_iconam/tests/test_mo_icon_interpolation_scalar_cells2verts_scalar_ri_dsl.py
@@ -17,8 +17,9 @@
     mo_icon_interpolation_scalar_cells2verts_scalar_ri_dsl,
 )
 from icon4py.common.dimension import CellDim, KDim, V2CDim, VertexDim
-from icon4py.testutils.simple_mesh import SimpleMesh
-from icon4py.testutils.utils import random_field, zero_field
+
+from .test_utils.helpers import random_field, zero_field
+from .test_utils.simple_mesh import SimpleMesh
 
 
 def mo_icon_interpolation_scalar_cells2verts_scalar_ri_dsl_numpy(