Remove docker from QGIS tests (#943)

The tests are now run via the qgis_testrunner script that is a clone
from the original QGIS runner:
https://github.com/qgis/QGIS/blob/4891c6bda8922ff5c6a70021f097ce43c486cddf/.docker/qgis_resources/test_runner/qgis_testrunner.py.
I removed the if check for being inside or outside of QGIS, because not
every module was available in Windows.

The QGIS profile information and the plugins themselves are now
installed inside of the .pixi folder, so that cleanup is easier. And
there is no need anymore for looking into the roaming folder.

The QGIS tests that really require the user interface are called via
`test-ribasim-qgis-ui`. This task is not automated.
The QGIS tests that do not require the user interface are called via
`test-ribasim-qgis` and are automated via github actions.

---------

Co-authored-by: Maarten Pronk <[email protected]>
Co-authored-by: Maarten Pronk <[email protected]>

.github/workflows/python_tests.yml

@@ -11,7 +11,7 @@ concurrency:
   cancel-in-progress: true
 jobs:
   test:
-    name: Python ${{ matrix.python-version }} - ${{ matrix.os }} - ${{ matrix.arch }}
+    name: Python ${{ matrix.python-version }} - ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -24,8 +24,6 @@ jobs:
           - "3.10"
           - "3.11"
           - "3.12"
-        arch:
-          - x86
     steps:
       - uses: actions/checkout@v4
 

.github/workflows/qgis.yml

@@ -2,26 +2,33 @@ name: QGIS Tests
 
 on:
     push:
-      branches: [main, update/pixi-lock]
+      branches: [main]
       paths-ignore: [".teamcity/**"]
       tags: ["*"]
     pull_request:
     merge_group:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 jobs:
-    test-qgis:
-        name: "Test"
-        runs-on: ubuntu-latest
-        if: false # Disable tests until we have time to fix them
-        defaults:
-          run:
-            working-directory: .docker
-        steps:
-
-        - name: Check out repository
-          uses: actions/checkout@v4
-        - name: Launching docker compose
-          run: ./start.sh
-        - name: Running tests
-          run: ./test.sh
-        - name: Stopping docker compose
-          run: ./stop.sh
+  test:
+    name: QGIS plugin ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macOS-latest
+          - windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: "latest"
+      - name: Prepare pixi
+        run: |
+          pixi run install-without-pre-commit
+      - name: Run tests
+        run: pixi run test-ribasim-qgis-cov
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3

docs/contribute/release.qmd

@@ -81,6 +81,14 @@ In order to publish Ribasim Python or Ribasim API follow the following steps:
 
 Our continuous integration (CI) should have caught most issues.
 A current weak spot in our testing is the QGIS plugin, so it is a good idea to do some manual checks to see if it works properly.
+Start with running the automated task to see if it can be correctly installed.
+
+```sh
+# This test might give a fatal error on the first run, this is most likely a timing issue.
+# Try to run it again when that happens.
+pixi run test-ribasim-qgis-ui
+```
+
 It is a good idea to load new test models if there are any, or test any other changed functionality.
 
 ## Announce release

pixi.toml

@@ -105,20 +105,26 @@ publish-ribasim-api = { cmd = "twine upload dist/*", cwd = "python/ribasim_api",
     "build-ribasim-api-wheel",
 ] }
 # QGIS
+qgis = "qgis --profiles-path .pixi/qgis_env"
 install-ribasim-qgis = "python ribasim_qgis/scripts/install_ribasim_qgis.py"
 install-imod-qgis = "python ribasim_qgis/scripts/install_qgis_plugin.py iMOD && python ribasim_qgis/scripts/enable_plugin.py imodqgis"
 install-plugin-reloader-qgis = "python ribasim_qgis/scripts/install_qgis_plugin.py \"Plugin Reloader\" && python ribasim_qgis/scripts/enable_plugin.py plugin_reloader"
 install-debugvs-qgis = "python ribasim_qgis/scripts/install_qgis_plugin.py debugvs==0.7 && python ribasim_qgis/scripts/enable_plugin.py debug_vs"
-start-docker-qgis = { cmd = "sh ./start.sh", cwd = ".docker" }
-test-ribasim-qgis = { cmd = "sh ./test.sh; sh ./stop.sh", cwd = ".docker", depends_on = [
-    "start-docker-qgis",
-] }
 install-qgis-plugins = { depends_on = [
     "install-plugin-reloader-qgis",
     "install-debugvs-qgis",
     "install-ribasim-qgis",
     "install-imod-qgis",
 ] }
+test-ribasim-qgis-ui = { cmd = "python ribasim_qgis/scripts/run_qgis_ui_tests.py", depends_on = [
+    "install-ribasim-qgis",
+] }
+test-ribasim-qgis = { cmd = "pytest --numprocesses=auto ribasim_qgis/tests", depends_on = [
+    "install-ribasim-qgis",
+]}
+test-ribasim-qgis-cov = { cmd = "pytest --numprocesses=auto --cov=ribasim_qgis --cov-report=xml --cov-config=ribasim_qgis/.coveragerc ribasim_qgis/tests", depends_on = [
+    "install-ribasim-qgis",
+]}
 mypy-ribasim-qgis = "mypy ribasim_qgis"
 # Run
 ribasim-model = "julia --project=core -e 'using Ribasim; Ribasim.main(ARGS)'"

ribasim_qgis/.coveragerc

@@ -0,0 +1,6 @@
+[run]
+omit =
+    ribasim_qgis/resources.py
+    ribasim_qgis/tests/*
+    ribasim_qgis/tomllib/*
+    ribasim_qgis/ui_tests/*

ribasim_qgis/scripts/enable_plugin.py

@@ -1,20 +1,11 @@
 import configparser
 import sys
-
-import platformdirs
+from pathlib import Path
 
 
 def enable_plugin(plugin_name: str) -> None:
-    config_file = (
-        platformdirs.user_state_path(roaming=True)
-        / "QGIS"
-        / "QGIS3"
-        / "profiles"
-        / "default"
-        / "QGIS"
-        / "QGIS3.ini"
-    )
-
+    config_file = Path(".pixi/qgis_env/profiles/default/QGIS/QGIS3.ini")
+    config_file.parent.mkdir(parents=True, exist_ok=True)
     config_file.touch()
 
     config = configparser.ConfigParser()

ribasim_qgis/scripts/install_qgis_plugin.py

@@ -6,7 +6,8 @@
 
 
 def install_qgis_plugin(plugin_name: str):
-    plugin_path = Path(".pixi/env/Library/python/plugins")
+    plugin_path = Path(".pixi/qgis_env/profiles/default/python/plugins")
+    plugin_path.mkdir(parents=True, exist_ok=True)
 
     try:
         subprocess.check_call(["qgis-plugin-manager", "init"], cwd=plugin_path)

ribasim_qgis/scripts/install_ribasim_qgis.py

@@ -3,8 +3,10 @@
 from enable_plugin import enable_plugin
 
 target_path = Path("ribasim_qgis").absolute()
-source_path = Path(".pixi/env/Library/python/plugins/ribasim_qgis")
+plugins_path = Path(".pixi/qgis_env/profiles/default/python/plugins")
+source_path = plugins_path / "ribasim_qgis"
 
+plugins_path.mkdir(parents=True, exist_ok=True)
 source_path.unlink(missing_ok=True)
 source_path.symlink_to(target_path, target_is_directory=True)
 

ribasim_qgis/scripts/qgis_testrunner.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+
+"""
+***************************************************************************
+    Launches a unit test inside QGIS and exit the application.
+
+    Arguments:
+
+    accepts a single argument with the package name in python dotted notation,
+    the program tries first to load the module and launch the `run_all`
+    function of the module, if that fails it considers the last part of
+    the dotted path to be the function name and the previous part to be the
+    module.
+
+    Extra options for QGIS command line can be passed in the env var
+    QGIS_EXTRA_OPTIONS
+
+    Example run:
+
+    # Will load geoserverexplorer.test.catalogtests and run `run_all`
+    QGIS_EXTRA_OPTIONS='--optionspath .' \
+        GSHOSTNAME=localhost \
+        python qgis_testrunner.py geoserverexplorer.test.catalogtests
+
+
+    GSHOSTNAME=localhost \
+         python qgis_testrunner.py geoserverexplorer.test.catalogtests.run_my
+
+
+    ---------------------
+    Date                 : May 2016
+    Copyright            : (C) 2016 by Alessandro Pasotti
+    Email                : apasotti at boundlessgeo dot com
+***************************************************************************
+*                                                                         *
+*   This program is free software; you can redistribute it and/or modify  *
+*   it under the terms of the GNU General Public License as published by  *
+*   the Free Software Foundation; either version 2 of the License, or     *
+*   (at your option) any later version.                                   *
+*                                                                         *
+***************************************************************************
+"""
+
+__author__ = "Alessandro Pasotti"
+__date__ = "May 2016"
+
+import importlib
+import logging
+import os
+import signal
+import sys
+import traceback
+
+from qgis.utils import iface
+
+assert iface is not None
+
+
+def __get_test_function(test_module_name):
+    """Load the test module and return the test function"""
+    print("QGIS Test Runner - Trying to import %s" % test_module_name)
+    try:
+        test_module = importlib.import_module(test_module_name)
+        function_name = "run_all"
+    except ImportError as e:
+        # traceback.print_exc(file=sys.stdout)
+        # Strip latest name
+        pos = test_module_name.rfind(".")
+        if pos <= 0:
+            raise e
+        test_module_name, function_name = (
+            test_module_name[:pos],
+            test_module_name[pos + 1 :],
+        )
+        print("QGIS Test Runner - Trying to import %s" % test_module_name)
+        sys.stdout.flush()
+        try:
+            test_module = importlib.import_module(test_module_name)
+        except ImportError as e:
+            # traceback.print_exc(file=sys.stdout)
+            raise e
+    return getattr(test_module, function_name, None)
+
+
+# Start as soon as the initializationCompleted signal is fired
+from qgis.core import QgsApplication, QgsProject, QgsProjectBadLayerHandler
+from qgis.PyQt.QtCore import QDir
+
+
+class QgsProjectBadLayerDefaultHandler(QgsProjectBadLayerHandler):
+    def handleBadLayers(self, layers, dom):
+        pass
+
+
+# Monkey patch QGIS Python console
+from console.console_output import writeOut
+
+
+def _write(self, m):
+    sys.stdout.write(m)
+
+
+writeOut.write = _write
+
+# Add current working dir to the python path
+sys.path.append(QDir.current().path())
+
+
+def __exit_qgis(error_code: int):
+    app = QgsApplication.instance()
+    os.kill(app.applicationPid(), error_code)
+
+
+def __run_test():
+    """Run the test specified as last argument in the command line."""
+    # Disable modal handler for bad layers
+    QgsProject.instance().setBadLayerHandler(QgsProjectBadLayerDefaultHandler())
+    print("QGIS Test Runner Inside - starting the tests ...")
+    try:
+        test_module_name = QgsApplication.instance().arguments()[-1]
+        function_name = __get_test_function(test_module_name)
+        print("QGIS Test Runner Inside - executing function %s" % function_name)
+        function_name()
+        __exit_qgis(signal.SIG_DFL)
+    except Exception as e:
+        logging.error("QGIS Test Runner Inside - [FAILED] Exception: %s" % e)
+        # Print tb
+        traceback.print_exc(file=sys.stderr)
+        __exit_qgis(signal.SIGTERM)
+
+
+iface.initializationCompleted.connect(__run_test)

ribasim_qgis/scripts/run_qgis_ui_tests.py

@@ -0,0 +1,24 @@
+import subprocess
+
+qgis_process = subprocess.run(
+    [
+        "qgis",
+        "--profiles-path",
+        ".pixi/qgis_env",
+        "--version-migration",
+        "--nologo",
+        "--code",
+        "ribasim_qgis/scripts/qgis_testrunner.py",
+        "ribasim_qgis.ui_tests",
+    ],
+    stdout=subprocess.PIPE,
+    stderr=subprocess.STDOUT,
+    text=True,
+)
+
+print(qgis_process.stdout)
+qgis_process.check_returncode()
+
+# QGIS always finishes with exit code 0, even when tests fail, so we have to check the output
+if any(s in qgis_process.stdout for s in ["QGIS died on signal", "FAILED"]):
+    exit(1)

ribasim_qgis/tests/test_load_plugin.py → ribasim_qgis/ui_tests/test_load_plugin.py

@@ -8,10 +8,8 @@ def test_plugin_is_loaded(self):
         plugin = plugins.get("ribasim_qgis")
         self.assertTrue(plugin, "Ribasim plugin not loaded")
 
-
-class TestDock(unittest.TestCase):
     def test_load_dock(self):
-        """Triggers Ribasim button and checks that Dock is added"""
+        """Triggers Ribasim button and checks that Dock is added."""
 
         # This checks the *actual* QGIS interface, not just a stub
         self.assertTrue(iface is not None, "QGIS interface not available")