[DPE-2684] Juju3 pipelines (#313)

* CI matrix + env.vars

* pyproject updates (libjuju version, pytest-mock, etc)

* tox.ini equipped with Juju version selection vars

* Juju3 compatible tests

* Fixtures that allow detection and 'switching' between Juju versions

* snap_microk8s instead of old microk8s user

.github/workflows/ci.yaml

@@ -19,6 +19,10 @@ jobs:
     uses: canonical/data-platform-workflows/.github/workflows/[email protected]
 
   unit-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        juju-version:  ["2.9", "3.1"]
     name: Unit test charm
     runs-on: ubuntu-latest
     timeout-minutes: 5
@@ -31,6 +35,12 @@ jobs:
           pipx install poetry
       - name: Run tests
         run: tox run -e unit
+        env:
+          # This env var is only to indicate Juju version to "simulate" in the unit tests
+          # No libjuju is being actually used in unit testing
+          LIBJUJU_VERSION_SPECIFIER: ${{ matrix.juju-version }}
+      - name: Upload Coverage to Codecov
+        uses: codecov/codecov-action@v3
 
   build:
     name: Build charm
@@ -75,7 +85,29 @@ jobs:
       fail-fast: false
       matrix:
         groups: ${{ fromJSON(needs.gh-hosted-collect-integration-tests.outputs.groups) }}
-    name: (GH hosted) ${{ matrix.groups.job_name }}
+        juju-snap-channel:  ["2.9/stable", "3.1/candidate"]
+        include:
+          - juju-snap-channel: "3.1/candidate"
+            agent-version: "3.1.6"
+            libjuju-version:  "3.2.2"
+          - juju-snap-channel: "2.9/stable"
+            agent-version: "2.9.44"
+            libjuju-version:  "2.9.44.1"
+        exclude:
+          # Disabling HA tests, as long as we want to have a limited pipeline on Juju3
+          - juju-snap-channel: "3.1/candidate"
+            groups:
+              job_name: "high_availability/test_replication.py | group 1"
+          - juju-snap-channel: "3.1/candidate"
+            groups:
+              job_name: "high_availability/test_self_healing.py | group 1"
+          - juju-snap-channel: "3.1/candidate"
+            groups:
+              job_name: "high_availability/test_upgrade.py | group 1"
+          - juju-snap-channel: "3.1/candidate"
+            groups:
+              job_name: "high_availability/test_upgrade_from_stable.py | group 1"
+    name: ${{ matrix.juju-snap-channel }} - (GH hosted) ${{ matrix.groups.job_name }}
     needs:
       - lint
       - unit-test
@@ -94,6 +126,12 @@ jobs:
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: microk8s
+          channel: "1.28-strict/stable"
+          bootstrap-options: "--agent-version ${{ matrix.agent-version }}"
+          juju-channel: ${{ matrix.juju-snap-channel }}
+      - name: Update python-libjuju version
+        if: ${{ matrix.libjuju-version != '2.9.44.1' }}
+        run: poetry add --lock --group integration juju@'${{ matrix.libjuju-version }}'
       - name: Download packed charm(s)
         uses: actions/download-artifact@v3
         with:
@@ -112,6 +150,7 @@ jobs:
       - name: Run integration tests
         run: tox run -e integration -- "${{ matrix.groups.path_to_test_file }}" --group="${{ matrix.groups.group_number }}" -m '${{ steps.select-test-stability.outputs.mark_expression }}'
         env:
+          LIBJUJU_VERSION_SPECIFIER: ${{ matrix.libjuju-version }}
           SECRETS_FROM_GITHUB: |
             {
               "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}",

pyproject.toml

@@ -49,6 +49,7 @@ shellcheck-py = "^0.9.0.5"
 
 [tool.poetry.group.unit.dependencies]
 pytest = "^7.4.0"
+pytest-mock = "^3.11.1"
 coverage = {extras = ["toml"], version = "^7.2.7"}
 
 [tool.poetry.group.integration.dependencies]
@@ -58,6 +59,8 @@ pytest-operator = "^0.28.0"
 pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v5.0.0", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
 pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v5.0.0", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
 juju = "^2.9.44.1"
+ops = "^2.5.0"
+pytest-mock = "^3.11.1"
 mysql-connector-python = "~8.0.33"
 tenacity = "^8.2.2"
 boto3 = "^1.28.22"

tests/conftest.py

@@ -0,0 +1,46 @@
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+import os
+from unittest.mock import PropertyMock
+
+import pytest
+from ops import JujuVersion
+from pytest_mock import MockerFixture
+
+
+@pytest.fixture(autouse=True)
+def juju_has_secrets(mocker: MockerFixture):
+    """This fixture will force the usage of secrets whenever run on Juju 3.x.
+
+    NOTE: This is needed, as normally JujuVersion is set to 0.0.0 in tests
+    (i.e. not the real juju version)
+    """
+    juju_version = os.environ["LIBJUJU_VERSION_SPECIFIER"].split("/")[0]
+    if juju_version < "3":
+        mocker.patch.object(
+            JujuVersion, "has_secrets", new_callable=PropertyMock
+        ).return_value = False
+        return False
+    else:
+        mocker.patch.object(
+            JujuVersion, "has_secrets", new_callable=PropertyMock
+        ).return_value = True
+        return True
+
+
+@pytest.fixture
+def only_with_juju_secrets(juju_has_secrets):
+    """Pretty way to skip Juju 3 tests."""
+    if not juju_has_secrets:
+        pytest.skip("Secrets test only applies on Juju 3.x")
+
+
+@pytest.fixture
+def only_without_juju_secrets(juju_has_secrets):
+    """Pretty way to skip Juju 2-specific tests.
+
+    Typically: to save CI time, when the same check were executed in a Juju 3-specific way already
+    """
+    if juju_has_secrets:
+        pytest.skip("Skipping legacy secrets tests")

tests/integration/high_availability/scripts/deploy_chaos_mesh.sh

@@ -12,10 +12,10 @@ fi
 
 deploy_chaos_mesh() {
 	echo "adding chaos-mesh helm repo"
-	sg microk8s -c "microk8s.helm3 repo add chaos-mesh https://charts.chaos-mesh.org"
+	sg snap_microk8s -c "microk8s.helm3 repo add chaos-mesh https://charts.chaos-mesh.org"
 
 	echo "installing chaos-mesh"
-        sg microk8s -c "microk8s.helm3 install chaos-mesh chaos-mesh/chaos-mesh \
+        sg snap_microk8s -c "microk8s.helm3 install chaos-mesh chaos-mesh/chaos-mesh \
           --namespace=\"${chaos_mesh_ns}\" \
           --set chaosDaemon.runtime=containerd \
           --set chaosDaemon.socketPath=/var/snap/microk8s/common/run/containerd.sock \

tests/integration/high_availability/scripts/destroy_chaos_mesh.sh

@@ -45,9 +45,9 @@ destroy_chaos_mesh() {
 		timeout 30 kubectl delete crd "${args[@]}" || true
 	fi
 
-	if [ -n "${chaos_mesh_ns}" ] && sg microk8s -c "microk8s.helm3 repo list --namespace=${chaos_mesh_ns}" | grep -q 'chaos-mesh'; then
+	if [ -n "${chaos_mesh_ns}" ] && sg snap_microk8s -c "microk8s.helm3 repo list --namespace=${chaos_mesh_ns}" | grep -q 'chaos-mesh'; then
 		echo "uninstalling chaos-mesh helm repo"
-		sg microk8s -c "microk8s.helm3 uninstall chaos-mesh --namespace=\"${chaos_mesh_ns}\"" || true
+		sg snap_microk8s -c "microk8s.helm3 uninstall chaos-mesh --namespace=\"${chaos_mesh_ns}\"" || true
 	fi
 }
 

tests/integration/test_backups.py

@@ -7,6 +7,7 @@
 
 import boto3
 import pytest
+from ops import JujuVersion
 from pytest_operator.plugin import OpsTest
 
 from .helpers import (
@@ -225,7 +226,10 @@ async def test_restore_on_same_cluster(ops_test: OpsTest, cloud_credentials) ->
             action_name="restore", **{"backup-id": backups_by_cloud[cloud_name]}
         )
         result = await action.wait()
-        assert result.results.get("Code") == "0"
+        if JujuVersion.from_environ().has_secrets:
+            assert result.results.get("return-code") == 0
+        else:
+            assert result.results.get("Code") == "0"
 
         # ensure the correct inserted values exist
         logger.info(
@@ -340,7 +344,10 @@ async def test_restore_on_new_cluster(ops_test: OpsTest, cloud_credentials) -> N
             action_name="restore", **{"backup-id": backups_by_cloud[cloud_name]}
         )
         result = await action.wait()
-        assert result.results.get("Code") == "0"
+        if JujuVersion.from_environ().has_secrets:
+            assert result.results.get("return-code") == 0
+        else:
+            assert result.results.get("Code") == "0"
 
         # ensure the correct inserted values exist
         logger.info(