Merge pull request #717 from google/py-gh-actions

gh actions: improve python testing

.github/workflows/python-e2e-test.yml

@@ -1,14 +1,16 @@
-name: Python - integration tests
+name: Python - build and test python package
 on:
   push:
     branches:
       - 'main'
     paths:
       - 'python/**'
+      - 'tests_data/**'
       - '.github/workflows/**'
   pull_request:
     paths:
       - 'python/**'
+      - 'tests_data/**'
       - '.github/workflows/**'
 
 permissions:
@@ -18,8 +20,10 @@ jobs:
   e2e-testing:
     strategy:
       matrix:
-        python-version: [ "3.8.x", "3.9.x", "3.10.x", "3.11.x", "3.12.x" ]
-        os: [ "ubuntu-latest", "macos-latest", "windows-latest" ]
+        # python-version: [ "3.8.x", "3.9.x", "3.10.x", "3.11.x", "3.12.x" ]
+        python-version: [ "3.12.x" ]  # FIXME: add back other python versions
+        # os: [ "ubuntu-latest", "macos-latest", "windows-latest" ]
+        os: [ "ubuntu-latest" ]  # FIXME: add back other OSes
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4
@@ -29,23 +33,37 @@ jobs:
         with:
           python-version: '${{ matrix.python-version }}'
 
-      - name: Install poetry
-        uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439 # pin@v3
-        with:
-          poetry-version: "1.7.1"
+      - name: Install uv
+        run: curl -LsSf https://astral.sh/uv/0.4.7/install.sh | sh
+
+      # It's OK to use the default python version and not the specified one
+      - name: Install all projects dependencies
+        working-directory: python
+        run: uv sync --all-extras --dev
 
-      - name: Install the project dependencies
+      - name: Build the python package
         working-directory: python
-        run: poetry build
+        run: uv run ./scripts/build_python_package.py
 
       - name: List wheels
         working-directory: python
         run: python -c "import os; print('\n'.join(os.listdir('dist')))"
 
-      - name: Install magika
+      - name: Install the magika package
         working-directory: python
         run: python -m pip install $(python -c "import glob; print(glob.glob('dist/magika-*.whl')[0])")
         shell: bash # Allows for cross-platform
 
+      - name: Run magika --version
+        working-directory: python
+        run: magika --version
+        shell: bash # Allows for cross-platform
+
       - name: Run magika with tests_data
-        run: magika -r tests_data/ || exit 1
+        run: magika -r tests_data/basic
+
+      - name: Run "magika cli" quick tests
+        run: python ./python/scripts/run_quick_test_magika_cli.py
+
+      - name: Run "magika module" quick tests
+        run: python ./python/scripts/run_quick_test_magika_module.py

.github/workflows/python-test-suite.yml

@@ -0,0 +1,57 @@
+name: Python - run test suite
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - 'python/**'
+      - 'tests_data/**'
+      - '.github/workflows/**'
+  pull_request:
+    paths:
+      - 'python/**'
+      - 'tests_data/**'
+      - '.github/workflows/**'
+
+permissions:
+  contents: read
+
+jobs:
+  unit-testing:
+    strategy:
+      matrix:
+        python-version: [ "3.8.x", "3.9.x", "3.10.x", "3.11.x", "3.12.x" ]
+        os: [ "ubuntu-latest", "macos-latest" ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # pin@v5
+        with:
+          python-version: '${{ matrix.python-version }}'
+
+      - name: Install uv
+        run: curl -LsSf https://astral.sh/uv/0.4.7/install.sh | sh
+
+      - name: Install all projects dependencies (with the requested python version)
+        working-directory: python
+        run: uv sync --python $(which python3) --all-extras --dev
+
+      - name: Run ruff check
+        working-directory: python
+        run: uv run --python $(which python3) ruff check --verbose
+
+      - name: Run ruff format --check
+        working-directory: python
+        run: uv run --python $(which python3) ruff format --check --verbose
+
+      - name: Run mypy
+        working-directory: python
+        run: uv run --python $(which python3) mypy src/magika tests
+
+      - name: Run the python tests suite
+        working-directory: python
+        run: uv run --python $(which python3) pytest tests -m "not slow"

python/scripts/build_python_package.py

@@ -53,15 +53,13 @@ def main() -> None:
         patch_cargo_toml_with_version(rust_cli_cargo_toml_path, python_version)
 
         # update Cargo.lock
-        r = subprocess.run(
-            ["cargo", "check"],
-            cwd=rust_root_dir / "cli",
-        )
+        r = subprocess.run(["cargo", "check"], cwd=rust_root_dir / "cli", check=True)
 
         # build the package
         r = subprocess.run(
             ["uv", "build", "--out-dir", str(dist_output_dir)],
             cwd=python_root_dir,
+            check=True,
         )
         print(f"Python packages built at {dist_output_dir}")
     finally:
@@ -76,8 +74,9 @@ def main() -> None:
                 str(rust_cli_cargo_toml_path),
                 str(rust_cli_cargo_lock_path),
             ],
-            capture_output=True,
             cwd=repo_root_dir,
+            capture_output=True,
+            check=True,
         )
 
 

python/scripts/run_quick_test_magika_cli.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# Copyright 2023-2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script assumes that the `magika` python package has been already installed,
+and that the `magika` client is available in the PATH. The script tests that the
+`magika` client appears functional.
+
+This script should only rely on dependencies installed with `pip install
+magika`; this script is used as part of "build & test package" github action,
+and the dev dependencies are not available.
+"""
+
+import subprocess
+import sys
+from pathlib import Path
+
+import click
+
+
+@click.command()
+def main() -> None:
+    basic_tests_dir = (
+        Path(__file__).parent.parent.parent / "tests_data" / "basic"
+    ).resolve()
+
+    p = subprocess.run(
+        ["magika", "-r", "--label", str(basic_tests_dir)],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+
+    assert p.stderr == ""
+
+    with_error = False
+    lines = p.stdout.split("\n")
+    for line in lines:
+        line = line.strip()
+        if line == "":
+            continue
+        file_path_str, file_output_str = line.split(": ", 1)
+        file_path = Path(file_path_str)
+        output_label = file_output_str.strip().split(" ", 1)[0]
+        expected_label = file_path.parent.name
+        if expected_label != output_label:
+            with_error = True
+            print(
+                f"ERROR: Misprediction for {file_path}: expected_label={expected_label}, output_label={output_label}"
+            )
+
+    if with_error:
+        print("ERROR: There was at least one misprediction")
+        sys.exit(1)
+
+    print("All examples were predicted correctly")
+
+
+if __name__ == "__main__":
+    main()

python/scripts/run_quick_test_magika_module.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# Copyright 2023-2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script should only rely on dependencies installed with `pip install
+magika`; this script is used as part of "build & test package" github action,
+and the dev dependencies are not available.
+"""
+
+import sys
+from pathlib import Path
+
+import click
+
+from magika import Magika
+from magika.types.content_type_label import ContentTypeLabel
+
+
+@click.command()
+def main() -> None:
+    m = Magika()
+
+    res = m.identify_bytes(b"text")
+    assert res.dl.label == ContentTypeLabel.UNDEFINED
+    assert res.output.label == ContentTypeLabel.TXT
+    assert res.score == 1.0
+
+    res = m.identify_bytes(b"\xff\xff\xff")
+    assert res.dl.label == ContentTypeLabel.UNDEFINED
+    assert res.output.label == ContentTypeLabel.UNKNOWN
+    assert res.score == 1.0
+
+    basic_tests_dir = (
+        Path(__file__).parent.parent.parent / "tests_data" / "basic"
+    ).resolve()
+
+    files_paths = sorted(filter(lambda p: p.is_file(), basic_tests_dir.rglob("*")))
+
+    with_error = False
+    for file_path in files_paths:
+        res = m.identify_path(file_path)
+        output_label = res.output.label
+        expected_label = file_path.parent.name
+        if expected_label != output_label:
+            with_error = True
+            print(
+                f"ERROR: Misprediction for {file_path}: expected_label={expected_label}, output_label={output_label}"
+            )
+
+    if with_error:
+        print("ERROR: There was at least one misprediction")
+        sys.exit(1)
+
+    print("All examples were predicted correctly")
+
+
+if __name__ == "__main__":
+    main()