Merge pull request #909 from google/pure-python-wheel

Build pure python wheel

.github/workflows/python-build-package.yml

@@ -16,7 +16,7 @@ permissions:
   contents: read
 
 jobs:
-  build:
+  build-wheels:
     runs-on: ${{ matrix.platform.runner }}
     strategy:
       matrix:
@@ -34,7 +34,7 @@ jobs:
         with:
           python-version: '3.12'
       - name: Install uv
-        run: curl -LsSf https://astral.sh/uv/0.4.7/install.sh | sh
+        run: curl -LsSf https://astral.sh/uv/0.5.22/install.sh | sh
       - if: matrix.platform.runner == 'ubuntu-latest'
         uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
         with:
@@ -76,35 +76,37 @@ jobs:
       - run: "python3 -c 'import magika; m = magika.Magika(); print(m)'"
       - run: magika -r tests_data/basic
       - run: python3 ./python/scripts/run_quick_test_magika_cli.py
-        # Windows' onnxruntime/ort returns different results wrt. Linux/MacOS. For
-        # now keep building even in case of misdetections.
-        # TODO(https://github.com/google/magika/issues/892): remove when this is fixed.
-        continue-on-error: ${{ matrix.platform.runner == 'windows-latest' }}
       - run: python3 ./python/scripts/run_quick_test_magika_module.py
-        # TODO(https://github.com/google/magika/issues/892): same as above,
-        # remove when this is fixed.
-        continue-on-error: ${{ matrix.platform.runner == 'windows-latest' }}
       - name: Upload wheels
         if: github.event_name != 'pull_request'
         uses: actions/upload-artifact@v4
         with:
           name: wheels-${{ matrix.platform.runner }}-${{ matrix.platform.target }}
           path: dist
 
-  sdist:
+  build-pure-python-wheel:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - run: python3 ./python/scripts/fix_package_version.py
-      - name: Build sdist
-        uses: PyO3/maturin-action@v1
-        with:
-          command: sdist
-          args: --out=../dist
-          working-directory: python
-      - name: Upload sdist
+      - name: Install uv
+        run: curl -LsSf https://astral.sh/uv/0.5.22/install.sh | sh
+      - run: uv run ./scripts/prepare_pyproject_for_pure_python_wheel.py
+        working-directory: python
+      - name: Build pure python wheel
+        run: uv build --wheel --out-dir ../dist
+        working-directory: python
+        name: Check that `uv add magika.whl` works
+        run: mkdir /tmp/test-uv && cp -vR dist/*.whl /tmp/test-uv && cd /tmp/test-uv && uv init && uv add ./$(\ls -1 *.whl | head -n 1)
+      - name: Install wheels
+        run: python3 -m pip install $(python -c "import glob; print(glob.glob('dist/*.whl')[0])")
+      - run: magika --version
+      - run: "python3 -c 'import magika; m = magika.Magika(); print(m)'"
+      - run: magika -r tests_data/basic
+      - run: python3 ./python/scripts/run_quick_test_magika_cli.py
+      - run: python3 ./python/scripts/run_quick_test_magika_module.py
+      - name: Upload wheel
         if: github.event_name != 'pull_request'
         uses: actions/upload-artifact@v4
         with:
-          name: wheels-sdist
+          name: wheels-pure
           path: dist

.github/workflows/python-test-suite.yml

@@ -34,7 +34,7 @@ jobs:
           python-version: '${{ matrix.python-version }}'
 
       - name: Install uv
-        run: curl -LsSf https://astral.sh/uv/0.4.7/install.sh | sh
+        run: curl -LsSf https://astral.sh/uv/0.5.22/install.sh | sh
 
       - name: Install all projects dependencies (with the requested python version)
         working-directory: python

python/pyproject.toml

@@ -2,10 +2,10 @@
 name = "magika"
 description = "A tool to determine the content type of a file with deep learning"
 authors = [
-    {author = "Magika Developers", email = "[email protected]"},
+    {name = "Magika Developers", email = "[email protected]"},
 ]
 readme = "README.md"
-license = "Apache-2.0"
+license = {"text" = "Apache-2.0"}
 requires-python = ">=3.8"
 keywords = ["content type detection", "machine learning"]
 classifiers = [
@@ -57,6 +57,8 @@ dev-dependencies = [
     "pytest>=8.3.2",
     "ruff>=0.6.3",
     "twine>=5.1.1",
+    "tomli-w>=1.0.0",
+    "tomli>=2.0.1",
 ]
 
 [build-system]

python/scripts/prepare_pyproject_for_pure_python_wheel.py

@@ -0,0 +1,45 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from pathlib import Path
+
+import tomli
+import tomli_w
+
+
+def main() -> None:
+    pyproject_toml_path = Path(__file__).parent.parent / "pyproject.toml"
+
+    pyproject_content = tomli.loads(pyproject_toml_path.read_text())
+
+    # Remove entry about maturin, we don't need it
+    _ = pyproject_content["tool"].pop("maturin")
+
+    # Tell uv we want to use the hatchling build system
+    pyproject_content["build-system"] = {
+        "requires": ["hatchling"],
+        "build-backend": "hatchling.build",
+    }
+
+    # Make the python's magika client available as a script
+    pyproject_content["project"]["scripts"] = {
+        "magika": "magika.cli.magika_client:main",
+    }
+
+    pyproject_toml_path.write_text(tomli_w.dumps(pyproject_content))
+
+
+if __name__ == "__main__":
+    main()

python/scripts/run_quick_test_magika_cli.py

@@ -26,19 +26,36 @@
 import subprocess
 import sys
 from pathlib import Path
+from typing import Optional
 
 import click
 
 
 @click.command()
-def main() -> None:
+@click.option(
+    "--client-path",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True),
+)
+def main(client_path: Optional[Path]) -> None:
+    """Tests the Rust or Python Magika client. By default, it runs "magika"
+    (expected in PATH). Use --client-path to specify a different client
+    executable.
+    """
+
     basic_tests_dir = (
         Path(__file__).resolve().parent.parent.parent / "tests_data" / "basic"
     )
     assert basic_tests_dir.is_dir()
 
+    if client_path is None:
+        client_path = Path("magika")
+    print(f'Testing client: "{client_path}"')
+
+    p = subprocess.run([str(client_path), "--version"], capture_output=True, text=True)
+    print(f'Output of "magika --version": {p.stdout.strip()}')
+
     p = subprocess.run(
-        ["magika", "-r", "--label", str(basic_tests_dir)],
+        [str(client_path), "-r", "--label", "--no-colors", str(basic_tests_dir)],
         capture_output=True,
         text=True,
     )

python/scripts/magika_python_module_tester.py → python/src/magika/cli/magika_client.py

@@ -154,16 +154,14 @@ def main(
         with_colors = False
 
     _l = get_logger(use_colors=with_colors)
-    _l.warning(
-        "This CLI is deprecated and only used for testing the python module! Use the Rust CLI instead."
-    )
 
     if verbose:
         _l.setLevel(logging.INFO)
     if debug:
         _l.setLevel(logging.DEBUG)
 
     if output_version:
+        _l.raw_print_to_stdout("Magika python client")
         _l.raw_print_to_stdout(f"Magika version: {VERSION}")
         _l.raw_print_to_stdout(f"Default model: {Magika._get_default_model_name()}")
         sys.exit(0)

python/tests/test_python_module_tester.py → python/tests/test_python_magika_client.py

@@ -16,16 +16,16 @@
 from pathlib import Path
 
 
-def test_python_module_tester() -> None:
+def test_python_magika_client() -> None:
     python_root_dir = Path(__file__).parent.parent
-    python_module_tester_path = (
-        python_root_dir / "scripts" / "magika_python_module_tester.py"
+    python_magika_client_path = (
+        python_root_dir / "src" / "magika" / "cli" / "magika_client.py"
     ).resolve()
 
     # quick test to check there are no obvious problems
-    cmd = [str(python_module_tester_path), "--help"]
+    cmd = [str(python_magika_client_path), "--help"]
     subprocess.run(cmd, capture_output=True, check=True)
 
     # quick test to check there are no crashes
-    cmd = [str(python_module_tester_path), str(python_module_tester_path)]
+    cmd = [str(python_magika_client_path), str(python_magika_client_path)]
     subprocess.run(cmd, capture_output=True, check=True)