Merge pull request #115 from jguhlin/jg-branch-4

Fix cI?
jguhlin · Jan 7, 2025 · 42ec3ac · 42ec3ac
2 parents ea69a01 + e50de47
commit 42ec3ac
Show file tree

Hide file tree

Showing 5 changed files with 127 additions and 40 deletions.
diff --git a/.github/workflows/create-python-release.yaml b/.github/workflows/create-python-release.yaml
@@ -28,38 +28,60 @@ jobs:
     strategy:
       matrix:
         platform:
-          - runner: ubuntu-22.04
+          - runner: ubuntu-24.04
             target: x86_64
-          - runner: ubuntu-22.04
+          - runner: ubuntu-24.04
             target: x86
-          - runner: ubuntu-22.04
+          - runner: ubuntu-24.04
             target: aarch64
-          - runner: ubuntu-22.04
-            target: armv7
-          - runner: ubuntu-22.04
-            target: s390x
-          - runner: ubuntu-22.04
-            target: ppc64le
     steps:
+      - run: sudo apt-get update && sudo apt-get install -y build-essential
       - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
       - run: cd minimappers2
       - uses: actions/setup-python@v5
         with:
           python-version: 3.x
+
+      # Conditionally install AArch64 cross-compiler and libraries
+      - name: Install AArch64 Cross-Compiler and Libraries
+        if: matrix.platform.target == 'aarch64'
+        run: |
+          sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
+          sudo apt-get update
+          sudo apt-get install -y gcc-13-aarch64-linux-gnu gcc-13-aarch64-linux-gnu g++-13-aarch64-linux-gnu libc6-dev-arm64-cross
+
+      # Add the AArch64 Rust target
+      - name: Add AArch64 Rust Target
+        if: matrix.platform.target == 'aarch64'
+        run: rustup target add aarch64-unknown-linux-gnu
+
+      # Set environment variables for cross-compilation
+#      - name: Set Environment Variables for AArch64
+#        if: matrix.platform.target == 'aarch64'
+#        run: |
+#          echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc-13" >> $GITHUB_ENV
+#          echo "CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc-13" >> $GITHUB_ENV
+#          echo "CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++-13" >> $GITHUB_ENV
+
+
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
           args: --release --out dist
           sccache: 'true'
           manylinux: auto
-      - name: Build free-threaded wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.platform.target }}
-          args: --release --out dist -i python3.13t
-          sccache: 'true'
-          manylinux: auto
+          working-directory: "minimappers2"
+#      - name: Build free-threaded wheels
+#        uses: PyO3/maturin-action@v1
+#        with:
+#          target: ${{ matrix.platform.target }}
+#          args: --release --out dist -i python3.13t
+#          sccache: 'true'
+#          manylinux: auto
+#          working-directory: "minimappers2"
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
@@ -71,16 +93,17 @@ jobs:
     strategy:
       matrix:
         platform:
-          - runner: ubuntu-22.04
+          - runner: ubuntu-24.04
             target: x86_64
-          - runner: ubuntu-22.04
+          - runner: ubuntu-24.04
             target: x86
-          - runner: ubuntu-22.04
+          - runner: ubuntu-24.04
             target: aarch64
-          - runner: ubuntu-22.04
-            target: armv7
     steps:
+      - run: sudo apt-get update && sudo apt-get install -y build-essential musl-tools
       - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
       - run: cd minimappers2
       - uses: actions/setup-python@v5
         with:
@@ -92,13 +115,15 @@ jobs:
           args: --release --out dist
           sccache: 'true'
           manylinux: musllinux_1_2
-      - name: Build free-threaded wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.platform.target }}
-          args: --release --out dist -i python3.13t
-          sccache: 'true'
-          manylinux: musllinux_1_2
+          working-directory: "minimappers2"
+#      - name: Build free-threaded wheels
+#        uses: PyO3/maturin-action@v1
+#        with:
+#          target: ${{ matrix.platform.target }}
+#          args: --release --out dist -i python3.13t
+#          sccache: 'true'
+#          manylinux: musllinux_1_2
+#          working-directory: "minimappers2"
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
@@ -118,6 +143,8 @@ jobs:
             target: aarch64
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
       - run: cd minimappers2
       - uses: actions/setup-python@v5
         with:
@@ -128,12 +155,14 @@ jobs:
           target: ${{ matrix.platform.target }}
           args: --release --out dist
           sccache: 'true'
-      - name: Build free-threaded wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.platform.target }}
-          args: --release --out dist -i python3.13t
-          sccache: 'true'
+          working-directory: "minimappers2"
+#      - name: Build free-threaded wheels
+#        uses: PyO3/maturin-action@v1
+#        with:
+#          target: ${{ matrix.platform.target }}
+#          args: --release --out dist -i python3.13t
+#          sccache: 'true'
+#          working-directory: "minimappers2"
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
@@ -143,12 +172,16 @@ jobs:
   sdist:
     runs-on: ubuntu-latest
     steps:
+      - run: sudo apt-get update && sudo apt-get install -y build-essential
       - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
       - run: cd minimappers2
       - name: Build sdist
         uses: PyO3/maturin-action@v1
         with:
           command: sdist
+          working-directory: "minimappers2"
           args: --out dist
       - name: Upload sdist
         uses: actions/upload-artifact@v4
@@ -169,6 +202,7 @@ jobs:
       # Used to generate artifact attestation
       attestations: write
     steps:
+      - run: sudo apt-get update && sudo apt-get install -y build-essential
       - uses: actions/download-artifact@v4
       - run: cd minimappers2
       - name: Generate artifact attestation
@@ -181,5 +215,6 @@ jobs:
         env:
           MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
         with:
+          working-directory: "minimappers2"
           command: upload
           args: --non-interactive --skip-existing wheels-*/*
diff --git a/minimappers2/Cargo.toml b/minimappers2/Cargo.toml
@@ -10,12 +10,20 @@ crate-type = ["cdylib", "rlib"]
 [dependencies]
 minimap2 = { version = "0.1.23", features = ["simde"], path = ".." }
 crossbeam = "0.8.4"
-mimalloc = {version = "0.1", default-features = false }
-
-pyo3 = { version = "0.22" }
+pyo3 = { version = "0.22", features = ["abi3-py39", "chrono", "extension-module"] }
 polars = "0.45"
 pyo3-polars = "0.19"
 
+[target.'cfg(all(any(not(target_family = "unix"), target_os = "emscripten", allocator = "mimalloc"), not(allocator = "default")))'.dependencies]
+mimalloc = { version = "0.1", default-features = false }
+
+# Feature background_threads is unsupported on MacOS (https://github.com/jemalloc/jemalloc/issues/843).
+[target.'cfg(all(target_family = "unix", not(target_os = "macos"), not(target_os = "emscripten"), not(allocator = "mimalloc"), not(allocator = "default")))'.dependencies]
+jemallocator = { version = "0.5", features = ["disable_initial_exec_tls", "background_threads"] }
+
+[target.'cfg(all(target_family = "unix", target_os = "macos", not(allocator = "mimalloc"), not(allocator = "default")))'.dependencies]
+jemallocator = { version = "0.5", features = ["disable_initial_exec_tls"] }
+
 [profile.release]
 opt-level = 3
 lto = "fat"

diff --git a/minimappers2/build.rs b/minimappers2/build.rs
@@ -0,0 +1,7 @@
+fn main() {
+    println!("cargo::rustc-check-cfg=cfg(allocator, values(\"default\", \"mimalloc\"))");
+    println!(
+        "cargo:rustc-env=TARGET={}",
+        std::env::var("TARGET").unwrap()
+    );
+}
diff --git a/minimappers2/pyproject.toml b/minimappers2/pyproject.toml
@@ -9,7 +9,7 @@ classifiers = [
     "Operating System :: POSIX :: Linux",
     "Programming Language :: Rust"
 ]
-license = { file = "LICENSE" }
+# license = { file = "LICENSE" }
 description = "A Python wrapper for minimap2-rs"
 keywords = ["minimap2", "bioinformatics", "alignment", "mapping"]
 dependencies = [

diff --git a/minimappers2/src/lib.rs b/minimappers2/src/lib.rs
@@ -2,15 +2,52 @@ use std::num::NonZeroI32;
 use std::sync::{Arc, Mutex};
 
 use crossbeam::queue::ArrayQueue;
-use mimalloc::MiMalloc;
 use minimap2::*;
 
 use polars::{df, prelude::*};
 use pyo3::prelude::*;
 use pyo3_polars::{error::PyPolarsErr, PyDataFrame};
 
+#[cfg(all(
+    target_family = "unix",
+    not(target_os = "emscripten"),
+    not(allocator = "default"),
+    not(allocator = "mimalloc"),
+))]
+use jemallocator::Jemalloc;
+#[cfg(all(
+    not(debug_assertions),
+    not(allocator = "default"),
+    any(
+        not(target_family = "unix"),
+        target_os = "emscripten",
+        allocator = "mimalloc"
+    ),
+))]
+use mimalloc::MiMalloc;
+
 #[global_allocator]
-static GLOBAL: MiMalloc = MiMalloc;
+#[cfg(all(
+    not(debug_assertions),
+    not(allocator = "mimalloc"),
+    not(allocator = "default"),
+    target_family = "unix",
+    not(target_os = "emscripten"),
+))]
+static ALLOC: Jemalloc = Jemalloc;
+
+#[global_allocator]
+#[cfg(all(
+    not(debug_assertions),
+    not(allocator = "default"),
+    any(
+        not(target_family = "unix"),
+        target_os = "emscripten",
+        allocator = "mimalloc"
+    ),
+))]
+static ALLOC: MiMalloc = MiMalloc;
+
 
 mod multithreading;