remove pathos

Jianhua-Wang · May 25, 2023 · ee8a639 · ee8a639
1 parent e5b6413
commit ee8a639
Show file tree

Hide file tree

Showing 5 changed files with 58 additions and 107 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,14 @@
 # Changelog
 
+## [0.3.2] - 2023-05-25
+
+### Added
+
+### Changed
+- remove pathos
+### Fixed
+
+
 ## [0.3.1] - 2023-05-25
 
 ### Added

diff --git a/easyfinemap/easyfinemap.py b/easyfinemap/easyfinemap.py
@@ -24,7 +24,7 @@
 
 import numpy as np
 import pandas as pd
-from pathos.pools import _ProcessPool as Pool
+from concurrent.futures import ProcessPoolExecutor
 from rich.progress import BarColumn, MofNCompleteColumn, Progress, TextColumn, TimeElapsedColumn
 import tabix
 import smunger as sg
@@ -689,6 +689,22 @@ def finemap_locus(
         credible_set[ColName.LEAD_SNP] = lead_snp
         return credible_set
 
+    def finemap_locus_parallel(self, kwargs):
+        """
+        Perform finemapping for a locus in parallel.
+
+        Parameters
+        ----------
+        kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        pd.DataFrame
+            Finemapping results.
+        """
+        return self.finemap_locus(**kwargs)
+
     def finemap_all_loci(
         self,
         sumstats: pd.DataFrame,
@@ -771,21 +787,35 @@ def finemap_all_loci(
             }
             kwargs_list.append(kwargs)
         ef = EasyFinemap()
-        output = []
-        with Progress(
-            TextColumn("{task.description}"),
-            BarColumn(),
-            MofNCompleteColumn(),
-            TimeElapsedColumn(),
-            auto_refresh=True,
-        ) as progress:
-            with Pool(threads) as p:
-                task = progress.add_task("Perform Fine-mapping...", total=len(loci))
-                results = [p.apply_async(ef.finemap_locus, kwds=kwargs) for kwargs in kwargs_list]
-                for res in results:
+        # output = []
+        # with Progress(
+        #     TextColumn("{task.description}"),
+        #     BarColumn(),
+        #     MofNCompleteColumn(),
+        #     TimeElapsedColumn(),
+        #     auto_refresh=True,
+        # ) as progress:
+        #     with Pool(threads) as p:
+        #         task = progress.add_task("Perform Fine-mapping...", total=len(loci))
+        #         results = [p.apply_async(ef.finemap_locus, kwds=kwargs) for kwargs in kwargs_list]
+        #         for res in results:
+        #             progress.update(task, advance=1)
+        #             progress.refresh()
+        #             output.append(res.get())
+        with ProcessPoolExecutor(max_workers=threads) as executor:
+            output = []
+            with Progress(
+                TextColumn("{task.description}"),
+                BarColumn(),
+                MofNCompleteColumn(),
+                TimeElapsedColumn(),
+                auto_refresh=True,
+            ) as progress:
+                task = progress.add_task("Perform Fine-mapping...", total=len(kwargs_list))
+                for _ in executor.map(ef.finemap_locus_parallel, kwargs_list):
                     progress.update(task, advance=1)
                     progress.refresh()
-                    output.append(res.get())
+                    output.append(_)
         output_df = pd.concat(output, ignore_index=True)
         if outfile:
             output_df.to_csv(outfile, sep="\t", index=False, float_format="%0.6g")

diff --git a/easyfinemap/loci.py b/easyfinemap/loci.py
@@ -441,7 +441,7 @@ def indep_snps_by_conditional(
                 BarColumn(),
                 MofNCompleteColumn(),
                 TimeElapsedColumn(),
-                auto_refresh=False,
+                auto_refresh=True,
             ) as progress:
                 task = progress.add_task("Run cojo-slct", total=len(args_list))
                 for _ in executor.map(loci.cojo_slct, *zip(*args_list)):

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -49,7 +49,6 @@ mkdocs-material-extensions = { version = "^1.1.1", optional = true }
 pandas = "^1.5.2"
 rich = "^13.3.1"
 jupyter = { version = "^1.0.0", optional = true }
-pathos = "^0.3.0"
 smunger = "0.0.23"
 rpy2 = "^3.5.10"