From 619ed4716c8e91e533b43802ecb1a459627c7172 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Aug 2023 12:40:49 +0000 Subject: [PATCH 01/58] Bump tqdm from 4.65.0 to 4.65.1 Bumps [tqdm](https://github.com/tqdm/tqdm) from 4.65.0 to 4.65.1. - [Release notes](https://github.com/tqdm/tqdm/releases) - [Commits](https://github.com/tqdm/tqdm/compare/v4.65.0...v4.65.1) --- updated-dependencies: - dependency-name: tqdm dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 379b4bb7..11b43be2 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires=[ 'cloudpickle==2.2.1', 'mpi4py==3.1.4', - 'tqdm==4.65.0', + 'tqdm==4.65.1', 'pyzmq==25.1.0' ], cmdclass=versioneer.get_cmdclass(), From 9590250a86df78730777f939ca3415c5d4286d4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Tue, 8 Aug 2023 06:45:04 -0600 Subject: [PATCH 02/58] update conda --- .ci_support/environment-mpich.yml | 2 +- .ci_support/environment-openmpi.yml | 2 +- .ci_support/environment-win.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci_support/environment-mpich.yml b/.ci_support/environment-mpich.yml index 9d2471d1..5adcfcc9 100644 --- a/.ci_support/environment-mpich.yml +++ b/.ci_support/environment-mpich.yml @@ -6,5 +6,5 @@ dependencies: - mpich - cloudpickle =2.2.1 - mpi4py =3.1.4 -- tqdm =4.65.0 +- tqdm =4.65.1 - pyzmq =25.1.0 diff --git a/.ci_support/environment-openmpi.yml b/.ci_support/environment-openmpi.yml index 501bf88c..053f81af 100644 --- a/.ci_support/environment-openmpi.yml +++ b/.ci_support/environment-openmpi.yml @@ -6,5 +6,5 @@ dependencies: - openmpi - cloudpickle =2.2.1 - mpi4py =3.1.4 -- tqdm =4.65.0 +- tqdm =4.65.1 - pyzmq =25.1.0 diff --git a/.ci_support/environment-win.yml b/.ci_support/environment-win.yml index a14be6b7..74616589 100644 --- a/.ci_support/environment-win.yml +++ b/.ci_support/environment-win.yml @@ -6,5 +6,5 @@ dependencies: - msmpi - cloudpickle =2.2.1 - mpi4py =3.1.4 -- tqdm =4.65.0 +- tqdm =4.65.1 - pyzmq =25.1.0 From b55ab62415096d9cfa62b1b3e6a4feeaac6d7c60 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Aug 2023 12:23:56 +0000 Subject: [PATCH 03/58] Bump tqdm from 4.65.1 to 4.66.0 Bumps [tqdm](https://github.com/tqdm/tqdm) from 4.65.1 to 4.66.0. - [Release notes](https://github.com/tqdm/tqdm/releases) - [Commits](https://github.com/tqdm/tqdm/compare/v4.65.1...v4.66.0) --- updated-dependencies: - dependency-name: tqdm dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 11b43be2..e3612dfc 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires=[ 'cloudpickle==2.2.1', 'mpi4py==3.1.4', - 'tqdm==4.65.1', + 'tqdm==4.66.0', 'pyzmq==25.1.0' ], cmdclass=versioneer.get_cmdclass(), From 6190ea0fa883be0f8d6746c4764cdef5a251ff30 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 9 Aug 2023 07:39:14 -0600 Subject: [PATCH 04/58] Update dependencies --- .ci_support/environment-mpich.yml | 2 +- .ci_support/environment-openmpi.yml | 2 +- .ci_support/environment-win.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci_support/environment-mpich.yml b/.ci_support/environment-mpich.yml index 5adcfcc9..37df26a6 100644 --- a/.ci_support/environment-mpich.yml +++ b/.ci_support/environment-mpich.yml @@ -6,5 +6,5 @@ dependencies: - mpich - cloudpickle =2.2.1 - mpi4py =3.1.4 -- tqdm =4.65.1 +- tqdm =4.66.0 - pyzmq =25.1.0 diff --git a/.ci_support/environment-openmpi.yml b/.ci_support/environment-openmpi.yml index 053f81af..da465c42 100644 --- a/.ci_support/environment-openmpi.yml +++ b/.ci_support/environment-openmpi.yml @@ -6,5 +6,5 @@ dependencies: - openmpi - cloudpickle =2.2.1 - mpi4py =3.1.4 -- tqdm =4.65.1 +- tqdm =4.66.0 - pyzmq =25.1.0 diff --git a/.ci_support/environment-win.yml b/.ci_support/environment-win.yml index 74616589..0d9a37bd 100644 --- a/.ci_support/environment-win.yml +++ b/.ci_support/environment-win.yml @@ -6,5 +6,5 @@ dependencies: - msmpi - cloudpickle =2.2.1 - mpi4py =3.1.4 -- tqdm =4.65.1 +- tqdm =4.66.0 - pyzmq =25.1.0 From a646c93ec1cea61ee086eebbab76abb156d3881b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Aug 2023 12:45:42 +0000 Subject: [PATCH 05/58] Bump pyzmq from 25.1.0 to 25.1.1 Bumps [pyzmq](https://github.com/zeromq/pyzmq) from 25.1.0 to 25.1.1. - [Release notes](https://github.com/zeromq/pyzmq/releases) - [Commits](https://github.com/zeromq/pyzmq/compare/v25.1.0...v25.1.1) --- updated-dependencies: - dependency-name: pyzmq dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e3612dfc..b5485119 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ 'cloudpickle==2.2.1', 'mpi4py==3.1.4', 'tqdm==4.66.0', - 'pyzmq==25.1.0' + 'pyzmq==25.1.1' ], cmdclass=versioneer.get_cmdclass(), ) From 6fe3d49582bc19c6a7b9debd084be60e3c3775d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Aug 2023 12:45:46 +0000 Subject: [PATCH 06/58] Bump tqdm from 4.66.0 to 4.66.1 Bumps [tqdm](https://github.com/tqdm/tqdm) from 4.66.0 to 4.66.1. - [Release notes](https://github.com/tqdm/tqdm/releases) - [Commits](https://github.com/tqdm/tqdm/compare/v4.66.0...v4.66.1) --- updated-dependencies: - dependency-name: tqdm dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e3612dfc..69abc6c0 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires=[ 'cloudpickle==2.2.1', 'mpi4py==3.1.4', - 'tqdm==4.66.0', + 'tqdm==4.66.1', 'pyzmq==25.1.0' ], cmdclass=versioneer.get_cmdclass(), From 4c64629339402225064f51fee5e661c8b18aa975 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Thu, 10 Aug 2023 07:14:00 -0600 Subject: [PATCH 07/58] update environments --- .ci_support/environment-mpich.yml | 2 +- .ci_support/environment-openmpi.yml | 2 +- .ci_support/environment-win.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci_support/environment-mpich.yml b/.ci_support/environment-mpich.yml index 37df26a6..b5e1bd45 100644 --- a/.ci_support/environment-mpich.yml +++ b/.ci_support/environment-mpich.yml @@ -7,4 +7,4 @@ dependencies: - cloudpickle =2.2.1 - mpi4py =3.1.4 - tqdm =4.66.0 -- pyzmq =25.1.0 +- pyzmq =25.1.1 diff --git a/.ci_support/environment-openmpi.yml b/.ci_support/environment-openmpi.yml index da465c42..1f8ab0f8 100644 --- a/.ci_support/environment-openmpi.yml +++ b/.ci_support/environment-openmpi.yml @@ -7,4 +7,4 @@ dependencies: - cloudpickle =2.2.1 - mpi4py =3.1.4 - tqdm =4.66.0 -- pyzmq =25.1.0 +- pyzmq =25.1.1 diff --git a/.ci_support/environment-win.yml b/.ci_support/environment-win.yml index 0d9a37bd..bbde74b3 100644 --- a/.ci_support/environment-win.yml +++ b/.ci_support/environment-win.yml @@ -7,4 +7,4 @@ dependencies: - cloudpickle =2.2.1 - mpi4py =3.1.4 - tqdm =4.66.0 -- pyzmq =25.1.0 +- pyzmq =25.1.1 From cb58fafd5eb118d91a94a0ce911ab0e8f7ff1156 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Thu, 10 Aug 2023 07:14:33 -0600 Subject: [PATCH 08/58] update environments --- .ci_support/environment-mpich.yml | 2 +- .ci_support/environment-openmpi.yml | 2 +- .ci_support/environment-win.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci_support/environment-mpich.yml b/.ci_support/environment-mpich.yml index 37df26a6..3c100618 100644 --- a/.ci_support/environment-mpich.yml +++ b/.ci_support/environment-mpich.yml @@ -6,5 +6,5 @@ dependencies: - mpich - cloudpickle =2.2.1 - mpi4py =3.1.4 -- tqdm =4.66.0 +- tqdm =4.66.1 - pyzmq =25.1.0 diff --git a/.ci_support/environment-openmpi.yml b/.ci_support/environment-openmpi.yml index da465c42..fae18170 100644 --- a/.ci_support/environment-openmpi.yml +++ b/.ci_support/environment-openmpi.yml @@ -6,5 +6,5 @@ dependencies: - openmpi - cloudpickle =2.2.1 - mpi4py =3.1.4 -- tqdm =4.66.0 +- tqdm =4.66.1 - pyzmq =25.1.0 diff --git a/.ci_support/environment-win.yml b/.ci_support/environment-win.yml index 0d9a37bd..56f80d75 100644 --- a/.ci_support/environment-win.yml +++ b/.ci_support/environment-win.yml @@ -6,5 +6,5 @@ dependencies: - msmpi - cloudpickle =2.2.1 - mpi4py =3.1.4 -- tqdm =4.66.0 +- tqdm =4.66.1 - pyzmq =25.1.0 From b80b2d7323327dace03ee7d7420fedbccfa3a01a Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Fri, 11 Aug 2023 16:12:44 -0600 Subject: [PATCH 09/58] Add interface_bootup() to pympipool.shared as well as the MpiExecInterface and the SlurmSubprocessInterface --- pympipool/shared/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pympipool/shared/__init__.py b/pympipool/shared/__init__.py index 9e765c95..73fadf43 100644 --- a/pympipool/shared/__init__.py +++ b/pympipool/shared/__init__.py @@ -1,5 +1,6 @@ from pympipool.shared.communication import ( SocketInterface, + interface_bootup, interface_connect, interface_send, interface_shutdown, @@ -7,3 +8,4 @@ ) from pympipool.shared.executorbase import cancel_items_in_queue from pympipool.shared.thread import RaisingThread +from pympipool.shared.interface import MpiExecInterface, SlurmSubprocessInterface From 014b2b658f9e7048bb224c373e46030a36484a35 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Fri, 11 Aug 2023 16:17:27 -0600 Subject: [PATCH 10/58] Clean up get_backend_path() --- pympipool/shared/executorbase.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pympipool/shared/executorbase.py b/pympipool/shared/executorbase.py index c2eea3f0..8f2aafc2 100644 --- a/pympipool/shared/executorbase.py +++ b/pympipool/shared/executorbase.py @@ -154,18 +154,16 @@ def execute_task_dict(task_dict, meta_future_lst): raise ValueError("Unrecognized Task in task_dict: ", task_dict) +def _get_command_path(executable): + return os.path.abspath(os.path.join(__file__, "..", "..", "backend", executable)) + + def get_backend_path(cores): command_lst = [sys.executable] if cores > 1: - command_lst += [ - os.path.abspath( - os.path.join(__file__, "..", "..", "backend", "mpiexec.py") - ), - ] + command_lst += [_get_command_path(executable="mpiexec.py")] else: - command_lst += [ - os.path.abspath(os.path.join(__file__, "..", "..", "backend", "serial.py")), - ] + command_lst += [_get_command_path(executable="serial.py")] return command_lst From c86e42201821dd88ba1b1f4d26425d8cffe59aae Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Fri, 11 Aug 2023 16:58:53 -0600 Subject: [PATCH 11/58] At least add the arguments for the executors --- pympipool/flux/fluxbroker.py | 11 +++++++++++ pympipool/mpi/mpibroker.py | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/pympipool/flux/fluxbroker.py b/pympipool/flux/fluxbroker.py index 0439549d..46e6fd76 100644 --- a/pympipool/flux/fluxbroker.py +++ b/pympipool/flux/fluxbroker.py @@ -8,6 +8,17 @@ class PyFluxExecutor(ExecutorBase): + """ + Args: + max_workers (int): defines the number workers which can execute functions in parallel + cores_per_worker (int): number of MPI cores to be used for each function call + threads_per_core (int): number of OpenMP threads to be used for each function call + gpus_per_worker (int): number of GPUs per worker - defaults to 0 + init_function (None): optional function to preset arguments for functions which are submitted later + cwd (str/None): current working directory where the parallel python task is executed + sleep_interval (float): synchronization interval - default 0.1 + executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux + """ def __init__( self, max_workers, diff --git a/pympipool/mpi/mpibroker.py b/pympipool/mpi/mpibroker.py index d879923c..4e22459f 100644 --- a/pympipool/mpi/mpibroker.py +++ b/pympipool/mpi/mpibroker.py @@ -8,6 +8,18 @@ class PyMPIExecutor(ExecutorBase): + """ + Args: + max_workers (int): defines the number workers which can execute functions in parallel + cores_per_worker (int): number of MPI cores to be used for each function call + threads_per_core (int): number of OpenMP threads to be used for each function call + gpus_per_worker (int): number of GPUs per worker - defaults to 0 + oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI only) - default False + init_function (None): optional function to preset arguments for functions which are submitted later + cwd (str/None): current working directory where the parallel python task is executed + sleep_interval (float): synchronization interval - default 0.1 + enable_slurm_backend (bool): enable the SLURM queueing system as backend - defaults to False + """ def __init__( self, max_workers, From edede6927954af108be4ddab3bb459be64b2868c Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Fri, 11 Aug 2023 17:13:36 -0600 Subject: [PATCH 12/58] black formatting --- pympipool/flux/fluxbroker.py | 1 + pympipool/mpi/mpibroker.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pympipool/flux/fluxbroker.py b/pympipool/flux/fluxbroker.py index 46e6fd76..ac96332f 100644 --- a/pympipool/flux/fluxbroker.py +++ b/pympipool/flux/fluxbroker.py @@ -19,6 +19,7 @@ class PyFluxExecutor(ExecutorBase): sleep_interval (float): synchronization interval - default 0.1 executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux """ + def __init__( self, max_workers, diff --git a/pympipool/mpi/mpibroker.py b/pympipool/mpi/mpibroker.py index 4e22459f..76c26797 100644 --- a/pympipool/mpi/mpibroker.py +++ b/pympipool/mpi/mpibroker.py @@ -20,6 +20,7 @@ class PyMPIExecutor(ExecutorBase): sleep_interval (float): synchronization interval - default 0.1 enable_slurm_backend (bool): enable the SLURM queueing system as backend - defaults to False """ + def __init__( self, max_workers, From 98a8652b34f1f0551fb7e07865254bd1de85173e Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Fri, 11 Aug 2023 17:40:11 -0600 Subject: [PATCH 13/58] Separate the input for the broker and the executor --- pympipool/flux/fluxbroker.py | 35 ++++++++++------------------ pympipool/mpi/mpibroker.py | 39 +++++++++++--------------------- pympipool/shared/executorbase.py | 26 ++++++++++++++------- tests/test_meta.py | 12 +++++----- 4 files changed, 49 insertions(+), 63 deletions(-) diff --git a/pympipool/flux/fluxbroker.py b/pympipool/flux/fluxbroker.py index ac96332f..330f7c5c 100644 --- a/pympipool/flux/fluxbroker.py +++ b/pympipool/flux/fluxbroker.py @@ -1,7 +1,6 @@ from pympipool.shared.executorbase import ( ExecutorBase, executor_broker, - get_executor_dict, ) from pympipool.shared.thread import RaisingThread from pympipool.flux.fluxtask import PyFluxSingleTaskExecutor @@ -37,13 +36,15 @@ def __init__( kwargs={ "future_queue": self._future_queue, "max_workers": max_workers, - "cores_per_worker": cores_per_worker, - "threads_per_core": threads_per_core, - "gpus_per_worker": gpus_per_worker, - "init_function": init_function, - "cwd": cwd, "sleep_interval": sleep_interval, - "executor": executor, + "executor_kwargs": { + "cores": cores_per_worker, + "threads_per_core": threads_per_core, + "gpus_per_task": int(gpus_per_worker / cores_per_worker), + "init_function": init_function, + "cwd": cwd, + "executor": executor, + }, }, ) self._process.start() @@ -52,25 +53,13 @@ def __init__( def _flux_executor_broker( future_queue, max_workers, - cores_per_worker=1, - threads_per_core=1, - gpus_per_worker=0, - init_function=None, - cwd=None, + executor_kwargs, sleep_interval=0.1, - executor=None, ): executor_broker( future_queue=future_queue, - meta_future_lst=get_executor_dict( - max_workers=max_workers, - executor_class=PyFluxSingleTaskExecutor, - cores=cores_per_worker, - threads_per_core=threads_per_core, - gpus_per_task=int(gpus_per_worker / cores_per_worker), - init_function=init_function, - cwd=cwd, - executor=executor, - ), + max_workers=max_workers, + executor_class=PyFluxSingleTaskExecutor, + executor_kwargs=executor_kwargs, sleep_interval=sleep_interval, ) diff --git a/pympipool/mpi/mpibroker.py b/pympipool/mpi/mpibroker.py index 76c26797..3a349085 100644 --- a/pympipool/mpi/mpibroker.py +++ b/pympipool/mpi/mpibroker.py @@ -1,7 +1,6 @@ from pympipool.shared.executorbase import ( ExecutorBase, executor_broker, - get_executor_dict, ) from pympipool.shared.thread import RaisingThread from pympipool.mpi.mpitask import PyMPISingleTaskExecutor @@ -50,14 +49,16 @@ def __init__( kwargs={ "future_queue": self._future_queue, "max_workers": max_workers, - "cores_per_worker": cores_per_worker, - "threads_per_core": threads_per_core, - "gpus_per_worker": gpus_per_worker, - "oversubscribe": oversubscribe, - "init_function": init_function, - "cwd": cwd, "sleep_interval": sleep_interval, - "enable_slurm_backend": enable_slurm_backend, + "executor_kwargs": { + "cores": cores_per_worker, + "threads_per_core": threads_per_core, + "gpus_per_task": int(gpus_per_worker / cores_per_worker), + "oversubscribe": oversubscribe, + "init_function": init_function, + "cwd": cwd, + "enable_slurm_backend": enable_slurm_backend, + }, }, ) self._process.start() @@ -66,27 +67,13 @@ def __init__( def _mpi_executor_broker( future_queue, max_workers, - cores_per_worker=1, - threads_per_core=1, - gpus_per_worker=0, - oversubscribe=False, - init_function=None, - cwd=None, + executor_kwargs, sleep_interval=0.1, - enable_slurm_backend=False, ): executor_broker( future_queue=future_queue, - meta_future_lst=get_executor_dict( - max_workers=max_workers, - executor_class=PyMPISingleTaskExecutor, - cores=cores_per_worker, - threads_per_core=threads_per_core, - gpus_per_task=int(gpus_per_worker / cores_per_worker), - oversubscribe=oversubscribe, - init_function=init_function, - cwd=cwd, - enable_slurm_backend=enable_slurm_backend, - ), + max_workers=max_workers, + executor_class=PyMPISingleTaskExecutor, + executor_kwargs=executor_kwargs, sleep_interval=sleep_interval, ) diff --git a/pympipool/shared/executorbase.py b/pympipool/shared/executorbase.py index 8f2aafc2..eb9d5c93 100644 --- a/pympipool/shared/executorbase.py +++ b/pympipool/shared/executorbase.py @@ -123,9 +123,16 @@ def execute_parallel_tasks_loop(interface, future_queue): def executor_broker( future_queue, - meta_future_lst, + max_workers, + executor_class, + executor_kwargs, sleep_interval=0.1, ): + meta_future_lst = _get_executor_dict( + max_workers=max_workers, + executor_class=executor_class, + executor_kwargs=executor_kwargs, + ) while True: try: task_dict = future_queue.get_nowait() @@ -154,10 +161,6 @@ def execute_task_dict(task_dict, meta_future_lst): raise ValueError("Unrecognized Task in task_dict: ", task_dict) -def _get_command_path(executable): - return os.path.abspath(os.path.join(__file__, "..", "..", "backend", executable)) - - def get_backend_path(cores): command_lst = [sys.executable] if cores > 1: @@ -167,11 +170,18 @@ def get_backend_path(cores): return command_lst -def get_executor_dict(max_workers, executor_class, **kwargs): - return {get_future_done(): executor_class(**kwargs) for _ in range(max_workers)} +def _get_command_path(executable): + return os.path.abspath(os.path.join(__file__, "..", "..", "backend", executable)) + + +def _get_executor_dict(max_workers, executor_class, executor_kwargs): + return { + _get_future_done(): executor_class(**executor_kwargs) + for _ in range(max_workers) + } -def get_future_done(): +def _get_future_done(): f = Future() f.set_result(True) return f diff --git a/tests/test_meta.py b/tests/test_meta.py index e11d13c1..70528f1a 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -3,8 +3,8 @@ import unittest from pympipool.shared.executorbase import ( execute_task_dict, - get_executor_dict, - get_future_done, + _get_executor_dict, + _get_future_done, ) from pympipool.mpi.mpitask import PyMPISingleTaskExecutor from pympipool.mpi.mpibroker import ( @@ -27,14 +27,14 @@ def mpi_funct(i): class TestFutureCreation(unittest.TestCase): def test_get_future_done(self): - f = get_future_done() + f = _get_future_done() self.assertTrue(isinstance(f, Future)) self.assertTrue(f.done()) class TestMetaExecutorFuture(unittest.TestCase): def test_meta_executor_future(self): - meta_future = get_executor_dict( + meta_future = _get_executor_dict( max_workers=1, executor_class=PyMPISingleTaskExecutor, ) @@ -47,7 +47,7 @@ def test_meta_executor_future(self): executor_obj.shutdown(wait=True) def test_execute_task_dict(self): - meta_future_lst = get_executor_dict( + meta_future_lst = _get_executor_dict( max_workers=1, executor_class=PyMPISingleTaskExecutor, ) @@ -68,7 +68,7 @@ def test_execute_task_dict(self): ) def test_execute_task_dict_error(self): - meta_future_lst = get_executor_dict( + meta_future_lst = _get_executor_dict( max_workers=1, executor_class=PyMPISingleTaskExecutor, ) From bbb6f955371013f5961dfee8e4929ae4a8fe198b Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Fri, 11 Aug 2023 17:49:16 -0600 Subject: [PATCH 14/58] fixes --- pympipool/mpi/mpibroker.py | 2 +- pympipool/shared/executorbase.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pympipool/mpi/mpibroker.py b/pympipool/mpi/mpibroker.py index 3a349085..f765cc1c 100644 --- a/pympipool/mpi/mpibroker.py +++ b/pympipool/mpi/mpibroker.py @@ -74,6 +74,6 @@ def _mpi_executor_broker( future_queue=future_queue, max_workers=max_workers, executor_class=PyMPISingleTaskExecutor, - executor_kwargs=executor_kwargs, sleep_interval=sleep_interval, + **executor_kwargs, ) diff --git a/pympipool/shared/executorbase.py b/pympipool/shared/executorbase.py index eb9d5c93..057efaf1 100644 --- a/pympipool/shared/executorbase.py +++ b/pympipool/shared/executorbase.py @@ -125,13 +125,13 @@ def executor_broker( future_queue, max_workers, executor_class, - executor_kwargs, sleep_interval=0.1, + **kwargs, ): meta_future_lst = _get_executor_dict( max_workers=max_workers, executor_class=executor_class, - executor_kwargs=executor_kwargs, + **kwargs, ) while True: try: @@ -174,9 +174,9 @@ def _get_command_path(executable): return os.path.abspath(os.path.join(__file__, "..", "..", "backend", executable)) -def _get_executor_dict(max_workers, executor_class, executor_kwargs): +def _get_executor_dict(max_workers, executor_class, **kwargs): return { - _get_future_done(): executor_class(**executor_kwargs) + _get_future_done(): executor_class(**kwargs) for _ in range(max_workers) } From a12bfaa13ceff120972e4216a8271adde7b54f7c Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Fri, 11 Aug 2023 17:54:58 -0600 Subject: [PATCH 15/58] Remove executor specific broker functions --- pympipool/flux/fluxbroker.py | 35 +++++++++++----------------------- pympipool/mpi/mpibroker.py | 37 ++++++++++++------------------------ tests/test_meta.py | 8 +++----- 3 files changed, 26 insertions(+), 54 deletions(-) diff --git a/pympipool/flux/fluxbroker.py b/pympipool/flux/fluxbroker.py index 330f7c5c..a4145cf6 100644 --- a/pympipool/flux/fluxbroker.py +++ b/pympipool/flux/fluxbroker.py @@ -32,34 +32,21 @@ def __init__( ): super().__init__() self._process = RaisingThread( - target=_flux_executor_broker, + target=executor_broker, kwargs={ + # Broker Arguments "future_queue": self._future_queue, "max_workers": max_workers, "sleep_interval": sleep_interval, - "executor_kwargs": { - "cores": cores_per_worker, - "threads_per_core": threads_per_core, - "gpus_per_task": int(gpus_per_worker / cores_per_worker), - "init_function": init_function, - "cwd": cwd, - "executor": executor, - }, + "executor_class": PyFluxSingleTaskExecutor, + + # Executor Arguments + "cores": cores_per_worker, + "threads_per_core": threads_per_core, + "gpus_per_task": int(gpus_per_worker / cores_per_worker), + "init_function": init_function, + "cwd": cwd, + "executor": executor, }, ) self._process.start() - - -def _flux_executor_broker( - future_queue, - max_workers, - executor_kwargs, - sleep_interval=0.1, -): - executor_broker( - future_queue=future_queue, - max_workers=max_workers, - executor_class=PyFluxSingleTaskExecutor, - executor_kwargs=executor_kwargs, - sleep_interval=sleep_interval, - ) diff --git a/pympipool/mpi/mpibroker.py b/pympipool/mpi/mpibroker.py index f765cc1c..64943e82 100644 --- a/pympipool/mpi/mpibroker.py +++ b/pympipool/mpi/mpibroker.py @@ -45,35 +45,22 @@ def __init__( + "to manage GPUs use the SLURM queuing system enable_slurm_backend=True ." ) self._process = RaisingThread( - target=_mpi_executor_broker, + target=executor_broker, kwargs={ + # Broker Arguments "future_queue": self._future_queue, "max_workers": max_workers, "sleep_interval": sleep_interval, - "executor_kwargs": { - "cores": cores_per_worker, - "threads_per_core": threads_per_core, - "gpus_per_task": int(gpus_per_worker / cores_per_worker), - "oversubscribe": oversubscribe, - "init_function": init_function, - "cwd": cwd, - "enable_slurm_backend": enable_slurm_backend, - }, + "executor_class": PyMPISingleTaskExecutor, + + # Executor Arguments + "cores": cores_per_worker, + "threads_per_core": threads_per_core, + "gpus_per_task": int(gpus_per_worker / cores_per_worker), + "oversubscribe": oversubscribe, + "init_function": init_function, + "cwd": cwd, + "enable_slurm_backend": enable_slurm_backend, }, ) self._process.start() - - -def _mpi_executor_broker( - future_queue, - max_workers, - executor_kwargs, - sleep_interval=0.1, -): - executor_broker( - future_queue=future_queue, - max_workers=max_workers, - executor_class=PyMPISingleTaskExecutor, - sleep_interval=sleep_interval, - **executor_kwargs, - ) diff --git a/tests/test_meta.py b/tests/test_meta.py index 70528f1a..2981204d 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -2,15 +2,13 @@ from queue import Queue import unittest from pympipool.shared.executorbase import ( + executor_broker, execute_task_dict, _get_executor_dict, _get_future_done, ) from pympipool.mpi.mpitask import PyMPISingleTaskExecutor -from pympipool.mpi.mpibroker import ( - PyMPIExecutor, - _mpi_executor_broker, -) +from pympipool.mpi.mpibroker import PyMPIExecutor def calc(i): @@ -81,7 +79,7 @@ def test_executor_broker(self): f = Future() q.put({"fn": calc, "args": (1,), "kwargs": {}, "future": f}) q.put({"shutdown": True, "wait": True}) - _mpi_executor_broker(future_queue=q, max_workers=1) + executor_broker(future_queue=q, max_workers=1, executor_class=PyMPISingleTaskExecutor) self.assertTrue(f.done()) self.assertEqual(f.result(), 1) q.join() From 7359e50858a08aa4ceddc7769d6022cffa9d79c6 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Fri, 11 Aug 2023 17:55:41 -0600 Subject: [PATCH 16/58] black formatting --- pympipool/flux/fluxbroker.py | 1 - pympipool/mpi/mpibroker.py | 1 - pympipool/shared/executorbase.py | 5 +---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/pympipool/flux/fluxbroker.py b/pympipool/flux/fluxbroker.py index a4145cf6..b0edafd2 100644 --- a/pympipool/flux/fluxbroker.py +++ b/pympipool/flux/fluxbroker.py @@ -39,7 +39,6 @@ def __init__( "max_workers": max_workers, "sleep_interval": sleep_interval, "executor_class": PyFluxSingleTaskExecutor, - # Executor Arguments "cores": cores_per_worker, "threads_per_core": threads_per_core, diff --git a/pympipool/mpi/mpibroker.py b/pympipool/mpi/mpibroker.py index 64943e82..0b660a21 100644 --- a/pympipool/mpi/mpibroker.py +++ b/pympipool/mpi/mpibroker.py @@ -52,7 +52,6 @@ def __init__( "max_workers": max_workers, "sleep_interval": sleep_interval, "executor_class": PyMPISingleTaskExecutor, - # Executor Arguments "cores": cores_per_worker, "threads_per_core": threads_per_core, diff --git a/pympipool/shared/executorbase.py b/pympipool/shared/executorbase.py index 057efaf1..5e0276d0 100644 --- a/pympipool/shared/executorbase.py +++ b/pympipool/shared/executorbase.py @@ -175,10 +175,7 @@ def _get_command_path(executable): def _get_executor_dict(max_workers, executor_class, **kwargs): - return { - _get_future_done(): executor_class(**kwargs) - for _ in range(max_workers) - } + return {_get_future_done(): executor_class(**kwargs) for _ in range(max_workers)} def _get_future_done(): From ef62a88e915a551a7869c6c4cc981a7bbdd13dff Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Fri, 11 Aug 2023 18:10:42 -0600 Subject: [PATCH 17/58] Fix flux tests --- tests/test_flux.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/test_flux.py b/tests/test_flux.py index 9ebd732c..c96e8b80 100644 --- a/tests/test_flux.py +++ b/tests/test_flux.py @@ -4,12 +4,12 @@ import numpy as np import unittest -from pympipool.shared.executorbase import cloudpickle_register +from pympipool.shared.executorbase import cloudpickle_register, executor_broker try: import flux.job - from pympipool.flux.fluxbroker import PyFluxExecutor, _flux_executor_broker + from pympipool.flux.fluxbroker import PyFluxExecutor from pympipool.flux.fluxtask import ( _flux_execute_parallel_tasks, PyFluxSingleTaskExecutor, @@ -119,7 +119,12 @@ def test_executor_broker(self): f = Future() q.put({"fn": calc, "args": (1,), "kwargs": {}, "future": f}) q.put({"shutdown": True, "wait": True}) - _flux_executor_broker(future_queue=q, max_workers=1, executor=self.executor) + executor_broker( + future_queue=q, + max_workers=1, + executor=self.executor, + executor_class=PyFluxSingleTaskExecutor, + ) self.assertTrue(f.done()) self.assertEqual(f.result(), 1) q.join() @@ -129,8 +134,12 @@ def test_executor_broker_threads(self): f = Future() q.put({"fn": calc, "args": (1,), "kwargs": {}, "future": f}) q.put({"shutdown": True, "wait": True}) - _flux_executor_broker( - future_queue=q, max_workers=1, threads_per_core=2, executor=self.executor + executor_broker( + future_queue=q, + max_workers=1, + threads_per_core=2, + executor=self.executor, + executor_class=PyFluxSingleTaskExecutor, ) self.assertTrue(f.done()) self.assertEqual(f.result(), 1) From f15ccf7e45921eaa645bfab591eb486fc3c2c59b Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Fri, 11 Aug 2023 19:03:49 -0600 Subject: [PATCH 18/58] remove _flux_execute_parallel_tasks() and _mpi_execute_parallel_tasks() --- pympipool/flux/fluxtask.py | 44 ++++-------------------------- pympipool/mpi/mpitask.py | 46 ++++---------------------------- pympipool/shared/executorbase.py | 27 ++++++++++++++++++- tests/test_flux.py | 22 +++++++++------ tests/test_worker.py | 16 ++++++----- tests/test_worker_memory.py | 7 ++--- 6 files changed, 64 insertions(+), 98 deletions(-) diff --git a/pympipool/flux/fluxtask.py b/pympipool/flux/fluxtask.py index 05fc3697..62d4b5c4 100644 --- a/pympipool/flux/fluxtask.py +++ b/pympipool/flux/fluxtask.py @@ -5,11 +5,9 @@ from pympipool.shared.executorbase import ( cloudpickle_register, ExecutorBase, - execute_parallel_tasks_loop, - get_backend_path, + execute_parallel_tasks, ) from pympipool.shared.interface import BaseInterface -from pympipool.shared.communication import interface_bootup from pympipool.shared.thread import RaisingThread @@ -61,10 +59,13 @@ def __init__( ): super().__init__() self._process = RaisingThread( - target=_flux_execute_parallel_tasks, + target=execute_parallel_tasks, kwargs={ + # Executor Arguments "future_queue": self._future_queue, "cores": cores, + "interface_class": FluxPythonInterface, + # Interface Arguments "threads_per_core": threads_per_core, "gpus_per_task": gpus_per_task, "cwd": cwd, @@ -129,38 +130,3 @@ def shutdown(self, wait=True): def poll(self): return self._future is not None and not self._future.done() - - -def _flux_execute_parallel_tasks( - future_queue, - cores, - threads_per_core=1, - gpus_per_task=0, - cwd=None, - executor=None, -): - """ - Execute a single tasks in parallel using the message passing interface (MPI). - - Args: - future_queue (queue.Queue): task queue of dictionary objects which are submitted to the parallel process - cores (int): defines the total number of MPI ranks to use - threads_per_core (int): number of OpenMP threads to be used for each function call - gpus_per_task (int): number of GPUs per MPI rank - defaults to 0 - cwd (str/None): current working directory where the parallel python task is executed - executor (flux.job.FluxExecutor/None): flux executor to submit tasks to - optional - """ - execute_parallel_tasks_loop( - interface=interface_bootup( - command_lst=get_backend_path(cores=cores), - connections=FluxPythonInterface( - cwd=cwd, - cores=cores, - threads_per_core=threads_per_core, - gpus_per_core=gpus_per_task, - oversubscribe=False, - executor=executor, - ), - ), - future_queue=future_queue, - ) diff --git a/pympipool/mpi/mpitask.py b/pympipool/mpi/mpitask.py index ec3acd4c..3f974009 100644 --- a/pympipool/mpi/mpitask.py +++ b/pympipool/mpi/mpitask.py @@ -1,11 +1,9 @@ from pympipool.shared.executorbase import ( cloudpickle_register, - execute_parallel_tasks_loop, + execute_parallel_tasks, ExecutorBase, - get_backend_path, ) from pympipool.shared.thread import RaisingThread -from pympipool.shared.communication import interface_bootup from pympipool.shared.interface import MpiExecInterface, SlurmSubprocessInterface @@ -59,10 +57,13 @@ def __init__( ): super().__init__() self._process = RaisingThread( - target=_mpi_execute_parallel_tasks, + target=execute_parallel_tasks, kwargs={ + # Executor Arguments "future_queue": self._future_queue, "cores": cores, + "interface_class": get_interface, + # Interface Arguments "threads_per_core": threads_per_core, "gpus_per_task": gpus_per_task, "cwd": cwd, @@ -78,43 +79,6 @@ def __init__( cloudpickle_register(ind=3) -def _mpi_execute_parallel_tasks( - future_queue, - cores, - threads_per_core=1, - gpus_per_task=0, - cwd=None, - oversubscribe=False, - enable_slurm_backend=False, -): - """ - Execute a single tasks in parallel using the message passing interface (MPI). - - Args: - future_queue (queue.Queue): task queue of dictionary objects which are submitted to the parallel process - cores (int): defines the total number of MPI ranks to use - threads_per_core (int): number of OpenMP threads to be used for each function call - gpus_per_task (int): number of GPUs per MPI rank - defaults to 0 - cwd (str/None): current working directory where the parallel python task is executed - oversubscribe (bool): enable of disable the oversubscribe feature of OpenMPI - defaults to False - enable_slurm_backend (bool): enable the SLURM queueing system as backend - defaults to False - """ - execute_parallel_tasks_loop( - interface=interface_bootup( - command_lst=get_backend_path(cores=cores), - connections=get_interface( - cores=cores, - threads_per_core=threads_per_core, - gpus_per_task=gpus_per_task, - cwd=cwd, - oversubscribe=oversubscribe, - enable_slurm_backend=enable_slurm_backend, - ), - ), - future_queue=future_queue, - ) - - def get_interface( cores=1, threads_per_core=1, diff --git a/pympipool/shared/executorbase.py b/pympipool/shared/executorbase.py index 5e0276d0..ae4b687b 100644 --- a/pympipool/shared/executorbase.py +++ b/pympipool/shared/executorbase.py @@ -11,6 +11,8 @@ import cloudpickle +from pympipool.shared.communication import interface_bootup + class ExecutorBase(FutureExecutor): def __init__(self): @@ -97,6 +99,29 @@ def cloudpickle_register(ind=2): pass +def execute_parallel_tasks( + future_queue, + cores, + interface_class, + **kwargs, +): + """ + Execute a single tasks in parallel using the message passing interface (MPI). + + Args: + future_queue (queue.Queue): task queue of dictionary objects which are submitted to the parallel process + cores (int): defines the total number of MPI ranks to use + interface_class: + """ + execute_parallel_tasks_loop( + interface=interface_bootup( + command_lst=_get_backend_path(cores=cores), + connections=interface_class(cores=cores, **kwargs), + ), + future_queue=future_queue, + ) + + def execute_parallel_tasks_loop(interface, future_queue): while True: task_dict = future_queue.get() @@ -161,7 +186,7 @@ def execute_task_dict(task_dict, meta_future_lst): raise ValueError("Unrecognized Task in task_dict: ", task_dict) -def get_backend_path(cores): +def _get_backend_path(cores): command_lst = [sys.executable] if cores > 1: command_lst += [_get_command_path(executable="mpiexec.py")] diff --git a/tests/test_flux.py b/tests/test_flux.py index c96e8b80..4cfdf5c9 100644 --- a/tests/test_flux.py +++ b/tests/test_flux.py @@ -4,16 +4,13 @@ import numpy as np import unittest -from pympipool.shared.executorbase import cloudpickle_register, executor_broker +from pympipool.shared.executorbase import cloudpickle_register, executor_broker, execute_parallel_tasks try: import flux.job from pympipool.flux.fluxbroker import PyFluxExecutor - from pympipool.flux.fluxtask import ( - _flux_execute_parallel_tasks, - PyFluxSingleTaskExecutor, - ) + from pympipool.flux.fluxtask import PyFluxSingleTaskExecutor, FluxPythonInterface skip_flux_test = False except ImportError: @@ -89,7 +86,12 @@ def test_execute_task(self): q.put({"fn": calc, "args": (), "kwargs": {"i": 2}, "future": f}) q.put({"shutdown": True, "wait": True}) cloudpickle_register(ind=1) - _flux_execute_parallel_tasks(future_queue=q, cores=1, executor=self.executor) + execute_parallel_tasks( + future_queue=q, + cores=1, + executor=self.executor, + interface_class=FluxPythonInterface, + ) self.assertEqual(f.result(), 2) q.join() @@ -99,8 +101,12 @@ def test_execute_task_threads(self): q.put({"fn": calc, "args": (), "kwargs": {"i": 2}, "future": f}) q.put({"shutdown": True, "wait": True}) cloudpickle_register(ind=1) - _flux_execute_parallel_tasks( - future_queue=q, cores=1, threads_per_core=1, executor=self.executor + execute_parallel_tasks( + future_queue=q, + cores=1, + threads_per_core=1, + executor=self.executor, + interface_class=FluxPythonInterface, ) self.assertEqual(f.result(), 2) q.join() diff --git a/tests/test_worker.py b/tests/test_worker.py index 546b5626..7a396701 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -3,8 +3,8 @@ from queue import Queue from time import sleep from concurrent.futures import CancelledError -from pympipool.mpi.mpitask import PyMPISingleTaskExecutor, _mpi_execute_parallel_tasks -from pympipool.shared.executorbase import cloudpickle_register +from pympipool.mpi.mpitask import PyMPISingleTaskExecutor, get_interface +from pympipool.shared.executorbase import cloudpickle_register, execute_parallel_tasks from concurrent.futures import Future @@ -103,10 +103,11 @@ def test_execute_task_failed_no_argument(self): q.put({"fn": calc, "args": (), "kwargs": {}, "future": f}) cloudpickle_register(ind=1) with self.assertRaises(TypeError): - _mpi_execute_parallel_tasks( + execute_parallel_tasks( future_queue=q, cores=1, oversubscribe=False, + interface_class=get_interface, ) q.join() @@ -116,10 +117,11 @@ def test_execute_task_failed_wrong_argument(self): q.put({"fn": calc, "args": (), "kwargs": {"j": 4}, "future": f}) cloudpickle_register(ind=1) with self.assertRaises(TypeError): - _mpi_execute_parallel_tasks( + execute_parallel_tasks( future_queue=q, cores=1, oversubscribe=False, + interface_class=get_interface, ) q.join() @@ -129,10 +131,11 @@ def test_execute_task(self): q.put({"fn": calc, "args": (), "kwargs": {"i": 2}, "future": f}) q.put({"shutdown": True, "wait": True}) cloudpickle_register(ind=1) - _mpi_execute_parallel_tasks( + execute_parallel_tasks( future_queue=q, cores=1, oversubscribe=False, + interface_class=get_interface, ) self.assertEqual(f.result(), np.array(4)) q.join() @@ -143,10 +146,11 @@ def test_execute_task_parallel(self): q.put({"fn": calc, "args": (), "kwargs": {"i": 2}, "future": f}) q.put({"shutdown": True, "wait": True}) cloudpickle_register(ind=1) - _mpi_execute_parallel_tasks( + execute_parallel_tasks( future_queue=q, cores=2, oversubscribe=False, + interface_class=get_interface, ) self.assertEqual(f.result(), [np.array(4), np.array(4)]) q.join() diff --git a/tests/test_worker_memory.py b/tests/test_worker_memory.py index f49ee1cf..3c668dff 100644 --- a/tests/test_worker_memory.py +++ b/tests/test_worker_memory.py @@ -2,8 +2,8 @@ import numpy as np from queue import Queue from pympipool.shared.backend import call_funct -from pympipool.shared.executorbase import cloudpickle_register -from pympipool.mpi.mpitask import PyMPISingleTaskExecutor, _mpi_execute_parallel_tasks +from pympipool.shared.executorbase import cloudpickle_register, execute_parallel_tasks +from pympipool.mpi.mpitask import PyMPISingleTaskExecutor, get_interface from concurrent.futures import Future @@ -39,10 +39,11 @@ def test_execute_task(self): q.put({"fn": get_global, "args": (), "kwargs": {}, "future": f}) q.put({"shutdown": True, "wait": True}) cloudpickle_register(ind=1) - _mpi_execute_parallel_tasks( + execute_parallel_tasks( future_queue=q, cores=1, oversubscribe=False, + interface_class=get_interface, ) self.assertEqual(f.result(), np.array([5])) q.join() From b1669dcf22cbbc267fa1af3638b2d769dd34b992 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Fri, 11 Aug 2023 19:07:04 -0600 Subject: [PATCH 19/58] Fix gpus_per_core and gpus_per_task --- pympipool/flux/fluxtask.py | 2 +- pympipool/mpi/mpitask.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pympipool/flux/fluxtask.py b/pympipool/flux/fluxtask.py index 62d4b5c4..89925cea 100644 --- a/pympipool/flux/fluxtask.py +++ b/pympipool/flux/fluxtask.py @@ -67,7 +67,7 @@ def __init__( "interface_class": FluxPythonInterface, # Interface Arguments "threads_per_core": threads_per_core, - "gpus_per_task": gpus_per_task, + "gpus_per_core": gpus_per_task, "cwd": cwd, "executor": executor, }, diff --git a/pympipool/mpi/mpitask.py b/pympipool/mpi/mpitask.py index 3f974009..61834f19 100644 --- a/pympipool/mpi/mpitask.py +++ b/pympipool/mpi/mpitask.py @@ -65,7 +65,7 @@ def __init__( "interface_class": get_interface, # Interface Arguments "threads_per_core": threads_per_core, - "gpus_per_task": gpus_per_task, + "gpus_per_core": gpus_per_task, "cwd": cwd, "oversubscribe": oversubscribe, "enable_slurm_backend": enable_slurm_backend, @@ -82,7 +82,7 @@ def __init__( def get_interface( cores=1, threads_per_core=1, - gpus_per_task=0, + gpus_per_core=0, cwd=None, oversubscribe=False, enable_slurm_backend=False, @@ -92,7 +92,7 @@ def get_interface( cwd=cwd, cores=cores, threads_per_core=threads_per_core, - gpus_per_core=gpus_per_task, + gpus_per_core=gpus_per_core, oversubscribe=oversubscribe, ) else: @@ -100,6 +100,6 @@ def get_interface( cwd=cwd, cores=cores, threads_per_core=threads_per_core, - gpus_per_core=gpus_per_task, + gpus_per_core=gpus_per_core, oversubscribe=oversubscribe, ) From a29f2b4335946d61b20f435bad802ffde596e56f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Fri, 11 Aug 2023 19:47:13 -0600 Subject: [PATCH 20/58] Implement a central _set_init_function() --- pympipool/flux/fluxtask.py | 5 +---- pympipool/mpi/mpitask.py | 5 +---- pympipool/shared/executorbase.py | 6 ++++++ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pympipool/flux/fluxtask.py b/pympipool/flux/fluxtask.py index 89925cea..b7a59331 100644 --- a/pympipool/flux/fluxtask.py +++ b/pympipool/flux/fluxtask.py @@ -73,10 +73,7 @@ def __init__( }, ) self._process.start() - if init_function is not None: - self._future_queue.put( - {"init": True, "fn": init_function, "args": (), "kwargs": {}} - ) + self._set_init_function(init_function=init_function) cloudpickle_register(ind=3) diff --git a/pympipool/mpi/mpitask.py b/pympipool/mpi/mpitask.py index 61834f19..fdcc17b7 100644 --- a/pympipool/mpi/mpitask.py +++ b/pympipool/mpi/mpitask.py @@ -72,10 +72,7 @@ def __init__( }, ) self._process.start() - if init_function is not None: - self._future_queue.put( - {"init": True, "fn": init_function, "args": (), "kwargs": {}} - ) + self._set_init_function(init_function=init_function) cloudpickle_register(ind=3) diff --git a/pympipool/shared/executorbase.py b/pympipool/shared/executorbase.py index ae4b687b..c950eae1 100644 --- a/pympipool/shared/executorbase.py +++ b/pympipool/shared/executorbase.py @@ -58,6 +58,12 @@ def shutdown(self, wait=True, *, cancel_futures=False): def __len__(self): return self._future_queue.qsize() + def _set_init_function(self, init_function): + if init_function is not None: + self._future_queue.put( + {"init": True, "fn": init_function, "args": (), "kwargs": {}} + ) + def cancel_items_in_queue(que): """ From e29dc91175753c4203726652bd6d02458d50c5d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Fri, 11 Aug 2023 19:55:48 -0600 Subject: [PATCH 21/58] Refactor the executor --- pympipool/flux/__init__.py | 2 +- pympipool/flux/{fluxtask.py => executor.py} | 48 ++++++++++++++- pympipool/flux/fluxbroker.py | 51 ---------------- pympipool/mpi/__init__.py | 2 +- pympipool/mpi/{mpitask.py => executor.py} | 64 +++++++++++++++++++- pympipool/mpi/mpibroker.py | 65 --------------------- tests/test_flux.py | 7 ++- tests/test_future.py | 2 +- tests/test_meta.py | 3 +- tests/test_task.py | 2 +- tests/test_worker.py | 2 +- tests/test_worker_memory.py | 2 +- 12 files changed, 121 insertions(+), 129 deletions(-) rename pympipool/flux/{fluxtask.py => executor.py} (70%) delete mode 100644 pympipool/flux/fluxbroker.py rename pympipool/mpi/{mpitask.py => executor.py} (58%) delete mode 100644 pympipool/mpi/mpibroker.py diff --git a/pympipool/flux/__init__.py b/pympipool/flux/__init__.py index 26f0a677..44e9e4f1 100644 --- a/pympipool/flux/__init__.py +++ b/pympipool/flux/__init__.py @@ -1 +1 @@ -from pympipool.flux.fluxbroker import PyFluxExecutor +from pympipool.flux.executor import PyFluxExecutor diff --git a/pympipool/flux/fluxtask.py b/pympipool/flux/executor.py similarity index 70% rename from pympipool/flux/fluxtask.py rename to pympipool/flux/executor.py index b7a59331..4c8930da 100644 --- a/pympipool/flux/fluxtask.py +++ b/pympipool/flux/executor.py @@ -5,12 +5,58 @@ from pympipool.shared.executorbase import ( cloudpickle_register, ExecutorBase, + executor_broker, execute_parallel_tasks, ) from pympipool.shared.interface import BaseInterface from pympipool.shared.thread import RaisingThread +class PyFluxExecutor(ExecutorBase): + """ + Args: + max_workers (int): defines the number workers which can execute functions in parallel + cores_per_worker (int): number of MPI cores to be used for each function call + threads_per_core (int): number of OpenMP threads to be used for each function call + gpus_per_worker (int): number of GPUs per worker - defaults to 0 + init_function (None): optional function to preset arguments for functions which are submitted later + cwd (str/None): current working directory where the parallel python task is executed + sleep_interval (float): synchronization interval - default 0.1 + executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux + """ + + def __init__( + self, + max_workers, + cores_per_worker=1, + threads_per_core=1, + gpus_per_worker=0, + init_function=None, + cwd=None, + sleep_interval=0.1, + executor=None, + ): + super().__init__() + self._process = RaisingThread( + target=executor_broker, + kwargs={ + # Broker Arguments + "future_queue": self._future_queue, + "max_workers": max_workers, + "sleep_interval": sleep_interval, + "executor_class": PyFluxSingleTaskExecutor, + # Executor Arguments + "cores": cores_per_worker, + "threads_per_core": threads_per_core, + "gpus_per_task": int(gpus_per_worker / cores_per_worker), + "init_function": init_function, + "cwd": cwd, + "executor": executor, + }, + ) + self._process.start() + + class PyFluxSingleTaskExecutor(ExecutorBase): """ The pympipool.Executor behaves like the concurrent.futures.Executor but it uses mpi4py to execute parallel tasks. @@ -29,7 +75,7 @@ class PyFluxSingleTaskExecutor(ExecutorBase): Examples: ``` >>> import numpy as np - >>> from pympipool.flux.fluxtask import PyFluxSingleTaskExecutor + >>> from pympipool.flux.executor import PyFluxSingleTaskExecutor >>> >>> def calc(i, j, k): >>> from mpi4py import MPI diff --git a/pympipool/flux/fluxbroker.py b/pympipool/flux/fluxbroker.py deleted file mode 100644 index b0edafd2..00000000 --- a/pympipool/flux/fluxbroker.py +++ /dev/null @@ -1,51 +0,0 @@ -from pympipool.shared.executorbase import ( - ExecutorBase, - executor_broker, -) -from pympipool.shared.thread import RaisingThread -from pympipool.flux.fluxtask import PyFluxSingleTaskExecutor - - -class PyFluxExecutor(ExecutorBase): - """ - Args: - max_workers (int): defines the number workers which can execute functions in parallel - cores_per_worker (int): number of MPI cores to be used for each function call - threads_per_core (int): number of OpenMP threads to be used for each function call - gpus_per_worker (int): number of GPUs per worker - defaults to 0 - init_function (None): optional function to preset arguments for functions which are submitted later - cwd (str/None): current working directory where the parallel python task is executed - sleep_interval (float): synchronization interval - default 0.1 - executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux - """ - - def __init__( - self, - max_workers, - cores_per_worker=1, - threads_per_core=1, - gpus_per_worker=0, - init_function=None, - cwd=None, - sleep_interval=0.1, - executor=None, - ): - super().__init__() - self._process = RaisingThread( - target=executor_broker, - kwargs={ - # Broker Arguments - "future_queue": self._future_queue, - "max_workers": max_workers, - "sleep_interval": sleep_interval, - "executor_class": PyFluxSingleTaskExecutor, - # Executor Arguments - "cores": cores_per_worker, - "threads_per_core": threads_per_core, - "gpus_per_task": int(gpus_per_worker / cores_per_worker), - "init_function": init_function, - "cwd": cwd, - "executor": executor, - }, - ) - self._process.start() diff --git a/pympipool/mpi/__init__.py b/pympipool/mpi/__init__.py index 5d4bea6a..24aa8369 100644 --- a/pympipool/mpi/__init__.py +++ b/pympipool/mpi/__init__.py @@ -1 +1 @@ -from pympipool.mpi.mpibroker import PyMPIExecutor +from pympipool.mpi.executor import PyMPIExecutor diff --git a/pympipool/mpi/mpitask.py b/pympipool/mpi/executor.py similarity index 58% rename from pympipool/mpi/mpitask.py rename to pympipool/mpi/executor.py index fdcc17b7..9aac7289 100644 --- a/pympipool/mpi/mpitask.py +++ b/pympipool/mpi/executor.py @@ -2,9 +2,69 @@ cloudpickle_register, execute_parallel_tasks, ExecutorBase, + executor_broker, ) -from pympipool.shared.thread import RaisingThread from pympipool.shared.interface import MpiExecInterface, SlurmSubprocessInterface +from pympipool.shared.thread import RaisingThread + + +class PyMPIExecutor(ExecutorBase): + """ + Args: + max_workers (int): defines the number workers which can execute functions in parallel + cores_per_worker (int): number of MPI cores to be used for each function call + threads_per_core (int): number of OpenMP threads to be used for each function call + gpus_per_worker (int): number of GPUs per worker - defaults to 0 + oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI only) - default False + init_function (None): optional function to preset arguments for functions which are submitted later + cwd (str/None): current working directory where the parallel python task is executed + sleep_interval (float): synchronization interval - default 0.1 + enable_slurm_backend (bool): enable the SLURM queueing system as backend - defaults to False + """ + + def __init__( + self, + max_workers, + cores_per_worker=1, + threads_per_core=1, + gpus_per_worker=0, + oversubscribe=False, + init_function=None, + cwd=None, + sleep_interval=0.1, + enable_slurm_backend=False, + ): + super().__init__() + if not enable_slurm_backend: + if threads_per_core != 1: + raise ValueError( + "The MPI backend only supports threads_per_core=1, " + + "to manage threads use the SLURM queuing system enable_slurm_backend=True ." + ) + elif gpus_per_worker != 0: + raise ValueError( + "The MPI backend only supports gpus_per_core=0, " + + "to manage GPUs use the SLURM queuing system enable_slurm_backend=True ." + ) + self._process = RaisingThread( + target=executor_broker, + kwargs={ + # Broker Arguments + "future_queue": self._future_queue, + "max_workers": max_workers, + "sleep_interval": sleep_interval, + "executor_class": PyMPISingleTaskExecutor, + # Executor Arguments + "cores": cores_per_worker, + "threads_per_core": threads_per_core, + "gpus_per_task": int(gpus_per_worker / cores_per_worker), + "oversubscribe": oversubscribe, + "init_function": init_function, + "cwd": cwd, + "enable_slurm_backend": enable_slurm_backend, + }, + ) + self._process.start() class PyMPISingleTaskExecutor(ExecutorBase): @@ -27,7 +87,7 @@ class PyMPISingleTaskExecutor(ExecutorBase): Examples: ``` >>> import numpy as np - >>> from pympipool.mpi.mpitask import PyMPISingleTaskExecutor + >>> from pympipool.mpi.executor import PyMPISingleTaskExecutor >>> >>> def calc(i, j, k): >>> from mpi4py import MPI diff --git a/pympipool/mpi/mpibroker.py b/pympipool/mpi/mpibroker.py deleted file mode 100644 index 0b660a21..00000000 --- a/pympipool/mpi/mpibroker.py +++ /dev/null @@ -1,65 +0,0 @@ -from pympipool.shared.executorbase import ( - ExecutorBase, - executor_broker, -) -from pympipool.shared.thread import RaisingThread -from pympipool.mpi.mpitask import PyMPISingleTaskExecutor - - -class PyMPIExecutor(ExecutorBase): - """ - Args: - max_workers (int): defines the number workers which can execute functions in parallel - cores_per_worker (int): number of MPI cores to be used for each function call - threads_per_core (int): number of OpenMP threads to be used for each function call - gpus_per_worker (int): number of GPUs per worker - defaults to 0 - oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI only) - default False - init_function (None): optional function to preset arguments for functions which are submitted later - cwd (str/None): current working directory where the parallel python task is executed - sleep_interval (float): synchronization interval - default 0.1 - enable_slurm_backend (bool): enable the SLURM queueing system as backend - defaults to False - """ - - def __init__( - self, - max_workers, - cores_per_worker=1, - threads_per_core=1, - gpus_per_worker=0, - oversubscribe=False, - init_function=None, - cwd=None, - sleep_interval=0.1, - enable_slurm_backend=False, - ): - super().__init__() - if not enable_slurm_backend: - if threads_per_core != 1: - raise ValueError( - "The MPI backend only supports threads_per_core=1, " - + "to manage threads use the SLURM queuing system enable_slurm_backend=True ." - ) - elif gpus_per_worker != 0: - raise ValueError( - "The MPI backend only supports gpus_per_core=0, " - + "to manage GPUs use the SLURM queuing system enable_slurm_backend=True ." - ) - self._process = RaisingThread( - target=executor_broker, - kwargs={ - # Broker Arguments - "future_queue": self._future_queue, - "max_workers": max_workers, - "sleep_interval": sleep_interval, - "executor_class": PyMPISingleTaskExecutor, - # Executor Arguments - "cores": cores_per_worker, - "threads_per_core": threads_per_core, - "gpus_per_task": int(gpus_per_worker / cores_per_worker), - "oversubscribe": oversubscribe, - "init_function": init_function, - "cwd": cwd, - "enable_slurm_backend": enable_slurm_backend, - }, - ) - self._process.start() diff --git a/tests/test_flux.py b/tests/test_flux.py index 4cfdf5c9..4d84511d 100644 --- a/tests/test_flux.py +++ b/tests/test_flux.py @@ -9,8 +9,11 @@ try: import flux.job - from pympipool.flux.fluxbroker import PyFluxExecutor - from pympipool.flux.fluxtask import PyFluxSingleTaskExecutor, FluxPythonInterface + from pympipool.flux.executor import ( + PyFluxExecutor, + PyFluxSingleTaskExecutor, + FluxPythonInterface, + ) skip_flux_test = False except ImportError: diff --git a/tests/test_future.py b/tests/test_future.py index cd96441e..1d59d4ee 100644 --- a/tests/test_future.py +++ b/tests/test_future.py @@ -1,7 +1,7 @@ import numpy as np import unittest from time import sleep -from pympipool.mpi.mpitask import PyMPISingleTaskExecutor +from pympipool.mpi.executor import PyMPISingleTaskExecutor from concurrent.futures import Future diff --git a/tests/test_meta.py b/tests/test_meta.py index 2981204d..643883c8 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -7,8 +7,7 @@ _get_executor_dict, _get_future_done, ) -from pympipool.mpi.mpitask import PyMPISingleTaskExecutor -from pympipool.mpi.mpibroker import PyMPIExecutor +from pympipool.mpi.executor import PyMPIExecutor, PyMPISingleTaskExecutor def calc(i): diff --git a/tests/test_task.py b/tests/test_task.py index bf1e166b..1585e6d3 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -1,5 +1,5 @@ import unittest -from pympipool.mpi.mpitask import PyMPISingleTaskExecutor +from pympipool.mpi.executor import PyMPISingleTaskExecutor def echo_funct(i): diff --git a/tests/test_worker.py b/tests/test_worker.py index 7a396701..5bc26747 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -3,7 +3,7 @@ from queue import Queue from time import sleep from concurrent.futures import CancelledError -from pympipool.mpi.mpitask import PyMPISingleTaskExecutor, get_interface +from pympipool.mpi.executor import PyMPISingleTaskExecutor, get_interface from pympipool.shared.executorbase import cloudpickle_register, execute_parallel_tasks from concurrent.futures import Future diff --git a/tests/test_worker_memory.py b/tests/test_worker_memory.py index 3c668dff..f8aaa7cb 100644 --- a/tests/test_worker_memory.py +++ b/tests/test_worker_memory.py @@ -3,7 +3,7 @@ from queue import Queue from pympipool.shared.backend import call_funct from pympipool.shared.executorbase import cloudpickle_register, execute_parallel_tasks -from pympipool.mpi.mpitask import PyMPISingleTaskExecutor, get_interface +from pympipool.mpi.executor import PyMPISingleTaskExecutor, get_interface from concurrent.futures import Future From 3f315bc07646cd13a118d1fa4c50f2d6dbd29116 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 12 Sep 2023 13:08:26 +0200 Subject: [PATCH 22/58] No longer surpress error messages --- pympipool/shared/interface.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pympipool/shared/interface.py b/pympipool/shared/interface.py index af5a75c9..8885cb36 100644 --- a/pympipool/shared/interface.py +++ b/pympipool/shared/interface.py @@ -43,9 +43,6 @@ def __init__( def bootup(self, command_lst): self._process = subprocess.Popen( args=self.generate_command(command_lst=command_lst), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE, cwd=self._cwd, ) @@ -54,9 +51,6 @@ def generate_command(self, command_lst): def shutdown(self, wait=True): self._process.terminate() - self._process.stdout.close() - self._process.stdin.close() - self._process.stderr.close() if wait: self._process.wait() self._process = None From 37d123d44b6f4d2c62c6d7e3ba9b42d0cf031e2e Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 3 Oct 2023 14:38:24 +0200 Subject: [PATCH 23/58] Update environment-mpich.yml --- .ci_support/environment-mpich.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-mpich.yml b/.ci_support/environment-mpich.yml index 4be67de9..04e28fc8 100644 --- a/.ci_support/environment-mpich.yml +++ b/.ci_support/environment-mpich.yml @@ -7,4 +7,4 @@ dependencies: - cloudpickle =2.2.1 - mpi4py =3.1.4 - tqdm =4.66.1 -- pyzmq =25.1.1 \ No newline at end of file +- pyzmq =22.3.0 From 17610e529ff2f687d4ed603c3cc6291bb430d36a Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 3 Oct 2023 14:38:36 +0200 Subject: [PATCH 24/58] Update environment-openmpi.yml --- .ci_support/environment-openmpi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-openmpi.yml b/.ci_support/environment-openmpi.yml index a9b18b78..bcb06341 100644 --- a/.ci_support/environment-openmpi.yml +++ b/.ci_support/environment-openmpi.yml @@ -7,4 +7,4 @@ dependencies: - cloudpickle =2.2.1 - mpi4py =3.1.4 - tqdm =4.66.1 -- pyzmq =25.1.1 \ No newline at end of file +- pyzmq =22.3.0 From f892723f20afe1be3bb3211b59580effa8d3da28 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 3 Oct 2023 14:38:46 +0200 Subject: [PATCH 25/58] Update environment-win.yml --- .ci_support/environment-win.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-win.yml b/.ci_support/environment-win.yml index fe3dbe56..56e93e4e 100644 --- a/.ci_support/environment-win.yml +++ b/.ci_support/environment-win.yml @@ -7,4 +7,4 @@ dependencies: - cloudpickle =2.2.1 - mpi4py =3.1.4 - tqdm =4.66.1 -- pyzmq =25.1.1 \ No newline at end of file +- pyzmq =22.3.0 From 3499d604b42d5e6cbe0ad131575875609f42732f Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 3 Oct 2023 14:39:00 +0200 Subject: [PATCH 26/58] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9ee0f78d..af7f948f 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ 'cloudpickle==2.2.1', 'mpi4py==3.1.4', 'tqdm==4.66.1', - 'pyzmq==25.1.1', + 'pyzmq==22.3.0', ], cmdclass=versioneer.get_cmdclass(), ) From 60dd94d0ea2def9643e91f9556dc1ee280cefc16 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 3 Oct 2023 14:45:45 +0200 Subject: [PATCH 27/58] Update environment-mpich.yml --- .ci_support/environment-mpich.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-mpich.yml b/.ci_support/environment-mpich.yml index 04e28fc8..46e3b286 100644 --- a/.ci_support/environment-mpich.yml +++ b/.ci_support/environment-mpich.yml @@ -7,4 +7,4 @@ dependencies: - cloudpickle =2.2.1 - mpi4py =3.1.4 - tqdm =4.66.1 -- pyzmq =22.3.0 +- pyzmq =24.0.1 From 21e646a736b83cf79d99af97340f45ac38435a59 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 3 Oct 2023 14:46:03 +0200 Subject: [PATCH 28/58] Update environment-openmpi.yml --- .ci_support/environment-openmpi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-openmpi.yml b/.ci_support/environment-openmpi.yml index bcb06341..e4a2f862 100644 --- a/.ci_support/environment-openmpi.yml +++ b/.ci_support/environment-openmpi.yml @@ -7,4 +7,4 @@ dependencies: - cloudpickle =2.2.1 - mpi4py =3.1.4 - tqdm =4.66.1 -- pyzmq =22.3.0 +- pyzmq =24.0.1 From ec8cebe77e9b8c43e9b145e0cb9a3bc749bc55da Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 3 Oct 2023 14:46:19 +0200 Subject: [PATCH 29/58] Update environment-win.yml --- .ci_support/environment-win.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-win.yml b/.ci_support/environment-win.yml index 56e93e4e..a7356492 100644 --- a/.ci_support/environment-win.yml +++ b/.ci_support/environment-win.yml @@ -7,4 +7,4 @@ dependencies: - cloudpickle =2.2.1 - mpi4py =3.1.4 - tqdm =4.66.1 -- pyzmq =22.3.0 +- pyzmq =24.0.1 From 761d45cbe29aa5b7ffafe86316935a9e8f460df2 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 3 Oct 2023 14:46:35 +0200 Subject: [PATCH 30/58] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index af7f948f..7f02c491 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ 'cloudpickle==2.2.1', 'mpi4py==3.1.4', 'tqdm==4.66.1', - 'pyzmq==22.3.0', + 'pyzmq==24.0.1', ], cmdclass=versioneer.get_cmdclass(), ) From dce7b4aa4ad8f8dc60a3330889e8019b9ad3cb88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Tue, 3 Oct 2023 16:02:25 +0200 Subject: [PATCH 31/58] Update versioneer to 0.29 --- pympipool/_version.py | 318 +++++++++--- versioneer.py | 1135 +++++++++++++++++++++++++++++------------ 2 files changed, 1034 insertions(+), 419 deletions(-) diff --git a/pympipool/_version.py b/pympipool/_version.py index ed7ec936..7b1a0f2f 100644 --- a/pympipool/_version.py +++ b/pympipool/_version.py @@ -4,8 +4,9 @@ # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) +# This file is released into the public domain. +# Generated by versioneer-0.29 +# https://github.com/python-versioneer/python-versioneer """Git implementation of _version.py.""" @@ -14,9 +15,11 @@ import re import subprocess import sys +from typing import Any, Callable, Dict, List, Optional, Tuple +import functools -def get_keywords(): +def get_keywords() -> Dict[str, str]: """Get the keywords needed to look up the version information.""" # these strings will be replaced by git during git-archive. # setup.py/versioneer.py will grep for the variable names, so they must @@ -32,8 +35,15 @@ def get_keywords(): class VersioneerConfig: """Container for Versioneer configuration parameters.""" + VCS: str + style: str + tag_prefix: str + parentdir_prefix: str + versionfile_source: str + verbose: bool -def get_config(): + +def get_config() -> VersioneerConfig: """Create, populate and return the VersioneerConfig() object.""" # these strings are filled in when 'setup.py versioneer' creates # _version.py @@ -51,14 +61,14 @@ class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" -LONG_VERSION_PY = {} -HANDLERS = {} +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" +def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator + """Create decorator to mark a method as the handler of a VCS.""" - def decorate(f): + def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} @@ -68,24 +78,39 @@ def decorate(f): return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): +def run_command( + commands: List[str], + args: List[str], + cwd: Optional[str] = None, + verbose: bool = False, + hide_stderr: bool = False, + env: Optional[Dict[str, str]] = None, +) -> Tuple[Optional[str], Optional[int]]: """Call the given command(s).""" assert isinstance(commands, list) - p = None - for c in commands: + process = None + + popen_kwargs: Dict[str, Any] = {} + if sys.platform == "win32": + # This hides the console window if pythonw.exe is used + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + popen_kwargs["startupinfo"] = startupinfo + + for command in commands: try: - dispcmd = str([c] + args) + dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, + process = subprocess.Popen( + [command] + args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), + **popen_kwargs, ) break - except EnvironmentError: - e = sys.exc_info()[1] + except OSError as e: if e.errno == errno.ENOENT: continue if verbose: @@ -96,18 +121,20 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env= if verbose: print("unable to find command, tried %s" % (commands,)) return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode + return None, process.returncode + return stdout, process.returncode -def versions_from_parentdir(parentdir_prefix, root, verbose): +def versions_from_parentdir( + parentdir_prefix: str, + root: str, + verbose: bool, +) -> Dict[str, Any]: """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both @@ -116,7 +143,7 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): """ rootdirs = [] - for i in range(3): + for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return { @@ -126,9 +153,8 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): "error": None, "date": None, } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level + rootdirs.append(root) + root = os.path.dirname(root) # up a level if verbose: print( @@ -139,41 +165,48 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): @register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): +def git_get_keywords(versionfile_abs: str) -> Dict[str, str]: """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. - keywords = {} + keywords: Dict[str, str] = {} try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: pass return keywords @register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): +def git_versions_from_keywords( + keywords: Dict[str, str], + tag_prefix: str, + verbose: bool, +) -> Dict[str, Any]: """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because @@ -186,11 +219,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -199,7 +232,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) + tags = {r for r in refs if re.search(r"\d", r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -208,6 +241,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix) :] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r"\d", r): + continue if verbose: print("picking %s" % r) return { @@ -230,7 +268,9 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): @register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): +def git_pieces_from_vcs( + tag_prefix: str, root: str, verbose: bool, runner: Callable = run_command +) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* @@ -241,7 +281,14 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) + # GIT_DIR can interfere with correct operation of Versioneer. + # It may be intended to be passed to the Versioneer-versioned project, + # but that should not change where we get our version from. + env = os.environ.copy() + env.pop("GIT_DIR", None) + runner = functools.partial(runner, env=env) + + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -249,7 +296,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( + describe_out, rc = runner( GITS, [ "describe", @@ -258,7 +305,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): "--always", "--long", "--match", - "%s*" % tag_prefix, + f"{tag_prefix}[[:digit:]]*", ], cwd=root, ) @@ -266,16 +313,48 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() - pieces = {} + pieces: Dict[str, Any] = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out @@ -292,7 +371,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # TAG-NUM-gHEX mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: - # unparseable. Maybe git-describe is misbehaving? + # unparsable. Maybe git-describe is misbehaving? pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out return pieces @@ -318,26 +397,27 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits + out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) + pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() + date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces -def plus_or_dot(pieces): +def plus_or_dot(pieces: Dict[str, Any]) -> str: """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" -def render_pep440(pieces): +def render_pep440(pieces: Dict[str, Any]) -> str: """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you @@ -361,23 +441,70 @@ def render_pep440(pieces): return rendered -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. +def render_pep440_branch(pieces: Dict[str, Any]) -> str: + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). Exceptions: - 1: no tags. 0.post.devDISTANCE + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver: str) -> Tuple[str, Optional[int]]: + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces: Dict[str, Any]) -> str: + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) + else: + rendered += ".post0.dev%d" % (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] else: # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] + rendered = "0.post0.dev%d" % pieces["distance"] return rendered -def render_pep440_post(pieces): +def render_pep440_post(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards @@ -404,12 +531,41 @@ def render_pep440_post(pieces): return rendered -def render_pep440_old(pieces): +def render_pep440_post_branch(pieces: Dict[str, Any]) -> str: + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_old(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -426,7 +582,7 @@ def render_pep440_old(pieces): return rendered -def render_git_describe(pieces): +def render_git_describe(pieces: Dict[str, Any]) -> str: """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. @@ -446,7 +602,7 @@ def render_git_describe(pieces): return rendered -def render_git_describe_long(pieces): +def render_git_describe_long(pieces: Dict[str, Any]) -> str: """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. @@ -466,7 +622,7 @@ def render_git_describe_long(pieces): return rendered -def render(pieces, style): +def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: return { @@ -482,10 +638,14 @@ def render(pieces, style): if style == "pep440": rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": @@ -504,7 +664,7 @@ def render(pieces, style): } -def get_versions(): +def get_versions() -> Dict[str, Any]: """Get version information or return default if unable to do so.""" # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have # __file__, we can work backwards from there to the root. Some @@ -524,7 +684,7 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split("/"): + for _ in cfg.versionfile_source.split("/"): root = os.path.dirname(root) except NameError: return { diff --git a/versioneer.py b/versioneer.py index 64fea1c8..1e3753e6 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,5 +1,5 @@ -# Version: 0.18 +# Version: 0.29 """The Versioneer - like a rocketeer, but for versions. @@ -7,18 +7,14 @@ ============== * like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer +* https://github.com/python-versioneer/python-versioneer * Brian Warner -* License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) - -This is a tool for managing a recorded version number in distutils-based +* License: Public Domain (Unlicense) +* Compatible with: Python 3.7, 3.8, 3.9, 3.10, 3.11 and pypy3 +* [![Latest Version][pypi-image]][pypi-url] +* [![Build Status][travis-image]][travis-url] + +This is a tool for managing a recorded version number in setuptools-based python projects. The goal is to remove the tedious and error-prone "update the embedded version string" step from your release process. Making a new release should be as easy as recording a new tag in your version-control @@ -27,9 +23,38 @@ ## Quick Install -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) -* run `versioneer install` in your source tree, commit the results +Versioneer provides two installation modes. The "classic" vendored mode installs +a copy of versioneer into your repository. The experimental build-time dependency mode +is intended to allow you to skip this step and simplify the process of upgrading. + +### Vendored mode + +* `pip install versioneer` to somewhere in your $PATH + * A [conda-forge recipe](https://github.com/conda-forge/versioneer-feedstock) is + available, so you can also use `conda install -c conda-forge versioneer` +* add a `[tool.versioneer]` section to your `pyproject.toml` or a + `[versioneer]` section to your `setup.cfg` (see [Install](INSTALL.md)) + * Note that you will need to add `tomli; python_version < "3.11"` to your + build-time dependencies if you use `pyproject.toml` +* run `versioneer install --vendor` in your source tree, commit the results +* verify version information with `python setup.py version` + +### Build-time dependency mode + +* `pip install versioneer` to somewhere in your $PATH + * A [conda-forge recipe](https://github.com/conda-forge/versioneer-feedstock) is + available, so you can also use `conda install -c conda-forge versioneer` +* add a `[tool.versioneer]` section to your `pyproject.toml` or a + `[versioneer]` section to your `setup.cfg` (see [Install](INSTALL.md)) +* add `versioneer` (with `[toml]` extra, if configuring in `pyproject.toml`) + to the `requires` key of the `build-system` table in `pyproject.toml`: + ```toml + [build-system] + requires = ["setuptools", "versioneer[toml]"] + build-backend = "setuptools.build_meta" + ``` +* run `versioneer install --no-vendor` in your source tree, commit the results +* verify version information with `python setup.py version` ## Version Identifiers @@ -61,7 +86,7 @@ for example `git describe --tags --dirty --always` reports things like "0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the 0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. +uncommitted changes). The version identifier is used for multiple purposes: @@ -166,7 +191,7 @@ Some situations are known to cause problems for Versioneer. This details the most significant ones. More can be found on Github -[issues page](https://github.com/warner/python-versioneer/issues). +[issues page](https://github.com/python-versioneer/python-versioneer/issues). ### Subprojects @@ -180,7 +205,7 @@ `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI distributions (and upload multiple independently-installable tarballs). * Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other langauges) in subdirectories. + provide bindings to Python (and perhaps other languages) in subdirectories. Versioneer will look for `.git` in parent directories, and most operations should get the right version string. However `pip` and `setuptools` have bugs @@ -194,9 +219,9 @@ Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in some later version. -[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking +[Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) is tracking this issue. The discussion in -[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the +[PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) describes the issue from the Versioneer side in more detail. [pip PR#3176](https://github.com/pypa/pip/pull/3176) and [pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve @@ -224,31 +249,20 @@ cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into a different virtualenv), so this can be surprising. -[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes +[Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) describes this one, but upgrading to a newer version of setuptools should probably resolve it. -### Unicode version strings - -While Versioneer works (and is continually tested) with both Python 2 and -Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. -Newer releases probably generate unicode version strings on py2. It's not -clear that this is wrong, but it may be surprising for applications when then -write these strings to a network connection or include them in bytes-oriented -APIs like cryptographic checksums. - -[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates -this question. - ## Updating Versioneer To upgrade your project to a new release of Versioneer, do the following: * install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace +* edit `setup.cfg` and `pyproject.toml`, if necessary, + to include any new configuration settings indicated by the release notes. + See [UPGRADING](./UPGRADING.md) for details. +* re-run `versioneer install --[no-]vendor` in your source tree, to replace `SRC/_version.py` * commit any changed files @@ -265,35 +279,70 @@ direction and include code from all supported VCS systems, reducing the number of intermediate scripts. +## Similar projects + +* [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored build-time + dependency +* [minver](https://github.com/jbweston/miniver) - a lightweight reimplementation of + versioneer +* [versioningit](https://github.com/jwodder/versioningit) - a PEP 518-based setuptools + plugin ## License To make Versioneer easier to embed, all its code is dedicated to the public domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . +Specifically, both are released under the "Unlicense", as described in +https://unlicense.org/. + +[pypi-image]: https://img.shields.io/pypi/v/versioneer.svg +[pypi-url]: https://pypi.python.org/pypi/versioneer/ +[travis-image]: +https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg +[travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer """ +# pylint:disable=invalid-name,import-outside-toplevel,missing-function-docstring +# pylint:disable=missing-class-docstring,too-many-branches,too-many-statements +# pylint:disable=raise-missing-from,too-many-lines,too-many-locals,import-error +# pylint:disable=too-few-public-methods,redefined-outer-name,consider-using-with +# pylint:disable=attribute-defined-outside-init,too-many-arguments -from __future__ import print_function -try: - import configparser -except ImportError: - import ConfigParser as configparser +import configparser import errno import json import os import re import subprocess import sys +from pathlib import Path +from typing import Any, Callable, cast, Dict, List, Optional, Tuple, Union +from typing import NoReturn +import functools + +have_tomllib = True +if sys.version_info >= (3, 11): + import tomllib +else: + try: + import tomli as tomllib + except ImportError: + have_tomllib = False class VersioneerConfig: """Container for Versioneer configuration parameters.""" + VCS: str + style: str + tag_prefix: str + versionfile_source: str + versionfile_build: Optional[str] + parentdir_prefix: Optional[str] + verbose: Optional[bool] + -def get_root(): +def get_root() -> str: """Get the project root directory. We require that all commands are run from the project root, i.e. the @@ -301,13 +350,23 @@ def get_root(): """ root = os.path.realpath(os.path.abspath(os.getcwd())) setup_py = os.path.join(root, "setup.py") + pyproject_toml = os.path.join(root, "pyproject.toml") versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + if not ( + os.path.exists(setup_py) + or os.path.exists(pyproject_toml) + or os.path.exists(versioneer_py) + ): # allow 'python path/to/setup.py COMMAND' root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) setup_py = os.path.join(root, "setup.py") + pyproject_toml = os.path.join(root, "pyproject.toml") versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + if not ( + os.path.exists(setup_py) + or os.path.exists(pyproject_toml) + or os.path.exists(versioneer_py) + ): err = ("Versioneer was unable to run the project root directory. " "Versioneer requires setup.py to be executed from " "its immediate directory (like 'python setup.py COMMAND'), " @@ -321,43 +380,62 @@ def get_root(): # module-import table will cache the first one. So we can't use # os.path.dirname(__file__), as that will find whichever # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(me)[0]) + my_path = os.path.realpath(os.path.abspath(__file__)) + me_dir = os.path.normcase(os.path.splitext(my_path)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: + if me_dir != vsr_dir and "VERSIONEER_PEP518" not in globals(): print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py)) + % (os.path.dirname(my_path), versioneer_py)) except NameError: pass return root -def get_config_from_root(root): +def get_config_from_root(root: str) -> VersioneerConfig: """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or + # This might raise OSError (if setup.cfg is missing), or # configparser.NoSectionError (if it lacks a [versioneer] section), or # configparser.NoOptionError (if it lacks "VCS="). See the docstring at # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) - VCS = parser.get("versioneer", "VCS") # mandatory - - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None + root_pth = Path(root) + pyproject_toml = root_pth / "pyproject.toml" + setup_cfg = root_pth / "setup.cfg" + section: Union[Dict[str, Any], configparser.SectionProxy, None] = None + if pyproject_toml.exists() and have_tomllib: + try: + with open(pyproject_toml, 'rb') as fobj: + pp = tomllib.load(fobj) + section = pp['tool']['versioneer'] + except (tomllib.TOMLDecodeError, KeyError) as e: + print(f"Failed to load config from {pyproject_toml}: {e}") + print("Try to load it from setup.cfg") + if not section: + parser = configparser.ConfigParser() + with open(setup_cfg) as cfg_file: + parser.read_file(cfg_file) + parser.get("versioneer", "VCS") # raise error if missing + + section = parser["versioneer"] + + # `cast`` really shouldn't be used, but its simplest for the + # common VersioneerConfig users at the moment. We verify against + # `None` values elsewhere where it matters + cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") - if cfg.tag_prefix in ("''", '""'): + cfg.VCS = section['VCS'] + cfg.style = section.get("style", "") + cfg.versionfile_source = cast(str, section.get("versionfile_source")) + cfg.versionfile_build = section.get("versionfile_build") + cfg.tag_prefix = cast(str, section.get("tag_prefix")) + if cfg.tag_prefix in ("''", '""', None): cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") + cfg.parentdir_prefix = section.get("parentdir_prefix") + if isinstance(section, configparser.SectionProxy): + # Make sure configparser translates to bool + cfg.verbose = section.getboolean("verbose") + else: + cfg.verbose = section.get("verbose") + return cfg @@ -366,37 +444,48 @@ class NotThisMethod(Exception): # these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): +def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator + """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f + HANDLERS.setdefault(vcs, {})[method] = f return f return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command( + commands: List[str], + args: List[str], + cwd: Optional[str] = None, + verbose: bool = False, + hide_stderr: bool = False, + env: Optional[Dict[str, str]] = None, +) -> Tuple[Optional[str], Optional[int]]: """Call the given command(s).""" assert isinstance(commands, list) - p = None - for c in commands: + process = None + + popen_kwargs: Dict[str, Any] = {} + if sys.platform == "win32": + # This hides the console window if pythonw.exe is used + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + popen_kwargs["startupinfo"] = startupinfo + + for command in commands: try: - dispcmd = str([c] + args) + dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None), **popen_kwargs) break - except EnvironmentError: - e = sys.exc_info()[1] + except OSError as e: if e.errno == errno.ENOENT: continue if verbose: @@ -407,26 +496,25 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, if verbose: print("unable to find command, tried %s" % (commands,)) return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode + return None, process.returncode + return stdout, process.returncode -LONG_VERSION_PY['git'] = ''' +LONG_VERSION_PY['git'] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) +# This file is released into the public domain. +# Generated by versioneer-0.29 +# https://github.com/python-versioneer/python-versioneer """Git implementation of _version.py.""" @@ -435,9 +523,11 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, import re import subprocess import sys +from typing import Any, Callable, Dict, List, Optional, Tuple +import functools -def get_keywords(): +def get_keywords() -> Dict[str, str]: """Get the keywords needed to look up the version information.""" # these strings will be replaced by git during git-archive. # setup.py/versioneer.py will grep for the variable names, so they must @@ -453,8 +543,15 @@ def get_keywords(): class VersioneerConfig: """Container for Versioneer configuration parameters.""" + VCS: str + style: str + tag_prefix: str + parentdir_prefix: str + versionfile_source: str + verbose: bool + -def get_config(): +def get_config() -> VersioneerConfig: """Create, populate and return the VersioneerConfig() object.""" # these strings are filled in when 'setup.py versioneer' creates # _version.py @@ -472,13 +569,13 @@ class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" -LONG_VERSION_PY = {} -HANDLERS = {} +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): +def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator + """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} @@ -487,22 +584,35 @@ def decorate(f): return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command( + commands: List[str], + args: List[str], + cwd: Optional[str] = None, + verbose: bool = False, + hide_stderr: bool = False, + env: Optional[Dict[str, str]] = None, +) -> Tuple[Optional[str], Optional[int]]: """Call the given command(s).""" assert isinstance(commands, list) - p = None - for c in commands: + process = None + + popen_kwargs: Dict[str, Any] = {} + if sys.platform == "win32": + # This hides the console window if pythonw.exe is used + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + popen_kwargs["startupinfo"] = startupinfo + + for command in commands: try: - dispcmd = str([c] + args) + dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None), **popen_kwargs) break - except EnvironmentError: - e = sys.exc_info()[1] + except OSError as e: if e.errno == errno.ENOENT: continue if verbose: @@ -513,18 +623,20 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, if verbose: print("unable to find command, tried %%s" %% (commands,)) return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: if verbose: print("unable to run %%s (error)" %% dispcmd) print("stdout was %%s" %% stdout) - return None, p.returncode - return stdout, p.returncode + return None, process.returncode + return stdout, process.returncode -def versions_from_parentdir(parentdir_prefix, root, verbose): +def versions_from_parentdir( + parentdir_prefix: str, + root: str, + verbose: bool, +) -> Dict[str, Any]: """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both @@ -533,15 +645,14 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): """ rootdirs = [] - for i in range(3): + for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level + rootdirs.append(root) + root = os.path.dirname(root) # up a level if verbose: print("Tried directories %%s but none started with prefix %%s" %% @@ -550,41 +661,48 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): @register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): +def git_get_keywords(versionfile_abs: str) -> Dict[str, str]: """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. - keywords = {} + keywords: Dict[str, str] = {} try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: pass return keywords @register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): +def git_versions_from_keywords( + keywords: Dict[str, str], + tag_prefix: str, + verbose: bool, +) -> Dict[str, Any]: """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because @@ -597,11 +715,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %%d @@ -610,7 +728,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%%s', no digits" %% ",".join(refs - tags)) if verbose: @@ -619,6 +737,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r'\d', r): + continue if verbose: print("picking %%s" %% r) return {"version": r, @@ -634,7 +757,12 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): @register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): +def git_pieces_from_vcs( + tag_prefix: str, + root: str, + verbose: bool, + runner: Callable = run_command +) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* @@ -645,8 +773,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + # GIT_DIR can interfere with correct operation of Versioneer. + # It may be intended to be passed to the Versioneer-versioned project, + # but that should not change where we get our version from. + env = os.environ.copy() + env.pop("GIT_DIR", None) + runner = functools.partial(runner, env=env) + + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %%s not under git control" %% root) @@ -654,24 +789,57 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) + describe_out, rc = runner(GITS, [ + "describe", "--tags", "--dirty", "--always", "--long", + "--match", f"{tag_prefix}[[:digit:]]*" + ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() - pieces = {} + pieces: Dict[str, Any] = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out @@ -688,7 +856,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: - # unparseable. Maybe git-describe is misbehaving? + # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%%s'" %% describe_out) return pieces @@ -713,26 +881,27 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits + out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) + pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() + date = runner(GITS, ["show", "-s", "--format=%%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces -def plus_or_dot(pieces): +def plus_or_dot(pieces: Dict[str, Any]) -> str: """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" -def render_pep440(pieces): +def render_pep440(pieces: Dict[str, Any]) -> str: """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you @@ -757,23 +926,71 @@ def render_pep440(pieces): return rendered -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. +def render_pep440_branch(pieces: Dict[str, Any]) -> str: + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). Exceptions: - 1: no tags. 0.post.devDISTANCE + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver: str) -> Tuple[str, Optional[int]]: + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces: Dict[str, Any]) -> str: + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%%d.dev%%d" %% (post_version + 1, pieces["distance"]) + else: + rendered += ".post0.dev%%d" %% (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] else: # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] + rendered = "0.post0.dev%%d" %% pieces["distance"] return rendered -def render_pep440_post(pieces): +def render_pep440_post(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards @@ -800,12 +1017,41 @@ def render_pep440_post(pieces): return rendered -def render_pep440_old(pieces): +def render_pep440_post_branch(pieces: Dict[str, Any]) -> str: + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_old(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -822,7 +1068,7 @@ def render_pep440_old(pieces): return rendered -def render_git_describe(pieces): +def render_git_describe(pieces: Dict[str, Any]) -> str: """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. @@ -842,7 +1088,7 @@ def render_git_describe(pieces): return rendered -def render_git_describe_long(pieces): +def render_git_describe_long(pieces: Dict[str, Any]) -> str: """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. @@ -862,7 +1108,7 @@ def render_git_describe_long(pieces): return rendered -def render(pieces, style): +def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", @@ -876,10 +1122,14 @@ def render(pieces, style): if style == "pep440": rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": @@ -894,7 +1144,7 @@ def render(pieces, style): "date": pieces.get("date")} -def get_versions(): +def get_versions() -> Dict[str, Any]: """Get version information or return default if unable to do so.""" # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have # __file__, we can work backwards from there to the root. Some @@ -915,7 +1165,7 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): + for _ in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: return {"version": "0+unknown", "full-revisionid": None, @@ -942,41 +1192,48 @@ def get_versions(): @register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): +def git_get_keywords(versionfile_abs: str) -> Dict[str, str]: """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. - keywords = {} + keywords: Dict[str, str] = {} try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: pass return keywords @register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): +def git_versions_from_keywords( + keywords: Dict[str, str], + tag_prefix: str, + verbose: bool, +) -> Dict[str, Any]: """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because @@ -989,11 +1246,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1002,7 +1259,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1011,6 +1268,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r'\d', r): + continue if verbose: print("picking %s" % r) return {"version": r, @@ -1026,7 +1288,12 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): @register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): +def git_pieces_from_vcs( + tag_prefix: str, + root: str, + verbose: bool, + runner: Callable = run_command +) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* @@ -1037,8 +1304,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + # GIT_DIR can interfere with correct operation of Versioneer. + # It may be intended to be passed to the Versioneer-versioned project, + # but that should not change where we get our version from. + env = os.environ.copy() + env.pop("GIT_DIR", None) + runner = functools.partial(runner, env=env) + + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1046,24 +1320,57 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = runner(GITS, [ + "describe", "--tags", "--dirty", "--always", "--long", + "--match", f"{tag_prefix}[[:digit:]]*" + ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() - pieces = {} + pieces: Dict[str, Any] = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out @@ -1080,7 +1387,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: - # unparseable. Maybe git-describe is misbehaving? + # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%s'" % describe_out) return pieces @@ -1105,19 +1412,20 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits + out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) + pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces -def do_vcs_install(manifest_in, versionfile_source, ipy): +def do_vcs_install(versionfile_source: str, ipy: Optional[str]) -> None: """Git-specific installation logic for Versioneer. For Git, this means creating/changing .gitattributes to mark _version.py @@ -1126,36 +1434,40 @@ def do_vcs_install(manifest_in, versionfile_source, ipy): GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] + files = [versionfile_source] if ipy: files.append(ipy) - try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) + if "VERSIONEER_PEP518" not in globals(): + try: + my_path = __file__ + if my_path.endswith((".pyc", ".pyo")): + my_path = os.path.splitext(my_path)[0] + ".py" + versioneer_file = os.path.relpath(my_path) + except NameError: + versioneer_file = "versioneer.py" + files.append(versioneer_file) present = False try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: + with open(".gitattributes", "r") as fobj: + for line in fobj: + if line.strip().startswith(versionfile_source): + if "export-subst" in line.strip().split()[1:]: + present = True + break + except OSError: pass if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() + with open(".gitattributes", "a+") as fobj: + fobj.write(f"{versionfile_source} export-subst\n") files.append(".gitattributes") run_command(GITS, ["add", "--"] + files) -def versions_from_parentdir(parentdir_prefix, root, verbose): +def versions_from_parentdir( + parentdir_prefix: str, + root: str, + verbose: bool, +) -> Dict[str, Any]: """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both @@ -1164,15 +1476,14 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): """ rootdirs = [] - for i in range(3): + for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level + rootdirs.append(root) + root = os.path.dirname(root) # up a level if verbose: print("Tried directories %s but none started with prefix %s" % @@ -1181,7 +1492,7 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.18) from +# This file was generated by 'versioneer.py' (0.29) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. @@ -1198,12 +1509,12 @@ def get_versions(): """ -def versions_from_file(filename): +def versions_from_file(filename: str) -> Dict[str, Any]: """Try to determine the version from _version.py if present.""" try: with open(filename) as f: contents = f.read() - except EnvironmentError: + except OSError: raise NotThisMethod("unable to read _version.py") mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) @@ -1215,9 +1526,8 @@ def versions_from_file(filename): return json.loads(mo.group(1)) -def write_to_version_file(filename, versions): +def write_to_version_file(filename: str, versions: Dict[str, Any]) -> None: """Write the given version number to the given _version.py file.""" - os.unlink(filename) contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) with open(filename, "w") as f: @@ -1226,14 +1536,14 @@ def write_to_version_file(filename, versions): print("set %s to '%s'" % (filename, versions["version"])) -def plus_or_dot(pieces): +def plus_or_dot(pieces: Dict[str, Any]) -> str: """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" -def render_pep440(pieces): +def render_pep440(pieces: Dict[str, Any]) -> str: """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you @@ -1258,23 +1568,71 @@ def render_pep440(pieces): return rendered -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. +def render_pep440_branch(pieces: Dict[str, Any]) -> str: + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). Exceptions: - 1: no tags. 0.post.devDISTANCE + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver: str) -> Tuple[str, Optional[int]]: + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces: Dict[str, Any]) -> str: + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) + else: + rendered += ".post0.dev%d" % (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] else: # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] + rendered = "0.post0.dev%d" % pieces["distance"] return rendered -def render_pep440_post(pieces): +def render_pep440_post(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards @@ -1301,12 +1659,41 @@ def render_pep440_post(pieces): return rendered -def render_pep440_old(pieces): +def render_pep440_post_branch(pieces: Dict[str, Any]) -> str: + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_old(pieces: Dict[str, Any]) -> str: """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -1323,7 +1710,7 @@ def render_pep440_old(pieces): return rendered -def render_git_describe(pieces): +def render_git_describe(pieces: Dict[str, Any]) -> str: """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. @@ -1343,7 +1730,7 @@ def render_git_describe(pieces): return rendered -def render_git_describe_long(pieces): +def render_git_describe_long(pieces: Dict[str, Any]) -> str: """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. @@ -1363,7 +1750,7 @@ def render_git_describe_long(pieces): return rendered -def render(pieces, style): +def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", @@ -1377,10 +1764,14 @@ def render(pieces, style): if style == "pep440": rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": @@ -1399,7 +1790,7 @@ class VersioneerBadRootError(Exception): """The project root directory is unknown or missing key files.""" -def get_versions(verbose=False): +def get_versions(verbose: bool = False) -> Dict[str, Any]: """Get the project version from whatever source is available. Returns dict with two keys: 'version' and 'full'. @@ -1414,7 +1805,7 @@ def get_versions(verbose=False): assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose + verbose = verbose or bool(cfg.verbose) # `bool()` used to avoid `None` assert cfg.versionfile_source is not None, \ "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" @@ -1475,13 +1866,17 @@ def get_versions(verbose=False): "date": None} -def get_version(): +def get_version() -> str: """Get the short version string for this project.""" return get_versions()["version"] -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" +def get_cmdclass(cmdclass: Optional[Dict[str, Any]] = None): + """Get the custom setuptools subclasses used by Versioneer. + + If the package uses a different cmdclass (e.g. one from numpy), it + should be provide as an argument. + """ if "versioneer" in sys.modules: del sys.modules["versioneer"] # this fixes the "python setup.py develop" case (also 'install' and @@ -1495,25 +1890,25 @@ def get_cmdclass(): # parent is protected against the child's "import versioneer". By # removing ourselves from sys.modules here, before the child build # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 + # Also see https://github.com/python-versioneer/python-versioneer/issues/52 - cmds = {} + cmds = {} if cmdclass is None else cmdclass.copy() - # we add "version" to both distutils and setuptools - from distutils.core import Command + # we add "version" to setuptools + from setuptools import Command class cmd_version(Command): description = "report generated version string" - user_options = [] - boolean_options = [] + user_options: List[Tuple[str, str, str]] = [] + boolean_options: List[str] = [] - def initialize_options(self): + def initialize_options(self) -> None: pass - def finalize_options(self): + def finalize_options(self) -> None: pass - def run(self): + def run(self) -> None: vers = get_versions(verbose=True) print("Version: %s" % vers["version"]) print(" full-revisionid: %s" % vers.get("full-revisionid")) @@ -1523,7 +1918,7 @@ def run(self): print(" error: %s" % vers["error"]) cmds["version"] = cmd_version - # we override "build_py" in both distutils and setuptools + # we override "build_py" in setuptools # # most invocation pathways end up running build_py: # distutils/build -> build_py @@ -1538,18 +1933,25 @@ def run(self): # then does setup.py bdist_wheel, or sometimes setup.py install # setup.py egg_info -> ? + # pip install -e . and setuptool/editable_wheel will invoke build_py + # but the build_py command is not expected to copy any files. + # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py + if 'build_py' in cmds: + _build_py: Any = cmds['build_py'] else: - from distutils.command.build_py import build_py as _build_py + from setuptools.command.build_py import build_py as _build_py class cmd_build_py(_build_py): - def run(self): + def run(self) -> None: root = get_root() cfg = get_config_from_root(root) versions = get_versions() _build_py.run(self) + if getattr(self, "editable_mode", False): + # During editable installs `.py` and data files are + # not copied to build_lib + return # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: @@ -1559,8 +1961,40 @@ def run(self): write_to_version_file(target_versionfile, versions) cmds["build_py"] = cmd_build_py + if 'build_ext' in cmds: + _build_ext: Any = cmds['build_ext'] + else: + from setuptools.command.build_ext import build_ext as _build_ext + + class cmd_build_ext(_build_ext): + def run(self) -> None: + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_ext.run(self) + if self.inplace: + # build_ext --inplace will only build extensions in + # build/lib<..> dir with no _version.py to write to. + # As in place builds will already have a _version.py + # in the module dir, we do not need to write one. + return + # now locate _version.py in the new build/ directory and replace + # it with an updated value + if not cfg.versionfile_build: + return + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) + if not os.path.exists(target_versionfile): + print(f"Warning: {target_versionfile} does not exist, skipping " + "version update. This can happen if you are running build_ext " + "without first running build_py.") + return + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + cmds["build_ext"] = cmd_build_ext + if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe + from cx_Freeze.dist import build_exe as _build_exe # type: ignore # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ @@ -1569,7 +2003,7 @@ def run(self): # ... class cmd_build_exe(_build_exe): - def run(self): + def run(self) -> None: root = get_root() cfg = get_config_from_root(root) versions = get_versions() @@ -1593,12 +2027,12 @@ def run(self): if 'py2exe' in sys.modules: # py2exe enabled? try: - from py2exe.distutils_buildexe import py2exe as _py2exe # py3 + from py2exe.setuptools_buildexe import py2exe as _py2exe # type: ignore except ImportError: - from py2exe.build_exe import py2exe as _py2exe # py2 + from py2exe.distutils_buildexe import py2exe as _py2exe # type: ignore class cmd_py2exe(_py2exe): - def run(self): + def run(self) -> None: root = get_root() cfg = get_config_from_root(root) versions = get_versions() @@ -1619,14 +2053,51 @@ def run(self): }) cmds["py2exe"] = cmd_py2exe + # sdist farms its file list building out to egg_info + if 'egg_info' in cmds: + _egg_info: Any = cmds['egg_info'] + else: + from setuptools.command.egg_info import egg_info as _egg_info + + class cmd_egg_info(_egg_info): + def find_sources(self) -> None: + # egg_info.find_sources builds the manifest list and writes it + # in one shot + super().find_sources() + + # Modify the filelist and normalize it + root = get_root() + cfg = get_config_from_root(root) + self.filelist.append('versioneer.py') + if cfg.versionfile_source: + # There are rare cases where versionfile_source might not be + # included by default, so we must be explicit + self.filelist.append(cfg.versionfile_source) + self.filelist.sort() + self.filelist.remove_duplicates() + + # The write method is hidden in the manifest_maker instance that + # generated the filelist and was thrown away + # We will instead replicate their final normalization (to unicode, + # and POSIX-style paths) + from setuptools import unicode_utils + normalized = [unicode_utils.filesys_decode(f).replace(os.sep, '/') + for f in self.filelist.files] + + manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt') + with open(manifest_filename, 'w') as fobj: + fobj.write('\n'.join(normalized)) + + cmds['egg_info'] = cmd_egg_info + # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist + if 'sdist' in cmds: + _sdist: Any = cmds['sdist'] else: - from distutils.command.sdist import sdist as _sdist + from setuptools.command.sdist import sdist as _sdist class cmd_sdist(_sdist): - def run(self): + def run(self) -> None: versions = get_versions() self._versioneer_generated_versions = versions # unless we update this, the command will keep using the old @@ -1634,7 +2105,7 @@ def run(self): self.distribution.metadata.version = versions["version"] return _sdist.run(self) - def make_release_tree(self, base_dir, files): + def make_release_tree(self, base_dir: str, files: List[str]) -> None: root = get_root() cfg = get_config_from_root(root) _sdist.make_release_tree(self, base_dir, files) @@ -1687,21 +2158,26 @@ def make_release_tree(self, base_dir, files): """ -INIT_PY_SNIPPET = """ +OLD_SNIPPET = """ from ._version import get_versions __version__ = get_versions()['version'] del get_versions """ +INIT_PY_SNIPPET = """ +from . import {0} +__version__ = {0}.get_versions()['version'] +""" -def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" + +def do_setup() -> int: + """Do main VCS-independent setup function for installing Versioneer.""" root = get_root() try: cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, + except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + if isinstance(e, (OSError, configparser.NoSectionError)): print("Adding sample versioneer config to setup.cfg", file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: @@ -1721,62 +2197,37 @@ def do_setup(): ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") + maybe_ipy: Optional[str] = ipy if os.path.exists(ipy): try: with open(ipy, "r") as f: old = f.read() - except EnvironmentError: + except OSError: old = "" - if INIT_PY_SNIPPET not in old: + module = os.path.splitext(os.path.basename(cfg.versionfile_source))[0] + snippet = INIT_PY_SNIPPET.format(module) + if OLD_SNIPPET in old: + print(" replacing boilerplate in %s" % ipy) + with open(ipy, "w") as f: + f.write(old.replace(OLD_SNIPPET, snippet)) + elif snippet not in old: print(" appending to %s" % ipy) with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) + f.write(snippet) else: print(" %s unmodified" % ipy) else: print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in, "r") as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % - cfg.versionfile_source) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") + maybe_ipy = None # Make VCS-specific changes. For git, this means creating/changing # .gitattributes to mark _version.py for export-subst keyword # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) + do_vcs_install(cfg.versionfile_source, maybe_ipy) return 0 -def scan_setup_py(): +def scan_setup_py() -> int: """Validate the contents of setup.py against Versioneer's expectations.""" found = set() setters = False @@ -1813,10 +2264,14 @@ def scan_setup_py(): return errors +def setup_command() -> NoReturn: + """Set up Versioneer and exit with appropriate error code.""" + errors = do_setup() + errors += scan_setup_py() + sys.exit(1 if errors else 0) + + if __name__ == "__main__": cmd = sys.argv[1] if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) + setup_command() From 2a86aebebf74be0aba3b77672e5790718e1a2726 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:01:37 +0000 Subject: [PATCH 32/58] Bump mpi4py from 3.1.4 to 3.1.5 Bumps [mpi4py](https://github.com/mpi4py/mpi4py) from 3.1.4 to 3.1.5. - [Release notes](https://github.com/mpi4py/mpi4py/releases) - [Changelog](https://github.com/mpi4py/mpi4py/blob/master/CHANGES.rst) - [Commits](https://github.com/mpi4py/mpi4py/compare/3.1.4...3.1.5) --- updated-dependencies: - dependency-name: mpi4py dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7f02c491..9331b686 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ packages=find_packages(exclude=["*tests*", "*.ci_support*"]), install_requires=[ 'cloudpickle==2.2.1', - 'mpi4py==3.1.4', + 'mpi4py==3.1.5', 'tqdm==4.66.1', 'pyzmq==24.0.1', ], From 596ef881198dfaae1e3a978259d1c7ef2d5097b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:01:43 +0000 Subject: [PATCH 33/58] Bump pyzmq from 24.0.1 to 25.1.1 Bumps [pyzmq](https://github.com/zeromq/pyzmq) from 24.0.1 to 25.1.1. - [Release notes](https://github.com/zeromq/pyzmq/releases) - [Commits](https://github.com/zeromq/pyzmq/compare/v24.0.1...v25.1.1) --- updated-dependencies: - dependency-name: pyzmq dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7f02c491..9ee0f78d 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ 'cloudpickle==2.2.1', 'mpi4py==3.1.4', 'tqdm==4.66.1', - 'pyzmq==24.0.1', + 'pyzmq==25.1.1', ], cmdclass=versioneer.get_cmdclass(), ) From 595902ff541d67f505ce3ca6e117453fe40dd8b7 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 4 Oct 2023 15:49:59 +0200 Subject: [PATCH 34/58] Update environment-mpich.yml --- .ci_support/environment-mpich.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-mpich.yml b/.ci_support/environment-mpich.yml index 46e3b286..e5f40b21 100644 --- a/.ci_support/environment-mpich.yml +++ b/.ci_support/environment-mpich.yml @@ -5,6 +5,6 @@ dependencies: - numpy - mpich - cloudpickle =2.2.1 -- mpi4py =3.1.4 +- mpi4py =3.1.5 - tqdm =4.66.1 - pyzmq =24.0.1 From af70ceb88beffda7eacfd98c05f06a904c110fd3 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 4 Oct 2023 15:50:10 +0200 Subject: [PATCH 35/58] Update environment-openmpi.yml --- .ci_support/environment-openmpi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-openmpi.yml b/.ci_support/environment-openmpi.yml index e4a2f862..7cb87df5 100644 --- a/.ci_support/environment-openmpi.yml +++ b/.ci_support/environment-openmpi.yml @@ -5,6 +5,6 @@ dependencies: - numpy - openmpi - cloudpickle =2.2.1 -- mpi4py =3.1.4 +- mpi4py =3.1.5 - tqdm =4.66.1 - pyzmq =24.0.1 From 16a3e724f8eed9e68348277c5ab82b9ab1b87631 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 4 Oct 2023 15:50:21 +0200 Subject: [PATCH 36/58] Update environment-win.yml --- .ci_support/environment-win.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-win.yml b/.ci_support/environment-win.yml index a7356492..6b93f026 100644 --- a/.ci_support/environment-win.yml +++ b/.ci_support/environment-win.yml @@ -5,6 +5,6 @@ dependencies: - numpy - msmpi - cloudpickle =2.2.1 -- mpi4py =3.1.4 +- mpi4py =3.1.5 - tqdm =4.66.1 - pyzmq =24.0.1 From 2d316fb25cf9fb20cbe5d20345cf9f02236bd905 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 4 Oct 2023 16:05:05 +0200 Subject: [PATCH 37/58] Update environment-mpich.yml --- .ci_support/environment-mpich.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-mpich.yml b/.ci_support/environment-mpich.yml index 46e3b286..cba47606 100644 --- a/.ci_support/environment-mpich.yml +++ b/.ci_support/environment-mpich.yml @@ -7,4 +7,4 @@ dependencies: - cloudpickle =2.2.1 - mpi4py =3.1.4 - tqdm =4.66.1 -- pyzmq =24.0.1 +- pyzmq =25.1.1 From efc542438677b36187d5ae3a85966a685f500dbc Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 4 Oct 2023 16:05:15 +0200 Subject: [PATCH 38/58] Update environment-openmpi.yml --- .ci_support/environment-openmpi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-openmpi.yml b/.ci_support/environment-openmpi.yml index e4a2f862..3dcad16c 100644 --- a/.ci_support/environment-openmpi.yml +++ b/.ci_support/environment-openmpi.yml @@ -7,4 +7,4 @@ dependencies: - cloudpickle =2.2.1 - mpi4py =3.1.4 - tqdm =4.66.1 -- pyzmq =24.0.1 +- pyzmq =25.1.1 From ae6a0cf19bd11b8d5e2fffdd3c7d5aa6b6fb1408 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 4 Oct 2023 16:05:28 +0200 Subject: [PATCH 39/58] Update environment-win.yml --- .ci_support/environment-win.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-win.yml b/.ci_support/environment-win.yml index a7356492..d4ac426c 100644 --- a/.ci_support/environment-win.yml +++ b/.ci_support/environment-win.yml @@ -7,4 +7,4 @@ dependencies: - cloudpickle =2.2.1 - mpi4py =3.1.4 - tqdm =4.66.1 -- pyzmq =24.0.1 +- pyzmq =25.1.1 From 9dbcaa107ebf7c428418b18f3ad89da87cad7951 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Thu, 5 Oct 2023 22:16:02 +0200 Subject: [PATCH 40/58] Update unittest-flux.yml --- .github/workflows/unittest-flux.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/unittest-flux.yml b/.github/workflows/unittest-flux.yml index 9b1a096f..66021ac1 100644 --- a/.github/workflows/unittest-flux.yml +++ b/.github/workflows/unittest-flux.yml @@ -64,15 +64,11 @@ jobs: run: > flux start coverage run --omit pympipool/_version.py -m unittest tests/test_flux.py; + coverage combine env: OMPI_MCA_plm: 'isolated' OMPI_MCA_rmaps_base_oversubscribe: 'yes' OMPI_MCA_btl_vader_single_copy_mechanism: 'none' - - name: Coverage + - name: Coveralls if: matrix.label == 'linux-64-py-3-11-openmpi' - continue-on-error: True - shell: bash -l {0} - run: | - coverage combine - coveralls - coverage xml + uses: coverallsapp/github-action@v2 From 4ceb2f5fed86ee81660482e222a15b2a0b14ecbd Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Thu, 5 Oct 2023 22:16:40 +0200 Subject: [PATCH 41/58] Update unittest-flux.yml --- .github/workflows/unittest-flux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittest-flux.yml b/.github/workflows/unittest-flux.yml index 66021ac1..4b990797 100644 --- a/.github/workflows/unittest-flux.yml +++ b/.github/workflows/unittest-flux.yml @@ -40,7 +40,7 @@ jobs: environment-file: ${{ matrix.environment-file }} - name: Install flux shell: bash -l {0} - run: mamba install -y flux-core coverage coveralls=3.3.1 + run: mamba install -y flux-core coverage - name: Setup shell: bash -l {0} run: pip install --no-deps . From b4d60c6639cd273981bec0c48320966b7c9fc65b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:00:38 +0000 Subject: [PATCH 42/58] Bump cloudpickle from 2.2.1 to 3.0.0 Bumps [cloudpickle](https://github.com/cloudpipe/cloudpickle) from 2.2.1 to 3.0.0. - [Release notes](https://github.com/cloudpipe/cloudpickle/releases) - [Changelog](https://github.com/cloudpipe/cloudpickle/blob/master/CHANGES.md) - [Commits](https://github.com/cloudpipe/cloudpickle/compare/v2.2.1...3.0.0) --- updated-dependencies: - dependency-name: cloudpickle dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9ee0f78d..fd740c24 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ ], packages=find_packages(exclude=["*tests*", "*.ci_support*"]), install_requires=[ - 'cloudpickle==2.2.1', + 'cloudpickle==3.0.0', 'mpi4py==3.1.4', 'tqdm==4.66.1', 'pyzmq==25.1.1', From 77a424a338ea7f512fe1abf69c00865cbc660b29 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Mon, 16 Oct 2023 15:15:22 +0200 Subject: [PATCH 43/58] Update environment-mpich.yml --- .ci_support/environment-mpich.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-mpich.yml b/.ci_support/environment-mpich.yml index cba47606..e4d97ded 100644 --- a/.ci_support/environment-mpich.yml +++ b/.ci_support/environment-mpich.yml @@ -4,7 +4,7 @@ dependencies: - python - numpy - mpich -- cloudpickle =2.2.1 +- cloudpickle =3.0.0 - mpi4py =3.1.4 - tqdm =4.66.1 - pyzmq =25.1.1 From f96c33b245f04eadad95e7a9e005b9bb55f7082d Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Mon, 16 Oct 2023 15:15:38 +0200 Subject: [PATCH 44/58] Update environment-openmpi.yml --- .ci_support/environment-openmpi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-openmpi.yml b/.ci_support/environment-openmpi.yml index 3dcad16c..996a6d95 100644 --- a/.ci_support/environment-openmpi.yml +++ b/.ci_support/environment-openmpi.yml @@ -4,7 +4,7 @@ dependencies: - python - numpy - openmpi -- cloudpickle =2.2.1 +- cloudpickle =3.0.0 - mpi4py =3.1.4 - tqdm =4.66.1 - pyzmq =25.1.1 From 8bf2c5583713c58c7e1522d9e687866ad1d13ef3 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Mon, 16 Oct 2023 15:16:22 +0200 Subject: [PATCH 45/58] Update environment-win.yml --- .ci_support/environment-win.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment-win.yml b/.ci_support/environment-win.yml index d4ac426c..9e84be7f 100644 --- a/.ci_support/environment-win.yml +++ b/.ci_support/environment-win.yml @@ -4,7 +4,7 @@ dependencies: - python - numpy - msmpi -- cloudpickle =2.2.1 +- cloudpickle =3.0.0 - mpi4py =3.1.4 - tqdm =4.66.1 - pyzmq =25.1.1 From 367b427d0e5ba9bcef91a276bba952db32b51486 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 17 Oct 2023 15:40:57 +0200 Subject: [PATCH 46/58] Update unittest-openmpi.yml --- .github/workflows/unittest-openmpi.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/unittest-openmpi.yml b/.github/workflows/unittest-openmpi.yml index 7a99f59d..5b1899ff 100644 --- a/.github/workflows/unittest-openmpi.yml +++ b/.github/workflows/unittest-openmpi.yml @@ -24,21 +24,6 @@ jobs: python-version: '3.11' label: linux-64-py-3-11-openmpi prefix: /usr/share/miniconda3/envs/my-env - - - operating-system: ubuntu-latest - python-version: '3.10' - label: linux-64-py-3-10-openmpi - prefix: /usr/share/miniconda3/envs/my-env - - - operating-system: ubuntu-latest - python-version: 3.9 - label: linux-64-py-3-9-openmpi - prefix: /usr/share/miniconda3/envs/my-env - - - operating-system: ubuntu-latest - python-version: 3.8 - label: linux-64-py-3-8-openmpi - prefix: /usr/share/miniconda3/envs/my-env steps: - uses: actions/checkout@v2 From ab86f2b571ebd9576f7885733a69e7282ed78dee Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Thu, 19 Oct 2023 14:15:06 +0200 Subject: [PATCH 47/58] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fd740c24..d8f33265 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='pympipool', version=versioneer.get_version(), - description='pympipool - scale python functions over multiple compute nodes', + description='pympipool - Scale serial and MPI-parallel python functions over hundreds of compute nodes all from within a jupyter notebook or serial python process.', long_description=Path("README.md").read_text(), long_description_content_type='text/markdown', url='https://github.com/jan-janssen/pympipool', From 6c3a1c4293126232e4f9a21a981490007d410b75 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Thu, 19 Oct 2023 14:31:29 +0200 Subject: [PATCH 48/58] Update unittest-openmpi.yml --- .github/workflows/unittest-openmpi.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/unittest-openmpi.yml b/.github/workflows/unittest-openmpi.yml index 5b1899ff..616e4d74 100644 --- a/.github/workflows/unittest-openmpi.yml +++ b/.github/workflows/unittest-openmpi.yml @@ -25,6 +25,21 @@ jobs: label: linux-64-py-3-11-openmpi prefix: /usr/share/miniconda3/envs/my-env + - operating-system: ubuntu-latest + python-version: '3.10' + label: linux-64-py-3-10-openmpi + prefix: /usr/share/miniconda3/envs/my-env + + - operating-system: ubuntu-latest + python-version: 3.9 + label: linux-64-py-3-9-openmpi + prefix: /usr/share/miniconda3/envs/my-env + + - operating-system: ubuntu-latest + python-version: 3.8 + label: linux-64-py-3-8-openmpi + prefix: /usr/share/miniconda3/envs/my-env + steps: - uses: actions/checkout@v2 - uses: conda-incubator/setup-miniconda@v2.2.0 From 6d265452f076706001e937bb56e4ac6d2a1f4eea Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 31 Oct 2023 11:26:48 +0100 Subject: [PATCH 49/58] Split Slurm and MPI interface --- pympipool/mpi/executor.py | 59 +---------------- pympipool/shared/interface.py | 31 +++++---- pympipool/slurm/__init__.py | 1 + pympipool/slurm/executor.py | 119 ++++++++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+), 69 deletions(-) create mode 100644 pympipool/slurm/__init__.py create mode 100644 pympipool/slurm/executor.py diff --git a/pympipool/mpi/executor.py b/pympipool/mpi/executor.py index 9aac7289..5d137789 100644 --- a/pympipool/mpi/executor.py +++ b/pympipool/mpi/executor.py @@ -4,7 +4,7 @@ ExecutorBase, executor_broker, ) -from pympipool.shared.interface import MpiExecInterface, SlurmSubprocessInterface +from pympipool.shared.interface import MpiExecInterface from pympipool.shared.thread import RaisingThread @@ -13,39 +13,22 @@ class PyMPIExecutor(ExecutorBase): Args: max_workers (int): defines the number workers which can execute functions in parallel cores_per_worker (int): number of MPI cores to be used for each function call - threads_per_core (int): number of OpenMP threads to be used for each function call - gpus_per_worker (int): number of GPUs per worker - defaults to 0 oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI only) - default False init_function (None): optional function to preset arguments for functions which are submitted later cwd (str/None): current working directory where the parallel python task is executed sleep_interval (float): synchronization interval - default 0.1 - enable_slurm_backend (bool): enable the SLURM queueing system as backend - defaults to False """ def __init__( self, max_workers, cores_per_worker=1, - threads_per_core=1, - gpus_per_worker=0, oversubscribe=False, init_function=None, cwd=None, sleep_interval=0.1, - enable_slurm_backend=False, ): super().__init__() - if not enable_slurm_backend: - if threads_per_core != 1: - raise ValueError( - "The MPI backend only supports threads_per_core=1, " - + "to manage threads use the SLURM queuing system enable_slurm_backend=True ." - ) - elif gpus_per_worker != 0: - raise ValueError( - "The MPI backend only supports gpus_per_core=0, " - + "to manage GPUs use the SLURM queuing system enable_slurm_backend=True ." - ) self._process = RaisingThread( target=executor_broker, kwargs={ @@ -56,12 +39,9 @@ def __init__( "executor_class": PyMPISingleTaskExecutor, # Executor Arguments "cores": cores_per_worker, - "threads_per_core": threads_per_core, - "gpus_per_task": int(gpus_per_worker / cores_per_worker), "oversubscribe": oversubscribe, "init_function": init_function, "cwd": cwd, - "enable_slurm_backend": enable_slurm_backend, }, ) self._process.start() @@ -77,12 +57,9 @@ class PyMPISingleTaskExecutor(ExecutorBase): Args: cores (int): defines the number of MPI ranks to use for each function call - threads_per_core (int): number of OpenMP threads to be used for each function call - gpus_per_task (int): number of GPUs per MPI rank - defaults to 0 oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI only) - default False init_function (None): optional function to preset arguments for functions which are submitted later cwd (str/None): current working directory where the parallel python task is executed - enable_slurm_backend (bool): enable the SLURM queueing system as backend - defaults to False Examples: ``` @@ -108,12 +85,9 @@ class PyMPISingleTaskExecutor(ExecutorBase): def __init__( self, cores=1, - threads_per_core=1, - gpus_per_task=0, oversubscribe=False, init_function=None, cwd=None, - enable_slurm_backend=False, ): super().__init__() self._process = RaisingThread( @@ -122,41 +96,12 @@ def __init__( # Executor Arguments "future_queue": self._future_queue, "cores": cores, - "interface_class": get_interface, + "interface_class": MpiExecInterface, # Interface Arguments - "threads_per_core": threads_per_core, - "gpus_per_core": gpus_per_task, "cwd": cwd, "oversubscribe": oversubscribe, - "enable_slurm_backend": enable_slurm_backend, }, ) self._process.start() self._set_init_function(init_function=init_function) cloudpickle_register(ind=3) - - -def get_interface( - cores=1, - threads_per_core=1, - gpus_per_core=0, - cwd=None, - oversubscribe=False, - enable_slurm_backend=False, -): - if not enable_slurm_backend: - return MpiExecInterface( - cwd=cwd, - cores=cores, - threads_per_core=threads_per_core, - gpus_per_core=gpus_per_core, - oversubscribe=oversubscribe, - ) - else: - return SlurmSubprocessInterface( - cwd=cwd, - cores=cores, - threads_per_core=threads_per_core, - gpus_per_core=gpus_per_core, - oversubscribe=oversubscribe, - ) diff --git a/pympipool/shared/interface.py b/pympipool/shared/interface.py index 8885cb36..147ca578 100644 --- a/pympipool/shared/interface.py +++ b/pympipool/shared/interface.py @@ -4,12 +4,10 @@ class BaseInterface(ABC): def __init__( - self, cwd, cores=1, threads_per_core=1, gpus_per_core=0, oversubscribe=False + self, cwd, cores=1, oversubscribe=False ): self._cwd = cwd self._cores = cores - self._threads_per_core = threads_per_core - self._gpus_per_core = gpus_per_core self._oversubscribe = oversubscribe def bootup(self, command_lst): @@ -27,15 +25,11 @@ def __init__( self, cwd=None, cores=1, - threads_per_core=1, - gpus_per_core=0, oversubscribe=False, ): super().__init__( cwd=cwd, cores=cores, - threads_per_core=threads_per_core, - gpus_per_core=gpus_per_core, oversubscribe=oversubscribe, ) self._process = None @@ -63,7 +57,6 @@ class MpiExecInterface(SubprocessInterface): def generate_command(self, command_lst): command_prepend_lst = generate_mpiexec_command( cores=self._cores, - gpus_per_core=self._gpus_per_core, oversubscribe=self._oversubscribe, ) return super().generate_command( @@ -71,7 +64,23 @@ def generate_command(self, command_lst): ) -class SlurmSubprocessInterface(SubprocessInterface): +class SrunInterface(SubprocessInterface): + def __init__( + self, + cwd=None, + cores=1, + threads_per_core=1, + gpus_per_core=0, + oversubscribe=False, + ): + super().__init__( + cwd=cwd, + cores=cores, + oversubscribe=oversubscribe, + ) + self._threads_per_core = threads_per_core + self._gpus_per_core = gpus_per_core + def generate_command(self, command_lst): command_prepend_lst = generate_slurm_command( cores=self._cores, @@ -85,12 +94,10 @@ def generate_command(self, command_lst): ) -def generate_mpiexec_command(cores, gpus_per_core=0, oversubscribe=False): +def generate_mpiexec_command(cores, oversubscribe=False): command_prepend_lst = ["mpiexec", "-n", str(cores)] if oversubscribe: command_prepend_lst += ["--oversubscribe"] - if gpus_per_core > 0: - raise ValueError() return command_prepend_lst diff --git a/pympipool/slurm/__init__.py b/pympipool/slurm/__init__.py new file mode 100644 index 00000000..d9534d99 --- /dev/null +++ b/pympipool/slurm/__init__.py @@ -0,0 +1 @@ +from pympipool.slurm.executor import PySlurmExecutor \ No newline at end of file diff --git a/pympipool/slurm/executor.py b/pympipool/slurm/executor.py new file mode 100644 index 00000000..2adf0264 --- /dev/null +++ b/pympipool/slurm/executor.py @@ -0,0 +1,119 @@ +from pympipool.shared.executorbase import ( + cloudpickle_register, + execute_parallel_tasks, + ExecutorBase, + executor_broker, +) +from pympipool.shared.interface import SrunInterface +from pympipool.shared.thread import RaisingThread + + +class PySlurmExecutor(ExecutorBase): + """ + Args: + max_workers (int): defines the number workers which can execute functions in parallel + cores_per_worker (int): number of MPI cores to be used for each function call + threads_per_core (int): number of OpenMP threads to be used for each function call + gpus_per_worker (int): number of GPUs per worker - defaults to 0 + oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI only) - default False + init_function (None): optional function to preset arguments for functions which are submitted later + cwd (str/None): current working directory where the parallel python task is executed + sleep_interval (float): synchronization interval - default 0.1 + """ + + def __init__( + self, + max_workers, + cores_per_worker=1, + threads_per_core=1, + gpus_per_worker=0, + oversubscribe=False, + init_function=None, + cwd=None, + sleep_interval=0.1, + ): + super().__init__() + self._process = RaisingThread( + target=executor_broker, + kwargs={ + # Broker Arguments + "future_queue": self._future_queue, + "max_workers": max_workers, + "sleep_interval": sleep_interval, + "executor_class": PySlurmSingleTaskExecutor, + # Executor Arguments + "cores": cores_per_worker, + "threads_per_core": threads_per_core, + "gpus_per_task": int(gpus_per_worker / cores_per_worker), + "oversubscribe": oversubscribe, + "init_function": init_function, + "cwd": cwd, + }, + ) + self._process.start() + + +class PySlurmSingleTaskExecutor(ExecutorBase): + """ + The pympipool.Executor behaves like the concurrent.futures.Executor but it uses mpi4py to execute parallel tasks. + In contrast to the mpi4py.futures.MPIPoolExecutor the pympipool.Executor can be executed in a serial python process + and does not require the python script to be executed with MPI. Still internally the pympipool.Executor uses the + mpi4py.futures.MPIPoolExecutor, consequently it is primarily an abstraction of its functionality to improve the + usability in particular when used in combination with Jupyter notebooks. + + Args: + cores (int): defines the number of MPI ranks to use for each function call + threads_per_core (int): number of OpenMP threads to be used for each function call + gpus_per_task (int): number of GPUs per MPI rank - defaults to 0 + oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI only) - default False + init_function (None): optional function to preset arguments for functions which are submitted later + cwd (str/None): current working directory where the parallel python task is executed + + Examples: + ``` + >>> import numpy as np + >>> from pympipool.mpi.executor import PyMPISingleTaskExecutor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with PyMPISingleTaskExecutor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` + """ + + def __init__( + self, + cores=1, + threads_per_core=1, + gpus_per_task=0, + oversubscribe=False, + init_function=None, + cwd=None, + ): + super().__init__() + self._process = RaisingThread( + target=execute_parallel_tasks, + kwargs={ + # Executor Arguments + "future_queue": self._future_queue, + "cores": cores, + "interface_class": SrunInterface, + # Interface Arguments + "threads_per_core": threads_per_core, + "gpus_per_core": gpus_per_task, + "cwd": cwd, + "oversubscribe": oversubscribe, + }, + ) + self._process.start() + self._set_init_function(init_function=init_function) + cloudpickle_register(ind=3) From 8189887381b5b5b51e7c41ec072ec96f03a50ec6 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 31 Oct 2023 11:30:37 +0100 Subject: [PATCH 50/58] update tests --- tests/test_parse.py | 2 +- tests/test_worker.py | 10 +++++----- tests/test_worker_memory.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_parse.py b/tests/test_parse.py index 32123162..a94c048d 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -2,7 +2,7 @@ import sys import unittest from pympipool.shared.backend import parse_arguments -from pympipool.shared.interface import SlurmSubprocessInterface, MpiExecInterface +from pympipool.shared.interface import SrunInterface, MpiExecInterface class TestParser(unittest.TestCase): diff --git a/tests/test_worker.py b/tests/test_worker.py index 5bc26747..386fb16f 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -3,7 +3,7 @@ from queue import Queue from time import sleep from concurrent.futures import CancelledError -from pympipool.mpi.executor import PyMPISingleTaskExecutor, get_interface +from pympipool.mpi.executor import PyMPISingleTaskExecutor, MpiExecInterface from pympipool.shared.executorbase import cloudpickle_register, execute_parallel_tasks from concurrent.futures import Future @@ -107,7 +107,7 @@ def test_execute_task_failed_no_argument(self): future_queue=q, cores=1, oversubscribe=False, - interface_class=get_interface, + interface_class=MpiExecInterface, ) q.join() @@ -121,7 +121,7 @@ def test_execute_task_failed_wrong_argument(self): future_queue=q, cores=1, oversubscribe=False, - interface_class=get_interface, + interface_class=MpiExecInterface, ) q.join() @@ -135,7 +135,7 @@ def test_execute_task(self): future_queue=q, cores=1, oversubscribe=False, - interface_class=get_interface, + interface_class=MpiExecInterface, ) self.assertEqual(f.result(), np.array(4)) q.join() @@ -150,7 +150,7 @@ def test_execute_task_parallel(self): future_queue=q, cores=2, oversubscribe=False, - interface_class=get_interface, + interface_class=MpiExecInterface, ) self.assertEqual(f.result(), [np.array(4), np.array(4)]) q.join() diff --git a/tests/test_worker_memory.py b/tests/test_worker_memory.py index f8aaa7cb..2d4da738 100644 --- a/tests/test_worker_memory.py +++ b/tests/test_worker_memory.py @@ -3,7 +3,7 @@ from queue import Queue from pympipool.shared.backend import call_funct from pympipool.shared.executorbase import cloudpickle_register, execute_parallel_tasks -from pympipool.mpi.executor import PyMPISingleTaskExecutor, get_interface +from pympipool.mpi.executor import PyMPISingleTaskExecutor, MpiExecInterface from concurrent.futures import Future @@ -43,7 +43,7 @@ def test_execute_task(self): future_queue=q, cores=1, oversubscribe=False, - interface_class=get_interface, + interface_class=MpiExecInterface, ) self.assertEqual(f.result(), np.array([5])) q.join() From c6bf57659e40791435b41c9b73558135de6dd177 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 31 Oct 2023 11:31:09 +0100 Subject: [PATCH 51/58] black formatting --- pympipool/shared/interface.py | 4 +--- pympipool/slurm/__init__.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pympipool/shared/interface.py b/pympipool/shared/interface.py index 147ca578..54818c79 100644 --- a/pympipool/shared/interface.py +++ b/pympipool/shared/interface.py @@ -3,9 +3,7 @@ class BaseInterface(ABC): - def __init__( - self, cwd, cores=1, oversubscribe=False - ): + def __init__(self, cwd, cores=1, oversubscribe=False): self._cwd = cwd self._cores = cores self._oversubscribe = oversubscribe diff --git a/pympipool/slurm/__init__.py b/pympipool/slurm/__init__.py index d9534d99..ed4aeab3 100644 --- a/pympipool/slurm/__init__.py +++ b/pympipool/slurm/__init__.py @@ -1 +1 @@ -from pympipool.slurm.executor import PySlurmExecutor \ No newline at end of file +from pympipool.slurm.executor import PySlurmExecutor From cf737272571a3e37eb241ed0360cf5f79ced9aa1 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 31 Oct 2023 11:34:13 +0100 Subject: [PATCH 52/58] Fix import --- pympipool/shared/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pympipool/shared/__init__.py b/pympipool/shared/__init__.py index 73fadf43..2daf2f44 100644 --- a/pympipool/shared/__init__.py +++ b/pympipool/shared/__init__.py @@ -8,4 +8,4 @@ ) from pympipool.shared.executorbase import cancel_items_in_queue from pympipool.shared.thread import RaisingThread -from pympipool.shared.interface import MpiExecInterface, SlurmSubprocessInterface +from pympipool.shared.interface import MpiExecInterface, SrunInterface From faa6643e92edba3f990d755710adead07a85885e Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 31 Oct 2023 11:36:56 +0100 Subject: [PATCH 53/58] no more GPU support for MPI --- tests/test_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_interface.py b/tests/test_interface.py index 149e5f03..b8cac32e 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -18,7 +18,7 @@ def test_interface(self): task_dict = {"fn": calc, "args": (), "kwargs": {"i": 2}} interface = SocketInterface( interface=MpiExecInterface( - cwd=None, cores=1, gpus_per_core=0, oversubscribe=False + cwd=None, cores=1, oversubscribe=False ) ) interface.bootup( From cf7f52250456abf685b2eb1854207a4e931a054d Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 31 Oct 2023 11:40:17 +0100 Subject: [PATCH 54/58] Error type changes --- tests/test_meta.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_meta.py b/tests/test_meta.py index 643883c8..1bba9a07 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -110,7 +110,7 @@ def test_meta_executor_parallel(self): self.assertTrue(fs_1.done()) def test_errors(self): - with self.assertRaises(ValueError): + with self.assertRaises(TypeError): PyMPIExecutor(max_workers=1, cores_per_worker=1, threads_per_core=2) - with self.assertRaises(ValueError): + with self.assertRaises(TypeError): PyMPIExecutor(max_workers=1, cores_per_worker=1, gpus_per_worker=1) From 29ae9103d264a573dc3d3c3e4177c18e5c35a1e2 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 31 Oct 2023 11:44:11 +0100 Subject: [PATCH 55/58] more fixes --- tests/test_parse.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tests/test_parse.py b/tests/test_parse.py index a94c048d..eaa953d1 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -22,7 +22,7 @@ def test_command_local(self): result_dict["zmqport"], ] interface = MpiExecInterface( - cwd=None, cores=2, gpus_per_core=0, oversubscribe=True + cwd=None, cores=2, oversubscribe=True ) self.assertEqual( command_lst, @@ -32,13 +32,6 @@ def test_command_local(self): ) self.assertEqual(result_dict, parse_arguments(command_lst)) - def test_mpiexec_gpu(self): - interface = MpiExecInterface( - cwd=os.path.abspath("."), cores=2, gpus_per_core=1, oversubscribe=True - ) - with self.assertRaises(ValueError): - interface.bootup(command_lst=[]) - def test_command_slurm(self): result_dict = { "host": "127.0.0.1", @@ -59,7 +52,7 @@ def test_command_slurm(self): "--zmqport", result_dict["zmqport"], ] - interface = SlurmSubprocessInterface( + interface = SrunInterface( cwd=os.path.abspath("."), cores=2, gpus_per_core=1, oversubscribe=True ) self.assertEqual( From 95206700a4351a4396b4937eebd71eff0919fe48 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 31 Oct 2023 11:58:05 +0100 Subject: [PATCH 56/58] fixes for flux --- pympipool/flux/executor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pympipool/flux/executor.py b/pympipool/flux/executor.py index 4c8930da..95425118 100644 --- a/pympipool/flux/executor.py +++ b/pympipool/flux/executor.py @@ -136,10 +136,10 @@ def __init__( super().__init__( cwd=cwd, cores=cores, - gpus_per_core=gpus_per_core, - threads_per_core=threads_per_core, oversubscribe=oversubscribe, ) + self._threads_per_core = threads_per_core + self._gpus_per_core = gpus_per_core self._executor = executor self._future = None From eba13b5701069f30547cbc3556a9e081e86c2244 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 31 Oct 2023 12:17:03 +0100 Subject: [PATCH 57/58] Fix interface shutdown --- pympipool/shared/executorbase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pympipool/shared/executorbase.py b/pympipool/shared/executorbase.py index c950eae1..cf1a27c5 100644 --- a/pympipool/shared/executorbase.py +++ b/pympipool/shared/executorbase.py @@ -53,6 +53,7 @@ def shutdown(self, wait=True, *, cancel_futures=False): if cancel_futures: cancel_items_in_queue(que=self._future_queue) self._future_queue.put({"shutdown": True, "wait": wait}) + self._future_queue.join() self._process.join() def __len__(self): From 7d7bd99d7113e740adffad5e7b08b18edd05e893 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 31 Oct 2023 12:26:38 +0100 Subject: [PATCH 58/58] Fix order --- pympipool/shared/executorbase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pympipool/shared/executorbase.py b/pympipool/shared/executorbase.py index cf1a27c5..232367fa 100644 --- a/pympipool/shared/executorbase.py +++ b/pympipool/shared/executorbase.py @@ -53,8 +53,8 @@ def shutdown(self, wait=True, *, cancel_futures=False): if cancel_futures: cancel_items_in_queue(que=self._future_queue) self._future_queue.put({"shutdown": True, "wait": wait}) - self._future_queue.join() self._process.join() + self._future_queue.join() def __len__(self): return self._future_queue.qsize()