From 60bf0abe2da68aea22016b079f1403b286dd7832 Mon Sep 17 00:00:00 2001 From: Roberto Di Remigio Date: Thu, 11 Aug 2022 17:40:15 +0200 Subject: [PATCH 1/3] Format flake.nix --- flake.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index d150de1a..c41909b5 100644 --- a/flake.nix +++ b/flake.nix @@ -19,9 +19,9 @@ pkgs = nixpkgs.legacyPackages.${system}; pythonEnv = mach-nix.lib."${system}".mkPython { requirements = builtins.readFile ./requirements.txt + '' - # additional dependencies for local work - #jupyterlab - pre-commit + # additional dependencies for local work + #jupyterlab + pre-commit ''; }; in From 67ad0031b5842eee6c60e1b31146a0ca522ae017 Mon Sep 17 00:00:00 2001 From: Roberto Di Remigio Date: Thu, 11 Aug 2022 17:59:11 +0200 Subject: [PATCH 2/3] Add mpi4py as dependency --- flake.lock | 20 ++++++++++---------- flake.nix | 6 +++--- pyproject.toml | 2 +- requirements.txt | 1 + setup.cfg | 4 ++-- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/flake.lock b/flake.lock index 702ac8ce..eea09d13 100644 --- a/flake.lock +++ b/flake.lock @@ -39,16 +39,16 @@ ] }, "locked": { - "lastModified": 1643953409, - "narHash": "sha256-CJDg/RpZdUVyI3QIAXUqIoYDl7VkxFtNE4JWih0ucKc=", + "lastModified": 1654084003, + "narHash": "sha256-j/XrVVistvM+Ua+0tNFvO5z83isL+LBgmBi9XppxuKA=", "owner": "DavHau", "repo": "mach-nix", - "rev": "fe5255e6fd8df57e9507b7af82fc59dda9e9ff2b", + "rev": "7e14360bde07dcae32e5e24f366c83272f52923f", "type": "github" }, "original": { "id": "mach-nix", - "ref": "3.4.0", + "ref": "3.5.0", "type": "indirect" } }, @@ -69,11 +69,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1659981942, - "narHash": "sha256-uCFiP/B/NXOWzhN6TKfMbSxtVMk1bVnCrnJRjCF6RmU=", + "lastModified": 1660162369, + "narHash": "sha256-pZukMP4zCA1FaBg0xHxf7KdE/Nv/C5YbDID7h2L8O7A=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "39d7f929fbcb1446ad7aa7441b04fb30625a4190", + "rev": "3a11db5f408095b8f08b098ec2066947f4b72ce2", "type": "github" }, "original": { @@ -86,11 +86,11 @@ "pypi-deps-db": { "flake": false, "locked": { - "lastModified": 1660122052, - "narHash": "sha256-CSjuNhW3X69BCWb8NGh3i3NugTSvCAlUu0yNN2fw5qk=", + "lastModified": 1660206848, + "narHash": "sha256-yJmDGjRHolfqz/y7iSvSzhM+EdZY/iJINjy9BT/72pA=", "owner": "DavHau", "repo": "pypi-deps-db", - "rev": "97944878d94a0f7515884ca1f03d4337acad0324", + "rev": "fe1bc73f6e12878796267ac1fb0046cb765778e5", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index c41909b5..9815e6e8 100644 --- a/flake.nix +++ b/flake.nix @@ -8,7 +8,7 @@ flake = false; }; mach-nix = { - url = "mach-nix/3.4.0"; + url = "mach-nix/3.5.0"; inputs.pypi-deps-db.follows = "pypi-deps-db"; }; }; @@ -19,8 +19,8 @@ pkgs = nixpkgs.legacyPackages.${system}; pythonEnv = mach-nix.lib."${system}".mkPython { requirements = builtins.readFile ./requirements.txt + '' - # additional dependencies for local work - #jupyterlab + # additional dependencies for local work + #jupyterlab pre-commit ''; }; diff --git a/pyproject.toml b/pyproject.toml index a772a6eb..e75a5315 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ norecursedirs = [ [build-system] requires = [ + "mpi4py>=3.0", "pybind11[global]>=2.6", "setuptools>=46.4.0", "wheel", @@ -24,5 +25,4 @@ requires = [ "cmake>=3.17", "ninja; platform_system!='Windows'", ] - build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index f835b807..ec2dfa9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ numpy +mpi4py>=3.0,<3.1.0 pytest pybind11[global] diff --git a/setup.cfg b/setup.cfg index ce97c2b1..02d11954 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,8 +32,7 @@ setup_requires = ninja; platform_system!='Windows' install_requires = - #mpi4py >= 3.0 - numpy >= 1.15.0 + mpi4py >= 3.0 test_suite = tests @@ -44,6 +43,7 @@ docs = sphinx_rtd_theme test = + numpy >= 1.15.0 pytest pytest-cov From c5945d1ecd9c9351c675167f4207f571a9ce65ad Mon Sep 17 00:00:00 2001 From: Roberto Di Remigio Date: Tue, 16 Aug 2022 14:17:42 +0200 Subject: [PATCH 3/3] Try with MPI --- CMakeLists.txt | 2 + cmake/custom/FindPythonModule.cmake | 92 +++++++++++++++++++++++++++++ cmake/custom/fetch_mrcpp.cmake | 16 ++++- cmake/custom/mpi.cmake | 64 ++++++++++++++++++++ flake.lock | 12 ++-- flake.nix | 7 +-- src/vampyr/CMakeLists.txt | 3 + src/vampyr/export_vampyr.cpp | 70 ++++++++++++++++++++++ 8 files changed, 254 insertions(+), 12 deletions(-) create mode 100644 cmake/custom/FindPythonModule.cmake create mode 100644 cmake/custom/mpi.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 777e9857..e04a3921 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,6 +54,8 @@ include(${PROJECT_SOURCE_DIR}/cmake/downloaded/autocmake_safeguards.cmake) include(${PROJECT_SOURCE_DIR}/cmake/downloaded/autocmake_code_coverage.cmake) find_package(Python 3.7 REQUIRED COMPONENTS Interpreter Development) +# find MPI and mpi4py +include(${PROJECT_SOURCE_DIR}/cmake/custom/mpi.cmake) # download pybind11 if not available on the system include(${PROJECT_SOURCE_DIR}/cmake/custom/fetch_pybind11.cmake) # download MRCPP if not available on the system diff --git a/cmake/custom/FindPythonModule.cmake b/cmake/custom/FindPythonModule.cmake new file mode 100644 index 00000000..da3d9412 --- /dev/null +++ b/cmake/custom/FindPythonModule.cmake @@ -0,0 +1,92 @@ +# Downloaded from +# http://www.cmake.org/pipermail/cmake/2011-January/041666.html +# * Added FORCE to location var +# * Function to macro so module_FOUND shows up +# * Remove ``if(NOT PY_${module})`` so runs each time, also remove module caps +# * Module name, not path in fphsa +# * Added version handling via parse_version call + +#.rst: +# +# Find if a Python module is installed. +# Usage: find_python_module( [[ATLEAST | EXACT] version] [QUIET] [REQUIRED]) + +macro(find_python_module module) + cmake_parse_arguments(ARG + "QUIET;REQUIRED" # options + "ATLEAST;EXACT" # one-value arguments + "" # multi-value arguments + ${ARGN} # everything else + ) + + if(ARG_QUIET) + set(${module}_FIND_QUIETLY TRUE) + endif() + + if(ARG_REQUIRED) + set(${module}_FIND_REQUIRED TRUE) + endif() + + if(ARG_ATLEAST AND ARG_EXACT) + message(FATAL_ERROR "Can't be both ATLEAST and EXACT") + endif() + if(ARG_ATLEAST) + set(_op ">=") + set(${module}_tgtver ${ARG_ATLEAST}) + elseif(ARG_EXACT) + set(_op "==") + set(${module}_tgtver ${ARG_EXACT}) + else() + # deceive handle_standard_arguments into not caring about version + set(_${module}_requested_version_found "${Python_EXECUTABLE}") + endif() + + unset(PY_${module} CACHE) + unset(${module}_VERSION CACHE) + + execute_process( + COMMAND "${Python_EXECUTABLE}" "-c" + "import re; \ + import ${module}; \ + print(re.compile('/__init__.py.*').sub('', ${module}.__file__))" + RESULT_VARIABLE _${module}_status + OUTPUT_VARIABLE _${module}_location + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(NOT ${_${module}_status}) + set(PY_${module} ${_${module}_location} CACHE STRING "Location of Python module ${module}" FORCE) + execute_process( + COMMAND "${Python_EXECUTABLE}" "-c" + "import sys; \ + import ${module}; \ + print(${module}.__version__)" + RESULT_VARIABLE _${module}_ver_status + OUTPUT_VARIABLE _${module}_version + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(NOT ${_${module}_ver_status}) + set(${module}_VERSION ${_${module}_version} CACHE STRING + "Version of Python module ${module}" FORCE) + + if(${module}_tgtver) + execute_process( + COMMAND "${Python_EXECUTABLE}" "-c" + "from pkg_resources import parse_version; \ + print(parse_version('${${module}_VERSION}') ${_op} parse_version('${${module}_tgtver}'))" + RESULT_VARIABLE _${module}_verenuf_status + OUTPUT_VARIABLE _${module}_verenuf + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(NOT ${_${module}_verenuf_status}) + if(${_${module}_verenuf} STREQUAL "True") + set(_${module}_requested_version_found "${Python_EXECUTABLE}") + endif() + endif() + endif() + endif() + endif() + find_package_handle_standard_args(${module} DEFAULT_MSG PY_${module} _${module}_requested_version_found) +endmacro() diff --git a/cmake/custom/fetch_mrcpp.cmake b/cmake/custom/fetch_mrcpp.cmake index 02100299..1b08c8b0 100644 --- a/cmake/custom/fetch_mrcpp.cmake +++ b/cmake/custom/fetch_mrcpp.cmake @@ -9,7 +9,19 @@ set(MRCPP_FETCHED FALSE) if(TARGET MRCPP::mrcpp) get_property(_loc TARGET MRCPP::mrcpp PROPERTY LOCATION) - message(STATUS "Found MRCPP: ${_loc} (found version ${MRCPP_VERSION})") + + get_target_property(MRCPP_HAS_OMP MRCPP::mrcpp MRCPP_HAS_OMP) + get_target_property(MRCPP_HAS_MPI MRCPP::mrcpp MRCPP_HAS_MPI) + + message(STATUS "Found MRCPP (OpenMP: ${MRCPP_HAS_OMP}; MPI: ${MRCPP_HAS_MPI}): ${_loc} (found version ${MRCPP_VERSION})") + + if(NOT MRCPP_HAS_OMP) + message(FATAL_ERROR "VAMPyR needs MRCPP with OpenMP features enabled!") + endif() + + if(NOT MRCPP_HAS_MPI AND VAMPYR_WITH_MPI) + message(FATAL_ERROR "You enabled MPI for VAMPyR, but MRCPP does not have MPI features enabled!") + endif() else() message(STATUS "Suitable MRCPP could not be located. Fetching and building!") include(FetchContent) @@ -24,7 +36,7 @@ else() set(CMAKE_CXX_COMPILER ${CMAKE_CXX_COMPILER}) # Always build with OpenMP and without MPI set(ENABLE_OPENMP TRUE CACHE BOOL "") - set(ENABLE_MPI FALSE CACHE BOOL "") + set(ENABLE_MPI ${VAMPYR_WITH_MPI} CACHE BOOL "") set(ENABLE_TESTS FALSE CACHE BOOL "") set(ENABLE_EXAMPLES FALSE CACHE BOOL "") diff --git a/cmake/custom/mpi.cmake b/cmake/custom/mpi.cmake new file mode 100644 index 00000000..505e1df7 --- /dev/null +++ b/cmake/custom/mpi.cmake @@ -0,0 +1,64 @@ +#.rst: +# +# Enables MPI support. +# This was adapted from Autocmake +# +# Variables used:: +# +# VAMPYR_WITH_MPI +# +# autocmake.yml configuration:: +# +# docopt: "--mpi Enable MPI parallelization [default: False]." +# define: "'-DVAMPYR_WITH_MPI={0}'.format(arguments['--mpi'])" + +option(VAMPYR_WITH_MPI "Enable MPI parallelization" OFF) + +if(VAMPYR_WITH_MPI) + find_package(MPI REQUIRED COMPONENTS CXX) + + if(NOT TARGET MPI::MPI_CXX) + message(FATAL_ERROR "MPI-enabled VAMPyR requires a working MPI installation.") + endif() + + # find mpi4py + include(${PROJECT_SOURCE_DIR}/cmake/custom/FindPythonModule.cmake) + + find_python_module(mpi4py REQUIRED) + + # we also need the include directories for mpi4py + if(mpi4py_FOUND) + execute_process( + COMMAND + "${Python_EXECUTABLE}" "-c" + "import mpi4py as m; print(m.__version__); print(m.get_include());" + RESULT_VARIABLE + _mpi4py_SEARCH_SUCCESS + OUTPUT_VARIABLE + _mpi4py_VALUES + ERROR_VARIABLE + _mpi4py_ERROR_VALUE + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + # Convert the process output into a list + string(REGEX REPLACE ";" "\\\\;" _mpi4py_VALUES ${_mpi4py_VALUES}) + string(REGEX REPLACE "\n" ";" _mpi4py_VALUES ${_mpi4py_VALUES}) + list(GET _mpi4py_VALUES 0 mpi4py_VERSION) + list(GET _mpi4py_VALUES 1 mpi4py_INCLUDE_DIRS) + + # Make sure all directory separators are '/' + string(REGEX REPLACE "\\\\" "/" mpi4py_INCLUDE_DIRS ${mpi4py_INCLUDE_DIRS}) + + # Get the major and minor version numbers + string(REGEX REPLACE "\\." ";" _mpi4py_VERSION_LIST ${mpi4py_VERSION}) + list(GET _mpi4py_VERSION_LIST 0 mpi4py_VERSION_MAJOR) + list(GET _mpi4py_VERSION_LIST 1 mpi4py_VERSION_MINOR) + list(GET _mpi4py_VERSION_LIST 2 mpi4py_VERSION_PATCH) + string(REGEX MATCH "[0-9]*" mpi4py_VERSION_PATCH ${mpi4py_VERSION_PATCH}) + math(EXPR mpi4py_VERSION_DECIMAL + "(${mpi4py_VERSION_MAJOR} * 10000) + (${mpi4py_VERSION_MINOR} * 100) + ${mpi4py_VERSION_PATCH}") + else() + message(FATAL_ERROR "MPI-enabled VAMPyR requires a working mpi4py installation.") + endif() +endif() diff --git a/flake.lock b/flake.lock index eea09d13..b5cc789e 100644 --- a/flake.lock +++ b/flake.lock @@ -69,11 +69,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1660162369, - "narHash": "sha256-pZukMP4zCA1FaBg0xHxf7KdE/Nv/C5YbDID7h2L8O7A=", + "lastModified": 1660485612, + "narHash": "sha256-sSLW1KaB1adKTJn9+Ja3h3AaS7QCZyhUKiSUStcLg80=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3a11db5f408095b8f08b098ec2066947f4b72ce2", + "rev": "6512b21eabb4d52e87ea2edcf31a288e67b2e4f8", "type": "github" }, "original": { @@ -86,11 +86,11 @@ "pypi-deps-db": { "flake": false, "locked": { - "lastModified": 1660206848, - "narHash": "sha256-yJmDGjRHolfqz/y7iSvSzhM+EdZY/iJINjy9BT/72pA=", + "lastModified": 1660639261, + "narHash": "sha256-m+13X54HOwUUpeahGUe+xywerPaU0LwtBGBy9ahfVLs=", "owner": "DavHau", "repo": "pypi-deps-db", - "rev": "fe1bc73f6e12878796267ac1fb0046cb765778e5", + "rev": "f014c49221a49dae6f4c5f72b51b8b8486dbf326", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 9815e6e8..22bbeb46 100644 --- a/flake.nix +++ b/flake.nix @@ -27,16 +27,15 @@ in { devShell = pkgs.mkShell { - nativeBuildInputs = with pkgs; [ + buildInputs = with pkgs; [ cmake eigen gcc - openmpi ninja - ]; - buildInputs = [ + openmpi pythonEnv ]; + hardeningDisable = [ "all" ]; NINJA_STATUS = "[Built edge %f of %t in %e sec] "; }; }); diff --git a/src/vampyr/CMakeLists.txt b/src/vampyr/CMakeLists.txt index 3977443f..ccbe8f91 100644 --- a/src/vampyr/CMakeLists.txt +++ b/src/vampyr/CMakeLists.txt @@ -11,11 +11,14 @@ target_compile_definitions(_vampyr target_include_directories(_vampyr PUBLIC ${CMAKE_CURRENT_LIST_DIR} + SYSTEM PRIVATE + ${mpi4py_INCLUDE_DIRS} ) target_link_libraries(_vampyr PUBLIC MRCPP::mrcpp + $:MPI::MPI_CXX>> ) # handle RPATH diff --git a/src/vampyr/export_vampyr.cpp b/src/vampyr/export_vampyr.cpp index 96b391c7..3b1e0a33 100644 --- a/src/vampyr/export_vampyr.cpp +++ b/src/vampyr/export_vampyr.cpp @@ -5,6 +5,12 @@ * UiT - The Arctic University of Norway */ +#include +// see here: https://github.com/mpi4py/mpi4py/issues/19#issuecomment-768143143 +#ifdef MSMPI_VER +#define PyMPI_HAVE_MPI_Message 1 +#endif +#include #include #include @@ -28,6 +34,45 @@ namespace py = pybind11; using namespace mrcpp; using namespace pybind11::literals; +struct mpi4py_comm { + mpi4py_comm() = default; + mpi4py_comm(MPI_Comm value) + : value(value) {} + operator MPI_Comm() { return value; } + + MPI_Comm value; +}; + +namespace pybind11 { +namespace detail { +template <> struct type_caster { +public: + PYBIND11_TYPE_CASTER(mpi4py_comm, _("mpi4py_comm")); + + // Python -> C++ + bool load(handle src, bool) { + PyObject *py_src = src.ptr(); + + // Check that we have been passed an mpi4py communicator + if (PyObject_TypeCheck(py_src, &PyMPIComm_Type)) { + // Convert to regular MPI communicator + value.value = *PyMPIComm_Get(py_src); + } else { + return false; + } + + return !PyErr_Occurred(); + } + + // C++ -> Python + static handle cast(mpi4py_comm src, return_value_policy /* policy */, handle /* parent */) { + // Create an mpi4py handle + return PyMPIComm_New(src.value); + } +}; +} // namespace detail +} // namespace pybind11 + namespace vampyr { void constants(py::module &m) { @@ -69,7 +114,32 @@ template void bind_vampyr(py::module &mod) noexcept { bind_advanced(sub_mod); } +// recieve a communicator and check if it equals MPI_COMM_WORLD +void print_comm(mpi4py_comm comm) { + if (comm == MPI_COMM_WORLD) { + std::cout << "Received the world." << std::endl; + } else { + std::cout << "Received something else." << std::endl; + } +} + +mpi4py_comm get_comm() { + return MPI_COMM_WORLD; // Just return MPI_COMM_WORLD for demonstration +} + PYBIND11_MODULE(_vampyr, m) { + + // initialize mpi4py's C-API + if (import_mpi4py() < 0) { + // mpi4py calls the Python C API + // we let pybind11 give us the detailed traceback + throw py::error_already_set(); + } + + // register the test functions + m.def("print_comm", &print_comm, "Do something with the mpi4py communicator."); + m.def("get_comm", &get_comm, "Return some communicator."); + m.doc() = R"pbdoc( VAMPyR ------