Skip to content

Commit

Permalink
Merge pull request #1 from cwpearson/feature/packers
Browse files Browse the repository at this point in the history
More PR tests, non-contiguous mdspan support, documentation
  • Loading branch information
cwpearson authored Mar 13, 2024
2 parents 4a63b78 + a927558 commit c8fc55e
Show file tree
Hide file tree
Showing 44 changed files with 890 additions and 319 deletions.
18 changes: 18 additions & 0 deletions .github/workflows/docs-build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
name: Docs build
run-name: Docs build
on: [push]


jobs:
# Build job
build:
runs-on: ubuntu-latest
env:
MDBOOK_VERSION: 0.4.36
steps:
- uses: actions/checkout@v4
- name: Install sphinx
run: |
pip install -U -r docs/requirements.txt
- name: Build with sphinx
run: make -C docs html
File renamed without changes.
16 changes: 16 additions & 0 deletions .github/workflows/formatting.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
name: clang-format check
run-name: clang-format check

on: [push, pull_request]

permissions: read-all

jobs:
formatting-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Run clang-format style check.
uses: DoozyX/[email protected]
with:
clangFormatVersion: 8
36 changes: 36 additions & 0 deletions .github/workflows/linux-mdspan.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
name: Linux (mdspan)
run-name: Linux (mdspan)
on: [push]
jobs:
Kokkos-040200-OpenMPI-OpenMP-kokkosmdspan-Release:
env:
KOKKOS_SRC: ${{ github.workspace }}/_deps/kokkos
KOKKOS_BUILD: ${{ github.workspace }}/_deps/kokkos-build
KOKKOS_INSTALL: ${{ github.workspace }}/_deps/kokkos-install
COMM_SRC: ${{ github.workspace }}
COMM_BUILD: ${{ github.workspace }}/build
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Install MPI
run: |
sudo apt-get update && sudo apt-get install -y libopenmpi-dev openmpi-bin cmake
which mpirun
mpirun --version
which mpicxx
mpicxx --version
- name: Check out repository code
uses: actions/checkout@v4
- name: Build Kokkos
run: |
git clone https://github.com/kokkos/kokkos.git "$KOKKOS_SRC"
cd "$KOKKOS_SRC" && git checkout 4.2.00
cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" -DCMAKE_INSTALL_PREFIX="$KOKKOS_INSTALL" -DCMAKE_BUILD_TYPE=Release -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=ON -DKokkos_ARCH_NATIVE=ON -DCMAKE_CXX_COMPILER=mpicxx
cmake --build "$KOKKOS_BUILD" --parallel $(nproc) -t install
- name: Build KokkosComm
run: |
cmake -S "$COMM_SRC" -B "$COMM_BUILD" -DKokkos_DIR="$KOKKOS_INSTALL/lib/cmake/Kokkos" -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-Wall -Wextra -Wshadow -Wpedantic" -DKokkosComm_ENABLE_MDSPAN=ON -DKokkosComm_USE_KOKKOS_MDSPAN=ON
VERBOSE=1 cmake --build "$COMM_BUILD"
- name: Test KokkosComm
run: |
ctest -V --test-dir "$COMM_BUILD"
36 changes: 36 additions & 0 deletions .github/workflows/osx-mdspan.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
name: macOS (mdspan)
run-name: macOS (mdspan)
on: [push]
jobs:
Kokkos-040200-MPICH-Threads-kokkosmdspan-Release:
env:
KOKKOS_SRC: ${{ github.workspace }}/_deps/kokkos
KOKKOS_BUILD: ${{ github.workspace }}/_deps/kokkos-build
KOKKOS_INSTALL: ${{ github.workspace }}/_deps/kokkos-install
COMM_SRC: ${{ github.workspace }}
COMM_BUILD: ${{ github.workspace }}/build
runs-on: macos-latest
timeout-minutes: 5
steps:
- name: Install MPI
run: |
brew install mpich cmake
which mpirun
mpirun --version
which mpicxx
mpicxx --version
- name: Check out repository code
uses: actions/checkout@v4
- name: Build Kokkos
run: |
git clone https://github.com/kokkos/kokkos.git "$KOKKOS_SRC"
cd "$KOKKOS_SRC" && git checkout 4.2.00
cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" -DCMAKE_INSTALL_PREFIX="$KOKKOS_INSTALL" -DCMAKE_BUILD_TYPE=Release -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_THREADS=ON -DKokkos_ARCH_NATIVE=ON -DCMAKE_CXX_COMPILER=mpicxx
cmake --build "$KOKKOS_BUILD" --parallel $(nproc) -t install
- name: Build KokkosComm
run: |
cmake -S "$COMM_SRC" -B "$COMM_BUILD" -DKokkos_DIR="$KOKKOS_INSTALL/lib/cmake/Kokkos" -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-Wall -Wextra -Wshadow -Wpedantic" -DKokkosComm_ENABLE_MDSPAN=ON -DKokkosComm_USE_KOKKOS_MDSPAN=ON
VERBOSE=1 cmake --build "$COMM_BUILD"
- name: Test KokkosComm
run: |
ctest -V --test-dir "$COMM_BUILD"
2 changes: 1 addition & 1 deletion .github/workflows/osx.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ jobs:
steps:
- name: Install MPI
run: |
brew install open-mpi cmake
brew install mpich cmake
which mpirun
mpirun --version
which mpicxx
Expand Down
41 changes: 27 additions & 14 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,36 +5,49 @@ cmake_minimum_required(VERSION 3.12)
project(KokkosComm VERSION 0.0.2)

option(KokkosComm_ENABLE_PERFTESTS "Build KokkosComm perf tests" ON)
set(KOKKOSCOMM_ENABLE_PERFTESTS ${KokkosComm_ENABLE_PERFTESTS} CACHE BOOL "" FORCE)
option(KokkosComm_ENABLE_TESTS "Build KokkosComm perf tests" ON)
option(KokkosComm_ENABLE_MDSPAN "Build mdspan overloads" OFF)
option(KokkosComm_MDSPAN_IMPL "mdspan implementation to use [std, kokkos]" "kokkos")
option(KokkosComm_USE_STD_MDSPAN "Use std::mdspan (requires c++23)" OFF)
option(KokkosComm_USE_KOKKOS_MDSPAN "Use Kokkos mdspan implementation" OFF)


## resolve options
set(KOKKOSCOMM_ENABLE_PERFTESTS ${KokkosComm_ENABLE_PERFTESTS} CACHE BOOL "" FORCE)
set(KOKKOSCOMM_ENABLE_TESTS ${KokkosComm_ENABLE_TESTS} CACHE BOOL "" FORCE)
option(KokkosComm_ENABLE_MDSPAN "Build std::mdspan overloads (requires c++23)" OFF)
set(KOKKOSCOMM_ENABLE_MDSPAN ${KokkosComm_ENABLE_MDSPAN} CACHE BOOL "" FORCE)
option(KokkosComm_USE_KOKKOS_MDSPAN "Use Kokkos mdspan implementation" OFF)
set(KOKKOSCOMM_USE_KOKKOS_MDSPAN ${KokkosComm_USE_KOKKOS_MDSPAN} CACHE BOOL "" FORCE)
set(KOKKOSCOMM_USE_STD_MDSPAN ${KokkosComm_USE_STD_MDSPAN} CACHE BOOL "" FORCE)

find_package(Kokkos REQUIRED)
find_package(MPI REQUIRED)

message(STATUS "MPI_VERSION = ${MPI_VERSION}")
message(STATUS "MPI_CXX_COMPILER = ${MPI_CXX_COMPILER}")
message(STATUS "MPI_CXX_COMPILE_OPTIONS = ${MPI_CXX_COMPILE_OPTIONS}")
message(STATUS "MPI_CXX_COMPILE_DEFINITIONS = ${MPI_CXX_COMPILE_DEFINITIONS}")
message(STATUS "MPI_CXX_INCLUDE_DIRS = ${MPI_CXX_INCLUDE_DIRS}")
message(STATUS "MPI_CXX_LINK_FLAGS = ${MPI_CXX_LINK_FLAGS}")
message(STATUS "MPI_CXX_LIBRARIES = ${MPI_CXX_LIBRARIES}")
message(STATUS "Kokkos MPI: MPI_VERSION = ${MPI_VERSION}")
message(STATUS "Kokkos MPI: MPI_CXX_COMPILER = ${MPI_CXX_COMPILER}")
message(STATUS "Kokkos MPI: MPI_CXX_COMPILE_OPTIONS = ${MPI_CXX_COMPILE_OPTIONS}")
message(STATUS "Kokkos MPI: MPI_CXX_COMPILE_DEFINITIONS = ${MPI_CXX_COMPILE_DEFINITIONS}")
message(STATUS "Kokkos MPI: MPI_CXX_INCLUDE_DIRS = ${MPI_CXX_INCLUDE_DIRS}")
message(STATUS "Kokkos MPI: MPI_CXX_LINK_FLAGS = ${MPI_CXX_LINK_FLAGS}")
message(STATUS "Kokkos MPI: MPI_CXX_LIBRARIES = ${MPI_CXX_LIBRARIES}")
if (KOKKOSCOMM_ENABLE_MDSPAN)
if (KOKKOSCOMM_USE_STD_MDSPAN)
message(STATUS "Kokkos MPI: mdspan support enabled (std::mdspan)")
elseif(KOKKOSCOMM_USE_KOKKOS_MDSPAN)
message(STATUS "Kokkos MPI: mdspan support enabled (kokkos/mdspan)")
else()
message(FATAL_ERROR "Kokkos MPI: mdspan support enabled but no MDSPAN implementation enabled!")
endif()
endif()

if(KOKKOSCOMM_USE_KOKKOS_MDSPAN)
include(FetchContent)
include(FetchContent)
if(KOKKOSCOMM_ENABLE_MDSPAN AND KOKKOSCOMM_USE_KOKKOS_MDSPAN)
message(STATUS "retrieving kokkos/mdspan...")
FetchContent_Declare(
kokkos_mdspan
GIT_REPOSITORY https://github.com/kokkos/mdspan.git
GIT_TAG 9ceface91483775a6c74d06ebf717bbb2768452f # mdspan-0.6.0
)
FetchContent_MakeAvailable(kokkos_mdspan)
FetchContent_GetProperties(kokkos_mdspan)
# add_subdirectory(${kokkos_mdspan_SOURCE_DIR}) # ${kokkos_mdspan_BINARY_DIR}
endif()

include(cmake/flags.cmake)
Expand Down
13 changes: 5 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@
## Getting Started

* **Requires c++20**
* **Requires c++23 for std::mdspan**
[cwpearson.github.io/kokkos-mpi/](https://cwpearson.github.io/kokkos-mpi/)

### macOS

Expand All @@ -31,9 +30,6 @@ ctest

## Documentation

[cwpearson.github.io/kokkos-mpi/](https://cwpearson.github.io/kokkos-mpi/)

https://www.sphinx-doc.org/en/master/usage/domains/cpp.html


## Design
Expand All @@ -49,9 +45,10 @@ https://www.sphinx-doc.org/en/master/usage/domains/cpp.html
- [ ] MPI Communicator wrapper

- [ ] Packing
- [ ] first pass could be a MpiDatatypePacker which just constructs an MPI Datatype matching the mdspan and hands it off to MPI to deal with the non-contiguous data
- [ ] second pass would be to somehow associate a Kokkos memory space with the `mdspan` so we know how to allocate intermediate packing buffers
- [x] use `Kokkos::deep_copy` to handle packing and unpacking of non-contiguous `Kokkos::View`
- [x] Tentative `Packer` interface.
- [x] Packer::MpiDatatype which just constructs an MPI Datatype matching the mdspan and hands it off to MPI to deal with the non-contiguous data
- [x] Packer::DeepCopy uses `Kokkos::deep_copy` to handle packing and unpacking of non-contiguous `Kokkos::View`
- [ ] second pass would be to somehow associate a Kokkos memory space with the `mdspan` so we know how to allocate intermediate packing buffers
- When non-contiguous views are passed to an MPI function, a temporary contiguous view of matching extent is allocated, and `Kokkos::deep_copy` is used to pack the data.
- [x] "Immediate" functions (e.g. `isend`) return a `KokkosComm::Req`, which can be `wait()`-ed to block until the input view can be reused. `Req` also manages the lifetimes of any intermediate views needed for packing the data, releasing those views when `wait()` is complete.
- [x] `KokkosComm::Traits` is specialized for `Kokkos::View` and `mdspan`
Expand Down
26 changes: 18 additions & 8 deletions cmake/flags.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,17 @@ check_cxx_compiler_flag(-Wformat=2 CXX_HAS_WFORMAT2)
check_cxx_compiler_flag(-Wmissing-include-dirs CXX_HAS_WMISSING_INCLUDE_DIRS)
check_cxx_compiler_flag(-Wno-gnu-zero-variadic-macro-arguments CXX_HAS_NO_GNU_ZERO_VARIADIC_MACRO_ARGUMENTS)


function(kokkoscomm_add_cxx_flags)

cmake_parse_arguments(ADD_CXX_FLAGS "INTERFACE" "TARGET" "" ${ARGN})

if(ADD_CXX_FLAGS_INTERFACE)
set(TARGET_COMPILE_OPTIONS_KEYWORD INTERFACE)
set(TARGET_COMPILE_FEATURES_KEYWORD INTERFACE)
else()
set(TARGET_COMPILE_OPTIONS_KEYWORD PRIVATE)
set(TARGET_COMPILE_FEATURES_KEYWORD PRIVATE)
endif()

if(CXX_HAS_WEXTRA)
Expand Down Expand Up @@ -56,17 +59,24 @@ function(kokkoscomm_add_cxx_flags)
target_compile_options(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_OPTIONS_KEYWORD} $<BUILD_INTERFACE:-Wno-gnu-zero-variadic-macro-arguments>)
endif()

# mdspan-related definitions
if (KOKKOSCOMM_ENABLE_MDSPAN)
# message(STATUS "set c++23 on ${ADD_CXX_FLAGS_TARGET}")
set_property(TARGET ${ADD_CXX_FLAGS_TARGET} PROPERTY CXX_STANDARD 23)
target_compile_definitions(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_OPTIONS_KEYWORD} KOKKOSCOMM_ENABLE_MDSPAN)
if(KOKKOSCOMM_USE_STD_MDSPAN)
target_compile_definitions(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_OPTIONS_KEYWORD} KOKKOSCOMM_USE_STD_MDSPAN)
elseif(KOKKOSCOMM_USE_KOKKOS_MDSPAN)
target_compile_definitions(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_OPTIONS_KEYWORD} KOKKOSCOMM_USE_KOKKOS_MDSPAN)
target_compile_definitions(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_OPTIONS_KEYWORD} KOKKOSCOMM_MDSPAN_IN_EXPERIMENTAL)
endif()
endif()

# choose cxx standard
set_target_properties(${ADD_CXX_FLAGS_TARGET} PROPERTIES CXX_EXTENSIONS OFF)
if (KOKKOSCOMM_ENABLE_MDSPAN AND KOKKOSCOMM_USE_STD_MDSPAN)
target_compile_features(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_FEATURES_KEYWORD} cxx_std_23)
else()
set_property(TARGET ${ADD_CXX_FLAGS_TARGET} PROPERTY CXX_STANDARD 20)
endif()
set_property(TARGET ${ADD_CXX_FLAGS_TARGET} PROPERTY CXX_EXTENSIONS OFF)
# kokkos/mdspan is experimental
if (KOKKOSCOMM_USE_KOKKOS_MDSPAN)
target_compile_definitions(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_OPTIONS_KEYWORD} KOKKOSCOMM_EXPERIMENTAL_MDSPAN)
target_compile_features(${ADD_CXX_FLAGS_TARGET} ${TARGET_COMPILE_FEATURES_KEYWORD} cxx_std_20)
endif()


endfunction()
2 changes: 0 additions & 2 deletions docs/api/collective.rst

This file was deleted.

86 changes: 86 additions & 0 deletions docs/api/core.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
Core
====


Point-to-point
--------------

.. cpp:function:: template <typename SendView, typename ExecSpace> \
Req KokkosComm::isend(const ExecSpace &space, const SendView &sv, int dest, int tag, MPI_Comm comm)

MPI_Isend wrapper

:param space: The execution space to operate in
:param sv: The data to send
:param dest: the destination rank
:param tag: the MPI tag
:param comm: the MPI communicator
:tparam SendView: A Kokkos::View to send
:tparam ExecSpace: A Kokkos execution space to operate in
:returns: A KokkosComm::Req representing the asynchronous communication and any lifetime-extended views.

.. cpp:function:: template <typename SendView, typename ExecSpace> \
void KokkosComm::send(const ExecSpace &space, const SendView &sv, int dest, int tag, MPI_Comm comm)

MPI_Send wrapper

:param space: The execution space to operate in
:param sv: The data to send
:param dest: the destination rank
:param tag: the MPI tag
:param comm: the MPI communicator
:tparam SendView: A Kokkos::View to send
:tparam ExecSpace: A Kokkos execution space to operate in

.. cpp:function:: template <typename Recv, typename ExecSpace> \
void KokkosComm::recv(const ExecSpace &space, Recv &rv, int src, int tag, MPI_Comm comm)

MPI_Recv wrapper

:param space: The execution space to operate in
:param srv: The data to recv
:param src: the source rank
:param tag: the MPI tag
:param comm: the MPI communicator
:tparam Recv: A Kokkos::View to send
:tparam ExecSpace: A Kokkos execution space to operate in


Collective
----------

.. cpp:function:: template <typename SendView, typename RecvView, typename ExecSpace> \
void KokkosComm::reduce(const ExecSpace &space, const SendView &sv, const RecvView &rv, MPI_Op op, int root, MPI_Comm comm)

MPI_Reduce wrapper

:param space: The execution space to operate in
:param sv: The data to send
:param rv: The view to receive into
:param op: The MPI_Op to use in the reduction
:param root: The root rank for the reduction
:param comm: the MPI communicator
:tparam SendView: A Kokkos::View to send
:tparam RecvView: A Kokkos::View to recv
:tparam ExecSpace: A Kokkos execution space to operate in

Related Types
-------------

.. cpp:class:: KokkosComm::Req

A wrapper around an MPI_Request that can also extend the lifetime of Views.

.. cpp:function:: MPI_Request &KokkosComm::Req::mpi_req()

Retrieve a reference to the held MPI_Request.

.. cpp:function:: void KokkosComm::Req::wait()

Call MPI_Wait on the held MPI_Request and drop copies of any previous arguments to Req::keep_until_wait().

.. cpp:function:: template<typename View> \
void KokkosComm::Req::keep_until_wait(const View &v)

Extend the lifetime of v at least until Req::wait() is called.
This is useful to prevent a View from being destroyed during an asynchronous MPI operation.
15 changes: 0 additions & 15 deletions docs/api/p2p.rst

This file was deleted.

Loading

0 comments on commit c8fc55e

Please sign in to comment.