Enable parallel mode (#3875)

This PR enables parallel mode


Closes rapidsai/graph_dl#328

Authors:
  - Joseph Nke (https://github.com/jnke2016)
  - Rick Ratzel (https://github.com/rlratzel)

Approvers:
  - Chuck Hastings (https://github.com/ChuckHastings)
  - Naim (https://github.com/naimnv)
  - Brad Rees (https://github.com/BradReesWork)
  - Rick Ratzel (https://github.com/rlratzel)
  - Jake Awe (https://github.com/AyodeAwe)

URL: #3875

cpp/CMakeLists.txt

@@ -192,6 +192,7 @@ set(CUGRAPH_SOURCES
     src/detail/shuffle_vertex_pairs.cu
     src/detail/collect_local_vertex_values.cu
     src/detail/groupby_and_count.cu
+    src/detail/collect_comm_wrapper.cu
     src/sampling/random_walks_mg.cu
     src/community/detail/common_methods_mg.cu
     src/community/detail/common_methods_sg.cu
@@ -443,6 +444,7 @@ add_library(cugraph_c
         src/c_api/labeling_result.cpp
         src/c_api/weakly_connected_components.cpp
         src/c_api/strongly_connected_components.cpp
+        src/c_api/allgather.cpp
         src/c_api/legacy_k_truss.cpp
         )
 add_library(cugraph::cugraph_c ALIAS cugraph_c)

cpp/include/cugraph/detail/collect_comm_wrapper.hpp

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2023, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <raft/core/handle.hpp>
+#include <rmm/device_uvector.hpp>
+
+#include <optional>
+
+namespace cugraph {
+namespace detail {
+
+/**
+ * @brief Gather the span of data from all ranks and broadcast the combined data to all ranks.
+ *
+ * @param[in] handle RAFT handle object to encapsulate resources (e.g. CUDA stream, communicator,
+ * and handles to various CUDA libraries) to run graph algorithms.
+ * @param[in] comm Raft comms that manages underlying NCCL comms handles across the ranks.
+ * @param[in] d_input The span of data to perform the 'allgatherv'.
+ *
+ * @return A vector containing the combined data of all ranks.
+ */
+template <typename T>
+rmm::device_uvector<T> device_allgatherv(raft::handle_t const& handle,
+                                         raft::comms::comms_t const& comm,
+                                         raft::device_span<T const> d_input);
+
+}  // namespace detail
+}  // namespace cugraph

cpp/include/cugraph_c/graph_functions.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022, NVIDIA CORPORATION.
+ * Copyright (c) 2022-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -136,6 +136,24 @@ cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_destinatio
 cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_weights(
   cugraph_induced_subgraph_result_t* induced_subgraph);
 
+/**
+ * @brief       Get the edge ids
+ *
+ * @param [in]     induced_subgraph   Opaque pointer to induced subgraph
+ * @return type erased array view of edge ids
+ */
+cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_ids(
+  cugraph_induced_subgraph_result_t* induced_subgraph);
+
+/**
+ * @brief       Get the edge types
+ *
+ * @param [in]     induced_subgraph   Opaque pointer to induced subgraph
+ * @return type erased array view of edge types
+ */
+cugraph_type_erased_device_array_view_t* cugraph_induced_subgraph_get_edge_type_ids(
+  cugraph_induced_subgraph_result_t* induced_subgraph);
+
 /**
  * @brief       Get the subgraph offsets
  *
@@ -184,6 +202,33 @@ cugraph_error_code_t cugraph_extract_induced_subgraph(
   cugraph_induced_subgraph_result_t** result,
   cugraph_error_t** error);
 
+// FIXME: Rename the return type
+/**
+ * @brief      Gather edgelist
+ *
+ * This function collects the edgelist from all ranks and stores the combine edgelist
+ * in each rank
+ *
+ * @param [in]  handle            Handle for accessing resources.
+ * @param [in]  src               Device array containing the source vertex ids.
+ * @param [in]  dst               Device array containing the destination vertex ids
+ * @param [in]  weights           Optional device array containing the edge weights
+ * @param [in]  edge_ids          Optional device array containing the edge ids for each edge.
+ * @param [in]  edge_type_ids     Optional device array containing the edge types for each edge
+ * @param [out] result            Opaque pointer to gathered edgelist result
+ * @param [out] error             Pointer to an error object storing details of any error.  Will
+ *                                be populated if error code is not CUGRAPH_SUCCESS
+ * @return error code
+ */
+cugraph_error_code_t cugraph_allgather(const cugraph_resource_handle_t* handle,
+                                       const cugraph_type_erased_device_array_view_t* src,
+                                       const cugraph_type_erased_device_array_view_t* dst,
+                                       const cugraph_type_erased_device_array_view_t* weights,
+                                       const cugraph_type_erased_device_array_view_t* edge_ids,
+                                       const cugraph_type_erased_device_array_view_t* edge_type_ids,
+                                       cugraph_induced_subgraph_result_t** result,
+                                       cugraph_error_t** error);
+
 #ifdef __cplusplus
 }
 #endif