Skip to content

Commit

Permalink
Added fallback for GraphBLAS
Browse files Browse the repository at this point in the history
  • Loading branch information
@JoBuRo
JoBuRo committed Jan 31, 2024
1 parent 92621d0 commit b3d71bd
Show file tree
Hide file tree
Showing 2 changed files with 399 additions and 11 deletions.
292 changes: 281 additions & 11 deletions src/engine/TransitivePath.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,30 @@ void TransitivePath::computeTransitivePathBound(
*dynRes = std::move(res).toDynamic();
}

// _____________________________________________________________________________
template <size_t RES_WIDTH, size_t SUB_WIDTH, size_t SIDE_WIDTH>
void TransitivePath::computeTransitivePathBoundFallback(
IdTable* dynRes, const IdTable& dynSub, const TransitivePathSide& startSide,
const TransitivePathSide& targetSide, const IdTable& startSideTable) const {
IdTableStatic<RES_WIDTH> res = std::move(*dynRes).toStatic<RES_WIDTH>();

auto [edges, nodes] = setupMapAndNodes<SUB_WIDTH, SIDE_WIDTH>(
dynSub, startSide, targetSide, startSideTable);

Map hull(allocator());
if (!targetSide.isVariable()) {
hull = transitiveHull(edges, nodes, std::get<Id>(targetSide.value_));
} else {
hull = transitiveHull(edges, nodes, std::nullopt);
}

TransitivePath::fillTableWithHull<RES_WIDTH, SIDE_WIDTH>(
res, hull, nodes, startSide.outputCol_, targetSide.outputCol_,
startSideTable, startSide.treeAndCol_.value().second);

*dynRes = std::move(res).toDynamic();
}

// _____________________________________________________________________________
template <size_t RES_WIDTH, size_t SUB_WIDTH>
void TransitivePath::computeTransitivePath(
Expand Down Expand Up @@ -261,6 +285,29 @@ void TransitivePath::computeTransitivePath(
*dynRes = std::move(res).toDynamic();
}

// _____________________________________________________________________________
template <size_t RES_WIDTH, size_t SUB_WIDTH>
void TransitivePath::computeTransitivePathFallback(
IdTable* dynRes, const IdTable& dynSub, const TransitivePathSide& startSide,
const TransitivePathSide& targetSide) const {
IdTableStatic<RES_WIDTH> res = std::move(*dynRes).toStatic<RES_WIDTH>();

auto [edges, nodes] =
setupMapAndNodes<SUB_WIDTH>(dynSub, startSide, targetSide);

Map hull{allocator()};
if (!targetSide.isVariable()) {
hull = transitiveHull(edges, nodes, std::get<Id>(targetSide.value_));
} else {
hull = transitiveHull(edges, nodes, std::nullopt);
}

TransitivePath::fillTableWithHull<RES_WIDTH>(res, hull, startSide.outputCol_,
targetSide.outputCol_);

*dynRes = std::move(res).toDynamic();
}

// _____________________________________________________________________________
ResultTable TransitivePath::computeResult() {
if (minDist_ == 0 && !isBoundOrId() && lhs_.isVariable() &&
Expand All @@ -277,17 +324,26 @@ ResultTable TransitivePath::computeResult() {

size_t subWidth = subRes->idTable().numColumns();

auto computeForOneSide = [this, &idTable, subRes, subWidth](
bool useFallback = false;

auto computeForOneSide = [this, &idTable, subRes, subWidth, useFallback](
auto& boundSide,
auto& otherSide) -> ResultTable {
shared_ptr<const ResultTable> sideRes =
boundSide.treeAndCol_.value().first->getResult();
size_t sideWidth = sideRes->idTable().numColumns();

CALL_FIXED_SIZE((std::array{resultWidth_, subWidth, sideWidth}),
&TransitivePath::computeTransitivePathBound, this, &idTable,
subRes->idTable(), boundSide, otherSide,
sideRes->idTable());
if (useFallback) {
CALL_FIXED_SIZE((std::array{resultWidth_, subWidth, sideWidth}),
&TransitivePath::computeTransitivePathBoundFallback, this,
&idTable, subRes->idTable(), boundSide, otherSide,
sideRes->idTable());
} else {
CALL_FIXED_SIZE((std::array{resultWidth_, subWidth, sideWidth}),
&TransitivePath::computeTransitivePathBound, this,
&idTable, subRes->idTable(), boundSide, otherSide,
sideRes->idTable());
}

return {std::move(idTable), resultSortedOn(),
ResultTable::getSharedLocalVocabFromNonEmptyOf(*sideRes, *subRes)};
Expand All @@ -299,15 +355,27 @@ ResultTable TransitivePath::computeResult() {
return computeForOneSide(rhs_, lhs_);
// Right side is an Id
} else if (!rhs_.isVariable()) {
CALL_FIXED_SIZE((std::array{resultWidth_, subWidth}),
&TransitivePath::computeTransitivePath, this, &idTable,
subRes->idTable(), rhs_, lhs_);
if (useFallback) {
CALL_FIXED_SIZE((std::array{resultWidth_, subWidth}),
&TransitivePath::computeTransitivePathFallback, this,
&idTable, subRes->idTable(), rhs_, lhs_);
} else {
CALL_FIXED_SIZE((std::array{resultWidth_, subWidth}),
&TransitivePath::computeTransitivePath, this, &idTable,
subRes->idTable(), rhs_, lhs_);
}
// No side is a bound variable, the right side is an unbound variable
// and the left side is either an unbound Variable or an ID.
} else {
CALL_FIXED_SIZE((std::array{resultWidth_, subWidth}),
&TransitivePath::computeTransitivePath, this, &idTable,
subRes->idTable(), lhs_, rhs_);
if (useFallback) {
CALL_FIXED_SIZE((std::array{resultWidth_, subWidth}),
&TransitivePath::computeTransitivePathFallback, this,
&idTable, subRes->idTable(), lhs_, rhs_);
} else {
CALL_FIXED_SIZE((std::array{resultWidth_, subWidth}),
&TransitivePath::computeTransitivePath, this, &idTable,
subRes->idTable(), lhs_, rhs_);
}
}

// NOTE: The only place, where the input to a transitive path operation is not
Expand Down Expand Up @@ -405,6 +473,84 @@ GrbMatrix TransitivePath::transitiveHull(
return std::move(*result);
}

// _____________________________________________________________________________
TransitivePath::Map TransitivePath::transitiveHull(
const Map& edges, const std::vector<Id>& startNodes,
std::optional<Id> target) const {
using MapIt = Map::const_iterator;
// For every node do a dfs on the graph
Map hull{allocator()};

// Stores nodes we already have a path to. This avoids cycles.
ad_utility::HashSetWithMemoryLimit<Id> marks{
getExecutionContext()->getAllocator()};

// The stack used to store the dfs' progress
std::vector<Set::const_iterator> positions;

// Used to store all edges leading away from a node for every level.
// Reduces access to the hashmap, and is safe as the map will not
// be modified after this point.
std::vector<const Set*> edgeCache;

for (Id currentStartNode : startNodes) {
if (hull.contains(currentStartNode)) {
// We have already computed the hull for this node
continue;
}

// Reset for this iteration
marks.clear();

MapIt rootEdges = edges.find(currentStartNode);
if (rootEdges != edges.end()) {
positions.push_back(rootEdges->second.begin());
edgeCache.push_back(&rootEdges->second);
}
if (minDist_ == 0 &&
(!target.has_value() || currentStartNode == target.value())) {
insertIntoMap(hull, currentStartNode, currentStartNode);
}

// While we have not found the entire transitive hull and have not reached
// the max step limit
while (!positions.empty()) {
checkCancellation();
size_t stackIndex = positions.size() - 1;
// Process the next child of the node at the top of the stack
Set::const_iterator& pos = positions[stackIndex];
const Set* nodeEdges = edgeCache.back();

if (pos == nodeEdges->end()) {
// We finished processing this node
positions.pop_back();
edgeCache.pop_back();
continue;
}

Id child = *pos;
++pos;
size_t childDepth = positions.size();
if (childDepth <= maxDist_ && marks.count(child) == 0) {
// process the child
if (childDepth >= minDist_) {
marks.insert(child);
if (!target.has_value() || child == target.value()) {
insertIntoMap(hull, currentStartNode, child);
}
}
// Add the child to the stack
MapIt it = edges.find(child);
if (it != edges.end()) {
positions.push_back(it->second.begin());
edgeCache.push_back(&it->second);
}
}
}
}
return hull;
}

// _____________________________________________________________________________
template <size_t WIDTH>
void TransitivePath::fillTableWithHull(IdTableStatic<WIDTH>& table,
Expand Down Expand Up @@ -477,6 +623,124 @@ void TransitivePath::fillTableWithHull(IdTableStatic<WIDTH>& table,
}
}

// _____________________________________________________________________________
template <size_t WIDTH, size_t START_WIDTH>
void TransitivePath::fillTableWithHull(IdTableStatic<WIDTH>& table,
const Map& hull, std::vector<Id>& nodes,
size_t startSideCol,
size_t targetSideCol,
const IdTable& startSideTable,
size_t skipCol) {
IdTableView<START_WIDTH> startView =
startSideTable.asStaticView<START_WIDTH>();

size_t rowIndex = 0;
for (size_t i = 0; i < nodes.size(); i++) {
Id node = nodes[i];
auto it = hull.find(node);
if (it == hull.end()) {
continue;
}

for (Id otherNode : it->second) {
table.emplace_back();
table(rowIndex, startSideCol) = node;
table(rowIndex, targetSideCol) = otherNode;

TransitivePath::copyColumns<START_WIDTH, WIDTH>(startView, table, i,
rowIndex, skipCol);

rowIndex++;
}
}
}

// _____________________________________________________________________________
template <size_t WIDTH>
void TransitivePath::fillTableWithHull(IdTableStatic<WIDTH>& table,
const Map& hull, size_t startSideCol,
size_t targetSideCol) {
size_t rowIndex = 0;
for (auto const& [node, linkedNodes] : hull) {
for (Id linkedNode : linkedNodes) {
table.emplace_back();
table(rowIndex, startSideCol) = node;
table(rowIndex, targetSideCol) = linkedNode;

rowIndex++;
}
}
}

// _____________________________________________________________________________
template <size_t SUB_WIDTH, size_t SIDE_WIDTH>
std::pair<TransitivePath::Map, std::vector<Id>>
TransitivePath::setupMapAndNodes(const IdTable& sub,
const TransitivePathSide& startSide,
const TransitivePathSide& targetSide,
const IdTable& startSideTable) const {
std::vector<Id> nodes;
Map edges = setupEdgesMap<SUB_WIDTH>(sub, startSide, targetSide);

// Bound -> var|id
std::span<const Id> startNodes = setupNodes<SIDE_WIDTH>(
startSideTable, startSide.treeAndCol_.value().second);
nodes.insert(nodes.end(), startNodes.begin(), startNodes.end());

return {std::move(edges), std::move(nodes)};
}

// _____________________________________________________________________________
template <size_t SUB_WIDTH>
std::pair<TransitivePath::Map, std::vector<Id>>
TransitivePath::setupMapAndNodes(const IdTable& sub,
const TransitivePathSide& startSide,
const TransitivePathSide& targetSide) const {
std::vector<Id> nodes;
Map edges = setupEdgesMap<SUB_WIDTH>(sub, startSide, targetSide);

// id -> var|id
if (!startSide.isVariable()) {
nodes.push_back(std::get<Id>(startSide.value_));
// var -> var
} else {
std::span<const Id> startNodes =
setupNodes<SUB_WIDTH>(sub, startSide.subCol_);
nodes.insert(nodes.end(), startNodes.begin(), startNodes.end());
if (minDist_ == 0) {
std::span<const Id> targetNodes =
setupNodes<SUB_WIDTH>(sub, targetSide.subCol_);
nodes.insert(nodes.end(), targetNodes.begin(), targetNodes.end());
}
}

return {std::move(edges), std::move(nodes)};
}

// _____________________________________________________________________________
template <size_t SUB_WIDTH>
TransitivePath::Map TransitivePath::setupEdgesMap(
const IdTable& dynSub, const TransitivePathSide& startSide,
const TransitivePathSide& targetSide) const {
const IdTableView<SUB_WIDTH> sub = dynSub.asStaticView<SUB_WIDTH>();
Map edges{allocator()};
decltype(auto) startCol = sub.getColumn(startSide.subCol_);
decltype(auto) targetCol = sub.getColumn(targetSide.subCol_);

for (size_t i = 0; i < sub.size(); i++) {
checkCancellation();
insertIntoMap(edges, startCol[i], targetCol[i]);
}
return edges;
}

// _____________________________________________________________________________
template <size_t WIDTH>
std::span<const Id> TransitivePath::setupNodes(const IdTable& table,
size_t col) {
return table.getColumn(col);
}

// _____________________________________________________________________________
GrbMatrix TransitivePath::getTargetRow(GrbMatrix& hull,
size_t targetIndex) const {
Expand Down Expand Up @@ -547,3 +811,9 @@ void TransitivePath::copyColumns(const IdTableView<INPUT_WIDTH>& inputTable,
outCol++;
}
}

// _____________________________________________________________________________
void TransitivePath::insertIntoMap(Map& map, Id key, Id value) const {
auto [it, success] = map.try_emplace(key, allocator());
it->second.insert(value);
}
Loading

0 comments on commit b3d71bd

Please sign in to comment.