libhns: Bugfix for flush cqe in case multi-process

[ Upstream commit f1a80cc ] For the case one process modify qp to error while the other process is posting send or recv, the race of flush cqe will happen. To solve this problem, the lock of sq and rq should be hold in modify qp. Fixes: e1726e9 ("libhns: Support flush cqe for hip08 in user space") Signed-off-by: Yixian Liu <[email protected]> Signed-off-by: Nicolas Morey-Chaisemartin <[email protected]>
linux-rdma · May 22, 2019 · 9f71327 · 9f71327
1 parent a17190e
commit 9f71327
Showing 1 changed file with 30 additions and 24 deletions.
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
@@ -859,22 +859,19 @@ int hns_roce_u_v2_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
 			*(qp->sdb) = qp->sq.head & 0xffff;
 
 		qp->next_sge = ind_sge;
-
-		if (ibvqp->state == IBV_QPS_ERR) {
-			attr_mask = IBV_QP_STATE;
-			attr.qp_state = IBV_QPS_ERR;
-
-			ret = hns_roce_u_v2_modify_qp(ibvqp, &attr, attr_mask);
-			if (ret) {
-				pthread_spin_unlock(&qp->sq.lock);
-				*bad_wr = wr;
-				return ret;
-			}
-		}
 	}
 
 	pthread_spin_unlock(&qp->sq.lock);
 
+	if (ibvqp->state == IBV_QPS_ERR) {
+		attr_mask = IBV_QP_STATE;
+		attr.qp_state = IBV_QPS_ERR;
+
+		ret = hns_roce_u_v2_modify_qp(ibvqp, &attr, attr_mask);
+		if (ret)
+			*bad_wr = wr;
+	}
+
 	return ret;
 }
 
@@ -960,22 +957,19 @@ static int hns_roce_u_v2_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr,
 		else
 			hns_roce_update_rq_db(ctx, qp->ibv_qp.qp_num,
 				     qp->rq.head & ((qp->rq.wqe_cnt << 1) - 1));
-
-		if (ibvqp->state == IBV_QPS_ERR) {
-			attr_mask = IBV_QP_STATE;
-			attr.qp_state = IBV_QPS_ERR;
-
-			ret = hns_roce_u_v2_modify_qp(ibvqp, &attr, attr_mask);
-			if (ret) {
-				pthread_spin_unlock(&qp->rq.lock);
-				*bad_wr = wr;
-				return ret;
-			}
-		}
 	}
 
 	pthread_spin_unlock(&qp->rq.lock);
 
+	if (ibvqp->state == IBV_QPS_ERR) {
+		attr_mask = IBV_QP_STATE;
+		attr.qp_state = IBV_QPS_ERR;
+
+		ret = hns_roce_u_v2_modify_qp(ibvqp, &attr, attr_mask);
+		if (ret)
+			*bad_wr = wr;
+	}
+
 	return ret;
 }
 
@@ -1030,9 +1024,21 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 	int ret;
 	struct ibv_modify_qp cmd;
 	struct hns_roce_qp *hr_qp = to_hr_qp(qp);
+	bool flag = false; /* modify qp to error */
+
+	if ((attr_mask & IBV_QP_STATE) && (attr->qp_state == IBV_QPS_ERR)) {
+		pthread_spin_lock(&hr_qp->sq.lock);
+		pthread_spin_lock(&hr_qp->rq.lock);
+		flag = true;
+	}
 
 	ret = ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof(cmd));
 
+	if (flag) {
+		pthread_spin_unlock(&hr_qp->rq.lock);
+		pthread_spin_unlock(&hr_qp->sq.lock);
+	}
+
 	if (!ret && (attr_mask & IBV_QP_STATE) &&
 	    attr->qp_state == IBV_QPS_RESET) {
 		hns_roce_v2_cq_clean(to_hr_cq(qp->recv_cq), qp->qp_num,