bnxt_re/lib: Avoid getting push buffer from data path.

Initialize the qp with a push buffer during QP create. Use the push buffer directly in post_send, instead of getting it using a bit map. This avoids few cpu cycles in data path. This is adding a limiation where the first 15 QPs of any application works in push mode. Reviewed-by: Kalesh AP <[email protected]> Reviewed-by: Ajit Khaparde <[email protected]> Signed-off-by: Selvin Xavier <[email protected]>
linux-rdma · Dec 19, 2024 · 0fe89fc · 0fe89fc
1 parent c8517c5
commit 0fe89fc
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/providers/bnxt_re/main.h b/providers/bnxt_re/main.h
@@ -204,6 +204,7 @@ struct bnxt_re_qp {
 	uint32_t tbl_indx;
 	uint32_t sq_psn;
 	uint32_t pending_db;
+	void *pbuf;
 	uint64_t wqe_cnt;
 	uint16_t mtu;
 	uint16_t qpst;

diff --git a/providers/bnxt_re/verbs.c b/providers/bnxt_re/verbs.c
@@ -1557,6 +1557,7 @@ struct ibv_qp *bnxt_re_create_qp(struct ibv_pd *ibvpd,
 	if (qp->cctx->gen_p5_p7 && cntx->udpi.wcdpi) {
 		qp->push_st_en = 1;
 		qp->max_push_sz = BNXT_RE_MAX_INLINE_SIZE;
+		qp->pbuf = bnxt_re_get_pbuf(&qp->push_st_en, cntx);
 	}
 
 	return &qp->ibvqp;
@@ -1628,6 +1629,10 @@ int bnxt_re_destroy_qp(struct ibv_qp *ibvqp)
 	if (status)
 		return status;
 
+	if (qp->pbuf) {
+		bnxt_re_put_pbuf(qp->cntx, qp->pbuf);
+		qp->pbuf = NULL;
+	}
 	bnxt_re_cleanup_cq(qp, qp->rcq);
 	bnxt_re_cleanup_cq(qp, qp->scq);
 	mem = qp->mem;
@@ -1739,7 +1744,7 @@ static int bnxt_re_required_slots(struct bnxt_re_qp *qp, struct ibv_send_wr *wr,
 		if (ilsize > qp->cap.max_inline)
 			return -EINVAL;
 		if (qp->push_st_en && ilsize <= qp->max_push_sz)
-			*pbuf = bnxt_re_get_pbuf(&qp->push_st_en, qp->cntx);
+			*pbuf = qp->pbuf;
 		wqe_byte = (ilsize + bnxt_re_get_sqe_hdr_sz());
 	} else {
 		wqe_byte = bnxt_re_calc_wqe_sz(wr->num_sge);
@@ -2064,7 +2069,6 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
 			ring_db = false;
 			pbuf->tail = *sq->dbtail;
 			bnxt_re_fill_push_wcb(qp, pbuf, idx);
-			bnxt_re_put_pbuf(qp->cntx, pbuf);
 			pbuf = NULL;
 		}
 
@@ -2080,8 +2084,6 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
 	if (ring_db)
 		bnxt_re_ring_sq_db(qp);
 
-	if (pbuf)
-		bnxt_re_put_pbuf(qp->cntx, pbuf);
 
 	pthread_spin_unlock(&sq->qlock);
 	return ret;