diff --git a/providers/bnxt_re/bnxt_re-abi.h b/providers/bnxt_re/bnxt_re-abi.h index 6382dcb74..f294da2d8 100644 --- a/providers/bnxt_re/bnxt_re-abi.h +++ b/providers/bnxt_re/bnxt_re-abi.h @@ -167,6 +167,15 @@ enum bnxt_re_psns_mask { BNXT_RE_PSNS_FLAGS_SHIFT = 0x18 }; +enum bnxt_re_msns_mask { + BNXT_RE_SQ_MSN_SEARCH_START_PSN_MASK = 0xFFFFFFUL, + BNXT_RE_SQ_MSN_SEARCH_START_PSN_SHIFT = 0, + BNXT_RE_SQ_MSN_SEARCH_NEXT_PSN_MASK = 0xFFFFFF000000ULL, + BNXT_RE_SQ_MSN_SEARCH_NEXT_PSN_SHIFT = 0x18, + BNXT_RE_SQ_MSN_SEARCH_START_IDX_MASK = 0xFFFF000000000000ULL, + BNXT_RE_SQ_MSN_SEARCH_START_IDX_SHIFT = 0x30 +}; + enum bnxt_re_bcqe_mask { BNXT_RE_BCQE_PH_MASK = 0x01, BNXT_RE_BCQE_TYPE_MASK = 0x0F, @@ -301,6 +310,11 @@ struct bnxt_re_psns_ext { __u32 rsvd1; }; +/* sq_msn_search (size:64b/8B) */ +struct bnxt_re_msns { + __le64 start_idx_next_psn_start_psn; +}; + struct bnxt_re_sge { __le64 pa; __le32 lkey; diff --git a/providers/bnxt_re/main.h b/providers/bnxt_re/main.h index 031db17d2..f51ad1955 100644 --- a/providers/bnxt_re/main.h +++ b/providers/bnxt_re/main.h @@ -601,4 +601,6 @@ static inline void bnxt_re_sub_sec_busy_wait(uint32_t nsec) break; } } + +#define BNXT_RE_HW_RETX(a) ((a)->comp_mask & BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED) #endif diff --git a/providers/bnxt_re/memory.h b/providers/bnxt_re/memory.h index 329e14a8e..a3c14631b 100644 --- a/providers/bnxt_re/memory.h +++ b/providers/bnxt_re/memory.h @@ -51,6 +51,8 @@ struct bnxt_re_queue { uint32_t head; uint32_t tail; uint32_t stride; + void *pad; /* to hold the padding area */ + uint32_t pad_stride_log2; /* Represents the difference between the real queue depth allocated in * HW and the user requested queue depth and is used to correctly flag * queue full condition based on user supplied queue depth. @@ -62,6 +64,8 @@ struct bnxt_re_queue { uint32_t esize; uint32_t max_slots; pthread_spinlock_t qlock; + uint32_t msn; + uint32_t msn_tbl_sz; }; int bnxt_re_alloc_aligned(struct bnxt_re_queue *que, uint32_t pg_size); diff --git a/providers/bnxt_re/verbs.c b/providers/bnxt_re/verbs.c index 709abff56..7d4a8a9fa 100644 --- a/providers/bnxt_re/verbs.c +++ b/providers/bnxt_re/verbs.c @@ -47,6 +47,7 @@ #include #include #include +#include #include #include @@ -1182,10 +1183,11 @@ static int bnxt_re_alloc_queues(struct bnxt_re_context *cntx, que->depth = nslots; que->diff = (diff * que->esize) / que->stride; + que->pad = (que->va + que->depth * que->stride); /* psn_depth extra entries of size que->stride */ - psn_size = qp->cctx->gen_p5_p7 ? sizeof(struct bnxt_re_psns_ext) : - sizeof(struct bnxt_re_psns); + psn_size = bnxt_re_get_psne_size(qp->cntx); psn_depth = (nswr * psn_size) / que->stride; + que->pad_stride_log2 = (uint32_t)ilog32(psn_size); if ((nswr * psn_size) % que->stride) psn_depth++; que->depth += psn_depth; @@ -1218,6 +1220,17 @@ static int bnxt_re_alloc_queues(struct bnxt_re_context *cntx, swque[indx].psns = (struct bnxt_re_psns *)psns_ext; } } + /* Init and adjust MSN table size according to qp mode */ + if (!BNXT_RE_HW_RETX(qp->cntx)) + goto skip_msn; + que->msn = 0; + que->msn_tbl_sz = 0; + if (qp->qpmode & BNXT_RE_WQE_MODE_VARIABLE) + que->msn_tbl_sz = roundup_pow_of_two(nslots) / 2; + else + que->msn_tbl_sz = roundup_pow_of_two(nswr); +skip_msn: + qp->cap.max_swr = nswr; pthread_spin_init(&que->qlock, PTHREAD_PROCESS_PRIVATE); @@ -1588,6 +1601,44 @@ static int bnxt_re_build_tx_sge(struct bnxt_re_queue *que, uint32_t *idx, return bnxt_re_put_tx_sge(que, idx, wr->sg_list, wr->num_sge); } +static void *bnxt_re_pull_psn_buff(struct bnxt_re_queue *que, bool hw_retx) +{ + if (hw_retx) + return (void *)(que->pad + ((que->msn) << que->pad_stride_log2)); + return (void *)(que->pad + ((*que->dbtail) << que->pad_stride_log2)); +} + +static void bnxt_re_fill_psns_for_msntbl(struct bnxt_re_qp *qp, uint32_t len, uint32_t st_idx) +{ + uint32_t npsn = 0, start_psn = 0, next_psn = 0; + struct bnxt_re_msns *msns; + uint32_t pkt_cnt = 0; + + msns = bnxt_re_pull_psn_buff(qp->jsqq->hwque, true); + msns->start_idx_next_psn_start_psn = 0; + + if (qp->qptyp == IBV_QPT_RC) { + start_psn = qp->sq_psn; + pkt_cnt = (len / qp->mtu); + if (len % qp->mtu) + pkt_cnt++; + /* Increment the psn even for 0 len packets + * e.g. for opcode rdma-write-with-imm-data + * with length field = 0 + */ + if (len == 0) + pkt_cnt = 1; + /* make it 24 bit */ + next_psn = qp->sq_psn + pkt_cnt; + npsn = next_psn; + qp->sq_psn = next_psn; + msns->start_idx_next_psn_start_psn |= + bnxt_re_update_msn_tbl(st_idx, npsn, start_psn); + qp->jsqq->hwque->msn++; + qp->jsqq->hwque->msn %= qp->jsqq->hwque->msn_tbl_sz; + } +} + static void bnxt_re_fill_psns(struct bnxt_re_qp *qp, struct bnxt_re_wrid *wrid, uint32_t len) { @@ -1810,7 +1861,10 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, bnxt_re_fill_wrid(wrid, wr->wr_id, bytes, sig, sq->tail, slots); wrid->wc_opcd = bnxt_re_ibv_wr_to_wc_opcd(wr->opcode); - bnxt_re_fill_psns(qp, wrid, bytes); + if (BNXT_RE_HW_RETX(qp->cntx)) + bnxt_re_fill_psns_for_msntbl(qp, bytes, *sq->dbtail); + else + bnxt_re_fill_psns(qp, wrid, bytes); bnxt_re_jqq_mod_start(qp->jsqq, swq_idx); bnxt_re_incr_tail(sq, slots); ring_db = true; diff --git a/providers/bnxt_re/verbs.h b/providers/bnxt_re/verbs.h index c42c2926f..52adf28ff 100644 --- a/providers/bnxt_re/verbs.h +++ b/providers/bnxt_re/verbs.h @@ -60,6 +60,20 @@ struct bnxt_re_work_compl { struct ibv_wc wc; }; +static inline uint8_t bnxt_re_get_psne_size(struct bnxt_re_context *cntx) +{ + return (BNXT_RE_HW_RETX(cntx)) ? sizeof(struct bnxt_re_msns) : + (cntx->cctx.gen_p5_p7) ? + sizeof(struct bnxt_re_psns_ext) : + sizeof(struct bnxt_re_psns); +} + +static inline uint32_t bnxt_re_get_npsn(uint8_t mode, uint32_t nwr, + uint32_t slots) +{ + return mode == BNXT_RE_WQE_MODE_VARIABLE ? slots : nwr; +} + int bnxt_re_query_device(struct ibv_context *context, const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *attr, size_t attr_size); @@ -107,4 +121,14 @@ int bnxt_re_destroy_ah(struct ibv_ah *ibvah); void bnxt_re_async_event(struct ibv_context *context, struct ibv_async_event *event); +static inline __le64 bnxt_re_update_msn_tbl(uint32_t st_idx, uint32_t npsn, uint32_t start_psn) +{ + /* Adjust the field values to their respective ofsets */ + return htole64((((uint64_t)(st_idx) << BNXT_RE_SQ_MSN_SEARCH_START_IDX_SHIFT) & + BNXT_RE_SQ_MSN_SEARCH_START_IDX_MASK) | + (((uint64_t)(npsn) << BNXT_RE_SQ_MSN_SEARCH_NEXT_PSN_SHIFT) & + BNXT_RE_SQ_MSN_SEARCH_NEXT_PSN_MASK) | + (((start_psn) << BNXT_RE_SQ_MSN_SEARCH_START_PSN_SHIFT) & + BNXT_RE_SQ_MSN_SEARCH_START_PSN_MASK)); +} #endif /* __BNXT_RE_VERBS_H__ */