Merge pull request #1445 from hginjgerx/dscp

libhns: Support DSCP for userspace
linux-rdma · Apr 15, 2024 · 3360630 · 3360630
2 parents 8364b11 + f2e92c3
commit 3360630
Show file tree

Hide file tree

Showing 5 changed files with 36 additions and 6 deletions.
diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h
@@ -109,6 +109,12 @@ struct hns_roce_ib_create_qp_resp {
  __aligned_u64 dwqe_mmap_key;
 };
 
+struct hns_roce_ib_modify_qp_resp {
+ __u8 tc_mode;
+ __u8 priority;
+ __u8 reserved[6];
+};
+
 enum {
  HNS_ROCE_EXSGE_FLAGS = 1 << 0,
  HNS_ROCE_RQ_INLINE_FLAGS = 1 << 1,
@@ -143,7 +149,8 @@ struct hns_roce_ib_alloc_pd_resp {
 
 struct hns_roce_ib_create_ah_resp {
  __u8 dmac[6];
- __u8 reserved[2];
+ __u8 priority;
+ __u8 tc_mode;
 };
 
 #endif /* HNS_ABI_USER_H */
diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
@@ -183,6 +183,11 @@ enum hns_roce_pktype {
  HNS_ROCE_PKTYPE_ROCE_V2_IPV4,
 };
 
+enum hns_roce_tc_map_mode {
+ HNS_ROCE_TC_MAP_MODE_PRIO,
+ HNS_ROCE_TC_MAP_MODE_DSCP,
+};
+
 struct hns_roce_db_page {
  struct hns_roce_db_page *prev, *next;
  struct hns_roce_buf buf;
@@ -324,6 +329,8 @@ struct hns_roce_qp {
  unsigned int next_sge;
  int port_num;
  uint8_t sl;
+ uint8_t tc_mode;
+ uint8_t priority;
  unsigned int qkey;
  enum ibv_mtu path_mtu;
 

diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h
@@ -65,4 +65,7 @@ DECLARE_DRV_CMD(hns_roce_create_srq_ex, IB_USER_VERBS_CMD_CREATE_XSRQ,
 DECLARE_DRV_CMD(hns_roce_create_ah, IB_USER_VERBS_CMD_CREATE_AH, empty,
  hns_roce_ib_create_ah_resp);
 
+DECLARE_DRV_CMD(hns_roce_modify_qp_ex, IB_USER_VERBS_EX_CMD_MODIFY_QP,
+ empty, hns_roce_ib_modify_qp_resp);
+
 #endif /* _HNS_ROCE_U_ABI_H */
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
@@ -1523,7 +1523,9 @@ static void record_qp_attr(struct ibv_qp *qp, struct ibv_qp_attr *attr,
  if (attr_mask & IBV_QP_PORT)
  hr_qp->port_num = attr->port_num;
 
- if (attr_mask & IBV_QP_AV)
+ if (hr_qp->tc_mode == HNS_ROCE_TC_MAP_MODE_DSCP)
+ hr_qp->sl = hr_qp->priority;
+ else if (attr_mask & IBV_QP_AV)
  hr_qp->sl = attr->ah_attr.sl;
 
  if (attr_mask & IBV_QP_QKEY)
@@ -1538,18 +1540,21 @@ static void record_qp_attr(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
  int attr_mask)
 {
- int ret;
- struct ibv_modify_qp cmd;
+ struct hns_roce_modify_qp_ex_resp resp_ex = {};
+ struct hns_roce_modify_qp_ex cmd_ex = {};
  struct hns_roce_qp *hr_qp = to_hr_qp(qp);
  bool flag = false; /* modify qp to error */
+ int ret;
 
  if ((attr_mask & IBV_QP_STATE) && (attr->qp_state == IBV_QPS_ERR)) {
  pthread_spin_lock(&hr_qp->sq.lock);
  pthread_spin_lock(&hr_qp->rq.lock);
  flag = true;
  }
 
- ret = ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof(cmd));
+ ret = ibv_cmd_modify_qp_ex(qp, attr, attr_mask, &cmd_ex.ibv_cmd,
+ sizeof(cmd_ex), &resp_ex.ibv_resp,
+ sizeof(resp_ex));
 
  if (flag) {
  if (!ret)
@@ -1561,8 +1566,13 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
  if (ret)
  return ret;
 
- if (attr_mask & IBV_QP_STATE)
+ if (attr_mask & IBV_QP_STATE) {
  qp->state = attr->qp_state;
+ if (attr->qp_state == IBV_QPS_RTR) {
+ hr_qp->tc_mode = resp_ex.drv_payload.tc_mode;
+ hr_qp->priority = resp_ex.drv_payload.priority;
+ }
+ }
 
  if ((attr_mask & IBV_QP_STATE) && attr->qp_state == IBV_QPS_RESET) {
  if (qp->recv_cq)

diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
@@ -1580,6 +1580,9 @@ struct ibv_ah *hns_roce_u_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
  ah->av.mac, NULL))
  goto err;
 
+ if (resp.tc_mode == HNS_ROCE_TC_MAP_MODE_DSCP)
+ ah->av.sl = resp.priority;
+
  ah->av.udp_sport = get_ah_udp_sport(attr);
 
  return &ah->ibv_ah;