Merge pull request #1410 from yishaih/mlx5_misc

mlx5:  Extend to support 'reg c0' and SW encap ICM

kernel-headers/rdma/bnxt_re-abi.h

@@ -55,6 +55,7 @@ enum {
  BNXT_RE_UCNTX_CMASK_WC_DPI_ENABLED = 0x04ULL,
  BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED = 0x08ULL,
  BNXT_RE_UCNTX_CMASK_POW2_DISABLED = 0x10ULL,
+ BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40,
 };
 
 enum bnxt_re_wqe_mode {

kernel-headers/rdma/hns-abi.h

@@ -125,4 +125,9 @@ struct hns_roce_ib_alloc_pd_resp {
  __u32 pdn;
 };
 
+struct hns_roce_ib_create_ah_resp {
+ __u8 dmac[6];
+ __u8 reserved[2];
+};
+
 #endif /* HNS_ABI_USER_H */

kernel-headers/rdma/mlx5-abi.h

@@ -37,6 +37,7 @@
 #include <linux/types.h>
 #include <linux/if_ether.h> /* For ETH_ALEN. */
 #include <rdma/ib_user_ioctl_verbs.h>
+#include <rdma/mlx5_user_ioctl_verbs.h>
 
 enum {
  MLX5_QP_FLAG_SIGNATURE = 1 << 0,
@@ -275,6 +276,7 @@ struct mlx5_ib_query_device_resp {
  __u32 tunnel_offloads_caps; /* enum mlx5_ib_tunnel_offloads */
  struct mlx5_ib_dci_streams_caps dci_streams_caps;
  __u16 reserved;
+ struct mlx5_ib_uapi_reg reg_c0;
 };
 
 enum mlx5_ib_create_cq_flags {

kernel-headers/rdma/mlx5_user_ioctl_verbs.h

@@ -64,6 +64,7 @@ enum mlx5_ib_uapi_dm_type {
  MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM,
  MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM,
  MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_PATTERN_SW_ICM,
+ MLX5_IB_UAPI_DM_TYPE_ENCAP_SW_ICM,
 };
 
 enum mlx5_ib_uapi_devx_create_event_channel_flags {

providers/mlx5/dr_action.c

@@ -831,7 +831,7 @@ int dr_actions_build_ste_arr(struct mlx5dv_dr_matcher *matcher,
  }
 
  attr.reformat_size = action->reformat.reformat_size;
- attr.reformat_id = action->reformat.dvo->object_id;
+ attr.reformat_id = dr_actions_reformat_get_id(action);
  attr.prio_tag_required = dmn->info.caps.prio_tag_required;
  break;
  case DR_ACTION_TYP_METER:
@@ -1527,8 +1527,9 @@ dr_action_verify_reformat_params(enum mlx5dv_flow_action_packet_reformat_type re
  size_t data_sz,
  void *data)
 {
- if ((!data && data_sz) || (data && !data_sz) || reformat_type >
- MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL) {
+ if ((!data && data_sz) || (data && !data_sz) ||
+ (dr_domain_is_support_sw_encap(dmn) && (data_sz > dmn->info.caps.max_encap_size)) ||
+ reformat_type > MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL) {
  dr_dbg(dmn, "Invalid reformat parameter!\n");
  goto out_err;
  }
@@ -1557,31 +1558,88 @@ dr_action_verify_reformat_params(enum mlx5dv_flow_action_packet_reformat_type re
  return errno;
 }
 
+static int dr_action_create_sw_reformat(struct mlx5dv_dr_domain *dmn,
+ struct mlx5dv_dr_action *action,
+ size_t data_sz, void *data)
+{
+ uint8_t *reformat_data;
+ int ret;
+
+ reformat_data = calloc(1, data_sz);
+ if (!reformat_data) {
+ errno = ENOMEM;
+ return errno;
+ }
+ memcpy(reformat_data, data, data_sz);
+ action->reformat.data = reformat_data;
+ action->reformat.reformat_size = data_sz;
+ ret = dr_ste_alloc_encap(action);
+ if (ret)
+ goto free_reformat_data;
+
+ return 0;
+
+free_reformat_data:
+ free(reformat_data);
+ action->reformat.data = NULL;
+
+ return ret;
+}
+
+static void dr_action_destroy_sw_reformat(struct mlx5dv_dr_action *action)
+{
+ dr_ste_free_encap(action);
+ free(action->reformat.data);
+}
+
+static int dr_action_create_devx_reformat(struct mlx5dv_dr_domain *dmn,
+ struct mlx5dv_dr_action *action,
+ size_t data_sz, void *data)
+{
+ struct mlx5dv_devx_obj *obj;
+ enum reformat_type rt;
+
+ if (action->action_type == DR_ACTION_TYP_L2_TO_TNL_L2)
+ rt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;
+ else
+ rt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
+
+ obj = dr_devx_create_reformat_ctx(dmn->ctx, rt, data_sz, data);
+ if (!obj)
+ return errno;
+
+ action->reformat.dvo = obj;
+ action->reformat.reformat_size = data_sz;
+
+ return 0;
+}
+
+static void dr_action_destroy_devx_reformat(struct mlx5dv_dr_action *action)
+{
+ mlx5dv_devx_obj_destroy(action->reformat.dvo);
+}
+
 static int
 dr_action_create_reformat_action(struct mlx5dv_dr_domain *dmn,
  size_t data_sz, void *data,
  struct mlx5dv_dr_action *action)
 {
- struct mlx5dv_devx_obj *obj;
  uint8_t *hw_actions;
 
  switch (action->action_type) {
  case DR_ACTION_TYP_L2_TO_TNL_L2:
  case DR_ACTION_TYP_L2_TO_TNL_L3:
  {
- enum reformat_type rt;
-
- if (action->action_type == DR_ACTION_TYP_L2_TO_TNL_L2)
- rt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;
- else
- rt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
+ if (dr_domain_is_support_sw_encap(dmn) &&
+ !dr_action_create_sw_reformat(dmn, action, data_sz, data))
+ return 0;
 
- obj = dr_devx_create_reformat_ctx(dmn->ctx, rt, data_sz, data);
- if (!obj)
+ /* When failed creating sw encap, fallback to
+ * use devx to try again.
+ */
+ if (dr_action_create_devx_reformat(dmn, action, data_sz, data))
  return errno;
 
- action->reformat.dvo = obj;
- action->reformat.reformat_size = data_sz;
  return 0;
  }
  case DR_ACTION_TYP_TNL_L2_TO_L2:
@@ -1720,6 +1778,14 @@ struct mlx5dv_dr_action *mlx5dv_dr_action_create_push_vlan(struct mlx5dv_dr_doma
  return action;
 }
 
+uint32_t dr_actions_reformat_get_id(struct mlx5dv_dr_action *action)
+{
+ if (action->reformat.chunk)
+ return action->reformat.index;
+
+ return action->reformat.dvo->object_id;
+}
+
 static int
 dr_action_modify_sw_to_hw_add(struct mlx5dv_dr_domain *dmn,
  __be64 *sw_action,
@@ -2442,12 +2508,25 @@ dr_action_convert_to_fte_dest(struct mlx5dv_dr_domain *dmn,
  }
 
  if (dest_reformat) {
+ int ret = 0;
+
  switch (dest_reformat->action_type) {
  case DR_ACTION_TYP_L2_TO_TNL_L2:
  case DR_ACTION_TYP_L2_TO_TNL_L3:
  if (dest_reformat->reformat.is_root_level)
  goto err_exit;
 
+ dr_domain_lock(dmn);
+ if (!dest_reformat->reformat.dvo) {
+ ret = dr_action_create_devx_reformat(dmn,
+ dest_reformat,
+ dest_reformat->reformat.reformat_size,
+ dest_reformat->reformat.data);
+ }
+ dr_domain_unlock(dmn);
+ if (ret)
+ goto err_exit;
+
  fte_attr->extended_dest = true;
  dest_info->has_reformat = true;
  dest_info->reformat_id = dest_reformat->reformat.dvo->object_id;
@@ -3013,10 +3092,15 @@ int mlx5dv_dr_action_destroy(struct mlx5dv_dr_action *action)
  break;
  case DR_ACTION_TYP_L2_TO_TNL_L2:
  case DR_ACTION_TYP_L2_TO_TNL_L3:
- if (action->reformat.is_root_level)
+ if (action->reformat.is_root_level) {
  mlx5_destroy_flow_action(action->reformat.flow_action);
- else
- mlx5dv_devx_obj_destroy(action->reformat.dvo);
+ } else {
+ if (action->reformat.chunk)
+ dr_action_destroy_sw_reformat(action);
+
+ if (action->reformat.dvo)
+ dr_action_destroy_devx_reformat(action);
+ }
  atomic_fetch_sub(&action->reformat.dmn->refcount, 1);
  break;
  case DR_ACTION_TYP_MODIFY_HDR:

providers/mlx5/dr_dbg.c

@@ -186,12 +186,12 @@ static int dr_dump_rule_action(FILE *f, const uint64_t rule_id,
  case DR_ACTION_TYP_L2_TO_TNL_L2:
  ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",0x%x\n",
  DR_DUMP_REC_TYPE_ACTION_ENCAP_L2, action_id,
- rule_id, action->reformat.dvo->object_id);
+ rule_id, dr_actions_reformat_get_id(action));
  break;
  case DR_ACTION_TYP_L2_TO_TNL_L3:
  ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",0x%x\n",
  DR_DUMP_REC_TYPE_ACTION_ENCAP_L3, action_id,
- rule_id, action->reformat.dvo->object_id);
+ rule_id, dr_actions_reformat_get_id(action));
  break;
  case DR_ACTION_TYP_METER:
  ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",0x%" PRIx64 ",0x%x,0x%" PRIx64 ",0x%" PRIx64 "\n",

providers/mlx5/dr_devx.c

@@ -328,6 +328,8 @@ int dr_devx_query_device(struct ibv_context *ctx, struct dr_devx_caps *caps)
  return err;
  }
 
+ caps->max_encap_size = DEVX_GET(query_hca_cap_out, out,
+ capability.flow_table_nic_cap.max_encap_header_size);
  caps->nic_rx_drop_address = DEVX_GET64(query_hca_cap_out, out,
  capability.flow_table_nic_cap.
  sw_steering_nic_rx_action_drop_icm_address);
@@ -437,6 +439,15 @@ int dr_devx_query_device(struct ibv_context *ctx, struct dr_devx_caps *caps)
  DEVX_GET64(query_hca_cap_out, out,
  capability.device_mem_cap.header_modify_pattern_sw_icm_start_address);
 
+ caps->log_sw_encap_icm_size =
+ DEVX_GET(query_hca_cap_out, out,
+ capability.device_mem_cap.log_indirect_encap_sw_icm_size);
+
+ if (caps->log_sw_encap_icm_size)
+ caps->indirect_encap_icm_base =
+ DEVX_GET64(query_hca_cap_out, out,
+ capability.device_mem_cap.indirect_encap_icm_base);
+
  /* RoCE caps */
  if (roce) {
  err = dr_devx_query_nic_vport_context(ctx, &caps->roce_caps.roce_en);

providers/mlx5/dr_domain.c

@@ -41,6 +41,34 @@ enum {
  MLX5DV_DR_DOMAIN_SYNC_FLAGS_MEM),
 };
 
+bool dr_domain_is_support_sw_encap(struct mlx5dv_dr_domain *dmn)
+{
+ return !!dmn->info.caps.log_sw_encap_icm_size;
+}
+
+static int dr_domain_init_sw_encap_resources(struct mlx5dv_dr_domain *dmn)
+{
+ if (!dr_domain_is_support_sw_encap(dmn))
+ return 0;
+
+ dmn->encap_icm_pool = dr_icm_pool_create(dmn, DR_ICM_TYPE_ENCAP);
+ if (!dmn->encap_icm_pool) {
+ dr_dbg(dmn, "Couldn't get sw-encap icm memory for %s\n",
+ ibv_get_device_name(dmn->ctx->device));
+ return errno;
+ }
+
+ return 0;
+}
+
+static void dr_domain_destroy_sw_encap_resources(struct mlx5dv_dr_domain *dmn)
+{
+ if (!dr_domain_is_support_sw_encap(dmn))
+ return;
+
+ dr_icm_pool_destroy(dmn->encap_icm_pool);
+}
+
 bool dr_domain_is_support_modify_hdr_cache(struct mlx5dv_dr_domain *dmn)
 {
  return dmn->info.caps.sw_format_ver >= MLX5_HW_CONNECTX_6DX &&
@@ -110,15 +138,24 @@ static int dr_domain_init_resources(struct mlx5dv_dr_domain *dmn)
  }
  }
 
+ ret = dr_domain_init_sw_encap_resources(dmn);
+ if (ret) {
+ dr_dbg(dmn, "Couldn't create sw-encap resource for %s\n",
+ ibv_get_device_name(dmn->ctx->device));
+ goto free_modify_header_ptrn_arg_mngr;
+ }
+
  ret = dr_send_ring_alloc(dmn);
  if (ret) {
  dr_dbg(dmn, "Couldn't create send-ring for %s\n",
  ibv_get_device_name(dmn->ctx->device));
- goto free_modify_header_ptrn_arg_mngr;
+ goto free_sw_encap_resources;
  }
 
  return 0;
 
+free_sw_encap_resources:
+ dr_domain_destroy_sw_encap_resources(dmn);
 free_modify_header_ptrn_arg_mngr:
  dr_ptrn_mngr_destroy(dmn->modify_header_ptrn_mngr);
  dr_arg_mngr_destroy(dmn->modify_header_arg_mngr);
@@ -136,6 +173,7 @@ static int dr_domain_init_resources(struct mlx5dv_dr_domain *dmn)
 static void dr_free_resources(struct mlx5dv_dr_domain *dmn)
 {
  dr_send_ring_free(dmn);
+ dr_domain_destroy_sw_encap_resources(dmn);
  dr_ptrn_mngr_destroy(dmn->modify_header_ptrn_mngr);
  dr_arg_mngr_destroy(dmn->modify_header_arg_mngr);
  dr_icm_pool_destroy(dmn->action_icm_pool);
@@ -446,6 +484,15 @@ static int dr_domain_check_icm_memory_caps(struct mlx5dv_dr_domain *dmn)
  dmn->info.max_log_modify_hdr_pattern_icm_sz = DR_CHUNK_SIZE_4K;
  }
 
+ if (dr_domain_is_support_sw_encap(dmn)) {
+ if (dmn->info.caps.log_sw_encap_icm_size <
+ (DR_CHUNK_SIZE_4K + DR_SW_ENCAP_ENTRY_LOG_SIZE)) {
+ errno = ENOMEM;
+ return errno;
+ }
+ dmn->info.max_log_sw_encap_icm_sz = DR_CHUNK_SIZE_4K;
+ }
+
  return 0;
 }
 
@@ -555,6 +602,12 @@ int mlx5dv_dr_domain_sync(struct mlx5dv_dr_domain *dmn, uint32_t flags)
  return ret;
  }
 
+ if (dmn->encap_icm_pool) {
+ ret = dr_icm_pool_sync_pool(dmn->encap_icm_pool);
+ if (ret)
+ return ret;
+ }
+
  if (dmn->action_icm_pool) {
  ret = dr_icm_pool_sync_pool(dmn->action_icm_pool);
  if (ret)

providers/mlx5/dr_icm_pool.c

@@ -84,6 +84,10 @@ dr_icm_allocate_aligned_dm(struct dr_icm_pool *pool,
  /* Align base is 64B */
  log_align_base = ilog32(DR_ICM_MODIFY_HDR_ALIGN_BASE - 1);
  break;
+ case DR_ICM_TYPE_ENCAP:
+ mlx5_dm_attr.type = MLX5_IB_UAPI_DM_TYPE_ENCAP_SW_ICM;
+ log_align_base = DR_SW_ENCAP_ENTRY_LOG_SIZE;
+ break;
  default:
  assert(false);
  errno = EINVAL;
@@ -577,6 +581,11 @@ struct dr_icm_pool *dr_icm_pool_create(struct mlx5dv_dr_domain *dmn,
  pool->th = dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
  pool->icm_type) / 2;
  break;
+ case DR_ICM_TYPE_ENCAP:
+ pool->max_log_chunk_sz = dmn->info.max_log_sw_encap_icm_sz;
+ pool->th = dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
+ pool->icm_type) / 2;
+ break;
  default:
  assert(false);
  }