libhns: Introduce hns direct verbs

Introduce the frame of hns direct verbs, including hnsdv_is_supported(),
hnsdv_create_qp() and hnsdv_query_device().

Signed-off-by: Junxian Huang <[email protected]>

debian/ibverbs-providers.install

@@ -1,6 +1,7 @@
 etc/libibverbs.d/
 usr/lib/*/libefa.so.*
 usr/lib/*/libibverbs/lib*-rdmav*.so
+usr/lib/*/libhns.so.*
 usr/lib/*/libmana.so.*
 usr/lib/*/libmlx4.so.*
 usr/lib/*/libmlx5.so.*

debian/ibverbs-providers.lintian-overrides

@@ -1,2 +1,2 @@
-# libefa, libmana, libmlx4 and libmlx5 are ibverbs provider that provides more functions.
-ibverbs-providers: package-name-doesnt-match-sonames libefa1 libmana1 libmlx4-1 libmlx5-1
+# libefa, libhns, libmana, libmlx4 and libmlx5 are ibverbs provider that provides more functions.
+ibverbs-providers: package-name-doesnt-match-sonames libefa1 libhns-1 libmana1 libmlx4-1 libmlx5-1

debian/ibverbs-providers.symbols

@@ -174,6 +174,12 @@ libefa.so.1 ibverbs-providers #MINVER#
  efadv_cq_from_ibv_cq_ex@EFA_1.2 43
  efadv_create_cq@EFA_1.2 43
  efadv_query_mr@EFA_1.3 50
+libhns.so.1 ibverbs-providers #MINVER#
+* Build-Depends-Package: libibverbs-dev
+ HNS_1.0@HNS_1.0 51
+ hnsdv_is_supported@HNS_1.0 51
+ hnsdv_create_qp@HNS_1.0 51
+ hnsdv_query_device@HNS_1.0 51
 libmana.so.1 ibverbs-providers #MINVER#
 * Build-Depends-Package: libibverbs-dev
  MANA_1.0@MANA_1.0 41

debian/libibverbs-dev.install

@@ -1,5 +1,6 @@
 usr/include/infiniband/arch.h
 usr/include/infiniband/efadv.h
+usr/include/infiniband/hnsdv.h
 usr/include/infiniband/ib_user_ioctl_verbs.h
 usr/include/infiniband/manadv.h
 usr/include/infiniband/mlx4dv.h
@@ -15,6 +16,8 @@ usr/include/infiniband/verbs_api.h
 usr/lib/*/lib*-rdmav*.a
 usr/lib/*/libefa.a
 usr/lib/*/libefa.so
+usr/lib/*/libhns.a
+usr/lib/*/libhns.so
 usr/lib/*/libibverbs*.so
 usr/lib/*/libibverbs.a
 usr/lib/*/libmana.a
@@ -24,6 +27,7 @@ usr/lib/*/libmlx4.so
 usr/lib/*/libmlx5.a
 usr/lib/*/libmlx5.so
 usr/lib/*/pkgconfig/libefa.pc
+usr/lib/*/pkgconfig/libhns.pc
 usr/lib/*/pkgconfig/libibverbs.pc
 usr/lib/*/pkgconfig/libmana.pc
 usr/lib/*/pkgconfig/libmlx4.pc

providers/hns/CMakeLists.txt

@@ -1,7 +1,14 @@
-rdma_provider(hns
+rdma_shared_provider(hns libhns.map
+ 1 1.0.${PACKAGE_VERSION}
  hns_roce_u.c
  hns_roce_u_buf.c
  hns_roce_u_db.c
  hns_roce_u_hw_v2.c
  hns_roce_u_verbs.c
 )
+
+publish_headers(infiniband
+ hnsdv.h
+)
+
+rdma_pkg_config("hns" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")

providers/hns/hns_roce_u.c

@@ -216,4 +216,17 @@ static const struct verbs_device_ops hns_roce_dev_ops = {
  .uninit_device = hns_uninit_device,
  .alloc_context = hns_roce_alloc_context,
 };
+
+bool is_hns_dev(struct ibv_device *device)
+{
+ struct verbs_device *verbs_device = verbs_get_device(device);
+
+ return verbs_device->ops == &hns_roce_dev_ops;
+}
+
+bool hnsdv_is_supported(struct ibv_device *device)
+{
+ return is_hns_dev(device);
+}
+
 PROVIDER_DRIVER(hns, hns_roce_dev_ops);

providers/hns/hns_roce_u.h

@@ -494,6 +494,8 @@ void hns_roce_free_qp_buf(struct hns_roce_qp *qp, struct hns_roce_context *ctx);
 
 void hns_roce_init_qp_indices(struct hns_roce_qp *qp);
 
+bool is_hns_dev(struct ibv_device *device);
+
 extern const struct hns_roce_u_hw hns_roce_u_hw_v2;
 
 #endif /* _HNS_ROCE_U_H */

providers/hns/hns_roce_u_abi.h

@@ -36,6 +36,7 @@
 #include <infiniband/kern-abi.h>
 #include <rdma/hns-abi.h>
 #include <kernel-abi/hns-abi.h>
+#include "hnsdv.h"
 
 DECLARE_DRV_CMD(hns_roce_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
  empty, hns_roce_ib_alloc_pd_resp);

providers/hns/hns_roce_u_verbs.c

@@ -785,6 +785,25 @@ int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq)
  return 0;
 }
 
+enum {
+ HNSDV_QP_SUP_COMP_MASK = 0,
+};
+
+static int check_hnsdv_qp_attr(struct hns_roce_context *ctx,
+ struct hnsdv_qp_init_attr *hns_attr)
+{
+ if (!hns_attr)
+ return 0;
+
+ if (!check_comp_mask(hns_attr->comp_mask, HNSDV_QP_SUP_COMP_MASK)) {
+ verbs_err(&ctx->ibv_ctx, "invalid hnsdv comp_mask 0x%x.\n",
+ hns_attr->comp_mask);
+ return EINVAL;
+ }
+
+ return 0;
+}
+
 enum {
  CREATE_QP_SUP_COMP_MASK = IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_XRCD |
  IBV_QP_INIT_ATTR_SEND_OPS_FLAGS,
@@ -866,14 +885,19 @@ static int verify_qp_create_cap(struct hns_roce_context *ctx,
 }
 
 static int verify_qp_create_attr(struct hns_roce_context *ctx,
- struct ibv_qp_init_attr_ex *attr)
+ struct ibv_qp_init_attr_ex *attr,
+ struct hnsdv_qp_init_attr *hns_attr)
 {
  int ret;
 
  ret = check_qp_create_mask(ctx, attr);
  if (ret)
  return ret;
 
+ ret = check_hnsdv_qp_attr(ctx, hns_attr);
+ if (ret)
+ return ret;
+
  return verify_qp_create_cap(ctx, attr);
 }
 
@@ -1274,14 +1298,15 @@ static int mmap_dwqe(struct ibv_context *ibv_ctx, struct hns_roce_qp *qp,
 }
 
 static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx,
- struct ibv_qp_init_attr_ex *attr)
+ struct ibv_qp_init_attr_ex *attr,
+ struct hnsdv_qp_init_attr *hns_attr)
 {
  struct hns_roce_context *context = to_hr_ctx(ibv_ctx);
  struct hns_roce_qp *qp;
  uint64_t dwqe_mmap_key;
  int ret;
 
- ret = verify_qp_create_attr(context, attr);
+ ret = verify_qp_create_attr(context, attr, hns_attr);
  if (ret)
  goto err;
 
@@ -1345,7 +1370,7 @@ struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd,
  attrx.comp_mask = IBV_QP_INIT_ATTR_PD;
  attrx.pd = pd;
 
- qp = create_qp(pd->context, &attrx);
+ qp = create_qp(pd->context, &attrx, NULL);
  if (qp)
  memcpy(attr, &attrx, sizeof(*attr));
 
@@ -1355,7 +1380,41 @@ struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd,
 struct ibv_qp *hns_roce_u_create_qp_ex(struct ibv_context *context,
  struct ibv_qp_init_attr_ex *attr)
 {
- return create_qp(context, attr);
+ return create_qp(context, attr, NULL);
+}
+
+struct ibv_qp *hnsdv_create_qp(struct ibv_context *context,
+ struct ibv_qp_init_attr_ex *qp_attr,
+ struct hnsdv_qp_init_attr *hns_attr)
+{
+ if (!context || !qp_attr) {
+ errno = EINVAL;
+ return NULL;
+ }
+
+ if (!is_hns_dev(context->device)) {
+ errno = EOPNOTSUPP;
+ return NULL;
+ }
+
+ return create_qp(context, qp_attr, hns_attr);
+}
+
+int hnsdv_query_device(struct ibv_context *context,
+ struct hnsdv_context *attrs_out)
+{
+ struct hns_roce_context *ctx = context ? to_hr_ctx(context) : NULL;
+
+ if (!ctx || !attrs_out)
+ return EINVAL;
+
+ if (!is_hns_dev(context->device)) {
+ verbs_err(verbs_get_ctx(context), "not a HNS RoCE device!\n");
+ return EOPNOTSUPP;
+ }
+ memset(attrs_out, 0, sizeof(*attrs_out));
+
+ return 0;
 }
 
 struct ibv_qp *hns_roce_u_open_qp(struct ibv_context *context,

providers/hns/hnsdv.h

@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright (c) 2024 Hisilicon Limited.
+ */
+
+#ifndef __HNSDV_H__
+#define __HNSDV_H__
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <sys/types.h>
+#include <infiniband/verbs.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct hnsdv_qp_init_attr {
+ uint64_t comp_mask;
+};
+
+struct hnsdv_context {
+ uint64_t comp_mask;
+};
+
+bool hnsdv_is_supported(struct ibv_device *device);
+int hnsdv_query_device(struct ibv_context *ctx_in,
+ struct hnsdv_context *attrs_out);
+struct ibv_qp *hnsdv_create_qp(struct ibv_context *context,
+ struct ibv_qp_init_attr_ex *qp_attr,
+ struct hnsdv_qp_init_attr *hns_qp_attr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __HNSDV_H__ */

providers/hns/libhns.map

@@ -0,0 +1,9 @@
+/* Export symbols should be added below according to
+ Documentation/versioning.md document. */
+HNS_1.0 {
+ global:
+ hnsdv_is_supported;
+ hnsdv_create_qp;
+ hnsdv_query_device;
+ local: *;
+};

redhat/rdma-core.spec

@@ -158,6 +158,8 @@ Provides: liberdma = %{version}-%{release}
 Obsoletes: liberdma < %{version}-%{release}
 Provides: libhfi1 = %{version}-%{release}
 Obsoletes: libhfi1 < %{version}-%{release}
+Provides: libhns = %{version}-%{release}
+Obsoletes: libhns < %{version}-%{release}
 Provides: libipathverbs = %{version}-%{release}
 Obsoletes: libipathverbs < %{version}-%{release}
 Provides: libirdma = %{version}-%{release}
@@ -188,7 +190,7 @@ Device-specific plug-in ibverbs userspace drivers are included:
 - libefa: Amazon Elastic Fabric Adapter
 - liberdma: Alibaba Elastic RDMA (iWarp) Adapter
 - libhfi1: Intel Omni-Path HFI
-- libhns: HiSilicon Hip06 SoC
+- libhns: HiSilicon Hip08+ SoC
 - libipathverbs: QLogic InfiniPath HCA
 - libirdma: Intel Ethernet Connection RDMA
 - libmana: Microsoft Azure Network Adapter
@@ -577,6 +579,7 @@ fi
 %dir %{_sysconfdir}/libibverbs.d
 %dir %{_libdir}/libibverbs
 %{_libdir}/libefa.so.*
+%{_libdir}/libhns.so.*
 %{_libdir}/libibverbs*.so.*
 %{_libdir}/libibverbs/*.so
 %{_libdir}/libmana.so.*

suse/rdma-core.spec

@@ -35,6 +35,7 @@ License: BSD-2-Clause OR GPL-2.0-only
 Group: Productivity/Networking/Other
 
 %define efa_so_major 1
+%define hns_so_major 1
 %define verbs_so_major 1
 %define rdmacm_so_major 1
 %define umad_so_major 3
@@ -45,6 +46,7 @@ Group: Productivity/Networking/Other
 %define mad_major 5
 
 %define efa_lname libefa%{efa_so_major}
+%define hns_lname libhns%{hns_so_major}
 %define verbs_lname libibverbs%{verbs_so_major}
 %define rdmacm_lname librdmacm%{rdmacm_so_major}
 %define umad_lname libibumad%{umad_so_major}
@@ -159,6 +161,7 @@ Requires: %{umad_lname} = %{version}-%{release}
 Requires: %{verbs_lname} = %{version}-%{release}
 %if 0%{?dma_coherent}
 Requires: %{efa_lname} = %{version}-%{release}
+Requires: %{hns_lname} = %{version}-%{release}
 Requires: %{mana_lname} = %{version}-%{release}
 Requires: %{mlx4_lname} = %{version}-%{release}
 Requires: %{mlx5_lname} = %{version}-%{release}
@@ -200,6 +203,7 @@ Requires: %{name}%{?_isa} = %{version}-%{release}
 Obsoletes: libcxgb4-rdmav2 < %{version}-%{release}
 Obsoletes: libefa-rdmav2 < %{version}-%{release}
 Obsoletes: libhfi1verbs-rdmav2 < %{version}-%{release}
+Obsoletes: libhns-rdmav2 < %{version}-%{release}
 Obsoletes: libipathverbs-rdmav2 < %{version}-%{release}
 Obsoletes: libmana-rdmav2 < %{version}-%{release}
 Obsoletes: libmlx4-rdmav2 < %{version}-%{release}
@@ -209,6 +213,7 @@ Obsoletes: libocrdma-rdmav2 < %{version}-%{release}
 Obsoletes: librxe-rdmav2 < %{version}-%{release}
 %if 0%{?dma_coherent}
 Requires: %{efa_lname} = %{version}-%{release}
+Requires: %{hns_lname} = %{version}-%{release}
 Requires: %{mana_lname} = %{version}-%{release}
 Requires: %{mlx4_lname} = %{version}-%{release}
 Requires: %{mlx5_lname} = %{version}-%{release}
@@ -228,7 +233,7 @@ Device-specific plug-in ibverbs userspace drivers are included:
 - libcxgb4: Chelsio T4 iWARP HCA
 - libefa: Amazon Elastic Fabric Adapter
 - libhfi1: Intel Omni-Path HFI
-- libhns: HiSilicon Hip06 SoC
+- libhns: HiSilicon Hip08+ SoC
 - libipathverbs: QLogic InfiniPath HCA
 - libirdma: Intel Ethernet Connection RDMA
 - libmana: Microsoft Azure Network Adapter
@@ -256,6 +261,13 @@ Group: System/Libraries
 %description -n %efa_lname
 This package contains the efa runtime library.
 
+%package -n %hns_lname
+Summary: HNS runtime library
+Group: System/Libraries
+
+%description -n %hns_lname
+This package contains the hns runtime library.
+
 %package -n %mana_lname
 Summary: MANA runtime library
 Group: System/Libraries
@@ -508,6 +520,9 @@ rm -rf %{buildroot}/%{_sbindir}/srp_daemon.sh
 %post -n %efa_lname -p /sbin/ldconfig
 %postun -n %efa_lname -p /sbin/ldconfig
 
+%post -n %hns_lname -p /sbin/ldconfig
+%postun -n %hns_lname -p /sbin/ldconfig
+
 %post -n %mana_lname -p /sbin/ldconfig
 %postun -n %mana_lname -p /sbin/ldconfig
 
@@ -702,6 +717,10 @@ done
 %files -n %efa_lname
 %{_libdir}/libefa*.so.*
 
+%files -n %hns_lname
+%defattr(-,root,root)
+%{_libdir}/libhns*.so.*
+
 %files -n %mana_lname
 %{_libdir}/libmana*.so.*