test_parent_domain.py

# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file
"""
Test module for Pyverbs' ParentDomain.
"""
from pyverbs.pd import ParentDomainInitAttr, ParentDomain, ParentDomainContext
from tests.base import RCResources, UDResources, RDMATestCase
from pyverbs.pyverbs_error import PyverbsRDMAError
from pyverbs.cq import CqInitAttrEx, CQEX
import pyverbs.mem_alloc as mem
import pyverbs.enums as e
import tests.utils as u

import unittest
import errno


HUGE_PAGE_SIZE = 0x200000


def default_allocator(pd, context, size, alignment, resource_type):
    return e._IBV_ALLOCATOR_USE_DEFAULT


def default_free(pd, context, ptr, resource_type):
    return e._IBV_ALLOCATOR_USE_DEFAULT


def mem_align_allocator(pd, context, size, alignment, resource_type):
    p = mem.posix_memalign(size, alignment)
    return p


def free_func(pd, context, ptr, resource_type):
    mem.free(ptr)


def huge_page_alloc(pd, context, size, alignment, resource_type):
    ptr = context.user_data
    remainder = ptr % alignment
    ptr += 0 if remainder == 0 else (alignment - remainder)
    context.user_data += size
    return ptr


def huge_page_free(pd, context, ptr, resource_type):
    """
    No need to free memory, since this allocator assumes the huge page was
    externally mapped (and will be externally un-mapped).
    """
    pass


def create_parent_domain_with_allocators(res):
    """
    Creates parent domain for res instance. The allocators themselves are taken
    from res.allocator_func and res.free_func.
    :param res: The resources instance to work on (an instance of BaseResources)
    """
    if res.allocator_func and res.free_func:
        res.pd_ctx = ParentDomainContext(res.pd, res.allocator_func,
                                         res.free_func, res.user_data)
    pd_attr = ParentDomainInitAttr(pd=res.pd, pd_context=res.pd_ctx)
    try:
        res.pd = ParentDomain(res.ctx, attr=pd_attr)
    except PyverbsRDMAError as ex:
        if ex.error_code == errno.EOPNOTSUPP:
            raise unittest.SkipTest('Parent Domain is not supported on this device')
        raise ex


def parent_domain_res_cls(base_class):
    """
    This is a factory function which creates a class that inherits base_class of
    any BaseResources type. Its purpose is to behave exactly as base_class does,
    except for creating a parent domain with custom allocators.
    Hence the returned class must be initialized with (alloc_func, free_func,
    user_data, **kwargs), while kwargs are the arguments needed (if any) for
    base_class.
    :param base_class: The base resources class to inherit from
    :return: ParentDomainRes(alloc_func=None, free_func=None, **kwargs) class
    """
    class ParentDomainRes(base_class):
        def __init__(self, alloc_func=None, free_func=None, user_data=None, **kwargs):
            self.pd_ctx = None
            self.protection_domain = None
            self.allocator_func = alloc_func
            self.free_func = free_func
            self.user_data = user_data
            super().__init__(**kwargs)

        def create_pd(self):
            super().create_pd()
            self.protection_domain = self.pd
            create_parent_domain_with_allocators(self)

    return ParentDomainRes


class ParentDomainHugePageRcRes(parent_domain_res_cls(RCResources)):
    def __init__(self, alloc_func=None, free_func=None, **kwargs):
        user_data = mem.mmap(length=HUGE_PAGE_SIZE,
                             flags=mem.MAP_ANONYMOUS_ | mem.MAP_PRIVATE_ | mem.MAP_HUGETLB_)
        super().__init__(alloc_func=alloc_func, free_func=free_func,
                         user_data=user_data, **kwargs)

    def __del__(self):
        mem.munmap(self.user_data, HUGE_PAGE_SIZE)


class ParentDomainCqExSrqRes(parent_domain_res_cls(RCResources)):
    """
    Parent domain resources. Based on RCResources.
    This includes a parent domain created with the given allocators, in addition
    it creates an extended CQ and a SRQ for RC traffic.
    :param dev_name: Device name to be used
    :param ib_port: IB port of the device to use
    :param gid_index: Which GID index to use
    :param alloc_func: Custom allocator function
    :param free_func: Custom free function
    """
    def __init__(self, dev_name, ib_port=None, gid_index=None, alloc_func=None,
                 free_func=None):
        super().__init__(dev_name=dev_name, ib_port=ib_port,
                         gid_index=gid_index, alloc_func=alloc_func,
                         free_func=free_func, with_srq=True)

    def create_cq(self):
        wc_flags = e.IBV_WC_STANDARD_FLAGS
        cia = CqInitAttrEx(cqe=2000, wc_flags=wc_flags, parent_domain=self.pd,
                           comp_mask=e.IBV_CQ_INIT_ATTR_MASK_FLAGS |
                                     e.IBV_CQ_INIT_ATTR_MASK_PD)
        try:
            self.cq = CQEX(self.ctx, cia)
        except PyverbsRDMAError as ex:
            if ex.error_code == errno.EOPNOTSUPP:
                raise unittest.SkipTest('Extended CQ with Parent Domain is not supported')
            raise ex


class ParentDomainTrafficTest(RDMATestCase):
    def setUp(self):
        super().setUp()
        self.iters = 10
        self.server = None
        self.client = None

    def test_without_allocators_rc_traffic(self):
        parent_domain_rc_res = parent_domain_res_cls(RCResources)
        self.create_players(parent_domain_rc_res)
        u.traffic(**self.traffic_args)

    def test_default_allocators_rc_traffic(self):
        parent_domain_rc_res = parent_domain_res_cls(RCResources)
        self.create_players(parent_domain_rc_res, alloc_func=default_allocator,
                            free_func=default_free)
        u.traffic(**self.traffic_args)

    def test_mem_align_rc_traffic(self):
        parent_domain_rc_res = parent_domain_res_cls(RCResources)
        self.create_players(parent_domain_rc_res,
                            alloc_func=mem_align_allocator, free_func=free_func)
        u.traffic(**self.traffic_args)

    def test_mem_align_ud_traffic(self):
        parent_domain_ud_res = parent_domain_res_cls(UDResources)
        self.create_players(parent_domain_ud_res,
                            alloc_func=mem_align_allocator, free_func=free_func)
        u.traffic(**self.traffic_args)

    def test_mem_align_srq_excq_rc_traffic(self):
        self.create_players(ParentDomainCqExSrqRes,
                            alloc_func=mem_align_allocator, free_func=free_func)
        u.traffic(**self.traffic_args, is_cq_ex=True)

    @u.requires_huge_pages()
    def test_huge_page_traffic(self):
        self.create_players(ParentDomainHugePageRcRes,
                            alloc_func=huge_page_alloc, free_func=huge_page_free)
        u.traffic(**self.traffic_args)