LU-17837 kfilnd: Set dev_cpt

The dev_cpt value was not being set by kfilnd.

Query the kfabric provider to get the low level
device. Using the device, determine the dev_cpt.

This change is backwards compatible with older
versions of the kfabric provider. If the query
is not supported the dev_cpt is set to
CFS_CPT_ANY.

HPE-bug-id: LUS-11352
Test-Parameters: trivial
Signed-off-by: Ron Gredvig <ron.gredvig@hpe.com>
Signed-off-by: Chris Horn <chris.horn@hpe.com>
Change-Id: Id8af36b7aa5e89969de93dc8db9c0bba03236140
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/55068
Reviewed-by: Ian Ziemba <ian.ziemba@hpe.com>
Reviewed-by: Caleb Carlson <caleb.carlson@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4
index 7498ac0..54a95a6 100644 (file)
--- a/lnet/autoconf/lustre-lnet.m4
+++ b/lnet/autoconf/lustre-lnet.m4
@@ -992,6 +992,21 @@ AC_ARG_WITH([kfi],
                ],[
                        AC_MSG_ERROR(["$with_kfi/Module.symvers does not exist"])
                ])
+               # at this point, we have kfilnd basic support,
+               # now check for extra features
+               LB_CHECK_COMPILE([if kfi_cxi domain ops are available],
+               KFI_CXI_dom_ops, [
+                       #include <kfi_endpoint.h>
+                       #include <kfi_cxi_ext.h>
+               ],[
+                       struct kfid *fid;
+                       struct kfi_cxi_domain_ops *dom_ops;
+                       kfi_open_ops(fid, KFI_CXI_DOM_OPS_1, 0,
+                               (void **)&dom_ops, NULL);
+               ],[
+                       AC_DEFINE(HAVE_KFI_CXI_DOM_OPS, 1,
+                               [kfi_cxi domain ops are available])
+               ])
        ],[])
 AC_DEFINE(HAVE_KFILND, 1, [support kfabric LND])
 AC_SUBST(KFICPPFLAGS)

diff --git a/lnet/klnds/kfilnd/kfilnd.c b/lnet/klnds/kfilnd/kfilnd.c
index df57836..53bf5ca 100644 (file)
--- a/lnet/klnds/kfilnd/kfilnd.c
+++ b/lnet/klnds/kfilnd/kfilnd.c
@@ -445,6 +445,8 @@ static int kfilnd_startup(struct lnet_ni *ni)
        const char *node;
        int rc;
        struct kfilnd_dev *kfdev;
+       int node_id;
+       int cpt = CFS_CPT_ANY;
 
        if (!ni)
                return -EINVAL;
@@ -477,6 +479,13 @@ static int kfilnd_startup(struct lnet_ni *ni)
                goto err;
        }
 
+       if (kfdev->device) {
+               node_id = dev_to_node(kfdev->device);
+               cpt = cfs_cpt_of_node(lnet_cpt_table(), node_id);
+       }
+
+       ni->ni_dev_cpt = cpt;
+
        /* Post a series of immediate receive buffers */
        rc = kfilnd_dev_post_imm_buffers(kfdev);
        if (rc) {

diff --git a/lnet/klnds/kfilnd/kfilnd.h b/lnet/klnds/kfilnd/kfilnd.h
index 35e63ed..93c7ad2 100644 (file)
--- a/lnet/klnds/kfilnd/kfilnd.h
+++ b/lnet/klnds/kfilnd/kfilnd.h
@@ -449,6 +449,9 @@ struct kfilnd_dev {
        /* Physical NIC address. */
        unsigned int nic_addr;
        atomic_t session_keys;
+
+       /* Physical device. */
+       struct device *device;
 };
 
 /* Invalid checksum value is treated as no checksum. */

diff --git a/lnet/klnds/kfilnd/kfilnd_dev.c b/lnet/klnds/kfilnd/kfilnd_dev.c
index 3f20036..2731c4c 100644 (file)
--- a/lnet/klnds/kfilnd/kfilnd_dev.c
+++ b/lnet/klnds/kfilnd/kfilnd_dev.c
@@ -135,6 +135,9 @@ struct kfilnd_dev *kfilnd_dev_alloc(struct lnet_ni *ni,
        int cpt;
        int lnet_ncpts;
        struct kfilnd_dev *dev;
+#ifdef HAVE_KFI_CXI_DOM_OPS
+       struct kfi_cxi_domain_ops *dom_ops;
+#endif
 
        if (!ni) {
                rc = -EINVAL;
@@ -177,6 +180,19 @@ struct kfilnd_dev *kfilnd_dev_alloc(struct lnet_ni *ni,
 
        dev->nic_addr = ((struct kcxi_addr *)dev_info->src_addr)->nic;
 
+       /* Get the device struct */
+       dev->device = NULL;
+#ifdef HAVE_KFI_CXI_DOM_OPS
+       rc = kfi_open_ops(&dev->dom->domain->fid, KFI_CXI_DOM_OPS_1, 0,
+                               (void **)&dom_ops, NULL);
+       if (!rc) {
+               rc = dom_ops->get_device(&dev->dom->domain->fid,
+                                          &dev->device);
+               if (!rc)
+                       CDEBUG(D_NET, "get_device failed\n");
+       }
+#endif
+
        /* Create an AV for this device */
        av_attr.type = KFI_AV_UNSPEC;
        av_attr.rx_ctx_bits = KFILND_FAB_RX_CTX_BITS;