From: Ron Gredvig <ron.gredvig@hpe.com>
Date: Fri, 20 Oct 2023 19:46:48 +0000 (+0000)
Subject: LU-17837 kfilnd: Set dev_cpt
X-Git-Tag: 2.15.64~166
X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=d1fd0115a4af30356b812d7cb49dec6a76c4cb72;p=fs%2Flustre-release.git

LU-17837 kfilnd: Set dev_cpt

The dev_cpt value was not being set by kfilnd.

Query the kfabric provider to get the low level
device. Using the device, determine the dev_cpt.

This change is backwards compatible with older
versions of the kfabric provider. If the query
is not supported the dev_cpt is set to
CFS_CPT_ANY.

HPE-bug-id: LUS-11352
Test-Parameters: trivial
Signed-off-by: Ron Gredvig <ron.gredvig@hpe.com>
Signed-off-by: Chris Horn <chris.horn@hpe.com>
Change-Id: Id8af36b7aa5e89969de93dc8db9c0bba03236140
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/55068
Reviewed-by: Ian Ziemba <ian.ziemba@hpe.com>
Reviewed-by: Caleb Carlson <caleb.carlson@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
---

diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4
index 7498ac0..54a95a6 100644
--- a/lnet/autoconf/lustre-lnet.m4
+++ b/lnet/autoconf/lustre-lnet.m4
@@ -992,6 +992,21 @@ AC_ARG_WITH([kfi],
 		],[
 			AC_MSG_ERROR(["$with_kfi/Module.symvers does not exist"])
 		])
+		# at this point, we have kfilnd basic support,
+		# now check for extra features
+		LB_CHECK_COMPILE([if kfi_cxi domain ops are available],
+		KFI_CXI_dom_ops, [
+			#include <kfi_endpoint.h>
+			#include <kfi_cxi_ext.h>
+		],[
+			struct kfid *fid;
+			struct kfi_cxi_domain_ops *dom_ops;
+			kfi_open_ops(fid, KFI_CXI_DOM_OPS_1, 0,
+				(void **)&dom_ops, NULL);
+		],[
+			AC_DEFINE(HAVE_KFI_CXI_DOM_OPS, 1,
+				[kfi_cxi domain ops are available])
+		])
 	],[])
 AC_DEFINE(HAVE_KFILND, 1, [support kfabric LND])
 AC_SUBST(KFICPPFLAGS)
diff --git a/lnet/klnds/kfilnd/kfilnd.c b/lnet/klnds/kfilnd/kfilnd.c
index df57836..53bf5ca 100644
--- a/lnet/klnds/kfilnd/kfilnd.c
+++ b/lnet/klnds/kfilnd/kfilnd.c
@@ -445,6 +445,8 @@ static int kfilnd_startup(struct lnet_ni *ni)
 	const char *node;
 	int rc;
 	struct kfilnd_dev *kfdev;
+	int node_id;
+	int cpt = CFS_CPT_ANY;
 
 	if (!ni)
 		return -EINVAL;
@@ -477,6 +479,13 @@ static int kfilnd_startup(struct lnet_ni *ni)
 		goto err;
 	}
 
+	if (kfdev->device) {
+		node_id = dev_to_node(kfdev->device);
+		cpt = cfs_cpt_of_node(lnet_cpt_table(), node_id);
+	}
+
+	ni->ni_dev_cpt = cpt;
+
 	/* Post a series of immediate receive buffers */
 	rc = kfilnd_dev_post_imm_buffers(kfdev);
 	if (rc) {
diff --git a/lnet/klnds/kfilnd/kfilnd.h b/lnet/klnds/kfilnd/kfilnd.h
index 35e63ed..93c7ad2 100644
--- a/lnet/klnds/kfilnd/kfilnd.h
+++ b/lnet/klnds/kfilnd/kfilnd.h
@@ -449,6 +449,9 @@ struct kfilnd_dev {
 	/* Physical NIC address. */
 	unsigned int nic_addr;
 	atomic_t session_keys;
+
+	/* Physical device. */
+	struct device *device;
 };
 
 /* Invalid checksum value is treated as no checksum. */
diff --git a/lnet/klnds/kfilnd/kfilnd_dev.c b/lnet/klnds/kfilnd/kfilnd_dev.c
index 3f20036..2731c4c 100644
--- a/lnet/klnds/kfilnd/kfilnd_dev.c
+++ b/lnet/klnds/kfilnd/kfilnd_dev.c
@@ -135,6 +135,9 @@ struct kfilnd_dev *kfilnd_dev_alloc(struct lnet_ni *ni,
 	int cpt;
 	int lnet_ncpts;
 	struct kfilnd_dev *dev;
+#ifdef HAVE_KFI_CXI_DOM_OPS
+	struct kfi_cxi_domain_ops *dom_ops;
+#endif
 
 	if (!ni) {
 		rc = -EINVAL;
@@ -177,6 +180,19 @@ struct kfilnd_dev *kfilnd_dev_alloc(struct lnet_ni *ni,
 
 	dev->nic_addr = ((struct kcxi_addr *)dev_info->src_addr)->nic;
 
+	/* Get the device struct */
+	dev->device = NULL;
+#ifdef HAVE_KFI_CXI_DOM_OPS
+	rc = kfi_open_ops(&dev->dom->domain->fid, KFI_CXI_DOM_OPS_1, 0,
+				(void **)&dom_ops, NULL);
+	if (!rc) {
+		rc = dom_ops->get_device(&dev->dom->domain->fid,
+					   &dev->device);
+		if (!rc)
+			CDEBUG(D_NET, "get_device failed\n");
+	}
+#endif
+
 	/* Create an AV for this device */
 	av_attr.type = KFI_AV_UNSPEC;
 	av_attr.rx_ctx_bits = KFILND_FAB_RX_CTX_BITS;