From: Chris Horn <chris.horn@hpe.com>
Date: Mon, 1 May 2023 19:25:47 +0000 (-0600)
Subject: LU-16797 lnet: improve numeric NID to CPT hashing
X-Git-Tag: 2.15.56~58
X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=bd12731f2f15b390345a3b58947325885346b072;p=fs%2Flustre-release.git

LU-16797 lnet: improve numeric NID to CPT hashing

Testing shows that the sum-by-multiplication of nid bytes method for
hashing a NID to a CPT does not have good distribution for gnilnd
and kfilnd NIDs. For these LNDs, use the old hash_long() method.

Also modify lustre_lnet_calc_cpt_of_nid() to ensure the specified
number of CPTs is greater than 0. This avoids a divide by zero.

Test-Parameters: trivial
HPE-bug-id: LUS-11576
Signed-off-by: Chris Horn <chris.horn@hpe.com>
Change-Id: I77c6a5c3bc6b773e97f0a3c87182c31674d284e0
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/50862
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
---

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index 36fa905..dfffd17 100644
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -1610,20 +1610,31 @@ static unsigned int
 lnet_nid4_cpt_hash(lnet_nid_t nid, unsigned int number)
 {
 	__u64 key = nid;
-	__u64 pair_bits = 0x0001000100010001LLU;
-	__u64 mask = pair_bits * 0xFF;
-	__u64 pair_sum;
+	__u16 lnd = LNET_NETTYP(LNET_NIDNET(nid));
+	unsigned int cpt;
 
-	/* Use (sum-by-multiplication of nid bytes) mod (number of CPTs)
-	 * to match nid to a CPT.
-	 */
-	pair_sum = (key & mask) + ((key >> 8) & mask);
-	pair_sum = (pair_sum * pair_bits) >> 48;
+	if (lnd == KFILND || lnd == GNILND) {
+		cpt = hash_long(key, LNET_CPT_BITS);
+
+		/* NB: The number of CPTs needn't be a power of 2 */
+		if (cpt >= number)
+			cpt = (key + cpt + (cpt >> 1)) % number;
+	} else {
+		__u64 pair_bits = 0x0001000100010001LLU;
+		__u64 mask = pair_bits * 0xFF;
+		__u64 pair_sum;
+		/* For ipv4 NIDs, use (sum-by-multiplication of nid bytes) mod
+		 * (number of CPTs) to match nid to a CPT.
+		 */
+		pair_sum = (key & mask) + ((key >> 8) & mask);
+		pair_sum = (pair_sum * pair_bits) >> 48;
+		cpt = (unsigned int)(pair_sum) % number;
+	}
 
 	CDEBUG(D_NET, "Match nid %s to cpt %u\n",
-	       libcfs_nid2str(nid), (unsigned int)(pair_sum) % number);
+	       libcfs_nid2str(nid), cpt);
 
-	return (unsigned int)(pair_sum) % number;
+	return cpt;
 }
 
 unsigned int
diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c
index b198a6d..dc29ac8 100644
--- a/lnet/utils/lnetconfig/liblnetconfig.c
+++ b/lnet/utils/lnetconfig/liblnetconfig.c
@@ -44,6 +44,7 @@
 #include <sys/ioctl.h>
 #include <net/if.h>
 #include <libcfs/util/ioctl.h>
+#include <libcfs/util/hash.h>
 #include <linux/lnet/lnetctl.h>
 #include "liblnd.h"
 #include <sys/types.h>
@@ -3872,26 +3873,41 @@ out:
 }
 
 unsigned int
-lnet_nid_cpt_hash(lnet_nid_t nid, long int number)
+lnet_nid_cpt_hash(lnet_nid_t nid, long int number, unsigned int cpt_bits)
 {
 	__u64 key = nid;
-	__u64 pair_bits = 0x0001000100010001LLU;
-	__u64 mask = pair_bits * 0xFF;
-	__u64 pair_sum;
+	__u16 lnd = LNET_NETTYP(LNET_NIDNET(nid));
+	unsigned int cpt;
 
 	if (number == 1)
 		return 0;
 
-	pair_sum = (key & mask) + ((key >> 8) & mask);
-	pair_sum = (pair_sum * pair_bits) >> 48;
+	if (lnd == KFILND || lnd == GNILND) {
+		cpt = hash_long(key, cpt_bits);
 
-	return (unsigned int)(pair_sum) % number;
+		/* NB: The number of CPTs needn't be a power of 2 */
+		if (cpt >= number)
+			cpt = (key + cpt + (cpt >> 1)) % number;
+	} else {
+		__u64 pair_bits = 0x0001000100010001LLU;
+		__u64 mask = pair_bits * 0xFF;
+		__u64 pair_sum;
+		/* For ipv4 NIDs, use (sum-by-multiplication of nid bytes) mod
+		 * (number of CPTs) to match nid to a CPT.
+		 */
+		pair_sum = (key & mask) + ((key >> 8) & mask);
+		pair_sum = (pair_sum * pair_bits) >> 48;
+		cpt = (unsigned int)(pair_sum) % number;
+	}
+
+	return cpt;
 }
 
 int lustre_lnet_calc_cpt_of_nid(char *nidc, long int ncpts)
 {
 	int rc = LUSTRE_CFG_RC_BAD_PARAM;
 	lnet_nid_t nid;
+	unsigned int cpt_bits;
 
 	if (!nidc) {
 		fprintf(stderr, "error:\n    msg: \"no NID provided\"\n");
@@ -3903,6 +3919,15 @@ int lustre_lnet_calc_cpt_of_nid(char *nidc, long int ncpts)
 		return rc;
 	}
 
+	if (ncpts < 1) {
+		fprintf(stderr, "error:\n    msg: \"number of CPTs must be >= 1\"\n");
+		return rc;
+	}
+
+	cpt_bits = 1;
+	while ((1 << cpt_bits) < ncpts)
+		cpt_bits++;
+
 	nid = libcfs_str2nid(nidc);
 	if (nid == LNET_NID_ANY) {
 		fprintf(stderr, "error:\n    msg: \"bad NID provided %s\"\n",
@@ -3910,7 +3935,7 @@ int lustre_lnet_calc_cpt_of_nid(char *nidc, long int ncpts)
 		return rc;
 	}
 
-	return (int)lnet_nid_cpt_hash(nid, ncpts);
+	return (int)lnet_nid_cpt_hash(nid, ncpts, cpt_bits);
 }
 
 int show_recovery_queue(enum lnet_health_type type, char *name, int seq_no,