From: Alexey Lyashkov <c17817@cray.com>
Date: Fri, 20 Dec 2019 12:40:37 +0000 (+0300)
Subject: LU-13036 lnet: avoid extra memory consumption
X-Git-Tag: 2.13.52~51
X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=refs%2Fchanges%2F97%2F36897%2F3;p=fs%2Flustre-release.git

LU-13036 lnet: avoid extra memory consumption

use slab allocation for the rsp_tracker and lnet_message
structs to avoid memory fragmnetation.

Test-parameters: trivial

Cray-bug-id: LUS-8190
Signed-off-by: Alexey Lyashkov <c17817@cray.com>
Change-Id: I67ec8f8fe4da4c646241d551e0a23745cae8ed00
Reviewed-on: https://review.whamcloud.com/36897
Reviewed-by: Alexandr Boyko <c17825@cray.com>
Reviewed-by: Alexander Zarochentsev <c17826@cray.com>
Reviewed-by: Chris Horn <hornc@cray.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Neil Brown <neilb@suse.de>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
---

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index 62ccd52..f0c551e 100644
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -194,6 +194,8 @@ lnet_net_lock_current(void)
 extern struct kmem_cache *lnet_mes_cachep;	 /* MEs kmem_cache */
 extern struct kmem_cache *lnet_small_mds_cachep; /* <= LNET_SMALL_MD_SIZE bytes
 						  * MDs kmem_cache */
+extern struct kmem_cache *lnet_rspt_cachep;
+extern struct kmem_cache *lnet_msg_cachep;
 
 static inline struct lnet_eq *
 lnet_eq_alloc (void)
@@ -462,9 +464,8 @@ lnet_msg_alloc(void)
 {
 	struct lnet_msg *msg;
 
-	LIBCFS_ALLOC(msg, sizeof(*msg));
+	msg = kmem_cache_alloc(lnet_msg_cachep, GFP_NOFS | __GFP_ZERO);
 
-	/* no need to zero, LIBCFS_ALLOC does for us */
 	return (msg);
 }
 
@@ -472,26 +473,30 @@ static inline void
 lnet_msg_free(struct lnet_msg *msg)
 {
 	LASSERT(!msg->msg_onactivelist);
-	LIBCFS_FREE(msg, sizeof(*msg));
+	kmem_cache_free(lnet_msg_cachep, msg);
 }
 
 static inline struct lnet_rsp_tracker *
 lnet_rspt_alloc(int cpt)
 {
 	struct lnet_rsp_tracker *rspt;
-	LIBCFS_ALLOC(rspt, sizeof(*rspt));
+
+	rspt = kmem_cache_alloc(lnet_rspt_cachep, GFP_NOFS | __GFP_ZERO);
 	if (rspt) {
 		lnet_net_lock(cpt);
 		the_lnet.ln_counters[cpt]->lct_health.lch_rst_alloc++;
 		lnet_net_unlock(cpt);
 	}
+	CDEBUG(D_MALLOC, "rspt alloc %p\n", rspt);
 	return rspt;
 }
 
 static inline void
 lnet_rspt_free(struct lnet_rsp_tracker *rspt, int cpt)
 {
-	LIBCFS_FREE(rspt, sizeof(*rspt));
+	CDEBUG(D_MALLOC, "rspt free %p\n", rspt);
+
+	kmem_cache_free(lnet_rspt_cachep, rspt);
 	lnet_net_lock(cpt);
 	the_lnet.ln_counters[cpt]->lct_health.lch_rst_alloc--;
 	lnet_net_unlock(cpt);
diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index 485ef06..a3a7e75 100644
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -553,9 +553,11 @@ lnet_init_locks(void)
 struct kmem_cache *lnet_mes_cachep;	   /* MEs kmem_cache */
 struct kmem_cache *lnet_small_mds_cachep;  /* <= LNET_SMALL_MD_SIZE bytes
 					    *  MDs kmem_cache */
+struct kmem_cache *lnet_rspt_cachep;	   /* response tracker cache */
+struct kmem_cache *lnet_msg_cachep;
 
 static int
-lnet_descriptor_setup(void)
+lnet_slab_setup(void)
 {
 	/* create specific kmem_cache for MEs and small MDs (i.e., originally
 	 * allocated in <size-xxx> kmem_cache).
@@ -571,12 +573,32 @@ lnet_descriptor_setup(void)
 	if (!lnet_small_mds_cachep)
 		return -ENOMEM;
 
+	lnet_rspt_cachep = kmem_cache_create("lnet_rspt", sizeof(struct lnet_rsp_tracker),
+					    0, 0, NULL);
+	if (!lnet_rspt_cachep)
+		return -ENOMEM;
+
+	lnet_msg_cachep = kmem_cache_create("lnet_msg", sizeof(struct lnet_msg),
+					    0, 0, NULL);
+	if (!lnet_msg_cachep)
+		return -ENOMEM;
+
 	return 0;
 }
 
 static void
-lnet_descriptor_cleanup(void)
+lnet_slab_cleanup(void)
 {
+	if (lnet_msg_cachep) {
+		kmem_cache_destroy(lnet_msg_cachep);
+		lnet_msg_cachep = NULL;
+	}
+
+
+	if (lnet_rspt_cachep) {
+		kmem_cache_destroy(lnet_rspt_cachep);
+		lnet_rspt_cachep = NULL;
+	}
 
 	if (lnet_small_mds_cachep) {
 		kmem_cache_destroy(lnet_small_mds_cachep);
@@ -1148,7 +1170,7 @@ lnet_prepare(lnet_pid_t requested_pid)
 	LNetInvalidateEQHandle(&the_lnet.ln_mt_eqh);
 	init_completion(&the_lnet.ln_started);
 
-	rc = lnet_descriptor_setup();
+	rc = lnet_slab_setup();
 	if (rc != 0)
 		goto failed;
 
@@ -1255,7 +1277,7 @@ lnet_unprepare (void)
 		the_lnet.ln_counters = NULL;
 	}
 	lnet_destroy_remote_nets_table();
-	lnet_descriptor_cleanup();
+	lnet_slab_cleanup();
 
 	return 0;
 }