LU-4330 lnet: Allocate MEs and small MDs in own kmem_caches

As part of LU-3848 and LU-4330, it has been discovered that LNET
MEs and small MDs (<=128 Bytes) are allocated in <size-128> kmem_cache
and thus can suffer quite frequent corruptions, from other modules or
Kernel parts, that occur there. To avoid this, MEs and small-MDs
specific kmem_cache have been created.

Also, lnet_libmd struct fields have been re-ordered to optimize its
memory foot-print.

Signed-off-by: Bruno Faccini <bruno.faccini@intel.com>
Change-Id: I6cfbed0c3f0a4c2447fe22b2d0d30dba8e3f3173
Reviewed-on: http://review.whamcloud.com/18586
Tested-by: Jenkins
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index b36f25d..9bf2cd8 100644 (file)
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -182,6 +182,11 @@ lnet_net_lock_current(void)
 
 #define MAX_PORTALS     64
 
 
 #define MAX_PORTALS     64
 

+#define LNET_SMALL_MD_SIZE   offsetof(lnet_libmd_t, md_iov.iov[1])
+extern struct kmem_cache *lnet_mes_cachep;      /* MEs kmem_cache */
+extern struct kmem_cache *lnet_small_mds_cachep; /* <= LNET_SMALL_MD_SIZE bytes
+                                                 * MDs kmem_cache */
+

 static inline lnet_eq_t *
 lnet_eq_alloc (void)
 {
 static inline lnet_eq_t *
 lnet_eq_alloc (void)
 {


@@ -213,7 +218,20 @@ lnet_md_alloc (lnet_md_t *umd)
                 size = offsetof(lnet_libmd_t, md_iov.iov[niov]);
         }
 
                 size = offsetof(lnet_libmd_t, md_iov.iov[niov]);
         }
 

-        LIBCFS_ALLOC(md, size);
+       if (size <= LNET_SMALL_MD_SIZE) {
+               md = kmem_cache_alloc(lnet_small_mds_cachep,
+                                     GFP_NOFS | __GFP_ZERO);
+               if (md) {
+                       CDEBUG(D_MALLOC, "slab-alloced 'md' of size %u at "
+                              "%p.\n", size, md);
+               } else {
+                       CDEBUG(D_MALLOC, "failed to allocate 'md' of size %u\n",
+                              size);
+                       return NULL;
+               }
+       } else {
+               LIBCFS_ALLOC(md, size);
+       }

 
        if (md != NULL) {
                /* Set here in case of early free */
 
        if (md != NULL) {
                /* Set here in case of early free */


@@ -235,22 +253,34 @@ lnet_md_free(lnet_libmd_t *md)
        else
                size = offsetof(lnet_libmd_t, md_iov.iov[md->md_niov]);
 
        else
                size = offsetof(lnet_libmd_t, md_iov.iov[md->md_niov]);
 

-       LIBCFS_FREE(md, size);
+       if (size <= LNET_SMALL_MD_SIZE) {
+               CDEBUG(D_MALLOC, "slab-freed 'md' at %p.\n", md);
+               kmem_cache_free(lnet_small_mds_cachep, md);
+       } else {
+               LIBCFS_FREE(md, size);
+       }

 }
 
 static inline lnet_me_t *
 lnet_me_alloc (void)
 {
 }
 
 static inline lnet_me_t *
 lnet_me_alloc (void)
 {

-        lnet_me_t *me;
+       lnet_me_t *me;
+
+       me = kmem_cache_alloc(lnet_mes_cachep, GFP_NOFS | __GFP_ZERO);
+
+       if (me)
+               CDEBUG(D_MALLOC, "slab-alloced 'me' at %p.\n", me);
+       else
+               CDEBUG(D_MALLOC, "failed to allocate 'me'\n");

 
 

-        LIBCFS_ALLOC(me, sizeof(*me));
-        return (me);
+       return me;

 }
 
 static inline void
 lnet_me_free(lnet_me_t *me)
 {
 }
 
 static inline void
 lnet_me_free(lnet_me_t *me)
 {

-       LIBCFS_FREE(me, sizeof(*me));
+       CDEBUG(D_MALLOC, "slab-freed 'me' at %p.\n", me);
+       kmem_cache_free(lnet_mes_cachep, me);

 }
 
 static inline lnet_msg_t *
 }
 
 static inline lnet_msg_t *

diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h
index c61e669..1a1b090 100644 (file)
--- a/lnet/include/lnet/lib-types.h
+++ b/lnet/include/lnet/lib-types.h
@@ -165,9 +165,9 @@ typedef struct lnet_libmd {
        int                     md_refcount;
        unsigned int            md_options;
        unsigned int            md_flags;
        int                     md_refcount;
        unsigned int            md_options;
        unsigned int            md_flags;

+       unsigned int            md_niov;        /* # frags at end of struct */

        void                   *md_user_ptr;
        lnet_eq_t              *md_eq;
        void                   *md_user_ptr;
        lnet_eq_t              *md_eq;

-       unsigned int            md_niov;        /* # frags */

        union {
                struct kvec     iov[LNET_MAX_IOV];
                lnet_kiov_t     kiov[LNET_MAX_IOV];
        union {
                struct kvec     iov[LNET_MAX_IOV];
                lnet_kiov_t     kiov[LNET_MAX_IOV];

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index 4a4c933..ec777fe 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -106,6 +106,45 @@ lnet_fini_locks(void)
 {
 }
 
 {
 }
 

+struct kmem_cache *lnet_mes_cachep;       /* MEs kmem_cache */
+struct kmem_cache *lnet_small_mds_cachep;  /* <= LNET_SMALL_MD_SIZE bytes
+                                           *  MDs kmem_cache */
+
+static int
+lnet_descriptor_setup(void)
+{
+       /* create specific kmem_cache for MEs and small MDs (i.e., originally
+        * allocated in <size-xxx> kmem_cache).
+        */
+       lnet_mes_cachep = kmem_cache_create("lnet_MEs", sizeof(lnet_me_t),
+                                           0, 0, NULL);
+       if (!lnet_mes_cachep)
+               return -ENOMEM;
+
+       lnet_small_mds_cachep = kmem_cache_create("lnet_small_MDs",
+                                                 LNET_SMALL_MD_SIZE, 0, 0,
+                                                 NULL);
+       if (!lnet_small_mds_cachep)
+               return -ENOMEM;
+
+       return 0;
+}
+
+static void
+lnet_descriptor_cleanup(void)
+{
+
+       if (lnet_small_mds_cachep) {
+               kmem_cache_destroy(lnet_small_mds_cachep);
+               lnet_small_mds_cachep = NULL;
+       }
+
+       if (lnet_mes_cachep) {
+               kmem_cache_destroy(lnet_mes_cachep);
+               lnet_mes_cachep = NULL;
+       }
+}
+

 static int
 lnet_create_remote_nets_table(void)
 {
 static int
 lnet_create_remote_nets_table(void)
 {


@@ -569,6 +608,10 @@ lnet_prepare(lnet_pid_t requested_pid)
        INIT_LIST_HEAD(&the_lnet.ln_drop_rules);
        INIT_LIST_HEAD(&the_lnet.ln_delay_rules);
 
        INIT_LIST_HEAD(&the_lnet.ln_drop_rules);
        INIT_LIST_HEAD(&the_lnet.ln_delay_rules);
 

+       rc = lnet_descriptor_setup();
+       if (rc != 0)
+               goto failed;
+

        rc = lnet_create_remote_nets_table();
        if (rc != 0)
                goto failed;
        rc = lnet_create_remote_nets_table();
        if (rc != 0)
                goto failed;


@@ -664,6 +707,7 @@ lnet_unprepare (void)
                the_lnet.ln_counters = NULL;
        }
        lnet_destroy_remote_nets_table();
                the_lnet.ln_counters = NULL;
        }
        lnet_destroy_remote_nets_table();

+       lnet_descriptor_cleanup();

 
        return 0;
 }
 
        return 0;
 }