LU-56 libcfs: NUMA allocator and code cleanup

This patch covered a few things:
- Implementation of NUMA allocaters based on concept of CPU
  partition
- Add macros of NUMA allocators for both libcfs and OBD
- cleanup OBD allocaters
- A minor fix for llite, it's using GFP flag which should be
  CFS_ALLOC flag

Signed-off-by: Liang Zhen <liang@whamcloud.com>
Change-Id: I5441a504d5df217b59b5db59e37ddb4855e7ae49
Reviewed-on: http://review.whamcloud.com/2523
Tested-by: Hudson
Reviewed-by: Doug Oucharek <doug@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/libcfs/include/libcfs/libcfs_private.h b/libcfs/include/libcfs/libcfs_private.h
index abcc7b8..7946ebc 100644 (file)
--- a/libcfs/include/libcfs/libcfs_private.h
+++ b/libcfs/include/libcfs/libcfs_private.h
@@ -134,51 +134,92 @@ extern cfs_atomic_t libcfs_kmemory;
  */
 #ifdef LIBCFS_DEBUG
 
  */
 #ifdef LIBCFS_DEBUG
 

-# define libcfs_kmem_inc(ptr, size)             \
-do {                                            \
-        cfs_atomic_add(size, &libcfs_kmemory);  \
+# define libcfs_kmem_inc(ptr, size)            \
+do {                                           \
+       cfs_atomic_add(size, &libcfs_kmemory);  \

 } while (0)
 
 } while (0)
 

-# define libcfs_kmem_dec(ptr, size) do {        \
-        cfs_atomic_sub(size, &libcfs_kmemory);  \
+# define libcfs_kmem_dec(ptr, size)            \
+do {                                           \
+       cfs_atomic_sub(size, &libcfs_kmemory);  \

 } while (0)
 
 } while (0)
 

+# define libcfs_kmem_read()                    \
+       cfs_atomic_read(&libcfs_kmemory)
+

 #else
 # define libcfs_kmem_inc(ptr, size) do {} while (0)
 # define libcfs_kmem_dec(ptr, size) do {} while (0)
 #else
 # define libcfs_kmem_inc(ptr, size) do {} while (0)
 # define libcfs_kmem_dec(ptr, size) do {} while (0)

+# define libcfs_kmem_read()    (0)

 #endif /* LIBCFS_DEBUG */
 
 #ifndef LIBCFS_VMALLOC_SIZE
 #define LIBCFS_VMALLOC_SIZE        (2 << CFS_PAGE_SHIFT) /* 2 pages */
 #endif
 
 #endif /* LIBCFS_DEBUG */
 
 #ifndef LIBCFS_VMALLOC_SIZE
 #define LIBCFS_VMALLOC_SIZE        (2 << CFS_PAGE_SHIFT) /* 2 pages */
 #endif
 

-#define LIBCFS_ALLOC_GFP(ptr, size, mask)                                 \
-do {                                                                      \
-        LASSERT(!cfs_in_interrupt() ||                                    \
-               (size <= LIBCFS_VMALLOC_SIZE && mask == CFS_ALLOC_ATOMIC));\
-        if (unlikely((size) > LIBCFS_VMALLOC_SIZE))                       \
-                (ptr) = cfs_alloc_large(size);                            \
-        else                                                              \
-                (ptr) = cfs_alloc((size), (mask));                        \
-        if (unlikely((ptr) == NULL)) {                                    \
-                CERROR("LNET: out of memory at %s:%d (tried to alloc '"   \
-                       #ptr "' = %d)\n", __FILE__, __LINE__, (int)(size));\
-                CERROR("LNET: %d total bytes allocated by lnet\n",        \
-                       cfs_atomic_read(&libcfs_kmemory));                 \
-                break;                                                    \
-        }                                                                 \
-        libcfs_kmem_inc((ptr), (size));                                   \
-        memset((ptr), 0, (size));                                         \
-        CDEBUG(D_MALLOC, "kmalloced '" #ptr "': %d at %p (tot %d).\n",    \
-               (int)(size), (ptr), cfs_atomic_read (&libcfs_kmemory));    \
+#define LIBCFS_ALLOC_PRE(size, mask)                                       \
+do {                                                                       \
+       LASSERT(!cfs_in_interrupt() ||                                      \
+               ((size) <= LIBCFS_VMALLOC_SIZE &&                           \
+                ((mask) & CFS_ALLOC_ATOMIC)) != 0);                        \
+} while (0)
+
+#define LIBCFS_ALLOC_POST(ptr, size)                                       \
+do {                                                                       \
+       if (unlikely((ptr) == NULL)) {                                      \
+               CERROR("LNET: out of memory at %s:%d (tried to alloc '"     \
+                      #ptr "' = %d)\n", __FILE__, __LINE__, (int)(size));  \
+               CERROR("LNET: %d total bytes allocated by lnet\n",          \
+                      libcfs_kmem_read());                                 \
+       } else {                                                            \
+               memset((ptr), 0, (size));                                   \
+               libcfs_kmem_inc((ptr), (size));                             \
+               CDEBUG(D_MALLOC, "alloc '" #ptr "': %d at %p (tot %d).\n",  \
+                      (int)(size), (ptr), libcfs_kmem_read());             \
+       }                                                                   \

 } while (0)
 
 } while (0)
 

+/**
+ * allocate memory with GFP flags @mask
+ */
+#define LIBCFS_ALLOC_GFP(ptr, size, mask)                                  \
+do {                                                                       \
+       LIBCFS_ALLOC_PRE((size), (mask));                                   \
+       (ptr) = (size) <= LIBCFS_VMALLOC_SIZE ?                             \
+               cfs_alloc((size), (mask)) : cfs_alloc_large(size);          \
+       LIBCFS_ALLOC_POST((ptr), (size));                                   \
+} while (0)
+
+/**
+ * default allocator
+ */

 #define LIBCFS_ALLOC(ptr, size) \
         LIBCFS_ALLOC_GFP(ptr, size, CFS_ALLOC_IO)
 
 #define LIBCFS_ALLOC(ptr, size) \
         LIBCFS_ALLOC_GFP(ptr, size, CFS_ALLOC_IO)
 

+/**
+ * non-sleeping allocator
+ */

 #define LIBCFS_ALLOC_ATOMIC(ptr, size) \
         LIBCFS_ALLOC_GFP(ptr, size, CFS_ALLOC_ATOMIC)
 
 #define LIBCFS_ALLOC_ATOMIC(ptr, size) \
         LIBCFS_ALLOC_GFP(ptr, size, CFS_ALLOC_ATOMIC)
 

+/**
+ * allocate memory for specified CPU partition
+ *   \a cptab != NULL, \a cpt is CPU partition id of \a cptab
+ *   \a cptab == NULL, \a cpt is HW NUMA node id
+ */
+#define LIBCFS_CPT_ALLOC_GFP(ptr, cptab, cpt, size, mask)                  \
+do {                                                                       \
+       LIBCFS_ALLOC_PRE((size), (mask));                                   \
+       (ptr) = (size) <= LIBCFS_VMALLOC_SIZE ?                             \
+               cfs_cpt_malloc((cptab), (cpt), (size), (mask)) :            \
+               cfs_cpt_vmalloc((cptab), (cpt), (size));                    \
+       LIBCFS_ALLOC_POST((ptr), (size));                                   \
+} while (0)
+
+/** default numa allocator */
+#define LIBCFS_CPT_ALLOC(ptr, cptab, cpt, size)                                    \
+       LIBCFS_CPT_ALLOC_GFP(ptr, cptab, cpt, size, CFS_ALLOC_IO)
+

 #define LIBCFS_FREE(ptr, size)                                          \
 do {                                                                    \
         int s = (size);                                                 \
 #define LIBCFS_FREE(ptr, size)                                          \
 do {                                                                    \
         int s = (size);                                                 \


@@ -189,7 +230,7 @@ do {                                                                    \
         }                                                               \
         libcfs_kmem_dec((ptr), s);                                      \
         CDEBUG(D_MALLOC, "kfreed '" #ptr "': %d at %p (tot %d).\n",     \
         }                                                               \
         libcfs_kmem_dec((ptr), s);                                      \
         CDEBUG(D_MALLOC, "kfreed '" #ptr "': %d at %p (tot %d).\n",     \

-               s, (ptr), cfs_atomic_read(&libcfs_kmemory));             \
+              s, (ptr), libcfs_kmem_read());                           \

         if (unlikely(s > LIBCFS_VMALLOC_SIZE))                          \
                 cfs_free_large(ptr);                                    \
         else                                                            \
         if (unlikely(s > LIBCFS_VMALLOC_SIZE))                          \
                 cfs_free_large(ptr);                                    \
         else                                                            \


@@ -247,15 +288,24 @@ do {                                                                           \
 # define KLASSERT(e) ((void)0)
 # define printk printf
 # ifdef CRAY_XT3                                /* buggy calloc! */
 # define KLASSERT(e) ((void)0)
 # define printk printf
 # ifdef CRAY_XT3                                /* buggy calloc! */

-#  define LIBCFS_ALLOC(ptr, size)               \
-   do {                                         \
-        (ptr) = malloc(size);                   \
-        memset(ptr, 0, size);                   \
+#  define LIBCFS_ALLOC_GFP(ptr, size, mask)    \
+   do {                                                \
+       (ptr) = malloc(size);                   \
+       memset(ptr, 0, size);                   \

    } while (0)
 # else
    } while (0)
 # else

-#  define LIBCFS_ALLOC(ptr, size) do { (ptr) = calloc(1,size); } while (0)
+#  define LIBCFS_ALLOC_GFP(ptr, size, mask)    \
+   do {                                                \
+       (ptr) = calloc(1, size);                \
+   } while (0)

 # endif
 # define LIBCFS_FREE(ptr, size) do { free(ptr); } while((size) - (size))
 # endif
 # define LIBCFS_FREE(ptr, size) do { free(ptr); } while((size) - (size))

+# define LIBCFS_ALLOC(ptr, size)                               \
+        LIBCFS_ALLOC_GFP(ptr, size, 0)
+# define LIBCFS_CPT_ALLOC_GFP(ptr, cptab, cpt, size, mask)     \
+        LIBCFS_ALLOC(ptr, size)
+# define LIBCFS_CPT_ALLOC(ptr, cptab, cpt, size)               \
+        LIBCFS_ALLOC(ptr, size)

 
 void libcfs_debug_dumplog(void);
 int libcfs_debug_init(unsigned long bufsize);
 
 void libcfs_debug_dumplog(void);
 int libcfs_debug_init(unsigned long bufsize);

diff --git a/libcfs/include/libcfs/linux/linux-mem.h b/libcfs/include/libcfs/linux/linux-mem.h
index a79d6a9..1d0f03d 100644 (file)
--- a/libcfs/include/libcfs/linux/linux-mem.h
+++ b/libcfs/include/libcfs/linux/linux-mem.h
@@ -153,6 +153,24 @@ extern int cfs_mem_is_in_cache(const void *addr, const cfs_mem_cache_t *kmem);
 #define CFS_SLAB_NOFS                   SLAB_NOFS
 
 /*
 #define CFS_SLAB_NOFS                   SLAB_NOFS
 
 /*

+ * NUMA allocators
+ *
+ * NB: we will rename these functions in a separate patch:
+ * - rename cfs_alloc to cfs_malloc
+ * - rename cfs_alloc/free_page to cfs_page_alloc/free
+ * - rename cfs_alloc/free_large to cfs_vmalloc/vfree
+ */
+extern void *cfs_cpt_malloc(struct cfs_cpt_table *cptab, int cpt,
+                           size_t nr_bytes, unsigned int flags);
+extern void *cfs_cpt_vmalloc(struct cfs_cpt_table *cptab, int cpt,
+                            size_t nr_bytes);
+extern cfs_page_t *cfs_page_cpt_alloc(struct cfs_cpt_table *cptab,
+                                     int cpt, unsigned int flags);
+extern void *cfs_mem_cache_cpt_alloc(cfs_mem_cache_t *cachep,
+                                    struct cfs_cpt_table *cptab,
+                                    int cpt, unsigned int flags);
+
+/*

  * Shrinker
  */
 #define cfs_shrinker    shrinker
  * Shrinker
  */
 #define cfs_shrinker    shrinker

diff --git a/libcfs/include/libcfs/user-mem.h b/libcfs/include/libcfs/user-mem.h
index 7c5a1ef..2c5533c 100644 (file)
--- a/libcfs/include/libcfs/user-mem.h
+++ b/libcfs/include/libcfs/user-mem.h
@@ -88,6 +88,18 @@ void cfs_mem_cache_free(cfs_mem_cache_t *c, void *addr);
 int cfs_mem_is_in_cache(const void *addr, const cfs_mem_cache_t *kmem);
 
 /*
 int cfs_mem_is_in_cache(const void *addr, const cfs_mem_cache_t *kmem);
 
 /*

+ * NUMA allocators
+ */
+#define cfs_cpt_malloc(cptab, cpt, bytes, flags)       \
+       cfs_alloc(bytes, flags)
+#define cfs_cpt_vmalloc(cptab, cpt, bytes)             \
+       cfs_alloc(bytes)
+#define cfs_page_cpt_alloc(cptab, cpt, mask)           \
+       cfs_alloc_page(mask)
+#define cfs_mem_cache_cpt_alloc(cache, cptab, cpt, gfp)        \
+       cfs_mem_cache_alloc(cache, gfp)
+
+/*

  * Copy to/from user
  */
 static inline int cfs_copy_from_user(void *a,void *b, int c)
  * Copy to/from user
  */
 static inline int cfs_copy_from_user(void *a,void *b, int c)

diff --git a/libcfs/libcfs/linux/linux-mem.c b/libcfs/libcfs/linux/linux-mem.c
index 8ae2063..1708a12 100644 (file)
--- a/libcfs/libcfs/linux/linux-mem.c
+++ b/libcfs/libcfs/linux/linux-mem.c
@@ -171,3 +171,49 @@ EXPORT_SYMBOL(cfs_mem_cache_create);
 EXPORT_SYMBOL(cfs_mem_cache_destroy);
 EXPORT_SYMBOL(cfs_mem_cache_alloc);
 EXPORT_SYMBOL(cfs_mem_cache_free);
 EXPORT_SYMBOL(cfs_mem_cache_destroy);
 EXPORT_SYMBOL(cfs_mem_cache_alloc);
 EXPORT_SYMBOL(cfs_mem_cache_free);

+
+/*
+ * NB: we will rename some of above functions in another patch:
+ * - rename cfs_alloc to cfs_malloc
+ * - rename cfs_alloc/free_page to cfs_page_alloc/free
+ * - rename cfs_alloc/free_large to cfs_vmalloc/vfree
+ */
+
+void *
+cfs_cpt_malloc(struct cfs_cpt_table *cptab, int cpt,
+              size_t nr_bytes, unsigned int flags)
+{
+       void    *ptr;
+
+       ptr = kmalloc_node(nr_bytes, cfs_alloc_flags_to_gfp(flags),
+                          cfs_cpt_spread_node(cptab, cpt));
+       if (ptr != NULL && (flags & CFS_ALLOC_ZERO) != 0)
+               memset(ptr, 0, nr_bytes);
+
+       return ptr;
+}
+EXPORT_SYMBOL(cfs_cpt_malloc);
+
+void *
+cfs_cpt_vmalloc(struct cfs_cpt_table *cptab, int cpt, size_t nr_bytes)
+{
+       return vmalloc_node(nr_bytes, cfs_cpt_spread_node(cptab, cpt));
+}
+EXPORT_SYMBOL(cfs_cpt_vmalloc);
+
+cfs_page_t *
+cfs_page_cpt_alloc(struct cfs_cpt_table *cptab, int cpt, unsigned int flags)
+{
+       return alloc_pages_node(cfs_cpt_spread_node(cptab, cpt),
+                               cfs_alloc_flags_to_gfp(flags), 0);
+}
+EXPORT_SYMBOL(cfs_page_cpt_alloc);
+
+void *
+cfs_mem_cache_cpt_alloc(cfs_mem_cache_t *cachep, struct cfs_cpt_table *cptab,
+                       int cpt, unsigned int flags)
+{
+       return kmem_cache_alloc_node(cachep, cfs_alloc_flags_to_gfp(flags),
+                                    cfs_cpt_spread_node(cptab, cpt));
+}
+EXPORT_SYMBOL(cfs_mem_cache_cpt_alloc);

diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h
index 39b4728..800721e 100644 (file)
--- a/lustre/include/obd_support.h
+++ b/lustre/include/obd_support.h
@@ -546,18 +546,21 @@ static inline void obd_pages_sub(int order)
 #define OBD_ALLOC_FAIL_MULT (OBD_ALLOC_FAIL_MASK / 100)
 
 #if defined(LUSTRE_UTILS) /* this version is for utils only */
 #define OBD_ALLOC_FAIL_MULT (OBD_ALLOC_FAIL_MASK / 100)
 
 #if defined(LUSTRE_UTILS) /* this version is for utils only */

-#define OBD_ALLOC_GFP(ptr, size, gfp_mask)                                    \
-do {                                                                          \
-        (ptr) = cfs_alloc(size, (gfp_mask));                                  \
-        if (unlikely((ptr) == NULL)) {                                        \
-                CERROR("kmalloc of '" #ptr "' (%d bytes) failed at %s:%d\n",  \
-                       (int)(size), __FILE__, __LINE__);                      \
-        } else {                                                              \
-                memset(ptr, 0, size);                                         \
-                CDEBUG(D_MALLOC, "kmalloced '" #ptr "': %d at %p\n",          \
-                       (int)(size), ptr);                                     \
-        }                                                                     \
+#define __OBD_MALLOC_VERBOSE(ptr, cptab, cpt, size, flags)                   \
+do {                                                                         \
+       (ptr) = (cptab) == NULL ?                                             \
+               cfs_alloc(size, flags) :                                      \
+               cfs_cpt_malloc(cptab, cpt, size, flags);                      \
+       if (unlikely((ptr) == NULL)) {                                        \
+               CERROR("kmalloc of '" #ptr "' (%d bytes) failed at %s:%d\n",  \
+                      (int)(size), __FILE__, __LINE__);                      \
+       } else {                                                              \
+               memset(ptr, 0, size);                                         \
+               CDEBUG(D_MALLOC, "kmalloced '" #ptr "': %d at %p\n",          \
+                      (int)(size), ptr);                                     \
+       }                                                                     \

 } while (0)
 } while (0)

+

 #else /* this version is for the kernel and liblustre */
 #define OBD_FREE_RTN0(ptr)                                                    \
 ({                                                                            \
 #else /* this version is for the kernel and liblustre */
 #define OBD_FREE_RTN0(ptr)                                                    \
 ({                                                                            \


@@ -565,9 +568,12 @@ do {                                                                          \
         (ptr) = NULL;                                                         \
         0;                                                                    \
 })
         (ptr) = NULL;                                                         \
         0;                                                                    \
 })

-#define OBD_ALLOC_GFP(ptr, size, gfp_mask)                                    \
-do {                                                                          \
-        (ptr) = cfs_alloc(size, (gfp_mask));                                  \
+
+#define __OBD_MALLOC_VERBOSE(ptr, cptab, cpt, size, flags)                   \
+do {                                                                         \
+       (ptr) = (cptab) == NULL ?                                             \
+               cfs_alloc(size, flags) :                                      \
+               cfs_cpt_malloc(cptab, cpt, size, flags);                      \

         if (likely((ptr) != NULL &&                                           \
                    (!HAS_FAIL_ALLOC_FLAG || obd_alloc_fail_rate == 0 ||       \
                     !obd_alloc_fail(ptr, #ptr, "km", size,                    \
         if (likely((ptr) != NULL &&                                           \
                    (!HAS_FAIL_ALLOC_FLAG || obd_alloc_fail_rate == 0 ||       \
                     !obd_alloc_fail(ptr, #ptr, "km", size,                    \


@@ -579,31 +585,53 @@ do {                                                                          \
 } while (0)
 #endif
 
 } while (0)
 #endif
 

-#ifndef OBD_ALLOC_MASK
-# define OBD_ALLOC_MASK CFS_ALLOC_IO
-#endif
+#define OBD_ALLOC_GFP(ptr, size, gfp_mask)                                   \
+       __OBD_MALLOC_VERBOSE(ptr, NULL, 0, size, gfp_mask)

 
 

-#define OBD_ALLOC(ptr, size) OBD_ALLOC_GFP(ptr, size, OBD_ALLOC_MASK)
+#define OBD_ALLOC(ptr, size) OBD_ALLOC_GFP(ptr, size, CFS_ALLOC_IO)

 #define OBD_ALLOC_WAIT(ptr, size) OBD_ALLOC_GFP(ptr, size, CFS_ALLOC_STD)
 #define OBD_ALLOC_PTR(ptr) OBD_ALLOC(ptr, sizeof *(ptr))
 #define OBD_ALLOC_PTR_WAIT(ptr) OBD_ALLOC_WAIT(ptr, sizeof *(ptr))
 
 #define OBD_ALLOC_WAIT(ptr, size) OBD_ALLOC_GFP(ptr, size, CFS_ALLOC_STD)
 #define OBD_ALLOC_PTR(ptr) OBD_ALLOC(ptr, sizeof *(ptr))
 #define OBD_ALLOC_PTR_WAIT(ptr) OBD_ALLOC_WAIT(ptr, sizeof *(ptr))
 

+#define OBD_CPT_ALLOC_GFP(ptr, cptab, cpt, size, gfp_mask)                   \
+       __OBD_MALLOC_VERBOSE(ptr, cptab, cpt, size, gfp_mask)
+
+#define OBD_CPT_ALLOC(ptr, cptab, cpt, size)                                 \
+       OBD_CPT_ALLOC_GFP(ptr, cptab, cpt, size, CFS_ALLOC_IO)
+
+#define OBD_CPT_ALLOC_PTR(ptr, cptab, cpt)                                   \
+       OBD_CPT_ALLOC(ptr, cptab, cpt, sizeof *(ptr))
+

 #ifdef __arch_um__
 #ifdef __arch_um__

-# define OBD_VMALLOC(ptr, size) OBD_ALLOC(ptr, size)
-#else
-# define OBD_VMALLOC(ptr, size)                                               \
-do {                                                                          \
-        (ptr) = cfs_alloc_large(size);                                        \
+
+# define OBD_VMALLOC(ptr, size)                                                      \
+        OBD_ALLOC(ptr, size)
+# define OBD_CPT_VMALLOC(ptr, cptab, cpt, size)                                      \
+        OBD_CPT_ALLOC(ptr, cptab, cpt, size)
+
+#else /* !__arch_um__ */
+
+# define __OBD_VMALLOC_VEROBSE(ptr, cptab, cpt, size)                        \
+do {                                                                         \
+       (ptr) = cptab == NULL ?                                               \
+               cfs_alloc_large(size) :                                       \
+               cfs_cpt_vmalloc(cptab, cpt, size);                            \

         if (unlikely((ptr) == NULL)) {                                        \
                 CERROR("vmalloc of '" #ptr "' (%d bytes) failed\n",           \
                        (int)(size));                                          \
                 CERROR(LPU64" total bytes allocated by Lustre, %d by LNET\n", \
                        obd_memory_sum(), cfs_atomic_read(&libcfs_kmemory));   \
         } else {                                                              \
         if (unlikely((ptr) == NULL)) {                                        \
                 CERROR("vmalloc of '" #ptr "' (%d bytes) failed\n",           \
                        (int)(size));                                          \
                 CERROR(LPU64" total bytes allocated by Lustre, %d by LNET\n", \
                        obd_memory_sum(), cfs_atomic_read(&libcfs_kmemory));   \
         } else {                                                              \

-                memset(ptr, 0, size);                                         \
+               memset(ptr, 0, size);                                         \

                 OBD_ALLOC_POST(ptr, size, "vmalloced");                       \
         }                                                                     \
 } while(0)
                 OBD_ALLOC_POST(ptr, size, "vmalloced");                       \
         }                                                                     \
 } while(0)

+
+# define OBD_VMALLOC(ptr, size)                                                      \
+        __OBD_VMALLOC_VEROBSE(ptr, NULL, 0, size)
+# define OBD_CPT_VMALLOC(ptr, cptab, cpt, size)                                      \
+        __OBD_VMALLOC_VEROBSE(ptr, cptab, cpt, size)
+

 #endif
 
 #ifdef __KERNEL__
 #endif
 
 #ifdef __KERNEL__


@@ -625,6 +653,14 @@ do {                                                                          \
                 OBD_ALLOC(ptr, size);                                         \
 } while (0)
 
                 OBD_ALLOC(ptr, size);                                         \
 } while (0)
 

+#define OBD_CPT_ALLOC_LARGE(ptr, cptab, cpt, size)                           \
+do {                                                                         \
+       if (size > OBD_ALLOC_BIG)                                             \
+               OBD_CPT_VMALLOC(ptr, cptab, cpt, size);                       \
+       else                                                                  \
+               OBD_CPT_ALLOC(ptr, cptab, cpt, size);                         \
+} while (0)
+

 #define OBD_FREE_LARGE(ptr, size)                                             \
 do {                                                                          \
         if (size > OBD_ALLOC_BIG)                                             \
 #define OBD_FREE_LARGE(ptr, size)                                             \
 do {                                                                          \
         if (size > OBD_ALLOC_BIG)                                             \


@@ -632,10 +668,17 @@ do {                                                                          \
         else                                                                  \
                 OBD_FREE(ptr, size);                                          \
 } while (0)
         else                                                                  \
                 OBD_FREE(ptr, size);                                          \
 } while (0)

-#else
-#define OBD_ALLOC_LARGE(ptr, size) OBD_ALLOC(ptr, size)
-#define OBD_FREE_LARGE(ptr, size) OBD_FREE(ptr,size)
-#endif
+
+#else /* !__KERNEL__ */
+
+#define OBD_ALLOC_LARGE(ptr, size)                                           \
+       OBD_ALLOC(ptr, size)
+#define OBD_CPT_ALLOC_LARGE(ptr, cptab, cpt, size)                           \
+       OBD_ALLOC(ptr, size)
+#define OBD_FREE_LARGE(ptr, size)                                            \
+       OBD_FREE(ptr, size)
+
+#endif /* __KERNEL__ */

 
 #ifdef CONFIG_DEBUG_SLAB
 #define POISON(ptr, c, s) do {} while (0)
 
 #ifdef CONFIG_DEBUG_SLAB
 #define POISON(ptr, c, s) do {} while (0)


@@ -707,10 +750,13 @@ do {                                                                          \
         (ptr) = NULL;                                                         \
         0;                                                                    \
 })
         (ptr) = NULL;                                                         \
         0;                                                                    \
 })

-#define OBD_SLAB_ALLOC(ptr, slab, type, size)                                 \
-do {                                                                          \
+
+#define __OBD_SLAB_ALLOC_VERBOSE(ptr, slab, cptab, cpt, size, type)          \
+do {                                                                         \

         LASSERT(ergo(type != CFS_ALLOC_ATOMIC, !cfs_in_interrupt()));         \
         LASSERT(ergo(type != CFS_ALLOC_ATOMIC, !cfs_in_interrupt()));         \

-        (ptr) = cfs_mem_cache_alloc(slab, (type));                            \
+       (ptr) = (cptab) == NULL ?                                             \
+               cfs_mem_cache_alloc(slab, type) :                             \
+               cfs_mem_cache_cpt_alloc(slab, cptab, cpt, type);              \

         if (likely((ptr) != NULL &&                                           \
                    (!HAS_FAIL_ALLOC_FLAG || obd_alloc_fail_rate == 0 ||       \
                     !obd_alloc_fail(ptr, #ptr, "slab-", size,                 \
         if (likely((ptr) != NULL &&                                           \
                    (!HAS_FAIL_ALLOC_FLAG || obd_alloc_fail_rate == 0 ||       \
                     !obd_alloc_fail(ptr, #ptr, "slab-", size,                 \


@@ -721,6 +767,11 @@ do {                                                                          \
         }                                                                     \
 } while(0)
 
         }                                                                     \
 } while(0)
 

+#define OBD_SLAB_ALLOC_GFP(ptr, slab, size, flags)                           \
+       __OBD_SLAB_ALLOC_VERBOSE(ptr, slab, NULL, 0, size, flags)
+#define OBD_SLAB_CPT_ALLOC_GFP(ptr, slab, cptab, cpt, size, flags)           \
+       __OBD_SLAB_ALLOC_VERBOSE(ptr, slab, cptab, cpt, size, flags)
+

 #define OBD_FREE_PTR(ptr) OBD_FREE(ptr, sizeof *(ptr))
 
 #define OBD_SLAB_FREE(ptr, slab, size)                                        \
 #define OBD_FREE_PTR(ptr) OBD_FREE(ptr, sizeof *(ptr))
 
 #define OBD_SLAB_FREE(ptr, slab, size)                                        \


@@ -730,20 +781,36 @@ do {                                                                          \
         POISON_PTR(ptr);                                                      \
 } while(0)
 
         POISON_PTR(ptr);                                                      \
 } while(0)
 

-#define OBD_SLAB_ALLOC_PTR(ptr, slab)                                         \
-        OBD_SLAB_ALLOC((ptr), (slab), CFS_ALLOC_STD, sizeof *(ptr))
-#define OBD_SLAB_FREE_PTR(ptr, slab)                                          \
-        OBD_SLAB_FREE((ptr), (slab), sizeof *(ptr))
-#define OBD_SLAB_ALLOC_PTR_GFP(ptr, slab, gfp)                              \
-        OBD_SLAB_ALLOC((ptr), (slab), (gfp), sizeof *(ptr))
+#define OBD_SLAB_ALLOC(ptr, slab, size)                                              \
+       OBD_SLAB_ALLOC_GFP(ptr, slab, size, CFS_ALLOC_IO)
+
+#define OBD_SLAB_CPT_ALLOC(ptr, slab, cptab, cpt, size)                              \
+       OBD_SLAB_CPT_ALLOC_GFP(ptr, slab, cptab, cpt, size, CFS_ALLOC_IO)
+
+#define OBD_SLAB_ALLOC_PTR(ptr, slab)                                        \
+       OBD_SLAB_ALLOC(ptr, slab, sizeof *(ptr))
+
+#define OBD_SLAB_CPT_ALLOC_PTR(ptr, slab, cptab, cpt)                        \
+       OBD_SLAB_CPT_ALLOC(ptr, slab, cptab, cpt, sizeof *(ptr))
+
+#define OBD_SLAB_ALLOC_PTR_GFP(ptr, slab, flags)                             \
+       OBD_SLAB_ALLOC_GFP(ptr, slab, sizeof *(ptr), flags)
+
+#define OBD_SLAB_CPT_ALLOC_PTR_GFP(ptr, slab, ctab, cpt, flags)                      \
+       OBD_SLAB_CPT_ALLOC_GFP(ptr, slab, cptab, cpt, sizeof *(ptr), flags)
+
+#define OBD_SLAB_FREE_PTR(ptr, slab)                                         \
+       OBD_SLAB_FREE((ptr), (slab), sizeof *(ptr))

 
 #define KEY_IS(str) \
         (keylen >= (sizeof(str)-1) && memcmp(key, str, (sizeof(str)-1)) == 0)
 
 /* Wrapper for contiguous page frame allocation */
 
 #define KEY_IS(str) \
         (keylen >= (sizeof(str)-1) && memcmp(key, str, (sizeof(str)-1)) == 0)
 
 /* Wrapper for contiguous page frame allocation */

-#define OBD_PAGE_ALLOC(ptr, gfp_mask)                                         \
-do {                                                                          \
-        (ptr) = cfs_alloc_page(gfp_mask);                                     \
+#define __OBD_PAGE_ALLOC_VERBOSE(ptr, cptab, cpt, gfp_mask)                  \
+do {                                                                         \
+       (ptr) = (cptab) == NULL ?                                             \
+               cfs_alloc_page(gfp_mask) :                                    \
+               cfs_page_cpt_alloc(cptab, cpt, gfp_mask);                     \

         if (unlikely((ptr) == NULL)) {                                        \
                 CERROR("alloc_pages of '" #ptr "' %d page(s) / "LPU64" bytes "\
                        "failed\n", (int)1,                                    \
         if (unlikely((ptr) == NULL)) {                                        \
                 CERROR("alloc_pages of '" #ptr "' %d page(s) / "LPU64" bytes "\
                        "failed\n", (int)1,                                    \


@@ -764,6 +831,11 @@ do {                                                                          \
         }                                                                     \
 } while (0)
 
         }                                                                     \
 } while (0)
 

+#define OBD_PAGE_ALLOC(ptr, gfp_mask)                                        \
+       __OBD_PAGE_ALLOC_VERBOSE(ptr, NULL, 0, gfp_mask)
+#define OBD_PAGE_CPT_ALLOC(ptr, cptab, cpt, gfp_mask)                        \
+       __OBD_PAGE_ALLOC_VERBOSE(ptr, cptab, cpt, gfp_mask)
+

 #define OBD_PAGE_FREE(ptr)                                                    \
 do {                                                                          \
         LASSERT(ptr);                                                         \
 #define OBD_PAGE_FREE(ptr)                                                    \
 do {                                                                          \
         LASSERT(ptr);                                                         \

diff --git a/lustre/llite/remote_perm.c b/lustre/llite/remote_perm.c
index 6febb2e..64dfd64 100644 (file)
--- a/lustre/llite/remote_perm.c
+++ b/lustre/llite/remote_perm.c
@@ -83,9 +83,9 @@ cfs_hlist_head_t *alloc_rmtperm_hash(void)
         cfs_hlist_head_t *hash;
         int i;
 
         cfs_hlist_head_t *hash;
         int i;
 

-        OBD_SLAB_ALLOC(hash, ll_rmtperm_hash_cachep, GFP_KERNEL,
-                       REMOTE_PERM_HASHSIZE * sizeof(*hash));
-
+       OBD_SLAB_ALLOC_GFP(hash, ll_rmtperm_hash_cachep,
+                          REMOTE_PERM_HASHSIZE * sizeof(*hash),
+                          CFS_ALLOC_STD);

         if (!hash)
                 return NULL;
 
         if (!hash)
                 return NULL;
 

diff --git a/lustre/osc/osc_quota.c b/lustre/osc/osc_quota.c
index efc0bbd..3d2b45b 100644 (file)
--- a/lustre/osc/osc_quota.c
+++ b/lustre/osc/osc_quota.c
@@ -97,7 +97,7 @@ static struct osc_quota_info *alloc_qinfo(struct client_obd *cli,
         struct osc_quota_info *oqi;
         ENTRY;
 
         struct osc_quota_info *oqi;
         ENTRY;
 

-        OBD_SLAB_ALLOC(oqi, qinfo_cachep, CFS_ALLOC_IO, sizeof(*oqi));
+       OBD_SLAB_ALLOC_PTR(oqi, qinfo_cachep);

         if(!oqi)
                 RETURN(NULL);
 
         if(!oqi)
                 RETURN(NULL);
 

diff --git a/lustre/ptlrpc/recov_thread.c b/lustre/ptlrpc/recov_thread.c
index c10331f..4d552cf 100644 (file)
--- a/lustre/ptlrpc/recov_thread.c
+++ b/lustre/ptlrpc/recov_thread.c
@@ -108,7 +108,7 @@ static struct llog_canceld_ctxt *llcd_alloc(struct llog_commit_master *lcm)
          */
         size = CFS_PAGE_SIZE - lustre_msg_size(LUSTRE_MSG_MAGIC_V2, 1, NULL);
         overhead =  offsetof(struct llog_canceld_ctxt, llcd_cookies);
          */
         size = CFS_PAGE_SIZE - lustre_msg_size(LUSTRE_MSG_MAGIC_V2, 1, NULL);
         overhead =  offsetof(struct llog_canceld_ctxt, llcd_cookies);

-        OBD_SLAB_ALLOC(llcd, llcd_cache, CFS_ALLOC_STD, size + overhead);
+       OBD_SLAB_ALLOC_GFP(llcd, llcd_cache, size + overhead, CFS_ALLOC_STD);

         if (!llcd)
                 return NULL;
 
         if (!llcd)
                 return NULL;
 

diff --git a/lustre/quota/quota_context.c b/lustre/quota/quota_context.c
index e53929a..3989ed9 100644 (file)
--- a/lustre/quota/quota_context.c
+++ b/lustre/quota/quota_context.c
@@ -525,7 +525,7 @@ void* quota_barrier(struct lustre_quota_ctxt *qctxt,
         struct lustre_qunit *qunit, *find_qunit;
         int cycle = 1;
 
         struct lustre_qunit *qunit, *find_qunit;
         int cycle = 1;
 

-        OBD_SLAB_ALLOC(qunit, qunit_cachep, CFS_ALLOC_IO, sizeof(*qunit));
+       OBD_SLAB_ALLOC_PTR(qunit, qunit_cachep);

         if (qunit == NULL) {
                 CERROR("locating %sunit failed for %sid %u\n",
                        isblk ? "b" : "i", oqctl->qc_type ? "g" : "u",
         if (qunit == NULL) {
                 CERROR("locating %sunit failed for %sid %u\n",
                        isblk ? "b" : "i", oqctl->qc_type ? "g" : "u",