Use kmalloc() instead of slab to allocate the MD and MSG portal descriptors.
On x86_64 the MD and MSG structs got quite large, into the 2k range, though
pages are still 4k there. This tricked slab into trying to back their slabs
with multi-page allocations. These allocations failed under high load which
lead to LBUG()s and timeouts.
While we're in here, properly initialize the in_use counts and stop using
sleeping allocations while holding the state lock.
s, (ptr), atomic_read(&portal_kmemory)); \
} while (0)
-#define PORTAL_SLAB_ALLOC(ptr, slab, size) \
-do { \
- LASSERT(!in_interrupt()); \
- (ptr) = kmem_cache_alloc((slab), SLAB_NOFS); \
- if ((ptr) == NULL) { \
- CERROR("PORTALS: out of memory at %s:%d (tried to alloc" \
- " '" #ptr "' from slab '" #slab "')\n", __FILE__, \
- __LINE__); \
- CERROR("PORTALS: %d total bytes allocated by portals\n", \
- atomic_read(&portal_kmemory)); \
- } else { \
- portal_kmem_inc((ptr), (size)); \
- memset((ptr), 0, (size)); \
- } \
- CDEBUG(D_MALLOC, "kmalloced '" #ptr "': %d at %p (tot %d).\n", \
- (int)(size), (ptr), atomic_read(&portal_kmemory)); \
-} while (0)
-
-#define PORTAL_SLAB_FREE(ptr, slab, size) \
-do { \
- int s = (size); \
- if ((ptr) == NULL) { \
- CERROR("PORTALS: free NULL '" #ptr "' (%d bytes) at " \
- "%s:%d\n", s, __FILE__, __LINE__); \
- break; \
- } \
- memset((ptr), 0x5a, s); \
- kmem_cache_free((slab), ptr); \
- portal_kmem_dec((ptr), s); \
- CDEBUG(D_MALLOC, "kfreed '" #ptr "': %d at %p (tot %d).\n", \
- s, (ptr), atomic_read (&portal_kmemory)); \
-} while (0)
-
/* ------------------------------------------------------------------- */
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
nal->cb_sti(nal, flagsp); \
}
-#ifndef PTL_USE_SLAB_CACHE
+#ifdef PTL_USE_DESC_LISTS
#define MAX_MES 2048
#define MAX_MDS 2048
#else
-extern kmem_cache_t *ptl_md_slab;
-extern kmem_cache_t *ptl_msg_slab;
-extern kmem_cache_t *ptl_me_slab;
-extern kmem_cache_t *ptl_eq_slab;
extern atomic_t md_in_use_count;
extern atomic_t msg_in_use_count;
extern atomic_t me_in_use_count;
{
/* NEVER called with statelock held */
lib_eq_t *eq;
- PORTAL_SLAB_ALLOC(eq, ptl_eq_slab, sizeof(*eq));
+ PORTAL_ALLOC(eq, sizeof(*eq));
if (eq == NULL)
return (NULL);
{
/* ALWAYS called with statelock held */
atomic_dec (&eq_in_use_count);
- PORTAL_SLAB_FREE(eq, ptl_eq_slab, sizeof(*eq));
+ PORTAL_FREE(eq, sizeof(*eq));
}
static inline lib_md_t *
{
/* NEVER called with statelock held */
lib_md_t *md;
- PORTAL_SLAB_ALLOC(md, ptl_md_slab, sizeof(*md));
+ PORTAL_ALLOC(md, sizeof(*md));
if (md == NULL)
return (NULL);
{
/* ALWAYS called with statelock held */
atomic_dec (&md_in_use_count);
- PORTAL_SLAB_FREE(md, ptl_md_slab, sizeof(*md));
+ PORTAL_FREE(md, sizeof(*md));
}
static inline lib_me_t *
{
/* NEVER called with statelock held */
lib_me_t *me;
- PORTAL_SLAB_ALLOC(me, ptl_me_slab, sizeof(*me));
+ PORTAL_ALLOC(me, sizeof(*me));
if (me == NULL)
return (NULL);
{
/* ALWAYS called with statelock held */
atomic_dec (&me_in_use_count);
- PORTAL_SLAB_FREE(me, ptl_me_slab, sizeof(*me));
+ PORTAL_FREE(me, sizeof(*me));
}
static inline lib_msg_t *
{
/* ALWAYS called with statelock held */
lib_msg_t *msg;
- PORTAL_SLAB_ALLOC(msg, ptl_msg_slab, sizeof(*msg));
+ PORTAL_ALLOC_ATOMIC(msg, sizeof(*msg));
if (msg == NULL)
return (NULL);
{
/* ALWAYS called with statelock held */
atomic_dec (&msg_in_use_count);
- PORTAL_SLAB_FREE(msg, ptl_msg_slab, sizeof(*msg));
+ PORTAL_FREE(msg, sizeof(*msg));
}
#endif
nal->cb_sti(nal, flagsp); \
}
-#ifndef PTL_USE_SLAB_CACHE
+#ifdef PTL_USE_DESC_LISTS
#define MAX_MES 2048
#define MAX_MDS 2048
#else
-extern kmem_cache_t *ptl_md_slab;
-extern kmem_cache_t *ptl_msg_slab;
-extern kmem_cache_t *ptl_me_slab;
-extern kmem_cache_t *ptl_eq_slab;
extern atomic_t md_in_use_count;
extern atomic_t msg_in_use_count;
extern atomic_t me_in_use_count;
{
/* NEVER called with statelock held */
lib_eq_t *eq;
- PORTAL_SLAB_ALLOC(eq, ptl_eq_slab, sizeof(*eq));
+ PORTAL_ALLOC(eq, sizeof(*eq));
if (eq == NULL)
return (NULL);
{
/* ALWAYS called with statelock held */
atomic_dec (&eq_in_use_count);
- PORTAL_SLAB_FREE(eq, ptl_eq_slab, sizeof(*eq));
+ PORTAL_FREE(eq, sizeof(*eq));
}
static inline lib_md_t *
{
/* NEVER called with statelock held */
lib_md_t *md;
- PORTAL_SLAB_ALLOC(md, ptl_md_slab, sizeof(*md));
+ PORTAL_ALLOC(md, sizeof(*md));
if (md == NULL)
return (NULL);
{
/* ALWAYS called with statelock held */
atomic_dec (&md_in_use_count);
- PORTAL_SLAB_FREE(md, ptl_md_slab, sizeof(*md));
+ PORTAL_FREE(md, sizeof(*md));
}
static inline lib_me_t *
{
/* NEVER called with statelock held */
lib_me_t *me;
- PORTAL_SLAB_ALLOC(me, ptl_me_slab, sizeof(*me));
+ PORTAL_ALLOC(me, sizeof(*me));
if (me == NULL)
return (NULL);
{
/* ALWAYS called with statelock held */
atomic_dec (&me_in_use_count);
- PORTAL_SLAB_FREE(me, ptl_me_slab, sizeof(*me));
+ PORTAL_FREE(me, sizeof(*me));
}
static inline lib_msg_t *
{
/* ALWAYS called with statelock held */
lib_msg_t *msg;
- PORTAL_SLAB_ALLOC(msg, ptl_msg_slab, sizeof(*msg));
+ PORTAL_ALLOC_ATOMIC(msg, sizeof(*msg));
if (msg == NULL)
return (NULL);
{
/* ALWAYS called with statelock held */
atomic_dec (&msg_in_use_count);
- PORTAL_SLAB_FREE(msg, ptl_msg_slab, sizeof(*msg));
+ PORTAL_FREE(msg, sizeof(*msg));
}
#endif
#include <portals/types.h>
#ifdef __KERNEL__
-# define PTL_USE_SLAB_CACHE
# include <linux/uio.h>
# include <linux/smp_lock.h>
# include <linux/types.h>
#else
+# define PTL_USE_DESC_LISTS
# include <sys/types.h>
#endif
#define PTL_MD_FLAG_UNLINK (1 << 0)
#define PTL_MD_FLAG_AUTO_UNLINKED (1 << 1)
-#ifndef PTL_USE_SLAB_CACHE
+#ifdef PTL_USE_DESC_LISTS
typedef struct
{
void *fl_objs; /* single contiguous array of objects */
struct list_head ni_test_peers;
-#ifndef PTL_USE_SLAB_CACHE
+#ifdef PTL_USE_DESC_LISTS
lib_freelist_t ni_free_mes;
lib_freelist_t ni_free_msgs;
lib_freelist_t ni_free_mds;
# include <sys/time.h>
#endif
-#ifdef PTL_USE_SLAB_CACHE
+#ifndef PTL_USE_DESC_LISTS
static int ptl_slab_users;
-kmem_cache_t *ptl_md_slab;
-kmem_cache_t *ptl_msg_slab;
-kmem_cache_t *ptl_me_slab;
-kmem_cache_t *ptl_eq_slab;
-
-atomic_t md_in_use_count;
-atomic_t msg_in_use_count;
-atomic_t me_in_use_count;
-atomic_t eq_in_use_count;
-
-/* NB zeroing in ctor and on freeing ensures items that
- * kmem_cache_validate() OK, but haven't been initialised
- * as an MD/ME/EQ can't have valid handles
- */
-static void
-ptl_md_slab_ctor (void *obj, kmem_cache_t *slab, unsigned long flags)
-{
- memset (obj, 0, sizeof (lib_md_t));
-}
-
-static void
-ptl_me_slab_ctor (void *obj, kmem_cache_t *slab, unsigned long flags)
-{
- memset (obj, 0, sizeof (lib_me_t));
-}
-
-static void
-ptl_eq_slab_ctor (void *obj, kmem_cache_t *slab, unsigned long flags)
-{
- memset (obj, 0, sizeof (lib_eq_t));
-}
+atomic_t md_in_use_count = ATOMIC_INIT(0);
+atomic_t msg_in_use_count = ATOMIC_INIT(0);
+atomic_t me_in_use_count = ATOMIC_INIT(0);
+atomic_t eq_in_use_count = ATOMIC_INIT(0);
int
kportal_descriptor_setup (nal_cb_t *nal)
{
- /* NB on failure caller must still call kportal_descriptor_cleanup */
- /* ****** */
-
- /* We'll have 1 set of slabs for ALL the nals :) */
-
- if (ptl_slab_users++)
- return 0;
-
- ptl_md_slab = kmem_cache_create("portals_MD",
- sizeof(lib_md_t), 0,
- SLAB_HWCACHE_ALIGN,
- ptl_md_slab_ctor, NULL);
- if (!ptl_md_slab) {
- CERROR("couldn't allocate ptl_md_t slab");
- RETURN (PTL_NOSPACE);
- }
-
- /* NB no ctor for msgs; they don't need handle verification */
- ptl_msg_slab = kmem_cache_create("portals_MSG",
- sizeof(lib_msg_t), 0,
- SLAB_HWCACHE_ALIGN,
- NULL, NULL);
- if (!ptl_msg_slab) {
- CERROR("couldn't allocate ptl_msg_t slab");
- RETURN (PTL_NOSPACE);
- }
-
- ptl_me_slab = kmem_cache_create("portals_ME",
- sizeof(lib_me_t), 0,
- SLAB_HWCACHE_ALIGN,
- ptl_me_slab_ctor, NULL);
- if (!ptl_me_slab) {
- CERROR("couldn't allocate ptl_me_t slab");
- RETURN (PTL_NOSPACE);
- }
-
- ptl_eq_slab = kmem_cache_create("portals_EQ",
- sizeof(lib_eq_t), 0,
- SLAB_HWCACHE_ALIGN,
- ptl_eq_slab_ctor, NULL);
- if (!ptl_eq_slab) {
- CERROR("couldn't allocate ptl_eq_t slab");
- RETURN (PTL_NOSPACE);
- }
-
+ ptl_slab_users++;
RETURN(PTL_OK);
}
void
kportal_descriptor_cleanup (nal_cb_t *nal)
{
- int rc;
-
if (--ptl_slab_users != 0)
return;
LASSERT (atomic_read (&me_in_use_count) == 0);
LASSERT (atomic_read (&eq_in_use_count) == 0);
LASSERT (atomic_read (&msg_in_use_count) == 0);
-
- if (ptl_md_slab != NULL) {
- rc = kmem_cache_destroy(ptl_md_slab);
- if (rc != 0)
- CERROR("unable to free MD slab\n");
- }
- if (ptl_msg_slab != NULL) {
- rc = kmem_cache_destroy(ptl_msg_slab);
- if (rc != 0)
- CERROR("unable to free MSG slab\n");
- }
- if (ptl_me_slab != NULL) {
- rc = kmem_cache_destroy(ptl_me_slab);
- if (rc != 0)
- CERROR("unable to free ME slab\n");
- }
- if (ptl_eq_slab != NULL) {
- rc = kmem_cache_destroy(ptl_eq_slab);
- if (rc != 0)
- CERROR("unable to free EQ slab\n");
- }
}
#else
s, (ptr), atomic_read(&portal_kmemory)); \
} while (0)
-#define PORTAL_SLAB_ALLOC(ptr, slab, size) \
-do { \
- LASSERT(!in_interrupt()); \
- (ptr) = kmem_cache_alloc((slab), SLAB_NOFS); \
- if ((ptr) == NULL) { \
- CERROR("PORTALS: out of memory at %s:%d (tried to alloc" \
- " '" #ptr "' from slab '" #slab "')\n", __FILE__, \
- __LINE__); \
- CERROR("PORTALS: %d total bytes allocated by portals\n", \
- atomic_read(&portal_kmemory)); \
- } else { \
- portal_kmem_inc((ptr), (size)); \
- memset((ptr), 0, (size)); \
- } \
- CDEBUG(D_MALLOC, "kmalloced '" #ptr "': %d at %p (tot %d).\n", \
- (int)(size), (ptr), atomic_read(&portal_kmemory)); \
-} while (0)
-
-#define PORTAL_SLAB_FREE(ptr, slab, size) \
-do { \
- int s = (size); \
- if ((ptr) == NULL) { \
- CERROR("PORTALS: free NULL '" #ptr "' (%d bytes) at " \
- "%s:%d\n", s, __FILE__, __LINE__); \
- break; \
- } \
- memset((ptr), 0x5a, s); \
- kmem_cache_free((slab), ptr); \
- portal_kmem_dec((ptr), s); \
- CDEBUG(D_MALLOC, "kfreed '" #ptr "': %d at %p (tot %d).\n", \
- s, (ptr), atomic_read (&portal_kmemory)); \
-} while (0)
-
/* ------------------------------------------------------------------- */
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
nal->cb_sti(nal, flagsp); \
}
-#ifndef PTL_USE_SLAB_CACHE
+#ifdef PTL_USE_DESC_LISTS
#define MAX_MES 2048
#define MAX_MDS 2048
#else
-extern kmem_cache_t *ptl_md_slab;
-extern kmem_cache_t *ptl_msg_slab;
-extern kmem_cache_t *ptl_me_slab;
-extern kmem_cache_t *ptl_eq_slab;
extern atomic_t md_in_use_count;
extern atomic_t msg_in_use_count;
extern atomic_t me_in_use_count;
{
/* NEVER called with statelock held */
lib_eq_t *eq;
- PORTAL_SLAB_ALLOC(eq, ptl_eq_slab, sizeof(*eq));
+ PORTAL_ALLOC(eq, sizeof(*eq));
if (eq == NULL)
return (NULL);
{
/* ALWAYS called with statelock held */
atomic_dec (&eq_in_use_count);
- PORTAL_SLAB_FREE(eq, ptl_eq_slab, sizeof(*eq));
+ PORTAL_FREE(eq, sizeof(*eq));
}
static inline lib_md_t *
{
/* NEVER called with statelock held */
lib_md_t *md;
- PORTAL_SLAB_ALLOC(md, ptl_md_slab, sizeof(*md));
+ PORTAL_ALLOC(md, sizeof(*md));
if (md == NULL)
return (NULL);
{
/* ALWAYS called with statelock held */
atomic_dec (&md_in_use_count);
- PORTAL_SLAB_FREE(md, ptl_md_slab, sizeof(*md));
+ PORTAL_FREE(md, sizeof(*md));
}
static inline lib_me_t *
{
/* NEVER called with statelock held */
lib_me_t *me;
- PORTAL_SLAB_ALLOC(me, ptl_me_slab, sizeof(*me));
+ PORTAL_ALLOC(me, sizeof(*me));
if (me == NULL)
return (NULL);
{
/* ALWAYS called with statelock held */
atomic_dec (&me_in_use_count);
- PORTAL_SLAB_FREE(me, ptl_me_slab, sizeof(*me));
+ PORTAL_FREE(me, sizeof(*me));
}
static inline lib_msg_t *
{
/* ALWAYS called with statelock held */
lib_msg_t *msg;
- PORTAL_SLAB_ALLOC(msg, ptl_msg_slab, sizeof(*msg));
+ PORTAL_ALLOC_ATOMIC(msg, sizeof(*msg));
if (msg == NULL)
return (NULL);
{
/* ALWAYS called with statelock held */
atomic_dec (&msg_in_use_count);
- PORTAL_SLAB_FREE(msg, ptl_msg_slab, sizeof(*msg));
+ PORTAL_FREE(msg, sizeof(*msg));
}
#endif
#include <portals/types.h>
#ifdef __KERNEL__
-# define PTL_USE_SLAB_CACHE
# include <linux/uio.h>
# include <linux/smp_lock.h>
# include <linux/types.h>
#else
+# define PTL_USE_DESC_LISTS
# include <sys/types.h>
#endif
#define PTL_MD_FLAG_UNLINK (1 << 0)
#define PTL_MD_FLAG_AUTO_UNLINKED (1 << 1)
-#ifndef PTL_USE_SLAB_CACHE
+#ifdef PTL_USE_DESC_LISTS
typedef struct
{
void *fl_objs; /* single contiguous array of objects */
struct list_head ni_test_peers;
-#ifndef PTL_USE_SLAB_CACHE
+#ifdef PTL_USE_DESC_LISTS
lib_freelist_t ni_free_mes;
lib_freelist_t ni_free_msgs;
lib_freelist_t ni_free_mds;
# include <sys/time.h>
#endif
-#ifdef PTL_USE_SLAB_CACHE
+#ifndef PTL_USE_DESC_LISTS
static int ptl_slab_users;
-kmem_cache_t *ptl_md_slab;
-kmem_cache_t *ptl_msg_slab;
-kmem_cache_t *ptl_me_slab;
-kmem_cache_t *ptl_eq_slab;
-
-atomic_t md_in_use_count;
-atomic_t msg_in_use_count;
-atomic_t me_in_use_count;
-atomic_t eq_in_use_count;
-
-/* NB zeroing in ctor and on freeing ensures items that
- * kmem_cache_validate() OK, but haven't been initialised
- * as an MD/ME/EQ can't have valid handles
- */
-static void
-ptl_md_slab_ctor (void *obj, kmem_cache_t *slab, unsigned long flags)
-{
- memset (obj, 0, sizeof (lib_md_t));
-}
-
-static void
-ptl_me_slab_ctor (void *obj, kmem_cache_t *slab, unsigned long flags)
-{
- memset (obj, 0, sizeof (lib_me_t));
-}
-
-static void
-ptl_eq_slab_ctor (void *obj, kmem_cache_t *slab, unsigned long flags)
-{
- memset (obj, 0, sizeof (lib_eq_t));
-}
+atomic_t md_in_use_count = ATOMIC_INIT(0);
+atomic_t msg_in_use_count = ATOMIC_INIT(0);
+atomic_t me_in_use_count = ATOMIC_INIT(0);
+atomic_t eq_in_use_count = ATOMIC_INIT(0);
int
kportal_descriptor_setup (nal_cb_t *nal)
{
- /* NB on failure caller must still call kportal_descriptor_cleanup */
- /* ****** */
-
- /* We'll have 1 set of slabs for ALL the nals :) */
-
- if (ptl_slab_users++)
- return 0;
-
- ptl_md_slab = kmem_cache_create("portals_MD",
- sizeof(lib_md_t), 0,
- SLAB_HWCACHE_ALIGN,
- ptl_md_slab_ctor, NULL);
- if (!ptl_md_slab) {
- CERROR("couldn't allocate ptl_md_t slab");
- RETURN (PTL_NOSPACE);
- }
-
- /* NB no ctor for msgs; they don't need handle verification */
- ptl_msg_slab = kmem_cache_create("portals_MSG",
- sizeof(lib_msg_t), 0,
- SLAB_HWCACHE_ALIGN,
- NULL, NULL);
- if (!ptl_msg_slab) {
- CERROR("couldn't allocate ptl_msg_t slab");
- RETURN (PTL_NOSPACE);
- }
-
- ptl_me_slab = kmem_cache_create("portals_ME",
- sizeof(lib_me_t), 0,
- SLAB_HWCACHE_ALIGN,
- ptl_me_slab_ctor, NULL);
- if (!ptl_me_slab) {
- CERROR("couldn't allocate ptl_me_t slab");
- RETURN (PTL_NOSPACE);
- }
-
- ptl_eq_slab = kmem_cache_create("portals_EQ",
- sizeof(lib_eq_t), 0,
- SLAB_HWCACHE_ALIGN,
- ptl_eq_slab_ctor, NULL);
- if (!ptl_eq_slab) {
- CERROR("couldn't allocate ptl_eq_t slab");
- RETURN (PTL_NOSPACE);
- }
-
+ ptl_slab_users++;
RETURN(PTL_OK);
}
void
kportal_descriptor_cleanup (nal_cb_t *nal)
{
- int rc;
-
if (--ptl_slab_users != 0)
return;
LASSERT (atomic_read (&me_in_use_count) == 0);
LASSERT (atomic_read (&eq_in_use_count) == 0);
LASSERT (atomic_read (&msg_in_use_count) == 0);
-
- if (ptl_md_slab != NULL) {
- rc = kmem_cache_destroy(ptl_md_slab);
- if (rc != 0)
- CERROR("unable to free MD slab\n");
- }
- if (ptl_msg_slab != NULL) {
- rc = kmem_cache_destroy(ptl_msg_slab);
- if (rc != 0)
- CERROR("unable to free MSG slab\n");
- }
- if (ptl_me_slab != NULL) {
- rc = kmem_cache_destroy(ptl_me_slab);
- if (rc != 0)
- CERROR("unable to free ME slab\n");
- }
- if (ptl_eq_slab != NULL) {
- rc = kmem_cache_destroy(ptl_eq_slab);
- if (rc != 0)
- CERROR("unable to free EQ slab\n");
- }
}
#else
git://git.whamcloud.com / fs / lustre-release.git / commitdiff