From 5b6be598455fa90e43c8c1511057e035ccb3d901 Mon Sep 17 00:00:00 2001 From: bobijam Date: Thu, 26 Jul 2007 06:24:20 +0000 Subject: [PATCH] Branch b1_6 b=12211 i=adilger i=green Description: randomly memory allocation failure util Details : Make lustre randomly failed allocating memory for testing purpose. --- lustre/ChangeLog | 5 ++++ lustre/autoconf/lustre-core.m4 | 10 ++++++++ lustre/include/linux/lvfs.h | 4 +++ lustre/include/obd_support.h | 48 +++++++++++++++++++++++++++--------- lustre/lvfs/Makefile.in | 2 +- lustre/lvfs/autoMakefile.am | 2 +- lustre/lvfs/fsfilt_ext3.c | 1 - lustre/lvfs/fsfilt_reiserfs.c | 1 - lustre/lvfs/lvfs_lib.c | 46 ++++++++++++++++++++++++++++++++++ lustre/lvfs/lvfs_linux.c | 8 +++++- lustre/{obdclass => lvfs}/prng.c | 0 lustre/obdclass/Makefile.in | 2 +- lustre/obdclass/autoMakefile.am | 4 +-- lustre/obdclass/class_obd.c | 7 +++--- lustre/obdclass/linux/linux-sysctl.c | 41 ++++++++++++++++++++++++++++++ 15 files changed, 157 insertions(+), 24 deletions(-) create mode 100644 lustre/lvfs/lvfs_lib.c rename lustre/{obdclass => lvfs}/prng.c (100%) diff --git a/lustre/ChangeLog b/lustre/ChangeLog index 4e238cb..d01b4c8 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -28,6 +28,11 @@ Details : llog ctxt refcount was added to avoide the race between ctxt free before it access the llog. And the llog ctxt can only be freed when its refcount is zero. +Severity : enhancement +Bugzilla : 12211 +Description: randomly memory allocation failure util +Details : Make lustre randomly failed allocating memory for testing purpose. + -------------------------------------------------------------------------------- 2007-07-30 Cluster File Systems, Inc. diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4 index fbe108fb..068a8ac 100644 --- a/lustre/autoconf/lustre-core.m4 +++ b/lustre/autoconf/lustre-core.m4 @@ -1370,6 +1370,16 @@ if test "$enable_mindf" = "yes" ; then AC_DEFINE([MIN_DF], 1, [Report minimum OST free space]) fi +AC_ARG_ENABLE([fail_alloc], + AC_HELP_STRING([--disable-fail-alloc], + [disable randomly alloc failure]), + [],[enable_fail_alloc=yes]) +AC_MSG_CHECKING([whether to randomly failing memory alloc]) +AC_MSG_RESULT([$enable_fail_alloc]) +if test x$enable_fail_alloc != xno ; then + AC_DEFINE([RANDOM_FAIL_ALLOC], 1, [enable randomly alloc failure]) +fi + ]) # diff --git a/lustre/include/linux/lvfs.h b/lustre/include/linux/lvfs.h index 085ffb5..04829c5 100644 --- a/lustre/include/linux/lvfs.h +++ b/lustre/include/linux/lvfs.h @@ -37,6 +37,10 @@ struct group_info { /* unused */ }; #define LLOG_LVFS +/* lvfs.c */ +int obd_alloc_fail(const void *ptr, const char *name, const char *type, + size_t size, const char *file, int line); + /* simple.c */ struct lvfs_ucred { diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index 3fdc2fa..430a059 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -24,6 +24,7 @@ #define _OBD_SUPPORT #include +#include /* global variables */ extern atomic_t obd_memory; @@ -44,6 +45,7 @@ extern unsigned int obd_max_dirty_pages; extern atomic_t obd_dirty_pages; extern cfs_waitq_t obd_race_waitq; extern int obd_race_state; +extern unsigned int obd_alloc_fail_rate; #define OBD_FAIL_MDS 0x100 @@ -211,6 +213,8 @@ extern int obd_race_state; #define OBD_FAIL_LPROC_REMOVE 0xB00 +#define OBD_FAIL_GENERAL_ALLOC 0xC00 + /* Failure injection control */ #define OBD_FAIL_MASK_SYS 0x0000FF00 #define OBD_FAIL_MASK_LOC (0x000000FF | OBD_FAIL_MASK_SYS) @@ -344,6 +348,16 @@ do { \ extern atomic_t libcfs_kmemory; +#ifdef RANDOM_FAIL_ALLOC +#define HAS_FAIL_ALLOC_FLAG OBD_FAIL_CHECK(OBD_FAIL_GENERAL_ALLOC) +#else +#define HAS_FAIL_ALLOC_FLAG 0 +#endif + +#define OBD_ALLOC_FAIL_BITS 24 +#define OBD_ALLOC_FAIL_MASK ((1 << OBD_ALLOC_FAIL_BITS) - 1) +#define OBD_ALLOC_FAIL_MULT (OBD_ALLOC_FAIL_MASK / 100) + #if defined(LUSTRE_UTILS) /* this version is for utils only */ #define OBD_ALLOC_GFP(ptr, size, gfp_mask) \ do { \ @@ -358,15 +372,20 @@ do { \ } \ } while (0) #else /* this version is for the kernel and liblustre */ +#define OBD_FREE_RTN0(ptr) \ +({ \ + cfs_free(ptr); \ + (ptr) = NULL; \ + 0; \ +}) #define OBD_ALLOC_GFP(ptr, size, gfp_mask) \ do { \ (ptr) = cfs_alloc(size, (gfp_mask)); \ - if ((ptr) == NULL) { \ - CERROR("kmalloc of '" #ptr "' (%d bytes) failed at %s:%d\n", \ - (int)(size), __FILE__, __LINE__); \ - CERROR("%d total bytes allocated by Lustre, %d by Portals\n", \ - atomic_read(&obd_memory), atomic_read(&libcfs_kmemory));\ - } else { \ + if (likely((ptr) != NULL && \ + (!HAS_FAIL_ALLOC_FLAG || obd_alloc_fail_rate == 0 || \ + !obd_alloc_fail(ptr, #ptr, "km", size, \ + __FILE__, __LINE__) || \ + OBD_FREE_RTN0(ptr)))){ \ memset(ptr, 0, size); \ atomic_add(size, &obd_memory); \ if (atomic_read(&obd_memory) > obd_memmax) \ @@ -478,16 +497,21 @@ do { \ /* we memset() the slab object to 0 when allocation succeeds, so DO NOT * HAVE A CTOR THAT DOES ANYTHING. its work will be cleared here. we'd * love to assert on that, but slab.c keeps kmem_cache_s all to itself. */ +#define OBD_SLAB_FREE_RTN0(ptr, slab) \ +({ \ + cfs_mem_cache_free((slab), (ptr)); \ + (ptr) = NULL; \ + 0; \ +}) #define OBD_SLAB_ALLOC(ptr, slab, type, size) \ do { \ LASSERT(!in_interrupt()); \ (ptr) = cfs_mem_cache_alloc(slab, (type)); \ - if ((ptr) == NULL) { \ - CERROR("slab-alloc of '"#ptr"' (%d bytes) failed at %s:%d\n", \ - (int)(size), __FILE__, __LINE__); \ - CERROR("%d total bytes allocated by Lustre, %d by Portals\n", \ - atomic_read(&obd_memory), atomic_read(&libcfs_kmemory));\ - } else { \ + if (likely((ptr) != NULL && \ + (!HAS_FAIL_ALLOC_FLAG || obd_alloc_fail_rate == 0 || \ + !obd_alloc_fail(ptr, #ptr, "slab-", size, \ + __FILE__, __LINE__) || \ + OBD_SLAB_FREE_RTN0(ptr, slab)))) { \ memset(ptr, 0, size); \ atomic_add(size, &obd_memory); \ if (atomic_read(&obd_memory) > obd_memmax) \ diff --git a/lustre/lvfs/Makefile.in b/lustre/lvfs/Makefile.in index afa2511..f103b62 100644 --- a/lustre/lvfs/Makefile.in +++ b/lustre/lvfs/Makefile.in @@ -2,7 +2,7 @@ MODULES := lvfs @SERVER_TRUE@MODULES += fsfilt_@BACKINGFS@ @QUOTA_TRUE@MODULES += quotafmt_test -lvfs-objs := lvfs_common.o lvfs_linux.o fsfilt.o upcall_cache.o +lvfs-objs := lvfs_common.o lvfs_linux.o fsfilt.o upcall_cache.o prng.o lvfs_lib.o @QUOTA_TRUE@lvfs-objs += lustre_quota_fmt.o @QUOTA_TRUE@quotafmt-objs := quotafmt_test.o diff --git a/lustre/lvfs/autoMakefile.am b/lustre/lvfs/autoMakefile.am index 644e10a..b16f2a3 100644 --- a/lustre/lvfs/autoMakefile.am +++ b/lustre/lvfs/autoMakefile.am @@ -4,7 +4,7 @@ # See the file COPYING in this distribution if LIBLUSTRE noinst_LIBRARIES = liblvfs.a -liblvfs_a_SOURCES = lvfs_userfs.c +liblvfs_a_SOURCES = lvfs_userfs.c prng.c lvfs_lib.c liblvfs_a_CFLAGS = $(LLCFLAGS) liblvfs_a_CPPFLAGS = $(LLCPPFLAGS) diff --git a/lustre/lvfs/fsfilt_ext3.c b/lustre/lvfs/fsfilt_ext3.c index 261d439..035a355 100644 --- a/lustre/lvfs/fsfilt_ext3.c +++ b/lustre/lvfs/fsfilt_ext3.c @@ -48,7 +48,6 @@ #include #include #include -#include #include #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0)) #include diff --git a/lustre/lvfs/fsfilt_reiserfs.c b/lustre/lvfs/fsfilt_reiserfs.c index 3441c96..d575bfe 100644 --- a/lustre/lvfs/fsfilt_reiserfs.c +++ b/lustre/lvfs/fsfilt_reiserfs.c @@ -44,7 +44,6 @@ #include #include #include -#include #include #include diff --git a/lustre/lvfs/lvfs_lib.c b/lustre/lvfs/lvfs_lib.c new file mode 100644 index 0000000..9a185c8 --- /dev/null +++ b/lustre/lvfs/lvfs_lib.c @@ -0,0 +1,46 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * lustre/lvfs/lvfs_lib.c + * Lustre filesystem abstraction routines + * + * Copyright (C) 2007 Cluster File Systems, Inc. + * Author: Andreas Dilger + * + * This file is part of Lustre, http://www.lustre.org. + * + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Lustre is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ +#ifdef __KERNEL__ +#include +#include +#else +#include +#endif +#include + +int obd_alloc_fail(const void *ptr, const char *name, const char *type, + size_t size, const char *file, int line) +{ + if (ptr == NULL || + (ll_rand() & OBD_ALLOC_FAIL_MASK) < obd_alloc_fail_rate) { + CERROR("%s%salloc of %s (%u bytes) failed at %s:%d\n", + ptr ? "force " :"", type, name, size, file, line); + CERROR("%d total bytes allocated by Lustre, %d by Portals\n", + atomic_read(&obd_memory), atomic_read(&libcfs_kmemory)); + return 1; + } + return 0; +} +EXPORT_SYMBOL(obd_alloc_fail); diff --git a/lustre/lvfs/lvfs_linux.c b/lustre/lvfs/lvfs_linux.c index 53147a4..f7cfe13 100644 --- a/lustre/lvfs/lvfs_linux.c +++ b/lustre/lvfs/lvfs_linux.c @@ -39,7 +39,6 @@ #include #include #include -#include #include #include #include @@ -52,6 +51,9 @@ atomic_t obd_memory; int obd_memmax; +unsigned int obd_fail_val; +unsigned int obd_fail_loc; +unsigned int obd_alloc_fail_rate = 0; /* Debugging check only needed during development */ #ifdef OBD_CTXT_DEBUG @@ -528,6 +530,10 @@ static void __exit lvfs_linux_exit(void) return; } +EXPORT_SYMBOL(obd_fail_loc); +EXPORT_SYMBOL(obd_alloc_fail_rate); +EXPORT_SYMBOL(obd_fail_val); + MODULE_AUTHOR("Cluster File Systems, Inc. "); MODULE_DESCRIPTION("Lustre VFS Filesystem Helper v0.1"); MODULE_LICENSE("GPL"); diff --git a/lustre/obdclass/prng.c b/lustre/lvfs/prng.c similarity index 100% rename from lustre/obdclass/prng.c rename to lustre/lvfs/prng.c diff --git a/lustre/obdclass/Makefile.in b/lustre/obdclass/Makefile.in index 1019c7e..d81ee84 100644 --- a/lustre/obdclass/Makefile.in +++ b/lustre/obdclass/Makefile.in @@ -23,7 +23,7 @@ obdclass-all-objs := llog.o llog_cat.o llog_lvfs.o llog_obd.o llog_swab.o obdclass-all-objs += class_obd.o class_hash.o obdclass-all-objs += debug.o genops.o uuid.o llog_ioctl.o obdclass-all-objs += lprocfs_status.o lustre_handles.o lustre_peer.o -obdclass-all-objs += statfs_pack.o obdo.o obd_config.o obd_mount.o prng.o +obdclass-all-objs += statfs_pack.o obdo.o obd_config.o obd_mount.o obdclass-objs := $(obdclass-linux-objs) $(obdclass-all-objs) diff --git a/lustre/obdclass/autoMakefile.am b/lustre/obdclass/autoMakefile.am index 956b153..40d964c 100644 --- a/lustre/obdclass/autoMakefile.am +++ b/lustre/obdclass/autoMakefile.am @@ -11,7 +11,7 @@ liblustreclass_a_SOURCES = class_obd.c debug.c genops.c statfs_pack.c uuid.c liblustreclass_a_SOURCES += lustre_handles.c lustre_peer.c lprocfs_status.c class_hash.c liblustreclass_a_SOURCES += obdo.c obd_config.c llog.c llog_obd.c llog_cat.c liblustreclass_a_SOURCES += llog_lvfs.c llog_swab.c -liblustreclass_a_SOURCES += prng.c #llog_ioctl.c rbtree.c +liblustreclass_a_SOURCES += #llog_ioctl.c rbtree.c liblustreclass_a_CPPFLAGS = $(LLCPPFLAGS) liblustreclass_a_CFLAGS = $(LLCFLAGS) @@ -32,7 +32,7 @@ obdclass_SOURCES := \ darwin/darwin-module.c darwin/darwin-sysctl.c \ class_obd.c genops.c lprocfs_status.c \ lustre_handles.c lustre_peer.c obd_config.c \ - obdo.c debug.c llog_ioctl.c uuid.c prng.c \ + obdo.c debug.c llog_ioctl.c uuid.c \ llog_swab.c llog_obd.c llog.c llog_cat.c llog_lvfs.c obdclass_CFLAGS := $(EXTRA_KCFLAGS) diff --git a/lustre/obdclass/class_obd.c b/lustre/obdclass/class_obd.c index 9e6144e..76a65e2 100644 --- a/lustre/obdclass/class_obd.c +++ b/lustre/obdclass/class_obd.c @@ -54,11 +54,12 @@ spinlock_t obd_dev_lock = SPIN_LOCK_UNLOCKED; #ifndef __KERNEL__ atomic_t obd_memory; int obd_memmax; +unsigned int obd_fail_val; +unsigned int obd_fail_loc; +unsigned int obd_alloc_fail_rate; #endif /* The following are visible and mutable through /proc/sys/lustre/. */ -unsigned int obd_fail_loc; -unsigned int obd_fail_val; unsigned int obd_debug_peer_on_timeout; unsigned int obd_dump_on_timeout; unsigned int obd_dump_on_eviction; @@ -375,8 +376,6 @@ void *obd_psdev = NULL; #endif EXPORT_SYMBOL(obd_devs); -EXPORT_SYMBOL(obd_fail_loc); -EXPORT_SYMBOL(obd_fail_val); EXPORT_SYMBOL(obd_print_fail_loc); EXPORT_SYMBOL(obd_race_waitq); EXPORT_SYMBOL(obd_race_state); diff --git a/lustre/obdclass/linux/linux-sysctl.c b/lustre/obdclass/linux/linux-sysctl.c index 52253af..19fc3bb 100644 --- a/lustre/obdclass/linux/linux-sysctl.c +++ b/lustre/obdclass/linux/linux-sysctl.c @@ -45,6 +45,7 @@ #define DEBUG_SUBSYSTEM S_CLASS #include +#include struct ctl_table_header *obd_table_header = NULL; @@ -60,6 +61,7 @@ enum { OBD_LDLM_TIMEOUT, /* LDLM timeout for ASTs before client eviction */ OBD_DUMP_ON_EVICTION, /* dump kernel debug log upon eviction */ OBD_DEBUG_PEER_ON_TIMEOUT, /* dump peer debug when RPC times out */ + OBD_ALLOC_FAIL_RATE, /* memory allocation random failure rate */ }; int LL_PROC_PROTO(proc_fail_loc) @@ -83,6 +85,41 @@ int LL_PROC_PROTO(proc_set_timeout) return rc; } +#ifdef RANDOM_FAIL_ALLOC +int LL_PROC_PROTO(proc_alloc_fail_rate) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,8) + loff_t *ppos = &filp->f_pos; +#endif + int rc = 0; + + if (!table->data || !table->maxlen || !*lenp || (*ppos && !write)) { + *lenp = 0; + return 0; + } + if (write) { + rc = lprocfs_write_frac_helper(buffer, *lenp, + (unsigned int*)table->data, + OBD_ALLOC_FAIL_MULT); + } else { + char buf[21]; + int len; + + len = lprocfs_read_frac_helper(buf, 21, + *(unsigned int*)table->data, + OBD_ALLOC_FAIL_MULT); + if (len > *lenp) + len = *lenp; + buf[len] = '\0'; + if (copy_to_user(buffer, buf, len)) + return -EFAULT; + *lenp = len; + } + *ppos += *lenp; + return rc; +} +#endif + static ctl_table obd_table[] = { {OBD_FAIL_LOC, "fail_loc", &obd_fail_loc, sizeof(int), 0644, NULL, &proc_fail_loc}, @@ -101,6 +138,10 @@ static ctl_table obd_table[] = { sizeof(int), 0644, NULL, &proc_dointvec}, {OBD_LDLM_TIMEOUT, "ldlm_timeout", &ldlm_timeout, sizeof(int), 0644, NULL, &proc_set_timeout}, +#ifdef RANDOM_FAIL_ALLOC + {OBD_ALLOC_FAIL_RATE, "alloc_fail_rate", &obd_alloc_fail_rate, + sizeof(int), 0644, NULL, &proc_alloc_fail_rate}, +#endif { 0 } }; -- 1.8.3.1