Whamcloud - gitweb
Branch HEAD
authorfanyong <fanyong>
Thu, 13 Dec 2007 16:02:35 +0000 (16:02 +0000)
committerfanyong <fanyong>
Thu, 13 Dec 2007 16:02:35 +0000 (16:02 +0000)
b=12974
i=nikita
i=adilger

1) set as "inline" for such large function ("obd_fail_check()") is not suitable,
   replaced with normal function "__obd_fail_check_set()".
2) atomic operations for obd_fail_count (replace "static count" in obd_fail_check).
3) use "test_and_set_bit()" for "OBD_FAILED/OBD_FAIL_ONCE" check and set to avoid
   unexpected racing condition between multi-thread ops.
4) some cleanup for OBD_FAIL_XXX macro.

22 files changed:
lustre/include/linux/obd_support.h
lustre/include/obd_support.h
lustre/ldlm/ldlm_lib.c
lustre/ldlm/ldlm_lockd.c
lustre/lvfs/lvfs_lib.c
lustre/mdd/mdd_lov.c
lustre/mds/handler.c
lustre/mds/mds_fs.c
lustre/mds/mds_open.c
lustre/mdt/mdt_handler.c
lustre/mdt/mdt_internal.h
lustre/mdt/mdt_open.c
lustre/mdt/mdt_recovery.c
lustre/obdclass/class_obd.c
lustre/obdclass/darwin/darwin-sysctl.c
lustre/obdclass/linux/linux-sysctl.c
lustre/obdclass/llog_cat.c
lustre/osc/osc_request.c
lustre/ost/ost_handler.c
lustre/ptlrpc/gss/sec_gss.c
lustre/ptlrpc/niobuf.c
lustre/quota/quota_master.c

index 858a04b..5f89994 100644 (file)
@@ -74,25 +74,7 @@ static inline __u32 crc32_le(__u32 crc, unsigned char const *p, size_t len)
 # include <linux/types.h>
 # include <linux/blkdev.h>
 # include <lvfs.h>
-
-static inline void OBD_FAIL_WRITE(int id, struct super_block *sb)
-{
-        if (OBD_FAIL_CHECK(id)) {
-#ifdef LIBCFS_DEBUG
-                BDEVNAME_DECLARE_STORAGE(tmp);
-                CERROR("obd_fail_loc=%x, fail write operation on %s\n",
-                       id, ll_bdevname(sb, tmp));
-#endif
-                /* TODO-CMD: fix getting jdev */
-                __lvfs_set_rdonly(lvfs_sbdev(sb), (lvfs_sbdev_type)0);
-                /* We set FAIL_ONCE because we never "un-fail" a device */
-                obd_fail_loc |= OBD_FAILED | OBD_FAIL_ONCE;
-        }
-}
-
-#define OBD_SLEEP_ON(wq, state)  wait_event_interruptible(wq, state)
-
-
+# define OBD_SLEEP_ON(wq, state)  wait_event_interruptible(wq, state)
 #else /* !__KERNEL__ */
 # define LTIME_S(time) (time)
 /* for obd_class.h */
index befcde8..8d86823 100644 (file)
 #include <lvfs.h>
 #include <lprocfs_status.h>
 
+#if defined(__linux__)
+#include <linux/obd_support.h>
+#elif defined(__APPLE__)
+#include <darwin/obd_support.h>
+#elif defined(__WINNT__)
+#include <winnt/obd_support.h>
+#else
+#error Unsupported operating system.
+#endif
+
 /* global variables */
 extern struct lprocfs_stats *obd_memory;
 enum {
@@ -35,7 +45,13 @@ enum {
         OBD_STATS_NUM,
 };
 
-extern unsigned int obd_fail_loc;
+enum {
+        OBD_FAIL_LOC_NOSET      = 0,
+        OBD_FAIL_LOC_ORSET      = 1,
+        OBD_FAIL_LOC_RESET      = 2
+};
+
+extern unsigned long obd_fail_loc;
 extern unsigned int obd_fail_val;
 extern unsigned int obd_debug_peer_on_timeout;
 extern unsigned int obd_dump_on_timeout;
@@ -52,6 +68,8 @@ extern cfs_waitq_t obd_race_waitq;
 extern int obd_race_state;
 extern unsigned int obd_alloc_fail_rate;
 
+int __obd_fail_check_set(__u32 id, __u32 value, int set);
+
 /* Timeout definitions */
 #define LDLM_TIMEOUT_DEFAULT 20
 #define OBD_TIMEOUT_DEFAULT 100
@@ -240,128 +258,109 @@ extern unsigned int obd_alloc_fail_rate;
 /* Failure injection control */
 #define OBD_FAIL_MASK_SYS    0x0000FF00
 #define OBD_FAIL_MASK_LOC   (0x000000FF | OBD_FAIL_MASK_SYS)
-#define OBD_FAIL_ONCE        0x80000000
-#define OBD_FAILED           0x40000000
+
+#define OBD_FAILED_BIT       30
+/* OBD_FAILED is 0x40000000 */
+#define OBD_FAILED          (1 << OBD_FAILED_BIT)
+
+#define OBD_FAIL_ONCE_BIT    31
+/* OBD_FAIL_ONCE is 0x80000000 */
+#define OBD_FAIL_ONCE       (1 << OBD_FAIL_ONCE_BIT)
 
 /* The following flags aren't made to be combined */
-#define OBD_FAIL_SKIP        0x20000000 /* skip N then fail */
-#define OBD_FAIL_SOME        0x10000000 /* fail N times */
-#define OBD_FAIL_RAND        0x08000000 /* fail 1/N of the time */
+#define OBD_FAIL_SKIP        0x20000000 /* skip N times then fail */
+#define OBD_FAIL_SOME        0x10000000 /* only fail N times */
+#define OBD_FAIL_RAND        0x08000000 /* fail 1/N of the times */
 #define OBD_FAIL_USR1        0x04000000 /* user flag */
 
-static inline int obd_fail_check(__u32 id)
+#define OBD_FAIL_PRECHECK(id) (obd_fail_loc &&                                \
+                              (obd_fail_loc & OBD_FAIL_MASK_LOC) ==           \
+                              ((id) & OBD_FAIL_MASK_LOC))
+
+static inline int obd_fail_check_set(__u32 id, __u32 value, int set)
 {
-        static int count = 0;
-        if (likely((obd_fail_loc & OBD_FAIL_MASK_LOC) != 
-                   (id & OBD_FAIL_MASK_LOC)))
-                return 0;
-        
-        if ((obd_fail_loc & (OBD_FAILED | OBD_FAIL_ONCE)) ==
-            (OBD_FAILED | OBD_FAIL_ONCE)) {
-                count = 0; /* paranoia */
-                return 0;
+        int ret = 0;
+        if (unlikely(OBD_FAIL_PRECHECK(id) &&
+            (ret = __obd_fail_check_set(id, value, set)))) {
+                CERROR("*** obd_fail_loc=%x ***\n", id);
         }
+        return ret;
+}
 
-        if (obd_fail_loc & OBD_FAIL_RAND) {
-                unsigned int ll_rand(void);
-                if (obd_fail_val < 2)
-                        return 0;
-                if (ll_rand() % obd_fail_val > 0)
-                        return 0;
-        }
+/* If id hit obd_fail_loc, return 1, otherwise return 0 */
+#define OBD_FAIL_CHECK(id) \
+        obd_fail_check_set(id, 0, OBD_FAIL_LOC_NOSET)
 
-        if (obd_fail_loc & OBD_FAIL_SKIP) {
-                count++;
-                if (count < obd_fail_val) 
-                        return 0;
-                count = 0;
-        }
+/* If id hit obd_fail_loc, obd_fail_loc |= value and return 1,
+ * otherwise return 0 */
+#define OBD_FAIL_CHECK_ORSET(id, value) \
+        obd_fail_check_set(id, value, OBD_FAIL_LOC_ORSET)
 
-        /* Overridden by FAIL_ONCE */
-        if (obd_fail_loc & OBD_FAIL_SOME) {
-                count++;
-                if (count >= obd_fail_val) {
-                        count = 0;
-                        /* Don't fail anymore */
-                        obd_fail_loc |= OBD_FAIL_ONCE;
-                }
-        }
+/* If id hit obd_fail_loc, obd_fail_loc = value and return 1,
+ * otherwise return 0 */
+#define OBD_FAIL_CHECK_RESET(id, value) \
+        obd_fail_check_set(id, value, OBD_FAIL_LOC_RESET)
 
-        obd_fail_loc |= OBD_FAILED;
-        /* Handle old checks that OR in this */
-        if (id & OBD_FAIL_ONCE)
-                obd_fail_loc |= OBD_FAIL_ONCE;
 
-        return 1;
+static inline int obd_fail_timeout_set(__u32 id, __u32 value, int secs, int set)
+{
+        int ret = 0;
+        if (unlikely(OBD_FAIL_PRECHECK(id) &&
+            (ret = __obd_fail_check_set(id, value, set)))) {
+                CERROR("obd_fail_timeout id %x sleeping for %d secs\n",
+                       id, secs);
+                set_current_state(TASK_UNINTERRUPTIBLE);
+                cfs_schedule_timeout(CFS_TASK_UNINT,  cfs_time_seconds(secs));
+                set_current_state(TASK_RUNNING);
+                CERROR("obd_fail_timeout id %x awake\n", id);
+        }
+        return ret;
 }
 
-#define OBD_FAIL_CHECK(id)                                                   \
-({                                                                           \
-        int _ret_ = 0;                                                       \
-        if (unlikely(obd_fail_loc && (_ret_ = obd_fail_check(id)))) {        \
-                CERROR("*** obd_fail_loc=%x ***\n", id);                     \
-        }                                                                    \
-        _ret_;                                                               \
-})
+/* If id hit obd_fail_loc, sleep secs */
+#define OBD_FAIL_TIMEOUT(id, secs) \
+        obd_fail_timeout_set(id, 0, secs, OBD_FAIL_LOC_NOSET)
 
-/* deprecated - just use OBD_FAIL_CHECK */
-#define OBD_FAIL_CHECK_ONCE OBD_FAIL_CHECK
-
-#define OBD_FAIL_RETURN(id, ret)                                             \
-do {                                                                         \
-        if (unlikely(obd_fail_loc && obd_fail_check(id))) {                  \
-                CERROR("*** obd_fail_return=%x rc=%d ***\n", id, ret);       \
-                RETURN(ret);                                                 \
-        }                                                                    \
-} while(0)
-
-#define OBD_FAIL_TIMEOUT(id, secs)                                           \
-({      int _ret_ = 0;                                                       \
-        if (unlikely(obd_fail_loc && (_ret_ = obd_fail_check(id)))) {        \
-                CERROR("obd_fail_timeout id %x sleeping for %d secs\n",      \
-                       (id), (secs));                                        \
-                set_current_state(TASK_UNINTERRUPTIBLE);                     \
-                cfs_schedule_timeout(CFS_TASK_UNINT,                         \
-                                    cfs_time_seconds(secs));                 \
-                set_current_state(TASK_RUNNING);                             \
-                CERROR("obd_fail_timeout id %x awake\n", (id));              \
-        }                                                                    \
-        _ret_;                                                               \
-})
-
-#define OBD_FAIL_TIMEOUT_MS(id, ms)                                          \
-({      int _ret_ = 0;                                                       \
-        if (unlikely(obd_fail_loc && (_ret_ = obd_fail_check(id)))) {        \
-                CERROR("obd_fail_timeout id %x sleeping for %d ms\n",        \
-                       (id), (ms));                                          \
-                set_current_state(TASK_UNINTERRUPTIBLE);                     \
-                cfs_schedule_timeout(CFS_TASK_UNINT,                         \
-                                     cfs_time_seconds(ms)/1000);             \
-                set_current_state(TASK_RUNNING);                             \
-                CERROR("obd_fail_timeout id %x awake\n", (id));              \
-        }                                                                    \
-        _ret_;                                                               \
-})
+/* If id hit obd_fail_loc, obd_fail_loc |= value and sleep secs */
+#define OBD_FAIL_TIMEOUT_ORSET(id, value, secs) \
+        obd_fail_timeout_set(id, value, secs, OBD_FAIL_LOC_ORSET)
 
 #ifdef __KERNEL__
+static inline void obd_fail_write(int id, struct super_block *sb)
+{
+        /* We set FAIL_ONCE because we never "un-fail" a device */
+        if (OBD_FAIL_CHECK_ORSET(id & ~OBD_FAIL_ONCE, OBD_FAIL_ONCE)) {
+#ifdef LIBCFS_DEBUG
+                BDEVNAME_DECLARE_STORAGE(tmp);
+                CERROR("obd_fail_loc=%x, fail write operation on %s\n",
+                       id, ll_bdevname(sb, tmp));
+#endif
+                /* TODO-CMD: fix getting jdev */
+                __lvfs_set_rdonly(lvfs_sbdev(sb), (lvfs_sbdev_type)0);
+        }
+}
+#define OBD_FAIL_WRITE(id, sb) obd_fail_write(id, sb)
+
 /* The idea here is to synchronise two threads to force a race. The
  * first thread that calls this with a matching fail_loc is put to
  * sleep. The next thread that calls with the same fail_loc wakes up
  * the first and continues. */
-#define OBD_RACE(id)                                                         \
-do {                                                                         \
-        if (unlikely(obd_fail_loc && obd_fail_check(id))) {                  \
-                obd_race_state = 0;                                          \
-                CERROR("obd_race id %x sleeping\n", (id));                   \
-                OBD_SLEEP_ON(obd_race_waitq, obd_race_state != 0);           \
-                CERROR("obd_fail_race id %x awake\n", (id));                 \
-        } else if ((obd_fail_loc & OBD_FAIL_MASK_LOC) ==                     \
-                    ((id) & OBD_FAIL_MASK_LOC)) {                            \
-                CERROR("obd_fail_race id %x waking\n", (id));                \
-                obd_race_state = 1;                                          \
-                wake_up(&obd_race_waitq);                                    \
-        }                                                                    \
-} while(0)
+static inline void obd_race(__u32 id)
+{
+        if (OBD_FAIL_PRECHECK(id)) {
+                if (unlikely(__obd_fail_check_set(id, 0, OBD_FAIL_LOC_NOSET))) {
+                        obd_race_state = 0;
+                        CERROR("obd_race id %x sleeping\n", id);
+                        OBD_SLEEP_ON(obd_race_waitq, obd_race_state != 0);
+                        CERROR("obd_fail_race id %x awake\n", id);
+                } else {
+                        CERROR("obd_fail_race id %x waking\n", id);
+                        obd_race_state = 1;
+                        wake_up(&obd_race_waitq);
+                }
+        }
+}
+#define OBD_RACE(id) obd_race(id)
 #else
 /* sigh.  an expedient fix until OBD_RACE is fixed up */
 #define OBD_RACE(foo) do {} while(0)
@@ -782,14 +781,4 @@ do {                                                                          \
 
 #define OBD_PAGE_FREE(ptr) OBD_PAGES_FREE(ptr, 0)
 
-#if defined(__linux__)
-#include <linux/obd_support.h>
-#elif defined(__APPLE__)
-#include <darwin/obd_support.h>
-#elif defined(__WINNT__)
-#include <winnt/obd_support.h>
-#else
-#error Unsupported operating system.
-#endif
-
 #endif
index 3303ab5..7d4d43a 100644 (file)
@@ -1828,8 +1828,7 @@ int target_pack_pool_reply(struct ptlrpc_request *req)
 
 int target_send_reply_msg(struct ptlrpc_request *req, int rc, int fail_id)
 {
-        if (OBD_FAIL_CHECK(fail_id | OBD_FAIL_ONCE)) {
-                obd_fail_loc |= OBD_FAIL_ONCE | OBD_FAILED;
+        if (OBD_FAIL_CHECK_ORSET(fail_id & ~OBD_FAIL_ONCE, OBD_FAIL_ONCE)) {
                 DEBUG_REQ(D_ERROR, req, "dropping reply");
                 return (-ECOMM);
         }
index ef065d5..1f63e1b 100644 (file)
@@ -949,7 +949,7 @@ existing_lock:
                 }
                 unlock_res_and_lock(lock);
 
-                if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_LDLM_ENQUEUE_EXTENT_ERR))
+                if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_ENQUEUE_EXTENT_ERR))
                         GOTO(out, rc = -ENOMEM);
 
                 rc = lustre_pack_reply(req, buffers, size, NULL);
@@ -985,7 +985,7 @@ existing_lock:
         /* Don't move a pending lock onto the export if it has already
          * been evicted.  Cancel it now instead. (bug 5683) */
         if (unlikely(req->rq_export->exp_failed ||
-                     OBD_FAIL_CHECK_ONCE(OBD_FAIL_LDLM_ENQUEUE_OLD_EXPORT))) {
+                     OBD_FAIL_CHECK(OBD_FAIL_LDLM_ENQUEUE_OLD_EXPORT))) {
                 LDLM_ERROR(lock, "lock on destroyed export %p", req->rq_export);
                 rc = -ENOTCONN;
         } else if (lock->l_flags & LDLM_FL_AST_SENT) {
@@ -1502,21 +1502,26 @@ static int ldlm_callback_handler(struct ptlrpc_request *req)
 
         switch (lustre_msg_get_opc(req->rq_reqmsg)) {
         case LDLM_BL_CALLBACK:
-                OBD_FAIL_RETURN(OBD_FAIL_LDLM_BL_CALLBACK, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_BL_CALLBACK))
+                        RETURN(0);
                 break;
         case LDLM_CP_CALLBACK:
-                OBD_FAIL_RETURN(OBD_FAIL_LDLM_CP_CALLBACK, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_CP_CALLBACK))
+                        RETURN(0);
                 break;
         case LDLM_GL_CALLBACK:
-                OBD_FAIL_RETURN(OBD_FAIL_LDLM_GL_CALLBACK, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_GL_CALLBACK))
+                        RETURN(0);
                 break;
         case OBD_LOG_CANCEL: /* remove this eventually - for 1.4.0 compat */
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOG_CANCEL_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOG_CANCEL_NET))
+                        RETURN(0);
                 rc = llog_origin_handle_cancel(req);
                 ldlm_callback_reply(req, rc);
                 RETURN(0);
         case OBD_QC_CALLBACK:
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_QC_CALLBACK_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_QC_CALLBACK_NET))
+                        RETURN(0);
                 rc = target_handle_qc_callback(req);
                 ldlm_callback_reply(req, rc);
                 RETURN(0);
@@ -1526,22 +1531,26 @@ static int ldlm_callback_handler(struct ptlrpc_request *req)
                 rc = target_handle_dqacq_callback(req);
                 RETURN(0);
         case LLOG_ORIGIN_HANDLE_CREATE:
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOGD_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOGD_NET))
+                        RETURN(0);
                 rc = llog_origin_handle_create(req);
                 ldlm_callback_reply(req, rc);
                 RETURN(0);
         case LLOG_ORIGIN_HANDLE_NEXT_BLOCK:
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOGD_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOGD_NET))
+                        RETURN(0);
                 rc = llog_origin_handle_next_block(req);
                 ldlm_callback_reply(req, rc);
                 RETURN(0);
         case LLOG_ORIGIN_HANDLE_READ_HEADER:
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOGD_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOGD_NET))
+                        RETURN(0);
                 rc = llog_origin_handle_read_header(req);
                 ldlm_callback_reply(req, rc);
                 RETURN(0);
         case LLOG_ORIGIN_HANDLE_CLOSE:
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOGD_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOGD_NET))
+                        RETURN(0);
                 rc = llog_origin_handle_close(req);
                 ldlm_callback_reply(req, rc);
                 RETURN(0);
@@ -1661,13 +1670,15 @@ static int ldlm_cancel_handler(struct ptlrpc_request *req)
         /* XXX FIXME move this back to mds/handler.c, bug 249 */
         case LDLM_CANCEL:
                 CDEBUG(D_INODE, "cancel\n");
-                OBD_FAIL_RETURN(OBD_FAIL_LDLM_CANCEL, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_CANCEL))
+                        RETURN(0);
                 rc = ldlm_handle_cancel(req);
                 if (rc)
                         break;
                 RETURN(0);
         case OBD_LOG_CANCEL:
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOG_CANCEL_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOG_CANCEL_NET))
+                        RETURN(0);
                 rc = llog_origin_handle_cancel(req);
                 ldlm_callback_reply(req, rc);
                 RETURN(0);
index 8ea2133..1306c06 100644 (file)
@@ -31,7 +31,7 @@
 #include <lustre_lib.h>
 
 unsigned int obd_fail_val = 0;
-unsigned int obd_fail_loc = 0;
+unsigned long obd_fail_loc = 0;
 unsigned int obd_alloc_fail_rate = 0;
 
 int obd_alloc_fail(const void *ptr, const char *name, const char *type,
@@ -55,6 +55,74 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type,
 }
 EXPORT_SYMBOL(obd_alloc_fail);
 
+int __obd_fail_check_set(__u32 id, __u32 value, int set)
+{
+        static atomic_t obd_fail_count = ATOMIC_INIT(0);
+
+        LASSERT(!(id & OBD_FAIL_ONCE));
+
+        if ((obd_fail_loc & (OBD_FAILED | OBD_FAIL_ONCE)) ==
+            (OBD_FAILED | OBD_FAIL_ONCE)) {
+                atomic_set(&obd_fail_count, 0); /* paranoia */
+                return 0;
+        }
+
+        /* Fail 1/obd_fail_val times */
+        if (obd_fail_loc & OBD_FAIL_RAND) {
+                if (obd_fail_val < 2 || ll_rand() % obd_fail_val > 0)
+                        return 0;
+        }
+
+        /* Skip the first obd_fail_val, then fail */
+        if (obd_fail_loc & OBD_FAIL_SKIP) {
+                if (atomic_inc_return(&obd_fail_count) <= obd_fail_val)
+                        return 0;
+        }
+
+        /* Fail obd_fail_val times, overridden by FAIL_ONCE */
+        if (obd_fail_loc & OBD_FAIL_SOME &&
+            (!(obd_fail_loc & OBD_FAIL_ONCE) || obd_fail_val <= 1)) { 
+                int count = atomic_inc_return(&obd_fail_count);
+
+                if (count >= obd_fail_val) {
+                        set_bit(OBD_FAIL_ONCE_BIT, &obd_fail_loc);
+                        atomic_set(&obd_fail_count, 0);
+                        /* we are lost race to increase obd_fail_count */
+                        if (count > obd_fail_val)
+                                return 0;
+                }
+        }
+
+        if ((set == OBD_FAIL_LOC_ORSET || set == OBD_FAIL_LOC_RESET) &&
+            (value & OBD_FAIL_ONCE))
+                set_bit(OBD_FAIL_ONCE_BIT, &obd_fail_loc);
+
+        /* Lost race to set OBD_FAILED_BIT. */
+        if (test_and_set_bit(OBD_FAILED_BIT, &obd_fail_loc)) {
+                /* If OBD_FAIL_ONCE is valid, only one process can fail,
+                 * otherwise multi-process can fail at the same time. */
+                if (obd_fail_loc & OBD_FAIL_ONCE)
+                        return 0;
+        }
+
+        switch (set) {
+                case OBD_FAIL_LOC_NOSET:
+                        break;
+                case OBD_FAIL_LOC_ORSET:
+                        obd_fail_loc |= value & ~(OBD_FAILED | OBD_FAIL_ONCE);
+                        break;
+                case OBD_FAIL_LOC_RESET:
+                        obd_fail_loc = value;
+                        break;
+                default:
+                        LASSERTF(0, "called with bad set %u\n", set);
+                        break;
+        }
+
+        return 1;
+}
+EXPORT_SYMBOL(__obd_fail_check_set);
+
 EXPORT_SYMBOL(obd_fail_loc);
 EXPORT_SYMBOL(obd_alloc_fail_rate);
 EXPORT_SYMBOL(obd_fail_val);
index 42d8fb5..b3ecae2 100644 (file)
@@ -382,7 +382,7 @@ int mdd_lov_create(const struct lu_env *env, struct mdd_device *mdd,
                 RETURN(0);
         }
 
-        if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_MDS_ALLOC_OBDO))
+        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_ALLOC_OBDO))
                 GOTO(out_ids, rc = -ENOMEM);
 
         LASSERT(lov_exp != NULL);
index bf008b0..21ae791 100644 (file)
@@ -519,7 +519,8 @@ static int mds_getstatus(struct ptlrpc_request *req)
         int rc, size[2] = { sizeof(struct ptlrpc_body), sizeof(*body) };
         ENTRY;
 
-        OBD_FAIL_RETURN(OBD_FAIL_MDS_GETSTATUS_PACK, req->rq_status = -ENOMEM);
+        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_GETSTATUS_PACK))
+                RETURN(req->rq_status = -ENOMEM);
         rc = lustre_pack_reply(req, 2, size, NULL);
         if (rc)
                 RETURN(req->rq_status = rc);
@@ -1177,7 +1178,8 @@ static int mds_readpage(struct ptlrpc_request *req, int offset)
         struct lvfs_ucred uc = {0,};
         ENTRY;
 
-        OBD_FAIL_RETURN(OBD_FAIL_MDS_READPAGE_PACK, -ENOMEM);
+        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_READPAGE_PACK))
+                RETURN(-ENOMEM);
         rc = lustre_pack_reply(req, 2, size, NULL);
         if (rc)
                 GOTO(out, rc);
@@ -1468,7 +1470,8 @@ int mds_handle(struct ptlrpc_request *req)
         struct obd_device *obd = NULL;
         ENTRY;
 
-        OBD_FAIL_RETURN(OBD_FAIL_MDS_ALL_REQUEST_NET | OBD_FAIL_ONCE, 0);
+        if (OBD_FAIL_CHECK_ORSET(OBD_FAIL_MDS_ALL_REQUEST_NET, OBD_FAIL_ONCE))
+                RETURN(0);
 
         LASSERT(current->journal_info == NULL);
 
@@ -1534,7 +1537,8 @@ int mds_handle(struct ptlrpc_request *req)
         switch (lustre_msg_get_opc(req->rq_reqmsg)) {
         case MDS_CONNECT:
                 DEBUG_REQ(D_INODE, req, "connect");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_CONNECT_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_CONNECT_NET))
+                        RETURN(0);
                 rc = target_handle_connect(req);
                 if (!rc) {
                         /* Now that we have an export, set mds. */
@@ -1551,39 +1555,45 @@ int mds_handle(struct ptlrpc_request *req)
 
         case MDS_DISCONNECT:
                 DEBUG_REQ(D_INODE, req, "disconnect");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_DISCONNECT_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_DISCONNECT_NET))
+                        RETURN(0);
                 rc = target_handle_disconnect(req);
                 req->rq_status = rc;            /* superfluous? */
                 break;
 
         case MDS_GETSTATUS:
                 DEBUG_REQ(D_INODE, req, "getstatus");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_GETSTATUS_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_GETSTATUS_NET))
+                        RETURN(0);
                 rc = mds_getstatus(req);
                 break;
 
         case MDS_GETATTR:
                 DEBUG_REQ(D_INODE, req, "getattr");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_GETATTR_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_GETATTR_NET))
+                        RETURN(0);
                 rc = mds_getattr(req, REQ_REC_OFF);
                 break;
 
         case MDS_SETXATTR:
                 DEBUG_REQ(D_INODE, req, "setxattr");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_SETXATTR_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_SETXATTR_NET))
+                        RETURN(0);
                 rc = mds_setxattr(req);
                 break;
 
         case MDS_GETXATTR:
                 DEBUG_REQ(D_INODE, req, "getxattr");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_GETXATTR_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_GETXATTR_NET))
+                        RETURN(0);
                 rc = mds_getxattr(req);
                 break;
 
         case MDS_GETATTR_NAME: {
                 struct lustre_handle lockh = { 0 };
                 DEBUG_REQ(D_INODE, req, "getattr_name");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_GETATTR_NAME_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_GETATTR_NAME_NET))
+                        RETURN(0);
 
                 /* If this request gets a reconstructed reply, we won't be
                  * acquiring any new locks in mds_getattr_lock, so we don't
@@ -1599,16 +1609,18 @@ int mds_handle(struct ptlrpc_request *req)
         }
         case MDS_STATFS:
                 DEBUG_REQ(D_INODE, req, "statfs");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_STATFS_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_STATFS_NET))
+                        RETURN(0);
                 rc = mds_statfs(req);
                 break;
 
         case MDS_READPAGE:
                 DEBUG_REQ(D_INODE, req, "readpage");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_READPAGE_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_READPAGE_NET))
+                        RETURN(0);
                 rc = mds_readpage(req, REQ_REC_OFF);
 
-                if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_MDS_SENDPAGE)) {
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_SENDPAGE)) {
                         RETURN(0);
                 }
 
@@ -1639,7 +1651,8 @@ int mds_handle(struct ptlrpc_request *req)
                            reint_names[opc] == NULL) ? reint_names[opc] :
                                                        "unknown opcode");
 
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_REINT_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_REINT_NET))
+                        RETURN(0);
 
                 if (opc == REINT_UNLINK || opc == REINT_RENAME)
                         bufcount = 4;
@@ -1659,25 +1672,29 @@ int mds_handle(struct ptlrpc_request *req)
 
         case MDS_CLOSE:
                 DEBUG_REQ(D_INODE, req, "close");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_CLOSE_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_CLOSE_NET))
+                        RETURN(0);
                 rc = mds_close(req, REQ_REC_OFF);
                 break;
 
         case MDS_DONE_WRITING:
                 DEBUG_REQ(D_INODE, req, "done_writing");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_DONE_WRITING_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_DONE_WRITING_NET))
+                        RETURN(0);
                 rc = mds_done_writing(req, REQ_REC_OFF);
                 break;
 
         case MDS_PIN:
                 DEBUG_REQ(D_INODE, req, "pin");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_PIN_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_PIN_NET))
+                        RETURN(0);
                 rc = mds_pin(req, REQ_REC_OFF);
                 break;
 
         case MDS_SYNC:
                 DEBUG_REQ(D_INODE, req, "sync");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_SYNC_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_SYNC_NET))
+                        RETURN(0);
                 rc = mds_sync(req, REQ_REC_OFF);
                 break;
 
@@ -1688,13 +1705,15 @@ int mds_handle(struct ptlrpc_request *req)
 
         case MDS_QUOTACHECK:
                 DEBUG_REQ(D_INODE, req, "quotacheck");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_QUOTACHECK_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_QUOTACHECK_NET))
+                        RETURN(0);
                 rc = mds_handle_quotacheck(req);
                 break;
 
         case MDS_QUOTACTL:
                 DEBUG_REQ(D_INODE, req, "quotactl");
-                OBD_FAIL_RETURN(OBD_FAIL_MDS_QUOTACTL_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_MDS_QUOTACTL_NET))
+                        RETURN(0);
                 rc = mds_handle_quotactl(req);
                 break;
 
@@ -1705,20 +1724,23 @@ int mds_handle(struct ptlrpc_request *req)
 
         case OBD_LOG_CANCEL:
                 CDEBUG(D_INODE, "log cancel\n");
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOG_CANCEL_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOG_CANCEL_NET))
+                        RETURN(0);
                 rc = -ENOTSUPP; /* la la la */
                 break;
 
         case LDLM_ENQUEUE:
                 DEBUG_REQ(D_INODE, req, "enqueue");
-                OBD_FAIL_RETURN(OBD_FAIL_LDLM_ENQUEUE, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_ENQUEUE))
+                        RETURN(0);
                 rc = ldlm_handle_enqueue(req, ldlm_server_completion_ast,
                                          ldlm_server_blocking_ast, NULL);
                 fail = OBD_FAIL_LDLM_REPLY;
                 break;
         case LDLM_CONVERT:
                 DEBUG_REQ(D_INODE, req, "convert");
-                OBD_FAIL_RETURN(OBD_FAIL_LDLM_CONVERT, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_CONVERT))
+                        RETURN(0);
                 rc = ldlm_handle_convert(req);
                 break;
         case LDLM_BL_CALLBACK:
@@ -1726,41 +1748,49 @@ int mds_handle(struct ptlrpc_request *req)
                 DEBUG_REQ(D_INODE, req, "callback");
                 CERROR("callbacks should not happen on MDS\n");
                 LBUG();
-                OBD_FAIL_RETURN(OBD_FAIL_LDLM_BL_CALLBACK, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_BL_CALLBACK))
+                        RETURN(0);
                 break;
         case LLOG_ORIGIN_HANDLE_CREATE:
                 DEBUG_REQ(D_INODE, req, "llog_init");
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOGD_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOGD_NET))
+                        RETURN(0);
                 rc = llog_origin_handle_create(req);
                 break;
         case LLOG_ORIGIN_HANDLE_DESTROY:
                 DEBUG_REQ(D_INODE, req, "llog_init");
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOGD_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOGD_NET))
+                        RETURN(0);
                 rc = llog_origin_handle_destroy(req);
                 break;
         case LLOG_ORIGIN_HANDLE_NEXT_BLOCK:
                 DEBUG_REQ(D_INODE, req, "llog next block");
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOGD_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOGD_NET))
+                        RETURN(0);
                 rc = llog_origin_handle_next_block(req);
                 break;
         case LLOG_ORIGIN_HANDLE_PREV_BLOCK:
                 DEBUG_REQ(D_INODE, req, "llog prev block");
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOGD_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOGD_NET))
+                        RETURN(0);
                 rc = llog_origin_handle_prev_block(req);
                 break;
         case LLOG_ORIGIN_HANDLE_READ_HEADER:
                 DEBUG_REQ(D_INODE, req, "llog read header");
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOGD_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOGD_NET))
+                        RETURN(0);
                 rc = llog_origin_handle_read_header(req);
                 break;
         case LLOG_ORIGIN_HANDLE_CLOSE:
                 DEBUG_REQ(D_INODE, req, "llog close");
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOGD_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOGD_NET))
+                        RETURN(0);
                 rc = llog_origin_handle_close(req);
                 break;
         case LLOG_CATINFO:
                 DEBUG_REQ(D_INODE, req, "llog catinfo");
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOGD_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOGD_NET))
+                        RETURN(0);
                 rc = llog_catinfo(req);
                 break;
         default:
index f4083dd..0bed7d1 100644 (file)
@@ -98,7 +98,7 @@ int mds_client_add(struct obd_device *obd, struct obd_export *exp,
                 cl_idx = find_first_zero_bit(bitmap, LR_MAX_CLIENTS);
         repeat:
                 if (cl_idx >= LR_MAX_CLIENTS ||
-                    OBD_FAIL_CHECK_ONCE(OBD_FAIL_MDS_CLIENT_ADD)) {
+                    OBD_FAIL_CHECK(OBD_FAIL_MDS_CLIENT_ADD)) {
                         CERROR("no room for %u client - fix LR_MAX_CLIENTS\n",
                                cl_idx);
                         return -EOVERFLOW;
@@ -486,7 +486,8 @@ int mds_fs_setup(struct obd_device *obd, struct vfsmount *mnt)
         int rc;
         ENTRY;
 
-        OBD_FAIL_RETURN(OBD_FAIL_MDS_FS_SETUP, -ENOENT);
+        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_FS_SETUP))
+                RETURN(-ENOENT);
         
         rc = cleanup_group_info();
         if (rc)
index f4e86be..bb6607b 100644 (file)
@@ -357,7 +357,7 @@ static int mds_create_objects(struct ptlrpc_request *req, int offset,
                 RETURN(rc);
         }
 
-        if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_MDS_ALLOC_OBDO))
+        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_ALLOC_OBDO))
                 GOTO(out_ids, rc = -ENOMEM);
 
         OBDO_ALLOC(oinfo.oi_oa);
@@ -872,8 +872,8 @@ int mds_open(struct mds_update_record *rec, int offset,
 
         mds_counter_incr(req->rq_export, LPROC_MDS_OPEN);
 
-        OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_PAUSE_OPEN | OBD_FAIL_ONCE,
-                         (obd_timeout + 1) / 4);
+        OBD_FAIL_TIMEOUT_ORSET(OBD_FAIL_MDS_PAUSE_OPEN, OBD_FAIL_ONCE,
+                               (obd_timeout + 1) / 4);
 
         CLASSERT(MAXQUOTAS < 4);
         if (offset == DLM_INTENT_REC_OFF) { /* intent */
@@ -1128,8 +1128,8 @@ found_child:
                 GOTO(cleanup, rc = -ENOTDIR);
         }
 
-        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_OPEN_CREATE)) {
-                obd_fail_loc = OBD_FAIL_LDLM_REPLY | OBD_FAIL_ONCE;
+        if (OBD_FAIL_CHECK_RESET(OBD_FAIL_MDS_OPEN_CREATE,
+                                 OBD_FAIL_LDLM_REPLY | OBD_FAIL_ONCE)) {
                 GOTO(cleanup, rc = -EAGAIN);
         }
 
index 1446971..9ba14df 100644 (file)
@@ -1384,7 +1384,8 @@ free_rdpg:
                         __cfs_free_page(rdpg->rp_pages[i]);
         OBD_FREE(rdpg->rp_pages, rdpg->rp_npages * sizeof rdpg->rp_pages[0]);
 
-        OBD_FAIL_RETURN(OBD_FAIL_MDS_SENDPAGE, 0);
+        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_SENDPAGE))
+                RETURN(0);
 
         return rc;
 }
@@ -2027,7 +2028,7 @@ static int mdt_req_handle(struct mdt_thread_info *info,
         /*
          * Mask out OBD_FAIL_ONCE, because that will stop
          * correct handling of failed req later in ldlm due to doing
-         * obd_fail_loc |= OBD_FAIL_ONCE | OBD_FAILED without actually
+         * obd_fail_loc |= OBD_FAIL_ONCE without actually
          * correct actions like it is done in target_send_reply_msg().
          */
         if (h->mh_fail_id != 0) {
@@ -2035,7 +2036,8 @@ static int mdt_req_handle(struct mdt_thread_info *info,
                  * Set to info->mti_fail_id to handler fail_id, it will be used
                  * later, and better than use default fail_id.
                  */
-                if (OBD_FAIL_CHECK(h->mh_fail_id && OBD_FAIL_MASK_LOC)) {
+                if (OBD_FAIL_CHECK_RESET(h->mh_fail_id && OBD_FAIL_MASK_LOC,
+                                         h->mh_fail_id & ~OBD_FAILED)) {
                         info->mti_fail_id = h->mh_fail_id;
                         RETURN(0);
                 }
@@ -2321,7 +2323,8 @@ static int mdt_handle0(struct ptlrpc_request *req,
 
         ENTRY;
 
-        OBD_FAIL_RETURN(OBD_FAIL_MDS_ALL_REQUEST_NET | OBD_FAIL_ONCE, 0);
+        if (OBD_FAIL_CHECK_ORSET(OBD_FAIL_MDS_ALL_REQUEST_NET, OBD_FAIL_ONCE))
+                RETURN(0);
 
         LASSERT(current->journal_info == NULL);
 
index fee383f..1252e81 100644 (file)
@@ -661,12 +661,11 @@ extern struct lu_context_key       mdt_thread_key;
 static inline void mdt_fail_write(const struct lu_env *env,
                                   struct dt_device *dd, int id)
 {
-        if (OBD_FAIL_CHECK(id)) {
+        if (OBD_FAIL_CHECK_ORSET(id, OBD_FAIL_ONCE)) {
                 CERROR(LUSTRE_MDT_NAME": obd_fail_loc=%x, fail write ops\n",
                        id);
                 dd->dd_ops->dt_ro(env, dd);
                 /* We set FAIL_ONCE because we never "un-fail" a device */
-                obd_fail_loc |= OBD_FAILED | OBD_FAIL_ONCE;
         }
 }
 
index 05c83c7..ed74be0 100644 (file)
@@ -604,8 +604,8 @@ static int mdt_finish_open(struct mdt_thread_info *info,
         } else if (flags & MDS_OPEN_DIRECTORY)
                 RETURN(-ENOTDIR);
 
-        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_OPEN_CREATE)) {
-                obd_fail_loc = OBD_FAIL_LDLM_REPLY | OBD_FAIL_ONCE;
+        if (OBD_FAIL_CHECK_RESET(OBD_FAIL_MDS_OPEN_CREATE,
+                                 OBD_FAIL_LDLM_REPLY | OBD_FAIL_ONCE)) {
                 RETURN(-EAGAIN);
         }
 
@@ -845,8 +845,8 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
         int                      created = 0;
         ENTRY;
 
-        OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_PAUSE_OPEN | OBD_FAIL_ONCE,
-                         (obd_timeout + 1) / 4);
+        OBD_FAIL_TIMEOUT_ORSET(OBD_FAIL_MDS_PAUSE_OPEN, OBD_FAIL_ONCE,
+                               (obd_timeout + 1) / 4);
 
         repbody = req_capsule_server_get(&info->mti_pill, &RMF_MDT_BODY);
 
index 23830f0..ed05c2a 100644 (file)
@@ -581,7 +581,7 @@ int mdt_client_new(const struct lu_env *env, struct mdt_device *mdt)
         spin_lock(&mdt->mdt_client_bitmap_lock);
         cl_idx = find_first_zero_bit(bitmap, LR_MAX_CLIENTS);
         if (cl_idx >= LR_MAX_CLIENTS ||
-            OBD_FAIL_CHECK_ONCE(OBD_FAIL_MDS_CLIENT_ADD)) {
+            OBD_FAIL_CHECK(OBD_FAIL_MDS_CLIENT_ADD)) {
                 CERROR("no room for %u clients - fix LR_MAX_CLIENTS\n",
                        cl_idx);
                 spin_unlock(&mdt->mdt_client_bitmap_lock);
@@ -937,7 +937,8 @@ int mdt_fs_setup(const struct lu_env *env, struct mdt_device *mdt,
         int rc = 0;
         ENTRY;
 
-        OBD_FAIL_RETURN(OBD_FAIL_MDS_FS_SETUP, -ENOENT);
+        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_FS_SETUP))
+                RETURN(-ENOENT);
 
         /* prepare transactions callbacks */
         mdt->mdt_txn_cb.dtc_txn_start = mdt_txn_start_cb;
index 243063d..6122cfe 100644 (file)
@@ -33,6 +33,8 @@
 #endif
 #ifndef __KERNEL__
 # include <liblustre.h>
+#else
+# include <asm/atomic.h>
 #endif
 
 #include <obd_support.h>
@@ -73,9 +75,9 @@ cfs_waitq_t obd_race_waitq;
 int obd_race_state;
 
 #ifdef __KERNEL__
-unsigned int obd_print_fail_loc(void)
+unsigned long obd_print_fail_loc(void)
 {
-        CWARN("obd_fail_loc = %x\n", obd_fail_loc);
+        CWARN("obd_fail_loc = %lx\n", obd_fail_loc);
         return obd_fail_loc;
 }
 
index 7f69593..8c5b7ba 100644 (file)
@@ -22,7 +22,7 @@ cfs_sysctl_table_header_t *obd_table_header = NULL;
 
 int proc_fail_loc SYSCTL_HANDLER_ARGS;
 int proc_obd_timeout SYSCTL_HANDLER_ARGS;
-extern unsigned int obd_fail_loc;
+extern unsigned long obd_fail_loc;
 extern unsigned int obd_dump_on_timeout;
 extern unsigned int obd_timeout;
 extern unsigned int ldlm_timeout;
@@ -76,7 +76,7 @@ extern cfs_waitq_t obd_race_waitq;
 int proc_fail_loc SYSCTL_HANDLER_ARGS
 { 
        int error = 0; 
-       int old_fail_loc = obd_fail_loc;
+       long old_fail_loc = obd_fail_loc;
        
        error = sysctl_handle_long(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 
        if (!error && req->newptr != USER_ADDR_NULL) {
index 121fdf9..c7d7ef0 100644 (file)
@@ -67,7 +67,7 @@ enum {
 int LL_PROC_PROTO(proc_fail_loc)
 {
         int rc;
-        int old_fail_loc = obd_fail_loc;
+        long old_fail_loc = obd_fail_loc;
 
         rc = ll_proc_dointvec(table, write, filp, buffer, lenp, ppos);
         if (old_fail_loc != obd_fail_loc)
index 64de7e0..8e39f4b 100644 (file)
@@ -67,7 +67,7 @@ static struct llog_handle *llog_cat_new_log(struct llog_handle *cathandle)
                 RETURN(ERR_PTR(-ENOSPC));
         }
 
-        if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_MDS_LLOG_CREATE_FAILED))
+        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_LLOG_CREATE_FAILED))
                 RETURN(ERR_PTR(-ENOSPC));
  
         rc = llog_create(cathandle->lgh_ctxt, &loghandle, NULL, NULL);
index 63c86a0..177002b 100644 (file)
@@ -913,7 +913,7 @@ static obd_count osc_checksum_bulk(int nob, obd_count pg_count,
                 /* corrupt the data before we compute the checksum, to
                  * simulate an OST->client data error */
                 if (i == 0 && opc == OST_READ &&
-                    OBD_FAIL_CHECK_ONCE(OBD_FAIL_OSC_CHECKSUM_RECEIVE))
+                    OBD_FAIL_CHECK(OBD_FAIL_OSC_CHECKSUM_RECEIVE))
                         memcpy(ptr + off, "bad1", min(4, nob));
                 cksum = crc32_le(cksum, ptr + off, count);
                 cfs_kunmap(pga[i]->pg);
@@ -926,7 +926,7 @@ static obd_count osc_checksum_bulk(int nob, obd_count pg_count,
         }
         /* For sending we only compute the wrong checksum instead
          * of corrupting the data so it is still correct on a redo */
-        if (opc == OST_WRITE && OBD_FAIL_CHECK_ONCE(OBD_FAIL_OSC_CHECKSUM_SEND))
+        if (opc == OST_WRITE && OBD_FAIL_CHECK(OBD_FAIL_OSC_CHECKSUM_SEND))
                 cksum++;
 
         return cksum;
@@ -950,8 +950,10 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,struct obdo *oa,
         struct osc_brw_async_args *aa;
 
         ENTRY;
-        OBD_FAIL_RETURN(OBD_FAIL_OSC_BRW_PREP_REQ, -ENOMEM); /* Recoverable */
-        OBD_FAIL_RETURN(OBD_FAIL_OSC_BRW_PREP_REQ2, -EINVAL); /* Fatal */
+        if (OBD_FAIL_CHECK(OBD_FAIL_OSC_BRW_PREP_REQ))
+                RETURN(-ENOMEM); /* Recoverable */
+        if (OBD_FAIL_CHECK(OBD_FAIL_OSC_BRW_PREP_REQ2))
+                RETURN(-EINVAL); /* Fatal */
 
         if ((cmd & OBD_BRW_WRITE) != 0) {
                 opc = OST_WRITE;
@@ -3008,7 +3010,8 @@ static int osc_match(struct obd_export *exp, struct lov_stripe_md *lsm,
         res_id.name[0] = lsm->lsm_object_id;
         res_id.name[2] = lsm->lsm_object_gr;
 
-        OBD_FAIL_RETURN(OBD_FAIL_OSC_MATCH, -EIO);
+        if (OBD_FAIL_CHECK(OBD_FAIL_OSC_MATCH))
+                RETURN(-EIO);
 
         /* Filesystem lock extents are extended to page boundaries so that
          * dealing with the page cache is a little smoother */
index 8008711..3dcc729 100644 (file)
@@ -160,7 +160,7 @@ static int ost_statfs(struct ptlrpc_request *req)
 
         req->rq_status = obd_statfs(req->rq_export->exp_obd, osfs,
                                     cfs_time_current_64() - HZ);
-        if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_OST_ENOSPC))
+        if (OBD_FAIL_CHECK(OBD_FAIL_OST_ENOSPC))
                 osfs->os_bfree = osfs->os_bavail = 64;
         if (req->rq_status != 0)
                 CERROR("ost: statfs failed: rc %d\n", req->rq_status);
@@ -481,13 +481,13 @@ static __u32 ost_checksum_bulk(struct ptlrpc_bulk_desc *desc, int opc)
                 /* corrupt the data before we compute the checksum, to
                  * simulate a client->OST data error */
                 if (i == 0 && opc == OST_WRITE &&
-                    OBD_FAIL_CHECK_ONCE(OBD_FAIL_OST_CHECKSUM_RECEIVE))
+                    OBD_FAIL_CHECK(OBD_FAIL_OST_CHECKSUM_RECEIVE))
                         memcpy(ptr, "bad3", min(4, len));
                 cksum = crc32_le(cksum, ptr, len);
                 /* corrupt the data after we compute the checksum, to
                  * simulate an OST->client data error */
                 if (i == 0 && opc == OST_READ &&
-                    OBD_FAIL_CHECK_ONCE(OBD_FAIL_OST_CHECKSUM_SEND))
+                    OBD_FAIL_CHECK(OBD_FAIL_OST_CHECKSUM_SEND))
                         memcpy(ptr, "bad4", min(4, len));
                 kunmap(page);
         }
@@ -1505,7 +1505,8 @@ int ost_handle(struct ptlrpc_request *req)
         switch (lustre_msg_get_opc(req->rq_reqmsg)) {
         case OST_CONNECT: {
                 CDEBUG(D_INODE, "connect\n");
-                OBD_FAIL_RETURN(OBD_FAIL_OST_CONNECT_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_CONNECT_NET))
+                        RETURN(0);
                 rc = target_handle_connect(req);
                 if (!rc)
                         obd = req->rq_export->exp_obd;
@@ -1513,33 +1514,38 @@ int ost_handle(struct ptlrpc_request *req)
         }
         case OST_DISCONNECT:
                 CDEBUG(D_INODE, "disconnect\n");
-                OBD_FAIL_RETURN(OBD_FAIL_OST_DISCONNECT_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_DISCONNECT_NET))
+                        RETURN(0);
                 rc = target_handle_disconnect(req);
                 break;
         case OST_CREATE:
                 CDEBUG(D_INODE, "create\n");
-                OBD_FAIL_RETURN(OBD_FAIL_OST_CREATE_NET, 0);
-                if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_OST_ENOSPC))
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_CREATE_NET))
+                        RETURN(0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_ENOSPC))
                         GOTO(out, rc = -ENOSPC);
-                if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_OST_EROFS))
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_EROFS))
                         GOTO(out, rc = -EROFS);
                 rc = ost_create(req->rq_export, req, oti);
                 break;
         case OST_DESTROY:
                 CDEBUG(D_INODE, "destroy\n");
-                OBD_FAIL_RETURN(OBD_FAIL_OST_DESTROY_NET, 0);
-                if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_OST_EROFS))
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_DESTROY_NET))
+                        RETURN(0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_EROFS))
                         GOTO(out, rc = -EROFS);
                 rc = ost_destroy(req->rq_export, req, oti);
                 break;
         case OST_GETATTR:
                 CDEBUG(D_INODE, "getattr\n");
-                OBD_FAIL_RETURN(OBD_FAIL_OST_GETATTR_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_GETATTR_NET))
+                        RETURN(0);
                 rc = ost_getattr(req->rq_export, req);
                 break;
         case OST_SETATTR:
                 CDEBUG(D_INODE, "setattr\n");
-                OBD_FAIL_RETURN(OBD_FAIL_OST_SETATTR_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_SETATTR_NET))
+                        RETURN(0);
                 rc = ost_setattr(req->rq_export, req, oti);
                 break;
         case OST_WRITE:
@@ -1552,10 +1558,11 @@ int ost_handle(struct ptlrpc_request *req)
                                req->rq_rqbd->rqbd_service->srv_req_portal);
                         GOTO(out, rc = -EPROTO);
                 }
-                OBD_FAIL_RETURN(OBD_FAIL_OST_BRW_NET, 0);
-                if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_OST_ENOSPC))
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_BRW_NET))
+                        RETURN(0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_ENOSPC))
                         GOTO(out, rc = -ENOSPC);
-                if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_OST_EROFS))
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_EROFS))
                         GOTO(out, rc = -EROFS);
                 rc = ost_brw_write(req, oti);
                 LASSERT(current->journal_info == NULL);
@@ -1571,26 +1578,30 @@ int ost_handle(struct ptlrpc_request *req)
                                req->rq_rqbd->rqbd_service->srv_req_portal);
                         GOTO(out, rc = -EPROTO);
                 }
-                OBD_FAIL_RETURN(OBD_FAIL_OST_BRW_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_BRW_NET))
+                        RETURN(0);
                 rc = ost_brw_read(req, oti);
                 LASSERT(current->journal_info == NULL);
                 /* ost_brw_read sends its own replies */
                 RETURN(rc);
         case OST_PUNCH:
                 CDEBUG(D_INODE, "punch\n");
-                OBD_FAIL_RETURN(OBD_FAIL_OST_PUNCH_NET, 0);
-                if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_OST_EROFS))
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_PUNCH_NET))
+                        RETURN(0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_EROFS))
                         GOTO(out, rc = -EROFS);
                 rc = ost_punch(req->rq_export, req, oti);
                 break;
         case OST_STATFS:
                 CDEBUG(D_INODE, "statfs\n");
-                OBD_FAIL_RETURN(OBD_FAIL_OST_STATFS_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_STATFS_NET))
+                        RETURN(0);
                 rc = ost_statfs(req);
                 break;
         case OST_SYNC:
                 CDEBUG(D_INODE, "sync\n");
-                OBD_FAIL_RETURN(OBD_FAIL_OST_SYNC_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_SYNC_NET))
+                        RETURN(0);
                 rc = ost_sync(req->rq_export, req);
                 break;
         case OST_SET_INFO:
@@ -1603,12 +1614,14 @@ int ost_handle(struct ptlrpc_request *req)
                 break;
         case OST_QUOTACHECK:
                 CDEBUG(D_INODE, "quotacheck\n");
-                OBD_FAIL_RETURN(OBD_FAIL_OST_QUOTACHECK_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_QUOTACHECK_NET))
+                        RETURN(0);
                 rc = ost_handle_quotacheck(req);
                 break;
         case OST_QUOTACTL:
                 CDEBUG(D_INODE, "quotactl\n");
-                OBD_FAIL_RETURN(OBD_FAIL_OST_QUOTACTL_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OST_QUOTACTL_NET))
+                        RETURN(0);
                 rc = ost_handle_quotactl(req);
                 break;
         case OBD_PING:
@@ -1626,7 +1639,8 @@ int ost_handle(struct ptlrpc_request *req)
                 RETURN(ptlrpc_reply(req));
         case OBD_LOG_CANCEL:
                 CDEBUG(D_INODE, "log cancel\n");
-                OBD_FAIL_RETURN(OBD_FAIL_OBD_LOG_CANCEL_NET, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_OBD_LOG_CANCEL_NET))
+                        RETURN(0);
                 rc = llog_origin_handle_cancel(req);
                 req->rq_status = rc;
                 rc = lustre_pack_reply(req, 1, NULL, NULL);
@@ -1635,7 +1649,8 @@ int ost_handle(struct ptlrpc_request *req)
                 RETURN(ptlrpc_reply(req));
         case LDLM_ENQUEUE:
                 CDEBUG(D_INODE, "enqueue\n");
-                OBD_FAIL_RETURN(OBD_FAIL_LDLM_ENQUEUE, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_ENQUEUE))
+                        RETURN(0);
                 rc = ldlm_handle_enqueue(req, ldlm_server_completion_ast,
                                          ldlm_server_blocking_ast,
                                          ldlm_server_glimpse_ast);
@@ -1643,12 +1658,14 @@ int ost_handle(struct ptlrpc_request *req)
                 break;
         case LDLM_CONVERT:
                 CDEBUG(D_INODE, "convert\n");
-                OBD_FAIL_RETURN(OBD_FAIL_LDLM_CONVERT, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_CONVERT))
+                        RETURN(0);
                 rc = ldlm_handle_convert(req);
                 break;
         case LDLM_CANCEL:
                 CDEBUG(D_INODE, "cancel\n");
-                OBD_FAIL_RETURN(OBD_FAIL_LDLM_CANCEL, 0);
+                if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_CANCEL))
+                        RETURN(0);
                 rc = ldlm_handle_cancel(req);
                 break;
         case LDLM_BL_CALLBACK:
index 1042f32..1337b8e 100644 (file)
@@ -1830,7 +1830,8 @@ int gss_pack_err_notify(struct ptlrpc_request *req, __u32 major, __u32 minor)
         int                        rc;
         ENTRY;
 
-        //OBD_FAIL_RETURN(OBD_FAIL_SVCGSS_ERR_NOTIFY|OBD_FAIL_ONCE, -EINVAL);
+        //if (OBD_FAIL_CHECK_ORSET(OBD_FAIL_SVCGSS_ERR_NOTIFY, OBD_FAIL_ONCE))
+        //      RETURN(-EINVAL);
 
         grctx->src_err_notify = 1;
         grctx->src_reserve_len = 0;
index 5da190c..726cae0 100644 (file)
@@ -52,10 +52,9 @@ static int ptl_send_buf (lnet_handle_md_t *mdh, void *base, int len,
         md.eq_handle = ptlrpc_eq_h;
 
         if (unlikely(ack == LNET_ACK_REQ &&
-                     OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_ACK | OBD_FAIL_ONCE))) {
+                     OBD_FAIL_CHECK_ORSET(OBD_FAIL_PTLRPC_ACK, OBD_FAIL_ONCE))){
                 /* don't ask for the ack to simulate failing client */
                 ack = LNET_NOACK_REQ;
-                obd_fail_loc |= OBD_FAIL_ONCE | OBD_FAILED;
         }
 
         rc = LNetMDBind (md, LNET_UNLINK, mdh);
@@ -93,7 +92,7 @@ int ptlrpc_start_bulk_transfer (struct ptlrpc_bulk_desc *desc)
         __u64                     xid;
         ENTRY;
 
-        if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_PTLRPC_BULK_PUT_NET))
+        if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_BULK_PUT_NET))
                 RETURN(0);
 
         /* NB no locking required until desc is on the network */
@@ -198,7 +197,7 @@ int ptlrpc_register_bulk (struct ptlrpc_request *req)
         lnet_md_t         md;
         ENTRY;
 
-        if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_PTLRPC_BULK_GET_NET))
+        if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_BULK_GET_NET))
                 RETURN(0);
 
         /* NB no locking required until desc is on the network */
@@ -405,7 +404,8 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
         lnet_md_t         reply_md;
         ENTRY;
 
-        OBD_FAIL_RETURN(OBD_FAIL_PTLRPC_DROP_RPC, 0);
+        if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_DROP_RPC))
+                RETURN(0);
 
         LASSERT (request->rq_type == PTL_RPC_MSG_REQUEST);
 
@@ -552,7 +552,7 @@ int ptlrpc_register_rqbd (struct ptlrpc_request_buffer_desc *rqbd)
         CDEBUG(D_NET, "LNetMEAttach: portal %d\n",
                service->srv_req_portal);
 
-        if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_PTLRPC_RQBD))
+        if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_RQBD))
                 return (-ENOMEM);
 
         rc = LNetMEAttach(service->srv_req_portal,
index f09b7c3..d47f5f9 100644 (file)
@@ -212,7 +212,8 @@ int dqacq_handler(struct obd_device *obd, struct qunit_data *qdata, int opc)
         int rc = 0;
         ENTRY;
 
-        OBD_FAIL_RETURN(OBD_FAIL_OBD_DQACQ, -EIO);
+        if (OBD_FAIL_CHECK(OBD_FAIL_OBD_DQACQ))
+                RETURN(-EIO);
 
         dquot = lustre_dqget(obd, info, qdata->qd_id, qdata_type);
         if (IS_ERR(dquot))