Whamcloud - gitweb
LU-224 Move fail_loc handling from lustre to libcfs
authorJames Simmons <uja.ornl@gmail.com>
Wed, 18 May 2011 17:54:44 +0000 (13:54 -0400)
committerOleg Drokin <green@whamcloud.com>
Fri, 20 May 2011 15:19:16 +0000 (08:19 -0700)
This patch goal is to move the fail_loc handling from lustre to libcfs. The
reason being is that currently you can only do fault testing in the lustre
core code. By moving this code to libcfs all sub systems will be able to
inject
faults into the system for testing purposes. This is in particular needed to
support the cray gemini klnd driver since one can inject faults for testing
failure conditions.

Change-Id: I438c30d0c7fe6987ad3fcfa5442366d4307a4d9f
Signed-off-by: James Simmons <uja.ornl@gmail.com>
Reviewed-on: http://review.whamcloud.com/450
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Liang Zhen <liang@whamcloud.com>
30 files changed:
libcfs/include/libcfs/Makefile.am
libcfs/include/libcfs/libcfs.h
libcfs/include/libcfs/libcfs_fail.h [new file with mode: 0644]
libcfs/libcfs/Makefile.in
libcfs/libcfs/autoMakefile.am
libcfs/libcfs/darwin/darwin-proc.c
libcfs/libcfs/fail.c [new file with mode: 0644]
libcfs/libcfs/linux/linux-proc.c
libcfs/libcfs/module.c
lustre/include/darwin/obd_support.h
lustre/include/linux/obd_support.h
lustre/include/obd_support.h
lustre/ldlm/ldlm_request.c
lustre/liblustre/tests/recovery_small.c
lustre/llite/file.c
lustre/lov/lov_request.c
lustre/lvfs/lvfs_lib.c
lustre/mdt/mdt_handler.c
lustre/mdt/mdt_internal.h
lustre/mgs/mgs_handler.c
lustre/obdclass/class_obd.c
lustre/obdclass/darwin/darwin-sysctl.c
lustre/obdclass/linux/linux-sysctl.c
lustre/obdfilter/filter.c
lustre/ost/ost_handler.c
lustre/ptlrpc/client.c
lustre/ptlrpc/service.c
lustre/tests/conf-sanity.sh
lustre/tests/replay-single.sh
lustre/tests/sanity-gss.sh

index f84ed14..cd61d27 100644 (file)
@@ -11,4 +11,4 @@ EXTRA_DIST := curproc.h libcfs_private.h libcfs.h list.h lltrace.h \
                libcfs_debug.h libcfsutil.h libcfs_ioctl.h \
                libcfs_pack.h libcfs_unpack.h libcfs_string.h \
                libcfs_kernelcomm.h libcfs_workitem.h lucache.h \
                libcfs_debug.h libcfsutil.h libcfs_ioctl.h \
                libcfs_pack.h libcfs_unpack.h libcfs_string.h \
                libcfs_kernelcomm.h libcfs_workitem.h lucache.h \
-               params_tree.h
+               libcfs_fail.h params_tree.h
index 4a96608..8732d98 100644 (file)
@@ -308,6 +308,7 @@ void cfs_get_random_bytes(void *buf, int size);
 #include <libcfs/libcfs_kernelcomm.h>
 #include <libcfs/libcfs_workitem.h>
 #include <libcfs/libcfs_hash.h>
 #include <libcfs/libcfs_kernelcomm.h>
 #include <libcfs/libcfs_workitem.h>
 #include <libcfs/libcfs_hash.h>
+#include <libcfs/libcfs_fail.h>
 #include <libcfs/params_tree.h>
 
 /* container_of depends on "likely" which is defined in libcfs_private.h */
 #include <libcfs/params_tree.h>
 
 /* container_of depends on "likely" which is defined in libcfs_private.h */
diff --git a/libcfs/include/libcfs/libcfs_fail.h b/libcfs/include/libcfs/libcfs_fail.h
new file mode 100644 (file)
index 0000000..b9dcd6b
--- /dev/null
@@ -0,0 +1,151 @@
+/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
+ * vim:expandtab:shiftwidth=8:tabstop=8:
+ *
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * Please contact Oracle Corporation, Inc., 500 Oracle Parkway, Redwood Shores,
+ * CA 94065 USA or visit www.oracle.com if you need additional information or
+ * have any questions.
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ */
+/*
+ * This file is part of Lustre, http://www.lustre.org/
+ * Lustre is a trademark of Oracle Corporation, Inc.
+ */
+
+#ifndef _LIBCFS_FAIL_H
+#define _LIBCFS_FAIL_H
+
+extern unsigned long cfs_fail_loc;
+extern unsigned int cfs_fail_val;
+
+extern cfs_waitq_t cfs_race_waitq;
+extern int cfs_race_state;
+
+int __cfs_fail_check_set(__u32 id, __u32 value, int set);
+int __cfs_fail_timeout_set(__u32 id, __u32 value, int ms, int set);
+
+enum {
+        CFS_FAIL_LOC_NOSET      = 0,
+        CFS_FAIL_LOC_ORSET      = 1,
+        CFS_FAIL_LOC_RESET      = 2
+};
+
+/* Failure injection control */
+#define CFS_FAIL_MASK_SYS    0x0000FF00
+#define CFS_FAIL_MASK_LOC   (0x000000FF | CFS_FAIL_MASK_SYS)
+
+#define CFS_FAILED_BIT       30
+/* CFS_FAILED is 0x40000000 */
+#define CFS_FAILED          (1 << CFS_FAILED_BIT)
+
+#define CFS_FAIL_ONCE_BIT    31
+/* CFS_FAIL_ONCE is 0x80000000 */
+#define CFS_FAIL_ONCE       (1 << CFS_FAIL_ONCE_BIT)
+
+/* The following flags aren't made to be combined */
+#define CFS_FAIL_SKIP        0x20000000 /* skip N times then fail */
+#define CFS_FAIL_SOME        0x10000000 /* only fail N times */
+#define CFS_FAIL_RAND        0x08000000 /* fail 1/N of the times */
+#define CFS_FAIL_USR1        0x04000000 /* user flag */
+
+#define CFS_FAIL_PRECHECK(id) (cfs_fail_loc &&                                \
+                              (cfs_fail_loc & CFS_FAIL_MASK_LOC) ==           \
+                              ((id) & CFS_FAIL_MASK_LOC))
+
+static inline int cfs_fail_check_set(__u32 id, __u32 value, int set)
+{
+        int ret = 0;
+        if (unlikely(CFS_FAIL_PRECHECK(id) &&
+            (ret = __cfs_fail_check_set(id, value, set)))) {
+                CERROR("*** cfs_fail_loc=%x ***\n", id);
+        }
+        return ret;
+}
+
+/* If id hit cfs_fail_loc, return 1, otherwise return 0 */
+#define CFS_FAIL_CHECK(id) \
+        cfs_fail_check_set(id, 0, CFS_FAIL_LOC_NOSET)
+
+/* If id hit cfs_fail_loc, cfs_fail_loc |= value and return 1,
+ * otherwise return 0 */
+#define CFS_FAIL_CHECK_ORSET(id, value) \
+        cfs_fail_check_set(id, value, CFS_FAIL_LOC_ORSET)
+
+/* If id hit cfs_fail_loc, cfs_fail_loc = value and return 1,
+ * otherwise return 0 */
+#define CFS_FAIL_CHECK_RESET(id, value) \
+        cfs_fail_check_set(id, value, CFS_FAIL_LOC_RESET)
+
+static inline int cfs_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
+{
+        if (unlikely(CFS_FAIL_PRECHECK(id)))
+                return __cfs_fail_timeout_set(id, value, ms, set);
+        else
+                return 0;
+}
+
+/* If id hit cfs_fail_loc, sleep for seconds or milliseconds */
+#define CFS_FAIL_TIMEOUT(id, secs) \
+        cfs_fail_timeout_set(id, 0, secs * 1000, CFS_FAIL_LOC_NOSET)
+
+#define CFS_FAIL_TIMEOUT_MS(id, ms) \
+        cfs_fail_timeout_set(id, 0, ms, CFS_FAIL_LOC_NOSET)
+
+/* If id hit cfs_fail_loc, cfs_fail_loc |= value and
+ * sleep seconds or milliseconds */
+#define CFS_FAIL_TIMEOUT_ORSET(id, value, secs) \
+        cfs_fail_timeout_set(id, value, secs * 1000, CFS_FAIL_LOC_ORSET)
+
+#define CFS_FAIL_TIMEOUT_MS_ORSET(id, value, ms) \
+        cfs_fail_timeout_set(id, value, ms, CFS_FAIL_LOC_ORSET)
+
+#ifdef __KERNEL__
+/* The idea here is to synchronise two threads to force a race. The
+ * first thread that calls this with a matching fail_loc is put to
+ * sleep. The next thread that calls with the same fail_loc wakes up
+ * the first and continues. */
+static inline void cfs_race(__u32 id)
+{
+        int rc;
+
+        if (CFS_FAIL_PRECHECK(id)) {
+                if (unlikely(__cfs_fail_check_set(id, 0, CFS_FAIL_LOC_NOSET))) {
+                        cfs_race_state = 0;
+                        CERROR("cfs_race id %x sleeping\n", id);
+                        cfs_wait_event_interruptible(cfs_race_waitq, cfs_race_state != 0, rc);
+                        CERROR("cfs_fail_race id %x awake\n", id);
+                } else {
+                        CERROR("cfs_fail_race id %x waking\n", id);
+                        cfs_race_state = 1;
+                        cfs_waitq_signal(&cfs_race_waitq);
+                }
+        }
+}
+#define CFS_RACE(id) cfs_race(id)
+#else
+/* sigh.  an expedient fix until CFS_RACE is fixed up */
+#define CFS_RACE(foo) do {} while(0)
+#endif
+
+#endif /* _LIBCFS_FAIL_H */
index 2537b0a..24bc9fd 100644 (file)
@@ -24,7 +24,7 @@ sources:
 
 endif
 
 
 endif
 
-libcfs-all-objs := debug.o nidstrings.o lwt.o module.o tracefile.o watchdog.o \
+libcfs-all-objs := debug.o fail.o nidstrings.o lwt.o module.o tracefile.o watchdog.o \
                libcfs_string.o hash.o kernel_user_comm.o prng.o workitem.o \
                upcall_cache.o
 
                libcfs_string.o hash.o kernel_user_comm.o prng.o workitem.o \
                upcall_cache.o
 
index 7712a64..1f2b97f 100644 (file)
@@ -44,7 +44,7 @@ if LIBLUSTRE
 noinst_LIBRARIES= libcfs.a
 libcfs_a_SOURCES= posix/posix-debug.c user-prim.c user-lock.c user-tcpip.c \
                   prng.c user-bitops.c user-mem.c hash.c kernel_user_comm.c \
 noinst_LIBRARIES= libcfs.a
 libcfs_a_SOURCES= posix/posix-debug.c user-prim.c user-lock.c user-tcpip.c \
                   prng.c user-bitops.c user-mem.c hash.c kernel_user_comm.c \
-                  workitem.c
+                  workitem.c fail.c
 libcfs_a_CPPFLAGS = $(LLCPPFLAGS)
 libcfs_a_CFLAGS = $(LLCFLAGS)
 endif
 libcfs_a_CPPFLAGS = $(LLCPPFLAGS)
 libcfs_a_CFLAGS = $(LLCFLAGS)
 endif
@@ -89,6 +89,6 @@ install-data-hook: $(install_data_hook)
 EXTRA_DIST := Info.plist
 
 MOSTLYCLEANFILES := @MOSTLYCLEANFILES@ linux-*.c linux/*.o darwin/*.o libcfs
 EXTRA_DIST := Info.plist
 
 MOSTLYCLEANFILES := @MOSTLYCLEANFILES@ linux-*.c linux/*.o darwin/*.o libcfs
-DIST_SOURCES := $(libcfs-all-objs:%.o=%.c) tracefile.h user-prim.c prng.c \
-      user-lock.c user-tcpip.c user-bitops.c workitem.c \
-      user-mem.c kernel_user_comm.c linux/linux-tracefile.h
+DIST_SOURCES := $(libcfs-all-objs:%.o=%.c) tracefile.h prng.c \
+      user-lock.c user-tcpip.c user-bitops.c user-prim.c workitem.c \
+      user-mem.c kernel_user_comm.c fail.c linux/linux-tracefile.h
index 9ce6c79..dca51e0 100644 (file)
@@ -144,6 +144,25 @@ static int sysctl_debug_mb SYSCTL_HANDLER_ARGS
        return error;
 }
 
        return error;
 }
 
+static int proc_fail_loc SYSCTL_HANDLER_ARGS
+{
+        int error = 0;
+        long old_fail_loc = cfs_fail_loc;
+
+        error = sysctl_handle_long(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
+        if (!error && req->newptr != USER_ADDR_NULL) {
+                if (old_fail_loc != cfs_fail_loc)
+                        cfs_waitq_signal(&cfs_race_waitq);
+        } else  if (req->newptr != USER_ADDR_NULL) {
+                /* Something was wrong with the write request */
+                printf ("sysctl fail loc fault: %d.\n", error);
+        } else {
+                /* Read request */
+                error = SYSCTL_OUT(req, &cfs_fail_loc, sizeof cfs_fail_loc);
+        }
+        return error;
+}
+
 /*
  * sysctl table for lnet
  */
 /*
  * sysctl table for lnet
  */
@@ -183,7 +202,9 @@ SYSCTL_PROC(_lnet,                  OID_AUTO,       daemon_file,
 SYSCTL_PROC(_lnet,                     OID_AUTO,       debug_mb,
             CTLTYPE_INT | CTLFLAG_RW,                  0,
             0,         &sysctl_debug_mb,               "L",    "debug_mb");
 SYSCTL_PROC(_lnet,                     OID_AUTO,       debug_mb,
             CTLTYPE_INT | CTLFLAG_RW,                  0,
             0,         &sysctl_debug_mb,               "L",    "debug_mb");
-
+SYSCTL_PROC(_lnet,                      OID_AUTO,       fail_loc, 
+             CTLTYPE_INT | CTLFLAG_RW ,                 &cfs_fail_loc,
+             0,         &proc_fail_loc,                 "I",    "cfs_fail_loc");
 
 static cfs_sysctl_table_t      top_table[] = {
        &sysctl__lnet,
 
 static cfs_sysctl_table_t      top_table[] = {
        &sysctl__lnet,
@@ -197,6 +218,7 @@ static cfs_sysctl_table_t   top_table[] = {
        &sysctl__lnet_debug_kernel,
        &sysctl__lnet_daemon_file,
        &sysctl__lnet_debug_mb,
        &sysctl__lnet_debug_kernel,
        &sysctl__lnet_daemon_file,
        &sysctl__lnet_debug_mb,
+        &sysctl__lnet_cfs_fail_loc
        NULL
 };
 
        NULL
 };
 
diff --git a/libcfs/libcfs/fail.c b/libcfs/libcfs/fail.c
new file mode 100644 (file)
index 0000000..d653ee6
--- /dev/null
@@ -0,0 +1,134 @@
+/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
+ * vim:expandtab:shiftwidth=8:tabstop=8:
+ *
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * Please contact Oracle Corporation, Inc., 500 Oracle Parkway, Redwood Shores,
+ * CA 94065 USA or visit www.oracle.com if you need additional information or
+ * have any questions.
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ */
+/*
+ * This file is part of Lustre, http://www.lustre.org/
+ * Lustre is a trademark of Oracle Corporation, Inc.
+ */
+
+#ifndef __KERNEL__
+#include <liblustre.h>
+#else
+#include <libcfs/libcfs.h>
+#endif
+
+unsigned long cfs_fail_loc = 0;
+unsigned int cfs_fail_val = 0;
+cfs_waitq_t cfs_race_waitq;
+int cfs_race_state;
+
+CFS_EXPORT_SYMBOL(cfs_fail_loc);
+CFS_EXPORT_SYMBOL(cfs_fail_val);
+CFS_EXPORT_SYMBOL(cfs_race_waitq);
+CFS_EXPORT_SYMBOL(cfs_race_state);
+
+int __cfs_fail_check_set(__u32 id, __u32 value, int set)
+{
+        static cfs_atomic_t cfs_fail_count = CFS_ATOMIC_INIT(0);
+
+        LASSERT(!(id & CFS_FAIL_ONCE));
+
+        if ((cfs_fail_loc & (CFS_FAILED | CFS_FAIL_ONCE)) ==
+            (CFS_FAILED | CFS_FAIL_ONCE)) {
+                cfs_atomic_set(&cfs_fail_count, 0); /* paranoia */
+                return 0;
+        }
+
+        /* Fail 1/cfs_fail_val times */
+        if (cfs_fail_loc & CFS_FAIL_RAND) {
+                if (cfs_fail_val < 2 || cfs_rand() % cfs_fail_val > 0)
+                        return 0;
+        }
+
+        /* Skip the first cfs_fail_val, then fail */
+        if (cfs_fail_loc & CFS_FAIL_SKIP) {
+                if (cfs_atomic_inc_return(&cfs_fail_count) <= cfs_fail_val)
+                        return 0;
+        }
+
+        /* Fail cfs_fail_val times, overridden by FAIL_ONCE */
+        if (cfs_fail_loc & CFS_FAIL_SOME &&
+            (!(cfs_fail_loc & CFS_FAIL_ONCE) || cfs_fail_val <= 1)) {
+                int count = cfs_atomic_inc_return(&cfs_fail_count);
+
+                if (count >= cfs_fail_val) {
+                        cfs_set_bit(CFS_FAIL_ONCE_BIT, &cfs_fail_loc);
+                        cfs_atomic_set(&cfs_fail_count, 0);
+                        /* we are lost race to increase  */
+                        if (count > cfs_fail_val)
+                                return 0;
+                }
+        }
+
+        if ((set == CFS_FAIL_LOC_ORSET || set == CFS_FAIL_LOC_RESET) &&
+            (value & CFS_FAIL_ONCE))
+                cfs_set_bit(CFS_FAIL_ONCE_BIT, &cfs_fail_loc);
+        /* Lost race to set CFS_FAILED_BIT. */
+        if (cfs_test_and_set_bit(CFS_FAILED_BIT, &cfs_fail_loc)) {
+                /* If CFS_FAIL_ONCE is valid, only one process can fail,
+                 * otherwise multi-process can fail at the same time. */
+                if (cfs_fail_loc & CFS_FAIL_ONCE)
+                        return 0;
+        }
+
+        switch (set) {
+                case CFS_FAIL_LOC_NOSET:
+                        break;
+                case CFS_FAIL_LOC_ORSET:
+                        cfs_fail_loc |= value & ~(CFS_FAILED | CFS_FAIL_ONCE);
+                        break;
+                case CFS_FAIL_LOC_RESET:
+                        cfs_fail_loc = value;
+                        break;
+                default:
+                        LASSERTF(0, "called with bad set %u\n", set);
+                        break;
+        }
+
+        return 1;
+}
+CFS_EXPORT_SYMBOL(__cfs_fail_check_set);
+
+int __cfs_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
+{
+        int ret = 0;
+
+        ret = __cfs_fail_check_set(id, value, set);
+        if (ret) {
+                CERROR("cfs_fail_timeout id %x sleeping for %dms\n",
+                       id, ms);
+                cfs_schedule_timeout_and_set_state(CFS_TASK_UNINT,
+                                                   cfs_time_seconds(ms) / 1000);
+                cfs_set_current_state(CFS_TASK_RUNNING);
+                CERROR("cfs_fail_timeout id %x awake\n", id);
+        }
+        return ret;
+}
+CFS_EXPORT_SYMBOL(__cfs_fail_timeout_set);
index 220c671..9777ffb 100644 (file)
@@ -107,6 +107,8 @@ enum {
         PSDEV_LNET_DEBUG_LOG_UPCALL, /* debug log upcall script */
         PSDEV_LNET_WATCHDOG_RATELIMIT,  /* ratelimit watchdog messages  */
         PSDEV_LNET_FORCE_LBUG,    /* hook to force an LBUG */
         PSDEV_LNET_DEBUG_LOG_UPCALL, /* debug log upcall script */
         PSDEV_LNET_WATCHDOG_RATELIMIT,  /* ratelimit watchdog messages  */
         PSDEV_LNET_FORCE_LBUG,    /* hook to force an LBUG */
+        PSDEV_LNET_FAIL_LOC,      /* control test failures instrumentation */
+        PSDEV_LNET_FAIL_VAL,      /* userdata for fail loc */
 };
 #else
 #define CTL_LNET                        CTL_UNNUMBERED
 };
 #else
 #define CTL_LNET                        CTL_UNNUMBERED
@@ -129,9 +131,10 @@ enum {
 #define PSDEV_LNET_DEBUG_LOG_UPCALL     CTL_UNNUMBERED
 #define PSDEV_LNET_WATCHDOG_RATELIMIT   CTL_UNNUMBERED
 #define PSDEV_LNET_FORCE_LBUG           CTL_UNNUMBERED
 #define PSDEV_LNET_DEBUG_LOG_UPCALL     CTL_UNNUMBERED
 #define PSDEV_LNET_WATCHDOG_RATELIMIT   CTL_UNNUMBERED
 #define PSDEV_LNET_FORCE_LBUG           CTL_UNNUMBERED
+#define PSDEV_LNET_FAIL_LOC             CTL_UNNUMBERED
+#define PSDEV_LNET_FAIL_VAL             CTL_UNNUMBERED
 #endif
 
 #endif
 
-
 int
 proc_call_handler(void *data, int write,
                   loff_t *ppos, void *buffer, size_t *lenp,
 int
 proc_call_handler(void *data, int write,
                   loff_t *ppos, void *buffer, size_t *lenp,
@@ -342,6 +345,17 @@ int LL_PROC_PROTO(libcfs_force_lbug)
         return 0;
 }
 
         return 0;
 }
 
+int LL_PROC_PROTO(proc_fail_loc)
+{
+        int rc;
+        long old_fail_loc = cfs_fail_loc;
+
+        rc = ll_proc_dolongvec(table, write, filp, buffer, lenp, ppos);
+        if (old_fail_loc != cfs_fail_loc)
+                cfs_waitq_signal(&cfs_race_waitq);
+        return rc;
+}
+
 static cfs_sysctl_table_t lnet_table[] = {
         /*
          * NB No .strategy entries have been provided since sysctl(8) prefers
 static cfs_sysctl_table_t lnet_table[] = {
         /*
          * NB No .strategy entries have been provided since sysctl(8) prefers
@@ -490,6 +504,22 @@ static cfs_sysctl_table_t lnet_table[] = {
                 .mode     = 0200,
                 .proc_handler = &libcfs_force_lbug
         },
                 .mode     = 0200,
                 .proc_handler = &libcfs_force_lbug
         },
+        {
+                .ctl_name = PSDEV_LNET_FAIL_LOC,
+                .procname = "fail_loc",
+                .data     = &cfs_fail_loc,
+                .maxlen   = sizeof(cfs_fail_loc),
+                .mode     = 0644,
+                .proc_handler = &proc_fail_loc
+        },
+        {
+                .ctl_name = PSDEV_LNET_FAIL_VAL,
+                .procname = "fail_val",
+                .data     = &cfs_fail_val,
+                .maxlen   = sizeof(int),
+                .mode     = 0644,
+                .proc_handler = &proc_dointvec
+        },
         {0}
 };
 
         {0}
 };
 
index a25df4a..8278b4c 100644 (file)
@@ -386,6 +386,7 @@ static int init_libcfs_module(void)
         cfs_init_mutex(&cfs_trace_thread_sem);
         cfs_init_rwsem(&ioctl_list_sem);
         CFS_INIT_LIST_HEAD(&ioctl_list);
         cfs_init_mutex(&cfs_trace_thread_sem);
         cfs_init_rwsem(&ioctl_list_sem);
         CFS_INIT_LIST_HEAD(&ioctl_list);
+        cfs_waitq_init(&cfs_race_waitq);
 
         rc = libcfs_debug_init(5 * 1024 * 1024);
         if (rc < 0) {
 
         rc = libcfs_debug_init(5 * 1024 * 1024);
         if (rc < 0) {
index 5172a77..149b613 100644 (file)
@@ -43,8 +43,6 @@
 
 #include <darwin/lustre_compat.h>
 
 
 #include <darwin/lustre_compat.h>
 
-#define OBD_SLEEP_ON(wq)        sleep_on(wq)
-
 /* for obd_class.h */
 # ifndef ERR_PTR
 #  define ERR_PTR(a) ((void *)(a))
 /* for obd_class.h */
 # ifndef ERR_PTR
 #  define ERR_PTR(a) ((void *)(a))
index 771285e..beae348 100644 (file)
@@ -61,7 +61,6 @@
 # include <linux/types.h>
 # include <linux/blkdev.h>
 # include <lvfs.h>
 # include <linux/types.h>
 # include <linux/blkdev.h>
 # include <lvfs.h>
-# define OBD_SLEEP_ON(wq, state)  wait_event_interruptible(wq, state)
 #else /* !__KERNEL__ */
 # define LTIME_S(time) (time)
 /* for obd_class.h */
 #else /* !__KERNEL__ */
 # define LTIME_S(time) (time)
 /* for obd_class.h */
index dd1d373..43e5ad4 100644 (file)
@@ -59,14 +59,6 @@ enum {
         OBD_STATS_NUM,
 };
 
         OBD_STATS_NUM,
 };
 
-enum {
-        OBD_FAIL_LOC_NOSET      = 0,
-        OBD_FAIL_LOC_ORSET      = 1,
-        OBD_FAIL_LOC_RESET      = 2
-};
-
-extern unsigned long obd_fail_loc;
-extern unsigned int obd_fail_val;
 extern unsigned int obd_debug_peer_on_timeout;
 extern unsigned int obd_dump_on_timeout;
 extern unsigned int obd_dump_on_eviction;
 extern unsigned int obd_debug_peer_on_timeout;
 extern unsigned int obd_dump_on_timeout;
 extern unsigned int obd_dump_on_eviction;
@@ -83,13 +75,8 @@ extern unsigned int obd_sync_filter;
 extern unsigned int obd_max_dirty_pages;
 extern cfs_atomic_t obd_dirty_pages;
 extern cfs_atomic_t obd_dirty_transit_pages;
 extern unsigned int obd_max_dirty_pages;
 extern cfs_atomic_t obd_dirty_pages;
 extern cfs_atomic_t obd_dirty_transit_pages;
-extern cfs_waitq_t obd_race_waitq;
-extern int obd_race_state;
 extern unsigned int obd_alloc_fail_rate;
 
 extern unsigned int obd_alloc_fail_rate;
 
-int __obd_fail_check_set(__u32 id, __u32 value, int set);
-int __obd_fail_timeout_set(__u32 id, __u32 value, int ms, int set);
-
 /* lvfs.c */
 int obd_alloc_fail(const void *ptr, const char *name, const char *type,
                    size_t size, const char *file, int line);
 /* lvfs.c */
 int obd_alloc_fail(const void *ptr, const char *name, const char *type,
                    size_t size, const char *file, int line);
@@ -426,75 +413,18 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type,
 #define OBD_FAIL_MDS_SYNC_CAPA_SL                   0x1310
 #define OBD_FAIL_SEQ_ALLOC                          0x1311
 
 #define OBD_FAIL_MDS_SYNC_CAPA_SL                   0x1310
 #define OBD_FAIL_SEQ_ALLOC                          0x1311
 
-/* Failure injection control */
-#define OBD_FAIL_MASK_SYS    0x0000FF00
-#define OBD_FAIL_MASK_LOC   (0x000000FF | OBD_FAIL_MASK_SYS)
-
-#define OBD_FAILED_BIT       30
-/* OBD_FAILED is 0x40000000 */
-#define OBD_FAILED          (1 << OBD_FAILED_BIT)
-
-#define OBD_FAIL_ONCE_BIT    31
-/* OBD_FAIL_ONCE is 0x80000000 */
-#define OBD_FAIL_ONCE       (1 << OBD_FAIL_ONCE_BIT)
-
-/* The following flags aren't made to be combined */
-#define OBD_FAIL_SKIP        0x20000000 /* skip N times then fail */
-#define OBD_FAIL_SOME        0x10000000 /* only fail N times */
-#define OBD_FAIL_RAND        0x08000000 /* fail 1/N of the times */
-#define OBD_FAIL_USR1        0x04000000 /* user flag */
-
-#define OBD_FAIL_PRECHECK(id) (obd_fail_loc &&                                \
-                              (obd_fail_loc & OBD_FAIL_MASK_LOC) ==           \
-                              ((id) & OBD_FAIL_MASK_LOC))
-
-static inline int obd_fail_check_set(__u32 id, __u32 value, int set)
-{
-        int ret = 0;
-        if (unlikely(OBD_FAIL_PRECHECK(id) &&
-            (ret = __obd_fail_check_set(id, value, set)))) {
-                CERROR("*** obd_fail_loc=%x ***\n", id);
-        }
-        return ret;
-}
-
-/* If id hit obd_fail_loc, return 1, otherwise return 0 */
-#define OBD_FAIL_CHECK(id) \
-        obd_fail_check_set(id, 0, OBD_FAIL_LOC_NOSET)
-
-/* If id hit obd_fail_loc, obd_fail_loc |= value and return 1,
- * otherwise return 0 */
-#define OBD_FAIL_CHECK_ORSET(id, value) \
-        obd_fail_check_set(id, value, OBD_FAIL_LOC_ORSET)
-
-/* If id hit obd_fail_loc, obd_fail_loc = value and return 1,
- * otherwise return 0 */
-#define OBD_FAIL_CHECK_RESET(id, value) \
-        obd_fail_check_set(id, value, OBD_FAIL_LOC_RESET)
-
-
-static inline int obd_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
-{
-        if (unlikely(OBD_FAIL_PRECHECK(id)))
-                return __obd_fail_timeout_set(id, value, ms, set);
-        else
-                return 0;
-}
-
-/* If id hit obd_fail_loc, sleep for seconds or milliseconds */
-#define OBD_FAIL_TIMEOUT(id, secs) \
-        obd_fail_timeout_set(id, 0, secs * 1000, OBD_FAIL_LOC_NOSET)
-
-#define OBD_FAIL_TIMEOUT_MS(id, ms) \
-        obd_fail_timeout_set(id, 0, ms, OBD_FAIL_LOC_NOSET)
-
-/* If id hit obd_fail_loc, obd_fail_loc |= value and
- * sleep seconds or milliseconds */
-#define OBD_FAIL_TIMEOUT_ORSET(id, value, secs) \
-        obd_fail_timeout_set(id, value, secs * 1000, OBD_FAIL_LOC_ORSET)
-
-#define OBD_FAIL_TIMEOUT_MS_ORSET(id, value, ms) \
-        obd_fail_timeout_set(id, value, ms, OBD_FAIL_LOC_ORSET)
+/* Assign references to moved code to reduce code changes */
+#define OBD_FAIL_PRECHECK(id)                   CFS_FAIL_PRECHECK(id)
+#define OBD_FAIL_CHECK(id)                      CFS_FAIL_CHECK(id)
+#define OBD_FAIL_CHECK_ORSET(id, value)         CFS_FAIL_CHECK_ORSET(id, value)
+#define OBD_FAIL_CHECK_RESET(id, value)         CFS_FAIL_CHECK_RESET(id, value)
+#define OBD_FAIL_RETURN(id, ret)                CFS_FAIL_RETURN(id, ret)
+#define OBD_FAIL_TIMEOUT(id, secs)              CFS_FAIL_TIMEOUT(id, secs)
+#define OBD_FAIL_TIMEOUT_MS(id, ms)             CFS_FAIL_TIMEOUT_MS(id, ms)
+#define OBD_FAIL_TIMEOUT_ORSET(id, value, secs) CFS_FAIL_TIMEOUT_ORSET(id, value, secs)
+#define OBD_RACE(id)                            CFS_RACE(id)
+#define OBD_FAIL_ONCE                           CFS_FAIL_ONCE
+#define OBD_FAILED                              CFS_FAILED
 
 #ifdef __KERNEL__
 static inline void obd_fail_write(int id, struct super_block *sb)
 
 #ifdef __KERNEL__
 static inline void obd_fail_write(int id, struct super_block *sb)
@@ -503,7 +433,7 @@ static inline void obd_fail_write(int id, struct super_block *sb)
         if (OBD_FAIL_CHECK_ORSET(id & ~OBD_FAIL_ONCE, OBD_FAIL_ONCE)) {
 #ifdef LIBCFS_DEBUG
                 BDEVNAME_DECLARE_STORAGE(tmp);
         if (OBD_FAIL_CHECK_ORSET(id & ~OBD_FAIL_ONCE, OBD_FAIL_ONCE)) {
 #ifdef LIBCFS_DEBUG
                 BDEVNAME_DECLARE_STORAGE(tmp);
-                CERROR("obd_fail_loc=%x, fail write operation on %s\n",
+                CERROR("cfs_fail_loc=%x, fail write operation on %s\n",
                        id, ll_bdevname(sb, tmp));
 #endif
                 /* TODO-CMD: fix getting jdev */
                        id, ll_bdevname(sb, tmp));
 #endif
                 /* TODO-CMD: fix getting jdev */
@@ -511,30 +441,6 @@ static inline void obd_fail_write(int id, struct super_block *sb)
         }
 }
 #define OBD_FAIL_WRITE(id, sb) obd_fail_write(id, sb)
         }
 }
 #define OBD_FAIL_WRITE(id, sb) obd_fail_write(id, sb)
-
-/* The idea here is to synchronise two threads to force a race. The
- * first thread that calls this with a matching fail_loc is put to
- * sleep. The next thread that calls with the same fail_loc wakes up
- * the first and continues. */
-static inline void obd_race(__u32 id)
-{
-        if (OBD_FAIL_PRECHECK(id)) {
-                if (unlikely(__obd_fail_check_set(id, 0, OBD_FAIL_LOC_NOSET))) {
-                        obd_race_state = 0;
-                        CERROR("obd_race id %x sleeping\n", id);
-                        OBD_SLEEP_ON(obd_race_waitq, obd_race_state != 0);
-                        CERROR("obd_fail_race id %x awake\n", id);
-                } else {
-                        CERROR("obd_fail_race id %x waking\n", id);
-                        obd_race_state = 1;
-                        cfs_waitq_signal(&obd_race_waitq);
-                }
-        }
-}
-#define OBD_RACE(id) obd_race(id)
-#else
-/* sigh.  an expedient fix until OBD_RACE is fixed up */
-#define OBD_RACE(foo) do {} while(0)
 #endif
 
 #define fixme() CDEBUG(D_OTHER, "FIXME\n");
 #endif
 
 #define fixme() CDEBUG(D_OTHER, "FIXME\n");
index d03e0eb..767ca01 100644 (file)
@@ -1104,9 +1104,9 @@ int ldlm_cli_cancel_req(struct obd_export *exp, cfs_list_t *cancels,
         LASSERT(exp != NULL);
         LASSERT(count > 0);
 
         LASSERT(exp != NULL);
         LASSERT(count > 0);
 
-        OBD_FAIL_TIMEOUT(OBD_FAIL_LDLM_PAUSE_CANCEL, obd_fail_val);
+        CFS_FAIL_TIMEOUT(OBD_FAIL_LDLM_PAUSE_CANCEL, cfs_fail_val);
 
 
-        if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_CANCEL_RACE))
+        if (CFS_FAIL_CHECK(OBD_FAIL_LDLM_CANCEL_RACE))
                 RETURN(count);
 
         free = ldlm_format_handles_avail(class_exp2cliimp(exp),
                 RETURN(count);
 
         free = ldlm_format_handles_avail(class_exp2cliimp(exp),
index 4946e33..03d25bc 100644 (file)
@@ -148,7 +148,7 @@ void cleanup_dir(const char *path)
         if (drop_arr[drop_index].name) {                                   \
             printf("server drops next %s\n", drop_arr[drop_index].name);   \
             sprintf(cmd,                                                   \
         if (drop_arr[drop_index].name) {                                   \
             printf("server drops next %s\n", drop_arr[drop_index].name);   \
             sprintf(cmd,                                                   \
-                    "%s %s \"echo %lu > /proc/sys/lustre/fail_loc\"",      \
+                    "%s %s \"lctl set_param fail_loc=%lu\"",               \
                     ssh_cmd, mds_server, drop_arr[drop_index].code);       \
             if ((rc = system(cmd)) != 0) {                                 \
                 rc = WEXITSTATUS(rc);                                      \
                     ssh_cmd, mds_server, drop_arr[drop_index].code);       \
             if ((rc = system(cmd)) != 0) {                                 \
                 rc = WEXITSTATUS(rc);                                      \
@@ -163,7 +163,7 @@ void cleanup_dir(const char *path)
         char cmd[1024];                                                    \
                                                                            \
         if (drop_arr[drop_index].name) {                                   \
         char cmd[1024];                                                    \
                                                                            \
         if (drop_arr[drop_index].name) {                                   \
-            sprintf(cmd, "%s %s \"echo 0 > /proc/sys/lustre/fail_loc\"",   \
+            sprintf(cmd, "%s %s \"lctl set_param fail_loc=0\"",            \
                     ssh_cmd, mds_server);                                  \
             if (!system(cmd)) {}                                           \
         }                                                                  \
                     ssh_cmd, mds_server);                                  \
             if (!system(cmd)) {}                                           \
         }                                                                  \
index e8a0cc1..22f5958 100644 (file)
@@ -331,7 +331,7 @@ int ll_file_release(struct inode *inode, struct file *file)
 
         rc = ll_md_close(sbi->ll_md_exp, inode, file);
 
 
         rc = ll_md_close(sbi->ll_md_exp, inode, file);
 
-        if (OBD_FAIL_TIMEOUT_MS(OBD_FAIL_PTLRPC_DUMP_LOG, obd_fail_val))
+        if (CFS_FAIL_TIMEOUT_MS(OBD_FAIL_PTLRPC_DUMP_LOG, cfs_fail_val))
                 libcfs_debug_dumplog();
 
         RETURN(rc);
                 libcfs_debug_dumplog();
 
         RETURN(rc);
index a8c350d..9e499e9 100644 (file)
@@ -716,8 +716,8 @@ int cb_create_update(void *cookie, int rc)
 
         lovreq = container_of(oinfo, struct lov_request, rq_oi);
 
 
         lovreq = container_of(oinfo, struct lov_request, rq_oi);
 
-        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_OSC_CREATE_FAIL))
-                if (lovreq->rq_idx == obd_fail_val)
+        if (CFS_FAIL_CHECK(OBD_FAIL_MDS_OSC_CREATE_FAIL))
+                if (lovreq->rq_idx == cfs_fail_val)
                         rc = -ENOTCONN;
 
         rc= lov_update_create_set(lovreq->rq_rqset, lovreq, rc);
                         rc = -ENOTCONN;
 
         rc= lov_update_create_set(lovreq->rq_rqset, lovreq, rc);
index 7bfe92d..9660b90 100644 (file)
@@ -47,8 +47,6 @@
 #include <lustre_lib.h>
 #include <lprocfs_status.h>
 
 #include <lustre_lib.h>
 #include <lprocfs_status.h>
 
-unsigned int obd_fail_val = 0;
-unsigned long obd_fail_loc = 0;
 unsigned int obd_alloc_fail_rate = 0;
 
 int obd_alloc_fail(const void *ptr, const char *name, const char *type,
 unsigned int obd_alloc_fail_rate = 0;
 
 int obd_alloc_fail(const void *ptr, const char *name, const char *type,
@@ -72,91 +70,6 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type,
 }
 EXPORT_SYMBOL(obd_alloc_fail);
 
 }
 EXPORT_SYMBOL(obd_alloc_fail);
 
-int __obd_fail_check_set(__u32 id, __u32 value, int set)
-{
-        static cfs_atomic_t obd_fail_count = CFS_ATOMIC_INIT(0);
-
-        LASSERT(!(id & OBD_FAIL_ONCE));
-
-        if ((obd_fail_loc & (OBD_FAILED | OBD_FAIL_ONCE)) ==
-            (OBD_FAILED | OBD_FAIL_ONCE)) {
-                cfs_atomic_set(&obd_fail_count, 0); /* paranoia */
-                return 0;
-        }
-
-        /* Fail 1/obd_fail_val times */
-        if (obd_fail_loc & OBD_FAIL_RAND) {
-                if (obd_fail_val < 2 || cfs_rand() % obd_fail_val > 0)
-                        return 0;
-        }
-
-        /* Skip the first obd_fail_val, then fail */
-        if (obd_fail_loc & OBD_FAIL_SKIP) {
-                if (cfs_atomic_inc_return(&obd_fail_count) <= obd_fail_val)
-                        return 0;
-        }
-
-        /* Fail obd_fail_val times, overridden by FAIL_ONCE */
-        if (obd_fail_loc & OBD_FAIL_SOME &&
-            (!(obd_fail_loc & OBD_FAIL_ONCE) || obd_fail_val <= 1)) {
-                int count = cfs_atomic_inc_return(&obd_fail_count);
-
-                if (count >= obd_fail_val) {
-                        cfs_set_bit(OBD_FAIL_ONCE_BIT, &obd_fail_loc);
-                        cfs_atomic_set(&obd_fail_count, 0);
-                        /* we are lost race to increase obd_fail_count */
-                        if (count > obd_fail_val)
-                                return 0;
-                }
-        }
-
-        if ((set == OBD_FAIL_LOC_ORSET || set == OBD_FAIL_LOC_RESET) &&
-            (value & OBD_FAIL_ONCE))
-                cfs_set_bit(OBD_FAIL_ONCE_BIT, &obd_fail_loc);
-
-        /* Lost race to set OBD_FAILED_BIT. */
-        if (cfs_test_and_set_bit(OBD_FAILED_BIT, &obd_fail_loc)) {
-                /* If OBD_FAIL_ONCE is valid, only one process can fail,
-                 * otherwise multi-process can fail at the same time. */
-                if (obd_fail_loc & OBD_FAIL_ONCE)
-                        return 0;
-        }
-
-        switch (set) {
-                case OBD_FAIL_LOC_NOSET:
-                        break;
-                case OBD_FAIL_LOC_ORSET:
-                        obd_fail_loc |= value & ~(OBD_FAILED | OBD_FAIL_ONCE);
-                        break;
-                case OBD_FAIL_LOC_RESET:
-                        obd_fail_loc = value;
-                        break;
-                default:
-                        LASSERTF(0, "called with bad set %u\n", set);
-                        break;
-        }
-
-        return 1;
-}
-EXPORT_SYMBOL(__obd_fail_check_set);
-
-int __obd_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
-{
-        int ret = 0;
-
-        ret = __obd_fail_check_set(id, value, set);
-        if (ret) {
-                CERROR("obd_fail_timeout id %x sleeping for %dms\n",
-                       id, ms);
-                cfs_schedule_timeout_and_set_state(CFS_TASK_UNINT,
-                                                   cfs_time_seconds(ms) / 1000);
-                cfs_set_current_state(CFS_TASK_RUNNING);
-                CERROR("obd_fail_timeout id %x awake\n", id);
-        }
-        return ret;
-}
-EXPORT_SYMBOL(__obd_fail_timeout_set);
-
 #ifdef LPROCFS
 void lprocfs_counter_add(struct lprocfs_stats *stats, int idx,
                                        long amount)
 #ifdef LPROCFS
 void lprocfs_counter_add(struct lprocfs_stats *stats, int idx,
                                        long amount)
@@ -231,6 +144,4 @@ void lprocfs_counter_sub(struct lprocfs_stats *stats, int idx,
 EXPORT_SYMBOL(lprocfs_counter_sub);
 #endif  /* LPROCFS */
 
 EXPORT_SYMBOL(lprocfs_counter_sub);
 #endif  /* LPROCFS */
 
-EXPORT_SYMBOL(obd_fail_loc);
 EXPORT_SYMBOL(obd_alloc_fail_rate);
 EXPORT_SYMBOL(obd_alloc_fail_rate);
-EXPORT_SYMBOL(obd_fail_val);
index dc5e254..5d08c4e 100644 (file)
@@ -2052,7 +2052,7 @@ static int mdt_sec_ctx_handle(struct mdt_thread_info *info)
                         sptlrpc_svc_ctx_invalidate(req);
         }
 
                         sptlrpc_svc_ctx_invalidate(req);
         }
 
-        OBD_FAIL_TIMEOUT(OBD_FAIL_SEC_CTX_HDL_PAUSE, obd_fail_val);
+        CFS_FAIL_TIMEOUT(OBD_FAIL_SEC_CTX_HDL_PAUSE, cfs_fail_val);
 
         return rc;
 }
 
         return rc;
 }
index 44aa75b..7210e57 100644 (file)
@@ -690,7 +690,7 @@ static inline void mdt_fail_write(const struct lu_env *env,
                                   struct dt_device *dd, int id)
 {
         if (OBD_FAIL_CHECK_ORSET(id, OBD_FAIL_ONCE)) {
                                   struct dt_device *dd, int id)
 {
         if (OBD_FAIL_CHECK_ORSET(id, OBD_FAIL_ONCE)) {
-                CERROR(LUSTRE_MDT_NAME": obd_fail_loc=%x, fail write ops\n",
+                CERROR(LUSTRE_MDT_NAME": cfs_fail_loc=%x, fail write ops\n",
                        id);
                 dd->dd_ops->dt_ro(env, dd);
                 /* We set FAIL_ONCE because we never "un-fail" a device */
                        id);
                 dd->dd_ops->dt_ro(env, dd);
                 /* We set FAIL_ONCE because we never "un-fail" a device */
index e0b8469..266c835 100644 (file)
@@ -659,8 +659,8 @@ int mgs_handle(struct ptlrpc_request *req)
         ENTRY;
 
         req_capsule_init(&req->rq_pill, req, RCL_SERVER);
         ENTRY;
 
         req_capsule_init(&req->rq_pill, req, RCL_SERVER);
-        OBD_FAIL_TIMEOUT_MS(OBD_FAIL_MGS_PAUSE_REQ, obd_fail_val);
-        if (OBD_FAIL_CHECK(OBD_FAIL_MGS_ALL_REQUEST_NET))
+        CFS_FAIL_TIMEOUT_MS(OBD_FAIL_MGS_PAUSE_REQ, cfs_fail_val);
+        if (CFS_FAIL_CHECK(OBD_FAIL_MGS_ALL_REQUEST_NET))
                 RETURN(0);
 
         LASSERT(current->journal_info == NULL);
                 RETURN(0);
 
         LASSERT(current->journal_info == NULL);
index c1ada33..6b2227d 100644 (file)
@@ -86,17 +86,6 @@ int at_extra = 30;
 cfs_atomic_t obd_dirty_pages;
 cfs_atomic_t obd_dirty_transit_pages;
 
 cfs_atomic_t obd_dirty_pages;
 cfs_atomic_t obd_dirty_transit_pages;
 
-cfs_waitq_t obd_race_waitq;
-int obd_race_state;
-
-#ifdef __KERNEL__
-unsigned long obd_print_fail_loc(void)
-{
-        CWARN("obd_fail_loc = %lx\n", obd_fail_loc);
-        return obd_fail_loc;
-}
-#endif
-
 static inline void obd_data2conn(struct lustre_handle *conn,
                                  struct obd_ioctl_data *data)
 {
 static inline void obd_data2conn(struct lustre_handle *conn,
                                  struct obd_ioctl_data *data)
 {
@@ -357,9 +346,6 @@ void *obd_psdev = NULL;
 #endif
 
 EXPORT_SYMBOL(obd_devs);
 #endif
 
 EXPORT_SYMBOL(obd_devs);
-EXPORT_SYMBOL(obd_print_fail_loc);
-EXPORT_SYMBOL(obd_race_waitq);
-EXPORT_SYMBOL(obd_race_state);
 EXPORT_SYMBOL(obd_debug_peer_on_timeout);
 EXPORT_SYMBOL(obd_dump_on_timeout);
 EXPORT_SYMBOL(obd_dump_on_eviction);
 EXPORT_SYMBOL(obd_debug_peer_on_timeout);
 EXPORT_SYMBOL(obd_dump_on_timeout);
 EXPORT_SYMBOL(obd_dump_on_eviction);
@@ -524,7 +510,6 @@ int init_obdclass(void)
 #endif
 
         cfs_spin_lock_init(&obd_types_lock);
 #endif
 
         cfs_spin_lock_init(&obd_types_lock);
-        cfs_waitq_init(&obd_race_waitq);
         obd_zombie_impexp_init();
 #ifdef LPROCFS
         obd_memory = lprocfs_alloc_stats(OBD_STATS_NUM,
         obd_zombie_impexp_init();
 #ifdef LPROCFS
         obd_memory = lprocfs_alloc_stats(OBD_STATS_NUM,
index 0820b20..2536efe 100644 (file)
@@ -56,9 +56,7 @@
 
 cfs_sysctl_table_header_t *obd_table_header = NULL;
 
 
 cfs_sysctl_table_header_t *obd_table_header = NULL;
 
-int proc_fail_loc SYSCTL_HANDLER_ARGS;
 int proc_obd_timeout SYSCTL_HANDLER_ARGS;
 int proc_obd_timeout SYSCTL_HANDLER_ARGS;
-extern unsigned long obd_fail_loc;
 extern unsigned int obd_dump_on_timeout;
 extern unsigned int obd_timeout;
 extern unsigned int ldlm_timeout;
 extern unsigned int obd_dump_on_timeout;
 extern unsigned int obd_timeout;
 extern unsigned int ldlm_timeout;
@@ -70,9 +68,6 @@ int read_lustre_kernel_version SYSCTL_HANDLER_ARGS;
 
 SYSCTL_NODE (,                  OID_AUTO,       lustre,            CTLFLAG_RW,
             0,                 "lustre sysctl top");
 
 SYSCTL_NODE (,                  OID_AUTO,       lustre,            CTLFLAG_RW,
             0,                 "lustre sysctl top");
-SYSCTL_PROC(_lustre,           OID_AUTO,       fail_loc, 
-           CTLTYPE_INT | CTLFLAG_RW ,          &obd_fail_loc,
-           0,          &proc_fail_loc,         "I",    "obd_fail_loc");
 SYSCTL_PROC(_lustre,           OID_AUTO,       timeout, 
            CTLTYPE_INT | CTLFLAG_RW ,          &obd_timeout,
            0,          &proc_obd_timeout,      "I",    "obd_timeout");
 SYSCTL_PROC(_lustre,           OID_AUTO,       timeout, 
            CTLTYPE_INT | CTLFLAG_RW ,          &obd_timeout,
            0,          &proc_obd_timeout,      "I",    "obd_timeout");
@@ -97,7 +92,6 @@ SYSCTL_INT(_lustre,           OID_AUTO,       ldlm_timeout,
 
 static cfs_sysctl_table_t      parent_table[] = {
        &sysctl__lustre,
 
 static cfs_sysctl_table_t      parent_table[] = {
        &sysctl__lustre,
-       &sysctl__lustre_fail_loc,
        &sysctl__lustre_timeout,
        &sysctl__lustre_dump_on_timeout,
         &sysctl__lustre_debug_peer_on_timeout,
        &sysctl__lustre_timeout,
        &sysctl__lustre_dump_on_timeout,
         &sysctl__lustre_debug_peer_on_timeout,
@@ -107,27 +101,6 @@ static cfs_sysctl_table_t      parent_table[] = {
        &sysctl__lustre_ldlm_timeout,
 };
 
        &sysctl__lustre_ldlm_timeout,
 };
 
-extern cfs_waitq_t obd_race_waitq;
-
-int proc_fail_loc SYSCTL_HANDLER_ARGS
-{ 
-       int error = 0; 
-       long old_fail_loc = obd_fail_loc;
-       
-       error = sysctl_handle_long(oidp, oidp->oid_arg1, oidp->oid_arg2, req); 
-       if (!error && req->newptr != USER_ADDR_NULL) {
-               if (old_fail_loc != obd_fail_loc) 
-                       cfs_waitq_signal(&obd_race_waitq);
-       } else  if (req->newptr != USER_ADDR_NULL) { 
-               /* Something was wrong with the write request */ 
-               printf ("sysctl fail loc fault: %d.\n", error);
-       } else { 
-               /* Read request */ 
-               error = SYSCTL_OUT(req, &obd_fail_loc, sizeof obd_fail_loc);
-       }
-       return error;
-}
-
 int proc_obd_timeout SYSCTL_HANDLER_ARGS
 { 
        int error = 0;
 int proc_obd_timeout SYSCTL_HANDLER_ARGS
 { 
        int error = 0;
index dc3b1fb..db6edd8 100644 (file)
@@ -63,9 +63,7 @@ cfs_sysctl_table_header_t *obd_table_header = NULL;
 #define OBD_SYSCTL 300
 
 enum {
 #define OBD_SYSCTL 300
 
 enum {
-        OBD_FAIL_LOC = 1,       /* control test failures instrumentation */
-        OBD_FAIL_VAL,           /* userdata for fail loc */
-        OBD_TIMEOUT,            /* RPC timeout before recovery/intr */
+        OBD_TIMEOUT = 3,        /* RPC timeout before recovery/intr */
         OBD_DUMP_ON_TIMEOUT,    /* dump kernel debug log upon eviction */
         OBD_MEMUSED,            /* bytes currently OBD_ALLOCated */
         OBD_PAGESUSED,          /* pages currently OBD_PAGE_ALLOCated */
         OBD_DUMP_ON_TIMEOUT,    /* dump kernel debug log upon eviction */
         OBD_MEMUSED,            /* bytes currently OBD_ALLOCated */
         OBD_PAGESUSED,          /* pages currently OBD_PAGE_ALLOCated */
@@ -110,18 +108,6 @@ enum {
 
 #endif
 
 
 #endif
 
-
-int LL_PROC_PROTO(proc_fail_loc)
-{
-        int rc;
-        long old_fail_loc = obd_fail_loc;
-
-        rc = ll_proc_dolongvec(table, write, filp, buffer, lenp, ppos);
-        if (old_fail_loc != obd_fail_loc)
-                cfs_waitq_signal(&obd_race_waitq);
-        return rc;
-}
-
 int LL_PROC_PROTO(proc_set_timeout)
 {
         int rc;
 int LL_PROC_PROTO(proc_set_timeout)
 {
         int rc;
@@ -327,22 +313,6 @@ int LL_PROC_PROTO(proc_at_history)
 #ifdef CONFIG_SYSCTL
 static cfs_sysctl_table_t obd_table[] = {
         {
 #ifdef CONFIG_SYSCTL
 static cfs_sysctl_table_t obd_table[] = {
         {
-                .ctl_name = OBD_FAIL_LOC,
-                .procname = "fail_loc",
-                .data     = &obd_fail_loc,
-                .maxlen   = sizeof(obd_fail_loc),
-                .mode     = 0644,
-                .proc_handler = &proc_fail_loc
-        },
-        {
-                .ctl_name = OBD_FAIL_VAL,
-                .procname = "fail_val",
-                .data     = &obd_fail_val,
-                .maxlen   = sizeof(int),
-                .mode     = 0644,
-                .proc_handler = &proc_dointvec
-        },
-        {
                 .ctl_name = OBD_TIMEOUT,
                 .procname = "timeout",
                 .data     = &obd_timeout,
                 .ctl_name = OBD_TIMEOUT,
                 .procname = "timeout",
                 .data     = &obd_timeout,
index 1ba0bca..7d499fd 100644 (file)
@@ -1472,7 +1472,7 @@ struct dentry *filter_fid2dentry(struct obd_device *obd,
             obd->u.filter.fo_destroys_in_progress == 0) {
                 /* don't fail lookups for orphan recovery, it causes
                  * later LBUGs when objects still exist during precreate */
             obd->u.filter.fo_destroys_in_progress == 0) {
                 /* don't fail lookups for orphan recovery, it causes
                  * later LBUGs when objects still exist during precreate */
-                CDEBUG(D_INFO, "*** obd_fail_loc=%x ***\n",OBD_FAIL_OST_ENOENT);
+                CDEBUG(D_INFO, "*** cfs_fail_loc=%x ***\n",OBD_FAIL_OST_ENOENT);
                 RETURN(ERR_PTR(-ENOENT));
         }
         if (id == 0) {
                 RETURN(ERR_PTR(-ENOENT));
         }
         if (id == 0) {
@@ -3700,11 +3700,10 @@ static int filter_statfs(struct obd_device *obd, struct obd_statfs *osfs,
                 struct lr_server_data *lsd = class_server_data(obd);
                 int index = le32_to_cpu(lsd->lsd_ost_index);
 
                 struct lr_server_data *lsd = class_server_data(obd);
                 int index = le32_to_cpu(lsd->lsd_ost_index);
 
-                if (obd_fail_val == -1 ||
-                    index == obd_fail_val)
+                if (cfs_fail_val == -1 || index == cfs_fail_val)
                         osfs->os_bfree = osfs->os_bavail = 2;
                         osfs->os_bfree = osfs->os_bavail = 2;
-                else if (obd_fail_loc & OBD_FAIL_ONCE)
-                        obd_fail_loc &= ~OBD_FAILED; /* reset flag */
+                else if (cfs_fail_loc & OBD_FAIL_ONCE)
+                        cfs_fail_loc &= ~OBD_FAILED; /* reset flag */
         }
 
         /* set EROFS to state field if FS is mounted as RDONLY. The goal is to
         }
 
         /* set EROFS to state field if FS is mounted as RDONLY. The goal is to
index e039671..135d423 100644 (file)
@@ -1112,7 +1112,7 @@ static int ost_brw_write(struct ptlrpc_request *req, struct obd_trans_info *oti)
         rc = req_capsule_server_pack(&req->rq_pill);
         if (rc != 0)
                 GOTO(out, rc);
         rc = req_capsule_server_pack(&req->rq_pill);
         if (rc != 0)
                 GOTO(out, rc);
-        OBD_FAIL_TIMEOUT(OBD_FAIL_OST_BRW_PAUSE_PACK, obd_fail_val);
+        CFS_FAIL_TIMEOUT(OBD_FAIL_OST_BRW_PAUSE_PACK, cfs_fail_val);
         rcs = req_capsule_server_get(&req->rq_pill, &RMF_RCS);
 
         tls = ost_tls_get(req);
         rcs = req_capsule_server_get(&req->rq_pill, &RMF_RCS);
 
         tls = ost_tls_get(req);
index ae3565f..7c0fb49 100644 (file)
@@ -1246,7 +1246,7 @@ static int after_reply(struct ptlrpc_request *req)
         }
 
         if (lustre_msg_get_opc(req->rq_reqmsg) != OBD_PING)
         }
 
         if (lustre_msg_get_opc(req->rq_reqmsg) != OBD_PING)
-                OBD_FAIL_TIMEOUT(OBD_FAIL_PTLRPC_PAUSE_REP, obd_fail_val);
+                CFS_FAIL_TIMEOUT(OBD_FAIL_PTLRPC_PAUSE_REP, cfs_fail_val);
         ptlrpc_at_adj_service(req, lustre_msg_get_timeout(req->rq_repmsg));
         ptlrpc_at_adj_net_latency(req,
                                   lustre_msg_get_service_time(req->rq_repmsg));
         ptlrpc_at_adj_service(req, lustre_msg_get_timeout(req->rq_repmsg));
         ptlrpc_at_adj_net_latency(req,
                                   lustre_msg_get_service_time(req->rq_repmsg));
index 1e3c9d9..902cdfd 100644 (file)
@@ -1495,9 +1495,9 @@ ptlrpc_server_handle_req_in(struct ptlrpc_service *svc)
         }
 
         if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_DROP_REQ_OPC) &&
         }
 
         if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_DROP_REQ_OPC) &&
-            lustre_msg_get_opc(req->rq_reqmsg) == obd_fail_val) {
+            lustre_msg_get_opc(req->rq_reqmsg) == cfs_fail_val) {
                 CERROR("drop incoming rpc opc %u, x"LPU64"\n",
                 CERROR("drop incoming rpc opc %u, x"LPU64"\n",
-                       obd_fail_val, req->rq_xid);
+                       cfs_fail_val, req->rq_xid);
                 goto err_req;
         }
 
                 goto err_req;
         }
 
@@ -1705,7 +1705,7 @@ ptlrpc_server_handle_request(struct ptlrpc_service *svc,
                lustre_msg_get_opc(request->rq_reqmsg));
 
         if (lustre_msg_get_opc(request->rq_reqmsg) != OBD_PING)
                lustre_msg_get_opc(request->rq_reqmsg));
 
         if (lustre_msg_get_opc(request->rq_reqmsg) != OBD_PING)
-                OBD_FAIL_TIMEOUT_MS(OBD_FAIL_PTLRPC_PAUSE_REQ, obd_fail_val);
+                CFS_FAIL_TIMEOUT_MS(OBD_FAIL_PTLRPC_PAUSE_REQ, cfs_fail_val);
 
         rc = svc->srv_handler(request);
 
 
         rc = svc->srv_handler(request);
 
index 13a6bfd..b3d4920 100644 (file)
@@ -1707,7 +1707,7 @@ run_test 39 "leak_finder recognizes both LUSTRE and LNET malloc messages"
 test_40() { # bug 15759
        start_ost
        #define OBD_FAIL_TGT_TOOMANY_THREADS     0x706
 test_40() { # bug 15759
        start_ost
        #define OBD_FAIL_TGT_TOOMANY_THREADS     0x706
-       do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000706"
+       do_facet $SINGLEMDS "$LCTL set_param fail_loc=0x80000706"
        start_mds
        cleanup
 }
        start_mds
        cleanup
 }
index e98a284..4ea7e29 100755 (executable)
@@ -1581,7 +1581,7 @@ test_65a() #bug 3055
 
     do_facet $SINGLEMDS lctl set_param fail_val=$((${REQ_DELAY} * 1000))
 #define OBD_FAIL_PTLRPC_PAUSE_REQ        0x50a
 
     do_facet $SINGLEMDS lctl set_param fail_val=$((${REQ_DELAY} * 1000))
 #define OBD_FAIL_PTLRPC_PAUSE_REQ        0x50a
-    do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0x8000050a
+    do_facet $SINGLEMDS $LCTL set_param fail_loc=0x8000050a
     createmany -o $DIR/$tfile 10 > /dev/null
     unlinkmany $DIR/$tfile 10 > /dev/null
     # check for log message
     createmany -o $DIR/$tfile 10 > /dev/null
     unlinkmany $DIR/$tfile 10 > /dev/null
     # check for log message
@@ -1613,14 +1613,14 @@ test_65b() #bug 3055
 
     do_facet ost1 lctl set_param fail_val=${REQ_DELAY}
 #define OBD_FAIL_OST_BRW_PAUSE_PACK      0x224
 
     do_facet ost1 lctl set_param fail_val=${REQ_DELAY}
 #define OBD_FAIL_OST_BRW_PAUSE_PACK      0x224
-    do_facet ost1 sysctl -w lustre.fail_loc=0x224
+    do_facet ost1 $LCTL set_param fail_loc=0x224
 
     rm -f $DIR/$tfile
     lfs setstripe $DIR/$tfile --index=0 --count=1
     # force some real bulk transfer
     multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
 
 
     rm -f $DIR/$tfile
     lfs setstripe $DIR/$tfile --index=0 --count=1
     # force some real bulk transfer
     multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
 
-    do_facet ost1 sysctl -w lustre.fail_loc=0
+    do_facet ost1 $LCTL set_param fail_loc=0
     # check for log message
     $LCTL dk | grep "Early reply #" || error "No early reply"
     debugrestore
     # check for log message
     $LCTL dk | grep "Early reply #" || error "No early reply"
     debugrestore
@@ -1636,18 +1636,18 @@ test_66a() #bug 3055
     at_start || return 0
     lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
     # adjust 5s at a time so no early reply is sent (within deadline)
     at_start || return 0
     lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
     # adjust 5s at a time so no early reply is sent (within deadline)
-    do_facet $SINGLEMDS "sysctl -w lustre.fail_val=5000"
+    do_facet $SINGLEMDS "$LCTL set_param fail_val=5000"
 #define OBD_FAIL_PTLRPC_PAUSE_REQ        0x50a
 #define OBD_FAIL_PTLRPC_PAUSE_REQ        0x50a
-    do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x8000050a"
+    do_facet $SINGLEMDS "$LCTL set_param fail_loc=0x8000050a"
     createmany -o $DIR/$tfile 20 > /dev/null
     unlinkmany $DIR/$tfile 20 > /dev/null
     lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
     createmany -o $DIR/$tfile 20 > /dev/null
     unlinkmany $DIR/$tfile 20 > /dev/null
     lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
-    do_facet $SINGLEMDS "sysctl -w lustre.fail_val=10000"
-    do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x8000050a"
+    do_facet $SINGLEMDS "$LCTL set_param fail_val=10000"
+    do_facet $SINGLEMDS "$LCTL set_param fail_loc=0x8000050a"
     createmany -o $DIR/$tfile 20 > /dev/null
     unlinkmany $DIR/$tfile 20 > /dev/null
     lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
     createmany -o $DIR/$tfile 20 > /dev/null
     unlinkmany $DIR/$tfile 20 > /dev/null
     lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
-    do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
+    do_facet $SINGLEMDS "$LCTL set_param fail_loc=0"
     sleep 9
     createmany -o $DIR/$tfile 20 > /dev/null
     unlinkmany $DIR/$tfile 20 > /dev/null
     sleep 9
     createmany -o $DIR/$tfile 20 > /dev/null
     unlinkmany $DIR/$tfile 20 > /dev/null
@@ -1665,11 +1665,11 @@ test_66b() #bug 3055
 
     at_start || return 0
     ORIG=$(lctl get_param -n mdc.${FSNAME}-*.timeouts | awk '/network/ {print $4}')
 
     at_start || return 0
     ORIG=$(lctl get_param -n mdc.${FSNAME}-*.timeouts | awk '/network/ {print $4}')
-    sysctl -w lustre.fail_val=$(($ORIG + 5))
+    $LCTL set_param fail_val=$(($ORIG + 5))
 #define OBD_FAIL_PTLRPC_PAUSE_REP      0x50c
 #define OBD_FAIL_PTLRPC_PAUSE_REP      0x50c
-    sysctl -w lustre.fail_loc=0x50c
+    $LCTL set_param fail_loc=0x50c
     ls $DIR/$tfile > /dev/null 2>&1
     ls $DIR/$tfile > /dev/null 2>&1
-    sysctl -w lustre.fail_loc=0
+    $LCTL set_param fail_loc=0
     CUR=$(lctl get_param -n mdc.${FSNAME}-*.timeouts | awk '/network/ {print $4}')
     WORST=$(lctl get_param -n mdc.${FSNAME}-*.timeouts | awk '/network/ {print $6}')
     echo "network timeout orig $ORIG, cur $CUR, worst $WORST"
     CUR=$(lctl get_param -n mdc.${FSNAME}-*.timeouts | awk '/network/ {print $4}')
     WORST=$(lctl get_param -n mdc.${FSNAME}-*.timeouts | awk '/network/ {print $6}')
     echo "network timeout orig $ORIG, cur $CUR, worst $WORST"
@@ -1684,12 +1684,12 @@ test_67a() #bug 3055
     at_start || return 0
     CONN1=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
     # sleeping threads may drive values above this
     at_start || return 0
     CONN1=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
     # sleeping threads may drive values above this
-    do_facet ost1 "sysctl -w lustre.fail_val=400"
+    do_facet ost1 "$LCTL set_param fail_val=400"
 #define OBD_FAIL_PTLRPC_PAUSE_REQ    0x50a
 #define OBD_FAIL_PTLRPC_PAUSE_REQ    0x50a
-    do_facet ost1 "sysctl -w lustre.fail_loc=0x50a"
+    do_facet ost1 "$LCTL set_param fail_loc=0x50a"
     createmany -o $DIR/$tfile 20 > /dev/null
     unlinkmany $DIR/$tfile 20 > /dev/null
     createmany -o $DIR/$tfile 20 > /dev/null
     unlinkmany $DIR/$tfile 20 > /dev/null
-    do_facet ost1 "sysctl -w lustre.fail_loc=0"
+    do_facet ost1 "$LCTL set_param fail_loc=0"
     CONN2=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
     ATTEMPTS=$(($CONN2 - $CONN1))
     echo "$ATTEMPTS osc reconnect attempts on gradual slow"
     CONN2=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
     ATTEMPTS=$(($CONN2 - $CONN1))
     echo "$ATTEMPTS osc reconnect attempts on gradual slow"
@@ -1717,8 +1717,8 @@ test_67b() #bug 3055
     lfs setstripe $DIR/$tdir/${OST} -o 0 -c 1 || error "setstripe"
     echo "Creating to objid $last_id on ost $OST..."
 #define OBD_FAIL_OST_PAUSE_CREATE        0x223
     lfs setstripe $DIR/$tdir/${OST} -o 0 -c 1 || error "setstripe"
     echo "Creating to objid $last_id on ost $OST..."
 #define OBD_FAIL_OST_PAUSE_CREATE        0x223
-    do_facet ost1 "sysctl -w lustre.fail_val=20000"
-    do_facet ost1 "sysctl -w lustre.fail_loc=0x80000223"
+    do_facet ost1 "$LCTL set_param fail_val=20000"
+    do_facet ost1 "$LCTL set_param fail_loc=0x80000223"
     createmany -o $DIR/$tdir/${OST}/f $next_id $((last_id - next_id + 2))
 
     client_reconnect
     createmany -o $DIR/$tdir/${OST}/f $next_id $((last_id - next_id + 2))
 
     client_reconnect
@@ -1728,9 +1728,9 @@ test_67b() #bug 3055
     ATTEMPTS=$(($CONN2 - $CONN1))
     echo "$ATTEMPTS osc reconnect attempts on instant slow"
     # do it again; should not timeout
     ATTEMPTS=$(($CONN2 - $CONN1))
     echo "$ATTEMPTS osc reconnect attempts on instant slow"
     # do it again; should not timeout
-    do_facet ost1 "sysctl -w lustre.fail_loc=0x80000223"
+    do_facet ost1 "$LCTL set_param fail_loc=0x80000223"
     cp /etc/profile $DIR/$tfile || error "cp failed"
     cp /etc/profile $DIR/$tfile || error "cp failed"
-    do_facet ost1 "sysctl -w lustre.fail_loc=0"
+    do_facet ost1 "$LCTL set_param fail_loc=0"
     client_reconnect
     do_facet ost1 "lctl get_param -n ost.OSS.ost_create.timeouts"
     CONN3=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
     client_reconnect
     do_facet ost1 "lctl get_param -n ost.OSS.ost_create.timeouts"
     CONN3=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
@@ -1759,13 +1759,13 @@ test_68 () #bug 13813
     mkdir -p $DIR/$tdir
     lfs setstripe $DIR/$tdir --index=0 --count=1
 #define OBD_FAIL_LDLM_PAUSE_CANCEL       0x312
     mkdir -p $DIR/$tdir
     lfs setstripe $DIR/$tdir --index=0 --count=1
 #define OBD_FAIL_LDLM_PAUSE_CANCEL       0x312
-    sysctl -w lustre.fail_val=$(($TIMEOUT - 1))
-    sysctl -w lustre.fail_loc=0x80000312
+    $LCTL set_param fail_val=$(($TIMEOUT - 1))
+    $LCTL set_param fail_loc=0x80000312
     cp /etc/profile $DIR/$tdir/${tfile}_1 || error "1st cp failed $?"
     cp /etc/profile $DIR/$tdir/${tfile}_1 || error "1st cp failed $?"
-    sysctl -w lustre.fail_val=$((TIMEOUT * 5 / 4))
-    sysctl -w lustre.fail_loc=0x80000312
+    $LCTL set_param fail_val=$((TIMEOUT * 5 / 4))
+    $LCTL set_param fail_loc=0x80000312
     cp /etc/profile $DIR/$tdir/${tfile}_2 || error "2nd cp failed $?"
     cp /etc/profile $DIR/$tdir/${tfile}_2 || error "2nd cp failed $?"
-    sysctl -w lustre.fail_loc=0
+    $LCTL set_param fail_loc=0
 
     echo $ENQ_MIN >> $ldlm_enqueue_min
     do_facet ost1 "echo $ENQ_MIN_R >> $ldlm_enqueue_min_r"
 
     echo $ENQ_MIN >> $ldlm_enqueue_min
     do_facet ost1 "echo $ENQ_MIN_R >> $ldlm_enqueue_min_r"
index 575dda4..b3062ed 100644 (file)
@@ -421,7 +421,7 @@ test_8()
     TOUCHPID=$!
     echo "waiting for touch (pid $TOUCHPID) to finish..."
     sleep 2 # give it a chance to really trigger context init rpc
     TOUCHPID=$!
     echo "waiting for touch (pid $TOUCHPID) to finish..."
     sleep 2 # give it a chance to really trigger context init rpc
-    do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0
+    do_facet $SINGLEMDS $LCTL set_param fail_loc=0
     wait $TOUCHPID || error "touch should have succeeded"
 
     $LCTL dk | grep "Early reply #" || error "No early reply"
     wait $TOUCHPID || error "touch should have succeeded"
 
     $LCTL dk | grep "Early reply #" || error "No early reply"