libcfs_debug.h libcfsutil.h libcfs_ioctl.h \
libcfs_pack.h libcfs_unpack.h libcfs_string.h \
libcfs_kernelcomm.h libcfs_workitem.h lucache.h \
- params_tree.h
+ libcfs_fail.h params_tree.h
#include <libcfs/libcfs_kernelcomm.h>
#include <libcfs/libcfs_workitem.h>
#include <libcfs/libcfs_hash.h>
+#include <libcfs/libcfs_fail.h>
#include <libcfs/params_tree.h>
/* container_of depends on "likely" which is defined in libcfs_private.h */
--- /dev/null
+/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
+ * vim:expandtab:shiftwidth=8:tabstop=8:
+ *
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * Please contact Oracle Corporation, Inc., 500 Oracle Parkway, Redwood Shores,
+ * CA 94065 USA or visit www.oracle.com if you need additional information or
+ * have any questions.
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ */
+/*
+ * This file is part of Lustre, http://www.lustre.org/
+ * Lustre is a trademark of Oracle Corporation, Inc.
+ */
+
+#ifndef _LIBCFS_FAIL_H
+#define _LIBCFS_FAIL_H
+
+extern unsigned long cfs_fail_loc;
+extern unsigned int cfs_fail_val;
+
+extern cfs_waitq_t cfs_race_waitq;
+extern int cfs_race_state;
+
+int __cfs_fail_check_set(__u32 id, __u32 value, int set);
+int __cfs_fail_timeout_set(__u32 id, __u32 value, int ms, int set);
+
+enum {
+ CFS_FAIL_LOC_NOSET = 0,
+ CFS_FAIL_LOC_ORSET = 1,
+ CFS_FAIL_LOC_RESET = 2
+};
+
+/* Failure injection control */
+#define CFS_FAIL_MASK_SYS 0x0000FF00
+#define CFS_FAIL_MASK_LOC (0x000000FF | CFS_FAIL_MASK_SYS)
+
+#define CFS_FAILED_BIT 30
+/* CFS_FAILED is 0x40000000 */
+#define CFS_FAILED (1 << CFS_FAILED_BIT)
+
+#define CFS_FAIL_ONCE_BIT 31
+/* CFS_FAIL_ONCE is 0x80000000 */
+#define CFS_FAIL_ONCE (1 << CFS_FAIL_ONCE_BIT)
+
+/* The following flags aren't made to be combined */
+#define CFS_FAIL_SKIP 0x20000000 /* skip N times then fail */
+#define CFS_FAIL_SOME 0x10000000 /* only fail N times */
+#define CFS_FAIL_RAND 0x08000000 /* fail 1/N of the times */
+#define CFS_FAIL_USR1 0x04000000 /* user flag */
+
+#define CFS_FAIL_PRECHECK(id) (cfs_fail_loc && \
+ (cfs_fail_loc & CFS_FAIL_MASK_LOC) == \
+ ((id) & CFS_FAIL_MASK_LOC))
+
+static inline int cfs_fail_check_set(__u32 id, __u32 value, int set)
+{
+ int ret = 0;
+ if (unlikely(CFS_FAIL_PRECHECK(id) &&
+ (ret = __cfs_fail_check_set(id, value, set)))) {
+ CERROR("*** cfs_fail_loc=%x ***\n", id);
+ }
+ return ret;
+}
+
+/* If id hit cfs_fail_loc, return 1, otherwise return 0 */
+#define CFS_FAIL_CHECK(id) \
+ cfs_fail_check_set(id, 0, CFS_FAIL_LOC_NOSET)
+
+/* If id hit cfs_fail_loc, cfs_fail_loc |= value and return 1,
+ * otherwise return 0 */
+#define CFS_FAIL_CHECK_ORSET(id, value) \
+ cfs_fail_check_set(id, value, CFS_FAIL_LOC_ORSET)
+
+/* If id hit cfs_fail_loc, cfs_fail_loc = value and return 1,
+ * otherwise return 0 */
+#define CFS_FAIL_CHECK_RESET(id, value) \
+ cfs_fail_check_set(id, value, CFS_FAIL_LOC_RESET)
+
+static inline int cfs_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
+{
+ if (unlikely(CFS_FAIL_PRECHECK(id)))
+ return __cfs_fail_timeout_set(id, value, ms, set);
+ else
+ return 0;
+}
+
+/* If id hit cfs_fail_loc, sleep for seconds or milliseconds */
+#define CFS_FAIL_TIMEOUT(id, secs) \
+ cfs_fail_timeout_set(id, 0, secs * 1000, CFS_FAIL_LOC_NOSET)
+
+#define CFS_FAIL_TIMEOUT_MS(id, ms) \
+ cfs_fail_timeout_set(id, 0, ms, CFS_FAIL_LOC_NOSET)
+
+/* If id hit cfs_fail_loc, cfs_fail_loc |= value and
+ * sleep seconds or milliseconds */
+#define CFS_FAIL_TIMEOUT_ORSET(id, value, secs) \
+ cfs_fail_timeout_set(id, value, secs * 1000, CFS_FAIL_LOC_ORSET)
+
+#define CFS_FAIL_TIMEOUT_MS_ORSET(id, value, ms) \
+ cfs_fail_timeout_set(id, value, ms, CFS_FAIL_LOC_ORSET)
+
+#ifdef __KERNEL__
+/* The idea here is to synchronise two threads to force a race. The
+ * first thread that calls this with a matching fail_loc is put to
+ * sleep. The next thread that calls with the same fail_loc wakes up
+ * the first and continues. */
+static inline void cfs_race(__u32 id)
+{
+ int rc;
+
+ if (CFS_FAIL_PRECHECK(id)) {
+ if (unlikely(__cfs_fail_check_set(id, 0, CFS_FAIL_LOC_NOSET))) {
+ cfs_race_state = 0;
+ CERROR("cfs_race id %x sleeping\n", id);
+ cfs_wait_event_interruptible(cfs_race_waitq, cfs_race_state != 0, rc);
+ CERROR("cfs_fail_race id %x awake\n", id);
+ } else {
+ CERROR("cfs_fail_race id %x waking\n", id);
+ cfs_race_state = 1;
+ cfs_waitq_signal(&cfs_race_waitq);
+ }
+ }
+}
+#define CFS_RACE(id) cfs_race(id)
+#else
+/* sigh. an expedient fix until CFS_RACE is fixed up */
+#define CFS_RACE(foo) do {} while(0)
+#endif
+
+#endif /* _LIBCFS_FAIL_H */
endif
-libcfs-all-objs := debug.o nidstrings.o lwt.o module.o tracefile.o watchdog.o \
+libcfs-all-objs := debug.o fail.o nidstrings.o lwt.o module.o tracefile.o watchdog.o \
libcfs_string.o hash.o kernel_user_comm.o prng.o workitem.o \
upcall_cache.o
noinst_LIBRARIES= libcfs.a
libcfs_a_SOURCES= posix/posix-debug.c user-prim.c user-lock.c user-tcpip.c \
prng.c user-bitops.c user-mem.c hash.c kernel_user_comm.c \
- workitem.c
+ workitem.c fail.c
libcfs_a_CPPFLAGS = $(LLCPPFLAGS)
libcfs_a_CFLAGS = $(LLCFLAGS)
endif
EXTRA_DIST := Info.plist
MOSTLYCLEANFILES := @MOSTLYCLEANFILES@ linux-*.c linux/*.o darwin/*.o libcfs
-DIST_SOURCES := $(libcfs-all-objs:%.o=%.c) tracefile.h user-prim.c prng.c \
- user-lock.c user-tcpip.c user-bitops.c workitem.c \
- user-mem.c kernel_user_comm.c linux/linux-tracefile.h
+DIST_SOURCES := $(libcfs-all-objs:%.o=%.c) tracefile.h prng.c \
+ user-lock.c user-tcpip.c user-bitops.c user-prim.c workitem.c \
+ user-mem.c kernel_user_comm.c fail.c linux/linux-tracefile.h
return error;
}
+static int proc_fail_loc SYSCTL_HANDLER_ARGS
+{
+ int error = 0;
+ long old_fail_loc = cfs_fail_loc;
+
+ error = sysctl_handle_long(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
+ if (!error && req->newptr != USER_ADDR_NULL) {
+ if (old_fail_loc != cfs_fail_loc)
+ cfs_waitq_signal(&cfs_race_waitq);
+ } else if (req->newptr != USER_ADDR_NULL) {
+ /* Something was wrong with the write request */
+ printf ("sysctl fail loc fault: %d.\n", error);
+ } else {
+ /* Read request */
+ error = SYSCTL_OUT(req, &cfs_fail_loc, sizeof cfs_fail_loc);
+ }
+ return error;
+}
+
/*
* sysctl table for lnet
*/
SYSCTL_PROC(_lnet, OID_AUTO, debug_mb,
CTLTYPE_INT | CTLFLAG_RW, 0,
0, &sysctl_debug_mb, "L", "debug_mb");
-
+SYSCTL_PROC(_lnet, OID_AUTO, fail_loc,
+ CTLTYPE_INT | CTLFLAG_RW , &cfs_fail_loc,
+ 0, &proc_fail_loc, "I", "cfs_fail_loc");
static cfs_sysctl_table_t top_table[] = {
&sysctl__lnet,
&sysctl__lnet_debug_kernel,
&sysctl__lnet_daemon_file,
&sysctl__lnet_debug_mb,
+ &sysctl__lnet_cfs_fail_loc
NULL
};
--- /dev/null
+/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
+ * vim:expandtab:shiftwidth=8:tabstop=8:
+ *
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * Please contact Oracle Corporation, Inc., 500 Oracle Parkway, Redwood Shores,
+ * CA 94065 USA or visit www.oracle.com if you need additional information or
+ * have any questions.
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ */
+/*
+ * This file is part of Lustre, http://www.lustre.org/
+ * Lustre is a trademark of Oracle Corporation, Inc.
+ */
+
+#ifndef __KERNEL__
+#include <liblustre.h>
+#else
+#include <libcfs/libcfs.h>
+#endif
+
+unsigned long cfs_fail_loc = 0;
+unsigned int cfs_fail_val = 0;
+cfs_waitq_t cfs_race_waitq;
+int cfs_race_state;
+
+CFS_EXPORT_SYMBOL(cfs_fail_loc);
+CFS_EXPORT_SYMBOL(cfs_fail_val);
+CFS_EXPORT_SYMBOL(cfs_race_waitq);
+CFS_EXPORT_SYMBOL(cfs_race_state);
+
+int __cfs_fail_check_set(__u32 id, __u32 value, int set)
+{
+ static cfs_atomic_t cfs_fail_count = CFS_ATOMIC_INIT(0);
+
+ LASSERT(!(id & CFS_FAIL_ONCE));
+
+ if ((cfs_fail_loc & (CFS_FAILED | CFS_FAIL_ONCE)) ==
+ (CFS_FAILED | CFS_FAIL_ONCE)) {
+ cfs_atomic_set(&cfs_fail_count, 0); /* paranoia */
+ return 0;
+ }
+
+ /* Fail 1/cfs_fail_val times */
+ if (cfs_fail_loc & CFS_FAIL_RAND) {
+ if (cfs_fail_val < 2 || cfs_rand() % cfs_fail_val > 0)
+ return 0;
+ }
+
+ /* Skip the first cfs_fail_val, then fail */
+ if (cfs_fail_loc & CFS_FAIL_SKIP) {
+ if (cfs_atomic_inc_return(&cfs_fail_count) <= cfs_fail_val)
+ return 0;
+ }
+
+ /* Fail cfs_fail_val times, overridden by FAIL_ONCE */
+ if (cfs_fail_loc & CFS_FAIL_SOME &&
+ (!(cfs_fail_loc & CFS_FAIL_ONCE) || cfs_fail_val <= 1)) {
+ int count = cfs_atomic_inc_return(&cfs_fail_count);
+
+ if (count >= cfs_fail_val) {
+ cfs_set_bit(CFS_FAIL_ONCE_BIT, &cfs_fail_loc);
+ cfs_atomic_set(&cfs_fail_count, 0);
+ /* we are lost race to increase */
+ if (count > cfs_fail_val)
+ return 0;
+ }
+ }
+
+ if ((set == CFS_FAIL_LOC_ORSET || set == CFS_FAIL_LOC_RESET) &&
+ (value & CFS_FAIL_ONCE))
+ cfs_set_bit(CFS_FAIL_ONCE_BIT, &cfs_fail_loc);
+ /* Lost race to set CFS_FAILED_BIT. */
+ if (cfs_test_and_set_bit(CFS_FAILED_BIT, &cfs_fail_loc)) {
+ /* If CFS_FAIL_ONCE is valid, only one process can fail,
+ * otherwise multi-process can fail at the same time. */
+ if (cfs_fail_loc & CFS_FAIL_ONCE)
+ return 0;
+ }
+
+ switch (set) {
+ case CFS_FAIL_LOC_NOSET:
+ break;
+ case CFS_FAIL_LOC_ORSET:
+ cfs_fail_loc |= value & ~(CFS_FAILED | CFS_FAIL_ONCE);
+ break;
+ case CFS_FAIL_LOC_RESET:
+ cfs_fail_loc = value;
+ break;
+ default:
+ LASSERTF(0, "called with bad set %u\n", set);
+ break;
+ }
+
+ return 1;
+}
+CFS_EXPORT_SYMBOL(__cfs_fail_check_set);
+
+int __cfs_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
+{
+ int ret = 0;
+
+ ret = __cfs_fail_check_set(id, value, set);
+ if (ret) {
+ CERROR("cfs_fail_timeout id %x sleeping for %dms\n",
+ id, ms);
+ cfs_schedule_timeout_and_set_state(CFS_TASK_UNINT,
+ cfs_time_seconds(ms) / 1000);
+ cfs_set_current_state(CFS_TASK_RUNNING);
+ CERROR("cfs_fail_timeout id %x awake\n", id);
+ }
+ return ret;
+}
+CFS_EXPORT_SYMBOL(__cfs_fail_timeout_set);
PSDEV_LNET_DEBUG_LOG_UPCALL, /* debug log upcall script */
PSDEV_LNET_WATCHDOG_RATELIMIT, /* ratelimit watchdog messages */
PSDEV_LNET_FORCE_LBUG, /* hook to force an LBUG */
+ PSDEV_LNET_FAIL_LOC, /* control test failures instrumentation */
+ PSDEV_LNET_FAIL_VAL, /* userdata for fail loc */
};
#else
#define CTL_LNET CTL_UNNUMBERED
#define PSDEV_LNET_DEBUG_LOG_UPCALL CTL_UNNUMBERED
#define PSDEV_LNET_WATCHDOG_RATELIMIT CTL_UNNUMBERED
#define PSDEV_LNET_FORCE_LBUG CTL_UNNUMBERED
+#define PSDEV_LNET_FAIL_LOC CTL_UNNUMBERED
+#define PSDEV_LNET_FAIL_VAL CTL_UNNUMBERED
#endif
-
int
proc_call_handler(void *data, int write,
loff_t *ppos, void *buffer, size_t *lenp,
return 0;
}
+int LL_PROC_PROTO(proc_fail_loc)
+{
+ int rc;
+ long old_fail_loc = cfs_fail_loc;
+
+ rc = ll_proc_dolongvec(table, write, filp, buffer, lenp, ppos);
+ if (old_fail_loc != cfs_fail_loc)
+ cfs_waitq_signal(&cfs_race_waitq);
+ return rc;
+}
+
static cfs_sysctl_table_t lnet_table[] = {
/*
* NB No .strategy entries have been provided since sysctl(8) prefers
.mode = 0200,
.proc_handler = &libcfs_force_lbug
},
+ {
+ .ctl_name = PSDEV_LNET_FAIL_LOC,
+ .procname = "fail_loc",
+ .data = &cfs_fail_loc,
+ .maxlen = sizeof(cfs_fail_loc),
+ .mode = 0644,
+ .proc_handler = &proc_fail_loc
+ },
+ {
+ .ctl_name = PSDEV_LNET_FAIL_VAL,
+ .procname = "fail_val",
+ .data = &cfs_fail_val,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = &proc_dointvec
+ },
{0}
};
cfs_init_mutex(&cfs_trace_thread_sem);
cfs_init_rwsem(&ioctl_list_sem);
CFS_INIT_LIST_HEAD(&ioctl_list);
+ cfs_waitq_init(&cfs_race_waitq);
rc = libcfs_debug_init(5 * 1024 * 1024);
if (rc < 0) {
#include <darwin/lustre_compat.h>
-#define OBD_SLEEP_ON(wq) sleep_on(wq)
-
/* for obd_class.h */
# ifndef ERR_PTR
# define ERR_PTR(a) ((void *)(a))
# include <linux/types.h>
# include <linux/blkdev.h>
# include <lvfs.h>
-# define OBD_SLEEP_ON(wq, state) wait_event_interruptible(wq, state)
#else /* !__KERNEL__ */
# define LTIME_S(time) (time)
/* for obd_class.h */
OBD_STATS_NUM,
};
-enum {
- OBD_FAIL_LOC_NOSET = 0,
- OBD_FAIL_LOC_ORSET = 1,
- OBD_FAIL_LOC_RESET = 2
-};
-
-extern unsigned long obd_fail_loc;
-extern unsigned int obd_fail_val;
extern unsigned int obd_debug_peer_on_timeout;
extern unsigned int obd_dump_on_timeout;
extern unsigned int obd_dump_on_eviction;
extern unsigned int obd_max_dirty_pages;
extern cfs_atomic_t obd_dirty_pages;
extern cfs_atomic_t obd_dirty_transit_pages;
-extern cfs_waitq_t obd_race_waitq;
-extern int obd_race_state;
extern unsigned int obd_alloc_fail_rate;
-int __obd_fail_check_set(__u32 id, __u32 value, int set);
-int __obd_fail_timeout_set(__u32 id, __u32 value, int ms, int set);
-
/* lvfs.c */
int obd_alloc_fail(const void *ptr, const char *name, const char *type,
size_t size, const char *file, int line);
#define OBD_FAIL_MDS_SYNC_CAPA_SL 0x1310
#define OBD_FAIL_SEQ_ALLOC 0x1311
-/* Failure injection control */
-#define OBD_FAIL_MASK_SYS 0x0000FF00
-#define OBD_FAIL_MASK_LOC (0x000000FF | OBD_FAIL_MASK_SYS)
-
-#define OBD_FAILED_BIT 30
-/* OBD_FAILED is 0x40000000 */
-#define OBD_FAILED (1 << OBD_FAILED_BIT)
-
-#define OBD_FAIL_ONCE_BIT 31
-/* OBD_FAIL_ONCE is 0x80000000 */
-#define OBD_FAIL_ONCE (1 << OBD_FAIL_ONCE_BIT)
-
-/* The following flags aren't made to be combined */
-#define OBD_FAIL_SKIP 0x20000000 /* skip N times then fail */
-#define OBD_FAIL_SOME 0x10000000 /* only fail N times */
-#define OBD_FAIL_RAND 0x08000000 /* fail 1/N of the times */
-#define OBD_FAIL_USR1 0x04000000 /* user flag */
-
-#define OBD_FAIL_PRECHECK(id) (obd_fail_loc && \
- (obd_fail_loc & OBD_FAIL_MASK_LOC) == \
- ((id) & OBD_FAIL_MASK_LOC))
-
-static inline int obd_fail_check_set(__u32 id, __u32 value, int set)
-{
- int ret = 0;
- if (unlikely(OBD_FAIL_PRECHECK(id) &&
- (ret = __obd_fail_check_set(id, value, set)))) {
- CERROR("*** obd_fail_loc=%x ***\n", id);
- }
- return ret;
-}
-
-/* If id hit obd_fail_loc, return 1, otherwise return 0 */
-#define OBD_FAIL_CHECK(id) \
- obd_fail_check_set(id, 0, OBD_FAIL_LOC_NOSET)
-
-/* If id hit obd_fail_loc, obd_fail_loc |= value and return 1,
- * otherwise return 0 */
-#define OBD_FAIL_CHECK_ORSET(id, value) \
- obd_fail_check_set(id, value, OBD_FAIL_LOC_ORSET)
-
-/* If id hit obd_fail_loc, obd_fail_loc = value and return 1,
- * otherwise return 0 */
-#define OBD_FAIL_CHECK_RESET(id, value) \
- obd_fail_check_set(id, value, OBD_FAIL_LOC_RESET)
-
-
-static inline int obd_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
-{
- if (unlikely(OBD_FAIL_PRECHECK(id)))
- return __obd_fail_timeout_set(id, value, ms, set);
- else
- return 0;
-}
-
-/* If id hit obd_fail_loc, sleep for seconds or milliseconds */
-#define OBD_FAIL_TIMEOUT(id, secs) \
- obd_fail_timeout_set(id, 0, secs * 1000, OBD_FAIL_LOC_NOSET)
-
-#define OBD_FAIL_TIMEOUT_MS(id, ms) \
- obd_fail_timeout_set(id, 0, ms, OBD_FAIL_LOC_NOSET)
-
-/* If id hit obd_fail_loc, obd_fail_loc |= value and
- * sleep seconds or milliseconds */
-#define OBD_FAIL_TIMEOUT_ORSET(id, value, secs) \
- obd_fail_timeout_set(id, value, secs * 1000, OBD_FAIL_LOC_ORSET)
-
-#define OBD_FAIL_TIMEOUT_MS_ORSET(id, value, ms) \
- obd_fail_timeout_set(id, value, ms, OBD_FAIL_LOC_ORSET)
+/* Assign references to moved code to reduce code changes */
+#define OBD_FAIL_PRECHECK(id) CFS_FAIL_PRECHECK(id)
+#define OBD_FAIL_CHECK(id) CFS_FAIL_CHECK(id)
+#define OBD_FAIL_CHECK_ORSET(id, value) CFS_FAIL_CHECK_ORSET(id, value)
+#define OBD_FAIL_CHECK_RESET(id, value) CFS_FAIL_CHECK_RESET(id, value)
+#define OBD_FAIL_RETURN(id, ret) CFS_FAIL_RETURN(id, ret)
+#define OBD_FAIL_TIMEOUT(id, secs) CFS_FAIL_TIMEOUT(id, secs)
+#define OBD_FAIL_TIMEOUT_MS(id, ms) CFS_FAIL_TIMEOUT_MS(id, ms)
+#define OBD_FAIL_TIMEOUT_ORSET(id, value, secs) CFS_FAIL_TIMEOUT_ORSET(id, value, secs)
+#define OBD_RACE(id) CFS_RACE(id)
+#define OBD_FAIL_ONCE CFS_FAIL_ONCE
+#define OBD_FAILED CFS_FAILED
#ifdef __KERNEL__
static inline void obd_fail_write(int id, struct super_block *sb)
if (OBD_FAIL_CHECK_ORSET(id & ~OBD_FAIL_ONCE, OBD_FAIL_ONCE)) {
#ifdef LIBCFS_DEBUG
BDEVNAME_DECLARE_STORAGE(tmp);
- CERROR("obd_fail_loc=%x, fail write operation on %s\n",
+ CERROR("cfs_fail_loc=%x, fail write operation on %s\n",
id, ll_bdevname(sb, tmp));
#endif
/* TODO-CMD: fix getting jdev */
}
}
#define OBD_FAIL_WRITE(id, sb) obd_fail_write(id, sb)
-
-/* The idea here is to synchronise two threads to force a race. The
- * first thread that calls this with a matching fail_loc is put to
- * sleep. The next thread that calls with the same fail_loc wakes up
- * the first and continues. */
-static inline void obd_race(__u32 id)
-{
- if (OBD_FAIL_PRECHECK(id)) {
- if (unlikely(__obd_fail_check_set(id, 0, OBD_FAIL_LOC_NOSET))) {
- obd_race_state = 0;
- CERROR("obd_race id %x sleeping\n", id);
- OBD_SLEEP_ON(obd_race_waitq, obd_race_state != 0);
- CERROR("obd_fail_race id %x awake\n", id);
- } else {
- CERROR("obd_fail_race id %x waking\n", id);
- obd_race_state = 1;
- cfs_waitq_signal(&obd_race_waitq);
- }
- }
-}
-#define OBD_RACE(id) obd_race(id)
-#else
-/* sigh. an expedient fix until OBD_RACE is fixed up */
-#define OBD_RACE(foo) do {} while(0)
#endif
#define fixme() CDEBUG(D_OTHER, "FIXME\n");
LASSERT(exp != NULL);
LASSERT(count > 0);
- OBD_FAIL_TIMEOUT(OBD_FAIL_LDLM_PAUSE_CANCEL, obd_fail_val);
+ CFS_FAIL_TIMEOUT(OBD_FAIL_LDLM_PAUSE_CANCEL, cfs_fail_val);
- if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_CANCEL_RACE))
+ if (CFS_FAIL_CHECK(OBD_FAIL_LDLM_CANCEL_RACE))
RETURN(count);
free = ldlm_format_handles_avail(class_exp2cliimp(exp),
if (drop_arr[drop_index].name) { \
printf("server drops next %s\n", drop_arr[drop_index].name); \
sprintf(cmd, \
- "%s %s \"echo %lu > /proc/sys/lustre/fail_loc\"", \
+ "%s %s \"lctl set_param fail_loc=%lu\"", \
ssh_cmd, mds_server, drop_arr[drop_index].code); \
if ((rc = system(cmd)) != 0) { \
rc = WEXITSTATUS(rc); \
char cmd[1024]; \
\
if (drop_arr[drop_index].name) { \
- sprintf(cmd, "%s %s \"echo 0 > /proc/sys/lustre/fail_loc\"", \
+ sprintf(cmd, "%s %s \"lctl set_param fail_loc=0\"", \
ssh_cmd, mds_server); \
if (!system(cmd)) {} \
} \
rc = ll_md_close(sbi->ll_md_exp, inode, file);
- if (OBD_FAIL_TIMEOUT_MS(OBD_FAIL_PTLRPC_DUMP_LOG, obd_fail_val))
+ if (CFS_FAIL_TIMEOUT_MS(OBD_FAIL_PTLRPC_DUMP_LOG, cfs_fail_val))
libcfs_debug_dumplog();
RETURN(rc);
lovreq = container_of(oinfo, struct lov_request, rq_oi);
- if (OBD_FAIL_CHECK(OBD_FAIL_MDS_OSC_CREATE_FAIL))
- if (lovreq->rq_idx == obd_fail_val)
+ if (CFS_FAIL_CHECK(OBD_FAIL_MDS_OSC_CREATE_FAIL))
+ if (lovreq->rq_idx == cfs_fail_val)
rc = -ENOTCONN;
rc= lov_update_create_set(lovreq->rq_rqset, lovreq, rc);
#include <lustre_lib.h>
#include <lprocfs_status.h>
-unsigned int obd_fail_val = 0;
-unsigned long obd_fail_loc = 0;
unsigned int obd_alloc_fail_rate = 0;
int obd_alloc_fail(const void *ptr, const char *name, const char *type,
}
EXPORT_SYMBOL(obd_alloc_fail);
-int __obd_fail_check_set(__u32 id, __u32 value, int set)
-{
- static cfs_atomic_t obd_fail_count = CFS_ATOMIC_INIT(0);
-
- LASSERT(!(id & OBD_FAIL_ONCE));
-
- if ((obd_fail_loc & (OBD_FAILED | OBD_FAIL_ONCE)) ==
- (OBD_FAILED | OBD_FAIL_ONCE)) {
- cfs_atomic_set(&obd_fail_count, 0); /* paranoia */
- return 0;
- }
-
- /* Fail 1/obd_fail_val times */
- if (obd_fail_loc & OBD_FAIL_RAND) {
- if (obd_fail_val < 2 || cfs_rand() % obd_fail_val > 0)
- return 0;
- }
-
- /* Skip the first obd_fail_val, then fail */
- if (obd_fail_loc & OBD_FAIL_SKIP) {
- if (cfs_atomic_inc_return(&obd_fail_count) <= obd_fail_val)
- return 0;
- }
-
- /* Fail obd_fail_val times, overridden by FAIL_ONCE */
- if (obd_fail_loc & OBD_FAIL_SOME &&
- (!(obd_fail_loc & OBD_FAIL_ONCE) || obd_fail_val <= 1)) {
- int count = cfs_atomic_inc_return(&obd_fail_count);
-
- if (count >= obd_fail_val) {
- cfs_set_bit(OBD_FAIL_ONCE_BIT, &obd_fail_loc);
- cfs_atomic_set(&obd_fail_count, 0);
- /* we are lost race to increase obd_fail_count */
- if (count > obd_fail_val)
- return 0;
- }
- }
-
- if ((set == OBD_FAIL_LOC_ORSET || set == OBD_FAIL_LOC_RESET) &&
- (value & OBD_FAIL_ONCE))
- cfs_set_bit(OBD_FAIL_ONCE_BIT, &obd_fail_loc);
-
- /* Lost race to set OBD_FAILED_BIT. */
- if (cfs_test_and_set_bit(OBD_FAILED_BIT, &obd_fail_loc)) {
- /* If OBD_FAIL_ONCE is valid, only one process can fail,
- * otherwise multi-process can fail at the same time. */
- if (obd_fail_loc & OBD_FAIL_ONCE)
- return 0;
- }
-
- switch (set) {
- case OBD_FAIL_LOC_NOSET:
- break;
- case OBD_FAIL_LOC_ORSET:
- obd_fail_loc |= value & ~(OBD_FAILED | OBD_FAIL_ONCE);
- break;
- case OBD_FAIL_LOC_RESET:
- obd_fail_loc = value;
- break;
- default:
- LASSERTF(0, "called with bad set %u\n", set);
- break;
- }
-
- return 1;
-}
-EXPORT_SYMBOL(__obd_fail_check_set);
-
-int __obd_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
-{
- int ret = 0;
-
- ret = __obd_fail_check_set(id, value, set);
- if (ret) {
- CERROR("obd_fail_timeout id %x sleeping for %dms\n",
- id, ms);
- cfs_schedule_timeout_and_set_state(CFS_TASK_UNINT,
- cfs_time_seconds(ms) / 1000);
- cfs_set_current_state(CFS_TASK_RUNNING);
- CERROR("obd_fail_timeout id %x awake\n", id);
- }
- return ret;
-}
-EXPORT_SYMBOL(__obd_fail_timeout_set);
-
#ifdef LPROCFS
void lprocfs_counter_add(struct lprocfs_stats *stats, int idx,
long amount)
EXPORT_SYMBOL(lprocfs_counter_sub);
#endif /* LPROCFS */
-EXPORT_SYMBOL(obd_fail_loc);
EXPORT_SYMBOL(obd_alloc_fail_rate);
-EXPORT_SYMBOL(obd_fail_val);
sptlrpc_svc_ctx_invalidate(req);
}
- OBD_FAIL_TIMEOUT(OBD_FAIL_SEC_CTX_HDL_PAUSE, obd_fail_val);
+ CFS_FAIL_TIMEOUT(OBD_FAIL_SEC_CTX_HDL_PAUSE, cfs_fail_val);
return rc;
}
struct dt_device *dd, int id)
{
if (OBD_FAIL_CHECK_ORSET(id, OBD_FAIL_ONCE)) {
- CERROR(LUSTRE_MDT_NAME": obd_fail_loc=%x, fail write ops\n",
+ CERROR(LUSTRE_MDT_NAME": cfs_fail_loc=%x, fail write ops\n",
id);
dd->dd_ops->dt_ro(env, dd);
/* We set FAIL_ONCE because we never "un-fail" a device */
ENTRY;
req_capsule_init(&req->rq_pill, req, RCL_SERVER);
- OBD_FAIL_TIMEOUT_MS(OBD_FAIL_MGS_PAUSE_REQ, obd_fail_val);
- if (OBD_FAIL_CHECK(OBD_FAIL_MGS_ALL_REQUEST_NET))
+ CFS_FAIL_TIMEOUT_MS(OBD_FAIL_MGS_PAUSE_REQ, cfs_fail_val);
+ if (CFS_FAIL_CHECK(OBD_FAIL_MGS_ALL_REQUEST_NET))
RETURN(0);
LASSERT(current->journal_info == NULL);
cfs_atomic_t obd_dirty_pages;
cfs_atomic_t obd_dirty_transit_pages;
-cfs_waitq_t obd_race_waitq;
-int obd_race_state;
-
-#ifdef __KERNEL__
-unsigned long obd_print_fail_loc(void)
-{
- CWARN("obd_fail_loc = %lx\n", obd_fail_loc);
- return obd_fail_loc;
-}
-#endif
-
static inline void obd_data2conn(struct lustre_handle *conn,
struct obd_ioctl_data *data)
{
#endif
EXPORT_SYMBOL(obd_devs);
-EXPORT_SYMBOL(obd_print_fail_loc);
-EXPORT_SYMBOL(obd_race_waitq);
-EXPORT_SYMBOL(obd_race_state);
EXPORT_SYMBOL(obd_debug_peer_on_timeout);
EXPORT_SYMBOL(obd_dump_on_timeout);
EXPORT_SYMBOL(obd_dump_on_eviction);
#endif
cfs_spin_lock_init(&obd_types_lock);
- cfs_waitq_init(&obd_race_waitq);
obd_zombie_impexp_init();
#ifdef LPROCFS
obd_memory = lprocfs_alloc_stats(OBD_STATS_NUM,
cfs_sysctl_table_header_t *obd_table_header = NULL;
-int proc_fail_loc SYSCTL_HANDLER_ARGS;
int proc_obd_timeout SYSCTL_HANDLER_ARGS;
-extern unsigned long obd_fail_loc;
extern unsigned int obd_dump_on_timeout;
extern unsigned int obd_timeout;
extern unsigned int ldlm_timeout;
SYSCTL_NODE (, OID_AUTO, lustre, CTLFLAG_RW,
0, "lustre sysctl top");
-SYSCTL_PROC(_lustre, OID_AUTO, fail_loc,
- CTLTYPE_INT | CTLFLAG_RW , &obd_fail_loc,
- 0, &proc_fail_loc, "I", "obd_fail_loc");
SYSCTL_PROC(_lustre, OID_AUTO, timeout,
CTLTYPE_INT | CTLFLAG_RW , &obd_timeout,
0, &proc_obd_timeout, "I", "obd_timeout");
static cfs_sysctl_table_t parent_table[] = {
&sysctl__lustre,
- &sysctl__lustre_fail_loc,
&sysctl__lustre_timeout,
&sysctl__lustre_dump_on_timeout,
&sysctl__lustre_debug_peer_on_timeout,
&sysctl__lustre_ldlm_timeout,
};
-extern cfs_waitq_t obd_race_waitq;
-
-int proc_fail_loc SYSCTL_HANDLER_ARGS
-{
- int error = 0;
- long old_fail_loc = obd_fail_loc;
-
- error = sysctl_handle_long(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
- if (!error && req->newptr != USER_ADDR_NULL) {
- if (old_fail_loc != obd_fail_loc)
- cfs_waitq_signal(&obd_race_waitq);
- } else if (req->newptr != USER_ADDR_NULL) {
- /* Something was wrong with the write request */
- printf ("sysctl fail loc fault: %d.\n", error);
- } else {
- /* Read request */
- error = SYSCTL_OUT(req, &obd_fail_loc, sizeof obd_fail_loc);
- }
- return error;
-}
-
int proc_obd_timeout SYSCTL_HANDLER_ARGS
{
int error = 0;
#define OBD_SYSCTL 300
enum {
- OBD_FAIL_LOC = 1, /* control test failures instrumentation */
- OBD_FAIL_VAL, /* userdata for fail loc */
- OBD_TIMEOUT, /* RPC timeout before recovery/intr */
+ OBD_TIMEOUT = 3, /* RPC timeout before recovery/intr */
OBD_DUMP_ON_TIMEOUT, /* dump kernel debug log upon eviction */
OBD_MEMUSED, /* bytes currently OBD_ALLOCated */
OBD_PAGESUSED, /* pages currently OBD_PAGE_ALLOCated */
#endif
-
-int LL_PROC_PROTO(proc_fail_loc)
-{
- int rc;
- long old_fail_loc = obd_fail_loc;
-
- rc = ll_proc_dolongvec(table, write, filp, buffer, lenp, ppos);
- if (old_fail_loc != obd_fail_loc)
- cfs_waitq_signal(&obd_race_waitq);
- return rc;
-}
-
int LL_PROC_PROTO(proc_set_timeout)
{
int rc;
#ifdef CONFIG_SYSCTL
static cfs_sysctl_table_t obd_table[] = {
{
- .ctl_name = OBD_FAIL_LOC,
- .procname = "fail_loc",
- .data = &obd_fail_loc,
- .maxlen = sizeof(obd_fail_loc),
- .mode = 0644,
- .proc_handler = &proc_fail_loc
- },
- {
- .ctl_name = OBD_FAIL_VAL,
- .procname = "fail_val",
- .data = &obd_fail_val,
- .maxlen = sizeof(int),
- .mode = 0644,
- .proc_handler = &proc_dointvec
- },
- {
.ctl_name = OBD_TIMEOUT,
.procname = "timeout",
.data = &obd_timeout,
obd->u.filter.fo_destroys_in_progress == 0) {
/* don't fail lookups for orphan recovery, it causes
* later LBUGs when objects still exist during precreate */
- CDEBUG(D_INFO, "*** obd_fail_loc=%x ***\n",OBD_FAIL_OST_ENOENT);
+ CDEBUG(D_INFO, "*** cfs_fail_loc=%x ***\n",OBD_FAIL_OST_ENOENT);
RETURN(ERR_PTR(-ENOENT));
}
if (id == 0) {
struct lr_server_data *lsd = class_server_data(obd);
int index = le32_to_cpu(lsd->lsd_ost_index);
- if (obd_fail_val == -1 ||
- index == obd_fail_val)
+ if (cfs_fail_val == -1 || index == cfs_fail_val)
osfs->os_bfree = osfs->os_bavail = 2;
- else if (obd_fail_loc & OBD_FAIL_ONCE)
- obd_fail_loc &= ~OBD_FAILED; /* reset flag */
+ else if (cfs_fail_loc & OBD_FAIL_ONCE)
+ cfs_fail_loc &= ~OBD_FAILED; /* reset flag */
}
/* set EROFS to state field if FS is mounted as RDONLY. The goal is to
rc = req_capsule_server_pack(&req->rq_pill);
if (rc != 0)
GOTO(out, rc);
- OBD_FAIL_TIMEOUT(OBD_FAIL_OST_BRW_PAUSE_PACK, obd_fail_val);
+ CFS_FAIL_TIMEOUT(OBD_FAIL_OST_BRW_PAUSE_PACK, cfs_fail_val);
rcs = req_capsule_server_get(&req->rq_pill, &RMF_RCS);
tls = ost_tls_get(req);
}
if (lustre_msg_get_opc(req->rq_reqmsg) != OBD_PING)
- OBD_FAIL_TIMEOUT(OBD_FAIL_PTLRPC_PAUSE_REP, obd_fail_val);
+ CFS_FAIL_TIMEOUT(OBD_FAIL_PTLRPC_PAUSE_REP, cfs_fail_val);
ptlrpc_at_adj_service(req, lustre_msg_get_timeout(req->rq_repmsg));
ptlrpc_at_adj_net_latency(req,
lustre_msg_get_service_time(req->rq_repmsg));
}
if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_DROP_REQ_OPC) &&
- lustre_msg_get_opc(req->rq_reqmsg) == obd_fail_val) {
+ lustre_msg_get_opc(req->rq_reqmsg) == cfs_fail_val) {
CERROR("drop incoming rpc opc %u, x"LPU64"\n",
- obd_fail_val, req->rq_xid);
+ cfs_fail_val, req->rq_xid);
goto err_req;
}
lustre_msg_get_opc(request->rq_reqmsg));
if (lustre_msg_get_opc(request->rq_reqmsg) != OBD_PING)
- OBD_FAIL_TIMEOUT_MS(OBD_FAIL_PTLRPC_PAUSE_REQ, obd_fail_val);
+ CFS_FAIL_TIMEOUT_MS(OBD_FAIL_PTLRPC_PAUSE_REQ, cfs_fail_val);
rc = svc->srv_handler(request);
test_40() { # bug 15759
start_ost
#define OBD_FAIL_TGT_TOOMANY_THREADS 0x706
- do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x80000706"
+ do_facet $SINGLEMDS "$LCTL set_param fail_loc=0x80000706"
start_mds
cleanup
}
do_facet $SINGLEMDS lctl set_param fail_val=$((${REQ_DELAY} * 1000))
#define OBD_FAIL_PTLRPC_PAUSE_REQ 0x50a
- do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0x8000050a
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0x8000050a
createmany -o $DIR/$tfile 10 > /dev/null
unlinkmany $DIR/$tfile 10 > /dev/null
# check for log message
do_facet ost1 lctl set_param fail_val=${REQ_DELAY}
#define OBD_FAIL_OST_BRW_PAUSE_PACK 0x224
- do_facet ost1 sysctl -w lustre.fail_loc=0x224
+ do_facet ost1 $LCTL set_param fail_loc=0x224
rm -f $DIR/$tfile
lfs setstripe $DIR/$tfile --index=0 --count=1
# force some real bulk transfer
multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
- do_facet ost1 sysctl -w lustre.fail_loc=0
+ do_facet ost1 $LCTL set_param fail_loc=0
# check for log message
$LCTL dk | grep "Early reply #" || error "No early reply"
debugrestore
at_start || return 0
lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
# adjust 5s at a time so no early reply is sent (within deadline)
- do_facet $SINGLEMDS "sysctl -w lustre.fail_val=5000"
+ do_facet $SINGLEMDS "$LCTL set_param fail_val=5000"
#define OBD_FAIL_PTLRPC_PAUSE_REQ 0x50a
- do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x8000050a"
+ do_facet $SINGLEMDS "$LCTL set_param fail_loc=0x8000050a"
createmany -o $DIR/$tfile 20 > /dev/null
unlinkmany $DIR/$tfile 20 > /dev/null
lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
- do_facet $SINGLEMDS "sysctl -w lustre.fail_val=10000"
- do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0x8000050a"
+ do_facet $SINGLEMDS "$LCTL set_param fail_val=10000"
+ do_facet $SINGLEMDS "$LCTL set_param fail_loc=0x8000050a"
createmany -o $DIR/$tfile 20 > /dev/null
unlinkmany $DIR/$tfile 20 > /dev/null
lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
- do_facet $SINGLEMDS "sysctl -w lustre.fail_loc=0"
+ do_facet $SINGLEMDS "$LCTL set_param fail_loc=0"
sleep 9
createmany -o $DIR/$tfile 20 > /dev/null
unlinkmany $DIR/$tfile 20 > /dev/null
at_start || return 0
ORIG=$(lctl get_param -n mdc.${FSNAME}-*.timeouts | awk '/network/ {print $4}')
- sysctl -w lustre.fail_val=$(($ORIG + 5))
+ $LCTL set_param fail_val=$(($ORIG + 5))
#define OBD_FAIL_PTLRPC_PAUSE_REP 0x50c
- sysctl -w lustre.fail_loc=0x50c
+ $LCTL set_param fail_loc=0x50c
ls $DIR/$tfile > /dev/null 2>&1
- sysctl -w lustre.fail_loc=0
+ $LCTL set_param fail_loc=0
CUR=$(lctl get_param -n mdc.${FSNAME}-*.timeouts | awk '/network/ {print $4}')
WORST=$(lctl get_param -n mdc.${FSNAME}-*.timeouts | awk '/network/ {print $6}')
echo "network timeout orig $ORIG, cur $CUR, worst $WORST"
at_start || return 0
CONN1=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
# sleeping threads may drive values above this
- do_facet ost1 "sysctl -w lustre.fail_val=400"
+ do_facet ost1 "$LCTL set_param fail_val=400"
#define OBD_FAIL_PTLRPC_PAUSE_REQ 0x50a
- do_facet ost1 "sysctl -w lustre.fail_loc=0x50a"
+ do_facet ost1 "$LCTL set_param fail_loc=0x50a"
createmany -o $DIR/$tfile 20 > /dev/null
unlinkmany $DIR/$tfile 20 > /dev/null
- do_facet ost1 "sysctl -w lustre.fail_loc=0"
+ do_facet ost1 "$LCTL set_param fail_loc=0"
CONN2=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
ATTEMPTS=$(($CONN2 - $CONN1))
echo "$ATTEMPTS osc reconnect attempts on gradual slow"
lfs setstripe $DIR/$tdir/${OST} -o 0 -c 1 || error "setstripe"
echo "Creating to objid $last_id on ost $OST..."
#define OBD_FAIL_OST_PAUSE_CREATE 0x223
- do_facet ost1 "sysctl -w lustre.fail_val=20000"
- do_facet ost1 "sysctl -w lustre.fail_loc=0x80000223"
+ do_facet ost1 "$LCTL set_param fail_val=20000"
+ do_facet ost1 "$LCTL set_param fail_loc=0x80000223"
createmany -o $DIR/$tdir/${OST}/f $next_id $((last_id - next_id + 2))
client_reconnect
ATTEMPTS=$(($CONN2 - $CONN1))
echo "$ATTEMPTS osc reconnect attempts on instant slow"
# do it again; should not timeout
- do_facet ost1 "sysctl -w lustre.fail_loc=0x80000223"
+ do_facet ost1 "$LCTL set_param fail_loc=0x80000223"
cp /etc/profile $DIR/$tfile || error "cp failed"
- do_facet ost1 "sysctl -w lustre.fail_loc=0"
+ do_facet ost1 "$LCTL set_param fail_loc=0"
client_reconnect
do_facet ost1 "lctl get_param -n ost.OSS.ost_create.timeouts"
CONN3=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
mkdir -p $DIR/$tdir
lfs setstripe $DIR/$tdir --index=0 --count=1
#define OBD_FAIL_LDLM_PAUSE_CANCEL 0x312
- sysctl -w lustre.fail_val=$(($TIMEOUT - 1))
- sysctl -w lustre.fail_loc=0x80000312
+ $LCTL set_param fail_val=$(($TIMEOUT - 1))
+ $LCTL set_param fail_loc=0x80000312
cp /etc/profile $DIR/$tdir/${tfile}_1 || error "1st cp failed $?"
- sysctl -w lustre.fail_val=$((TIMEOUT * 5 / 4))
- sysctl -w lustre.fail_loc=0x80000312
+ $LCTL set_param fail_val=$((TIMEOUT * 5 / 4))
+ $LCTL set_param fail_loc=0x80000312
cp /etc/profile $DIR/$tdir/${tfile}_2 || error "2nd cp failed $?"
- sysctl -w lustre.fail_loc=0
+ $LCTL set_param fail_loc=0
echo $ENQ_MIN >> $ldlm_enqueue_min
do_facet ost1 "echo $ENQ_MIN_R >> $ldlm_enqueue_min_r"
TOUCHPID=$!
echo "waiting for touch (pid $TOUCHPID) to finish..."
sleep 2 # give it a chance to really trigger context init rpc
- do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0
+ do_facet $SINGLEMDS $LCTL set_param fail_loc=0
wait $TOUCHPID || error "touch should have succeeded"
$LCTL dk | grep "Early reply #" || error "No early reply"