set -e
+export REFORMAT=""
+export VERBOSE=false
+
# eg, assert_env LUSTRE MDSNODES OSTNODES CLIENTS
assert_env() {
local failed=""
[ -d /r ] && export ROOT=/r
export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests
+ export LLMOUNT=${LLMOUNT:-"llmount"}
export LCONF=${LCONF:-"lconf"}
export LMC=${LMC:-"lmc"}
- export LCTL=${LCTL:-"lctl"}
- export CHECKSTAT="${CHECKSTAT:-checkstat} -v"
+ export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
+ export CHECKSTAT="${CHECKSTAT:-checkstat} "
# Paths on remote nodes, if different
export RLUSTRE=${RLUSTRE:-$LUSTRE}
# command line
- while getopts "rf:" opt $*; do
+ while getopts "rvf:" opt $*; do
case $opt in
f) CONFIG=$OPTARG;;
r) REFORMAT=--reformat;;
+ v) VERBOSE=true;;
\?) usage;;
esac
done
# save the name of the config file for the upcall
echo "XMLCONFIG=$LUSTRE/tests/$XMLCONFIG" > $LUSTRE/tests/XMLCONFIG
+# echo "CONFIG=`canonical_path $CONFIG`" > $LUSTRE/tests/CONFIG
}
# Facet functions
facet=$1
shift
active=`facet_active $facet`
- do_facet $facet $LCONF --select ${facet}_svc=${active}_facet --node ${active}_facet --ptldebug $PTLDEBUG $@ $XMLCONFIG
+ do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \
+ --node ${active}_facet --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
+ $@ $XMLCONFIG
}
stop() {
facet=$1
active=`facet_active $facet`
shift
- do_facet $facet $LCONF --select ${facet}_svc=${active}_facet --node ${active}_facet --ptldebug $PTLDEBUG $@ --cleanup $XMLCONFIG
+ do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \
+ --node ${active}_facet --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
+ $@ --cleanup $XMLCONFIG
}
zconf_mount() {
- mnt=$1
+ client=$1
+ mnt=$2
+
+ do_node $client mkdir $mnt 2> /dev/null || :
- [ -d $mnt ] || mkdir $mnt
-
if [ -x /sbin/mount.lustre ] ; then
- mount -t lustre -o nettype=$NETTYPE \
- `facet_host mds`:/mds_svc/client_facet $mnt
+ do_node $client mount -t lustre -o nettype=$NETTYPE `facet_active_host mds`:/mds_svc/client_facet $mnt || return 1
else
# this is so cheating
+ do_node $client $LCONF --nosetup --node client_facet $XMLCONFIG > /dev/null || return 2
$LCONF --nosetup --node client_facet $XMLCONFIG
- $LUSTRE/utils/llmount `facet_host mds`:/mds_svc/client_facet $mnt \
- -o nettype=$NETTYPE
+ do_node $client $LLMOUNT `facet_active_host mds`:/mds_svc/client_facet $mnt -o nettype=$NETTYPE|| return 4
fi
[ -d /r ] && $LCTL modules > /r/tmp/ogdb-`hostname`
}
zconf_umount() {
- mnt=$1
- umount $mnt || :
- $LCONF --cleanup --nosetup --node client_facet $XMLCONFIG || :
+ client=$1
+ mnt=$2
+ [ "$3" ] && force=-f
+ do_node $client umount $force $mnt || :
+ do_node $client $LCONF --cleanup --nosetup --node client_facet $XMLCONFIG > /dev/null || :
}
shutdown_facet() {
wait_for_host() {
HOST=$1
check_network $HOST 900
- while ! $PDSH $HOST "$CHECKSTAT -t dir $LUSTRE"; do sleep 5; done
+ while ! do_node $HOST "$CHECKSTAT -t dir $LUSTRE"; do sleep 5; done
+ while ! do_node $HOST "ls -d $LUSTRE " > /dev/null; do sleep 5; done
}
wait_for() {
client_df() {
# not every config has many clients
if [ ! -z "$CLIENTS" ]; then
- $PDSH $CLIENTS "df $MOUNT" | dshbak -c
+ $PDSH $CLIENTS "df $MOUNT" > /dev/null
fi
}
reboot_facet $facet
client_df &
DFPID=$!
+ echo "df pid is $DFPID"
change_active $facet
TO=`facet_active_host $facet`
- echo "Failover MDS to $TO"
+ echo "Failover $facet to $TO"
wait_for $facet
start $facet
}
$LMC -m ${XMLCONFIG} $@
}
+h2gm () {
+ if [ "$1" = "client" ]; then echo \'*\'; else
+ $PDSH $1 $GMNALNID -l | cut -d\ -f2
+ fi
+}
+
h2tcp() {
if [ "$1" = "client" ]; then echo \'*\'; else
echo $1
facet_active_host() {
local facet=$1
local active=`facet_active $facet`
- echo `facet_host $active`
+ if [ "$facet" == client ]; then
+ hostname
+ else
+ echo `facet_host $active`
+ fi
}
change_active() {
do_node() {
HOST=$1
shift
+
+ if $VERBOSE; then
+ echo "CMD: $HOST $@"
+ $PDSH $HOST $LCTL mark "$@" > /dev/null 2>&1 || :
+ fi
$PDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests; cd $RPWD; sh -c \"$@\")"
}
do_facet() {
local facet=$1
shift
echo "add facet $facet: `facet_host $facet`"
- do_lmc --add node --node ${facet}_facet $@ --timeout $TIMEOUT
+ do_lmc --add node --node ${facet}_facet $@ --timeout $TIMEOUT \
+ --lustre_upcall $UPCALL --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM
do_lmc --add net --node ${facet}_facet --nid `facet_nid $facet` \
--nettype $NETTYPE
}
facet=$1
shift
rm -f ${facet}active
- add_facet $facet --lustre_upcall $UPCALL
+ add_facet $facet
do_lmc --add mds --node ${facet}_facet --mds ${facet}_svc $*
}
}
##################################
+# OBD_FAIL funcs
+
+drop_request() {
+# OBD_FAIL_MDS_ALL_REQUEST_NET
+ RC=0
+ do_facet mds "echo 0x123 > /proc/sys/lustre/fail_loc"
+ do_facet client "$1" || RC=$?
+ do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
+ return $RC
+}
+
+drop_reply() {
+# OBD_FAIL_MDS_ALL_REPLY_NET
+ RC=0
+ do_facet mds "echo 0x122 > /proc/sys/lustre/fail_loc"
+ do_facet client "$@" || RC=$?
+ do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
+ return $RC
+}
+
+pause_bulk() {
+#define OBD_FAIL_OST_BRW_PAUSE_BULK 0x214
+ RC=0
+ do_facet ost "echo 0x214 > /proc/sys/lustre/fail_loc"
+ do_facet client "$1" || RC=$?
+ do_facet client "sync"
+ do_facet ost "echo 0 > /proc/sys/lustre/fail_loc"
+ return $RC
+}
+
+drop_ldlm_cancel() {
+#define OBD_FAIL_LDLM_CANCEL 0x304
+ RC=0
+ do_facet client "echo 0x304 > /proc/sys/lustre/fail_loc"
+ do_facet client "$@" || RC=$?
+ do_facet client "echo 0 > /proc/sys/lustre/fail_loc"
+ return $RC
+}
+
+drop_bl_callback() {
+#define OBD_FAIL_LDLM_BL_CALLBACK 0x305
+ RC=0
+ do_facet client "echo 0x305 > /proc/sys/lustre/fail_loc"
+ do_facet client "$@" || RC=$?
+ do_facet client "echo 0 > /proc/sys/lustre/fail_loc"
+ return $RC
+}
+
+clear_failloc() {
+ facet=$1
+ pause=$2
+ sleep $pause
+ echo "clearing fail_loc on $facet"
+ do_facet $facet "sysctl -w lustre.fail_loc=0"
+}
+
+cancel_lru_locks() {
+ $LCTL mark cancel_lru_locks
+ for d in /proc/fs/lustre/ldlm/namespaces/$1*; do
+ if [ -f $d/lru_size ]; then
+ echo clear > $d/lru_size
+ grep [0-9] $d/lock_unused_count
+ fi
+ done
+}
+
+##################################
# Test interface
error() {
echo "${TESTSUITE}: **** FAIL:" $@
run_one() {
testnum=$1
message=$2
- tfile=f$base
- tdir=d$base
+ tfile=f${testnum}
+ tdir=d${base}
# Pretty tests run faster.
equals_msg $testnum: $message
test_${testnum} || error "test_$testnum failed with $?"
}
+
+canonical_path() {
+ (cd `dirname $1`; echo $PWD/`basename $1`)
+}
+
git://git.whamcloud.com / fs / lustre-release.git / blobdiff