. ${CONFIG:=$LUSTRE/tests/cfg/insanity-local.sh}
-ALWAYS_EXCEPT=""
+ALWAYS_EXCEPT="10"
build_test_filter
fail_clients() {
num=$1
- if [ -z "$num" -o $num -gt $((FAIL_NUM - DOWN_NUM)) ]; then
+ if [ -z "$num" ] || [ "$num" -gt $((FAIL_NUM - DOWN_NUM)) ]; then
num=$((FAIL_NUM - DOWN_NUM))
fi
- if [ -z "$num" -o $num -le 0 ]; then
+ if [ -z "$num" ] || [ "$num" -le 0 ]; then
return
fi
[ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
wait_for mds
start mds $MDSLCONFARGS ${REFORMAT}
- while ! $PDSH $HOST "$CHECKSTAT -t dir $LUSTRE"; do sleep 5; done
+ while ! do_node $HOST "$CHECKSTAT -t dir $LUSTRE"; do sleep 5; done
do_node $CLIENTS lconf --node client_facet \
--select mds_service=$ACTIVEMDS $XMLCONFIG
}
clients_recover_osts() {
facet=$1
- $PDSH $CLIENTS "$LCTL "'--device %OSC_`hostname`_OST_'"${facet}_svc_MNT_client recover"
+ $PDSH $CLIENTS "$LCTL "'--device %OSC_`hostname`_'"${facet}_svc_MNT_client_facet recover"
}
if [ "$ONLY" == "cleanup" ]; then
wait $DFPID
clients_recover_osts ost1
echo "Verify reintegration"
- client_df
+ client_df || return 1
}
run_test 2 "Second Failure Mode: MDS/OST `date`"
echo "Reintegrating CLIENTS"
reintegrate_clients
- client_df
+ client_df || return 1
}
run_test 3 "Thirdb Failure Mode: MDS/CLIENT `date`"
###################################################
wait $DFPID
clients_recover_osts ost1
echo "Test Lustre stability after MDS failover"
- client_df
+ client_df || return 1
}
run_test 4 "Fourth Failure Mode: OST/MDS `date`"
###################################################
clients_recover_osts ost1
clients_recover_osts ost2
- client_df
+ sleep 5
+ client_df || return 1
}
run_test 5 "Fifth Failure Mode: OST/OST `date`"
###################################################
#Create files
echo "Verify Lustre filesystem is up and running"
- client_df
- $PDSH $CLIENTS "/bin/touch $MOUNT/\`hostname\`_testfile"
+ client_df || return 1
+ $PDSH $CLIENTS "/bin/touch $MOUNT/\`hostname\`_testfile" || return 2
#OST Portion
echo "Failing OST"
wait_for ost1
start ost1
reintegrate_clients
-
+ sleep 5
+
echo "Verifying mount"
- client_df
+ client_df || return 3
}
run_test 6 "Sixth Failure Mode: OST/CLIENT `date`"
###################################################
#Reintegration
echo "Reintegrating CLIENTs"
reintegrate_clients
- client_df
+ client_df || return 1
#Sleep
echo "wait 1 minutes"
echo "Reintegrating CLIENTs/OST"
reintegrate_clients
start ost1
- client_df
- $PDSH $CLIENTS "/bin/touch $MOUNT/CLIENT_OST_2\`hostname\`_testfile"
+ client_df || return 1
+ $PDSH $CLIENTS "/bin/touch $MOUNT/CLIENT_OST_2\`hostname\`_testfile" || return 2
#Sleep
echo "Wait 1 minutes"
#Check FS
echo "Test Lustre stability after CLIENTs failure"
client_df
- $PDSH $LIVE_CLIENT "ls -l $MOUNT"
- $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
+ $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 1
+ $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 2
#Sleep
echo "Wait 1 minutes"
#Create files
echo "Verify Lustre filesystem is up and running"
- client_df
- $PDSH $CLIENTS "/bin/touch $MOUNT/\`hostname\`_testfile"
+ client_df || return 3
+ $PDSH $CLIENTS "/bin/touch $MOUNT/\`hostname\`_testfile" || return 4
#CLIENT Portion
echo "Failing CLIENTs"
#Check FS
echo "Test Lustre stability after CLIENTs failure"
client_df
- $PDSH $LIVE_CLIENT "ls -l $MOUNT"
- $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
+ $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 5
+ $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 6
#Reintegration
echo "Reintegrating CLIENTs/CLIENTs"
reintegrate_clients
- client_df
+ client_df || return 7
#Sleep
echo "Wait 1 minutes"
export LCONF=${LCONF:-"lconf"}
export LMC=${LMC:-"lmc"}
export LCTL=${LCTL:-"lctl"}
- export CHECKSTAT="${CHECKSTAT:-checkstat} -v"
+ export CHECKSTAT="${CHECKSTAT:-checkstat} "
# Paths on remote nodes, if different
export RLUSTRE=${RLUSTRE:-$LUSTRE}
wait_for_host() {
HOST=$1
check_network $HOST 900
- while ! $PDSH $HOST "$CHECKSTAT -t dir $LUSTRE"; do sleep 5; done
+ while ! do_node $HOST "$CHECKSTAT -t dir $LUSTRE"; do sleep 5; done
}
wait_for() {
client_df() {
# not every config has many clients
if [ ! -z "$CLIENTS" ]; then
- $PDSH $CLIENTS "df $MOUNT" | dshbak -c
+ $PDSH $CLIENTS "df $MOUNT" > /dev/null
fi
}