2 # Test multiple failures, AKA Test 17
6 LUSTRE=${LUSTRE:-`dirname $0`/..}
7 . $LUSTRE/tests/test-framework.sh
11 . ${CONFIG:=$LUSTRE/tests/cfg/insanity-local.sh}
13 ALWAYS_EXCEPT="10 $INSANITY_EXCEPT"
16 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
19 CLEANUP=${CLEANUP:-""}
23 assert_env mds_HOST MDS_MKFS_OPTS MDSDEV
24 assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
25 assert_env LIVE_CLIENT FSNAME
28 # This can be a regexp, to allow more clients
29 CLIENTS=${CLIENTS:-"`comma_list $LIVE_CLIENT $FAIL_CLIENTS $EXTRA_CLIENTS`"}
34 # fail clients round robin
36 # list of failable clients
37 FAIL_LIST=($FAIL_CLIENTS)
38 FAIL_NUM=${#FAIL_LIST[*]}
41 DOWN_NUM=0 # number of nodes currently down
43 # set next client to fail
45 FAIL_CLIENT=${FAIL_LIST[$FAIL_NEXT]}
46 FAIL_NEXT=$(( (FAIL_NEXT+1) % FAIL_NUM ))
47 echo "fail $FAIL_CLIENT, next is $FAIL_NEXT"
52 if [ "$FAILURE_MODE" = HARD ]; then
54 while ping -w 3 -c 1 $client > /dev/null 2>&1; do
55 echo "waiting for node $client to fail"
58 elif [ "$FAILURE_MODE" = SOFT ]; then
59 zconf_umount $client $MOUNT -f
65 if [ "$FAILURE_MODE" = HARD ]; then
72 if [ -z "$num" ] || [ "$num" -gt $((FAIL_NUM - DOWN_NUM)) ]; then
73 num=$((FAIL_NUM - DOWN_NUM))
76 if [ -z "$num" ] || [ "$num" -le 0 ]; then
82 for i in `seq $num`; do
85 DOWN_CLIENTS="$DOWN_CLIENTS $client"
86 shutdown_client $client
89 echo "down clients: $DOWN_CLIENTS"
91 for client in $DOWN_CLIENTS; do
94 DOWN_NUM=`echo $DOWN_CLIENTS | wc -w`
98 reintegrate_clients() {
99 for client in $DOWN_CLIENTS; do
100 wait_for_host $client
101 echo "Restarting $client"
102 zconf_mount $client $MOUNT || return 1
109 start ost$1 `ostdevname $1` $OST_MOUNT_OPTS
116 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
117 if echo $DOWN_CLIENTS | grep -q $c; then continue; fi
118 $PDSH $c touch $MOUNT/${c}_$file || return 1
124 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
125 $PDSH $c rm $MOUNT/${c}_$file
130 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
131 echo "$c mkdir $MOUNT/$c"
132 $PDSH $c "mkdir $MOUNT/$c"
133 $PDSH $c "ls -l $MOUNT/$c"
138 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
139 echo "rmdir $MOUNT/$c"
140 $PDSH $LIVE_CLIENT "rmdir $MOUNT/$c"
144 clients_recover_osts() {
146 # do_node $CLIENTS "$LCTL "'--device %OSC_`hostname`_'"${facet}_svc_MNT_client_facet recover"
149 cleanup_and_setup_lustre
151 # 9 Different Failure Modes Combinations
152 echo "Starting Test 17 at `date`"
156 echo "Waiting for df pid: $DFPID"
157 wait $DFPID || { echo "df returned $?" && return 1; }
159 facet_failover ost1 || return 4
160 echo "Waiting for df pid: $DFPID"
161 wait $DFPID || { echo "df returned $?" && return 2; }
163 if [ $OSTCOUNT -gt 1 ]; then
164 facet_failover ost2 || return 5
165 echo "Waiting for df pid: $DFPID"
166 wait $DFPID || { echo "df returned $?" && return 3; }
170 run_test 0 "Fail all nodes, independently"
172 ############### First Failure Mode ###############
174 echo "Don't do a MDS - MDS Failure Case"
175 echo "This makes no sense"
177 run_test 1 "MDS/MDS failure"
178 ###################################################
180 ############### Second Failure Mode ###############
182 echo "Verify Lustre filesystem is up and running"
183 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
190 # prepare for MDS failover
200 echo "Reintegrating OST"
203 start_ost 1 || return 2
206 start mds $MDSDEV $MDS_MOUNT_OPTS || return $?
210 clients_recover_osts ost1
211 echo "Verify reintegration"
212 client_df || return 1
215 run_test 2 "Second Failure Mode: MDS/OST `date`"
216 ###################################################
219 ############### Third Failure Mode ###############
222 echo "Verify Lustre filesystem is up and running"
223 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
227 wait $DFPID || echo df failed: $?
230 echo "Test Lustre stability after MDS failover"
234 echo "Failing 2 CLIENTS"
238 echo "Test Lustre stability after CLIENT failure"
242 echo "Reintegrating CLIENTS"
243 reintegrate_clients || return 1
245 client_df || return 3
247 run_test 3 "Thirdb Failure Mode: MDS/CLIENT `date`"
248 ###################################################
250 ############### Fourth Failure Mode ###############
252 echo "Fourth Failure Mode: OST/MDS `date`"
258 echo "Test Lustre stability after OST failure"
267 # prepare for MDS failover
276 echo "Reintegrating OST"
282 start mds $MDSDEV $MDS_MOUNT_OPTS
287 clients_recover_osts ost1
288 echo "Test Lustre stability after MDS failover"
289 client_df || return 1
291 run_test 4 "Fourth Failure Mode: OST/MDS `date`"
292 ###################################################
294 ############### Fifth Failure Mode ###############
296 [ $OSTCOUNT -lt 1 ] && skip "$OSTCOUNT < 1, not enough OSTs" && return 0
298 echo "Fifth Failure Mode: OST/OST `date`"
301 echo "Verify Lustre filesystem is up and running"
302 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
311 echo "Test Lustre stability after OST failure"
321 echo "Test Lustre stability after OST failure"
327 echo "Reintegrating OSTs"
333 clients_recover_osts ost1
334 clients_recover_osts ost2
339 client_df || return 2
341 run_test 5 "Fifth Failure Mode: OST/OST `date`"
342 ###################################################
344 ############### Sixth Failure Mode ###############
346 echo "Sixth Failure Mode: OST/CLIENT `date`"
349 echo "Verify Lustre filesystem is up and running"
350 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
353 client_touch testfile || return 2
360 echo "Test Lustre stability after OST failure"
366 echo "Failing CLIENTs"
370 echo "Test Lustre stability after CLIENTs failure"
376 echo "Reintegrating OST/CLIENTs"
384 echo "Verifying mount"
385 [ -z "$(mounted_lustre_filesystems)" ] && return 3
388 run_test 6 "Sixth Failure Mode: OST/CLIENT `date`"
389 ###################################################
392 ############### Seventh Failure Mode ###############
394 echo "Seventh Failure Mode: CLIENT/MDS `date`"
397 echo "Verify Lustre filesystem is up and running"
398 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
401 client_touch testfile || return 1
404 echo "Part 1: Failing CLIENT"
408 echo "Test Lustre stability after CLIENTs failure"
410 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
411 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
414 echo "Wait 1 minutes"
418 echo "Verify Lustre filesystem is up and running"
419 [ -z "$(mounted_lustre_filesystems)" ] && return 2
428 echo "Test Lustre stability after MDS failover"
429 wait $DFPID || echo "df on down clients fails " || return 1
430 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
431 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
434 echo "Reintegrating CLIENTs"
435 reintegrate_clients || return 2
439 echo "wait 1 minutes"
442 run_test 7 "Seventh Failure Mode: CLIENT/MDS `date`"
443 ###################################################
446 ############### Eighth Failure Mode ###############
448 echo "Eighth Failure Mode: CLIENT/OST `date`"
451 echo "Verify Lustre filesystem is up and running"
452 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
455 client_touch testfile
458 echo "Failing CLIENTs"
462 echo "Test Lustre stability after CLIENTs failure"
464 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
465 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
468 echo "Wait 1 minutes"
472 echo "Verify Lustre filesystem is up and running"
473 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
476 client_touch testfile
484 echo "Test Lustre stability after OST failure"
488 #non-failout hangs forever here
489 #$PDSH $LIVE_CLIENT "ls -l $MOUNT"
490 #$PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
493 echo "Reintegrating CLIENTs/OST"
494 reintegrate_clients || return 3
498 client_df || return 1
499 client_touch testfile2 || return 2
502 echo "Wait 1 minutes"
505 run_test 8 "Eighth Failure Mode: CLIENT/OST `date`"
506 ###################################################
509 ############### Ninth Failure Mode ###############
514 echo "Verify Lustre filesystem is up and running"
515 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
518 client_touch testfile || return 1
521 echo "Failing CLIENTs"
525 echo "Test Lustre stability after CLIENTs failure"
527 $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 1
528 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 2
531 echo "Wait 1 minutes"
535 echo "Verify Lustre filesystem is up and running"
536 $PDSH $LIVE_CLIENT "grep -e $DIR /proc/mounts" || return 3
537 $PDSH $LIVE_CLIENT df $MOUNT
538 client_touch testfile || return 4
541 echo "Failing CLIENTs"
545 echo "Test Lustre stability after CLIENTs failure"
547 $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 5
548 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 6
551 echo "Reintegrating CLIENTs/CLIENTs"
552 reintegrate_clients || return 7
556 echo "Wait 1 minutes"
559 run_test 9 "Ninth Failure Mode: CLIENT/CLIENT `date`"
560 ###################################################
563 #Run availability after all failures
564 DURATION=${DURATION:-$((2 * 60 * 60))} # 6 hours default
565 LOADTEST=${LOADTEST:-metadata-load.py}
566 $PWD/availability.sh $CONFIG $DURATION $CLIENTS || return 1
568 run_test 10 "Running Availability for 6 hours..."
570 equals_msg `basename $0`: test complete, cleaning up
571 check_and_cleanup_lustre
572 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true