2 # Test multiple failures, AKA Test 17
6 LUSTRE=${LUSTRE:-`dirname $0`/..}
7 . $LUSTRE/tests/test-framework.sh
11 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
13 ALWAYS_EXCEPT="10 $INSANITY_EXCEPT"
15 if [ "$FAILURE_MODE" = "HARD" ]; then
16 mixed_ost_devs && CONFIG_EXCEPTIONS="0 2 4 5 6 8" && \
17 echo -n "Several ost services on one ost node are used with FAILURE_MODE=$FAILURE_MODE. " && \
18 echo "Except the tests: $CONFIG_EXCEPTIONS" && \
19 ALWAYS_EXCEPT="$ALWAYS_EXCEPT $CONFIG_EXCEPTIONS"
23 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
26 CLEANUP=${CLEANUP:-""}
30 SINGLECLIENT=${SINGLECLIENT:-$HOSTNAME}
31 LIVE_CLIENT=${LIVE_CLIENT:-$SINGLECLIENT}
32 FAIL_CLIENTS=${FAIL_CLIENTS:-$RCLIENTS}
34 assert_env mds_HOST MDS_MKFS_OPTS
35 assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
36 assert_env LIVE_CLIENT FSNAME
39 # This can be a regexp, to allow more clients
40 CLIENTS=${CLIENTS:-"`comma_list $LIVE_CLIENT $FAIL_CLIENTS`"}
45 # fail clients round robin
47 # list of failable clients
48 FAIL_LIST=($FAIL_CLIENTS)
49 FAIL_NUM=${#FAIL_LIST[*]}
52 DOWN_NUM=0 # number of nodes currently down
54 # set next client to fail
56 FAIL_CLIENT=${FAIL_LIST[$FAIL_NEXT]}
57 FAIL_NEXT=$(( (FAIL_NEXT+1) % FAIL_NUM ))
58 echo "fail $FAIL_CLIENT, next is $FAIL_NEXT"
63 if [ "$FAILURE_MODE" = HARD ]; then
65 while ping -w 3 -c 1 $client > /dev/null 2>&1; do
66 echo "waiting for node $client to fail"
69 elif [ "$FAILURE_MODE" = SOFT ]; then
70 zconf_umount $client $MOUNT -f
76 if [ "$FAILURE_MODE" = HARD ]; then
84 log "Request clients to fail: ${num}. Num of clients to fail: ${FAIL_NUM}, already failed: $DOWN_NUM"
85 if [ -z "$num" ] || [ "$num" -gt $((FAIL_NUM - DOWN_NUM)) ]; then
86 num=$((FAIL_NUM - DOWN_NUM))
89 if [ -z "$num" ] || [ "$num" -le 0 ]; then
90 log "No clients failed!"
96 for i in `seq $num`; do
99 DOWN_CLIENTS="$DOWN_CLIENTS $client"
100 shutdown_client $client
103 echo "down clients: $DOWN_CLIENTS"
105 for client in $DOWN_CLIENTS; do
108 DOWN_NUM=`echo $DOWN_CLIENTS | wc -w`
112 reintegrate_clients() {
113 for client in $DOWN_CLIENTS; do
114 wait_for_host $client
115 echo "Restarting $client"
116 zconf_mount $client $MOUNT || return 1
123 start ost$1 `ostdevname $1` $OST_MOUNT_OPTS
130 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
131 if echo $DOWN_CLIENTS | grep -q $c; then continue; fi
132 $PDSH $c touch $MOUNT/${c}_$file || return 1
138 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
139 $PDSH $c rm $MOUNT/${c}_$file
144 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
145 echo "$c mkdir $MOUNT/$c"
146 $PDSH $c "mkdir $MOUNT/$c"
147 $PDSH $c "ls -l $MOUNT/$c"
152 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
153 echo "rmdir $MOUNT/$c"
154 $PDSH $LIVE_CLIENT "rmdir $MOUNT/$c"
158 clients_recover_osts() {
160 # do_node $CLIENTS "$LCTL "'--device %OSC_`hostname`_'"${facet}_svc_MNT_client_facet recover"
163 cleanup_and_setup_lustre
165 # 9 Different Failure Modes Combinations
166 echo "Starting Test 17 at `date`"
169 facet_failover $SINGLEMDS
170 echo "Waiting for df pid: $DFPID"
171 wait $DFPID || { echo "df returned $?" && return 1; }
173 for i in $(seq $OSTCOUNT) ; do
174 facet_failover ost$i || return 4
175 echo "Waiting for df pid: $DFPID"
176 wait $DFPID || { echo "df returned $?" && return 3; }
180 run_test 0 "Fail all nodes, independently"
182 ############### First Failure Mode ###############
184 echo "Don't do a MDS - MDS Failure Case"
185 echo "This makes no sense"
187 run_test 1 "MDS/MDS failure"
188 ###################################################
190 ############### Second Failure Mode ###############
192 echo "Verify Lustre filesystem is up and running"
193 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
197 shutdown_facet $SINGLEMDS
198 reboot_facet $SINGLEMDS
200 # prepare for MDS failover
201 change_active $SINGLEMDS
202 reboot_facet $SINGLEMDS
210 echo "Reintegrating OST"
213 start_ost 1 || return 2
216 start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS || return $?
220 clients_recover_osts ost1
221 echo "Verify reintegration"
222 client_df || return 1
225 run_test 2 "Second Failure Mode: MDS/OST `date`"
226 ###################################################
229 ############### Third Failure Mode ###############
232 echo "Verify Lustre filesystem is up and running"
233 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
236 facet_failover $SINGLEMDS
237 wait $DFPID || echo df failed: $?
240 echo "Test Lustre stability after MDS failover"
244 echo "Failing 2 CLIENTS"
248 echo "Test Lustre stability after CLIENT failure"
252 echo "Reintegrating CLIENTS"
253 reintegrate_clients || return 1
255 client_df || return 3
256 sleep 2 # give it a little time for fully recovered before next test
258 run_test 3 "Thirdb Failure Mode: MDS/CLIENT `date`"
259 ###################################################
261 ############### Fourth Failure Mode ###############
263 echo "Fourth Failure Mode: OST/MDS `date`"
269 echo "Test Lustre stability after OST failure"
275 shutdown_facet $SINGLEMDS
276 reboot_facet $SINGLEMDS
278 # prepare for MDS failover
279 change_active $SINGLEMDS
280 reboot_facet $SINGLEMDS
287 echo "Reintegrating OST"
293 start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS
298 clients_recover_osts ost1
299 echo "Test Lustre stability after MDS failover"
300 client_df || return 1
302 run_test 4 "Fourth Failure Mode: OST/MDS `date`"
303 ###################################################
305 ############### Fifth Failure Mode ###############
307 [ $OSTCOUNT -lt 2 ] && skip "$OSTCOUNT < 2, not enough OSTs" && return 0
309 echo "Fifth Failure Mode: OST/OST `date`"
312 echo "Verify Lustre filesystem is up and running"
313 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
322 echo "Test Lustre stability after OST failure"
332 echo "Test Lustre stability after OST failure"
338 echo "Reintegrating OSTs"
344 clients_recover_osts ost1
345 clients_recover_osts ost2
350 client_df || return 2
352 run_test 5 "Fifth Failure Mode: OST/OST `date`"
353 ###################################################
355 ############### Sixth Failure Mode ###############
357 echo "Sixth Failure Mode: OST/CLIENT `date`"
360 echo "Verify Lustre filesystem is up and running"
361 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
364 client_touch testfile || return 2
371 echo "Test Lustre stability after OST failure"
377 echo "Failing CLIENTs"
381 echo "Test Lustre stability after CLIENTs failure"
387 echo "Reintegrating OST/CLIENTs"
395 echo "Verifying mount"
396 [ -z "$(mounted_lustre_filesystems)" ] && return 3
399 run_test 6 "Sixth Failure Mode: OST/CLIENT `date`"
400 ###################################################
403 ############### Seventh Failure Mode ###############
405 echo "Seventh Failure Mode: CLIENT/MDS `date`"
408 echo "Verify Lustre filesystem is up and running"
409 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
412 client_touch testfile || return 1
415 echo "Part 1: Failing CLIENT"
419 echo "Test Lustre stability after CLIENTs failure"
421 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
422 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
425 echo "Wait 1 minutes"
429 echo "Verify Lustre filesystem is up and running"
430 [ -z "$(mounted_lustre_filesystems)" ] && return 2
436 facet_failover $SINGLEMDS
439 echo "Test Lustre stability after MDS failover"
440 wait $DFPID || echo "df on down clients fails " || return 1
441 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
442 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
445 echo "Reintegrating CLIENTs"
446 reintegrate_clients || return 2
450 echo "wait 1 minutes"
453 run_test 7 "Seventh Failure Mode: CLIENT/MDS `date`"
454 ###################################################
457 ############### Eighth Failure Mode ###############
459 echo "Eighth Failure Mode: CLIENT/OST `date`"
462 echo "Verify Lustre filesystem is up and running"
463 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
466 client_touch testfile
469 echo "Failing CLIENTs"
473 echo "Test Lustre stability after CLIENTs failure"
475 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
476 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
479 echo "Wait 1 minutes"
483 echo "Verify Lustre filesystem is up and running"
484 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
487 client_touch testfile
495 echo "Test Lustre stability after OST failure"
499 #non-failout hangs forever here
500 #$PDSH $LIVE_CLIENT "ls -l $MOUNT"
501 #$PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
504 echo "Reintegrating CLIENTs/OST"
505 reintegrate_clients || return 3
509 client_df || return 1
510 client_touch testfile2 || return 2
513 echo "Wait 1 minutes"
516 run_test 8 "Eighth Failure Mode: CLIENT/OST `date`"
517 ###################################################
520 ############### Ninth Failure Mode ###############
525 echo "Verify Lustre filesystem is up and running"
526 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
529 client_touch testfile || return 1
532 echo "Failing CLIENTs"
536 echo "Test Lustre stability after CLIENTs failure"
538 $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 1
539 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 2
542 echo "Wait 1 minutes"
546 echo "Verify Lustre filesystem is up and running"
547 $PDSH $LIVE_CLIENT df $MOUNT || return 3
548 client_touch testfile || return 4
551 echo "Failing CLIENTs"
555 echo "Test Lustre stability after CLIENTs failure"
557 $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 5
558 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 6
561 echo "Reintegrating CLIENTs/CLIENTs"
562 reintegrate_clients || return 7
566 echo "Wait 1 minutes"
569 run_test 9 "Ninth Failure Mode: CLIENT/CLIENT `date`"
570 ###################################################
573 #Run availability after all failures
574 DURATION=${DURATION:-$((2 * 60 * 60))} # 6 hours default
575 LOADTEST=${LOADTEST:-metadata-load.py}
576 $PWD/availability.sh $CONFIG $DURATION $CLIENTS || return 1
578 run_test 10 "Running Availability for 6 hours..."
580 equals_msg `basename $0`: test complete, cleaning up
581 check_and_cleanup_lustre
582 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true