2 # Test multiple failures, AKA Test 17
6 LUSTRE=${LUSTRE:-`dirname $0`/..}
7 . $LUSTRE/tests/test-framework.sh
11 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
13 ALWAYS_EXCEPT="10 $INSANITY_EXCEPT"
15 if [ "$FAILURE_MODE" = "HARD" ]; then
16 mixed_ost_devs && CONFIG_EXCEPTIONS="0 2 4 5 6 8" && \
17 echo -n "Several ost services on one ost node are used with FAILURE_MODE=$FAILURE_MODE. " && \
18 echo "Except the tests: $CONFIG_EXCEPTIONS" && \
19 ALWAYS_EXCEPT="$ALWAYS_EXCEPT $CONFIG_EXCEPTIONS"
23 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
26 CLEANUP=${CLEANUP:-""}
30 SINGLECLIENT=${SINGLECLIENT:-$HOSTNAME}
31 LIVE_CLIENT=${LIVE_CLIENT:-$SINGLECLIENT}
32 FAIL_CLIENTS=${FAIL_CLIENTS:-$RCLIENTS}
34 assert_env mds_HOST MDS_MKFS_OPTS
35 assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
36 assert_env LIVE_CLIENT FSNAME
38 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
39 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
41 # FAIL_CLIENTS list should not contain the LIVE_CLIENT
42 FAIL_CLIENTS=$(echo " $FAIL_CLIENTS " | sed -re "s/\s+$LIVE_CLIENT\s+/ /g")
47 # fail clients round robin
49 # list of failable clients
50 FAIL_LIST=($FAIL_CLIENTS)
51 FAIL_NUM=${#FAIL_LIST[*]}
54 DOWN_NUM=0 # number of nodes currently down
56 # set next client to fail
58 FAIL_CLIENT=${FAIL_LIST[$FAIL_NEXT]}
59 FAIL_NEXT=$(( (FAIL_NEXT+1) % FAIL_NUM ))
60 echo "fail $FAIL_CLIENT, next is $FAIL_NEXT"
66 log "Request clients to fail: ${num}. Num of clients to fail: ${FAIL_NUM}, already failed: $DOWN_NUM"
67 if [ -z "$num" ] || [ "$num" -gt $((FAIL_NUM - DOWN_NUM)) ]; then
68 num=$((FAIL_NUM - DOWN_NUM))
71 if [ -z "$num" ] || [ "$num" -le 0 ]; then
72 log "No clients failed!"
78 for i in `seq $num`; do
81 DOWN_CLIENTS="$DOWN_CLIENTS $client"
82 shutdown_client $client
85 echo "down clients: $DOWN_CLIENTS"
87 for client in $DOWN_CLIENTS; do
90 DOWN_NUM=`echo $DOWN_CLIENTS | wc -w`
94 reintegrate_clients() {
95 for client in $DOWN_CLIENTS; do
97 echo "Restarting $client"
98 zconf_mount $client $MOUNT || return 1
105 start ost$1 `ostdevname $1` $OST_MOUNT_OPTS
112 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
113 if echo $DOWN_CLIENTS | grep -q $c; then continue; fi
114 $PDSH $c touch $MOUNT/${c}_$file || return 1
120 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
121 $PDSH $c rm $MOUNT/${c}_$file
126 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
127 echo "$c mkdir $MOUNT/$c"
128 $PDSH $c "mkdir $MOUNT/$c"
129 $PDSH $c "ls -l $MOUNT/$c"
134 for c in $LIVE_CLIENT $FAIL_CLIENTS; do
135 echo "rmdir $MOUNT/$c"
136 $PDSH $LIVE_CLIENT "rmdir $MOUNT/$c"
140 clients_recover_osts() {
142 # do_node $CLIENTS "$LCTL "'--device %OSC_`hostname`_'"${facet}_svc_MNT_client_facet recover"
145 check_and_setup_lustre
147 # 9 Different Failure Modes Combinations
148 echo "Starting Test 17 at `date`"
153 for i in $(seq $OSTCOUNT) ; do
158 run_test 0 "Fail all nodes, independently"
160 ############### First Failure Mode ###############
162 echo "Don't do a MDS - MDS Failure Case"
163 echo "This makes no sense"
165 run_test 1 "MDS/MDS failure"
166 ###################################################
168 ############### Second Failure Mode ###############
170 echo "Verify Lustre filesystem is up and running"
171 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
175 shutdown_facet $SINGLEMDS
176 reboot_facet $SINGLEMDS
178 # prepare for MDS failover
179 change_active $SINGLEMDS
180 reboot_facet $SINGLEMDS
188 echo "Reintegrating OST"
191 start_ost 1 || return 2
194 start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS || return $?
198 clients_recover_osts ost1
199 echo "Verify reintegration"
200 client_df || return 1
203 run_test 2 "Second Failure Mode: MDS/OST `date`"
204 ###################################################
207 ############### Third Failure Mode ###############
210 echo "Verify Lustre filesystem is up and running"
211 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
217 echo "Test Lustre stability after MDS failover"
221 echo "Failing 2 CLIENTS"
225 echo "Test Lustre stability after CLIENT failure"
229 echo "Reintegrating CLIENTS"
230 reintegrate_clients || return 1
232 client_df || return 3
233 sleep 2 # give it a little time for fully recovered before next test
235 run_test 3 "Thirdb Failure Mode: MDS/CLIENT `date`"
236 ###################################################
238 ############### Fourth Failure Mode ###############
240 echo "Fourth Failure Mode: OST/MDS `date`"
246 echo "Test Lustre stability after OST failure"
252 shutdown_facet $SINGLEMDS
253 reboot_facet $SINGLEMDS
255 # prepare for MDS failover
256 change_active $SINGLEMDS
257 reboot_facet $SINGLEMDS
264 echo "Reintegrating OST"
270 start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS
275 clients_recover_osts ost1
276 echo "Test Lustre stability after MDS failover"
277 client_df || return 1
279 run_test 4 "Fourth Failure Mode: OST/MDS `date`"
280 ###################################################
282 ############### Fifth Failure Mode ###############
284 [ $OSTCOUNT -lt 2 ] && skip "$OSTCOUNT < 2, not enough OSTs" && return 0
286 echo "Fifth Failure Mode: OST/OST `date`"
289 echo "Verify Lustre filesystem is up and running"
290 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
299 echo "Test Lustre stability after OST failure"
309 echo "Test Lustre stability after OST failure"
315 echo "Reintegrating OSTs"
321 clients_recover_osts ost1
322 clients_recover_osts ost2
327 client_df || return 2
329 run_test 5 "Fifth Failure Mode: OST/OST `date`"
330 ###################################################
332 ############### Sixth Failure Mode ###############
334 echo "Sixth Failure Mode: OST/CLIENT `date`"
337 echo "Verify Lustre filesystem is up and running"
338 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
341 client_touch testfile || return 2
348 echo "Test Lustre stability after OST failure"
355 echo "Failing CLIENTs"
359 echo "Test Lustre stability after CLIENTs failure"
366 echo "Reintegrating OST/CLIENTs"
369 reintegrate_clients || return 1
372 wait_remote_prog df $((TIMEOUT * 3 + 10))
376 echo "Verifying mount"
377 [ -z "$(mounted_lustre_filesystems)" ] && return 3
380 run_test 6 "Sixth Failure Mode: OST/CLIENT `date`"
381 ###################################################
384 ############### Seventh Failure Mode ###############
386 echo "Seventh Failure Mode: CLIENT/MDS `date`"
389 echo "Verify Lustre filesystem is up and running"
390 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
393 client_touch testfile || return 1
396 echo "Part 1: Failing CLIENT"
400 echo "Test Lustre stability after CLIENTs failure"
402 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
403 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
406 echo "Wait 1 minutes"
410 echo "Verify Lustre filesystem is up and running"
411 [ -z "$(mounted_lustre_filesystems)" ] && return 2
419 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
420 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
423 echo "Reintegrating CLIENTs"
424 reintegrate_clients || return 2
428 echo "wait 1 minutes"
431 run_test 7 "Seventh Failure Mode: CLIENT/MDS `date`"
432 ###################################################
435 ############### Eighth Failure Mode ###############
437 echo "Eighth Failure Mode: CLIENT/OST `date`"
440 echo "Verify Lustre filesystem is up and running"
441 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
444 client_touch testfile
447 echo "Failing CLIENTs"
451 echo "Test Lustre stability after CLIENTs failure"
453 $PDSH $LIVE_CLIENT "ls -l $MOUNT"
454 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
457 echo "Wait 1 minutes"
461 echo "Verify Lustre filesystem is up and running"
462 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
465 client_touch testfile
473 echo "Test Lustre stability after OST failure"
477 #non-failout hangs forever here
478 #$PDSH $LIVE_CLIENT "ls -l $MOUNT"
479 #$PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile"
482 echo "Reintegrating CLIENTs/OST"
483 reintegrate_clients || return 3
487 client_df || return 1
488 client_touch testfile2 || return 2
491 echo "Wait 1 minutes"
494 run_test 8 "Eighth Failure Mode: CLIENT/OST `date`"
495 ###################################################
498 ############### Ninth Failure Mode ###############
503 echo "Verify Lustre filesystem is up and running"
504 [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
507 client_touch testfile || return 1
510 echo "Failing CLIENTs"
514 echo "Test Lustre stability after CLIENTs failure"
516 $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 1
517 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 2
520 echo "Wait 1 minutes"
524 echo "Verify Lustre filesystem is up and running"
525 $PDSH $LIVE_CLIENT df $MOUNT || return 3
526 client_touch testfile || return 4
529 echo "Failing CLIENTs"
533 echo "Test Lustre stability after CLIENTs failure"
535 $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 5
536 $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 6
539 echo "Reintegrating CLIENTs/CLIENTs"
540 reintegrate_clients || return 7
544 echo "Wait 1 minutes"
547 run_test 9 "Ninth Failure Mode: CLIENT/CLIENT `date`"
548 ###################################################
551 #Run availability after all failures
552 DURATION=${DURATION:-$((2 * 60 * 60))} # 6 hours default
553 LOADTEST=${LOADTEST:-metadata-load.py}
554 $PWD/availability.sh $CONFIG $DURATION $CLIENTS || return 1
556 run_test 10 "Running Availability for 6 hours..."
558 equals_msg `basename $0`: test complete, cleaning up
559 check_and_cleanup_lustre
560 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true