LU-1187 test: add dne test to insanity.sh

1. Add test_10 (fail MDT0/OST/MDT1) and
test_11(fail MDT0/client/MDT1), test_12(MDTs/OSTs/Clients),
test_13(MDTs/Clients/OSTs) and test_14(OSTs/Clients/MDTs).

2. Replace fail SINGLEMDS with fail all MDTs to
fully test DNE.

Change-Id: Ic3cae011d6ede5b0b9616608cc840a0565777ecd
Signed-off-by: Wang Di <di.wang@whamcloud.com>
Reviewed-on: http://review.whamcloud.com/4319
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Jian Yu <jian.yu@intel.com>

diff --git a/lustre/tests/insanity.sh b/lustre/tests/insanity.sh
index 74ca4a2..32701f9 100755 (executable)
--- a/lustre/tests/insanity.sh
+++ b/lustre/tests/insanity.sh
@@ -14,7 +14,7 @@ init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 init_logging
 #              
-ALWAYS_EXCEPT="10  $INSANITY_EXCEPT"
+ALWAYS_EXCEPT="15  $INSANITY_EXCEPT"
 
 if [ "$FAILURE_MODE" = "HARD" ]; then
        skip_env "$TESTSUITE: is not functional with FAILURE_MODE = HARD, " \
@@ -33,7 +33,7 @@ SINGLECLIENT=${SINGLECLIENT:-$HOSTNAME}
 LIVE_CLIENT=${LIVE_CLIENT:-$SINGLECLIENT}
 FAIL_CLIENTS=${FAIL_CLIENTS:-$RCLIENTS}
 
-assert_env mds_HOST
+assert_env mds_HOST MDSCOUNT
 assert_env ost_HOST OSTCOUNT
 assert_env LIVE_CLIENT FSNAME
 
@@ -109,6 +109,10 @@ start_ost() {
        start ost$1 `ostdevname $1` $OST_MOUNT_OPTS
 }
 
+start_mdt() {
+       start mds$1 $(mdsdevname $1) $MDS_MOUNT_OPTS
+}
+
 trap exit INT
 
 client_touch() {
@@ -154,7 +158,9 @@ mkdir -p $TESTDIR
 echo "Starting Test 17 at `date`"
 
 test_0() {
-    fail $SINGLEMDS
+       for i in $(seq $MDSCOUNT) ; do
+               fail mds$i
+       done
 
     for i in $(seq $OSTCOUNT) ; do
         fail ost$i
@@ -165,8 +171,35 @@ run_test 0 "Fail all nodes, independently"
 
 ############### First Failure Mode ###############
 test_1() {
-echo "Don't do a MDS - MDS Failure Case"
-echo "This makes no sense"
+       [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+
+       clients_up
+
+       shutdown_facet mds1
+       reboot_facet mds1
+
+       # prepare for MDS failover
+       change_active mds1
+       reboot_facet mds1
+
+       clients_up &
+       DFPID=$!
+       sleep 5
+
+       shutdown_facet mds2
+
+       echo "Reintegrating MDS2"
+       reboot_facet mds2
+       wait_for_facet mds2
+       start_mdt 2 || return 2
+
+       wait_for_facet mds1
+       start_mdt 1 || return $?
+
+       #Check FS
+       wait $DFPID
+       echo "Verify reintegration"
+       clients_up || return 1
 }
 run_test 1 "MDS/MDS failure"
 ###################################################
@@ -181,12 +214,14 @@ test_2() {
 
     clients_up
 
-    shutdown_facet $SINGLEMDS
-    reboot_facet $SINGLEMDS
+       for i in $(seq $MDSCOUNT) ; do
+               shutdown_facet mds$i
+               reboot_facet mds$i
 
-    # prepare for MDS failover
-    change_active $SINGLEMDS
-    reboot_facet $SINGLEMDS
+               # prepare for MDS failover
+               change_active mds$i
+               reboot_facet mds$i
+       done
 
     clients_up &
     DFPID=$!
@@ -199,8 +234,10 @@ test_2() {
     wait_for_facet ost1
     start_ost 1 || return 2
 
-    wait_for_facet $SINGLEMDS
-    start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS || return $?
+       for i in $(seq $MDSCOUNT) ; do
+               wait_for_facet mds$i
+               start_mdt $i || return $?
+       done
 
     #Check FS
     wait $DFPID
@@ -212,15 +249,16 @@ test_2() {
 run_test 2 "Second Failure Mode: MDS/OST `date`"
 ###################################################
 
-
 ############### Third Failure Mode ###############
 test_3() {
     #Create files
     echo "Verify Lustre filesystem is up and running"
     [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
-    
+
     #MDS Portion
-    fail $SINGLEMDS
+       for i in $(seq $MDSCOUNT) ; do
+               fail mds$i
+       done
     #Check FS
 
     echo "Test Lustre stability after MDS failover"
@@ -253,20 +291,21 @@ test_4() {
 
     #OST Portion
     shutdown_facet ost1
- 
+
     #Check FS
     echo "Test Lustre stability after OST failure"
     clients_up &
     DFPIDA=$!
     sleep 5
 
-    #MDS Portion
-    shutdown_facet $SINGLEMDS
-    reboot_facet $SINGLEMDS
+       for i in $(seq $MDSCOUNT) ; do
+               shutdown_facet mds$i
+               reboot_facet mds$i
 
-    # prepare for MDS failover
-    change_active $SINGLEMDS
-    reboot_facet $SINGLEMDS
+               # prepare for MDS failover
+               change_active mds$i
+               reboot_facet mds$i
+       done
 
     clients_up &
     DFPIDB=$!
@@ -278,10 +317,12 @@ test_4() {
     wait_for_facet ost1
     start_ost 1
 
-    wait_for_facet $SINGLEMDS
-    start $SINGLEMDS `mdsdevname 1` $MDS_MOUNT_OPTS
+       for i in $(seq $MDSCOUNT) ; do
+               wait_for_facet mds$i
+               start_mdt $i || return $?
+       done
     #Check FS
-    
+
     wait $DFPIDA
     wait $DFPIDB
     clients_recover_osts ost1
@@ -426,7 +467,9 @@ test_7() {
     client_rm testfile
 
     #MDS Portion
-    fail $SINGLEMDS
+       for i in $(seq $MDSCOUNT) ; do
+               fail mds$i
+       done
 
     $PDSH $LIVE_CLIENT "ls -l $TESTDIR"
     $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile"
@@ -559,13 +602,183 @@ test_9() {
 run_test 9 "Ninth Failure Mode: CLIENT/CLIENT `date`"
 ###################################################
 
+############### Tenth Failure Mode ###############
 test_10() {
+       [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+
+       shutdown_facet mds1
+       reboot_facet mds1
+
+       # prepare for MDS failover
+       change_active mds1
+       reboot_facet mds1
+
+       clients_up &
+       DFPID=$!
+       sleep 5
+
+       shutdown_facet ost1
+
+       echo "Reintegrating OST"
+       reboot_facet ost1
+       wait_for_facet ost1
+       start_ost 1 || return 2
+
+       shutdown_facet mds2
+       reboot_facet mds2
+
+       # prepare for MDS failover
+       change_active mds2
+       reboot_facet mds2
+
+       wait_for_facet mds1
+       start_mdt 1 || return $?
+
+       wait_for_facet mds2
+       start_mdt 2 || return $?
+
+       #Check FS
+       wait $DFPID
+       clients_recover_osts ost1
+       echo "Verify reintegration"
+       clients_up || return 1
+}
+run_test 10 "Tenth Failure Mode: MDT0/OST/MDT1 `date`"
+###################################################
+
+############### Seventh Failure Mode ###############
+test_11() {
+       [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+       echo "Verify Lustre filesystem is up and running"
+       [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
+
+       #MDS Portion
+       fail mds1
+       #Check FS
+
+       echo "Test Lustre stability after MDS failover"
+       clients_up
+
+       #CLIENT Portion
+       echo "Failing 2 CLIENTS"
+       fail_clients 2
+
+       #Check FS
+       echo "Test Lustre stability after CLIENT failure"
+       clients_up
+
+       #Reintegration
+       echo "Reintegrating CLIENTS"
+       reintegrate_clients || return 1
+
+       fail mds2
+
+       clients_up || return 3
+       sleep 2 # give it a little time for fully recovered before next test
+}
+run_test 11 "Eleventh Failure Mode: MDS0/CLIENT/MDS1 `date`"
+###################################################
+
+test_12() {
+       [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+       echo "Verify Lustre filesystem is up and running"
+       [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
+
+       #MDS Portion
+       fail mds1,mds2
+       clients_up
+
+       #OSS Portion
+       fail ost1,ost2
+       clients_up
+
+       #CLIENT Portion
+       echo "Failing 2 CLIENTS"
+       fail_clients 2
+
+       #Check FS
+       echo "Test Lustre stability after CLIENT failure"
+       clients_up
+
+       #Reintegration
+       echo "Reintegrating CLIENTS"
+       reintegrate_clients || return 1
+
+       clients_up || return 3
+       sleep 2 # give it a little time for fully recovered before next test
+}
+run_test 12 "Twelve Failure Mode: MDS0,MDS1/OST0, OST1/CLIENTS `date`"
+###################################################
+
+test_13() {
+       [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+       echo "Verify Lustre filesystem is up and running"
+       [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
+
+       #MDS Portion
+       fail mds1,mds2
+       clients_up
+
+       #CLIENT Portion
+       echo "Failing 2 CLIENTS"
+       fail_clients 2
+
+       #Check FS
+       echo "Test Lustre stability after CLIENT failure"
+       clients_up
+
+       #Reintegration
+       echo "Reintegrating CLIENTS"
+       reintegrate_clients || return 1
+
+       clients_up || return 3
+       sleep 2 # give it a little time for fully recovered before next test
+
+       #OSS Portion
+       fail ost1,ost2
+       clients_up || return 4
+}
+run_test 13 "Thirteen Failure Mode: MDS0,MDS1/CLIENTS/OST0,OST1 `date`"
+###################################################
+
+test_14() {
+       [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+       echo "Verify Lustre filesystem is up and running"
+       [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
+
+       #OST Portion
+       fail ost1,ost2
+       clients_up
+
+       #CLIENT Portion
+       echo "Failing 2 CLIENTS"
+       fail_clients 2
+
+       #Check FS
+       echo "Test Lustre stability after CLIENT failure"
+       clients_up
+
+       #Reintegration
+       echo "Reintegrating CLIENTS"
+       reintegrate_clients || return 1
+
+       clients_up || return 3
+       sleep 2 # give it a little time for fully recovered before next test
+
+       #OSS Portion
+       fail mds1,mds2
+       clients_up || return 4
+}
+run_test 14 "Fourteen Failure Mode: OST0,OST1/CLIENTS/MDS0,MDS1 `date`"
+###################################################
+
+test_15() {
     #Run availability after all failures
     DURATION=${DURATION:-$((2 * 60 * 60))} # 6 hours default
     LOADTEST=${LOADTEST:-metadata-load.py}
     $PWD/availability.sh $CONFIG $DURATION $CLIENTS || return 1
 }
-run_test 10 "Running Availability for 6 hours..."
+run_test 15 "Running Availability for 6 hours..."
 
 complete $SECONDS
 check_and_cleanup_lustre