X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Finsanity.sh;h=497887f21f4e2d038e2e5078247f36124d780e6e;hb=f8995c83720e999b13f057739f8217822a3951fa;hp=f4b775422543d6a0fe23573a249ebfa52c63794a;hpb=be5a77ae66eab1841f67da28ac9bca10a038a1fb;p=fs%2Flustre-release.git diff --git a/lustre/tests/insanity.sh b/lustre/tests/insanity.sh index f4b7754..497887f 100755 --- a/lustre/tests/insanity.sh +++ b/lustre/tests/insanity.sh @@ -9,6 +9,7 @@ LUSTRE=${LUSTRE:-`dirname $0`/..} init_test_env $@ . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} +init_logging # ALWAYS_EXCEPT="10 $INSANITY_EXCEPT" @@ -35,11 +36,14 @@ assert_env mds_HOST MDS_MKFS_OPTS assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT assert_env LIVE_CLIENT FSNAME +require_dsh_mds || exit 0 +require_dsh_ost || exit 0 -# This can be a regexp, to allow more clients -CLIENTS=${CLIENTS:-"`comma_list $LIVE_CLIENT $FAIL_CLIENTS`"} +# FAIL_CLIENTS list should not contain the LIVE_CLIENT +FAIL_CLIENTS=$(echo " $FAIL_CLIENTS " | sed -re "s/\s+$LIVE_CLIENT\s+/ /g") DIR=${DIR:-$MOUNT} +TESTDIR=$DIR/d0.$(basename $0 .sh) ##### # fail clients round robin @@ -58,26 +62,6 @@ set_fail_client() { echo "fail $FAIL_CLIENT, next is $FAIL_NEXT" } -shutdown_client() { - client=$1 - if [ "$FAILURE_MODE" = HARD ]; then - $POWER_DOWN $client - while ping -w 3 -c 1 $client > /dev/null 2>&1; do - echo "waiting for node $client to fail" - sleep 1 - done - elif [ "$FAILURE_MODE" = SOFT ]; then - zconf_umount $client $MOUNT -f - fi -} - -reboot_node() { - NODE=$1 - if [ "$FAILURE_MODE" = HARD ]; then - $POWER_UP $NODE - fi -} - fail_clients() { num=$1 @@ -103,7 +87,7 @@ fail_clients() { echo "down clients: $DOWN_CLIENTS" for client in $DOWN_CLIENTS; do - reboot_node $client + boot_node $client done DOWN_NUM=`echo $DOWN_CLIENTS | wc -w` client_rmdirs @@ -129,29 +113,28 @@ client_touch() { file=$1 for c in $LIVE_CLIENT $FAIL_CLIENTS; do if echo $DOWN_CLIENTS | grep -q $c; then continue; fi - $PDSH $c touch $MOUNT/${c}_$file || return 1 + $PDSH $c touch $TESTDIR/${c}_$file || return 1 done } client_rm() { file=$1 for c in $LIVE_CLIENT $FAIL_CLIENTS; do - $PDSH $c rm $MOUNT/${c}_$file + $PDSH $c rm $TESTDIR/${c}_$file done } client_mkdirs() { for c in $LIVE_CLIENT $FAIL_CLIENTS; do - echo "$c mkdir $MOUNT/$c" - $PDSH $c "mkdir $MOUNT/$c" - $PDSH $c "ls -l $MOUNT/$c" + echo "$c mkdir $TESTDIR/$c" + $PDSH $c "mkdir $TESTDIR/$c && ls -l $TESTDIR/$c" done } client_rmdirs() { for c in $LIVE_CLIENT $FAIL_CLIENTS; do - echo "rmdir $MOUNT/$c" - $PDSH $LIVE_CLIENT "rmdir $MOUNT/$c" + echo "rmdir $TESTDIR/$c" + $PDSH $LIVE_CLIENT "rmdir $TESTDIR/$c" done } @@ -160,20 +143,19 @@ clients_recover_osts() { # do_node $CLIENTS "$LCTL "'--device %OSC_`hostname`_'"${facet}_svc_MNT_client_facet recover" } -cleanup_and_setup_lustre +check_and_setup_lustre + +rm -rf $TESTDIR +mkdir -p $TESTDIR # 9 Different Failure Modes Combinations echo "Starting Test 17 at `date`" test_0() { - facet_failover $SINGLEMDS - echo "Waiting for df pid: $DFPID" - wait $DFPID || { echo "df returned $?" && return 1; } + fail $SINGLEMDS for i in $(seq $OSTCOUNT) ; do - facet_failover ost$i || return 4 - echo "Waiting for df pid: $DFPID" - wait $DFPID || { echo "df returned $?" && return 3; } + fail ost$i done return 0 } @@ -192,7 +174,7 @@ test_2() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up shutdown_facet $SINGLEMDS reboot_facet $SINGLEMDS @@ -201,7 +183,7 @@ test_2() { change_active $SINGLEMDS reboot_facet $SINGLEMDS - client_df & + clients_up & DFPID=$! sleep 5 @@ -219,7 +201,7 @@ test_2() { wait $DFPID clients_recover_osts ost1 echo "Verify reintegration" - client_df || return 1 + clients_up || return 1 } run_test 2 "Second Failure Mode: MDS/OST `date`" @@ -233,12 +215,11 @@ test_3() { [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" #MDS Portion - facet_failover $SINGLEMDS - wait $DFPID || echo df failed: $? + fail $SINGLEMDS #Check FS echo "Test Lustre stability after MDS failover" - client_df + clients_up #CLIENT Portion echo "Failing 2 CLIENTS" @@ -246,13 +227,13 @@ test_3() { #Check FS echo "Test Lustre stability after CLIENT failure" - client_df + clients_up #Reintegration echo "Reintegrating CLIENTS" reintegrate_clients || return 1 - client_df || return 3 + clients_up || return 3 sleep 2 # give it a little time for fully recovered before next test } run_test 3 "Thirdb Failure Mode: MDS/CLIENT `date`" @@ -267,7 +248,7 @@ test_4() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPIDA=$! sleep 5 @@ -279,7 +260,7 @@ test_4() { change_active $SINGLEMDS reboot_facet $SINGLEMDS - client_df & + clients_up & DFPIDB=$! sleep 5 @@ -297,14 +278,14 @@ test_4() { wait $DFPIDB clients_recover_osts ost1 echo "Test Lustre stability after MDS failover" - client_df || return 1 + clients_up || return 1 } run_test 4 "Fourth Failure Mode: OST/MDS `date`" ################################################### ############### Fifth Failure Mode ############### test_5() { - [ $OSTCOUNT -lt 2 ] && skip "$OSTCOUNT < 2, not enough OSTs" && return 0 + [ $OSTCOUNT -lt 2 ] && skip_env "$OSTCOUNT < 2, not enough OSTs" && return 0 echo "Fifth Failure Mode: OST/OST `date`" @@ -312,7 +293,7 @@ test_5() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up #OST Portion shutdown_facet ost1 @@ -320,7 +301,7 @@ test_5() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPIDA=$! sleep 5 @@ -330,7 +311,7 @@ test_5() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPIDB=$! sleep 5 @@ -347,7 +328,7 @@ test_5() { wait $DFPIDA wait $DFPIDB - client_df || return 2 + clients_up || return 2 } run_test 5 "Fifth Failure Mode: OST/OST `date`" ################################################### @@ -360,7 +341,7 @@ test_6() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile || return 2 #OST Portion @@ -369,8 +350,9 @@ test_6() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPIDA=$! + echo DFPIDA=$DFPIDA sleep 5 #CLIENT Portion @@ -379,22 +361,25 @@ test_6() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df & + clients_up & DFPIDB=$! + echo DFPIDB=$DFPIDB sleep 5 #Reintegration echo "Reintegrating OST/CLIENTs" wait_for ost1 start_ost 1 - reintegrate_clients + reintegrate_clients || return 1 sleep 5 + wait_remote_prog "stat -f" $((TIMEOUT * 3 + 20)) wait $DFPIDA wait $DFPIDB + echo "Verifying mount" [ -z "$(mounted_lustre_filesystems)" ] && return 3 - client_df + clients_up } run_test 6 "Sixth Failure Mode: OST/CLIENT `date`" ################################################### @@ -408,7 +393,7 @@ test_7() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile || return 1 #CLIENT Portion @@ -417,9 +402,9 @@ test_7() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df - $PDSH $LIVE_CLIENT "ls -l $MOUNT" - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" + clients_up + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" #Sleep echo "Wait 1 minutes" @@ -429,22 +414,19 @@ test_7() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && return 2 - client_df + clients_up client_rm testfile #MDS Portion - facet_failover $SINGLEMDS + fail $SINGLEMDS - #Check FS - echo "Test Lustre stability after MDS failover" - wait $DFPID || echo "df on down clients fails " || return 1 - $PDSH $LIVE_CLIENT "ls -l $MOUNT" - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" #Reintegration echo "Reintegrating CLIENTs" reintegrate_clients || return 2 - client_df + clients_up #Sleep echo "wait 1 minutes" @@ -462,7 +444,7 @@ test_8() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile #CLIENT Portion @@ -471,9 +453,9 @@ test_8() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df - $PDSH $LIVE_CLIENT "ls -l $MOUNT" - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" + clients_up + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" #Sleep echo "Wait 1 minutes" @@ -483,7 +465,7 @@ test_8() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile @@ -493,12 +475,12 @@ test_8() { #Check FS echo "Test Lustre stability after OST failure" - client_df & + clients_up & DFPID=$! sleep 5 #non-failout hangs forever here - #$PDSH $LIVE_CLIENT "ls -l $MOUNT" - #$PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" + #$PDSH $LIVE_CLIENT "ls -l $TESTDIR" + #$PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" #Reintegration echo "Reintegrating CLIENTs/OST" @@ -506,7 +488,7 @@ test_8() { wait_for ost1 start_ost 1 wait $DFPID - client_df || return 1 + clients_up || return 1 client_touch testfile2 || return 2 #Sleep @@ -525,7 +507,7 @@ test_9() { echo "Verify Lustre filesystem is up and running" [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running" - client_df + clients_up client_touch testfile || return 1 #CLIENT Portion @@ -534,9 +516,9 @@ test_9() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df - $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 1 - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 2 + clients_up + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" || return 1 + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" || return 2 #Sleep echo "Wait 1 minutes" @@ -544,7 +526,7 @@ test_9() { #Create files echo "Verify Lustre filesystem is up and running" - $PDSH $LIVE_CLIENT df $MOUNT || return 3 + client_up $LIVE_CLIENT || return 3 client_touch testfile || return 4 #CLIENT Portion @@ -553,14 +535,14 @@ test_9() { #Check FS echo "Test Lustre stability after CLIENTs failure" - client_df - $PDSH $LIVE_CLIENT "ls -l $MOUNT" || return 5 - $PDSH $LIVE_CLIENT "rm -f $MOUNT/*_testfile" || return 6 + clients_up + $PDSH $LIVE_CLIENT "ls -l $TESTDIR" || return 5 + $PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" || return 6 #Reintegration echo "Reintegrating CLIENTs/CLIENTs" reintegrate_clients || return 7 - client_df + clients_up #Sleep echo "Wait 1 minutes" @@ -579,4 +561,4 @@ run_test 10 "Running Availability for 6 hours..." equals_msg `basename $0`: test complete, cleaning up check_and_cleanup_lustre -[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true +[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true