X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Frecovery-small.sh;h=2f84c01c517dfd8d5b9c4df2fd7f0c6f3bbfa314;hb=fbb7ead129258897f5a5d5c9ce28d31fbbe5bca2;hp=605c092cf3ff9b3b94f86a850a7abea7b2d9b5e3;hpb=f329de1394e4df6635c5c3bc8b392f1e530cac03;p=fs%2Flustre-release.git diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh index 605c092..2f84c01 100755 --- a/lustre/tests/recovery-small.sh +++ b/lustre/tests/recovery-small.sh @@ -2,12 +2,12 @@ set -e -# bug 2986 -ALWAYS_EXCEPT="20b" +# 20b: bug 2986 +ALWAYS_EXCEPT=" 20b" LUSTRE=${LUSTRE:-`dirname $0`/..} -UPCALL=${UPCALL:-$PWD/recovery-small-upcall.sh} + . $LUSTRE/tests/test-framework.sh init_test_env $@ @@ -27,12 +27,12 @@ gen_config() { rm -f $XMLCONFIG if [ "$MDSCOUNT" -gt 1 ]; then - add_lmv lmv1 + add_lmv lmv1_svc for mds in `mds_list`; do MDSDEV=$TMP/${mds}-`hostname` - add_mds $mds --dev $MDSDEV --size $MDSSIZE --lmv lmv1 + add_mds $mds --dev $MDSDEV --size $MDSSIZE --lmv lmv1_svc done - add_lov_to_lmv lov1 lmv1 --stripe_sz $STRIPE_BYTES \ + add_lov_to_lmv lov1 lmv1_svc --stripe_sz $STRIPE_BYTES \ --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 MDS=lmv1 else @@ -45,18 +45,20 @@ gen_config() { add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE - add_client client --mds ${MDS} --lov lov1 --path $MOUNT + add_client client ${MDS} --lov lov1 --path $MOUNT } setup() { gen_config + start_krb5_kdc || exit 1 start ost --reformat $OSTLCONFARGS start ost2 --reformat $OSTLCONFARGS + start_lsvcgssd || exit 2 + start_lgssd || exit 3 [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE for mds in `mds_list`; do start $mds --reformat $MDSLCONFARGS done - grep " $MOUNT " /proc/mounts || zconf_mount `hostname` $MOUNT } @@ -65,6 +67,8 @@ cleanup() { for mds in `mds_list`; do stop $mds ${FORCE} $MDSLCONFARGS done + stop_lgssd + stop_lsvcgssd stop ost2 ${FORCE} --dump cleanup.log stop ost ${FORCE} --dump cleanup.log } @@ -88,6 +92,8 @@ fi REFORMAT=--reformat $SETUP unset REFORMAT +[ "$ONLY" == "setup" ] && exit + test_1() { drop_request "mcreate $MOUNT/1" || return 1 drop_reint_reply "mcreate $MOUNT/2" || return 2 @@ -186,6 +192,7 @@ test_12(){ #define OBD_FAIL_MDS_CLOSE_NET 0x115 sleep 2 kill -USR1 $PID + cancel_lru_locks MDC # force the close echo "waiting for multiop $PID" wait $PID || return 2 do_facet client munlink $MOUNT/$tfile || return 3 @@ -194,13 +201,13 @@ run_test 12 "recover from timed out resend in ptlrpcd (b=2494)" # Bug 113, check that readdir lost recv timeout works. test_13() { - mkdir /mnt/lustre/readdir - touch /mnt/lustre/readdir/newentry + mkdir /mnt/lustre/readdir || return 1 + touch /mnt/lustre/readdir/newentry || return # OBD_FAIL_MDS_READPAGE_NET|OBD_FAIL_ONCE do_facet mds "sysctl -w lustre.fail_loc=0x80000104" - ls /mnt/lustre/readdir || return 1 + ls /mnt/lustre/readdir || return 3 do_facet mds "sysctl -w lustre.fail_loc=0" - rm -rf /mnt/lustre/readdir + rm -rf /mnt/lustre/readdir || return 4 } run_test 13 "mdc_readpage restart test (bug 1138)" @@ -234,6 +241,10 @@ start_read_ahead() { done } +# recovery timeout. This actually should be taken from +# obd_timeout +RECOV_TIMEOUT=30 + test_16() { do_facet client cp /etc/termcap $MOUNT sync @@ -246,7 +257,7 @@ test_16() { do_facet client "cmp /etc/termcap $MOUNT/termcap" && return 1 sysctl -w lustre.fail_loc=0 # give recovery a chance to finish (shouldn't take long) - sleep $TIMEOUT + sleep $RECOV_TIMEOUT do_facet client "cmp /etc/termcap $MOUNT/termcap" || return 2 start_read_ahead } @@ -257,9 +268,10 @@ test_17() { # client will get evicted here sysctl -w lustre.fail_loc=0x80000503 do_facet client cp /etc/termcap $DIR/$tfile - sysctl -w lustre.fail_loc=0 - sleep $TIMEOUT + sleep $RECOV_TIMEOUT + sysctl -w lustre.fail_loc=0 + do_facet client "df $DIR" # expect cmp to fail do_facet client "cmp /etc/termcap $DIR/$tfile" && return 1 do_facet client "rm $DIR/$tfile" || return 2 @@ -329,6 +341,8 @@ test_19a() { drop_ldlm_cancel "chmod 0777 $f" || echo evicted do_facet client checkstat -v -p 0777 $f || echo evicted + # let the client reconnect + sleep 5 do_facet client "munlink $f" } run_test 19a "test expired_lock_main on mds (2867)" @@ -351,7 +365,7 @@ test_20a() { # bug 2983 - ldlm_handle_enqueue cleanup mkdir -p $DIR/$tdir multiop $DIR/$tdir/${tfile} O_wc & MULTI_PID=$! - usleep 500 + sleep 1 cancel_lru_locks OSC #define OBD_FAIL_LDLM_ENQUEUE_EXTENT_ERR 0x308 do_facet ost sysctl -w lustre.fail_loc=0x80000308 @@ -373,4 +387,233 @@ test_20b() { # bug 2986 - ldlm_handle_enqueue error during open } run_test 20b "ldlm_handle_enqueue error (should return error)" +test_21a() { + mkdir -p $DIR/$tdir-1 + mkdir -p $DIR/$tdir-2 + multiop $DIR/$tdir-1/f O_c & + close_pid=$! + + do_facet mds "sysctl -w lustre.fail_loc=0x80000129" + multiop $DIR/$tdir-2/f Oc & + open_pid=$! + sleep 1 + do_facet mds "sysctl -w lustre.fail_loc=0" + + do_facet mds "sysctl -w lustre.fail_loc=0x80000115" + kill -USR1 $close_pid + cancel_lru_locks MDC # force the close + wait $close_pid || return 1 + wait $open_pid || return 2 + do_facet mds "sysctl -w lustre.fail_loc=0" + + $CHECKSTAT -t file $DIR/$tdir-1/f || return 3 + $CHECKSTAT -t file $DIR/$tdir-2/f || return 4 + + rm -rf $DIR/$tdir-* +} +run_test 21a "drop close request while close and open are both in flight" + +test_21b() { + mkdir -p $DIR/$tdir-1 + mkdir -p $DIR/$tdir-2 + multiop $DIR/$tdir-1/f O_c & + close_pid=$! + + do_facet mds "sysctl -w lustre.fail_loc=0x80000107" + mcreate $DIR/$tdir-2/f & + open_pid=$! + sleep 1 + do_facet mds "sysctl -w lustre.fail_loc=0" + + kill -USR1 $close_pid + cancel_lru_locks MDC # force the close + wait $close_pid || return 1 + wait $open_pid || return 3 + + $CHECKSTAT -t file $DIR/$tdir-1/f || return 4 + $CHECKSTAT -t file $DIR/$tdir-2/f || return 5 + rm -rf $DIR/$tdir-* +} +run_test 21b "drop open request while close and open are both in flight" + +test_21c() { + mkdir -p $DIR/$tdir-1 + mkdir -p $DIR/$tdir-2 + multiop $DIR/$tdir-1/f O_c & + close_pid=$! + + do_facet mds "sysctl -w lustre.fail_loc=0x80000107" + mcreate $DIR/$tdir-2/f & + open_pid=$! + sleep 3 + do_facet mds "sysctl -w lustre.fail_loc=0" + + do_facet mds "sysctl -w lustre.fail_loc=0x80000115" + kill -USR1 $close_pid + cancel_lru_locks MDC # force the close + wait $close_pid || return 1 + wait $open_pid || return 2 + + do_facet mds "sysctl -w lustre.fail_loc=0" + + $CHECKSTAT -t file $DIR/$tdir-1/f || return 2 + $CHECKSTAT -t file $DIR/$tdir-2/f || return 3 + rm -rf $DIR/$tdir-* +} +run_test 21c "drop both request while close and open are both in flight" + +test_21d() { + mkdir -p $DIR/$tdir-1 + mkdir -p $DIR/$tdir-2 + multiop $DIR/$tdir-1/f O_c & + pid=$! + + do_facet mds "sysctl -w lustre.fail_loc=0x80000129" + multiop $DIR/$tdir-2/f Oc & + sleep 1 + do_facet mds "sysctl -w lustre.fail_loc=0" + + do_facet mds "sysctl -w lustre.fail_loc=0x80000122" + kill -USR1 $pid + cancel_lru_locks MDC # force the close + wait $pid || return 1 + do_facet mds "sysctl -w lustre.fail_loc=0" + + $CHECKSTAT -t file $DIR/$tdir-1/f || return 2 + $CHECKSTAT -t file $DIR/$tdir-2/f || return 3 + + rm -rf $DIR/$tdir-* +} +run_test 21d "drop close reply while close and open are both in flight" + +test_21e() { + mkdir -p $DIR/$tdir-1 + mkdir -p $DIR/$tdir-2 + multiop $DIR/$tdir-1/f O_c & + pid=$! + + do_facet mds "sysctl -w lustre.fail_loc=0x80000119" + touch $DIR/$tdir-2/f & + sleep 1 + do_facet mds "sysctl -w lustre.fail_loc=0" + + kill -USR1 $pid + cancel_lru_locks MDC # force the close + wait $pid || return 1 + + sleep $TIMEOUT + $CHECKSTAT -t file $DIR/$tdir-1/f || return 2 + $CHECKSTAT -t file $DIR/$tdir-2/f || return 3 + rm -rf $DIR/$tdir-* +} +run_test 21e "drop open reply while close and open are both in flight" + +test_21f() { + mkdir -p $DIR/$tdir-1 + mkdir -p $DIR/$tdir-2 + multiop $DIR/$tdir-1/f O_c & + pid=$! + + do_facet mds "sysctl -w lustre.fail_loc=0x80000119" + touch $DIR/$tdir-2/f & + sleep 1 + do_facet mds "sysctl -w lustre.fail_loc=0" + + do_facet mds "sysctl -w lustre.fail_loc=0x80000122" + kill -USR1 $pid + cancel_lru_locks MDC # force the close + wait $pid || return 1 + do_facet mds "sysctl -w lustre.fail_loc=0" + + $CHECKSTAT -t file $DIR/$tdir-1/f || return 2 + $CHECKSTAT -t file $DIR/$tdir-2/f || return 3 + rm -rf $DIR/$tdir-* +} +run_test 21f "drop both reply while close and open are both in flight" + +test_21g() { + mkdir -p $DIR/$tdir-1 + mkdir -p $DIR/$tdir-2 + multiop $DIR/$tdir-1/f O_c & + pid=$! + + do_facet mds "sysctl -w lustre.fail_loc=0x80000119" + touch $DIR/$tdir-2/f & + sleep 1 + do_facet mds "sysctl -w lustre.fail_loc=0" + + do_facet mds "sysctl -w lustre.fail_loc=0x80000115" + kill -USR1 $pid + cancel_lru_locks MDC # force the close + wait $pid || return 1 + do_facet mds "sysctl -w lustre.fail_loc=0" + + $CHECKSTAT -t file $DIR/$tdir-1/f || return 2 + $CHECKSTAT -t file $DIR/$tdir-2/f || return 3 + rm -rf $DIR/$tdir-* +} +run_test 21g "drop open reply and close request while close and open are both in flight" + +test_21h() { + mkdir -p $DIR/$tdir-1 + mkdir -p $DIR/$tdir-2 + multiop $DIR/$tdir-1/f O_c & + pid=$! + + do_facet mds "sysctl -w lustre.fail_loc=0x80000107" + touch $DIR/$tdir-2/f & + touch_pid=$! + sleep 1 + do_facet mds "sysctl -w lustre.fail_loc=0" + + do_facet mds "sysctl -w lustre.fail_loc=0x80000122" + cancel_lru_locks MDC # force the close + kill -USR1 $pid + wait $pid || return 1 + do_facet mds "sysctl -w lustre.fail_loc=0" + + wait $touch_pid || return 2 + + $CHECKSTAT -t file $DIR/$tdir-1/f || return 3 + $CHECKSTAT -t file $DIR/$tdir-2/f || return 4 + rm -rf $DIR/$tdir-* +} +run_test 21h "drop open request and close reply while close and open are both in flight" + +# bug 3462 - multiple MDC requests +test_22() { + f1=$DIR/${tfile}-1 + f2=$DIR/${tfile}-2 + + do_facet mds "sysctl -w lustre.fail_loc=0x80000115" + multiop $f2 Oc & + close_pid=$! + + sleep 1 + multiop $f1 msu || return 1 + + cancel_lru_locks MDC # force the close + do_facet mds "sysctl -w lustre.fail_loc=0" + + wait $close_pid || return 2 + rm -rf $f2 || return 4 +} +run_test 22 "drop close request and do mknod" + +test_23() { #b=4561 + multiop $DIR/$tfile O_c & + pid=$! + # give a chance for open + sleep 5 + + # try the close + drop_request "kill -USR1 $pid" + + fail mds + wait $pid || return 1 + return 0 +} +#run_test 23 "client hang when close a file after mds crash" + + $CLEANUP