X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Freplay-dual.sh;h=7c5a908930be3646cc996a57a62ebd6933d36e34;hb=0c6b5334c2355e37c1f2db871ca1c91d7ec5536d;hp=ee249b636940cc90184da30a2c4321655378f7bc;hpb=69426b8d6812bb72ddc7290bd054233f175e27f3;p=fs%2Flustre-release.git diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh index ee249b6..7c5a908 100755 --- a/lustre/tests/replay-dual.sh +++ b/lustre/tests/replay-dual.sh @@ -1,121 +1,49 @@ -#!/bin/sh +#!/bin/bash set -e +# bug number: 6088 10124 10800 +ALWAYS_EXCEPT="8 15c 17 $REPLAY_DUAL_EXCEPT" + +PTLDEBUG=${PTLDEBUG:--1} LUSTRE=${LUSTRE:-`dirname $0`/..} . $LUSTRE/tests/test-framework.sh init_test_env $@ -. ${CONFIG:=$LUSTRE/tests/cfg/lmv.sh} - -# Skip these tests -# 21 - open vs. unlink out of order replay: isn't solved yet -ALWAYS_EXCEPT="21" +. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} SETUP=${SETUP:-"setup"} CLEANUP=${CLEANUP:-"cleanup"} -gen_config() { - rm -f $XMLCONFIG - if [ "$MDSCOUNT" -gt 1 ]; then - add_lmv lmv1_svc - for mds in `mds_list`; do - MDSDEV=$TMP/${mds}-`hostname` - add_mds $mds --dev $MDSDEV --size $MDSSIZE --lmv lmv1_svc - done - add_lov_to_lmv lov1 lmv1_svc --stripe_sz $STRIPE_BYTES \ - --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 - MDS=lmv1 - else - add_mds mds1 --dev $MDSDEV --size $MDSSIZE - add_lov lov1 mds1 --stripe_sz $STRIPE_BYTES \ - --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 - MDS=mds1 - fi - - add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE --failover - add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE --failover - add_client client ${MDS} --lov lov1 --path $MOUNT -} - build_test_filter -SETUP=${SETUP:-"setup"} -CLEANUP=${CLEANUP:-"cleanup"} - cleanup() { - # make sure we are using the primary MDS, so the config log will - # be able to clean up properly. - activemds=`facet_active mds1` - if [ $activemds != "mds1" ]; then - fail mds1 - fi - - umount $MOUNT2 || true - umount $MOUNT || true - rmmod llite - - # b=3941 - # In mds recovery, the mds will clear orphans in ost by - # mds_lov_clear_orphan, which will sent the request to ost and waiting for - # the reply, if we stop mds at this time, we will got the obd_refcount > 1 - # errors, because mds_lov_clear_orphan grab a export of mds, - # so the obd_refcount of mds will not be zero. So, wait a while before - # stop mds. This bug needs further work. - for mds in `mds_list`; do - sleep 5 - stop $mds ${FORCE} $MDSLCONFARGS - done - stop ost2 ${FORCE} - stop ost ${FORCE} --dump cleanup-dual.log - stop_lgssd - stop_lsvcgssd + stopall } if [ "$ONLY" == "cleanup" ]; then - sysctl -w portals.debug=0 + sysctl -w lnet.debug=0 cleanup exit fi setup() { - gen_config - - start_krb5_kdc || exit 1 - start_lsvcgssd || exit 2 - start_lgssd || exit 3 - start ost --reformat $OSTLCONFARGS - PINGER=`cat /proc/fs/lustre/pinger` - - if [ "$PINGER" != "on" ]; then - echo "ERROR: Lustre must be built with --enable-pinger for replay-dual" - stop ost - exit 1 - fi - - start ost2 --reformat $OSTLCONFARGS - [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE - for mds in `mds_list`; do - start $mds --reformat $MDSLCONFARGS - done - grep " $MOUNT " /proc/mounts || zconf_mount `hostname` $MOUNT - grep " $MOUNT2 " /proc/mounts || zconf_mount `hostname` $MOUNT2 - - echo $TIMEOUT > /proc/sys/lustre/timeout - echo $UPCALL > /proc/sys/lustre/upcall + cleanup + [ "$REFORMAT" ] && formatall + setupall + mount_client $MOUNT2 } $SETUP [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE - test_1() { touch $MOUNT1/a - replay_barrier mds1 + replay_barrier $SINGLEMDS touch $MOUNT2/b - fail mds1 + fail $SINGLEMDS checkstat $MOUNT2/a || return 1 checkstat $MOUNT1/b || return 2 rm $MOUNT2/a $MOUNT1/b @@ -128,24 +56,23 @@ run_test 1 "|X| simple create" test_2() { - replay_barrier mds1 + replay_barrier $SINGLEMDS mkdir $MOUNT1/adir - fail mds1 + fail $SINGLEMDS checkstat $MOUNT2/adir || return 1 rmdir $MOUNT2/adir checkstat $MOUNT2/adir && return 2 return 0 } - run_test 2 "|X| mkdir adir" test_3() { - replay_barrier mds1 + replay_barrier $SINGLEMDS mkdir $MOUNT1/adir mkdir $MOUNT2/adir/bdir - fail mds1 + fail $SINGLEMDS checkstat $MOUNT2/adir || return 1 checkstat $MOUNT1/adir/bdir || return 2 rmdir $MOUNT2/adir/bdir $MOUNT1/adir @@ -153,16 +80,15 @@ test_3() { checkstat $MOUNT2/adir/bdir && return 4 return 0 } - run_test 3 "|X| mkdir adir, mkdir adir/bdir " test_4() { mkdir $MOUNT1/adir - replay_barrier mds1 + replay_barrier $SINGLEMDS mkdir $MOUNT1/adir && return 1 mkdir $MOUNT2/adir/bdir - fail mds1 + fail $SINGLEMDS checkstat $MOUNT2/adir || return 2 checkstat $MOUNT1/adir/bdir || return 3 @@ -171,7 +97,6 @@ test_4() { checkstat $MOUNT2/adir/bdir && return 5 return 0 } - run_test 4 "|X| mkdir adir (-EEXIST), mkdir adir/bdir " @@ -183,11 +108,11 @@ test_5() { # give multiop a chance to open sleep 1 rm -f $MOUNT1/a - replay_barrier mds1 + replay_barrier $SINGLEMDS kill -USR1 $pid wait $pid || return 1 - fail mds1 + fail $SINGLEMDS [ -e $MOUNT2/a ] && return 2 return 0 } @@ -203,43 +128,22 @@ test_6() { # give multiop a chance to open sleep 1 rm -f $MOUNT1/a - replay_barrier mds1 + replay_barrier $SINGLEMDS kill -USR1 $pid1 wait $pid1 || return 1 - fail mds1 - kill -USR1 $pid2 - wait $pid2 || return 1 - [ -e $MOUNT2/a ] && return 2 - return 0 -} -run_test 6 "open1, open2, unlink |X| close1 [fail mds] close2" - -test_6b() { - mcreate $MOUNT1/a - multiop $MOUNT2/a o_c & - pid1=$! - multiop $MOUNT1/a o_c & - pid2=$! - # give multiop a chance to open - sleep 1 - rm -f $MOUNT1/a - replay_barrier mds1 + fail $SINGLEMDS kill -USR1 $pid2 wait $pid2 || return 1 - - fail mds1 - kill -USR1 $pid1 - wait $pid1 || return 1 [ -e $MOUNT2/a ] && return 2 return 0 } -run_test 6b "open1, open2, unlink |X| close2 [fail mds] close1" +run_test 6 "open1, open2, unlink |X| close1 [fail $SINGLEMDS] close2" test_8() { - replay_barrier mds1 + replay_barrier $SINGLEMDS drop_reint_reply "mcreate $MOUNT1/$tfile" || return 1 - fail mds1 + fail $SINGLEMDS checkstat $MOUNT2/$tfile || return 2 rm $MOUNT1/$tfile || return 3 @@ -248,13 +152,13 @@ test_8() { run_test 8 "replay of resent request" test_9() { - replay_barrier mds1 + replay_barrier $SINGLEMDS mcreate $MOUNT1/$tfile-1 mcreate $MOUNT2/$tfile-2 # drop first reint reply - sysctl -w lustre.fail_loc=0x80000119 - fail mds1 - sysctl -w lustre.fail_loc=0 + do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0x80000119 + fail $SINGLEMDS + do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0 rm $MOUNT1/$tfile-[1,2] || return 1 @@ -264,13 +168,13 @@ run_test 9 "resending a replayed create" test_10() { mcreate $MOUNT1/$tfile-1 - replay_barrier mds1 + replay_barrier $SINGLEMDS munlink $MOUNT1/$tfile-1 mcreate $MOUNT2/$tfile-2 # drop first reint reply - sysctl -w lustre.fail_loc=0x80000119 - fail mds1 - sysctl -w lustre.fail_loc=0 + do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0x80000119 + fail $SINGLEMDS + do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0 checkstat $MOUNT1/$tfile-1 && return 1 checkstat $MOUNT1/$tfile-2 || return 2 @@ -281,18 +185,18 @@ test_10() { run_test 10 "resending a replayed unlink" test_11() { - replay_barrier mds1 + replay_barrier $SINGLEMDS mcreate $MOUNT1/$tfile-1 mcreate $MOUNT2/$tfile-2 mcreate $MOUNT1/$tfile-3 mcreate $MOUNT2/$tfile-4 mcreate $MOUNT1/$tfile-5 # drop all reint replies for a while - sysctl -w lustre.fail_loc=0x0119 - facet_failover mds1 + do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0x0119 + facet_failover $SINGLEMDS #sleep for while, let both clients reconnect and timeout sleep $((TIMEOUT * 2)) - sysctl -w lustre.fail_loc=0 + do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0 rm $MOUNT1/$tfile-[1-5] || return 1 @@ -301,22 +205,22 @@ test_11() { run_test 11 "both clients timeout during replay" test_12() { - replay_barrier mds1 + replay_barrier $SINGLEMDS multiop $DIR/$tfile mo_c & MULTIPID=$! sleep 5 - # drop first enqueue - sysctl -w lustre.fail_loc=0x80000302 - facet_failover mds1 +#define OBD_FAIL_LDLM_ENQUEUE 0x302 + do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0x80000302 + facet_failover $SINGLEMDS df $MOUNT || return 1 - sysctl -w lustre.fail_loc=0 + do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0 ls $DIR/$tfile - $CHECKSTAT -t file $DIR/$tfile || return 2 kill -USR1 $MULTIPID || return 3 wait $MULTIPID || return 4 + $CHECKSTAT -t file $DIR/$tfile || return 2 rm $DIR/$tfile return 0 @@ -328,16 +232,16 @@ test_13() { MULTIPID=$! sleep 5 - replay_barrier mds1 + replay_barrier $SINGLEMDS kill -USR1 $MULTIPID || return 3 wait $MULTIPID || return 4 # drop close - sysctl -w lustre.fail_loc=0x80000115 - facet_failover mds1 + do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0x80000115 + facet_failover $SINGLEMDS df $MOUNT || return 1 - sysctl -w lustre.fail_loc=0 + do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0 ls $DIR/$tfile $CHECKSTAT -t file $DIR/$tfile || return 2 @@ -347,208 +251,204 @@ test_13() { } run_test 13 "close resend timeout" - test_14() { - replay_barrier mds1 + replay_barrier $SINGLEMDS createmany -o $MOUNT1/$tfile- 25 createmany -o $MOUNT2/$tfile-2- 1 createmany -o $MOUNT1/$tfile-3- 25 umount $MOUNT2 - facet_failover mds1 + facet_failover $SINGLEMDS # expect failover to fail df $MOUNT && return 1 sleep 1 - # first 25 files shouuld have been - # replayed - sleep 2 + # first 25 files should have been replayed unlinkmany $MOUNT1/$tfile- 25 || return 2 - zconf_mount `hostname` $MOUNT2 + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" return 0 } run_test 14 "timeouts waiting for lost client during replay" test_15() { - replay_barrier mds1 + replay_barrier $SINGLEMDS createmany -o $MOUNT1/$tfile- 25 createmany -o $MOUNT2/$tfile-2- 1 umount $MOUNT2 - facet_failover mds1 + facet_failover $SINGLEMDS df $MOUNT || return 1 - sleep 1 unlinkmany $MOUNT1/$tfile- 25 || return 2 + [ -e $MOUNT1/$tfile-2-0 ] && error "$tfile-2-0 exists" - zconf_mount `hostname` $MOUNT2 + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" return 0 } run_test 15 "timeout waiting for lost client during replay, 1 client completes" + +test_15a() { + local ost_last_id="" + local osc_last_id="" + + replay_barrier $SINGLEMDS + echo "data" > "$MOUNT2/${tfile}-m2" + + umount $MOUNT2 + facet_failover $SINGLEMDS + df $MOUNT || return 1 + + ost_last_id=`cat /proc/fs/lustre/obdfilter/*/last_id` + mds_last_id=`cat /proc/fs/lustre/osc/*mds*/last_id` + + echo "Ids after MDS<->OST synchonizing" + echo "--------------------------------" + echo "MDS last_id:" + echo $mds_last_id + echo "OST last_id:" + echo $ost_last_id + + local i=0 + echo $ost_last_id | while read id; do + ost_ids[$i]=$id + ((i++)) + done + + i=0 + echo $mds_last_id | while read id; do + mds_ids[$i]=$id + ((i++)) + done + + local arr_len=${#mds_ids[*]} + for ((i=0;i<$arr_len;i++)); do + mds_id=${mds_ids[i]} + ost_id=${ost_ids[i]} + + test $mds_id -ge $ost_id || { + echo "MDS last id ($mds_id) is smaller than OST one ($ost_id)" + return 2 + } + done + + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" + return 0 +} +#CROW run_test 15a "OST clear orphans - synchronize ids on MDS and OST" + +test_15b() { + replay_barrier $SINGLEMDS + echo "data" > "$MOUNT2/${tfile}-m2" + umount $MOUNT2 + + do_facet ost1 "sysctl -w lustre.fail_loc=0x80000802" + facet_failover $SINGLEMDS + + df $MOUNT || return 1 + do_facet ost1 "sysctl -w lustre.fail_loc=0" + + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" + return 0 +} +#CROW run_test 15b "multiple delayed OST clear orphans" + +test_15c() { + replay_barrier $SINGLEMDS + for ((i = 0; i < 2000; i++)); do + echo "data" > "$MOUNT2/${tfile}-$i" || error "create ${tfile}-$i failed" + done + + umount $MOUNT2 + facet_failover $SINGLEMDS + + df $MOUNT || return 1 + + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" + return 0 +} +run_test 15c "remove multiple OST orphans" + test_16() { - replay_barrier mds1 + replay_barrier $SINGLEMDS createmany -o $MOUNT1/$tfile- 25 createmany -o $MOUNT2/$tfile-2- 1 umount $MOUNT2 - facet_failover mds1 + facet_failover $SINGLEMDS sleep $TIMEOUT - facet_failover mds1 + facet_failover $SINGLEMDS df $MOUNT || return 1 - sleep 1 unlinkmany $MOUNT1/$tfile- 25 || return 2 - zconf_mount `hostname` $MOUNT2 + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" return 0 } -#run_test 16 "fail MDS during recovery (3571)" +run_test 16 "fail MDS during recovery (3571)" test_17() { createmany -o $MOUNT1/$tfile- 25 createmany -o $MOUNT2/$tfile-2- 1 # Make sure the disconnect is lost - replay_barrier ost + replay_barrier ost1 umount $MOUNT2 - echo -1 > /proc/sys/portals/debug - facet_failover ost + facet_failover ost1 sleep $TIMEOUT - facet_failover ost + facet_failover ost1 df $MOUNT || return 1 - sleep 1 unlinkmany $MOUNT1/$tfile- 25 || return 2 - zconf_mount `hostname` $MOUNT2 + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" return 0 } -#Still not support ost fail over -#run_test 17 "fail OST during recovery (3571)" - -test_18 () { - replay_barrier mds1 - multiop $MOUNT2/$tfile O_c & - pid2=$! - multiop $MOUNT1/$tfile O_c & - pid1=$! - # give multiop a chance to open - sleep 1 - kill -USR1 $pid2 - kill -USR1 $pid1 - sleep 1 - umount $MOUNT2 - facet_failover mds1 - df || df || return 1 - zconf_mount `hostname` $MOUNT2 -} -run_test 18 "replay open, Abort recovery, don't assert (3892)" +run_test 17 "fail OST during recovery (3571)" -# cleanup with blocked enqueue fails until timer elapses (MDS busy), wait for -# itexport NOW=0 +# cleanup with blocked enqueue fails until timer elapses (MDS busy), wait for it +export NOW=0 -test_20() { # bug 3822 - evicting client with enqueued lock - mkdir -p $MOUNT1/$tdir - touch $MOUNT1/$tdir/f0 +test_18() { # bug 3822 - evicting client with enqueued lock + #set -vx + mkdir -p $MOUNT1/$tdir + touch $MOUNT1/$tdir/f0 #define OBD_FAIL_LDLM_ENQUEUE_BLOCKED 0x30b - statmany -s $MOUNT1/$tdir/f 500 & - OPENPID=$! - NOW=`date +%s` - do_facet mds1 sysctl -w lustre.fail_loc=0x8000030b # hold enqueue - sleep 1 + statmany -s $MOUNT1/$tdir/f 1 500 & + OPENPID=$! + NOW=`date +%s` + do_facet $SINGLEMDS sysctl -w lustre.fail_loc=0x8000030b # hold enqueue + sleep 1 #define OBD_FAIL_LDLM_BL_CALLBACK 0x305 - do_facet client sysctl -w lustre.fail_loc=0x80000305 # drop cb, evict - cancel_lru_locks MDC - usleep 500 # wait to ensure first client is one that will be evicted - openfile -f O_RDONLY $MOUNT2/$tdir/f0 - wait $OPENPID - dmesg | grep "entering recovery in server" && \ - error "client not evicted" || true -} -run_test 20 "ldlm_handle_enqueue succeeds on evicted export (3822)" - -# $1 - fs num (1, 2, ...) -# $2 - mds -function find_dev_for_fs_and_mds() -{ - local fs=`ls /proc/fs/lustre/llite|head -n $1|tail -n1` - local fsuuid=`cat /proc/fs/lustre/llite/$fs/uuid` - $LCTL device_list | awk "/mdc.*$2.*$fsuuid/ {print \$4}" + do_facet client sysctl -w lustre.fail_loc=0x80000305 # drop cb, evict + cancel_lru_locks mdc + usleep 500 # wait to ensure first client is one that will be evicted + openfile -f O_RDONLY $MOUNT2/$tdir/f0 + wait $OPENPID + dmesg | grep "entering recovery in server" && \ + error "client not evicted" || true } +run_test 18 "ldlm_handle_enqueue succeeds on evicted export (3822)" -test_21() { - mdc1dev=`find_dev_for_fs_and_mds 1 mds1` - mdc2dev=`find_dev_for_fs_and_mds 2 mds1` - multiop $MOUNT1/f21 O - cancel_lru_locks MDC - # generate IT_OPEN to be replayed against existing file - multiop $MOUNT1/f21 o_Sc & - pid=$! - - # IT_OPEN will be committed by the failover time - replay_barrier mds1 - - # generate MDS_REINT_UNLINK to be replayed - rm -f $MOUNT2/f21 || return 1 - - # disable recovery on the both clients - $LCTL --device %$mdc1dev disable_recovery - $LCTL --device %$mdc2dev disable_recovery - facet_failover mds1 - - # let unlink to be replayed first - $LCTL --device %$mdc2dev enable_recovery - sleep $((TIMEOUT/2)) - - # now let open to be replaye - $LCTL --device %$mdc1dev enable_recovery - kill -USR1 $pid - wait $pid || return 2 -} -run_test 21 "open vs. unlink out of order replay" - -test_22() { # bug 6063 - AST during recovery - cancel_lru_locks MDC - cat /proc/fs/lustre/ldlm/namespaces/mds-*/lock_count - mdc1dev=`find_dev_for_fs_and_mds 1 mds1` - mdc2dev=`find_dev_for_fs_and_mds 2 mds1` - $LCTL --device %$mdc1dev disable_recovery - $LCTL --device %$mdc2dev disable_recovery - - replay_barrier mds1 - mkdir $MOUNT1/${tdir}-1 # client1: request to be replayed - ls $MOUNT2 # client2: take lock needed for - facet_failover mds1 - - # let's recover 2nd connection with granted UPDATE lock - $LCTL --device %$mdc2dev enable_recovery - sleep $((TIMEOUT / 2)) - - LOCKS=`grep -v '^0$' /proc/fs/lustre/ldlm/namespaces/mds-*/lock_count` - if [ "$LOCKS" != "" ]; then - echo "The lock got replayed before mkdir is replayed: $LOCKS" - $LCTL --device %$mdc1dev enable_recovery - return 1 - fi - - # let's recover 1st connection with mkdir replay that needs the lock - $LCTL --device %$mdc1dev enable_recovery - sleep $TIMEOUT - - df $MOUNT || return 2 - return 0 +test_19() { # Bug 10991 - resend of open request does not fail assertion. + replay_barrier $SINGLEMDS + drop_ldlm_reply "createmany -o $DIR/$tfile 1" || return 1 + fail $SINGLEMDS + checkstat $DIR2/${tfile}0 || return 2 + rm $DIR/${tfile}0 || return 3 + + return 0 } -run_test 22 "AST during recovery" +run_test 19 "resend of open request" if [ "$ONLY" != "setup" ]; then - equals_msg test complete, cleaning up - if [ $NOW ]; then - SLEEP=$((`date +%s` - $NOW)) - [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP - fi - $CLEANUP + equals_msg `basename $0`: test complete, cleaning up + SLEEP=$((`date +%s` - $NOW)) + [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP + $CLEANUP fi +[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true +