X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Freplay-dual.sh;h=820a97b584e1929d31ff00d70b8c9b842c7ae05c;hb=a64f43dfeb7b30b5f94cee813f57845c76feeeac;hp=8e1063198a52e2bc666c98a3ee49c348c8a3df94;hpb=ff17cc7e0282d9b1522810e0c5d12171c4d46a2d;p=fs%2Flustre-release.git diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh index 8e10631..820a97b 100755 --- a/lustre/tests/replay-dual.sh +++ b/lustre/tests/replay-dual.sh @@ -1,73 +1,41 @@ -#!/bin/sh +#!/bin/bash set -e +# bug number: 6088 10124 10800 +ALWAYS_EXCEPT="8 15c 17 $REPLAY_DUAL_EXCEPT" + +PTLDEBUG=${PTLDEBUG:--1} LUSTRE=${LUSTRE:-`dirname $0`/..} . $LUSTRE/tests/test-framework.sh init_test_env $@ -. ${CONFIG:=$LUSTRE/tests/cfg/local.sh} - -gen_config() { - rm -f $XMLCONFIG - add_mds mds --dev $MDSDEV --size $MDSSIZE - if [ ! -z "$mdsfailover_HOST" ]; then - add_mdsfailover mds --dev $MDSDEV --size $MDSSIZE - fi - - add_lov lov1 mds --stripe_sz $STRIPE_BYTES\ - --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 - add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE - add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE - add_client client mds --lov lov1 --path $MOUNT -} - +. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} +SETUP=${SETUP:-"setup"} +CLEANUP=${CLEANUP:-"cleanup"} build_test_filter cleanup() { - # make sure we are using the primary MDS, so the config log will - # be able to clean up properly. - activemds=`facet_active mds` - if [ $activemds != "mds" ]; then - fail mds - fi - - umount $MOUNT2 - umount $MOUNT - rmmod llite - stop mds ${FORCE} - stop ost2 ${FORCE} - stop ost ${FORCE} --dump cleanup-dual.log + stopall } if [ "$ONLY" == "cleanup" ]; then - sysctl -w portals.debug=0 + sysctl -w lnet.debug=0 cleanup exit fi -gen_config -start ost --reformat $OSTLCONFARGS -PINGER=`cat /proc/fs/lustre/pinger` - -if [ "$PINGER" != "on" ]; then - echo "ERROR: Lustre must be built with --enable-pinger for replay-dual" - stop mds - exit 1 -fi - -start ost2 --reformat $OSTLCONFARGS -[ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE -start mds $MDSLCONFARGS --reformat -zconf_mount `hostname` $MOUNT -zconf_mount `hostname` $MOUNT2 - -echo $TIMEOUT > /proc/sys/lustre/timeout -echo $UPCALL > /proc/sys/lustre/upcall +setup() { + cleanup + formatall + setupall + mount_client $MOUNT2 +} +$SETUP [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE test_1() { @@ -97,7 +65,6 @@ test_2() { checkstat $MOUNT2/adir && return 2 return 0 } - run_test 2 "|X| mkdir adir" test_3() { @@ -113,7 +80,6 @@ test_3() { checkstat $MOUNT2/adir/bdir && return 4 return 0 } - run_test 3 "|X| mkdir adir, mkdir adir/bdir " test_4() { @@ -131,7 +97,6 @@ test_4() { checkstat $MOUNT2/adir/bdir && return 5 return 0 } - run_test 4 "|X| mkdir adir (-EEXIST), mkdir adir/bdir " @@ -175,7 +140,318 @@ test_6() { } run_test 6 "open1, open2, unlink |X| close1 [fail mds] close2" +test_8() { + replay_barrier mds + drop_reint_reply "mcreate $MOUNT1/$tfile" || return 1 + fail mds + checkstat $MOUNT2/$tfile || return 2 + rm $MOUNT1/$tfile || return 3 + + return 0 +} +run_test 8 "replay of resent request" + +test_9() { + replay_barrier mds + mcreate $MOUNT1/$tfile-1 + mcreate $MOUNT2/$tfile-2 + # drop first reint reply + sysctl -w lustre.fail_loc=0x80000119 + fail mds + sysctl -w lustre.fail_loc=0 + + rm $MOUNT1/$tfile-[1,2] || return 1 + + return 0 +} +run_test 9 "resending a replayed create" + +test_10() { + mcreate $MOUNT1/$tfile-1 + replay_barrier mds + munlink $MOUNT1/$tfile-1 + mcreate $MOUNT2/$tfile-2 + # drop first reint reply + sysctl -w lustre.fail_loc=0x80000119 + fail mds + sysctl -w lustre.fail_loc=0 + + checkstat $MOUNT1/$tfile-1 && return 1 + checkstat $MOUNT1/$tfile-2 || return 2 + rm $MOUNT1/$tfile-2 + + return 0 +} +run_test 10 "resending a replayed unlink" + +test_11() { + replay_barrier mds + mcreate $MOUNT1/$tfile-1 + mcreate $MOUNT2/$tfile-2 + mcreate $MOUNT1/$tfile-3 + mcreate $MOUNT2/$tfile-4 + mcreate $MOUNT1/$tfile-5 + # drop all reint replies for a while + sysctl -w lustre.fail_loc=0x0119 + facet_failover mds + #sleep for while, let both clients reconnect and timeout + sleep $((TIMEOUT * 2)) + sysctl -w lustre.fail_loc=0 + while [ -z "$(ls $MOUNT1/$tfile-[1-5] 2>/dev/null)" ]; do + sleep 5 + echo -n "." + done + ls $MOUNT1/$tfile-[1-5] + rm $MOUNT1/$tfile-[1-5] || return 1 + + return 0 +} +run_test 11 "both clients timeout during replay" + +test_12() { + replay_barrier mds + + multiop $DIR/$tfile mo_c & + MULTIPID=$! + sleep 5 + +#define OBD_FAIL_LDLM_ENQUEUE 0x302 + sysctl -w lustre.fail_loc=0x80000302 + facet_failover mds + df $MOUNT || return 1 + sysctl -w lustre.fail_loc=0 + + ls $DIR/$tfile + kill -USR1 $MULTIPID || return 3 + wait $MULTIPID || return 4 + $CHECKSTAT -t file $DIR/$tfile || return 2 + rm $DIR/$tfile + + return 0 +} +run_test 12 "open resend timeout" + +test_13() { + multiop $DIR/$tfile mo_c & + MULTIPID=$! + sleep 5 + + replay_barrier mds + + kill -USR1 $MULTIPID || return 3 + wait $MULTIPID || return 4 + + # drop close + sysctl -w lustre.fail_loc=0x80000115 + facet_failover mds + df $MOUNT || return 1 + sysctl -w lustre.fail_loc=0 + + ls $DIR/$tfile + $CHECKSTAT -t file $DIR/$tfile || return 2 + rm $DIR/$tfile + + return 0 +} +run_test 13 "close resend timeout" + +test_14() { + replay_barrier mds + createmany -o $MOUNT1/$tfile- 25 + createmany -o $MOUNT2/$tfile-2- 1 + createmany -o $MOUNT1/$tfile-3- 25 + umount $MOUNT2 + + facet_failover mds + # expect failover to fail + df $MOUNT && return 1 + sleep 1 + + # first 25 files should have been replayed + unlinkmany $MOUNT1/$tfile- 25 || return 2 + + zconf_mount `hostname` $MOUNT2 + return 0 +} +run_test 14 "timeouts waiting for lost client during replay" + +test_15() { + replay_barrier mds + createmany -o $MOUNT1/$tfile- 25 + createmany -o $MOUNT2/$tfile-2- 1 + umount $MOUNT2 + + facet_failover mds + df $MOUNT || return 1 + + unlinkmany $MOUNT1/$tfile- 25 || return 2 + [ -e $MOUNT1/$tfile-2-0 ] && error "$tfile-2-0 exists" + + zconf_mount `hostname` $MOUNT2 + return 0 +} +run_test 15 "timeout waiting for lost client during replay, 1 client completes" + +test_15a() { + local ost_last_id="" + local osc_last_id="" + + replay_barrier mds + echo "data" > "$MOUNT2/${tfile}-m2" + + umount $MOUNT2 + facet_failover mds + df $MOUNT || return 1 + + ost_last_id=`cat /proc/fs/lustre/obdfilter/*/last_id` + mds_last_id=`cat /proc/fs/lustre/osc/*mds*/last_id` + + echo "Ids after MDS<->OST synchonizing" + echo "--------------------------------" + echo "MDS last_id:" + echo $mds_last_id + echo "OST last_id:" + echo $ost_last_id + + local i=0 + echo $ost_last_id | while read id; do + ost_ids[$i]=$id + ((i++)) + done + + i=0 + echo $mds_last_id | while read id; do + mds_ids[$i]=$id + ((i++)) + done + + local arr_len=${#mds_ids[*]} + for ((i=0;i<$arr_len;i++)); do + mds_id=${mds_ids[i]} + ost_id=${ost_ids[i]} + + test $mds_id -ge $ost_id || { + echo "MDS last id ($mds_id) is smaller than OST one ($ost_id)" + return 2 + } + done + + zconf_mount `hostname` $MOUNT2 + return 0 +} +#CROW run_test 15a "OST clear orphans - synchronize ids on MDS and OST" + +test_15b() { + replay_barrier mds + echo "data" > "$MOUNT2/${tfile}-m2" + umount $MOUNT2 + + do_facet ost1 "sysctl -w lustre.fail_loc=0x80000802" + facet_failover mds + + df $MOUNT || return 1 + do_facet ost1 "sysctl -w lustre.fail_loc=0" + + zconf_mount `hostname` $MOUNT2 + return 0 +} +#CROW run_test 15b "multiple delayed OST clear orphans" + +test_15c() { + replay_barrier mds + for ((i = 0; i < 2000; i++)); do + echo "data" > "$MOUNT2/${tfile}-$i" || error "create ${tfile}-$i failed" + done + + umount $MOUNT2 + facet_failover mds + + df $MOUNT || return 1 + + zconf_mount `hostname` $MOUNT2 + return 0 +} +run_test 15c "remove multiple OST orphans" + +test_16() { + replay_barrier mds + createmany -o $MOUNT1/$tfile- 25 + createmany -o $MOUNT2/$tfile-2- 1 + umount $MOUNT2 + + facet_failover mds + sleep $TIMEOUT + facet_failover mds + df $MOUNT || return 1 + + unlinkmany $MOUNT1/$tfile- 25 || return 2 + + zconf_mount `hostname` $MOUNT2 + return 0 + +} +run_test 16 "fail MDS during recovery (3571)" + +test_17() { + createmany -o $MOUNT1/$tfile- 25 + createmany -o $MOUNT2/$tfile-2- 1 + + # Make sure the disconnect is lost + replay_barrier ost1 + umount $MOUNT2 + + facet_failover ost1 + sleep $TIMEOUT + facet_failover ost1 + df $MOUNT || return 1 + + unlinkmany $MOUNT1/$tfile- 25 || return 2 + + zconf_mount `hostname` $MOUNT2 + return 0 + +} +run_test 17 "fail OST during recovery (3571)" + +# cleanup with blocked enqueue fails until timer elapses (MDS busy), wait for it +export NOW=0 + +test_18() { # bug 3822 - evicting client with enqueued lock + #set -vx + mkdir -p $MOUNT1/$tdir + touch $MOUNT1/$tdir/f0 +#define OBD_FAIL_LDLM_ENQUEUE_BLOCKED 0x30b + statmany -s $MOUNT1/$tdir/f 1 500 & + OPENPID=$! + NOW=`date +%s` + do_facet mds sysctl -w lustre.fail_loc=0x8000030b # hold enqueue + sleep 1 +#define OBD_FAIL_LDLM_BL_CALLBACK 0x305 + do_facet client sysctl -w lustre.fail_loc=0x80000305 # drop cb, evict + cancel_lru_locks mdc + usleep 500 # wait to ensure first client is one that will be evicted + openfile -f O_RDONLY $MOUNT2/$tdir/f0 + wait $OPENPID + dmesg | grep "entering recovery in server" && \ + error "client not evicted" || true +} +run_test 18 "ldlm_handle_enqueue succeeds on evicted export (3822)" + +test_19() { # Bug 10991 - resend of open request does not fail assertion. + replay_barrier mds + drop_ldlm_reply "createmany -o $DIR/$tfile 1" || return 1 + fail mds + checkstat $DIR2/${tfile}0 || return 2 + rm $DIR/${tfile}0 || return 3 + + return 0 +} +run_test 19 "resend of open request" + if [ "$ONLY" != "setup" ]; then - equals_msg test complete, cleaning up - cleanup + equals_msg test complete, cleaning up + SLEEP=$((`date +%s` - $NOW)) + [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP + $CLEANUP fi +echo "$0: completed"