X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Freplay-dual.sh;h=49cad57938bab04db4f6e48f4c313825237d90e1;hb=4570931c81d68a6db23aeef769b565686f62707f;hp=8c60328cb9d4605813885256d0287bf0df528310;hpb=113303973ec9f8484eb2355a1a6ef3c4c7fd6a56;p=fs%2Flustre-release.git diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh index 8c60328..49cad57 100755 --- a/lustre/tests/replay-dual.sh +++ b/lustre/tests/replay-dual.sh @@ -2,48 +2,38 @@ set -e -# bug number: 6088 10124 10800 -ALWAYS_EXCEPT="8 15c 17 $REPLAY_DUAL_EXCEPT" +# bug number: 13129 13129 10124 +ALWAYS_EXCEPT="2 3 15c $REPLAY_DUAL_EXCEPT" +SAVE_PWD=$PWD PTLDEBUG=${PTLDEBUG:--1} -LUSTRE=${LUSTRE:-`dirname $0`/..} +LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} +SETUP=${SETUP:-""} +CLEANUP=${CLEANUP:-""} +MOUNT_2=${MOUNT_2:-"yes"} . $LUSTRE/tests/test-framework.sh init_test_env $@ . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} -SETUP=${SETUP:-"setup"} -CLEANUP=${CLEANUP:-"cleanup"} +# +[ "$SLOW" = "no" ] && EXCEPT_SLOW="1 2 3 4 5 14" build_test_filter -cleanup() { - stopall -} - -if [ "$ONLY" == "cleanup" ]; then - sysctl -w lnet.debug=0 - cleanup - exit -fi - -setup() { - cleanup - formatall - setupall - mount_client $MOUNT2 -} +cleanup_and_setup_lustre +assert_DIR +rm -rf $DIR/[df][0-9]* -$SETUP [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE test_1() { touch $MOUNT1/a - replay_barrier mds + replay_barrier $SINGLEMDS touch $MOUNT2/b - fail mds + fail $SINGLEMDS checkstat $MOUNT2/a || return 1 checkstat $MOUNT1/b || return 2 rm $MOUNT2/a $MOUNT1/b @@ -56,24 +46,23 @@ run_test 1 "|X| simple create" test_2() { - replay_barrier mds + replay_barrier $SINGLEMDS mkdir $MOUNT1/adir - fail mds + fail $SINGLEMDS checkstat $MOUNT2/adir || return 1 rmdir $MOUNT2/adir checkstat $MOUNT2/adir && return 2 return 0 } - run_test 2 "|X| mkdir adir" test_3() { - replay_barrier mds + replay_barrier $SINGLEMDS mkdir $MOUNT1/adir mkdir $MOUNT2/adir/bdir - fail mds + fail $SINGLEMDS checkstat $MOUNT2/adir || return 1 checkstat $MOUNT1/adir/bdir || return 2 rmdir $MOUNT2/adir/bdir $MOUNT1/adir @@ -81,16 +70,15 @@ test_3() { checkstat $MOUNT2/adir/bdir && return 4 return 0 } - run_test 3 "|X| mkdir adir, mkdir adir/bdir " test_4() { mkdir $MOUNT1/adir - replay_barrier mds + replay_barrier $SINGLEMDS mkdir $MOUNT1/adir && return 1 mkdir $MOUNT2/adir/bdir - fail mds + fail $SINGLEMDS checkstat $MOUNT2/adir || return 2 checkstat $MOUNT1/adir/bdir || return 3 @@ -99,23 +87,20 @@ test_4() { checkstat $MOUNT2/adir/bdir && return 5 return 0 } - run_test 4 "|X| mkdir adir (-EEXIST), mkdir adir/bdir " test_5() { # multiclient version of replay_single.sh/test_8 mcreate $MOUNT1/a - multiop $MOUNT2/a o_tSc & + multiop_bg_pause $MOUNT2/a o_tSc || return 1 pid=$! - # give multiop a chance to open - sleep 1 rm -f $MOUNT1/a - replay_barrier mds + replay_barrier $SINGLEMDS kill -USR1 $pid wait $pid || return 1 - fail mds + fail $SINGLEMDS [ -e $MOUNT2/a ] && return 2 return 0 } @@ -124,29 +109,27 @@ run_test 5 "open, unlink |X| close" test_6() { mcreate $MOUNT1/a - multiop $MOUNT2/a o_c & + multiop_bg_pause $MOUNT2/a o_c || return 1 pid1=$! - multiop $MOUNT1/a o_c & + multiop_bg_pause $MOUNT1/a o_c || return 1 pid2=$! - # give multiop a chance to open - sleep 1 rm -f $MOUNT1/a - replay_barrier mds + replay_barrier $SINGLEMDS kill -USR1 $pid1 wait $pid1 || return 1 - fail mds + fail $SINGLEMDS kill -USR1 $pid2 wait $pid2 || return 1 [ -e $MOUNT2/a ] && return 2 return 0 } -run_test 6 "open1, open2, unlink |X| close1 [fail mds] close2" +run_test 6 "open1, open2, unlink |X| close1 [fail $SINGLEMDS] close2" test_8() { - replay_barrier mds + replay_barrier $SINGLEMDS drop_reint_reply "mcreate $MOUNT1/$tfile" || return 1 - fail mds + fail $SINGLEMDS checkstat $MOUNT2/$tfile || return 2 rm $MOUNT1/$tfile || return 3 @@ -155,13 +138,13 @@ test_8() { run_test 8 "replay of resent request" test_9() { - replay_barrier mds + replay_barrier $SINGLEMDS mcreate $MOUNT1/$tfile-1 mcreate $MOUNT2/$tfile-2 # drop first reint reply - sysctl -w lustre.fail_loc=0x80000119 - fail mds - sysctl -w lustre.fail_loc=0 + do_facet $SINGLEMDS lctl set_param fail_loc=0x80000119 + fail $SINGLEMDS + do_facet $SINGLEMDS lctl set_param fail_loc=0 rm $MOUNT1/$tfile-[1,2] || return 1 @@ -171,13 +154,13 @@ run_test 9 "resending a replayed create" test_10() { mcreate $MOUNT1/$tfile-1 - replay_barrier mds + replay_barrier $SINGLEMDS munlink $MOUNT1/$tfile-1 mcreate $MOUNT2/$tfile-2 # drop first reint reply - sysctl -w lustre.fail_loc=0x80000119 - fail mds - sysctl -w lustre.fail_loc=0 + do_facet $SINGLEMDS lctl set_param fail_loc=0x80000119 + fail $SINGLEMDS + do_facet $SINGLEMDS lctl set_param fail_loc=0 checkstat $MOUNT1/$tfile-1 && return 1 checkstat $MOUNT1/$tfile-2 || return 2 @@ -188,18 +171,19 @@ test_10() { run_test 10 "resending a replayed unlink" test_11() { - replay_barrier mds + replay_barrier $SINGLEMDS mcreate $MOUNT1/$tfile-1 mcreate $MOUNT2/$tfile-2 mcreate $MOUNT1/$tfile-3 mcreate $MOUNT2/$tfile-4 mcreate $MOUNT1/$tfile-5 # drop all reint replies for a while - sysctl -w lustre.fail_loc=0x0119 - facet_failover mds + do_facet $SINGLEMDS lctl set_param fail_loc=0x0119 + # note that with this fail_loc set, facet_failover df will fail + facet_failover $SINGLEMDS #sleep for while, let both clients reconnect and timeout sleep $((TIMEOUT * 2)) - sysctl -w lustre.fail_loc=0 + do_facet $SINGLEMDS lctl set_param fail_loc=0 rm $MOUNT1/$tfile-[1-5] || return 1 @@ -208,17 +192,16 @@ test_11() { run_test 11 "both clients timeout during replay" test_12() { - replay_barrier mds + replay_barrier $SINGLEMDS - multiop $DIR/$tfile mo_c & + multiop_bg_pause $DIR/$tfile mo_c || return 1 MULTIPID=$! - sleep 5 #define OBD_FAIL_LDLM_ENQUEUE 0x302 - sysctl -w lustre.fail_loc=0x80000302 - facet_failover mds + do_facet $SINGLEMDS lctl set_param fail_loc=0x80000302 + facet_failover $SINGLEMDS + do_facet $SINGLEMDS lctl set_param fail_loc=0 df $MOUNT || return 1 - sysctl -w lustre.fail_loc=0 ls $DIR/$tfile kill -USR1 $MULTIPID || return 3 @@ -231,20 +214,19 @@ test_12() { run_test 12 "open resend timeout" test_13() { - multiop $DIR/$tfile mo_c & + multiop_bg_pause $DIR/$tfile mo_c || return 1 MULTIPID=$! - sleep 5 - replay_barrier mds + replay_barrier $SINGLEMDS kill -USR1 $MULTIPID || return 3 wait $MULTIPID || return 4 # drop close - sysctl -w lustre.fail_loc=0x80000115 - facet_failover mds + do_facet $SINGLEMDS lctl set_param fail_loc=0x80000115 + facet_failover $SINGLEMDS + do_facet $SINGLEMDS lctl set_param fail_loc=0 df $MOUNT || return 1 - sysctl -w lustre.fail_loc=0 ls $DIR/$tfile $CHECKSTAT -t file $DIR/$tfile || return 2 @@ -255,137 +237,72 @@ test_13() { run_test 13 "close resend timeout" test_14() { - replay_barrier mds + replay_barrier $SINGLEMDS createmany -o $MOUNT1/$tfile- 25 createmany -o $MOUNT2/$tfile-2- 1 createmany -o $MOUNT1/$tfile-3- 25 umount $MOUNT2 - facet_failover mds - # expect failover to fail + facet_failover $SINGLEMDS + # expect failover to fail due to missing client 2 df $MOUNT && return 1 sleep 1 # first 25 files should have been replayed unlinkmany $MOUNT1/$tfile- 25 || return 2 - zconf_mount `hostname` $MOUNT2 + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" return 0 } run_test 14 "timeouts waiting for lost client during replay" -test_15() { - replay_barrier mds +test_15a() { # was test_15 + replay_barrier $SINGLEMDS createmany -o $MOUNT1/$tfile- 25 createmany -o $MOUNT2/$tfile-2- 1 umount $MOUNT2 - facet_failover mds + facet_failover $SINGLEMDS df $MOUNT || return 1 unlinkmany $MOUNT1/$tfile- 25 || return 2 [ -e $MOUNT1/$tfile-2-0 ] && error "$tfile-2-0 exists" - zconf_mount `hostname` $MOUNT2 - return 0 -} -run_test 15 "timeout waiting for lost client during replay, 1 client completes" - -test_15a() { - local ost_last_id="" - local osc_last_id="" - - replay_barrier mds - echo "data" > "$MOUNT2/${tfile}-m2" - - umount $MOUNT2 - facet_failover mds - df $MOUNT || return 1 - - ost_last_id=`cat /proc/fs/lustre/obdfilter/*/last_id` - mds_last_id=`cat /proc/fs/lustre/osc/*mds*/last_id` - - echo "Ids after MDS<->OST synchonizing" - echo "--------------------------------" - echo "MDS last_id:" - echo $mds_last_id - echo "OST last_id:" - echo $ost_last_id - - local i=0 - echo $ost_last_id | while read id; do - ost_ids[$i]=$id - ((i++)) - done - - i=0 - echo $mds_last_id | while read id; do - mds_ids[$i]=$id - ((i++)) - done - - local arr_len=${#mds_ids[*]} - for ((i=0;i<$arr_len;i++)); do - mds_id=${mds_ids[i]} - ost_id=${ost_ids[i]} - - test $mds_id -ge $ost_id || { - echo "MDS last id ($mds_id) is smaller than OST one ($ost_id)" - return 2 - } - done - - zconf_mount `hostname` $MOUNT2 + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" return 0 } -#CROW run_test 15a "OST clear orphans - synchronize ids on MDS and OST" - -test_15b() { - replay_barrier mds - echo "data" > "$MOUNT2/${tfile}-m2" - umount $MOUNT2 - - do_facet ost1 "sysctl -w lustre.fail_loc=0x80000802" - facet_failover mds - - df $MOUNT || return 1 - do_facet ost1 "sysctl -w lustre.fail_loc=0" - - zconf_mount `hostname` $MOUNT2 - return 0 -} -#CROW run_test 15b "multiple delayed OST clear orphans" +run_test 15a "timeout waiting for lost client during replay, 1 client completes" test_15c() { - replay_barrier mds + replay_barrier $SINGLEMDS for ((i = 0; i < 2000; i++)); do echo "data" > "$MOUNT2/${tfile}-$i" || error "create ${tfile}-$i failed" done umount $MOUNT2 - facet_failover mds + facet_failover $SINGLEMDS df $MOUNT || return 1 - zconf_mount `hostname` $MOUNT2 + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" return 0 } run_test 15c "remove multiple OST orphans" test_16() { - replay_barrier mds + replay_barrier $SINGLEMDS createmany -o $MOUNT1/$tfile- 25 createmany -o $MOUNT2/$tfile-2- 1 umount $MOUNT2 - facet_failover mds + facet_failover $SINGLEMDS sleep $TIMEOUT - facet_failover mds + facet_failover $SINGLEMDS df $MOUNT || return 1 unlinkmany $MOUNT1/$tfile- 25 || return 2 - zconf_mount `hostname` $MOUNT2 + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" return 0 } @@ -406,7 +323,7 @@ test_17() { unlinkmany $MOUNT1/$tfile- 25 || return 2 - zconf_mount `hostname` $MOUNT2 + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" return 0 } @@ -423,23 +340,25 @@ test_18() { # bug 3822 - evicting client with enqueued lock statmany -s $MOUNT1/$tdir/f 1 500 & OPENPID=$! NOW=`date +%s` - do_facet mds sysctl -w lustre.fail_loc=0x8000030b # hold enqueue + do_facet $SINGLEMDS lctl set_param fail_loc=0x8000030b # hold enqueue sleep 1 #define OBD_FAIL_LDLM_BL_CALLBACK 0x305 - do_facet client sysctl -w lustre.fail_loc=0x80000305 # drop cb, evict + do_facet client lctl set_param fail_loc=0x80000305 # drop cb, evict cancel_lru_locks mdc usleep 500 # wait to ensure first client is one that will be evicted openfile -f O_RDONLY $MOUNT2/$tdir/f0 wait $OPENPID dmesg | grep "entering recovery in server" && \ error "client not evicted" || true + do_facet client "lctl set_param fail_loc=0" + do_facet $SINGLEMDS "lctl set_param fail_loc=0" } run_test 18 "ldlm_handle_enqueue succeeds on evicted export (3822)" test_19() { # Bug 10991 - resend of open request does not fail assertion. - replay_barrier mds + replay_barrier $SINGLEMDS drop_ldlm_reply "createmany -o $DIR/$tfile 1" || return 1 - fail mds + fail $SINGLEMDS checkstat $DIR2/${tfile}0 || return 2 rm $DIR/${tfile}0 || return 3 @@ -447,10 +366,9 @@ test_19() { # Bug 10991 - resend of open request does not fail assertion. } run_test 19 "resend of open request" -if [ "$ONLY" != "setup" ]; then - equals_msg test complete, cleaning up - SLEEP=$((`date +%s` - $NOW)) - [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP - $CLEANUP -fi -echo "$0: completed" +equals_msg `basename $0`: test complete, cleaning up +SLEEP=$((`date +%s` - $NOW)) +[ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP +check_and_cleanup_lustre +[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true +