X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Freplay-dual.sh;h=1d3f1c391e729787e4e62e259356eb2a152fe228;hp=4482716fa9bcae99e02b3e392cbe3a272b7883e4;hb=5d0201db65d366d8905a28d103a2a9b511c22ca7;hpb=3f12dbd8a4fa651a23ca3d9201f301e535234d71 diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh index 4482716..1d3f1c3 100755 --- a/lustre/tests/replay-dual.sh +++ b/lustre/tests/replay-dual.sh @@ -2,8 +2,8 @@ set -e -# bug number: 13129 13129 10124 -ALWAYS_EXCEPT="2 3 15c $REPLAY_DUAL_EXCEPT" +# bug number: 10124 +ALWAYS_EXCEPT="15c $REPLAY_DUAL_EXCEPT" SAVE_PWD=$PWD PTLDEBUG=${PTLDEBUG:--1} @@ -13,28 +13,67 @@ CLEANUP=${CLEANUP:-""} MOUNT_2=${MOUNT_2:-"yes"} . $LUSTRE/tests/test-framework.sh -if [ "$FAILURE_MODE" = "HARD" ] && mixed_ost_devs; then - CONFIG_EXCEPTIONS="17" - echo -n "Several ost services on one ost node are used with FAILURE_MODE=$FAILURE_MODE. " - echo "Except the tests: $CONFIG_EXCEPTIONS" - ALWAYS_EXCEPT="$ALWAYS_EXCEPT $CONFIG_EXCEPTIONS" -fi - init_test_env $@ - . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} +init_logging -# -[ "$SLOW" = "no" ] && EXCEPT_SLOW="1 2 3 4 5 14" +remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0 + +[ "$SLOW" = "no" ] && EXCEPT_SLOW="21b" build_test_filter -cleanup_and_setup_lustre +check_and_setup_lustre +MOUNTED=$(mounted_lustre_filesystems) +if ! $(echo $MOUNTED | grep -w -q $MOUNT2); then + zconf_mount $HOSTNAME $MOUNT2 + MOUNTED2=yes +fi + assert_DIR rm -rf $DIR/[df][0-9]* [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE +# LU-482 Avert LVM and VM inability to flush caches in pre .33 kernels +if [ $LINUX_VERSION_CODE -lt $(version_code 2.6.33) ]; then + sync + do_facet $SINGLEMDS sync +fi + +test_0a() { + touch $MOUNT2/$tfile-A # force sync FLD/SEQ update before barrier + replay_barrier $SINGLEMDS +#define OBD_FAIL_PTLRPC_FINISH_REPLAY | OBD_FAIL_ONCE + touch $MOUNT2/$tfile + createmany -o $MOUNT1/$tfile- 50 + $LCTL set_param fail_loc=0x80000514 + facet_failover $SINGLEMDS + client_up || return 1 + umount -f $MOUNT2 + client_up || return 1 + zconf_mount `hostname` $MOUNT2 || error "mount2 fais" + unlinkmany $MOUNT1/$tfile- 50 || return 2 + rm $MOUNT2/$tfile || return 3 + rm $MOUNT2/$tfile-A || return 4 +} +run_test 0a "expired recovery with lost client" + +test_0b() { + replay_barrier $SINGLEMDS + touch $MOUNT2/$tfile + touch $MOUNT1/$tfile-2 + umount $MOUNT2 + facet_failover $SINGLEMDS + umount -f $MOUNT1 + zconf_mount `hostname` $MOUNT1 || error "mount1 fais" + zconf_mount `hostname` $MOUNT2 || error "mount2 fais" + checkstat $MOUNT1/$tfile-2 && return 1 + checkstat $MOUNT2/$tfile && return 2 + return 0 +} +run_test 0b "lost client during waiting for next transno" + test_1() { touch $MOUNT1/a replay_barrier $SINGLEMDS @@ -208,7 +247,7 @@ test_12() { do_facet $SINGLEMDS lctl set_param fail_loc=0x80000302 facet_failover $SINGLEMDS do_facet $SINGLEMDS lctl set_param fail_loc=0 - df $MOUNT || return 1 + clients_up || return 1 ls $DIR/$tfile kill -USR1 $MULTIPID || return 3 @@ -229,11 +268,11 @@ test_13() { kill -USR1 $MULTIPID || return 3 wait $MULTIPID || return 4 - # drop close + # drop close do_facet $SINGLEMDS lctl set_param fail_loc=0x80000115 facet_failover $SINGLEMDS do_facet $SINGLEMDS lctl set_param fail_loc=0 - df $MOUNT || return 1 + clients_up || return 1 ls $DIR/$tfile $CHECKSTAT -t file $DIR/$tfile || return 2 @@ -243,34 +282,50 @@ test_13() { } run_test 13 "close resend timeout" -test_14() { +# test 14a removed after 18143 because it shouldn't fail anymore and do the same +# as test_15a + +test_14b() { + wait_mds_ost_sync + wait_destroy_complete + BEFOREUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'` + mkdir -p $MOUNT1/$tdir + $SETSTRIPE -i 0 $MOUNT1/$tdir replay_barrier $SINGLEMDS - createmany -o $MOUNT1/$tfile- 25 - createmany -o $MOUNT2/$tfile-2- 1 - createmany -o $MOUNT1/$tfile-3- 25 + createmany -o $MOUNT1/$tdir/$tfile- 5 + + $SETSTRIPE -i 0 $MOUNT2/f14b-3 + echo "data" > $MOUNT2/f14b-3 + createmany -o $MOUNT1/$tdir/$tfile-3- 5 umount $MOUNT2 - facet_failover $SINGLEMDS - # expect failover to fail due to missing client 2 - df $MOUNT && return 1 - sleep 1 + fail $SINGLEMDS + wait_recovery_complete $SINGLEMDS || error "MDS recovery not done" - # first 25 files should have been replayed - unlinkmany $MOUNT1/$tfile- 25 || return 2 + # first 25 files should have been replayed + unlinkmany $MOUNT1/$tdir/$tfile- 5 || return 2 + unlinkmany $MOUNT1/$tdir/$tfile-3- 5 || return 3 - zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" + + wait_mds_ost_sync || return 4 + wait_destroy_complete || return 5 + + AFTERUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'` + log "before $BEFOREUSED, after $AFTERUSED" + [ $AFTERUSED -ne $BEFOREUSED ] && \ + error "after $AFTERUSED > before $BEFOREUSED" && return 4 return 0 } -run_test 14 "timeouts waiting for lost client during replay" +run_test 14b "delete ost orphans if gap occured in objids due to VBR" -test_15a() { # was test_15 +test_15a() { # was test_15 replay_barrier $SINGLEMDS createmany -o $MOUNT1/$tfile- 25 createmany -o $MOUNT2/$tfile-2- 1 umount $MOUNT2 - facet_failover $SINGLEMDS - df $MOUNT || return 1 + fail $SINGLEMDS unlinkmany $MOUNT1/$tfile- 25 || return 2 [ -e $MOUNT1/$tfile-2-0 ] && error "$tfile-2-0 exists" @@ -283,14 +338,12 @@ run_test 15a "timeout waiting for lost client during replay, 1 client completes" test_15c() { replay_barrier $SINGLEMDS for ((i = 0; i < 2000; i++)); do - echo "data" > "$MOUNT2/${tfile}-$i" || error "create ${tfile}-$i failed" + echo "data" > "$MOUNT2/${tfile}-$i" || error "create ${tfile}-$i failed" done - umount $MOUNT2 - facet_failover $SINGLEMDS - df $MOUNT || return 1 - + fail $SINGLEMDS + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" return 0 } @@ -304,8 +357,7 @@ test_16() { facet_failover $SINGLEMDS sleep $TIMEOUT - facet_failover $SINGLEMDS - df $MOUNT || return 1 + fail $SINGLEMDS unlinkmany $MOUNT1/$tfile- 25 || return 2 @@ -316,6 +368,8 @@ test_16() { run_test 16 "fail MDS during recovery (3571)" test_17() { + remote_ost_nodsh && skip "remote OST with nodsh" && return 0 + createmany -o $MOUNT1/$tfile- 25 createmany -o $MOUNT2/$tfile-2- 1 @@ -325,8 +379,7 @@ test_17() { facet_failover ost1 sleep $TIMEOUT - facet_failover ost1 - df $MOUNT || return 1 + fail ost1 unlinkmany $MOUNT1/$tfile- 25 || return 2 @@ -373,9 +426,135 @@ test_19() { # Bug 10991 - resend of open request does not fail assertion. } run_test 19 "resend of open request" -equals_msg `basename $0`: test complete, cleaning up +test_20() { #16389 + BEFORE=`date +%s` + replay_barrier $SINGLEMDS + touch $MOUNT1/a + touch $MOUNT2/b + umount $MOUNT2 + fail $SINGLEMDS + rm $MOUNT1/a + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" + TIER1=$((`date +%s` - BEFORE)) + BEFORE=`date +%s` + replay_barrier $SINGLEMDS + touch $MOUNT1/a + touch $MOUNT2/b + umount $MOUNT2 + fail $SINGLEMDS + rm $MOUNT1/a + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" + TIER2=$((`date +%s` - BEFORE)) + [ $TIER2 -ge $((TIER1 * 2)) ] && \ + error "recovery time is growing $TIER2 > $TIER1" + return 0 +} +run_test 20 "recovery time is not increasing" + +# commit on sharing tests +test_21a() { + local param_file=$TMP/$tfile-params + + save_lustre_params $(facet_active_host $SINGLEMDS) "mdt.*.commit_on_sharing" > $param_file + do_facet $SINGLEMDS lctl set_param mdt.*.commit_on_sharing=1 + touch $MOUNT1/$tfile-1 + mv $MOUNT2/$tfile-1 $MOUNT2/$tfile-2 + mv $MOUNT1/$tfile-2 $MOUNT1/$tfile-3 + replay_barrier_nosync $SINGLEMDS + umount $MOUNT2 + + facet_failover $SINGLEMDS + + # all renames are replayed + unlink $MOUNT1/$tfile-3 || return 2 + + zconf_mount `hostname` $MOUNT2 || error "mount $MOUNT2 fail" + + do_facet $SINGLEMDS lctl set_param mdt.*.commit_on_sharing=0 + rm -rf $MOUNT1/$tfile-* + restore_lustre_params < $param_file + rm -f $param_file + return 0 +} +run_test 21a "commit on sharing" + +test_21b_sub () { + local mds=$1 + do_node $CLIENT1 rm -f $MOUNT1/$tfile-* + + do_facet $mds sync + do_node $CLIENT1 touch $MOUNT1/$tfile-1 + do_node $CLIENT2 mv $MOUNT1/$tfile-1 $MOUNT1/$tfile-2 + do_node $CLIENT1 mv $MOUNT1/$tfile-2 $MOUNT1/$tfile-3 + + replay_barrier_nosync $mds + shutdown_client $CLIENT2 $MOUNT1 + + facet_failover $mds + + # were renames replayed? + local rc=0 + echo UNLINK $MOUNT1/$tfile-3 + do_node $CLIENT1 unlink $MOUNT1/$tfile-3 || \ + { echo "unlink $tfile-3 fail!" && rc=1; } + + boot_node $CLIENT2 + zconf_mount_clients $CLIENT2 $MOUNT1 || error "mount $CLIENT2 $MOUNT1 fail" + + return $rc +} + +test_21b() { + [ -z "$CLIENTS" ] && skip "Need two or more clients." && return + [ $CLIENTCOUNT -lt 2 ] && \ + { skip "Need two or more clients, have $CLIENTCOUNT" && return; } + + if [ "$FAILURE_MODE" = "HARD" ] && mixed_mdt_devs; then + skip "Several mdt services on one mds node are used with FAILURE_MODE=$FAILURE_MODE. " + return 0 + fi + + + zconf_umount_clients $CLIENTS $MOUNT2 + zconf_mount_clients $CLIENTS $MOUNT1 + + local param_file=$TMP/$tfile-params + + local num=$(get_mds_dir $MOUNT1) + + save_lustre_params $(facet_active_host mds$num) "mdt.*.commit_on_sharing" > $param_file + + # COS enabled + local COS=1 + do_facet mds$num lctl set_param mdt.*.commit_on_sharing=$COS + + test_21b_sub mds$num || error "Not all renames are replayed. COS=$COS" + + # COS disabled (should fail) + COS=0 + do_facet mds$num lctl set_param mdt.*.commit_on_sharing=$COS + + # there is still a window when transactions may be written to disk before + # the mds device is set R/O. To avoid such a rare test failure, the check + # is repeated several times. + local n_attempts=1 + while true; do + test_21b_sub mds$num || break; + let n_attempts=n_attempts+1 + [ $n_attempts -gt 3 ] && + error "The test cannot check whether COS works or not: all renames are replied w/o COS" + done + restore_lustre_params < $param_file + rm -f $param_file + return 0 +} +run_test 21b "commit on sharing, two clients" + +# end commit on sharing tests + +complete $(basename $0) $SECONDS SLEEP=$((`date +%s` - $NOW)) [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP +[ "$MOUNTED2" = yes ] && zconf_umount $HOSTNAME $MOUNT2 || true check_and_cleanup_lustre -[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true - +exit_status