LU-3626 tests: More time to allow mds-ost sync being finished

Because mds-ost unlink log synchronization is not enforced by any
timeout, so we will add more time to allow mds-ost sync being
finished in replay-single test_20b. And also add more information
in the tests to help us understand which MDT slow down the
synchronziation.

Signed-off-by: wang di <di.wang@intel.com>
Change-Id: I5f63540e602ad29cd221de6dac0c22546629a9cd
Reviewed-on: http://review.whamcloud.com/7676
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Jian Yu <jian.yu@intel.com>

diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index 5c695d8..3086976 100755 (executable)
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -424,27 +424,30 @@ test_20a() {      # was test_20
 run_test 20a "|X| open(O_CREAT), unlink, replay, close (test mds_cleanup_orphans)"
 
 test_20b() { # bug 10480
-    BEFOREUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'`
+       local wait_timeout=$((TIMEOUT * 4))
+       local BEFOREUSED
+       local AFTERUSED
 
-    dd if=/dev/zero of=$DIR/$tfile bs=4k count=10000 &
-    pid=$!
-    while [ ! -e $DIR/$tfile ] ; do
-        usleep 60                           # give dd a chance to start
-    done
-
-    $GETSTRIPE $DIR/$tfile || return 1
-    rm -f $DIR/$tfile || return 2       # make it an orphan
-    mds_evict_client
-    client_up || client_up || true    # reconnect
+       BEFOREUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'`
+       dd if=/dev/zero of=$DIR/$tfile bs=4k count=10000 &
+       pid=$!
+       while [ ! -e $DIR/$tfile ] ; do
+       usleep 60                           # give dd a chance to start
+       done
 
-    fail $SINGLEMDS                            # start orphan recovery
-    wait_recovery_complete $SINGLEMDS || error "MDS recovery not done"
-    wait_mds_ost_sync || return 3
-    AFTERUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'`
-    log "before $BEFOREUSED, after $AFTERUSED"
-    (( $AFTERUSED > $BEFOREUSED + $(fs_log_size) )) &&
-        error "after $AFTERUSED > before $BEFOREUSED"
-    return 0
+       $GETSTRIPE $DIR/$tfile || return 1
+       rm -f $DIR/$tfile || return 2       # make it an orphan
+       mds_evict_client
+       client_up || client_up || true    # reconnect
+
+       fail $SINGLEMDS                            # start orphan recovery
+       wait_recovery_complete $SINGLEMDS || error "MDS recovery not done"
+       wait_delete_completed_mds $wait_timeout || return 3
+       AFTERUSED=$(df -P $DIR | tail -1 | awk '{ print $3 }')
+       log "before $BEFOREUSED, after $AFTERUSED"
+       (( $AFTERUSED > $BEFOREUSED + $(fs_log_size) )) &&
+               error "after $AFTERUSED > before $BEFOREUSED"
+       return 0
 }
 run_test 20b "write, unlink, eviction, replay, (test mds_cleanup_orphans)"
 

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 190026d..d516313 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -2068,6 +2068,7 @@ wait_mds_ost_sync () {
        echo "Waiting for orphan cleanup..."
        # MAX value includes time needed for MDS-OST reconnection
        local MAX=$(( TIMEOUT * 2 ))
+       local WAIT_TIMEOUT=${1:-$MAX}
        local WAIT=0
        local new_wait=true
        local list=$(comma_list $(mdts_nodes))
@@ -2080,7 +2081,9 @@ wait_mds_ost_sync () {
                list=$(comma_list $(osts_nodes))
                cmd="$LCTL get_param -n obdfilter.*.mds_sync"
        fi
-       while [ $WAIT -lt $MAX ]; do
+
+       echo "wait $WAIT_TIMEOUT secs maximumly for $list mds-ost sync done."
+       while [ $WAIT -lt $WAIT_TIMEOUT ]; do
                local -a sync=($(do_nodes $list "$cmd"))
                local con=1
                local i
@@ -2096,10 +2099,13 @@ wait_mds_ost_sync () {
                done
                sleep 2 # increase waiting time and cover statfs cache
                [ ${con} -eq 1 ] && return 0
-               echo "Waiting $WAIT secs for $facet mds-ost sync done."
+               echo "Waiting $WAIT secs for $list $i mds-ost sync done."
                WAIT=$((WAIT + 2))
        done
-       echo "$facet recovery not done in $MAX sec. $STATUS"
+
+       # show which nodes are not finished.
+       do_nodes $list "$cmd"
+       echo "$facet recovery node $i not done in $WAIT_TIMEOUT sec. $STATUS"
        return 1
 }