LU-3626 tests: More time to allow mds-ost sync being finished

[fs/lustre-release.git] / lustre / tests / replay-single.sh

diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index 3a6299c..3086976 100755 (executable)
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -21,16 +21,20 @@ GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""}
 require_dsh_mds || exit 0
 
 # Skip these tests
-# bug number:  17466 18857      LU1867
-ALWAYS_EXCEPT="61d   33a 33b    89      $REPLAY_SINGLE_EXCEPT"
+# bug number:  17466 18857      LU-1867 LU-1473
+ALWAYS_EXCEPT="61d   33a 33b    89      62     $REPLAY_SINGLE_EXCEPT"
 
 [ $(facet_fstype $SINGLEMDS) = "zfs" ] &&
-# bug number for skipped test:        LU-2342  LU-951
-       ALWAYS_EXCEPT="$ALWAYS_EXCEPT 20b 70a  73a"
+# bug number for skipped test:        LU-951
+       ALWAYS_EXCEPT="$ALWAYS_EXCEPT 73a"
 
 #                                                  63 min  7 min  AT AT AT AT"
 [ "$SLOW" = "no" ] && EXCEPT_SLOW="1 2 3 4 6 12 16 44a      44b    65 66 67 68"
 
+[ $(facet_fstype $SINGLEMDS) = "zfs" ] &&
+# bug number for skipped test:        LU-3127
+        ALWAYS_EXCEPT="$ALWAYS_EXCEPT 73b"
+
 build_test_filter
 
 check_and_setup_lustre
@@ -420,27 +424,30 @@ test_20a() {      # was test_20
 run_test 20a "|X| open(O_CREAT), unlink, replay, close (test mds_cleanup_orphans)"
 
 test_20b() { # bug 10480
-    BEFOREUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'`
-
-    dd if=/dev/zero of=$DIR/$tfile bs=4k count=10000 &
-    pid=$!
-    while [ ! -e $DIR/$tfile ] ; do
-        usleep 60                           # give dd a chance to start
-    done
+       local wait_timeout=$((TIMEOUT * 4))
+       local BEFOREUSED
+       local AFTERUSED
 
-    $GETSTRIPE $DIR/$tfile || return 1
-    rm -f $DIR/$tfile || return 2       # make it an orphan
-    mds_evict_client
-    client_up || client_up || true    # reconnect
+       BEFOREUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'`
+       dd if=/dev/zero of=$DIR/$tfile bs=4k count=10000 &
+       pid=$!
+       while [ ! -e $DIR/$tfile ] ; do
+       usleep 60                           # give dd a chance to start
+       done
 
-    fail $SINGLEMDS                            # start orphan recovery
-    wait_recovery_complete $SINGLEMDS || error "MDS recovery not done"
-    wait_mds_ost_sync || return 3
-    AFTERUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'`
-    log "before $BEFOREUSED, after $AFTERUSED"
-    [ $AFTERUSED -gt $((BEFOREUSED + 20)) ] && \
-        error "after $AFTERUSED > before $BEFOREUSED"
-    return 0
+       $GETSTRIPE $DIR/$tfile || return 1
+       rm -f $DIR/$tfile || return 2       # make it an orphan
+       mds_evict_client
+       client_up || client_up || true    # reconnect
+
+       fail $SINGLEMDS                            # start orphan recovery
+       wait_recovery_complete $SINGLEMDS || error "MDS recovery not done"
+       wait_delete_completed_mds $wait_timeout || return 3
+       AFTERUSED=$(df -P $DIR | tail -1 | awk '{ print $3 }')
+       log "before $BEFOREUSED, after $AFTERUSED"
+       (( $AFTERUSED > $BEFOREUSED + $(fs_log_size) )) &&
+               error "after $AFTERUSED > before $BEFOREUSED"
+       return 0
 }
 run_test 20b "write, unlink, eviction, replay, (test mds_cleanup_orphans)"
 
@@ -796,36 +803,40 @@ count_ost_writes() {
 
 #b=2477,2532
 test_40(){
-    $LCTL mark multiop $MOUNT/$tfile OS_c
-    multiop $MOUNT/$tfile OS_c  &
-    PID=$!
-    writeme -s $MOUNT/${tfile}-2 &
-    WRITE_PID=$!
-    sleep 1
-    facet_failover $SINGLEMDS
+       # always need connection to MDS to verify layout during IO. LU-2628.
+       lctl get_param mdc.*.connect_flags | grep -q layout_lock &&
+               skip "layout_lock needs MDS connection for IO" && return 0
+
+       $LCTL mark multiop $MOUNT/$tfile OS_c
+       multiop $MOUNT/$tfile OS_c  &
+       PID=$!
+       writeme -s $MOUNT/${tfile}-2 &
+       WRITE_PID=$!
+       sleep 1
+       facet_failover $SINGLEMDS
 #define OBD_FAIL_MDS_CONNECT_NET         0x117
-    do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000117"
-    kill -USR1 $PID
-    stat1=`count_ost_writes`
-    sleep $TIMEOUT
-    stat2=`count_ost_writes`
-    echo "$stat1, $stat2"
-    if [ $stat1 -lt $stat2 ]; then
-       echo "writes continuing during recovery"
-       RC=0
-    else
-       echo "writes not continuing during recovery, bug 2477"
-       RC=4
-    fi
-    echo "waiting for writeme $WRITE_PID"
-    kill $WRITE_PID
-    wait $WRITE_PID
+       do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000117"
+       kill -USR1 $PID
+       stat1=`count_ost_writes`
+       sleep $TIMEOUT
+       stat2=`count_ost_writes`
+       echo "$stat1, $stat2"
+       if [ $stat1 -lt $stat2 ]; then
+               echo "writes continuing during recovery"
+               RC=0
+       else
+               echo "writes not continuing during recovery, bug 2477"
+               RC=4
+       fi
+       echo "waiting for writeme $WRITE_PID"
+       kill $WRITE_PID
+       wait $WRITE_PID
 
-    echo "waiting for multiop $PID"
-    wait $PID || return 2
-    do_facet client munlink $MOUNT/$tfile  || return 3
-    do_facet client munlink $MOUNT/${tfile}-2  || return 3
-    return $RC
+       echo "waiting for multiop $PID"
+       wait $PID || return 2
+       do_facet client munlink $MOUNT/$tfile  || return 3
+       do_facet client munlink $MOUNT/${tfile}-2  || return 3
+       return $RC
 }
 run_test 40 "cause recovery in ptlrpc, ensure IO continues"
 
@@ -900,8 +911,8 @@ run_test 43 "mds osc import failure during recovery; don't LBUG"
 test_44a() { # was test_44
        local at_max_saved=0
 
-       local mdcdev=$($LCTL get_param -n devices |
-               awk "/ ${FSNAME}-MDT0000-mdc-/ {print \$1}")
+       local mdcdev=$($LCTL dl |
+               awk "/${FSNAME}-MDT0000-mdc-/ {if (\$2 == \"UP\") {print \$1}}")
        [ "$mdcdev" ] || return 2
        [ $(echo $mdcdev | wc -w) -eq 1 ] ||
                { echo mdcdev=$mdcdev; $LCTL dl; return 3; }
@@ -914,7 +925,8 @@ test_44a() { # was test_44
 
        for i in `seq 1 10`; do
                echo "$i of 10 ($(date +%s))"
-               do_facet $SINGLEMDS "lctl get_param -n mdt.*.mdt.timeouts | grep service"
+               do_facet $SINGLEMDS \
+                       "lctl get_param -n md[ts].*.mdt.timeouts | grep service"
 #define OBD_FAIL_TGT_CONN_RACE     0x701
                do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000701"
                 # lctl below may fail, it is valid case
@@ -928,15 +940,16 @@ test_44a() { # was test_44
 run_test 44a "race in target handle connect"
 
 test_44b() {
-       local mdcdev=$($LCTL get_param -n devices |
-               awk "/ ${FSNAME}-MDT0000-mdc-/ {print \$1}")
+       local mdcdev=$($LCTL dl |
+               awk "/${FSNAME}-MDT0000-mdc-/ {if (\$2 == \"UP\") {print \$1}}")
        [ "$mdcdev" ] || return 2
        [ $(echo $mdcdev | wc -w) -eq 1 ] ||
                { echo mdcdev=$mdcdev; $LCTL dl; return 3; }
 
        for i in `seq 1 10`; do
                echo "$i of 10 ($(date +%s))"
-               do_facet $SINGLEMDS "lctl get_param -n mdt.*.mdt.timeouts | grep service"
+               do_facet $SINGLEMDS \
+                       "lctl get_param -n md[ts].*.mdt.timeouts | grep service"
         #define OBD_FAIL_TGT_DELAY_RECONNECT 0x704
                do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000704"
         # lctl below may fail, it is valid case
@@ -1383,6 +1396,11 @@ test_57() {
 }
 run_test 57 "test recovery from llog for setattr op"
 
+cleanup_58() {
+    zconf_umount `hostname` $MOUNT2
+    trap - EXIT
+}
+
 #recovery many mds-ost setattr from llog
 test_58a() {
     mkdir -p $DIR/$tdir
@@ -1403,6 +1421,8 @@ test_58b() {
     local orig
     local new
 
+    trap cleanup_58 EXIT
+
     large_xattr_enabled &&
         orig="$(generate_string $(max_xattr_size))" || orig="bar"
 
@@ -1416,7 +1436,7 @@ test_58b() {
     [[ "$new" = "$orig" ]] || return 1
     rm -f $DIR/$tdir/$tfile
     rmdir $DIR/$tdir
-    zconf_umount `hostname` $MOUNT2
+    cleanup_58
 }
 run_test 58b "test replay of setxattr op"
 
@@ -1425,6 +1445,8 @@ test_58c() { # bug 16570
     local orig1
     local new
 
+    trap cleanup_58 EXIT
+
     if large_xattr_enabled; then
         local xattr_size=$(max_xattr_size)
         orig="$(generate_string $((xattr_size / 2)))"
@@ -1447,7 +1469,7 @@ test_58c() { # bug 16570
     [[ "$new" = "$orig1" ]] || return 4
     rm -f $DIR/$tdir/$tfile
     rmdir $DIR/$tdir
-    zconf_umount $HOSTNAME $MOUNT2
+    cleanup_58
 }
 run_test 58c "resend/reconstruct setxattr op"
 
@@ -1888,6 +1910,7 @@ test_70b () {
        # set duration to 900 because it takes some time to boot node
        [ "$FAILURE_MODE" = HARD ] && duration=900
 
+       local elapsed
        local start_ts=$(date +%s)
        local cmd="rundbench 1 -t $duration"
        local pid=""
@@ -1896,15 +1919,24 @@ test_70b () {
                DBENCH_LIB=$DBENCH_LIB TESTSUITE=$TESTSUITE TESTNAME=$TESTNAME \
                MOUNT=$MOUNT DIR=$DIR/$tdir/\\\$(hostname) LCTL=$LCTL $cmd" &
        pid=$!
+
+       #LU-1897 wait for all dbench copies to start
+       while ! check_for_process $clients dbench; do
+               elapsed=$(($(date +%s) - start_ts))
+               if [ $elapsed -gt $duration ]; then
+                       killall_process $clients dbench
+                       error "dbench failed to start on $clients!"
+               fi
+               sleep 1
+       done
+
        log "Started rundbench load pid=$pid ..."
 
-       # give rundbench a chance to start, bug 24118
-       sleep 12
-       local elapsed=$(($(date +%s) - start_ts))
+       elapsed=$(($(date +%s) - start_ts))
        local num_failovers=0
        while [ $elapsed -lt $duration ]; do
                if ! check_for_process $clients dbench; then
-                       error_noexit "dbench not running on some of $clients!"
+                       error_noexit "dbench stopped on some of $clients!"
                        killall_process $clients dbench
                        break
                fi
@@ -2010,11 +2042,10 @@ test_80a() {
        local remote_dir=$DIR/$tdir/remote_dir
 
        mkdir -p $DIR/$tdir
-       # OBD_FAIL_UPDATE_OBJ_NET    0x1500
-       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x1500
+       #define OBD_FAIL_UPDATE_OBJ_NET_REP     0x1701
+       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x1701
        $LFS mkdir -i $MDTIDX $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
 
        fail mds$((MDTIDX + 1))
 
@@ -2038,11 +2069,10 @@ test_80b() {
        local remote_dir=$DIR/$tdir/remote_dir
 
        mkdir -p $DIR/$tdir
-       # OBD_FAIL_UPDATE_OBJ_NET    0x1500
-       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x1500
+       #define OBD_FAIL_UPDATE_OBJ_NET_REP     0x1701
+       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x1701
        $LFS mkdir -i $MDTIDX $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
 
        fail mds${MDTIDX}
 
@@ -2066,11 +2096,10 @@ test_80c() {
        local remote_dir=$DIR/$tdir/remote_dir
 
        mkdir -p $DIR/$tdir
-       # OBD_FAIL_UPDATE_OBJ_NET    0x1500
-       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x1500
+       #define OBD_FAIL_UPDATE_OBJ_NET_REP     0x1701
+       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x1701
        $LFS mkdir -i $MDTIDX $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
 
        fail mds${MDTIDX}
        fail mds$((MDTIDX + 1))
@@ -2090,11 +2119,10 @@ test_80d() {
        local remote_dir=$DIR/$tdir/remote_dir
 
        mkdir -p $DIR/$tdir
-       # OBD_FAIL_UPDATE_OBJ_NET    0x1500
-       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x1500
+       #define OBD_FAIL_UPDATE_OBJ_NET_REP     0x1701
+       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x1701
        $LFS mkdir -i $MDTIDX $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
 
        fail mds${MDTIDX},mds$((MDTIDX + 1))
 
@@ -2122,7 +2150,6 @@ test_80e() {
        do_facet mds${MDTIDX} lctl set_param fail_loc=0x119
        $LFS mkdir -i $MDTIDX $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds${MDTIDX} lctl set_param fail_loc=0
 
        fail mds${MDTIDX}
 
@@ -2149,7 +2176,6 @@ test_80f() {
        do_facet mds${MDTIDX} lctl set_param fail_loc=0x119
        $LFS mkdir -i $MDTIDX $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds${MDTIDX} lctl set_param fail_loc=0
 
        fail mds$((MDTIDX + 1))
 
@@ -2177,7 +2203,6 @@ test_80g() {
        do_facet mds${MDTIDX} lctl set_param fail_loc=0x119
        $LFS mkdir -i $MDTIDX $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds${MDTIDX} lctl set_param fail_loc=0
 
        fail mds${MDTIDX}
        fail mds$((MDTIDX + 1))
@@ -2201,7 +2226,6 @@ test_80h() {
        do_facet mds${MDTIDX} lctl set_param fail_loc=0x119
        $LFS mkdir -i $MDTIDX $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds${MDTIDX} lctl set_param fail_loc=0
 
        fail mds${MDTIDX},mds$((MDTIDX + 1))
 
@@ -2227,11 +2251,11 @@ test_81a() {
        mkdir -p $DIR/$tdir
        $LFS mkdir -i $MDTIDX $remote_dir || error "lfs mkdir failed"
 
-       # OBD_FAIL_UPDATE_OBJ_NET    0x1500
-       do_facet mds${MDTIDX} lctl set_param fail_loc=0x1500
+       touch $remote_dir
+       # OBD_FAIL_OBJ_UPDATE_NET_REP       0x1701
+       do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
        rmdir $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds${MDTIDX} lctl set_param fail_loc=0
 
        fail mds$((MDTIDX + 1))
 
@@ -2257,11 +2281,10 @@ test_81b() {
        mkdir -p $DIR/$tdir
        $LFS mkdir -i $MDTIDX $remote_dir || error "lfs mkdir failed"
 
-       # OBD_FAIL_UPDATE_OBJ_NET    0x1500
-       do_facet mds${MDTIDX} lctl set_param fail_loc=0x1500
+       # OBD_FAIL_OBJ_UPDATE_NET_REP       0x1701
+       do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
        rmdir $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds${MDTIDX} lctl set_param fail_loc=0
 
        fail mds${MDTIDX}
 
@@ -2288,11 +2311,10 @@ test_81c() {
        mkdir -p $DIR/$tdir
        $LFS mkdir -i $MDTIDX $remote_dir || error "lfs mkdir failed"
 
-       # OBD_FAIL_UPDATE_OBJ_NET    0x1500
-       do_facet mds${MDTIDX} lctl set_param fail_loc=0x1500
+       # OBD_FAIL_OBJ_UPDATE_NET_REP       0x1701
+       do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
        rmdir $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds${MDTIDX} lctl set_param fail_loc=0
 
        fail mds${MDTIDX}
        fail mds$((MDTIDX + 1))
@@ -2315,11 +2337,10 @@ test_81d() {
        mkdir -p $DIR/$tdir
        $LFS mkdir -i $MDTIDX $remote_dir || error "lfs mkdir failed"
 
-       # OBD_FAIL_UPDATE_OBJ_NET    0x1500
-       do_facet mds${MDTIDX} lctl set_param fail_loc=0x1500
+       # OBD_FAIL_OBJ_UPDATE_NET_REP       0x1701
+       do_facet mds${MDTIDX} lctl set_param fail_loc=0x1701
        rmdir $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds${MDTIDX} lctl set_param fail_loc=0
 
        fail mds${MDTIDX},mds$((MDTIDX + 1))
 
@@ -2381,7 +2402,6 @@ test_81f() {
        do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
        rmdir $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
 
        fail mds$((MDTIDX + 1))
 
@@ -2412,7 +2432,6 @@ test_81g() {
        do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
        rmdir $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
 
        fail mds${MDTIDX}
        fail mds$((MDTIDX + 1))
@@ -2439,7 +2458,6 @@ test_81h() {
        do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0x119
        rmdir $remote_dir &
        local CLIENT_PID=$!
-       do_facet mds$((MDTIDX + 1)) lctl set_param fail_loc=0
 
        fail mds${MDTIDX},mds$((MDTIDX + 1))
 
@@ -2738,8 +2756,8 @@ test_90() { # bug 19494
         local uuid=$(ostuuid_from_index $i)
         for file in f$i all; do
             if [[ $dir/$file != $($LFS find --obd $uuid --name $file $dir) ]]; then
-                $GETSTRIPE $dir/file
-                error wrong stripe: $file, uuid: $uuid
+               $GETSTRIPE $dir/$file
+               error wrong stripe: $file, uuid: $uuid
             fi
         done
     done
@@ -2747,7 +2765,7 @@ test_90() { # bug 19494
     # Before failing an OST, get its obd name and index
     local varsvc=${ostfail}_svc
     local obd=$(do_facet $ostfail lctl get_param -n obdfilter.${!varsvc}.uuid)
-    local index=${obd:(-6):1}
+       local index=$(($(facet_number $ostfail) - 1))
 
     echo "Fail $ostfail $obd, display the list of affected files"
     shutdown_facet $ostfail || return 2