b=2432,2494

test cases for these bugs.
Also some test enhancments
  - add more lctl marks
  - add SETUP CLEANUP options to replay-single

diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 7e37260..d5dae90 100755 (executable)
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -3,6 +3,7 @@
 set -e
 
 LUSTRE=${LUSTRE:-`dirname $0`/..}
+UPCALL=${UPCALL:-$PWD/recovery-small-upcall.sh}
 . $LUSTRE/tests/test-framework.sh
 
 init_test_env $@
@@ -150,5 +151,30 @@ test_11(){
     do_facet client munlink $MOUNT/$tfile  || return 4
 }
 run_test 11 "wake up a thead waiting for completion after eviction (b=2460)"
+
+clear_failloc() {
+    facet=$1
+    pause=$2
+    sleep $pause
+    echo "clearing fail_loc on $facet"
+    do_facet $facet "sysctl -w lustre.fail_loc=0"
+}
+
+#b=2494
+test_12(){
+    $LCTL mark multiop $MOUNT/$tfile OS_c 
+    multiop $MOUNT/$tfile OS_c  &
+    PID=$!
+#define OBD_FAIL_MDS_CLOSE_NET           0x115
+    DDPID=$!
+    do_facet mds "sysctl -w lustre.fail_loc=0x115"
+    clear_failloc mds $((TIMEOUT * 2)) &
+    kill -USR1 $PID
+    echo "waiting for multiop $PID"
+    wait $PID || return 2
+    do_facet client munlink $MOUNT/$tfile  || return 3
+}
+run_test 12 "recover from timed out resend in ptlrpcd (b=2494)"
+
 $CLEANUP
     

diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index 20aecdf..9292b0a 100755 (executable)
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -52,13 +52,20 @@ if [ "$ONLY" == "cleanup" ]; then
     exit
 fi
 
-gen_config
+SETUP=${SETUP:-"setup"}
+CLEANUP=${CLEANUP:-"cleanup"}
 
-start ost --reformat $OSTLCONFARGS 
-start ost2 --reformat $OSTLCONFARGS 
-[ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
-start mds $MDSLCONFARGS --reformat
-zconf_mount $MOUNT
+setup() {
+    gen_config
+
+    start ost --reformat $OSTLCONFARGS 
+    start ost2 --reformat $OSTLCONFARGS 
+    [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
+    start mds $MDSLCONFARGS --reformat
+    zconf_mount $MOUNT
+}
+
+$SETUP
 
 if [ "$ONLY" == "setup" ]; then
     exit 0
@@ -665,5 +672,20 @@ test_35() {
 }
 run_test 35 "test recovery from llog for unlink op"
 
+# b=2432 resent cancel after replay uses wrong cookie,
+# so don't resend cancels
+test_36() {
+    replay_barrier mds
+    touch $DIR/$tfile
+    checkstat $DIR/$tfile
+    facet_failover mds
+    cancel_lru_locks MDC
+    if dmesg | grep "unknown lock cookie"; then 
+       echo "cancel after replay failed"
+       return 1
+    fi
+}
+run_test 36 "don't resend cancel"
+
 equals_msg test complete, cleaning up
-cleanup
+$CLEANUP

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 1ea2be7..9c65868 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -255,6 +255,7 @@ do_node() {
 
     if $VERBOSE; then
        echo "CMD: $HOST $@"
+       $PDSH $HOST $LCTL mark "$@" || :
     fi
     $PDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests; cd $RPWD; sh -c \"$@\")"
 }
@@ -416,12 +417,13 @@ drop_bl_callback() {
 }
 
 cancel_lru_locks() {
-       for d in /proc/fs/lustre/ldlm/namespaces/$1*; do
-           if [ -f $d/lru_size ]; then
-               echo clear > $d/lru_size
-               grep [0-9] $d/lock_unused_count
-           fi
-       done
+    $LCTL mark cancel_lru_locks
+    for d in /proc/fs/lustre/ldlm/namespaces/$1*; do
+       if [ -f $d/lru_size ]; then
+           echo clear > $d/lru_size
+           grep [0-9] $d/lock_unused_count
+       fi
+    done
 }
 
 ##################################