Branch b1_4

Forceful test cleanup (from b_cray)
On tests that evict clients, make sure clients are reconnected at the end
of the test.

diff --git a/lustre/tests/recovery-cleanup.sh b/lustre/tests/recovery-cleanup.sh
index c0d8a51..8c3c625 100755 (executable)
--- a/lustre/tests/recovery-cleanup.sh
+++ b/lustre/tests/recovery-cleanup.sh
@@ -142,4 +142,4 @@ try_to_cleanup
 drop_request "munlink /mnt/lustre/link1" & wait_for_timeout
 try_to_cleanup
 
-$CLEANUP '--dump $TMP/recovery-cleanup-`hostname`.log'
+FORCE=--force $CLEANUP '--dump $TMP/recovery-cleanup-`hostname`.log'

diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 21012f7..d7480b6 100755 (executable)
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -66,7 +66,7 @@ fi
 
 if [ "$ONLY" == "cleanup" ]; then
     sysctl -w portals.debug=0 || true
-    cleanup
+    FORCE=--force cleanup
     exit
 fi
 
@@ -388,7 +388,7 @@ test_24() { # bug 2248 - eviction fails writeback but app doesn't see it
        wait $MULTI_PID
        rc=$?
        sysctl -w lustre.fail_loc=0x0
-       client_df
+       client_reconnect
        [ $rc -eq 0 ] && error "multiop didn't fail fsync: rc $rc" || true
 }
 run_test 24 "fsync error (should return error)" 
@@ -473,8 +473,9 @@ test_25c() {
        sleep 30
        test_25c_guts
        rc=$?
+       client_reconnect
        return $rc
 }
 run_test 25c "failover OST under load"
 
-$CLEANUP
+FORCE=--force $CLEANUP

diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh
index 9f6b778..4920a19 100755 (executable)
--- a/lustre/tests/replay-dual.sh
+++ b/lustre/tests/replay-dual.sh
@@ -17,11 +17,11 @@ gen_config() {
     rm -f $XMLCONFIG
     add_mds mds --dev $MDSDEV --size $MDSSIZE
     if [ ! -z "$mdsfailover_HOST" ]; then
-        add_mdsfailover mds --dev $MDSDEV --size $MDSSIZE
+    add_mdsfailover mds --dev $MDSDEV --size $MDSSIZE
     fi
     
     add_lov lov1 mds --stripe_sz $STRIPE_BYTES \
-       --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
+   --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
     add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE --failover
     add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE  --failover
     add_client client mds --lov lov1 --path $MOUNT
@@ -49,7 +49,7 @@ cleanup() {
 
 if [ "$ONLY" == "cleanup" ]; then
     sysctl -w portals.debug=0
-    cleanup
+    FORCE=--force cleanup
     exit
 fi
 
@@ -362,30 +362,30 @@ run_test 17 "fail OST during recovery (3571)"
 # cleanup with blocked enqueue fails until timer elapses (MDS busy), wait for it
 export NOW=0
 
-test_18() {    # bug 3822 - evicting client with enqueued lock
-       #set -vx
-       mkdir -p $MOUNT1/$tdir
-       touch $MOUNT1/$tdir/f0
+test_18() { # bug 3822 - evicting client with enqueued lock
+   #set -vx
+   mkdir -p $MOUNT1/$tdir
+   touch $MOUNT1/$tdir/f0
 #define OBD_FAIL_LDLM_ENQUEUE_BLOCKED    0x30b
-       statmany -s $MOUNT1/$tdir/f 1 500 &
-       OPENPID=$!
-       NOW=`date +%s`
-       do_facet mds sysctl -w lustre.fail_loc=0x8000030b  # hold enqueue
-       sleep 1
+   statmany -s $MOUNT1/$tdir/f 1 500 &
+   OPENPID=$!
+   NOW=`date +%s`
+   do_facet mds sysctl -w lustre.fail_loc=0x8000030b  # hold enqueue
+   sleep 1
 #define OBD_FAIL_LDLM_BL_CALLBACK        0x305
-       do_facet client sysctl -w lustre.fail_loc=0x80000305  # drop cb, evict
-       cancel_lru_locks MDC
-       usleep 500 # wait to ensure first client is one that will be evicted
-       openfile -f O_RDONLY $MOUNT2/$tdir/f0
-       wait $OPENPID
-       dmesg | grep "entering recovery in server" && \
-               error "client not evicted" || true
+   do_facet client sysctl -w lustre.fail_loc=0x80000305  # drop cb, evict
+   cancel_lru_locks MDC
+   usleep 500 # wait to ensure first client is one that will be evicted
+   openfile -f O_RDONLY $MOUNT2/$tdir/f0
+   wait $OPENPID
+   dmesg | grep "entering recovery in server" && \
+      error "client not evicted" || true
 }
 run_test 18 "ldlm_handle_enqueue succeeds on evicted export (3822)"
 
 if [ "$ONLY" != "setup" ]; then
-       equals_msg test complete, cleaning up
-       SLEEP=$((`date +%s` - $NOW))
-       [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
-       $CLEANUP
+   equals_msg test complete, cleaning up
+   SLEEP=$((`date +%s` - $NOW))
+   [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
+   FORCE=--force $CLEANUP
 fi

diff --git a/lustre/tests/replay-ost-single.sh b/lustre/tests/replay-ost-single.sh
index da1f2a9..753b858 100755 (executable)
--- a/lustre/tests/replay-ost-single.sh
+++ b/lustre/tests/replay-ost-single.sh
@@ -41,7 +41,7 @@ cleanup() {
 
 if [ "$ONLY" == "cleanup" ]; then
     sysctl -w portals.debug=0
-    cleanup
+    FORCE=--force cleanup
     exit
 fi
 
@@ -197,4 +197,4 @@ test_7() {
 run_test 7 "Fail OST before obd_destroy"
 
 equals_msg test complete, cleaning up
-$CLEANUP
+FORCE=--force $CLEANUP

diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index 98a7c03..15a2c99 100755 (executable)
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -48,7 +48,7 @@ cleanup() {
 
 if [ "$ONLY" == "cleanup" ]; then
     sysctl -w portals.debug=0 || true
-    cleanup
+    FORCE=--force cleanup
     exit
 fi
 
@@ -1031,4 +1031,4 @@ test_58() {
 run_test 58 "test recovery from llog for setattr op (test llog_gen_rec)"
 
 equals_msg test complete, cleaning up
-$CLEANUP
+FORCE=--force $CLEANUP

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 2e82e0f..6173ecd 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -150,6 +150,17 @@ client_df() {
     fi
 }
 
+client_reconnect() {
+    df $MOUNT > /dev/null
+    uname -n >> $MOUNT/recon
+    if [ ! -z "$CLIENTS" ]; then
+       $PDSH $CLIENTS "df $MOUNT; uname -n >> $MOUNT/recon" > /dev/null
+    fi
+    echo Connected clients:
+    cat $MOUNT/recon
+    rm $MOUNT/recon
+}
+
 facet_failover() {
     facet=$1
     echo "Failing $facet node `facet_active_host $facet`"
@@ -582,6 +593,12 @@ pass() {
        echo PASS $@
 }
 
+check_mds() {
+    FFREE=`cat /proc/fs/lustre/mds/*/filesfree`
+    FTOTAL=`cat /proc/fs/lustre/mds/*/filestotal`
+    [ $FFREE -ge $FTOTAL ] && error "files free $FFREE > total $FTOTAL" || true
+}
+
 run_one() {
     testnum=$1
     message=$2
@@ -593,7 +610,9 @@ run_one() {
 
     BEFORE=`date +%s`
     log "== test $testnum: $message ============ `date +%H:%M:%S` ($BEFORE)"
+    #check_mds
     test_${testnum} || error "test_$testnum failed with $?"
+    #check_mds
     pass "($((`date +%s` - $BEFORE))s)"
 }