From fdfefb0534a6eb4f9eebcfbebabefef0f0af5117 Mon Sep 17 00:00:00 2001
From: nathan <nathan>
Date: Fri, 8 Apr 2005 17:01:38 +0000
Subject: [PATCH] Branch b1_4 Forceful test cleanup (from b_cray) On tests that
 evict clients, make sure clients are reconnected at the end of the test.

---
 lustre/tests/recovery-cleanup.sh  |  2 +-
 lustre/tests/recovery-small.sh    |  7 +++---
 lustre/tests/replay-dual.sh       | 46 +++++++++++++++++++--------------------
 lustre/tests/replay-ost-single.sh |  4 ++--
 lustre/tests/replay-single.sh     |  4 ++--
 lustre/tests/test-framework.sh    | 19 ++++++++++++++++
 6 files changed, 51 insertions(+), 31 deletions(-)

diff --git a/lustre/tests/recovery-cleanup.sh b/lustre/tests/recovery-cleanup.sh
index c0d8a51..8c3c625 100755
--- a/lustre/tests/recovery-cleanup.sh
+++ b/lustre/tests/recovery-cleanup.sh
@@ -142,4 +142,4 @@ try_to_cleanup
 drop_request "munlink /mnt/lustre/link1" & wait_for_timeout
 try_to_cleanup
 
-$CLEANUP '--dump $TMP/recovery-cleanup-`hostname`.log'
+FORCE=--force $CLEANUP '--dump $TMP/recovery-cleanup-`hostname`.log'
diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 21012f7..d7480b6 100755
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -66,7 +66,7 @@ fi
 
 if [ "$ONLY" == "cleanup" ]; then
     sysctl -w portals.debug=0 || true
-    cleanup
+    FORCE=--force cleanup
     exit
 fi
 
@@ -388,7 +388,7 @@ test_24() {	# bug 2248 - eviction fails writeback but app doesn't see it
 	wait $MULTI_PID
 	rc=$?
 	sysctl -w lustre.fail_loc=0x0
-	client_df
+	client_reconnect
 	[ $rc -eq 0 ] && error "multiop didn't fail fsync: rc $rc" || true
 }
 run_test 24 "fsync error (should return error)" 
@@ -473,8 +473,9 @@ test_25c() {
 	sleep 30
 	test_25c_guts
 	rc=$?
+	client_reconnect
 	return $rc
 }
 run_test 25c "failover OST under load"
 
-$CLEANUP
+FORCE=--force $CLEANUP
diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh
index 9f6b778..4920a19 100755
--- a/lustre/tests/replay-dual.sh
+++ b/lustre/tests/replay-dual.sh
@@ -17,11 +17,11 @@ gen_config() {
     rm -f $XMLCONFIG
     add_mds mds --dev $MDSDEV --size $MDSSIZE
     if [ ! -z "$mdsfailover_HOST" ]; then
-	 add_mdsfailover mds --dev $MDSDEV --size $MDSSIZE
+    add_mdsfailover mds --dev $MDSDEV --size $MDSSIZE
     fi
     
     add_lov lov1 mds --stripe_sz $STRIPE_BYTES \
-	--stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
+   --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
     add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE --failover
     add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE  --failover
     add_client client mds --lov lov1 --path $MOUNT
@@ -49,7 +49,7 @@ cleanup() {
 
 if [ "$ONLY" == "cleanup" ]; then
     sysctl -w portals.debug=0
-    cleanup
+    FORCE=--force cleanup
     exit
 fi
 
@@ -362,30 +362,30 @@ run_test 17 "fail OST during recovery (3571)"
 # cleanup with blocked enqueue fails until timer elapses (MDS busy), wait for it
 export NOW=0
 
-test_18() {	# bug 3822 - evicting client with enqueued lock
-	#set -vx
-	mkdir -p $MOUNT1/$tdir
-	touch $MOUNT1/$tdir/f0
+test_18() { # bug 3822 - evicting client with enqueued lock
+   #set -vx
+   mkdir -p $MOUNT1/$tdir
+   touch $MOUNT1/$tdir/f0
 #define OBD_FAIL_LDLM_ENQUEUE_BLOCKED    0x30b
-	statmany -s $MOUNT1/$tdir/f 1 500 &
-	OPENPID=$!
-	NOW=`date +%s`
-	do_facet mds sysctl -w lustre.fail_loc=0x8000030b  # hold enqueue
-	sleep 1
+   statmany -s $MOUNT1/$tdir/f 1 500 &
+   OPENPID=$!
+   NOW=`date +%s`
+   do_facet mds sysctl -w lustre.fail_loc=0x8000030b  # hold enqueue
+   sleep 1
 #define OBD_FAIL_LDLM_BL_CALLBACK        0x305
-	do_facet client sysctl -w lustre.fail_loc=0x80000305  # drop cb, evict
-	cancel_lru_locks MDC
-	usleep 500 # wait to ensure first client is one that will be evicted
-	openfile -f O_RDONLY $MOUNT2/$tdir/f0
-	wait $OPENPID
-	dmesg | grep "entering recovery in server" && \
-		error "client not evicted" || true
+   do_facet client sysctl -w lustre.fail_loc=0x80000305  # drop cb, evict
+   cancel_lru_locks MDC
+   usleep 500 # wait to ensure first client is one that will be evicted
+   openfile -f O_RDONLY $MOUNT2/$tdir/f0
+   wait $OPENPID
+   dmesg | grep "entering recovery in server" && \
+      error "client not evicted" || true
 }
 run_test 18 "ldlm_handle_enqueue succeeds on evicted export (3822)"
 
 if [ "$ONLY" != "setup" ]; then
-	equals_msg test complete, cleaning up
-	SLEEP=$((`date +%s` - $NOW))
-	[ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
-	$CLEANUP
+   equals_msg test complete, cleaning up
+   SLEEP=$((`date +%s` - $NOW))
+   [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
+   FORCE=--force $CLEANUP
 fi
diff --git a/lustre/tests/replay-ost-single.sh b/lustre/tests/replay-ost-single.sh
index da1f2a9..753b858 100755
--- a/lustre/tests/replay-ost-single.sh
+++ b/lustre/tests/replay-ost-single.sh
@@ -41,7 +41,7 @@ cleanup() {
 
 if [ "$ONLY" == "cleanup" ]; then
     sysctl -w portals.debug=0
-    cleanup
+    FORCE=--force cleanup
     exit
 fi
 
@@ -197,4 +197,4 @@ test_7() {
 run_test 7 "Fail OST before obd_destroy"
 
 equals_msg test complete, cleaning up
-$CLEANUP
+FORCE=--force $CLEANUP
diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index 98a7c03..15a2c99 100755
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -48,7 +48,7 @@ cleanup() {
 
 if [ "$ONLY" == "cleanup" ]; then
     sysctl -w portals.debug=0 || true
-    cleanup
+    FORCE=--force cleanup
     exit
 fi
 
@@ -1031,4 +1031,4 @@ test_58() {
 run_test 58 "test recovery from llog for setattr op (test llog_gen_rec)"
 
 equals_msg test complete, cleaning up
-$CLEANUP
+FORCE=--force $CLEANUP
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 2e82e0f..6173ecd 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -150,6 +150,17 @@ client_df() {
     fi
 }
 
+client_reconnect() {
+    df $MOUNT > /dev/null
+    uname -n >> $MOUNT/recon
+    if [ ! -z "$CLIENTS" ]; then
+	$PDSH $CLIENTS "df $MOUNT; uname -n >> $MOUNT/recon" > /dev/null
+    fi
+    echo Connected clients:
+    cat $MOUNT/recon
+    rm $MOUNT/recon
+}
+
 facet_failover() {
     facet=$1
     echo "Failing $facet node `facet_active_host $facet`"
@@ -582,6 +593,12 @@ pass() {
 	echo PASS $@
 }
 
+check_mds() {
+    FFREE=`cat /proc/fs/lustre/mds/*/filesfree`
+    FTOTAL=`cat /proc/fs/lustre/mds/*/filestotal`
+    [ $FFREE -ge $FTOTAL ] && error "files free $FFREE > total $FTOTAL" || true
+}
+
 run_one() {
     testnum=$1
     message=$2
@@ -593,7 +610,9 @@ run_one() {
 
     BEFORE=`date +%s`
     log "== test $testnum: $message ============ `date +%H:%M:%S` ($BEFORE)"
+    #check_mds
     test_${testnum} || error "test_$testnum failed with $?"
+    #check_mds
     pass "($((`date +%s` - $BEFORE))s)"
 }
 
-- 
1.8.3.1