From 0c6b5334c2355e37c1f2db871ca1c91d7ec5536d Mon Sep 17 00:00:00 2001
From: grev <grev>
Date: Thu, 30 Aug 2007 21:34:29 +0000
Subject: [PATCH] b=12560 b=13367 i=Adilger i=Nathan

t-f skip(); use logs for all test scripts
helper remote_mds() remote_ost()
---
 lustre/tests/acceptance-small.sh  |  9 ++++++---
 lustre/tests/conf-sanity.sh       | 20 +++++++++----------
 lustre/tests/insanity.sh          |  3 ++-
 lustre/tests/recovery-small.sh    | 17 +++++++---------
 lustre/tests/replay-dual.sh       |  5 +++--
 lustre/tests/replay-ost-single.sh |  3 ++-
 lustre/tests/replay-single.sh     |  5 +++++
 lustre/tests/sanity-quota.sh      | 38 +++++++++++------------------------
 lustre/tests/sanity.sh            | 37 +++++++++++++++++-----------------
 lustre/tests/sanityN.sh           | 11 ++++++----
 lustre/tests/test-framework.sh    | 42 ++++++++++++++++++++++++++++++++-------
 11 files changed, 108 insertions(+), 82 deletions(-)

diff --git a/lustre/tests/acceptance-small.sh b/lustre/tests/acceptance-small.sh
index 2f753a0..4e203a6 100755
--- a/lustre/tests/acceptance-small.sh
+++ b/lustre/tests/acceptance-small.sh
@@ -78,7 +78,7 @@ for NAME in $CONFIGS; do
 
 	if [ "$SANITY" != "no" ]; then
 	        title sanity
-		SANITYLOG=/tmp/sanity.log sh sanity.sh
+		sh sanity.sh
 		$CLEANUP
 		$SETUP
 	fi
@@ -189,7 +189,7 @@ for NAME in $CONFIGS; do
 		mkdir -p $MOUNT2
 		mount_client $MOUNT2
 		#echo "can't mount2 for '$NAME', skipping sanityN.sh"
-		SANITYLOG=$TMP/sanity.log START=: CLEAN=: sh sanityN.sh
+		START=: CLEAN=: sh sanityN.sh
 		umount $MOUNT2
 
 		$DEBUG_ON
@@ -219,7 +219,10 @@ for NAME in $CONFIGS; do
 		$CLEANUP
 		unload_modules
 		# Liblustre needs accept=all, noacl
-		LNETOPTS="accept=all" MDS_MOUNT_OPTS="${MDS_MOUNT_OPTS},noacl" $SETUP
+		[ -f /etc/modprobe.conf ] && MODPROBECONF=/etc/modprobe.conf
+		[ -f /etc/modprobe.d/Lustre ] && MODPROBECONF=/etc/modprobe.d/Lustre
+
+		LNETOPTS="$(awk '/^options lnet/ { print $0}' $MODPROBECONF | sed 's/^options lnet //g') accept=all" MDS_MOUNT_OPTS="${MDS_MOUNT_OPTS},noacl" $SETUP
 		export LIBLUSTRE_MOUNT_POINT=$MOUNT2
 		export LIBLUSTRE_MOUNT_TARGET=$MGSNID:/$FSNAME
 		export LIBLUSTRE_TIMEOUT=`cat /proc/sys/lustre/timeout`
diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 6118cc8..46f36ed 100644
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -602,7 +602,7 @@ test_15() {
 		echo "save $MOUNTLUSTRE to $MOUNTLUSTRE.sav"
 		mv $MOUNTLUSTRE $MOUNTLUSTRE.sav && trap cleanup_15 EXIT INT
 		if [ -f $MOUNTLUSTRE ]; then
-			echo "$MOUNTLUSTRE cannot be moved, skipping test"
+			skip "$MOUNTLUSTRE cannot be moved, skipping test"
 			return 0
 		fi
 	fi
@@ -980,7 +980,7 @@ test_28() {
 run_test 28 "permanent parameter setting"
 
 test_29() {
-	[ "$OSTCOUNT" -lt "2" ] && echo "skipping deactivate test" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "$OSTCOUNT < 2, skipping" && return
         setup > /dev/null 2>&1
 	start_ost2
 	sleep 10
@@ -1070,9 +1070,9 @@ test_32a() {
         #       there appears to be a lot of assumption here about loopback
         #       devices
         # or maybe this test is just totally useless on a client-only system
-        [ -z "$TUNEFS" ] && echo "No tunefs" && return
-        [ ! -r disk1_4.zip ] && echo "Cant find disk1_4.zip, skipping" && return
-	unzip -o -j -d $TMP/$tdir disk1_4.zip || { echo "Cant unzip disk1_4, skipping" && return ; }
+        [ -z "$TUNEFS" ] && skip "No tunefs" && return
+        [ ! -r disk1_4.zip ] && skip "Cant find disk1_4.zip, skipping" && return
+	unzip -o -j -d $TMP/$tdir disk1_4.zip || { skip "Cant unzip disk1_4, skipping" && return ; }
 	load_modules
 	sysctl lnet.debug=$PTLDEBUG
 
@@ -1126,9 +1126,9 @@ test_32b() {
         #       there appears to be a lot of assumption here about loopback
         #       devices
         # or maybe this test is just totally useless on a client-only system
-        [ -z "$TUNEFS" ] && echo "No tunefs" && return
-        [ ! -r disk1_4.zip ] && echo "Cant find disk1_4.zip, skipping" && return
-	unzip -o -j -d $TMP/$tdir disk1_4.zip || { echo "Cant unzip disk1_4, skipping" && return ; }
+        [ -z "$TUNEFS" ] && skip "No tunefs" && return
+        [ ! -r disk1_4.zip ] && skip "Cant find disk1_4.zip, skipping" && return
+	unzip -o -j -d $TMP/$tdir disk1_4.zip || { skip "Cant unzip disk1_4, skipping" && return ; }
 	load_modules
 	sysctl lnet.debug=$PTLDEBUG
 
@@ -1197,5 +1197,5 @@ umount_client $MOUNT
 cleanup_nocli
 cleanup_krb5_env
 
-equals_msg "Done"
-echo "$0: completed"
+equals_msg `basename $0`: test complete
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
diff --git a/lustre/tests/insanity.sh b/lustre/tests/insanity.sh
index 47b2933..6e37027 100755
--- a/lustre/tests/insanity.sh
+++ b/lustre/tests/insanity.sh
@@ -571,5 +571,6 @@ test_10() {
 }
 run_test 10 "Running Availability for 6 hours..."
 
-equals_msg "Done, cleaning up"
+equals_msg `basename $0`: test complete, cleaning up
 $CLEANUP
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 0cdca40..4a2b57a 100755
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -228,7 +228,7 @@ test_17() {
 run_test 17 "timeout bulk get, don't evict client (2732)"
 
 test_18a() {
-    [ -z ${ost2_svc} ] && echo Skipping, needs 2 osts && return 0
+    [ -z ${ost2_svc} ] && skip "needs 2 osts" && return 0
 
     do_facet client mkdir -p $MOUNT/$tdir
     f=$MOUNT/$tdir/$tfile
@@ -578,10 +578,8 @@ run_test 24 "fsync error (should return error)"
 
 test_26() {      # bug 5921 - evict dead exports by pinger
 # this test can only run from a client on a separate node.
-	[ "`lsmod | grep obdfilter`" ] && \
-	    echo "skipping test 26 (local OST)" && return
-	[ "`lsmod | grep mds`" ] && \
-	    echo "skipping test 26 (local MDS)" && return
+	remote_ost || skip "local OST" && return
+	remote_mds || skip "local MDS" && return
 	OST_FILE=$LPROC/obdfilter/${ost1_svc}/num_exports
         OST_EXP="`do_facet ost1 cat $OST_FILE`"
 	OST_NEXP1=`echo $OST_EXP | cut -d' ' -f2`
@@ -626,8 +624,7 @@ test_26b() {      # bug 10140 - evict dead exports by pinger
 run_test 26b "evict dead exports"
 
 test_27() {
-	[ "`lsmod | grep mds`" ] || \
-	    { echo "skipping test 27 (non-local MDS)" && return 0; }
+	remote_mds && { skip "remote MDS" && return 0; }
 	mkdir -p $DIR/$tdir
 	writemany -q -a $DIR/$tdir/$tfile 0 5 &
 	CLIENT_PID=$!
@@ -773,8 +770,7 @@ run_test 54 "back in time"
 
 # bug 11330 - liblustre application death during I/O locks up OST
 test_55() {
-	[ "`lsmod | grep obdfilter`" ] || \
-	    { echo "skipping test 55 (non-local OST)" && return 0; }	
+	remote_ost && { skip "remote OST" && return 0; }
 
 	mkdir -p $DIR/$tdir
 
@@ -880,5 +876,6 @@ test_58() { # bug 11546
 }
 run_test 58 "Eviction in the middle of open RPC reply processing"
 
+equals_msg `basename $0`: test complete, cleaning up
 $CLEANUP
-echo "$0: completed"
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh
index 15ca9e8..7c5a908 100755
--- a/lustre/tests/replay-dual.sh
+++ b/lustre/tests/replay-dual.sh
@@ -445,9 +445,10 @@ test_19() { # Bug 10991 - resend of open request does not fail assertion.
 run_test 19 "resend of open request"
 
 if [ "$ONLY" != "setup" ]; then
-   equals_msg test complete, cleaning up
+   equals_msg `basename $0`: test complete, cleaning up
    SLEEP=$((`date +%s` - $NOW))
    [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
    $CLEANUP
 fi
-echo "$0: completed"
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
+
diff --git a/lustre/tests/replay-ost-single.sh b/lustre/tests/replay-ost-single.sh
index 6a3b903..9fe3f77 100755
--- a/lustre/tests/replay-ost-single.sh
+++ b/lustre/tests/replay-ost-single.sh
@@ -201,5 +201,6 @@ test_7() {
 }
 run_test 7 "Fail OST before obd_destroy"
 
-equals_msg test complete, cleaning up
+equals_msg `basename $0`: test complete, cleaning up
 $CLEANUP
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index c969222..7455edc 100755
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -930,6 +930,10 @@ run_test 40 "cause recovery in ptlrpc, ensure IO continues"
 # the page, guarnateeing that the unlock from the RPC completion would
 # assert on trying to unlock the unlocked page.
 test_41() {
+    [ $OSTCOUNT -lt 2 ] && \
+	skip "skipping test 41: we don't have a second OST to test with" && \
+	return
+
     local f=$MOUNT/$tfile
     # make sure the start of the file is ost1
     lfs setstripe $f $((128 * 1024)) 0 0 
@@ -1198,3 +1202,4 @@ run_test 60 "test llog post recovery init vs llog unlink"
 
 equals_msg `basename $0`: test complete, cleaning up
 $CLEANUP
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh
index 92a0080..f491b0e 100644
--- a/lustre/tests/sanity-quota.sh
+++ b/lustre/tests/sanity-quota.sh
@@ -56,6 +56,9 @@ ITUNE_SZ=${ITUNE_SZ:-5}		# default 50% of IUNIT_SZ
 MAX_DQ_TIME=604800
 MAX_IQ_TIME=604800
 
+QUOTALOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
+FAIL_ON_ERROR=false
+
 log() {
 	echo "$*"
 	$LCTL mark "$*" 2> /dev/null || true
@@ -121,12 +124,12 @@ run_test() {
  	fi
         testname=EXCEPT_$1
         if [ ${!testname}x != x ]; then
-                 echo "skipping excluded test $1"
+                 TESTNAME=test_$1 skip "skipping excluded test $1"
                  return 0
         fi
         testname=EXCEPT_$base
         if [ ${!testname}x != x ]; then
-                 echo "skipping excluded test $1 (base $base)"
+                 TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)"
                  return 0
         fi
         run_one $1 "$2"
@@ -135,20 +138,6 @@ run_test() {
 
 [ "$QUOTALOG" ] && rm -f $QUOTALOG || true
 
-error() { 
-	sysctl -w lustre.fail_loc=0
-	log "FAIL: $TESTNAME $@"
-	if [ "$QUOTALOG" ]; then
-		echo "FAIL: $TESTNAME $@" >> $QUOTALOG
-	else
-		exit 1
-	fi
-}
-
-pass() { 
-	echo PASS $@
-}
-
 mounted_lustre_filesystems() {
 	awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts | sed -n ${1}p
 }
@@ -613,7 +602,7 @@ run_test 5 "Chown & chgrp successfully even out of block/file quota ==="
 # block quota acquire & release
 test_6() {
 	if [ $OSTCOUNT -lt 2 ]; then
-		echo "WARN: too few osts, skip this test."
+		skip "$OSTCOUNT < 2, too few osts"
 		return 0;
 	fi
 
@@ -674,10 +663,7 @@ run_test 6 "Block quota acquire & release ========="
 # quota recovery (block quota only by now)
 test_7()
 {
-	if [ -z "`lsmod|grep mds`" ]; then 
-		echo "WARN: no local mds, skip this test"
-		return 0
-	fi
+	remote_mds && skip "remote mds"
 
 	LIMIT=$(( $BUNIT_SZ * $(($OSTCOUNT + 1)) * 10)) # 10 bunits each sever
 	TESTFILE="$TSTDIR/quota_tst70"
@@ -729,7 +715,7 @@ test_8() {
 	FILE_LIMIT=1000000
 	DBENCH_LIB=${DBENCH_LIB:-/usr/lib/dbench}
 	
-	[ ! -d $DBENCH_LIB ] && echo "dbench not installed, skip this test" && return 0
+	[ ! -d $DBENCH_LIB ] && skip "dbench not installed" && return 0
 	
 	echo "  Set enough high limit for user: $TSTUSR"
 	$LFS setquota -u $TSTUSR 0 $BLK_LIMIT 0 $FILE_LIMIT $MOUNT
@@ -760,7 +746,7 @@ test_9() {
         size_file=$((1024 * 1024 * 9 / 2 * $OSTCOUNT))
         echo "lustrefs_size:$lustrefs_size  size_file:$size_file"
         if [ $lustrefs_size -lt $size_file ]; then
-	    echo "WARN: less than $size_file free, skip this test."
+	    skip "less than $size_file free"
 	    return 0;
         fi
 
@@ -808,12 +794,12 @@ test_10() {
 	size_file=$((1024 * 1024 * 9 / 2 * $OSTCOUNT))
 	echo "lustrefs_size:$lustrefs_size  size_file:$size_file"
 	if [ $lustrefs_size -lt $size_file ]; then
-		echo "WARN: less than $size_file free, skip this test."
+		skip "less than $size_file free"
 		return 0;
 	fi
 
 	if [ ! -d /proc/fs/lustre/ost/ -o ! -d /proc/fs/lustre/mds ]; then
-		echo "WARN: mds or ost isn't local, skip this test."
+		skip "mds or ost isn't local"
 		return 0;
 	fi
 
@@ -1124,5 +1110,5 @@ if [ "`mount | grep ^$NAME`" ]; then
 fi
 
 echo '=========================== finished ==============================='
-[ -f "$QUOTALOG" ] && cat $QUOTALOG && exit 1
+[ -f "$QUOTALOG" ] && cat $QUOTALOG && grep -q FAIL $QUOTALOG && exit 1 || true
 echo "$0: completed"
diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 81183a1..c8bfcc5 100644
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -76,8 +76,6 @@ else
 	fi
 fi
 
-SANITYLOG=${SANITYLOG:-/tmp/sanity.log}
-
 export NAME=${NAME:-local}
 
 SAVE_PWD=$PWD
@@ -92,6 +90,9 @@ if [ ! -z "$USING_KRB5" ]; then
     $RUNAS -u $(($RUNAS_ID + 1)) krb5_login.sh || exit 1
 fi
 
+SANITYLOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
+FAIL_ON_ERROR=false
+
 cleanup() {
 	echo -n "cln.."
 	cleanupall ${FORCE} $* || { echo "FAILed to clean up"; exit 20; }
@@ -253,7 +254,6 @@ STRIPECOUNT=`cat $LPROC/lov/$LOVNAME/stripecount`
 STRIPESIZE=`cat $LPROC/lov/$LOVNAME/stripesize`
 ORIGFREE=`cat $LPROC/lov/$LOVNAME/kbytesavail`
 MAXFREE=${MAXFREE:-$((200000 * $OSTCOUNT))}
-MDS=$(\ls $LPROC/mdt 2> /dev/null | grep -v num_refs | tail -n 1)
 
 [ -f $DIR/d52a/foo ] && chattr -a $DIR/d52a/foo
 [ -f $DIR/d52b/foo ] && chattr -i $DIR/d52b/foo
@@ -1070,8 +1070,8 @@ exhaust_all_precreations() {
 }
 
 test_27n() {
-	[ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-		skip "too few OSTs, or remote MDS" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+	remote_mds && skip "remote MDS" && return
 
 	reset_enospc
 	rm -f $DIR/d27/f27n
@@ -1084,8 +1084,8 @@ test_27n() {
 run_test 27n "create file with some full OSTs =================="
 
 test_27o() {
-	[ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-		skip "too few OSTs, or remote MDS" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+	remote_mds && skip "remote MDS" && return
 
 	reset_enospc
 	rm -f $DIR/d27/f27o
@@ -1100,8 +1100,8 @@ test_27o() {
 run_test 27o "create file with all full OSTs (should error) ===="
 
 test_27p() {
-	[ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-		skip "too few OSTs, or remote MDS" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+	remote_mds && skip "remote MDS" && return
 
 	reset_enospc
 	rm -f $DIR/d27/f27p
@@ -1119,8 +1119,8 @@ test_27p() {
 run_test 27p "append to a truncated file with some full OSTs ==="
 
 test_27q() {
-	[ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-		skip "too few OSTs, or remote MDS" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+	remote_mds && skip "remote MDS" && return
 
 	reset_enospc
 	rm -f $DIR/d27/f27q
@@ -1139,8 +1139,8 @@ test_27q() {
 run_test 27q "append to truncated file with all OSTs full (should error) ==="
 
 test_27r() {
-	[ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-		skip "too few OSTs, or remote MDS" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+	remote_mds && skip "remote MDS" && return
 
 	reset_enospc
 	rm -f $DIR/d27/f27r
@@ -1782,9 +1782,7 @@ test_36f() {
 }
 run_test 36f "utime on file racing with OST BRW write =========="
 
-if [ -d $LPROC/obdfilter ]; then
-export FMD_MAX_AGE=`cat $LPROC/obdfilter/*/client_cache_seconds | head -n 1`
-fi
+export FMD_MAX_AGE=`do_facet ost1 cat $LPROC/obdfilter/*/client_cache_seconds | head -n 1`
 test_36g() {
 	[ -z "$FMD_MAX_AGE" ] && skip "skip test for remote OST" && return
 	FMD_BEFORE="`awk '/ll_fmd_cache/ { print $2 }' /proc/slabinfo`"
@@ -2367,6 +2365,8 @@ test_52b() {
 run_test 52b "immutable flag test (should return errors) ======="
 
 test_53() {
+	remote_mds && skip "remote MDS" && return
+
 	# only test MDT0000 
         for i in `ls -d $LPROC/osc/*-osc-MDT0000 2> /dev/null` ; do
                 ostname=`basename $i | cut -d - -f 1-2`
@@ -2580,6 +2580,8 @@ run_test 56h "check lfs find ! -name ============================="
 
 test_57a() {
 	# note test will not do anything if MDS is not local
+	remote_mds && skip "remote MDS" && return
+
 	[ -z "$MDS" ] && skip "skipping test for remote MDS" && return
 	for DEV in `cat $LPROC/mds/*/mntdev`; do
 		dumpe2fs -h $DEV > $TMP/t57a.dump || error "can't access $DEV"
@@ -3949,8 +3951,7 @@ free_min_max () {
 
 test_116() {
 	[ "$OSTCOUNT" -lt "2" ] && skip "$OSTCOUNT < 2 OSTs" && return
-	[ $(grep -c obdfilter $LPROC/devices) -eq 0 ] &&
-		skip "remote MDS, skipping test" && return
+	remote_mds && skip "remote MDS" && return
 
 	echo -n "Free space priority "
 	cat $LPROC/lov/*-clilov-*/qos_prio_free
diff --git a/lustre/tests/sanityN.sh b/lustre/tests/sanityN.sh
index 6d45392..8c1b4e0 100644
--- a/lustre/tests/sanityN.sh
+++ b/lustre/tests/sanityN.sh
@@ -52,6 +52,9 @@ LUSTRE=${LUSTRE:-`dirname $0`/..}
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 
+SANITYLOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
+FAIL_ON_ERROR=false
+
 if [ ! -z "$USING_KRB5" ]; then
     $RUNAS krb5_login.sh || exit 1
 fi
@@ -533,7 +536,7 @@ test_16() {
 run_test 16 "2500 iterations of dual-mount fsx ================="
 
 test_17() { # bug 3513, 3667
-	[ ! -d /proc/fs/lustre/ost ] && echo "skipping OST-only test" && return
+	[ ! -d /proc/fs/lustre/ost ] && skip "remote OST, skipping OST-only test" && return
 
 	cp /etc/termcap $DIR1/f17
 	cancel_lru_locks osc > /dev/null
@@ -677,7 +680,8 @@ test_24() {
 run_test 24 "lfs df [-ih] [path] test ========================="
 
 test_25() {
-	[ `cat $LPROC/mdc/*-mdc-*/connect_flags | grep -c acl` -lt 2 ] && echo "skipping $TESTNAME (must have acl)" && return
+	[ `cat $LPROC/mdc/*-mdc-*/connect_flags | grep -c acl` -lt 2 ] && \
+	    skip "must have acl, skipping" && return
 
 	mkdir $DIR1/$tdir || error "mkdir $DIR1/$tdir"
 	touch $DIR1/$tdir/f1 || error "touch $DIR1/$tdir/f1"
@@ -807,6 +811,5 @@ if [ "$I_MOUNTED" = "yes" ]; then
 fi
 
 echo '=========================== finished ==============================='
-[ -f "$SANITYLOG" ] && cat $SANITYLOG && exit 1 || true
+[ -f "$SANITYLOG" ] && cat $SANITYLOG && grep -q FAIL $SANITYLOG && exit 1 || true
 echo "$0: completed"
-
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 5769f4b..fad8246 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -38,6 +38,7 @@ init_test_env() {
 
     [ -d /r ] && export ROOT=${ROOT:-/r}
     export TMP=${TMP:-$ROOT/tmp}
+    export TESTSUITELOG=${TMP}/${TESTSUITE}.log
 
     export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/utils/gss:$LUSTRE/tests
     export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
@@ -82,6 +83,9 @@ init_test_env() {
 
     shift $((OPTIND - 1))
     ONLY=${ONLY:-$*}
+
+    [ "$TESTSUITELOG" ] && rm -f $TESTSUITELOG || true
+
 }
 
 load_module() {
@@ -944,15 +948,29 @@ pgcache_empty() {
     return 0
 }
 
+FAIL_ON_ERROR=true
 ##################################
 # Test interface 
 error() {
+    local ERRLOG
     sysctl -w lustre.fail_loc=0 2> /dev/null || true
-    log "${TESTSUITE}: **** FAIL:" $@
-    $LCTL dk $TMP/lustre-log-$TESTNAME.log
-    log "FAIL: $TESTNAME $@"
-    $LCTL dk $TMP/lustrefail_${TESTSUITE}_${TESTNAME}.$(date +%s)
-    exit 1
+    log "${TESTSUITE} ${TESTNAME}: **** FAIL:" $@
+    ERRLOG=$TMP/lustre_${TESTSUITE}_${TESTNAME}.$(date +%s)
+    echo "Dumping lctl log to $ERRLOG"
+    # We need to dump the logs on all nodes
+    $LCTL dk $ERRLOG
+    [ ! "$mds_HOST" = "$(hostname)" ] && do_node $mds_HOST $LCTL dk $ERRLOG
+    [ ! "$ost_HOST" = "$(hostname)" -a ! "$ost_HOST" = "$mds_HOST" ] && do_node $ost_HOST $LCTL dk $ERRLOG
+    debugrestore
+    [ "$TESTSUITELOG" ] && echo "$0: FAIL: $TESTNAME $@" >> $TESTSUITELOG
+    if $FAIL_ON_ERROR; then
+	exit 1
+    fi
+}
+
+skip () {
+	log " SKIP: ${TESTSUITE} ${TESTNAME} $@"
+	[ "$TESTSUITELOG" ] && echo "${TESTSUITE}: SKIP: $TESTNAME $@" >> $TESTSUITELOG
 }
 
 build_test_filter() {
@@ -993,12 +1011,12 @@ run_test() {
     fi
     testname=EXCEPT_$1
     if [ ${!testname}x != x ]; then
-        log "skipping excluded test $1"
+        TESTNAME=test_$1 skip "skipping excluded test $1"
         return 0
     fi
     testname=EXCEPT_$base
     if [ ${!testname}x != x ]; then
-        log "skipping excluded test $1 (base $base)"
+        TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)"
         return 0
     fi
     run_one $1 "$2"
@@ -1068,3 +1086,13 @@ osc_to_ost()
     fi
     echo $ost
 }
+
+remote_mds ()
+{
+    [ ! -e /proc/fs/lustre/mdt/*MDT* ]
+}
+
+remote_ost ()
+{
+    [ $(grep -c obdfilter $LPROC/devices) -eq 0 ]
+}
-- 
1.8.3.1