b=12560

b=13367
i=Adilger
i=Nathan

t-f skip(); use logs for all test scripts
helper remote_mds() remote_ost()

diff --git a/lustre/tests/acceptance-small.sh b/lustre/tests/acceptance-small.sh
index 2f753a0..4e203a6 100755 (executable)
--- a/lustre/tests/acceptance-small.sh
+++ b/lustre/tests/acceptance-small.sh
@@ -78,7 +78,7 @@ for NAME in $CONFIGS; do
 
        if [ "$SANITY" != "no" ]; then
                title sanity
-               SANITYLOG=/tmp/sanity.log sh sanity.sh
+               sh sanity.sh
                $CLEANUP
                $SETUP
        fi
@@ -189,7 +189,7 @@ for NAME in $CONFIGS; do
                mkdir -p $MOUNT2
                mount_client $MOUNT2
                #echo "can't mount2 for '$NAME', skipping sanityN.sh"
-               SANITYLOG=$TMP/sanity.log START=: CLEAN=: sh sanityN.sh
+               START=: CLEAN=: sh sanityN.sh
                umount $MOUNT2
 
                $DEBUG_ON
@@ -219,7 +219,10 @@ for NAME in $CONFIGS; do
                $CLEANUP
                unload_modules
                # Liblustre needs accept=all, noacl
-               LNETOPTS="accept=all" MDS_MOUNT_OPTS="${MDS_MOUNT_OPTS},noacl" $SETUP
+               [ -f /etc/modprobe.conf ] && MODPROBECONF=/etc/modprobe.conf
+               [ -f /etc/modprobe.d/Lustre ] && MODPROBECONF=/etc/modprobe.d/Lustre
+
+               LNETOPTS="$(awk '/^options lnet/ { print $0}' $MODPROBECONF | sed 's/^options lnet //g') accept=all" MDS_MOUNT_OPTS="${MDS_MOUNT_OPTS},noacl" $SETUP
                export LIBLUSTRE_MOUNT_POINT=$MOUNT2
                export LIBLUSTRE_MOUNT_TARGET=$MGSNID:/$FSNAME
                export LIBLUSTRE_TIMEOUT=`cat /proc/sys/lustre/timeout`

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 6118cc8..46f36ed 100644 (file)
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -602,7 +602,7 @@ test_15() {
                echo "save $MOUNTLUSTRE to $MOUNTLUSTRE.sav"
                mv $MOUNTLUSTRE $MOUNTLUSTRE.sav && trap cleanup_15 EXIT INT
                if [ -f $MOUNTLUSTRE ]; then
-                       echo "$MOUNTLUSTRE cannot be moved, skipping test"
+                       skip "$MOUNTLUSTRE cannot be moved, skipping test"
                        return 0
                fi
        fi
@@ -980,7 +980,7 @@ test_28() {
 run_test 28 "permanent parameter setting"
 
 test_29() {
-       [ "$OSTCOUNT" -lt "2" ] && echo "skipping deactivate test" && return
+       [ "$OSTCOUNT" -lt "2" ] && skip "$OSTCOUNT < 2, skipping" && return
         setup > /dev/null 2>&1
        start_ost2
        sleep 10
@@ -1070,9 +1070,9 @@ test_32a() {
         #       there appears to be a lot of assumption here about loopback
         #       devices
         # or maybe this test is just totally useless on a client-only system
-        [ -z "$TUNEFS" ] && echo "No tunefs" && return
-        [ ! -r disk1_4.zip ] && echo "Cant find disk1_4.zip, skipping" && return
-       unzip -o -j -d $TMP/$tdir disk1_4.zip || { echo "Cant unzip disk1_4, skipping" && return ; }
+        [ -z "$TUNEFS" ] && skip "No tunefs" && return
+        [ ! -r disk1_4.zip ] && skip "Cant find disk1_4.zip, skipping" && return
+       unzip -o -j -d $TMP/$tdir disk1_4.zip || { skip "Cant unzip disk1_4, skipping" && return ; }
        load_modules
        sysctl lnet.debug=$PTLDEBUG
 
@@ -1126,9 +1126,9 @@ test_32b() {
         #       there appears to be a lot of assumption here about loopback
         #       devices
         # or maybe this test is just totally useless on a client-only system
-        [ -z "$TUNEFS" ] && echo "No tunefs" && return
-        [ ! -r disk1_4.zip ] && echo "Cant find disk1_4.zip, skipping" && return
-       unzip -o -j -d $TMP/$tdir disk1_4.zip || { echo "Cant unzip disk1_4, skipping" && return ; }
+        [ -z "$TUNEFS" ] && skip "No tunefs" && return
+        [ ! -r disk1_4.zip ] && skip "Cant find disk1_4.zip, skipping" && return
+       unzip -o -j -d $TMP/$tdir disk1_4.zip || { skip "Cant unzip disk1_4, skipping" && return ; }
        load_modules
        sysctl lnet.debug=$PTLDEBUG
 
@@ -1197,5 +1197,5 @@ umount_client $MOUNT
 cleanup_nocli
 cleanup_krb5_env
 
-equals_msg "Done"
-echo "$0: completed"
+equals_msg `basename $0`: test complete
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true

diff --git a/lustre/tests/insanity.sh b/lustre/tests/insanity.sh
index 47b2933..6e37027 100755 (executable)
--- a/lustre/tests/insanity.sh
+++ b/lustre/tests/insanity.sh
@@ -571,5 +571,6 @@ test_10() {
 }
 run_test 10 "Running Availability for 6 hours..."
 
-equals_msg "Done, cleaning up"
+equals_msg `basename $0`: test complete, cleaning up
 $CLEANUP
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true

diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 0cdca40..4a2b57a 100755 (executable)
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -228,7 +228,7 @@ test_17() {
 run_test 17 "timeout bulk get, don't evict client (2732)"
 
 test_18a() {
-    [ -z ${ost2_svc} ] && echo Skipping, needs 2 osts && return 0
+    [ -z ${ost2_svc} ] && skip "needs 2 osts" && return 0
 
     do_facet client mkdir -p $MOUNT/$tdir
     f=$MOUNT/$tdir/$tfile
@@ -578,10 +578,8 @@ run_test 24 "fsync error (should return error)"
 
 test_26() {      # bug 5921 - evict dead exports by pinger
 # this test can only run from a client on a separate node.
-       [ "`lsmod | grep obdfilter`" ] && \
-           echo "skipping test 26 (local OST)" && return
-       [ "`lsmod | grep mds`" ] && \
-           echo "skipping test 26 (local MDS)" && return
+       remote_ost || skip "local OST" && return
+       remote_mds || skip "local MDS" && return
        OST_FILE=$LPROC/obdfilter/${ost1_svc}/num_exports
         OST_EXP="`do_facet ost1 cat $OST_FILE`"
        OST_NEXP1=`echo $OST_EXP | cut -d' ' -f2`
@@ -626,8 +624,7 @@ test_26b() {      # bug 10140 - evict dead exports by pinger
 run_test 26b "evict dead exports"
 
 test_27() {
-       [ "`lsmod | grep mds`" ] || \
-           { echo "skipping test 27 (non-local MDS)" && return 0; }
+       remote_mds && { skip "remote MDS" && return 0; }
        mkdir -p $DIR/$tdir
        writemany -q -a $DIR/$tdir/$tfile 0 5 &
        CLIENT_PID=$!
@@ -773,8 +770,7 @@ run_test 54 "back in time"
 
 # bug 11330 - liblustre application death during I/O locks up OST
 test_55() {
-       [ "`lsmod | grep obdfilter`" ] || \
-           { echo "skipping test 55 (non-local OST)" && return 0; }    
+       remote_ost && { skip "remote OST" && return 0; }
 
        mkdir -p $DIR/$tdir
 
@@ -880,5 +876,6 @@ test_58() { # bug 11546
 }
 run_test 58 "Eviction in the middle of open RPC reply processing"
 
+equals_msg `basename $0`: test complete, cleaning up
 $CLEANUP
-echo "$0: completed"
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true

diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh
index 15ca9e8..7c5a908 100755 (executable)
--- a/lustre/tests/replay-dual.sh
+++ b/lustre/tests/replay-dual.sh
@@ -445,9 +445,10 @@ test_19() { # Bug 10991 - resend of open request does not fail assertion.
 run_test 19 "resend of open request"
 
 if [ "$ONLY" != "setup" ]; then
-   equals_msg test complete, cleaning up
+   equals_msg `basename $0`: test complete, cleaning up
    SLEEP=$((`date +%s` - $NOW))
    [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
    $CLEANUP
 fi
-echo "$0: completed"
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
+

diff --git a/lustre/tests/replay-ost-single.sh b/lustre/tests/replay-ost-single.sh
index 6a3b903..9fe3f77 100755 (executable)
--- a/lustre/tests/replay-ost-single.sh
+++ b/lustre/tests/replay-ost-single.sh
@@ -201,5 +201,6 @@ test_7() {
 }
 run_test 7 "Fail OST before obd_destroy"
 
-equals_msg test complete, cleaning up
+equals_msg `basename $0`: test complete, cleaning up
 $CLEANUP
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true

diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index c969222..7455edc 100755 (executable)
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -930,6 +930,10 @@ run_test 40 "cause recovery in ptlrpc, ensure IO continues"
 # the page, guarnateeing that the unlock from the RPC completion would
 # assert on trying to unlock the unlocked page.
 test_41() {
+    [ $OSTCOUNT -lt 2 ] && \
+       skip "skipping test 41: we don't have a second OST to test with" && \
+       return
+
     local f=$MOUNT/$tfile
     # make sure the start of the file is ost1
     lfs setstripe $f $((128 * 1024)) 0 0 
@@ -1198,3 +1202,4 @@ run_test 60 "test llog post recovery init vs llog unlink"
 
 equals_msg `basename $0`: test complete, cleaning up
 $CLEANUP
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true

diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh
index 92a0080..f491b0e 100644 (file)
--- a/lustre/tests/sanity-quota.sh
+++ b/lustre/tests/sanity-quota.sh
@@ -56,6 +56,9 @@ ITUNE_SZ=${ITUNE_SZ:-5}               # default 50% of IUNIT_SZ
 MAX_DQ_TIME=604800
 MAX_IQ_TIME=604800
 
+QUOTALOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
+FAIL_ON_ERROR=false
+
 log() {
        echo "$*"
        $LCTL mark "$*" 2> /dev/null || true
@@ -121,12 +124,12 @@ run_test() {
        fi
         testname=EXCEPT_$1
         if [ ${!testname}x != x ]; then
-                 echo "skipping excluded test $1"
+                 TESTNAME=test_$1 skip "skipping excluded test $1"
                  return 0
         fi
         testname=EXCEPT_$base
         if [ ${!testname}x != x ]; then
-                 echo "skipping excluded test $1 (base $base)"
+                 TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)"
                  return 0
         fi
         run_one $1 "$2"
@@ -135,20 +138,6 @@ run_test() {
 
 [ "$QUOTALOG" ] && rm -f $QUOTALOG || true
 
-error() { 
-       sysctl -w lustre.fail_loc=0
-       log "FAIL: $TESTNAME $@"
-       if [ "$QUOTALOG" ]; then
-               echo "FAIL: $TESTNAME $@" >> $QUOTALOG
-       else
-               exit 1
-       fi
-}
-
-pass() { 
-       echo PASS $@
-}
-
 mounted_lustre_filesystems() {
        awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts | sed -n ${1}p
 }
@@ -613,7 +602,7 @@ run_test 5 "Chown & chgrp successfully even out of block/file quota ==="
 # block quota acquire & release
 test_6() {
        if [ $OSTCOUNT -lt 2 ]; then
-               echo "WARN: too few osts, skip this test."
+               skip "$OSTCOUNT < 2, too few osts"
                return 0;
        fi
 
@@ -674,10 +663,7 @@ run_test 6 "Block quota acquire & release ========="
 # quota recovery (block quota only by now)
 test_7()
 {
-       if [ -z "`lsmod|grep mds`" ]; then 
-               echo "WARN: no local mds, skip this test"
-               return 0
-       fi
+       remote_mds && skip "remote mds"
 
        LIMIT=$(( $BUNIT_SZ * $(($OSTCOUNT + 1)) * 10)) # 10 bunits each sever
        TESTFILE="$TSTDIR/quota_tst70"
@@ -729,7 +715,7 @@ test_8() {
        FILE_LIMIT=1000000
        DBENCH_LIB=${DBENCH_LIB:-/usr/lib/dbench}
        
-       [ ! -d $DBENCH_LIB ] && echo "dbench not installed, skip this test" && return 0
+       [ ! -d $DBENCH_LIB ] && skip "dbench not installed" && return 0
        
        echo "  Set enough high limit for user: $TSTUSR"
        $LFS setquota -u $TSTUSR 0 $BLK_LIMIT 0 $FILE_LIMIT $MOUNT
@@ -760,7 +746,7 @@ test_9() {
         size_file=$((1024 * 1024 * 9 / 2 * $OSTCOUNT))
         echo "lustrefs_size:$lustrefs_size  size_file:$size_file"
         if [ $lustrefs_size -lt $size_file ]; then
-           echo "WARN: less than $size_file free, skip this test."
+           skip "less than $size_file free"
            return 0;
         fi
 
@@ -808,12 +794,12 @@ test_10() {
        size_file=$((1024 * 1024 * 9 / 2 * $OSTCOUNT))
        echo "lustrefs_size:$lustrefs_size  size_file:$size_file"
        if [ $lustrefs_size -lt $size_file ]; then
-               echo "WARN: less than $size_file free, skip this test."
+               skip "less than $size_file free"
                return 0;
        fi
 
        if [ ! -d /proc/fs/lustre/ost/ -o ! -d /proc/fs/lustre/mds ]; then
-               echo "WARN: mds or ost isn't local, skip this test."
+               skip "mds or ost isn't local"
                return 0;
        fi
 
@@ -1124,5 +1110,5 @@ if [ "`mount | grep ^$NAME`" ]; then
 fi
 
 echo '=========================== finished ==============================='
-[ -f "$QUOTALOG" ] && cat $QUOTALOG && exit 1
+[ -f "$QUOTALOG" ] && cat $QUOTALOG && grep -q FAIL $QUOTALOG && exit 1 || true
 echo "$0: completed"

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 81183a1..c8bfcc5 100644 (file)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -76,8 +76,6 @@ else
        fi
 fi
 
-SANITYLOG=${SANITYLOG:-/tmp/sanity.log}
-
 export NAME=${NAME:-local}
 
 SAVE_PWD=$PWD
@@ -92,6 +90,9 @@ if [ ! -z "$USING_KRB5" ]; then
     $RUNAS -u $(($RUNAS_ID + 1)) krb5_login.sh || exit 1
 fi
 
+SANITYLOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
+FAIL_ON_ERROR=false
+
 cleanup() {
        echo -n "cln.."
        cleanupall ${FORCE} $* || { echo "FAILed to clean up"; exit 20; }
@@ -253,7 +254,6 @@ STRIPECOUNT=`cat $LPROC/lov/$LOVNAME/stripecount`
 STRIPESIZE=`cat $LPROC/lov/$LOVNAME/stripesize`
 ORIGFREE=`cat $LPROC/lov/$LOVNAME/kbytesavail`
 MAXFREE=${MAXFREE:-$((200000 * $OSTCOUNT))}
-MDS=$(\ls $LPROC/mdt 2> /dev/null | grep -v num_refs | tail -n 1)
 
 [ -f $DIR/d52a/foo ] && chattr -a $DIR/d52a/foo
 [ -f $DIR/d52b/foo ] && chattr -i $DIR/d52b/foo
@@ -1070,8 +1070,8 @@ exhaust_all_precreations() {
 }
 
 test_27n() {
-       [ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-               skip "too few OSTs, or remote MDS" && return
+       [ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+       remote_mds && skip "remote MDS" && return
 
        reset_enospc
        rm -f $DIR/d27/f27n
@@ -1084,8 +1084,8 @@ test_27n() {
 run_test 27n "create file with some full OSTs =================="
 
 test_27o() {
-       [ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-               skip "too few OSTs, or remote MDS" && return
+       [ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+       remote_mds && skip "remote MDS" && return
 
        reset_enospc
        rm -f $DIR/d27/f27o
@@ -1100,8 +1100,8 @@ test_27o() {
 run_test 27o "create file with all full OSTs (should error) ===="
 
 test_27p() {
-       [ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-               skip "too few OSTs, or remote MDS" && return
+       [ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+       remote_mds && skip "remote MDS" && return
 
        reset_enospc
        rm -f $DIR/d27/f27p
@@ -1119,8 +1119,8 @@ test_27p() {
 run_test 27p "append to a truncated file with some full OSTs ==="
 
 test_27q() {
-       [ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-               skip "too few OSTs, or remote MDS" && return
+       [ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+       remote_mds && skip "remote MDS" && return
 
        reset_enospc
        rm -f $DIR/d27/f27q
@@ -1139,8 +1139,8 @@ test_27q() {
 run_test 27q "append to truncated file with all OSTs full (should error) ==="
 
 test_27r() {
-       [ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-               skip "too few OSTs, or remote MDS" && return
+       [ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+       remote_mds && skip "remote MDS" && return
 
        reset_enospc
        rm -f $DIR/d27/f27r
@@ -1782,9 +1782,7 @@ test_36f() {
 }
 run_test 36f "utime on file racing with OST BRW write =========="
 
-if [ -d $LPROC/obdfilter ]; then
-export FMD_MAX_AGE=`cat $LPROC/obdfilter/*/client_cache_seconds | head -n 1`
-fi
+export FMD_MAX_AGE=`do_facet ost1 cat $LPROC/obdfilter/*/client_cache_seconds | head -n 1`
 test_36g() {
        [ -z "$FMD_MAX_AGE" ] && skip "skip test for remote OST" && return
        FMD_BEFORE="`awk '/ll_fmd_cache/ { print $2 }' /proc/slabinfo`"
@@ -2367,6 +2365,8 @@ test_52b() {
 run_test 52b "immutable flag test (should return errors) ======="
 
 test_53() {
+       remote_mds && skip "remote MDS" && return
+
        # only test MDT0000 
         for i in `ls -d $LPROC/osc/*-osc-MDT0000 2> /dev/null` ; do
                 ostname=`basename $i | cut -d - -f 1-2`
@@ -2580,6 +2580,8 @@ run_test 56h "check lfs find ! -name ============================="
 
 test_57a() {
        # note test will not do anything if MDS is not local
+       remote_mds && skip "remote MDS" && return
+
        [ -z "$MDS" ] && skip "skipping test for remote MDS" && return
        for DEV in `cat $LPROC/mds/*/mntdev`; do
                dumpe2fs -h $DEV > $TMP/t57a.dump || error "can't access $DEV"
@@ -3949,8 +3951,7 @@ free_min_max () {
 
 test_116() {
        [ "$OSTCOUNT" -lt "2" ] && skip "$OSTCOUNT < 2 OSTs" && return
-       [ $(grep -c obdfilter $LPROC/devices) -eq 0 ] &&
-               skip "remote MDS, skipping test" && return
+       remote_mds && skip "remote MDS" && return
 
        echo -n "Free space priority "
        cat $LPROC/lov/*-clilov-*/qos_prio_free

diff --git a/lustre/tests/sanityN.sh b/lustre/tests/sanityN.sh
index 6d45392..8c1b4e0 100644 (file)
--- a/lustre/tests/sanityN.sh
+++ b/lustre/tests/sanityN.sh
@@ -52,6 +52,9 @@ LUSTRE=${LUSTRE:-`dirname $0`/..}
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 
+SANITYLOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
+FAIL_ON_ERROR=false
+
 if [ ! -z "$USING_KRB5" ]; then
     $RUNAS krb5_login.sh || exit 1
 fi
@@ -533,7 +536,7 @@ test_16() {
 run_test 16 "2500 iterations of dual-mount fsx ================="
 
 test_17() { # bug 3513, 3667
-       [ ! -d /proc/fs/lustre/ost ] && echo "skipping OST-only test" && return
+       [ ! -d /proc/fs/lustre/ost ] && skip "remote OST, skipping OST-only test" && return
 
        cp /etc/termcap $DIR1/f17
        cancel_lru_locks osc > /dev/null
@@ -677,7 +680,8 @@ test_24() {
 run_test 24 "lfs df [-ih] [path] test ========================="
 
 test_25() {
-       [ `cat $LPROC/mdc/*-mdc-*/connect_flags | grep -c acl` -lt 2 ] && echo "skipping $TESTNAME (must have acl)" && return
+       [ `cat $LPROC/mdc/*-mdc-*/connect_flags | grep -c acl` -lt 2 ] && \
+           skip "must have acl, skipping" && return
 
        mkdir $DIR1/$tdir || error "mkdir $DIR1/$tdir"
        touch $DIR1/$tdir/f1 || error "touch $DIR1/$tdir/f1"
@@ -807,6 +811,5 @@ if [ "$I_MOUNTED" = "yes" ]; then
 fi
 
 echo '=========================== finished ==============================='
-[ -f "$SANITYLOG" ] && cat $SANITYLOG && exit 1 || true
+[ -f "$SANITYLOG" ] && cat $SANITYLOG && grep -q FAIL $SANITYLOG && exit 1 || true
 echo "$0: completed"
-

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 5769f4b..fad8246 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -38,6 +38,7 @@ init_test_env() {
 
     [ -d /r ] && export ROOT=${ROOT:-/r}
     export TMP=${TMP:-$ROOT/tmp}
+    export TESTSUITELOG=${TMP}/${TESTSUITE}.log
 
     export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/utils/gss:$LUSTRE/tests
     export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
@@ -82,6 +83,9 @@ init_test_env() {
 
     shift $((OPTIND - 1))
     ONLY=${ONLY:-$*}
+
+    [ "$TESTSUITELOG" ] && rm -f $TESTSUITELOG || true
+
 }
 
 load_module() {
@@ -944,15 +948,29 @@ pgcache_empty() {
     return 0
 }
 
+FAIL_ON_ERROR=true
 ##################################
 # Test interface 
 error() {
+    local ERRLOG
     sysctl -w lustre.fail_loc=0 2> /dev/null || true
-    log "${TESTSUITE}: **** FAIL:" $@
-    $LCTL dk $TMP/lustre-log-$TESTNAME.log
-    log "FAIL: $TESTNAME $@"
-    $LCTL dk $TMP/lustrefail_${TESTSUITE}_${TESTNAME}.$(date +%s)
-    exit 1
+    log "${TESTSUITE} ${TESTNAME}: **** FAIL:" $@
+    ERRLOG=$TMP/lustre_${TESTSUITE}_${TESTNAME}.$(date +%s)
+    echo "Dumping lctl log to $ERRLOG"
+    # We need to dump the logs on all nodes
+    $LCTL dk $ERRLOG
+    [ ! "$mds_HOST" = "$(hostname)" ] && do_node $mds_HOST $LCTL dk $ERRLOG
+    [ ! "$ost_HOST" = "$(hostname)" -a ! "$ost_HOST" = "$mds_HOST" ] && do_node $ost_HOST $LCTL dk $ERRLOG
+    debugrestore
+    [ "$TESTSUITELOG" ] && echo "$0: FAIL: $TESTNAME $@" >> $TESTSUITELOG
+    if $FAIL_ON_ERROR; then
+       exit 1
+    fi
+}
+
+skip () {
+       log " SKIP: ${TESTSUITE} ${TESTNAME} $@"
+       [ "$TESTSUITELOG" ] && echo "${TESTSUITE}: SKIP: $TESTNAME $@" >> $TESTSUITELOG
 }
 
 build_test_filter() {
@@ -993,12 +1011,12 @@ run_test() {
     fi
     testname=EXCEPT_$1
     if [ ${!testname}x != x ]; then
-        log "skipping excluded test $1"
+        TESTNAME=test_$1 skip "skipping excluded test $1"
         return 0
     fi
     testname=EXCEPT_$base
     if [ ${!testname}x != x ]; then
-        log "skipping excluded test $1 (base $base)"
+        TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)"
         return 0
     fi
     run_one $1 "$2"
@@ -1068,3 +1086,13 @@ osc_to_ost()
     fi
     echo $ost
 }
+
+remote_mds ()
+{
+    [ ! -e /proc/fs/lustre/mdt/*MDT* ]
+}
+
+remote_ost ()
+{
+    [ $(grep -c obdfilter $LPROC/devices) -eq 0 ]
+}