From e4cf956f93a4384d19ea73e601a6651710703492 Mon Sep 17 00:00:00 2001
From: Manoj Joseph <manoj.joseph@sun.com>
Date: Wed, 20 Jan 2010 02:06:26 -0700
Subject: [PATCH] b=20057 Autovetting and test-framework enhancements

Test-framework and script changes to support autovetting and buffalo V2

i=rread
i=grev
---
 lustre/tests/Makefile.am                |   8 +-
 lustre/tests/acceptance-small.sh        | 519 ++++++--------------------------
 lustre/tests/conf-sanity.sh             |   6 +-
 lustre/tests/insanity.sh                |   5 +-
 lustre/tests/large-scale.sh             |   1 +
 lustre/tests/lfsck.sh                   | 249 +++++++++++++++
 lustre/tests/lfscktest.sh               | 242 ---------------
 lustre/tests/liblustre.sh               |  40 +++
 lustre/tests/lustre_rsync-test.sh       |   2 +-
 lustre/tests/metadata-updates.sh        |   1 +
 lustre/tests/ost-pools.sh               |   1 +
 lustre/tests/parallel-scale.sh          |   1 +
 lustre/tests/performance-sanity.sh      |   1 +
 lustre/tests/racer.sh                   |   3 +
 lustre/tests/recovery-double-scale.sh   |   1 +
 lustre/tests/recovery-mds-scale.sh      |   1 +
 lustre/tests/recovery-random-scale.sh   |   1 +
 lustre/tests/recovery-small.sh          |   3 +-
 lustre/tests/replay-dual.sh             |   2 +-
 lustre/tests/replay-ost-single.sh       |   3 +-
 lustre/tests/replay-single.sh           |   3 +-
 lustre/tests/rpc.sh                     |   5 +
 lustre/tests/runracer                   |  68 +++--
 lustre/tests/runtests                   |   1 +
 lustre/tests/sanity-benchmark.sh        |   3 +
 lustre/tests/sanity-gss.sh              |   3 +-
 lustre/tests/sanity-quota.sh            |   5 +-
 lustre/tests/sanity-sec.sh              |   5 +-
 lustre/tests/sanity.sh                  |   1 +
 lustre/tests/{sanityN.sh => sanityn.sh} |   5 +
 lustre/tests/test-framework.sh          | 334 ++++++++++++--------
 lustre/tests/yaml.sh                    | 182 +++++++++++
 32 files changed, 874 insertions(+), 831 deletions(-)
 create mode 100644 lustre/tests/lfsck.sh
 delete mode 100755 lustre/tests/lfscktest.sh
 create mode 100644 lustre/tests/liblustre.sh
 create mode 100644 lustre/tests/racer.sh
 rename lustre/tests/{sanityN.sh => sanityn.sh} (99%)
 create mode 100644 lustre/tests/yaml.sh

diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am
index b92e942..0b12c21 100644
--- a/lustre/tests/Makefile.am
+++ b/lustre/tests/Makefile.am
@@ -10,10 +10,10 @@ noinst_DATA = disk1_8.tar.bz2
 noinst_SCRIPTS = leak_finder.pl llmount.sh llmountcleanup.sh functions.sh
 noinst_SCRIPTS += test-framework.sh runvmstat runiozone runtests
 noinst_SCRIPTS += sanity.sh rundbench acceptance-small.sh compile.sh
-noinst_SCRIPTS += conf-sanity.sh insanity.sh lfscktest.sh oos.sh oos2.sh
+noinst_SCRIPTS += conf-sanity.sh insanity.sh lfsck.sh oos.sh oos2.sh
 noinst_SCRIPTS += llog-test.sh recovery-small.sh replay-dual.sh sanity-quota.sh
-noinst_SCRIPTS += replay-ost-single.sh replay-single.sh run-llog.sh sanityN.sh
-noinst_SCRIPTS += large-scale.sh runracer replay-vbr.sh
+noinst_SCRIPTS += replay-ost-single.sh replay-single.sh run-llog.sh sanityn.sh
+noinst_SCRIPTS += large-scale.sh racer.sh runracer replay-vbr.sh
 noinst_SCRIPTS += performance-sanity.sh mdsrate-create-small.sh
 noinst_SCRIPTS += mdsrate-create-large.sh mdsrate-lookup-1dir.sh
 noinst_SCRIPTS += mdsrate-lookup-10dirs.sh sanity-benchmark.sh
@@ -23,7 +23,7 @@ noinst_SCRIPTS += sanity-sec.sh sanity-gss.sh krb5_login.sh setup_kerberos.sh
 noinst_SCRIPTS += recovery-mds-scale.sh run_dd.sh run_tar.sh run_iozone.sh
 noinst_SCRIPTS += run_dbench.sh run_IOR.sh recovery-double-scale.sh
 noinst_SCRIPTS += recovery-random-scale.sh parallel-scale.sh metadata-updates.sh
-noinst_SCRIPTS += lustre_rsync-test.sh ost-pools.sh rpc.sh
+noinst_SCRIPTS += lustre_rsync-test.sh ost-pools.sh rpc.sh yaml.sh liblustre.sh
 nobase_noinst_SCRIPTS = cfg/local.sh
 nobase_noinst_SCRIPTS += acl/make-tree acl/run cfg/ncli.sh
 nobase_noinst_SCRIPTS += racer/dir_create.sh racer/file_create.sh racer/file_list.sh
diff --git a/lustre/tests/acceptance-small.sh b/lustre/tests/acceptance-small.sh
index be833a8..ed7562f 100755
--- a/lustre/tests/acceptance-small.sh
+++ b/lustre/tests/acceptance-small.sh
@@ -2,16 +2,23 @@
 # script which _must_ complete successfully (at minimum) before checkins to
 # the CVS HEAD are allowed.
 #set -vx
-set -e
+set -eTE
 
-[ -z "$CONFIG" -a "$NAME" ] && CONFIGS=$NAME
-[ "$CONFIGS" ] || CONFIGS="local"  #"local lov"
-[ "$MAX_THREADS" ] || MAX_THREADS=20
-RAMKB=`awk '/MemTotal:/ { print $2 }' /proc/meminfo`
-if [ -z "$THREADS" ]; then
-	THREADS=$((RAMKB / 16384))
-	[ $THREADS -gt $MAX_THREADS ] && THREADS=$MAX_THREADS
+export MSKIPPED=0
+export OSKIPPED=0
+
+# This is the default set of tests to run.
+DEFAULT_SUITES="runtests sanity sanity-benchmark sanityn lfsck liblustre
+                runracer replay-single conf-sanity recovery-small
+                replay-ost-single replay-dual insanity sanity-quota sanity-sec
+                sanity-gss performance-sanity large-scale recovery-mds-scale 
+                recovery-double-scale recovery-random-scale parallel-scale 
+                lustre_rsync-test metadata-updates ost-pools"
+
+if [[ -n $@ ]]; then
+    ACC_SM_ONLY="${ACC_SM_ONLY} $@"
 fi
+
 [ "$SIZE" ] || SIZE=$((RAMKB * 2))
 [ "$RSIZE" ] || RSIZE=512
 [ "$UID" ] || UID=1000
@@ -23,28 +30,24 @@ fi
 [ "$DEBUG_OFF" ] || DEBUG_OFF="eval lctl set_param debug=\"$DEBUG_LVL\""
 [ "$DEBUG_ON" ] || DEBUG_ON="eval lctl set_param debug=0x33f0484"
 
-export TESTSUITE_LIST="RUNTESTS SANITY DBENCH BONNIE IOZONE FSX SANITYN LFSCK LIBLUSTRE RACER REPLAY_SINGLE CONF_SANITY RECOVERY_SMALL REPLAY_OST_SINGLE REPLAY_DUAL REPLAY_VBR INSANITY SANITY_QUOTA SANITY_SEC SANITY_GSS PERFORMANCE_SANITY LARGE_SCALE RECOVERY_MDS_SCALE RECOVERY_DOUBLE_SCALE RECOVERY_RANDOM_SCALE PARALLEL_SCALE LUSTRE_RSYNC_TEST METADATA_UPDATES OST_POOLS SANITY_BENCHMARK"
-
 if [ "$ACC_SM_ONLY" ]; then
-    for O in $TESTSUITE_LIST; do
-	export ${O}="no"
+    for O in $DEFAULT_SUITES; do
+        O=$(echo $O | tr "-" "_" | tr "[:lower:]" "[:upper:]")
+        export ${O}="no"
     done
     for O in $ACC_SM_ONLY; do
-	O=`echo ${O%.sh} | tr "-" "_"`
-	O=`echo $O | tr "[:lower:]" "[:upper:]"`
-	export ${O}="yes"
+        O=`echo ${O%.sh} | tr "-" "_"`
+        O=`echo $O | tr "[:lower:]" "[:upper:]"`
+        export ${O}="yes"
     done
 fi
 LFSCK="no" # bug 13698
 
-LIBLUSTRETESTS=${LIBLUSTRETESTS:-../liblustre/tests}
-
 STARTTIME=`date +%s`
-RANTEST=""
 
 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 . $LUSTRE/tests/test-framework.sh
-init_test_env $@
+init_test_env
 
 if $GSS; then
     # liblustre doesn't support GSS
@@ -79,431 +82,97 @@ setup_if_needed() {
     fi
 }
 
+find_in_path() {
+    target=$1
+    for dir in $(tr : " " <<< $PATH); do
+      if [ -e $dir/$target ]; then
+          echo "$dir/$target found in PATH"
+          return 0
+      fi
+    done
+    return 1
+}
+
 title() {
     # update titlebar if stdin is attaached to an xterm
     if ${UPDATE_TITLEBAR:-false}; then
-	if tty -s; then
-	    case $TERM in 
-		xterm*)
-		    echo -ne "\033]2; acceptance-small: $* \007" >&0
-		    ;;
-	    esac
-	fi
+        if tty -s; then
+            case $TERM in 
+                xterm*)
+                    echo -ne "\033]2; acceptance-small: $* \007" >&0
+                    ;;
+            esac
+        fi
     fi 
     log "-----============= acceptance-small: "$*" ============----- `date`"
-    RANTEST=${RANTEST}$*", "
 }
 
-skip_remost()
-{
-	remote_ost_nodsh && log "SKIP: $1: remote OST with nodsh" && return 0
-	return 1
-}
-
-skip_remmds()
-{
-	remote_mds_nodsh && log "SKIP: $1: remote MDS with nodsh" && return 0
-	return 1
+is_sanity_benchmark() {
+    local benchmarks="dbench bonnie iozone fsx"
+    local suite=$1
+    for b in $benchmarks; do
+        if [ "$b" == "$suite" ]; then
+            return 0
+        fi
+    done
+    return 1
 }
 
-for NAME in $CONFIGS; do
-	export NAME MOUNT START CLEAN
-	. $LUSTRE/tests/cfg/$NAME.sh
-
-	assert_env mds_HOST MDS_MKFS_OPTS 
-	assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
-	assert_env FSNAME MOUNT MOUNT2
-
-	setup_if_needed
-
-	MSKIPPED=0
-	OSKIPPED=0
-	if [ "$RUNTESTS" != "no" ]; then
-	        title runtests
-		bash runtests
-		$CLEANUP
-		$SETUP
-		RUNTESTS="done"
-	fi
-
-	if [ "$SANITY" != "no" ]; then
-	        title sanity
-		MOUNT2="" bash sanity.sh
-		$CLEANUP
-		$SETUP
-		SANITY="done"
-	fi
-
-	which dbench > /dev/null 2>&1 || DBENCH=no
-	if [ "$DBENCH" != "no" ]; then
-	        title dbench
-		DBENCHDIR=$MOUNT/$HOSTNAME
-		mkdir -p $DBENCHDIR
-		SPACE=`df -P $MOUNT | tail -n 1 | awk '{ print $4 }'`
-		DB_THREADS=$((SPACE / 50000))
-		[ $THREADS -lt $DB_THREADS ] && DB_THREADS=$THREADS
-
-		$DEBUG_OFF
-		myUID=$RUNAS_ID
-		myRUNAS=$RUNAS
-		FAIL_ON_ERROR=false check_runas_id_ret $myUID $myUID $myRUNAS || { myRUNAS="" && myUID=$UID; }
-		chown $myUID:$myUID $DBENCHDIR
-		duration=""
-		[ "$SLOW" = "no" ] && duration=" -t 120"
-		if [ "$SLOW" != "no" -o $DB_THREADS -eq 1 ]; then
-			$myRUNAS bash rundbench -D $DBENCHDIR 1 $duration || error "dbench failed!"
-			$DEBUG_ON
-			$CLEANUP
-			$SETUP
-		fi
-		if [ $DB_THREADS -gt 1 ]; then
-			$DEBUG_OFF
-			$myRUNAS bash rundbench -D $DBENCHDIR $DB_THREADS $duration
-			$DEBUG_ON
-			$CLEANUP
-			$SETUP
-		fi
-		rm -rf $DBENCHDIR
-		DBENCH="done"
-	fi
-
-	which bonnie++ > /dev/null 2>&1 || BONNIE=no
-	if [ "$BONNIE" != "no" ]; then
-	        title bonnie
-		BONDIR=$MOUNT/d0.bonnie
-		mkdir -p $BONDIR
-		$LFS setstripe -c -1 $BONDIR
-		sync
-		MIN=`lctl get_param -n osc.*.kbytesavail | sort -n | head -n1`
-		SPACE=$(( OSTCOUNT * MIN ))
-		[ $SPACE -lt $SIZE ] && SIZE=$((SPACE * 3 / 4))
-		log "min OST has ${MIN}kB available, using ${SIZE}kB file size"
-		$DEBUG_OFF
-		myUID=$RUNAS_ID
-		myRUNAS=$RUNAS
-		FAIL_ON_ERROR=false check_runas_id_ret $myUID $myUID $myRUNAS || { myRUNAS="" && myUID=$UID; }
-		chown $myUID:$myUID $BONDIR		
-		$myRUNAS bonnie++ -f -r 0 -s$((SIZE / 1024)) -n 10 -u$myUID:$myUID -d$BONDIR
-		$DEBUG_ON
-		$CLEANUP
-		$SETUP
-		BONNIE="done"
-	fi
-
-	export O_DIRECT
-	[ "$SLOW" = "no" ] && export IOZONE=no # 5 minutes
-
-	which iozone > /dev/null 2>&1 || IOZONE=no
-	if [ "$IOZONE" != "no" ]; then
-	        title iozone
-		IOZDIR=$MOUNT/d0.iozone
-		mkdir -p $IOZDIR
-		$LFS setstripe -c -1 $IOZDIR
-		sync
-		MIN=`lctl get_param -n osc.*.kbytesavail | sort -n | head -n1`
-		SPACE=$(( OSTCOUNT * MIN ))
-		[ $SPACE -lt $SIZE ] && SIZE=$((SPACE * 3 / 4))
-		log "min OST has ${MIN}kB available, using ${SIZE}kB file size"
-		IOZONE_OPTS="-i 0 -i 1 -i 2 -e -+d -r $RSIZE"
-		IOZFILE="$IOZDIR/iozone"
-		IOZLOG=$TMP/iozone.log
-		# $SPACE was calculated with all OSTs
-		$DEBUG_OFF
-		myUID=$RUNAS_ID
-		myRUNAS=$RUNAS
-		FAIL_ON_ERROR=false check_runas_id_ret $myUID $myUID $myRUNAS || { myRUNAS="" && myUID=$UID; }
-		chown $myUID:$myUID $IOZDIR
-		$myRUNAS iozone $IOZONE_OPTS -s $SIZE -f $IOZFILE 2>&1 | tee $IOZLOG
-		tail -1 $IOZLOG | grep -q complete || \
-			{ error "iozone (1) failed" && false; }
-		rm -f $IOZLOG
-		$DEBUG_ON
-		$CLEANUP
-		$SETUP
-
-		# check if O_DIRECT support is implemented in kernel
-		if [ -z "$O_DIRECT" ]; then
-			touch $MOUNT/f.iozone
-			if ! ./directio write $MOUNT/f.iozone 0 1; then
-				log "SKIP iozone DIRECT IO test"
-				O_DIRECT=no
-			fi
-			rm -f $MOUNT/f.iozone
-		fi
-		if [ "$O_DIRECT" != "no" -a "$IOZONE_DIR" != "no" ]; then
-			$DEBUG_OFF
-			$myRUNAS iozone -I $IOZONE_OPTS -s $SIZE -f $IOZFILE.odir 2>&1 | tee $IOZLOG
-			tail -1 $IOZLOG | grep -q complete || \
-				{ error "iozone (2) failed" && false; }
-			rm -f $IOZLOG
-			$DEBUG_ON
-			$CLEANUP
-			$SETUP
-		fi
-
-		SPACE=`df -P $MOUNT | tail -n 1 | awk '{ print $4 }'`
-		IOZ_THREADS=$((SPACE / SIZE * 2 / 3 ))
-		[ $THREADS -lt $IOZ_THREADS ] && IOZ_THREADS=$THREADS
-		IOZVER=`iozone -v | awk '/Revision:/ {print $3}' | tr -d .`
-		if [ "$IOZ_THREADS" -gt 1 -a "$IOZVER" -ge 3145 ]; then
-			$LFS setstripe -c -1 $IOZDIR
-			$DEBUG_OFF
-			THREAD=1
-			IOZFILE=" "
-			while [ $THREAD -le $IOZ_THREADS ]; do
-				IOZFILE="$IOZFILE $IOZDIR/iozone.$THREAD"
-				THREAD=$((THREAD + 1))
-			done
-			$myRUNAS iozone $IOZONE_OPTS -s $((SIZE / IOZ_THREADS)) -t $IOZ_THREADS -F $IOZFILE 2>&1 | tee $IOZLOG
-			tail -1 $IOZLOG | grep -q complete || \
-				{ error "iozone (3) failed" && false; }
-			rm -f $IOZLOG
-			$DEBUG_ON
-			$CLEANUP
-			$SETUP
-		elif [ $IOZVER -lt 3145 ]; then
-			VER=`iozone -v | awk '/Revision:/ { print $3 }'`
-			echo "iozone $VER too old for multi-thread test"
-		fi
-		IOZONE="done"
-	fi
-
-	if [ "$FSX" != "no" ]; then
-	        title fsx
-		FSX_SIZE=$((RAMKB / 2))
-		SPACE=`df -P $MOUNT | tail -n 1 | awk '{ print $4 }'`
-		[ $SPACE -lt $FSX_SIZE ] && FSX_SIZE=$((SPACE * 3 / 4))
-		$DEBUG_OFF
-		FSX_SEED=${FSX_SEED:-$RANDOM}
-		rm -f $MOUNT/fsxfile
-		$LFS setstripe -c -1 $MOUNT/fsxfile
-		echo Using FSX_SEED=$FSX_SEED FSX_SIZE=$FSX_SIZE COUNT=$COUNT
-		./fsx -c 50 -p 1000 -S $FSX_SEED -P $TMP -l $FSX_SIZE \
-			-N $(($COUNT * 100)) $MOUNT/fsxfile
-		$DEBUG_ON
-		$CLEANUP
-		$SETUP
-		FSX="done"
-	fi	
-
-	if [ "$SANITYN" != "no" ]; then
-	        title sanityN
-		bash sanityN.sh
-		$CLEANUP
-		$SETUP
-		SANITYN="done"
-	fi
-
-	[ "$LFSCK" != "no" ] && remote_mds && log "Remote MDS, skipping LFSCK test" && LFSCK=no && MSKIPPED=1
-	[ "$LFSCK" != "no" ] && remote_ost && log "Remote OST, skipping LFSCK test" && LFSCK=no && OSKIPPED=1
-	if [ "$LFSCK" != "no" ]; then
-	        title lfsck
-		if [ -x /usr/sbin/lfsck ]; then
-			bash lfscktest.sh
-		else
-			log "$($E2FSCK -V)"
-			log "SKIP: $E2FSCK does not support lfsck"
-		fi
-		LFSCK="done"
-	fi
-
-	[ "$NETTYPE" = "tcp" -o "$NETTYPE" = "ptl" ] || LIBLUSTRE=no # bug 15660
-	if [ "$LIBLUSTRE" != "no" ] && ! check_versions ; then
-		skip liblustre version mismatch: cli $CLIVER, mds $MDSVER, ost $OSTVER
-		LIBLUSTRE=no	# bug 17696
-	fi
-	if [ "$LIBLUSTRE" != "no" ]; then
-		title liblustre
-		assert_env MGSNID MOUNT2
-		export LIBLUSTRE_MOUNT_POINT=$MOUNT2
-		export LIBLUSTRE_MOUNT_RETRY=5
-		export LIBLUSTRE_MOUNT_TARGET=$MGSNID:/$FSNAME
-		export LIBLUSTRE_TIMEOUT=`lctl get_param -n timeout`
-		#export LIBLUSTRE_DEBUG_MASK=`lctl get_param -n debug`
-		if [ -x $LIBLUSTRETESTS/sanity ]; then
-			mkdir -p $MOUNT2
-			if [ "$LIBLUSTRE_EXCEPT" ]; then
-				LIBLUSTRE_OPT="$LIBLUSTRE_OPT \
-					$(echo ' '$LIBLUSTRE_EXCEPT  | sed -re 's/\s+/ -e /g')"
-			fi
-			echo $LIBLUSTRETESTS/sanity --target=$LIBLUSTRE_MOUNT_TARGET $LIBLUSTRE_OPT
-			$LIBLUSTRETESTS/sanity --target=$LIBLUSTRE_MOUNT_TARGET $LIBLUSTRE_OPT
-		fi
-		$CLEANUP
-		#$SETUP
-		LIBLUSTRE="done"
-	fi
-
-	[ "$RACER" != "no" ] && [ -n "$CLIENTS" -a "$PDSH" = "no_dsh" ] && log "Remote client with no_dsh" && RACER=no 
-	if [ "$RACER" != "no" ]; then
-        	title racer
-		setup_if_needed
-		DURATION=${DURATION:-900}
-		[ "$SLOW" = "no" ] && DURATION=300
-		RACERCLIENTS=$HOSTNAME
-		[ ! -z ${CLIENTS} ] && RACERCLIENTS=$CLIENTS
-		log "racer on clients: $RACERCLIENTS DURATION=$DURATION RACERDIRS=$RACERDIRS"
-		CLIENTS=${RACERCLIENTS} DURATION=$DURATION bash runracer $RACERDIRS
-		$CLEANUP
-		$SETUP
-		RACER="done"
-	fi
-done
-
-if [ "$SANITY_BENCHMARK" != "no" ]; then
-        title sanity-benchmark
-	bash sanity-benchmark.sh
-	SANITY_BENCHMARK="done"
-fi
-
-[ "$REPLAY_SINGLE" != "no" ] && skip_remmds replay-single && REPLAY_SINGLE=no && MSKIPPED=1
-if [ "$REPLAY_SINGLE" != "no" ]; then
-        title replay-single
-	bash replay-single.sh
-	REPLAY_SINGLE="done"
-fi
-
-[ "$CONF_SANITY" != "no" ] && skip_remmds conf-sanity && CONF_SANITY=no && MSKIPPED=1
-[ "$CONF_SANITY" != "no" ] && skip_remost conf-sanity && CONF_SANITY=no && OSKIPPED=1
-if [ "$CONF_SANITY" != "no" ]; then
-        title conf-sanity
-        bash conf-sanity.sh
-        CONF_SANITY="done"
-fi
-
-[ "$RECOVERY_SMALL" != "no" ] && skip_remmds recover-small && RECOVERY_SMALL=no && MSKIPPED=1
-if [ "$RECOVERY_SMALL" != "no" ]; then
-        title recovery-small
-        bash recovery-small.sh
-        RECOVERY_SMALL="done"
-fi
-
-[ "$REPLAY_OST_SINGLE" != "no" ] && skip_remost replay-ost-single && REPLAY_OST_SINGLE=no && OSKIPPED=1
-if [ "$REPLAY_OST_SINGLE" != "no" ]; then
-        title replay-ost-single
-        bash replay-ost-single.sh
-        REPLAY_OST_SINGLE="done"
-fi
-
-[ "$REPLAY_DUAL" != "no" ] && skip_remost replay-dual && REPLAY_DUAL=no && OSKIPPED=1
-if [ "$REPLAY_DUAL" != "no" ]; then
-        title replay-dual
-        bash replay-dual.sh
-        REPLAY_DUAL="done"
-fi
-
-[ "$REPLAY_VBR" != "no" ] && skip_remmds replay-vbr && REPLAY_VBR=no && MSKIPPED=1
-if [ "$REPLAY_VBR" != "no" ]; then
-        title replay-vbr
-        bash replay-vbr.sh
-        REPLAY_VBR="done"
-fi
-
-[ "$INSANITY" != "no" ] && skip_remmds insanity && INSANITY=no && MSKIPPED=1
-[ "$INSANITY" != "no" ] && skip_remost insanity && INSANITY=no && OSKIPPED=1
-if [ "$INSANITY" != "no" ]; then
-        title insanity
-        bash insanity.sh -r
-        INSANITY="done"
-fi
-
-[ "$SANITY_QUOTA" != "no" ] && skip_remmds sanity-quota && SANITY_QUOTA=no && MSKIPPED=1
-[ "$SANITY_QUOTA" != "no" ] && skip_remost sanity-quota && SANITY_QUOTA=no && OSKIPPED=1
-if [ "$SANITY_QUOTA" != "no" ]; then
-        title sanity-quota
-        bash sanity-quota.sh
-        SANITY_QUOTA="done"
-fi
-
-[ "$SANITY_SEC" != "no" ] && skip_remmds sanity-sec && SANITY_SEC=no && MSKIPPED=1
-[ "$SANITY_SEC" != "no" ] && skip_remost sanity-sec && SANITY_SEC=no && OSKIPPED=1
-if [ "$SANITY_SEC" != "no" ]; then
-        title sanity-sec
-        bash sanity-sec.sh
-        SANITY_SEC="done"
-fi
-
-[ "$SANITY_GSS" != "no" ] && skip_remmds sanity-gss && SANITY_GSS=no && MSKIPPED=1
-if [ "$SANITY_GSS" != "no" ]; then
-        title sanity-gss
-        bash sanity-gss.sh
-        SANITY_GSS="done"
-fi
-
-
-[ "$LUSTRE_RSYNC_TEST" != "no" ] && skip_remmds lustre_rsync-test && LUSTRE_RSYNC_TEST=no && MSKIPPED=1
-[ "$LUSTRE_RSYNC_TEST" != "no" ] && skip_remost lustre_rsync-test && LUSTRE_RSYNC_TEST=no && OSKIPPED=1
-if [ "$LUSTRE_RSYNC_TEST" != "no" ]; then
-        title lustre_rsync-test
-        bash lustre_rsync-test.sh
-        LUSTRE_RSYNC_TEST="done"
-fi
-
-[ "$OST_POOLS" != "no" ] && skip_remmds ost-pools && OST_POOLS=no && MSKIPPED=1
-[ "$OST_POOLS" != "no" ] && skip_remost ost-pools && OST_POOLS=no && OSKIPPED=1
-if [ "$OST_POOLS" != "no" ]; then
-        title ost-pools
-        bash ost-pools.sh
-        OST_POOLS="done"
-fi
-
-
-[ "$SLOW" = no ] && PERFORMANCE_SANITY="no"
-[ -x "$MDSRATE" ] || PERFORMANCE_SANITY="no"
-which mpirun > /dev/null 2>&1 || PERFORMANCE_SANITY="no"
-if [ "$PERFORMANCE_SANITY" != "no" ]; then
-        title performance-sanity
-        bash performance-sanity.sh
-        PERFORMANCE_SANITY="done"
-fi
+run_suite() {
+    local suite_name=$(echo ${1%.sh} | tr "[:upper:]_" "[:lower:]-" )
+    local suite=$(echo ${suite_name} | tr "[:lower:]-" "[:upper:]_")
+    local suite_only=ONLY # Change to ${suite}_ONLY after fixing YALA
+
+    if is_sanity_benchmark ${suite_name}; then
+        suite_only=suite_name
+        suite_script=$LUSTRE/tests/sanity-benchmark.sh
+    elif [ -e $LUSTRE/tests/${suite_name}.sh ]; then
+        suite_script=$LUSTRE/tests/${suite_name}.sh
+    elif [ -e $LUSTRE/tests/$suite_name ]; then
+        suite_script=$LUSTRE/tests/$suite_name
+    elif find_in_path $suite_name; then
+        suite_script=${suite_name}
+    elif find_in_path ${suite_name}.sh; then
+        suite_script=${suite_name}.sh
+    else
+        echo "Can't find test script for $suite_name"
+        return 1
+    fi
 
-[ "$LARGE_SCALE" != "no" ] && skip_remmds large-scale && LARGE_SCALE=no && MSKIPPED=1
-if [ "$LARGE_SCALE" != "no" ]; then
-        title large-scale
-        bash large-scale.sh
-        LARGE_SCALE="done"
-fi
+    echo "$suite_script located."
+    if [[ ${!suite} != no ]]; then
+        title $suite_name
+        log_test $suite_name
+        bash $suite_script ${!suite_only}
+        $CLEANUP
+        $SETUP
+        eval ${suite}="done"
+    else
+        echo "Skipping $suite_name"
+    fi
+}
 
-[ "$RECOVERY_MDS_SCALE" != "no" ] && skip_remmds recovery-mds-scale && RECOVERY_MDS_SCALE=no && MSKIPPED=1
-[ "$RECOVERY_MDS_SCALE" != "no" ] && skip_remost recovery-mds-scale && RECOVERY_MDS_SCALE=no && OSKIPPED=1
-if [ "$RECOVERY_MDS_SCALE" != "no" ]; then
-        title recovery-mds-scale
-        bash recovery-mds-scale.sh
-        RECOVERY_MDS_SCALE="done"
-fi
+run_suites() {
+    for suite in $*; do
+        run_suite $suite
+    done
+}
 
-[ "$RECOVERY_DOUBLE_SCALE" != "no" ] && skip_remmds recovery-double-scale && RECOVERY_DOUBLE_SCALE=no && MSKIPPED=1
-[ "$RECOVERY_DOUBLE_SCALE" != "no" ] && skip_remost recovery-double-scale && RECOVERY_DOUBLE_SCALE=no && OSKIPPED=1
-if [ "$RECOVERY_DOUBLE_SCALE" != "no" ]; then
-        title recovery-double-scale
-        bash recovery-double-scale.sh
-        RECOVERY_DOUBLE_SCALE="done"
-fi
+export NAME MOUNT START CLEAN
+. $LUSTRE/tests/cfg/$NAME.sh
 
-[ "$RECOVERY_RANDOM_SCALE" != "no" ] && skip_remmds recovery-random-scale && RECOVERY_RANDOM_SCALE=no && MSKIPPED=1
-if [ "$RECOVERY_RANDOM_SCALE" != "no" ]; then
-        title recovery-random-scale
-        bash recovery-random-scale.sh
-        RECOVERY_RANDOM_SCALE="done"
-fi
+assert_env mds_HOST MDS_MKFS_OPTS 
+assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
+assert_env FSNAME MOUNT MOUNT2
 
-which mpirun > /dev/null 2>&1 || PARALLEL_SCALE="no"
-if [ "$PARALLEL_SCALE" != "no" ]; then
-        title parallel-scale
-        bash parallel-scale.sh
-        PARALLEL_SCALE="done"
-fi
+setup_if_needed
+init_logging
 
-if [ "$METADATA_UPDATES" != "no" ]; then
-        title metadata-updates
-        bash metadata-updates.sh
-        METADATA_UPDATES="done"
-fi
+run_suites ${ACC_SM_ONLY:-$DEFAULT_SUITES}
 
 RC=$?
 title FINISHED
 echo "Finished at `date` in $((`date +%s` - $STARTTIME))s"
-echo "Tests ran: $RANTEST"
 print_summary
 [ "$MSKIPPED" = 1 ] && log "FAIL: remote MDS tests skipped" && RC=1
 [ "$OSKIPPED" = 1 ] && log "FAIL: remote OST tests skipped" && RC=1
diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 303d37c..005a2e5 100644
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -26,6 +26,7 @@ RLUSTRE=${RLUSTRE:-$LUSTRE}
 
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
+init_logging
 # STORED_MDSSIZE is used in test_18
 if [ -n "$MDSSIZE" ]; then
     STORED_MDSSIZE=$MDSSIZE
@@ -35,10 +36,9 @@ MDSSIZE=40000
 OSTSIZE=40000
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 
-remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
-remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
+require_dsh_mds || exit 0
+require_dsh_ost || exit 0
 
-#
 [ "$SLOW" = "no" ] && EXCEPT_SLOW="0 1 2 3 6 7 15 18 24b 25 30 31 32 33 34a 45"
 
 assert_DIR
diff --git a/lustre/tests/insanity.sh b/lustre/tests/insanity.sh
index e28d33e..497887f 100755
--- a/lustre/tests/insanity.sh
+++ b/lustre/tests/insanity.sh
@@ -9,6 +9,7 @@ LUSTRE=${LUSTRE:-`dirname $0`/..}
 init_test_env $@
 
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 #              
 ALWAYS_EXCEPT="10  $INSANITY_EXCEPT"
 
@@ -35,8 +36,8 @@ assert_env mds_HOST MDS_MKFS_OPTS
 assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
 assert_env LIVE_CLIENT FSNAME
 
-remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
-remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
+require_dsh_mds || exit 0
+require_dsh_ost || exit 0
 
 # FAIL_CLIENTS list should not contain the LIVE_CLIENT
 FAIL_CLIENTS=$(echo " $FAIL_CLIENTS " | sed -re "s/\s+$LIVE_CLIENT\s+/ /g")
diff --git a/lustre/tests/large-scale.sh b/lustre/tests/large-scale.sh
index 4fba840..9614290 100644
--- a/lustre/tests/large-scale.sh
+++ b/lustre/tests/large-scale.sh
@@ -15,6 +15,7 @@ CLEANUP=${CLEANUP:-""}
 init_test_env $@
 
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 remote_mds_nodsh && log "SKIP: remote MDS with nodsh" && exit 0
 
diff --git a/lustre/tests/lfsck.sh b/lustre/tests/lfsck.sh
new file mode 100644
index 0000000..2e48bad
--- /dev/null
+++ b/lustre/tests/lfsck.sh
@@ -0,0 +1,249 @@
+#!/bin/bash
+#set -vx
+set -e
+
+TESTNAME="lfsck"
+TMP=${TMP:-/tmp}
+MDSDB=${MDSDB:-$TMP/mdsdb}
+OSTDB=${OSTDB:-$TMP/ostdb}
+LOG=${LOG:-"$TMP/lfsck.log"}
+L2FSCK_PATH=${L2FSCK_PATH:-""}
+NUMFILES=${NUMFILES:-10}
+NUMDIRS=${NUMDIRS:-4}
+LFIND=${LFIND:-"lfs find"}
+GETFATTR=${GETFATTR:-getfattr}
+SETFATTR=${SETFATTR:-setfattr}
+MAX_ERR=1
+
+export PATH=$LFSCK_PATH:`dirname $0`:`dirname $0`/../utils:$PATH
+
+[ -z "`which $GETFATTR`" ] && echo "$0: $GETFATTR not found" && exit 5
+[ -z "`which $SETFATTR`" ] && echo "$0: $SETFATTR not found" && exit 6
+
+LUSTRE=${LUSTRE:-`dirname $0`/..}
+. $LUSTRE/tests/test-framework.sh
+init_test_env $@
+. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
+
+require_dsh_mds || exit 0
+require_dsh_ost || exit 0
+
+if [ ! -x /usr/sbin/lfsck ]; then
+    log "$($E2FSCK -V)"
+    log "SKIP: $E2FSCK does not support lfsck"
+    exit 0
+fi
+
+# if nothing mounted, don't nuke MOUNT variable needed in llmount.sh
+WAS_MOUNTED=$(mounted_lustre_filesystems | head -1)
+if [ -z "$WAS_MOUNTED" ]; then
+       # This code doesn't handle multiple mounts well, so nuke MOUNT2 variable
+        MOUNT2="" sh llmount.sh
+        MOUNT=$(mounted_lustre_filesystems)
+        [ -z "$MOUNT" ] && echo "NAME=$NAME not mounted" && exit 2
+else
+        MOUNT=${WAS_MOUNTED}
+fi
+
+DIR=${DIR:-$MOUNT/$TESTNAME}
+[ -z "`echo $DIR | grep $MOUNT`" ] && echo "$DIR not in $MOUNT" && exit 3
+
+if [ "$WAS_MOUNTED" ]; then
+        LFSCK_SETUP=no
+        MAX_ERR=4               # max expected error from e2fsck
+fi
+
+get_mnt_devs() {
+        DEVS=`lctl get_param -n $1.*.mntdev`
+        for DEV in $DEVS; do
+                case $DEV in
+                *loop*) losetup $DEV | sed -e "s/.*(//" -e "s/).*//" ;;
+                *) echo $DEV ;;
+                esac
+        done
+}
+
+if [ "$LFSCK_SETUP" != "no" ]; then
+        #Create test directory 
+        # -- can't remove the mountpoint...
+        [ -z "$DIR" ] && rm -rf $DIR/*
+        mkdir -p $DIR
+        OSTCOUNT=`$LFIND $MOUNT | grep -c "^[0-9]*: "`
+
+        # Create some files on the filesystem
+        for d in `seq -f d%g $NUMDIRS`; do
+                echo "creating files in $DIR/$d"
+                for e in `seq -f d%g $NUMDIRS`; do
+                        mkdir -p  $DIR/$d/$e
+                        for f in `seq -f test%g $NUMDIRS`; do
+                                cp /etc/fstab $DIR/$d/$e/$f ||exit 5
+                        done
+                done
+        done
+
+        # Create Files to be modified
+        for f in `seq -f $DIR/testfile.%g $((NUMFILES * 3))`; do
+                echo "creating $f"
+                cp /etc/termcap $f || exit 10
+        done
+
+        #Create some more files
+        for d in `seq -f d%g $((NUMDIRS * 2 + 1)) $((NUMDIRS * 2 + 3))`; do
+                echo "creating files in $DIR/$d"
+                for e in `seq -f d%g $NUMDIRS`; do
+                        mkdir -p  $DIR/$d/$e
+                        for f in `seq -f test%g $NUMDIRS`; do
+                                cp /etc/hosts $DIR/$d/$e/$f ||exit 15
+                        done
+                done
+        done
+
+        # these should NOT be taken as duplicates
+        for f in `seq -f $DIR/$d/linkfile.%g $NUMFILES`; do
+                echo "linking files in $DIR/$d"
+                cp /etc/hosts $f
+                ln $f $f.link
+        done
+
+        # Get objids for a file on the OST
+        OST_FILES=`seq -f $DIR/testfile.%g $NUMFILES`
+        OST_REMOVE=`$LFIND $OST_FILES | awk '$1 == 0 { print $2 }' | head -n $NUMFILES`
+
+        export MDS_DUPE=""
+        for f in `seq -f testfile.%g $((NUMFILES + 1)) $((NUMFILES * 2))`; do
+                TEST_FILE=$DIR/$f
+                echo "DUPLICATING MDS file $TEST_FILE"
+                $LFIND -v $TEST_FILE >> $LOG || exit 20
+                MDS_DUPE="$MDS_DUPE $TEST_FILE"
+        done
+        MDS_DUPE=`echo $MDS_DUPE | sed "s#$MOUNT/##g"`
+
+        export MDS_REMOVE=""
+        for f in `seq -f testfile.%g $((NUMFILES * 2 + 1)) $((NUMFILES * 3))`; do
+                TEST_FILE=$DIR/$f
+                echo "REMOVING MDS file $TEST_FILE which has info:"
+                $LFIND -v $TEST_FILE >> $LOG || exit 30
+                MDS_REMOVE="$MDS_REMOVE $TEST_FILE"
+        done
+        MDS_REMOVE=`echo $MDS_REMOVE | sed "s#$MOUNT/##g"`
+
+        # when the OST is also using an OSD this needs to be fixed
+        MDTDEVS=`get_mnt_devs osd`
+        OSTDEVS=`get_mnt_devs obdfilter`
+        OSTCOUNT=`echo $OSTDEVS | wc -w`
+        sh llmountcleanup.sh || exit 40
+
+        # Remove objects associated with files
+        echo "removing objects: `echo $OST_REMOVE`"
+        DEBUGTMP=`mktemp $TMP/debugfs.XXXXXXXXXX`
+        for i in $OST_REMOVE; do
+                echo "rm O/0/d$((i % 32))/$i" >> $DEBUGTMP
+        done
+        $DEBUGFS -w -f $DEBUGTMP `echo $OSTDEVS | cut -d' ' -f 1`
+        RET=$?
+        rm $DEBUGTMP
+        [ $RET -ne 0 ] && exit 50
+
+        SAVE_PWD=$PWD
+        mount -t $FSTYPE -o loop $MDSDEV $MOUNT || exit 60
+        do_umount() {
+                trap 0
+                cd $SAVE_PWD
+                umount -f $MOUNT
+        }
+        trap do_umount EXIT
+
+        #Remove files from mds
+        for f in $MDS_REMOVE; do
+                rm $MOUNT/ROOT/$f || exit 70
+        done
+
+        #Create EAs on files so objects are referenced from different files
+        ATTRTMP=`mktemp $TMP/setfattr.XXXXXXXXXX`
+        cd $MOUNT/ROOT || exit 78
+        for f in $MDS_DUPE; do
+                touch $f.bad || exit 74
+                getfattr -n trusted.lov $f | sed "s#$f#&.bad#" > $ATTRTMP
+                setfattr --restore $ATTRTMP || exit 80
+        done
+        cd $SAVE_PWD
+        rm $ATTRTMP
+
+        do_umount
+else
+        # when the OST is also using an OSD this needs to be fixed
+        MDTDEVS=`get_mnt_devs osd`
+        OSTDEVS=`get_mnt_devs obdfilter`
+        OSTCOUNT=`echo $OSTDEVS | wc -w`
+fi # LFSCK_SETUP
+
+# Run e2fsck to get mds and ost info
+# a return status of 1 indicates e2fsck successfuly fixed problems found
+set +e
+
+echo "$E2FSCK -d -v -fn --mdsdb $MDSDB $MDSDEV"
+df > /dev/null  # update statfs data on disk
+$E2FSCK -d -v -fn --mdsdb $MDSDB $MDSDEV
+RET=$?
+[ $RET -gt $MAX_ERR ] && echo "$E2FSCK returned $RET" && exit 90 || true
+
+export OSTDB_LIST=""
+i=0
+for OSTDEV in $OSTDEVS; do
+        df > /dev/null  # update statfs data on disk
+        $E2FSCK -d -v -fn --mdsdb $MDSDB --ostdb $OSTDB-$i $OSTDEV
+        RET=$?
+        [ $RET -gt $MAX_ERR ] && echo "$E2FSCK returned $RET" && exit 100
+        OSTDB_LIST="$OSTDB_LIST $OSTDB-$i"
+        i=$((i + 1))
+done
+
+#Remount filesystem
+[ "`mount | grep $MOUNT`" ] || $SETUP
+
+# need to turn off shell error detection to get proper error return
+# lfsck will return 1 if the filesystem had errors fixed
+echo "LFSCK TEST 1"
+echo "lfsck -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT"
+echo y | lfsck -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT
+RET=$?
+[ $RET -eq 0 ] && echo "clean after first check" && exit 0
+echo "LFSCK TEST 1 - finished with rc=$RET"
+[ $RET -gt $MAX_ERR ] && exit 110 || true
+
+# make sure everything gets to the backing store
+sync; sleep 2; sync
+
+echo "LFSCK TEST 2"
+echo "$E2FSCK -d -v -fn --mdsdb $MDSDB $MDSDEV"
+df > /dev/null  # update statfs data on disk
+$E2FSCK -d -v -fn --mdsdb $MDSDB $MDSDEV
+RET=$?
+[ $RET -gt $MAX_ERR ] && echo "$E2FSCK returned $RET" && exit 123 || true
+
+i=0
+export OSTDB_LIST=""
+for OSTDEV in $OSTDEVS; do
+        df > /dev/null  # update statfs data on disk
+        $E2FSCK -d -v -fn --mdsdb $MDSDB --ostdb $OSTDB-$i $OSTDEV
+        RET=$?
+        [ $RET -gt $MAX_ERR ] && echo "$E2FSCK returned $RET" && exit 124
+        OSTDB_LIST="$OSTDB_LIST $OSTDB-$i"
+        i=$((i + 1))
+done
+
+echo "LFSCK TEST 2"
+echo "lfsck -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT"
+lfsck -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT
+RET=$?
+echo "LFSCK TEST 2 - finished with rc=$RET"
+[ $RET -ne 0 ] && exit 125 || true
+if [ -z "$WAS_MOUNTED" ]; then
+        sh llmountcleanup.sh || exit 120
+fi
+
+#Cleanup 
+rm -f $MDSDB $OSTDB-* || true
+
+echo "$0: completed"
diff --git a/lustre/tests/lfscktest.sh b/lustre/tests/lfscktest.sh
deleted file mode 100755
index af7d794..0000000
--- a/lustre/tests/lfscktest.sh
+++ /dev/null
@@ -1,242 +0,0 @@
-#!/bin/bash
-#set -vx
-set -e
-
-TESTNAME="lfscktest"
-TMP=${TMP:-/tmp}
-MDSDB=${MDSDB:-$TMP/mdsdb}
-OSTDB=${OSTDB:-$TMP/ostdb}
-LOG=${LOG:-"$TMP/lfscktest.log"}
-L2FSCK_PATH=${L2FSCK_PATH:-""}
-NUMFILES=${NUMFILES:-10}
-NUMDIRS=${NUMDIRS:-4}
-LFIND=${LFIND:-"lfs find"}
-GETFATTR=${GETFATTR:-getfattr}
-SETFATTR=${SETFATTR:-setfattr}
-MAX_ERR=1
-
-export PATH=$LFSCK_PATH:`dirname $0`:`dirname $0`/../utils:$PATH
-
-[ -z "`which $GETFATTR`" ] && echo "$0: $GETFATTR not found" && exit 5
-[ -z "`which $SETFATTR`" ] && echo "$0: $SETFATTR not found" && exit 6
-
-LUSTRE=${LUSTRE:-`dirname $0`/..}
-. $LUSTRE/tests/test-framework.sh
-init_test_env $@
-. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
-
-remote_mds && skip "remote MDS" && exit 0
-remote_ost && skip "remote OST" && exit 0
-
-# if nothing mounted, don't nuke MOUNT variable needed in llmount.sh
-WAS_MOUNTED=$(mounted_lustre_filesystems | head -1)
-if [ -z "$WAS_MOUNTED" ]; then
-       # This code doesn't handle multiple mounts well, so nuke MOUNT2 variable
-        MOUNT2="" sh llmount.sh
-        MOUNT=$(mounted_lustre_filesystems)
-        [ -z "$MOUNT" ] && echo "NAME=$NAME not mounted" && exit 2
-else
-        MOUNT=${WAS_MOUNTED}
-fi
-
-DIR=${DIR:-$MOUNT/$TESTNAME}
-[ -z "`echo $DIR | grep $MOUNT`" ] && echo "$DIR not in $MOUNT" && exit 3
-
-if [ "$WAS_MOUNTED" ]; then
-	LFSCK_SETUP=no
-	MAX_ERR=4		# max expected error from e2fsck
-fi
-
-get_mnt_devs() {
-	DEVS=`lctl get_param -n $1.*.mntdev`
-	for DEV in $DEVS; do
-		case $DEV in
-		*loop*) losetup $DEV | sed -e "s/.*(//" -e "s/).*//" ;;
-		*) echo $DEV ;;
-		esac
-	done
-}
-
-if [ "$LFSCK_SETUP" != "no" ]; then
-	#Create test directory 
-	# -- can't remove the mountpoint...
-	[ -z "$DIR" ] && rm -rf $DIR/*
-	mkdir -p $DIR
-	OSTCOUNT=`$LFIND $MOUNT | grep -c "^[0-9]*: "`
-
-	# Create some files on the filesystem
-	for d in `seq -f d%g $NUMDIRS`; do
-		echo "creating files in $DIR/$d"
-		for e in `seq -f d%g $NUMDIRS`; do
-			mkdir -p  $DIR/$d/$e
-			for f in `seq -f test%g $NUMDIRS`; do
-				cp /etc/fstab $DIR/$d/$e/$f ||exit 5
-			done
-		done
-	done
-
-	# Create Files to be modified
-	for f in `seq -f $DIR/testfile.%g $((NUMFILES * 3))`; do
-		echo "creating $f"
-		cp /etc/termcap $f || exit 10
-	done
-
-	#Create some more files
-	for d in `seq -f d%g $((NUMDIRS * 2 + 1)) $((NUMDIRS * 2 + 3))`; do
-		echo "creating files in $DIR/$d"
-		for e in `seq -f d%g $NUMDIRS`; do
-			mkdir -p  $DIR/$d/$e
-			for f in `seq -f test%g $NUMDIRS`; do
-				cp /etc/hosts $DIR/$d/$e/$f ||exit 15
-			done
-		done
-	done
-
-	# these should NOT be taken as duplicates
-	for f in `seq -f $DIR/$d/linkfile.%g $NUMFILES`; do
-		echo "linking files in $DIR/$d"
-		cp /etc/hosts $f
-		ln $f $f.link
-	done
-
-	# Get objids for a file on the OST
-	OST_FILES=`seq -f $DIR/testfile.%g $NUMFILES`
-	OST_REMOVE=`$LFIND $OST_FILES | awk '$1 == 0 { print $2 }' | head -n $NUMFILES`
-
-	export MDS_DUPE=""
-	for f in `seq -f testfile.%g $((NUMFILES + 1)) $((NUMFILES * 2))`; do
-		TEST_FILE=$DIR/$f
-		echo "DUPLICATING MDS file $TEST_FILE"
-		$LFIND -v $TEST_FILE >> $LOG || exit 20
-		MDS_DUPE="$MDS_DUPE $TEST_FILE"
-	done
-	MDS_DUPE=`echo $MDS_DUPE | sed "s#$MOUNT/##g"`
-
-	export MDS_REMOVE=""
-	for f in `seq -f testfile.%g $((NUMFILES * 2 + 1)) $((NUMFILES * 3))`; do
-		TEST_FILE=$DIR/$f
-		echo "REMOVING MDS file $TEST_FILE which has info:"
-		$LFIND -v $TEST_FILE >> $LOG || exit 30
-		MDS_REMOVE="$MDS_REMOVE $TEST_FILE"
-	done
-	MDS_REMOVE=`echo $MDS_REMOVE | sed "s#$MOUNT/##g"`
-
-	# when the OST is also using an OSD this needs to be fixed
-	MDTDEVS=`get_mnt_devs osd`
-	OSTDEVS=`get_mnt_devs obdfilter`
-	OSTCOUNT=`echo $OSTDEVS | wc -w`
-	sh llmountcleanup.sh || exit 40
-
-	# Remove objects associated with files
-	echo "removing objects: `echo $OST_REMOVE`"
-	DEBUGTMP=`mktemp $TMP/debugfs.XXXXXXXXXX`
-	for i in $OST_REMOVE; do
-		echo "rm O/0/d$((i % 32))/$i" >> $DEBUGTMP
-	done
-	$DEBUGFS -w -f $DEBUGTMP `echo $OSTDEVS | cut -d' ' -f 1`
-	RET=$?
-	rm $DEBUGTMP
-	[ $RET -ne 0 ] && exit 50
-
-	SAVE_PWD=$PWD
-	mount -t $FSTYPE -o loop $MDSDEV $MOUNT || exit 60
-	do_umount() {
-		trap 0
-		cd $SAVE_PWD
-		umount -f $MOUNT
-	}
-	trap do_umount EXIT
-
-	#Remove files from mds
-	for f in $MDS_REMOVE; do
-		rm $MOUNT/ROOT/$f || exit 70
-	done
-
-	#Create EAs on files so objects are referenced from different files
-	ATTRTMP=`mktemp $TMP/setfattr.XXXXXXXXXX`
-	cd $MOUNT/ROOT || exit 78
-	for f in $MDS_DUPE; do
-		touch $f.bad || exit 74
-		getfattr -n trusted.lov $f | sed "s#$f#&.bad#" > $ATTRTMP
-		setfattr --restore $ATTRTMP || exit 80
-	done
-	cd $SAVE_PWD
-	rm $ATTRTMP
-
-	do_umount
-else
-	# when the OST is also using an OSD this needs to be fixed
-	MDTDEVS=`get_mnt_devs osd`
-	OSTDEVS=`get_mnt_devs obdfilter`
-	OSTCOUNT=`echo $OSTDEVS | wc -w`
-fi # LFSCK_SETUP
-
-# Run e2fsck to get mds and ost info
-# a return status of 1 indicates e2fsck successfuly fixed problems found
-set +e
-
-echo "$E2FSCK -d -v -fn --mdsdb $MDSDB $MDSDEV"
-df > /dev/null	# update statfs data on disk
-$E2FSCK -d -v -fn --mdsdb $MDSDB $MDSDEV
-RET=$?
-[ $RET -gt $MAX_ERR ] && echo "$E2FSCK returned $RET" && exit 90 || true
-
-export OSTDB_LIST=""
-i=0
-for OSTDEV in $OSTDEVS; do
-	df > /dev/null	# update statfs data on disk
-	$E2FSCK -d -v -fn --mdsdb $MDSDB --ostdb $OSTDB-$i $OSTDEV
-	RET=$?
-	[ $RET -gt $MAX_ERR ] && echo "$E2FSCK returned $RET" && exit 100
-	OSTDB_LIST="$OSTDB_LIST $OSTDB-$i"
-	i=$((i + 1))
-done
-
-#Remount filesystem
-[ "`mount | grep $MOUNT`" ] || $SETUP
-
-# need to turn off shell error detection to get proper error return
-# lfsck will return 1 if the filesystem had errors fixed
-echo "LFSCK TEST 1"
-echo "lfsck -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT"
-echo y | lfsck -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT
-RET=$?
-[ $RET -eq 0 ] && echo "clean after first check" && exit 0
-echo "LFSCK TEST 1 - finished with rc=$RET"
-[ $RET -gt $MAX_ERR ] && exit 110 || true
-
-# make sure everything gets to the backing store
-sync; sleep 2; sync
-
-echo "LFSCK TEST 2"
-echo "$E2FSCK -d -v -fn --mdsdb $MDSDB $MDSDEV"
-df > /dev/null	# update statfs data on disk
-$E2FSCK -d -v -fn --mdsdb $MDSDB $MDSDEV
-RET=$?
-[ $RET -gt $MAX_ERR ] && echo "$E2FSCK returned $RET" && exit 123 || true
-
-i=0
-export OSTDB_LIST=""
-for OSTDEV in $OSTDEVS; do
-	df > /dev/null	# update statfs data on disk
-	$E2FSCK -d -v -fn --mdsdb $MDSDB --ostdb $OSTDB-$i $OSTDEV
-	RET=$?
-	[ $RET -gt $MAX_ERR ] && echo "$E2FSCK returned $RET" && exit 124
-	OSTDB_LIST="$OSTDB_LIST $OSTDB-$i"
-	i=$((i + 1))
-done
-
-echo "LFSCK TEST 2"
-echo "lfsck -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT"
-lfsck -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT
-RET=$?
-echo "LFSCK TEST 2 - finished with rc=$RET"
-[ $RET -ne 0 ] && exit 125 || true
-if [ -z "$WAS_MOUNTED" ]; then
-	sh llmountcleanup.sh || exit 120
-fi
-
-#Cleanup 
-rm -f $MDSDB $OSTDB-* || true
-
-echo "$0: completed"
diff --git a/lustre/tests/liblustre.sh b/lustre/tests/liblustre.sh
new file mode 100644
index 0000000..daf7f5a
--- /dev/null
+++ b/lustre/tests/liblustre.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+#set -vx
+
+set -e
+
+LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
+. $LUSTRE/tests/test-framework.sh
+init_test_env $@
+. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
+
+LIBLUSTRETESTS=${LIBLUSTRETESTS:-$LUSTRE/liblustre/tests}
+
+assert_env MGSNID MOUNT2
+export LIBLUSTRE_MOUNT_POINT=$MOUNT2
+export LIBLUSTRE_MOUNT_RETRY=5
+export LIBLUSTRE_MOUNT_TARGET=$MGSNID:/$FSNAME
+export LIBLUSTRE_TIMEOUT=`lctl get_param -n timeout`
+#export LIBLUSTRE_DEBUG_MASK=`lctl get_param -n debug`
+
+test_1() {
+    if ! check_versions; then
+	skip "liblustre version mismatch: cli $CLIVER, mds $MDSVER, ost $OSTVER"
+    elif ! [ "$NETTYPE" = "tcp" -o "$NETTYPE" = "ptl" ]; then
+	skip "NETTYPE=$NETTYPE unsupported for liblustre"
+    elif [ ! -x $LIBLUSTRETESTS/sanity ]; then
+	skip "$LIBLUSTRETESTS/sanity: not found"
+    else
+	mkdir -p $MOUNT2
+	$LIBLUSTRETESTS/sanity --target=$LIBLUSTRE_MOUNT_TARGET
+	if [ "$LIBLUSTRE_EXCEPT" ]; then
+	    LIBLUSTRE_OPT="$LIBLUSTRE_OPT \
+			$(echo ' '$LIBLUSTRE_EXCEPT  | sed -re 's/\s+/ -e /g')"
+	fi
+	echo $LIBLUSTRETESTS/sanity --target=$LIBLUSTRE_MOUNT_TARGET $LIBLUSTRE_OPT
+	$LIBLUSTRETESTS/sanity --target=$LIBLUSTRE_MOUNT_TARGET $LIBLUSTRE_OPT
+    fi
+}
+run_test 1 "liblustre sanity"
diff --git a/lustre/tests/lustre_rsync-test.sh b/lustre/tests/lustre_rsync-test.sh
index ef24c38..576bc4c 100644
--- a/lustre/tests/lustre_rsync-test.sh
+++ b/lustre/tests/lustre_rsync-test.sh
@@ -27,7 +27,7 @@ LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
-
+init_logging
 
 REPLLOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
 
diff --git a/lustre/tests/metadata-updates.sh b/lustre/tests/metadata-updates.sh
index 432e96d..c2c6457 100755
--- a/lustre/tests/metadata-updates.sh
+++ b/lustre/tests/metadata-updates.sh
@@ -10,6 +10,7 @@ LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 TRACE=${TRACE:-"+x"}
 
diff --git a/lustre/tests/ost-pools.sh b/lustre/tests/ost-pools.sh
index 99c1073..81c45c5 100644
--- a/lustre/tests/ost-pools.sh
+++ b/lustre/tests/ost-pools.sh
@@ -25,6 +25,7 @@ LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 check_and_setup_lustre
 
diff --git a/lustre/tests/parallel-scale.sh b/lustre/tests/parallel-scale.sh
index 0bbca75..e6c41ff 100644
--- a/lustre/tests/parallel-scale.sh
+++ b/lustre/tests/parallel-scale.sh
@@ -6,6 +6,7 @@ LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 #              bug 20670           21255 
 ALWAYS_EXCEPT="parallel_grouplock  statahead $PARALLEL_SCALE_EXCEPT"
diff --git a/lustre/tests/performance-sanity.sh b/lustre/tests/performance-sanity.sh
index beabcc4..cae7b00 100644
--- a/lustre/tests/performance-sanity.sh
+++ b/lustre/tests/performance-sanity.sh
@@ -11,6 +11,7 @@ LUSTRE=${LUSTRE:-`dirname $0`/..}
 init_test_env $@
 
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 [ -x "$MDSRATE" ] || FAIL_ON_ERROR=true error "No mdsrate program. Aborting."
 which mpirun > /dev/null 2>&1 || \
diff --git a/lustre/tests/racer.sh b/lustre/tests/racer.sh
new file mode 100644
index 0000000..f862012
--- /dev/null
+++ b/lustre/tests/racer.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+bash $(dirname $0)/runracer $@
diff --git a/lustre/tests/recovery-double-scale.sh b/lustre/tests/recovery-double-scale.sh
index 1f84f92..7e26239 100644
--- a/lustre/tests/recovery-double-scale.sh
+++ b/lustre/tests/recovery-double-scale.sh
@@ -17,6 +17,7 @@ CLEANUP=${CLEANUP:-""}
 init_test_env $@
 
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 TESTSUITELOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh)}
 DEBUGLOG=$TESTSUITELOG.debug
 
diff --git a/lustre/tests/recovery-mds-scale.sh b/lustre/tests/recovery-mds-scale.sh
index d4867f5..59c368e 100644
--- a/lustre/tests/recovery-mds-scale.sh
+++ b/lustre/tests/recovery-mds-scale.sh
@@ -14,6 +14,7 @@ CLEANUP=${CLEANUP:-""}
 init_test_env $@
 
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 TESTSUITELOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh)}
 DEBUGLOG=$TESTSUITELOG.debug
diff --git a/lustre/tests/recovery-random-scale.sh b/lustre/tests/recovery-random-scale.sh
index 2458aff..65de6c1 100644
--- a/lustre/tests/recovery-random-scale.sh
+++ b/lustre/tests/recovery-random-scale.sh
@@ -18,6 +18,7 @@ CLEANUP=${CLEANUP:-""}
 init_test_env $@
 
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 TESTSUITELOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh)}
 DEBUGLOG=$TESTSUITELOG.debug
diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index f910d25..8617876 100755
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -10,6 +10,7 @@ LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 if [ "$FAILURE_MODE" = "HARD" ] && mixed_ost_devs; then
     CONFIG_EXCEPTIONS="52"
@@ -18,7 +19,7 @@ if [ "$FAILURE_MODE" = "HARD" ] && mixed_ost_devs; then
     ALWAYS_EXCEPT="$ALWAYS_EXCEPT $CONFIG_EXCEPTIONS"
 fi
 
-remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
+require_dsh_mds || exit 0
 
 # also long tests: 19, 21a, 21e, 21f, 23, 27
 #                                   1  2.5  2.5    4    4          (min)"
diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh
index cdeae9f..f450f70 100755
--- a/lustre/tests/replay-dual.sh
+++ b/lustre/tests/replay-dual.sh
@@ -21,8 +21,8 @@ if [ "$FAILURE_MODE" = "HARD" ] && mixed_ost_devs; then
 fi
 
 init_test_env $@
-
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
 
diff --git a/lustre/tests/replay-ost-single.sh b/lustre/tests/replay-ost-single.sh
index 1bdbcbf..3325fd1 100755
--- a/lustre/tests/replay-ost-single.sh
+++ b/lustre/tests/replay-ost-single.sh
@@ -8,13 +8,14 @@ CLEANUP=${CLEANUP:-""}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 # While we do not use OSTCOUNT=1 setup anymore,
 # ost1failover_HOST is used
 #ostfailover_HOST=${ostfailover_HOST:-$ost_HOST}
 #failover= must be defined in OST_MKFS_OPTIONS if ostfailover_HOST != ost_HOST
 
-remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
+require_dsh_ost || exit 0
 
 if [ "$FAILURE_MODE" = "HARD" ] && mixed_ost_devs; then
     skip_env "$0: Several ost services on one ost node are used with FAILURE_MODE=$FAILURE_MODE. "
diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index 2ddf988..36dc419 100755
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -13,10 +13,11 @@ CLEANUP=${CLEANUP:-}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 CHECK_GRANT=${CHECK_GRANT:-"yes"}
 GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""}
 
-remote_mds_nodsh && log "SKIP: remote MDS with nodsh" && exit 0
+require_dsh_mds || exit 0
 
 # Skip these tests
 # bug number:  17466 18857,15962
diff --git a/lustre/tests/rpc.sh b/lustre/tests/rpc.sh
index 11c5181..e1127d2 100755
--- a/lustre/tests/rpc.sh
+++ b/lustre/tests/rpc.sh
@@ -3,6 +3,11 @@ export PATH=`dirname $0`/../utils:$PATH
 NAME=${NAME:-local}
 
 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
+
+if [ ! -f $LUSTRE/tests/rpc.sh ]; then
+    LUSTRE=$(cd $(dirname $(which $0))/..; echo $PWD)
+fi
+
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
diff --git a/lustre/tests/runracer b/lustre/tests/runracer
index b67a819..083a56e 100644
--- a/lustre/tests/runracer
+++ b/lustre/tests/runracer
@@ -2,16 +2,18 @@
 #set -vx
 set -e
 
+ONLY=${ONLY:-"$*"}
 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
-racer=`which racer.sh`
+racer=`which racer.sh 2> /dev/null`
+echo racer: $racer
 [ -z "$racer" ] && echo racer is not installed && exit 1
 
 CLIENTS=${CLIENTS:-$HOSTNAME}
-RACERDIRS=$@
 RACERDIRS=${RACERDIRS:-$DIR}
 echo RACERDIRS=$RACERDIRS
 for d in ${RACERDIRS}; do
@@ -21,6 +23,7 @@ for d in ${RACERDIRS}; do
 done
 
 DURATION=${DURATION:-120}
+PIDFILE=$TMP/racer.$$
 
 assert_env CLIENTS
 
@@ -105,7 +108,7 @@ racer_cleanup () {
 		done
 	else
 		echo "Racer completed before DURATION=$DURATION expired. Cleaning up..."
-		kill $TIMERPID
+		kill $TIMERPID || true
 		for dir in $RDIRS; do
 			do_racer_cleanup $dir
 		done
@@ -114,36 +117,47 @@ racer_cleanup () {
 
 racer_timeout () {
 	timeout="timeout"
+	RACERPID=$(cat $PIDFILE)
+	rm -f $PIDFILE
 	racer_cleanup
 	echo "$0: completed $RC"
-	exit $RC
+	return $RC
 }
 
-# run racer
-log "Start racer on clients: $CLIENTS DURATION=$DURATION"
-RC=0
-
+build_test_filter
+check_and_setup_lustre
 trap racer_timeout ALRM
 
-timer_on $((DURATION + 5))
+# run racer
+test_1() {
+    RC=0
+
+    timer_on $((DURATION + 5))
+
+    RACERPID=""
+    for rdir in $RDIRS; do
+        do_nodes $CLIENTS "DURATION=$DURATION $racer $rdir $NUM_RACER_THREADS" &
+        pid=$!
+        RACERPID="$RACERPID $pid"
+    done
+
+    echo RACERPID=$RACERPID
+    echo $RACERPID > $PIDFILE
+    for rpid in $RACERPID; do
+        wait $rpid
+        rc=$?
+        echo "rpid=$rpid rc=$rc"
+        if [ $rc != 0 ]; then
+                RC=$((RC + 1))
+        fi
+    done
 
-RACERPID=""
-for rdir in $RDIRS; do
-	do_nodes $CLIENTS "DURATION=$DURATION $racer $rdir $NUM_RACER_THREADS" &
-	pid=$!
-	RACERPID="$RACERPID $pid"
-done
+    racer_cleanup
 
-echo RACERPID=$RACERPID
-for rpid in $RACERPID; do
-	wait $rpid
-	rc=$?
-	echo "rpid=$rpid rc=$rc"
-	if [ $rc != 0 ]; then
-		RC=$((RC + 1))
-	fi
-done
+    return $RC
+}
+run_test 1 "racer on clients: $CLIENTS DURATION=$DURATION"
 
-racer_cleanup
-echo "$0: completed $RC"
-exit $RC
+equals_msg `basename $0`: test complete, cleaning up
+check_and_cleanup_lustre
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true
diff --git a/lustre/tests/runtests b/lustre/tests/runtests
index ad0778b..34d24d4 100755
--- a/lustre/tests/runtests
+++ b/lustre/tests/runtests
@@ -13,6 +13,7 @@ export NAME=${NAME:-local}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 RUNTESTS_SRC=${RUNTESTS_SRC:-"/etc /bin"}
 [ "$COUNT" ] || COUNT=1000
diff --git a/lustre/tests/sanity-benchmark.sh b/lustre/tests/sanity-benchmark.sh
index cfce876..0cbbf02 100644
--- a/lustre/tests/sanity-benchmark.sh
+++ b/lustre/tests/sanity-benchmark.sh
@@ -6,10 +6,13 @@
 
 set -e
 
+ONLY=${ONLY:-"$*"}
+
 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 MAX_THREADS=${MAX_THREADS:-20}
 RAMKB=`awk '/MemTotal:/ { print $2 }' /proc/meminfo`
diff --git a/lustre/tests/sanity-gss.sh b/lustre/tests/sanity-gss.sh
index 01e6b1d..171ea7c 100644
--- a/lustre/tests/sanity-gss.sh
+++ b/lustre/tests/sanity-gss.sh
@@ -33,8 +33,9 @@ LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
-remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
+require_dsh_mds || exit 0
 
 [ "$SLOW" = "no" ] && EXCEPT_SLOW="100 101"
 
diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh
index 3ef3ca7..b3abd20 100644
--- a/lustre/tests/sanity-quota.sh
+++ b/lustre/tests/sanity-quota.sh
@@ -48,12 +48,13 @@ LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 DIRECTIO=${DIRECTIO:-$LUSTRE/tests/directio}
 
 [ $MDSCOUNT -gt 1 ] && skip "CMD case" && exit 0
 
-remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
-remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
+require_dsh_mds || exit 0
+require_dsh_ost || exit 0
 
 [ "$SLOW" = "no" ] && EXCEPT_SLOW="9 10 11 18b 21"
 
diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh
index fe1db87..0c99c11 100644
--- a/lustre/tests/sanity-sec.sh
+++ b/lustre/tests/sanity-sec.sh
@@ -22,6 +22,7 @@ LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 RUNAS=${RUNAS:-"$LUSTRE/tests/runas"}
 WTL=${WTL:-"$LUSTRE/tests/write_time_limit"}
@@ -31,8 +32,8 @@ PERM_CONF=$CONFDIR/perm.conf
 SANITYSECLOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
 FAIL_ON_ERROR=false
 
-remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
-remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
+require_dsh_mds || exit 0
+require_dsh_ost || exit 0
 
 ID0=${ID0:-500}
 ID1=${ID1:-501}
diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index a24bd44..b5e58c9 100644
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -72,6 +72,7 @@ LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/${NAME}.sh}
+init_logging
 
 [ "$SLOW" = "no" ] && EXCEPT_SLOW="24o 24v 27m 36f 36g 51b 51c 60c 63 64b 68 71 73 77f 78 101 103 115 120g 124b"
 
diff --git a/lustre/tests/sanityN.sh b/lustre/tests/sanityn.sh
similarity index 99%
rename from lustre/tests/sanityN.sh
rename to lustre/tests/sanityn.sh
index 68a98d7..35d9d3c 100644
--- a/lustre/tests/sanityN.sh
+++ b/lustre/tests/sanityn.sh
@@ -42,6 +42,7 @@ CLEANUP=${CLEANUP:-:}
 SETUP=${SETUP:-:}
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
 
 [ "$SLOW" = "no" ] && EXCEPT_SLOW="12 16 23 33a"
 
@@ -68,6 +69,9 @@ check_runas_id $RUNAS_ID $RUNAS_ID $RUNAS
 
 build_test_filter
 
+mkdir -p $MOUNT2
+mount_client $MOUNT2
+
 test_1a() {
 	touch $DIR1/f1
 	[ -f $DIR2/f1 ] || error
@@ -916,6 +920,7 @@ run_test 37 "check i_size is not updated for directory on close (bug 18695) ====
 
 log "cleanup: ======================================================"
 
+[ "$(mount | grep $MOUNT2)" ] && umount $MOUNT2
 check_and_cleanup_lustre
 
 echo '=========================== finished ==============================='
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 296073b..abb6531 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -22,6 +22,7 @@ export QUOTA_AUTO=1
 # function used by scripts run on remote nodes
 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 . $LUSTRE/tests/functions.sh
+. $LUSTRE/tests/yaml.sh
 
 LUSTRE_TESTS_CFG_DIR=${LUSTRE_TESTS_CFG_DIR:-${LUSTRE}/tests/cfg}
 
@@ -54,17 +55,17 @@ usage() {
 
 print_summary () {
     trap 0
-    [ "$TESTSUITE" == "lfscktest" ] && return 0
+    [ "$TESTSUITE" == "lfsck" ] && return 0
     [ -n "$ONLY" ] && echo "WARNING: ONLY is set to ${ONLY}."
     local form="%-13s %-17s %s\n"
     printf "$form" "status" "script" "skipped tests E(xcluded) S(low)"
     echo "------------------------------------------------------------------------------------"
-    for O in $TESTSUITE_LIST; do
+    for O in $DEFAULT_SUITES; do
         local skipped=""
         local slow=""
-        local o=$(echo $O | tr "[:upper:]" "[:lower:]")
+        O=$(echo $O  | tr "-" "_" | tr "[:lower:]" "[:upper:]")
+        local o=$(echo $O  | tr "[:upper:]" "[:lower:]")
         o=${o//_/-}
-        o=${o//tyn/tyN}
         local log=${TMP}/${o}.log
         [ -f $log ] && skipped=$(grep excluded $log | awk '{ printf " %s", $3 }' | sed 's/test_//g')
         [ -f $log ] && slow=$(grep SLOW $log | awk '{ printf " %s", $3 }' | sed 's/test_//g')
@@ -74,7 +75,8 @@ print_summary () {
 
     done
 
-    for O in $TESTSUITE_LIST; do
+    for O in $DEFAULT_SUITES; do
+        O=$(echo $O  | tr "-" "_" | tr "[:lower:]" "[:upper:]")
         if [ "${!O}" = "no" ]; then
             # FIXME.
             # only for those tests suits which are run directly from acc-sm script:
@@ -87,7 +89,8 @@ print_summary () {
         fi
     done
 
-    for O in $TESTSUITE_LIST; do
+    for O in $DEFAULT_SUITES; do
+        O=$(echo $O  | tr "-" "_" | tr "[:lower:]" "[:upper:]")
         [ "${!O}" = "done" -o "${!O}" = "no" ] || \
             printf "$form" "UNFINISHED" "$O" ""
     done
@@ -109,6 +112,10 @@ init_test_env() {
     #[ -d /r ] && export ROOT=${ROOT:-/r}
     export TMP=${TMP:-$ROOT/tmp}
     export TESTSUITELOG=${TMP}/${TESTSUITE}.log
+    if [[ -z $LOGDIRSET ]]; then
+        export LOGDIR=${LOGDIR:-${TMP}/test_logs/}/$(date +%s)
+        export LOGDIRSET=true
+    fi
     export HOSTNAME=${HOSTNAME:-`hostname`}
     if ! echo $PATH | grep -q $LUSTRE/utils; then
         export PATH=$PATH:$LUSTRE/utils
@@ -117,12 +124,12 @@ init_test_env() {
         export PATH=$PATH:$LUSTRE/utils/gss
     fi
     if ! echo $PATH | grep -q $LUSTRE/tests; then
-	export PATH=$PATH:$LUSTRE/tests
+        export PATH=$PATH:$LUSTRE/tests
     fi
     export MDSRATE=${MDSRATE:-"$LUSTRE/tests/mpi/mdsrate"}
     [ ! -f "$MDSRATE" ] && export MDSRATE=$(which mdsrate 2> /dev/null)
     if ! echo $PATH | grep -q $LUSTRE/tests/racer; then
-        export PATH=$PATH:$LUSTRE/tests/racer
+        export PATH=$LUSTRE/tests/racer:$PATH:
     fi
     if ! echo $PATH | grep -q $LUSTRE/tests/mpi; then
         export PATH=$PATH:$LUSTRE/tests/mpi
@@ -1464,12 +1471,12 @@ do_node() {
 
     if [ "$myPDSH" = "rsh" ]; then
 # we need this because rsh does not return exit code of an executed command
-	local command_status="$TMP/cs"
-	rsh $HOST ":> $command_status"
-	rsh $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin;
-		    cd $RPWD; sh -c \"$@\") ||
-		    echo command failed >$command_status"
-	[ -n "$($myPDSH $HOST cat $command_status)" ] && return 1 || true
+        local command_status="$TMP/cs"
+        rsh $HOST ":> $command_status"
+        rsh $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin;
+                    cd $RPWD; sh -c \"$@\") ||
+                    echo command failed >$command_status"
+        [ -n "$($myPDSH $HOST cat $command_status)" ] && return 1 || true
         return 0
     fi
 
@@ -1501,7 +1508,7 @@ do_nodes() {
     local rnodes=$1
     shift
 
-    if $(single_local_node $rnodes); then
+    if single_local_node $rnodes; then
         if $verbose; then
            do_node --verbose $rnodes $@
         else
@@ -1687,8 +1694,8 @@ switch_identity() {
 
 remount_client()
 {
-	zconf_umount `hostname` $1 || error "umount failed"
-	zconf_mount `hostname` $1 || error "mount failed"
+        zconf_umount `hostname` $1 || error "umount failed"
+        zconf_mount `hostname` $1 || error "mount failed"
 }
 
 writeconf_facet () {
@@ -1737,9 +1744,9 @@ setupall() {
                 eval mds${num}failover_HOST=$(facet_host mds$num)
             fi
 
-	    if [ $IDENTITY_UPCALL != "default" ]; then
+            if [ $IDENTITY_UPCALL != "default" ]; then
                 switch_identity $num $IDENTITY_UPCALL
-	    fi
+            fi
         done
         for num in `seq $OSTCOUNT`; do
             DEVNAME=$(ostdevname $num)
@@ -1768,7 +1775,7 @@ setupall() {
     [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT
 
     if [ "$MOUNT_2" ]; then
-	mount_client $MOUNT2
+        mount_client $MOUNT2
         [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT2
     fi
 
@@ -1788,7 +1795,7 @@ setupall() {
 }
 
 mounted_lustre_filesystems() {
-	awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts
+        awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts
 }
 
 init_facet_vars () {
@@ -2070,7 +2077,7 @@ cleanup_and_setup_lustre() {
         lctl set_param debug=0 || true
         cleanupall
         if [ "$ONLY" == "cleanup" ]; then
-    	    exit 0
+            exit 0
         fi
     fi
     check_and_setup_lustre
@@ -2211,9 +2218,9 @@ at_max_get() {
 
     # suppose that all ost-s has the same at_max set
     if [ $facet == "ost" ]; then
-	do_facet ost1 "lctl get_param -n at_max"
+        do_facet ost1 "lctl get_param -n at_max"
     else
-	do_facet $facet "lctl get_param -n at_max"
+        do_facet $facet "lctl get_param -n at_max"
     fi
 }
 
@@ -2225,15 +2232,15 @@ at_max_set() {
     for facet in $@; do
         if [ $facet == "ost" ]; then
             for i in `seq $OSTCOUNT`; do
-		do_facet ost$i "lctl set_param at_max=$at_max"
+                do_facet ost$i "lctl set_param at_max=$at_max"
 
             done
         elif [ $facet == "mds" ]; then
             for i in `seq $MDSCOUNT`; do
-		do_facet mds$i "lctl set_param at_max=$at_max"
+                do_facet mds$i "lctl set_param at_max=$at_max"
             done
         else
-	    do_facet $facet "lctl set_param at_max=$at_max"
+            do_facet $facet "lctl set_param at_max=$at_max"
         fi
     done
 }
@@ -2401,7 +2408,6 @@ stop_full_debug_logging() {
 
 error_noexit() {
     local TYPE=${TYPE:-"FAIL"}
-    local ERRLOG
 
     local dump=true
     # do not dump logs if $1=false
@@ -2412,15 +2418,14 @@ error_noexit() {
 
     log " ${TESTSUITE} ${TESTNAME}: @@@@@@ ${TYPE}: $@ "
 
+    # We need to dump the logs on all nodes
     if $dump; then
-        ERRLOG=$TMP/lustre_${TESTSUITE}_${TESTNAME}.$(date +%s)
-        echo "Dumping lctl log to $ERRLOG"
-        # We need to dump the logs on all nodes
-        do_nodes $(comma_list $(nodes_list)) $NODE $LCTL dk $ERRLOG
+        gather_logs $(comma_list $(nodes_list))
     fi
+
     debugrestore
     [ "$TESTSUITELOG" ] && echo "$0: ${TYPE}: $TESTNAME $@" >> $TESTSUITELOG
-    TEST_FAILED=true
+    echo "$@" > $LOGDIR/err
 }
 
 error() {
@@ -2475,7 +2480,7 @@ build_test_filter() {
     done
     for G in $GRANT_CHECK_LIST; do
         eval GCHECK_ONLY_${G}=true
-   	done
+        done
 }
 
 basetest() {
@@ -2496,13 +2501,13 @@ run_test() {
         testname=ONLY_$1
         if [ ${!testname}x != x ]; then
             [ "$LAST_SKIPPED" ] && echo "" && LAST_SKIPPED=
-            run_one $1 "$2"
+            run_one_logged $1 "$2"
             return $?
         fi
         testname=ONLY_$base
         if [ ${!testname}x != x ]; then
             [ "$LAST_SKIPPED" ] && echo "" && LAST_SKIPPED=
-            run_one $1 "$2"
+            run_one_logged $1 "$2"
             return $?
         fi
         LAST_SKIPPED="y"
@@ -2535,7 +2540,7 @@ run_test() {
     fi
 
     LAST_SKIPPED=
-    run_one $1 "$2"
+    run_one_logged $1 "$2"
 
     return $?
 }
@@ -2567,16 +2572,21 @@ log() {
 }
 
 trace() {
-	log "STARTING: $*"
-	strace -o $TMP/$1.strace -ttt $*
-	RC=$?
-	log "FINISHED: $*: rc $RC"
-	return 1
+        log "STARTING: $*"
+        strace -o $TMP/$1.strace -ttt $*
+        RC=$?
+        log "FINISHED: $*: rc $RC"
+        return 1
 }
 
 pass() {
-    $TEST_FAILED && echo -n "FAIL " || echo -n "PASS " 
-    echo $@
+    # Set TEST_STATUS here; will be used for logging the result
+    if [ -f $LOGDIR/err ]; then
+        TEST_STATUS="FAIL"
+    else
+        TEST_STATUS="PASS"
+    fi
+    echo $TEST_STATUS " " $@
 }
 
 check_mds() {
@@ -2596,28 +2606,47 @@ run_one() {
     message=$2
     tfile=f${testnum}
     export tdir=d0.${TESTSUITE}/d${base}
-
+    export TESTNAME=test_$testnum
     local SAVE_UMASK=`umask`
     umask 0022
 
-    local BEFORE=`date +%s`
     echo
-    log "== test $testnum: $message == `date +%H:%M:%S` ($BEFORE)"
-    #check_mds
-    export TESTNAME=test_$testnum
-    TEST_FAILED=false
+    log "== test $testnum: $message == `date +%H:%M:%S`"
     test_${testnum} || error "test_$testnum failed with $?"
-    #check_mds
     cd $SAVE_PWD
     reset_fail_loc
     check_grant ${testnum} || error "check_grant $testnum failed with $?"
     check_catastrophe || error "LBUG/LASSERT detected"
     ps auxww | grep -v grep | grep -q multiop && error "multiop still running"
-    pass "($((`date +%s` - $BEFORE))s)"
-    TEST_FAILED=false
     unset TESTNAME
     unset tdir
     umask $SAVE_UMASK
+    return 0
+}
+
+run_one_logged() {
+    local BEFORE=`date +%s`
+    local TEST_ERROR
+    local name=${TESTSUITE}.test_${1}.test_log.$(hostname).log
+    local test_log=$LOGDIR/$name
+    rm -rf $LOGDIR/err
+
+    run_one $1 "$2" 2>&1 | tee $test_log
+    local RC=${PIPESTATUS[0]}
+
+    [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \
+        echo "test_$1 returned $RC" | tee $LOGDIR/err
+
+    duration=$((`date +%s` - $BEFORE))
+    pass "(${duration}s)"
+    [ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err)
+    log_sub_test test_${1} $TEST_STATUS $duration "$RC" "$TEST_ERROR"
+
+    if [ -f $LOGDIR/err ]; then
+        $FAIL_ON_ERROR && exit $RC
+    fi
+
+    return 0
 }
 
 canonical_path() {
@@ -2627,50 +2656,50 @@ canonical_path() {
 sync_clients() {
     [ -d $DIR1 ] && cd $DIR1 && sync; sleep 1; sync
     [ -d $DIR2 ] && cd $DIR2 && sync; sleep 1; sync
-	cd $SAVE_PWD
+        cd $SAVE_PWD
 }
 
 check_grant() {
     export base=`basetest $1`
     [ "$CHECK_GRANT" == "no" ] && return 0
 
-	testname=GCHECK_ONLY_${base}
+        testname=GCHECK_ONLY_${base}
         [ ${!testname}x == x ] && return 0
 
     echo -n "checking grant......"
-	cd $SAVE_PWD
-	# write some data to sync client lost_grant
-	rm -f $DIR1/${tfile}_check_grant_* 2>&1
-	for i in `seq $OSTCOUNT`; do
-		$LFS setstripe $DIR1/${tfile}_check_grant_$i -i $(($i -1)) -c 1
-		dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \
-					      count=1 > /dev/null 2>&1
-	done
-	# sync all the data and make sure no pending data on server
-	sync_clients
-	
-	#get client grant and server grant
-	client_grant=0
+        cd $SAVE_PWD
+        # write some data to sync client lost_grant
+        rm -f $DIR1/${tfile}_check_grant_* 2>&1
+        for i in `seq $OSTCOUNT`; do
+                $LFS setstripe $DIR1/${tfile}_check_grant_$i -i $(($i -1)) -c 1
+                dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \
+                                              count=1 > /dev/null 2>&1
+        done
+        # sync all the data and make sure no pending data on server
+        sync_clients
+        
+        #get client grant and server grant
+        client_grant=0
     for d in `lctl get_param -n osc.*.cur_grant_bytes`; do
-		client_grant=$((client_grant + $d))
-	done
-	server_grant=0
-	for d in `lctl get_param -n obdfilter.*.tot_granted`; do
-		server_grant=$((server_grant + $d))
-	done
-
-	# cleanup the check_grant file
-	for i in `seq $OSTCOUNT`; do
-	        rm $DIR1/${tfile}_check_grant_$i
-	done
-
-	#check whether client grant == server grant
-	if [ $client_grant != $server_grant ]; then
-		echo "failed: client:${client_grant} server: ${server_grant}"
-		return 1
-	else
-		echo "pass"
-	fi
+                client_grant=$((client_grant + $d))
+        done
+        server_grant=0
+        for d in `lctl get_param -n obdfilter.*.tot_granted`; do
+                server_grant=$((server_grant + $d))
+        done
+
+        # cleanup the check_grant file
+        for i in `seq $OSTCOUNT`; do
+                rm $DIR1/${tfile}_check_grant_$i
+        done
+
+        #check whether client grant == server grant
+        if [ $client_grant != $server_grant ]; then
+                echo "failed: client:${client_grant} server: ${server_grant}"
+                return 1
+        else
+                echo "pass"
+        fi
 }
 
 ########################
@@ -2706,6 +2735,13 @@ remote_mds_nodsh()
     remote_mds && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$mds_HOST" ]
 }
 
+require_dsh_mds()
+{
+        remote_mds_nodsh && echo "SKIP: $TESTSUITE: remote MDS with nodsh" && \
+            MSKIPPED=1 && return 1
+        return 0
+}
+
 remote_ost ()
 {
     local node
@@ -2721,6 +2757,13 @@ remote_ost_nodsh()
     remote_ost && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$ost_HOST" ]
 }
 
+require_dsh_ost()
+{
+        remote_ost_nodsh && echo "SKIP: $TESTSUITE: remote OST with nodsh" && \
+            OSKIPPED=1 && return 1
+        return 0
+}
+
 remote_mgs_nodsh()
 {
     local MGS 
@@ -3034,19 +3077,19 @@ exit \\\$rc;"
 # $2 file
 # $3 $RUNAS
 get_stripe_info() {
-	local tmp_file
+        local tmp_file
 
-	stripe_size=0
-	stripe_count=0
-	stripe_index=0
-	tmp_file=$(mktemp)
+        stripe_size=0
+        stripe_count=0
+        stripe_index=0
+        tmp_file=$(mktemp)
 
-	do_facet $1 $3 lfs getstripe -v $2 > $tmp_file
+        do_facet $1 $3 lfs getstripe -v $2 > $tmp_file
 
-	stripe_size=`awk '$1 ~ /size/ {print $2}' $tmp_file`
-	stripe_count=`awk '$1 ~ /count/ {print $2}' $tmp_file`
-	stripe_index=`awk '$1 ~ /stripe_offset/ {print $2}' $tmp_file`
-	rm -f $tmp_file
+        stripe_size=`awk '$1 ~ /size/ {print $2}' $tmp_file`
+        stripe_count=`awk '$1 ~ /count/ {print $2}' $tmp_file`
+        stripe_index=`awk '$1 ~ /stripe_offset/ {print $2}' $tmp_file`
+        rm -f $tmp_file
 }
 
 # CMD: determine mds index where directory inode presents
@@ -3209,7 +3252,6 @@ wait_osc_import_state() {
     log "${ost_facet} now in ${CONN_STATE} state"
     return 0
 }
-
 get_clientmdc_proc_path() {
     echo "${1}-mdc-*"
 }
@@ -3218,7 +3260,9 @@ do_rpc_nodes () {
     local list=$1
     shift
 
-    do_nodes --verbose $list "PATH=$LUSTRE/tests/:$PATH sh rpc.sh $@ " 
+    # Add paths to lustre tests for 32 and 64 bit systems.
+    local RPATH="$LUSTRE/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:$PATH"
+    do_nodes --verbose $list "PATH=$RPATH sh rpc.sh $@ " 
 }
 
 wait_clients_import_state () {
@@ -3364,27 +3408,30 @@ gather_logs () {
     # of writing the file to an NFS directory so it doesn't need to be copied.
     local tmp=$TMP
     local docp=true
-    [ -d "$SHARED_DIR_LOGS" ] && tmp=$SHARED_DIR_LOGS && docp=false
-
+    [ -f $LOGDIR/shared ] && docp=false
+ 
     # dump lustre logs, dmesg
-    do_nodes $list "log=$tmp/\\\$(hostname)-debug-$ts.log ;
-lctl dk \\\$log >/dev/null;
-log=$tmp/\\\$(hostname)-dmesg-$ts.log;
-dmesg > \\\$log; "
-
-    # FIXME: does it make sense to collect the logs for $ts only, but all
-    # TESTSUITE logs?
-    # rsync $TMP/*${TESTSUITE}* to gather the logs dumped by error fn
-    local logs=$TMP/'*'${TESTSUITE}'*'
-    if $docp; then
-        logs=$logs' '$tmp/'*'$ts'*'
-    fi
-    for node in ${list//,/ }; do
-        rsync -az $node:"$logs" $TMP
-    done
 
-    local archive=$TMP/${TESTSUITE}-$ts.tar.bz2
-    tar -jcf $archive $tmp/*$ts* $TMP/*${TESTSUITE}*
+    prefix="$LOGDIR/${TESTSUITE}.${TESTNAME}"
+    suffix="$ts.log"
+    echo "Dumping lctl log to ${prefix}.*.${suffix}"
+
+    if [ "$CLIENTONLY" -o "$PDSH" == "no_dsh" ]; then
+        echo "Dumping logs only on local client."
+        $LCTL dk > ${prefix}.debug_log.$(hostname).${suffix}
+        dmesg > ${prefix}.dmesg.$(hostname).${suffix}
+        return
+    fi
+
+    do_nodes --verbose $list \
+        "$LCTL dk > ${prefix}.debug_log.\\\$(hostname).${suffix};
+         dmesg > ${prefix}.dmesg.\\\$(hostname).${suffix}"
+    if [ ! -f $LOGDIR/shared ]; then
+        do_nodes $list rsync -az "${prefix}.*.${suffix}" $HOSTNAME:$LOGDIR
+      fi
+
+    local archive=$LOGDIR/${TESTSUITE}-$ts.tar.bz2
+    tar -jcf $archive $LOGDIR/*$ts* $LOGDIR/*${TESTSUITE}*
 
     echo $archive
 }
@@ -3712,3 +3759,54 @@ set_flavor_all()
 }
 
 
+check_logdir() {
+    local dir=$1
+    # Checking for shared logdir
+    if [ ! -d $dir ]; then
+        # Not found. Create local logdir
+        mkdir -p $dir
+    else
+        touch $dir/node.$(hostname).yml
+    fi
+    return 0
+}
+
+check_write_access() {
+    local dir=$1
+    for node in $(nodes_list); do
+        if [ ! -f "$dir/node.${node}.yml" ]; then
+            # Logdir not accessible/writable from this node.
+            return 1
+        fi
+    done
+    return 0
+}
+
+init_logging() {
+    if [[ -n $YAML_LOG ]]; then
+        return
+    fi
+    export YAML_LOG=${LOGDIR}/results.yml
+    mkdir -p $LOGDIR
+    init_clients_lists
+
+    do_rpc_nodes $(comma_list $(nodes_list)) check_logdir $LOGDIR
+    if check_write_access $LOGDIR; then
+        touch $LOGDIR/shared
+        echo "Logging to shared log directory: $LOGDIR"
+    else
+        echo "Logging to local directory: $LOGDIR"
+    fi
+
+    yml_nodes_file $LOGDIR
+    yml_results_file >> $YAML_LOG
+}
+
+log_test() {
+    yml_log_test $1 >> $YAML_LOG
+}
+
+log_sub_test() {
+    yml_log_sub_test $@ >> $YAML_LOG
+}
+
diff --git a/lustre/tests/yaml.sh b/lustre/tests/yaml.sh
new file mode 100644
index 0000000..689f474
--- /dev/null
+++ b/lustre/tests/yaml.sh
@@ -0,0 +1,182 @@
+#!/bin/bash
+# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
+
+#
+# Shell routines for logging results to a yaml file.
+#
+
+split_output() {
+    while read line; do
+        host=${line%%:*};
+        echo "$line" | sed "s/^${host}: //" | sed "s/^${host}://" \
+            >> $logdir/node.$host.yml;
+    done
+}
+
+yml_nodes_file() {
+    export logdir=$1
+
+    if [ -f $logdir/shared ]; then
+        do_rpc_nodes $(comma_list $(nodes_list)) \
+            "yml_node >> $logdir/node.\\\$(hostname).yml"
+    else
+        do_rpc_nodes $(comma_list $(nodes_list)) yml_node | split_output
+    fi
+    yml_entities
+}
+
+yml_results_file() {
+    export logdir=$1
+
+    #TestGroup
+    yml_test_group
+
+    # Tests
+    printf "Tests:\n"
+}
+
+# Called on the node for which we the info is needed.
+yml_node() {
+    local node=$(hostname)
+    logdir=$1
+
+    printf "Build:\n"
+    yml_build_info
+    printf "\n"
+
+    printf "Node:\n"
+    yml_node_info
+    printf "\n"
+
+    printf "LustreEntities:\n"
+}
+
+yml_test_group() {
+    TEST_GROUP=${TEST_GROUP:-"acc-sm-$(hostname)"}
+    TEST_HOST=${TEST_HOST:-$(hostname)}
+    TEST_USER=${TEST_USER:-$USER}
+
+    # TestGroup information
+    cat <<EOF
+TestGroup:
+    test_group: $TEST_GROUP
+    testhost: $TEST_HOST
+    submission: $(date)
+    user_name: $TEST_USER
+
+EOF
+}
+
+release() {
+   if [ -r /etc/lsb-release ]; then
+      dist=$(grep 'DISTRIB_ID' /etc/lsb-release | sed 's/DISTRIB_ID=//' | head -1)
+   elif [ -r /etc/redhat-release ]; then
+       dist=$(awk '/release/ { printf("%s %s %s", $1, $2, $3)}' /etc/redhat-release)
+   elif [ -r /etc/*-release ]; then
+       dist=$(find /etc/ -maxdepth 1 -name '*release' 2> /dev/null | \
+           sed -e 's/\/etc\///' -e 's/-release//' | head -1)
+   else
+       dist="UNKNOWN"
+   fi
+
+   echo $dist
+}
+
+yml_build_info() {
+    TEST_DISTRO=$(release)
+    LUSTRE_VERSION=$(lctl lustre_build_version | awk '/Lustre version:/ {print $3}')
+    LUSTRE_BUILD=$(sed 's/-.*//' <<<$LUSTRE_VERSION)
+
+cat <<EOF
+    lbats_build_id: $LBATS_ID
+    lbats_build_name: $LBATS_NAME
+    architecture: $(uname -m)
+    os: $(uname -o)
+    os_distribution: $TEST_DISTRO
+    lustre_version: $LUSTRE_VERSION
+    lustre_build: $LUSTRE_BUILD
+    kernel_version: $(uname -r)
+EOF
+}
+
+yml_node_info()
+{
+    mem=$(awk '/MemTotal:/ {print $2 " " $3}' /proc/meminfo)
+cat <<EOF
+    node_name: $(hostname)
+    mem_size: $mem
+    architecture: $(uname -m)
+    networks:
+EOF
+    for nw in $(lctl list_nids | grep -v @lo | cut -f 2 -d '@' | uniq); do
+        printf "        - $nw\n"
+    done
+}
+
+yml_entity() {
+    cat<<EOF
+-
+    node_type: $1
+    node_name: $2
+EOF
+}
+
+yml_entities() {
+    local host
+    for num in $(seq $MDSCOUNT); do
+        host=$(facet_active_host mds$num)
+        yml_entity "MDS $num" $host >> $logdir/node.$host.yml
+    done
+
+    for num in $(seq $OSTCOUNT); do
+        host=$(facet_active_host ost$num)
+        yml_entity "OST $num" $host >> $logdir/node.$host.yml
+    done
+
+    i=1
+    for host in ${CLIENTS//,/ }; do
+        yml_entity "Client $i" $host >> $logdir/node.$host.yml
+        i=$((i+1))
+    done
+}
+
+yml_log_test() {
+    if [ $1 != "FINISHED" ]; then
+        cat <<EOF
+-
+        name: $1
+        description: $TESTSUITE $1
+        submission: $(date)
+        SubTests:
+EOF
+    fi
+}
+
+yml_log_test_status() {
+    cat <<EOF
+    duration: $1
+    status: $2
+EOF
+}
+
+yml_log_sub_test() {
+    cat <<EOF
+        -
+            name: $1
+            status: $2
+            duration: $3
+            return_code: $4
+EOF
+    shift 4
+    printf "            error: "
+    echo $@
+}
+
+yml_log_sub_test_log() {
+    cat <<EOF
+        -
+            name: $1
+            type: $2
+            location: $3
+EOF
+}
-- 
1.8.3.1