From 883189869c2f7055c2df6f1592a8efd0495baaaf Mon Sep 17 00:00:00 2001
From: yury <yury>
Date: Mon, 2 Oct 2006 13:40:54 +0000
Subject: [PATCH] - added lmv support to many local tests including recovery
 ones.

---
 lustre/fid/fid_handler.c          |   6 +-
 lustre/tests/cfg/lmv.sh           |  81 ++++++++
 lustre/tests/cfg/local.sh         |   7 +-
 lustre/tests/recovery-small.sh    |   2 +-
 lustre/tests/replay-dual.sh       |  58 +++---
 lustre/tests/replay-ost-single.sh |   2 +-
 lustre/tests/replay-single-lmv.sh | 114 +++++++++++
 lustre/tests/replay-single.sh     | 200 ++++++++++----------
 lustre/tests/sanity-lmv.sh        | 388 ++++++++++++++++++++++++++++++++++++++
 lustre/tests/test-framework.sh    |  64 +++++--
 10 files changed, 770 insertions(+), 152 deletions(-)
 create mode 100644 lustre/tests/cfg/lmv.sh
 create mode 100755 lustre/tests/replay-single-lmv.sh
 create mode 100644 lustre/tests/sanity-lmv.sh

diff --git a/lustre/fid/fid_handler.c b/lustre/fid/fid_handler.c
index ec7f968..c09adc5 100644
--- a/lustre/fid/fid_handler.c
+++ b/lustre/fid/fid_handler.c
@@ -201,7 +201,7 @@ static int __seq_server_alloc_meta(struct lu_server_seq *seq,
                         LASSERT(in->lr_start > super->lr_start);
 
                         /*
-                         * Server cannot send to client empty range, this is why
+                         * Server cannot send empty range to client, this is why
                          * we check here that range from client is "newer" than
                          * exhausted super.
                          */
@@ -224,7 +224,7 @@ static int __seq_server_alloc_meta(struct lu_server_seq *seq,
                        "super: "DRANGE"\n", seq->lss_name, PRANGE(super));
         } else {
                 /*
-                 * XXX: avoid cascading RPCs using kind of async preallocation
+                 * XXX: Avoid cascading RPCs using kind of async preallocation
                  * when meta-sequence is close to exhausting.
                  */
                 if (range_is_exhausted(super)) {
@@ -241,7 +241,7 @@ static int __seq_server_alloc_meta(struct lu_server_seq *seq,
                                 RETURN(rc);
                         }
 
-                        /* saving new range into allocation space. */
+                        /* Saving new range to allocation space. */
                         *super = seq->lss_cli->lcs_range;
                         LASSERT(range_is_sane(super));
                 }
diff --git a/lustre/tests/cfg/lmv.sh b/lustre/tests/cfg/lmv.sh
new file mode 100644
index 0000000..915dcfa
--- /dev/null
+++ b/lustre/tests/cfg/lmv.sh
@@ -0,0 +1,81 @@
+FSNAME=lustre
+
+# facet hosts
+mds_HOST=${mds_HOST:-`hostname`}
+mdsfailover_HOST=${mdsfailover_HOST}
+mgs_HOST=${mgs_HOST:-$mds_HOST}
+ost_HOST=${ost_HOST:-`hostname`}
+ostfailover_HOST=${ostfailover_HOST}
+
+mds1_HOST=${mds1_HOST:-$mds_HOST}
+mds2_HOST=$mds1_HOST
+mds3_HOST=$mds1_HOST
+ost2_HOST=${ost2_HOST:-$ost_HOST}
+gks_HOST=${gks_HOST:-$mds_HOST}
+
+NETTYPE=${NETTYPE:-tcp}
+MGSNID=${MGSNID:-`h2$NETTYPE $mgs_HOST`}
+FSTYPE=${FSTYPE:-ldiskfs}
+STRIPE_BYTES=${STRIPE_BYTES:-1048576}
+STRIPES_PER_OBJ=${STRIPES_PER_OBJ:-$((OSTCOUNT -1))}
+TIMEOUT=${TIMEOUT:-20}
+PTLDEBUG=${PTLDEBUG:-0x33f0404}
+SUBSYSTEM=${SUBSYSTEM:- 0xffb7e3ff}
+
+TMP=${TMP:-/tmp}
+
+MDSCOUNT=${MDSCOUNT:-3}
+MDSDEVBASE=${MDSDEVBASE:-$TMP/${FSNAME}-mds}
+MDSSIZE=${MDSSIZE:-100000}
+
+OSTCOUNT=${OSTCOUNT:-2}
+OSTDEVBASE=${OSTDEVBASE:-$TMP/${FSNAME}-ost}
+OSTSIZE=${OSTSIZE:-200000}
+
+#client
+MOUNT=${MOUNT:-/mnt/${FSNAME}}
+MOUNT1=${MOUNT1:-$MOUNT}
+MOUNT2=${MOUNT2:-${MOUNT}2}
+MOUNTOPT=${MOUNTOPT:-"user_xattr,"}
+[ "x$RMTCLIENT" != "x" ] &&
+	MOUNTOPT=$MOUNTOPT",remote_client"
+DIR=${DIR:-$MOUNT}
+DIR1=${DIR:-$MOUNT1}
+DIR2=${DIR2:-$MOUNT2}
+
+PDSH=${PDSH:-no_dsh}
+FAILURE_MODE=${FAILURE_MODE:-SOFT} # or HARD
+POWER_DOWN=${POWER_DOWN:-"powerman --off"}
+POWER_UP=${POWER_UP:-"powerman --on"}
+
+MKFSOPT=""
+MOUNTOPT=""
+[ "x$MDSJOURNALSIZE" != "x" ] &&
+    MKFSOPT=$MKFSOPT" -J size=$MDSJOURNALSIZE"
+[ "x$MDSISIZE" != "x" ] &&
+    MKFSOPT=$MKFSOPT" -i $MDSISIZE"
+[ "x$MKFSOPT" != "x" ] &&
+    MKFSOPT="--mkfsoptions=\"$MKFSOPT\""
+[ "x$mdsfailover_HOST" != "x" ] &&
+    MOUNTOPT=$MOUNTOPT" --failnode=`h2$NETTYPE $mdsfailover_HOST`"
+[ "x$STRIPE_BYTES" != "x" ] &&
+    MOUNTOPT=$MOUNTOPT" --param lov.stripesize=$STRIPE_BYTES"
+[ "x$STRIPES_PER_OBJ" != "x" ] &&
+    MOUNTOPT=$MOUNTOPT" --param lov.stripecount=$STRIPES_PER_OBJ"
+MDS_MKFS_OPTS="--mgs --mdt --fsname=$FSNAME --device-size=$MDSSIZE --param sys.timeout=$TIMEOUT $MKFSOPT $MOUNTOPT $MDSOPT"
+MDSn_MKFS_OPTS="--mgsnode=$MGSNID --mdt --fsname=$FSNAME --device-size=$MDSSIZE --param sys.timeout=$TIMEOUT $MKFSOPT $MOUNTOPT $MDSOPT"
+
+MKFSOPT=""
+MOUNTOPT=""
+[ "x$OSTJOURNALSIZE" != "x" ] &&
+    MKFSOPT=$MKFSOPT" -J size=$OSTJOURNALSIZE"
+[ "x$MKFSOPT" != "x" ] &&
+    MKFSOPT="--mkfsoptions=\"$MKFSOPT\""
+[ "x$ostfailover_HOST" != "x" ] &&
+    MOUNTOPT=$MOUNTOPT" --failnode=`h2$NETTYPE $ostfailover_HOST`"
+OST_MKFS_OPTS="--ost --fsname=$FSNAME --device-size=$OSTSIZE --mgsnode=$MGSNID --param sys.timeout=$TIMEOUT $MKFSOPT $MOUNTOPT $OSTOPT"
+
+MDS_MOUNT_OPTS="-o loop"
+OST_MOUNT_OPTS="-o loop"
+
+SINGLEMDS=${SINGLEMDS:-"mds1"}
diff --git a/lustre/tests/cfg/local.sh b/lustre/tests/cfg/local.sh
index 59eb56c..61df8dc 100644
--- a/lustre/tests/cfg/local.sh
+++ b/lustre/tests/cfg/local.sh
@@ -3,13 +3,17 @@ FSNAME=lustre
 # facet hosts
 mds_HOST=${mds_HOST:-`hostname`}
 mdsfailover_HOST=${mdsfailover_HOST}
+mds1_HOST=${mds1_HOST:-$mds_HOST}
+mds1failover_HOST=${mds1failover_HOST:-$mdsfailover_HOST}
 mgs_HOST=${mgs_HOST:-$mds_HOST}
 ost_HOST=${ost_HOST:-`hostname`}
 ostfailover_HOST=${ostfailover_HOST}
 
 TMP=${TMP:-/tmp}
 
-MDSDEV=${MDSDEV:-$TMP/${FSNAME}-mdt}
+MDSDEV=${MDSDEV:-$TMP/mds1-`hostname`}
+MDSCOUNT=${MDSCOUNT:-1}
+MDSDEVBASE=${MDSDEVBASE:-$TMP/${FSNAME}-mds}
 MDSSIZE=${MDSSIZE:-100000}
 MDSOPT=${MDSOPT:-"--mountfsoptions=acl"}
 
@@ -27,6 +31,7 @@ MGSNID=${MGSNID:-`h2$NETTYPE $mgs_HOST`}
 FSTYPE=${FSTYPE:-ldiskfs}
 STRIPE_BYTES=${STRIPE_BYTES:-1048576}
 STRIPES_PER_OBJ=${STRIPES_PER_OBJ:-0}
+SINGLEMDS=${SINGLEMDS:-"mds1"}
 TIMEOUT=${TIMEOUT:-20}
 PTLDEBUG=${PTLDEBUG:-0x33f0404}
 SUBSYSTEM=${SUBSYSTEM:- 0xffb7e3ff}
diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 72510d0..2ea5518 100755
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -9,7 +9,7 @@ PTLDEBUG=${PTLDEBUG:--1}
 LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
-. ${CONFIG:=$LUSTRE/tests/cfg/local.sh}
+. ${CONFIG:=$LUSTRE/tests/cfg/lmv.sh}
 
 build_test_filter
 
diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh
index 3ec7485..71184a7 100755
--- a/lustre/tests/replay-dual.sh
+++ b/lustre/tests/replay-dual.sh
@@ -11,7 +11,7 @@ LUSTRE=${LUSTRE:-`dirname $0`/..}
 
 init_test_env $@
 
-. ${CONFIG:=$LUSTRE/tests/cfg/local.sh}
+. ${CONFIG:=$LUSTRE/tests/cfg/lmv.sh}
 
 SETUP=${SETUP:-"setup"}
 CLEANUP=${CLEANUP:-"cleanup"}
@@ -40,10 +40,10 @@ $SETUP
 
 test_1() {
     touch $MOUNT1/a
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     touch $MOUNT2/b
 
-    fail mds
+    fail $SINGLEMDS
     checkstat $MOUNT2/a || return 1
     checkstat $MOUNT1/b || return 2
     rm $MOUNT2/a $MOUNT1/b
@@ -56,10 +56,10 @@ run_test 1 "|X| simple create"
 
 
 test_2() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     mkdir $MOUNT1/adir
 
-    fail mds
+    fail $SINGLEMDS
     checkstat $MOUNT2/adir || return 1
     rmdir $MOUNT2/adir
     checkstat $MOUNT2/adir && return 2
@@ -69,11 +69,11 @@ test_2() {
 run_test 2 "|X| mkdir adir"
 
 test_3() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     mkdir $MOUNT1/adir
     mkdir $MOUNT2/adir/bdir
 
-    fail mds
+    fail $SINGLEMDS
     checkstat $MOUNT2/adir      || return 1
     checkstat $MOUNT1/adir/bdir || return 2
     rmdir $MOUNT2/adir/bdir $MOUNT1/adir
@@ -86,11 +86,11 @@ run_test 3 "|X| mkdir adir, mkdir adir/bdir "
 
 test_4() {
     mkdir $MOUNT1/adir
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     mkdir $MOUNT1/adir  && return 1
     mkdir $MOUNT2/adir/bdir
 
-    fail mds
+    fail $SINGLEMDS
     checkstat $MOUNT2/adir      || return 2
     checkstat $MOUNT1/adir/bdir || return 3
 
@@ -111,11 +111,11 @@ test_5() {
     # give multiop a chance to open
     sleep 1 
     rm -f $MOUNT1/a
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     kill -USR1 $pid
     wait $pid || return 1
 
-    fail mds
+    fail $SINGLEMDS
     [ -e $MOUNT2/a ] && return 2
     return 0
 }
@@ -131,22 +131,22 @@ test_6() {
     # give multiop a chance to open
     sleep 1 
     rm -f $MOUNT1/a
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     kill -USR1 $pid1
     wait $pid1 || return 1
 
-    fail mds
+    fail $SINGLEMDS
     kill -USR1 $pid2
     wait $pid2 || return 1
     [ -e $MOUNT2/a ] && return 2
     return 0
 }
-run_test 6 "open1, open2, unlink |X| close1 [fail mds] close2"
+run_test 6 "open1, open2, unlink |X| close1 [fail $SINGLEMDS] close2"
 
 test_8() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     drop_reint_reply "mcreate $MOUNT1/$tfile"    || return 1
-    fail mds
+    fail $SINGLEMDS
     checkstat $MOUNT2/$tfile || return 2
     rm $MOUNT1/$tfile || return 3
 
@@ -155,12 +155,12 @@ test_8() {
 run_test 8 "replay of resent request"
 
 test_9() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     mcreate $MOUNT1/$tfile-1
     mcreate $MOUNT2/$tfile-2
     # drop first reint reply
     sysctl -w lustre.fail_loc=0x80000119
-    fail mds
+    fail $SINGLEMDS
     sysctl -w lustre.fail_loc=0
 
     rm $MOUNT1/$tfile-[1,2] || return 1
@@ -171,12 +171,12 @@ run_test 9 "resending a replayed create"
 
 test_10() {
     mcreate $MOUNT1/$tfile-1
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     munlink $MOUNT1/$tfile-1
     mcreate $MOUNT2/$tfile-2
     # drop first reint reply
     sysctl -w lustre.fail_loc=0x80000119
-    fail mds
+    fail $SINGLEMDS
     sysctl -w lustre.fail_loc=0
 
     checkstat $MOUNT1/$tfile-1 && return 1
@@ -188,7 +188,7 @@ test_10() {
 run_test 10 "resending a replayed unlink"
 
 test_11() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     mcreate $MOUNT1/$tfile-1
     mcreate $MOUNT2/$tfile-2
     mcreate $MOUNT1/$tfile-3
@@ -208,7 +208,7 @@ test_11() {
 run_test 11 "both clients timeout during replay"
 
 test_12() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
 
     multiop $DIR/$tfile mo_c &
     MULTIPID=$!
@@ -235,7 +235,7 @@ test_13() {
     MULTIPID=$!
     sleep 5
 
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
 
     kill -USR1 $MULTIPID || return 3
     wait $MULTIPID || return 4
@@ -255,7 +255,7 @@ test_13() {
 run_test 13 "close resend timeout"
 
 test_14() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     createmany -o $MOUNT1/$tfile- 25
     createmany -o $MOUNT2/$tfile-2- 1
     createmany -o $MOUNT1/$tfile-3- 25
@@ -275,7 +275,7 @@ test_14() {
 run_test 14 "timeouts waiting for lost client during replay"
 
 test_15() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     createmany -o $MOUNT1/$tfile- 25
     createmany -o $MOUNT2/$tfile-2- 1
     umount $MOUNT2
@@ -295,7 +295,7 @@ test_15a() {
     local ost_last_id=""
     local osc_last_id=""
     
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     echo "data" > "$MOUNT2/${tfile}-m2"
 
     umount $MOUNT2
@@ -341,7 +341,7 @@ test_15a() {
 #CROW run_test 15a "OST clear orphans - synchronize ids on MDS and OST"
 
 test_15b() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     echo "data" > "$MOUNT2/${tfile}-m2"
     umount $MOUNT2
 
@@ -357,7 +357,7 @@ test_15b() {
 #CROW run_test 15b "multiple delayed OST clear orphans"
 
 test_15c() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     for ((i = 0; i < 2000; i++)); do
 	echo "data" > "$MOUNT2/${tfile}-$i" || error "create ${tfile}-$i failed"
     done
@@ -373,7 +373,7 @@ test_15c() {
 run_test 15c "remove multiple OST orphans"
 
 test_16() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     createmany -o $MOUNT1/$tfile- 25
     createmany -o $MOUNT2/$tfile-2- 1
     umount $MOUNT2
diff --git a/lustre/tests/replay-ost-single.sh b/lustre/tests/replay-ost-single.sh
index 0caba0b..1accb15 100755
--- a/lustre/tests/replay-ost-single.sh
+++ b/lustre/tests/replay-ost-single.sh
@@ -6,7 +6,7 @@ PTLDEBUG=${PTLDEBUG:--1}
 LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
-. ${CONFIG:=$LUSTRE/tests/cfg/local.sh}
+. ${CONFIG:=$LUSTRE/tests/cfg/lmv.sh}
 
 ostfailover_HOST=${ostfailover_HOST:-$ost_HOST}
 #failover= must be defined in OST_MKFS_OPTIONS if ostfailover_HOST != ost_HOST
diff --git a/lustre/tests/replay-single-lmv.sh b/lustre/tests/replay-single-lmv.sh
new file mode 100755
index 0000000..107095e
--- /dev/null
+++ b/lustre/tests/replay-single-lmv.sh
@@ -0,0 +1,114 @@
+#!/bin/sh
+
+set -e
+#set -v
+
+#
+# This test needs to be run on the client
+#
+
+LUSTRE=${LUSTRE:-`dirname $0`/..}
+. $LUSTRE/tests/test-framework.sh
+init_test_env $@
+. ${CONFIG:=$LUSTRE/tests/cfg/lmv.sh}
+
+
+# Skip these tests
+ALWAYS_EXCEPT=""
+build_test_filter
+
+SETUP=${SETUP:-"setup"}
+CLEANUP=${CLEANUP:-"stopall"}
+
+if [ "$ONLY" == "cleanup" ]; then
+    sysctl -w lnet.debug=0 || true
+    $CLEANUP
+    exit 0
+fi
+
+setup() {
+    formatall
+    setupall
+}
+
+$SETUP
+
+if [ "$ONLY" == "setup" ]; then
+    exit 0
+fi
+
+mkdir -p $DIR
+
+
+test_0() {
+    replay_barrier mds1
+    fail mds1
+}
+run_test 0 "empty replay"
+
+test_0b() {
+    # this test attempts to trigger a race in the precreation code, 
+    # and must run before any other objects are created on the filesystem
+    fail ost1
+    createmany -o $DIR/$tfile 20 || return 1
+    unlinkmany $DIR/$tfile 20 || return 2
+}
+run_test 0b "ensure object created after recover exists. (3284)"
+
+test_1a() {
+    mkdir $DIR/dir01
+    replay_barrier mds2
+    $CHECKSTAT -t dir $DIR/dir01 || return 1
+    rmdir $DIR/dir01
+    fail mds2
+    stat $DIR/dir01
+}
+run_test 1a "unlink cross-node dir (fail mds with inode)"
+
+test_1b() {
+    mkdir $DIR/dir11
+    replay_barrier mds1
+    $CHECKSTAT -t dir $DIR/dir11 || return 1
+    rmdir $DIR/dir11
+    fail mds1
+    stat $DIR/dir11
+}
+run_test 1b "unlink cross-node dir (fail mds with name)"
+
+test_2a() {
+    mkdir $DIR/dir21
+    createmany -o $DIR/dir21/f 3000
+    sleep 10
+    $CHECKSTAT -t dir $DIR/dir21 || return 1
+    $CHECKSTAT -t file $DIR/dir21/f1002 || return 1
+    replay_barrier mds1
+    rm $DIR/dir21/f1002
+    fail mds1
+    stat $DIR/dir21/f1002
+}
+run_test 2a "unlink cross-node file (fail mds with name)"
+
+
+test_3a() {
+    replay_barrier mds2
+    mkdir $DIR/dir3a1
+    $LCTL mark "FAILOVER mds2"
+    fail mds2
+    stat $DIR
+    $CHECKSTAT -t dir $DIR/dir3a1 || return 1
+}
+run_test 3a "mkdir cross-node dir (fail mds with inode)"
+
+test_3b() {
+    mkdir $DIR/dir13
+    replay_barrier mds1
+    $CHECKSTAT -t dir $DIR/dir13 || return 1
+    rmdir $DIR/dir13
+    fail mds1
+    stat $DIR/dir13
+}
+run_test 3b "mkdir cross-node dir (fail mds with name)"
+
+equals_msg test complete, cleaning up
+$CLEANUP
+
diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index 0463e41..a87e8a8 100755
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -10,7 +10,7 @@ set -e
 LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
-. ${CONFIG:=$LUSTRE/tests/cfg/local.sh}
+. ${CONFIG:=$LUSTRE/tests/cfg/lmv.sh}
 
 
 # Skip these tests
@@ -48,8 +48,8 @@ fi
 mkdir -p $DIR
 
 test_0() {
-    replay_barrier mds
-    fail mds
+    replay_barrier $SINGLEMDS
+    fail $SINGLEMDS
 }
 run_test 0 "empty replay"
 
@@ -75,7 +75,7 @@ seq_get_width()
 }
 
 test_0c() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
 
     local seq_width
     
@@ -89,8 +89,8 @@ test_0c() {
     touch $DIR/$tfile || return 1
     seq_set_width $seq_width
     
-    # fail mds and start recovery, replay RPCs, etc.
-    fail mds
+    # fail $SINGLEMDS and start recovery, replay RPCs, etc.
+    fail $SINGLEMDS
     
     # wait for recovery finish
     sleep 10
@@ -110,18 +110,18 @@ test_0c() {
 run_test 0c "fld create"
 
 test_1() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     mcreate $DIR/$tfile
-    fail mds
+    fail $SINGLEMDS
     $CHECKSTAT -t file $DIR/$tfile || return 1
     rm $DIR/$tfile
 }
 run_test 1 "simple create"
 
 test_2a() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     touch $DIR/$tfile
-    fail mds
+    fail $SINGLEMDS
     $CHECKSTAT -t file $DIR/$tfile || return 1
     rm $DIR/$tfile
 }
@@ -129,43 +129,43 @@ run_test 2a "touch"
 
 test_2b() {
     ./mcreate $DIR/$tfile
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     touch $DIR/$tfile
-    fail mds
+    fail $SINGLEMDS
     $CHECKSTAT -t file $DIR/$tfile || return 1
     rm $DIR/$tfile
 }
 run_test 2b "touch"
 
 test_3a() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     mcreate $DIR/$tfile
     o_directory $DIR/$tfile
-    fail mds
+    fail $SINGLEMDS
     $CHECKSTAT -t file $DIR/$tfile || return 2
     rm $DIR/$tfile
 }
 run_test 3a "replay failed open(O_DIRECTORY)"
 
 test_3b() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
 #define OBD_FAIL_MDS_OPEN_PACK | OBD_FAIL_ONCE
     do_facet mds "sysctl -w lustre.fail_loc=0x80000114"
     touch $DIR/$tfile
     do_facet mds "sysctl -w lustre.fail_loc=0"
-    fail mds
+    fail $SINGLEMDS
     $CHECKSTAT -t file $DIR/$tfile && return 2
     return 0
 }
 run_test 3b "replay failed open -ENOMEM"
 
 test_3c() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
 #define OBD_FAIL_MDS_ALLOC_OBDO | OBD_FAIL_ONCE
     do_facet mds "sysctl -w lustre.fail_loc=0x80000128"
     touch $DIR/$tfile
     do_facet mds "sysctl -w lustre.fail_loc=0"
-    fail mds
+    fail $SINGLEMDS
 
     $CHECKSTAT -t file $DIR/$tfile && return 2
     return 0
@@ -173,11 +173,11 @@ test_3c() {
 run_test 3c "replay failed open -ENOMEM"
 
 test_4() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     for i in `seq 10`; do
         echo "tag-$i" > $DIR/$tfile-$i
     done 
-    fail mds
+    fail $SINGLEMDS
     for i in `seq 10`; do
       grep -q "tag-$i" $DIR/$tfile-$i || error "$tfile-$i"
     done 
@@ -185,9 +185,9 @@ test_4() {
 run_test 4 "|x| 10 open(O_CREAT)s"
 
 test_4b() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     rm -rf $DIR/$tfile-*
-    fail mds
+    fail $SINGLEMDS
     $CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
 }
 run_test 4b "|x| rm 10 files"
@@ -195,11 +195,11 @@ run_test 4b "|x| rm 10 files"
 # The idea is to get past the first block of precreated files on both 
 # osts, and then replay.
 test_5() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     for i in `seq 220`; do
         echo "tag-$i" > $DIR/$tfile-$i
     done 
-    fail mds
+    fail $SINGLEMDS
     for i in `seq 220`; do
       grep -q "tag-$i" $DIR/$tfile-$i || error "f1c-$i"
     done 
@@ -211,10 +211,10 @@ run_test 5 "|x| 220 open(O_CREAT)"
 
 
 test_6() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     mkdir $DIR/$tdir
     mcreate $DIR/$tdir/$tfile
-    fail mds
+    fail $SINGLEMDS
     $CHECKSTAT -t dir $DIR/$tdir || return 1
     $CHECKSTAT -t file $DIR/$tdir/$tfile || return 2
     sleep 2
@@ -223,18 +223,18 @@ test_6() {
 run_test 6 "mkdir + contained create"
 
 test_6b() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     rm -rf $DIR/$tdir
-    fail mds
+    fail $SINGLEMDS
     $CHECKSTAT -t dir $DIR/$tdir && return 1 || true 
 }
 run_test 6b "|X| rmdir"
 
 test_7() {
     mkdir $DIR/$tdir
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     mcreate $DIR/$tdir/$tfile
-    fail mds
+    fail $SINGLEMDS
     $CHECKSTAT -t dir $DIR/$tdir || return 1
     $CHECKSTAT -t file $DIR/$tdir/$tfile || return 2
     rm -fr $DIR/$tdir
@@ -242,11 +242,11 @@ test_7() {
 run_test 7 "mkdir |X| contained create"
 
 test_8() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     multiop $DIR/$tfile mo_c &
     MULTIPID=$!
     sleep 1
-    fail mds
+    fail $SINGLEMDS
     ls $DIR/$tfile
     $CHECKSTAT -t file $DIR/$tfile || return 1
     kill -USR1 $MULTIPID || return 2
@@ -256,10 +256,10 @@ test_8() {
 run_test 8 "creat open |X| close"
 
 test_9() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     mcreate $DIR/$tfile
     local old_inum=`ls -i $DIR/$tfile | awk '{print $1}'`
-    fail mds
+    fail $SINGLEMDS
     local new_inum=`ls -i $DIR/$tfile | awk '{print $1}'`
 
     echo " old_inum == $old_inum, new_inum == $new_inum"
@@ -276,10 +276,10 @@ run_test 9  "|X| create (same inum/gen)"
 
 test_10() {
     mcreate $DIR/$tfile
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     mv $DIR/$tfile $DIR/$tfile-2
     rm -f $DIR/$tfile
-    fail mds
+    fail $SINGLEMDS
     $CHECKSTAT $DIR/$tfile && return 1
     $CHECKSTAT $DIR/$tfile-2 ||return 2
     rm $DIR/$tfile-2
@@ -291,11 +291,11 @@ test_11() {
     mcreate $DIR/$tfile
     echo "old" > $DIR/$tfile
     mv $DIR/$tfile $DIR/$tfile-2
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     echo "new" > $DIR/$tfile
     grep new $DIR/$tfile 
     grep old $DIR/$tfile-2
-    fail mds
+    fail $SINGLEMDS
     grep new $DIR/$tfile || return 1
     grep old $DIR/$tfile-2 || return 2
 }
@@ -308,11 +308,11 @@ test_12() {
     # give multiop a chance to open
     sleep 1
     rm -f $DIR/$tfile
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     kill -USR1 $pid
     wait $pid || return 1
 
-    fail mds
+    fail $SINGLEMDS
     [ -e $DIR/$tfile ] && return 2
     return 0
 }
@@ -329,8 +329,8 @@ test_13() {
     sleep 1 
     chmod 0 $DIR/$tfile
     $CHECKSTAT -p 0 $DIR/$tfile
-    replay_barrier mds
-    fail mds
+    replay_barrier $SINGLEMDS
+    fail $SINGLEMDS
     kill -USR1 $pid
     wait $pid || return 1
 
@@ -345,11 +345,11 @@ test_14() {
     # give multiop a chance to open
     sleep 1 
     rm -f $DIR/$tfile
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     kill -USR1 $pid || return 1
     wait $pid || return 2
 
-    fail mds
+    fail $SINGLEMDS
     [ -e $DIR/$tfile ] && return 3
     return 0
 }
@@ -361,12 +361,12 @@ test_15() {
     # give multiop a chance to open
     sleep 1 
     rm -f $DIR/$tfile
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     touch $DIR/g11 || return 1
     kill -USR1 $pid
     wait $pid || return 2
 
-    fail mds
+    fail $SINGLEMDS
     [ -e $DIR/$tfile ] && return 3
     touch $DIR/h11 || return 4
     return 0
@@ -375,11 +375,11 @@ run_test 15 "open(O_CREAT), unlink |X|  touch new, close"
 
 
 test_16() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     mcreate $DIR/$tfile
     munlink $DIR/$tfile
     mcreate $DIR/$tfile-2
-    fail mds
+    fail $SINGLEMDS
     [ -e $DIR/$tfile ] && return 1
     [ -e $DIR/$tfile-2 ] || return 2
     munlink $DIR/$tfile-2 || return 3
@@ -387,12 +387,12 @@ test_16() {
 run_test 16 "|X| open(O_CREAT), unlink, touch new,  unlink new"
 
 test_17() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     multiop $DIR/$tfile O_c &
     pid=$!
     # give multiop a chance to open
     sleep 1 
-    fail mds
+    fail $SINGLEMDS
     kill -USR1 $pid || return 1
     wait $pid || return 2
     $CHECKSTAT -t file $DIR/$tfile || return 3
@@ -401,7 +401,7 @@ test_17() {
 run_test 17 "|X| open(O_CREAT), |replay| close"
 
 test_18() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     multiop $DIR/$tfile O_tSc &
     pid=$!
     # give multiop a chance to open
@@ -412,7 +412,7 @@ test_18() {
     kill -USR1 $pid
     wait $pid || return 2
 
-    fail mds
+    fail $SINGLEMDS
     [ -e $DIR/$tfile ] && return 3
     [ -e $DIR/$tfile-2 ] || return 4
     # this touch frequently fails
@@ -425,25 +425,25 @@ run_test 18 "|X| open(O_CREAT), unlink, touch new, close, touch, unlink"
 
 # bug 1855 (a simpler form of test_11 above)
 test_19() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     mcreate $DIR/$tfile
     echo "old" > $DIR/$tfile
     mv $DIR/$tfile $DIR/$tfile-2
     grep old $DIR/$tfile-2
-    fail mds
+    fail $SINGLEMDS
     grep old $DIR/$tfile-2 || return 2
 }
 run_test 19 "|X| mcreate, open, write, rename "
 
 test_20() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     multiop $DIR/$tfile O_tSc &
     pid=$!
     # give multiop a chance to open
     sleep 1 
     rm -f $DIR/$tfile
 
-    fail mds
+    fail $SINGLEMDS
     kill -USR1 $pid
     wait $pid || return 1
     [ -e $DIR/$tfile ] && return 2
@@ -461,7 +461,7 @@ test_20b() {
     mds_evict_client
     df -P $DIR || df -P $DIR || true    # reconnect
 
-    fail mds                            # start orphan recovery
+    fail $SINGLEMDS                            # start orphan recovery
     df -P $DIR || df -P $DIR || true    # reconnect
     sleep 2
     AFTERSPACE=`df -P $DIR | tail -1 | awk '{ print $4 }'`
@@ -472,7 +472,7 @@ test_20b() {
 run_test 20b "write, unlink, eviction, replay, (test mds_cleanup_orphans)"
 
 test_21() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     multiop $DIR/$tfile O_tSc &
     pid=$!
     # give multiop a chance to open
@@ -480,7 +480,7 @@ test_21() {
     rm -f $DIR/$tfile
     touch $DIR/g11 || return 1
 
-    fail mds
+    fail $SINGLEMDS
     kill -USR1 $pid
     wait $pid || return 2
     [ -e $DIR/$tfile ] && return 3
@@ -495,10 +495,10 @@ test_22() {
     # give multiop a chance to open
     sleep 1 
 
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     rm -f $DIR/$tfile
 
-    fail mds
+    fail $SINGLEMDS
     kill -USR1 $pid
     wait $pid || return 1
     [ -e $DIR/$tfile ] && return 2
@@ -512,11 +512,11 @@ test_23() {
     # give multiop a chance to open
     sleep 1 
 
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     rm -f $DIR/$tfile
     touch $DIR/g11 || return 1
 
-    fail mds
+    fail $SINGLEMDS
     kill -USR1 $pid
     wait $pid || return 2
     [ -e $DIR/$tfile ] && return 3
@@ -531,8 +531,8 @@ test_24() {
     # give multiop a chance to open
     sleep 1 
 
-    replay_barrier mds
-    fail mds
+    replay_barrier $SINGLEMDS
+    fail $SINGLEMDS
     rm -f $DIR/$tfile
     kill -USR1 $pid
     wait $pid || return 1
@@ -548,8 +548,8 @@ test_25() {
     sleep 1 
     rm -f $DIR/$tfile
 
-    replay_barrier mds
-    fail mds
+    replay_barrier $SINGLEMDS
+    fail $SINGLEMDS
     kill -USR1 $pid
     wait $pid || return 1
     [ -e $DIR/$tfile ] && return 2
@@ -558,7 +558,7 @@ test_25() {
 run_test 25 "open(O_CREAT), unlink, replay, close (test mds_cleanup_orphans)"
 
 test_26() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     multiop $DIR/$tfile-1 O_tSc &
     pid1=$!
     multiop $DIR/$tfile-2 O_tSc &
@@ -570,7 +570,7 @@ test_26() {
     kill -USR1 $pid2
     wait $pid2 || return 1
 
-    fail mds
+    fail $SINGLEMDS
     kill -USR1 $pid1
     wait $pid1 || return 2
     [ -e $DIR/$tfile-1 ] && return 3
@@ -580,7 +580,7 @@ test_26() {
 run_test 26 "|X| open(O_CREAT), unlink two, close one, replay, close one (test mds_cleanup_orphans)"
 
 test_27() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     multiop $DIR/$tfile-1 O_tSc &
     pid1=$!
     multiop $DIR/$tfile-2 O_tSc &
@@ -590,7 +590,7 @@ test_27() {
     rm -f $DIR/$tfile-1
     rm -f $DIR/$tfile-2
 
-    fail mds
+    fail $SINGLEMDS
     kill -USR1 $pid1
     wait $pid1 || return 1
     kill -USR1 $pid2
@@ -608,13 +608,13 @@ test_28() {
     pid2=$!
     # give multiop a chance to open
     sleep 1 
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     rm -f $DIR/$tfile-1
     rm -f $DIR/$tfile-2
     kill -USR1 $pid2
     wait $pid2 || return 1
 
-    fail mds
+    fail $SINGLEMDS
     kill -USR1 $pid1
     wait $pid1 || return 2
     [ -e $DIR/$tfile-1 ] && return 3
@@ -630,11 +630,11 @@ test_29() {
     pid2=$!
     # give multiop a chance to open
     sleep 1 
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     rm -f $DIR/$tfile-1
     rm -f $DIR/$tfile-2
 
-    fail mds
+    fail $SINGLEMDS
     kill -USR1 $pid1
     wait $pid1 || return 1
     kill -USR1 $pid2
@@ -655,8 +655,8 @@ test_30() {
     rm -f $DIR/$tfile-1
     rm -f $DIR/$tfile-2
 
-    replay_barrier mds
-    fail mds
+    replay_barrier $SINGLEMDS
+    fail $SINGLEMDS
     kill -USR1 $pid1
     wait $pid1 || return 1
     kill -USR1 $pid2
@@ -676,9 +676,9 @@ test_31() {
     sleep 1 
     rm -f $DIR/$tfile-1
 
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     rm -f $DIR/$tfile-2
-    fail mds
+    fail $SINGLEMDS
     kill -USR1 $pid1
     wait $pid1 || return 1
     kill -USR1 $pid2
@@ -709,7 +709,7 @@ run_test 32 "close() notices client eviction; close() after client eviction"
 
 # Abort recovery before client complete
 test_33() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     touch $DIR/$tfile
     fail_abort mds
     # this file should be gone, because the replay was aborted
@@ -725,7 +725,7 @@ test_34() {
     sleep 1 
     rm -f $DIR/$tfile
 
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     fail_abort mds
     kill -USR1 $pid
     [ -e $DIR/$tfile ] && return 1
@@ -753,7 +753,7 @@ run_test 35 "test recovery from llog for unlink op"
 # b=2432 resent cancel after replay uses wrong cookie,
 # so don't resend cancels
 test_36() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     touch $DIR/$tfile
     checkstat $DIR/$tfile
     facet_failover mds
@@ -775,7 +775,7 @@ test_37() {
     sleep 1 
     rmdir $DIR/$tfile
 
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     # clear the dmesg buffer so we only see errors from this recovery
     dmesg -c >/dev/null
     fail_abort mds
@@ -789,8 +789,8 @@ run_test 37 "abort recovery before client does replay (test mds_cleanup_orphans
 test_38() {
     createmany -o $DIR/$tfile-%d 800
     unlinkmany $DIR/$tfile-%d 0 400
-    replay_barrier mds
-    fail mds
+    replay_barrier $SINGLEMDS
+    fail $SINGLEMDS
     unlinkmany $DIR/$tfile-%d 400 400
     sleep 2
     $CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
@@ -799,9 +799,9 @@ run_test 38 "test recovery from unlink llog (test llog_gen_rec) "
 
 test_39() { # bug 4176
     createmany -o $DIR/$tfile-%d 800
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     unlinkmany $DIR/$tfile-%d 0 400
-    fail mds
+    fail $SINGLEMDS
     unlinkmany $DIR/$tfile-%d 400 400
     sleep 2
     $CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
@@ -894,11 +894,11 @@ run_test 42 "recovery after ost failure"
 
 # timeout in MDS/OST recovery RPC will LBUG MDS
 test_43() { # bug 2530
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
 
     # OBD_FAIL_OST_CREATE_NET 0x204
     do_facet ost1 "sysctl -w lustre.fail_loc=0x80000204"
-    fail mds
+    fail $SINGLEMDS
     sleep 10
     do_facet ost1 "sysctl -w lustre.fail_loc=0"
 
@@ -963,7 +963,7 @@ run_test 45 "Handle failed close"
 test_46() {
     dmesg -c >/dev/null
     drop_reply "touch $DIR/$tfile"
-    fail mds
+    fail $SINGLEMDS
     # ironically, the previous test, 45, will cause a real forced close,
     # so just look for one for this test
     dmesg | grep -i "force closing client file handle for $tfile" && return 1
@@ -995,10 +995,10 @@ test_47() { # bug 2824
 run_test 47 "MDS->OSC failure during precreate cleanup (2824)"
 
 test_48() {
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     createmany -o $DIR/$tfile 20  || return 1
     # OBD_FAIL_OST_EROFS 0x216
-    fail mds
+    fail $SINGLEMDS
     do_facet ost1 "sysctl -w lustre.fail_loc=0x80000216"
     df $MOUNT || return 2
 
@@ -1025,9 +1025,9 @@ test_52() {
     cancel_lru_locks mdc
 
     multiop $DIR/$tfile s
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     do_facet mds "sysctl -w lustre.fail_loc=0x8000030c"
-    fail mds
+    fail $SINGLEMDS
     do_facet mds "sysctl -w lustre.fail_loc=0x0"
 
     $CHECKSTAT -t file $DIR/$tfile-* && return 3 || true
@@ -1053,9 +1053,9 @@ run_test 55 "let MDS_CHECK_RESENT return the original return code instead of 0"
 #b3440 ASSERTION(rec->ur_fid2->id) failed
 test_56() {
     ln -s foo $DIR/$tfile
-    replay_barrier mds
+    replay_barrier $SINGLEMDS
     #drop_reply "cat $DIR/$tfile"
-    fail mds
+    fail $SINGLEMDS
     sleep 10
 }
 run_test 56 "don't replay a symlink open request (3440)"
@@ -1065,8 +1065,8 @@ test_57() {
 #define OBD_FAIL_MDS_OST_SETATTR       0x12c
     do_facet mds "sysctl -w lustre.fail_loc=0x8000012c"
     touch $DIR/$tfile
-    replay_barrier mds
-    fail mds
+    replay_barrier $SINGLEMDS
+    fail $SINGLEMDS
     sleep 1
     $CHECKSTAT -t file $DIR/$tfile || return 1
     do_facet mds "sysctl -w lustre.fail_loc=0x0"
@@ -1080,8 +1080,8 @@ test_58() {
     do_facet mds "sysctl -w lustre.fail_loc=0x8000012c"
     mkdir $DIR/$tdir
     createmany -o $DIR/$tdir/$tfile-%d 2500
-    replay_barrier mds
-    fail mds
+    replay_barrier $SINGLEMDS
+    fail $SINGLEMDS
     sleep 2
     $CHECKSTAT -t file $DIR/$tdir/$tfile-* || return 1
     do_facet mds "sysctl -w lustre.fail_loc=0x0"
diff --git a/lustre/tests/sanity-lmv.sh b/lustre/tests/sanity-lmv.sh
new file mode 100644
index 0000000..e20b1c1
--- /dev/null
+++ b/lustre/tests/sanity-lmv.sh
@@ -0,0 +1,388 @@
+#!/bin/bash
+#
+# Run select tests by setting ONLY, or as arguments to the script.
+# Skip specific tests by setting EXCEPT.
+#
+# e.g. ONLY="22 23" or ONLY="`seq 32 39`" or EXCEPT="31"
+set -e
+
+ONLY=${ONLY:-"$*"}
+# bug number for skipped test: 2108
+ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-""}
+# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
+#case `uname -r` in
+#2.6.*) ALWAYS_EXCEPT="$ALWAYS_EXCEPT 54c 55" # bug 3117
+#esac
+
+[ "$ALWAYS_EXCEPT$EXCEPT" ] && echo "Skipping tests: $ALWAYS_EXCEPT $EXCEPT"
+
+SRCDIR=`dirname $0`
+export PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$PATH
+export SECURITY=${SECURITY:-"null"}
+
+TMP=${TMP:-/tmp}
+FSTYPE=${FSTYPE:-ldiskfs}
+
+CHECKSTAT=${CHECKSTAT:-"checkstat -v"}
+CREATETEST=${CREATETEST:-createtest}
+LFS=${LFS:-lfs}
+LSTRIPE=${LSTRIPE:-"$LFS setstripe"}
+LFIND=${LFIND:-"$LFS find"}
+LVERIFY=${LVERIFY:-ll_dirstripe_verify}
+LCTL=${LCTL:-lctl}
+MCREATE=${MCREATE:-mcreate}
+OPENFILE=${OPENFILE:-openfile}
+OPENUNLINK=${OPENUNLINK:-openunlink}
+TOEXCL=${TOEXCL:-toexcl}
+TRUNCATE=${TRUNCATE:-truncate}
+MUNLINK=${MUNLINK:-munlink}
+SOCKETSERVER=${SOCKETSERVER:-socketserver}
+SOCKETCLIENT=${SOCKETCLIENT:-socketclient}
+IOPENTEST1=${IOPENTEST1:-iopentest1}
+IOPENTEST2=${IOPENTEST2:-iopentest2}
+PTLDEBUG=${PTLDEBUG:-0}
+
+export NAME=${NAME:-lmv}
+
+SAVE_PWD=$PWD
+
+LUSTRE=${LUSTRE:-`dirname $0`/..}
+. $LUSTRE/tests/test-framework.sh
+init_test_env $@
+. ${CONFIG:=$LUSTRE/tests/cfg/lmv.sh}
+
+if [ ! -z "$USING_KRB5" ]; then
+    $RUNAS krb5_login.sh || exit 1
+    $RUNAS -u $(($RUNAS_ID + 1)) krb5_login.sh || exit 1
+fi
+
+cleanup() {
+	echo -n "cln.."
+	cleanupall ${FORCE} $* || { echo "FAILed to clean up"; exit 20; }
+}
+CLEANUP=${CLEANUP:-:}
+
+setup() {
+	echo -n "mnt.."
+        load_modules
+	setupall || exit 10
+	echo "done"
+}
+
+SETUP=${SETUP:-:}
+
+log() {
+	echo "$*"
+	$LCTL mark "$*" 2> /dev/null || true
+}
+
+trace() {
+	log "STARTING: $*"
+	strace -o $TMP/$1.strace -ttt $*
+	RC=$?
+	log "FINISHED: $*: rc $RC"
+	return 1
+}
+TRACE=${TRACE:-""}
+
+check_kernel_version() {
+	VERSION_FILE=$LPROC/version
+	WANT_VER=$1
+	[ ! -f $VERSION_FILE ] && echo "can't find kernel version" && return 1
+	GOT_VER=$(awk '/kernel:/ {print $2}' $VERSION_FILE)
+	[ $GOT_VER -ge $WANT_VER ] && return 0
+	log "test needs at least kernel version $WANT_VER, running $GOT_VER"
+	return 1
+}
+
+_basetest() {
+    echo $*
+}
+
+basetest() {
+    IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
+}
+
+run_one() {
+	if ! grep -q $DIR /proc/mounts; then
+		$SETUP
+	fi
+	testnum=$1
+	message=$2
+	BEFORE=`date +%s`
+	log "== test $testnum: $message= `date +%H:%M:%S` ($BEFORE)"
+	export TESTNAME=test_$testnum
+	export tfile=f${testnum}
+	export tdir=d${base}
+	test_${testnum} || error "exit with rc=$?"
+	unset TESTNAME
+	pass "($((`date +%s` - $BEFORE))s)"
+	cd $SAVE_PWD
+	$CLEANUP
+}
+
+build_test_filter() {
+	[ "$ALWAYS_EXCEPT$EXCEPT$SANITY_EXCEPT" ] && \
+	    echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT $SANITY_EXCEPT`"
+
+        for O in $ONLY; do
+            eval ONLY_${O}=true
+        done
+        for E in $EXCEPT $ALWAYS_EXCEPT $SANITY_EXCEPT; do
+            eval EXCEPT_${E}=true
+        done
+}
+
+_basetest() {
+	echo $*
+}
+
+basetest() {
+	IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
+}
+
+run_test() {
+         export base=`basetest $1`
+         if [ "$ONLY" ]; then
+                 testname=ONLY_$1
+                 if [ ${!testname}x != x ]; then
+ 			run_one $1 "$2"
+ 			return $?
+                 fi
+                 testname=ONLY_$base
+                 if [ ${!testname}x != x ]; then
+                         run_one $1 "$2"
+                         return $?
+                 fi
+                 echo -n "."
+                 return 0
+ 	fi
+        testname=EXCEPT_$1
+        if [ ${!testname}x != x ]; then
+                 echo "skipping excluded test $1"
+                 return 0
+        fi
+        testname=EXCEPT_$base
+        if [ ${!testname}x != x ]; then
+                 echo "skipping excluded test $1 (base $base)"
+                 return 0
+        fi
+        run_one $1 "$2"
+ 	return $?
+}
+
+[ "$SANITYLOG" ] && rm -f $SANITYLOG || true
+
+error() { 
+	sysctl -w lustre.fail_loc=0
+	log "FAIL: $TESTNAME $@"
+	$LCTL dk $TMP/lustre-log-$TESTNAME.log
+	if [ "$SANITYLOG" ]; then
+		echo "FAIL: $TESTNAME $@" >> $SANITYLOG
+	else
+		exit 1
+	fi
+}
+
+pass() { 
+	echo PASS $@
+}
+
+mounted_lustre_filesystems() {
+    awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts
+}
+
+MOUNTED="`mounted_lustre_filesystems`"
+if [ -z "$MOUNTED" ]; then
+        formatall
+	setupall
+	MOUNTED="`mounted_lustre_filesystems`"
+	[ -z "$MOUNTED" ] && error "NAME=$NAME not mounted"
+	I_MOUNTED=yes
+fi
+
+[ `echo $MOUNT | wc -w` -gt 1 ] && error "NAME=$NAME mounted more than once"
+
+DIR=${DIR:-$MOUNT}
+[ -z "`echo $DIR | grep $MOUNT`" ] && echo "$DIR not in $MOUNT" && exit 99
+
+LOVNAME=`cat $LPROC/llite/*/lov/common_name | tail -n 1`
+OSTCOUNT=`cat $LPROC/lov/$LOVNAME/numobd`
+STRIPECOUNT=`cat $LPROC/lov/$LOVNAME/stripecount`
+STRIPESIZE=`cat $LPROC/lov/$LOVNAME/stripesize`
+ORIGFREE=`cat $LPROC/lov/$LOVNAME/kbytesavail`
+MAXFREE=${MAXFREE:-$((200000 * $OSTCOUNT))}
+MDS=$(\ls $LPROC/mdt 2> /dev/null | grep -v num_refs | tail -n 1)
+
+[ -f $DIR/d52a/foo ] && chattr -a $DIR/d52a/foo
+[ -f $DIR/d52b/foo ] && chattr -i $DIR/d52b/foo
+rm -rf $DIR/[Rdfs][1-9]*
+
+build_test_filter
+
+echo "preparing for tests involving mounts"
+EXT2_DEV=${EXT2_DEV:-$TMP/SANITY.LOOP}
+touch $EXT2_DEV
+mke2fs -j -F $EXT2_DEV 8000 > /dev/null
+echo # add a newline after mke2fs.
+
+umask 077
+
+test_1a() {
+	mkdir $DIR/1a0 || error 
+	createmany -o $DIR/1a0/f 4000
+	rmdir $DIR/1a0 && error
+	rm -rf $DIR/1a0 || error
+}
+run_test 1a " remove splitted dir ============================="
+
+test_1b() {
+	mkdir $DIR/1b0 || error
+	createmany -o $DIR/1b0/f 4000
+	find $DIR/1b0 -type f | xargs rm -f
+	NUM=`ls $DIR/1b0 | wc -l`
+	if [ $NUM -ne 0 ] ; then
+		echo "dir must be empty"
+		error
+	fi
+	touch $DIR/1b0/file0
+	touch $DIR/1b0/file1
+	touch $DIR/1b0/file2
+
+	echo "3 files left"
+	rmdir $DIR/1b0 && error
+	rm -f $DIR/1b0/file0
+
+	echo "2 files left"
+	rmdir $DIR/1b0 && error
+	rm -f $DIR/1b0/file1
+
+	echo "1 files left"
+	rmdir $DIR/1b0 && error
+	rm -f $DIR/1b0/file2
+
+	echo "0 files left"
+	rmdir $DIR/1b0 || error
+}
+run_test 1b " remove splitted dir ============================="
+
+test_1c() {
+	mkdir $DIR/1b1 || error
+	createmany -o $DIR/1b1/f 4000
+	find $DIR/1b1 -type f | xargs rm -f
+	NUM=`ls $DIR/1b1 | wc -l`
+	if [ $NUM -ne 0 ] ; then
+		echo "dir must be empty"
+		error
+	fi
+	touch $DIR/1b1/file0
+	touch $DIR/1b1/file1
+	touch $DIR/1b1/file2
+
+	ls $DIR/1b1/
+	log "3 files left"
+	rmdir $DIR/1b1 && error
+	rm -f $DIR/1b1/file0
+
+	ls $DIR/1b1/
+	log "2 files left"
+	rmdir $DIR/1b1 && error
+	rm -f $DIR/1b1/file1
+
+	ls $DIR/1b1/
+	log "1 files left"
+	rmdir $DIR/1b1 && error
+	rm -f $DIR/1b1/file2
+
+	ls $DIR/1b1/
+	log "0 files left"
+	rmdir $DIR/1b1 || error
+}
+run_test 1c " remove splitted cross-node dir ============================="
+
+test_2a() {
+	mkdir $DIR/2a0 || error 
+	createmany -o $DIR/2a0/f 5000
+	NUM=`ls $DIR/2a0 | wc -l`
+	echo "found $NUM files"
+	if [ $NUM -ne 5000 ]; then
+		echo "wrong number of files: $NUM"
+		error
+	fi
+	rm -rf $DIR/2a0 || error
+}
+run_test 2a " list splitted dir ============================="
+
+test_2b() {
+	mkdir $DIR/2b1 || error 
+	createmany -o $DIR/2b1/f 5000
+	$CLEAN
+	$START
+	statmany -l $DIR/2b1/f 5000 5000 || error
+	statmany -s $DIR/2b1/f 5000 5000 || error
+	rm -rf $DIR/2b1 || error
+}
+run_test 2b " list splitted dir after remount ============================="
+
+test_3a() {
+	mkdir $DIR/3a0 || error
+	for i in `seq 100`; do
+		mkdir $DIR/3a0/d${i} || error
+	done
+	createmany -o $DIR/3a0/f 5000 || error
+	rm -rf $DIR/3a0 || error
+}
+run_test 3a " dir splitting with cross-ref ============================="
+
+test_3b() {
+	mkdir $DIR/3b1 || error
+	createmany -m $DIR/3b1/f 5000 || error
+	rm -rf $DIR/3b1 || error
+}
+run_test 3b " dir splitting via createmany -m ============================="
+
+test_3c() {
+	mkdir $DIR/3c1 || error
+	echo "MDS nodes: $MDSCOUNT"
+	for j in `seq 3`; do
+		for i in `seq 10`; do
+			$LFS dirstripe $DIR/3c1/d-${j}-${i} $j || error
+			createmany -m $DIR/3c1/d-${j}-${i}/m 200 || error
+			createmany -o $DIR/3c1/d-${j}-${i}/o 200 || error
+		done
+	done
+	rm -rf $DIR/3c1 || error
+}
+
+run_test 3c " dir splitting via lfs stripe ============================="
+
+test_4a() {
+	let rr=0
+	while let "rr < 33000"; do
+		if let "rr % 2000 == 0"; then
+			echo "$rr"
+		fi
+		mkdir $DIR/4a1 || error
+		rm -rf $DIR/4a1
+		let "rr = rr + 1"
+	done
+}
+
+## this test is very time-consuming, don't run it by default
+#run_test 4a " FIDS/ nlink overflow test  ============================="
+
+
+TMPDIR=$OLDTMPDIR
+TMP=$OLDTMP
+HOME=$OLDHOME
+
+log "cleanup: ======================================================"
+if [ "`mount | grep ^$NAME`" ]; then
+	rm -rf $DIR/[Rdfs][1-9]*
+	if [ "$I_MOUNTED" = "yes" ]; then
+		sh llmountcleanup.sh || error
+	fi
+fi
+
+echo '=========================== finished ==============================='
+[ -f "$SANITYLOG" ] && cat $SANITYLOG && exit 1 || true
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index e13e76d..5fbcac3 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -231,8 +231,10 @@ send_sigint() {
 
 start_gss_daemons() {
     # starting on MDT
-    do_facet mds "$LSVCGSSD -v"
-    do_facet mds "$LGSSD -v"
+    for num in `seq $MDSCOUNT`; do
+        do_facet mds$num "$LSVCGSSD -v"
+        do_facet mds$num "$LGSSD -v"
+    done
     # starting on OSTs
     for num in `seq $OSTCOUNT`; do
         do_facet ost$num "$LSVCGSSD -v"
@@ -247,8 +249,10 @@ start_gss_daemons() {
     #
     # check daemons are running
     #
-    check_gss_daemon_facet mds lsvcgssd
-    check_gss_daemon_facet mds lgssd
+    for num in `seq $MDSCOUNT`; do
+        check_gss_daemon_facet mds$num lsvcgssd
+        check_gss_daemon_facet mds$num lgssd
+    done
     for num in `seq $OSTCOUNT`; do
         check_gss_daemon_facet ost$num lsvcgssd
     done
@@ -256,7 +260,9 @@ start_gss_daemons() {
 }
 
 stop_gss_daemons() {
-    send_sigint mds lsvcgssd lgssd
+    for num in `seq $MDSCOUNT`; do
+        send_sigint mds$num lsvcgssd lgssd
+    done
     for num in `seq $OSTCOUNT`; do
         send_sigint ost$num lsvcgssd
     done
@@ -641,21 +647,31 @@ ostdevname() {
     echo -n $DEVPTR
 }
 
+mdsdevname() {
+    num=$1
+    DEVNAME=MDSDEV$num
+    #if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
+    eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}}
+    echo -n $DEVPTR
+}
+
 ########
 ## MountConf setup
 
 stopall() {
     # make sure we are using the primary server, so test-framework will
     # be able to clean up properly.
-    activemds=`facet_active mds`
-    if [ $activemds != "mds" ]; then
-        fail mds
+    activemds=`facet_active mds1`
+    if [ $activemds != "mds1" ]; then
+        fail mds1
     fi
     
     # assume client mount is local 
     grep " $MOUNT " /proc/mounts && zconf_umount `hostname` $MOUNT $*
     grep " $MOUNT2 " /proc/mounts && zconf_umount `hostname` $MOUNT2 $*
-    stop mds -f
+    for num in `seq $MDSCOUNT`; do
+        stop mds$num -f
+    done
     for num in `seq $OSTCOUNT`; do
         stop ost$num -f
     done
@@ -668,18 +684,29 @@ cleanupall() {
     cleanup_krb5_env
 }
 
+mdsmkfsopts()
+{
+    local nr=$1
+    test $nr = 1 && echo -n $MDS_MKFS_OPTS || echo -n $MDSn_MKFS_OPTS
+}
+
 formatall() {
     stopall
+
     # We need ldiskfs here, may as well load them all
     load_modules
-    echo Formatting mds, osts
-    if $VERBOSE; then
-        add mds $MDS_MKFS_OPTS --reformat $MDSDEV || exit 10
-    else
-        add mds $MDS_MKFS_OPTS --reformat $MDSDEV > /dev/null || exit 10
-    fi
+    echo "Formatting mdts, osts"
+    for num in `seq $MDSCOUNT`; do
+        echo "Format mds$num: $(mdsdevname $num)"
+        if $VERBOSE; then
+            add mds$num `mdsmkfsopts $num` --reformat `mdsdevname $num` || exit 9
+        else
+            add mds$num `mdsmkfsopts $num` --reformat `mdsdevname $num` > /dev/null || exit 9
+        fi
+    done
 
     for num in `seq $OSTCOUNT`; do
+        echo "Format ost$num: $(ostdevname $num)"
         if $VERBOSE; then
             add ost$num $OST_MKFS_OPTS --reformat `ostdevname $num` || exit 10
         else
@@ -695,8 +722,11 @@ mount_client() {
 setupall() {
     load_modules
     init_krb5_env
-    echo Setup mdt, osts
-    start mds $MDSDEV $MDS_MOUNT_OPTS
+    echo "Setup mdts, osts"
+    for num in `seq $MDSCOUNT`; do
+        DEVNAME=`mdsdevname $num`
+        start mds$num $DEVNAME $MDS_MOUNT_OPTS
+    done
     for num in `seq $OSTCOUNT`; do
         DEVNAME=`ostdevname $num`
         start ost$num $DEVNAME $OST_MOUNT_OPTS
-- 
1.8.3.1