From 2dcd6d7cad48c6c037dd4b9a6db8eb463c66dd07 Mon Sep 17 00:00:00 2001
From: nathan <nathan>
Date: Thu, 23 Feb 2006 23:22:29 +0000
Subject: [PATCH] Branch b1_4_mountconf b=9858 regression test mods b=8007 need
 refcount around config_llog_data for arbitrary cleanup

---
 lustre/include/linux/obd_class.h |   2 +
 lustre/mgc/mgc_request.c         | 165 +++++++++++++++++++++++++++------------
 lustre/tests/acceptance-small.sh |  18 ++---
 lustre/tests/cfg/local.sh        |  37 ++++++---
 lustre/tests/llmountcleanup.sh   |   6 +-
 lustre/tests/llrmount.sh         |  43 ----------
 lustre/tests/local.sh            |  61 ++++++---------
 lustre/tests/replay-single.sh    | 137 ++++++++++++++++----------------
 lustre/tests/runtests            |   4 +-
 lustre/tests/sanity.sh           |   2 +-
 lustre/tests/sanityN.sh          |   2 +-
 lustre/tests/test-framework.sh   |  88 ++++++++-------------
 lustre/utils/mkfs_lustre.c       |   6 +-
 13 files changed, 287 insertions(+), 284 deletions(-)
 delete mode 100755 lustre/tests/llrmount.sh

diff --git a/lustre/include/linux/obd_class.h b/lustre/include/linux/obd_class.h
index a966a55..79c6ee7 100644
--- a/lustre/include/linux/obd_class.h
+++ b/lustre/include/linux/obd_class.h
@@ -130,6 +130,8 @@ struct config_llog_data {
         struct ldlm_res_id  cld_resid;
         struct config_llog_instance cld_cfg;
         struct list_head    cld_list_chain;
+        atomic_t            cld_refcount;
+        unsigned int        cld_stopping:1;
 };
 
 struct lustre_profile {
diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c
index 9cdc305..8aa55db1 100644
--- a/lustre/mgc/mgc_request.c
+++ b/lustre/mgc/mgc_request.c
@@ -76,14 +76,39 @@ int mgc_logname2resid(char *logname, struct ldlm_res_id *res_id)
 EXPORT_SYMBOL(mgc_logname2resid);
 
 /********************** config llog list **********************/
-DECLARE_MUTEX(config_llog_lock);
-struct list_head config_llog_list = LIST_HEAD_INIT(config_llog_list);
-
-/* Find log and take the global log sem.  I don't want mutliple processes
-   running process_log at once -- sounds like badness.  It actually might be
-   fine, as long as we're not trying to update from the same log
-   simultaneously (in which case we should use a per-log sem.) */
-static struct config_llog_data *config_log_get(char *logname, 
+static struct list_head config_llog_list = LIST_HEAD_INIT(config_llog_list);
+static spinlock_t       config_list_lock = SPIN_LOCK_UNLOCKED;
+
+static int config_log_get(struct config_llog_data *cld)
+{
+        ENTRY;
+        CDEBUG(D_MGC, "log %s refs %d\n", cld->cld_logname,
+               atomic_read(&cld->cld_refcount));
+        atomic_inc(&cld->cld_refcount);
+        if (cld->cld_stopping) {
+                atomic_dec(&cld->cld_refcount);
+                RETURN(1);
+        }
+        RETURN(0);
+}
+
+static void config_log_put(struct config_llog_data *cld)
+{
+        ENTRY;
+        CDEBUG(D_MGC, "log %s refs %d\n", cld->cld_logname,
+               atomic_read(&cld->cld_refcount));
+        if (atomic_dec_and_test(&cld->cld_refcount)) {
+                CDEBUG(D_MGC, "dropping config log %s\n", cld->cld_logname);
+                OBD_FREE(cld->cld_logname, strlen(cld->cld_logname) + 1);
+                if (cld->cld_cfg.cfg_instance != NULL)
+                        OBD_FREE(cld->cld_cfg.cfg_instance, 
+                                 strlen(cld->cld_cfg.cfg_instance) + 1);
+                OBD_FREE(cld, sizeof(*cld));
+        }
+        EXIT;
+}
+
+static struct config_llog_data *config_log_find(char *logname, 
                                                struct config_llog_instance *cfg)
 {
         struct list_head *tmp;
@@ -97,25 +122,25 @@ static struct config_llog_data *config_log_get(char *logname,
                         match_instance++;
         }
 
-        down(&config_llog_lock);
+        spin_lock(&config_list_lock);
         list_for_each(tmp, &config_llog_list) {
                 cld = list_entry(tmp, struct config_llog_data, cld_list_chain);
                 if (match_instance && 
                     strcmp(cfg->cfg_instance, cld->cld_cfg.cfg_instance) == 0) 
-                        return(cld);
+                        goto out_found;
                 
                 if (!match_instance && 
                     strcmp(logname, cld->cld_logname) == 0) 
-                        return(cld);
+                        goto out_found;
         }
-        up(&config_llog_lock);
+        spin_unlock(&config_list_lock);
+
         CERROR("can't get log %s\n", logname);
         return(ERR_PTR(-ENOENT));
-}
-
-static void config_log_put(void)
-{
-        up(&config_llog_lock);
+out_found:
+        atomic_inc(&cld->cld_refcount);
+        spin_unlock(&config_list_lock);
+        return(cld);
 }
 
 /* Add this log to our list of active logs. 
@@ -130,76 +155,71 @@ static int config_log_add(char *logname, struct config_llog_instance *cfg,
 
         CDEBUG(D_MGC, "adding config log %s:%s\n", logname, cfg->cfg_instance);
         
-        down(&config_llog_lock);
         OBD_ALLOC(cld, sizeof(*cld));
         if (!cld) 
-                GOTO(out, rc = -ENOMEM);
+                RETURN(-ENOMEM);
         OBD_ALLOC(cld->cld_logname, strlen(logname) + 1);
         if (!cld->cld_logname) { 
                 OBD_FREE(cld, sizeof(*cld));
-                GOTO(out, rc = -ENOMEM);
+                RETURN(-ENOMEM);
         }
         strcpy(cld->cld_logname, logname);
         cld->cld_cfg = *cfg;
         cld->cld_cfg.cfg_last_idx = 0;
         cld->cld_cfg.cfg_flags = 0;
         cld->cld_cfg.cfg_sb = sb;
+        atomic_set(&cld->cld_refcount, 1);
         if (cfg->cfg_instance != NULL) {
                 OBD_ALLOC(cld->cld_cfg.cfg_instance, 
                           strlen(cfg->cfg_instance) + 1);
                 strcpy(cld->cld_cfg.cfg_instance, cfg->cfg_instance);
         }
         mgc_logname2resid(logname, &cld->cld_resid);
+        spin_lock(&config_list_lock);
         list_add(&cld->cld_list_chain, &config_llog_list);
-out:
-        up(&config_llog_lock);
+        spin_unlock(&config_list_lock);
+        
         RETURN(rc);
 }
 
-/* Stop watching for updates on this log. 2 clients on the same node
-   may be at different gens, so we need different log info (eg. 
-   already mounted client is at gen 10, but must start a new client
-   from gen 0.)*/
+/* Stop watching for updates on this log. */
 static int config_log_end(char *logname, struct config_llog_instance *cfg)
 {       
         struct config_llog_data *cld;
         int rc = 0;
         ENTRY;
                                        
-        cld = config_log_get(logname, cfg);
+        cld = config_log_find(logname, cfg);
         if (IS_ERR(cld)) 
                 RETURN(PTR_ERR(cld));
+        /* drop the ref from the find */
+        config_log_put(cld);
 
-        OBD_FREE(cld->cld_logname, strlen(cld->cld_logname) + 1);
-        if (cld->cld_cfg.cfg_instance != NULL)
-                OBD_FREE(cld->cld_cfg.cfg_instance, 
-                         strlen(cfg->cfg_instance) + 1);
-
+        cld->cld_stopping = 1;
+        spin_lock(&config_list_lock);
         list_del(&cld->cld_list_chain);
-        OBD_FREE(cld, sizeof(*cld));
-        config_log_put();
-        CDEBUG(D_MGC, "dropped config log %s (%d)\n", logname, rc);
+        spin_unlock(&config_list_lock);
+        /* drop the start ref */
+        config_log_put(cld);
+        CDEBUG(D_MGC, "end config log %s (%d)\n", logname, rc);
         RETURN(rc);
 }
 
+/* Failsafe */
 static void config_log_end_all(void)
 {
         struct list_head *tmp, *n;
         struct config_llog_data *cld;
         ENTRY;
         
-        down(&config_llog_lock);
+        spin_lock(&config_list_lock);
         list_for_each_safe(tmp, n, &config_llog_list) {
                 cld = list_entry(tmp, struct config_llog_data, cld_list_chain);
                 CERROR("conflog failsafe %s\n", cld->cld_logname);
-                OBD_FREE(cld->cld_logname, strlen(cld->cld_logname) + 1);
-                if (cld->cld_cfg.cfg_instance != NULL)
-                        OBD_FREE(cld->cld_cfg.cfg_instance, 
-                                 strlen(cld->cld_cfg.cfg_instance) + 1);
                 list_del(&cld->cld_list_chain);
-                OBD_FREE(cld, sizeof(*cld));
+                config_log_put(cld);
         }
-        up(&config_llog_lock);
+        spin_unlock(&config_list_lock);
         EXIT;
 }
 
@@ -350,11 +370,13 @@ static int mgc_async_requeue(void *data)
         struct l_wait_info  lwi;
         struct config_llog_data *cld = (struct config_llog_data *)data;
         unsigned long flags;
-        int rc;
+        int rc = 0;
         ENTRY;
 
         if (!data) 
                 RETURN(-EINVAL);
+        if (cld->cld_stopping) 
+                GOTO(out, rc = 0);
 
         lock_kernel();
         ptlrpc_daemonize();
@@ -389,6 +411,10 @@ static int mgc_async_requeue(void *data)
 #endif 
         rc = mgc_process_log(the_mgc, cld);
         class_export_put(the_mgc->obd_self_export);
+out:
+        /* Whether we enqueued again or not in mgc_process_log, 
+           we're done with the ref from the old mgc_blocking_ast */        
+        config_log_put(cld);                                                    
 
         RETURN(rc);
 }
@@ -398,6 +424,7 @@ static int mgc_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
                             void *data, int flag)
 {
         struct lustre_handle lockh;
+        struct config_llog_data *cld = (struct config_llog_data *)data;
         int rc = 0;
         ENTRY;
 
@@ -421,31 +448,43 @@ static int mgc_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
                 if (!lock->l_conn_export ||
                     !lock->l_conn_export->exp_obd->u.cli.cl_conn_count) {
                         CDEBUG(D_MGC, "Disconnecting, don't requeue\n");
-                        break;
+                        goto out_drop;
                 }
                 if (lock->l_req_mode != lock->l_granted_mode) {
                         CERROR("original grant failed, won't requeue\n");
-                        break;
+                        goto out_drop;
                 }
                 if (!data) {
                         CERROR("missing data, won't requeue\n");
-                        break;
+                        goto out_drop;
+                }
+                if (cld->cld_stopping) {
+                        CERROR("stopping, won't requeue\n");
+                        goto out_drop;
                 }
 
                 /* Re-enqueue the lock in a separate thread, because we must
                    return from this fn before that lock can be taken. */
                 rc = kernel_thread(mgc_async_requeue, data,
                                    CLONE_VM | CLONE_FS);
-                if (rc < 0) 
+                if (rc < 0) {
                         CERROR("Cannot re-enqueue thread: %d\n", rc);
-                else 
+                } else {
                         rc = 0;
+                        break;
+                }
+out_drop:
+                /* Drop this here or in mgc_async_requeue,
+                   in either case, we're done with the reference
+                   after this. */
+                config_log_put(cld);    
                 break;
         }
         default:
                 LBUG();
         }
 
+
         if (rc) {
                 CERROR("%s CB failed %d:\n", flag == LDLM_CB_BLOCKING ? 
                        "blocking" : "cancel", rc);
@@ -468,6 +507,10 @@ static int mgc_enqueue(struct obd_export *exp, struct lov_stripe_md *lsm,
 
         CDEBUG(D_MGC, "Enqueue for %s (res "LPX64")\n", cld->cld_logname,
                cld->cld_resid.name[0]);
+                
+        /* We can only drop this when we drop the lock */
+        if (config_log_get(cld))
+                RETURN(ELDLM_LOCK_ABORTED);
 
         /* We need a callback for every lockholder, so don't try to
            ldlm_lock_match (see rev 1.1.2.11.2.47) */
@@ -819,6 +862,8 @@ out_closel:
         RETURN(rc);
 }
 
+DECLARE_MUTEX(llog_process_lock);
+
 /* Get a config log from the MGS and process it.
    This func is called for both clients and servers. */
 static int mgc_process_log(struct obd_device *mgc, 
@@ -828,10 +873,20 @@ static int mgc_process_log(struct obd_device *mgc,
         struct lustre_handle lockh;
         struct client_obd *cli = &mgc->u.cli;
         struct lvfs_run_ctxt saved;
-        struct lustre_sb_info *lsi = s2lsi(cld->cld_cfg.cfg_sb);
+        struct lustre_sb_info *lsi;
         int rc, rcl, flags = 0, must_pop = 0;
         ENTRY;
 
+        if (!cld || !cld->cld_cfg.cfg_sb) {
+                /* This should never happen */
+                CERROR("Missing cld, aborting log update\n");
+                RETURN(-EINVAL);
+        }
+        if (cld->cld_stopping) 
+                RETURN(0);
+
+        lsi = s2lsi(cld->cld_cfg.cfg_sb);
+
         CDEBUG(D_MGC, "Process log %s:%s from %d\n", cld->cld_logname, 
                cld->cld_cfg.cfg_instance, cld->cld_cfg.cfg_last_idx + 1);
 
@@ -841,6 +896,12 @@ static int mgc_process_log(struct obd_device *mgc,
                 RETURN(-EINVAL);
         }
 
+        /* I don't want mutliple processes running process_log at once -- 
+           sounds like badness.  It actually might be fine, as long as 
+           we're not trying to update from the same log
+           simultaneously (in which case we should use a per-log sem.) */
+        down(&llog_process_lock);
+
         /* Get the cfg lock on the llog */
         rcl = mgc_enqueue(mgc->u.cli.cl_mgc_mgsexp, NULL, LDLM_PLAIN, NULL, 
                           LCK_CR, &flags, NULL, NULL, NULL, 
@@ -898,6 +959,8 @@ static int mgc_process_log(struct obd_device *mgc,
                        "(%d) from the MGS.\n",
                        mgc->obd_name, cld->cld_logname, rc);
         }
+
+        up(&llog_process_lock);
         
         RETURN(rc);
 }
@@ -942,7 +1005,7 @@ static int mgc_process_config(struct obd_device *obd, obd_count len, void *buf)
                 /* We're only called through here on the initial mount */
                 config_log_add(logname, cfg, sb);
 
-                cld = config_log_get(logname, cfg);
+                cld = config_log_find(logname, cfg);
                 if (IS_ERR(cld)) {
                         rc = PTR_ERR(cld);
                 } else {
@@ -952,8 +1015,8 @@ static int mgc_process_config(struct obd_device *obd, obd_count len, void *buf)
                         cld->cld_cfg.cfg_flags |= CFG_F_MARKER;
 
                         rc = mgc_process_log(obd, cld);
+                        config_log_put(cld);
                 }
-                config_log_put();
                 break;       
         }
         case LCFG_LOG_END: {
diff --git a/lustre/tests/acceptance-small.sh b/lustre/tests/acceptance-small.sh
index c769134..8c11365 100755
--- a/lustre/tests/acceptance-small.sh
+++ b/lustre/tests/acceptance-small.sh
@@ -51,13 +51,13 @@ for NAME in $CONFIGS; do
 		sh rundbench 1
 		$DEBUG_ON
 		sh llmountcleanup.sh
-		sh llrmount.sh
+		sh llmount.sh
 		if [ $DB_THREADS -gt 1 ]; then
 			$DEBUG_OFF
 			sh rundbench $DB_THREADS
 			$DEBUG_ON
 			sh llmountcleanup.sh
-			sh llrmount.sh
+			sh llmount.sh
 		fi
 		rm -f /mnt/lustre/`hostname`/client.txt
 	fi
@@ -68,7 +68,7 @@ for NAME in $CONFIGS; do
 		bonnie++ -f -r 0 -s $(($SIZE / 1024)) -n 10 -u $UID -d $MOUNT
 		$DEBUG_ON
 		sh llmountcleanup.sh
-		sh llrmount.sh
+		sh llmount.sh
 	fi
 
 	IOZONE_OPTS="-i 0 -i 1 -i 2 -e -+d -r $RSIZE -s $SIZE"
@@ -79,14 +79,14 @@ for NAME in $CONFIGS; do
 		iozone $IOZONE_OPTS $IOZFILE
 		$DEBUG_ON
 		sh llmountcleanup.sh
-		sh llrmount.sh
+		sh llmount.sh
 
 		if [ "$O_DIRECT" != "no" -a "$IOZONE_DIR" != "no" ]; then
 			$DEBUG_OFF
 			iozone -I $IOZONE_OPTS $IOZFILE.odir
 			$DEBUG_ON
 			sh llmountcleanup.sh
-			sh llrmount.sh
+			sh llmount.sh
 		fi
 
 		SPACE=`df -P $MOUNT | tail -n 1 | awk '{ print $4 }'`
@@ -104,7 +104,7 @@ for NAME in $CONFIGS; do
 			iozone $IOZONE_OPTS -t $IOZ_THREADS $IOZFILE
 			$DEBUG_ON
 			sh llmountcleanup.sh
-			sh llrmount.sh
+			sh llmount.sh
 		elif [ $IOZVER -lt 3145 ]; then
 			VER=`iozone -v | awk '/Revision:/ { print $3 }'`
 			echo "iozone $VER too old for multi-thread test"
@@ -117,7 +117,7 @@ for NAME in $CONFIGS; do
 			-N $(($COUNT * 100)) $MOUNT/fsxfile
 		$DEBUG_ON
 		sh llmountcleanup.sh
-		sh llrmount.sh
+		sh llmount.sh
 	fi	
 
 	mkdir -p $MOUNT2
@@ -146,7 +146,7 @@ for NAME in $CONFIGS; do
 
 		$DEBUG_ON
 		sh llmountcleanup.sh
-		sh llrmount.sh
+		sh llmount.sh
 	fi
 
 	if [ "$LIBLUSTRE" != "no" ]; then
@@ -159,7 +159,7 @@ for NAME in $CONFIGS; do
 			$LIBLUSTRETESTS/sanity --target=$LIBLUSTRE_MOUNT_TARGET
 		fi
 		sh llmountcleanup.sh
-		#sh llrmount.sh
+		#sh llmount.sh
 	fi
 
 	mount | grep $MOUNT && sh llmountcleanup.sh
diff --git a/lustre/tests/cfg/local.sh b/lustre/tests/cfg/local.sh
index 66d48f6..0d10302 100644
--- a/lustre/tests/cfg/local.sh
+++ b/lustre/tests/cfg/local.sh
@@ -3,33 +3,44 @@ MDSNODE=${MDSNODE:-`hostname`}
 OSTNODE=${OSTNODE:-`hostname`}
 CLIENT=${CLIENT:-client}
 
+FSNAME=lustre
 mds_HOST=${mds_HOST:-$MDSNODE}
 mdsfailover_HOST=${mdsfailover_HOST}
+mgs_HOST=${mgs_HOST:-$MDSNODE}
 ost_HOST=${ost_HOST:-$OSTNODE}
 ost2_HOST=${ost2_HOST:-$ost_HOST}
 client_HOST=${client_HOST:-$CLIENT}
 NETTYPE=${NETTYPE:-tcp}
+MGSNID=`h2$NETTYPE $HOSTNAME`
 
-MOUNT=${MOUNT:-"/mnt/lustre"}
+MDSDEV=${MDSDEV:-$ROOT/tmp/${FSNAME}-mdt}
+MDSSIZE=${MDSSIZE:-100000}
+MDSOPT=${MDSOPT:-"--mountfsoptions=acl"}
+OSTDEV=${OSTDEV:-$ROOT/tmp/${FSNAME}-ost0}
+OSTSIZE=${OSTSIZE:-200000}
+OSTDEV2=${OSTDEV2:-$ROOT/tmp/${FSNAME}-ost1}
+FSTYPE=${FSTYPE:-ext3}
+
+MDS_MKFS_OPTS="--mgs --mdt --device-size=$MDSSIZE $MDSOPT"
+OST_MKFS_OPTS="--ost --device-size=$OSTSIZE --mgsnid=`h2$NETTYPE $HOSTNAME` $OSTOPT"
+OST2_MKFS_OPTS="--ost --device-size=$OSTSIZE --mgsnid=`h2$NETTYPE $HOSTNAME` $OSTOPT"
+
+MDS_MOUNT_OPTS="-o loop"
+OST_MOUNT_OPTS="-o loop"
+OST2_MOUNT_OPTS="-o loop"
+
+MOUNT=${MOUNT:-/mnt/${FSNAME}}
 MOUNT1=${MOUNT1:-$MOUNT}
 MOUNT2=${MOUNT2:-${MOUNT}2}
 DIR=${DIR:-$MOUNT}
 DIR2=${DIR2:-$MOUNT1}
-PTLDEBUG=${PTLDEBUG:-0x3f0400}
-SUBSYSTEM=${SUBSYSTEM:- 0xffb7e3ff}
-PDSH=${PDSH:-no_dsh}
+MOUNTOPT=${MOUNTOPT:-"user_xattr,acl"}
 
-MDSDEV=${MDSDEV:-$ROOT/tmp/mds1-`hostname`}
-MDSSIZE=${MDSSIZE:-100000}
-OSTDEV=${OSTDEV:-$ROOT/tmp/ost1-`hostname`}
-OSTSIZE=${OSTSIZE:-200000}
-FSTYPE=${FSTYPE:-ext3}
 TIMEOUT=${TIMEOUT:-20}
 UPCALL=${UPCALL:-DEFAULT}
-
-MDSOPT=${MDSOPT:-"user_xattr,acl"}
-CLIENTOPT=${CLIENTOPT:-"user_xattr,acl"}
-MOUNTOPT=${MOUNTOPT:-"user_xattr,acl"}
+PTLDEBUG=${PTLDEBUG:-0x3f0400}
+SUBSYSTEM=${SUBSYSTEM:- 0xffb7e3ff}
+PDSH=${PDSH:-no_dsh}
 
 STRIPE_BYTES=${STRIPE_BYTES:-1048576}
 STRIPES_PER_OBJ=${STRIPES_PER_OBJ:-0}
diff --git a/lustre/tests/llmountcleanup.sh b/lustre/tests/llmountcleanup.sh
index b0d703b..35dbadd 100755
--- a/lustre/tests/llmountcleanup.sh
+++ b/lustre/tests/llmountcleanup.sh
@@ -30,8 +30,12 @@ fi
 
 [ "$MOUNT2" ] && umount $MOUNT2
 
-${LCONF} $NOMOD $portals_opt $lustre_opt $node_opt --cleanup $@ \
+#${LCONF} $NOMOD $portals_opt $lustre_opt $node_opt --cleanup $@ \
     --dump $TMP/debug $conf_opt
+
+echo FIXME this must be umount of some sort
+exit 1
+
 rc=$?
 echo "lconf DONE"
 BUSY=`dmesg | grep -i destruct`
diff --git a/lustre/tests/llrmount.sh b/lustre/tests/llrmount.sh
deleted file mode 100755
index 434ef44..0000000
--- a/lustre/tests/llrmount.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/sh
-# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
-
-export PATH=`dirname $0`/../utils:$PATH
-
-LCONF=${LCONF:-lconf}
-NAME=${NAME:-local}
-LLMOUNT=${LLMOUNT:-llmount}
-
-config=$NAME.xml
-mkconfig=$NAME.sh
-
-if [ "$PORTALS" ]; then
-  portals_opt="--portals=$PORTALS"
-fi
-
-if [ "$LUSTRE" ]; then
-  lustre_opt="--lustre=$LUSTRE"
-fi
-
-if [ "$LDAPURL" ]; then
-    conf_opt="--ldapurl $LDAPURL --config $NAME"
-else
-    if [ ! -f $config -o $mkconfig -nt $config ]; then
-	sh $mkconfig $config || exit 1
-    fi
-    conf_opt="$config"
-fi    
-
-[ "$NODE" ] && node_opt="--node $NODE"
-[ "$DEBUG" ] && portals_opt="$portals_opt --ptldebug=$DEBUG"
-[ "$PTLDEBUG" ] && portals_opt="$portals_opt --ptldebug=$PTLDEBUG"
-
-${LCONF} $NOMOD $portals_opt $lustre_opt $node_opt $@ $conf_opt || {
-    # maybe acceptor error, dump tcp port usage
-    netstat -tpn
-    exit 2
-}
-
-
-if [ "$MOUNT2" ]; then
-	$LLMOUNT -v -o user_xattr,acl `hostname`:/mds1/client $MOUNT2 || exit 3
-fi
diff --git a/lustre/tests/local.sh b/lustre/tests/local.sh
index 0a8cc71..28748f2 100755
--- a/lustre/tests/local.sh
+++ b/lustre/tests/local.sh
@@ -4,29 +4,30 @@ export PATH=`dirname $0`/../utils:$PATH
 
 config=${1:-`basename $0 .sh`.xml}
 
-LMC="${LMC:-lmc} -m $config"
+LMC=echo 
 TMP=${TMP:-/tmp}
 
+FSNAME=lustre
 HOSTNAME=`hostname`
-MDSDEV=${MDSDEV:-$TMP/mds1-`hostname`}
+MDSDEV=${MDSDEV:-$TMP/mdt-${FSNAME}}
 MDSSIZE=${MDSSIZE:-400000}
-FSTYPE=${FSTYPE:-ext3}
-MOUNT=${MOUNT:-/mnt/lustre}
+MOUNT=${MOUNT:-/mnt/${FSNAME}}
 MOUNT2=${MOUNT2:-${MOUNT}2}
 NETTYPE=${NETTYPE:-tcp}
 [ "$ACCEPTOR_PORT" ] && PORT_OPT="--port $ACCEPTOR_PORT"
 
-OSTDEV=${OSTDEV:-$TMP/ost1-`hostname`}
+OSTDEV=${OSTDEV:-$TMP/ost0-${FSNAME}}
 OSTSIZE=${OSTSIZE:-400000}
+OSTDEV2=${OSTDEV2:-$TMP/ost1-${FSNAME}}
 
 MDS_MOUNT_OPTS="user_xattr,acl,${MDS_MOUNT_OPTS:-""}"
 CLIENTOPT="user_xattr,acl,${CLIENTOPT:-""}"
 
 # specific journal size for the ost, in MB
 JSIZE=${JSIZE:-0}
-[ "$JSIZE" -gt 0 ] && JARG="--journal_size $JSIZE"
+[ "$JSIZE" -gt 0 ] && OST_MKFS_OPTS=$OST_MKFS_OPTS" -J size=$JSIZE"
 MDSISIZE=${MDSISIZE:-0}
-[ "$MDSISIZE" -gt 0 ] && IARG="--inode_size $MDSISIZE"
+[ "$MDSISIZE" -gt 0 ] && MDS_MKFS_OPTS=$MDS_MKFS_OPTS" -i $MDSISIZE"
 
 STRIPE_BYTES=${STRIPE_BYTES:-1048576}
 STRIPES_PER_OBJ=1	# 0 means stripe over all OSTs
@@ -58,38 +59,28 @@ h2iib () {
 	esac
 }
 
-# create nodes
-${LMC} --add node --node $HOSTNAME || exit 10
-${LMC} --add net --node $HOSTNAME --nid `h2$NETTYPE $HOSTNAME` \
-    --nettype $NETTYPE $PORT_OPT || exit 11
-${LMC} --add net --node client --nid '*' --nettype $NETTYPE $PORT_OPT|| exit 12
+MGSNID=`h2$NETTYPE $HOSTNAME`
 
 # configure mds server
 [ "x$MDS_MOUNT_OPTS" != "x" ] &&
-    MDS_MOUNT_OPTS="--mountfsoptions $MDS_MOUNT_OPTS"
-
+    MDS_MOUNT_OPTS="--mountfsoptions=$MDS_MOUNT_OPTS"
+[ "x$MDS_MKFS_OPTS" != "x" ] &&
+    MDS_MOUNT_OPTS="--mkfsoptions=\"$MDS_MOUNT_OPTS\""
 [ "x$QUOTA_OPTS" != "x" ] &&
     QUOTA_OPTS="--quota $QUOTA_OPTS"
-    
-# configure mds server
-${LMC} --add mds --node $HOSTNAME --mds mds1 --fstype $FSTYPE \
-	--dev $MDSDEV $MDS_MOUNT_OPTS $QUOTA_OPTS\
-	--size $MDSSIZE $JARG $IARG $MDSOPT || exit 20
+[ ! -z "$mdsfailover_HOST" ] && MDS_FAIL_OPT="--failover=$mdsfailover_HOST"    
+
+MDS_OPTS="--mgs $MDS_FAIL_OPT --device-size=$MDSSIZE $MDS_MOUNT_OPTS $MDS_MKFS_OPTS"
+echo mkfs.lustre --mdt $MDS_OPTS --reformat $MDSDEV
 
 [ "x$OST_MOUNT_OPTS" != "x" ] &&
-    OST_MOUNT_OPTS="--mountfsoptions $OST_MOUNT_OPTS"
-
-# configure ost
-${LMC} --add lov --lov lov1 --mds mds1 --stripe_sz $STRIPE_BYTES \
-	--stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0 $LOVOPT || exit 20
-
-${LMC} --add ost --node $HOSTNAME --lov lov1 --fstype $FSTYPE \
-	--dev $OSTDEV $QUOTA_OPTS\
-	$OST_MOUNT_OPTS --size $OSTSIZE $JARG $OSTOPT || exit 30
-
-# create client config
-[ "x$CLIENTOPT" != "x" ] && CLIENTOPT="--clientoptions $CLIENTOPT"
-${LMC} --add mtpt --node $HOSTNAME --path $MOUNT \
-	--mds mds1 --lov lov1 $CLIENTOPT || exit 40
-${LMC} --add mtpt --node client --path $MOUNT2 \
-	--mds mds1 --lov lov1 $CLIENTOPT || exit 41
+    OST_MOUNT_OPTS="--mountfsoptions=$OST_MOUNT_OPTS"
+[ "x$OST_MKFS_OPTS" != "x" ] &&
+    OST_MOUNT_OPTS="--mkfsoptions=\"$OST_MOUNT_OPTS\""
+
+OST_OPTS="--mgsnid=`h2$NETTYPE $HOSTNAME` $OST_FAIL_OPT --device-size=$OSTSIZE $OST_MOUNT_OPTS $OST_MKFS_OPTS"
+echo mkfs.lustre --ost $OST_OPTS --reformat $OSTDEV
+
+OST2_OPTS="--mgsnid=`h2$NETTYPE $HOSTNAME` $OST_FAIL_OPT --device-size=$OSTSIZE $OST_MOUNT_OPTS $OST_MKFS_OPTS"
+echo mkfs.lustre --ost $OST2_OPTS --reformat $OSTDEV2
+
diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index 6aed831..cff5e11 100755
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -1,6 +1,7 @@
 #!/bin/sh
 
 set -e
+set -v
 
 #
 # This test needs to be run on the client
@@ -17,20 +18,6 @@ init_test_env $@
 # bug number: 2766 
 ALWAYS_EXCEPT="0b  $REPLAY_SINGLE_EXCEPT"
 
-gen_config() {
-    rm -f $XMLCONFIG
-    add_mds mds --dev $MDSDEV --size $MDSSIZE
-    if [ ! -z "$mdsfailover_HOST" ]; then
-	 add_mdsfailover mds --dev $MDSDEV --size $MDSSIZE
-    fi
-    
-    add_lov lov1 mds --stripe_sz $STRIPE_BYTES \
-	--stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
-    add_ost ost --lov lov1 --dev $OSTDEV --size $OSTSIZE
-    add_ost ost2 --lov lov1 --dev ${OSTDEV}-2 --size $OSTSIZE
-    add_client client mds --lov lov1 --path $MOUNT
-}
-
 build_test_filter
 
 cleanup() {
@@ -38,7 +25,7 @@ cleanup() {
     # be able to clean up properly.
     activemds=`facet_active mds`
     if [ $activemds != "mds" ]; then
-        fail mds
+        fail mds $MDS_MOUNT_OPTS
     fi
     zconf_umount `hostname` $MOUNT
     stop mds ${FORCE} $MDSLCONFARGS
@@ -56,13 +43,21 @@ SETUP=${SETUP:-"setup"}
 CLEANUP=${CLEANUP:-"cleanup"}
 
 setup() {
-    gen_config
-
-    start ost --reformat $OSTLCONFARGS 
-    start ost2 --reformat $OSTLCONFARGS 
+    stop mds || 1
+    stop ost || 1
+    stop ost2 || 1
+    add mds $MDS_MKFS_OPTS --reformat $MDSDEV
+    add ost $OST_MKFS_OPTS --reformat $OSTDEV
+    add ost2 $OST2_MKFS_OPTS --reformat $OSTDEV2
+    start mds $MDSDEV $MDS_MOUNT_OPTS
+    start ost $OSTDEV $OST_MOUNT_OPTS
+    start ost2 $OSTDEV2 $OST2_MOUNT_OPTS
     [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
-    start mds $MDSLCONFARGS --reformat
+
+    #add_lov lov1 mds --stripe_sz $STRIPE_BYTES --stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
+
     grep " $MOUNT " /proc/mounts || zconf_mount `hostname` $MOUNT
+    sleep 10
 }
 
 $SETUP
@@ -75,14 +70,14 @@ mkdir -p $DIR
 
 test_0() {
     replay_barrier mds
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
 }
 run_test 0 "empty replay"
 
 test_0b() {
     # this test attempts to trigger a race in the precreation code, 
     # and must run before any other objects are created on the filesystem
-    fail ost
+    fail ost $OST_MOUNT_OPTS
     createmany -o $DIR/$tfile 20 || return 1
     unlinkmany $DIR/$tfile 20 || return 2
 }
@@ -91,7 +86,7 @@ run_test 0b "ensure object created after recover exists. (3284)"
 test_1() {
     replay_barrier mds
     mcreate $DIR/$tfile
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     $CHECKSTAT -t file $DIR/$tfile || return 1
     rm $DIR/$tfile
 }
@@ -144,7 +139,7 @@ test_1a() {
 test_2a() {
     replay_barrier mds
     touch $DIR/$tfile
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     $CHECKSTAT -t file $DIR/$tfile || return 1
     rm $DIR/$tfile
 }
@@ -154,7 +149,7 @@ test_2b() {
     ./mcreate $DIR/$tfile
     replay_barrier mds
     touch $DIR/$tfile
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     $CHECKSTAT -t file $DIR/$tfile || return 1
     rm $DIR/$tfile
 }
@@ -164,7 +159,7 @@ test_3a() {
     replay_barrier mds
     mcreate $DIR/$tfile
     o_directory $DIR/$tfile
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     $CHECKSTAT -t file $DIR/$tfile || return 2
     rm $DIR/$tfile
 }
@@ -176,7 +171,7 @@ test_3b() {
     do_facet mds "sysctl -w lustre.fail_loc=0x80000114"
     touch $DIR/$tfile
     do_facet mds "sysctl -w lustre.fail_loc=0"
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     $CHECKSTAT -t file $DIR/$tfile && return 2
     return 0
 }
@@ -188,7 +183,7 @@ test_3c() {
     do_facet mds "sysctl -w lustre.fail_loc=0x80000128"
     touch $DIR/$tfile
     do_facet mds "sysctl -w lustre.fail_loc=0"
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
 
     $CHECKSTAT -t file $DIR/$tfile && return 2
     return 0
@@ -200,7 +195,7 @@ test_4() {
     for i in `seq 10`; do
         echo "tag-$i" > $DIR/$tfile-$i
     done 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     for i in `seq 10`; do
       grep -q "tag-$i" $DIR/$tfile-$i || error "$tfile-$i"
     done 
@@ -210,7 +205,7 @@ run_test 4 "|x| 10 open(O_CREAT)s"
 test_4b() {
     replay_barrier mds
     rm -rf $DIR/$tfile-*
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     $CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
 }
 run_test 4b "|x| rm 10 files"
@@ -222,7 +217,7 @@ test_5() {
     for i in `seq 220`; do
         echo "tag-$i" > $DIR/$tfile-$i
     done 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     for i in `seq 220`; do
       grep -q "tag-$i" $DIR/$tfile-$i || error "f1c-$i"
     done 
@@ -237,7 +232,7 @@ test_6() {
     replay_barrier mds
     mkdir $DIR/$tdir
     mcreate $DIR/$tdir/$tfile
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     $CHECKSTAT -t dir $DIR/$tdir || return 1
     $CHECKSTAT -t file $DIR/$tdir/$tfile || return 2
     sleep 2
@@ -248,7 +243,7 @@ run_test 6 "mkdir + contained create"
 test_6b() {
     replay_barrier mds
     rm -rf $DIR/$tdir
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     $CHECKSTAT -t dir $DIR/$tdir && return 1 || true 
 }
 run_test 6b "|X| rmdir"
@@ -257,7 +252,7 @@ test_7() {
     mkdir $DIR/$tdir
     replay_barrier mds
     mcreate $DIR/$tdir/$tfile
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     $CHECKSTAT -t dir $DIR/$tdir || return 1
     $CHECKSTAT -t file $DIR/$tdir/$tfile || return 2
     rm -fr $DIR/$tdir
@@ -269,7 +264,7 @@ test_8() {
     multiop $DIR/$tfile mo_c &
     MULTIPID=$!
     sleep 1
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     ls $DIR/$tfile
     $CHECKSTAT -t file $DIR/$tfile || return 1
     kill -USR1 $MULTIPID || return 2
@@ -282,7 +277,7 @@ test_9() {
     replay_barrier mds
     mcreate $DIR/$tfile
     local old_inum=`ls -i $DIR/$tfile | awk '{print $1}'`
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     local new_inum=`ls -i $DIR/$tfile | awk '{print $1}'`
 
     echo " old_inum == $old_inum, new_inum == $new_inum"
@@ -302,7 +297,7 @@ test_10() {
     replay_barrier mds
     mv $DIR/$tfile $DIR/$tfile-2
     rm -f $DIR/$tfile
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     $CHECKSTAT $DIR/$tfile && return 1
     $CHECKSTAT $DIR/$tfile-2 ||return 2
     rm $DIR/$tfile-2
@@ -318,7 +313,7 @@ test_11() {
     echo "new" > $DIR/$tfile
     grep new $DIR/$tfile 
     grep old $DIR/$tfile-2
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     grep new $DIR/$tfile || return 1
     grep old $DIR/$tfile-2 || return 2
 }
@@ -335,7 +330,7 @@ test_12() {
     kill -USR1 $pid
     wait $pid || return 1
 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     [ -e $DIR/$tfile ] && return 2
     return 0
 }
@@ -353,7 +348,7 @@ test_13() {
     chmod 0 $DIR/$tfile
     $CHECKSTAT -p 0 $DIR/$tfile
     replay_barrier mds
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     kill -USR1 $pid
     wait $pid || return 1
 
@@ -372,7 +367,7 @@ test_14() {
     kill -USR1 $pid || return 1
     wait $pid || return 2
 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     [ -e $DIR/$tfile ] && return 3
     return 0
 }
@@ -389,7 +384,7 @@ test_15() {
     kill -USR1 $pid
     wait $pid || return 2
 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     [ -e $DIR/$tfile ] && return 3
     touch $DIR/h11 || return 4
     return 0
@@ -402,7 +397,7 @@ test_16() {
     mcreate $DIR/$tfile
     munlink $DIR/$tfile
     mcreate $DIR/$tfile-2
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     [ -e $DIR/$tfile ] && return 1
     [ -e $DIR/$tfile-2 ] || return 2
     munlink $DIR/$tfile-2 || return 3
@@ -415,7 +410,7 @@ test_17() {
     pid=$!
     # give multiop a chance to open
     sleep 1 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     kill -USR1 $pid || return 1
     wait $pid || return 2
     $CHECKSTAT -t file $DIR/$tfile || return 3
@@ -435,7 +430,7 @@ test_18() {
     kill -USR1 $pid
     wait $pid || return 2
 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     [ -e $DIR/$tfile ] && return 3
     [ -e $DIR/$tfile-2 ] || return 4
     # this touch frequently fails
@@ -453,7 +448,7 @@ test_19() {
     echo "old" > $DIR/$tfile
     mv $DIR/$tfile $DIR/$tfile-2
     grep old $DIR/$tfile-2
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     grep old $DIR/$tfile-2 || return 2
 }
 run_test 19 "|X| mcreate, open, write, rename "
@@ -466,7 +461,7 @@ test_20() {
     sleep 1 
     rm -f $DIR/$tfile
 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     kill -USR1 $pid
     wait $pid || return 1
     [ -e $DIR/$tfile ] && return 2
@@ -483,7 +478,7 @@ test_21() {
     rm -f $DIR/$tfile
     touch $DIR/g11 || return 1
 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     kill -USR1 $pid
     wait $pid || return 2
     [ -e $DIR/$tfile ] && return 3
@@ -501,7 +496,7 @@ test_22() {
     replay_barrier mds
     rm -f $DIR/$tfile
 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     kill -USR1 $pid
     wait $pid || return 1
     [ -e $DIR/$tfile ] && return 2
@@ -519,7 +514,7 @@ test_23() {
     rm -f $DIR/$tfile
     touch $DIR/g11 || return 1
 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     kill -USR1 $pid
     wait $pid || return 2
     [ -e $DIR/$tfile ] && return 3
@@ -535,7 +530,7 @@ test_24() {
     sleep 1 
 
     replay_barrier mds
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     rm -f $DIR/$tfile
     kill -USR1 $pid
     wait $pid || return 1
@@ -552,7 +547,7 @@ test_25() {
     rm -f $DIR/$tfile
 
     replay_barrier mds
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     kill -USR1 $pid
     wait $pid || return 1
     [ -e $DIR/$tfile ] && return 2
@@ -573,7 +568,7 @@ test_26() {
     kill -USR1 $pid2
     wait $pid2 || return 1
 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     kill -USR1 $pid1
     wait $pid1 || return 2
     [ -e $DIR/$tfile-1 ] && return 3
@@ -593,7 +588,7 @@ test_27() {
     rm -f $DIR/$tfile-1
     rm -f $DIR/$tfile-2
 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     kill -USR1 $pid1
     wait $pid1 || return 1
     kill -USR1 $pid2
@@ -617,7 +612,7 @@ test_28() {
     kill -USR1 $pid2
     wait $pid2 || return 1
 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     kill -USR1 $pid1
     wait $pid1 || return 2
     [ -e $DIR/$tfile-1 ] && return 3
@@ -637,7 +632,7 @@ test_29() {
     rm -f $DIR/$tfile-1
     rm -f $DIR/$tfile-2
 
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     kill -USR1 $pid1
     wait $pid1 || return 1
     kill -USR1 $pid2
@@ -659,7 +654,7 @@ test_30() {
     rm -f $DIR/$tfile-2
 
     replay_barrier mds
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     kill -USR1 $pid1
     wait $pid1 || return 1
     kill -USR1 $pid2
@@ -681,7 +676,7 @@ test_31() {
 
     replay_barrier mds
     rm -f $DIR/$tfile-2
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     kill -USR1 $pid1
     wait $pid1 || return 1
     kill -USR1 $pid2
@@ -759,7 +754,7 @@ test_36() {
     replay_barrier mds
     touch $DIR/$tfile
     checkstat $DIR/$tfile
-    facet_failover mds
+    facet_failover mds $MDS_MOUNT_OPTS
     cancel_lru_locks MDC
     if dmesg | grep "unknown lock cookie"; then 
 	echo "cancel after replay failed"
@@ -793,7 +788,7 @@ test_38() {
     createmany -o $DIR/$tfile-%d 800
     unlinkmany $DIR/$tfile-%d 0 400
     replay_barrier mds
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     unlinkmany $DIR/$tfile-%d 400 400
     sleep 2
     $CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
@@ -804,7 +799,7 @@ test_39() { # bug 4176
     createmany -o $DIR/$tfile-%d 800
     replay_barrier mds
     unlinkmany $DIR/$tfile-%d 0 400
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     unlinkmany $DIR/$tfile-%d 400 400
     sleep 2
     $CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
@@ -824,7 +819,7 @@ test_40(){
     writeme -s $MOUNT/${tfile}-2 &
     WRITE_PID=$!
     sleep 1
-    facet_failover mds
+    facet_failover mds $MDS_MOUNT_OPTS
 #define OBD_FAIL_MDS_CONNECT_NET         0x117
     do_facet mds "sysctl -w lustre.fail_loc=0x80000117"
     kill -USR1 $PID
@@ -883,7 +878,7 @@ test_42() {
     unlinkmany $DIR/$tfile-%d 0 400
     DEBUG42=`sysctl -n lnet.debug`
     sysctl -w lnet.debug=-1
-    facet_failover ost
+    facet_failover ost $OST_MOUNT_OPTS
     
     # osc is evicted, fs is smaller (but only with failout OSTs (bug 7287)
     #blocks_after=`df -P $MOUNT | tail -n 1 | awk '{ print $2 }'`
@@ -902,7 +897,7 @@ test_43() { # bug 2530
 
     # OBD_FAIL_OST_CREATE_NET 0x204
     do_facet ost "sysctl -w lustre.fail_loc=0x80000204"
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     sleep 10
     do_facet ost "sysctl -w lustre.fail_loc=0"
 
@@ -948,7 +943,7 @@ run_test 45 "Handle failed close"
 test_46() {
     dmesg -c >/dev/null
     drop_reply "touch $DIR/$tfile"
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     # ironically, the previous test, 45, will cause a real forced close,
     # so just look for one for this test
     dmesg | grep -i "force closing client file handle for $tfile" && return 1
@@ -962,7 +957,7 @@ test_47() { # bug 2824
     createmany -o $DIR/$tfile 20  || return 1
 
     # OBD_FAIL_OST_CREATE_NET 0x204
-    fail ost
+    fail ost $OST_MOUNT_OPTS
     do_facet ost "sysctl -w lustre.fail_loc=0x80000204"
     df $MOUNT || return 2
 
@@ -983,7 +978,7 @@ test_48() {
     replay_barrier mds
     createmany -o $DIR/$tfile 20  || return 1
     # OBD_FAIL_OST_EROFS 0x216
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     do_facet ost "sysctl -w lustre.fail_loc=0x80000216"
     df $MOUNT || return 2
 
@@ -1012,7 +1007,7 @@ test_52() {
     multiop $DIR/$tfile s
     replay_barrier mds
     do_facet mds "sysctl -w lustre.fail_loc=0x8000030c"
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     do_facet mds "sysctl -w lustre.fail_loc=0x0"
 
     $CHECKSTAT -t file $DIR/$tfile-* && return 3 || true
@@ -1040,7 +1035,7 @@ test_56() {
     ln -s foo $DIR/$tfile
     replay_barrier mds
     #drop_reply "cat $DIR/$tfile"
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     sleep 10
 }
 run_test 56 "don't replay a symlink open request (3440)"
@@ -1051,7 +1046,7 @@ test_57() {
     do_facet mds "sysctl -w lustre.fail_loc=0x8000012c"
     touch $DIR/$tfile
     replay_barrier mds
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     sleep 1
     $CHECKSTAT -t file $DIR/$tfile || return 1
     do_facet mds "sysctl -w lustre.fail_loc=0x0"
@@ -1066,7 +1061,7 @@ test_58() {
     mkdir $DIR/$tdir
     createmany -o $DIR/$tdir/$tfile-%d 2500
     replay_barrier mds
-    fail mds
+    fail mds $MDS_MOUNT_OPTS
     sleep 2
     $CHECKSTAT -t file $DIR/$tdir/$tfile-* || return 1
     do_facet mds "sysctl -w lustre.fail_loc=0x0"
diff --git a/lustre/tests/runtests b/lustre/tests/runtests
index 75f8765..7ed88cb3 100755
--- a/lustre/tests/runtests
+++ b/lustre/tests/runtests
@@ -91,7 +91,7 @@ done
 [ "$ERROR" ] && fail "old and new files are different" $ERROR
 
 sh llmountcleanup.sh || exit 19
-sh llrmount.sh $OPTS || exit 20
+sh llmount.sh $OPTS || exit 20
 
 log "comparing previously copied files"
 for f in $FILES; do
@@ -102,7 +102,7 @@ done
 [ "$ERROR" ] && fail "old and new files are different on second diff" $ERROR
 
 sh llmountcleanup.sh || exit 19
-sh llrmount.sh $OPTS || exit 20
+sh llmount.sh $OPTS || exit 20
 
 log "removing $DST"
 rm -r $V $DST || fail "can't remove $DST" 37
diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index b413655..80fc412 100644
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -77,7 +77,7 @@ CLEAN=${CLEAN:-:}
 
 start() {
 	echo -n "mnt.."
-	sh llrmount.sh > /dev/null || exit 10
+	sh llmount.sh > /dev/null || exit 10
 	echo "done"
 }
 START=${START:-:}
diff --git a/lustre/tests/sanityN.sh b/lustre/tests/sanityN.sh
index 9b753a3..0de58a6 100644
--- a/lustre/tests/sanityN.sh
+++ b/lustre/tests/sanityN.sh
@@ -44,7 +44,7 @@ CLEAN=${CLEAN:-}
 
 start() {
 	echo -n "mnt.."
-	sh llrmount.sh > /dev/null || exit 10
+	sh llmount.sh > /dev/null || exit 10
 	echo "done"
 }
 START=${START:-}
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index dd63d65..2d568fe 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -2,6 +2,7 @@
 # vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
 
 set -e
+#set -vx
 
 export REFORMAT=""
 export VERBOSE=false
@@ -71,10 +72,15 @@ init_test_env() {
 start() {
     facet=$1
     shift
+    device=$1
+    shift
     active=`facet_active $facet`
-    do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \
-        --node ${active}_facet  --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
-        $@ $XMLCONFIG
+    echo "mount active=${active}, facet=${facet}"
+    mkdir -p /mnt/${facet}
+    do_facet ${facet} mount -t lustre $@ ${device} /mnt/${facet} 
+    #do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \
+    #    --node ${active}_facet  --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
+    #    $@ $XMLCONFIG
     RC=${PIPESTATUS[0]}
     if [ $RC -ne 0 ]; then
         # maybe acceptor error, dump tcp port usage
@@ -87,9 +93,12 @@ stop() {
     facet=$1
     active=`facet_active $facet`
     shift
-    do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \
-        --node ${active}_facet  --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
-        $@ --cleanup $XMLCONFIG
+    echo "mount active=${active}, facet=${facet}"
+    do_facet ${facet} umount $@ /mnt/${facet}
+    #do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \
+    #    --node ${active}_facet  --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
+    #    $@ --cleanup $XMLCONFIG
+    return 0
 }
 
 zconf_mount() {
@@ -104,12 +113,8 @@ zconf_mount() {
         OPTIONS="-o $MOUNTOPT"
     fi
 
-    if [ -x /sbin/mount.lustre ] ; then
-	do_node $client mount -t lustre $OPTIONS \
-		`facet_nid mgs`:/lustre-client $mnt || return 1
-    else
-	return 4
-    fi
+    do_node $client mount -t lustre $OPTIONS \
+	`facet_nid mgs`:/lustre-client $mnt || return 1
 
     [ -d /r ] && $LCTL modules > /r/tmp/ogdb-`hostname`
     return 0
@@ -118,9 +123,9 @@ zconf_mount() {
 zconf_umount() {
     client=$1
     mnt=$2
-    [ "$3" ] && force=-f
-    do_node $client umount $force  $mnt || :
-    do_node $client $LCONF --cleanup --nosetup --node client_facet $XMLCONFIG > /dev/null || :
+    [ "$3" ] && failover=-f
+    # force is the default for umount 
+    do_node $client umount $mnt
 }
 
 shutdown_facet() {
@@ -129,7 +134,7 @@ shutdown_facet() {
        $POWER_DOWN `facet_active_host $facet`
        sleep 2 
     elif [ "$FAILURE_MODE" = SOFT ]; then
-       stop $facet --force --failover --nomod
+       stop $facet -f
     fi
 }
 
@@ -176,7 +181,7 @@ facet_failover() {
     facet=$1
     echo "Failing $facet node `facet_active_host $facet`"
     shutdown_facet $facet
-    reboot_facet $facet
+    reboot_facet $*
     client_df &
     DFPID=$!
     echo "df pid is $DFPID"
@@ -213,7 +218,7 @@ mds_evict_client() {
 
 fail() {
     local facet=$1
-    facet_failover $facet
+    facet_failover $*
     df $MOUNT || error "post-failover df: $?"
 }
 
@@ -229,7 +234,8 @@ fail_abort() {
 }
 
 do_lmc() {
-    $LMC -m ${XMLCONFIG} $@
+    echo There is no lmc.  This is mountconf, baby.
+    exit 1
 }
 
 h2gm () {
@@ -341,48 +347,22 @@ do_facet() {
     do_node $HOST $@
 }
 
-add_mds() {
-    local MOUNT_OPTS
+add_facet() {
     local facet=$1
     shift
-    rm -f ${facet}active
-    [ "x$MDSOPT" != "x" ] && MOUNT_OPTS="--mountfsoptions $MDSOPT"
-    do_lmc --add mds --node ${facet}_facet --mds ${facet}_svc \
-    	--fstype $FSTYPE $* $MOUNT_OPTS
+    echo "add facet $facet: `facet_host $facet`"
+    do_lmc --add node --node ${facet}_facet $@ --timeout $TIMEOUT \
+        --lustre_upcall $UPCALL --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM
+    do_lmc --add net --node ${facet}_facet --nid `facet_nid $facet` \
+        --nettype lnet $PORT_OPT
 }
 
-add_mdsfailover() {
-    local MOUNT_OPTS
+add() {
     local facet=$1
     shift
-    add_facet ${facet}failover  --lustre_upcall $UPCALL
-    [ "x$MDSOPT" != "x" ] && MOUNT_OPTS="--mountfsoptions $MDSOPT"
-    do_lmc --add mds  --node ${facet}failover_facet --mds ${facet}_svc \
-    	--fstype $FSTYPE $* $MOUNT_OPTS
-}
-
-add_ost() {
-    facet=$1
-    shift
+    umount /mnt/${facet} || true
     rm -f ${facet}active
-    add_facet $facet
-    do_lmc --add ost --node ${facet}_facet --ost ${facet}_svc \
-    	--fstype $FSTYPE $* $OSTOPT
-}
-
-add_ostfailover() {
-    facet=$1
-    shift
-    add_facet ${facet}failover
-    do_lmc --add ost --failover --node ${facet}failover_facet \
-    	--ost ${facet}_svc --fstype $FSTYPE $* $OSTOPT
-}
-
-add_lov() {
-    lov=$1
-    mds_facet=$2
-    shift; shift
-    do_lmc --add lov --mds ${mds_facet}_svc --lov $lov $* $LOVOPT
+    mkfs.lustre $*
 }
 
 add_client() {
diff --git a/lustre/utils/mkfs_lustre.c b/lustre/utils/mkfs_lustre.c
index 1ca6a48..87254622 100644
--- a/lustre/utils/mkfs_lustre.c
+++ b/lustre/utils/mkfs_lustre.c
@@ -802,7 +802,7 @@ static inline void badopt(const char *opt, char *type)
 }
 
 int parse_opts(int argc, char *const argv[], struct mkfs_opts *mop,
-               char *mountopts)
+               char **mountopts)
 {
         static struct option long_opt[] = {
                 {"backfstype", 1, 0, 'b'},
@@ -944,7 +944,7 @@ int parse_opts(int argc, char *const argv[], struct mkfs_opts *mop,
                         mop->mo_ldd.ldd_flags &= ~LDD_F_SV_TYPE_MGS;
                         break;
                 case 'o':
-                        mountopts = optarg;
+                        *mountopts = optarg;
                         break;
                 case 'O':
                         mop->mo_ldd.ldd_flags |= LDD_F_SV_TYPE_OST;
@@ -1052,7 +1052,7 @@ int main(int argc, char *const argv[])
                 print_ldd("Read previous values", &(mop.mo_ldd));
 #endif
 
-        ret = parse_opts(argc, argv, &mop, mountopts);
+        ret = parse_opts(argc, argv, &mop, &mountopts);
         if (ret) 
                 goto out;
 
-- 
1.8.3.1