From c04dddd71d140c61f654681c1993f5bb1ded4f48 Mon Sep 17 00:00:00 2001
From: Li Wei <liwei@whamcloud.com>
Date: Tue, 12 Jun 2012 21:54:47 +0800
Subject: [PATCH] LU-1415 tests: Basic support for ZFS-based servers

This patch extends Test Framework to support formatting and mounting
ZFS-based servers.

Current global FSTYPE is no longer flexible enough, as different
facets can use different types of back ends.  Such "mixed-fstype"
configurations are firstly resulted from the incremental landing of
Orion changes.  For example, when we have OFD and ZFS OSD, although
OSTs will be able to use ZFS, MGS and MDTs can still only use LDiskFS.
Secondly, to reduce the test matrix, we may also want to have both
OSTs using LDiskFS and OSTs using ZFS in the same test cluster.
Hence, this patch makes back end file system type a per-facet
attribute.  The flexible configuration variables in local.sh are
reserved for users; all other scripts shall use facet_fstype()
instead.

The major differences between using an LDiskFS-based target and a
ZFS-based one are on the mkfs.lustre(8) command line:

  - ZFS-based targets shall have "--backfstype=zfs", while
    LDiskFS-based ones shall have "--backfstype=ldiskfs".

  - LDiskFS-specific "--mkfsoptions" arguments shall not be given to
    ZFS-based targets, and vice versa.

  - ZFS-based targets have different device specifications.  See
    mkfs.lustre(8).

In addition, we will make "--index" mandatory.  Naturally, format
options have to be generated for each facet dynamically, based on its
back end type, index, etc.  This patch changes mkfs_opts() to do
exactly that.  All test scripts shall use mkfs_opts() instead of
reading related environment variables directly.

Similarly, mount options should also be generated on a per-facet, or
at least per-fstype, basis.  Nevertheless, this patch takes a shortcut
by keeping current per-facet-type {MGS,MDS,OST}_MOUNT_OPTSs and
generating the "loop" mount option dynamically in mount_facet().  I
believe this solution introduces fewer changes comparing to a pure
per-facet one and is sufficient for current use cases.

This patch is based on Brian Behlendorf's work under ORI-155.  See
http://review.whamcloud.com/1417.

Change-Id: Ifcce9b10179dd1b4992a30d10df13ea10bc34548
Whamcloud-bug-id: ORI-155
Test-Parameters: testgroup=full
Test-Parameters: testgroup=full envdefinitions=USE_OFD=yes,LOAD_MODULES_REMOTE=true
Signed-off-by: Li Wei <liwei@whamcloud.com>
Reviewed-on: http://review.whamcloud.com/2907
Tested-by: Hudson
Reviewed-by: Yu Jian <yujian@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
---
 lustre/tests/acceptance-small.sh |   4 +-
 lustre/tests/auster              |   4 +-
 lustre/tests/cfg/local.sh        |  93 +++---
 lustre/tests/conf-sanity.sh      | 222 ++++++++------
 lustre/tests/insanity.sh         |   4 +-
 lustre/tests/mmp.sh              |  14 +-
 lustre/tests/sanity-quota.sh     |  12 -
 lustre/tests/sanity.sh           |  19 +-
 lustre/tests/test-framework.sh   | 624 ++++++++++++++++++++++++++++-----------
 9 files changed, 663 insertions(+), 333 deletions(-)

diff --git a/lustre/tests/acceptance-small.sh b/lustre/tests/acceptance-small.sh
index e69ede2..a863f23 100755
--- a/lustre/tests/acceptance-small.sh
+++ b/lustre/tests/acceptance-small.sh
@@ -168,8 +168,8 @@ run_suites() {
 export NAME MOUNT START CLEAN
 . $LUSTRE/tests/cfg/$NAME.sh
 
-assert_env mds_HOST MDS_MKFS_OPTS 
-assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
+assert_env mds_HOST
+assert_env ost_HOST OSTCOUNT
 assert_env FSNAME MOUNT MOUNT2
 
 setup_if_needed
diff --git a/lustre/tests/auster b/lustre/tests/auster
index 7b69bb0..07e6f43 100755
--- a/lustre/tests/auster
+++ b/lustre/tests/auster
@@ -319,8 +319,8 @@ fi
 export NAME MOUNT START CLEAN
 . ${CONFIG:-$LUSTRE/tests/cfg/$NAME.sh}
 
-assert_env mds_HOST MDS_MKFS_OPTS
-assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
+assert_env mds_HOST
+assert_env ost_HOST OSTCOUNT
 assert_env FSNAME MOUNT MOUNT2
 
 echo "Started at `date`"
diff --git a/lustre/tests/cfg/local.sh b/lustre/tests/cfg/local.sh
index 121ad75..7a6184d 100644
--- a/lustre/tests/cfg/local.sh
+++ b/lustre/tests/cfg/local.sh
@@ -22,15 +22,36 @@ for num in $(seq $MDSCOUNT); do
 done
 MDSDEVBASE=${MDSDEVBASE:-$TMP/${FSNAME}-mdt}
 MDSSIZE=${MDSSIZE:-200000}
-MDSOPT=${MDSOPT:-"--mountfsoptions=user_xattr"}
+#
+# Format options of facets can be specified with these variables:
+#
+#   - <facet_type>OPT
+#
+# Arguments for "--mkfsoptions" shall be specified with these
+# variables:
+#
+#   - <fstype>_MKFS_OPTS
+#   - <facet_type>_FS_MKFS_OPTS
+#
+# A number of other options have their own specific variables.  See
+# mkfs_opts().
+#
+MDSOPT=${MDSOPT:-}
+MDS_FS_MKFS_OPTS=${MDS_FS_MKFS_OPTS:-}
+MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-}
 
 MGSDEV=${MGSDEV:-$MDSDEV1}
 MGSSIZE=${MGSSIZE:-$MDSSIZE}
+MGSOPT=${MGSOPT:-}
+MGS_FS_MKFS_OPTS=${MGS_FS_MKFS_OPTS:-}
+MGS_MOUNT_OPTS=${MGS_MOUNT_OPTS:-}
 
 OSTCOUNT=${OSTCOUNT:-2}
 OSTDEVBASE=${OSTDEVBASE:-$TMP/${FSNAME}-ost}
 OSTSIZE=${OSTSIZE:-200000}
-OSTOPT=""
+OSTOPT=${OSTOPT:-}
+OST_FS_MKFS_OPTS=${OST_FS_MKFS_OPTS:-}
+OST_MOUNT_OPTS=${OST_MOUNT_OPTS:-}
 # Can specify individual ost devs with
 # OSTDEV1="/dev/sda"
 # on specific hosts with
@@ -38,7 +59,30 @@ OSTOPT=""
 
 NETTYPE=${NETTYPE:-tcp}
 MGSNID=${MGSNID:-`h2$NETTYPE $mgs_HOST`}
+
+#
+# Back end file system type(s) of facets can be specified with these
+# variables:
+#
+#   1. <facet>_FSTYPE
+#   2. <facet_type>FSTYPE
+#   3. FSTYPE
+#
+# More specific ones override more general ones.  See facet_fstype().
+#
 FSTYPE=${FSTYPE:-ldiskfs}
+
+LDISKFS_MKFS_OPTS=${LDISKFS_MKFS_OPTS:-}
+ZFS_MKFS_OPTS=${ZFS_MKFS_OPTS:-}
+
+#
+# If any OST is "remote" and the non-default implementation (e.g.,
+# current OFD) is desired, then make sure that either a)
+# LOAD_MODULES_REMOTE is true or b) modprobe(8) is configured to
+# blacklist the undesired (and aliased the other, if necessary).
+#
+USE_OFD=${USE_OFD:-no}
+
 STRIPE_BYTES=${STRIPE_BYTES:-1048576}
 STRIPES_PER_OBJ=${STRIPES_PER_OBJ:-0}
 SINGLEMDS=${SINGLEMDS:-"mds1"}
@@ -60,51 +104,6 @@ QUOTA_TYPE="ug3"
 QUOTA_USERS=${QUOTA_USERS:-"quota_usr quota_2usr sanityusr sanityusr1"}
 LQUOTAOPTS=${LQUOTAOPTS:-"hash_lqs_cur_bits=3"}
 
-MKFSOPT=""
-[ "x$MDSJOURNALSIZE" != "x" ] &&
-    MKFSOPT=$MKFSOPT" -J size=$MDSJOURNALSIZE"
-[ "x$MDSISIZE" != "x" ] &&
-    MKFSOPT=$MKFSOPT" -i $MDSISIZE"
-[ "x$MKFSOPT" != "x" ] &&
-    MKFSOPT="--mkfsoptions=\\\"$MKFSOPT\\\""
-[ "x$SECLEVEL" != "x" ] &&
-    MKFSOPT=$MKFSOPT" --param mdt.sec_level=$SECLEVEL"
-[ "x$MDSCAPA" != "x" ] &&
-    MKFSOPT=$MKFSOPT" --param mdt.capa=$MDSCAPA"
-[ "x$mdsfailover_HOST" != "x" ] &&
-    MDSOPT=$MDSOPT" --failnode=`h2$NETTYPE $mdsfailover_HOST`"
-[ "x$STRIPE_BYTES" != "x" ] &&
-    MDSOPT=$MDSOPT" --param lov.stripesize=$STRIPE_BYTES"
-[ "x$STRIPES_PER_OBJ" != "x" ] &&
-    MDSOPT=$MDSOPT" --param lov.stripecount=$STRIPES_PER_OBJ"
-[ "x$L_GETIDENTITY" != "x" ] &&
-    MDSOPT=$MDSOPT" --param mdt.identity_upcall=$L_GETIDENTITY"
-
-MDS_MKFS_OPTS="--mdt --fsname=$FSNAME --device-size=$MDSSIZE --param sys.timeout=$TIMEOUT $MKFSOPT $MDSOPT $MDS_MKFS_OPTS"
-if [[ $mds1_HOST == $mgs_HOST ]] && [[ $MDSDEV1 == $MGSDEV ]]; then
-    MDS_MKFS_OPTS="--mgs $MDS_MKFS_OPTS"
-else
-    MDS_MKFS_OPTS="--mgsnode=$MGSNID $MDS_MKFS_OPTS"
-    MGS_MKFS_OPTS="--mgs --device-size=$MGSSIZE"
-fi
-
-MKFSOPT=""
-[ "x$OSTJOURNALSIZE" != "x" ] &&
-    MKFSOPT=$MKFSOPT" -J size=$OSTJOURNALSIZE"
-[ "x$MKFSOPT" != "x" ] &&
-    MKFSOPT="--mkfsoptions=\\\"$MKFSOPT\\\""
-[ "x$SECLEVEL" != "x" ] &&
-    MKFSOPT=$MKFSOPT" --param ost.sec_level=$SECLEVEL"
-[ "x$OSSCAPA" != "x" ] &&
-    MKFSOPT=$MKFSOPT" --param ost.capa=$OSSCAPA"
-[ "x$ostfailover_HOST" != "x" ] &&
-    OSTOPT=$OSTOPT" --failnode=`h2$NETTYPE $ostfailover_HOST`"
-OST_MKFS_OPTS="--ost --fsname=$FSNAME --device-size=$OSTSIZE --mgsnode=$MGSNID --param sys.timeout=$TIMEOUT $MKFSOPT $OSTOPT $OST_MKFS_OPTS"
-
-MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-"-o loop,user_xattr"}
-OST_MOUNT_OPTS=${OST_MOUNT_OPTS:-"-o loop"}
-MGS_MOUNT_OPTS=${MGS_MOUNT_OPTS:-$MDS_MOUNT_OPTS}
-
 #client
 MOUNT=${MOUNT:-/mnt/${FSNAME}}
 MOUNT1=${MOUNT1:-$MOUNT}
diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index b387f3b..701e58a 100644
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -49,18 +49,9 @@ if [ -n "$MDSSIZE" ]; then
 fi
 
 # pass "-E lazy_itable_init" to mke2fs to speed up the formatting time
-for facet in MGS MDS OST; do
-    opts=${facet}_MKFS_OPTS
-    if [[ ${!opts} != *lazy_itable_init* ]]; then
-        eval SAVED_${facet}_MKFS_OPTS=\"${!opts}\"
-        if [[ ${!opts} != *mkfsoptions* ]]; then
-            eval ${facet}_MKFS_OPTS=\"${!opts} --mkfsoptions='\\\"-E lazy_itable_init\\\"'\"
-        else
-            val=${!opts//--mkfsoptions=\\\"/--mkfsoptions=\\\"-E lazy_itable_init }
-            eval ${facet}_MKFS_OPTS='${val}'
-        fi
-    fi
-done
+if [[ "$LDISKFS_MKFS_OPTS" != *lazy_itable_init* ]]; then
+	LDISKFS_MKFS_OPTS=$(csa_add "$LDISKFS_MKFS_OPTS" -E lazy_itable_init)
+fi
 
 init_logging
 
@@ -590,7 +581,10 @@ test_17() {
 run_test 17 "Verify failed mds_postsetup won't fail assertion (2936) (should return errs)"
 
 test_18() {
-        [ "$FSTYPE" != "ldiskfs" ] && skip "not needed for FSTYPE=$FSTYPE" && return
+	if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
+		skip "Only applicable to ldiskfs-based MDTs"
+		return
+	fi
 
         local MDSDEV=$(mdsdevname ${SINGLEMDS//mds/})
 
@@ -624,15 +618,9 @@ test_18() {
 
 
         echo "mount mds with large journal..."
-        local OLD_MDS_MKFS_OPTS=$MDS_MKFS_OPTS
 
-        local opts="--mdt --fsname=$FSNAME --device-size=$myMDSSIZE --param sys.timeout=$TIMEOUT $MDSOPT"
-
-        if combined_mgs_mds ; then
-            MDS_MKFS_OPTS="--mgs $opts"
-        else
-            MDS_MKFS_OPTS="--mgsnode=$MGSNID $opts"
-        fi
+	local OLD_MDSSIZE=$MDSSIZE
+	MDSSIZE=$myMDSSIZE
 
         reformat_and_config
         echo "mount lustre system..."
@@ -649,8 +637,8 @@ test_18() {
 
         cleanup || return $?
 
-        MDS_MKFS_OPTS=$OLD_MDS_MKFS_OPTS
-        reformat_and_config
+	MDSSIZE=$OLD_MDSSIZE
+	reformat_and_config
 }
 run_test 18 "check mkfs creates large journals"
 
@@ -846,6 +834,10 @@ run_test 23b "Simulate -EINTR during mount"
 fs2mds_HOST=$mds_HOST
 fs2ost_HOST=$ost_HOST
 
+MDSDEV1_2=$fs2mds_DEV
+OSTDEV1_2=$fs2ost_DEV
+OSTDEV2_2=$fs3ost_DEV
+
 cleanup_24a() {
 	trap 0
 	echo "umount $MOUNT2 ..."
@@ -866,14 +858,18 @@ test_24a() {
 
 	[ -n "$ost1_HOST" ] && fs2ost_HOST=$ost1_HOST
 
-	local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
-	local fs2ostdev=${fs2ost_DEV:-$(ostdevname 1)_2}
+	local fs2mdsdev=$(mdsdevname 1_2)
+	local fs2ostdev=$(ostdevname 1_2)
+	local fs2mdsvdev=$(mdsvdevname 1_2)
+	local fs2ostvdev=$(ostvdevname 1_2)
 
 	# test 8-char fsname as well
 	local FSNAME2=test1234
-	add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME2} --nomgs --mgsnode=$MGSNID --reformat $fs2mdsdev || exit 10
+	add fs2mds $(mkfs_opts mds1) --nomgs --mgsnode=$MGSNID \
+		--fsname=${FSNAME2} --reformat $fs2mdsdev $fs2mdsvdev || exit 10
 
-	add fs2ost $OST_MKFS_OPTS --fsname=${FSNAME2} --reformat $fs2ostdev || exit 10
+	add fs2ost $(mkfs_opts ost1) --fsname=${FSNAME2} --reformat \
+		$fs2ostdev $fs2ostvdev || exit 10
 
 	setup
 	start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
@@ -916,9 +912,11 @@ test_24b() {
 		skip_env "mixed loopback and real device not working" && return
 	fi
 
-	local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
+	local fs2mdsdev=$(mdsdevname 1_2)
+	local fs2mdsvdev=$(mdsvdevname 1_2)
 
-	add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME}2 --mgs --reformat $fs2mdsdev || exit 10
+	add fs2mds $(mkfs_opts mds1) --mgs --fsname=${FSNAME}2 --reformat \
+		$fs2mdsdev $fs2mdsvdev || exit 10
 	setup
 	start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && return 2
 	cleanup || return 6
@@ -1343,6 +1341,7 @@ test_33a() { # bug 12333, was test_33
         local rc=0
         local FSNAME2=test-123
         local MDSDEV=$(mdsdevname ${SINGLEMDS//mds/})
+	local mkfsoptions
 
         [ -n "$ost1_HOST" ] && fs2ost_HOST=$ost1_HOST
 
@@ -1353,12 +1352,19 @@ test_33a() { # bug 12333, was test_33
                 skip_env "mixed loopback and real device not working" && return
         fi
 
-        combined_mgs_mds || mkfs_opts="$mkfs_opts --nomgs"
+	local fs2mdsdev=$(mdsdevname 1_2)
+	local fs2ostdev=$(ostdevname 1_2)
+	local fs2mdsvdev=$(mdsvdevname 1_2)
+	local fs2ostvdev=$(ostvdevname 1_2)
 
-        local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
-        local fs2ostdev=${fs2ost_DEV:-$(ostdevname 1)_2}
-        add fs2mds $MDS_MKFS_OPTS --mkfsoptions='\"-J size=8\"' --fsname=${FSNAME2} --reformat $fs2mdsdev || exit 10
-        add fs2ost $OST_MKFS_OPTS --fsname=${FSNAME2} --index=8191 --mgsnode=$MGSNID --reformat $fs2ostdev || exit 10
+	if [ $(facet_fstype mds1) == ldiskfs ]; then
+		mkfsoptions="--mkfsoptions=\\\"-J size=8\\\"" # See bug 17931.
+	fi
+
+	add fs2mds $(mkfs_opts mds1) --fsname=${FSNAME2} --reformat \
+		$mkfsoptions $fs2mdsdev $fs2mdsvdev || exit 10
+	add fs2ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
+		--index=8191 --reformat $fs2ostdev $fs2ostvdev || exit 10
 
         start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
         start fs2ost $fs2ostdev $OST_MOUNT_OPTS
@@ -1373,7 +1379,6 @@ test_33a() { # bug 12333, was test_33
         umount -d $MOUNT2
         stop fs2ost -f
         stop fs2mds -f
-        rm -rf $MOUNT2 $fs2mdsdev $fs2ostdev
         cleanup_nocli || rc=6
         return $rc
 }
@@ -1586,14 +1591,21 @@ test_36() { # 12743
 		skip_env "mixed loopback and real device not working" && return
         fi
 
-        local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
-        local fs2ostdev=${fs2ost_DEV:-$(ostdevname 1)_2}
-        local fs3ostdev=${fs3ost_DEV:-$(ostdevname 2)_2}
-        add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME2} --reformat $fs2mdsdev || exit 10
-        # XXX after we support non 4K disk blocksize, change following --mkfsoptions with
-        # other argument
-        add fs2ost $OST_MKFS_OPTS --mkfsoptions='-b4096' --fsname=${FSNAME2} --mgsnode=$MGSNID --reformat $fs2ostdev || exit 10
-        add fs3ost $OST_MKFS_OPTS --mkfsoptions='-b4096' --fsname=${FSNAME2} --mgsnode=$MGSNID --reformat $fs3ostdev || exit 10
+	local fs2mdsdev=$(mdsdevname 1_2)
+	local fs2ostdev=$(ostdevname 1_2)
+	local fs3ostdev=$(ostdevname 2_2)
+	local fs2mdsvdev=$(mdsvdevname 1_2)
+	local fs2ostvdev=$(ostvdevname 1_2)
+	local fs3ostvdev=$(ostvdevname 2_2)
+
+	add fs2mds $(mkfs_opts mds1) --fsname=${FSNAME2} --reformat \
+		$fs2mdsdev $fs2mdsvdev || exit 10
+	# XXX after we support non 4K disk blocksize in ldiskfs, specify a
+	#     different one than the default value here.
+	add fs2ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
+		--reformat $fs2ostdev $fs2ostvdev || exit 10
+	add fs3ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
+		--reformat $fs3ostdev $fs3ostvdev || exit 10
 
         start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS
         start fs2ost $fs2ostdev $OST_MOUNT_OPTS
@@ -1636,7 +1648,6 @@ test_36() { # 12743
         stop fs3ost -f || return 200
         stop fs2ost -f || return 201
         stop fs2mds -f || return 202
-        rm -rf $MOUNT2 $fs2mdsdev $fs2ostdev $fs3ostdev
         unload_modules_conf || return 203
         return $rc
 }
@@ -1646,6 +1657,13 @@ test_37() {
 	local mntpt=$(facet_mntpt $SINGLEMDS)
 	local mdsdev=$(mdsdevname ${SINGLEMDS//mds/})
 	local mdsdev_sym="$TMP/sym_mdt.img"
+	local opts=$MDS_MOUNT_OPTS
+	local rc=0
+
+	if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
+		skip "Currently only applicable to ldiskfs-based MDTs"
+		return
+	fi
 
 	echo "MDS :     $mdsdev"
 	echo "SYMLINK : $mdsdev_sym"
@@ -1655,8 +1673,11 @@ test_37() {
 
 	echo "mount symlink device - $mdsdev_sym"
 
-	local rc=0
-	mount_op=$(do_facet $SINGLEMDS mount -v -t lustre $MDS_MOUNT_OPTS  $mdsdev_sym $mntpt 2>&1 )
+	if ! do_facet $SINGLEMDS test -b $mdsdev; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
+	mount_op=$(do_facet $SINGLEMDS mount -v -t lustre $opts \
+		$mdsdev_sym $mntpt 2>&1)
 	rc=${PIPESTATUS[0]}
 
 	echo mount_op=$mount_op
@@ -1749,7 +1770,11 @@ test_40() { # bug 15759
 run_test 40 "race during service thread startup"
 
 test_41a() { #bug 14134
-        echo $MDS_MOUNT_OPTS | grep "loop" && skip " loop devices does not work with nosvc option" && return
+	if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&
+	   ! do_facet $SINGLEMDS test -b $(mdsdevname 1); then
+		skip "Loop devices does not work with nosvc option"
+		return
+	fi
 
         local rc
         local MDSDEV=$(mdsdevname ${SINGLEMDS//mds/})
@@ -1774,7 +1799,11 @@ test_41a() { #bug 14134
 run_test 41a "mount mds with --nosvc and --nomgs"
 
 test_41b() {
-        echo $MDS_MOUNT_OPTS | grep "loop" && skip " loop devices does not work with nosvc option" && return
+	if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&
+	   ! do_facet $SINGLEMDS test -b $(mdsdevname 1); then
+		skip "Loop devices does not work with nosvc option"
+		return
+	fi
 
         ! combined_mgs_mds && skip "needs combined mgs device" && return 0
 
@@ -2087,12 +2116,12 @@ run_test 48 "too many acls on file"
 
 # check PARAM_SYS_LDLM_TIMEOUT option of MKFS.LUSTRE
 test_49() { # bug 17710
-	local OLD_MDS_MKFS_OPTS=$MDS_MKFS_OPTS
-	local OLD_OST_MKFS_OPTS=$OST_MKFS_OPTS
+	local timeout_orig=$TIMEOUT
+	local ldlm_timeout_orig=$LDLM_TIMEOUT
 	local LOCAL_TIMEOUT=20
 
-
-	OST_MKFS_OPTS="--ost --fsname=$FSNAME --device-size=$OSTSIZE --mgsnode=$MGSNID --param sys.timeout=$LOCAL_TIMEOUT --param sys.ldlm_timeout=$LOCAL_TIMEOUT $MKFSOPT $OSTOPT"
+	LDLM_TIMEOUT=$LOCAL_TIMEOUT
+	TIMEOUT=$LOCAL_TIMEOUT
 
 	reformat
 	setup_noconfig
@@ -2115,7 +2144,7 @@ test_49() { # bug 17710
 	stop_ost || return 2
 	stop_mds || return 3
 
-	OST_MKFS_OPTS="--ost --fsname=$FSNAME --device-size=$OSTSIZE --mgsnode=$MGSNID --param sys.timeout=$LOCAL_TIMEOUT --param sys.ldlm_timeout=$((LOCAL_TIMEOUT - 1)) $MKFSOPT $OSTOPT"
+	LDLM_TIMEOUT=$((LOCAL_TIMEOUT - 1))
 
 	reformat
 	setup_noconfig
@@ -2135,8 +2164,8 @@ test_49() { # bug 17710
 
 	cleanup || return $?
 
-	MDS_MKFS_OPTS=$OLD_MDS_MKFS_OPTS
-	OST_MKFS_OPTS=$OLD_OST_MKFS_OPTS
+	LDLM_TIMEOUT=$ldlm_timeout_orig
+	TIMEOUT=$timeout_orig
 }
 run_test 49 "check PARAM_SYS_LDLM_TIMEOUT option of MKFS.LUSTRE"
 
@@ -2427,6 +2456,7 @@ test_52() {
 	local ost1mnt=$(facet_mntpt ost1)
 	local ost1node=$(facet_active_host ost1)
 	local ost1tmp=$TMP/conf52
+	local loop
 
 	mkdir -p $DIR/$tdir
 	[ $? -eq 0 ] || { error "Unable to create tdir"; return 4; }
@@ -2462,7 +2492,11 @@ test_52() {
 	echo mount ost1 as ldiskfs
 	do_node $ost1node mkdir -p $ost1mnt
 	[ $? -eq 0 ] || { error "Unable to create $ost1mnt"; return 23; }
-	do_node $ost1node mount -t $FSTYPE $ost1_dev $ost1mnt $OST_MOUNT_OPTS
+	if ! do_node $ost1node test -b $ost1_dev; then
+		loop="-o loop"
+	fi
+	do_node $ost1node mount -t $(facet_fstype ost1) $loop $ost1_dev \
+		$ost1mnt
 	[ $? -eq 0 ] || { error "Unable to mount ost1 as ldiskfs"; return 12; }
 
 	# backup objects
@@ -2624,19 +2658,19 @@ lov_objid_size()
 
 test_55() {
 	local mdsdev=$(mdsdevname 1)
-	local ostdev=$(ostdevname 1)
-	local saved_opts=$OST_MKFS_OPTS
+	local mdsvdev=$(mdsvdevname 1)
 
 	for i in 1023 2048
 	do
-		OST_MKFS_OPTS="$saved_opts --index $i"
-		reformat
-
+		add mds1 $(mkfs_opts mds1) --reformat $mdsdev $mdsvdev ||
+			exit 10
+		add ost1 $(mkfs_opts ost1) --index=$i --reformat \
+			$(ostdevname 1) $(ostvdevname 1)
 		setup_noconfig
 		stopall
-
 		setup_noconfig
 		sync
+
 		echo checking size of lov_objid for ost index $i
 		LOV_OBJID_SIZE=$(do_facet mds1 "$DEBUGFS -R 'stat lov_objid' $mdsdev 2>/dev/null" | grep ^User | awk '{print $6}')
 		if [ "$LOV_OBJID_SIZE" != $(lov_objid_size $i) ]; then
@@ -2647,15 +2681,19 @@ test_55() {
 		stopall
 	done
 
-	OST_MKFS_OPTS=$saved_opts
 	reformat
 }
 run_test 55 "check lov_objid size"
 
 test_56() {
-	add mds1 $MDS_MKFS_OPTS --mkfsoptions='\"-J size=16\"' --reformat $(mdsdevname 1)
-	add ost1 $OST_MKFS_OPTS --index=1000 --reformat $(ostdevname 1)
-	add ost2 $OST_MKFS_OPTS --index=10000 --reformat $(ostdevname 2)
+	local mds_journal_size_orig=$MDSJOURNALSIZE
+
+	MDSJOURNALSIZE=16
+	add mds1 $(mkfs_opts mds1) --reformat $(mdsdevname 1) $(mdsvdevname 1)
+	add ost1 $(mkfs_opts ost1) --index=1000 --reformat \
+		$(ostdevname 1) $(ostvdevname 1)
+	add ost2 $(mkfs_opts ost2) --index=10000 --reformat \
+		$(ostdevname 2) $(ostvdevname 2)
 
 	start_mgsmds
 	start_ost
@@ -2665,6 +2703,7 @@ test_56() {
 	$LFS osts
 	[ -n "$ENABLE_QUOTA" ] && { $LFS quotacheck -ug $MOUNT || error "quotacheck has failed" ; }
 	stopall
+	MDSJOURNALSIZE=$mds_journal_size_orig
 	reformat
 }
 run_test 56 "check big indexes"
@@ -2694,7 +2733,10 @@ count_osts() {
 }
 
 test_58() { # bug 22658
-	[ "$FSTYPE" != "ldiskfs" ] && skip "not supported for $FSTYPE" && return
+	if [ $(facet_fstype mds) == zfs ]; then
+		skip "Does not work with ZFS-based MDTs yet"
+		return
+	fi
 	setup_noconfig
 	mkdir -p $DIR/$tdir
 	createmany -o $DIR/$tdir/$tfile-%d 100
@@ -2748,7 +2790,14 @@ test_59() {
 run_test 59 "writeconf mount option"
 
 test_60() { # LU-471
-	add mds1 $MDS_MKFS_OPTS --mkfsoptions='\" -E stride=64 -O ^uninit_bg\"' --reformat $(mdsdevname 1)
+	if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
+		skip "Only applicable to ldiskfs-based MDTs"
+		return
+	fi
+
+	add mds1 $(mkfs_opts mds1) \
+		--mkfsoptions='\" -E stride=64 -O ^uninit_bg\"' --reformat \
+		$(mdsdevname 1) $(mdsvdevname 1) || exit 10
 
 	dump=$(do_facet $SINGLEMDS dumpe2fs $(mdsdevname 1))
 	rc=${PIPESTATUS[0]}
@@ -2765,17 +2814,19 @@ test_60() { # LU-471
 run_test 60 "check mkfs.lustre --mkfsoptions -E -O options setting"
 
 test_61() { # LU-80
-    local reformat=false
-
-    [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.1.53) ] ||
-        { skip "Need MDS version at least 2.1.53"; return 0; }
-
-    if ! large_xattr_enabled; then
-        reformat=true
-        local mds_dev=$(mdsdevname ${SINGLEMDS//mds/})
-        add $SINGLEMDS $MDS_MKFS_OPTS --mkfsoptions='\"-O large_xattr\"' \
-            --reformat $mds_dev || error "reformatting $mds_dev failed"
-    fi
+	local reformat=false
+
+	[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.1.53) ] ||
+		{ skip "Need MDS version at least 2.1.53"; return 0; }
+
+	if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&
+	   ! large_xattr_enabled; then
+		reformat=true
+		local mds_dev=$(mdsdevname ${SINGLEMDS//mds/})
+		LDISKFS_MKFS_OPTS+=" -O large_xattr"
+		add $SINGLEMDS $(mkfs_opts $SINGLEMDS) --reformat $mds_dev ||
+			error "reformatting $mds_dev failed"
+	fi
 
     setup_noconfig || error "setting up the filesystem failed"
     client_up || error "starting client failed"
@@ -2822,7 +2873,10 @@ test_61() { # LU-80
 
     rm -f $file
     stopall
-    $reformat && reformat
+	if $reformat; then
+		LDISKFS_MKFS_OPTS=${LDISKFS_MKFS_OPTS% -O large_xattr}
+		reformat
+	fi
 }
 run_test 61 "large xattr"
 
@@ -2848,13 +2902,5 @@ fi
 
 cleanup_gss
 
-# restore the ${facet}_MKFS_OPTS variables
-for facet in MGS MDS OST; do
-    opts=SAVED_${facet}_MKFS_OPTS
-    if [[ -n ${!opts} ]]; then
-        eval ${facet}_MKFS_OPTS=\"${!opts}\"
-    fi
-done
-
 complete $(basename $0) $SECONDS
 exit_status
diff --git a/lustre/tests/insanity.sh b/lustre/tests/insanity.sh
index f0cb403..e113e34 100755
--- a/lustre/tests/insanity.sh
+++ b/lustre/tests/insanity.sh
@@ -30,8 +30,8 @@ SINGLECLIENT=${SINGLECLIENT:-$HOSTNAME}
 LIVE_CLIENT=${LIVE_CLIENT:-$SINGLECLIENT}
 FAIL_CLIENTS=${FAIL_CLIENTS:-$RCLIENTS}
 
-assert_env mds_HOST MDS_MKFS_OPTS
-assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
+assert_env mds_HOST
+assert_env ost_HOST OSTCOUNT
 assert_env LIVE_CLIENT FSNAME
 
 require_dsh_mds || exit 0
diff --git a/lustre/tests/mmp.sh b/lustre/tests/mmp.sh
index b99b4b6..59fede1 100755
--- a/lustre/tests/mmp.sh
+++ b/lustre/tests/mmp.sh
@@ -27,8 +27,6 @@ init_logging
 
 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
-[ "$MDSFSTYPE" != "ldiskfs" ] && skip "MDS not running ldiskfs" && exit 0
-[ "$OSTFSTYPE" != "ldiskfs" ] && skip "OST not running ldiskfs" && exit 0
 
 # unmount and cleanup the Lustre filesystem
 MMP_RESTORE_MOUNT=false
@@ -164,7 +162,17 @@ get_mmp_check_interval() {
 
 # Enable the MMP feature on the Lustre server targets.
 mmp_init() {
-    init_vars
+	init_vars
+
+	if [ $(facet_fstype $MMP_MDS) != ldiskfs ]; then
+		skip "Only applicable to ldiskfs-based MDTs"
+		exit
+	fi
+
+	if [ $(facet_fstype $MMP_OSS) != ldiskfs ]; then
+		skip "Only applicable to ldiskfs-based OSTs"
+		exit
+	fi
 
     # The MMP feature is automatically enabled by mkfs.lustre for
     # new file system at format time if failover is being used.
diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh
index 96d1ab0..da93c05 100644
--- a/lustre/tests/sanity-quota.sh
+++ b/lustre/tests/sanity-quota.sh
@@ -6,13 +6,6 @@
 # Run test by setting NOSETUP=true when ltest has setup env for us
 set -e
 
-#kernel 2.4.x doesn't support quota
-K_VER=`uname --kernel-release | cut -b 1-3`
-if [ $K_VER = "2.4" ]; then
-    echo "Kernel 2.4 doesn't support quota"
-    exit 0
-fi
-
 SRCDIR=`dirname $0`
 export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin
 
@@ -24,11 +17,6 @@ ONLY=${ONLY:-"$*"}
 ALWAYS_EXCEPT="10 12 $SANITY_QUOTA_EXCEPT"
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
 
-case `uname -r` in
-2.6*) FSTYPE=${FSTYPE:-ldiskfs};;
-*) error "unsupported kernel" ;;
-esac
-
 [ "$ALWAYS_EXCEPT$EXCEPT" ] && \
 	echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT`"
 
diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index b7201d7..5815b5f 100644
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -16,12 +16,6 @@ CPU=`awk '/model/ {print $4}' /proc/cpuinfo`
 #                                    buffer i/o errs             sock spc runas
 [ "$CPU" = "UML" ] && EXCEPT="$EXCEPT 27m 27n 27o 27p 27q 27r 31d 54a  64b 99a 99b 99c 99d 99e 99f 101a"
 
-case `uname -r` in
-2.4*) FSTYPE=${FSTYPE:-ext3} ;;
-2.6*) FSTYPE=${FSTYPE:-ldiskfs} ;;
-*) error "unsupported kernel" ;;
-esac
-
 SRCDIR=$(cd $(dirname $0); echo $PWD)
 export PATH=$PATH:/sbin
 
@@ -1357,9 +1351,6 @@ check_seq_oid()
         # compare lmm_object_id and lu_fid->oid
         [ $lmm_oid = ${fid[2]} ] || { error "OID mismatch"; return 2; }
 
-        [ "$FSTYPE" != "ldiskfs" ] &&
-                skip "cannot check filter fid FSTYPE=$FSTYPE" && return 0
-
         # check the trusted.fid attribute of the OST objects of the file
         local have_obdidx=false
         local stripe_nr=0
@@ -1372,6 +1363,11 @@ check_seq_oid()
                 local ost=$((obdidx + 1))
                 local dev=$(ostdevname $ost)
 
+		if [ $(facet_fstype ost$ost) != ldiskfs ]; then
+			echo "Currently only works with ldiskfs-based OSTs"
+			continue
+		fi
+
                 log "want: stripe:$stripe_nr ost:$obdidx oid:$oid/$hex seq:$seq"
 
                 #don't unmount/remount the OSTs if we don't need to do that
@@ -6938,7 +6934,10 @@ set_dir_limits () {
 	done
 }
 test_129() {
-	[ "$FSTYPE" != "ldiskfs" ] && skip "not needed for FSTYPE=$FSTYPE" && return 0
+	if [ "$(facet_type_fstype MDS)" != ldiskfs ]; then
+		skip "Only applicable to ldiskfs-based MDTs"
+		return
+	fi
 	remote_mds_nodsh && skip "remote MDS with nodsh" && return
 
 	EFBIG=27
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 80bac33..1562de6 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -129,6 +129,10 @@ init_test_env() {
     export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite
     export FSCK_MAX_ERR=4   # File system errors left uncorrected
 
+	export ZFS=${ZFS:-zfs}
+	export ZPOOL=${ZPOOL:-zpool}
+	export ZDB=${ZDB:-zdb}
+
     #[ -d /r ] && export ROOT=${ROOT:-/r}
     export TMP=${TMP:-$ROOT/tmp}
     export TESTSUITELOG=${TMP}/${TESTSUITE}.log
@@ -193,7 +197,6 @@ init_test_env() {
     export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate}
     [ ! -f "$LFS_MIGRATE" ] &&
         export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null)
-    export FSTYPE=${FSTYPE:-"ldiskfs"}
     export NAME=${NAME:-local}
     export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"}
     [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && \
@@ -272,12 +275,6 @@ version_code() {
 export LINUX_VERSION=$(uname -r | sed -e "s/[-.]/ /3" -e "s/ .*//")
 export LINUX_VERSION_CODE=$(version_code ${LINUX_VERSION//\./ })
 
-case `uname -r` in
-2.4.*) EXT=".o"; USE_QUOTA=no; [ ! "$CLIENTONLY" ] && FSTYPE=ext3;;
-    *) EXT=".ko"; USE_QUOTA=yes;;
-esac
-
-
 module_loaded () {
    /sbin/lsmod | grep -q "^\<$1\>"
 }
@@ -359,11 +356,27 @@ llite_lloop_enabled() {
 }
 
 load_modules_local() {
-    if [ -n "$MODPROBE" ]; then
-        # use modprobe
-        echo "Using modprobe to load modules"
-        return 0
-    fi
+	if [ "$USE_OFD" == yes ]; then
+		if module_loaded obdfilter; then
+			if ! $LUSTRE_RMMOD ldiskfs; then
+				echo "$HOSTNAME may still be using obdfilter.ko"
+				return 1
+			fi
+		fi
+	else
+		if module_loaded ofd; then
+			if ! $LUSTRE_RMMOD ldiskfs; then
+				echo "$HOSTNAME may still be using ofd.ko"
+				return 1
+			fi
+		fi
+	fi
+
+	if [ -n "$MODPROBE" ]; then
+		# use modprobe
+		echo "Using modprobe to load modules"
+		return 0
+	fi
 
     echo Loading modules from $LUSTRE
     load_module ../libcfs/libcfs/libcfs
@@ -388,21 +401,21 @@ load_modules_local() {
         grep -q crc16 $SYMLIST || { modprobe crc16 2>/dev/null || true; }
         grep -q -w jbd $SYMLIST || { modprobe jbd 2>/dev/null || true; }
         grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; }
-        if [ "$FSTYPE" = "ldiskfs" ]; then
-            grep -q exportfs_decode_fh $SYMLIST ||
-                { modprobe exportfs 2> /dev/null || true; }
-            load_module ../ldiskfs/ldiskfs/ldiskfs
-        fi
-        [ "$USE_QUOTA" = "yes" -a "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS
-        load_module mgs/mgs
-        load_module mds/mds
-        load_module mdd/mdd
-        load_module mdt/mdt
-        load_module lvfs/fsfilt_$FSTYPE
-        load_module cmm/cmm
-        load_module osd-ldiskfs/osd_ldiskfs
-        load_module ost/ost
-		if [ "x$USE_OFD" = "xyes" ]; then
+		if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then
+			grep -q exportfs_decode_fh $SYMLIST ||
+			{ modprobe exportfs 2> /dev/null || true; }
+			load_module ../ldiskfs/ldiskfs/ldiskfs
+		fi
+		[ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS
+		load_module mgs/mgs
+		load_module mds/mds
+		load_module mdd/mdd
+		load_module mdt/mdt
+		load_module lvfs/fsfilt_ldiskfs
+		load_module cmm/cmm
+		load_module osd-ldiskfs/osd_ldiskfs
+		load_module ost/ost
+		if [ "$USE_OFD" == yes ]; then
 			load_module ofd/ofd
 		else
 			load_module obdfilter/obdfilter
@@ -426,15 +439,15 @@ load_modules_local() {
 }
 
 load_modules () {
-    load_modules_local
-    # bug 19124
-    # load modules on remote nodes optionally
-    # lustre-tests have to be installed on these nodes
-    if $LOAD_MODULES_REMOTE ; then
-        local list=$(comma_list $(remote_nodes_list))
-        echo loading modules on $list
-        do_rpc_nodes $list load_modules 
-    fi
+	load_modules_local
+	# bug 19124
+	# load modules on remote nodes optionally
+	# lustre-tests have to be installed on these nodes
+	if $LOAD_MODULES_REMOTE ; then
+		local list=$(comma_list $(remote_nodes_list))
+		echo loading modules on $list
+		do_rpc_nodes $list load_modules_local
+	fi
 }
 
 check_mem_leak () {
@@ -451,18 +464,18 @@ check_mem_leak () {
 }
 
 unload_modules() {
-    wait_exit_ST client # bug 12845
+	wait_exit_ST client # bug 12845
 
-    $LUSTRE_RMMOD $FSTYPE || return 2
+	$LUSTRE_RMMOD ldiskfs || return 2
 
-    if $LOAD_MODULES_REMOTE ; then
-        local list=$(comma_list $(remote_nodes_list))
-        if [ ! -z $list ]; then
-            echo unloading modules on $list
-            do_rpc_nodes $list $LUSTRE_RMMOD $FSTYPE
-            do_rpc_nodes $list check_mem_leak
-        fi
-    fi
+	if $LOAD_MODULES_REMOTE ; then
+		local list=$(comma_list $(remote_nodes_list))
+		if [ ! -z $list ]; then
+			echo unloading modules on $list
+			do_rpc_nodes $list $LUSTRE_RMMOD ldiskfs
+			do_rpc_nodes $list check_mem_leak
+		fi
+	fi
 
     if grep -qe "/sbin/mount\.lustre" /proc/mounts; then
         umount /sbin/mount.lustre || true
@@ -588,18 +601,97 @@ cleanup_gss() {
     fi
 }
 
+facet_type() {
+	local facet=$1
+
+	echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9]\+//' |
+		tr '[:lower:]' '[:upper:]'
+}
+
+facet_number() {
+	local facet=$1
+
+	if [ $facet == mgs ]; then
+		return 1
+	fi
+
+	echo -n $facet | sed -e 's/^fs[0-9]\+//' | sed -e 's/^[a-z]\+//'
+}
+
+facet_fstype() {
+	local facet=$1
+	local var
+
+	var=${facet}_FSTYPE
+	if [ -n "${!var}" ]; then
+		echo -n ${!var}
+		return
+	fi
+
+	var=$(facet_type $facet)FSTYPE
+	if [ -n "${!var}" ]; then
+		echo -n ${!var}
+		return
+	fi
+
+	if [ -n "$FSTYPE" ]; then
+		echo -n $FSTYPE
+		return
+	fi
+
+	return 1
+}
+
+node_fstypes() {
+	local node=$1
+	local fstypes
+	local fstype
+	local facets=$(get_facets)
+	local facet
+
+	for facet in ${facets//,/ }; do
+		if [ $node == $(facet_host $facet) ] ||
+		   [ $node == "$(facet_failover_host $facet)" ]; then
+			fstype=$(facet_fstype $facet)
+			if [[ $fstypes != *$fstype* ]]; then
+				fstypes+="${fstypes:+,}$fstype"
+			fi
+		fi
+	done
+	echo -n $fstypes
+}
+
+devicelabel() {
+	local facet=$1
+	local dev=$2
+	local label
+	local fstype=$(facet_fstype $facet)
+
+	case $fstype in
+	ldiskfs)
+		label=$(do_facet ${facet} "$E2LABEL ${dev} 2>/dev/null");;
+	zfs)
+		label=$(do_facet ${facet} "$ZFS get -H -o value lustre:svname \
+		                           ${dev} 2>/dev/null");;
+	*)
+		error "unknown fstype!";;
+	esac
+
+	echo -n $label
+}
+
 mdsdevlabel() {
-    local num=$1
-    local device=`mdsdevname $num`
-    local label=`do_facet mds$num "e2label ${device}" | grep -v "CMD: "`
-    echo -n $label
+	local num=$1
+	local device=$(mdsdevname $num)
+	local label=$(devicelabel mds$num ${device} | grep -v "CMD: ")
+	echo -n $label
 }
 
 ostdevlabel() {
-    local num=$1
-    local device=`ostdevname $num`
-    local label=`do_facet ost$num "e2label ${device}" | grep -v "CMD: "`
-    echo -n $label
+	local num=$1
+	local device=$(ostdevname $num)
+	local label=$(devicelabel ost$num ${device} | grep -v "CMD: ")
+	echo -n $label
 }
 
 set_debug_size () {
@@ -669,20 +761,46 @@ mount_facets () {
 	done
 }
 
+#
+# Add argument "arg" (e.g., "loop") to the comma-separated list
+# of arguments for option "opt" (e.g., "-o") on command
+# line "opts" (e.g., "-o flock").
+#
+csa_add() {
+	local opts=$1
+	local opt=$2
+	local arg=$3
+	local opt_pattern="\([[:space:]]\+\|^\)$opt"
+
+	if echo "$opts" | grep -q $opt_pattern; then
+		opts=$(echo "$opts" | sed -e \
+			"s/$opt_pattern[[:space:]]*[^[:space:]]\+/&,$arg/")
+	else
+		opts+="${opts:+ }$opt $arg"
+	fi
+	echo -n "$opts"
+}
+
 mount_facet() {
 	local facet=$1
 	shift
 	local dev=$(facet_active $facet)_dev
 	local opt=${facet}_opt
 	local mntpt=$(facet_mntpt $facet)
+	local opts="${!opt} $@"
+
+	if [ $(facet_fstype $facet) == ldiskfs ] &&
+	   ! do_facet $facet test -b ${!dev}; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
 
-	echo "Starting ${facet}: ${!opt} $@ ${!dev} $mntpt"
+	echo "Starting ${facet}: $opts ${!dev} $mntpt"
 	# for testing LU-482 error handling in mount_facets() and test_0a()
 	if [ -f $TMP/test-lu482-trigger ]; then
 		RC=2
 	else
-		do_facet ${facet} "mkdir -p $mntpt; mount -t lustre ${!opt} \
-				   $@ ${!dev} $mntpt"
+		do_facet ${facet} "mkdir -p $mntpt; mount -t lustre $opts \
+		                   ${!dev} $mntpt"
 		RC=${PIPESTATUS[0]}
 	fi
 	if [ $RC -ne 0 ]; then
@@ -690,7 +808,7 @@ mount_facet() {
     else
         set_default_debug_facet $facet
 
-        label=$(do_facet ${facet} "$E2LABEL ${!dev}")
+		label=$(devicelabel ${facet} ${!dev})
         [ -z "$label" ] && echo no label for ${!dev} && exit 1
         eval export ${facet}_svc=${label}
         echo Started ${label}
@@ -1889,6 +2007,25 @@ facet_host() {
     echo -n ${!varname}
 }
 
+facet_failover_host() {
+	local facet=$1
+	local var
+
+	var=${facet}failover_HOST
+	if [ -n "${!var}" ]; then
+		echo ${!var}
+		return
+	fi
+
+	if [[ $facet == ost* ]]; then
+		var=ostfailover_HOST
+		if [ -n "${!var}" ]; then
+			echo ${!var}
+			return
+		fi
+	fi
+}
+
 facet_active() {
     local facet=$1
     local activevar=${facet}active
@@ -1998,13 +2135,15 @@ single_local_node () {
 
 # Outputs environment variable assignments that should be passed to remote nodes
 get_env_vars() {
-    local var
-    local value
+	local var
+	local value
 
-    for var in ${!MODOPTS_*}; do
-        value=${!var}
-        echo "${var}=\"$value\""
-    done
+	for var in ${!MODOPTS_*}; do
+		value=${!var}
+		echo -n " ${var}=\"$value\""
+	done
+
+	echo -n " USE_OFD=$USE_OFD"
 }
 
 do_nodes() {
@@ -2074,19 +2213,121 @@ add() {
 ostdevname() {
     num=$1
     DEVNAME=OSTDEV$num
-    #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
-    eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}}
+
+	local fstype=$(facet_fstype ost$num)
+
+	case $fstype in
+		ldiskfs )
+			#if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
+			eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};;
+		zfs )
+			#dataset name is independent of vdev device names
+			eval DEVPTR=${FSNAME}-ost${num}/ost${num};;
+		* )
+			error "unknown fstype!";;
+	esac
+
     echo -n $DEVPTR
 }
 
+ostvdevname() {
+	num=$1
+	DEVNAME=OSTDEV$num
+
+	local fstype=$(facet_fstype ost$num)
+
+	case $fstype in
+		ldiskfs )
+			# vdevs are not supported by ldiskfs
+			eval VDEVPTR="";;
+		zfs )
+			#if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
+			eval VDEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};;
+		* )
+			error "unknown fstype!";;
+	esac
+
+	echo -n $VDEVPTR
+}
+
 mdsdevname() {
     num=$1
     DEVNAME=MDSDEV$num
-    #if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
-    eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}}
+
+	local fstype=$(facet_fstype mds$num)
+
+	case $fstype in
+		ldiskfs )
+			#if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
+			eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};;
+		zfs )
+			#dataset name is independent of vdev device names
+			eval DEVPTR=${FSNAME}-mdt${num}/mdt${num};;
+		* )
+			error "unknown fstype!";;
+	esac
+
+	echo -n $DEVPTR
+}
+
+mdsvdevname() {
+	num=$1
+	DEVNAME=MDSDEV$num
+
+	local fstype=$(facet_fstype mds$num)
+
+	case $fstype in
+		ldiskfs )
+			# vdevs are not supported by ldiskfs
+			eval VDEVPTR="";;
+		zfs )
+			#if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
+			eval VDEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};;
+		* )
+			error "unknown fstype!";;
+	esac
+
+	echo -n $VDEVPTR
+}
+
+mgsdevname() {
+	DEVNAME=MGSDEV
+
+	local fstype=$(facet_fstype mds$num)
+
+	case $fstype in
+		ldiskfs )
+			#if $MGSDEV isn't defined, default is $MDSDEV1
+			eval DEVPTR=${!DEVNAME:=${MDSDEV1}};;
+		zfs )
+			#dataset name is independent of vdev device names
+			eval DEVPTR=${FSNAME}-mgs/mgs;;
+		* )
+			error "unknown fstype!";;
+	esac
+
     echo -n $DEVPTR
 }
 
+mgsvdevname() {
+	DEVNAME=MGSDEV
+
+	local fstype=$(facet_fstype mds$num)
+
+	case $fstype in
+		ldiskfs )
+			# vdevs are not supported by ldiskfs
+			eval VDEVPTR="";;
+		zfs )
+			#if $MGSDEV isn't defined, default is $MGSDEV1
+			eval VDEVPTR=${!DEVNAME:=${MDSDEV1}};;
+		* )
+			error "unknown fstype!";;
+	esac
+
+	echo -n $VDEVPTR
+}
+
 facet_mntpt () {
     local facet=$1
     [[ $facet = mgs ]] && combined_mgs_mds && facet="mds1"
@@ -2159,88 +2400,123 @@ combined_mgs_mds () {
     [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]]
 }
 
-mkfs_opts () {
-    local facet=$1
+lower() {
+	echo -n "$1" | tr '[:upper:]' '[:lower:]'
+}
 
-    local tgt=$(echo $facet | tr -d [:digit:] | tr "[:lower:]" "[:upper:]")
-    local optvar=${tgt}_MKFS_OPTS
-    local opt=${!optvar}
+upper() {
+	echo -n "$1" | tr '[:lower:]' '[:upper:]'
+}
 
-    # FIXME: ! combo  mgs/mds + mgsfailover is not supported yet
-    [[ $facet = mgs ]] && echo $opt && return
+mkfs_opts() {
+	local facet=$1
+	local type=$(facet_type $facet)
+	local index=$(($(facet_number $facet) - 1))
+	local fstype=$(facet_fstype $facet)
+	local opts
+	local fs_mkfs_opts
+	local var
+
+	if [ $type == MGS ] && combined_mgs_mds; then
+		return 1
+	fi
 
-    # 1.
-    # --failnode options 
-    local var=${facet}failover_HOST
-    if [ x"${!var}" != x ] && [ x"${!var}" != x$(facet_host $facet) ] ; then
-        local failnode=$(h2$NETTYPE ${!var})
-        failnode="--failnode=$failnode"
-        # options does not contain
-        # or contains wrong --failnode=
-        if [[ $opt != *${failnode}* ]]; then
-            opt=$(echo $opt | sed 's/--failnode=.* / /')
-            opt="$opt $failnode"
-        fi
-    fi
+	if [ $type == MGS ] || ( [ $type == MDS ] && combined_mgs_mds ); then
+		opts="--mgs"
+	else
+		opts="--mgsnode=$MGSNID"
+	fi
 
-    # 2.
-    # --mgsnode options
-    # no additional mkfs mds "--mgsnode" option for this configuration
-    if [[ $facet = mds ]] && combined_mgs_mds; then
-        echo $opt
-        return
-    fi
+	if [ $type != MGS ]; then
+		opts+=" --fsname=$FSNAME --$(lower ${type/MDS/MDT}) --index=$index"
+	fi
 
-    # additional mkfs "--mgsnode"
-    local mgsnode="--mgsnode=$MGSNID"
-    opt=${opt//$mgsnode }
-    for nid in ${MGSNID//:/ }; do
-        local mgsnode="--mgsnode=$nid"
-        # options does not contain
-        # --mgsnode=$nid
-        if [[ $opt != *${mgsnode}" "* ]]; then
-            opt="$opt --mgsnode=$nid"
-        fi
-    done
+	var=${facet}failover_HOST
+	if [ -n "${!var}" ] && [ ${!var} != $(facet_host $facet) ]; then
+		opts+=" --failnode=$(h2$NETTYPE ${!var})"
+	fi
+
+	opts+=${TIMEOUT:+" --param=sys.timeout=$TIMEOUT"}
+	opts+=${LDLM_TIMEOUT:+" --param=sys.ldlm_timeout=$LDLM_TIMEOUT"}
+
+	if [ $type == MDS ]; then
+		opts+=${SECLEVEL:+" --param=mdt.sec_level"}
+		opts+=${MDSCAPA:+" --param-mdt.capa=$MDSCAPA"}
+		opts+=${STRIPE_BYTES:+" --param=lov.stripesize=$STRIPE_BYTES"}
+		opts+=${STRIPES_PER_OBJ:+" --param=lov.stripecount=$STRIPES_PER_OBJ"}
+		opts+=${L_GETIDENTITY:+" --param=mdt.identity_upcall=$L_GETIDENTITY"}
+
+		if [ $fstype == ldiskfs ]; then
+			opts+=${IAMDIR:+" --iam-dir"}
+
+			fs_mkfs_opts+=${MDSJOURNALSIZE:+" -J size=$MDSJOURNALSIZE"}
+			fs_mkfs_opts+=${MDSISIZE:+" -i $MDSISIZE"}
+		fi
+	fi
+
+	if [ $type == OST ]; then
+		opts+=${SECLEVEL:+" --param=ost.sec_level"}
+		opts+=${OSSCAPA:+" --param=ost.capa=$OSSCAPA"}
+
+		if [ $fstype == ldiskfs ]; then
+			fs_mkfs_opts+=${OSTJOURNALSIZE:+" -J size=$OSTJOURNALSIZE"}
+		fi
+	fi
+
+	opts+=" --backfstype=$fstype"
 
-    echo $opt
+	var=${type}SIZE
+	if [ -n "${!var}" ]; then
+		opts+=" --device-size=${!var}"
+	fi
+
+	var=$(upper $fstype)_MKFS_OPTS
+	fs_mkfs_opts+=${!var:+" ${!var}"}
+
+	var=${type}_FS_MKFS_OPTS
+	fs_mkfs_opts+=${!var:+" ${!var}"}
+
+	if [ -n "${fs_mkfs_opts## }" ]; then
+		opts+=" --mkfsoptions=\\\"${fs_mkfs_opts## }\\\""
+	fi
+
+	var=${type}OPT
+	opts+=${!var:+" ${!var}"}
+
+	echo -n "$opts"
 }
 
 formatall() {
-    if [ "$IAMDIR" == "yes" ]; then
-        MDS_MKFS_OPTS="$MDS_MKFS_OPTS --iam-dir"
-    fi
+	local quiet
 
-    [ "$FSTYPE" ] && FSTYPE_OPT="--backfstype $FSTYPE"
+	if ! $VERBOSE; then
+		quiet=yes
+	fi
 
-    stopall
-    # We need ldiskfs here, may as well load them all
-    load_modules
-    [ "$CLIENTONLY" ] && return
-    echo Formatting mgs, mds, osts
-    if ! combined_mgs_mds ; then
-        add mgs $(mkfs_opts mgs) $FSTYPE_OPT --reformat $MGSDEV || exit 10
-    fi
+	stopall
+	# We need ldiskfs here, may as well load them all
+	load_modules
+	[ "$CLIENTONLY" ] && return
+	echo Formatting mgs, mds, osts
+	if ! combined_mgs_mds ; then
+		echo "Format mgs: $(mgsdevname)"
+		add mgs $(mkfs_opts mgs) --reformat $(mgsdevname) \
+			$(mgsvdevname) ${quiet:+>/dev/null} || exit 10
+		fi
 
-    for num in `seq $MDSCOUNT`; do
-        echo "Format mds$num: $(mdsdevname $num)"
-        if $VERBOSE; then
-            add mds$num $(mkfs_opts mds) $FSTYPE_OPT --reformat $(mdsdevname $num) || exit 10
-        else
-            add mds$num $(mkfs_opts mds) $FSTYPE_OPT --reformat $(mdsdevname $num) > /dev/null || exit 10
-        fi
-    done
+		for num in `seq $MDSCOUNT`; do
+			echo "Format mds$num: $(mdsdevname $num)"
+			add mds$num $(mkfs_opts mds$num) --reformat \
+			$(mdsdevname $num) $(mdsvdevname $num) \
+			${quiet:+>/dev/null} || exit 10
+		done
 
-    # the ost-s could have different OST_MKFS_OPTS
-    # because of different failnode-s
-    for num in `seq $OSTCOUNT`; do
-        echo "Format ost$num: $(ostdevname $num)"
-        if $VERBOSE; then
-            add ost$num $(mkfs_opts ost${num}) $FSTYPE_OPT --reformat `ostdevname $num` || exit 10
-        else
-            add ost$num $(mkfs_opts ost${num}) $FSTYPE_OPT --reformat `ostdevname $num` > /dev/null || exit 10
-        fi
-    done
+		for num in `seq $OSTCOUNT`; do
+			echo "Format ost$num: $(ostdevname $num)"
+			add ost$num $(mkfs_opts ost$num) --reformat \
+			$(ostdevname $num) $(ostvdevname ${num}) \
+			${quiet:+>/dev/null} || exit 10
+		done
 }
 
 mount_client() {
@@ -2320,7 +2596,7 @@ setupall() {
         echo $WRITECONF | grep -q "writeconf" && \
             writeconf_all
         if ! combined_mgs_mds ; then
-            start mgs $MGSDEV $MGS_MOUNT_OPTS
+			start mgs $(mgsdevname) $MGS_MOUNT_OPTS
         fi
 
         for num in `seq $MDSCOUNT`; do
@@ -2401,7 +2677,7 @@ init_facet_vars () {
     eval export ${facet}_opt=\"$@\"
 
     local dev=${facet}_dev
-    local label=$(do_facet ${facet} "$E2LABEL ${!dev}")
+	local label=$(devicelabel ${facet} ${!dev})
     [ -z "$label" ] && echo no label for ${!dev} && exit 1
 
     eval export ${facet}_svc=${label}
@@ -2441,7 +2717,7 @@ init_facets_vars () {
         done
     fi
 
-    combined_mgs_mds || init_facet_vars mgs $MGSDEV $MGS_MOUNT_OPTS
+	combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS
 
     remote_ost_nodsh && return
 
@@ -2479,7 +2755,7 @@ init_param_vars () {
     osc_ensure_active $SINGLEMDS $TIMEOUT
     osc_ensure_active client $TIMEOUT
 
-    if [ $QUOTA_AUTO -ne 0 ]; then
+	if [ $QUOTA_AUTO -ne 0 ] && [ "$USE_OFD" != yes ]; then
         if [ "$ENABLE_QUOTA" ]; then
             echo "enable quota as required"
             setup_quota $MOUNT || return 2
@@ -2653,12 +2929,14 @@ check_and_setup_lustre() {
         set_default_debug_nodes $(comma_list $(nodes_list))
     fi
 
-    init_gss
-    set_flavor_all $SEC
+	init_gss
+	if $GSS; then
+		set_flavor_all $SEC
+	fi
 
-    if [ "$ONLY" == "setup" ]; then
-        exit 0
-    fi
+	if [ "$ONLY" == "setup" ]; then
+		exit 0
+	fi
 }
 
 restore_mount () {
@@ -4790,32 +5068,44 @@ run_llverfs()
 }
 
 remove_mdt_files() {
-    local facet=$1
-    local mdtdev=$2
-    shift 2
-    local files="$@"
-    local mntpt=$(facet_mntpt $facet)
+	local facet=$1
+	local mdtdev=$2
+	shift 2
+	local files="$@"
+	local mntpt=$(facet_mntpt $facet)
+	local opts=$MDS_MOUNT_OPTS
 
-    echo "removing files from $mdtdev on $facet: $files"
-    mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $?
-    rc=0;
-    for f in $files; do
-        rm $mntpt/ROOT/$f || { rc=$?; break; }
-    done
-    umount -f $mntpt || return $?
-    return $rc
+	echo "removing files from $mdtdev on $facet: $files"
+	if [ $(facet_fstype $facet) == ldiskfs ] &&
+	   ! do_facet $facet test -b ${!dev}; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
+	mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
+		return $?
+	rc=0;
+	for f in $files; do
+		rm $mntpt/ROOT/$f || { rc=$?; break; }
+	done
+	umount -f $mntpt || return $?
+	return $rc
 }
 
 duplicate_mdt_files() {
-    local facet=$1
-    local mdtdev=$2
-    shift 2
-    local files="$@"
-    local mntpt=$(facet_mntpt $facet)
+	local facet=$1
+	local mdtdev=$2
+	shift 2
+	local files="$@"
+	local mntpt=$(facet_mntpt $facet)
+	local opts=$MDS_MOUNT_OPTS
 
-    echo "duplicating files on $mdtdev on $facet: $files"
-    mkdir -p $mntpt || return $?
-    mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $?
+	echo "duplicating files on $mdtdev on $facet: $files"
+	mkdir -p $mntpt || return $?
+	if [ $(facet_fstype $facet) == ldiskfs ] &&
+	   ! do_facet $facet test -b ${!dev}; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
+	mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
+		return $?
 
     do_umount() {
         trap 0
-- 
1.8.3.1