LU-1415 tests: Basic support for ZFS-based servers

This patch extends Test Framework to support formatting and mounting
ZFS-based servers.

Current global FSTYPE is no longer flexible enough, as different
facets can use different types of back ends.  Such "mixed-fstype"
configurations are firstly resulted from the incremental landing of
Orion changes.  For example, when we have OFD and ZFS OSD, although
OSTs will be able to use ZFS, MGS and MDTs can still only use LDiskFS.
Secondly, to reduce the test matrix, we may also want to have both
OSTs using LDiskFS and OSTs using ZFS in the same test cluster.
Hence, this patch makes back end file system type a per-facet
attribute.  The flexible configuration variables in local.sh are
reserved for users; all other scripts shall use facet_fstype()
instead.

The major differences between using an LDiskFS-based target and a
ZFS-based one are on the mkfs.lustre(8) command line:

  - ZFS-based targets shall have "--backfstype=zfs", while
    LDiskFS-based ones shall have "--backfstype=ldiskfs".

  - LDiskFS-specific "--mkfsoptions" arguments shall not be given to
    ZFS-based targets, and vice versa.

  - ZFS-based targets have different device specifications.  See
    mkfs.lustre(8).

In addition, we will make "--index" mandatory.  Naturally, format
options have to be generated for each facet dynamically, based on its
back end type, index, etc.  This patch changes mkfs_opts() to do
exactly that.  All test scripts shall use mkfs_opts() instead of
reading related environment variables directly.

Similarly, mount options should also be generated on a per-facet, or
at least per-fstype, basis.  Nevertheless, this patch takes a shortcut
by keeping current per-facet-type {MGS,MDS,OST}_MOUNT_OPTSs and
generating the "loop" mount option dynamically in mount_facet().  I
believe this solution introduces fewer changes comparing to a pure
per-facet one and is sufficient for current use cases.

This patch is based on Brian Behlendorf's work under ORI-155.  See
http://review.whamcloud.com/1417.

Change-Id: Ifcce9b10179dd1b4992a30d10df13ea10bc34548
Whamcloud-bug-id: ORI-155
Test-Parameters: testgroup=full
Test-Parameters: testgroup=full envdefinitions=USE_OFD=yes,LOAD_MODULES_REMOTE=true
Signed-off-by: Li Wei <liwei@whamcloud.com>
Reviewed-on: http://review.whamcloud.com/2907
Tested-by: Hudson
Reviewed-by: Yu Jian <yujian@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lustre/tests/acceptance-small.sh b/lustre/tests/acceptance-small.sh
index e69ede2..a863f23 100755 (executable)
--- a/lustre/tests/acceptance-small.sh
+++ b/lustre/tests/acceptance-small.sh
@@ -168,8 +168,8 @@ run_suites() {
 export NAME MOUNT START CLEAN
 . $LUSTRE/tests/cfg/$NAME.sh
 
 export NAME MOUNT START CLEAN
 . $LUSTRE/tests/cfg/$NAME.sh
 

-assert_env mds_HOST MDS_MKFS_OPTS 
-assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
+assert_env mds_HOST
+assert_env ost_HOST OSTCOUNT

 assert_env FSNAME MOUNT MOUNT2
 
 setup_if_needed
 assert_env FSNAME MOUNT MOUNT2
 
 setup_if_needed

diff --git a/lustre/tests/auster b/lustre/tests/auster
index 7b69bb0..07e6f43 100755 (executable)
--- a/lustre/tests/auster
+++ b/lustre/tests/auster
@@ -319,8 +319,8 @@ fi
 export NAME MOUNT START CLEAN
 . ${CONFIG:-$LUSTRE/tests/cfg/$NAME.sh}
 
 export NAME MOUNT START CLEAN
 . ${CONFIG:-$LUSTRE/tests/cfg/$NAME.sh}
 

-assert_env mds_HOST MDS_MKFS_OPTS
-assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
+assert_env mds_HOST
+assert_env ost_HOST OSTCOUNT

 assert_env FSNAME MOUNT MOUNT2
 
 echo "Started at `date`"
 assert_env FSNAME MOUNT MOUNT2
 
 echo "Started at `date`"

diff --git a/lustre/tests/cfg/local.sh b/lustre/tests/cfg/local.sh
index 121ad75..7a6184d 100644 (file)
--- a/lustre/tests/cfg/local.sh
+++ b/lustre/tests/cfg/local.sh
@@ -22,15 +22,36 @@ for num in $(seq $MDSCOUNT); do
 done
 MDSDEVBASE=${MDSDEVBASE:-$TMP/${FSNAME}-mdt}
 MDSSIZE=${MDSSIZE:-200000}
 done
 MDSDEVBASE=${MDSDEVBASE:-$TMP/${FSNAME}-mdt}
 MDSSIZE=${MDSSIZE:-200000}

-MDSOPT=${MDSOPT:-"--mountfsoptions=user_xattr"}
+#
+# Format options of facets can be specified with these variables:
+#
+#   - <facet_type>OPT
+#
+# Arguments for "--mkfsoptions" shall be specified with these
+# variables:
+#
+#   - <fstype>_MKFS_OPTS
+#   - <facet_type>_FS_MKFS_OPTS
+#
+# A number of other options have their own specific variables.  See
+# mkfs_opts().
+#
+MDSOPT=${MDSOPT:-}
+MDS_FS_MKFS_OPTS=${MDS_FS_MKFS_OPTS:-}
+MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-}

 
 MGSDEV=${MGSDEV:-$MDSDEV1}
 MGSSIZE=${MGSSIZE:-$MDSSIZE}
 
 MGSDEV=${MGSDEV:-$MDSDEV1}
 MGSSIZE=${MGSSIZE:-$MDSSIZE}

+MGSOPT=${MGSOPT:-}
+MGS_FS_MKFS_OPTS=${MGS_FS_MKFS_OPTS:-}
+MGS_MOUNT_OPTS=${MGS_MOUNT_OPTS:-}

 
 OSTCOUNT=${OSTCOUNT:-2}
 OSTDEVBASE=${OSTDEVBASE:-$TMP/${FSNAME}-ost}
 OSTSIZE=${OSTSIZE:-200000}
 
 OSTCOUNT=${OSTCOUNT:-2}
 OSTDEVBASE=${OSTDEVBASE:-$TMP/${FSNAME}-ost}
 OSTSIZE=${OSTSIZE:-200000}

-OSTOPT=""
+OSTOPT=${OSTOPT:-}
+OST_FS_MKFS_OPTS=${OST_FS_MKFS_OPTS:-}
+OST_MOUNT_OPTS=${OST_MOUNT_OPTS:-}

 # Can specify individual ost devs with
 # OSTDEV1="/dev/sda"
 # on specific hosts with
 # Can specify individual ost devs with
 # OSTDEV1="/dev/sda"
 # on specific hosts with


@@ -38,7 +59,30 @@ OSTOPT=""
 
 NETTYPE=${NETTYPE:-tcp}
 MGSNID=${MGSNID:-`h2$NETTYPE $mgs_HOST`}
 
 NETTYPE=${NETTYPE:-tcp}
 MGSNID=${MGSNID:-`h2$NETTYPE $mgs_HOST`}

+
+#
+# Back end file system type(s) of facets can be specified with these
+# variables:
+#
+#   1. <facet>_FSTYPE
+#   2. <facet_type>FSTYPE
+#   3. FSTYPE
+#
+# More specific ones override more general ones.  See facet_fstype().
+#

 FSTYPE=${FSTYPE:-ldiskfs}
 FSTYPE=${FSTYPE:-ldiskfs}

+
+LDISKFS_MKFS_OPTS=${LDISKFS_MKFS_OPTS:-}
+ZFS_MKFS_OPTS=${ZFS_MKFS_OPTS:-}
+
+#
+# If any OST is "remote" and the non-default implementation (e.g.,
+# current OFD) is desired, then make sure that either a)
+# LOAD_MODULES_REMOTE is true or b) modprobe(8) is configured to
+# blacklist the undesired (and aliased the other, if necessary).
+#
+USE_OFD=${USE_OFD:-no}
+

 STRIPE_BYTES=${STRIPE_BYTES:-1048576}
 STRIPES_PER_OBJ=${STRIPES_PER_OBJ:-0}
 SINGLEMDS=${SINGLEMDS:-"mds1"}
 STRIPE_BYTES=${STRIPE_BYTES:-1048576}
 STRIPES_PER_OBJ=${STRIPES_PER_OBJ:-0}
 SINGLEMDS=${SINGLEMDS:-"mds1"}


@@ -60,51 +104,6 @@ QUOTA_TYPE="ug3"
 QUOTA_USERS=${QUOTA_USERS:-"quota_usr quota_2usr sanityusr sanityusr1"}
 LQUOTAOPTS=${LQUOTAOPTS:-"hash_lqs_cur_bits=3"}
 
 QUOTA_USERS=${QUOTA_USERS:-"quota_usr quota_2usr sanityusr sanityusr1"}
 LQUOTAOPTS=${LQUOTAOPTS:-"hash_lqs_cur_bits=3"}
 

-MKFSOPT=""
-[ "x$MDSJOURNALSIZE" != "x" ] &&
-    MKFSOPT=$MKFSOPT" -J size=$MDSJOURNALSIZE"
-[ "x$MDSISIZE" != "x" ] &&
-    MKFSOPT=$MKFSOPT" -i $MDSISIZE"
-[ "x$MKFSOPT" != "x" ] &&
-    MKFSOPT="--mkfsoptions=\\\"$MKFSOPT\\\""
-[ "x$SECLEVEL" != "x" ] &&
-    MKFSOPT=$MKFSOPT" --param mdt.sec_level=$SECLEVEL"
-[ "x$MDSCAPA" != "x" ] &&
-    MKFSOPT=$MKFSOPT" --param mdt.capa=$MDSCAPA"
-[ "x$mdsfailover_HOST" != "x" ] &&
-    MDSOPT=$MDSOPT" --failnode=`h2$NETTYPE $mdsfailover_HOST`"
-[ "x$STRIPE_BYTES" != "x" ] &&
-    MDSOPT=$MDSOPT" --param lov.stripesize=$STRIPE_BYTES"
-[ "x$STRIPES_PER_OBJ" != "x" ] &&
-    MDSOPT=$MDSOPT" --param lov.stripecount=$STRIPES_PER_OBJ"
-[ "x$L_GETIDENTITY" != "x" ] &&
-    MDSOPT=$MDSOPT" --param mdt.identity_upcall=$L_GETIDENTITY"
-
-MDS_MKFS_OPTS="--mdt --fsname=$FSNAME --device-size=$MDSSIZE --param sys.timeout=$TIMEOUT $MKFSOPT $MDSOPT $MDS_MKFS_OPTS"
-if [[ $mds1_HOST == $mgs_HOST ]] && [[ $MDSDEV1 == $MGSDEV ]]; then
-    MDS_MKFS_OPTS="--mgs $MDS_MKFS_OPTS"
-else
-    MDS_MKFS_OPTS="--mgsnode=$MGSNID $MDS_MKFS_OPTS"
-    MGS_MKFS_OPTS="--mgs --device-size=$MGSSIZE"
-fi
-
-MKFSOPT=""
-[ "x$OSTJOURNALSIZE" != "x" ] &&
-    MKFSOPT=$MKFSOPT" -J size=$OSTJOURNALSIZE"
-[ "x$MKFSOPT" != "x" ] &&
-    MKFSOPT="--mkfsoptions=\\\"$MKFSOPT\\\""
-[ "x$SECLEVEL" != "x" ] &&
-    MKFSOPT=$MKFSOPT" --param ost.sec_level=$SECLEVEL"
-[ "x$OSSCAPA" != "x" ] &&
-    MKFSOPT=$MKFSOPT" --param ost.capa=$OSSCAPA"
-[ "x$ostfailover_HOST" != "x" ] &&
-    OSTOPT=$OSTOPT" --failnode=`h2$NETTYPE $ostfailover_HOST`"
-OST_MKFS_OPTS="--ost --fsname=$FSNAME --device-size=$OSTSIZE --mgsnode=$MGSNID --param sys.timeout=$TIMEOUT $MKFSOPT $OSTOPT $OST_MKFS_OPTS"
-
-MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-"-o loop,user_xattr"}
-OST_MOUNT_OPTS=${OST_MOUNT_OPTS:-"-o loop"}
-MGS_MOUNT_OPTS=${MGS_MOUNT_OPTS:-$MDS_MOUNT_OPTS}
-

 #client
 MOUNT=${MOUNT:-/mnt/${FSNAME}}
 MOUNT1=${MOUNT1:-$MOUNT}
 #client
 MOUNT=${MOUNT:-/mnt/${FSNAME}}
 MOUNT1=${MOUNT1:-$MOUNT}

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index b387f3b..701e58a 100644 (file)
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -49,18 +49,9 @@ if [ -n "$MDSSIZE" ]; then
 fi
 
 # pass "-E lazy_itable_init" to mke2fs to speed up the formatting time
 fi
 
 # pass "-E lazy_itable_init" to mke2fs to speed up the formatting time

-for facet in MGS MDS OST; do
-    opts=${facet}_MKFS_OPTS
-    if [[ ${!opts} != *lazy_itable_init* ]]; then
-        eval SAVED_${facet}_MKFS_OPTS=\"${!opts}\"
-        if [[ ${!opts} != *mkfsoptions* ]]; then
-            eval ${facet}_MKFS_OPTS=\"${!opts} --mkfsoptions='\\\"-E lazy_itable_init\\\"'\"
-        else
-            val=${!opts//--mkfsoptions=\\\"/--mkfsoptions=\\\"-E lazy_itable_init }
-            eval ${facet}_MKFS_OPTS='${val}'
-        fi
-    fi
-done
+if [[ "$LDISKFS_MKFS_OPTS" != *lazy_itable_init* ]]; then
+       LDISKFS_MKFS_OPTS=$(csa_add "$LDISKFS_MKFS_OPTS" -E lazy_itable_init)
+fi

 
 init_logging
 
 
 init_logging
 


@@ -590,7 +581,10 @@ test_17() {
 run_test 17 "Verify failed mds_postsetup won't fail assertion (2936) (should return errs)"
 
 test_18() {
 run_test 17 "Verify failed mds_postsetup won't fail assertion (2936) (should return errs)"
 
 test_18() {

-        [ "$FSTYPE" != "ldiskfs" ] && skip "not needed for FSTYPE=$FSTYPE" && return
+       if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
+               skip "Only applicable to ldiskfs-based MDTs"
+               return
+       fi

 
         local MDSDEV=$(mdsdevname ${SINGLEMDS//mds/})
 
 
         local MDSDEV=$(mdsdevname ${SINGLEMDS//mds/})
 


@@ -624,15 +618,9 @@ test_18() {
 
 
         echo "mount mds with large journal..."
 
 
         echo "mount mds with large journal..."

-        local OLD_MDS_MKFS_OPTS=$MDS_MKFS_OPTS

 
 

-        local opts="--mdt --fsname=$FSNAME --device-size=$myMDSSIZE --param sys.timeout=$TIMEOUT $MDSOPT"
-
-        if combined_mgs_mds ; then
-            MDS_MKFS_OPTS="--mgs $opts"
-        else
-            MDS_MKFS_OPTS="--mgsnode=$MGSNID $opts"
-        fi
+       local OLD_MDSSIZE=$MDSSIZE
+       MDSSIZE=$myMDSSIZE

 
         reformat_and_config
         echo "mount lustre system..."
 
         reformat_and_config
         echo "mount lustre system..."


@@ -649,8 +637,8 @@ test_18() {
 
         cleanup || return $?
 
 
         cleanup || return $?
 

-        MDS_MKFS_OPTS=$OLD_MDS_MKFS_OPTS
-        reformat_and_config
+       MDSSIZE=$OLD_MDSSIZE
+       reformat_and_config

 }
 run_test 18 "check mkfs creates large journals"
 
 }
 run_test 18 "check mkfs creates large journals"
 


@@ -846,6 +834,10 @@ run_test 23b "Simulate -EINTR during mount"
 fs2mds_HOST=$mds_HOST
 fs2ost_HOST=$ost_HOST
 
 fs2mds_HOST=$mds_HOST
 fs2ost_HOST=$ost_HOST
 

+MDSDEV1_2=$fs2mds_DEV
+OSTDEV1_2=$fs2ost_DEV
+OSTDEV2_2=$fs3ost_DEV
+

 cleanup_24a() {
        trap 0
        echo "umount $MOUNT2 ..."
 cleanup_24a() {
        trap 0
        echo "umount $MOUNT2 ..."


@@ -866,14 +858,18 @@ test_24a() {
 
        [ -n "$ost1_HOST" ] && fs2ost_HOST=$ost1_HOST
 
 
        [ -n "$ost1_HOST" ] && fs2ost_HOST=$ost1_HOST
 

-       local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
-       local fs2ostdev=${fs2ost_DEV:-$(ostdevname 1)_2}
+       local fs2mdsdev=$(mdsdevname 1_2)
+       local fs2ostdev=$(ostdevname 1_2)
+       local fs2mdsvdev=$(mdsvdevname 1_2)
+       local fs2ostvdev=$(ostvdevname 1_2)

 
        # test 8-char fsname as well
        local FSNAME2=test1234
 
        # test 8-char fsname as well
        local FSNAME2=test1234

-       add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME2} --nomgs --mgsnode=$MGSNID --reformat $fs2mdsdev || exit 10
+       add fs2mds $(mkfs_opts mds1) --nomgs --mgsnode=$MGSNID \
+               --fsname=${FSNAME2} --reformat $fs2mdsdev $fs2mdsvdev || exit 10

 
 

-       add fs2ost $OST_MKFS_OPTS --fsname=${FSNAME2} --reformat $fs2ostdev || exit 10
+       add fs2ost $(mkfs_opts ost1) --fsname=${FSNAME2} --reformat \
+               $fs2ostdev $fs2ostvdev || exit 10

 
        setup
        start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
 
        setup
        start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT


@@ -916,9 +912,11 @@ test_24b() {
                skip_env "mixed loopback and real device not working" && return
        fi
 
                skip_env "mixed loopback and real device not working" && return
        fi
 

-       local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
+       local fs2mdsdev=$(mdsdevname 1_2)
+       local fs2mdsvdev=$(mdsvdevname 1_2)

 
 

-       add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME}2 --mgs --reformat $fs2mdsdev || exit 10
+       add fs2mds $(mkfs_opts mds1) --mgs --fsname=${FSNAME}2 --reformat \
+               $fs2mdsdev $fs2mdsvdev || exit 10

        setup
        start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && return 2
        cleanup || return 6
        setup
        start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && return 2
        cleanup || return 6


@@ -1343,6 +1341,7 @@ test_33a() { # bug 12333, was test_33
         local rc=0
         local FSNAME2=test-123
         local MDSDEV=$(mdsdevname ${SINGLEMDS//mds/})
         local rc=0
         local FSNAME2=test-123
         local MDSDEV=$(mdsdevname ${SINGLEMDS//mds/})

+       local mkfsoptions

 
         [ -n "$ost1_HOST" ] && fs2ost_HOST=$ost1_HOST
 
 
         [ -n "$ost1_HOST" ] && fs2ost_HOST=$ost1_HOST
 


@@ -1353,12 +1352,19 @@ test_33a() { # bug 12333, was test_33
                 skip_env "mixed loopback and real device not working" && return
         fi
 
                 skip_env "mixed loopback and real device not working" && return
         fi
 

-        combined_mgs_mds || mkfs_opts="$mkfs_opts --nomgs"
+       local fs2mdsdev=$(mdsdevname 1_2)
+       local fs2ostdev=$(ostdevname 1_2)
+       local fs2mdsvdev=$(mdsvdevname 1_2)
+       local fs2ostvdev=$(ostvdevname 1_2)

 
 

-        local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
-        local fs2ostdev=${fs2ost_DEV:-$(ostdevname 1)_2}
-        add fs2mds $MDS_MKFS_OPTS --mkfsoptions='\"-J size=8\"' --fsname=${FSNAME2} --reformat $fs2mdsdev || exit 10
-        add fs2ost $OST_MKFS_OPTS --fsname=${FSNAME2} --index=8191 --mgsnode=$MGSNID --reformat $fs2ostdev || exit 10
+       if [ $(facet_fstype mds1) == ldiskfs ]; then
+               mkfsoptions="--mkfsoptions=\\\"-J size=8\\\"" # See bug 17931.
+       fi
+
+       add fs2mds $(mkfs_opts mds1) --fsname=${FSNAME2} --reformat \
+               $mkfsoptions $fs2mdsdev $fs2mdsvdev || exit 10
+       add fs2ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
+               --index=8191 --reformat $fs2ostdev $fs2ostvdev || exit 10

 
         start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
         start fs2ost $fs2ostdev $OST_MOUNT_OPTS
 
         start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
         start fs2ost $fs2ostdev $OST_MOUNT_OPTS


@@ -1373,7 +1379,6 @@ test_33a() { # bug 12333, was test_33
         umount -d $MOUNT2
         stop fs2ost -f
         stop fs2mds -f
         umount -d $MOUNT2
         stop fs2ost -f
         stop fs2mds -f

-        rm -rf $MOUNT2 $fs2mdsdev $fs2ostdev

         cleanup_nocli || rc=6
         return $rc
 }
         cleanup_nocli || rc=6
         return $rc
 }


@@ -1586,14 +1591,21 @@ test_36() { # 12743
                skip_env "mixed loopback and real device not working" && return
         fi
 
                skip_env "mixed loopback and real device not working" && return
         fi
 

-        local fs2mdsdev=${fs2mds_DEV:-${MDSDEV}_2}
-        local fs2ostdev=${fs2ost_DEV:-$(ostdevname 1)_2}
-        local fs3ostdev=${fs3ost_DEV:-$(ostdevname 2)_2}
-        add fs2mds $MDS_MKFS_OPTS --fsname=${FSNAME2} --reformat $fs2mdsdev || exit 10
-        # XXX after we support non 4K disk blocksize, change following --mkfsoptions with
-        # other argument
-        add fs2ost $OST_MKFS_OPTS --mkfsoptions='-b4096' --fsname=${FSNAME2} --mgsnode=$MGSNID --reformat $fs2ostdev || exit 10
-        add fs3ost $OST_MKFS_OPTS --mkfsoptions='-b4096' --fsname=${FSNAME2} --mgsnode=$MGSNID --reformat $fs3ostdev || exit 10
+       local fs2mdsdev=$(mdsdevname 1_2)
+       local fs2ostdev=$(ostdevname 1_2)
+       local fs3ostdev=$(ostdevname 2_2)
+       local fs2mdsvdev=$(mdsvdevname 1_2)
+       local fs2ostvdev=$(ostvdevname 1_2)
+       local fs3ostvdev=$(ostvdevname 2_2)
+
+       add fs2mds $(mkfs_opts mds1) --fsname=${FSNAME2} --reformat \
+               $fs2mdsdev $fs2mdsvdev || exit 10
+       # XXX after we support non 4K disk blocksize in ldiskfs, specify a
+       #     different one than the default value here.
+       add fs2ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
+               --reformat $fs2ostdev $fs2ostvdev || exit 10
+       add fs3ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
+               --reformat $fs3ostdev $fs3ostvdev || exit 10

 
         start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS
         start fs2ost $fs2ostdev $OST_MOUNT_OPTS
 
         start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS
         start fs2ost $fs2ostdev $OST_MOUNT_OPTS


@@ -1636,7 +1648,6 @@ test_36() { # 12743
         stop fs3ost -f || return 200
         stop fs2ost -f || return 201
         stop fs2mds -f || return 202
         stop fs3ost -f || return 200
         stop fs2ost -f || return 201
         stop fs2mds -f || return 202

-        rm -rf $MOUNT2 $fs2mdsdev $fs2ostdev $fs3ostdev

         unload_modules_conf || return 203
         return $rc
 }
         unload_modules_conf || return 203
         return $rc
 }


@@ -1646,6 +1657,13 @@ test_37() {
        local mntpt=$(facet_mntpt $SINGLEMDS)
        local mdsdev=$(mdsdevname ${SINGLEMDS//mds/})
        local mdsdev_sym="$TMP/sym_mdt.img"
        local mntpt=$(facet_mntpt $SINGLEMDS)
        local mdsdev=$(mdsdevname ${SINGLEMDS//mds/})
        local mdsdev_sym="$TMP/sym_mdt.img"

+       local opts=$MDS_MOUNT_OPTS
+       local rc=0
+
+       if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
+               skip "Currently only applicable to ldiskfs-based MDTs"
+               return
+       fi

 
        echo "MDS :     $mdsdev"
        echo "SYMLINK : $mdsdev_sym"
 
        echo "MDS :     $mdsdev"
        echo "SYMLINK : $mdsdev_sym"


@@ -1655,8 +1673,11 @@ test_37() {
 
        echo "mount symlink device - $mdsdev_sym"
 
 
        echo "mount symlink device - $mdsdev_sym"
 

-       local rc=0
-       mount_op=$(do_facet $SINGLEMDS mount -v -t lustre $MDS_MOUNT_OPTS  $mdsdev_sym $mntpt 2>&1 )
+       if ! do_facet $SINGLEMDS test -b $mdsdev; then
+               opts=$(csa_add "$opts" -o loop)
+       fi
+       mount_op=$(do_facet $SINGLEMDS mount -v -t lustre $opts \
+               $mdsdev_sym $mntpt 2>&1)

        rc=${PIPESTATUS[0]}
 
        echo mount_op=$mount_op
        rc=${PIPESTATUS[0]}
 
        echo mount_op=$mount_op


@@ -1749,7 +1770,11 @@ test_40() { # bug 15759
 run_test 40 "race during service thread startup"
 
 test_41a() { #bug 14134
 run_test 40 "race during service thread startup"
 
 test_41a() { #bug 14134

-        echo $MDS_MOUNT_OPTS | grep "loop" && skip " loop devices does not work with nosvc option" && return
+       if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&
+          ! do_facet $SINGLEMDS test -b $(mdsdevname 1); then
+               skip "Loop devices does not work with nosvc option"
+               return
+       fi

 
         local rc
         local MDSDEV=$(mdsdevname ${SINGLEMDS//mds/})
 
         local rc
         local MDSDEV=$(mdsdevname ${SINGLEMDS//mds/})


@@ -1774,7 +1799,11 @@ test_41a() { #bug 14134
 run_test 41a "mount mds with --nosvc and --nomgs"
 
 test_41b() {
 run_test 41a "mount mds with --nosvc and --nomgs"
 
 test_41b() {

-        echo $MDS_MOUNT_OPTS | grep "loop" && skip " loop devices does not work with nosvc option" && return
+       if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&
+          ! do_facet $SINGLEMDS test -b $(mdsdevname 1); then
+               skip "Loop devices does not work with nosvc option"
+               return
+       fi

 
         ! combined_mgs_mds && skip "needs combined mgs device" && return 0
 
 
         ! combined_mgs_mds && skip "needs combined mgs device" && return 0
 


@@ -2087,12 +2116,12 @@ run_test 48 "too many acls on file"
 
 # check PARAM_SYS_LDLM_TIMEOUT option of MKFS.LUSTRE
 test_49() { # bug 17710
 
 # check PARAM_SYS_LDLM_TIMEOUT option of MKFS.LUSTRE
 test_49() { # bug 17710

-       local OLD_MDS_MKFS_OPTS=$MDS_MKFS_OPTS
-       local OLD_OST_MKFS_OPTS=$OST_MKFS_OPTS
+       local timeout_orig=$TIMEOUT
+       local ldlm_timeout_orig=$LDLM_TIMEOUT

        local LOCAL_TIMEOUT=20
 
        local LOCAL_TIMEOUT=20
 

-
-       OST_MKFS_OPTS="--ost --fsname=$FSNAME --device-size=$OSTSIZE --mgsnode=$MGSNID --param sys.timeout=$LOCAL_TIMEOUT --param sys.ldlm_timeout=$LOCAL_TIMEOUT $MKFSOPT $OSTOPT"
+       LDLM_TIMEOUT=$LOCAL_TIMEOUT
+       TIMEOUT=$LOCAL_TIMEOUT

 
        reformat
        setup_noconfig
 
        reformat
        setup_noconfig


@@ -2115,7 +2144,7 @@ test_49() { # bug 17710
        stop_ost || return 2
        stop_mds || return 3
 
        stop_ost || return 2
        stop_mds || return 3
 

-       OST_MKFS_OPTS="--ost --fsname=$FSNAME --device-size=$OSTSIZE --mgsnode=$MGSNID --param sys.timeout=$LOCAL_TIMEOUT --param sys.ldlm_timeout=$((LOCAL_TIMEOUT - 1)) $MKFSOPT $OSTOPT"
+       LDLM_TIMEOUT=$((LOCAL_TIMEOUT - 1))

 
        reformat
        setup_noconfig
 
        reformat
        setup_noconfig


@@ -2135,8 +2164,8 @@ test_49() { # bug 17710
 
        cleanup || return $?
 
 
        cleanup || return $?
 

-       MDS_MKFS_OPTS=$OLD_MDS_MKFS_OPTS
-       OST_MKFS_OPTS=$OLD_OST_MKFS_OPTS
+       LDLM_TIMEOUT=$ldlm_timeout_orig
+       TIMEOUT=$timeout_orig

 }
 run_test 49 "check PARAM_SYS_LDLM_TIMEOUT option of MKFS.LUSTRE"
 
 }
 run_test 49 "check PARAM_SYS_LDLM_TIMEOUT option of MKFS.LUSTRE"
 


@@ -2427,6 +2456,7 @@ test_52() {
        local ost1mnt=$(facet_mntpt ost1)
        local ost1node=$(facet_active_host ost1)
        local ost1tmp=$TMP/conf52
        local ost1mnt=$(facet_mntpt ost1)
        local ost1node=$(facet_active_host ost1)
        local ost1tmp=$TMP/conf52

+       local loop

 
        mkdir -p $DIR/$tdir
        [ $? -eq 0 ] || { error "Unable to create tdir"; return 4; }
 
        mkdir -p $DIR/$tdir
        [ $? -eq 0 ] || { error "Unable to create tdir"; return 4; }


@@ -2462,7 +2492,11 @@ test_52() {
        echo mount ost1 as ldiskfs
        do_node $ost1node mkdir -p $ost1mnt
        [ $? -eq 0 ] || { error "Unable to create $ost1mnt"; return 23; }
        echo mount ost1 as ldiskfs
        do_node $ost1node mkdir -p $ost1mnt
        [ $? -eq 0 ] || { error "Unable to create $ost1mnt"; return 23; }

-       do_node $ost1node mount -t $FSTYPE $ost1_dev $ost1mnt $OST_MOUNT_OPTS
+       if ! do_node $ost1node test -b $ost1_dev; then
+               loop="-o loop"
+       fi
+       do_node $ost1node mount -t $(facet_fstype ost1) $loop $ost1_dev \
+               $ost1mnt

        [ $? -eq 0 ] || { error "Unable to mount ost1 as ldiskfs"; return 12; }
 
        # backup objects
        [ $? -eq 0 ] || { error "Unable to mount ost1 as ldiskfs"; return 12; }
 
        # backup objects


@@ -2624,19 +2658,19 @@ lov_objid_size()
 
 test_55() {
        local mdsdev=$(mdsdevname 1)
 
 test_55() {
        local mdsdev=$(mdsdevname 1)

-       local ostdev=$(ostdevname 1)
-       local saved_opts=$OST_MKFS_OPTS
+       local mdsvdev=$(mdsvdevname 1)

 
        for i in 1023 2048
        do
 
        for i in 1023 2048
        do

-               OST_MKFS_OPTS="$saved_opts --index $i"
-               reformat
-
+               add mds1 $(mkfs_opts mds1) --reformat $mdsdev $mdsvdev ||
+                       exit 10
+               add ost1 $(mkfs_opts ost1) --index=$i --reformat \
+                       $(ostdevname 1) $(ostvdevname 1)

                setup_noconfig
                stopall
                setup_noconfig
                stopall

-

                setup_noconfig
                sync
                setup_noconfig
                sync

+

                echo checking size of lov_objid for ost index $i
                LOV_OBJID_SIZE=$(do_facet mds1 "$DEBUGFS -R 'stat lov_objid' $mdsdev 2>/dev/null" | grep ^User | awk '{print $6}')
                if [ "$LOV_OBJID_SIZE" != $(lov_objid_size $i) ]; then
                echo checking size of lov_objid for ost index $i
                LOV_OBJID_SIZE=$(do_facet mds1 "$DEBUGFS -R 'stat lov_objid' $mdsdev 2>/dev/null" | grep ^User | awk '{print $6}')
                if [ "$LOV_OBJID_SIZE" != $(lov_objid_size $i) ]; then


@@ -2647,15 +2681,19 @@ test_55() {
                stopall
        done
 
                stopall
        done
 

-       OST_MKFS_OPTS=$saved_opts

        reformat
 }
 run_test 55 "check lov_objid size"
 
 test_56() {
        reformat
 }
 run_test 55 "check lov_objid size"
 
 test_56() {

-       add mds1 $MDS_MKFS_OPTS --mkfsoptions='\"-J size=16\"' --reformat $(mdsdevname 1)
-       add ost1 $OST_MKFS_OPTS --index=1000 --reformat $(ostdevname 1)
-       add ost2 $OST_MKFS_OPTS --index=10000 --reformat $(ostdevname 2)
+       local mds_journal_size_orig=$MDSJOURNALSIZE
+
+       MDSJOURNALSIZE=16
+       add mds1 $(mkfs_opts mds1) --reformat $(mdsdevname 1) $(mdsvdevname 1)
+       add ost1 $(mkfs_opts ost1) --index=1000 --reformat \
+               $(ostdevname 1) $(ostvdevname 1)
+       add ost2 $(mkfs_opts ost2) --index=10000 --reformat \
+               $(ostdevname 2) $(ostvdevname 2)

 
        start_mgsmds
        start_ost
 
        start_mgsmds
        start_ost


@@ -2665,6 +2703,7 @@ test_56() {
        $LFS osts
        [ -n "$ENABLE_QUOTA" ] && { $LFS quotacheck -ug $MOUNT || error "quotacheck has failed" ; }
        stopall
        $LFS osts
        [ -n "$ENABLE_QUOTA" ] && { $LFS quotacheck -ug $MOUNT || error "quotacheck has failed" ; }
        stopall

+       MDSJOURNALSIZE=$mds_journal_size_orig

        reformat
 }
 run_test 56 "check big indexes"
        reformat
 }
 run_test 56 "check big indexes"


@@ -2694,7 +2733,10 @@ count_osts() {
 }
 
 test_58() { # bug 22658
 }
 
 test_58() { # bug 22658

-       [ "$FSTYPE" != "ldiskfs" ] && skip "not supported for $FSTYPE" && return
+       if [ $(facet_fstype mds) == zfs ]; then
+               skip "Does not work with ZFS-based MDTs yet"
+               return
+       fi

        setup_noconfig
        mkdir -p $DIR/$tdir
        createmany -o $DIR/$tdir/$tfile-%d 100
        setup_noconfig
        mkdir -p $DIR/$tdir
        createmany -o $DIR/$tdir/$tfile-%d 100


@@ -2748,7 +2790,14 @@ test_59() {
 run_test 59 "writeconf mount option"
 
 test_60() { # LU-471
 run_test 59 "writeconf mount option"
 
 test_60() { # LU-471

-       add mds1 $MDS_MKFS_OPTS --mkfsoptions='\" -E stride=64 -O ^uninit_bg\"' --reformat $(mdsdevname 1)
+       if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
+               skip "Only applicable to ldiskfs-based MDTs"
+               return
+       fi
+
+       add mds1 $(mkfs_opts mds1) \
+               --mkfsoptions='\" -E stride=64 -O ^uninit_bg\"' --reformat \
+               $(mdsdevname 1) $(mdsvdevname 1) || exit 10

 
        dump=$(do_facet $SINGLEMDS dumpe2fs $(mdsdevname 1))
        rc=${PIPESTATUS[0]}
 
        dump=$(do_facet $SINGLEMDS dumpe2fs $(mdsdevname 1))
        rc=${PIPESTATUS[0]}


@@ -2765,17 +2814,19 @@ test_60() { # LU-471
 run_test 60 "check mkfs.lustre --mkfsoptions -E -O options setting"
 
 test_61() { # LU-80
 run_test 60 "check mkfs.lustre --mkfsoptions -E -O options setting"
 
 test_61() { # LU-80

-    local reformat=false
-
-    [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.1.53) ] ||
-        { skip "Need MDS version at least 2.1.53"; return 0; }
-
-    if ! large_xattr_enabled; then
-        reformat=true
-        local mds_dev=$(mdsdevname ${SINGLEMDS//mds/})
-        add $SINGLEMDS $MDS_MKFS_OPTS --mkfsoptions='\"-O large_xattr\"' \
-            --reformat $mds_dev || error "reformatting $mds_dev failed"
-    fi
+       local reformat=false
+
+       [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.1.53) ] ||
+               { skip "Need MDS version at least 2.1.53"; return 0; }
+
+       if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&
+          ! large_xattr_enabled; then
+               reformat=true
+               local mds_dev=$(mdsdevname ${SINGLEMDS//mds/})
+               LDISKFS_MKFS_OPTS+=" -O large_xattr"
+               add $SINGLEMDS $(mkfs_opts $SINGLEMDS) --reformat $mds_dev ||
+                       error "reformatting $mds_dev failed"
+       fi

 
     setup_noconfig || error "setting up the filesystem failed"
     client_up || error "starting client failed"
 
     setup_noconfig || error "setting up the filesystem failed"
     client_up || error "starting client failed"


@@ -2822,7 +2873,10 @@ test_61() { # LU-80
 
     rm -f $file
     stopall
 
     rm -f $file
     stopall

-    $reformat && reformat
+       if $reformat; then
+               LDISKFS_MKFS_OPTS=${LDISKFS_MKFS_OPTS% -O large_xattr}
+               reformat
+       fi

 }
 run_test 61 "large xattr"
 
 }
 run_test 61 "large xattr"
 


@@ -2848,13 +2902,5 @@ fi
 
 cleanup_gss
 
 
 cleanup_gss
 

-# restore the ${facet}_MKFS_OPTS variables
-for facet in MGS MDS OST; do
-    opts=SAVED_${facet}_MKFS_OPTS
-    if [[ -n ${!opts} ]]; then
-        eval ${facet}_MKFS_OPTS=\"${!opts}\"
-    fi
-done
-

 complete $(basename $0) $SECONDS
 exit_status
 complete $(basename $0) $SECONDS
 exit_status

diff --git a/lustre/tests/insanity.sh b/lustre/tests/insanity.sh
index f0cb403..e113e34 100755 (executable)
--- a/lustre/tests/insanity.sh
+++ b/lustre/tests/insanity.sh
@@ -30,8 +30,8 @@ SINGLECLIENT=${SINGLECLIENT:-$HOSTNAME}
 LIVE_CLIENT=${LIVE_CLIENT:-$SINGLECLIENT}
 FAIL_CLIENTS=${FAIL_CLIENTS:-$RCLIENTS}
 
 LIVE_CLIENT=${LIVE_CLIENT:-$SINGLECLIENT}
 FAIL_CLIENTS=${FAIL_CLIENTS:-$RCLIENTS}
 

-assert_env mds_HOST MDS_MKFS_OPTS
-assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
+assert_env mds_HOST
+assert_env ost_HOST OSTCOUNT

 assert_env LIVE_CLIENT FSNAME
 
 require_dsh_mds || exit 0
 assert_env LIVE_CLIENT FSNAME
 
 require_dsh_mds || exit 0

diff --git a/lustre/tests/mmp.sh b/lustre/tests/mmp.sh
index b99b4b6..59fede1 100755 (executable)
--- a/lustre/tests/mmp.sh
+++ b/lustre/tests/mmp.sh
@@ -27,8 +27,6 @@ init_logging
 
 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
 
 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0

-[ "$MDSFSTYPE" != "ldiskfs" ] && skip "MDS not running ldiskfs" && exit 0
-[ "$OSTFSTYPE" != "ldiskfs" ] && skip "OST not running ldiskfs" && exit 0

 
 # unmount and cleanup the Lustre filesystem
 MMP_RESTORE_MOUNT=false
 
 # unmount and cleanup the Lustre filesystem
 MMP_RESTORE_MOUNT=false


@@ -164,7 +162,17 @@ get_mmp_check_interval() {
 
 # Enable the MMP feature on the Lustre server targets.
 mmp_init() {
 
 # Enable the MMP feature on the Lustre server targets.
 mmp_init() {

-    init_vars
+       init_vars
+
+       if [ $(facet_fstype $MMP_MDS) != ldiskfs ]; then
+               skip "Only applicable to ldiskfs-based MDTs"
+               exit
+       fi
+
+       if [ $(facet_fstype $MMP_OSS) != ldiskfs ]; then
+               skip "Only applicable to ldiskfs-based OSTs"
+               exit
+       fi

 
     # The MMP feature is automatically enabled by mkfs.lustre for
     # new file system at format time if failover is being used.
 
     # The MMP feature is automatically enabled by mkfs.lustre for
     # new file system at format time if failover is being used.

diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh
index 96d1ab0..da93c05 100644 (file)
--- a/lustre/tests/sanity-quota.sh
+++ b/lustre/tests/sanity-quota.sh
@@ -6,13 +6,6 @@
 # Run test by setting NOSETUP=true when ltest has setup env for us
 set -e
 
 # Run test by setting NOSETUP=true when ltest has setup env for us
 set -e
 

-#kernel 2.4.x doesn't support quota
-K_VER=`uname --kernel-release | cut -b 1-3`
-if [ $K_VER = "2.4" ]; then
-    echo "Kernel 2.4 doesn't support quota"
-    exit 0
-fi
-

 SRCDIR=`dirname $0`
 export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin
 
 SRCDIR=`dirname $0`
 export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin
 


@@ -24,11 +17,6 @@ ONLY=${ONLY:-"$*"}
 ALWAYS_EXCEPT="10 12 $SANITY_QUOTA_EXCEPT"
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
 
 ALWAYS_EXCEPT="10 12 $SANITY_QUOTA_EXCEPT"
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
 

-case `uname -r` in
-2.6*) FSTYPE=${FSTYPE:-ldiskfs};;
-*) error "unsupported kernel" ;;
-esac
-

 [ "$ALWAYS_EXCEPT$EXCEPT" ] && \
        echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT`"
 
 [ "$ALWAYS_EXCEPT$EXCEPT" ] && \
        echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT`"
 

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index b7201d7..5815b5f 100644 (file)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -16,12 +16,6 @@ CPU=`awk '/model/ {print $4}' /proc/cpuinfo`
 #                                    buffer i/o errs             sock spc runas
 [ "$CPU" = "UML" ] && EXCEPT="$EXCEPT 27m 27n 27o 27p 27q 27r 31d 54a  64b 99a 99b 99c 99d 99e 99f 101a"
 
 #                                    buffer i/o errs             sock spc runas
 [ "$CPU" = "UML" ] && EXCEPT="$EXCEPT 27m 27n 27o 27p 27q 27r 31d 54a  64b 99a 99b 99c 99d 99e 99f 101a"
 

-case `uname -r` in
-2.4*) FSTYPE=${FSTYPE:-ext3} ;;
-2.6*) FSTYPE=${FSTYPE:-ldiskfs} ;;
-*) error "unsupported kernel" ;;
-esac
-

 SRCDIR=$(cd $(dirname $0); echo $PWD)
 export PATH=$PATH:/sbin
 
 SRCDIR=$(cd $(dirname $0); echo $PWD)
 export PATH=$PATH:/sbin
 


@@ -1357,9 +1351,6 @@ check_seq_oid()
         # compare lmm_object_id and lu_fid->oid
         [ $lmm_oid = ${fid[2]} ] || { error "OID mismatch"; return 2; }
 
         # compare lmm_object_id and lu_fid->oid
         [ $lmm_oid = ${fid[2]} ] || { error "OID mismatch"; return 2; }
 

-        [ "$FSTYPE" != "ldiskfs" ] &&
-                skip "cannot check filter fid FSTYPE=$FSTYPE" && return 0
-

         # check the trusted.fid attribute of the OST objects of the file
         local have_obdidx=false
         local stripe_nr=0
         # check the trusted.fid attribute of the OST objects of the file
         local have_obdidx=false
         local stripe_nr=0


@@ -1372,6 +1363,11 @@ check_seq_oid()
                 local ost=$((obdidx + 1))
                 local dev=$(ostdevname $ost)
 
                 local ost=$((obdidx + 1))
                 local dev=$(ostdevname $ost)
 

+               if [ $(facet_fstype ost$ost) != ldiskfs ]; then
+                       echo "Currently only works with ldiskfs-based OSTs"
+                       continue
+               fi
+

                 log "want: stripe:$stripe_nr ost:$obdidx oid:$oid/$hex seq:$seq"
 
                 #don't unmount/remount the OSTs if we don't need to do that
                 log "want: stripe:$stripe_nr ost:$obdidx oid:$oid/$hex seq:$seq"
 
                 #don't unmount/remount the OSTs if we don't need to do that


@@ -6938,7 +6934,10 @@ set_dir_limits () {
        done
 }
 test_129() {
        done
 }
 test_129() {

-       [ "$FSTYPE" != "ldiskfs" ] && skip "not needed for FSTYPE=$FSTYPE" && return 0
+       if [ "$(facet_type_fstype MDS)" != ldiskfs ]; then
+               skip "Only applicable to ldiskfs-based MDTs"
+               return
+       fi

        remote_mds_nodsh && skip "remote MDS with nodsh" && return
 
        EFBIG=27
        remote_mds_nodsh && skip "remote MDS with nodsh" && return
 
        EFBIG=27

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 80bac33..1562de6 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -129,6 +129,10 @@ init_test_env() {
     export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite
     export FSCK_MAX_ERR=4   # File system errors left uncorrected
 
     export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite
     export FSCK_MAX_ERR=4   # File system errors left uncorrected
 

+       export ZFS=${ZFS:-zfs}
+       export ZPOOL=${ZPOOL:-zpool}
+       export ZDB=${ZDB:-zdb}
+

     #[ -d /r ] && export ROOT=${ROOT:-/r}
     export TMP=${TMP:-$ROOT/tmp}
     export TESTSUITELOG=${TMP}/${TESTSUITE}.log
     #[ -d /r ] && export ROOT=${ROOT:-/r}
     export TMP=${TMP:-$ROOT/tmp}
     export TESTSUITELOG=${TMP}/${TESTSUITE}.log


@@ -193,7 +197,6 @@ init_test_env() {
     export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate}
     [ ! -f "$LFS_MIGRATE" ] &&
         export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null)
     export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate}
     [ ! -f "$LFS_MIGRATE" ] &&
         export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null)

-    export FSTYPE=${FSTYPE:-"ldiskfs"}

     export NAME=${NAME:-local}
     export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"}
     [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && \
     export NAME=${NAME:-local}
     export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"}
     [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && \


@@ -272,12 +275,6 @@ version_code() {
 export LINUX_VERSION=$(uname -r | sed -e "s/[-.]/ /3" -e "s/ .*//")
 export LINUX_VERSION_CODE=$(version_code ${LINUX_VERSION//\./ })
 
 export LINUX_VERSION=$(uname -r | sed -e "s/[-.]/ /3" -e "s/ .*//")
 export LINUX_VERSION_CODE=$(version_code ${LINUX_VERSION//\./ })
 

-case `uname -r` in
-2.4.*) EXT=".o"; USE_QUOTA=no; [ ! "$CLIENTONLY" ] && FSTYPE=ext3;;
-    *) EXT=".ko"; USE_QUOTA=yes;;
-esac
-
-

 module_loaded () {
    /sbin/lsmod | grep -q "^\<$1\>"
 }
 module_loaded () {
    /sbin/lsmod | grep -q "^\<$1\>"
 }


@@ -359,11 +356,27 @@ llite_lloop_enabled() {
 }
 
 load_modules_local() {
 }
 
 load_modules_local() {

-    if [ -n "$MODPROBE" ]; then
-        # use modprobe
-        echo "Using modprobe to load modules"
-        return 0
-    fi
+       if [ "$USE_OFD" == yes ]; then
+               if module_loaded obdfilter; then
+                       if ! $LUSTRE_RMMOD ldiskfs; then
+                               echo "$HOSTNAME may still be using obdfilter.ko"
+                               return 1
+                       fi
+               fi
+       else
+               if module_loaded ofd; then
+                       if ! $LUSTRE_RMMOD ldiskfs; then
+                               echo "$HOSTNAME may still be using ofd.ko"
+                               return 1
+                       fi
+               fi
+       fi
+
+       if [ -n "$MODPROBE" ]; then
+               # use modprobe
+               echo "Using modprobe to load modules"
+               return 0
+       fi

 
     echo Loading modules from $LUSTRE
     load_module ../libcfs/libcfs/libcfs
 
     echo Loading modules from $LUSTRE
     load_module ../libcfs/libcfs/libcfs


@@ -388,21 +401,21 @@ load_modules_local() {
         grep -q crc16 $SYMLIST || { modprobe crc16 2>/dev/null || true; }
         grep -q -w jbd $SYMLIST || { modprobe jbd 2>/dev/null || true; }
         grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; }
         grep -q crc16 $SYMLIST || { modprobe crc16 2>/dev/null || true; }
         grep -q -w jbd $SYMLIST || { modprobe jbd 2>/dev/null || true; }
         grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; }

-        if [ "$FSTYPE" = "ldiskfs" ]; then
-            grep -q exportfs_decode_fh $SYMLIST ||
-                { modprobe exportfs 2> /dev/null || true; }
-            load_module ../ldiskfs/ldiskfs/ldiskfs
-        fi
-        [ "$USE_QUOTA" = "yes" -a "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS
-        load_module mgs/mgs
-        load_module mds/mds
-        load_module mdd/mdd
-        load_module mdt/mdt
-        load_module lvfs/fsfilt_$FSTYPE
-        load_module cmm/cmm
-        load_module osd-ldiskfs/osd_ldiskfs
-        load_module ost/ost
-               if [ "x$USE_OFD" = "xyes" ]; then
+               if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then
+                       grep -q exportfs_decode_fh $SYMLIST ||
+                       { modprobe exportfs 2> /dev/null || true; }
+                       load_module ../ldiskfs/ldiskfs/ldiskfs
+               fi
+               [ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS
+               load_module mgs/mgs
+               load_module mds/mds
+               load_module mdd/mdd
+               load_module mdt/mdt
+               load_module lvfs/fsfilt_ldiskfs
+               load_module cmm/cmm
+               load_module osd-ldiskfs/osd_ldiskfs
+               load_module ost/ost
+               if [ "$USE_OFD" == yes ]; then

                        load_module ofd/ofd
                else
                        load_module obdfilter/obdfilter
                        load_module ofd/ofd
                else
                        load_module obdfilter/obdfilter


@@ -426,15 +439,15 @@ load_modules_local() {
 }
 
 load_modules () {
 }
 
 load_modules () {

-    load_modules_local
-    # bug 19124
-    # load modules on remote nodes optionally
-    # lustre-tests have to be installed on these nodes
-    if $LOAD_MODULES_REMOTE ; then
-        local list=$(comma_list $(remote_nodes_list))
-        echo loading modules on $list
-        do_rpc_nodes $list load_modules 
-    fi
+       load_modules_local
+       # bug 19124
+       # load modules on remote nodes optionally
+       # lustre-tests have to be installed on these nodes
+       if $LOAD_MODULES_REMOTE ; then
+               local list=$(comma_list $(remote_nodes_list))
+               echo loading modules on $list
+               do_rpc_nodes $list load_modules_local
+       fi

 }
 
 check_mem_leak () {
 }
 
 check_mem_leak () {


@@ -451,18 +464,18 @@ check_mem_leak () {
 }
 
 unload_modules() {
 }
 
 unload_modules() {

-    wait_exit_ST client # bug 12845
+       wait_exit_ST client # bug 12845

 
 

-    $LUSTRE_RMMOD $FSTYPE || return 2
+       $LUSTRE_RMMOD ldiskfs || return 2

 
 

-    if $LOAD_MODULES_REMOTE ; then
-        local list=$(comma_list $(remote_nodes_list))
-        if [ ! -z $list ]; then
-            echo unloading modules on $list
-            do_rpc_nodes $list $LUSTRE_RMMOD $FSTYPE
-            do_rpc_nodes $list check_mem_leak
-        fi
-    fi
+       if $LOAD_MODULES_REMOTE ; then
+               local list=$(comma_list $(remote_nodes_list))
+               if [ ! -z $list ]; then
+                       echo unloading modules on $list
+                       do_rpc_nodes $list $LUSTRE_RMMOD ldiskfs
+                       do_rpc_nodes $list check_mem_leak
+               fi
+       fi

 
     if grep -qe "/sbin/mount\.lustre" /proc/mounts; then
         umount /sbin/mount.lustre || true
 
     if grep -qe "/sbin/mount\.lustre" /proc/mounts; then
         umount /sbin/mount.lustre || true


@@ -588,18 +601,97 @@ cleanup_gss() {
     fi
 }
 
     fi
 }
 

+facet_type() {
+       local facet=$1
+
+       echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9]\+//' |
+               tr '[:lower:]' '[:upper:]'
+}
+
+facet_number() {
+       local facet=$1
+
+       if [ $facet == mgs ]; then
+               return 1
+       fi
+
+       echo -n $facet | sed -e 's/^fs[0-9]\+//' | sed -e 's/^[a-z]\+//'
+}
+
+facet_fstype() {
+       local facet=$1
+       local var
+
+       var=${facet}_FSTYPE
+       if [ -n "${!var}" ]; then
+               echo -n ${!var}
+               return
+       fi
+
+       var=$(facet_type $facet)FSTYPE
+       if [ -n "${!var}" ]; then
+               echo -n ${!var}
+               return
+       fi
+
+       if [ -n "$FSTYPE" ]; then
+               echo -n $FSTYPE
+               return
+       fi
+
+       return 1
+}
+
+node_fstypes() {
+       local node=$1
+       local fstypes
+       local fstype
+       local facets=$(get_facets)
+       local facet
+
+       for facet in ${facets//,/ }; do
+               if [ $node == $(facet_host $facet) ] ||
+                  [ $node == "$(facet_failover_host $facet)" ]; then
+                       fstype=$(facet_fstype $facet)
+                       if [[ $fstypes != *$fstype* ]]; then
+                               fstypes+="${fstypes:+,}$fstype"
+                       fi
+               fi
+       done
+       echo -n $fstypes
+}
+
+devicelabel() {
+       local facet=$1
+       local dev=$2
+       local label
+       local fstype=$(facet_fstype $facet)
+
+       case $fstype in
+       ldiskfs)
+               label=$(do_facet ${facet} "$E2LABEL ${dev} 2>/dev/null");;
+       zfs)
+               label=$(do_facet ${facet} "$ZFS get -H -o value lustre:svname \
+                                          ${dev} 2>/dev/null");;
+       *)
+               error "unknown fstype!";;
+       esac
+
+       echo -n $label
+}
+

 mdsdevlabel() {
 mdsdevlabel() {

-    local num=$1
-    local device=`mdsdevname $num`
-    local label=`do_facet mds$num "e2label ${device}" | grep -v "CMD: "`
-    echo -n $label
+       local num=$1
+       local device=$(mdsdevname $num)
+       local label=$(devicelabel mds$num ${device} | grep -v "CMD: ")
+       echo -n $label

 }
 
 ostdevlabel() {
 }
 
 ostdevlabel() {

-    local num=$1
-    local device=`ostdevname $num`
-    local label=`do_facet ost$num "e2label ${device}" | grep -v "CMD: "`
-    echo -n $label
+       local num=$1
+       local device=$(ostdevname $num)
+       local label=$(devicelabel ost$num ${device} | grep -v "CMD: ")
+       echo -n $label

 }
 
 set_debug_size () {
 }
 
 set_debug_size () {


@@ -669,20 +761,46 @@ mount_facets () {
        done
 }
 
        done
 }
 

+#
+# Add argument "arg" (e.g., "loop") to the comma-separated list
+# of arguments for option "opt" (e.g., "-o") on command
+# line "opts" (e.g., "-o flock").
+#
+csa_add() {
+       local opts=$1
+       local opt=$2
+       local arg=$3
+       local opt_pattern="\([[:space:]]\+\|^\)$opt"
+
+       if echo "$opts" | grep -q $opt_pattern; then
+               opts=$(echo "$opts" | sed -e \
+                       "s/$opt_pattern[[:space:]]*[^[:space:]]\+/&,$arg/")
+       else
+               opts+="${opts:+ }$opt $arg"
+       fi
+       echo -n "$opts"
+}
+

 mount_facet() {
        local facet=$1
        shift
        local dev=$(facet_active $facet)_dev
        local opt=${facet}_opt
        local mntpt=$(facet_mntpt $facet)
 mount_facet() {
        local facet=$1
        shift
        local dev=$(facet_active $facet)_dev
        local opt=${facet}_opt
        local mntpt=$(facet_mntpt $facet)

+       local opts="${!opt} $@"
+
+       if [ $(facet_fstype $facet) == ldiskfs ] &&
+          ! do_facet $facet test -b ${!dev}; then
+               opts=$(csa_add "$opts" -o loop)
+       fi

 
 

-       echo "Starting ${facet}: ${!opt} $@ ${!dev} $mntpt"
+       echo "Starting ${facet}: $opts ${!dev} $mntpt"

        # for testing LU-482 error handling in mount_facets() and test_0a()
        if [ -f $TMP/test-lu482-trigger ]; then
                RC=2
        else
        # for testing LU-482 error handling in mount_facets() and test_0a()
        if [ -f $TMP/test-lu482-trigger ]; then
                RC=2
        else

-               do_facet ${facet} "mkdir -p $mntpt; mount -t lustre ${!opt} \
-                                  $@ ${!dev} $mntpt"
+               do_facet ${facet} "mkdir -p $mntpt; mount -t lustre $opts \
+                                  ${!dev} $mntpt"

                RC=${PIPESTATUS[0]}
        fi
        if [ $RC -ne 0 ]; then
                RC=${PIPESTATUS[0]}
        fi
        if [ $RC -ne 0 ]; then


@@ -690,7 +808,7 @@ mount_facet() {
     else
         set_default_debug_facet $facet
 
     else
         set_default_debug_facet $facet
 

-        label=$(do_facet ${facet} "$E2LABEL ${!dev}")
+               label=$(devicelabel ${facet} ${!dev})

         [ -z "$label" ] && echo no label for ${!dev} && exit 1
         eval export ${facet}_svc=${label}
         echo Started ${label}
         [ -z "$label" ] && echo no label for ${!dev} && exit 1
         eval export ${facet}_svc=${label}
         echo Started ${label}


@@ -1889,6 +2007,25 @@ facet_host() {
     echo -n ${!varname}
 }
 
     echo -n ${!varname}
 }
 

+facet_failover_host() {
+       local facet=$1
+       local var
+
+       var=${facet}failover_HOST
+       if [ -n "${!var}" ]; then
+               echo ${!var}
+               return
+       fi
+
+       if [[ $facet == ost* ]]; then
+               var=ostfailover_HOST
+               if [ -n "${!var}" ]; then
+                       echo ${!var}
+                       return
+               fi
+       fi
+}
+

 facet_active() {
     local facet=$1
     local activevar=${facet}active
 facet_active() {
     local facet=$1
     local activevar=${facet}active


@@ -1998,13 +2135,15 @@ single_local_node () {
 
 # Outputs environment variable assignments that should be passed to remote nodes
 get_env_vars() {
 
 # Outputs environment variable assignments that should be passed to remote nodes
 get_env_vars() {

-    local var
-    local value
+       local var
+       local value

 
 

-    for var in ${!MODOPTS_*}; do
-        value=${!var}
-        echo "${var}=\"$value\""
-    done
+       for var in ${!MODOPTS_*}; do
+               value=${!var}
+               echo -n " ${var}=\"$value\""
+       done
+
+       echo -n " USE_OFD=$USE_OFD"

 }
 
 do_nodes() {
 }
 
 do_nodes() {


@@ -2074,19 +2213,121 @@ add() {
 ostdevname() {
     num=$1
     DEVNAME=OSTDEV$num
 ostdevname() {
     num=$1
     DEVNAME=OSTDEV$num

-    #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
-    eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}}
+
+       local fstype=$(facet_fstype ost$num)
+
+       case $fstype in
+               ldiskfs )
+                       #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
+                       eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};;
+               zfs )
+                       #dataset name is independent of vdev device names
+                       eval DEVPTR=${FSNAME}-ost${num}/ost${num};;
+               * )
+                       error "unknown fstype!";;
+       esac
+

     echo -n $DEVPTR
 }
 
     echo -n $DEVPTR
 }
 

+ostvdevname() {
+       num=$1
+       DEVNAME=OSTDEV$num
+
+       local fstype=$(facet_fstype ost$num)
+
+       case $fstype in
+               ldiskfs )
+                       # vdevs are not supported by ldiskfs
+                       eval VDEVPTR="";;
+               zfs )
+                       #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
+                       eval VDEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};;
+               * )
+                       error "unknown fstype!";;
+       esac
+
+       echo -n $VDEVPTR
+}
+

 mdsdevname() {
     num=$1
     DEVNAME=MDSDEV$num
 mdsdevname() {
     num=$1
     DEVNAME=MDSDEV$num

-    #if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
-    eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}}
+
+       local fstype=$(facet_fstype mds$num)
+
+       case $fstype in
+               ldiskfs )
+                       #if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
+                       eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};;
+               zfs )
+                       #dataset name is independent of vdev device names
+                       eval DEVPTR=${FSNAME}-mdt${num}/mdt${num};;
+               * )
+                       error "unknown fstype!";;
+       esac
+
+       echo -n $DEVPTR
+}
+
+mdsvdevname() {
+       num=$1
+       DEVNAME=MDSDEV$num
+
+       local fstype=$(facet_fstype mds$num)
+
+       case $fstype in
+               ldiskfs )
+                       # vdevs are not supported by ldiskfs
+                       eval VDEVPTR="";;
+               zfs )
+                       #if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
+                       eval VDEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};;
+               * )
+                       error "unknown fstype!";;
+       esac
+
+       echo -n $VDEVPTR
+}
+
+mgsdevname() {
+       DEVNAME=MGSDEV
+
+       local fstype=$(facet_fstype mds$num)
+
+       case $fstype in
+               ldiskfs )
+                       #if $MGSDEV isn't defined, default is $MDSDEV1
+                       eval DEVPTR=${!DEVNAME:=${MDSDEV1}};;
+               zfs )
+                       #dataset name is independent of vdev device names
+                       eval DEVPTR=${FSNAME}-mgs/mgs;;
+               * )
+                       error "unknown fstype!";;
+       esac
+

     echo -n $DEVPTR
 }
 
     echo -n $DEVPTR
 }
 

+mgsvdevname() {
+       DEVNAME=MGSDEV
+
+       local fstype=$(facet_fstype mds$num)
+
+       case $fstype in
+               ldiskfs )
+                       # vdevs are not supported by ldiskfs
+                       eval VDEVPTR="";;
+               zfs )
+                       #if $MGSDEV isn't defined, default is $MGSDEV1
+                       eval VDEVPTR=${!DEVNAME:=${MDSDEV1}};;
+               * )
+                       error "unknown fstype!";;
+       esac
+
+       echo -n $VDEVPTR
+}
+

 facet_mntpt () {
     local facet=$1
     [[ $facet = mgs ]] && combined_mgs_mds && facet="mds1"
 facet_mntpt () {
     local facet=$1
     [[ $facet = mgs ]] && combined_mgs_mds && facet="mds1"


@@ -2159,88 +2400,123 @@ combined_mgs_mds () {
     [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]]
 }
 
     [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]]
 }
 

-mkfs_opts () {
-    local facet=$1
+lower() {
+       echo -n "$1" | tr '[:upper:]' '[:lower:]'
+}

 
 

-    local tgt=$(echo $facet | tr -d [:digit:] | tr "[:lower:]" "[:upper:]")
-    local optvar=${tgt}_MKFS_OPTS
-    local opt=${!optvar}
+upper() {
+       echo -n "$1" | tr '[:lower:]' '[:upper:]'
+}

 
 

-    # FIXME: ! combo  mgs/mds + mgsfailover is not supported yet
-    [[ $facet = mgs ]] && echo $opt && return
+mkfs_opts() {
+       local facet=$1
+       local type=$(facet_type $facet)
+       local index=$(($(facet_number $facet) - 1))
+       local fstype=$(facet_fstype $facet)
+       local opts
+       local fs_mkfs_opts
+       local var
+
+       if [ $type == MGS ] && combined_mgs_mds; then
+               return 1
+       fi

 
 

-    # 1.
-    # --failnode options 
-    local var=${facet}failover_HOST
-    if [ x"${!var}" != x ] && [ x"${!var}" != x$(facet_host $facet) ] ; then
-        local failnode=$(h2$NETTYPE ${!var})
-        failnode="--failnode=$failnode"
-        # options does not contain
-        # or contains wrong --failnode=
-        if [[ $opt != *${failnode}* ]]; then
-            opt=$(echo $opt | sed 's/--failnode=.* / /')
-            opt="$opt $failnode"
-        fi
-    fi
+       if [ $type == MGS ] || ( [ $type == MDS ] && combined_mgs_mds ); then
+               opts="--mgs"
+       else
+               opts="--mgsnode=$MGSNID"
+       fi

 
 

-    # 2.
-    # --mgsnode options
-    # no additional mkfs mds "--mgsnode" option for this configuration
-    if [[ $facet = mds ]] && combined_mgs_mds; then
-        echo $opt
-        return
-    fi
+       if [ $type != MGS ]; then
+               opts+=" --fsname=$FSNAME --$(lower ${type/MDS/MDT}) --index=$index"
+       fi

 
 

-    # additional mkfs "--mgsnode"
-    local mgsnode="--mgsnode=$MGSNID"
-    opt=${opt//$mgsnode }
-    for nid in ${MGSNID//:/ }; do
-        local mgsnode="--mgsnode=$nid"
-        # options does not contain
-        # --mgsnode=$nid
-        if [[ $opt != *${mgsnode}" "* ]]; then
-            opt="$opt --mgsnode=$nid"
-        fi
-    done
+       var=${facet}failover_HOST
+       if [ -n "${!var}" ] && [ ${!var} != $(facet_host $facet) ]; then
+               opts+=" --failnode=$(h2$NETTYPE ${!var})"
+       fi
+
+       opts+=${TIMEOUT:+" --param=sys.timeout=$TIMEOUT"}
+       opts+=${LDLM_TIMEOUT:+" --param=sys.ldlm_timeout=$LDLM_TIMEOUT"}
+
+       if [ $type == MDS ]; then
+               opts+=${SECLEVEL:+" --param=mdt.sec_level"}
+               opts+=${MDSCAPA:+" --param-mdt.capa=$MDSCAPA"}
+               opts+=${STRIPE_BYTES:+" --param=lov.stripesize=$STRIPE_BYTES"}
+               opts+=${STRIPES_PER_OBJ:+" --param=lov.stripecount=$STRIPES_PER_OBJ"}
+               opts+=${L_GETIDENTITY:+" --param=mdt.identity_upcall=$L_GETIDENTITY"}
+
+               if [ $fstype == ldiskfs ]; then
+                       opts+=${IAMDIR:+" --iam-dir"}
+
+                       fs_mkfs_opts+=${MDSJOURNALSIZE:+" -J size=$MDSJOURNALSIZE"}
+                       fs_mkfs_opts+=${MDSISIZE:+" -i $MDSISIZE"}
+               fi
+       fi
+
+       if [ $type == OST ]; then
+               opts+=${SECLEVEL:+" --param=ost.sec_level"}
+               opts+=${OSSCAPA:+" --param=ost.capa=$OSSCAPA"}
+
+               if [ $fstype == ldiskfs ]; then
+                       fs_mkfs_opts+=${OSTJOURNALSIZE:+" -J size=$OSTJOURNALSIZE"}
+               fi
+       fi
+
+       opts+=" --backfstype=$fstype"

 
 

-    echo $opt
+       var=${type}SIZE
+       if [ -n "${!var}" ]; then
+               opts+=" --device-size=${!var}"
+       fi
+
+       var=$(upper $fstype)_MKFS_OPTS
+       fs_mkfs_opts+=${!var:+" ${!var}"}
+
+       var=${type}_FS_MKFS_OPTS
+       fs_mkfs_opts+=${!var:+" ${!var}"}
+
+       if [ -n "${fs_mkfs_opts## }" ]; then
+               opts+=" --mkfsoptions=\\\"${fs_mkfs_opts## }\\\""
+       fi
+
+       var=${type}OPT
+       opts+=${!var:+" ${!var}"}
+
+       echo -n "$opts"

 }
 
 formatall() {
 }
 
 formatall() {

-    if [ "$IAMDIR" == "yes" ]; then
-        MDS_MKFS_OPTS="$MDS_MKFS_OPTS --iam-dir"
-    fi
+       local quiet

 
 

-    [ "$FSTYPE" ] && FSTYPE_OPT="--backfstype $FSTYPE"
+       if ! $VERBOSE; then
+               quiet=yes
+       fi

 
 

-    stopall
-    # We need ldiskfs here, may as well load them all
-    load_modules
-    [ "$CLIENTONLY" ] && return
-    echo Formatting mgs, mds, osts
-    if ! combined_mgs_mds ; then
-        add mgs $(mkfs_opts mgs) $FSTYPE_OPT --reformat $MGSDEV || exit 10
-    fi
+       stopall
+       # We need ldiskfs here, may as well load them all
+       load_modules
+       [ "$CLIENTONLY" ] && return
+       echo Formatting mgs, mds, osts
+       if ! combined_mgs_mds ; then
+               echo "Format mgs: $(mgsdevname)"
+               add mgs $(mkfs_opts mgs) --reformat $(mgsdevname) \
+                       $(mgsvdevname) ${quiet:+>/dev/null} || exit 10
+               fi

 
 

-    for num in `seq $MDSCOUNT`; do
-        echo "Format mds$num: $(mdsdevname $num)"
-        if $VERBOSE; then
-            add mds$num $(mkfs_opts mds) $FSTYPE_OPT --reformat $(mdsdevname $num) || exit 10
-        else
-            add mds$num $(mkfs_opts mds) $FSTYPE_OPT --reformat $(mdsdevname $num) > /dev/null || exit 10
-        fi
-    done
+               for num in `seq $MDSCOUNT`; do
+                       echo "Format mds$num: $(mdsdevname $num)"
+                       add mds$num $(mkfs_opts mds$num) --reformat \
+                       $(mdsdevname $num) $(mdsvdevname $num) \
+                       ${quiet:+>/dev/null} || exit 10
+               done

 
 

-    # the ost-s could have different OST_MKFS_OPTS
-    # because of different failnode-s
-    for num in `seq $OSTCOUNT`; do
-        echo "Format ost$num: $(ostdevname $num)"
-        if $VERBOSE; then
-            add ost$num $(mkfs_opts ost${num}) $FSTYPE_OPT --reformat `ostdevname $num` || exit 10
-        else
-            add ost$num $(mkfs_opts ost${num}) $FSTYPE_OPT --reformat `ostdevname $num` > /dev/null || exit 10
-        fi
-    done
+               for num in `seq $OSTCOUNT`; do
+                       echo "Format ost$num: $(ostdevname $num)"
+                       add ost$num $(mkfs_opts ost$num) --reformat \
+                       $(ostdevname $num) $(ostvdevname ${num}) \
+                       ${quiet:+>/dev/null} || exit 10
+               done

 }
 
 mount_client() {
 }
 
 mount_client() {


@@ -2320,7 +2596,7 @@ setupall() {
         echo $WRITECONF | grep -q "writeconf" && \
             writeconf_all
         if ! combined_mgs_mds ; then
         echo $WRITECONF | grep -q "writeconf" && \
             writeconf_all
         if ! combined_mgs_mds ; then

-            start mgs $MGSDEV $MGS_MOUNT_OPTS
+                       start mgs $(mgsdevname) $MGS_MOUNT_OPTS

         fi
 
         for num in `seq $MDSCOUNT`; do
         fi
 
         for num in `seq $MDSCOUNT`; do


@@ -2401,7 +2677,7 @@ init_facet_vars () {
     eval export ${facet}_opt=\"$@\"
 
     local dev=${facet}_dev
     eval export ${facet}_opt=\"$@\"
 
     local dev=${facet}_dev

-    local label=$(do_facet ${facet} "$E2LABEL ${!dev}")
+       local label=$(devicelabel ${facet} ${!dev})

     [ -z "$label" ] && echo no label for ${!dev} && exit 1
 
     eval export ${facet}_svc=${label}
     [ -z "$label" ] && echo no label for ${!dev} && exit 1
 
     eval export ${facet}_svc=${label}


@@ -2441,7 +2717,7 @@ init_facets_vars () {
         done
     fi
 
         done
     fi
 

-    combined_mgs_mds || init_facet_vars mgs $MGSDEV $MGS_MOUNT_OPTS
+       combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS

 
     remote_ost_nodsh && return
 
 
     remote_ost_nodsh && return
 


@@ -2479,7 +2755,7 @@ init_param_vars () {
     osc_ensure_active $SINGLEMDS $TIMEOUT
     osc_ensure_active client $TIMEOUT
 
     osc_ensure_active $SINGLEMDS $TIMEOUT
     osc_ensure_active client $TIMEOUT
 

-    if [ $QUOTA_AUTO -ne 0 ]; then
+       if [ $QUOTA_AUTO -ne 0 ] && [ "$USE_OFD" != yes ]; then

         if [ "$ENABLE_QUOTA" ]; then
             echo "enable quota as required"
             setup_quota $MOUNT || return 2
         if [ "$ENABLE_QUOTA" ]; then
             echo "enable quota as required"
             setup_quota $MOUNT || return 2


@@ -2653,12 +2929,14 @@ check_and_setup_lustre() {
         set_default_debug_nodes $(comma_list $(nodes_list))
     fi
 
         set_default_debug_nodes $(comma_list $(nodes_list))
     fi
 

-    init_gss
-    set_flavor_all $SEC
+       init_gss
+       if $GSS; then
+               set_flavor_all $SEC
+       fi

 
 

-    if [ "$ONLY" == "setup" ]; then
-        exit 0
-    fi
+       if [ "$ONLY" == "setup" ]; then
+               exit 0
+       fi

 }
 
 restore_mount () {
 }
 
 restore_mount () {


@@ -4790,32 +5068,44 @@ run_llverfs()
 }
 
 remove_mdt_files() {
 }
 
 remove_mdt_files() {

-    local facet=$1
-    local mdtdev=$2
-    shift 2
-    local files="$@"
-    local mntpt=$(facet_mntpt $facet)
+       local facet=$1
+       local mdtdev=$2
+       shift 2
+       local files="$@"
+       local mntpt=$(facet_mntpt $facet)
+       local opts=$MDS_MOUNT_OPTS

 
 

-    echo "removing files from $mdtdev on $facet: $files"
-    mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $?
-    rc=0;
-    for f in $files; do
-        rm $mntpt/ROOT/$f || { rc=$?; break; }
-    done
-    umount -f $mntpt || return $?
-    return $rc
+       echo "removing files from $mdtdev on $facet: $files"
+       if [ $(facet_fstype $facet) == ldiskfs ] &&
+          ! do_facet $facet test -b ${!dev}; then
+               opts=$(csa_add "$opts" -o loop)
+       fi
+       mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
+               return $?
+       rc=0;
+       for f in $files; do
+               rm $mntpt/ROOT/$f || { rc=$?; break; }
+       done
+       umount -f $mntpt || return $?
+       return $rc

 }
 
 duplicate_mdt_files() {
 }
 
 duplicate_mdt_files() {

-    local facet=$1
-    local mdtdev=$2
-    shift 2
-    local files="$@"
-    local mntpt=$(facet_mntpt $facet)
+       local facet=$1
+       local mdtdev=$2
+       shift 2
+       local files="$@"
+       local mntpt=$(facet_mntpt $facet)
+       local opts=$MDS_MOUNT_OPTS

 
 

-    echo "duplicating files on $mdtdev on $facet: $files"
-    mkdir -p $mntpt || return $?
-    mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $?
+       echo "duplicating files on $mdtdev on $facet: $files"
+       mkdir -p $mntpt || return $?
+       if [ $(facet_fstype $facet) == ldiskfs ] &&
+          ! do_facet $facet test -b ${!dev}; then
+               opts=$(csa_add "$opts" -o loop)
+       fi
+       mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
+               return $?

 
     do_umount() {
         trap 0
 
     do_umount() {
         trap 0