X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=1d4c7a02b0e36f049f0d8cc879fa2803a9750104;hp=4b2cd84f06f254c6618a69c5386dd52c2c4d9fac;hb=9a9d09e8f7fbf1d03a8a02dd14b549ad2cd4be88;hpb=8a1606a7cbbcaf418fefa5d97a2335df0c0edb09

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 4b2cd84..1d4c7a02 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -1,11 +1,11 @@
 #!/bin/bash
-# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
 
 trap 'print_summary && touch $TF_FAIL && \
     echo "test-framework exiting on error"' ERR
 set -e
 #set -x
 
+export EJOURNAL=${EJOURNAL:-""}
 export REFORMAT=${REFORMAT:-""}
 export WRITECONF=${WRITECONF:-""}
 export VERBOSE=${VERBOSE:-false}
@@ -15,6 +15,8 @@ export GSS_KRB5=false
 export GSS_PIPEFS=false
 export IDENTITY_UPCALL=default
 export QUOTA_AUTO=1
+export JOBSTATS_AUTO=${JOBSTATS_AUTO:-1}
+export JOBID_VAR=${JOBID_VAR:-"procname_uid"}
 
 # LOAD_LLOOP: LU-409: only load llite_lloop module if kernel < 2.6.32 or
 #             LOAD_LLOOP is true. LOAD_LLOOP is false by default.
@@ -27,6 +29,8 @@ LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 . $LUSTRE/tests/functions.sh
 . $LUSTRE/tests/yaml.sh
 
+export LD_LIBRARY_PATH=${LUSTRE}/utils:${LD_LIBRARY_PATH}
+
 LUSTRE_TESTS_CFG_DIR=${LUSTRE_TESTS_CFG_DIR:-${LUSTRE}/tests/cfg}
 
 EXCEPT_LIST_FILE=${EXCEPT_LIST_FILE:-${LUSTRE_TESTS_CFG_DIR}/tests-to-skip.sh}
@@ -37,8 +41,13 @@ if [ -f "$EXCEPT_LIST_FILE" ]; then
     . $EXCEPT_LIST_FILE
 fi
 
-[ -z "$MODPROBECONF" -a -f /etc/modprobe.conf ] && MODPROBECONF=/etc/modprobe.conf
-[ -z "$MODPROBECONF" -a -f /etc/modprobe.d/Lustre ] && MODPROBECONF=/etc/modprobe.d/Lustre
+# check config files for options in decreasing order of preference
+[ -z "$MODPROBECONF" -a -f /etc/modprobe.d/lustre.conf ] &&
+    MODPROBECONF=/etc/modprobe.d/lustre.conf
+[ -z "$MODPROBECONF" -a -f /etc/modprobe.d/Lustre ] &&
+    MODPROBECONF=/etc/modprobe.d/Lustre
+[ -z "$MODPROBECONF" -a -f /etc/modprobe.conf ] &&
+    MODPROBECONF=/etc/modprobe.conf
 
 assert_DIR () {
     local failed=""
@@ -125,20 +134,16 @@ init_test_env() {
     export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite
     export FSCK_MAX_ERR=4   # File system errors left uncorrected
 
-    # This is used by a small number of tests to share state between the client
-    # running the tests, or in some cases between the servers (e.g. lfsck.sh).
-    # It needs to be a non-lustre filesystem that is available on all the nodes.
-    export SHARED_DIRECTORY=${SHARED_DIRECTORY:-"/tmp"}
-    export MDSDB=${MDSDB:-$SHARED_DIRECTORY/mdsdb}
-    export OSTDB=${OSTDB:-$SHARED_DIRECTORY/ostdb}
+	export ZFS=${ZFS:-zfs}
+	export ZPOOL=${ZPOOL:-zpool}
+	export ZDB=${ZDB:-zdb}
 
     #[ -d /r ] && export ROOT=${ROOT:-/r}
     export TMP=${TMP:-$ROOT/tmp}
     export TESTSUITELOG=${TMP}/${TESTSUITE}.log
-    if [[ -z $LOGDIRSET ]]; then
-        export LOGDIR=${LOGDIR:-${TMP}/test_logs/}/$(date +%s)
-        export LOGDIRSET=true
-    fi
+    export LOGDIR=${LOGDIR:-${TMP}/test_logs/$(date +%s)}
+    export TESTLOG_PREFIX=$LOGDIR/$TESTSUITE
+
     export HOSTNAME=${HOSTNAME:-$(hostname -s)}
     if ! echo $PATH | grep -q $LUSTRE/utils; then
         export PATH=$LUSTRE/utils:$PATH
@@ -168,10 +173,14 @@ init_test_env() {
         export PATH=$LUSTRE/tests/mpi:$PATH
     fi
     export RSYNC_RSH=${RSYNC_RSH:-rsh}
+
     export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
     [ ! -f "$LCTL" ] && export LCTL=$(which lctl)
     export LFS=${LFS:-"$LUSTRE/utils/lfs"}
     [ ! -f "$LFS" ] && export LFS=$(which lfs)
+    SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"}
+    GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"}
+
     export L_GETIDENTITY=${L_GETIDENTITY:-"$LUSTRE/utils/l_getidentity"}
     if [ ! -f "$L_GETIDENTITY" ]; then
         if `which l_getidentity > /dev/null 2>&1`; then
@@ -188,8 +197,11 @@ init_test_env() {
     [ ! -f "$TUNEFS" ] && export TUNEFS=$(which tunefs.lustre)
     export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} "
     export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod}
-    [ ! -f "$LUSTRE_RMMOD" ] && export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null)
-    export FSTYPE=${FSTYPE:-"ldiskfs"}
+    [ ! -f "$LUSTRE_RMMOD" ] &&
+        export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null)
+    export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate}
+    [ ! -f "$LFS_MIGRATE" ] &&
+        export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null)
     export NAME=${NAME:-local}
     export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"}
     [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && \
@@ -258,11 +270,29 @@ init_test_env() {
     rm -f $TMP/*active
 }
 
-case `uname -r` in
-2.4.*) EXT=".o"; USE_QUOTA=no; [ ! "$CLIENTONLY" ] && FSTYPE=ext3;;
-    *) EXT=".ko"; USE_QUOTA=yes;;
-esac
+check_cpt_number() {
+	local facet=$1
+	local ncpts
+
+	ncpts=$(do_facet $facet "lctl get_param -n " \
+		"cpu_partition_table 2>/dev/null| wc -l" || echo 1)
+
+	if [ $ncpts -eq 0 ]; then
+		echo "1"
+	else
+		echo $ncpts
+	fi
+}
+
+version_code() {
+    # split arguments like "1.8.6-wc3" into "1", "8", "6", "wc3"
+    eval set -- $(tr "[:punct:]" " " <<< $*)
+
+    echo -n $((($1 << 16) | ($2 << 8) | $3))
+}
 
+export LINUX_VERSION=$(uname -r | sed -e "s/[-.]/ /3" -e "s/ .*//")
+export LINUX_VERSION_CODE=$(version_code ${LINUX_VERSION//\./ })
 
 module_loaded () {
    /sbin/lsmod | grep -q "^\<$1\>"
@@ -345,14 +375,61 @@ llite_lloop_enabled() {
 }
 
 load_modules_local() {
-    if [ -n "$MODPROBE" ]; then
-        # use modprobe
-        echo "Using modprobe to load modules"
-        return 0
-    fi
+	if [ "$USE_OFD" == yes ]; then
+		if module_loaded obdfilter; then
+			if ! $LUSTRE_RMMOD ldiskfs; then
+				echo "$HOSTNAME may still be using obdfilter.ko"
+				return 1
+			fi
+		fi
+	else
+		if module_loaded ofd; then
+			if ! $LUSTRE_RMMOD ldiskfs; then
+				echo "$HOSTNAME may still be using ofd.ko"
+				return 1
+			fi
+		fi
+	fi
+
+	if [ -n "$MODPROBE" ]; then
+		# use modprobe
+		echo "Using modprobe to load modules"
+		return 0
+	fi
+
+	echo Loading modules from $LUSTRE
+
+	local ncpus
+
+	if [ -f /sys/devices/system/cpu/online ]; then
+		ncpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/online) + 1))
+		echo "detected $ncpus online CPUs by sysfs"
+	else
+		ncpus=$(getconf _NPROCESSORS_CONF 2>/dev/null)
+		local rc=$?
+		if [ $rc -eq 0 ]; then
+			echo "detected $ncpus online CPUs by getconf"
+		else
+			echo "Can't detect number of CPUs"
+			ncpus=1
+		fi
+	fi
+
+	local ncpts=0
+	# if there is only one CPU core, libcfs can only create one partition
+	# if there is more than 4 CPU cores, libcfs should create multiple CPU
+	# partitions. So we just force libcfs to create 2 partitions for
+	# system with 2 or 4 cores
+	if [ $ncpus -le 4 ] && [ $ncpus -gt 1 ]; then
+		# force to enable multiple CPU partitions
+		echo "Force libcfs to create 2 CPU partitions"
+		ncpts=2
+	else
+		echo "libcfs will create CPU partition based on online CPUs"
+	fi
+
+	load_module ../libcfs/libcfs/libcfs cpu_npartitions=$ncpts
 
-    echo Loading modules from $LUSTRE
-    load_module ../libcfs/libcfs/libcfs
     [ "$PTLDEBUG" ] && lctl set_param debug="$PTLDEBUG"
     [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug="${SUBSYSTEM# }"
     load_module ../lnet/lnet/lnet
@@ -362,7 +439,6 @@ load_modules_local() {
     load_module obdclass/obdclass
     load_module ptlrpc/ptlrpc
     load_module ptlrpc/gss/ptlrpc_gss
-    [ "$USE_QUOTA" = "yes" -a "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS
     load_module fld/fld
     load_module fid/fid
     load_module lmv/lmv
@@ -370,20 +446,38 @@ load_modules_local() {
     load_module osc/osc
     load_module lov/lov
     load_module mgc/mgc
+    load_module obdecho/obdecho
     if ! client_only; then
-        grep -q crc16 /proc/kallsyms || { modprobe crc16 2>/dev/null || true; }
-        grep -q -w jbd /proc/kallsyms || { modprobe jbd 2>/dev/null || true; }
-        grep -q -w jbd2 /proc/kallsyms || { modprobe jbd2 2>/dev/null || true; }
-        [ "$FSTYPE" = "ldiskfs" ] && load_module ../ldiskfs/ldiskfs/ldiskfs
-        load_module mgs/mgs
-        load_module mds/mds
-        load_module mdd/mdd
-        load_module mdt/mdt
-        load_module lvfs/fsfilt_$FSTYPE
-        load_module cmm/cmm
-        load_module osd-ldiskfs/osd_ldiskfs
-        load_module ost/ost
-        load_module obdfilter/obdfilter
+        SYMLIST=/proc/kallsyms
+        grep -q crc16 $SYMLIST || { modprobe crc16 2>/dev/null || true; }
+        grep -q -w jbd $SYMLIST || { modprobe jbd 2>/dev/null || true; }
+        grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; }
+		[ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS
+		if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then
+			load_module osd-zfs/osd_zfs
+		fi
+		load_module mgs/mgs
+		load_module mds/mds
+		load_module mdd/mdd
+		if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then
+			#
+			# This block shall be moved up beside osd-zfs as soon
+			# as osd-ldiskfs stops using mdd symbols.
+			#
+			grep -q exportfs_decode_fh $SYMLIST ||
+				{ modprobe exportfs 2> /dev/null || true; }
+			load_module ../ldiskfs/ldiskfs/ldiskfs
+			load_module lvfs/fsfilt_ldiskfs
+			load_module osd-ldiskfs/osd_ldiskfs
+		fi
+		load_module mdt/mdt
+		load_module cmm/cmm
+		load_module ost/ost
+		if [ "$USE_OFD" == yes ]; then
+			load_module ofd/ofd
+		else
+			load_module obdfilter/obdfilter
+		fi
     fi
 
 
@@ -403,15 +497,15 @@ load_modules_local() {
 }
 
 load_modules () {
-    load_modules_local
-    # bug 19124
-    # load modules on remote nodes optionally
-    # lustre-tests have to be installed on these nodes
-    if $LOAD_MODULES_REMOTE ; then
-        local list=$(comma_list $(remote_nodes_list))
-        echo loading modules on $list
-        do_rpc_nodes $list load_modules 
-    fi
+	load_modules_local
+	# bug 19124
+	# load modules on remote nodes optionally
+	# lustre-tests have to be installed on these nodes
+	if $LOAD_MODULES_REMOTE ; then
+		local list=$(comma_list $(remote_nodes_list))
+		echo loading modules on $list
+		do_rpc_nodes $list load_modules_local
+	fi
 }
 
 check_mem_leak () {
@@ -428,18 +522,18 @@ check_mem_leak () {
 }
 
 unload_modules() {
-    wait_exit_ST client # bug 12845
+	wait_exit_ST client # bug 12845
 
-    $LUSTRE_RMMOD $FSTYPE || return 2
+	$LUSTRE_RMMOD ldiskfs || return 2
 
-    if $LOAD_MODULES_REMOTE ; then
-        local list=$(comma_list $(remote_nodes_list))
-        if [ ! -z $list ]; then
-            echo unloading modules on $list
-            do_rpc_nodes $list $LUSTRE_RMMOD $FSTYPE
-            do_rpc_nodes $list check_mem_leak
-        fi
-    fi
+	if $LOAD_MODULES_REMOTE ; then
+		local list=$(comma_list $(remote_nodes_list))
+		if [ ! -z $list ]; then
+			echo unloading modules on $list
+			do_rpc_nodes $list $LUSTRE_RMMOD ldiskfs
+			do_rpc_nodes $list check_mem_leak
+		fi
+	fi
 
     if grep -qe "/sbin/mount\.lustre" /proc/mounts; then
         umount /sbin/mount.lustre || true
@@ -565,23 +659,135 @@ cleanup_gss() {
     fi
 }
 
+facet_type() {
+	local facet=$1
+
+	echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9]\+//' |
+		tr '[:lower:]' '[:upper:]'
+}
+
+facet_number() {
+	local facet=$1
+
+	if [ $facet == mgs ]; then
+		return 1
+	fi
+
+	echo -n $facet | sed -e 's/^fs[0-9]\+//' | sed -e 's/^[a-z]\+//'
+}
+
+facet_fstype() {
+	local facet=$1
+	local var
+
+	var=${facet}_FSTYPE
+	if [ -n "${!var}" ]; then
+		echo -n ${!var}
+		return
+	fi
+
+	var=$(facet_type $facet)FSTYPE
+	if [ -n "${!var}" ]; then
+		echo -n ${!var}
+		return
+	fi
+
+	if [ -n "$FSTYPE" ]; then
+		echo -n $FSTYPE
+		return
+	fi
+
+	return 1
+}
+
+node_fstypes() {
+	local node=$1
+	local fstypes
+	local fstype
+	local facets=$(get_facets)
+	local facet
+
+	for facet in ${facets//,/ }; do
+		if [ $node == $(facet_host $facet) ] ||
+		   [ $node == "$(facet_failover_host $facet)" ]; then
+			fstype=$(facet_fstype $facet)
+			if [[ $fstypes != *$fstype* ]]; then
+				fstypes+="${fstypes:+,}$fstype"
+			fi
+		fi
+	done
+	echo -n $fstypes
+}
+
+devicelabel() {
+	local facet=$1
+	local dev=$2
+	local label
+	local fstype=$(facet_fstype $facet)
+
+	case $fstype in
+	ldiskfs)
+		label=$(do_facet ${facet} "$E2LABEL ${dev} 2>/dev/null");;
+	zfs)
+		label=$(do_facet ${facet} "$ZFS get -H -o value lustre:svname \
+		                           ${dev} 2>/dev/null");;
+	*)
+		error "unknown fstype!";;
+	esac
+
+	echo -n $label
+}
+
 mdsdevlabel() {
-    local num=$1
-    local device=`mdsdevname $num`
-    local label=`do_facet mds$num "e2label ${device}" | grep -v "CMD: "`
-    echo -n $label
+	local num=$1
+	local device=$(mdsdevname $num)
+	local label=$(devicelabel mds$num ${device} | grep -v "CMD: ")
+	echo -n $label
 }
 
 ostdevlabel() {
-    local num=$1
-    local device=`ostdevname $num`
-    local label=`do_facet ost$num "e2label ${device}" | grep -v "CMD: "`
-    echo -n $label
+	local num=$1
+	local device=$(ostdevname $num)
+	local label=$(devicelabel ost$num ${device} | grep -v "CMD: ")
+	echo -n $label
+}
+
+#
+# This and set_obdfilter_param() shall be used to access OSD parameters
+# once existed under "obdfilter":
+#
+#   mntdev
+#   stats
+#   read_cache_enable
+#   writethrough_cache_enable
+#
+get_obdfilter_param() {
+	local nodes=$1
+	local device=${2:-$FSNAME-OST*}
+	local name=$3
+
+	do_nodes $nodes "$LCTL get_param -n obdfilter.$device.$name \
+		osd-*.$device.$name 2>&1" | grep -v 'Found no match'
+}
+
+set_obdfilter_param() {
+	local nodes=$1
+	local device=${2:-$FSNAME-OST*}
+	local name=$3
+	local value=$4
+
+	do_nodes $nodes "$LCTL set_param -n obdfilter.$device.$name=$value \
+		osd-*.$device.$name=$value 2>&1" | grep -v 'Found no match'
 }
 
 set_debug_size () {
     local dz=${1:-$DEBUG_SIZE}
-    local cpus=$(getconf _NPROCESSORS_CONF)
+
+    if [ -f /sys/devices/system/cpu/possible ]; then
+        local cpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/possible)+1))
+    else
+        local cpus=$(getconf _NPROCESSORS_CONF)
+    fi
 
     # bug 19944, adjust size to be -gt num_possible_cpus()
     # promise 2MB for every cpu at least
@@ -593,26 +799,25 @@ set_debug_size () {
 
 set_default_debug () {
     local debug=${1:-"$PTLDEBUG"}
-    local subsystem_debug=${2:-"$SUBSYSTEM"}
+    local subsys=${2:-"$SUBSYSTEM"}
     local debug_size=${3:-$DEBUG_SIZE}
 
-    lctl set_param debug="$debug"
-    lctl set_param subsystem_debug="${subsystem_debug# }"
+    [ -n "$debug" ] && lctl set_param debug="$debug" >/dev/null
+    [ -n "$subsys" ] && lctl set_param subsystem_debug="${subsys# }" >/dev/null
 
-    set_debug_size $debug_size
-    sync
+    [ -n "$debug_size" ] && set_debug_size $debug_size > /dev/null
 }
 
 set_default_debug_nodes () {
     local nodes=$1
 
     if [[ ,$nodes, = *,$HOSTNAME,* ]]; then
-	nodes=$(exclude_items_from_list "$nodes" "$HOSTNAME")
-	set_default_debug
+        nodes=$(exclude_items_from_list "$nodes" "$HOSTNAME")
+            set_default_debug
     fi
 
     [[ -n $nodes ]] && do_rpc_nodes $nodes set_default_debug \
-	\\\"$PTLDEBUG\\\" \\\"$SUBSYSTEM\\\" $DEBUG_SIZE || true
+        \\\"$PTLDEBUG\\\" \\\"$SUBSYSTEM\\\" $DEBUG_SIZE || true
 }
 
 set_default_debug_facet () {
@@ -625,31 +830,71 @@ set_default_debug_facet () {
 
 # Facet functions
 mount_facets () {
-    local facets=${1:-$(get_facets)}
-    local facet
+	local facets=${1:-$(get_facets)}
+	local facet
 
-    for facet in ${facets//,/ }; do
-        mount_facet $facet || error "Restart of $facet failed!"
-    done
+	for facet in ${facets//,/ }; do
+		mount_facet $facet
+		local RC=$?
+		[ $RC -eq 0 ] && continue
+
+		if [ "$TESTSUITE.$TESTNAME" = "replay-dual.test_0a" ]; then
+			skip "Restart of $facet failed!." && touch $LU482_FAILED
+		else
+			error "Restart of $facet failed!"
+		fi
+		return $RC
+	done
 }
 
-mount_facet() {
-    local facet=$1
-    shift
-    local dev=$(facet_active $facet)_dev
-    local opt=${facet}_opt
-    local mntpt=$(facet_mntpt $facet)
+#
+# Add argument "arg" (e.g., "loop") to the comma-separated list
+# of arguments for option "opt" (e.g., "-o") on command
+# line "opts" (e.g., "-o flock").
+#
+csa_add() {
+	local opts=$1
+	local opt=$2
+	local arg=$3
+	local opt_pattern="\([[:space:]]\+\|^\)$opt"
+
+	if echo "$opts" | grep -q $opt_pattern; then
+		opts=$(echo "$opts" | sed -e \
+			"s/$opt_pattern[[:space:]]*[^[:space:]]\+/&,$arg/")
+	else
+		opts+="${opts:+ }$opt $arg"
+	fi
+	echo -n "$opts"
+}
 
-    echo "Starting ${facet}: ${!opt} $@ ${!dev} $mntpt"
-    do_facet ${facet} "mkdir -p $mntpt; mount -t lustre ${!opt} $@ ${!dev} $mntpt"
-    RC=${PIPESTATUS[0]}
-    if [ $RC -ne 0 ]; then
-        echo "mount -t lustre $@ ${!dev} $mntpt"
-        echo "Start of ${!dev} on ${facet} failed ${RC}"
+mount_facet() {
+	local facet=$1
+	shift
+	local dev=$(facet_active $facet)_dev
+	local opt=${facet}_opt
+	local mntpt=$(facet_mntpt $facet)
+	local opts="${!opt} $@"
+
+	if [ $(facet_fstype $facet) == ldiskfs ] &&
+	   ! do_facet $facet test -b ${!dev}; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
+
+	echo "Starting ${facet}: $opts ${!dev} $mntpt"
+	# for testing LU-482 error handling in mount_facets() and test_0a()
+	if [ -f $TMP/test-lu482-trigger ]; then
+		RC=2
+	else
+		do_facet ${facet} "mkdir -p $mntpt; mount -t lustre $opts \
+		                   ${!dev} $mntpt"
+		RC=${PIPESTATUS[0]}
+	fi
+	if [ $RC -ne 0 ]; then
+		echo "Start of ${!dev} on ${facet} failed ${RC}"
     else
         set_default_debug_facet $facet
 
-        label=$(do_facet ${facet} "$E2LABEL ${!dev}")
+		label=$(devicelabel ${facet} ${!dev})
         [ -z "$label" ] && echo no label for ${!dev} && exit 1
         eval export ${facet}_svc=${label}
         echo Started ${label}
@@ -681,6 +926,39 @@ start() {
     return $RC
 }
 
+#
+# When a ZFS OSD is made read-only by replay_barrier(), its pool is "freezed".
+# Because stopping corresponding target may not clear this in-memory state, we
+# need to zap the pool from memory by exporting and reimporting the pool.
+#
+# Although the uberblocks are not updated when a pool is freezed, transactions
+# are still written to the disks.  Modified blocks may be cached in memory when
+# tests try reading them back.  The export-and-reimport process also evicts any
+# cached pool data from memory to provide the correct "data loss" semantics.
+#
+refresh_disk() {
+	local facet=$1
+	local fstype=$(facet_fstype $facet)
+	local _dev
+	local dev
+	local poolname
+
+	if [ "${fstype}" == "zfs" ]; then
+		_dev=$(facet_active $facet)_dev
+		dev=${!_dev} # expand _dev to its value, e.g. ${mds1_dev}
+		poolname="${dev%%/*}" # poolname is string before "/"
+
+		if [ "${poolname}" == "" ]; then
+			echo "invalid dataset name: $dev"
+			return
+		fi
+		do_facet $facet "cp /etc/zfs/zpool.cache /tmp/zpool.cache.back"
+		do_facet $facet "$ZPOOL export ${poolname}"
+		do_facet $facet "$ZPOOL import -f -c /tmp/zpool.cache.back \
+		                 ${poolname}"
+	fi
+}
+
 stop() {
     local running
     local facet=$1
@@ -702,6 +980,10 @@ stop() {
 
 # save quota version (both administrative and operational quotas)
 # add an additional parameter if mountpoint is ever different from $MOUNT
+#
+# XXX This function is kept for interoperability with old server (< 2.3.51),
+#     it should be removed whenever we drop the interoperability for such
+#     server.
 quota_save_version() {
     local fsname=${2:-$FSNAME}
     local spec=$1
@@ -721,7 +1003,11 @@ quota_save_version() {
     done
 }
 
-# client could mount several lustre 
+# client could mount several lustre
+#
+# XXX This function is kept for interoperability with old server (< 2.3.51),
+#     it should be removed whenever we drop the interoperability for such
+#     server.
 quota_type () {
     local fsname=${1:-$FSNAME}
     local rc=0
@@ -731,17 +1017,29 @@ quota_type () {
     return $rc 
 }
 
-restore_quota_type () {
-   local mntpt=${1:-$MOUNT}
-   local quota_type=$(quota_type $FSNAME | grep MDT | cut -d "=" -f2)
-   if [ ! "$old_QUOTA_TYPE" ] || [ "$quota_type" = "$old_QUOTA_TYPE" ]; then
-        return
-   fi
-   quota_save_version $old_QUOTA_TYPE
+# XXX This function is kept for interoperability with old server (< 2.3.51),
+#     it should be removed whenever we drop the interoperability for such
+#     server.
+restore_quota_old() {
+	local mntpt=${1:-$MOUNT}
+	local quota_type=$(quota_type $FSNAME | grep MDT | cut -d "=" -f2)
+	if [ ! "$old_QUOTA_TYPE" ] ||
+		[ "$quota_type" = "$old_QUOTA_TYPE" ]; then
+		return
+	fi
+	quota_save_version $old_QUOTA_TYPE
 }
 
-setup_quota(){
-    local mntpt=$1
+# XXX This function is kept for interoperability with old server (< 2.3.51),
+#     it should be removed whenever we drop the interoperability for such
+#     server.
+setup_quota_old(){
+	local mntpt=$1
+
+	if [ "$USE_OFD" = "yes" ]; then
+		$LFS quotacheck $mntpt || error "quotacheck failed"
+		return
+	fi
 
     # We need save the original quota_type params, and restore them after testing
 
@@ -785,6 +1083,93 @@ setup_quota(){
     done
 }
 
+# get mdt quota type
+mdt_quota_type() {
+	local varsvc=${SINGLEMDS}_svc
+	do_facet $SINGLEMDS $LCTL get_param -n \
+		osd-$FSTYPE.${!varsvc}.quota_slave.enabled
+}
+
+# get ost quota type
+ost_quota_type() {
+	# All OSTs should have same quota type
+	local varsvc=ost1_svc
+	do_facet ost1 $LCTL get_param -n \
+		osd-$FSTYPE.${!varsvc}.quota_slave.enabled
+}
+
+# restore old quota type settings
+restore_quota() {
+	if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.51) ]; then
+		restore_quota_old
+		return
+	fi
+
+	if [ "$old_MDT_QUOTA_TYPE" ]; then
+		do_facet mgs $LCTL conf_param \
+			$FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE
+	fi
+	if [ "$old_OST_QUOTA_TYPE" ]; then
+		do_facet mgs $LCTL conf_param \
+			$FSNAME.quota.ost=$old_OST_QUOTA_TYPE
+	fi
+}
+
+setup_quota(){
+	if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.51) ]; then
+		setup_quota_old $1
+		return
+	fi
+
+	local mntpt=$1
+
+	# save old quota type & set new quota type
+	local mdt_qtype=$(mdt_quota_type)
+	local ost_qtype=$(ost_quota_type)
+
+	echo "[HOST:$HOSTNAME] [old_mdt_qtype:$mdt_qtype]" \
+		"[old_ost_qtype:$ost_qtype] [new_qtype:$QUOTA_TYPE]"
+
+	export old_MDT_QUOTA_TYPE=$mdt_qtype
+	export old_OST_QUOTA_TYPE=$ost_qtype
+
+	do_facet mgs $LCTL conf_param $FSNAME.quota.mdt=$QUOTA_TYPE ||
+		error "set mdt quota type failed"
+	do_facet mgs $LCTL conf_param $FSNAME.quota.ost=$QUOTA_TYPE ||
+		error "set ost quota type failed"
+
+	local quota_usrs=$QUOTA_USERS
+
+	# get_filesystem_size
+	local disksz=$(lfs df $mntpt | grep "filesystem summary:" |
+		     awk '{print $3}')
+	local blk_soft=$((disksz + 1024))
+	local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over
+
+	local inodes=$(lfs df -i $mntpt | grep "filesystem summary:" |
+		     awk '{print $3}')
+	local i_soft=$inodes
+	local i_hard=$((i_soft + i_soft / 20))
+
+	echo "Total disk size: $disksz  block-softlimit: $blk_soft" \
+		"block-hardlimit: $blk_hard inode-softlimit: $i_soft" \
+		"inode-hardlimit: $i_hard"
+
+	local cmd
+	for usr in $quota_usrs; do
+		echo "Setting up quota on $HOSTNAME:$mntpt for $usr..."
+		for type in u g; do
+			cmd="$LFS setquota -$type $usr -b $blk_soft"
+			cmd="$cmd -B $blk_hard -i $i_soft -I $i_hard $mntpt"
+			echo "+ $cmd"
+			eval $cmd || error "$cmd FAILED!"
+		done
+		# display the quota status
+		echo "Quota settings for $usr : "
+		$LFS quota -v -u $usr $mntpt || true
+	done
+}
+
 zconf_mount() {
     local client=$1
     local mnt=$2
@@ -845,7 +1230,7 @@ sanity_mount_check_nodes () {
     local rc=0
     for mnt in $mnts ; do
         do_nodes $nodes "running=\\\$(grep -c $mnt' ' /proc/mounts);
-mpts=\\\$(mount | grep -w -c $mnt);
+mpts=\\\$(mount | grep -c $mnt' ');
 if [ \\\$running -ne \\\$mpts ]; then
     echo \\\$(hostname) env are INSANE!;
     exit 1;
@@ -915,7 +1300,7 @@ fi;
 exit \\\$rc" || return ${PIPESTATUS[0]}
 
     echo "Started clients $clients: "
-    do_nodes $clients "mount | grep -w $mnt"
+    do_nodes $clients "mount | grep $mnt' '"
 
     set_default_debug_nodes $clients
 
@@ -933,18 +1318,16 @@ zconf_umount_clients() {
     do_nodes $clients "running=\\\$(grep -c $mnt' ' /proc/mounts);
 if [ \\\$running -ne 0 ] ; then
 echo Stopping client \\\$(hostname) $mnt opts:$force;
-lsof -t $mnt || need_kill=no;
+lsof $mnt || need_kill=no;
 if [ "x$force" != "x" -a "x\\\$need_kill" != "xno" ]; then
     pids=\\\$(lsof -t $mnt | sort -u);
     if [ -n \\\"\\\$pids\\\" ]; then
              kill -9 \\\$pids;
     fi
 fi;
-busy=\\\$(umount $force $mnt 2>&1 | grep -c "busy");
-if [ \\\$busy -ne 0 ] ; then
+while umount $force $mnt 2>&1 | grep -q "busy"; do
     echo "$mnt is still busy, wait one second" && sleep 1;
-    umount $force $mnt;
-fi
+done;
 fi"
 }
 
@@ -996,12 +1379,12 @@ facets_on_host () {
     echo $(comma_list $affected)
 }
 
-facet_up () {
-    local facet=$1
-    local host=${2:-$(facet_host $facet)}
+facet_up() {
+	local facet=$1
+	local host=${2:-$(facet_host $facet)}
 
-    local label=$(convert_facet2label $facet)
-    do_node $host lctl dl | awk '{print $4}' | grep -q $label
+	local label=$(convert_facet2label $facet)
+	do_node $host $LCTL dl | awk '{print $4}' | grep -q -x $label
 }
 
 facets_up_on_host () {
@@ -1042,12 +1425,13 @@ remount_facet() {
 }
 
 reboot_facet() {
-    local facet=$1
-    if [ "$FAILURE_MODE" = HARD ]; then
-        reboot_node $(facet_active_host $facet)
-    else
-        sleep 10
-    fi
+	local facet=$1
+	if [ "$FAILURE_MODE" = HARD ]; then
+		reboot_node $(facet_active_host $facet)
+	else
+		refresh_disk ${facet}
+		sleep 10
+	fi
 }
 
 boot_node() {
@@ -1101,14 +1485,20 @@ start_client_load() {
     eval export ${var}=$load
 
     do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \
-                              BREAK_ON_ERROR=$BREAK_ON_ERROR \
-                              END_RUN_FILE=$END_RUN_FILE \
-                              LOAD_PID_FILE=$LOAD_PID_FILE \
-                              TESTSUITELOG=$TESTSUITELOG \
-                              run_${load}.sh" &
-    CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $!"
+BREAK_ON_ERROR=$BREAK_ON_ERROR \
+END_RUN_FILE=$END_RUN_FILE \
+LOAD_PID_FILE=$LOAD_PID_FILE \
+TESTLOG_PREFIX=$TESTLOG_PREFIX \
+TESTNAME=$TESTNAME \
+DBENCH_LIB=$DBENCH_LIB \
+DBENCH_SRC=$DBENCH_SRC \
+run_${load}.sh" &
+    local ppid=$!
     log "Started client load: ${load} on $client"
 
+    # get the children process IDs
+    local pids=$(ps --ppid $ppid -o pid= | xargs)
+    CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $ppid $pids"
     return 0
 }
 
@@ -1125,14 +1515,14 @@ start_client_loads () {
     sleep 2
 }
 
-# only for remote client 
+# only for remote client
 check_client_load () {
     local client=$1
     local var=$(node_var_name $client)_load
     local TESTLOAD=run_${!var}.sh
 
     ps auxww | grep -v grep | grep $client | grep -q "$TESTLOAD" || return 1
-    
+
     # bug 18914: try to connect several times not only when
     # check ps, but  while check_catastrophe also
     local tries=3
@@ -1210,7 +1600,7 @@ restart_client_loads () {
             if [ "$rc" != 0 ]; then
                 log "Client load failed to restart on node $client, rc=$rc"
                 # failure one client load means test fail
-                # we do not need to check other 
+                # we do not need to check other
                 return $rc
             fi
         else
@@ -1218,6 +1608,70 @@ restart_client_loads () {
         fi
     done
 }
+
+# Start vmstat and save its process ID in a file.
+start_vmstat() {
+    local nodes=$1
+    local pid_file=$2
+
+    [ -z "$nodes" -o -z "$pid_file" ] && return 0
+
+    do_nodes $nodes \
+        "vmstat 1 > $TESTLOG_PREFIX.$TESTNAME.vmstat.\\\$(hostname -s).log \
+        2>/dev/null </dev/null & echo \\\$! > $pid_file"
+}
+
+# Display the nodes on which client loads failed.
+print_end_run_file() {
+    local file=$1
+    local node
+
+    [ -s $file ] || return 0
+
+    echo "Found the END_RUN_FILE file: $file"
+    cat $file
+
+    # A client load will stop if it finds the END_RUN_FILE file.
+    # That does not mean the client load actually failed though.
+    # The first node in END_RUN_FILE is the one we are interested in.
+    read node < $file
+
+    if [ -n "$node" ]; then
+        local var=$(node_var_name $node)_load
+
+        local prefix=$TESTLOG_PREFIX
+        [ -n "$TESTNAME" ] && prefix=$prefix.$TESTNAME
+        local stdout_log=$prefix.run_${!var}_stdout.$node.log
+        local debug_log=$(echo $stdout_log | sed 's/\(.*\)stdout/\1debug/')
+
+        echo "Client load ${!var} failed on node $node:"
+        echo "$stdout_log"
+        echo "$debug_log"
+    fi
+}
+
+# Stop the process which had its PID saved in a file.
+stop_process() {
+    local nodes=$1
+    local pid_file=$2
+
+    [ -z "$nodes" -o -z "$pid_file" ] && return 0
+
+    do_nodes $nodes "test -f $pid_file &&
+        { kill -s TERM \\\$(cat $pid_file); rm -f $pid_file; }" || true
+}
+
+# Stop all client loads.
+stop_client_loads() {
+    local nodes=${1:-$CLIENTS}
+    local pid_file=$2
+
+    # stop the client loads
+    stop_process $nodes $pid_file
+
+    # clean up the processes that started them
+    [ -n "$CLIENT_LOAD_PIDS" ] && kill -9 $CLIENT_LOAD_PIDS 2>/dev/null || true
+}
 # End recovery-scale functions
 
 # verify that lustre actually cleaned up properly
@@ -1252,25 +1706,34 @@ wait_update () {
 
         local RESULT
         local WAIT=0
-        local sleep=5
+        local sleep=1
+        local print=10
         while [ true ]; do
             RESULT=$(do_node $node "$TEST")
             if [ "$RESULT" == "$FINAL" ]; then
-                echo "Updated after $WAIT sec: wanted '$FINAL' got '$RESULT'"
+                [ -z "$RESULT" -o $WAIT -le $sleep ] ||
+                    echo "Updated after ${WAIT}s: wanted '$FINAL' got '$RESULT'"
                 return 0
             fi
             [ $WAIT -ge $MAX ] && break
-            echo "Waiting $((MAX - WAIT)) secs for update"
+            [ $((WAIT % print)) -eq 0 ] &&
+                echo "Waiting $((MAX - WAIT)) secs for update"
             WAIT=$((WAIT + sleep))
             sleep $sleep
         done
-        echo "Update not seen after $MAX sec: wanted '$FINAL' got '$RESULT'"
+        echo "Update not seen after ${MAX}s: wanted '$FINAL' got '$RESULT'"
         return 3
 }
 
-wait_update_facet () {
-    local facet=$1
-    wait_update  $(facet_active_host $facet) "$@"
+wait_update_facet() {
+	local facet=$1
+	shift
+	wait_update $(facet_active_host $facet) "$@"
+}
+
+sync_all_data() {
+	do_node $(osts_nodes) "lctl set_param -n osd*.*OS*.force_sync 1" 2>&1 |
+		grep -v 'Found no match'
 }
 
 wait_delete_completed () {
@@ -1517,14 +1980,6 @@ facet_failover() {
 
     echo "Failing $facet on node $host"
 
-    # Make sure the client data is synced to disk. LU-924
-    #
-    # We don't write client data synchrnously (to avoid flooding sync writes
-    # when there are many clients connecting), so if the server reboots before
-    # the client data reachs disk, the client data will be lost and the client
-    # will be evicted after recovery, which is not what we expected.
-    do_facet $facet "sync; sync; sync"
-
     local affected=$(affected_facets $facet)
 
     shutdown_facet $facet
@@ -1609,13 +2064,14 @@ fail_nodf() {
 }
 
 fail_abort() {
-    local facet=$1
-    stop $facet
-    change_active $facet
-    wait_for_facet $facet
-    mount_facet $facet -o abort_recovery
-    clients_up || echo "first df failed: $?"
-    clients_up || error "post-failover df: $?"
+	local facet=$1
+	stop $facet
+	refresh_disk ${facet}
+	change_active $facet
+	wait_for_facet $facet
+	mount_facet $facet -o abort_recovery
+	clients_up || echo "first df failed: $?"
+	clients_up || error "post-failover df: $?"
 }
 
 do_lmc() {
@@ -1777,6 +2233,25 @@ facet_host() {
     echo -n ${!varname}
 }
 
+facet_failover_host() {
+	local facet=$1
+	local var
+
+	var=${facet}failover_HOST
+	if [ -n "${!var}" ]; then
+		echo ${!var}
+		return
+	fi
+
+	if [[ $facet == ost* ]]; then
+		var=ostfailover_HOST
+		if [ -n "${!var}" ]; then
+			echo ${!var}
+			return
+		fi
+	fi
+}
+
 facet_active() {
     local facet=$1
     local activevar=${facet}active
@@ -1886,13 +2361,34 @@ single_local_node () {
 
 # Outputs environment variable assignments that should be passed to remote nodes
 get_env_vars() {
-    local var
-    local value
+	local var
+	local value
+	local facets=$(get_facets)
+	local facet
 
-    for var in ${!MODOPTS_*}; do
-        value=${!var}
-        echo "${var}=\"$value\""
-    done
+	for var in ${!MODOPTS_*}; do
+		value=${!var}
+		echo -n " ${var}=\"$value\""
+	done
+
+	echo -n " USE_OFD=$USE_OFD"
+
+	for facet in ${facets//,/ }; do
+		var=${facet}_FSTYPE
+		if [ -n "${!var}" ]; then
+			echo -n " $var=${!var}"
+		fi
+	done
+
+	for var in MGSFSTYPE MDSFSTYPE OSTFSTYPE; do
+		if [ -n "${!var}" ]; then
+			echo -n " $var=${!var}"
+		fi
+	done
+
+	if [ -n "$FSTYPE" ]; then
+		echo -n " FSTYPE=$FSTYPE"
+	fi
 }
 
 do_nodes() {
@@ -1921,6 +2417,7 @@ do_nodes() {
     [ -z "$myPDSH" -o "$myPDSH" = "no_dsh" -o "$myPDSH" = "rsh" ] && \
         echo "cannot run remote command on $rnodes with $myPDSH" && return 128
 
+    export FANOUT=$(get_node_count "${rnodes//,/ }")
     if $VERBOSE; then
         echo "CMD: $rnodes $@" >&2
         $myPDSH $rnodes "$LCTL mark \"$@\"" > /dev/null 2>&1 || :
@@ -1944,6 +2441,25 @@ do_facet() {
     do_node $HOST "$@"
 }
 
+# Function: do_facet_random_file $FACET $FILE $SIZE
+# Creates FILE with random content on the given FACET of given SIZE
+
+do_facet_random_file() {
+	local facet="$1"
+	local fpath="$2"
+	local fsize="$3"
+	local cmd="dd if=/dev/urandom of='$fpath' bs=$fsize count=1"
+	do_facet $facet "$cmd 2>/dev/null"
+}
+
+do_facet_create_file() {
+	local facet="$1"
+	local fpath="$2"
+	local fsize="$3"
+	local cmd="dd if=/dev/zero of='$fpath' bs=$fsize count=1"
+	do_facet $facet "$cmd 2>/dev/null"
+}
+
 do_nodesv() {
     do_nodes --verbose "$@"
 }
@@ -1961,19 +2477,121 @@ add() {
 ostdevname() {
     num=$1
     DEVNAME=OSTDEV$num
-    #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
-    eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}}
+
+	local fstype=$(facet_fstype ost$num)
+
+	case $fstype in
+		ldiskfs )
+			#if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
+			eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};;
+		zfs )
+			#dataset name is independent of vdev device names
+			eval DEVPTR=${FSNAME}-ost${num}/ost${num};;
+		* )
+			error "unknown fstype!";;
+	esac
+
     echo -n $DEVPTR
 }
 
+ostvdevname() {
+	num=$1
+	DEVNAME=OSTDEV$num
+
+	local fstype=$(facet_fstype ost$num)
+
+	case $fstype in
+		ldiskfs )
+			# vdevs are not supported by ldiskfs
+			eval VDEVPTR="";;
+		zfs )
+			#if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
+			eval VDEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};;
+		* )
+			error "unknown fstype!";;
+	esac
+
+	echo -n $VDEVPTR
+}
+
 mdsdevname() {
     num=$1
     DEVNAME=MDSDEV$num
-    #if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
-    eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}}
+
+	local fstype=$(facet_fstype mds$num)
+
+	case $fstype in
+		ldiskfs )
+			#if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
+			eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};;
+		zfs )
+			#dataset name is independent of vdev device names
+			eval DEVPTR=${FSNAME}-mdt${num}/mdt${num};;
+		* )
+			error "unknown fstype!";;
+	esac
+
+	echo -n $DEVPTR
+}
+
+mdsvdevname() {
+	num=$1
+	DEVNAME=MDSDEV$num
+
+	local fstype=$(facet_fstype mds$num)
+
+	case $fstype in
+		ldiskfs )
+			# vdevs are not supported by ldiskfs
+			eval VDEVPTR="";;
+		zfs )
+			#if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
+			eval VDEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};;
+		* )
+			error "unknown fstype!";;
+	esac
+
+	echo -n $VDEVPTR
+}
+
+mgsdevname() {
+	DEVNAME=MGSDEV
+
+	local fstype=$(facet_fstype mds$num)
+
+	case $fstype in
+		ldiskfs )
+			#if $MGSDEV isn't defined, default is $MDSDEV1
+			eval DEVPTR=${!DEVNAME:=${MDSDEV1}};;
+		zfs )
+			#dataset name is independent of vdev device names
+			eval DEVPTR=${FSNAME}-mgs/mgs;;
+		* )
+			error "unknown fstype!";;
+	esac
+
     echo -n $DEVPTR
 }
 
+mgsvdevname() {
+	DEVNAME=MGSDEV
+
+	local fstype=$(facet_fstype mds$num)
+
+	case $fstype in
+		ldiskfs )
+			# vdevs are not supported by ldiskfs
+			eval VDEVPTR="";;
+		zfs )
+			#if $MGSDEV isn't defined, default is $MGSDEV1
+			eval VDEVPTR=${!DEVNAME:=${MDSDEV1}};;
+		* )
+			error "unknown fstype!";;
+	esac
+
+	echo -n $VDEVPTR
+}
+
 facet_mntpt () {
     local facet=$1
     [[ $facet = mgs ]] && combined_mgs_mds && facet="mds1"
@@ -2046,88 +2664,123 @@ combined_mgs_mds () {
     [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]]
 }
 
-mkfs_opts () {
-    local facet=$1
+lower() {
+	echo -n "$1" | tr '[:upper:]' '[:lower:]'
+}
 
-    local tgt=$(echo $facet | tr -d [:digit:] | tr "[:lower:]" "[:upper:]")
-    local optvar=${tgt}_MKFS_OPTS
-    local opt=${!optvar}
+upper() {
+	echo -n "$1" | tr '[:lower:]' '[:upper:]'
+}
 
-    # FIXME: ! combo  mgs/mds + mgsfailover is not supported yet
-    [[ $facet = mgs ]] && echo $opt && return
+mkfs_opts() {
+	local facet=$1
+	local type=$(facet_type $facet)
+	local index=$(($(facet_number $facet) - 1))
+	local fstype=$(facet_fstype $facet)
+	local opts
+	local fs_mkfs_opts
+	local var
 
-    # 1.
-    # --failnode options 
-    local var=${facet}failover_HOST
-    if [ x"${!var}" != x ] && [ x"${!var}" != x$(facet_host $facet) ] ; then
-        local failnode=$(h2$NETTYPE ${!var})
-        failnode="--failnode=$failnode"
-        # options does not contain
-        # or contains wrong --failnode=
-        if [[ $opt != *${failnode}* ]]; then
-            opt=$(echo $opt | sed 's/--failnode=.* / /')
-            opt="$opt $failnode"
-        fi
-    fi
+	if [ $type == MGS ] && combined_mgs_mds; then
+		return 1
+	fi
 
-    # 2.
-    # --mgsnode options
-    # no additional mkfs mds "--mgsnode" option for this configuration
-    if [[ $facet = mds ]] && combined_mgs_mds; then
-        echo $opt
-        return
-    fi
+	if [ $type == MGS ] || ( [ $type == MDS ] && combined_mgs_mds ); then
+		opts="--mgs"
+	else
+		opts="--mgsnode=$MGSNID"
+	fi
 
-    # additional mkfs "--mgsnode"
-    local mgsnode="--mgsnode=$MGSNID"
-    opt=${opt//$mgsnode }
-    for nid in ${MGSNID//:/ }; do
-        local mgsnode="--mgsnode=$nid"
-        # options does not contain
-        # --mgsnode=$nid
-        if [[ $opt != *${mgsnode}" "* ]]; then
-            opt="$opt --mgsnode=$nid"
-        fi
-    done
+	if [ $type != MGS ]; then
+		opts+=" --fsname=$FSNAME --$(lower ${type/MDS/MDT}) --index=$index"
+	fi
 
-    echo $opt
-}
+	var=${facet}failover_HOST
+	if [ -n "${!var}" ] && [ ${!var} != $(facet_host $facet) ]; then
+		opts+=" --failnode=$(h2$NETTYPE ${!var})"
+	fi
 
-formatall() {
-    if [ "$IAMDIR" == "yes" ]; then
-        MDS_MKFS_OPTS="$MDS_MKFS_OPTS --iam-dir"
-    fi
+	opts+=${TIMEOUT:+" --param=sys.timeout=$TIMEOUT"}
+	opts+=${LDLM_TIMEOUT:+" --param=sys.ldlm_timeout=$LDLM_TIMEOUT"}
 
-    [ "$FSTYPE" ] && FSTYPE_OPT="--backfstype $FSTYPE"
+	if [ $type == MDS ]; then
+		opts+=${SECLEVEL:+" --param=mdt.sec_level"}
+		opts+=${MDSCAPA:+" --param-mdt.capa=$MDSCAPA"}
+		opts+=${STRIPE_BYTES:+" --param=lov.stripesize=$STRIPE_BYTES"}
+		opts+=${STRIPES_PER_OBJ:+" --param=lov.stripecount=$STRIPES_PER_OBJ"}
+		opts+=${L_GETIDENTITY:+" --param=mdt.identity_upcall=$L_GETIDENTITY"}
 
-    stopall
-    # We need ldiskfs here, may as well load them all
-    load_modules
-    [ "$CLIENTONLY" ] && return
-    echo Formatting mgs, mds, osts
-    if ! combined_mgs_mds ; then
-        add mgs $(mkfs_opts mgs) $FSTYPE_OPT --reformat $MGSDEV || exit 10
-    fi
+		if [ $fstype == ldiskfs ]; then
+			opts+=${IAMDIR:+" --iam-dir"}
 
-    for num in `seq $MDSCOUNT`; do
-        echo "Format mds$num: $(mdsdevname $num)"
-        if $VERBOSE; then
-            add mds$num $(mkfs_opts mds) $FSTYPE_OPT --reformat $(mdsdevname $num) || exit 10
-        else
-            add mds$num $(mkfs_opts mds) $FSTYPE_OPT --reformat $(mdsdevname $num) > /dev/null || exit 10
-        fi
-    done
+			fs_mkfs_opts+=${MDSJOURNALSIZE:+" -J size=$MDSJOURNALSIZE"}
+			fs_mkfs_opts+=${MDSISIZE:+" -i $MDSISIZE"}
+		fi
+	fi
 
-    # the ost-s could have different OST_MKFS_OPTS
-    # because of different failnode-s
-    for num in `seq $OSTCOUNT`; do
-        echo "Format ost$num: $(ostdevname $num)"
-        if $VERBOSE; then
-            add ost$num $(mkfs_opts ost${num}) $FSTYPE_OPT --reformat `ostdevname $num` || exit 10
-        else
-            add ost$num $(mkfs_opts ost${num}) $FSTYPE_OPT --reformat `ostdevname $num` > /dev/null || exit 10
-        fi
-    done
+	if [ $type == OST ]; then
+		opts+=${SECLEVEL:+" --param=ost.sec_level"}
+		opts+=${OSSCAPA:+" --param=ost.capa=$OSSCAPA"}
+
+		if [ $fstype == ldiskfs ]; then
+			fs_mkfs_opts+=${OSTJOURNALSIZE:+" -J size=$OSTJOURNALSIZE"}
+		fi
+	fi
+
+	opts+=" --backfstype=$fstype"
+
+	var=${type}SIZE
+	if [ -n "${!var}" ]; then
+		opts+=" --device-size=${!var}"
+	fi
+
+	var=$(upper $fstype)_MKFS_OPTS
+	fs_mkfs_opts+=${!var:+" ${!var}"}
+
+	var=${type}_FS_MKFS_OPTS
+	fs_mkfs_opts+=${!var:+" ${!var}"}
+
+	if [ -n "${fs_mkfs_opts## }" ]; then
+		opts+=" --mkfsoptions=\\\"${fs_mkfs_opts## }\\\""
+	fi
+
+	var=${type}OPT
+	opts+=${!var:+" ${!var}"}
+
+	echo -n "$opts"
+}
+
+formatall() {
+	local quiet
+
+	if ! $VERBOSE; then
+		quiet=yes
+	fi
+
+	stopall
+	# We need ldiskfs here, may as well load them all
+	load_modules
+	[ "$CLIENTONLY" ] && return
+	echo Formatting mgs, mds, osts
+	if ! combined_mgs_mds ; then
+		echo "Format mgs: $(mgsdevname)"
+		add mgs $(mkfs_opts mgs) --reformat $(mgsdevname) \
+			$(mgsvdevname) ${quiet:+>/dev/null} || exit 10
+		fi
+
+		for num in `seq $MDSCOUNT`; do
+			echo "Format mds$num: $(mdsdevname $num)"
+			add mds$num $(mkfs_opts mds$num) --reformat \
+			$(mdsdevname $num) $(mdsvdevname $num) \
+			${quiet:+>/dev/null} || exit 10
+		done
+
+		for num in `seq $OSTCOUNT`; do
+			echo "Format ost$num: $(ostdevname $num)"
+			add ost$num $(mkfs_opts ost$num) --reformat \
+			$(ostdevname $num) $(ostvdevname ${num}) \
+			${quiet:+>/dev/null} || exit 10
+		done
 }
 
 mount_client() {
@@ -2207,7 +2860,7 @@ setupall() {
         echo $WRITECONF | grep -q "writeconf" && \
             writeconf_all
         if ! combined_mgs_mds ; then
-            start mgs $MGSDEV $MGS_MOUNT_OPTS
+			start mgs $(mgsdevname) $MGS_MOUNT_OPTS
         fi
 
         for num in `seq $MDSCOUNT`; do
@@ -2250,6 +2903,7 @@ setupall() {
     [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
     mount_client $MOUNT
     [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT
+    clients_up
 
     if [ "$MOUNT_2" ]; then
         mount_client $MOUNT2
@@ -2287,7 +2941,7 @@ init_facet_vars () {
     eval export ${facet}_opt=\"$@\"
 
     local dev=${facet}_dev
-    local label=$(do_facet ${facet} "$E2LABEL ${!dev}")
+	local label=$(devicelabel ${facet} ${!dev})
     [ -z "$label" ] && echo no label for ${!dev} && exit 1
 
     eval export ${facet}_svc=${label}
@@ -2327,7 +2981,7 @@ init_facets_vars () {
         done
     fi
 
-    combined_mgs_mds || init_facet_vars mgs $MGSDEV $MGS_MOUNT_OPTS
+	combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS
 
     remote_ost_nodsh && return
 
@@ -2357,31 +3011,46 @@ osc_ensure_active () {
 }
 
 init_param_vars () {
-    if ! remote_ost_nodsh && ! remote_mds_nodsh; then
-        export MDSVER=$(do_facet $SINGLEMDS "lctl get_param version" | cut -d. -f1,2)
-        export OSTVER=$(do_facet ost1 "lctl get_param version" | cut -d. -f1,2)
-        export CLIVER=$(lctl get_param version | cut -d. -f 1,2)
-    fi
-
-    remote_mds_nodsh ||
-        TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout")
-
-    log "Using TIMEOUT=$TIMEOUT"
-
-    osc_ensure_active $SINGLEMDS $TIMEOUT
-    osc_ensure_active client $TIMEOUT
-
-    if [ $QUOTA_AUTO -ne 0 ]; then
-        if [ "$ENABLE_QUOTA" ]; then
-            echo "enable quota as required"
-            setup_quota $MOUNT || return 2
-        else
-            echo "disable quota as required"
-            $LFS quotaoff -ug $MOUNT > /dev/null 2>&1
-        fi
-    fi
-
-    return 0
+	remote_mds_nodsh ||
+		TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout")
+
+	log "Using TIMEOUT=$TIMEOUT"
+
+	osc_ensure_active $SINGLEMDS $TIMEOUT
+	osc_ensure_active client $TIMEOUT
+
+	local jobid_var
+	if [ -z "$(lctl get_param -n mdc.*.connect_flags | grep jobstats)" ]; then
+		jobid_var="none"
+	elif [ $JOBSTATS_AUTO -ne 0 ]; then
+		echo "enable jobstats, set job scheduler as $JOBID_VAR"
+		jobid_var=$JOBID_VAR
+	else
+		jobid_var=`$LCTL get_param -n jobid_var`
+		if [ $jobid_var != "disable" ]; then
+			echo "disable jobstats as required"
+			jobid_var="disable"
+		else
+			jobid_var="none"
+		fi
+	fi
+
+	if [ $jobid_var == $JOBID_VAR -o $jobid_var == "disable" ]; then
+		do_facet mgs $LCTL conf_param $FSNAME.sys.jobid_var=$jobid_var
+		wait_update $HOSTNAME "$LCTL get_param -n jobid_var" \
+			$jobid_var || return 1
+	fi
+
+	if [ $QUOTA_AUTO -ne 0 ]; then
+		if [ "$ENABLE_QUOTA" ]; then
+			echo "enable quota as required"
+			setup_quota $MOUNT || return 2
+		else
+			echo "disable quota as required"
+			$LFS quotaoff -ug $MOUNT > /dev/null 2>&1
+		fi
+	fi
+	return 0
 }
 
 nfs_client_mode () {
@@ -2545,12 +3214,14 @@ check_and_setup_lustre() {
         set_default_debug_nodes $(comma_list $(nodes_list))
     fi
 
-    init_gss
-    set_flavor_all $SEC
+	init_gss
+	if $GSS; then
+		set_flavor_all $SEC
+	fi
 
-    if [ "$ONLY" == "setup" ]; then
-        exit 0
-    fi
+	if [ "$ONLY" == "setup" ]; then
+		exit 0
+	fi
 }
 
 restore_mount () {
@@ -2580,26 +3251,24 @@ cleanup_and_setup_lustre() {
 
 # Get all of the server target devices from a given server node and type.
 get_mnt_devs() {
-    local node=$1
-    local type=$2
-    local obd_type
-    local devs
-    local dev
-
-    case $type in
-    mdt) obd_type="osd" ;;
-    ost) obd_type="obdfilter" ;; # needs to be fixed when OST also uses an OSD
-    *) echo "invalid server type" && return 1 ;;
-    esac
-
-    devs=$(do_node $node "lctl get_param -n $obd_type*.*.mntdev")
-    for dev in $devs; do
-        case $dev in
-        *loop*) do_node $node "losetup $dev" | \
-                sed -e "s/.*(//" -e "s/).*//" ;;
-        *) echo $dev ;;
-        esac
-    done
+	local node=$1
+	local type=$2
+	local devs
+	local dev
+
+	if [ "$type" == ost ]; then
+		devs=$(get_obdfilter_param $node "" mntdev)
+	else
+		devs=$(do_node $node \
+		       "lctl get_param -n osd-*.$FSNAME-M*.mntdev")
+	fi
+	for dev in $devs; do
+		case $dev in
+		*loop*) do_node $node "losetup $dev" | \
+				sed -e "s/.*(//" -e "s/).*//" ;;
+		*) echo $dev ;;
+		esac
+	done
 }
 
 # Get all of the server target devices.
@@ -2633,24 +3302,32 @@ run_e2fsck() {
     return 0
 }
 
+# verify a directory is shared among nodes.
+check_shared_dir() {
+    local dir=$1
+
+    [ -z "$dir" ] && return 1
+    do_rpc_nodes $(comma_list $(nodes_list)) check_logdir $dir
+    check_write_access $dir || return 1
+    return 0
+}
+
 # Run e2fsck on MDT and OST(s) to generate databases used for lfsck.
 generate_db() {
     local i
     local ostidx
     local dev
-    local tmp_file
 
-    [ $MDSCOUNT -eq 1 ] || error "CMD is not supported"
-    tmp_file=$(mktemp -p $SHARED_DIRECTORY || 
-        error "fail to create file in $SHARED_DIRECTORY")
+	[[ $(lustre_version_code $SINGLEMDS) -ne $(version_code 2.2.0) ]] ||
+		{ skip "Lustre 2.2.0 lacks the patch for LU-1255"; exit 0; }
+
+    check_shared_dir $SHARED_DIRECTORY ||
+        error "$SHARED_DIRECTORY isn't a shared directory"
 
-    # make sure everything gets to the backing store
-    local list=$(comma_list $CLIENTS $(facet_host $SINGLEMDS) $(osts_nodes))
-    do_nodes $list "sync; sleep 2; sync"
+    export MDSDB=$SHARED_DIRECTORY/mdsdb
+    export OSTDB=$SHARED_DIRECTORY/ostdb
 
-    do_nodes $list ls $tmp_file || \
-        error "$SHARED_DIRECTORY is not a shared directory"
-    rm $tmp_file
+    [ $MDSCOUNT -eq 1 ] || error "CMD is not supported"
 
     run_e2fsck $(mdts_nodes) $MDTDEV "--mdsdb $MDSDB"
 
@@ -2687,10 +3364,11 @@ check_and_cleanup_lustre() {
         run_lfsck
     fi
 
-    if is_mounted $MOUNT; then
-        [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]*
-        [ "$ENABLE_QUOTA" ] && restore_quota_type || true
-    fi
+	if is_mounted $MOUNT; then
+		[ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* ||
+			error "remove sub-test dirs failed"
+		[ "$ENABLE_QUOTA" ] && restore_quota || true
+	fi
 
     if [ "$I_UMOUNTED2" = "yes" ]; then
         restore_mount $MOUNT2 || error "restore $MOUNT2 failed"
@@ -2863,6 +3541,10 @@ at_max_get() {
     at_get $1 at_max
 }
 
+at_min_get() {
+	at_get $1 at_min
+}
+
 at_max_set() {
     local at_max=$1
     shift
@@ -2923,11 +3605,14 @@ pause_bulk() {
 
 drop_ldlm_cancel() {
 #define OBD_FAIL_LDLM_CANCEL             0x304
-    RC=0
-    do_facet client lctl set_param fail_loc=0x304
-    do_facet client "$@" || RC=$?
-    do_facet client lctl set_param fail_loc=0
-    return $RC
+	local RC=0
+	local list=$(comma_list $(mdts_nodes) $(osts_nodes))
+	do_nodes $list lctl set_param fail_loc=0x304
+
+	do_facet client "$@" || RC=$?
+
+	do_nodes $list lctl set_param fail_loc=0
+	return $RC
 }
 
 drop_bl_callback() {
@@ -3034,6 +3719,17 @@ stop_full_debug_logging() {
     debugrestore
 }
 
+# prints bash call stack
+log_trace_dump() {
+	echo "  Trace dump:"
+	for (( i=1; i < ${#BASH_LINENO[*]} ; i++ )) ; do
+		local s=${BASH_SOURCE[$i]}
+		local l=${BASH_LINENO[$i-1]}
+		local f=${FUNCNAME[$i]}
+		echo "  = $s:$l:$f()"
+	done
+}
+
 ##################################
 # Test interface
 ##################################
@@ -3049,7 +3745,9 @@ error_noexit() {
     fi
 
     log " ${TESTSUITE} ${TESTNAME}: @@@@@@ ${TYPE}: $@ "
+    log_trace_dump
 
+    mkdir -p $LOGDIR
     # We need to dump the logs on all nodes
     if $dump; then
         gather_logs $(comma_list $(nodes_list))
@@ -3061,11 +3759,11 @@ error_noexit() {
 }
 
 exit_status () {
-    local status=0
-    local log=$TESTSUITELOG
+	local status=0
+	local log=$TESTSUITELOG
 
-    [ -f "$log" ] && grep -q FAIL: $log && status=1
-    exit $status
+	[ -f "$log" ] && grep -q FAIL $log && status=1
+	exit $status
 }
 
 error() {
@@ -3086,17 +3784,29 @@ error_ignore() {
     error_noexit "$@"
 }
 
+error_and_remount() {
+	error_noexit "$@"
+	remount_client $MOUNT
+	exit 1
+}
+
 skip_env () {
     $FAIL_ON_SKIP_ENV && error false $@ || skip $@
 }
 
-skip () {
-    echo
-    log " SKIP: ${TESTSUITE} ${TESTNAME} $@"
-    [ "$ALWAYS_SKIPPED" ] && \
-        skip_logged ${TESTNAME} "$@" || true
-    [ "$TESTSUITELOG" ] && \
-        echo "${TESTSUITE}: SKIP: $TESTNAME $@" >> $TESTSUITELOG || true
+skip() {
+	echo
+	log " SKIP: $TESTSUITE $TESTNAME $@"
+
+	if [[ -n "$ALWAYS_SKIPPED" ]]; then
+		skip_logged $TESTNAME "$@"
+	else
+		mkdir -p $LOGDIR
+		echo "$@" > $LOGDIR/skip
+	fi
+
+	[[ -n "$TESTSUITELOG" ]] &&
+		echo "$TESTSUITE: SKIP: $TESTNAME $@" >> $TESTSUITELOG || true
 }
 
 build_test_filter() {
@@ -3164,41 +3874,36 @@ run_test() {
         echo -n "."
         return 0
     fi
+
+	LAST_SKIPPED="y"
+	ALWAYS_SKIPPED="y"
     testname=EXCEPT_$1
     if [ ${!testname}x != x ]; then
-        LAST_SKIPPED="y"
         TESTNAME=test_$1 skip "skipping excluded test $1"
         return 0
     fi
     testname=EXCEPT_$base
     if [ ${!testname}x != x ]; then
-        LAST_SKIPPED="y"
         TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)"
         return 0
     fi
     testname=EXCEPT_ALWAYS_$1
     if [ ${!testname}x != x ]; then
-        LAST_SKIPPED="y"
-        ALWAYS_SKIPPED="y"
         TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1"
         return 0
     fi
     testname=EXCEPT_ALWAYS_$base
     if [ ${!testname}x != x ]; then
-        LAST_SKIPPED="y"
-        ALWAYS_SKIPPED="y"
         TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1 (base $base)"
         return 0
     fi
     testname=EXCEPT_SLOW_$1
     if [ ${!testname}x != x ]; then
-        LAST_SKIPPED="y"
         TESTNAME=test_$1 skip "skipping SLOW test $1"
         return 0
     fi
     testname=EXCEPT_SLOW_$base
     if [ ${!testname}x != x ]; then
-        LAST_SKIPPED="y"
         TESTNAME=test_$1 skip "skipping SLOW test $1 (base $base)"
         return 0
     fi
@@ -3246,13 +3951,15 @@ complete () {
 }
 
 pass() {
-    # Set TEST_STATUS here; will be used for logging the result
-    if [ -f $LOGDIR/err ]; then
-        TEST_STATUS="FAIL"
-    else
-        TEST_STATUS="PASS"
-    fi
-    echo "$TEST_STATUS $@" 2>&1 | tee -a $TESTSUITELOG
+	# Set TEST_STATUS here. It will be used for logging the result.
+	TEST_STATUS="PASS"
+
+	if [[ -f $LOGDIR/err ]]; then
+		TEST_STATUS="FAIL"
+	elif [[ -f $LOGDIR/skip ]]; then
+		TEST_STATUS="SKIP"
+	fi
+	echo "$TEST_STATUS $@" 2>&1 | tee -a $TESTSUITELOG
 }
 
 check_mds() {
@@ -3295,7 +4002,7 @@ banner() {
 run_one() {
     local testnum=$1
     local message=$2
-    tfile=f${testnum}
+    tfile=f.${TESTSUITE}.${testnum}
     export tdir=d0.${TESTSUITE}/d${base}
     export TESTNAME=test_$testnum
     local SAVE_UMASK=`umask`
@@ -3321,42 +4028,49 @@ run_one() {
 #  - test result is saved to data file
 #
 run_one_logged() {
-    local BEFORE=`date +%s`
-    local TEST_ERROR
-    local name=${TESTSUITE}.test_${1}.test_log.$(hostname -s).log
-    local test_log=$LOGDIR/$name
-    rm -rf $LOGDIR/err
-    local SAVE_UMASK=`umask`
-    umask 0022
+	local BEFORE=`date +%s`
+	local TEST_ERROR
+	local name=${TESTSUITE}.test_${1}.test_log.$(hostname -s).log
+	local test_log=$LOGDIR/$name
+	rm -rf $LOGDIR/err
+	rm -rf $LOGDIR/skip
+	local SAVE_UMASK=`umask`
+	umask 0022
 
-    echo
-    log_sub_test_begin test_${1}
-    (run_one $1 "$2") 2>&1 | tee $test_log
-    local RC=${PIPESTATUS[0]}
+	echo
+	log_sub_test_begin test_${1}
+	(run_one $1 "$2") 2>&1 | tee -i $test_log
+	local RC=${PIPESTATUS[0]}
 
-    [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \
-        echo "test_$1 returned $RC" | tee $LOGDIR/err
+	[ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \
+		echo "test_$1 returned $RC" | tee $LOGDIR/err
 
-    duration=$((`date +%s` - $BEFORE))
-    pass "$1" "(${duration}s)"
-    [ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err)
-    log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR"
+	duration=$((`date +%s` - $BEFORE))
+	pass "$1" "(${duration}s)"
 
-    if [ -f $LOGDIR/err ]; then
-        $FAIL_ON_ERROR && exit $RC
-    fi
+	if [[ -f $LOGDIR/err ]]; then
+		TEST_ERROR=$(cat $LOGDIR/err)
+	elif [[ -f $LOGDIR/skip ]]; then
+		TEST_ERROR=$(cat $LOGDIR/skip)
+	fi
+	log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR"
 
-    umask $SAVE_UMASK
+	if [ -f $LOGDIR/err ]; then
+		$FAIL_ON_ERROR && exit $RC
+	fi
 
-    return 0
+	umask $SAVE_UMASK
+
+	return 0
 }
 
 #
 # Print information of skipped tests to result.yml
 #
 skip_logged(){
-    log_sub_test_begin $1
-    log_sub_test_end "SKIP" "0" "0" "\"$2\""
+	log_sub_test_begin $1
+	shift
+	log_sub_test_end "SKIP" "0" "0" "$@"
 }
 
 canonical_path() {
@@ -3386,7 +4100,7 @@ check_grant() {
 
     # get server grant
     server_grant=`do_nodes $(comma_list $(osts_nodes)) \
-                    "$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" | \
+                    "$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" |
                     awk '{total += $1} END{print total}'`
 
     # check whether client grant == server grant
@@ -3430,6 +4144,11 @@ index_from_ostuuid()
     $LFS osts $2 | sed -ne "/${1}/s/\(.*\): .* .*$/\1/p"
 }
 
+mdtuuid_from_index()
+{
+    $LFS mdts $2 | sed -ne "/^$1: /s/.* \(.*\) .*$/\1/p"
+}
+
 remote_node () {
     local node=$1
     [ "$node" != "$(hostname)" ]
@@ -3600,7 +4319,8 @@ is_patchless ()
 }
 
 check_versions () {
-    [ "$MDSVER" = "$CLIVER" -a "$OSTVER" = "$CLIVER" ]
+    [ "$(lustre_version_code client)" = "$(lustre_version_code $SINGLEMDS)" -a \
+      "$(lustre_version_code client)" = "$(lustre_version_code ost1)" ]
 }
 
 get_node_count() {
@@ -3648,6 +4368,63 @@ setstripe_nfsserver () {
     do_nodev $nfsserver lfs setstripe "$@"
 }
 
+# Check and add a test group.
+add_group() {
+	local group_id=$1
+	local group_name=$2
+	local rc=0
+
+	local gid=$(getent group $group_name | cut -d: -f3)
+	if [[ -n "$gid" ]]; then
+		[[ "$gid" -eq "$group_id" ]] || {
+			error_noexit "inconsistent group ID:" \
+				     "new: $group_id, old: $gid"
+			rc=1
+		}
+	else
+		groupadd -g $group_id $group_name
+		rc=${PIPESTATUS[0]}
+	fi
+
+	return $rc
+}
+
+# Check and add a test user.
+add_user() {
+	local user_id=$1
+	shift
+	local user_name=$1
+	shift
+	local group_name=$1
+	shift
+	local home=$1
+	shift
+	local opts="$@"
+	local rc=0
+
+	local uid=$(getent passwd $user_name | cut -d: -f3)
+	if [[ -n "$uid" ]]; then
+		if [[ "$uid" -eq "$user_id" ]]; then
+			local dir=$(getent passwd $user_name | cut -d: -f6)
+			if [[ "$dir" != "$home" ]]; then
+				mkdir -p $home
+				usermod -d $home $user_name
+				rc=${PIPESTATUS[0]}
+			fi
+		else
+			error_noexit "inconsistent user ID:" \
+				     "new: $user_id, old: $uid"
+			rc=1
+		fi
+	else
+		mkdir -p $home
+		useradd -M -u $user_id -d $home -g $group_name $opts $user_name
+		rc=${PIPESTATUS[0]}
+	fi
+
+	return $rc
+}
+
 check_runas_id_ret() {
     local myRC=0
     local myRUNAS_UID=$1
@@ -3712,7 +4489,7 @@ fi"
 # Run multiop in the background, but wait for it to print
 # "PAUSING" to its stdout before returning from this function.
 multiop_bg_pause() {
-    MULTIOP_PROG=${MULTIOP_PROG:-multiop}
+    MULTIOP_PROG=${MULTIOP_PROG:-$MULTIOP}
     FILE=$1
     ARGS=$2
 
@@ -3823,25 +4600,6 @@ exit \\\$rc;"
     fi 
 }
 
-# $1 node
-# $2 file
-# $3 $RUNAS
-get_stripe_info() {
-        local tmp_file
-
-        stripe_size=0
-        stripe_count=0
-        stripe_index=0
-        tmp_file=$(mktemp)
-
-        do_facet $1 $3 lfs getstripe -v $2 > $tmp_file
-
-        stripe_size=`awk '$1 ~ /size/ {print $2}' $tmp_file`
-        stripe_count=`awk '$1 ~ /count/ {print $2}' $tmp_file`
-        stripe_index=`awk '$1 ~ /stripe_offset/ {print $2}' $tmp_file`
-        rm -f $tmp_file
-}
-
 # CMD: determine mds index where directory inode presents
 get_mds_dir () {
     local dir=$1
@@ -3905,19 +4663,12 @@ get_clientosc_proc_path() {
 
 get_lustre_version () {
     local facet=${1:-"$SINGLEMDS"}    
-    do_facet $facet $LCTL get_param -n version |  awk '/^lustre:/ {print $2}'
-}
-
-get_mds_version_major () {
-    local facet=${1:-"$SINGLEMDS"}
-    local version=$(get_lustre_version $facet)
-    echo $version | awk -F. '{print $1}'
+    do_facet $facet $LCTL get_param -n version | awk '/^lustre:/ {print $2}'
 }
 
-get_mds_version_minor () {
+lustre_version_code() {
     local facet=${1:-"$SINGLEMDS"}
-    local version=$(get_lustre_version $facet)
-    echo $version | awk -F. '{print $2}'
+    version_code $(get_lustre_version $1)
 }
 
 # If the 2.0 MDS was mounted on 1.8 device, then the OSC and LOV names
@@ -3926,10 +4677,8 @@ get_mds_version_minor () {
 # mdt osc: fsname-OSTXXXX-osc
 mds_on_old_device() {
     local mds=${1:-"$SINGLEMDS"}
-    local major=$(get_mds_version_major $mds)
-    local minor=$(get_mds_version_minor $mds)
 
-    if [ $major -ge 2 ] || [ $major -eq 1 -a $minor -gt 8 ]; then
+    if [ $(lustre_version_code $mds) -gt $(version_code 1.9.0) ]; then
         do_facet $mds "lctl list_param osc.$FSNAME-OST*-osc \
             > /dev/null 2>&1" && return 0
     fi
@@ -3944,9 +4693,8 @@ get_mdtosc_proc_path() {
     local mdt_label=$(convert_facet2label $mds_facet)
     local mdt_index=$(echo $mdt_label | sed -e 's/^.*-//')
 
-    local major=$(get_mds_version_major $mds_facet)
-    local minor=$(get_mds_version_minor $mds_facet)
-    if [ $major -le 1 -a $minor -le 8 ] || mds_on_old_device $mds_facet; then
+    if [ $(lustre_version_code $mds_facet) -le $(version_code 1.8.0) ] ||
+       mds_on_old_device $mds_facet; then
         echo "${ost_label}-osc"
     else
         echo "${ost_label}-osc-${mdt_index}"
@@ -4032,10 +4780,7 @@ wait_import_state() {
 # the value depends on configure options, and it is not stored in /proc.
 # obd_support.h:
 # #define CONNECTION_SWITCH_MIN 5U
-# #ifndef CRAY_XT3
 # #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/20)
-# #else
-# #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/2)
 
 request_timeout () {
     local facet=$1
@@ -4112,20 +4857,30 @@ wait_clients_import_state () {
 }
 
 oos_full() {
-        local -a AVAILA
-        local -a GRANTA
-        local OSCFULL=1
-        AVAILA=($(do_nodes $(comma_list $(osts_nodes)) \
-                  $LCTL get_param obdfilter.*.kbytesavail))
-        GRANTA=($(do_nodes $(comma_list $(osts_nodes)) \
-                  $LCTL get_param -n obdfilter.*.tot_granted))
-        for ((i=0; i<${#AVAILA[@]}; i++)); do
-                local -a AVAIL1=(${AVAILA[$i]//=/ })
-                GRANT=$((${GRANTA[$i]}/1024))
-                echo -n $(echo ${AVAIL1[0]} | cut -d"." -f2) avl=${AVAIL1[1]} grnt=$GRANT diff=$((AVAIL1[1] - GRANT))
-                [ $((AVAIL1[1] - GRANT)) -lt 400 ] && OSCFULL=0 && echo " FULL" || echo
-        done
-        return $OSCFULL
+	local -a AVAILA
+	local -a GRANTA
+	local -a TOTALA
+	local OSCFULL=1
+	AVAILA=($(do_nodes $(comma_list $(osts_nodes)) \
+	          $LCTL get_param obdfilter.*.kbytesavail))
+	GRANTA=($(do_nodes $(comma_list $(osts_nodes)) \
+	          $LCTL get_param -n obdfilter.*.tot_granted))
+	TOTALA=($(do_nodes $(comma_list $(osts_nodes)) \
+	          $LCTL get_param -n obdfilter.*.kbytestotal))
+	for ((i=0; i<${#AVAILA[@]}; i++)); do
+		local -a AVAIL1=(${AVAILA[$i]//=/ })
+		local -a TOTAL=(${TOTALA[$i]//=/ })
+		GRANT=$((${GRANTA[$i]}/1024))
+		# allow 1% of total space in bavail because of delayed
+		# allocation with ZFS which might release some free space after
+		# txg commit.  For small devices, we set a mininum of 8MB
+		local LIMIT=$((${TOTAL} / 100 + 8000))
+		echo -n $(echo ${AVAIL1[0]} | cut -d"." -f2) avl=${AVAIL1[1]} \
+			grnt=$GRANT diff=$((AVAIL1[1] - GRANT)) limit=${LIMIT}
+		[ $((AVAIL1[1] - GRANT)) -lt $LIMIT ] && OSCFULL=0 && \
+			echo " FULL" || echo
+	done
+	return $OSCFULL
 }
 
 pool_list () {
@@ -4214,7 +4969,7 @@ destroy_pools () {
 
     echo destroy the created pools: ${!listvar}
     for poolname in ${!listvar//,/ }; do
-        destroy_pool $fsname.$poolname 
+        destroy_pool $fsname.$poolname
     done
 }
 
@@ -4229,11 +4984,18 @@ gather_logs () {
 
     local ts=$(date +%s)
     local docp=true
+
+    if [[ ! -f "$YAML_LOG" ]]; then
+        # init_logging is not performed before gather_logs,
+        # so the $LOGDIR needs to be checked here
+        check_shared_dir $LOGDIR && touch $LOGDIR/shared
+    fi
+
     [ -f $LOGDIR/shared ] && docp=false
- 
+
     # dump lustre logs, dmesg
 
-    prefix="$LOGDIR/${TESTSUITE}.${TESTNAME}"
+    prefix="$TESTLOG_PREFIX.$TESTNAME"
     suffix="$ts.log"
     echo "Dumping lctl log to ${prefix}.*.${suffix}"
 
@@ -4249,18 +5011,7 @@ gather_logs () {
          dmesg > ${prefix}.dmesg.\\\$(hostname -s).${suffix}"
     if [ ! -f $LOGDIR/shared ]; then
         do_nodes $list rsync -az "${prefix}.*.${suffix}" $HOSTNAME:$LOGDIR
-      fi
-
-    local archive=$LOGDIR/${TESTSUITE}-$ts.tar.bz2
-    tar -jcf $archive $LOGDIR/*$ts* $LOGDIR/*${TESTSUITE}*
-
-    echo $archive
-}
-
-cleanup_logs () {
-    local list=${1:-$(comma_list $(nodes_list))}
-
-    [ -n ${TESTSUITE} ] && do_nodes $list "rm -f $TMP/*${TESTSUITE}*" || true
+    fi
 }
 
 do_ls () {
@@ -4536,7 +5287,7 @@ wait_flavor()
         echo -n "checking $dir..."
         res=$(do_check_flavor $dir $flavor)
         echo "found $res/$expect $flavor connections"
-        [ $res -eq $expect ] && return 0
+        [ $res -ge $expect ] && return 0
         sleep 4
     done
 
@@ -4605,18 +5356,23 @@ check_logdir() {
         # Not found. Create local logdir
         mkdir -p $dir
     else
-        touch $dir/node.$(hostname -s).yml
+        touch $dir/check_file.$(hostname -s)
     fi
     return 0
 }
 
 check_write_access() {
     local dir=$1
+    local node
+    local file
+
     for node in $(nodes_list); do
-        if [ ! -f "$dir/node.$(short_hostname ${node}).yml" ]; then
+        file=$dir/check_file.$(short_hostname $node)
+        if [[ ! -f "$file" ]]; then
             # Logdir not accessible/writable from this node.
             return 1
         fi
+        rm -f $file || return 1
     done
     return 0
 }
@@ -4632,16 +5388,17 @@ init_logging() {
     mkdir -p $LOGDIR
     init_clients_lists
 
-    do_rpc_nodes $(comma_list $(nodes_list)) check_logdir $LOGDIR
-    if check_write_access $LOGDIR; then
-        touch $LOGDIR/shared
-        echo "Logging to shared log directory: $LOGDIR"
-    else
-        echo "Logging to local directory: $LOGDIR"
-    fi
+    if [ ! -f $YAML_LOG ]; then       # If the yaml log already exists then we will just append to it
+      if check_shared_dir $LOGDIR; then
+          touch $LOGDIR/shared
+          echo "Logging to shared log directory: $LOGDIR"
+      else
+          echo "Logging to local directory: $LOGDIR"
+      fi
 
-    yml_nodes_file $LOGDIR >> $YAML_LOG
-    yml_results_file >> $YAML_LOG
+      yml_nodes_file $LOGDIR >> $YAML_LOG
+      yml_results_file >> $YAML_LOG
+    fi
 
     umask $SAVE_UMASK
 }
@@ -4696,32 +5453,44 @@ run_llverfs()
 }
 
 remove_mdt_files() {
-    local facet=$1
-    local mdtdev=$2
-    shift 2
-    local files="$@"
-    local mntpt=$(facet_mntpt $facet)
-
-    echo "removing files from $mdtdev on $facet: $files"
-    mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $?
-    rc=0;
-    for f in $files; do
-        rm $mntpt/ROOT/$f || { rc=$?; break; }
-    done
-    umount -f $mntpt || return $?
-    return $rc
+	local facet=$1
+	local mdtdev=$2
+	shift 2
+	local files="$@"
+	local mntpt=$(facet_mntpt $facet)
+	local opts=$MDS_MOUNT_OPTS
+
+	echo "removing files from $mdtdev on $facet: $files"
+	if [ $(facet_fstype $facet) == ldiskfs ] &&
+	   ! do_facet $facet test -b ${!dev}; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
+	mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
+		return $?
+	rc=0;
+	for f in $files; do
+		rm $mntpt/ROOT/$f || { rc=$?; break; }
+	done
+	umount -f $mntpt || return $?
+	return $rc
 }
 
 duplicate_mdt_files() {
-    local facet=$1
-    local mdtdev=$2
-    shift 2
-    local files="$@"
-    local mntpt=$(facet_mntpt $facet)
-
-    echo "duplicating files on $mdtdev on $facet: $files"
-    mkdir -p $mntpt || return $?
-    mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $?
+	local facet=$1
+	local mdtdev=$2
+	shift 2
+	local files="$@"
+	local mntpt=$(facet_mntpt $facet)
+	local opts=$MDS_MOUNT_OPTS
+
+	echo "duplicating files on $mdtdev on $facet: $files"
+	mkdir -p $mntpt || return $?
+	if [ $(facet_fstype $facet) == ldiskfs ] &&
+	   ! do_facet $facet test -b ${!dev}; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
+	mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
+		return $?
 
     do_umount() {
         trap 0
@@ -4838,3 +5607,101 @@ generate_string() {
 
     echo "$(head -c $size < /dev/zero | tr '\0' y)"
 }
+
+reformat_external_journal() {
+	if [ ! -z ${EJOURNAL} ]; then
+		local rcmd="do_facet ${SINGLEMDS}"
+
+		echo "reformat external journal on ${SINGLEMDS}:${EJOURNAL}"
+		${rcmd} mke2fs -O journal_dev ${EJOURNAL} || return 1
+	fi
+}
+
+# MDT file-level backup/restore
+mds_backup_restore() {
+	local devname=$(mdsdevname ${SINGLEMDS//mds/})
+	local mntpt=$(facet_mntpt brpt)
+	local rcmd="do_facet ${SINGLEMDS}"
+	local metaea=${TMP}/backup_restore.ea
+	local metadata=${TMP}/backup_restore.tgz
+	local opts=${MDS_MOUNT_OPTS}
+
+	if ! ${rcmd} test -b ${devname}; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
+
+	echo "file-level backup/restore on ${SINGLEMDS}:${devname}"
+
+	# step 1: build mount point
+	${rcmd} mkdir -p $mntpt
+	# step 2: cleanup old backup
+	${rcmd} rm -f $metaea $metadata
+	# step 3: mount dev
+	${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 1
+	# step 4: backup metaea
+	echo "backup EA"
+	${rcmd} "cd $mntpt && getfattr -R -d -m '.*' -P . > $metaea && cd -" ||
+		return 2
+	# step 5: backup metadata
+	echo "backup data"
+	${rcmd} tar zcf $metadata -C $mntpt/ . > /dev/null 2>&1 || return 3
+	# step 6: umount
+	${rcmd} umount -d $mntpt || return 4
+	# step 7: reformat external journal if needed
+	reformat_external_journal || return 5
+	# step 8: reformat dev
+	echo "reformat new device"
+	add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS}) --backfstype ldiskfs \
+		--reformat $devname > /dev/null || return 6
+	# step 9: mount dev
+	${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 7
+	# step 10: restore metadata
+	echo "restore data"
+	${rcmd} tar zxfp $metadata -C $mntpt > /dev/null 2>&1 || return 8
+	# step 11: restore metaea
+	echo "restore EA"
+	${rcmd} "cd $mntpt && setfattr --restore=$metaea && cd - " || return 9
+	# step 12: remove recovery logs
+	echo "remove recovery logs"
+	${rcmd} rm -fv $mntpt/OBJECTS/* $mntpt/CATALOGS
+	# step 13: umount dev
+	${rcmd} umount -d $mntpt || return 10
+	# step 14: cleanup tmp backup
+	${rcmd} rm -f $metaea $metadata
+}
+
+# remove OI files
+mds_remove_ois() {
+	local devname=$(mdsdevname ${SINGLEMDS//mds/})
+	local mntpt=$(facet_mntpt brpt)
+	local rcmd="do_facet ${SINGLEMDS}"
+	local idx=$1
+	local opts=${MDS_MOUNT_OPTS}
+
+	if ! ${rcmd} test -b ${devname}; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
+
+	echo "remove OI files: idx=${idx}"
+
+	# step 1: build mount point
+	${rcmd} mkdir -p $mntpt
+	# step 2: mount dev
+	${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 1
+	if [ -z $idx ]; then
+		# step 3: remove all OI files
+		${rcmd} rm -fv $mntpt/oi.16*
+	elif [ $idx -lt 2 ]; then
+		${rcmd} rm -fv $mntpt/oi.16.${idx}
+	else
+		local i
+
+		# others, rm oi.16.[idx, idx * idx, idx ** ...]
+		for ((i=${idx}; i<64; i=$((i * idx)))); do
+			${rcmd} rm -fv $mntpt/oi.16.${i}
+		done
+	fi
+	# step 4: umount
+	${rcmd} umount -d $mntpt || return 2
+	# OI files will be recreated when mounted as lustre next time.
+}