X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;ds=inline;f=lustre%2Ftests%2Ftest-framework.sh;h=b7216d25095334581485b72d22503749c2885c31;hb=50bbf65fec92adbcedf1706eb881e2a156874d0d;hp=ad6ad36e03eefed2ee25bdd220d479e5bd2e4c55;hpb=322968acf183ab16d952cd3026f6580957b31259;p=fs%2Flustre-release.git

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index ad6ad36..b7216d2 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -1,11 +1,11 @@
 #!/bin/bash
-# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
 
 trap 'print_summary && touch $TF_FAIL && \
     echo "test-framework exiting on error"' ERR
 set -e
 #set -x
 
+export EJOURNAL=${EJOURNAL:-""}
 export REFORMAT=${REFORMAT:-""}
 export WRITECONF=${WRITECONF:-""}
 export VERBOSE=${VERBOSE:-false}
@@ -130,13 +130,16 @@ init_test_env() {
     export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite
     export FSCK_MAX_ERR=4   # File system errors left uncorrected
 
+	export ZFS=${ZFS:-zfs}
+	export ZPOOL=${ZPOOL:-zpool}
+	export ZDB=${ZDB:-zdb}
+
     #[ -d /r ] && export ROOT=${ROOT:-/r}
     export TMP=${TMP:-$ROOT/tmp}
     export TESTSUITELOG=${TMP}/${TESTSUITE}.log
-    if [[ -z $LOGDIRSET ]]; then
-        export LOGDIR=${LOGDIR:-${TMP}/test_logs/}/$(date +%s)
-        export LOGDIRSET=true
-    fi
+    export LOGDIR=${LOGDIR:-${TMP}/test_logs/$(date +%s)}
+    export TESTLOG_PREFIX=$LOGDIR/$TESTSUITE
+
     export HOSTNAME=${HOSTNAME:-$(hostname -s)}
     if ! echo $PATH | grep -q $LUSTRE/utils; then
         export PATH=$LUSTRE/utils:$PATH
@@ -166,10 +169,14 @@ init_test_env() {
         export PATH=$LUSTRE/tests/mpi:$PATH
     fi
     export RSYNC_RSH=${RSYNC_RSH:-rsh}
+
     export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
     [ ! -f "$LCTL" ] && export LCTL=$(which lctl)
     export LFS=${LFS:-"$LUSTRE/utils/lfs"}
     [ ! -f "$LFS" ] && export LFS=$(which lfs)
+    SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"}
+    GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"}
+
     export L_GETIDENTITY=${L_GETIDENTITY:-"$LUSTRE/utils/l_getidentity"}
     if [ ! -f "$L_GETIDENTITY" ]; then
         if `which l_getidentity > /dev/null 2>&1`; then
@@ -186,8 +193,11 @@ init_test_env() {
     [ ! -f "$TUNEFS" ] && export TUNEFS=$(which tunefs.lustre)
     export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} "
     export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod}
-    [ ! -f "$LUSTRE_RMMOD" ] && export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null)
-    export FSTYPE=${FSTYPE:-"ldiskfs"}
+    [ ! -f "$LUSTRE_RMMOD" ] &&
+        export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null)
+    export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate}
+    [ ! -f "$LFS_MIGRATE" ] &&
+        export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null)
     export NAME=${NAME:-local}
     export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"}
     [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && \
@@ -256,11 +266,26 @@ init_test_env() {
     rm -f $TMP/*active
 }
 
-case `uname -r` in
-2.4.*) EXT=".o"; USE_QUOTA=no; [ ! "$CLIENTONLY" ] && FSTYPE=ext3;;
-    *) EXT=".ko"; USE_QUOTA=yes;;
-esac
+check_cpt_number() {
+	local ncpts
+
+	ncpts=$(lctl get_param -n cpu_partition_table 2>/dev/null| wc -l)
+	if [ $ncpts -eq 0 ]; then
+		echo "1"
+	else
+		echo $ncpts
+	fi
+}
+
+version_code() {
+    # split arguments like "1.8.6-wc3" into "1", "8", "6", "wc3"
+    eval set -- $(tr "[:punct:]" " " <<< $*)
 
+    echo -n $((($1 << 16) | ($2 << 8) | $3))
+}
+
+export LINUX_VERSION=$(uname -r | sed -e "s/[-.]/ /3" -e "s/ .*//")
+export LINUX_VERSION_CODE=$(version_code ${LINUX_VERSION//\./ })
 
 module_loaded () {
    /sbin/lsmod | grep -q "^\<$1\>"
@@ -343,14 +368,61 @@ llite_lloop_enabled() {
 }
 
 load_modules_local() {
-    if [ -n "$MODPROBE" ]; then
-        # use modprobe
-        echo "Using modprobe to load modules"
-        return 0
-    fi
+	if [ "$USE_OFD" == yes ]; then
+		if module_loaded obdfilter; then
+			if ! $LUSTRE_RMMOD ldiskfs; then
+				echo "$HOSTNAME may still be using obdfilter.ko"
+				return 1
+			fi
+		fi
+	else
+		if module_loaded ofd; then
+			if ! $LUSTRE_RMMOD ldiskfs; then
+				echo "$HOSTNAME may still be using ofd.ko"
+				return 1
+			fi
+		fi
+	fi
+
+	if [ -n "$MODPROBE" ]; then
+		# use modprobe
+		echo "Using modprobe to load modules"
+		return 0
+	fi
+
+	echo Loading modules from $LUSTRE
+
+	local ncpus
+
+	if [ -f /sys/devices/system/cpu/online ]; then
+		ncpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/online) + 1))
+		echo "detected $ncpus online CPUs by sysfs"
+	else
+		ncpus=$(getconf _NPROCESSORS_CONF 2>/dev/null)
+		local rc=$?
+		if [ $rc -eq 0 ]; then
+			echo "detected $ncpus online CPUs by getconf"
+		else
+			echo "Can't detect number of CPUs"
+			ncpus=1
+		fi
+	fi
+
+	local ncpts=0
+	# if there is only one CPU core, libcfs can only create one partition
+	# if there is more than 4 CPU cores, libcfs should create multiple CPU
+	# partitions. So we just force libcfs to create 2 partitions for
+	# system with 2 or 4 cores
+	if [ $ncpus -le 4 ] && [ $ncpus -gt 1 ]; then
+		# force to enable multiple CPU partitions
+		echo "Force libcfs to create 2 CPU partitions"
+		ncpts=2
+	else
+		echo "libcfs will create CPU partition based on online CPUs"
+	fi
+
+	load_module ../libcfs/libcfs/libcfs cpu_npartitions=$ncpts
 
-    echo Loading modules from $LUSTRE
-    load_module ../libcfs/libcfs/libcfs
     [ "$PTLDEBUG" ] && lctl set_param debug="$PTLDEBUG"
     [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug="${SUBSYSTEM# }"
     load_module ../lnet/lnet/lnet
@@ -360,7 +432,6 @@ load_modules_local() {
     load_module obdclass/obdclass
     load_module ptlrpc/ptlrpc
     load_module ptlrpc/gss/ptlrpc_gss
-    [ "$USE_QUOTA" = "yes" -a "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS
     load_module fld/fld
     load_module fid/fid
     load_module lmv/lmv
@@ -368,25 +439,38 @@ load_modules_local() {
     load_module osc/osc
     load_module lov/lov
     load_module mgc/mgc
+    load_module obdecho/obdecho
     if ! client_only; then
         SYMLIST=/proc/kallsyms
         grep -q crc16 $SYMLIST || { modprobe crc16 2>/dev/null || true; }
         grep -q -w jbd $SYMLIST || { modprobe jbd 2>/dev/null || true; }
         grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; }
-        if [ "$FSTYPE" = "ldiskfs" ]; then
-            grep -q exportfs_decode_fh $SYMLIST ||
-                { modprobe exportfs 2> /dev/null || true; }
-            load_module ../ldiskfs/ldiskfs/ldiskfs
-        fi
-        load_module mgs/mgs
-        load_module mds/mds
-        load_module mdd/mdd
-        load_module mdt/mdt
-        load_module lvfs/fsfilt_$FSTYPE
-        load_module cmm/cmm
-        load_module osd-ldiskfs/osd_ldiskfs
-        load_module ost/ost
-        load_module obdfilter/obdfilter
+		[ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS
+		if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then
+			load_module osd-zfs/osd_zfs
+		fi
+		load_module mgs/mgs
+		load_module mds/mds
+		load_module mdd/mdd
+		if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then
+			#
+			# This block shall be moved up beside osd-zfs as soon
+			# as osd-ldiskfs stops using mdd symbols.
+			#
+			grep -q exportfs_decode_fh $SYMLIST ||
+				{ modprobe exportfs 2> /dev/null || true; }
+			load_module ../ldiskfs/ldiskfs/ldiskfs
+			load_module lvfs/fsfilt_ldiskfs
+			load_module osd-ldiskfs/osd_ldiskfs
+		fi
+		load_module mdt/mdt
+		load_module cmm/cmm
+		load_module ost/ost
+		if [ "$USE_OFD" == yes ]; then
+			load_module ofd/ofd
+		else
+			load_module obdfilter/obdfilter
+		fi
     fi
 
 
@@ -406,15 +490,15 @@ load_modules_local() {
 }
 
 load_modules () {
-    load_modules_local
-    # bug 19124
-    # load modules on remote nodes optionally
-    # lustre-tests have to be installed on these nodes
-    if $LOAD_MODULES_REMOTE ; then
-        local list=$(comma_list $(remote_nodes_list))
-        echo loading modules on $list
-        do_rpc_nodes $list load_modules 
-    fi
+	load_modules_local
+	# bug 19124
+	# load modules on remote nodes optionally
+	# lustre-tests have to be installed on these nodes
+	if $LOAD_MODULES_REMOTE ; then
+		local list=$(comma_list $(remote_nodes_list))
+		echo loading modules on $list
+		do_rpc_nodes $list load_modules_local
+	fi
 }
 
 check_mem_leak () {
@@ -431,18 +515,18 @@ check_mem_leak () {
 }
 
 unload_modules() {
-    wait_exit_ST client # bug 12845
+	wait_exit_ST client # bug 12845
 
-    $LUSTRE_RMMOD $FSTYPE || return 2
+	$LUSTRE_RMMOD ldiskfs || return 2
 
-    if $LOAD_MODULES_REMOTE ; then
-        local list=$(comma_list $(remote_nodes_list))
-        if [ ! -z $list ]; then
-            echo unloading modules on $list
-            do_rpc_nodes $list $LUSTRE_RMMOD $FSTYPE
-            do_rpc_nodes $list check_mem_leak
-        fi
-    fi
+	if $LOAD_MODULES_REMOTE ; then
+		local list=$(comma_list $(remote_nodes_list))
+		if [ ! -z $list ]; then
+			echo unloading modules on $list
+			do_rpc_nodes $list $LUSTRE_RMMOD ldiskfs
+			do_rpc_nodes $list check_mem_leak
+		fi
+	fi
 
     if grep -qe "/sbin/mount\.lustre" /proc/mounts; then
         umount /sbin/mount.lustre || true
@@ -568,23 +652,135 @@ cleanup_gss() {
     fi
 }
 
+facet_type() {
+	local facet=$1
+
+	echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9]\+//' |
+		tr '[:lower:]' '[:upper:]'
+}
+
+facet_number() {
+	local facet=$1
+
+	if [ $facet == mgs ]; then
+		return 1
+	fi
+
+	echo -n $facet | sed -e 's/^fs[0-9]\+//' | sed -e 's/^[a-z]\+//'
+}
+
+facet_fstype() {
+	local facet=$1
+	local var
+
+	var=${facet}_FSTYPE
+	if [ -n "${!var}" ]; then
+		echo -n ${!var}
+		return
+	fi
+
+	var=$(facet_type $facet)FSTYPE
+	if [ -n "${!var}" ]; then
+		echo -n ${!var}
+		return
+	fi
+
+	if [ -n "$FSTYPE" ]; then
+		echo -n $FSTYPE
+		return
+	fi
+
+	return 1
+}
+
+node_fstypes() {
+	local node=$1
+	local fstypes
+	local fstype
+	local facets=$(get_facets)
+	local facet
+
+	for facet in ${facets//,/ }; do
+		if [ $node == $(facet_host $facet) ] ||
+		   [ $node == "$(facet_failover_host $facet)" ]; then
+			fstype=$(facet_fstype $facet)
+			if [[ $fstypes != *$fstype* ]]; then
+				fstypes+="${fstypes:+,}$fstype"
+			fi
+		fi
+	done
+	echo -n $fstypes
+}
+
+devicelabel() {
+	local facet=$1
+	local dev=$2
+	local label
+	local fstype=$(facet_fstype $facet)
+
+	case $fstype in
+	ldiskfs)
+		label=$(do_facet ${facet} "$E2LABEL ${dev} 2>/dev/null");;
+	zfs)
+		label=$(do_facet ${facet} "$ZFS get -H -o value lustre:svname \
+		                           ${dev} 2>/dev/null");;
+	*)
+		error "unknown fstype!";;
+	esac
+
+	echo -n $label
+}
+
 mdsdevlabel() {
-    local num=$1
-    local device=`mdsdevname $num`
-    local label=`do_facet mds$num "e2label ${device}" | grep -v "CMD: "`
-    echo -n $label
+	local num=$1
+	local device=$(mdsdevname $num)
+	local label=$(devicelabel mds$num ${device} | grep -v "CMD: ")
+	echo -n $label
 }
 
 ostdevlabel() {
-    local num=$1
-    local device=`ostdevname $num`
-    local label=`do_facet ost$num "e2label ${device}" | grep -v "CMD: "`
-    echo -n $label
+	local num=$1
+	local device=$(ostdevname $num)
+	local label=$(devicelabel ost$num ${device} | grep -v "CMD: ")
+	echo -n $label
+}
+
+#
+# This and set_obdfilter_param() shall be used to access OSD parameters
+# once existed under "obdfilter":
+#
+#   mntdev
+#   stats
+#   read_cache_enable
+#   writethrough_cache_enable
+#
+get_obdfilter_param() {
+	local nodes=$1
+	local device=${2:-$FSNAME-OST*}
+	local name=$3
+
+	do_nodes $nodes "$LCTL get_param -n obdfilter.$device.$name \
+		osd-*.$device.$name 2>&1" | grep -v 'Found no match'
+}
+
+set_obdfilter_param() {
+	local nodes=$1
+	local device=${2:-$FSNAME-OST*}
+	local name=$3
+	local value=$4
+
+	do_nodes $nodes "$LCTL set_param -n obdfilter.$device.$name=$value \
+		osd-*.$device.$name=$value 2>&1" | grep -v 'Found no match'
 }
 
 set_debug_size () {
     local dz=${1:-$DEBUG_SIZE}
-    local cpus=$(getconf _NPROCESSORS_CONF)
+
+    if [ -f /sys/devices/system/cpu/possible ]; then
+        local cpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/possible)+1))
+    else
+        local cpus=$(getconf _NPROCESSORS_CONF)
+    fi
 
     # bug 19944, adjust size to be -gt num_possible_cpus()
     # promise 2MB for every cpu at least
@@ -596,14 +792,13 @@ set_debug_size () {
 
 set_default_debug () {
     local debug=${1:-"$PTLDEBUG"}
-    local subsystem_debug=${2:-"$SUBSYSTEM"}
+    local subsys=${2:-"$SUBSYSTEM"}
     local debug_size=${3:-$DEBUG_SIZE}
 
-    lctl set_param debug="$debug"
-    lctl set_param subsystem_debug="${subsystem_debug# }"
+    [ -n "$debug" ] && lctl set_param debug="$debug" >/dev/null
+    [ -n "$subsys" ] && lctl set_param subsystem_debug="${subsys# }" >/dev/null
 
-    set_debug_size $debug_size
-    sync
+    [ -n "$debug_size" ] && set_debug_size $debug_size > /dev/null
 }
 
 set_default_debug_nodes () {
@@ -628,31 +823,71 @@ set_default_debug_facet () {
 
 # Facet functions
 mount_facets () {
-    local facets=${1:-$(get_facets)}
-    local facet
+	local facets=${1:-$(get_facets)}
+	local facet
 
-    for facet in ${facets//,/ }; do
-        mount_facet $facet || error "Restart of $facet failed!"
-    done
+	for facet in ${facets//,/ }; do
+		mount_facet $facet
+		local RC=$?
+		[ $RC -eq 0 ] && continue
+
+		if [ "$TESTSUITE.$TESTNAME" = "replay-dual.test_0a" ]; then
+			skip "Restart of $facet failed!." && touch $LU482_FAILED
+		else
+			error "Restart of $facet failed!"
+		fi
+		return $RC
+	done
 }
 
-mount_facet() {
-    local facet=$1
-    shift
-    local dev=$(facet_active $facet)_dev
-    local opt=${facet}_opt
-    local mntpt=$(facet_mntpt $facet)
+#
+# Add argument "arg" (e.g., "loop") to the comma-separated list
+# of arguments for option "opt" (e.g., "-o") on command
+# line "opts" (e.g., "-o flock").
+#
+csa_add() {
+	local opts=$1
+	local opt=$2
+	local arg=$3
+	local opt_pattern="\([[:space:]]\+\|^\)$opt"
 
-    echo "Starting ${facet}: ${!opt} $@ ${!dev} $mntpt"
-    do_facet ${facet} "mkdir -p $mntpt; mount -t lustre ${!opt} $@ ${!dev} $mntpt"
-    RC=${PIPESTATUS[0]}
-    if [ $RC -ne 0 ]; then
-        echo "mount -t lustre $@ ${!dev} $mntpt"
-        echo "Start of ${!dev} on ${facet} failed ${RC}"
+	if echo "$opts" | grep -q $opt_pattern; then
+		opts=$(echo "$opts" | sed -e \
+			"s/$opt_pattern[[:space:]]*[^[:space:]]\+/&,$arg/")
+	else
+		opts+="${opts:+ }$opt $arg"
+	fi
+	echo -n "$opts"
+}
+
+mount_facet() {
+	local facet=$1
+	shift
+	local dev=$(facet_active $facet)_dev
+	local opt=${facet}_opt
+	local mntpt=$(facet_mntpt $facet)
+	local opts="${!opt} $@"
+
+	if [ $(facet_fstype $facet) == ldiskfs ] &&
+	   ! do_facet $facet test -b ${!dev}; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
+
+	echo "Starting ${facet}: $opts ${!dev} $mntpt"
+	# for testing LU-482 error handling in mount_facets() and test_0a()
+	if [ -f $TMP/test-lu482-trigger ]; then
+		RC=2
+	else
+		do_facet ${facet} "mkdir -p $mntpt; mount -t lustre $opts \
+		                   ${!dev} $mntpt"
+		RC=${PIPESTATUS[0]}
+	fi
+	if [ $RC -ne 0 ]; then
+		echo "Start of ${!dev} on ${facet} failed ${RC}"
     else
         set_default_debug_facet $facet
 
-        label=$(do_facet ${facet} "$E2LABEL ${!dev}")
+		label=$(devicelabel ${facet} ${!dev})
         [ -z "$label" ] && echo no label for ${!dev} && exit 1
         eval export ${facet}_svc=${label}
         echo Started ${label}
@@ -684,6 +919,39 @@ start() {
     return $RC
 }
 
+#
+# When a ZFS OSD is made read-only by replay_barrier(), its pool is "freezed".
+# Because stopping corresponding target may not clear this in-memory state, we
+# need to zap the pool from memory by exporting and reimporting the pool.
+#
+# Although the uberblocks are not updated when a pool is freezed, transactions
+# are still written to the disks.  Modified blocks may be cached in memory when
+# tests try reading them back.  The export-and-reimport process also evicts any
+# cached pool data from memory to provide the correct "data loss" semantics.
+#
+refresh_disk() {
+	local facet=$1
+	local fstype=$(facet_fstype $facet)
+	local _dev
+	local dev
+	local poolname
+
+	if [ "${fstype}" == "zfs" ]; then
+		_dev=$(facet_active $facet)_dev
+		dev=${!_dev} # expand _dev to its value, e.g. ${mds1_dev}
+		poolname="${dev%%/*}" # poolname is string before "/"
+
+		if [ "${poolname}" == "" ]; then
+			echo "invalid dataset name: $dev"
+			return
+		fi
+		do_facet $facet "cp /etc/zfs/zpool.cache /tmp/zpool.cache.back"
+		do_facet $facet "$ZPOOL export ${poolname}"
+		do_facet $facet "$ZPOOL import -f -c /tmp/zpool.cache.back \
+		                 ${poolname}"
+	fi
+}
+
 stop() {
     local running
     local facet=$1
@@ -746,6 +1014,11 @@ restore_quota_type () {
 setup_quota(){
     local mntpt=$1
 
+	if [ "$USE_OFD" = "yes" ]; then
+		$LFS quotacheck $mntpt || error "quotacheck failed"
+		return
+	fi
+
     # We need save the original quota_type params, and restore them after testing
 
     # Suppose that quota type the same on mds and ost
@@ -848,7 +1121,7 @@ sanity_mount_check_nodes () {
     local rc=0
     for mnt in $mnts ; do
         do_nodes $nodes "running=\\\$(grep -c $mnt' ' /proc/mounts);
-mpts=\\\$(mount | grep -w -c $mnt);
+mpts=\\\$(mount | grep -c $mnt' ');
 if [ \\\$running -ne \\\$mpts ]; then
     echo \\\$(hostname) env are INSANE!;
     exit 1;
@@ -918,7 +1191,7 @@ fi;
 exit \\\$rc" || return ${PIPESTATUS[0]}
 
     echo "Started clients $clients: "
-    do_nodes $clients "mount | grep -w $mnt"
+    do_nodes $clients "mount | grep $mnt' '"
 
     set_default_debug_nodes $clients
 
@@ -936,18 +1209,16 @@ zconf_umount_clients() {
     do_nodes $clients "running=\\\$(grep -c $mnt' ' /proc/mounts);
 if [ \\\$running -ne 0 ] ; then
 echo Stopping client \\\$(hostname) $mnt opts:$force;
-lsof -t $mnt || need_kill=no;
+lsof $mnt || need_kill=no;
 if [ "x$force" != "x" -a "x\\\$need_kill" != "xno" ]; then
     pids=\\\$(lsof -t $mnt | sort -u);
     if [ -n \\\"\\\$pids\\\" ]; then
              kill -9 \\\$pids;
     fi
 fi;
-busy=\\\$(umount $force $mnt 2>&1 | grep -c "busy");
-if [ \\\$busy -ne 0 ] ; then
+while umount $force $mnt 2>&1 | grep -q "busy"; do
     echo "$mnt is still busy, wait one second" && sleep 1;
-    umount $force $mnt;
-fi
+done;
 fi"
 }
 
@@ -1045,12 +1316,13 @@ remount_facet() {
 }
 
 reboot_facet() {
-    local facet=$1
-    if [ "$FAILURE_MODE" = HARD ]; then
-        reboot_node $(facet_active_host $facet)
-    else
-        sleep 10
-    fi
+	local facet=$1
+	if [ "$FAILURE_MODE" = HARD ]; then
+		reboot_node $(facet_active_host $facet)
+	else
+		refresh_disk ${facet}
+		sleep 10
+	fi
 }
 
 boot_node() {
@@ -1104,14 +1376,20 @@ start_client_load() {
     eval export ${var}=$load
 
     do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \
-                              BREAK_ON_ERROR=$BREAK_ON_ERROR \
-                              END_RUN_FILE=$END_RUN_FILE \
-                              LOAD_PID_FILE=$LOAD_PID_FILE \
-                              TESTSUITELOG=$TESTSUITELOG \
-                              run_${load}.sh" &
-    CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $!"
+BREAK_ON_ERROR=$BREAK_ON_ERROR \
+END_RUN_FILE=$END_RUN_FILE \
+LOAD_PID_FILE=$LOAD_PID_FILE \
+TESTLOG_PREFIX=$TESTLOG_PREFIX \
+TESTNAME=$TESTNAME \
+DBENCH_LIB=$DBENCH_LIB \
+DBENCH_SRC=$DBENCH_SRC \
+run_${load}.sh" &
+    local ppid=$!
     log "Started client load: ${load} on $client"
 
+    # get the children process IDs
+    local pids=$(ps --ppid $ppid -o pid= | xargs)
+    CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $ppid $pids"
     return 0
 }
 
@@ -1128,14 +1406,14 @@ start_client_loads () {
     sleep 2
 }
 
-# only for remote client 
+# only for remote client
 check_client_load () {
     local client=$1
     local var=$(node_var_name $client)_load
     local TESTLOAD=run_${!var}.sh
 
     ps auxww | grep -v grep | grep $client | grep -q "$TESTLOAD" || return 1
-    
+
     # bug 18914: try to connect several times not only when
     # check ps, but  while check_catastrophe also
     local tries=3
@@ -1213,7 +1491,7 @@ restart_client_loads () {
             if [ "$rc" != 0 ]; then
                 log "Client load failed to restart on node $client, rc=$rc"
                 # failure one client load means test fail
-                # we do not need to check other 
+                # we do not need to check other
                 return $rc
             fi
         else
@@ -1221,6 +1499,70 @@ restart_client_loads () {
         fi
     done
 }
+
+# Start vmstat and save its process ID in a file.
+start_vmstat() {
+    local nodes=$1
+    local pid_file=$2
+
+    [ -z "$nodes" -o -z "$pid_file" ] && return 0
+
+    do_nodes $nodes \
+        "vmstat 1 > $TESTLOG_PREFIX.$TESTNAME.vmstat.\\\$(hostname -s).log \
+        2>/dev/null </dev/null & echo \\\$! > $pid_file"
+}
+
+# Display the nodes on which client loads failed.
+print_end_run_file() {
+    local file=$1
+    local node
+
+    [ -s $file ] || return 0
+
+    echo "Found the END_RUN_FILE file: $file"
+    cat $file
+
+    # A client load will stop if it finds the END_RUN_FILE file.
+    # That does not mean the client load actually failed though.
+    # The first node in END_RUN_FILE is the one we are interested in.
+    read node < $file
+
+    if [ -n "$node" ]; then
+        local var=$(node_var_name $node)_load
+
+        local prefix=$TESTLOG_PREFIX
+        [ -n "$TESTNAME" ] && prefix=$prefix.$TESTNAME
+        local stdout_log=$prefix.run_${!var}_stdout.$node.log
+        local debug_log=$(echo $stdout_log | sed 's/\(.*\)stdout/\1debug/')
+
+        echo "Client load ${!var} failed on node $node:"
+        echo "$stdout_log"
+        echo "$debug_log"
+    fi
+}
+
+# Stop the process which had its PID saved in a file.
+stop_process() {
+    local nodes=$1
+    local pid_file=$2
+
+    [ -z "$nodes" -o -z "$pid_file" ] && return 0
+
+    do_nodes $nodes "test -f $pid_file &&
+        { kill -s TERM \\\$(cat $pid_file); rm -f $pid_file; }" || true
+}
+
+# Stop all client loads.
+stop_client_loads() {
+    local nodes=${1:-$CLIENTS}
+    local pid_file=$2
+
+    # stop the client loads
+    stop_process $nodes $pid_file
+
+    # clean up the processes that started them
+    [ -n "$CLIENT_LOAD_PIDS" ] && kill -9 $CLIENT_LOAD_PIDS 2>/dev/null || true
+}
 # End recovery-scale functions
 
 # verify that lustre actually cleaned up properly
@@ -1255,7 +1597,8 @@ wait_update () {
 
         local RESULT
         local WAIT=0
-        local sleep=5
+        local sleep=1
+        local print=10
         while [ true ]; do
             RESULT=$(do_node $node "$TEST")
             if [ "$RESULT" == "$FINAL" ]; then
@@ -1264,7 +1607,8 @@ wait_update () {
                 return 0
             fi
             [ $WAIT -ge $MAX ] && break
-            echo "Waiting $((MAX - WAIT)) secs for update"
+            [ $((WAIT % print)) -eq 0 ] &&
+                echo "Waiting $((MAX - WAIT)) secs for update"
             WAIT=$((WAIT + sleep))
             sleep $sleep
         done
@@ -1277,6 +1621,11 @@ wait_update_facet () {
     wait_update  $(facet_active_host $facet) "$@"
 }
 
+sync_all_data() {
+	do_node $(osts_nodes) "lctl set_param -n osd*.*OS*.force_sync 1" 2>&1 |
+		grep -v 'Found no match'
+}
+
 wait_delete_completed () {
     local TOTALPREV=`lctl get_param -n osc.*.kbytesavail | \
                      awk 'BEGIN{total=0}; {total+=$1}; END{print total}'`
@@ -1521,14 +1870,6 @@ facet_failover() {
 
     echo "Failing $facet on node $host"
 
-    # Make sure the client data is synced to disk. LU-924
-    #
-    # We don't write client data synchrnously (to avoid flooding sync writes
-    # when there are many clients connecting), so if the server reboots before
-    # the client data reachs disk, the client data will be lost and the client
-    # will be evicted after recovery, which is not what we expected.
-    do_facet $facet "sync; sync; sync"
-
     local affected=$(affected_facets $facet)
 
     shutdown_facet $facet
@@ -1613,13 +1954,14 @@ fail_nodf() {
 }
 
 fail_abort() {
-    local facet=$1
-    stop $facet
-    change_active $facet
-    wait_for_facet $facet
-    mount_facet $facet -o abort_recovery
-    clients_up || echo "first df failed: $?"
-    clients_up || error "post-failover df: $?"
+	local facet=$1
+	stop $facet
+	refresh_disk ${facet}
+	change_active $facet
+	wait_for_facet $facet
+	mount_facet $facet -o abort_recovery
+	clients_up || echo "first df failed: $?"
+	clients_up || error "post-failover df: $?"
 }
 
 do_lmc() {
@@ -1781,6 +2123,25 @@ facet_host() {
     echo -n ${!varname}
 }
 
+facet_failover_host() {
+	local facet=$1
+	local var
+
+	var=${facet}failover_HOST
+	if [ -n "${!var}" ]; then
+		echo ${!var}
+		return
+	fi
+
+	if [[ $facet == ost* ]]; then
+		var=ostfailover_HOST
+		if [ -n "${!var}" ]; then
+			echo ${!var}
+			return
+		fi
+	fi
+}
+
 facet_active() {
     local facet=$1
     local activevar=${facet}active
@@ -1890,13 +2251,34 @@ single_local_node () {
 
 # Outputs environment variable assignments that should be passed to remote nodes
 get_env_vars() {
-    local var
-    local value
+	local var
+	local value
+	local facets=$(get_facets)
+	local facet
 
-    for var in ${!MODOPTS_*}; do
-        value=${!var}
-        echo "${var}=\"$value\""
-    done
+	for var in ${!MODOPTS_*}; do
+		value=${!var}
+		echo -n " ${var}=\"$value\""
+	done
+
+	echo -n " USE_OFD=$USE_OFD"
+
+	for facet in ${facets//,/ }; do
+		var=${facet}_FSTYPE
+		if [ -n "${!var}" ]; then
+			echo -n " $var=${!var}"
+		fi
+	done
+
+	for var in MGSFSTYPE MDSFSTYPE OSTFSTYPE; do
+		if [ -n "${!var}" ]; then
+			echo -n " $var=${!var}"
+		fi
+	done
+
+	if [ -n "$FSTYPE" ]; then
+		echo -n " FSTYPE=$FSTYPE"
+	fi
 }
 
 do_nodes() {
@@ -1925,6 +2307,7 @@ do_nodes() {
     [ -z "$myPDSH" -o "$myPDSH" = "no_dsh" -o "$myPDSH" = "rsh" ] && \
         echo "cannot run remote command on $rnodes with $myPDSH" && return 128
 
+    export FANOUT=$(get_node_count "${rnodes//,/ }")
     if $VERBOSE; then
         echo "CMD: $rnodes $@" >&2
         $myPDSH $rnodes "$LCTL mark \"$@\"" > /dev/null 2>&1 || :
@@ -1948,6 +2331,25 @@ do_facet() {
     do_node $HOST "$@"
 }
 
+# Function: do_facet_random_file $FACET $FILE $SIZE
+# Creates FILE with random content on the given FACET of given SIZE
+
+do_facet_random_file() {
+	local facet="$1"
+	local fpath="$2"
+	local fsize="$3"
+	local cmd="dd if=/dev/urandom of='$fpath' bs=$fsize count=1"
+	do_facet $facet "$cmd 2>/dev/null"
+}
+
+do_facet_create_file() {
+	local facet="$1"
+	local fpath="$2"
+	local fsize="$3"
+	local cmd="dd if=/dev/zero of='$fpath' bs=$fsize count=1"
+	do_facet $facet "$cmd 2>/dev/null"
+}
+
 do_nodesv() {
     do_nodes --verbose "$@"
 }
@@ -1965,19 +2367,121 @@ add() {
 ostdevname() {
     num=$1
     DEVNAME=OSTDEV$num
-    #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
-    eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}}
+
+	local fstype=$(facet_fstype ost$num)
+
+	case $fstype in
+		ldiskfs )
+			#if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
+			eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};;
+		zfs )
+			#dataset name is independent of vdev device names
+			eval DEVPTR=${FSNAME}-ost${num}/ost${num};;
+		* )
+			error "unknown fstype!";;
+	esac
+
     echo -n $DEVPTR
 }
 
+ostvdevname() {
+	num=$1
+	DEVNAME=OSTDEV$num
+
+	local fstype=$(facet_fstype ost$num)
+
+	case $fstype in
+		ldiskfs )
+			# vdevs are not supported by ldiskfs
+			eval VDEVPTR="";;
+		zfs )
+			#if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
+			eval VDEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};;
+		* )
+			error "unknown fstype!";;
+	esac
+
+	echo -n $VDEVPTR
+}
+
 mdsdevname() {
     num=$1
     DEVNAME=MDSDEV$num
-    #if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
-    eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}}
+
+	local fstype=$(facet_fstype mds$num)
+
+	case $fstype in
+		ldiskfs )
+			#if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
+			eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};;
+		zfs )
+			#dataset name is independent of vdev device names
+			eval DEVPTR=${FSNAME}-mdt${num}/mdt${num};;
+		* )
+			error "unknown fstype!";;
+	esac
+
+	echo -n $DEVPTR
+}
+
+mdsvdevname() {
+	num=$1
+	DEVNAME=MDSDEV$num
+
+	local fstype=$(facet_fstype mds$num)
+
+	case $fstype in
+		ldiskfs )
+			# vdevs are not supported by ldiskfs
+			eval VDEVPTR="";;
+		zfs )
+			#if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
+			eval VDEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};;
+		* )
+			error "unknown fstype!";;
+	esac
+
+	echo -n $VDEVPTR
+}
+
+mgsdevname() {
+	DEVNAME=MGSDEV
+
+	local fstype=$(facet_fstype mds$num)
+
+	case $fstype in
+		ldiskfs )
+			#if $MGSDEV isn't defined, default is $MDSDEV1
+			eval DEVPTR=${!DEVNAME:=${MDSDEV1}};;
+		zfs )
+			#dataset name is independent of vdev device names
+			eval DEVPTR=${FSNAME}-mgs/mgs;;
+		* )
+			error "unknown fstype!";;
+	esac
+
     echo -n $DEVPTR
 }
 
+mgsvdevname() {
+	DEVNAME=MGSDEV
+
+	local fstype=$(facet_fstype mds$num)
+
+	case $fstype in
+		ldiskfs )
+			# vdevs are not supported by ldiskfs
+			eval VDEVPTR="";;
+		zfs )
+			#if $MGSDEV isn't defined, default is $MGSDEV1
+			eval VDEVPTR=${!DEVNAME:=${MDSDEV1}};;
+		* )
+			error "unknown fstype!";;
+	esac
+
+	echo -n $VDEVPTR
+}
+
 facet_mntpt () {
     local facet=$1
     [[ $facet = mgs ]] && combined_mgs_mds && facet="mds1"
@@ -2050,88 +2554,123 @@ combined_mgs_mds () {
     [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]]
 }
 
-mkfs_opts () {
-    local facet=$1
+lower() {
+	echo -n "$1" | tr '[:upper:]' '[:lower:]'
+}
 
-    local tgt=$(echo $facet | tr -d [:digit:] | tr "[:lower:]" "[:upper:]")
-    local optvar=${tgt}_MKFS_OPTS
-    local opt=${!optvar}
+upper() {
+	echo -n "$1" | tr '[:lower:]' '[:upper:]'
+}
 
-    # FIXME: ! combo  mgs/mds + mgsfailover is not supported yet
-    [[ $facet = mgs ]] && echo $opt && return
+mkfs_opts() {
+	local facet=$1
+	local type=$(facet_type $facet)
+	local index=$(($(facet_number $facet) - 1))
+	local fstype=$(facet_fstype $facet)
+	local opts
+	local fs_mkfs_opts
+	local var
 
-    # 1.
-    # --failnode options 
-    local var=${facet}failover_HOST
-    if [ x"${!var}" != x ] && [ x"${!var}" != x$(facet_host $facet) ] ; then
-        local failnode=$(h2$NETTYPE ${!var})
-        failnode="--failnode=$failnode"
-        # options does not contain
-        # or contains wrong --failnode=
-        if [[ $opt != *${failnode}* ]]; then
-            opt=$(echo $opt | sed 's/--failnode=.* / /')
-            opt="$opt $failnode"
-        fi
-    fi
+	if [ $type == MGS ] && combined_mgs_mds; then
+		return 1
+	fi
 
-    # 2.
-    # --mgsnode options
-    # no additional mkfs mds "--mgsnode" option for this configuration
-    if [[ $facet = mds ]] && combined_mgs_mds; then
-        echo $opt
-        return
-    fi
+	if [ $type == MGS ] || ( [ $type == MDS ] && combined_mgs_mds ); then
+		opts="--mgs"
+	else
+		opts="--mgsnode=$MGSNID"
+	fi
 
-    # additional mkfs "--mgsnode"
-    local mgsnode="--mgsnode=$MGSNID"
-    opt=${opt//$mgsnode }
-    for nid in ${MGSNID//:/ }; do
-        local mgsnode="--mgsnode=$nid"
-        # options does not contain
-        # --mgsnode=$nid
-        if [[ $opt != *${mgsnode}" "* ]]; then
-            opt="$opt --mgsnode=$nid"
-        fi
-    done
+	if [ $type != MGS ]; then
+		opts+=" --fsname=$FSNAME --$(lower ${type/MDS/MDT}) --index=$index"
+	fi
 
-    echo $opt
-}
+	var=${facet}failover_HOST
+	if [ -n "${!var}" ] && [ ${!var} != $(facet_host $facet) ]; then
+		opts+=" --failnode=$(h2$NETTYPE ${!var})"
+	fi
 
-formatall() {
-    if [ "$IAMDIR" == "yes" ]; then
-        MDS_MKFS_OPTS="$MDS_MKFS_OPTS --iam-dir"
-    fi
+	opts+=${TIMEOUT:+" --param=sys.timeout=$TIMEOUT"}
+	opts+=${LDLM_TIMEOUT:+" --param=sys.ldlm_timeout=$LDLM_TIMEOUT"}
 
-    [ "$FSTYPE" ] && FSTYPE_OPT="--backfstype $FSTYPE"
+	if [ $type == MDS ]; then
+		opts+=${SECLEVEL:+" --param=mdt.sec_level"}
+		opts+=${MDSCAPA:+" --param-mdt.capa=$MDSCAPA"}
+		opts+=${STRIPE_BYTES:+" --param=lov.stripesize=$STRIPE_BYTES"}
+		opts+=${STRIPES_PER_OBJ:+" --param=lov.stripecount=$STRIPES_PER_OBJ"}
+		opts+=${L_GETIDENTITY:+" --param=mdt.identity_upcall=$L_GETIDENTITY"}
 
-    stopall
-    # We need ldiskfs here, may as well load them all
-    load_modules
-    [ "$CLIENTONLY" ] && return
-    echo Formatting mgs, mds, osts
-    if ! combined_mgs_mds ; then
-        add mgs $(mkfs_opts mgs) $FSTYPE_OPT --reformat $MGSDEV || exit 10
-    fi
+		if [ $fstype == ldiskfs ]; then
+			opts+=${IAMDIR:+" --iam-dir"}
 
-    for num in `seq $MDSCOUNT`; do
-        echo "Format mds$num: $(mdsdevname $num)"
-        if $VERBOSE; then
-            add mds$num $(mkfs_opts mds) $FSTYPE_OPT --reformat $(mdsdevname $num) || exit 10
-        else
-            add mds$num $(mkfs_opts mds) $FSTYPE_OPT --reformat $(mdsdevname $num) > /dev/null || exit 10
-        fi
-    done
+			fs_mkfs_opts+=${MDSJOURNALSIZE:+" -J size=$MDSJOURNALSIZE"}
+			fs_mkfs_opts+=${MDSISIZE:+" -i $MDSISIZE"}
+		fi
+	fi
 
-    # the ost-s could have different OST_MKFS_OPTS
-    # because of different failnode-s
-    for num in `seq $OSTCOUNT`; do
-        echo "Format ost$num: $(ostdevname $num)"
-        if $VERBOSE; then
-            add ost$num $(mkfs_opts ost${num}) $FSTYPE_OPT --reformat `ostdevname $num` || exit 10
-        else
-            add ost$num $(mkfs_opts ost${num}) $FSTYPE_OPT --reformat `ostdevname $num` > /dev/null || exit 10
-        fi
-    done
+	if [ $type == OST ]; then
+		opts+=${SECLEVEL:+" --param=ost.sec_level"}
+		opts+=${OSSCAPA:+" --param=ost.capa=$OSSCAPA"}
+
+		if [ $fstype == ldiskfs ]; then
+			fs_mkfs_opts+=${OSTJOURNALSIZE:+" -J size=$OSTJOURNALSIZE"}
+		fi
+	fi
+
+	opts+=" --backfstype=$fstype"
+
+	var=${type}SIZE
+	if [ -n "${!var}" ]; then
+		opts+=" --device-size=${!var}"
+	fi
+
+	var=$(upper $fstype)_MKFS_OPTS
+	fs_mkfs_opts+=${!var:+" ${!var}"}
+
+	var=${type}_FS_MKFS_OPTS
+	fs_mkfs_opts+=${!var:+" ${!var}"}
+
+	if [ -n "${fs_mkfs_opts## }" ]; then
+		opts+=" --mkfsoptions=\\\"${fs_mkfs_opts## }\\\""
+	fi
+
+	var=${type}OPT
+	opts+=${!var:+" ${!var}"}
+
+	echo -n "$opts"
+}
+
+formatall() {
+	local quiet
+
+	if ! $VERBOSE; then
+		quiet=yes
+	fi
+
+	stopall
+	# We need ldiskfs here, may as well load them all
+	load_modules
+	[ "$CLIENTONLY" ] && return
+	echo Formatting mgs, mds, osts
+	if ! combined_mgs_mds ; then
+		echo "Format mgs: $(mgsdevname)"
+		add mgs $(mkfs_opts mgs) --reformat $(mgsdevname) \
+			$(mgsvdevname) ${quiet:+>/dev/null} || exit 10
+		fi
+
+		for num in `seq $MDSCOUNT`; do
+			echo "Format mds$num: $(mdsdevname $num)"
+			add mds$num $(mkfs_opts mds$num) --reformat \
+			$(mdsdevname $num) $(mdsvdevname $num) \
+			${quiet:+>/dev/null} || exit 10
+		done
+
+		for num in `seq $OSTCOUNT`; do
+			echo "Format ost$num: $(ostdevname $num)"
+			add ost$num $(mkfs_opts ost$num) --reformat \
+			$(ostdevname $num) $(ostvdevname ${num}) \
+			${quiet:+>/dev/null} || exit 10
+		done
 }
 
 mount_client() {
@@ -2211,7 +2750,7 @@ setupall() {
         echo $WRITECONF | grep -q "writeconf" && \
             writeconf_all
         if ! combined_mgs_mds ; then
-            start mgs $MGSDEV $MGS_MOUNT_OPTS
+			start mgs $(mgsdevname) $MGS_MOUNT_OPTS
         fi
 
         for num in `seq $MDSCOUNT`; do
@@ -2254,6 +2793,7 @@ setupall() {
     [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
     mount_client $MOUNT
     [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT
+    clients_up
 
     if [ "$MOUNT_2" ]; then
         mount_client $MOUNT2
@@ -2291,7 +2831,7 @@ init_facet_vars () {
     eval export ${facet}_opt=\"$@\"
 
     local dev=${facet}_dev
-    local label=$(do_facet ${facet} "$E2LABEL ${!dev}")
+	local label=$(devicelabel ${facet} ${!dev})
     [ -z "$label" ] && echo no label for ${!dev} && exit 1
 
     eval export ${facet}_svc=${label}
@@ -2331,7 +2871,7 @@ init_facets_vars () {
         done
     fi
 
-    combined_mgs_mds || init_facet_vars mgs $MGSDEV $MGS_MOUNT_OPTS
+	combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS
 
     remote_ost_nodsh && return
 
@@ -2361,12 +2901,6 @@ osc_ensure_active () {
 }
 
 init_param_vars () {
-    if ! remote_ost_nodsh && ! remote_mds_nodsh; then
-        export MDSVER=$(do_facet $SINGLEMDS "lctl get_param version" | cut -d. -f1,2)
-        export OSTVER=$(do_facet ost1 "lctl get_param version" | cut -d. -f1,2)
-        export CLIVER=$(lctl get_param version | cut -d. -f 1,2)
-    fi
-
     remote_mds_nodsh ||
         TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout")
 
@@ -2375,7 +2909,7 @@ init_param_vars () {
     osc_ensure_active $SINGLEMDS $TIMEOUT
     osc_ensure_active client $TIMEOUT
 
-    if [ $QUOTA_AUTO -ne 0 ]; then
+	if [ $QUOTA_AUTO -ne 0 ]; then
         if [ "$ENABLE_QUOTA" ]; then
             echo "enable quota as required"
             setup_quota $MOUNT || return 2
@@ -2549,12 +3083,14 @@ check_and_setup_lustre() {
         set_default_debug_nodes $(comma_list $(nodes_list))
     fi
 
-    init_gss
-    set_flavor_all $SEC
+	init_gss
+	if $GSS; then
+		set_flavor_all $SEC
+	fi
 
-    if [ "$ONLY" == "setup" ]; then
-        exit 0
-    fi
+	if [ "$ONLY" == "setup" ]; then
+		exit 0
+	fi
 }
 
 restore_mount () {
@@ -2584,26 +3120,24 @@ cleanup_and_setup_lustre() {
 
 # Get all of the server target devices from a given server node and type.
 get_mnt_devs() {
-    local node=$1
-    local type=$2
-    local obd_type
-    local devs
-    local dev
-
-    case $type in
-    mdt) obd_type="osd" ;;
-    ost) obd_type="obdfilter" ;; # needs to be fixed when OST also uses an OSD
-    *) echo "invalid server type" && return 1 ;;
-    esac
-
-    devs=$(do_node $node "lctl get_param -n $obd_type*.*.mntdev")
-    for dev in $devs; do
-        case $dev in
-        *loop*) do_node $node "losetup $dev" | \
-                sed -e "s/.*(//" -e "s/).*//" ;;
-        *) echo $dev ;;
-        esac
-    done
+	local node=$1
+	local type=$2
+	local devs
+	local dev
+
+	if [ "$type" == ost ]; then
+		devs=$(get_obdfilter_param $node "" mntdev)
+	else
+		devs=$(do_node $node \
+		       "lctl get_param -n osd-*.$FSNAME-M*.mntdev")
+	fi
+	for dev in $devs; do
+		case $dev in
+		*loop*) do_node $node "losetup $dev" | \
+				sed -e "s/.*(//" -e "s/).*//" ;;
+		*) echo $dev ;;
+		esac
+	done
 }
 
 # Get all of the server target devices.
@@ -2697,7 +3231,8 @@ check_and_cleanup_lustre() {
     fi
 
     if is_mounted $MOUNT; then
-        [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]*
+        [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* ||
+            error "remove sub-test dirs failed"
         [ "$ENABLE_QUOTA" ] && restore_quota_type || true
     fi
 
@@ -2872,6 +3407,10 @@ at_max_get() {
     at_get $1 at_max
 }
 
+at_min_get() {
+	at_get $1 at_min
+}
+
 at_max_set() {
     local at_max=$1
     shift
@@ -2932,11 +3471,14 @@ pause_bulk() {
 
 drop_ldlm_cancel() {
 #define OBD_FAIL_LDLM_CANCEL             0x304
-    RC=0
-    do_facet client lctl set_param fail_loc=0x304
-    do_facet client "$@" || RC=$?
-    do_facet client lctl set_param fail_loc=0
-    return $RC
+	local RC=0
+	local list=$(comma_list $(mdts_nodes) $(osts_nodes))
+	do_nodes $list lctl set_param fail_loc=0x304
+
+	do_facet client "$@" || RC=$?
+
+	do_nodes $list lctl set_param fail_loc=0
+	return $RC
 }
 
 drop_bl_callback() {
@@ -3043,6 +3585,17 @@ stop_full_debug_logging() {
     debugrestore
 }
 
+# prints bash call stack
+log_trace_dump() {
+	echo "  Trace dump:"
+	for (( i=1; i < ${#BASH_LINENO[*]} ; i++ )) ; do
+		local s=${BASH_SOURCE[$i]}
+		local l=${BASH_LINENO[$i-1]}
+		local f=${FUNCNAME[$i]}
+		echo "  = $s:$l:$f()"
+	done
+}
+
 ##################################
 # Test interface
 ##################################
@@ -3058,10 +3611,12 @@ error_noexit() {
     fi
 
     log " ${TESTSUITE} ${TESTNAME}: @@@@@@ ${TYPE}: $@ "
+    log_trace_dump
 
+    mkdir -p $LOGDIR
     # We need to dump the logs on all nodes
     if $dump; then
-        gather_logs $(comma_list $(nodes_list)) 0
+        gather_logs $(comma_list $(nodes_list))
     fi
 
     debugrestore
@@ -3070,11 +3625,11 @@ error_noexit() {
 }
 
 exit_status () {
-    local status=0
-    local log=$TESTSUITELOG
+	local status=0
+	local log=$TESTSUITELOG
 
-    [ -f "$log" ] && grep -q FAIL: $log && status=1
-    exit $status
+	[ -f "$log" ] && grep -q FAIL $log && status=1
+	exit $status
 }
 
 error() {
@@ -3304,7 +3859,7 @@ banner() {
 run_one() {
     local testnum=$1
     local message=$2
-    tfile=f${testnum}
+    tfile=f.${TESTSUITE}.${testnum}
     export tdir=d0.${TESTSUITE}/d${base}
     export TESTNAME=test_$testnum
     local SAVE_UMASK=`umask`
@@ -3330,34 +3885,34 @@ run_one() {
 #  - test result is saved to data file
 #
 run_one_logged() {
-    local BEFORE=`date +%s`
-    local TEST_ERROR
-    local name=${TESTSUITE}.test_${1}.test_log.$(hostname -s).log
-    local test_log=$LOGDIR/$name
-    rm -rf $LOGDIR/err
-    local SAVE_UMASK=`umask`
-    umask 0022
+	local BEFORE=`date +%s`
+	local TEST_ERROR
+	local name=${TESTSUITE}.test_${1}.test_log.$(hostname -s).log
+	local test_log=$LOGDIR/$name
+	rm -rf $LOGDIR/err
+	local SAVE_UMASK=`umask`
+	umask 0022
 
-    echo
-    log_sub_test_begin test_${1}
-    (run_one $1 "$2") 2>&1 | tee -i $test_log
-    local RC=${PIPESTATUS[0]}
+	echo
+	log_sub_test_begin test_${1}
+	(run_one $1 "$2") 2>&1 | tee -i $test_log
+	local RC=${PIPESTATUS[0]}
 
-    [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \
-        echo "test_$1 returned $RC" | tee $LOGDIR/err
+	[ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \
+		echo "test_$1 returned $RC" | tee $LOGDIR/err
 
-    duration=$((`date +%s` - $BEFORE))
-    pass "$1" "(${duration}s)"
-    [ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err)
-    log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR"
+	duration=$((`date +%s` - $BEFORE))
+	pass "$1" "(${duration}s)"
+	[ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err)
+	log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR"
 
-    if [ -f $LOGDIR/err ]; then
-        $FAIL_ON_ERROR && exit $RC
-    fi
+	if [ -f $LOGDIR/err ]; then
+		$FAIL_ON_ERROR && exit $RC
+	fi
 
-    umask $SAVE_UMASK
+	umask $SAVE_UMASK
 
-    return 0
+	return 0
 }
 
 #
@@ -3395,7 +3950,7 @@ check_grant() {
 
     # get server grant
     server_grant=`do_nodes $(comma_list $(osts_nodes)) \
-                    "$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" | \
+                    "$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" |
                     awk '{total += $1} END{print total}'`
 
     # check whether client grant == server grant
@@ -3441,7 +3996,7 @@ index_from_ostuuid()
 
 mdtuuid_from_index()
 {
-    $LFS mdts $2 | awk '/^'$1'/ { print $2 }'
+    $LFS mdts $2 | sed -ne "/^$1: /s/.* \(.*\) .*$/\1/p"
 }
 
 remote_node () {
@@ -3614,7 +4169,8 @@ is_patchless ()
 }
 
 check_versions () {
-    [ "$MDSVER" = "$CLIVER" -a "$OSTVER" = "$CLIVER" ]
+    [ "$(lustre_version_code client)" = "$(lustre_version_code $SINGLEMDS)" -a \
+      "$(lustre_version_code client)" = "$(lustre_version_code ost1)" ]
 }
 
 get_node_count() {
@@ -3726,7 +4282,7 @@ fi"
 # Run multiop in the background, but wait for it to print
 # "PAUSING" to its stdout before returning from this function.
 multiop_bg_pause() {
-    MULTIOP_PROG=${MULTIOP_PROG:-multiop}
+    MULTIOP_PROG=${MULTIOP_PROG:-$MULTIOP}
     FILE=$1
     ARGS=$2
 
@@ -3837,25 +4393,6 @@ exit \\\$rc;"
     fi 
 }
 
-# $1 node
-# $2 file
-# $3 $RUNAS
-get_stripe_info() {
-        local tmp_file
-
-        stripe_size=0
-        stripe_count=0
-        stripe_index=0
-        tmp_file=$(mktemp)
-
-        do_facet $1 $3 lfs getstripe -v $2 > $tmp_file
-
-        stripe_size=`awk '$1 ~ /size/ {print $2}' $tmp_file`
-        stripe_count=`awk '$1 ~ /count/ {print $2}' $tmp_file`
-        stripe_index=`awk '$1 ~ /stripe_offset/ {print $2}' $tmp_file`
-        rm -f $tmp_file
-}
-
 # CMD: determine mds index where directory inode presents
 get_mds_dir () {
     local dir=$1
@@ -3919,19 +4456,12 @@ get_clientosc_proc_path() {
 
 get_lustre_version () {
     local facet=${1:-"$SINGLEMDS"}    
-    do_facet $facet $LCTL get_param -n version |  awk '/^lustre:/ {print $2}'
+    do_facet $facet $LCTL get_param -n version | awk '/^lustre:/ {print $2}'
 }
 
-get_mds_version_major () {
+lustre_version_code() {
     local facet=${1:-"$SINGLEMDS"}
-    local version=$(get_lustre_version $facet)
-    echo $version | awk -F. '{print $1}'
-}
-
-get_mds_version_minor () {
-    local facet=${1:-"$SINGLEMDS"}
-    local version=$(get_lustre_version $facet)
-    echo $version | awk -F. '{print $2}'
+    version_code $(get_lustre_version $1)
 }
 
 # If the 2.0 MDS was mounted on 1.8 device, then the OSC and LOV names
@@ -3940,10 +4470,8 @@ get_mds_version_minor () {
 # mdt osc: fsname-OSTXXXX-osc
 mds_on_old_device() {
     local mds=${1:-"$SINGLEMDS"}
-    local major=$(get_mds_version_major $mds)
-    local minor=$(get_mds_version_minor $mds)
 
-    if [ $major -ge 2 ] || [ $major -eq 1 -a $minor -gt 8 ]; then
+    if [ $(lustre_version_code $mds) -gt $(version_code 1.9.0) ]; then
         do_facet $mds "lctl list_param osc.$FSNAME-OST*-osc \
             > /dev/null 2>&1" && return 0
     fi
@@ -3958,9 +4486,8 @@ get_mdtosc_proc_path() {
     local mdt_label=$(convert_facet2label $mds_facet)
     local mdt_index=$(echo $mdt_label | sed -e 's/^.*-//')
 
-    local major=$(get_mds_version_major $mds_facet)
-    local minor=$(get_mds_version_minor $mds_facet)
-    if [ $major -le 1 -a $minor -le 8 ] || mds_on_old_device $mds_facet; then
+    if [ $(lustre_version_code $mds_facet) -le $(version_code 1.8.0) ] ||
+       mds_on_old_device $mds_facet; then
         echo "${ost_label}-osc"
     else
         echo "${ost_label}-osc-${mdt_index}"
@@ -4046,10 +4573,7 @@ wait_import_state() {
 # the value depends on configure options, and it is not stored in /proc.
 # obd_support.h:
 # #define CONNECTION_SWITCH_MIN 5U
-# #ifndef CRAY_XT3
 # #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/20)
-# #else
-# #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/2)
 
 request_timeout () {
     local facet=$1
@@ -4126,20 +4650,30 @@ wait_clients_import_state () {
 }
 
 oos_full() {
-        local -a AVAILA
-        local -a GRANTA
-        local OSCFULL=1
-        AVAILA=($(do_nodes $(comma_list $(osts_nodes)) \
-                  $LCTL get_param obdfilter.*.kbytesavail))
-        GRANTA=($(do_nodes $(comma_list $(osts_nodes)) \
-                  $LCTL get_param -n obdfilter.*.tot_granted))
-        for ((i=0; i<${#AVAILA[@]}; i++)); do
-                local -a AVAIL1=(${AVAILA[$i]//=/ })
-                GRANT=$((${GRANTA[$i]}/1024))
-                echo -n $(echo ${AVAIL1[0]} | cut -d"." -f2) avl=${AVAIL1[1]} grnt=$GRANT diff=$((AVAIL1[1] - GRANT))
-                [ $((AVAIL1[1] - GRANT)) -lt 400 ] && OSCFULL=0 && echo " FULL" || echo
-        done
-        return $OSCFULL
+	local -a AVAILA
+	local -a GRANTA
+	local -a TOTALA
+	local OSCFULL=1
+	AVAILA=($(do_nodes $(comma_list $(osts_nodes)) \
+	          $LCTL get_param obdfilter.*.kbytesavail))
+	GRANTA=($(do_nodes $(comma_list $(osts_nodes)) \
+	          $LCTL get_param -n obdfilter.*.tot_granted))
+	TOTALA=($(do_nodes $(comma_list $(osts_nodes)) \
+	          $LCTL get_param -n obdfilter.*.kbytestotal))
+	for ((i=0; i<${#AVAILA[@]}; i++)); do
+		local -a AVAIL1=(${AVAILA[$i]//=/ })
+		local -a TOTAL=(${TOTALA[$i]//=/ })
+		GRANT=$((${GRANTA[$i]}/1024))
+		# allow 1% of total space in bavail because of delayed
+		# allocation with ZFS which might release some free space after
+		# txg commit.  For small devices, we set a mininum of 8MB
+		local LIMIT=$((${TOTAL} / 100 + 8000))
+		echo -n $(echo ${AVAIL1[0]} | cut -d"." -f2) avl=${AVAIL1[1]} \
+			grnt=$GRANT diff=$((AVAIL1[1] - GRANT)) limit=${LIMIT}
+		[ $((AVAIL1[1] - GRANT)) -lt $LIMIT ] && OSCFULL=0 && \
+			echo " FULL" || echo
+	done
+	return $OSCFULL
 }
 
 pool_list () {
@@ -4228,7 +4762,7 @@ destroy_pools () {
 
     echo destroy the created pools: ${!listvar}
     for poolname in ${!listvar//,/ }; do
-        destroy_pool $fsname.$poolname 
+        destroy_pool $fsname.$poolname
     done
 }
 
@@ -4240,15 +4774,21 @@ cleanup_pools () {
 
 gather_logs () {
     local list=$1
-    local tar_logs=$2
 
     local ts=$(date +%s)
     local docp=true
+
+    if [[ ! -f "$YAML_LOG" ]]; then
+        # init_logging is not performed before gather_logs,
+        # so the $LOGDIR needs to be checked here
+        check_shared_dir $LOGDIR && touch $LOGDIR/shared
+    fi
+
     [ -f $LOGDIR/shared ] && docp=false
- 
+
     # dump lustre logs, dmesg
 
-    prefix="$LOGDIR/${TESTSUITE}.${TESTNAME}"
+    prefix="$TESTLOG_PREFIX.$TESTNAME"
     suffix="$ts.log"
     echo "Dumping lctl log to ${prefix}.*.${suffix}"
 
@@ -4264,22 +4804,9 @@ gather_logs () {
          dmesg > ${prefix}.dmesg.\\\$(hostname -s).${suffix}"
     if [ ! -f $LOGDIR/shared ]; then
         do_nodes $list rsync -az "${prefix}.*.${suffix}" $HOSTNAME:$LOGDIR
-      fi
-
-    if [ $tar_logs == 1 ]; then
-        local archive=$LOGDIR/${TESTSUITE}-$ts.tar.bz2
-        tar -jcf $archive $LOGDIR/*$ts* $LOGDIR/*${TESTSUITE}*
-
-        echo $archive
     fi
 }
 
-cleanup_logs () {
-    local list=${1:-$(comma_list $(nodes_list))}
-
-    [ -n ${TESTSUITE} ] && do_nodes $list "rm -f $TMP/*${TESTSUITE}*" || true
-}
-
 do_ls () {
     local mntpt_root=$1
     local num_mntpts=$2
@@ -4622,20 +5149,24 @@ check_logdir() {
         # Not found. Create local logdir
         mkdir -p $dir
     else
-        touch $dir/node.$(hostname -s).yml
+        touch $dir/check_file.$(hostname -s)
     fi
     return 0
 }
 
 check_write_access() {
     local dir=$1
+    local node
+    local file
+
     for node in $(nodes_list); do
-        if [ ! -f "$dir/node.$(short_hostname ${node}).yml" ]; then
+        file=$dir/check_file.$(short_hostname $node)
+        if [[ ! -f "$file" ]]; then
             # Logdir not accessible/writable from this node.
             return 1
         fi
+        rm -f $file || return 1
     done
-    rm -f $dir/node.*.yml
     return 0
 }
 
@@ -4650,15 +5181,17 @@ init_logging() {
     mkdir -p $LOGDIR
     init_clients_lists
 
-    if check_shared_dir $LOGDIR; then
-        touch $LOGDIR/shared
-        echo "Logging to shared log directory: $LOGDIR"
-    else
-        echo "Logging to local directory: $LOGDIR"
-    fi
+    if [ ! -f $YAML_LOG ]; then       # If the yaml log already exists then we will just append to it
+      if check_shared_dir $LOGDIR; then
+          touch $LOGDIR/shared
+          echo "Logging to shared log directory: $LOGDIR"
+      else
+          echo "Logging to local directory: $LOGDIR"
+      fi
 
-    yml_nodes_file $LOGDIR >> $YAML_LOG
-    yml_results_file >> $YAML_LOG
+      yml_nodes_file $LOGDIR >> $YAML_LOG
+      yml_results_file >> $YAML_LOG
+    fi
 
     umask $SAVE_UMASK
 }
@@ -4713,32 +5246,44 @@ run_llverfs()
 }
 
 remove_mdt_files() {
-    local facet=$1
-    local mdtdev=$2
-    shift 2
-    local files="$@"
-    local mntpt=$(facet_mntpt $facet)
-
-    echo "removing files from $mdtdev on $facet: $files"
-    mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $?
-    rc=0;
-    for f in $files; do
-        rm $mntpt/ROOT/$f || { rc=$?; break; }
-    done
-    umount -f $mntpt || return $?
-    return $rc
+	local facet=$1
+	local mdtdev=$2
+	shift 2
+	local files="$@"
+	local mntpt=$(facet_mntpt $facet)
+	local opts=$MDS_MOUNT_OPTS
+
+	echo "removing files from $mdtdev on $facet: $files"
+	if [ $(facet_fstype $facet) == ldiskfs ] &&
+	   ! do_facet $facet test -b ${!dev}; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
+	mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
+		return $?
+	rc=0;
+	for f in $files; do
+		rm $mntpt/ROOT/$f || { rc=$?; break; }
+	done
+	umount -f $mntpt || return $?
+	return $rc
 }
 
 duplicate_mdt_files() {
-    local facet=$1
-    local mdtdev=$2
-    shift 2
-    local files="$@"
-    local mntpt=$(facet_mntpt $facet)
-
-    echo "duplicating files on $mdtdev on $facet: $files"
-    mkdir -p $mntpt || return $?
-    mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $?
+	local facet=$1
+	local mdtdev=$2
+	shift 2
+	local files="$@"
+	local mntpt=$(facet_mntpt $facet)
+	local opts=$MDS_MOUNT_OPTS
+
+	echo "duplicating files on $mdtdev on $facet: $files"
+	mkdir -p $mntpt || return $?
+	if [ $(facet_fstype $facet) == ldiskfs ] &&
+	   ! do_facet $facet test -b ${!dev}; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
+	mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
+		return $?
 
     do_umount() {
         trap 0
@@ -4855,3 +5400,101 @@ generate_string() {
 
     echo "$(head -c $size < /dev/zero | tr '\0' y)"
 }
+
+reformat_external_journal() {
+	if [ ! -z ${EJOURNAL} ]; then
+		local rcmd="do_facet ${SINGLEMDS}"
+
+		echo "reformat external journal on ${SINGLEMDS}:${EJOURNAL}"
+		${rcmd} mke2fs -O journal_dev ${EJOURNAL} || return 1
+	fi
+}
+
+# MDT file-level backup/restore
+mds_backup_restore() {
+	local devname=$(mdsdevname ${SINGLEMDS//mds/})
+	local mntpt=$(facet_mntpt brpt)
+	local rcmd="do_facet ${SINGLEMDS}"
+	local metaea=${TMP}/backup_restore.ea
+	local metadata=${TMP}/backup_restore.tgz
+	local opts=${MDS_MOUNT_OPTS}
+
+	if ! ${rcmd} test -b ${devname}; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
+
+	echo "file-level backup/restore on ${SINGLEMDS}:${devname}"
+
+	# step 1: build mount point
+	${rcmd} mkdir -p $mntpt
+	# step 2: cleanup old backup
+	${rcmd} rm -f $metaea $metadata
+	# step 3: mount dev
+	${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 1
+	# step 4: backup metaea
+	echo "backup EA"
+	${rcmd} "cd $mntpt && getfattr -R -d -m '.*' -P . > $metaea && cd -" ||
+		return 2
+	# step 5: backup metadata
+	echo "backup data"
+	${rcmd} tar zcf $metadata -C $mntpt/ . > /dev/null 2>&1 || return 3
+	# step 6: umount
+	${rcmd} umount -d $mntpt || return 4
+	# step 7: reformat external journal if needed
+	reformat_external_journal || return 5
+	# step 8: reformat dev
+	echo "reformat new device"
+	add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS}) --backfstype ldiskfs \
+		--reformat $devname > /dev/null || return 6
+	# step 9: mount dev
+	${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 7
+	# step 10: restore metadata
+	echo "restore data"
+	${rcmd} tar zxfp $metadata -C $mntpt > /dev/null 2>&1 || return 8
+	# step 11: restore metaea
+	echo "restore EA"
+	${rcmd} "cd $mntpt && setfattr --restore=$metaea && cd - " || return 9
+	# step 12: remove recovery logs
+	echo "remove recovery logs"
+	${rcmd} rm -fv $mntpt/OBJECTS/* $mntpt/CATALOGS
+	# step 13: umount dev
+	${rcmd} umount -d $mntpt || return 10
+	# step 14: cleanup tmp backup
+	${rcmd} rm -f $metaea $metadata
+}
+
+# remove OI files
+mds_remove_ois() {
+	local devname=$(mdsdevname ${SINGLEMDS//mds/})
+	local mntpt=$(facet_mntpt brpt)
+	local rcmd="do_facet ${SINGLEMDS}"
+	local idx=$1
+	local opts=${MDS_MOUNT_OPTS}
+
+	if ! ${rcmd} test -b ${devname}; then
+		opts=$(csa_add "$opts" -o loop)
+	fi
+
+	echo "remove OI files: idx=${idx}"
+
+	# step 1: build mount point
+	${rcmd} mkdir -p $mntpt
+	# step 2: mount dev
+	${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 1
+	if [ -z $idx ]; then
+		# step 3: remove all OI files
+		${rcmd} rm -fv $mntpt/oi.16*
+	elif [ $idx -lt 2 ]; then
+		${rcmd} rm -fv $mntpt/oi.16.${idx}
+	else
+		local i
+
+		# others, rm oi.16.[idx, idx * idx, idx ** ...]
+		for ((i=${idx}; i<64; i=$((i * idx)))); do
+			${rcmd} rm -fv $mntpt/oi.16.${i}
+		done
+	fi
+	# step 4: umount
+	${rcmd} umount -d $mntpt || return 2
+	# OI files will be recreated when mounted as lustre next time.
+}