X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=2fa00075a68575adb7e31012a2786df656128d07;hp=9a1bcbd79fba8921a8e52901e5e374f251ab2bbd;hb=d25e7654f798060f6b671c3b65949a70b8c67f58;hpb=9c6c262ffd6ba46e666586cd492749ec9c063a71

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 9a1bcbd..2fa0007 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -282,8 +282,6 @@ init_test_env() {
             IDENTITY_UPCALL=false
             ;;
     esac
-    USE_OFD=${USE_OFD:-yes}
-    [ "$USE_OFD" = "yes" ] && LOAD_MODULES_REMOTE=true
 
     export LOAD_MODULES_REMOTE=${LOAD_MODULES_REMOTE:-false}
 
@@ -291,9 +289,9 @@ init_test_env() {
     export RLUSTRE=${RLUSTRE:-$LUSTRE}
     export RPWD=${RPWD:-$PWD}
     export I_MOUNTED=${I_MOUNTED:-"no"}
-    if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mds.ko -a \
-        ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mds.ko -a \
-        ! -f `dirname $0`/../mds/mds.ko ]; then
+    if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mdt.ko -a \
+        ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mdt.ko -a \
+        ! -f `dirname $0`/../mdt/mdt.ko ]; then
         export CLIENTMODSONLY=yes
     fi
 
@@ -338,7 +336,7 @@ version_code() {
     # split arguments like "1.8.6-wc3" into "1", "8", "6", "wc3"
     eval set -- $(tr "[:punct:]" " " <<< $*)
 
-    echo -n $((($1 << 16) | ($2 << 8) | $3))
+    echo -n "$((($1 << 16) | ($2 << 8) | $3))"
 }
 
 export LINUX_VERSION=$(uname -r | sed -e "s/[-.]/ /3" -e "s/ .*//")
@@ -373,20 +371,24 @@ load_module() {
         optvar="MODOPTS_$(basename $module | tr a-z A-Z)"
         eval set -- \$$optvar
         if [ $# -eq 0 -a -n "$MODPROBECONF" ]; then
-            # Nothing in $MODOPTS_<MODULE>; try modprobe.conf
-            set -- $(grep -P "^options\\s+${BASE}" $MODPROBECONF)
-            # Get rid of "options $module"
-            (($# > 0)) && shift 2
-
-            # Ensure we have accept=all for lnet
-            if [ $(basename $module) = lnet ]; then
-                # OK, this is a bit wordy...
-                local arg accept_all_present=false
-                for arg in "$@"; do
-                    [ "$arg" = accept=all ] && accept_all_present=true
-                done
-                $accept_all_present || set -- "$@" accept=all
-            fi
+		# Nothing in $MODOPTS_<MODULE>; try modprobe.conf
+		local opt
+		opt=$(awk -v var="^options $BASE" '$0 ~ var \
+			{gsub("'"options $BASE"'",""); print}' $MODPROBECONF)
+		set -- $(echo -n $opt)
+
+		# Ensure we have accept=all for lnet
+		if [ $(basename $module) = lnet ]; then
+			# OK, this is a bit wordy...
+			local arg accept_all_present=false
+
+			for arg in "$@"; do
+				[ "$arg" = accept=all ] && \
+					accept_all_present=true
+			done
+			$accept_all_present || set -- "$@" accept=all
+		fi
+		export $optvar="$*"
         fi
     fi
 
@@ -425,8 +427,6 @@ llite_lloop_enabled() {
 }
 
 load_modules_local() {
-	[ $(facet_fstype ost1) == "zfs" ] && export USE_OFD=yes
-
 	if [ -n "$MODPROBE" ]; then
 		# use modprobe
 		echo "Using modprobe to load modules"
@@ -493,7 +493,6 @@ load_modules_local() {
 			load_module osd-zfs/osd_zfs
 		fi
 		load_module mgs/mgs
-		load_module mds/mds
 		load_module mdd/mdd
 		if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then
 			#
@@ -698,7 +697,7 @@ cleanup_gss() {
 facet_type() {
 	local facet=$1
 
-	echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9]\+//' |
+	echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9_]\+//' |
 		tr '[:lower:]' '[:upper:]'
 }
 
@@ -1044,13 +1043,14 @@ quota_save_version() {
 # XXX This function is kept for interoperability with old server (< 2.3.50),
 #     it should be removed whenever we drop the interoperability for such
 #     server.
-quota_type () {
-    local fsname=${1:-$FSNAME}
-    local rc=0
-    do_facet mgs lctl get_param mdd.${fsname}-MDT*.quota_type || rc=$?
-    do_nodes $(comma_list $(osts_nodes)) \
-        lctl get_param obdfilter.${fsname}-OST*.quota_type || rc=$?
-    return $rc 
+quota_type() {
+	local fsname=${1:-$FSNAME}
+	local rc=0
+	do_facet $SINGLEMDS lctl get_param mdd.${fsname}-MDT*.quota_type ||
+		rc=$?
+	do_nodes $(comma_list $(osts_nodes)) \
+		lctl get_param obdfilter.${fsname}-OST*.quota_type || rc=$?
+	return $rc 
 }
 
 # XXX This function is kept for interoperability with old server (< 2.3.50),
@@ -1092,11 +1092,11 @@ setup_quota_old(){
     local quota_usrs=$QUOTA_USERS
 
     # get_filesystem_size
-    local disksz=$(lfs df $mntpt | grep "filesystem summary:"  | awk '{print $3}')
+    local disksz=$(lfs_df $mntpt | grep "summary"  | awk '{print $2}')
     local blk_soft=$((disksz + 1024))
     local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over
 
-    local Inodes=$(lfs df -i $mntpt | grep "filesystem summary:"  | awk '{print $3}')
+    local Inodes=$(lfs_df -i $mntpt | grep "summary"  | awk '{print $2}')
     local i_soft=$Inodes
     local i_hard=$((i_soft + i_soft / 20))
 
@@ -1121,7 +1121,7 @@ setup_quota_old(){
 mdt_quota_type() {
 	local varsvc=${SINGLEMDS}_svc
 	do_facet $SINGLEMDS $LCTL get_param -n \
-		osd-$FSTYPE.${!varsvc}.quota_slave.enabled
+		osd-$(facet_fstype $SINGLEMDS).${!varsvc}.quota_slave.enabled
 }
 
 # get ost quota type
@@ -1129,7 +1129,7 @@ ost_quota_type() {
 	# All OSTs should have same quota type
 	local varsvc=ost1_svc
 	do_facet ost1 $LCTL get_param -n \
-		osd-$FSTYPE.${!varsvc}.quota_slave.enabled
+		osd-$(facet_fstype ost1).${!varsvc}.quota_slave.enabled
 }
 
 # restore old quota type settings
@@ -1149,16 +1149,19 @@ restore_quota() {
 	fi
 }
 
+# Handle the case when there is a space in the lfs df
+# "filesystem summary" line the same as when there is no space.
+# This will allow fixing the "lfs df" summary line in the future.
+lfs_df() {
+	$LFS df $* | sed -e 's/filesystem /filesystem_/'
+}
+
 setup_quota(){
 	if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then
 		setup_quota_old $1
 		return
 	fi
 
-	# XXX remove it once all quota code landed
-	echo "skip quota setup"
-	return
-
 	local mntpt=$1
 
 	# save old quota type & set new quota type
@@ -1179,13 +1182,11 @@ setup_quota(){
 	local quota_usrs=$QUOTA_USERS
 
 	# get_filesystem_size
-	local disksz=$(lfs df $mntpt | grep "filesystem summary:" |
-		     awk '{print $3}')
+	local disksz=$(lfs_df $mntpt | grep "summary" | awk '{print $2}')
 	local blk_soft=$((disksz + 1024))
 	local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over
 
-	local inodes=$(lfs df -i $mntpt | grep "filesystem summary:" |
-		     awk '{print $3}')
+	local inodes=$(lfs_df -i $mntpt | grep "summary" | awk '{print $2}')
 	local i_soft=$inodes
 	local i_hard=$((i_soft + i_soft / 20))
 
@@ -1725,15 +1726,16 @@ cleanup_check() {
 
     check_mem_leak || exit 204
 
-    [ "`lctl dl 2> /dev/null | wc -l`" -gt 0 ] && lctl dl && \
-        echo "$0: lustre didn't clean up..." 1>&2 && return 202 || true
+	[ "`lctl dl 2> /dev/null | wc -l`" -gt 0 ] && lctl dl &&
+		echo "$TESTSUITE: lustre didn't clean up..." 1>&2 &&
+		return 202 || true
 
-    if module_loaded lnet || module_loaded libcfs; then
-        echo "$0: modules still loaded..." 1>&2
-        /sbin/lsmod 1>&2
-        return 203
-    fi
-    return 0
+	if module_loaded lnet || module_loaded libcfs; then
+		echo "$TESTSUITE: modules still loaded..." 1>&2
+		/sbin/lsmod 1>&2
+		return 203
+	fi
+	return 0
 }
 
 wait_update () {
@@ -2048,32 +2050,58 @@ affected_facets () {
 }
 
 facet_failover() {
-    local facet=$1
-    local sleep_time=$2
-    local host=$(facet_active_host $facet)
-
-    echo "Failing $facet on node $host"
-
-    local affected=$(affected_facets $facet)
+	local facets=$1
+	local sleep_time=$2
+	local -a affecteds
+	local facet
+	local total=0
+	local index=0
+	local skip
+
+	#Because it will only get up facets, we need get affected
+	#facets before shutdown
+	#For HARD Failure mode, it needs make sure facets on the same
+	#HOST will only be shutdown and reboot once
+	for facet in ${facets//,/ }; do
+		local affected_facet
+		skip=0
+		#check whether facet has been included in other affected facets
+		for ((index=0; index<$total; index++)); do
+			[[ *,$facet,* == ,${affecteds[index]}, ]] && skip=1
+		done
 
-    shutdown_facet $facet
+		if [ $skip -eq 0 ]; then
+			affecteds[$total]=$(affected_facets $facet)
+			total=$((total+1))
+		fi
+	done
 
-    echo affected facets: $affected
+	for ((index=0; index<$total; index++)); do
+		facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
+		local host=$(facet_active_host $facet)
+		echo "Failing ${affecteds[index]} on $host"
+		shutdown_facet $facet
+	done
 
-    [ -n "$sleep_time" ] && sleep $sleep_time
+	for ((index=0; index<$total; index++)); do
+		facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
+		echo reboot facets: ${affecteds[index]}
 
-    reboot_facet $facet
+		reboot_facet $facet
 
-    change_active $affected
+		change_active ${affecteds[index]}
 
-    wait_for_facet $affected
-    # start mgs first if it is affected
-    if ! combined_mgs_mds && list_member $affected mgs; then
-        mount_facet mgs || error "Restart of mgs failed"
-    fi
-    # FIXME; has to be changed to mount all facets concurrently
-    affected=$(exclude_items_from_list $affected mgs)
-    mount_facets $affected
+		wait_for_facet ${affecteds[index]}
+		# start mgs first if it is affected
+		if ! combined_mgs_mds &&
+			list_member ${affecteds[index]} mgs; then
+			mount_facet mgs || error "Restart of mgs failed"
+		fi
+		# FIXME; has to be changed to mount all facets concurrently
+		affected=$(exclude_items_from_list ${affecteds[index]} mgs)
+		echo mount facets: ${affecteds[index]}
+		mount_facets ${affecteds[index]}
+	done
 }
 
 obd_name() {
@@ -2081,42 +2109,42 @@ obd_name() {
 }
 
 replay_barrier() {
-    local facet=$1
-    do_facet $facet "sync; sync; sync"
-    df $MOUNT
+	local facet=$1
+	do_facet $facet "sync; sync; sync"
+	df $MOUNT
 
-    # make sure there will be no seq change
-    local clients=${CLIENTS:-$HOSTNAME}
-    local f=fsa-\\\$\(hostname\)
-    do_nodes $clients "mcreate $MOUNT/$f; rm $MOUNT/$f"
-    do_nodes $clients "if [ -d $MOUNT2 ]; then mcreate $MOUNT2/$f; rm $MOUNT2/$f; fi"
+        # make sure there will be no seq change
+	local clients=${CLIENTS:-$HOSTNAME}
+	local f=fsa-\\\$\(hostname\)
+	do_nodes $clients "mcreate $MOUNT/$f; rm $MOUNT/$f"
+	do_nodes $clients "if [ -d $MOUNT2 ]; then mcreate $MOUNT2/$f; rm $MOUNT2/$f; fi"
 
-    local svc=${facet}_svc
-    do_facet $facet $LCTL --device %${!svc} notransno
-    do_facet $facet $LCTL --device %${!svc} readonly
-    do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
-    $LCTL mark "local REPLAY BARRIER on ${!svc}"
+	local svc=${facet}_svc
+	do_facet $facet $LCTL --device ${!svc} notransno
+	do_facet $facet $LCTL --device ${!svc} readonly
+	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
+	$LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
 
 replay_barrier_nodf() {
-    local facet=$1    echo running=${running}
-    do_facet $facet "sync; sync; sync"
-    local svc=${facet}_svc
-    echo Replay barrier on ${!svc}
-    do_facet $facet $LCTL --device %${!svc} notransno
-    do_facet $facet $LCTL --device %${!svc} readonly
-    do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
-    $LCTL mark "local REPLAY BARRIER on ${!svc}"
+	local facet=$1    echo running=${running}
+	do_facet $facet "sync; sync; sync"
+	local svc=${facet}_svc
+	echo Replay barrier on ${!svc}
+	do_facet $facet $LCTL --device ${!svc} notransno
+	do_facet $facet $LCTL --device ${!svc} readonly
+	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
+	$LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
 
 replay_barrier_nosync() {
-    local facet=$1    echo running=${running}
-    local svc=${facet}_svc
-    echo Replay barrier on ${!svc}
-    do_facet $facet $LCTL --device %${!svc} notransno
-    do_facet $facet $LCTL --device %${!svc} readonly
-    do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
-    $LCTL mark "local REPLAY BARRIER on ${!svc}"
+	local facet=$1    echo running=${running}
+	local svc=${facet}_svc
+	echo Replay barrier on ${!svc}
+	do_facet $facet $LCTL --device ${!svc} notransno
+	do_facet $facet $LCTL --device ${!svc} readonly
+	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
+	$LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
 
 mds_evict_client() {
@@ -2297,21 +2325,26 @@ hostlist_expand() {
 }
 
 facet_host() {
-    local facet=$1
+	local facet=$1
+	local varname
 
-    [ "$facet" == client ] && echo -n $HOSTNAME && return
-    varname=${facet}_HOST
-    if [ -z "${!varname}" ]; then
-        if [ "${facet:0:3}" == "ost" ]; then
-            eval ${facet}_HOST=${ost_HOST}
-        fi
-    fi
-    echo -n ${!varname}
+	[ "$facet" == client ] && echo -n $HOSTNAME && return
+	varname=${facet}_HOST
+	if [ -z "${!varname}" ]; then
+		if [ "${facet:0:3}" == "ost" ]; then
+			eval export ${facet}_HOST=${ost_HOST}
+		elif [ "${facet:0:3}" == "mdt" -o \
+			"${facet:0:3}" == "mds" -o \
+			"${facet:0:3}" == "mgs" ]; then
+			eval export ${facet}_HOST=${mds_HOST}
+		fi
+	fi
+	echo -n ${!varname}
 }
 
 facet_failover_host() {
 	local facet=$1
-	local var
+	local varname
 
 	var=${facet}failover_HOST
 	if [ -n "${!var}" ]; then
@@ -2319,12 +2352,18 @@ facet_failover_host() {
 		return
 	fi
 
+	if [ "${facet:0:3}" == "mdt" -o "${facet:0:3}" == "mds" -o \
+	     "${facet:0:3}" == "mgs" ]; then
+
+		eval export ${facet}failover_host=${mds_HOST}
+		echo ${mds_HOST}
+		return
+	fi
+
 	if [[ $facet == ost* ]]; then
-		var=ostfailover_HOST
-		if [ -n "${!var}" ]; then
-			echo ${!var}
-			return
-		fi
+		eval export ${facet}failover_host=${ost_HOST}
+		echo ${ost_HOST}
+		return
 	fi
 }
 
@@ -2447,8 +2486,6 @@ get_env_vars() {
 		echo -n " ${var}=\"$value\""
 	done
 
-	echo -n " USE_OFD=$USE_OFD"
-
 	for facet in ${facets//,/ }; do
 		var=${facet}_FSTYPE
 		if [ -n "${!var}" ]; then
@@ -2561,8 +2598,9 @@ ostdevname() {
 			#if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
 			eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};;
 		zfs )
-			#dataset name is independent of vdev device names
-			eval DEVPTR=${FSNAME}-ost${num}/ost${num};;
+			#try $OSTDEVn then $OSTDEVBASE + num then zfs default
+			local foo=${OSTDEVBASE:-${FSNAME}-ost${num}/ost}${num}
+			eval DEVPTR=${!DEVNAME:=$foo};;
 		* )
 			error "unknown fstype!";;
 	esac
@@ -2601,8 +2639,9 @@ mdsdevname() {
 			#if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
 			eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};;
 		zfs )
-			#dataset name is independent of vdev device names
-			eval DEVPTR=${FSNAME}-mdt${num}/mdt${num};;
+			# try $MDSDEVn then $MDSDEVBASE + num then zfs default
+			local foo=${MDSDEVBASE:-${FSNAME}-mdt${num}/mdt}${num}
+			eval DEVPTR=${!DEVNAME:=$foo};;
 		* )
 			error "unknown fstype!";;
 	esac
@@ -2631,17 +2670,16 @@ mdsvdevname() {
 }
 
 mgsdevname() {
-	DEVNAME=MGSDEV
+	local DEVNAME=MGSDEV
+	local MDSDEV1=$(mdsdevname 1)
 
-	local fstype=$(facet_fstype mds$num)
+	local fstype=$(facet_fstype mds1)
 
 	case $fstype in
-		ldiskfs )
+		ldiskfs|zfs )
 			#if $MGSDEV isn't defined, default is $MDSDEV1
+			#ZFS independent mgsdev should be ${FSNAME}-mgs/mgs
 			eval DEVPTR=${!DEVNAME:=${MDSDEV1}};;
-		zfs )
-			#dataset name is independent of vdev device names
-			eval DEVPTR=${FSNAME}-mgs/mgs;;
 		* )
 			error "unknown fstype!";;
 	esac
@@ -2652,7 +2690,7 @@ mgsdevname() {
 mgsvdevname() {
 	DEVNAME=MGSDEV
 
-	local fstype=$(facet_fstype mds$num)
+	local fstype=$(facet_fstype mds1)
 
 	case $fstype in
 		ldiskfs )
@@ -2737,7 +2775,8 @@ cleanupall() {
 }
 
 combined_mgs_mds () {
-    [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]]
+	[[ "$(mdsdevname 1)" = "$(mgsdevname)" ]] &&
+		[[ "$(facet_host mds1)" = "$(facet_host mgs)" ]]
 }
 
 lower() {
@@ -2750,9 +2789,12 @@ upper() {
 
 mkfs_opts() {
 	local facet=$1
+	local dev=$2
+	local fsname=${3:-"$FSNAME"}
 	local type=$(facet_type $facet)
 	local index=$(($(facet_number $facet) - 1))
 	local fstype=$(facet_fstype $facet)
+	local host=$(facet_host $facet)
 	local opts
 	local fs_mkfs_opts
 	local var
@@ -2761,14 +2803,17 @@ mkfs_opts() {
 		return 1
 	fi
 
-	if [ $type == MGS ] || ( [ $type == MDS ] && combined_mgs_mds ); then
+	if [ $type == MGS ] || ( [ $type == MDS ] &&
+                                 [ "$dev" == $(mgsdevname) ] &&
+				 [ "$host" == "$(facet_host mgs)" ] ); then
 		opts="--mgs"
 	else
 		opts="--mgsnode=$MGSNID"
 	fi
 
 	if [ $type != MGS ]; then
-		opts+=" --fsname=$FSNAME --$(lower ${type/MDS/MDT}) --index=$index"
+		opts+=" --fsname=$fsname --$(lower ${type/MDS/MDT}) \
+			--index=$index"
 	fi
 
 	var=${facet}failover_HOST
@@ -2787,8 +2832,6 @@ mkfs_opts() {
 		opts+=${L_GETIDENTITY:+" --param=mdt.identity_upcall=$L_GETIDENTITY"}
 
 		if [ $fstype == ldiskfs ]; then
-			opts+=${IAMDIR:+" --iam-dir"}
-
 			fs_mkfs_opts+=${MDSJOURNALSIZE:+" -J size=$MDSJOURNALSIZE"}
 			fs_mkfs_opts+=${MDSISIZE:+" -i $MDSISIZE"}
 		fi
@@ -2840,23 +2883,24 @@ formatall() {
 	echo Formatting mgs, mds, osts
 	if ! combined_mgs_mds ; then
 		echo "Format mgs: $(mgsdevname)"
-		add mgs $(mkfs_opts mgs) --reformat $(mgsdevname) \
-			$(mgsvdevname) ${quiet:+>/dev/null} || exit 10
-		fi
+		add mgs $(mkfs_opts mgs $(mgsdevname)) --reformat \
+			$(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} ||
+			exit 10
+	fi
 
-		for num in `seq $MDSCOUNT`; do
-			echo "Format mds$num: $(mdsdevname $num)"
-			add mds$num $(mkfs_opts mds$num) --reformat \
-			$(mdsdevname $num) $(mdsvdevname $num) \
+	for num in $(seq $MDSCOUNT); do
+		echo "Format mds$num: $(mdsdevname $num)"
+		add mds$num $(mkfs_opts mds$num $(mdsdevname ${num})) \
+			--reformat $(mdsdevname $num) $(mdsvdevname $num) \
 			${quiet:+>/dev/null} || exit 10
-		done
+	done
 
-		for num in `seq $OSTCOUNT`; do
-			echo "Format ost$num: $(ostdevname $num)"
-			add ost$num $(mkfs_opts ost$num) --reformat \
-			$(ostdevname $num) $(ostvdevname ${num}) \
+	for num in $(seq $OSTCOUNT); do
+		echo "Format ost$num: $(ostdevname $num)"
+		add ost$num $(mkfs_opts ost$num $(ostdevname ${num})) \
+			--reformat $(ostdevname $num) $(ostvdevname ${num}) \
 			${quiet:+>/dev/null} || exit 10
-		done
+	done
 }
 
 mount_client() {
@@ -2904,23 +2948,31 @@ remount_client()
         zconf_mount `hostname` $1 || error "mount failed"
 }
 
-writeconf_facet () {
-    local facet=$1
-    local dev=$2
+writeconf_facet() {
+	local facet=$1
+	local dev=$2
 
-    do_facet $facet "$TUNEFS --writeconf $dev"
+	stop ${facet} -f
+	rm -f ${facet}active
+	do_facet ${facet} "$TUNEFS --quiet --writeconf $dev" || return 1
+	return 0
 }
 
 writeconf_all () {
-    for num in `seq $MDSCOUNT`; do
-        DEVNAME=$(mdsdevname $num)
-        writeconf_facet mds$num $DEVNAME
-    done
+	local mdt_count=${1:-$MDSCOUNT}
+	local ost_count=${2:-$OSTCOUNT}
+	local rc=0
 
-    for num in `seq $OSTCOUNT`; do
-        DEVNAME=$(ostdevname $num)
-        writeconf_facet ost$num $DEVNAME
-    done
+	for num in $(seq $mdt_count); do
+		DEVNAME=$(mdsdevname $num)
+		writeconf_facet mds$num $DEVNAME || rc=$?
+	done
+
+	for num in $(seq $ost_count); do
+		DEVNAME=$(ostdevname $num)
+		writeconf_facet ost$num $DEVNAME || rc=$?
+	done
+	return $rc
 }
 
 setupall() {
@@ -3049,9 +3101,14 @@ init_facet_vars () {
 
 	local varname=${facet}failover_HOST
 	if [ -z "${!varname}" ]; then
-		eval $varname=$(facet_host $facet)
+		eval export $varname=$(facet_host $facet)
 	fi
 
+	varname=${facet}_HOST
+	if [ -z "${!varname}" ]; then
+		eval export $varname=$(facet_host $facet)
+ 	fi
+
 	# ${facet}failover_dev is set in cfg file
 	varname=${facet}failover_dev
 	if [ -n "${!varname}" ] ; then
@@ -3073,23 +3130,26 @@ init_facet_vars () {
 }
 
 init_facets_vars () {
-    local DEVNAME
+	local DEVNAME
 
-    if ! remote_mds_nodsh; then 
-        for num in `seq $MDSCOUNT`; do
-            DEVNAME=`mdsdevname $num`
-            init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
-        done
-    fi
+	if ! remote_mds_nodsh; then
+		for num in $(seq $MDSCOUNT); do
+			DEVNAME=`mdsdevname $num`
+			eval export MDSDEV${num}=$DEVNAME
+			init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
+		done
+	fi
 
+	eval export MGSDEV=$(mgsdevname)
 	combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS
 
-    remote_ost_nodsh && return
-
-    for num in `seq $OSTCOUNT`; do
-        DEVNAME=`ostdevname $num`
-        init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
-    done
+	if ! remote_ost_nodsh; then
+		for num in $(seq $OSTCOUNT); do
+			DEVNAME=$(ostdevname $num)
+			eval export OSTDEV${num}=$DEVNAME
+			init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
+		done
+	fi
 }
 
 osc_ensure_active () {
@@ -3412,7 +3472,7 @@ run_e2fsck() {
     local extra_opts=$3
 
     df > /dev/null      # update statfs data on disk
-    local cmd="$E2FSCK -d -v -t -t -f -n $extra_opts $target_dev"
+    local cmd="$E2FSCK -d -v -t -t -f $extra_opts $target_dev"
     echo $cmd
     local rc=0
     do_node $node $cmd || rc=$?
@@ -3448,14 +3508,14 @@ generate_db() {
 
     [ $MDSCOUNT -eq 1 ] || error "CMD is not supported"
 
-    run_e2fsck $(mdts_nodes) $MDTDEV "--mdsdb $MDSDB"
+    run_e2fsck $(mdts_nodes) $MDTDEV "-n --mdsdb $MDSDB"
 
     i=0
     ostidx=0
     OSTDB_LIST=""
     for node in $(osts_nodes); do
         for dev in ${OSTDEVS[i]}; do
-            run_e2fsck $node $dev "--mdsdb $MDSDB --ostdb $OSTDB-$ostidx"
+            run_e2fsck $node $dev "-n --mdsdb $MDSDB --ostdb $OSTDB-$ostidx"
             OSTDB_LIST="$OSTDB_LIST $OSTDB-$ostidx"
             ostidx=$((ostidx + 1))
         done
@@ -3714,6 +3774,17 @@ drop_reint_reply() {
     return $RC
 }
 
+drop_update_reply() {
+# OBD_FAIL_MDS_OBJ_UPDATE_NET
+	local index=$1
+	shift 1
+	RC=0
+	do_facet mds${index} lctl set_param fail_loc=0x188
+	do_facet client "$@" || RC=$?
+	do_facet mds${index} lctl set_param fail_loc=0
+	return $RC
+}
+
 pause_bulk() {
 #define OBD_FAIL_OST_BRW_PAUSE_BULK      0x214
     RC=0
@@ -3738,11 +3809,11 @@ drop_ldlm_cancel() {
 
 drop_bl_callback() {
 #define OBD_FAIL_LDLM_BL_CALLBACK_NET			0x305
-    RC=0
-    do_facet client lctl set_param fail_loc=0x305
-    do_facet client "$@" || RC=$?
-    do_facet client lctl set_param fail_loc=0
-    return $RC
+	RC=0
+	do_facet client lctl set_param fail_loc=0x80000305
+	do_facet client "$@" || RC=$?
+	do_facet client lctl set_param fail_loc=0
+	return $RC
 }
 
 drop_ldlm_reply() {
@@ -3874,9 +3945,10 @@ error_noexit() {
         gather_logs $(comma_list $(nodes_list))
     fi
 
-    debugrestore
-    [ "$TESTSUITELOG" ] && echo "$0: ${TYPE}: $TESTNAME $@" >> $TESTSUITELOG
-    echo "$@" > $LOGDIR/err
+	debugrestore
+	[ "$TESTSUITELOG" ] &&
+		echo "$TESTSUITE: $TYPE: $TESTNAME $@" >> $TESTSUITELOG
+	echo "$@" > $LOGDIR/err
 }
 
 exit_status () {
@@ -4036,10 +4108,6 @@ run_test() {
     return $?
 }
 
-equals_msg() {
-    banner "$*"
-}
-
 log() {
     echo "$*"
     module_loaded lnet || load_modules
@@ -4066,9 +4134,11 @@ trace() {
 }
 
 complete () {
-    equals_msg $1 test complete, duration $2 sec
+    local duration=$1
+
+    banner test complete, duration $duration sec
     [ -f "$TESTSUITELOG" ] && egrep .FAIL $TESTSUITELOG || true
-    echo duration $2 >>$TESTSUITELOG
+    echo duration $duration >>$TESTSUITELOG
 }
 
 pass() {
@@ -4270,9 +4340,45 @@ mdtuuid_from_index()
     $LFS mdts $2 | sed -ne "/^$1: /s/.* \(.*\) .*$/\1/p"
 }
 
+# Description:
+#   Return unique identifier for given hostname
+host_id() {
+	local host_name=$1
+	echo $host_name | md5sum | cut -d' ' -f1
+}
+
+# Description:
+#   Returns list of ip addresses for each interface
+local_addr_list() {
+	ip addr | awk '/inet\ / {print $2}' | awk -F\/ '{print $1}'
+}
+
+is_local_addr() {
+	local addr=$1
+	# Cache address list to avoid mutiple execution of local_addr_list
+	LOCAL_ADDR_LIST=${LOCAL_ADDR_LIST:-$(local_addr_list)}
+	local i
+	for i in $LOCAL_ADDR_LIST ; do
+		[[ "$i" == "$addr" ]] && return 0
+	done
+	return 1
+}
+
+local_node() {
+	local host_name=$1
+	local is_local="IS_LOCAL_$(host_id $host_name)"
+	if [ -z "${!is_local-}" ] ; then
+		eval $is_local=0
+		local host_ip=$($LUSTRE/tests/resolveip $host_name)
+		is_local_addr "$host_ip" && eval $is_local=1
+	fi
+	[[ "${!is_local}" == "1" ]]
+}
+
 remote_node () {
-    local node=$1
-    [ "$node" != "$(hostname)" ]
+	local node=$1
+	local_node $node && return 1
+	return 0
 }
 
 remote_mds ()
@@ -5563,12 +5669,13 @@ run_llverfs()
 {
         local dir=$1
         local llverfs_opts=$2
+        local use_partial_arg=$3
         local partial_arg=""
         local size=$(df -B G $dir |tail -n 1 |awk '{print $2}' |sed 's/G//') #GB
 
         # Run in partial (fast) mode if the size
         # of a partition > 1 GB
-        [ $size -gt 1 ] && partial_arg="-p"
+        [ "x$use_partial_arg" != "xno" ] && [ $size -gt 1 ] && partial_arg="-p"
 
         llverfs $partial_arg $llverfs_opts $dir
 }
@@ -5689,10 +5796,12 @@ get_block_size() {
 
 # Check whether the "large_xattr" feature is enabled or not.
 large_xattr_enabled() {
-    local mds_dev=$(mdsdevname ${SINGLEMDS//mds/})
+	[[ $(facet_fstype $SINGLEMDS) == zfs ]] && return 0
 
-    do_facet $SINGLEMDS "$DUMPE2FS -h $mds_dev 2>&1 | grep -q large_xattr"
-    return ${PIPESTATUS[0]}
+	local mds_dev=$(mdsdevname ${SINGLEMDS//mds/})
+
+	do_facet $SINGLEMDS "$DUMPE2FS -h $mds_dev 2>&1 | grep -q large_xattr"
+	return ${PIPESTATUS[0]}
 }
 
 # Get the maximum xattr size supported by the filesystem.
@@ -5746,6 +5855,7 @@ mds_backup_restore() {
 	local metaea=${TMP}/backup_restore.ea
 	local metadata=${TMP}/backup_restore.tgz
 	local opts=${MDS_MOUNT_OPTS}
+	local svc=${SINGLEMDS}_svc
 
 	if ! ${rcmd} test -b ${devname}; then
 		opts=$(csa_add "$opts" -o loop)
@@ -5789,6 +5899,8 @@ mds_backup_restore() {
 	${rcmd} umount -d $mntpt || return 10
 	# step 14: cleanup tmp backup
 	${rcmd} rm -f $metaea $metadata
+	# step 15: reset device label - it's not virgin on
+	${rcmd} e2label $devname ${!svc}
 }
 
 # remove OI files
@@ -5834,3 +5946,64 @@ generate_logname() {
 
 	echo "$TESTLOG_PREFIX.$TESTNAME.$logname.$(hostname -s).log"
 }
+
+# mkdir directory on different MDTs
+test_mkdir() {
+	local option
+	local parent
+	local child
+	local path
+	local dir
+	local rc=0
+
+	if [ $# -eq 2 ]; then
+		option=$1
+		path=$2
+	else
+		path=$1
+	fi
+
+	child=${path##*/}
+	parent=${path%/*}
+
+	if [ "$parent" == "$child" ]; then
+		parent=$(pwd)
+	fi
+
+	if [ "$option" == "-p" -a -d ${parent}/${child} ]; then
+		return $rc
+	fi
+
+	# it needs to check whether there is further / in child
+	dir=$(echo $child | awk -F '/' '{print $2}')
+	if [ ! -z "$dir" ]; then
+		local subparent=$(echo $child | awk -F '/' '{ print $1 }')
+		parent=${parent}"/"${subparent}
+		child=$dir
+	fi
+
+	if [ ! -d ${parent} ]; then
+		if [ "$option" == "-p" ]; then
+			mkdir -p ${parent}
+		else
+			return 1
+		fi
+	fi
+
+	if [ $MDSCOUNT -le 1 ]; then
+		mkdir $option ${parent}/${child} || rc=$?
+	else
+		local mdt_idx=$($LFS getstripe -M $parent)
+
+		if [ "$mdt_idx" -ne 0 ]; then
+			mkdir $option ${parent}/${child} || rc=$?
+			return $rc
+		fi
+
+		local test_num=$(echo $testnum | sed -e 's/[^0-9]*//g')
+		local mdt_idx=$((test_num % MDSCOUNT))
+		echo "mkdir $mdt_idx for ${parent}/${child}"
+		$LFS setdirstripe -i $mdt_idx ${parent}/${child} || rc=$?
+	fi
+	return $rc
+}