X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;ds=sidebyside;f=lustre%2Ftests%2Ftest-framework.sh;h=2fa00075a68575adb7e31012a2786df656128d07;hb=d25e7654f798060f6b671c3b65949a70b8c67f58;hp=521a58a8c088a4c99ff41a288ed5c2f59669b4d5;hpb=6f2355435eca7541ee7b4804930ac34d580f71c9;p=fs%2Flustre-release.git

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 521a58a..2fa0007 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -282,8 +282,6 @@ init_test_env() {
             IDENTITY_UPCALL=false
             ;;
     esac
-    USE_OFD=${USE_OFD:-yes}
-    [ "$USE_OFD" = "yes" ] && LOAD_MODULES_REMOTE=true
 
     export LOAD_MODULES_REMOTE=${LOAD_MODULES_REMOTE:-false}
 
@@ -338,7 +336,7 @@ version_code() {
     # split arguments like "1.8.6-wc3" into "1", "8", "6", "wc3"
     eval set -- $(tr "[:punct:]" " " <<< $*)
 
-    echo -n $((($1 << 16) | ($2 << 8) | $3))
+    echo -n "$((($1 << 16) | ($2 << 8) | $3))"
 }
 
 export LINUX_VERSION=$(uname -r | sed -e "s/[-.]/ /3" -e "s/ .*//")
@@ -373,20 +371,24 @@ load_module() {
         optvar="MODOPTS_$(basename $module | tr a-z A-Z)"
         eval set -- \$$optvar
         if [ $# -eq 0 -a -n "$MODPROBECONF" ]; then
-            # Nothing in $MODOPTS_<MODULE>; try modprobe.conf
-            set -- $(grep -P "^options\\s+${BASE}" $MODPROBECONF)
-            # Get rid of "options $module"
-            (($# > 0)) && shift 2
-
-            # Ensure we have accept=all for lnet
-            if [ $(basename $module) = lnet ]; then
-                # OK, this is a bit wordy...
-                local arg accept_all_present=false
-                for arg in "$@"; do
-                    [ "$arg" = accept=all ] && accept_all_present=true
-                done
-                $accept_all_present || set -- "$@" accept=all
-            fi
+		# Nothing in $MODOPTS_<MODULE>; try modprobe.conf
+		local opt
+		opt=$(awk -v var="^options $BASE" '$0 ~ var \
+			{gsub("'"options $BASE"'",""); print}' $MODPROBECONF)
+		set -- $(echo -n $opt)
+
+		# Ensure we have accept=all for lnet
+		if [ $(basename $module) = lnet ]; then
+			# OK, this is a bit wordy...
+			local arg accept_all_present=false
+
+			for arg in "$@"; do
+				[ "$arg" = accept=all ] && \
+					accept_all_present=true
+			done
+			$accept_all_present || set -- "$@" accept=all
+		fi
+		export $optvar="$*"
         fi
     fi
 
@@ -425,8 +427,6 @@ llite_lloop_enabled() {
 }
 
 load_modules_local() {
-	[ $(facet_fstype ost1) == "zfs" ] && export USE_OFD=yes
-
 	if [ -n "$MODPROBE" ]; then
 		# use modprobe
 		echo "Using modprobe to load modules"
@@ -697,7 +697,7 @@ cleanup_gss() {
 facet_type() {
 	local facet=$1
 
-	echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9]\+//' |
+	echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9_]\+//' |
 		tr '[:lower:]' '[:upper:]'
 }
 
@@ -1092,11 +1092,11 @@ setup_quota_old(){
     local quota_usrs=$QUOTA_USERS
 
     # get_filesystem_size
-    local disksz=$(lfs df $mntpt | grep "filesystem summary:"  | awk '{print $3}')
+    local disksz=$(lfs_df $mntpt | grep "summary"  | awk '{print $2}')
     local blk_soft=$((disksz + 1024))
     local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over
 
-    local Inodes=$(lfs df -i $mntpt | grep "filesystem summary:"  | awk '{print $3}')
+    local Inodes=$(lfs_df -i $mntpt | grep "summary"  | awk '{print $2}')
     local i_soft=$Inodes
     local i_hard=$((i_soft + i_soft / 20))
 
@@ -1149,6 +1149,13 @@ restore_quota() {
 	fi
 }
 
+# Handle the case when there is a space in the lfs df
+# "filesystem summary" line the same as when there is no space.
+# This will allow fixing the "lfs df" summary line in the future.
+lfs_df() {
+	$LFS df $* | sed -e 's/filesystem /filesystem_/'
+}
+
 setup_quota(){
 	if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then
 		setup_quota_old $1
@@ -1175,13 +1182,11 @@ setup_quota(){
 	local quota_usrs=$QUOTA_USERS
 
 	# get_filesystem_size
-	local disksz=$(lfs df $mntpt | grep "filesystem summary:" |
-		     awk '{print $3}')
+	local disksz=$(lfs_df $mntpt | grep "summary" | awk '{print $2}')
 	local blk_soft=$((disksz + 1024))
 	local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over
 
-	local inodes=$(lfs df -i $mntpt | grep "filesystem summary:" |
-		     awk '{print $3}')
+	local inodes=$(lfs_df -i $mntpt | grep "summary" | awk '{print $2}')
 	local i_soft=$inodes
 	local i_hard=$((i_soft + i_soft / 20))
 
@@ -2045,32 +2050,58 @@ affected_facets () {
 }
 
 facet_failover() {
-    local facet=$1
-    local sleep_time=$2
-    local host=$(facet_active_host $facet)
-
-    echo "Failing $facet on node $host"
-
-    local affected=$(affected_facets $facet)
+	local facets=$1
+	local sleep_time=$2
+	local -a affecteds
+	local facet
+	local total=0
+	local index=0
+	local skip
+
+	#Because it will only get up facets, we need get affected
+	#facets before shutdown
+	#For HARD Failure mode, it needs make sure facets on the same
+	#HOST will only be shutdown and reboot once
+	for facet in ${facets//,/ }; do
+		local affected_facet
+		skip=0
+		#check whether facet has been included in other affected facets
+		for ((index=0; index<$total; index++)); do
+			[[ *,$facet,* == ,${affecteds[index]}, ]] && skip=1
+		done
 
-    shutdown_facet $facet
+		if [ $skip -eq 0 ]; then
+			affecteds[$total]=$(affected_facets $facet)
+			total=$((total+1))
+		fi
+	done
 
-    echo affected facets: $affected
+	for ((index=0; index<$total; index++)); do
+		facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
+		local host=$(facet_active_host $facet)
+		echo "Failing ${affecteds[index]} on $host"
+		shutdown_facet $facet
+	done
 
-    [ -n "$sleep_time" ] && sleep $sleep_time
+	for ((index=0; index<$total; index++)); do
+		facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
+		echo reboot facets: ${affecteds[index]}
 
-    reboot_facet $facet
+		reboot_facet $facet
 
-    change_active $affected
+		change_active ${affecteds[index]}
 
-    wait_for_facet $affected
-    # start mgs first if it is affected
-    if ! combined_mgs_mds && list_member $affected mgs; then
-        mount_facet mgs || error "Restart of mgs failed"
-    fi
-    # FIXME; has to be changed to mount all facets concurrently
-    affected=$(exclude_items_from_list $affected mgs)
-    mount_facets $affected
+		wait_for_facet ${affecteds[index]}
+		# start mgs first if it is affected
+		if ! combined_mgs_mds &&
+			list_member ${affecteds[index]} mgs; then
+			mount_facet mgs || error "Restart of mgs failed"
+		fi
+		# FIXME; has to be changed to mount all facets concurrently
+		affected=$(exclude_items_from_list ${affecteds[index]} mgs)
+		echo mount facets: ${affecteds[index]}
+		mount_facets ${affecteds[index]}
+	done
 }
 
 obd_name() {
@@ -2078,42 +2109,42 @@ obd_name() {
 }
 
 replay_barrier() {
-    local facet=$1
-    do_facet $facet "sync; sync; sync"
-    df $MOUNT
+	local facet=$1
+	do_facet $facet "sync; sync; sync"
+	df $MOUNT
 
-    # make sure there will be no seq change
-    local clients=${CLIENTS:-$HOSTNAME}
-    local f=fsa-\\\$\(hostname\)
-    do_nodes $clients "mcreate $MOUNT/$f; rm $MOUNT/$f"
-    do_nodes $clients "if [ -d $MOUNT2 ]; then mcreate $MOUNT2/$f; rm $MOUNT2/$f; fi"
+        # make sure there will be no seq change
+	local clients=${CLIENTS:-$HOSTNAME}
+	local f=fsa-\\\$\(hostname\)
+	do_nodes $clients "mcreate $MOUNT/$f; rm $MOUNT/$f"
+	do_nodes $clients "if [ -d $MOUNT2 ]; then mcreate $MOUNT2/$f; rm $MOUNT2/$f; fi"
 
-    local svc=${facet}_svc
-    do_facet $facet $LCTL --device %${!svc} notransno
-    do_facet $facet $LCTL --device %${!svc} readonly
-    do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
-    $LCTL mark "local REPLAY BARRIER on ${!svc}"
+	local svc=${facet}_svc
+	do_facet $facet $LCTL --device ${!svc} notransno
+	do_facet $facet $LCTL --device ${!svc} readonly
+	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
+	$LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
 
 replay_barrier_nodf() {
-    local facet=$1    echo running=${running}
-    do_facet $facet "sync; sync; sync"
-    local svc=${facet}_svc
-    echo Replay barrier on ${!svc}
-    do_facet $facet $LCTL --device %${!svc} notransno
-    do_facet $facet $LCTL --device %${!svc} readonly
-    do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
-    $LCTL mark "local REPLAY BARRIER on ${!svc}"
+	local facet=$1    echo running=${running}
+	do_facet $facet "sync; sync; sync"
+	local svc=${facet}_svc
+	echo Replay barrier on ${!svc}
+	do_facet $facet $LCTL --device ${!svc} notransno
+	do_facet $facet $LCTL --device ${!svc} readonly
+	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
+	$LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
 
 replay_barrier_nosync() {
-    local facet=$1    echo running=${running}
-    local svc=${facet}_svc
-    echo Replay barrier on ${!svc}
-    do_facet $facet $LCTL --device %${!svc} notransno
-    do_facet $facet $LCTL --device %${!svc} readonly
-    do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
-    $LCTL mark "local REPLAY BARRIER on ${!svc}"
+	local facet=$1    echo running=${running}
+	local svc=${facet}_svc
+	echo Replay barrier on ${!svc}
+	do_facet $facet $LCTL --device ${!svc} notransno
+	do_facet $facet $LCTL --device ${!svc} readonly
+	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
+	$LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
 
 mds_evict_client() {
@@ -2294,21 +2325,26 @@ hostlist_expand() {
 }
 
 facet_host() {
-    local facet=$1
+	local facet=$1
+	local varname
 
-    [ "$facet" == client ] && echo -n $HOSTNAME && return
-    varname=${facet}_HOST
-    if [ -z "${!varname}" ]; then
-        if [ "${facet:0:3}" == "ost" ]; then
-            eval ${facet}_HOST=${ost_HOST}
-        fi
-    fi
-    echo -n ${!varname}
+	[ "$facet" == client ] && echo -n $HOSTNAME && return
+	varname=${facet}_HOST
+	if [ -z "${!varname}" ]; then
+		if [ "${facet:0:3}" == "ost" ]; then
+			eval export ${facet}_HOST=${ost_HOST}
+		elif [ "${facet:0:3}" == "mdt" -o \
+			"${facet:0:3}" == "mds" -o \
+			"${facet:0:3}" == "mgs" ]; then
+			eval export ${facet}_HOST=${mds_HOST}
+		fi
+	fi
+	echo -n ${!varname}
 }
 
 facet_failover_host() {
 	local facet=$1
-	local var
+	local varname
 
 	var=${facet}failover_HOST
 	if [ -n "${!var}" ]; then
@@ -2316,12 +2352,18 @@ facet_failover_host() {
 		return
 	fi
 
+	if [ "${facet:0:3}" == "mdt" -o "${facet:0:3}" == "mds" -o \
+	     "${facet:0:3}" == "mgs" ]; then
+
+		eval export ${facet}failover_host=${mds_HOST}
+		echo ${mds_HOST}
+		return
+	fi
+
 	if [[ $facet == ost* ]]; then
-		var=ostfailover_HOST
-		if [ -n "${!var}" ]; then
-			echo ${!var}
-			return
-		fi
+		eval export ${facet}failover_host=${ost_HOST}
+		echo ${ost_HOST}
+		return
 	fi
 }
 
@@ -2444,8 +2486,6 @@ get_env_vars() {
 		echo -n " ${var}=\"$value\""
 	done
 
-	echo -n " USE_OFD=$USE_OFD"
-
 	for facet in ${facets//,/ }; do
 		var=${facet}_FSTYPE
 		if [ -n "${!var}" ]; then
@@ -2558,8 +2598,9 @@ ostdevname() {
 			#if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
 			eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};;
 		zfs )
-			#dataset name is independent of vdev device names
-			eval DEVPTR=${FSNAME}-ost${num}/ost${num};;
+			#try $OSTDEVn then $OSTDEVBASE + num then zfs default
+			local foo=${OSTDEVBASE:-${FSNAME}-ost${num}/ost}${num}
+			eval DEVPTR=${!DEVNAME:=$foo};;
 		* )
 			error "unknown fstype!";;
 	esac
@@ -2598,8 +2639,9 @@ mdsdevname() {
 			#if $MDSDEVn isn't defined, default is $MDSDEVBASE + num
 			eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};;
 		zfs )
-			#dataset name is independent of vdev device names
-			eval DEVPTR=${FSNAME}-mdt${num}/mdt${num};;
+			# try $MDSDEVn then $MDSDEVBASE + num then zfs default
+			local foo=${MDSDEVBASE:-${FSNAME}-mdt${num}/mdt}${num}
+			eval DEVPTR=${!DEVNAME:=$foo};;
 		* )
 			error "unknown fstype!";;
 	esac
@@ -2628,17 +2670,16 @@ mdsvdevname() {
 }
 
 mgsdevname() {
-	DEVNAME=MGSDEV
+	local DEVNAME=MGSDEV
+	local MDSDEV1=$(mdsdevname 1)
 
-	local fstype=$(facet_fstype mds$num)
+	local fstype=$(facet_fstype mds1)
 
 	case $fstype in
-		ldiskfs )
+		ldiskfs|zfs )
 			#if $MGSDEV isn't defined, default is $MDSDEV1
+			#ZFS independent mgsdev should be ${FSNAME}-mgs/mgs
 			eval DEVPTR=${!DEVNAME:=${MDSDEV1}};;
-		zfs )
-			#dataset name is independent of vdev device names
-			eval DEVPTR=${FSNAME}-mgs/mgs;;
 		* )
 			error "unknown fstype!";;
 	esac
@@ -2649,7 +2690,7 @@ mgsdevname() {
 mgsvdevname() {
 	DEVNAME=MGSDEV
 
-	local fstype=$(facet_fstype mds$num)
+	local fstype=$(facet_fstype mds1)
 
 	case $fstype in
 		ldiskfs )
@@ -2734,7 +2775,8 @@ cleanupall() {
 }
 
 combined_mgs_mds () {
-    [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]]
+	[[ "$(mdsdevname 1)" = "$(mgsdevname)" ]] &&
+		[[ "$(facet_host mds1)" = "$(facet_host mgs)" ]]
 }
 
 lower() {
@@ -2747,9 +2789,12 @@ upper() {
 
 mkfs_opts() {
 	local facet=$1
+	local dev=$2
+	local fsname=${3:-"$FSNAME"}
 	local type=$(facet_type $facet)
 	local index=$(($(facet_number $facet) - 1))
 	local fstype=$(facet_fstype $facet)
+	local host=$(facet_host $facet)
 	local opts
 	local fs_mkfs_opts
 	local var
@@ -2758,14 +2803,17 @@ mkfs_opts() {
 		return 1
 	fi
 
-	if [ $type == MGS ] || ( [ $type == MDS ] && combined_mgs_mds ); then
+	if [ $type == MGS ] || ( [ $type == MDS ] &&
+                                 [ "$dev" == $(mgsdevname) ] &&
+				 [ "$host" == "$(facet_host mgs)" ] ); then
 		opts="--mgs"
 	else
 		opts="--mgsnode=$MGSNID"
 	fi
 
 	if [ $type != MGS ]; then
-		opts+=" --fsname=$FSNAME --$(lower ${type/MDS/MDT}) --index=$index"
+		opts+=" --fsname=$fsname --$(lower ${type/MDS/MDT}) \
+			--index=$index"
 	fi
 
 	var=${facet}failover_HOST
@@ -2835,23 +2883,24 @@ formatall() {
 	echo Formatting mgs, mds, osts
 	if ! combined_mgs_mds ; then
 		echo "Format mgs: $(mgsdevname)"
-		add mgs $(mkfs_opts mgs) --reformat $(mgsdevname) \
-			$(mgsvdevname) ${quiet:+>/dev/null} || exit 10
-		fi
+		add mgs $(mkfs_opts mgs $(mgsdevname)) --reformat \
+			$(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} ||
+			exit 10
+	fi
 
-		for num in `seq $MDSCOUNT`; do
-			echo "Format mds$num: $(mdsdevname $num)"
-			add mds$num $(mkfs_opts mds$num) --reformat \
-			$(mdsdevname $num) $(mdsvdevname $num) \
+	for num in $(seq $MDSCOUNT); do
+		echo "Format mds$num: $(mdsdevname $num)"
+		add mds$num $(mkfs_opts mds$num $(mdsdevname ${num})) \
+			--reformat $(mdsdevname $num) $(mdsvdevname $num) \
 			${quiet:+>/dev/null} || exit 10
-		done
+	done
 
-		for num in `seq $OSTCOUNT`; do
-			echo "Format ost$num: $(ostdevname $num)"
-			add ost$num $(mkfs_opts ost$num) --reformat \
-			$(ostdevname $num) $(ostvdevname ${num}) \
+	for num in $(seq $OSTCOUNT); do
+		echo "Format ost$num: $(ostdevname $num)"
+		add ost$num $(mkfs_opts ost$num $(ostdevname ${num})) \
+			--reformat $(ostdevname $num) $(ostvdevname ${num}) \
 			${quiet:+>/dev/null} || exit 10
-		done
+	done
 }
 
 mount_client() {
@@ -2899,23 +2948,31 @@ remount_client()
         zconf_mount `hostname` $1 || error "mount failed"
 }
 
-writeconf_facet () {
-    local facet=$1
-    local dev=$2
+writeconf_facet() {
+	local facet=$1
+	local dev=$2
 
-    do_facet $facet "$TUNEFS --writeconf $dev"
+	stop ${facet} -f
+	rm -f ${facet}active
+	do_facet ${facet} "$TUNEFS --quiet --writeconf $dev" || return 1
+	return 0
 }
 
 writeconf_all () {
-    for num in `seq $MDSCOUNT`; do
-        DEVNAME=$(mdsdevname $num)
-        writeconf_facet mds$num $DEVNAME
-    done
+	local mdt_count=${1:-$MDSCOUNT}
+	local ost_count=${2:-$OSTCOUNT}
+	local rc=0
 
-    for num in `seq $OSTCOUNT`; do
-        DEVNAME=$(ostdevname $num)
-        writeconf_facet ost$num $DEVNAME
-    done
+	for num in $(seq $mdt_count); do
+		DEVNAME=$(mdsdevname $num)
+		writeconf_facet mds$num $DEVNAME || rc=$?
+	done
+
+	for num in $(seq $ost_count); do
+		DEVNAME=$(ostdevname $num)
+		writeconf_facet ost$num $DEVNAME || rc=$?
+	done
+	return $rc
 }
 
 setupall() {
@@ -3044,9 +3101,14 @@ init_facet_vars () {
 
 	local varname=${facet}failover_HOST
 	if [ -z "${!varname}" ]; then
-		eval $varname=$(facet_host $facet)
+		eval export $varname=$(facet_host $facet)
 	fi
 
+	varname=${facet}_HOST
+	if [ -z "${!varname}" ]; then
+		eval export $varname=$(facet_host $facet)
+ 	fi
+
 	# ${facet}failover_dev is set in cfg file
 	varname=${facet}failover_dev
 	if [ -n "${!varname}" ] ; then
@@ -3068,23 +3130,26 @@ init_facet_vars () {
 }
 
 init_facets_vars () {
-    local DEVNAME
+	local DEVNAME
 
-    if ! remote_mds_nodsh; then 
-        for num in `seq $MDSCOUNT`; do
-            DEVNAME=`mdsdevname $num`
-            init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
-        done
-    fi
+	if ! remote_mds_nodsh; then
+		for num in $(seq $MDSCOUNT); do
+			DEVNAME=`mdsdevname $num`
+			eval export MDSDEV${num}=$DEVNAME
+			init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
+		done
+	fi
 
+	eval export MGSDEV=$(mgsdevname)
 	combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS
 
-    remote_ost_nodsh && return
-
-    for num in `seq $OSTCOUNT`; do
-        DEVNAME=`ostdevname $num`
-        init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
-    done
+	if ! remote_ost_nodsh; then
+		for num in $(seq $OSTCOUNT); do
+			DEVNAME=$(ostdevname $num)
+			eval export OSTDEV${num}=$DEVNAME
+			init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
+		done
+	fi
 }
 
 osc_ensure_active () {
@@ -3709,6 +3774,17 @@ drop_reint_reply() {
     return $RC
 }
 
+drop_update_reply() {
+# OBD_FAIL_MDS_OBJ_UPDATE_NET
+	local index=$1
+	shift 1
+	RC=0
+	do_facet mds${index} lctl set_param fail_loc=0x188
+	do_facet client "$@" || RC=$?
+	do_facet mds${index} lctl set_param fail_loc=0
+	return $RC
+}
+
 pause_bulk() {
 #define OBD_FAIL_OST_BRW_PAUSE_BULK      0x214
     RC=0
@@ -4264,9 +4340,45 @@ mdtuuid_from_index()
     $LFS mdts $2 | sed -ne "/^$1: /s/.* \(.*\) .*$/\1/p"
 }
 
+# Description:
+#   Return unique identifier for given hostname
+host_id() {
+	local host_name=$1
+	echo $host_name | md5sum | cut -d' ' -f1
+}
+
+# Description:
+#   Returns list of ip addresses for each interface
+local_addr_list() {
+	ip addr | awk '/inet\ / {print $2}' | awk -F\/ '{print $1}'
+}
+
+is_local_addr() {
+	local addr=$1
+	# Cache address list to avoid mutiple execution of local_addr_list
+	LOCAL_ADDR_LIST=${LOCAL_ADDR_LIST:-$(local_addr_list)}
+	local i
+	for i in $LOCAL_ADDR_LIST ; do
+		[[ "$i" == "$addr" ]] && return 0
+	done
+	return 1
+}
+
+local_node() {
+	local host_name=$1
+	local is_local="IS_LOCAL_$(host_id $host_name)"
+	if [ -z "${!is_local-}" ] ; then
+		eval $is_local=0
+		local host_ip=$($LUSTRE/tests/resolveip $host_name)
+		is_local_addr "$host_ip" && eval $is_local=1
+	fi
+	[[ "${!is_local}" == "1" ]]
+}
+
 remote_node () {
-    local node=$1
-    [ "$node" != "$(hostname)" ]
+	local node=$1
+	local_node $node && return 1
+	return 0
 }
 
 remote_mds ()
@@ -5834,3 +5946,64 @@ generate_logname() {
 
 	echo "$TESTLOG_PREFIX.$TESTNAME.$logname.$(hostname -s).log"
 }
+
+# mkdir directory on different MDTs
+test_mkdir() {
+	local option
+	local parent
+	local child
+	local path
+	local dir
+	local rc=0
+
+	if [ $# -eq 2 ]; then
+		option=$1
+		path=$2
+	else
+		path=$1
+	fi
+
+	child=${path##*/}
+	parent=${path%/*}
+
+	if [ "$parent" == "$child" ]; then
+		parent=$(pwd)
+	fi
+
+	if [ "$option" == "-p" -a -d ${parent}/${child} ]; then
+		return $rc
+	fi
+
+	# it needs to check whether there is further / in child
+	dir=$(echo $child | awk -F '/' '{print $2}')
+	if [ ! -z "$dir" ]; then
+		local subparent=$(echo $child | awk -F '/' '{ print $1 }')
+		parent=${parent}"/"${subparent}
+		child=$dir
+	fi
+
+	if [ ! -d ${parent} ]; then
+		if [ "$option" == "-p" ]; then
+			mkdir -p ${parent}
+		else
+			return 1
+		fi
+	fi
+
+	if [ $MDSCOUNT -le 1 ]; then
+		mkdir $option ${parent}/${child} || rc=$?
+	else
+		local mdt_idx=$($LFS getstripe -M $parent)
+
+		if [ "$mdt_idx" -ne 0 ]; then
+			mkdir $option ${parent}/${child} || rc=$?
+			return $rc
+		fi
+
+		local test_num=$(echo $testnum | sed -e 's/[^0-9]*//g')
+		local mdt_idx=$((test_num % MDSCOUNT))
+		echo "mkdir $mdt_idx for ${parent}/${child}"
+		$LFS setdirstripe -i $mdt_idx ${parent}/${child} || rc=$?
+	fi
+	return $rc
+}