Whamcloud - gitweb
Revert "LU-2469 tests: use MDS/OSTDEV and MDS/OSTDEVBASE with zfs"
[fs/lustre-release.git] / lustre / tests / test-framework.sh
index 7baca91..8a08dff 100644 (file)
@@ -282,8 +282,6 @@ init_test_env() {
             IDENTITY_UPCALL=false
             ;;
     esac
-    USE_OFD=${USE_OFD:-yes}
-    [ "$USE_OFD" = "yes" ] && LOAD_MODULES_REMOTE=true
 
     export LOAD_MODULES_REMOTE=${LOAD_MODULES_REMOTE:-false}
 
@@ -338,7 +336,7 @@ version_code() {
     # split arguments like "1.8.6-wc3" into "1", "8", "6", "wc3"
     eval set -- $(tr "[:punct:]" " " <<< $*)
 
-    echo -n $((($1 << 16) | ($2 << 8) | $3))
+    echo -n "$((($1 << 16) | ($2 << 8) | $3))"
 }
 
 export LINUX_VERSION=$(uname -r | sed -e "s/[-.]/ /3" -e "s/ .*//")
@@ -373,20 +371,24 @@ load_module() {
         optvar="MODOPTS_$(basename $module | tr a-z A-Z)"
         eval set -- \$$optvar
         if [ $# -eq 0 -a -n "$MODPROBECONF" ]; then
-            # Nothing in $MODOPTS_<MODULE>; try modprobe.conf
-            set -- $(grep -P "^options\\s+${BASE}" $MODPROBECONF)
-            # Get rid of "options $module"
-            (($# > 0)) && shift 2
-
-            # Ensure we have accept=all for lnet
-            if [ $(basename $module) = lnet ]; then
-                # OK, this is a bit wordy...
-                local arg accept_all_present=false
-                for arg in "$@"; do
-                    [ "$arg" = accept=all ] && accept_all_present=true
-                done
-                $accept_all_present || set -- "$@" accept=all
-            fi
+               # Nothing in $MODOPTS_<MODULE>; try modprobe.conf
+               local opt
+               opt=$(awk -v var="^options $BASE" '$0 ~ var \
+                       {gsub("'"options $BASE"'",""); print}' $MODPROBECONF)
+               set -- $(echo -n $opt)
+
+               # Ensure we have accept=all for lnet
+               if [ $(basename $module) = lnet ]; then
+                       # OK, this is a bit wordy...
+                       local arg accept_all_present=false
+
+                       for arg in "$@"; do
+                               [ "$arg" = accept=all ] && \
+                                       accept_all_present=true
+                       done
+                       $accept_all_present || set -- "$@" accept=all
+               fi
+               export $optvar="$*"
         fi
     fi
 
@@ -425,8 +427,6 @@ llite_lloop_enabled() {
 }
 
 load_modules_local() {
-       [ $(facet_fstype ost1) == "zfs" ] && export USE_OFD=yes
-
        if [ -n "$MODPROBE" ]; then
                # use modprobe
                echo "Using modprobe to load modules"
@@ -697,7 +697,7 @@ cleanup_gss() {
 facet_type() {
        local facet=$1
 
-       echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9]\+//' |
+       echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9_]\+//' |
                tr '[:lower:]' '[:upper:]'
 }
 
@@ -1043,13 +1043,14 @@ quota_save_version() {
 # XXX This function is kept for interoperability with old server (< 2.3.50),
 #     it should be removed whenever we drop the interoperability for such
 #     server.
-quota_type () {
-    local fsname=${1:-$FSNAME}
-    local rc=0
-    do_facet mgs lctl get_param mdd.${fsname}-MDT*.quota_type || rc=$?
-    do_nodes $(comma_list $(osts_nodes)) \
-        lctl get_param obdfilter.${fsname}-OST*.quota_type || rc=$?
-    return $rc 
+quota_type() {
+       local fsname=${1:-$FSNAME}
+       local rc=0
+       do_facet $SINGLEMDS lctl get_param mdd.${fsname}-MDT*.quota_type ||
+               rc=$?
+       do_nodes $(comma_list $(osts_nodes)) \
+               lctl get_param obdfilter.${fsname}-OST*.quota_type || rc=$?
+       return $rc 
 }
 
 # XXX This function is kept for interoperability with old server (< 2.3.50),
@@ -1091,11 +1092,11 @@ setup_quota_old(){
     local quota_usrs=$QUOTA_USERS
 
     # get_filesystem_size
-    local disksz=$(lfs df $mntpt | grep "filesystem summary:"  | awk '{print $3}')
+    local disksz=$(lfs_df $mntpt | grep "summary"  | awk '{print $2}')
     local blk_soft=$((disksz + 1024))
     local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over
 
-    local Inodes=$(lfs df -i $mntpt | grep "filesystem summary:"  | awk '{print $3}')
+    local Inodes=$(lfs_df -i $mntpt | grep "summary"  | awk '{print $2}')
     local i_soft=$Inodes
     local i_hard=$((i_soft + i_soft / 20))
 
@@ -1120,7 +1121,7 @@ setup_quota_old(){
 mdt_quota_type() {
        local varsvc=${SINGLEMDS}_svc
        do_facet $SINGLEMDS $LCTL get_param -n \
-               osd-$FSTYPE.${!varsvc}.quota_slave.enabled
+               osd-$(facet_fstype $SINGLEMDS).${!varsvc}.quota_slave.enabled
 }
 
 # get ost quota type
@@ -1128,7 +1129,7 @@ ost_quota_type() {
        # All OSTs should have same quota type
        local varsvc=ost1_svc
        do_facet ost1 $LCTL get_param -n \
-               osd-$FSTYPE.${!varsvc}.quota_slave.enabled
+               osd-$(facet_fstype ost1).${!varsvc}.quota_slave.enabled
 }
 
 # restore old quota type settings
@@ -1148,6 +1149,13 @@ restore_quota() {
        fi
 }
 
+# Handle the case when there is a space in the lfs df
+# "filesystem summary" line the same as when there is no space.
+# This will allow fixing the "lfs df" summary line in the future.
+lfs_df() {
+       $LFS df $* | sed -e 's/filesystem /filesystem_/'
+}
+
 setup_quota(){
        if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then
                setup_quota_old $1
@@ -1174,13 +1182,11 @@ setup_quota(){
        local quota_usrs=$QUOTA_USERS
 
        # get_filesystem_size
-       local disksz=$(lfs df $mntpt | grep "filesystem summary:" |
-                    awk '{print $3}')
+       local disksz=$(lfs_df $mntpt | grep "summary" | awk '{print $2}')
        local blk_soft=$((disksz + 1024))
        local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over
 
-       local inodes=$(lfs df -i $mntpt | grep "filesystem summary:" |
-                    awk '{print $3}')
+       local inodes=$(lfs_df -i $mntpt | grep "summary" | awk '{print $2}')
        local i_soft=$inodes
        local i_hard=$((i_soft + i_soft / 20))
 
@@ -1525,6 +1531,7 @@ TESTLOG_PREFIX=$TESTLOG_PREFIX \
 TESTNAME=$TESTNAME \
 DBENCH_LIB=$DBENCH_LIB \
 DBENCH_SRC=$DBENCH_SRC \
+LFS=$LFS \
 run_${load}.sh" &
     local ppid=$!
     log "Started client load: ${load} on $client"
@@ -2044,32 +2051,58 @@ affected_facets () {
 }
 
 facet_failover() {
-    local facet=$1
-    local sleep_time=$2
-    local host=$(facet_active_host $facet)
-
-    echo "Failing $facet on node $host"
-
-    local affected=$(affected_facets $facet)
+       local facets=$1
+       local sleep_time=$2
+       local -a affecteds
+       local facet
+       local total=0
+       local index=0
+       local skip
+
+       #Because it will only get up facets, we need get affected
+       #facets before shutdown
+       #For HARD Failure mode, it needs make sure facets on the same
+       #HOST will only be shutdown and reboot once
+       for facet in ${facets//,/ }; do
+               local affected_facet
+               skip=0
+               #check whether facet has been included in other affected facets
+               for ((index=0; index<$total; index++)); do
+                       [[ *,$facet,* == ,${affecteds[index]}, ]] && skip=1
+               done
 
-    shutdown_facet $facet
+               if [ $skip -eq 0 ]; then
+                       affecteds[$total]=$(affected_facets $facet)
+                       total=$((total+1))
+               fi
+       done
 
-    echo affected facets: $affected
+       for ((index=0; index<$total; index++)); do
+               facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
+               local host=$(facet_active_host $facet)
+               echo "Failing ${affecteds[index]} on $host"
+               shutdown_facet $facet
+       done
 
-    [ -n "$sleep_time" ] && sleep $sleep_time
+       for ((index=0; index<$total; index++)); do
+               facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
+               echo reboot facets: ${affecteds[index]}
 
-    reboot_facet $facet
+               reboot_facet $facet
 
-    change_active $affected
+               change_active ${affecteds[index]}
 
-    wait_for_facet $affected
-    # start mgs first if it is affected
-    if ! combined_mgs_mds && list_member $affected mgs; then
-        mount_facet mgs || error "Restart of mgs failed"
-    fi
-    # FIXME; has to be changed to mount all facets concurrently
-    affected=$(exclude_items_from_list $affected mgs)
-    mount_facets $affected
+               wait_for_facet ${affecteds[index]}
+               # start mgs first if it is affected
+               if ! combined_mgs_mds &&
+                       list_member ${affecteds[index]} mgs; then
+                       mount_facet mgs || error "Restart of mgs failed"
+               fi
+               # FIXME; has to be changed to mount all facets concurrently
+               affected=$(exclude_items_from_list ${affecteds[index]} mgs)
+               echo mount facets: ${affecteds[index]}
+               mount_facets ${affecteds[index]}
+       done
 }
 
 obd_name() {
@@ -2077,42 +2110,42 @@ obd_name() {
 }
 
 replay_barrier() {
-    local facet=$1
-    do_facet $facet "sync; sync; sync"
-    df $MOUNT
+       local facet=$1
+       do_facet $facet "sync; sync; sync"
+       df $MOUNT
 
-    # make sure there will be no seq change
-    local clients=${CLIENTS:-$HOSTNAME}
-    local f=fsa-\\\$\(hostname\)
-    do_nodes $clients "mcreate $MOUNT/$f; rm $MOUNT/$f"
-    do_nodes $clients "if [ -d $MOUNT2 ]; then mcreate $MOUNT2/$f; rm $MOUNT2/$f; fi"
+        # make sure there will be no seq change
+       local clients=${CLIENTS:-$HOSTNAME}
+       local f=fsa-\\\$\(hostname\)
+       do_nodes $clients "mcreate $MOUNT/$f; rm $MOUNT/$f"
+       do_nodes $clients "if [ -d $MOUNT2 ]; then mcreate $MOUNT2/$f; rm $MOUNT2/$f; fi"
 
-    local svc=${facet}_svc
-    do_facet $facet $LCTL --device %${!svc} notransno
-    do_facet $facet $LCTL --device %${!svc} readonly
-    do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
-    $LCTL mark "local REPLAY BARRIER on ${!svc}"
+       local svc=${facet}_svc
+       do_facet $facet $LCTL --device ${!svc} notransno
+       do_facet $facet $LCTL --device ${!svc} readonly
+       do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
+       $LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
 
 replay_barrier_nodf() {
-    local facet=$1    echo running=${running}
-    do_facet $facet "sync; sync; sync"
-    local svc=${facet}_svc
-    echo Replay barrier on ${!svc}
-    do_facet $facet $LCTL --device %${!svc} notransno
-    do_facet $facet $LCTL --device %${!svc} readonly
-    do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
-    $LCTL mark "local REPLAY BARRIER on ${!svc}"
+       local facet=$1    echo running=${running}
+       do_facet $facet "sync; sync; sync"
+       local svc=${facet}_svc
+       echo Replay barrier on ${!svc}
+       do_facet $facet $LCTL --device ${!svc} notransno
+       do_facet $facet $LCTL --device ${!svc} readonly
+       do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
+       $LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
 
 replay_barrier_nosync() {
-    local facet=$1    echo running=${running}
-    local svc=${facet}_svc
-    echo Replay barrier on ${!svc}
-    do_facet $facet $LCTL --device %${!svc} notransno
-    do_facet $facet $LCTL --device %${!svc} readonly
-    do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
-    $LCTL mark "local REPLAY BARRIER on ${!svc}"
+       local facet=$1    echo running=${running}
+       local svc=${facet}_svc
+       echo Replay barrier on ${!svc}
+       do_facet $facet $LCTL --device ${!svc} notransno
+       do_facet $facet $LCTL --device ${!svc} readonly
+       do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
+       $LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
 
 mds_evict_client() {
@@ -2293,21 +2326,26 @@ hostlist_expand() {
 }
 
 facet_host() {
-    local facet=$1
+       local facet=$1
+       local varname
 
-    [ "$facet" == client ] && echo -n $HOSTNAME && return
-    varname=${facet}_HOST
-    if [ -z "${!varname}" ]; then
-        if [ "${facet:0:3}" == "ost" ]; then
-            eval ${facet}_HOST=${ost_HOST}
-        fi
-    fi
-    echo -n ${!varname}
+       [ "$facet" == client ] && echo -n $HOSTNAME && return
+       varname=${facet}_HOST
+       if [ -z "${!varname}" ]; then
+               if [ "${facet:0:3}" == "ost" ]; then
+                       eval export ${facet}_HOST=${ost_HOST}
+               elif [ "${facet:0:3}" == "mdt" -o \
+                       "${facet:0:3}" == "mds" -o \
+                       "${facet:0:3}" == "mgs" ]; then
+                       eval export ${facet}_HOST=${mds_HOST}
+               fi
+       fi
+       echo -n ${!varname}
 }
 
 facet_failover_host() {
        local facet=$1
-       local var
+       local varname
 
        var=${facet}failover_HOST
        if [ -n "${!var}" ]; then
@@ -2315,12 +2353,18 @@ facet_failover_host() {
                return
        fi
 
+       if [ "${facet:0:3}" == "mdt" -o "${facet:0:3}" == "mds" -o \
+            "${facet:0:3}" == "mgs" ]; then
+
+               eval export ${facet}failover_host=${mds_HOST}
+               echo ${mds_HOST}
+               return
+       fi
+
        if [[ $facet == ost* ]]; then
-               var=ostfailover_HOST
-               if [ -n "${!var}" ]; then
-                       echo ${!var}
-                       return
-               fi
+               eval export ${facet}failover_host=${ost_HOST}
+               echo ${ost_HOST}
+               return
        fi
 }
 
@@ -2443,8 +2487,6 @@ get_env_vars() {
                echo -n " ${var}=\"$value\""
        done
 
-       echo -n " USE_OFD=$USE_OFD"
-
        for facet in ${facets//,/ }; do
                var=${facet}_FSTYPE
                if [ -n "${!var}" ]; then
@@ -2627,17 +2669,16 @@ mdsvdevname() {
 }
 
 mgsdevname() {
-       DEVNAME=MGSDEV
+       local DEVNAME=MGSDEV
+       local MDSDEV1=$(mdsdevname 1)
 
-       local fstype=$(facet_fstype mds$num)
+       local fstype=$(facet_fstype mds1)
 
        case $fstype in
-               ldiskfs )
+               ldiskfs|zfs )
                        #if $MGSDEV isn't defined, default is $MDSDEV1
+                       #ZFS independent mgsdev should be ${FSNAME}-mgs/mgs
                        eval DEVPTR=${!DEVNAME:=${MDSDEV1}};;
-               zfs )
-                       #dataset name is independent of vdev device names
-                       eval DEVPTR=${FSNAME}-mgs/mgs;;
                * )
                        error "unknown fstype!";;
        esac
@@ -2648,7 +2689,7 @@ mgsdevname() {
 mgsvdevname() {
        DEVNAME=MGSDEV
 
-       local fstype=$(facet_fstype mds$num)
+       local fstype=$(facet_fstype mds1)
 
        case $fstype in
                ldiskfs )
@@ -2733,7 +2774,8 @@ cleanupall() {
 }
 
 combined_mgs_mds () {
-    [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]]
+       [[ "$(mdsdevname 1)" = "$(mgsdevname)" ]] &&
+               [[ "$(facet_host mds1)" = "$(facet_host mgs)" ]]
 }
 
 lower() {
@@ -2746,9 +2788,12 @@ upper() {
 
 mkfs_opts() {
        local facet=$1
+       local dev=$2
+       local fsname=${3:-"$FSNAME"}
        local type=$(facet_type $facet)
        local index=$(($(facet_number $facet) - 1))
        local fstype=$(facet_fstype $facet)
+       local host=$(facet_host $facet)
        local opts
        local fs_mkfs_opts
        local var
@@ -2757,14 +2802,17 @@ mkfs_opts() {
                return 1
        fi
 
-       if [ $type == MGS ] || ( [ $type == MDS ] && combined_mgs_mds ); then
+       if [ $type == MGS ] || ( [ $type == MDS ] &&
+                                 [ "$dev" == $(mgsdevname) ] &&
+                                [ "$host" == "$(facet_host mgs)" ] ); then
                opts="--mgs"
        else
                opts="--mgsnode=$MGSNID"
        fi
 
        if [ $type != MGS ]; then
-               opts+=" --fsname=$FSNAME --$(lower ${type/MDS/MDT}) --index=$index"
+               opts+=" --fsname=$fsname --$(lower ${type/MDS/MDT}) \
+                       --index=$index"
        fi
 
        var=${facet}failover_HOST
@@ -2834,23 +2882,24 @@ formatall() {
        echo Formatting mgs, mds, osts
        if ! combined_mgs_mds ; then
                echo "Format mgs: $(mgsdevname)"
-               add mgs $(mkfs_opts mgs) --reformat $(mgsdevname) \
-                       $(mgsvdevname) ${quiet:+>/dev/null} || exit 10
-               fi
+               add mgs $(mkfs_opts mgs $(mgsdevname)) --reformat \
+                       $(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} ||
+                       exit 10
+       fi
 
-               for num in `seq $MDSCOUNT`; do
-                       echo "Format mds$num: $(mdsdevname $num)"
-                       add mds$num $(mkfs_opts mds$num) --reformat \
-                       $(mdsdevname $num) $(mdsvdevname $num) \
+       for num in $(seq $MDSCOUNT); do
+               echo "Format mds$num: $(mdsdevname $num)"
+               add mds$num $(mkfs_opts mds$num $(mdsdevname ${num})) \
+                       --reformat $(mdsdevname $num) $(mdsvdevname $num) \
                        ${quiet:+>/dev/null} || exit 10
-               done
+       done
 
-               for num in `seq $OSTCOUNT`; do
-                       echo "Format ost$num: $(ostdevname $num)"
-                       add ost$num $(mkfs_opts ost$num) --reformat \
-                       $(ostdevname $num) $(ostvdevname ${num}) \
+       for num in $(seq $OSTCOUNT); do
+               echo "Format ost$num: $(ostdevname $num)"
+               add ost$num $(mkfs_opts ost$num $(ostdevname ${num})) \
+                       --reformat $(ostdevname $num) $(ostvdevname ${num}) \
                        ${quiet:+>/dev/null} || exit 10
-               done
+       done
 }
 
 mount_client() {
@@ -2898,23 +2947,31 @@ remount_client()
         zconf_mount `hostname` $1 || error "mount failed"
 }
 
-writeconf_facet () {
-    local facet=$1
-    local dev=$2
+writeconf_facet() {
+       local facet=$1
+       local dev=$2
 
-    do_facet $facet "$TUNEFS --writeconf $dev"
+       stop ${facet} -f
+       rm -f ${facet}active
+       do_facet ${facet} "$TUNEFS --quiet --writeconf $dev" || return 1
+       return 0
 }
 
 writeconf_all () {
-    for num in `seq $MDSCOUNT`; do
-        DEVNAME=$(mdsdevname $num)
-        writeconf_facet mds$num $DEVNAME
-    done
+       local mdt_count=${1:-$MDSCOUNT}
+       local ost_count=${2:-$OSTCOUNT}
+       local rc=0
 
-    for num in `seq $OSTCOUNT`; do
-        DEVNAME=$(ostdevname $num)
-        writeconf_facet ost$num $DEVNAME
-    done
+       for num in $(seq $mdt_count); do
+               DEVNAME=$(mdsdevname $num)
+               writeconf_facet mds$num $DEVNAME || rc=$?
+       done
+
+       for num in $(seq $ost_count); do
+               DEVNAME=$(ostdevname $num)
+               writeconf_facet ost$num $DEVNAME || rc=$?
+       done
+       return $rc
 }
 
 setupall() {
@@ -3043,9 +3100,14 @@ init_facet_vars () {
 
        local varname=${facet}failover_HOST
        if [ -z "${!varname}" ]; then
-               eval $varname=$(facet_host $facet)
+               eval export $varname=$(facet_host $facet)
        fi
 
+       varname=${facet}_HOST
+       if [ -z "${!varname}" ]; then
+               eval export $varname=$(facet_host $facet)
+       fi
+
        # ${facet}failover_dev is set in cfg file
        varname=${facet}failover_dev
        if [ -n "${!varname}" ] ; then
@@ -3067,23 +3129,26 @@ init_facet_vars () {
 }
 
 init_facets_vars () {
-    local DEVNAME
+       local DEVNAME
 
-    if ! remote_mds_nodsh; then 
-        for num in `seq $MDSCOUNT`; do
-            DEVNAME=`mdsdevname $num`
-            init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
-        done
-    fi
+       if ! remote_mds_nodsh; then
+               for num in $(seq $MDSCOUNT); do
+                       DEVNAME=`mdsdevname $num`
+                       eval export MDSDEV${num}=$DEVNAME
+                       init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
+               done
+       fi
 
+       eval export MGSDEV=$(mgsdevname)
        combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS
 
-    remote_ost_nodsh && return
-
-    for num in `seq $OSTCOUNT`; do
-        DEVNAME=`ostdevname $num`
-        init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
-    done
+       if ! remote_ost_nodsh; then
+               for num in $(seq $OSTCOUNT); do
+                       DEVNAME=$(ostdevname $num)
+                       eval export OSTDEV${num}=$DEVNAME
+                       init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
+               done
+       fi
 }
 
 osc_ensure_active () {
@@ -3708,6 +3773,17 @@ drop_reint_reply() {
     return $RC
 }
 
+drop_update_reply() {
+# OBD_FAIL_MDS_OBJ_UPDATE_NET
+       local index=$1
+       shift 1
+       RC=0
+       do_facet mds${index} lctl set_param fail_loc=0x188
+       do_facet client "$@" || RC=$?
+       do_facet mds${index} lctl set_param fail_loc=0
+       return $RC
+}
+
 pause_bulk() {
 #define OBD_FAIL_OST_BRW_PAUSE_BULK      0x214
     RC=0
@@ -3732,11 +3808,11 @@ drop_ldlm_cancel() {
 
 drop_bl_callback() {
 #define OBD_FAIL_LDLM_BL_CALLBACK_NET                  0x305
-    RC=0
-    do_facet client lctl set_param fail_loc=0x305
-    do_facet client "$@" || RC=$?
-    do_facet client lctl set_param fail_loc=0
-    return $RC
+       RC=0
+       do_facet client lctl set_param fail_loc=0x80000305
+       do_facet client "$@" || RC=$?
+       do_facet client lctl set_param fail_loc=0
+       return $RC
 }
 
 drop_ldlm_reply() {
@@ -4263,9 +4339,45 @@ mdtuuid_from_index()
     $LFS mdts $2 | sed -ne "/^$1: /s/.* \(.*\) .*$/\1/p"
 }
 
+# Description:
+#   Return unique identifier for given hostname
+host_id() {
+       local host_name=$1
+       echo $host_name | md5sum | cut -d' ' -f1
+}
+
+# Description:
+#   Returns list of ip addresses for each interface
+local_addr_list() {
+       ip addr | awk '/inet\ / {print $2}' | awk -F\/ '{print $1}'
+}
+
+is_local_addr() {
+       local addr=$1
+       # Cache address list to avoid mutiple execution of local_addr_list
+       LOCAL_ADDR_LIST=${LOCAL_ADDR_LIST:-$(local_addr_list)}
+       local i
+       for i in $LOCAL_ADDR_LIST ; do
+               [[ "$i" == "$addr" ]] && return 0
+       done
+       return 1
+}
+
+local_node() {
+       local host_name=$1
+       local is_local="IS_LOCAL_$(host_id $host_name)"
+       if [ -z "${!is_local-}" ] ; then
+               eval $is_local=0
+               local host_ip=$($LUSTRE/tests/resolveip $host_name)
+               is_local_addr "$host_ip" && eval $is_local=1
+       fi
+       [[ "${!is_local}" == "1" ]]
+}
+
 remote_node () {
-    local node=$1
-    [ "$node" != "$(hostname)" ]
+       local node=$1
+       local_node $node && return 1
+       return 0
 }
 
 remote_mds ()
@@ -5833,3 +5945,64 @@ generate_logname() {
 
        echo "$TESTLOG_PREFIX.$TESTNAME.$logname.$(hostname -s).log"
 }
+
+# mkdir directory on different MDTs
+test_mkdir() {
+       local option
+       local parent
+       local child
+       local path
+       local dir
+       local rc=0
+
+       if [ $# -eq 2 ]; then
+               option=$1
+               path=$2
+       else
+               path=$1
+       fi
+
+       child=${path##*/}
+       parent=${path%/*}
+
+       if [ "$parent" == "$child" ]; then
+               parent=$(pwd)
+       fi
+
+       if [ "$option" == "-p" -a -d ${parent}/${child} ]; then
+               return $rc
+       fi
+
+       # it needs to check whether there is further / in child
+       dir=$(echo $child | awk -F '/' '{print $2}')
+       if [ ! -z "$dir" ]; then
+               local subparent=$(echo $child | awk -F '/' '{ print $1 }')
+               parent=${parent}"/"${subparent}
+               child=$dir
+       fi
+
+       if [ ! -d ${parent} ]; then
+               if [ "$option" == "-p" ]; then
+                       mkdir -p ${parent}
+               else
+                       return 1
+               fi
+       fi
+
+       if [ $MDSCOUNT -le 1 ]; then
+               mkdir $option ${parent}/${child} || rc=$?
+       else
+               local mdt_idx=$($LFS getstripe -M $parent)
+
+               if [ "$mdt_idx" -ne 0 ]; then
+                       mkdir $option ${parent}/${child} || rc=$?
+                       return $rc
+               fi
+
+               local test_num=$(echo $testnum | sed -e 's/[^0-9]*//g')
+               local mdt_idx=$((test_num % MDSCOUNT))
+               echo "mkdir $mdt_idx for ${parent}/${child}"
+               $LFS setdirstripe -i $mdt_idx ${parent}/${child} || rc=$?
+       fi
+       return $rc
+}