X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=43b17e91cb47e313e5ce8a69d2139590ee25331f;hb=85cbe1a3ee6940f7468884bca43cd736a5365694;hp=be6d1ecde53c10bde000a9f774ce18df6196e03b;hpb=a633afc625197c853d3da2ec6bfca94de8bcd5ac;p=fs%2Flustre-release.git

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index be6d1ec..43b17e9 100755
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -23,6 +23,11 @@ export LOAD_LLOOP=${LOAD_LLOOP:-false}
 
 #export PDSH="pdsh -S -Rssh -w"
 export MOUNT_CMD=${MOUNT_CMD:-"mount -t lustre"}
+export UMOUNT=${UMOUNT:-"umount -d"}
+# sles12 umount has a issue with -d option
+[ -e /etc/SuSE-release ] && grep -w VERSION /etc/SuSE-release | grep -wq 12 && {
+	export UMOUNT="umount"
+}
 
 # function used by scripts run on remote nodes
 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
@@ -71,7 +76,6 @@ usage() {
 print_summary () {
     trap 0
 	[ -z "$DEFAULT_SUITES"] && return 0
-    [ "$TESTSUITE" == "lfsck" ] && return 0
     [ -n "$ONLY" ] && echo "WARNING: ONLY is set to $(echo $ONLY)"
     local details
     local form="%-13s %-17s %-9s %s %s\n"
@@ -193,9 +197,8 @@ init_test_env() {
 		fi
 	fi
 
-    export LFSCK_BIN=${LFSCK_BIN:-lfsck}
-    export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite
-    export FSCK_MAX_ERR=4   # File system errors left uncorrected
+	export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after test suite
+	export FSCK_MAX_ERR=4   # File system errors left uncorrected
 
 	export ZFS=${ZFS:-zfs}
 	export ZPOOL=${ZPOOL:-zpool}
@@ -594,7 +597,7 @@ load_modules_local() {
 	# 'mount' doesn't look in $PATH, just sbin
 	local mount_lustre=$LUSTRE/utils/mount.lustre
 	if [ -f $mount_lustre ]; then
-		local sbin_mount=/sbin/mount.lustre
+		local sbin_mount=$(readlink -f /sbin)/mount.lustre
 		if grep -qw "$sbin_mount" /proc/mounts; then
 			cmp -s $mount_lustre $sbin_mount || umount $sbin_mount
 		fi
@@ -658,7 +661,7 @@ unload_modules() {
 		fi
 	fi
 
-	local sbin_mount=/sbin/mount.lustre
+	local sbin_mount=$(readlink -f /sbin)/mount.lustre
 	if grep -qe "$sbin_mount " /proc/mounts; then
 		umount $sbin_mount || true
 		[ -s $sbin_mount ] && ! grep -q "STUB MARK" $sbin_mount ||
@@ -1113,7 +1116,7 @@ get_osd_param() {
 	local name=$3
 
 	do_nodes $nodes "$LCTL get_param -n obdfilter.$device.$name \
-		osd-*.$device.$name 2>&1" | grep -v 'Found no match'
+		osd-*.$device.$name 2>&1" | grep -v 'error:'
 }
 
 set_osd_param() {
@@ -1123,7 +1126,7 @@ set_osd_param() {
 	local value=$4
 
 	do_nodes $nodes "$LCTL set_param -n obdfilter.$device.$name=$value \
-		osd-*.$device.$name=$value 2>&1" | grep -v 'Found no match'
+		osd-*.$device.$name=$value 2>&1" | grep -v 'error:'
 }
 
 set_debug_size () {
@@ -1221,6 +1224,8 @@ mount_facet() {
 	local mntpt=$(facet_mntpt $facet)
 	local opts="${!opt} $@"
 
+	module_loaded lustre || load_modules
+
 	if [ $(facet_fstype $facet) == ldiskfs ] &&
 	   ! do_facet $facet test -b ${!dev}; then
 		opts=$(csa_add "$opts" -o loop)
@@ -1240,40 +1245,63 @@ mount_facet() {
 		                   ${!dev} $mntpt"
 		RC=${PIPESTATUS[0]}
 	fi
+
 	if [ $RC -ne 0 ]; then
 		echo "Start of ${!dev} on ${facet} failed ${RC}"
-    else
-        set_default_debug_facet $facet
+		return $RC
+	fi
+
+	set_default_debug_facet $facet
 
 	if [[ $facet == mds* ]]; then
 		do_facet $facet \
-			lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 \
-				2>/dev/null
+		lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 2>/dev/null
 	fi
 
-		label=$(devicelabel ${facet} ${!dev})
-        [ -z "$label" ] && echo no label for ${!dev} && exit 1
-        eval export ${facet}_svc=${label}
-        echo Started ${label}
-    fi
-    return $RC
+	if [[ $opts =~ .*nosvc.* ]]; then
+		echo "Start ${!dev} without service"
+	else
+		local fstype=$(facet_fstype $facet)
+
+		case $fstype in
+		ldiskfs)
+			wait_update_facet ${facet} "$E2LABEL ${!dev} \
+				2>/dev/null | grep -E ':[a-zA-Z]{3}[0-9]{4}'" \
+				"" || error "${!dev} failed to initialize!";;
+		zfs)
+			wait_update_facet ${facet} "$ZFS get -H -o value \
+				lustre:svname ${!dev} 2>/dev/null | \
+				grep -E ':[a-zA-Z]{3}[0-9]{4}'" "" ||
+				error "${!dev} failed to initialize!";;
+
+		*)
+			error "unknown fstype!";;
+		esac
+	fi
+
+	label=$(devicelabel ${facet} ${!dev})
+	[ -z "$label" ] && echo no label for ${!dev} && exit 1
+	eval export ${facet}_svc=${label}
+	echo Started ${label}
+
+	return $RC
 }
 
 # start facet device options
 start() {
-    local facet=$1
-    shift
-    local device=$1
-    shift
-    eval export ${facet}_dev=${device}
-    eval export ${facet}_opt=\"$@\"
+	local facet=$1
+	shift
+	local device=$1
+	shift
+	eval export ${facet}_dev=${device}
+	eval export ${facet}_opt=\"$@\"
 
-    local varname=${facet}failover_dev
-    if [ -n "${!varname}" ] ; then
-        eval export ${facet}failover_dev=${!varname}
-    else
-        eval export ${facet}failover_dev=$device
-    fi
+	local varname=${facet}failover_dev
+	if [ -n "${!varname}" ] ; then
+		eval export ${facet}failover_dev=${!varname}
+	else
+		eval export ${facet}failover_dev=$device
+	fi
 
 	local mntpt=$(facet_mntpt $facet)
 	do_facet ${facet} mkdir -p $mntpt
@@ -1301,7 +1329,7 @@ stop() {
     running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts") || true
     if [ ${running} -ne 0 ]; then
         echo "Stopping $mntpt (opts:$@) on $HOST"
-        do_facet ${facet} umount -d $@ $mntpt
+	do_facet ${facet} $UMOUNT $@ $mntpt
     fi
 
 	# umount should block, but we should wait for unrelated obd's
@@ -1354,70 +1382,6 @@ quota_type() {
 	return $rc
 }
 
-# XXX This function is kept for interoperability with old server (< 2.3.50),
-#     it should be removed whenever we drop the interoperability for such
-#     server.
-restore_quota_old() {
-	local mntpt=${1:-$MOUNT}
-	local quota_type=$(quota_type $FSNAME | grep MDT | cut -d "=" -f2)
-	if [ ! "$old_QUOTA_TYPE" ] ||
-		[ "$quota_type" = "$old_QUOTA_TYPE" ]; then
-		return
-	fi
-	quota_save_version $old_QUOTA_TYPE
-}
-
-# XXX This function is kept for interoperability with old server (< 2.3.50),
-#     it should be removed whenever we drop the interoperability for such
-#     server.
-setup_quota_old(){
-	local mntpt=$1
-
-	# no quota enforcement for now and accounting works out of the box
-	return
-
-    # We need save the original quota_type params, and restore them after testing
-
-    # Suppose that quota type the same on mds and ost
-    local quota_type=$(quota_type | grep MDT | cut -d "=" -f2)
-    [ ${PIPESTATUS[0]} -eq 0 ] || error "quota_type failed!"
-    echo "[HOST:$HOSTNAME] [old_quota_type:$quota_type] [new_quota_type:$QUOTA_TYPE]"
-    if [ "$quota_type" != "$QUOTA_TYPE" ]; then
-        export old_QUOTA_TYPE=$quota_type
-        quota_save_version $QUOTA_TYPE
-    else
-        qtype=$(tr -c -d "ug" <<< $QUOTA_TYPE)
-        $LFS quotacheck -$qtype $mntpt || error "quotacheck has failed for $type"
-    fi
-
-    local quota_usrs=$QUOTA_USERS
-
-    # get_filesystem_size
-    local disksz=$(lfs_df $mntpt | grep "summary"  | awk '{print $2}')
-    local blk_soft=$((disksz + 1024))
-    local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over
-
-    local Inodes=$(lfs_df -i $mntpt | grep "summary"  | awk '{print $2}')
-    local i_soft=$Inodes
-    local i_hard=$((i_soft + i_soft / 20))
-
-    echo "Total disk size: $disksz  block-softlimit: $blk_soft block-hardlimit:
-        $blk_hard inode-softlimit: $i_soft inode-hardlimit: $i_hard"
-
-    local cmd
-    for usr in $quota_usrs; do
-        echo "Setting up quota on $HOSTNAME:$mntpt for $usr..."
-        for type in u g; do
-            cmd="$LFS setquota -$type $usr -b $blk_soft -B $blk_hard -i $i_soft -I $i_hard $mntpt"
-            echo "+ $cmd"
-            eval $cmd || error "$cmd FAILED!"
-        done
-        # display the quota status
-        echo "Quota settings for $usr : "
-        $LFS quota -v -u $usr $mntpt || true
-    done
-}
-
 # get mdt quota type
 mdt_quota_type() {
 	local varsvc=${SINGLEMDS}_svc
@@ -1435,11 +1399,6 @@ ost_quota_type() {
 
 # restore old quota type settings
 restore_quota() {
-	if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then
-		restore_quota_old
-		return
-	fi
-
 	if [ "$old_MDT_QUOTA_TYPE" ]; then
 		do_facet mgs $LCTL conf_param \
 			$FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE
@@ -1475,11 +1434,6 @@ mdt_free_inodes() {
 }
 
 setup_quota(){
-	if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then
-		setup_quota_old $1
-		return
-	fi
-
 	local mntpt=$1
 
 	# save old quota type & set new quota type
@@ -2107,16 +2061,19 @@ wait_update () {
 }
 
 wait_update_facet() {
+	local verbose=
+	[ "$1" = "--verbose" ] && verbose="$1" && shift
+
 	local facet=$1
 	shift
-	wait_update $(facet_active_host $facet) "$@"
+	wait_update $verbose $(facet_active_host $facet) "$@"
 }
 
 sync_all_data() {
 	do_nodes $(comma_list $(mdts_nodes)) \
-	    "lctl set_param -n osd*.*MDT*.force_sync 1"
+	    "lctl set_param -n osd*.*MDT*.force_sync=1"
 	do_nodes $(comma_list $(osts_nodes)) \
-	    "lctl set_param -n osd*.*OS*.force_sync 1" 2>&1 |
+	    "lctl set_param -n osd*.*OS*.force_sync=1" 2>&1 |
 		grep -v 'Found no match'
 }
 
@@ -2432,6 +2389,13 @@ affected_facets () {
 }
 
 facet_failover() {
+	local E2FSCK_ON_MDT0=false
+	if [ "$1" == "--fsck" ]; then
+		shift
+		[ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&
+			E2FSCK_ON_MDT0=true
+	fi
+
 	local facets=$1
 	local sleep_time=$2
 	local -a affecteds
@@ -2465,6 +2429,9 @@ facet_failover() {
 		shutdown_facet $facet
 	done
 
+	$E2FSCK_ON_MDT0 && (run_e2fsck $(facet_active_host $SINGLEMDS) \
+		$(mdsdevname 1) "-n" || error "Running e2fsck")
+
 	for ((index=0; index<$total; index++)); do
 		facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
 		echo reboot facets: ${affecteds[index]}
@@ -3222,7 +3189,7 @@ unmount_ldiskfs() {
 	local dev=$(facet_device $facet)
 	local mnt=$(facet_mntpt $facet)
 
-	do_facet $facet umount -d $mnt
+	do_facet $facet $UMOUNT $mnt
 }
 
 var_name() {
@@ -4179,88 +4146,31 @@ check_shared_dir() {
 	return 0
 }
 
-# Run e2fsck on MDT and OST(s) to generate databases used for lfsck.
-generate_db() {
-	local i
-	local ostidx
-	local dev
-	local node
-
-	[[ $(lustre_version_code $SINGLEMDS) -ne $(version_code 2.2.0) ]] ||
-		{ skip "Lustre 2.2.0 lacks the patch for LU-1255"; exit 0; }
-
-	check_shared_dir $SHARED_DIRECTORY ||
-		error "$SHARED_DIRECTORY isn't a shared directory"
-
-	export MDSDB=$SHARED_DIRECTORY/mdsdb
-	export OSTDB=$SHARED_DIRECTORY/ostdb
-
-	# DNE is not supported, so when running e2fsck on a DNE filesystem,
-	# we only pass master MDS parameters.
-	run_e2fsck $MDTNODE $MDTDEV "-n --mdsdb $MDSDB"
-
-    i=0
-    ostidx=0
-    OSTDB_LIST=""
-    for node in $(osts_nodes); do
-        for dev in ${OSTDEVS[i]}; do
-            run_e2fsck $node $dev "-n --mdsdb $MDSDB --ostdb $OSTDB-$ostidx"
-            OSTDB_LIST="$OSTDB_LIST $OSTDB-$ostidx"
-            ostidx=$((ostidx + 1))
-        done
-        i=$((i + 1))
-    done
-}
-
-# Run lfsck on server node if lfsck can't be found on client (LU-2571)
-run_lfsck_remote() {
-	local cmd="$LFSCK_BIN -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT"
-	local client=$1
-	local mounted=true
-	local rc=0
-
-	#Check if lustre is already mounted
-	do_rpc_nodes $client is_mounted $MOUNT || mounted=false
-	if ! $mounted; then
-		zconf_mount $client $MOUNT ||
-			error "failed to mount Lustre on $client"
-	fi
-	#Run lfsck
-	echo $cmd
-	do_node $client $cmd || rc=$?
-	#Umount if necessary
-	if ! $mounted; then
-		zconf_umount $client $MOUNT ||
-			error "failed to unmount Lustre on $client"
-	fi
-
-	[ $rc -le $FSCK_MAX_ERR ] ||
-		error "$cmd returned $rc, should be <= $FSCK_MAX_ERR"
-	echo "lfsck finished with rc=$rc"
-
-	return $rc
-}
-
 run_lfsck() {
-	local facets="client $SINGLEMDS"
-	local found=false
-	local facet
-	local node
-	local rc=0
-
-	for facet in $facets; do
-		node=$(facet_active_host $facet)
-		if check_progs_installed $node $LFSCK_BIN; then
-			found=true
-			break
-		fi
+	do_nodes $(comma_list $(mdts_nodes) $(osts_nodes)) \
+		$LCTL set_param printk=+lfsck
+	do_facet $SINGLEMDS "$LCTL lfsck_start -M $FSNAME-MDT0000 -r -A -t all"
+
+	for k in $(seq $MDSCOUNT); do
+		# wait up to 10+1 minutes for LFSCK to complete
+		wait_update_facet --verbose mds${k} "$LCTL get_param -n \
+			mdd.$(facet_svc mds${k}).lfsck_layout |
+			awk '/^status/ { print \\\$2 }'" "completed" 600 ||
+			error "MDS${k} layout isn't the expected 'completed'"
+		wait_update_facet --verbose mds${k} "$LCTL get_param -n \
+			mdd.$(facet_svc mds${k}).lfsck_namespace |
+			awk '/^status/ { print \\\$2 }'" "completed" 60 ||
+			error "MDS${k} namespace isn't the expected 'completed'"
 	done
-	! $found && error "None of \"$facets\" supports lfsck"
-
-	run_lfsck_remote $node || rc=$?
-
-	rm -rvf $MDSDB* $OSTDB* || true
-	return $rc
+	local rep_mdt=$(do_nodes $(comma_list $(mdts_nodes)) \
+			$LCTL get_param -n mdd.$FSNAME-*.lfsck_* |
+			awk '/repaired/ { print $2 }' | calc_sum)
+	local rep_ost=$(do_nodes $(comma_list $(osts_nodes)) \
+			$LCTL get_param -n obdfilter.$FSNAME-*.lfsck_* |
+			awk '/repaired/ { print $2 }' | calc_sum)
+	local repaired=$((rep_mdt + rep_ost))
+	[ $repaired -eq 0 ] ||
+		error "lfsck repaired $rep_mdt MDT and $rep_ost OST errors"
 }
 
 dump_file_contents() {
@@ -4318,11 +4228,10 @@ log_zfs_info() {
 }
 
 check_and_cleanup_lustre() {
-    if [ "$LFSCK_ALWAYS" = "yes" -a "$TESTSUITE" != "lfsck" ]; then
-        get_svr_devs
-        generate_db
-        run_lfsck
-    fi
+	if [ "$LFSCK_ALWAYS" = "yes" -a "$TESTSUITE" != "sanity-lfsck" -a \
+	     "$TESTSUITE" != "sanity-scrub" ]; then
+		run_lfsck
+	fi
 
 	if is_mounted $MOUNT; then
 		[ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* ||
@@ -4653,33 +4562,10 @@ set_nodes_failloc () {
 }
 
 cancel_lru_locks() {
-    $LCTL mark "cancel_lru_locks $1 start"
-
-    if [ $1 != "MGC" ]; then
-	for d in $(lctl get_param -N ldlm.namespaces.*.lru_size |
-		 egrep -i $1); do
-	    $LCTL set_param -n $d=clear
-	done
-	$LCTL get_param ldlm.namespaces.*.lock_unused_count | egrep -i $1 |
-		grep -v '=0'
-    else
-	for d in $(find \
-		/{proc,sys}/fs/lustre/ldlm/namespaces/*$1*/lru_size \
-		2> /dev/null); do
-	    echo "clear" > $d
-	done
-
-	for d in $(find \
-		/{proc,sys}/fs/lustre/ldlm/namespaces/*$1*/lock_unused_count \
-		2> /dev/null); do
-	    if [ $(cat $d) != 0 ]; then
-		echo "ldlm.namespaces.$(echo "$d" |
-			cut -f 7 -d'/').lock_unused_count=$(cat $d)"
-	    fi
-	done
-    fi
-
-    $LCTL mark "cancel_lru_locks $1 stop"
+	#$LCTL mark "cancel_lru_locks $1 start"
+	$LCTL set_param -n ldlm.namespaces.*$1*.lru_size=clear
+	$LCTL get_param ldlm.namespaces.*$1*.lock_unused_count | grep -v '=0'
+	#$LCTL mark "cancel_lru_locks $1 stop"
 }
 
 default_lru_size()
@@ -5824,19 +5710,21 @@ mds_on_old_device() {
 }
 
 get_mdtosc_proc_path() {
-    local mds_facet=$1
-    local ost_label=${2:-"*OST*"}
-
-    [ "$mds_facet" = "mds" ] && mds_facet=$SINGLEMDS
-    local mdt_label=$(convert_facet2label $mds_facet)
-    local mdt_index=$(echo $mdt_label | sed -e 's/^.*-//')
-
-    if [ $(lustre_version_code $mds_facet) -le $(version_code 1.8.0) ] ||
-       mds_on_old_device $mds_facet; then
-        echo "${ost_label}-osc"
-    else
-        echo "${ost_label}-osc-${mdt_index}"
-    fi
+	local mds_facet=$1
+	local ost_label=${2:-"*OST*"}
+
+	[ "$mds_facet" = "mds" ] && mds_facet=$SINGLEMDS
+	local mdt_label=$(convert_facet2label $mds_facet)
+	local mdt_index=$(echo $mdt_label | sed -e 's/^.*-//')
+
+	if [ $(lustre_version_code $mds_facet) -le $(version_code 1.8.0) ] ||
+	   mds_on_old_device $mds_facet; then
+		echo "${ost_label}-osc"
+	elif [[ $ost_label = *OST* ]]; then
+		echo "${ost_label}-osc-${mdt_index}"
+	else
+		echo "${ost_label}-osp-${mdt_index}"
+	fi
 }
 
 get_osc_import_name() {
@@ -5950,8 +5838,8 @@ _wait_osc_import_state() {
 	local facet=$1
 	local ost_facet=$2
 	local expected=$3
-	local ost=$(get_osc_import_name $facet $ost_facet)
-	local param="osc.${ost}.ost_server_uuid"
+	local target=$(get_osc_import_name $facet $ost_facet)
+	local param="osc.${target}.ost_server_uuid"
 	local params=$param
 	local i=0
 
@@ -5975,6 +5863,16 @@ _wait_osc_import_state() {
 			params=$($LCTL list_param $param 2>/dev/null || true)
 		done
 	fi
+
+	if [[ $ost_facet = mds* ]]; then
+		# no OSP connection to itself
+		if [[ $facet = $ost_facet ]]; then
+			return 0
+		fi
+		param="osp.${target}.mdt_server_uuid"
+		params=$param
+	fi
+
 	if ! do_rpc_nodes "$(facet_active_host $facet)" \
 			wait_import_state $expected "$params" $maxtime; then
 		error "import is not in ${expected} state"
@@ -6056,6 +5954,16 @@ wait_mgc_import_state() {
 	fi
 }
 
+wait_dne_interconnect() {
+	local num
+
+	if [ $MDSCOUNT -gt 1 ]; then
+		for num in $(seq $MDSCOUNT); do
+			wait_osc_import_state mds mds$num FULL
+		done
+	fi
+}
+
 get_clientmdc_proc_path() {
     echo "${1}-mdc-*"
 }
@@ -6311,6 +6219,31 @@ max_recovery_time() {
 	echo -n $service_time
 }
 
+recovery_time_min() {
+	local connection_switch_min=5
+	local connection_switch_inc=5
+	local connection_switch_max
+	local reconnect_delay_max
+	local initial_connect_timeout
+	local max
+	local timout_20
+
+	#connection_switch_max=min(50, max($connection_switch_min,$TIMEOUT)
+	(($connection_switch_min > $TIMEOUT)) &&
+		max=$connection_switch_min || max=$TIMEOUT
+	(($max < 50)) && connection_switch_max=$max || connection_switch_max=50
+
+	#initial_connect_timeout = max(connection_switch_min, obd_timeout/20)
+	timeout_20=$((TIMEOUT/20))
+	(($connection_switch_min > $timeout_20)) &&
+		initial_connect_timeout=$connection_switch_min ||
+		initial_connect_timeout=$timeout_20
+
+	reconnect_delay_max=$((connection_switch_max + connection_switch_inc + \
+			       initial_connect_timeout))
+	echo $((2 * reconnect_delay_max))
+}
+
 get_clients_mount_count () {
     local clients=${CLIENTS:-`hostname`}
 
@@ -6984,7 +6917,7 @@ mds_backup_restore() {
 	echo "backup data"
 	${rcmd} tar zcf $metadata -C $mntpt/ . > /dev/null 2>&1 || return 3
 	# step 6: umount
-	${rcmd} umount -d $mntpt || return 4
+	${rcmd} $UMOUNT $mntpt || return 4
 	# step 8: reformat dev
 	echo "reformat new device"
 	format_mdt $(facet_number $facet)
@@ -7000,7 +6933,7 @@ mds_backup_restore() {
 	echo "remove recovery logs"
 	${rcmd} rm -fv $mntpt/OBJECTS/* $mntpt/CATALOGS
 	# step 13: umount dev
-	${rcmd} umount -d $mntpt || return 10
+	${rcmd} $UMOUNT $mntpt || return 10
 	# step 14: cleanup tmp backup
 	${rcmd} rm -f $metaea $metadata
 	# step 15: reset device label - it's not virgin on
@@ -7040,7 +6973,7 @@ mds_remove_ois() {
 		done
 	fi
 	# step 4: umount
-	${rcmd} umount -d $mntpt || return 2
+	${rcmd} $UMOUNT $mntpt || return 2
 	# OI files will be recreated when mounted as lustre next time.
 }