From: Hongchao Zhang <hongchao.zhang@intel.com>
Date: Sat, 3 Mar 2018 06:36:32 +0000 (-0800)
Subject: LU-684 tests: replace dev_read_only patch with dm-flakey
X-Git-Tag: 2.11.0-RC1~22
X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=54b9e3f789358bd9dfb94b77fe33a4faa1e28ab2

LU-684 tests: replace dev_read_only patch with dm-flakey

The dev_read_only kernel patch is mainly used for testing,
in order to simulate a server crash for ldiskfs by discarding
all of the writes to the device.

Since Linux kernel 3.0, this testing functionality can be
simulated by using "dm-flakey" target for device-mapper,
which supports a "drop_writes" parameter that could be used
in place of our dev_read_only kernel patch.

Signed-off-by: Hongchao Zhang <hongchao.zhang@intel.com>
Signed-off-by: James Simmons <uja.ornl@yahoo.com>
Signed-off-by: Jian Yu <jian.yu@intel.com>
Change-Id: I51ff9a1a10fb5bacdc1afa2716b769b5eda41863
Reviewed-on: https://review.whamcloud.com/7200
Tested-by: Jenkins
Reviewed-by: Fan Yong <fan.yong@intel.com>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
---

diff --git a/lustre/tests/auster b/lustre/tests/auster
index ef6ec12..5a4e73d 100755
--- a/lustre/tests/auster
+++ b/lustre/tests/auster
@@ -131,33 +131,33 @@ print_summary () {
 
 
 setup_if_needed() {
-    nfs_client_mode && return
-    auster_cleanup=false
-
-    local MOUNTED=$(mounted_lustre_filesystems)
-    if $(echo $MOUNTED' ' | grep -w -q $MOUNT' '); then
-        check_config_clients $MOUNT
-       # init_facets_vars
-       # init_param_vars
-        return
-    fi
+	nfs_client_mode && return
+	AUSTER_CLEANUP=false
+
+	local MOUNTED=$(mounted_lustre_filesystems)
+	if $(echo $MOUNTED' ' | grep -w -q $MOUNT' '); then
+		check_config_clients $MOUNT
+		# init_facets_vars
+		# init_param_vars
+		return
+	fi
 
-    echo "Lustre is not mounted, trying to do setup ... "
-    $reformat && formatall
-    setupall
+	echo "Lustre is not mounted, trying to do setup ... "
+	$reformat && CLEANUP_DM_DEV=true formatall
+	setupall
 
-    MOUNTED=$(mounted_lustre_filesystems)
-    if ! $(echo $MOUNTED' ' | grep -w -q $MOUNT' '); then
-        echo "Lustre is not mounted after setup! "
-        exit 1
-    fi
-    auster_cleanup=true
+	MOUNTED=$(mounted_lustre_filesystems)
+	if ! $(echo $MOUNTED' ' | grep -w -q $MOUNT' '); then
+		echo "Lustre is not mounted after setup! "
+		exit 1
+	fi
+	AUSTER_CLEANUP=true
 }
 
 cleanup_if_needed() {
-    if $auster_cleanup; then
-        cleanupall
-    fi
+	if $AUSTER_CLEANUP; then
+		cleanupall
+	fi
 }
 
 find_script_in_path() {
diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index cb2daaf..e1d9c0c 100644
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -571,22 +571,6 @@ test_9() {
 }
 run_test 9 "test ptldebug and subsystem for mkfs"
 
-is_blkdev () {
-	local facet=$1
-	local dev=$2
-	local size=${3:-""}
-
-	local rc=0
-	do_facet $facet "test -b $dev" || rc=1
-	if [[ "$size" ]]; then
-		local in=$(do_facet $facet "dd if=$dev of=/dev/null bs=1k \
-			   count=1 skip=$size 2>&1" |
-			awk '($3 == "in") { print $1 }')
-		[[ $in  = "1+0" ]] || rc=1
-	fi
-	return $rc
-}
-
 #
 # Test 16 was to "verify that lustre will correct the mode of OBJECTS".
 # But with new MDS stack we don't care about the mode of local objects
@@ -7553,7 +7537,7 @@ test_106() {
 #shows that osp code is buggy
 	do_facet mds1 $LCTL set_param fail_loc=0 fail_val=0
 
-	cleanupall
+	stopall
 }
 run_test 106 "check osp llog processing when catalog is wrapped"
 
@@ -8096,4 +8080,5 @@ OSTSIZE=$STORED_OSTSIZE
 reformat
 
 complete $SECONDS
+check_and_cleanup_lustre
 exit_status
diff --git a/lustre/tests/lfsck-performance.sh b/lustre/tests/lfsck-performance.sh
index d9ace79..dc873bf 100644
--- a/lustre/tests/lfsck-performance.sh
+++ b/lustre/tests/lfsck-performance.sh
@@ -90,7 +90,7 @@ lfsck_create() {
 
 lfsck_cleanup() {
 	do_rpc_nodes $(facet_active_host $SINGLEMDS) unload_modules
-	formatall
+	REFORMAT="yes" cleanup_and_setup_lustre
 }
 
 lfsck_create_nfiles() {
@@ -1041,4 +1041,5 @@ run_test 8 "lfsck namespace impact on create performance"
 # cleanup the system at last
 lfsck_cleanup
 complete $SECONDS
+check_and_cleanup_lustre
 exit_status
diff --git a/lustre/tests/lnet-selftest.sh b/lustre/tests/lnet-selftest.sh
index 0589a95..54053b8 100755
--- a/lustre/tests/lnet-selftest.sh
+++ b/lustre/tests/lnet-selftest.sh
@@ -183,4 +183,5 @@ run_test smoke "lst regression test"
 
 complete $SECONDS
 _restore_mount
+check_and_cleanup_lustre
 exit_status
diff --git a/lustre/tests/mmp.sh b/lustre/tests/mmp.sh
index b390c52..ba2dcbf 100755
--- a/lustre/tests/mmp.sh
+++ b/lustre/tests/mmp.sh
@@ -663,7 +663,7 @@ test_10() {
         error_noexit "e2fsck $MMP_OSTDEV on $MMP_OSS_FAILOVER returned $rc"
     fi
 
-    stop_services primary || return ${PIPESTATUS[0]}
+    CLEANUP_DM_DEV=true stop_services primary || return ${PIPESTATUS[0]}
     return 0
 }
 run_test 10 "e2fsck with mounted filesystem"
@@ -673,4 +673,5 @@ FAIL_ON_ERROR=$SAVED_FAIL_ON_ERROR
 
 complete $SECONDS
 $MMP_RESTORE_MOUNT && setupall
+check_and_cleanup_lustre
 exit_status
diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 87c9048..0a6204d 100755
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -2217,13 +2217,14 @@ test_111 ()
 	[[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.3.62) ]] ||
 		{ skip "Need MDS version at least 2.3.62"; return 0; }
 
-	local mdsdev=$(mdsdevname ${SINGLEMDS//mds/})
 #define OBD_FAIL_MDS_CHANGELOG_INIT 0x151
 	do_facet $SINGLEMDS lctl set_param fail_loc=0x151
 	stop $SINGLEMDS || error "stop MDS failed"
-	start $SINGLEMDS $mdsdev && error "start MDS should fail"
+	start $SINGLEMDS $(mdsdevname ${SINGLEMDS//mds/}) &&
+		error "start MDS should fail"
 	do_facet $SINGLEMDS lctl set_param fail_loc=0
-	start $SINGLEMDS $mdsdev || error "start MDS failed"
+	start $SINGLEMDS $(mdsdevname ${SINGLEMDS//mds/}) ||
+		error "start MDS failed"
 }
 run_test 111 "mdd setup fail should not cause umount oops"
 
diff --git a/lustre/tests/sanity-lfsck.sh b/lustre/tests/sanity-lfsck.sh
index 6bc7557..8c1ed01 100644
--- a/lustre/tests/sanity-lfsck.sh
+++ b/lustre/tests/sanity-lfsck.sh
@@ -42,11 +42,11 @@ SAVED_OSTCOUNT=${OSTCOUNT}
 MDSSIZE=100000
 OSTSIZE=100000
 # no need too many OSTs, to reduce the format/start/stop overhead
+cleanupall
 [ $OSTCOUNT -gt 4 ] && OSTCOUNT=4
 
 # build up a clean test environment.
-formatall
-setupall
+REFORMAT="yes" check_and_setup_lustre
 
 [[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.4.90) ]] &&
 	ALWAYS_EXCEPT="$ALWAYS_EXCEPT 2c"
@@ -5157,7 +5157,8 @@ OSTSIZE=${SAVED_OSTSIZE}
 OSTCOUNT=${SAVED_OSTCOUNT}
 
 # cleanup the system at last
-formatall
+REFORMAT="yes" cleanup_and_setup_lustre
 
 complete $SECONDS
+check_and_cleanup_lustre
 exit_status
diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh
index 9a507cd..6cc590b 100755
--- a/lustre/tests/sanity-quota.sh
+++ b/lustre/tests/sanity-quota.sh
@@ -2607,10 +2607,8 @@ test_35() {
 	fi
 
 	log "Restart..."
-	local ORIG_REFORMAT=$REFORMAT
-	REFORMAT=""
-	cleanup_and_setup_lustre
-	REFORMAT=$ORIG_REFORMAT
+	stopall
+	setupall
 	quota_init
 
 	echo "Verify disk usage after restart"
diff --git a/lustre/tests/sanity-scrub.sh b/lustre/tests/sanity-scrub.sh
index 24c81a9..b248d95 100644
--- a/lustre/tests/sanity-scrub.sh
+++ b/lustre/tests/sanity-scrub.sh
@@ -27,7 +27,7 @@ if ! check_versions; then
 	exit 0
 fi
 
-stopall
+cleanupall
 
 SAVED_MDSSIZE=${MDSSIZE}
 SAVED_OSTSIZE=${OSTSIZE}
@@ -43,8 +43,7 @@ OSTSIZE=200000
 [ $OSTCOUNT -gt 4 ] && OSTCOUNT=4
 
 # build up a clean test environment.
-formatall
-setupall
+REFORMAT="yes" check_and_setup_lustre
 
 build_test_filter
 
@@ -1249,7 +1248,8 @@ OSTSIZE=${SAVED_OSTSIZE}
 OSTCOUNT=${SAVED_OSTCOUNT}
 
 # cleanup the system at last
-formatall
+REFORMAT="yes" cleanup_and_setup_lustre
 
 complete $SECONDS
+check_and_cleanup_lustre
 exit_status
diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh
index 9d81a27..b2027ee 100755
--- a/lustre/tests/sanity-sec.sh
+++ b/lustre/tests/sanity-sec.sh
@@ -81,18 +81,7 @@ FOPS_IDMAPS=(
 
 check_and_setup_lustre
 
-sec_cleanup() {
-	if [ "$I_MOUNTED" = "yes" ]; then
-		cleanupall -f || error "sec_cleanup"
-	fi
-}
-
-DIR=${DIR:-$MOUNT}
-[ -z "$(echo $DIR | grep $MOUNT)" ] &&
-	error "$DIR not in $MOUNT" && sec_cleanup && exit 1
-
-[ $(echo $MOUNT | wc -w) -gt 1 ] &&
-	echo "NAME=$MOUNT mounted more than once" && sec_cleanup && exit 0
+assert_DIR
 
 # for GSS_SUP
 GSS_REF=$(lsmod | grep ^ptlrpc_gss | awk '{print $3}')
@@ -2085,7 +2074,6 @@ sec_unsetup() {
 }
 sec_unsetup
 
-sec_cleanup
-
 complete $SECONDS
+check_and_cleanup_lustre
 exit_status
diff --git a/lustre/tests/scrub-performance.sh b/lustre/tests/scrub-performance.sh
index 785335a..f04e08c 100644
--- a/lustre/tests/scrub-performance.sh
+++ b/lustre/tests/scrub-performance.sh
@@ -41,7 +41,7 @@ if [ ${NTHREADS} -eq 0 ]; then
 	NTHREADS=$((CPUCORE * 2))
 fi
 
-stopall
+cleanupall
 
 if ! combined_mgs_mds ; then
 	do_rpc_nodes $(facet_active_host mgs) load_modules_local
@@ -97,12 +97,12 @@ scrub_create() {
 }
 
 scrub_cleanup() {
-	stopall
+	cleanupall
 	do_rpc_nodes $(facet_active_host $SINGLEMDS) unload_modules
 	if ! combined_mgs_mds ; then
 		do_rpc_nodes $(facet_active_host mgs) unload_modules
 	fi
-	formatall
+	REFORMAT="yes" cleanup_and_setup_lustre
 }
 
 scrub_create_nfiles() {
@@ -196,4 +196,5 @@ run_test 0 "OI scrub performance test"
 # cleanup the system at last
 scrub_cleanup
 complete $SECONDS
+check_and_cleanup_lustre
 exit_status
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 0c99dca..4b7e0db 100755
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -154,6 +154,7 @@ init_test_env() {
 	export RPC_MODE=${RPC_MODE:-false}
 	export DO_CLEANUP=${DO_CLEANUP:-true}
 	export KEEP_ZPOOL=${KEEP_ZPOOL:-false}
+	export CLEANUP_DM_DEV=false
 
 	export MKE2FS=$MKE2FS
 	if [ -z "$MKE2FS" ]; then
@@ -315,6 +316,10 @@ init_test_env() {
 	export LDEV=${LDEV:-"$LUSTRE/scripts/ldev"}
 	[ ! -f "$LDEV" ] && export LDEV=$(which ldev 2> /dev/null)
 
+	export DMSETUP=${DMSETUP:-dmsetup}
+	export DM_DEV_PATH=${DM_DEV_PATH:-/dev/mapper}
+	export LOSETUP=${LOSETUP:-losetup}
+
 	if [ "$ACCEPTOR_PORT" ]; then
 		export PORT_OPT="--port $ACCEPTOR_PORT"
 	fi
@@ -353,6 +358,7 @@ init_test_env() {
 	export RLUSTRE=${RLUSTRE:-$LUSTRE}
 	export RPWD=${RPWD:-$PWD}
 	export I_MOUNTED=${I_MOUNTED:-"no"}
+	export AUSTER_CLEANUP=${AUSTER_CLEANUP:-false}
 	if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mdt.ko -a \
 	     ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mdt.ko -a \
 	     ! -f /lib/modules/$(uname -r)/extra/kernel/fs/lustre/mdt.ko -a \
@@ -1495,50 +1501,381 @@ csa_add() {
 	echo -n "$opts"
 }
 
+#
+# Associate loop device with a given regular file.
+# Return the loop device.
+#
+setup_loop_device() {
+	local facet=$1
+	local file=$2
+
+	do_facet $facet "loop_dev=\\\$($LOSETUP -j $file | cut -d : -f 1);
+			 if [[ -z \\\$loop_dev ]]; then
+				loop_dev=\\\$($LOSETUP -f);
+				$LOSETUP \\\$loop_dev $file || loop_dev=;
+			 fi;
+			 echo -n \\\$loop_dev"
+}
+
+#
+# Detach a loop device.
+#
+cleanup_loop_device() {
+	local facet=$1
+	local loop_dev=$2
+
+	do_facet $facet "! $LOSETUP $loop_dev >/dev/null 2>&1 ||
+			 $LOSETUP -d $loop_dev"
+}
+
+#
+# Check if a given device is a block device.
+#
+is_blkdev() {
+	local facet=$1
+	local dev=$2
+	local size=${3:-""}
+
+	[[ -n "$dev" ]] || return 1
+	do_facet $facet "test -b $dev" || return 1
+	if [[ -n "$size" ]]; then
+		local in=$(do_facet $facet "dd if=$dev of=/dev/null bs=1k \
+					    count=1 skip=$size 2>&1" |
+					    awk '($3 == "in") { print $1 }')
+		[[ "$in" = "1+0" ]] || return 1
+	fi
+}
+
+#
+# Check if a given device is a device-mapper device.
+#
+is_dm_dev() {
+	local facet=$1
+	local dev=$2
+
+	[[ -n "$dev" ]] || return 1
+	do_facet $facet "$DMSETUP status $dev >/dev/null 2>&1"
+}
+
+#
+# Check if a given device is a device-mapper flakey device.
+#
+is_dm_flakey_dev() {
+	local facet=$1
+	local dev=$2
+	local type
+
+	[[ -n "$dev" ]] || return 1
+
+	type=$(do_facet $facet "$DMSETUP status $dev 2>&1" |
+	       awk '{print $3}')
+	[[ $type = flakey ]] && return 0 || return 1
+}
+
+#
+# Check if device-mapper flakey device is supported by the kernel
+# of $facet node or not.
+#
+dm_flakey_supported() {
+	local facet=$1
+
+	do_facet $facet "modprobe dm-flakey;
+			 $DMSETUP targets | grep -q flakey" &> /dev/null
+}
+
+#
+# Get the device-mapper flakey device name of a given facet.
+#
+dm_facet_devname() {
+	local facet=$1
+	[[ $facet = mgs ]] && combined_mgs_mds && facet=mds1
+
+	echo -n ${facet}_flakey
+}
+
+#
+# Get the device-mapper flakey device of a given facet.
+# A device created by dmsetup will appear as /dev/mapper/<device-name>.
+#
+dm_facet_devpath() {
+	local facet=$1
+
+	echo -n $DM_DEV_PATH/$(dm_facet_devname $facet)
+}
+
+#
+# Set a device-mapper device with a new table.
+#
+# The table has the following format:
+# <logical_start_sector> <num_sectors> <target_type> <target_args>
+#
+# flakey <target_args> includes:
+# <destination_device> <offset> <up_interval> <down_interval> \
+# [<num_features> [<feature_arguments>]]
+#
+# linear <target_args> includes:
+# <destination_device> <start_sector>
+#
+dm_set_dev_table() {
+	local facet=$1
+	local dm_dev=$2
+	local target_type=$3
+	local num_sectors
+	local real_dev
+	local tmp
+	local table
+
+	read tmp num_sectors tmp real_dev tmp \
+		<<< $(do_facet $facet "$DMSETUP table $dm_dev")
+
+	case $target_type in
+	flakey)
+		table="0 $num_sectors flakey $real_dev 0 0 1800 1 drop_writes"
+		;;
+	linear)
+		table="0 $num_sectors linear $real_dev 0"
+		;;
+	*) error "invalid target type $target_type" ;;
+	esac
+
+	do_facet $facet "$DMSETUP suspend --nolockfs --noflush $dm_dev" ||
+		error "failed to suspend $dm_dev"
+	do_facet $facet "$DMSETUP load $dm_dev --table \\\"$table\\\"" ||
+		error "failed to load $target_type table into $dm_dev"
+	do_facet $facet "$DMSETUP resume $dm_dev" ||
+		error "failed to resume $dm_dev"
+}
+
+#
+# Set a device-mapper flakey device as "read-only" by using the "drop_writes"
+# feature parameter.
+#
+# drop_writes:
+#	All write I/O is silently ignored.
+#	Read I/O is handled correctly.
+#
+dm_set_dev_readonly() {
+	local facet=$1
+	local dm_dev=${2:-$(dm_facet_devpath $facet)}
+
+	dm_set_dev_table $facet $dm_dev flakey
+}
+
+#
+# Set a device-mapper device to traditional linear mapping mode.
+#
+dm_clear_dev_readonly() {
+	local facet=$1
+	local dm_dev=${2:-$(dm_facet_devpath $facet)}
+
+	dm_set_dev_table $facet $dm_dev linear
+}
+
+#
+# Set the device of a given facet as "read-only".
+#
+set_dev_readonly() {
+	local facet=$1
+	local svc=${facet}_svc
+
+	if [[ $(facet_fstype $facet) = zfs ]] ||
+	   ! dm_flakey_supported $facet; then
+		do_facet $facet $LCTL --device ${!svc} readonly
+	else
+		dm_set_dev_readonly $facet
+	fi
+}
+
+#
+# Get size in 512-byte sectors (BLKGETSIZE64 / 512) of a given device.
+#
+get_num_sectors() {
+	local facet=$1
+	local dev=$2
+	local num_sectors
+
+	num_sectors=$(do_facet $facet "blockdev --getsz $dev 2>/dev/null")
+	[[ ${PIPESTATUS[0]} = 0 && -n "$num_sectors" ]] || num_sectors=0
+	echo -n $num_sectors
+}
+
+#
+# Create a device-mapper device with a given block device or regular file (will
+# be associated with loop device).
+# Return the full path of the device-mapper device.
+#
+dm_create_dev() {
+	local facet=$1
+	local real_dev=$2				   # destination device
+	local dm_dev_name=${3:-$(dm_facet_devname $facet)} # device name
+	local dm_dev=$DM_DEV_PATH/$dm_dev_name		  # device-mapper device
+
+	# check if the device-mapper device to be created already exists
+	if is_dm_dev $facet $dm_dev; then
+		# if the existing device was set to "read-only", then clear it
+		! is_dm_flakey_dev $facet $dm_dev ||
+			dm_clear_dev_readonly $facet $dm_dev
+
+		echo -n $dm_dev
+		return 0
+	fi
+
+	# check if the destination device is a block device, and if not,
+	# associate it with a loop device
+	is_blkdev $facet $real_dev ||
+		real_dev=$(setup_loop_device $facet $real_dev)
+	[[ -n "$real_dev" ]] || { echo -n $real_dev; return 2; }
+
+	# now create the device-mapper device
+	local num_sectors=$(get_num_sectors $facet $real_dev)
+	local table="0 $num_sectors linear $real_dev 0"
+	local rc=0
+
+	do_facet $facet "$DMSETUP create $dm_dev_name --table \\\"$table\\\"" ||
+		{ rc=${PIPESTATUS[0]}; dm_dev=; }
+	do_facet $facet "$DMSETUP mknodes >/dev/null 2>&1"
+
+	echo -n $dm_dev
+	return $rc
+}
+
+#
+# Map the facet name to its device variable name.
+#
+facet_device_alias() {
+	local facet=$1
+	local dev_alias=$facet
+
+	case $facet in
+		fs2mds) dev_alias=mds1_2 ;;
+		fs2ost) dev_alias=ost1_2 ;;
+		fs3ost) dev_alias=ost2_2 ;;
+		*) ;;
+	esac
+
+	echo -n $dev_alias
+}
+
+#
+# Save the original value of the facet device and export the new value.
+#
+export_dm_dev() {
+	local facet=$1
+	local dm_dev=$2
+
+	local active_facet=$(facet_active $facet)
+	local dev_alias=$(facet_device_alias $active_facet)
+	local dev_name=${dev_alias}_dev
+	local dev=${!dev_name}
+
+	if [[ $active_facet = $facet ]]; then
+		local failover_dev=${dev_alias}failover_dev
+		if [[ ${!failover_dev} = $dev ]]; then
+			eval export ${failover_dev}_saved=$dev
+			eval export ${failover_dev}=$dm_dev
+		fi
+	else
+		dev_alias=$(facet_device_alias $facet)
+		local facet_dev=${dev_alias}_dev
+		if [[ ${!facet_dev} = $dev ]]; then
+			eval export ${facet_dev}_saved=$dev
+			eval export ${facet_dev}=$dm_dev
+		fi
+	fi
+
+	eval export ${dev_name}_saved=$dev
+	eval export ${dev_name}=$dm_dev
+}
+
+#
+# Restore the saved value of the facet device.
+#
+unexport_dm_dev() {
+	local facet=$1
+
+	[[ $facet = mgs ]] && combined_mgs_mds && facet=mds1
+	local dev_alias=$(facet_device_alias $facet)
+
+	local saved_dev=${dev_alias}_dev_saved
+	[[ -z ${!saved_dev} ]] ||
+		eval export ${dev_alias}_dev=${!saved_dev}
+
+	saved_dev=${dev_alias}failover_dev_saved
+	[[ -z ${!saved_dev} ]] ||
+		eval export ${dev_alias}failover_dev=${!saved_dev}
+}
+
+#
+# Remove a device-mapper device.
+# If the destination device is a loop device, then also detach it.
+#
+dm_cleanup_dev() {
+	local facet=$1
+	local dm_dev=${2:-$(dm_facet_devpath $facet)}
+	local major
+	local minor
+
+	is_dm_dev $facet $dm_dev || return 0
+
+	read major minor <<< $(do_facet $facet "$DMSETUP table $dm_dev" |
+		awk '{ print $4 }' | awk -F: '{ print $1" "$2 }')
+
+	do_facet $facet "$DMSETUP remove $dm_dev"
+	do_facet $facet "$DMSETUP mknodes >/dev/null 2>&1"
+
+	unexport_dm_dev $facet
+
+	# detach a loop device
+	[[ $major -ne 7 ]] || cleanup_loop_device $facet /dev/loop$minor
+}
+
 mount_facet() {
 	local facet=$1
 	shift
-	local dev=$(facet_active $facet)_dev
+	local active_facet=$(facet_active $facet)
+	local dev_alias=$(facet_device_alias $active_facet)
+	local dev=${dev_alias}_dev
 	local opt=${facet}_opt
 	local mntpt=$(facet_mntpt $facet)
 	local opts="${!opt} $@"
 	local fstype=$(facet_fstype $facet)
 	local devicelabel
+	local dm_dev=${!dev}
 
 	module_loaded lustre || load_modules
 
-	if [ $(facet_fstype $facet) == ldiskfs ] &&
-	   ! do_facet $facet test -b ${!dev}; then
-		opts=$(csa_add "$opts" -o loop)
-	fi
+	case $fstype in
+	ldiskfs)
+		if dm_flakey_supported $facet; then
+			dm_dev=$(dm_create_dev $facet ${!dev})
+			[[ -n "$dm_dev" ]] || dm_dev=${!dev}
+		fi
 
-	if [[ $(facet_fstype $facet) == zfs ]]; then
+		is_blkdev $facet $dm_dev || opts=$(csa_add "$opts" -o loop)
+
+		devicelabel=$(do_facet ${facet} "$E2LABEL $dm_dev");;
+	zfs)
 		# import ZFS storage pool
 		import_zpool $facet || return ${PIPESTATUS[0]}
-	fi
 
-	case $fstype in
-	ldiskfs)
-		devicelabel=$(do_facet ${facet} "$E2LABEL ${!dev}");;
-	zfs)
 		devicelabel=$(do_facet ${facet} "$ZFS get -H -o value \
-						lustre:svname ${!dev}");;
+						lustre:svname $dm_dev");;
 	*)
 		error "unknown fstype!";;
 	esac
 
-	echo "Starting ${facet}: $opts ${!dev} $mntpt"
+	echo "Starting ${facet}: $opts $dm_dev $mntpt"
 	# for testing LU-482 error handling in mount_facets() and test_0a()
 	if [ -f $TMP/test-lu482-trigger ]; then
 		RC=2
 	else
-		do_facet ${facet} "mkdir -p $mntpt; $MOUNT_CMD $opts \
-		                   ${!dev} $mntpt"
+		do_facet ${facet} \
+			"mkdir -p $mntpt; $MOUNT_CMD $opts $dm_dev $mntpt"
 		RC=${PIPESTATUS[0]}
 	fi
 
 	if [ $RC -ne 0 ]; then
-		echo "Start of ${!dev} on ${facet} failed ${RC}"
+		echo "Start of $dm_dev on ${facet} failed ${RC}"
 		return $RC
 	fi
 
@@ -1555,19 +1892,19 @@ mount_facet() {
 	fi
 
 	if [[ $opts =~ .*nosvc.* ]]; then
-		echo "Start ${!dev} without service"
+		echo "Start $dm_dev without service"
 	else
 
 		case $fstype in
 		ldiskfs)
-			wait_update_facet ${facet} "$E2LABEL ${!dev} \
+			wait_update_facet ${facet} "$E2LABEL $dm_dev \
 				2>/dev/null | grep -E ':[a-zA-Z]{3}[0-9]{4}'" \
-				"" || error "${!dev} failed to initialize!";;
+				"" || error "$dm_dev failed to initialize!";;
 		zfs)
 			wait_update_facet ${facet} "$ZFS get -H -o value \
-				lustre:svname ${!dev} 2>/dev/null | \
+				lustre:svname $dm_dev 2>/dev/null | \
 				grep -E ':[a-zA-Z]{3}[0-9]{4}'" "" ||
-				error "${!dev} failed to initialize!";;
+				error "$dm_dev failed to initialize!";;
 
 		*)
 			error "unknown fstype!";;
@@ -1581,11 +1918,13 @@ mount_facet() {
 	fi
 
 
-	label=$(devicelabel ${facet} ${!dev})
-	[ -z "$label" ] && echo no label for ${!dev} && exit 1
+	label=$(devicelabel ${facet} $dm_dev)
+	[ -z "$label" ] && echo no label for $dm_dev && exit 1
 	eval export ${facet}_svc=${label}
 	echo Started ${label}
 
+	export_dm_dev $facet $dm_dev
+
 	return $RC
 }
 
@@ -1595,14 +1934,16 @@ start() {
 	shift
 	local device=$1
 	shift
-	eval export ${facet}_dev=${device}
+	local dev_alias=$(facet_device_alias $facet)
+
+	eval export ${dev_alias}_dev=${device}
 	eval export ${facet}_opt=\"$@\"
 
-	local varname=${facet}failover_dev
+	local varname=${dev_alias}failover_dev
 	if [ -n "${!varname}" ] ; then
-		eval export ${facet}failover_dev=${!varname}
+		eval export ${dev_alias}failover_dev=${!varname}
 	else
-		eval export ${facet}failover_dev=$device
+		eval export ${dev_alias}failover_dev=$device
 	fi
 
 	local mntpt=$(facet_mntpt $facet)
@@ -1621,18 +1962,18 @@ start() {
 }
 
 stop() {
-    local running
-    local facet=$1
-    shift
-    local HOST=`facet_active_host $facet`
-    [ -z $HOST ] && echo stop: no host for $facet && return 0
+	local running
+	local facet=$1
+	shift
+	local HOST=$(facet_active_host $facet)
+	[[ -z $HOST ]] && echo stop: no host for $facet && return 0
 
-    local mntpt=$(facet_mntpt $facet)
+	local mntpt=$(facet_mntpt $facet)
 	running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts || true")
-    if [ ${running} -ne 0 ]; then
-        echo "Stopping $mntpt (opts:$@) on $HOST"
-	do_facet ${facet} $UMOUNT $@ $mntpt
-    fi
+	if [ ${running} -ne 0 ]; then
+		echo "Stopping $mntpt (opts:$@) on $HOST"
+		do_facet ${facet} $UMOUNT $@ $mntpt
+	fi
 
 	# umount should block, but we should wait for unrelated obd's
 	# like the MGS or MGC to also stop.
@@ -1641,6 +1982,13 @@ stop() {
 	if [[ $(facet_fstype $facet) == zfs ]]; then
 		# export ZFS storage pool
 		[ "$KEEP_ZPOOL" = "true" ] || export_zpool $facet
+	elif dm_flakey_supported $facet; then
+		local host=${facet}_HOST
+		local failover_host=${facet}failover_HOST
+		if [[ -n ${!failover_host} && ${!failover_host} != ${!host} ]]||
+			$CLEANUP_DM_DEV || [[ $facet = fs* ]]; then
+			dm_cleanup_dev $facet
+		fi
 	fi
 }
 
@@ -2159,13 +2507,23 @@ facets_up_on_host () {
 }
 
 shutdown_facet() {
-    local facet=$1
+	local facet=$1
+	local affected_facet
+	local affected_facets
+
+	if [[ "$FAILURE_MODE" = HARD ]]; then
+		if [[ $(facet_fstype $facet) = ldiskfs ]] &&
+			dm_flakey_supported $facet; then
+			affected_facets=$(affected_facets $facet)
+			for affected_facet in ${affected_facets//,/ }; do
+				unexport_dm_dev $affected_facet
+			done
+		fi
 
-    if [ "$FAILURE_MODE" = HARD ]; then
-        shutdown_node_hard $(facet_active_host $facet)
-    else
-        stop $facet
-    fi
+		shutdown_node_hard $(facet_active_host $facet)
+	else
+		stop $facet
+	fi
 }
 
 reboot_node() {
@@ -2956,7 +3314,7 @@ replay_barrier() {
 	# handled by stop() and mount_facet() separately, which are used
 	# inside fail() and fail_abort().
 	#
-	do_facet $facet $LCTL --device ${!svc} readonly
+	set_dev_readonly $facet
 	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
 	$LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
@@ -2967,7 +3325,7 @@ replay_barrier_nodf() {
 	local svc=${facet}_svc
 	echo Replay barrier on ${!svc}
 	do_facet $facet $LCTL --device ${!svc} notransno
-	do_facet $facet $LCTL --device ${!svc} readonly
+	set_dev_readonly $facet
 	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
 	$LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
@@ -2977,7 +3335,7 @@ replay_barrier_nosync() {
 	local svc=${facet}_svc
 	echo Replay barrier on ${!svc}
 	do_facet $facet $LCTL --device ${!svc} notransno
-	do_facet $facet $LCTL --device ${!svc} readonly
+	set_dev_readonly $facet
 	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
 	$LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
@@ -3348,16 +3706,20 @@ do_node() {
         $myPDSH $HOST "$LCTL mark \"$@\"" > /dev/null 2>&1 || :
     fi
 
-    if [ "$myPDSH" = "rsh" ]; then
-# we need this because rsh does not return exit code of an executed command
-        local command_status="$TMP/cs"
-        rsh $HOST ":> $command_status"
-        rsh $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin;
-                    cd $RPWD; LUSTRE=\"$RLUSTRE\" sh -c \"$@\") ||
-                    echo command failed >$command_status"
-        [ -n "$($myPDSH $HOST cat $command_status)" ] && return 1 || true
-        return 0
-    fi
+	if [[ "$myPDSH" == "rsh" ]] ||
+	   [[ "$myPDSH" == *pdsh* && "$myPDSH" != *-S* ]]; then
+		# we need this because rsh and pdsh do not return
+		# exit code of an executed command
+		local command_status="$TMP/cs"
+		eval $myPDSH $HOST ":> $command_status"
+		eval $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests;
+				     PATH=\$PATH:/sbin:/usr/sbin;
+				     cd $RPWD;
+				     LUSTRE=\"$RLUSTRE\" sh -c \"$@\") ||
+				     echo command failed >$command_status"
+		[[ -n "$($myPDSH $HOST cat $command_status)" ]] && return 1 ||
+			return 0
+	fi
 
     if $verbose ; then
         # print HOSTNAME for myPDSH="no_dsh"
@@ -3534,6 +3896,8 @@ ostdevname() {
 
 	case $fstype in
 		ldiskfs )
+			local dev=ost${num}_dev
+			[[ -n ${!dev} ]] && eval DEVPTR=${!dev} ||
 			#if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
 			eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};;
 		zfs )
@@ -3580,6 +3944,8 @@ mdsdevname() {
 
 	case $fstype in
 		ldiskfs )
+			local dev=mds${num}_dev
+			[[ -n ${!dev} ]] && eval DEVPTR=${!dev} ||
 			#if $MDSDEVn isn't defined, default is $MDSDEVBASE{n}
 			eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};;
 		zfs )
@@ -3622,9 +3988,10 @@ mgsdevname() {
 	case $fstype in
 	ldiskfs )
 		if [ $(facet_host mgs) = $(facet_host mds1) ] &&
-		   ( [ -z "$MGSDEV" ] || [ $MGSDEV = $(mdsdevname 1) ] ); then
+		   ( [ -z "$MGSDEV" ] || [ $MGSDEV = $MDSDEV1 ] ); then
 			DEVPTR=$(mdsdevname 1)
 		else
+			[[ -n $mgs_dev ]] && DEVPTR=$mgs_dev ||
 			DEVPTR=$MGSDEV
 		fi;;
 	zfs )
@@ -3681,11 +4048,16 @@ mount_ldiskfs() {
 	local dev=$(facet_device $facet)
 	local mnt=${2:-$(facet_mntpt $facet)}
 	local opts
+	local dm_dev=$dev
 
-	if ! do_facet $facet test -b $dev; then
-		opts="-o loop"
+	if dm_flakey_supported $facet; then
+		dm_dev=$(dm_create_dev $facet $dev)
+		[[ -n "$dm_dev" ]] || dm_dev=$dev
 	fi
-	do_facet $facet mount -t ldiskfs $opts $dev $mnt
+	is_blkdev $facet $dm_dev || opts=$(csa_add "$opts" -o loop)
+	export_dm_dev $facet $dm_dev
+
+	do_facet $facet mount -t ldiskfs $opts $dm_dev $mnt
 }
 
 unmount_ldiskfs() {
@@ -3804,7 +4176,7 @@ cleanupall() {
 	nfs_client_mode && return
 	cifs_client_mode && return
 
-	stopall $*
+	CLEANUP_DM_DEV=true stopall $*
 	cleanup_echo_devs
 
 	unload_modules
@@ -4132,18 +4504,23 @@ mountmgs() {
 }
 
 mountmds() {
+	local num
+	local devname
+	local host
+	local varname
 	for num in $(seq $MDSCOUNT); do
-		DEVNAME=$(mdsdevname $num)
-		start mds$num $DEVNAME $MDS_MOUNT_OPTS
-
-		# We started mds, now we should set failover variables properly.
-		# Set mds${num}failover_HOST if unset (the default
-		# failnode).
-		local varname=mds${num}failover_HOST
-		if [ -z "${!varname}" ]; then
-			eval mds${num}failover_HOST=$(facet_host mds$num)
-		fi
-
+		devname=$(mdsdevname $num)
+		start mds$num $devname $MDS_MOUNT_OPTS
+
+		# We started mds$num, now we should set mds${num}_HOST
+		# and mds${num}failover_HOST variables properly if they
+		# are not set.
+		host=$(facet_host mds$num)
+		for varname in mds${num}_HOST mds${num}failover_HOST; do
+			if [[ -z "${!varname}" ]]; then
+				eval $varname=$host
+			fi
+		done
 		if [ $IDENTITY_UPCALL != "default" ]; then
 			switch_identity $num $IDENTITY_UPCALL
 		fi
@@ -4151,18 +4528,23 @@ mountmds() {
 }
 
 mountoss() {
+	local num
+	local devname
+	local host
+	local varname
 	for num in $(seq $OSTCOUNT); do
-		DEVNAME=$(ostdevname $num)
-		start ost$num $DEVNAME $OST_MOUNT_OPTS
-
-		# We started ost$num, now we should set ost${num}failover
-		# variable properly. Set ost${num}failover_HOST if it is not
-		# set (the default failnode).
-		varname=ost${num}failover_HOST
-		if [ -z "${!varname}" ]; then
-			eval ost${num}failover_HOST=$(facet_host ost${num})
-		fi
-
+		devname=$(ostdevname $num)
+		start ost$num $devname $OST_MOUNT_OPTS
+
+		# We started ost$num, now we should set ost${num}_HOST
+		# and ost${num}failover_HOST variables properly if they
+		# are not set.
+		host=$(facet_host ost$num)
+		for varname in ost${num}_HOST ost${num}failover_HOST; do
+			if [[ -z "${!varname}" ]]; then
+				eval $varname=$host
+			fi
+		done
 	done
 }
 
@@ -4567,7 +4949,7 @@ check_and_setup_lustre() {
 	# 1.
 	# both MOUNT and MOUNT2 are not mounted
 	if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then
-		[ "$REFORMAT" = "yes" ] && formatall
+		[ "$REFORMAT" = "yes" ] && CLEANUP_DM_DEV=true formatall
 		# setupall mounts both MOUNT and MOUNT2 (if MOUNT_2 is set)
 		setupall
 		is_mounted $MOUNT || error "NAME=$NAME not mounted"
@@ -4881,7 +5263,7 @@ check_and_cleanup_lustre() {
 		cleanup_mount $MOUNT2
 	fi
 
-	if [ "$I_MOUNTED" = "yes" ]; then
+	if [[ "$I_MOUNTED" = "yes" ]] && ! $AUSTER_CLEANUP; then
 		cleanupall -f || error "cleanup failed"
 		unset I_MOUNTED
 	fi
@@ -7624,15 +8006,20 @@ run_sgpdd () {
 
 # returns the canonical name for an ldiskfs device
 ldiskfs_canon() {
-        local dev="$1"
-        local facet="$2"
-
-        do_facet $facet "dv=\\\$(lctl get_param -n $dev);
-if foo=\\\$(lvdisplay -c \\\$dv 2>/dev/null); then
-    echo dm-\\\${foo##*:};
-else
-    echo \\\$(basename \\\$dv);
-fi;"
+	local dev="$1"
+	local facet="$2"
+
+	do_facet $facet "dv=\\\$($LCTL get_param -n $dev);
+			 if foo=\\\$(lvdisplay -c \\\$dv 2>/dev/null); then
+				echo dm-\\\${foo##*:};
+			 else
+				name=\\\$(basename \\\$dv);
+				if [[ \\\$name = *flakey* ]]; then
+					name=\\\$(lsblk -o NAME,KNAME |
+						awk /\\\$name/'{print \\\$NF}');
+				fi;
+				echo \\\$name;
+			 fi;"
 }
 
 is_sanity_benchmark() {