X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=d82f4bd258c686b6c8b4e7a7f5f360dd33d1d09e;hp=6f89cc8640e520088f7c33280c5416c5854b3f86;hb=37f6357a5c9f;hpb=99420a1830b89a8aba6350b095065d65107f7c0f

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 6f89cc8..d82f4bd 100755
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -154,6 +154,7 @@ init_test_env() {
 	export RPC_MODE=${RPC_MODE:-false}
 	export DO_CLEANUP=${DO_CLEANUP:-true}
 	export KEEP_ZPOOL=${KEEP_ZPOOL:-false}
+	export CLEANUP_DM_DEV=false
 
 	export MKE2FS=$MKE2FS
 	if [ -z "$MKE2FS" ]; then
@@ -292,7 +293,7 @@ init_test_env() {
 	export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod}
 	[ ! -f "$LUSTRE_RMMOD" ] &&
 		export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null)
-	export LUSTRE_ROUTES_COVERSION=${LUSTRE_ROUTES_CONVERSION:-$LUSTRE/scripts/lustre_routes_conversion}
+	export LUSTRE_ROUTES_CONVERSION=${LUSTRE_ROUTES_CONVERSION:-$LUSTRE/scripts/lustre_routes_conversion}
 	[ ! -f "$LUSTRE_ROUTES_CONVERSION" ] &&
 		export LUSTRE_ROUTES_CONVERSION=$(which lustre_routes_conversion 2> /dev/null)
 	export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate}
@@ -315,6 +316,10 @@ init_test_env() {
 	export LDEV=${LDEV:-"$LUSTRE/scripts/ldev"}
 	[ ! -f "$LDEV" ] && export LDEV=$(which ldev 2> /dev/null)
 
+	export DMSETUP=${DMSETUP:-dmsetup}
+	export DM_DEV_PATH=${DM_DEV_PATH:-/dev/mapper}
+	export LOSETUP=${LOSETUP:-losetup}
+
 	if [ "$ACCEPTOR_PORT" ]; then
 		export PORT_OPT="--port $ACCEPTOR_PORT"
 	fi
@@ -353,6 +358,7 @@ init_test_env() {
 	export RLUSTRE=${RLUSTRE:-$LUSTRE}
 	export RPWD=${RPWD:-$PWD}
 	export I_MOUNTED=${I_MOUNTED:-"no"}
+	export AUSTER_CLEANUP=${AUSTER_CLEANUP:-false}
 	if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mdt.ko -a \
 	     ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mdt.ko -a \
 	     ! -f /lib/modules/$(uname -r)/extra/kernel/fs/lustre/mdt.ko -a \
@@ -750,8 +756,10 @@ fs_log_size() {
 
 	case $(facet_fstype $facet) in
 		ldiskfs) size=50;; # largest seen is 44, leave some headroom
-		zfs)     size=$(lctl get_param osc.$FSNAME-OST*.import |
-			awk '/grant_block_size:/ { print $2 * 2; exit; }');;
+		# grant_block_size is in bytes, allow at least 2x max blocksize
+		zfs)     size=$(lctl get_param osc.$FSNAME*.import |
+				awk '/grant_block_size:/ {print $2/512; exit;}')
+			  ;;
 	esac
 
 	echo -n $size
@@ -1467,7 +1475,8 @@ mount_facets () {
 		[ $RC -eq 0 ] && continue
 
 		if [ "$TESTSUITE.$TESTNAME" = "replay-dual.test_0a" ]; then
-			skip "Restart of $facet failed!." && touch $LU482_FAILED
+			skip_noexit "Restart of $facet failed!." &&
+				touch $LU482_FAILED
 		else
 			error "Restart of $facet failed!"
 		fi
@@ -1495,50 +1504,381 @@ csa_add() {
 	echo -n "$opts"
 }
 
+#
+# Associate loop device with a given regular file.
+# Return the loop device.
+#
+setup_loop_device() {
+	local facet=$1
+	local file=$2
+
+	do_facet $facet "loop_dev=\\\$($LOSETUP -j $file | cut -d : -f 1);
+			 if [[ -z \\\$loop_dev ]]; then
+				loop_dev=\\\$($LOSETUP -f);
+				$LOSETUP \\\$loop_dev $file || loop_dev=;
+			 fi;
+			 echo -n \\\$loop_dev"
+}
+
+#
+# Detach a loop device.
+#
+cleanup_loop_device() {
+	local facet=$1
+	local loop_dev=$2
+
+	do_facet $facet "! $LOSETUP $loop_dev >/dev/null 2>&1 ||
+			 $LOSETUP -d $loop_dev"
+}
+
+#
+# Check if a given device is a block device.
+#
+is_blkdev() {
+	local facet=$1
+	local dev=$2
+	local size=${3:-""}
+
+	[[ -n "$dev" ]] || return 1
+	do_facet $facet "test -b $dev" || return 1
+	if [[ -n "$size" ]]; then
+		local in=$(do_facet $facet "dd if=$dev of=/dev/null bs=1k \
+					    count=1 skip=$size 2>&1" |
+					    awk '($3 == "in") { print $1 }')
+		[[ "$in" = "1+0" ]] || return 1
+	fi
+}
+
+#
+# Check if a given device is a device-mapper device.
+#
+is_dm_dev() {
+	local facet=$1
+	local dev=$2
+
+	[[ -n "$dev" ]] || return 1
+	do_facet $facet "$DMSETUP status $dev >/dev/null 2>&1"
+}
+
+#
+# Check if a given device is a device-mapper flakey device.
+#
+is_dm_flakey_dev() {
+	local facet=$1
+	local dev=$2
+	local type
+
+	[[ -n "$dev" ]] || return 1
+
+	type=$(do_facet $facet "$DMSETUP status $dev 2>&1" |
+	       awk '{print $3}')
+	[[ $type = flakey ]] && return 0 || return 1
+}
+
+#
+# Check if device-mapper flakey device is supported by the kernel
+# of $facet node or not.
+#
+dm_flakey_supported() {
+	local facet=$1
+
+	do_facet $facet "modprobe dm-flakey;
+			 $DMSETUP targets | grep -q flakey" &> /dev/null
+}
+
+#
+# Get the device-mapper flakey device name of a given facet.
+#
+dm_facet_devname() {
+	local facet=$1
+	[[ $facet = mgs ]] && combined_mgs_mds && facet=mds1
+
+	echo -n ${facet}_flakey
+}
+
+#
+# Get the device-mapper flakey device of a given facet.
+# A device created by dmsetup will appear as /dev/mapper/<device-name>.
+#
+dm_facet_devpath() {
+	local facet=$1
+
+	echo -n $DM_DEV_PATH/$(dm_facet_devname $facet)
+}
+
+#
+# Set a device-mapper device with a new table.
+#
+# The table has the following format:
+# <logical_start_sector> <num_sectors> <target_type> <target_args>
+#
+# flakey <target_args> includes:
+# <destination_device> <offset> <up_interval> <down_interval> \
+# [<num_features> [<feature_arguments>]]
+#
+# linear <target_args> includes:
+# <destination_device> <start_sector>
+#
+dm_set_dev_table() {
+	local facet=$1
+	local dm_dev=$2
+	local target_type=$3
+	local num_sectors
+	local real_dev
+	local tmp
+	local table
+
+	read tmp num_sectors tmp real_dev tmp \
+		<<< $(do_facet $facet "$DMSETUP table $dm_dev")
+
+	case $target_type in
+	flakey)
+		table="0 $num_sectors flakey $real_dev 0 0 1800 1 drop_writes"
+		;;
+	linear)
+		table="0 $num_sectors linear $real_dev 0"
+		;;
+	*) error "invalid target type $target_type" ;;
+	esac
+
+	do_facet $facet "$DMSETUP suspend --nolockfs --noflush $dm_dev" ||
+		error "failed to suspend $dm_dev"
+	do_facet $facet "$DMSETUP load $dm_dev --table \\\"$table\\\"" ||
+		error "failed to load $target_type table into $dm_dev"
+	do_facet $facet "$DMSETUP resume $dm_dev" ||
+		error "failed to resume $dm_dev"
+}
+
+#
+# Set a device-mapper flakey device as "read-only" by using the "drop_writes"
+# feature parameter.
+#
+# drop_writes:
+#	All write I/O is silently ignored.
+#	Read I/O is handled correctly.
+#
+dm_set_dev_readonly() {
+	local facet=$1
+	local dm_dev=${2:-$(dm_facet_devpath $facet)}
+
+	dm_set_dev_table $facet $dm_dev flakey
+}
+
+#
+# Set a device-mapper device to traditional linear mapping mode.
+#
+dm_clear_dev_readonly() {
+	local facet=$1
+	local dm_dev=${2:-$(dm_facet_devpath $facet)}
+
+	dm_set_dev_table $facet $dm_dev linear
+}
+
+#
+# Set the device of a given facet as "read-only".
+#
+set_dev_readonly() {
+	local facet=$1
+	local svc=${facet}_svc
+
+	if [[ $(facet_fstype $facet) = zfs ]] ||
+	   ! dm_flakey_supported $facet; then
+		do_facet $facet $LCTL --device ${!svc} readonly
+	else
+		dm_set_dev_readonly $facet
+	fi
+}
+
+#
+# Get size in 512-byte sectors (BLKGETSIZE64 / 512) of a given device.
+#
+get_num_sectors() {
+	local facet=$1
+	local dev=$2
+	local num_sectors
+
+	num_sectors=$(do_facet $facet "blockdev --getsz $dev 2>/dev/null")
+	[[ ${PIPESTATUS[0]} = 0 && -n "$num_sectors" ]] || num_sectors=0
+	echo -n $num_sectors
+}
+
+#
+# Create a device-mapper device with a given block device or regular file (will
+# be associated with loop device).
+# Return the full path of the device-mapper device.
+#
+dm_create_dev() {
+	local facet=$1
+	local real_dev=$2				   # destination device
+	local dm_dev_name=${3:-$(dm_facet_devname $facet)} # device name
+	local dm_dev=$DM_DEV_PATH/$dm_dev_name		  # device-mapper device
+
+	# check if the device-mapper device to be created already exists
+	if is_dm_dev $facet $dm_dev; then
+		# if the existing device was set to "read-only", then clear it
+		! is_dm_flakey_dev $facet $dm_dev ||
+			dm_clear_dev_readonly $facet $dm_dev
+
+		echo -n $dm_dev
+		return 0
+	fi
+
+	# check if the destination device is a block device, and if not,
+	# associate it with a loop device
+	is_blkdev $facet $real_dev ||
+		real_dev=$(setup_loop_device $facet $real_dev)
+	[[ -n "$real_dev" ]] || { echo -n $real_dev; return 2; }
+
+	# now create the device-mapper device
+	local num_sectors=$(get_num_sectors $facet $real_dev)
+	local table="0 $num_sectors linear $real_dev 0"
+	local rc=0
+
+	do_facet $facet "$DMSETUP create $dm_dev_name --table \\\"$table\\\"" ||
+		{ rc=${PIPESTATUS[0]}; dm_dev=; }
+	do_facet $facet "$DMSETUP mknodes >/dev/null 2>&1"
+
+	echo -n $dm_dev
+	return $rc
+}
+
+#
+# Map the facet name to its device variable name.
+#
+facet_device_alias() {
+	local facet=$1
+	local dev_alias=$facet
+
+	case $facet in
+		fs2mds) dev_alias=mds1_2 ;;
+		fs2ost) dev_alias=ost1_2 ;;
+		fs3ost) dev_alias=ost2_2 ;;
+		*) ;;
+	esac
+
+	echo -n $dev_alias
+}
+
+#
+# Save the original value of the facet device and export the new value.
+#
+export_dm_dev() {
+	local facet=$1
+	local dm_dev=$2
+
+	local active_facet=$(facet_active $facet)
+	local dev_alias=$(facet_device_alias $active_facet)
+	local dev_name=${dev_alias}_dev
+	local dev=${!dev_name}
+
+	if [[ $active_facet = $facet ]]; then
+		local failover_dev=${dev_alias}failover_dev
+		if [[ ${!failover_dev} = $dev ]]; then
+			eval export ${failover_dev}_saved=$dev
+			eval export ${failover_dev}=$dm_dev
+		fi
+	else
+		dev_alias=$(facet_device_alias $facet)
+		local facet_dev=${dev_alias}_dev
+		if [[ ${!facet_dev} = $dev ]]; then
+			eval export ${facet_dev}_saved=$dev
+			eval export ${facet_dev}=$dm_dev
+		fi
+	fi
+
+	eval export ${dev_name}_saved=$dev
+	eval export ${dev_name}=$dm_dev
+}
+
+#
+# Restore the saved value of the facet device.
+#
+unexport_dm_dev() {
+	local facet=$1
+
+	[[ $facet = mgs ]] && combined_mgs_mds && facet=mds1
+	local dev_alias=$(facet_device_alias $facet)
+
+	local saved_dev=${dev_alias}_dev_saved
+	[[ -z ${!saved_dev} ]] ||
+		eval export ${dev_alias}_dev=${!saved_dev}
+
+	saved_dev=${dev_alias}failover_dev_saved
+	[[ -z ${!saved_dev} ]] ||
+		eval export ${dev_alias}failover_dev=${!saved_dev}
+}
+
+#
+# Remove a device-mapper device.
+# If the destination device is a loop device, then also detach it.
+#
+dm_cleanup_dev() {
+	local facet=$1
+	local dm_dev=${2:-$(dm_facet_devpath $facet)}
+	local major
+	local minor
+
+	is_dm_dev $facet $dm_dev || return 0
+
+	read major minor <<< $(do_facet $facet "$DMSETUP table $dm_dev" |
+		awk '{ print $4 }' | awk -F: '{ print $1" "$2 }')
+
+	do_facet $facet "$DMSETUP remove $dm_dev"
+	do_facet $facet "$DMSETUP mknodes >/dev/null 2>&1"
+
+	unexport_dm_dev $facet
+
+	# detach a loop device
+	[[ $major -ne 7 ]] || cleanup_loop_device $facet /dev/loop$minor
+}
+
 mount_facet() {
 	local facet=$1
 	shift
-	local dev=$(facet_active $facet)_dev
+	local active_facet=$(facet_active $facet)
+	local dev_alias=$(facet_device_alias $active_facet)
+	local dev=${dev_alias}_dev
 	local opt=${facet}_opt
 	local mntpt=$(facet_mntpt $facet)
 	local opts="${!opt} $@"
 	local fstype=$(facet_fstype $facet)
 	local devicelabel
+	local dm_dev=${!dev}
 
 	module_loaded lustre || load_modules
 
-	if [ $(facet_fstype $facet) == ldiskfs ] &&
-	   ! do_facet $facet test -b ${!dev}; then
-		opts=$(csa_add "$opts" -o loop)
-	fi
+	case $fstype in
+	ldiskfs)
+		if dm_flakey_supported $facet; then
+			dm_dev=$(dm_create_dev $facet ${!dev})
+			[[ -n "$dm_dev" ]] || dm_dev=${!dev}
+		fi
 
-	if [[ $(facet_fstype $facet) == zfs ]]; then
+		is_blkdev $facet $dm_dev || opts=$(csa_add "$opts" -o loop)
+
+		devicelabel=$(do_facet ${facet} "$E2LABEL $dm_dev");;
+	zfs)
 		# import ZFS storage pool
 		import_zpool $facet || return ${PIPESTATUS[0]}
-	fi
 
-	case $fstype in
-	ldiskfs)
-		devicelabel=$(do_facet ${facet} "$E2LABEL ${!dev}");;
-	zfs)
 		devicelabel=$(do_facet ${facet} "$ZFS get -H -o value \
-						lustre:svname ${!dev}");;
+						lustre:svname $dm_dev");;
 	*)
 		error "unknown fstype!";;
 	esac
 
-	echo "Starting ${facet}: $opts ${!dev} $mntpt"
+	echo "Starting ${facet}: $opts $dm_dev $mntpt"
 	# for testing LU-482 error handling in mount_facets() and test_0a()
 	if [ -f $TMP/test-lu482-trigger ]; then
 		RC=2
 	else
-		do_facet ${facet} "mkdir -p $mntpt; $MOUNT_CMD $opts \
-		                   ${!dev} $mntpt"
+		do_facet ${facet} \
+			"mkdir -p $mntpt; $MOUNT_CMD $opts $dm_dev $mntpt"
 		RC=${PIPESTATUS[0]}
 	fi
 
 	if [ $RC -ne 0 ]; then
-		echo "Start of ${!dev} on ${facet} failed ${RC}"
+		echo "Start of $dm_dev on ${facet} failed ${RC}"
 		return $RC
 	fi
 
@@ -1555,19 +1895,19 @@ mount_facet() {
 	fi
 
 	if [[ $opts =~ .*nosvc.* ]]; then
-		echo "Start ${!dev} without service"
+		echo "Start $dm_dev without service"
 	else
 
 		case $fstype in
 		ldiskfs)
-			wait_update_facet ${facet} "$E2LABEL ${!dev} \
+			wait_update_facet ${facet} "$E2LABEL $dm_dev \
 				2>/dev/null | grep -E ':[a-zA-Z]{3}[0-9]{4}'" \
-				"" || error "${!dev} failed to initialize!";;
+				"" || error "$dm_dev failed to initialize!";;
 		zfs)
 			wait_update_facet ${facet} "$ZFS get -H -o value \
-				lustre:svname ${!dev} 2>/dev/null | \
+				lustre:svname $dm_dev 2>/dev/null | \
 				grep -E ':[a-zA-Z]{3}[0-9]{4}'" "" ||
-				error "${!dev} failed to initialize!";;
+				error "$dm_dev failed to initialize!";;
 
 		*)
 			error "unknown fstype!";;
@@ -1581,11 +1921,13 @@ mount_facet() {
 	fi
 
 
-	label=$(devicelabel ${facet} ${!dev})
-	[ -z "$label" ] && echo no label for ${!dev} && exit 1
+	label=$(devicelabel ${facet} $dm_dev)
+	[ -z "$label" ] && echo no label for $dm_dev && exit 1
 	eval export ${facet}_svc=${label}
 	echo Started ${label}
 
+	export_dm_dev $facet $dm_dev
+
 	return $RC
 }
 
@@ -1595,14 +1937,16 @@ start() {
 	shift
 	local device=$1
 	shift
-	eval export ${facet}_dev=${device}
+	local dev_alias=$(facet_device_alias $facet)
+
+	eval export ${dev_alias}_dev=${device}
 	eval export ${facet}_opt=\"$@\"
 
-	local varname=${facet}failover_dev
+	local varname=${dev_alias}failover_dev
 	if [ -n "${!varname}" ] ; then
-		eval export ${facet}failover_dev=${!varname}
+		eval export ${dev_alias}failover_dev=${!varname}
 	else
-		eval export ${facet}failover_dev=$device
+		eval export ${dev_alias}failover_dev=$device
 	fi
 
 	local mntpt=$(facet_mntpt $facet)
@@ -1621,18 +1965,18 @@ start() {
 }
 
 stop() {
-    local running
-    local facet=$1
-    shift
-    local HOST=`facet_active_host $facet`
-    [ -z $HOST ] && echo stop: no host for $facet && return 0
-
-    local mntpt=$(facet_mntpt $facet)
-    running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts") || true
-    if [ ${running} -ne 0 ]; then
-        echo "Stopping $mntpt (opts:$@) on $HOST"
-	do_facet ${facet} $UMOUNT $@ $mntpt
-    fi
+	local running
+	local facet=$1
+	shift
+	local HOST=$(facet_active_host $facet)
+	[[ -z $HOST ]] && echo stop: no host for $facet && return 0
+
+	local mntpt=$(facet_mntpt $facet)
+	running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts || true")
+	if [ ${running} -ne 0 ]; then
+		echo "Stopping $mntpt (opts:$@) on $HOST"
+		do_facet ${facet} $UMOUNT $@ $mntpt
+	fi
 
 	# umount should block, but we should wait for unrelated obd's
 	# like the MGS or MGC to also stop.
@@ -1641,6 +1985,13 @@ stop() {
 	if [[ $(facet_fstype $facet) == zfs ]]; then
 		# export ZFS storage pool
 		[ "$KEEP_ZPOOL" = "true" ] || export_zpool $facet
+	elif dm_flakey_supported $facet; then
+		local host=${facet}_HOST
+		local failover_host=${facet}failover_HOST
+		if [[ -n ${!failover_host} && ${!failover_host} != ${!host} ]]||
+			$CLEANUP_DM_DEV || [[ $facet = fs* ]]; then
+			dm_cleanup_dev $facet
+		fi
 	fi
 }
 
@@ -2159,13 +2510,23 @@ facets_up_on_host () {
 }
 
 shutdown_facet() {
-    local facet=$1
+	local facet=$1
+	local affected_facet
+	local affected_facets
+
+	if [[ "$FAILURE_MODE" = HARD ]]; then
+		if [[ $(facet_fstype $facet) = ldiskfs ]] &&
+			dm_flakey_supported $facet; then
+			affected_facets=$(affected_facets $facet)
+			for affected_facet in ${affected_facets//,/ }; do
+				unexport_dm_dev $affected_facet
+			done
+		fi
 
-    if [ "$FAILURE_MODE" = HARD ]; then
-        shutdown_node_hard $(facet_active_host $facet)
-    else
-        stop $facet
-    fi
+		shutdown_node_hard $(facet_active_host $facet)
+	else
+		stop $facet
+	fi
 }
 
 reboot_node() {
@@ -2526,18 +2887,18 @@ sync_all_data() {
 }
 
 wait_zfs_commit() {
+	local zfs_wait=${2:-5}
+
 	# the occupied disk space will be released
-	# only after DMUs are committed
+	# only after TXGs are committed
 	if [[ $(facet_fstype $1) == zfs ]]; then
-		echo "sleep $2 for ZFS OSD"
-		sleep $2
+		echo "sleep $zfs_wait for ZFS $(facet_fstype $1)"
+		sleep $zfs_wait
 	fi
 }
 
 wait_delete_completed_mds() {
-	local MAX_WAIT=${1:-20}
-	# for ZFS, waiting more time for DMUs to be committed
-	local ZFS_WAIT=${2:-5}
+	local max_wait=${1:-20}
 	local mds2sync=""
 	local stime=$(date +%s)
 	local etime
@@ -2554,8 +2915,8 @@ wait_delete_completed_mds() {
 		mds2sync="$mds2sync $node"
 	done
 	if [ -z "$mds2sync" ]; then
-		wait_zfs_commit $SINGLEMDS $ZFS_WAIT
-		return
+		wait_zfs_commit $SINGLEMDS
+		return 0
 	fi
 	mds2sync=$(comma_list $mds2sync)
 
@@ -2567,21 +2928,26 @@ wait_delete_completed_mds() {
 	# do this upon commit
 
 	local WAIT=0
-	while [[ $WAIT -ne $MAX_WAIT ]]; do
+	while [[ $WAIT -ne $max_wait ]]; do
 		changes=$(do_nodes $mds2sync \
 			"$LCTL get_param -n osc.*MDT*.sync_*" | calc_sum)
 		#echo "$node: $changes changes on all"
 		if [[ $changes -eq 0 ]]; then
-			wait_zfs_commit $SINGLEMDS $ZFS_WAIT
-			return
+			wait_zfs_commit $SINGLEMDS
+
+			# the occupied disk space will be released
+			# only after TXGs are committed
+			wait_zfs_commit ost1
+			return 0
 		fi
 		sleep 1
-		WAIT=$(( WAIT + 1))
+		WAIT=$((WAIT + 1))
 	done
 
 	etime=$(date +%s)
 	echo "Delete is not completed in $((etime - stime)) seconds"
 	do_nodes $mds2sync "$LCTL get_param osc.*MDT*.sync_*"
+	return 1
 }
 
 wait_for_host() {
@@ -2689,6 +3055,7 @@ wait_mds_ost_sync () {
 	done
 
 	# show which nodes are not finished.
+	cmd=$(echo $cmd | sed '/-n//')
 	do_nodes $list "$cmd"
 	echo "$facet recovery node $i not done in $WAIT_TIMEOUT sec. $STATUS"
 	return 1
@@ -2736,7 +3103,7 @@ wait_destroy_complete () {
 
 wait_delete_completed() {
 	wait_delete_completed_mds $1 || return $?
-	wait_destroy_complete
+	wait_destroy_complete || return $?
 }
 
 wait_exit_ST () {
@@ -2747,7 +3114,8 @@ wait_exit_ST () {
     local running
     # conf-sanity 31 takes a long time cleanup
     while [ $WAIT -lt 300 ]; do
-        running=$(do_facet ${facet} "lsmod | grep lnet > /dev/null && lctl dl | grep ' ST '") || true
+	running=$(do_facet ${facet} "lsmod | grep lnet > /dev/null &&
+lctl dl | grep ' ST ' || true")
         [ -z "${running}" ] && return 0
         echo "waited $WAIT for${running}"
         [ $INTERVAL -lt 64 ] && INTERVAL=$((INTERVAL + INTERVAL))
@@ -2949,7 +3317,7 @@ replay_barrier() {
 	# handled by stop() and mount_facet() separately, which are used
 	# inside fail() and fail_abort().
 	#
-	do_facet $facet $LCTL --device ${!svc} readonly
+	set_dev_readonly $facet
 	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
 	$LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
@@ -2960,7 +3328,7 @@ replay_barrier_nodf() {
 	local svc=${facet}_svc
 	echo Replay barrier on ${!svc}
 	do_facet $facet $LCTL --device ${!svc} notransno
-	do_facet $facet $LCTL --device ${!svc} readonly
+	set_dev_readonly $facet
 	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
 	$LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
@@ -2970,7 +3338,7 @@ replay_barrier_nosync() {
 	local svc=${facet}_svc
 	echo Replay barrier on ${!svc}
 	do_facet $facet $LCTL --device ${!svc} notransno
-	do_facet $facet $LCTL --device ${!svc} readonly
+	set_dev_readonly $facet
 	do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
 	$LCTL mark "local REPLAY BARRIER on ${!svc}"
 }
@@ -3302,16 +3670,20 @@ do_node() {
         $myPDSH $HOST "$LCTL mark \"$@\"" > /dev/null 2>&1 || :
     fi
 
-    if [ "$myPDSH" = "rsh" ]; then
-# we need this because rsh does not return exit code of an executed command
-        local command_status="$TMP/cs"
-        rsh $HOST ":> $command_status"
-        rsh $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin;
-                    cd $RPWD; LUSTRE=\"$RLUSTRE\" sh -c \"$@\") ||
-                    echo command failed >$command_status"
-        [ -n "$($myPDSH $HOST cat $command_status)" ] && return 1 || true
-        return 0
-    fi
+	if [[ "$myPDSH" == "rsh" ]] ||
+	   [[ "$myPDSH" == *pdsh* && "$myPDSH" != *-S* ]]; then
+		# we need this because rsh and pdsh do not return
+		# exit code of an executed command
+		local command_status="$TMP/cs"
+		eval $myPDSH $HOST ":> $command_status"
+		eval $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests;
+				     PATH=\$PATH:/sbin:/usr/sbin;
+				     cd $RPWD;
+				     LUSTRE=\"$RLUSTRE\" sh -c \"$@\") ||
+				     echo command failed >$command_status"
+		[[ -n "$($myPDSH $HOST cat $command_status)" ]] && return 1 ||
+			return 0
+	fi
 
     if $verbose ; then
         # print HOSTNAME for myPDSH="no_dsh"
@@ -3488,6 +3860,8 @@ ostdevname() {
 
 	case $fstype in
 		ldiskfs )
+			local dev=ost${num}_dev
+			[[ -n ${!dev} ]] && eval DEVPTR=${!dev} ||
 			#if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
 			eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};;
 		zfs )
@@ -3534,6 +3908,8 @@ mdsdevname() {
 
 	case $fstype in
 		ldiskfs )
+			local dev=mds${num}_dev
+			[[ -n ${!dev} ]] && eval DEVPTR=${!dev} ||
 			#if $MDSDEVn isn't defined, default is $MDSDEVBASE{n}
 			eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};;
 		zfs )
@@ -3576,9 +3952,10 @@ mgsdevname() {
 	case $fstype in
 	ldiskfs )
 		if [ $(facet_host mgs) = $(facet_host mds1) ] &&
-		   ( [ -z "$MGSDEV" ] || [ $MGSDEV = $(mdsdevname 1) ] ); then
+		   ( [ -z "$MGSDEV" ] || [ $MGSDEV = $MDSDEV1 ] ); then
 			DEVPTR=$(mdsdevname 1)
 		else
+			[[ -n $mgs_dev ]] && DEVPTR=$mgs_dev ||
 			DEVPTR=$MGSDEV
 		fi;;
 	zfs )
@@ -3635,11 +4012,16 @@ mount_ldiskfs() {
 	local dev=$(facet_device $facet)
 	local mnt=${2:-$(facet_mntpt $facet)}
 	local opts
+	local dm_dev=$dev
 
-	if ! do_facet $facet test -b $dev; then
-		opts="-o loop"
+	if dm_flakey_supported $facet; then
+		dm_dev=$(dm_create_dev $facet $dev)
+		[[ -n "$dm_dev" ]] || dm_dev=$dev
 	fi
-	do_facet $facet mount -t ldiskfs $opts $dev $mnt
+	is_blkdev $facet $dm_dev || opts=$(csa_add "$opts" -o loop)
+	export_dm_dev $facet $dm_dev
+
+	do_facet $facet mount -t ldiskfs $opts $dm_dev $mnt
 }
 
 unmount_ldiskfs() {
@@ -3765,7 +4147,7 @@ cleanupall() {
 	nfs_client_mode && return
 	cifs_client_mode && return
 
-	stopall $*
+	CLEANUP_DM_DEV=true stopall $*
 	cleanup_echo_devs
 
 	unload_modules
@@ -3821,8 +4203,8 @@ mkfs_opts() {
 
 	if [ $type == MDS ]; then
 		opts+=${MDSCAPA:+" --param-mdt.capa=$MDSCAPA"}
-		opts+=${STRIPE_BYTES:+" --param=lov.stripesize=$STRIPE_BYTES"}
-		opts+=${STRIPES_PER_OBJ:+" --param=lov.stripecount=$STRIPES_PER_OBJ"}
+		opts+=${DEF_STRIPE_SIZE:+" --param=lov.stripesize=$DEF_STRIPE_SIZE"}
+		opts+=${DEF_STRIPE_COUNT:+" --param=lov.stripecount=$DEF_STRIPE_COUNT"}
 		opts+=${L_GETIDENTITY:+" --param=mdt.identity_upcall=$L_GETIDENTITY"}
 
 		if [ $fstype == ldiskfs ]; then
@@ -4093,18 +4475,23 @@ mountmgs() {
 }
 
 mountmds() {
+	local num
+	local devname
+	local host
+	local varname
 	for num in $(seq $MDSCOUNT); do
-		DEVNAME=$(mdsdevname $num)
-		start mds$num $DEVNAME $MDS_MOUNT_OPTS
-
-		# We started mds, now we should set failover variables properly.
-		# Set mds${num}failover_HOST if unset (the default
-		# failnode).
-		local varname=mds${num}failover_HOST
-		if [ -z "${!varname}" ]; then
-			eval mds${num}failover_HOST=$(facet_host mds$num)
-		fi
-
+		devname=$(mdsdevname $num)
+		start mds$num $devname $MDS_MOUNT_OPTS
+
+		# We started mds$num, now we should set mds${num}_HOST
+		# and mds${num}failover_HOST variables properly if they
+		# are not set.
+		host=$(facet_host mds$num)
+		for varname in mds${num}_HOST mds${num}failover_HOST; do
+			if [[ -z "${!varname}" ]]; then
+				eval $varname=$host
+			fi
+		done
 		if [ $IDENTITY_UPCALL != "default" ]; then
 			switch_identity $num $IDENTITY_UPCALL
 		fi
@@ -4112,18 +4499,23 @@ mountmds() {
 }
 
 mountoss() {
+	local num
+	local devname
+	local host
+	local varname
 	for num in $(seq $OSTCOUNT); do
-		DEVNAME=$(ostdevname $num)
-		start ost$num $DEVNAME $OST_MOUNT_OPTS
-
-		# We started ost$num, now we should set ost${num}failover
-		# variable properly. Set ost${num}failover_HOST if it is not
-		# set (the default failnode).
-		varname=ost${num}failover_HOST
-		if [ -z "${!varname}" ]; then
-			eval ost${num}failover_HOST=$(facet_host ost${num})
-		fi
-
+		devname=$(ostdevname $num)
+		start ost$num $devname $OST_MOUNT_OPTS
+
+		# We started ost$num, now we should set ost${num}_HOST
+		# and ost${num}failover_HOST variables properly if they
+		# are not set.
+		host=$(facet_host ost$num)
+		for varname in ost${num}_HOST ost${num}failover_HOST; do
+			if [[ -z "${!varname}" ]]; then
+				eval $varname=$host
+			fi
+		done
 	done
 }
 
@@ -4528,7 +4920,7 @@ check_and_setup_lustre() {
 	# 1.
 	# both MOUNT and MOUNT2 are not mounted
 	if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then
-		[ "$REFORMAT" = "yes" ] && formatall
+		[ "$REFORMAT" = "yes" ] && CLEANUP_DM_DEV=true formatall
 		# setupall mounts both MOUNT and MOUNT2 (if MOUNT_2 is set)
 		setupall
 		is_mounted $MOUNT || error "NAME=$NAME not mounted"
@@ -4698,7 +5090,7 @@ run_e2fsck() {
 	if [ -n "$(grep "DNE mode isn't supported" $log)" ]; then
 		rm -f $log
 		if [ $MDSCOUNT -gt 1 ]; then
-			skip "DNE mode isn't supported!"
+			skip_noexit "DNE mode isn't supported!"
 			cleanupall
 			exit_status
 		else
@@ -4842,7 +5234,7 @@ check_and_cleanup_lustre() {
 		cleanup_mount $MOUNT2
 	fi
 
-	if [ "$I_MOUNTED" = "yes" ]; then
+	if [[ "$I_MOUNTED" = "yes" ]] && ! $AUSTER_CLEANUP; then
 		cleanupall -f || error "cleanup failed"
 		unset I_MOUNTED
 	fi
@@ -5381,7 +5773,7 @@ skip_env () {
 	$FAIL_ON_SKIP_ENV && error false $@ || skip $@
 }
 
-skip() {
+skip_noexit() {
 	echo
 	log " SKIP: $TESTSUITE $TESTNAME $@"
 
@@ -5396,8 +5788,13 @@ skip() {
 		echo "$TESTSUITE: SKIP: $TESTNAME $@" >> $TESTSUITELOG || true
 }
 
+skip() {
+	skip_noexit
+	exit 0
+}
+
 build_test_filter() {
-    EXCEPT="$EXCEPT $(testslist_filter)"
+	EXCEPT="$EXCEPT $(testslist_filter)"
 
 	for O in $ONLY; do
 		if [[ $O = [0-9]*-[0-9]* ]]; then
@@ -5471,32 +5868,32 @@ run_test() {
 	ALWAYS_SKIPPED="y"
 	testname=EXCEPT_$1
 	if [ ${!testname}x != x ]; then
-		TESTNAME=test_$1 skip "skipping excluded test $1"
+		TESTNAME=test_$1 skip_noexit "skipping excluded test $1"
 		return 0
 	fi
 	testname=EXCEPT_$base
 	if [ ${!testname}x != x ]; then
-		TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)"
+		TESTNAME=test_$1 skip_noexit "skipping excluded test $1 (base $base)"
 		return 0
 	fi
 	testname=EXCEPT_ALWAYS_$1
 	if [ ${!testname}x != x ]; then
-		TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1"
+		TESTNAME=test_$1 skip_noexit "skipping ALWAYS excluded test $1"
 		return 0
 	fi
 	testname=EXCEPT_ALWAYS_$base
 	if [ ${!testname}x != x ]; then
-		TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1 (base $base)"
+		TESTNAME=test_$1 skip_noexit "skipping ALWAYS excluded test $1 (base $base)"
 		return 0
 	fi
 	testname=EXCEPT_SLOW_$1
 	if [ ${!testname}x != x ]; then
-		TESTNAME=test_$1 skip "skipping SLOW test $1"
+		TESTNAME=test_$1 skip_noexit "skipping SLOW test $1"
 		return 0
 	fi
 	testname=EXCEPT_SLOW_$base
 	if [ ${!testname}x != x ]; then
-		TESTNAME=test_$1 skip "skipping SLOW test $1 (base $base)"
+		TESTNAME=test_$1 skip_noexit "skipping SLOW test $1 (base $base)"
 		return 0
 	fi
 
@@ -6320,7 +6717,7 @@ restore_lustre_params() {
 	local val
 
 	while IFS=" =" read facet name val; do
-		do_facet $facet "$LCTL set_param -n $name $val"
+		do_facet $facet "$LCTL set_param -n $name=$val"
 	done
 }
 
@@ -6780,7 +7177,7 @@ create_pool() {
 	local fsname=${1%%.*}
 	local poolname=${1##$fsname.}
 
-	trap "destroy_test_pools $fsname" EXIT
+	stack_trap "destroy_test_pools $fsname" EXIT
 	do_facet mgs lctl pool_new $1
 	local RC=$?
 	# get param should return err unless pool is created
@@ -7585,15 +7982,20 @@ run_sgpdd () {
 
 # returns the canonical name for an ldiskfs device
 ldiskfs_canon() {
-        local dev="$1"
-        local facet="$2"
-
-        do_facet $facet "dv=\\\$(lctl get_param -n $dev);
-if foo=\\\$(lvdisplay -c \\\$dv 2>/dev/null); then
-    echo dm-\\\${foo##*:};
-else
-    echo \\\$(basename \\\$dv);
-fi;"
+	local dev="$1"
+	local facet="$2"
+
+	do_facet $facet "dv=\\\$($LCTL get_param -n $dev);
+			 if foo=\\\$(lvdisplay -c \\\$dv 2>/dev/null); then
+				echo dm-\\\${foo##*:};
+			 else
+				name=\\\$(basename \\\$dv);
+				if [[ \\\$name = *flakey* ]]; then
+					name=\\\$(lsblk -o NAME,KNAME |
+						awk /\\\$name/'{print \\\$NF}');
+				fi;
+				echo \\\$name;
+			 fi;"
 }
 
 is_sanity_benchmark() {
@@ -8336,8 +8738,7 @@ lss_gen_conf()
 
 	if ! combined_mgs_mds ; then
 		[ $(facet_fstype mgs) != zfs ] &&
-			skip "Lustre snapshot 1 only works for ZFS backend" &&
-			exit 0
+			skip "Lustre snapshot 1 only works for ZFS backend"
 
 		local host=$(facet_active_host mgs)
 		local dir=$(dirname $(facet_vdevice mgs))
@@ -8351,8 +8752,7 @@ lss_gen_conf()
 
 	for num in `seq $MDSCOUNT`; do
 		[ $(facet_fstype mds$num) != zfs ] &&
-			skip "Lustre snapshot 1 only works for ZFS backend" &&
-			exit 0
+			skip "Lustre snapshot 1 only works for ZFS backend"
 
 		lss_gen_conf_one mds$num MDT $((num - 1)) ||
 			lss_err "generate lss conf (mds$num)"
@@ -8360,8 +8760,7 @@ lss_gen_conf()
 
 	for num in `seq $OSTCOUNT`; do
 		[ $(facet_fstype ost$num) != zfs ] &&
-			skip "Lustre snapshot 1 only works for ZFS backend" &&
-			exit 0
+			skip "Lustre snapshot 1 only works for ZFS backend"
 
 		lss_gen_conf_one ost$num OST $((num - 1)) ||
 			lss_err "generate lss conf (ost$num)"
@@ -8524,19 +8923,173 @@ check_clients_full() {
 	done
 }
 
+#Changelogs
+__changelog_deregister() {
+	local facet=$1
+	local mdt="$(facet_svc $facet)"
+	local cl_user=$2
+	local rc=0
+
+	# skip cleanup if no user registered for this MDT
+	[ -z "$cl_user" ] && echo "$mdt: no changelog user" && return 0
+	# user is no longer registered, skip cleanup
+	changelog_users "$facet" | grep -q "$cl_user" ||
+		{ echo "$mdt: changelog user '$cl_user' not found"; return 0; }
+
+	# From this point, if any operation fails, it is an error
+	__changelog_clear $facet $cl_user 0 ||
+		error_noexit "$mdt: changelog_clear $cl_user 0 fail: $rc"
+	do_facet $facet $LCTL --device $mdt changelog_deregister $cl_user ||
+		error_noexit "$mdt: changelog_deregister '$cl_user' fail: $rc"
+}
+
+declare -Ax CL_USERS
+changelog_register() {
+	for M in $(seq $MDSCOUNT); do
+		local facet=mds$M
+		local mdt="$(facet_svc $facet)"
+		stack_trap "do_facet $facet $LCTL \
+			set_param mdd.$mdt.changelog_mask=-hsm" EXIT
+		do_facet $facet $LCTL set_param mdd.$mdt.changelog_mask=+hsm ||
+			error "$mdt: changelog_mask=+hsm failed: $?"
+
+		local cl_user
+		cl_user=$(do_facet $facet \
+				  $LCTL --device $mdt changelog_register -n) ||
+			error "$mdt: register changelog user failed: $?"
+		stack_trap "__changelog_deregister $facet $cl_user" EXIT
+
+		stack_trap "CL_USERS[$facet]='${CL_USERS[$facet]}'" EXIT
+		# Bash does not support nested arrays, but the format of a
+		# cl_user is constrained enough to use whitespaces as separators
+		CL_USERS[$facet]+="$cl_user "
+	done
+	echo "Registered $MDSCOUNT changelog users: '${CL_USERS[@]% }'"
+}
+
+changelog_deregister() {
+	local cl_user
+
+	for facet in "${!CL_USERS[@]}"; do
+		for cl_user in ${CL_USERS[$facet]}; do
+			__changelog_deregister $facet $cl_user || return $?
+		done
+		unset CL_USERS[$facet]
+	done
+}
+
+changelog_users() {
+	local facet=$1
+	local service=$(facet_svc $facet)
+
+	do_facet $facet $LCTL get_param -n mdd.$service.changelog_users
+}
+
+changelog_user_rec() {
+	local facet=$1
+	local cl_user=$2
+	local service=$(facet_svc $facet)
+
+	changelog_users $facet | awk '$1 == "'$cl_user'" { print $2 }'
+}
+
+changelog_chmask() {
+	local mask=$1
+
+	do_nodes $(comma_list $(mdts_nodes)) \
+		$LCTL set_param mdd.*.changelog_mask="$mask"
+}
+
+# usage: __changelog_clear FACET CL_USER [+]INDEX
+__changelog_clear()
+{
+	local facet=$1
+	local mdt="$(facet_svc $facet)"
+	local cl_user=$2
+	local -i rec
+
+	case "$3" in
+	+*)
+		# Remove the leading '+'
+		rec=${3:1}
+		rec+=$(changelog_user_rec $facet $cl_user)
+		;;
+	*)
+		rec=$3
+		;;
+	esac
+
+	if [ $rec -eq 0 ]; then
+		echo "$mdt: clear the changelog for $cl_user of all records"
+	else
+		echo "$mdt: clear the changelog for $cl_user to record #$rec"
+	fi
+	$LFS changelog_clear $mdt $cl_user $rec
+}
+
+# usage: changelog_clear [+]INDEX
+#
+# If INDEX is prefixed with '+', increment every changelog user's record index
+# by INDEX. Otherwise, clear the changelog up to INDEX for every changelog
+# users.
+changelog_clear() {
+	local rc
+	for facet in ${!CL_USERS[@]}; do
+		for cl_user in ${CL_USERS[$facet]}; do
+			__changelog_clear $facet $cl_user $1 || rc=${rc:-$?}
+		done
+	done
+
+	return ${rc:-0}
+}
+
+changelog_dump() {
+	for M in $(seq $MDSCOUNT); do
+		local facet=mds$M
+		local mdt="$(facet_svc $facet)"
+
+		$LFS changelog $mdt | sed -e 's/^/'$mdt'./'
+	done
+}
+
+changelog_extract_field() {
+	local cltype=$1
+	local file=$2
+	local identifier=$3
+
+	changelog_dump | gawk "/$cltype.*$file$/ {
+		print gensub(/^.* "$identifier'(\[[^\]]*\]).*$/,"\\1",1)}' |
+		tail -1
+}
+
 restore_layout() {
 	local dir=$1
 	local layout=$2
 
-	[ -z "$layout" ] && return
+	[ ! -d "$dir" ] && return
+
+	[ -z "$layout" ] && {
+		$LFS setstripe -d $dir || error "error deleting stripe '$dir'"
+		return
+	}
 
 	setfattr -n trusted.lov -v $layout $dir ||
-		error "error restoring layout \"$layout\" to \"$dir\""
+		error "error restoring layout '$layout' to '$dir'"
 }
 
+# save the layout of a directory, the returned string will be used by
+# restore_layout() to restore the layout
 save_layout() {
 	local dir=$1
-	local str=$(getfattr -n trusted.lov --absolute-names -e hex $dir |
-			awk -F'=' '/trusted.lov/{print $2}')
+	local str=$(getfattr -n trusted.lov --absolute-names -e hex $dir \
+		    2> /dev/null | awk -F'=' '/trusted.lov/{ print $2 }')
 	echo "$str"
 }
+
+# save layout of a directory and restore it at exit
+save_layout_restore_at_exit() {
+	local dir=$1
+	local layout=$(save_layout $dir)
+
+	stack_trap "restore_layout $dir $layout" EXIT
+}