X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Fconf-sanity.sh;h=acf020d790c8db0e7aedeb1aa745d3affe51ed2b;hp=7a9a105e600af71dddb2c63d99b791b73b14a821;hb=63e17799a369e2ff0b140fd41dc5d7d8656d2bf0;hpb=e26d7cc3992252e5fce5a51aee716f933b04c13a

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 7a9a105..acf020d 100644
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -8,7 +8,7 @@ SRCDIR=$(dirname $0)
 PTLDEBUG=${PTLDEBUG:--1}
 LUSTRE=${LUSTRE:-$(dirname $0)/..}
 . $LUSTRE/tests/test-framework.sh
-init_test_env $@
+init_test_env "$@"
 init_logging
 
 #                                  tool to create lustre filesystem images
@@ -98,7 +98,7 @@ reformat() {
 
 start_mgs () {
 	echo "start mgs service on $(facet_active_host mgs)"
-	start mgs $(mgsdevname) $MGS_MOUNT_OPTS $@
+	start mgs $(mgsdevname) $MGS_MOUNT_OPTS "$@"
 }
 
 start_mdt() {
@@ -108,7 +108,7 @@ start_mdt() {
 	shift 1
 
 	echo "start mds service on `facet_active_host $facet`"
-	start $facet ${dev} $MDS_MOUNT_OPTS $@ || return 94
+	start $facet ${dev} $MDS_MOUNT_OPTS "$@" || return 94
 }
 
 stop_mdt_no_force() {
@@ -136,7 +136,7 @@ start_mds() {
 	local num
 
 	for num in $(seq $MDSCOUNT); do
-		start_mdt $num $@ || return 94
+		start_mdt $num "$@" || return 94
 	done
 	for num in $(seq $MDSCOUNT); do
 		wait_clients_import_state ${CLIENTS:-$HOSTNAME} mds${num} FULL
@@ -147,7 +147,7 @@ start_mgsmds() {
 	if ! combined_mgs_mds ; then
 		start_mgs
 	fi
-	start_mds $@
+	start_mds "$@"
 }
 
 stop_mds() {
@@ -165,7 +165,7 @@ stop_mgs() {
 
 start_ost() {
 	echo "start ost1 service on `facet_active_host ost1`"
-	start ost1 $(ostdevname 1) $OST_MOUNT_OPTS $@ || return 95
+	start ost1 $(ostdevname 1) $OST_MOUNT_OPTS "$@" || return 95
 	wait_clients_import_state ${CLIENTS:-$HOSTNAME} ost1 FULL
 }
 
@@ -177,7 +177,7 @@ stop_ost() {
 
 start_ost2() {
 	echo "start ost2 service on `facet_active_host ost2`"
-	start ost2 $(ostdevname 2) $OST_MOUNT_OPTS $@ || return 92
+	start ost2 $(ostdevname 2) $OST_MOUNT_OPTS "$@" || return 92
 	wait_clients_import_state ${CLIENTS:-$HOSTNAME} ost2 FULL
 }
 
@@ -199,7 +199,7 @@ umount_client() {
 	local mountpath=$1
 	shift
 	echo "umount lustre on $mountpath....."
-	zconf_umount $HOSTNAME $mountpath $@ || return 97
+	zconf_umount $HOSTNAME $mountpath "$@" || return 97
 }
 
 manual_umount_client(){
@@ -332,7 +332,7 @@ test_4() {
 	setup
 	touch $DIR/$tfile || error "touch $DIR/$tfile failed"
 	stop_ost || error "Unable to stop OST1"
-	umount_client $MOUNT -f || error “unmount $MOUNT failed”
+	umount_client $MOUNT -f || error "unmount $MOUNT failed"
 	cleanup_nocli
 	eno=$?
 	# ok for ost to fail shutdown
@@ -704,13 +704,13 @@ test_18() {
 	setup
 	check_mount || error "check_mount failed"
 
-        echo "check journal size..."
-        local FOUNDSIZE=$(do_facet $SINGLEMDS "$DEBUGFS -c -R 'stat <8>' $MDSDEV" | awk '/Size: / { print $NF; exit;}')
-        if [ $FOUNDSIZE -gt $((32 * 1024 * 1024)) ]; then
-                log "Success: mkfs creates large journals. Size: $((FOUNDSIZE >> 20))M"
-        else
-                error "expected journal size > 32M, found $((FOUNDSIZE >> 20))M"
-        fi
+	echo "check journal size..."
+	local FOUNDSIZE=$(do_facet $SINGLEMDS "$DEBUGFS -c -R 'stat <8>' $MDSDEV" | awk '/Size: / { print $NF; exit;}')
+	if [ $FOUNDSIZE -gt $((32 * 1024 * 1024)) ]; then
+		log "Success: mkfs creates large journals. Size: $((FOUNDSIZE >> 20))M"
+	else
+		error "expected journal size > 32M, found $((FOUNDSIZE >> 20))M"
+	fi
 
 	cleanup || error "cleanup failed with rc $?"
 
@@ -1253,37 +1253,42 @@ test_30a() {
 	setup_noconfig
 
 	echo Big config llog
-	TEST="llite.$FSNAME-*.max_read_ahead_whole_mb"
-	ORIG=$($LCTL get_param -n $TEST)
-	LIST=(1 2 3 4 5 4 3 2 1 2 3 4 5 4 3 2 1 2 3 4 5)
-	for i in ${LIST[@]}; do
-		set_persistent_param_and_check client "$TEST" \
-			"$FSNAME.llite.max_read_ahead_whole_mb" $i
+	local path="llite.$FSNAME-*"
+	local cpath="$FSNAME.llite"
+	local param="max_read_ahead_whole_mb"
+
+	local test="${path}.$param"
+	local conf="${cpath}.$param"
+
+	local orig=$($LCTL get_param -n $test)
+	local list=(1 2 3 4 5 4 3 2 1 2 3 4 5 4 3 2 1 2 3 4 5)
+	for i in ${list[@]}; do
+		set_persistent_param_and_check client $test $conf $i
 	done
 	# make sure client restart still works
 	umount_client $MOUNT
 	mount_client $MOUNT || error "mount_client $MOUNT failed"
-	[ "$($LCTL get_param -n $TEST)" -ne "$i" ] &&
-		error "Param didn't stick across restart $($TEST) != $i"
+	[ "$($LCTL get_param -n $test)" -ne "$i" ] &&
+		error "Param didn't stick across restart $($test) != $i"
 	pass
 
 	echo Erase parameter setting
 	if [[ $PERM_CMD == *"set_param -P"* ]]; then
-		do_facet mgs "$PERM_CMD -d $TEST" ||
-			error "Erase param $TEST failed"
+		do_facet mgs "$PERM_CMD -d $test" ||
+			error "Erase param $test failed"
 	else
-		do_facet mgs "$PERM_CMD \
-			      -d $FSNAME.llite.max_read_ahead_whole_mb" ||
-			error "Erase param $FSNAME.llite.max_read_ahead_whole_mb failed"
+		do_facet mgs "$PERM_CMD -d $conf" ||
+			error "Erase param $conf failed"
 	fi
 	umount_client $MOUNT
 	mount_client $MOUNT || error "mount_client $MOUNT failed"
-	FINAL=$($LCTL get_param -n $TEST)
-	echo "deleted (default) value=$FINAL, orig=$ORIG"
-	ORIG=${ORIG%%.[0-9]*}
-	FINAL=${FINAL%%.[0-9]*}
+	local final=$($LCTL get_param -n $test)
+	echo "deleted (default) value=$final, orig=$orig"
+	orig=${orig%%.[0-9]*}
+	final=${final%%.[0-9]*}
 	# assumes this parameter started at the default value
-	[ "$FINAL" -eq "$ORIG" ] || fail "Deleted value=$FINAL, orig=$ORIG"
+	[ "$final" -eq "$orig" ] ||
+		error "Deleted value=$final -ne orig=$orig"
 
 	cleanup || error "cleanup failed with rc $?"
 }
@@ -1362,7 +1367,8 @@ run_test 31 "Connect to non-existent node (shouldn't crash)"
 T32_QID=60000
 T32_BLIMIT=40960 # Kbytes
 T32_ILIMIT=4
-
+T32_PRJID=1000
+T32_PROLIMIT=$((T32_BLIMIT/10))
 #
 # This is not really a test but a tool to create new disk
 # image tarballs for the upgrade tests.
@@ -1380,14 +1386,24 @@ test_32newtarball() {
 	local remote_dir
 	local striped_dir
 	local pushd_dir
+	local pfl_dir
+	local pfl_file
+	local dom_dir
+	local dom_file
+	local flr_dir
+	local flr_file
+	local pj_quota_dir
+	local pj_quota_file
+	local target_dir
 
 	if [ $FSNAME != t32fs -o \( -z "$MDSDEV" -a -z "$MDSDEV1" \) -o	\
-	     $OSTCOUNT -ne 1 -o	-z "$OSTDEV1" ]; then
-		error "Needs FSNAME=t32fs MDSCOUNT=2 "			\
-		      "MDSDEV1=<nonexistent_file>"			\
-		      "MDSDEV2=<nonexistent_file>"			\
-		      "(or MDSDEV, in the case of b1_8)"		\
-		      "OSTCOUNT=1 OSTDEV1=<nonexistent_file>"
+	$OSTCOUNT -ne 2 -o -z "$OSTDEV1" ]; then
+		error   "Needs FSNAME=t32fs MDSCOUNT=2 "		\
+			"MDSDEV1=<nonexistent_file> "			\
+			"MDSDEV2=<nonexistent_file> "			\
+			"(or MDSDEV, in the case of b1_8) "		\
+			"OSTCOUNT=2 OSTDEV1=<nonexistent_file> "	\
+			"OSTDEV2=<nonexistent_file>"
 	fi
 
 	mkdir $tmp || {
@@ -1397,9 +1413,6 @@ test_32newtarball() {
 
 	mkdir $tmp/src || return 1
 	tar cf - -C $src . | tar xf - -C $tmp/src
-	dd if=/dev/zero of=$tmp/src/t32_qf_old bs=1M \
-		count=$(($T32_BLIMIT / 1024 / 4))
-	chown $T32_QID.$T32_QID $tmp/src/t32_qf_old
 
 	# format ost with comma-separated NIDs to verify LU-4460
 	local failnid="$(h2nettype 1.2.3.4),$(h2nettype 4.3.2.1)"
@@ -1412,20 +1425,104 @@ test_32newtarball() {
 	$LFS setquota -u $T32_QID -b 0 -B $T32_BLIMIT -i 0 -I $T32_ILIMIT \
 		/mnt/$FSNAME
 
-	tar cf - -C $tmp/src . | tar xf - -C /mnt/$FSNAME
-
 	if [[ $MDSCOUNT -ge 2 ]]; then
 		remote_dir=/mnt/$FSNAME/remote_dir
 		$LFS mkdir -i 1 $remote_dir
 		tar cf - -C $tmp/src . | tar xf - -C $remote_dir
 
-		if [[ "$MDS1_VERSION" -ge $(version_code 2.7.0) ]]; then
+		target_dir=$remote_dir
+		if [[ $MDS1_VERSION -ge $(version_code 2.7.0) ]]; then
 			striped_dir=/mnt/$FSNAME/striped_dir_old
 			$LFS mkdir -i 1 -c 2 $striped_dir
 			tar cf - -C $tmp/src . | tar xf - -C $striped_dir
 		fi
+	else
+		tar cf - -C $tmp/src . | tar xf - -C /mnt/$FSNAME
+		target_dir=/mnt/$FSNAME
 	fi
 
+	# add project quota #
+	[[ $mds1_FSTYPE == "ldiskfs" &&
+		$MDS1_VERSION -gt $(version_code 2.9.55) ]] ||
+	[[ $mds1_FSTYPE == "zfs" &&
+		$MDS1_VERSION -gt $(version_code 2.10.53) ]] && {
+		pj_quota_dir=$target_dir/project_quota_dir
+		pj_quota_file_old=$pj_quota_dir/pj_quota_file_old
+
+		enable_project_quota
+		set_mdt_qtype ugp ||
+			error "enable mdt quota failed"
+
+		set_ost_qtype ugp ||
+			error "enable ost quota failed"
+
+		mkdir -p $pj_quota_dir
+		$LFS setquota -p $T32_PRJID -b 0 -B $T32_PROLIMIT -i 0 \
+			-I $T32_ILIMIT $pj_quota_dir ||
+			error "setquota -p $T32_PRJID failed"
+
+		$LFS setstripe $pj_quota_file_old -c 1 ||
+			error "setstripe $pj_quota_file_old failed"
+
+		chown $T32_QID:$T32_QID $pj_quota_file_old ||
+			error "chown $pj_quota_file_old failed"
+
+		change_project -p $T32_PRJID $pj_quota_file_old
+
+		mkdir -p $tmp/src/project_quota_dir
+		cp $pj_quota_file_old $tmp/src/project_quota_dir/
+	}
+
+	#####################
+	tar cf - -C $tmp/src . | tar xf - -C /mnt/$FSNAME
+
+	#if [[ $MDSCOUNT -ge 2 ]]; then
+	#	remote_dir=/mnt/$FSNAME/remote_dir
+	#	$LFS mkdir -i 1 $remote_dir
+	#	tar cf - -C $tmp/src . | tar xf - -C $remote_dir
+
+	#	if [[ "$MDS1_VERSION" -ge $(version_code 2.7.0) ]]; then
+	#		striped_dir=/mnt/$FSNAME/striped_dir_old
+	#		$LFS mkdir -i 1 -c 2 $striped_dir
+	#		tar cf - -C $tmp/src . | tar xf - -C $striped_dir
+	#	fi
+	#fi
+
+	# PFL file #
+	if [[ $MDS1_VERSION -ge $(version_code 2.9.51) ]]; then
+		pfl_dir=$target_dir/pfl_dir
+		pfl_file=$pfl_dir/pfl_file
+		mkdir -p $pfl_dir
+		$LFS setstripe -E 2M -c 1 -o 0 -E -1 -S 2M -c 1 -o 1 \
+					$pfl_file ||
+			error "Create PFL file failed"
+
+		dd if=/dev/urandom of=$pfl_file bs=1k count=3k
+		mkdir -p $tmp/src/pfl_dir
+		cp $pfl_file $tmp/src/pfl_dir/
+	fi
+
+	############
+	# DoM / FLR file #
+	if [[ $MDS1_VERSION -ge $(version_code 2.10.56) ]]; then
+		dom_dir=$target_dir/dom_dir
+		dom_file=$dom_dir/dom_file
+		flr_dir=$target_dir/flr_dir
+		flr_file=$flr_dir/flr_file
+
+		mkdir -p $dom_dir
+		$LFS setstripe -E 1M -L mdt -E -1 -S 4M $dom_file
+		dd if=/dev/urandom of=$dom_file bs=1k count=2k
+		mkdir -p $tmp/src/dom_dir
+		cp $dom_file $tmp/src/dom_dir
+	# FLR #
+		mkdir -p $flr_dir
+		LFS mirror create -N2 $flr_file
+		dd if=/dev/urandom of=$flr_file bs=1k count=1
+		mkdir -p $tmp/src/flr_dir
+		cp $flr_file $tmp/src/flr_dir
+	fi
+	############
 	stopall
 
 	mkdir $tmp/img || return 1
@@ -1474,6 +1571,7 @@ test_32newtarball() {
 	kill -s USR1 $pid
 	wait $pid
 
+	#################
 	stopall
 
 	pushd $tmp/src
@@ -1494,7 +1592,14 @@ test_32newtarball() {
 			devname=$(mdsvdevname $num)
 		mv $devname $tmp/img
 	done
-	mv $OSTDEV1 $tmp/img
+
+	for num in $(seq $OSTCOUNT); do
+		local devname=$(ostdevname $num)
+		local facet=oss$num
+		[[ $(facet_fstype $facet) != zfs ]] ||
+			devname=$(ostdevname $num)
+		mv $devname $tmp/img
+	done
 
 	version=$(sed -e 's/\(^[0-9]\+\.[0-9]\+\)\(.*$\)/\1/' $tmp/img/commit |
 			  sed -e 's/\./_/g')	# E.g., "1.8.7" -> "1_8"
@@ -1521,8 +1626,9 @@ t32_check() {
 
 	local IMGTYPE="$mds1_FSTYPE"
 
-	tarballs=$($r find $RLUSTRE/tests -maxdepth 1 \
-		   -name \'disk*-$IMGTYPE.tar.bz2\')
+	[[ -n "$T32IMAGE" ]] && tarballs=$T32IMAGE ||
+		tarballs=$($r find $RLUSTRE/tests -maxdepth 1 \
+			   -name \'disk*-$IMGTYPE.tar.bz2\')
 
 	[ -z "$tarballs" ] && skip "No applicable tarballs found"
 }
@@ -1532,6 +1638,7 @@ t32_test_cleanup() {
 	local facet=$SINGLEMDS
 	local rc=$?
 
+	echo "== cleanup with rc=$rc =="
 	if $shall_cleanup_lustre; then
 		umount $tmp/mnt/lustre || rc=$?
 	fi
@@ -1544,6 +1651,9 @@ t32_test_cleanup() {
 	if $shall_cleanup_ost; then
 		$r $UMOUNT $tmp/mnt/ost || rc=$?
 	fi
+	if $shall_cleanup_ost1; then
+		$r $UMOUNT $tmp/mnt/ost1 || rc=$?
+	fi
 
 	$r rm -rf $tmp
 	rm -rf $tmp
@@ -1703,6 +1813,21 @@ t32_verify_quota() {
 	return 0
 }
 
+getquota() {
+	local spec=$4
+	local uuid=$3
+	local mnt=$5
+
+	sync_all_data > /dev/null 2>&1 || true
+
+	[ "$uuid" != "global" ] || uuid=$mnt
+
+	$LFS quota -v "$1" "$2" $mnt |
+		awk 'BEGIN { num='$spec' } { if ($1 == "'$uuid'") \
+		{ if (NF == 1) { getline } else { num++ } ; print $num;} }' \
+		| tr -d "*"
+}
+
 t32_test() {
 	local tarball=$1
 	local writeconf=$2
@@ -1710,11 +1835,17 @@ t32_test() {
 	local dom_upgrade=${dom_upgrade:-"no"}
 	local project_upgrade=${project_upgrade:-"no"}
 	local ff_convert=${ff_convert:-"no"}
+	local pfl_upgrade=${pfl_upgrade:-"no"}
+	local project_quota_upgrade=${project_quota_upgrade:-"no"}
+	local dom_new_upgrade=${dom_new_upgrade:-"no"}
+	local flr_upgrade=${flr_upgrade:-"no"}
 	local shall_cleanup_mdt=false
 	local shall_cleanup_mdt1=false
 	local shall_cleanup_ost=false
+	local shall_cleanup_ost1=false
 	local shall_cleanup_lustre=false
 	local mdt2_is_available=false
+	local ost2_is_available=false
 	local node=$(facet_active_host $SINGLEMDS)
 	local r="do_node $node"
 	local tmp=$TMP/t32
@@ -1735,9 +1866,14 @@ t32_test() {
 	local mdt_dev=$tmp/mdt
 	local mdt2_dev=$tmp/mdt2
 	local ost_dev=$tmp/ost
+	local ost2_dev=$tmp/ost2
 	local stripe_index
 	local stripe_count
 	local dir
+	local pfl_file=$tmp/mnt/lustre/remote_dir/pfl_dir/pfl_file
+	local flr_file=$tmp/mnt/lustre/remote_dir/flr_dir/flr_file
+	local dom_file=$tmp/mnt/lustre/remote_dir/dom_dir/dom_file
+	local quota_dir=$tmp/mnt/lustre/remote_dir/project_quota_dir
 
 	combined_mgs_mds || stop_mgs || error "Unable to stop MGS"
 	trap 'trap - RETURN; t32_test_cleanup' RETURN
@@ -1746,7 +1882,7 @@ t32_test() {
 	nid=$($r $LCTL list_nids | head -1)
 
 	mkdir -p $tmp/mnt/lustre || error "mkdir $tmp/mnt/lustre failed"
-	$r mkdir -p $tmp/mnt/{mdt,mdt1,ost}
+	$r mkdir -p $tmp/mnt/{mdt,mdt1,ost,ost1}
 	$r tar xjvf $tarball -S -C $tmp || {
 		error_noexit "Unpacking the disk image tarball"
 		return 1
@@ -1779,8 +1915,10 @@ t32_test() {
 			ff_convert="no"
 
 	! $r test -f $mdt2_dev || mdt2_is_available=true
+	! $r test -f $ost2_dev || ost2_is_available=true
 
 	if [[ "$mds1_FSTYPE" == zfs ]]; then
+		echo "== fstype is zfs =="
 		# import pool first
 		local poolname
 		local poolname_list="t32fs-mdt1 t32fs-ost1"
@@ -1800,12 +1938,14 @@ t32_test() {
 		mdt_dev=t32fs-mdt1/mdt1
 		ost_dev=t32fs-ost1/ost1
 		! $mdt2_is_available || mdt2_dev=t32fs-mdt2/mdt2
+		! $ost2_is_available || ost2_dev=t32fs-ost2/ost2
 		wait_update_facet $SINGLEMDS "$ZPOOL list |
 			awk '/^t32fs-mdt1/ { print \\\$1 }'" "t32fs-mdt1" || {
 				error_noexit "import zfs pool failed"
 				return 1
 			}
 	elif [ "$project_upgrade" != "no" ]; then
+		echo "== project upgrade =="
 		! $r tune2fs -O project $mdt_dev &&
 			error_noexit "enable project on mdt0 failed" &&
 				return 1
@@ -1813,7 +1953,10 @@ t32_test() {
 			error_noexit "enable project on mdt failed" &&
 				return 1
 		! $r tune2fs -O project $ost_dev &&
-			error_noexit "enable project on mdt failed" &&
+			error_noexit "enable project on ost failed" &&
+				return 1
+		$ost2_is_available && ! $r tune2fs -O project $ost2_dev &&
+			error_noexit "enable project on ost2 failed" &&
 				return 1
 	fi
 
@@ -1834,6 +1977,7 @@ t32_test() {
 	fi
 
 	if [ "$writeconf" ]; then
+		echo "== writeconf mdt tunefs and quota =="
 		mopts=writeconf
 		if [ "$mds1_FSTYPE" == ldiskfs ]; then
 			mopts="loop,$mopts"
@@ -1842,32 +1986,58 @@ t32_test() {
 				error_noexit "Enable mdt quota feature"
 				return 1
 			}
+
+			if [ "$project_quota_upgrade" != "no" ]; then
+				echo "== enable mdt2 project quota =="
+				$r $TUNE2FS -O project $mdt_dev || {
+					$r losetup -a
+					error_noexit "tune2fs $mdt_dev failed"
+					return 1
+				}
+			fi
+
 			if $mdt2_is_available; then
 				$r $TUNEFS --quota $mdt2_dev || {
 					$r losetup -a
-					error_noexit "Enable mdt quota feature"
+					error_noexit "Enable mdt2 quota feature"
 					return 1
 				}
+				if [ "$project_quota_upgrade" != "no" ]; then
+					echo "== enable mdt2 project quota =="
+					$r $TUNE2FS -O project $mdt2_dev || {
+						$r losetup -a
+						error_noexit \
+						    "tune2fs $mdt2_dev failed"
+						return 1
+					}
+				fi
 			fi
 		fi
 	else
+		echo "== replace nids =="
 		if [ -n "$($LCTL list_nids | grep -v '\(tcp\|lo\)[[:digit:]]*$')" ]; then
 			[[ "$MGS_VERSION" -ge $(version_code 2.3.59) ]] ||
 			skip "LU-2200: Cannot run over IB w/o lctl replace_nids "
 				"(Need MGS version at least 2.3.59)"
+		fi
 
-			local osthost=$(facet_active_host ost1)
-			local ostnid=$(do_node $osthost $LCTL list_nids | head -1)
+		local osthost=$(facet_active_host ost1)
+		local ostnid=$(do_node $osthost $LCTL list_nids | head -1)
 
-			mopts=nosvc
-			if [ "$mds1_FSTYPE" == ldiskfs ]; then
-				mopts="loop,$mopts"
-			fi
-			$r $MOUNT_CMD -o $mopts $mdt_dev $tmp/mnt/mdt
-			$r $LCTL replace_nids $fsname-OST0000 $ostnid
-			$r $LCTL replace_nids $fsname-MDT0000 $nid
-			$r $UMOUNT $tmp/mnt/mdt
+		mopts=nosvc
+		if [ "$mds1_FSTYPE" == ldiskfs ]; then
+			mopts="loop,$mopts"
+		fi
+		$r $MOUNT_CMD -o $mopts $mdt_dev $tmp/mnt/mdt
+		$r $LCTL replace_nids $fsname-OST0000 $ostnid
+		if $ost2_is_available; then
+			$r $LCTL replace_nids $fsname-OST0001 $ostnid
+		fi
+		$r $LCTL replace_nids $fsname-MDT0000 $nid
+		if $mdt2_is_available; then
+			$r $LCTL replace_nids $fsname-MDT0001 $nid
 		fi
+		$r $UMOUNT $tmp/mnt/mdt
 
 		mopts=exclude=$fsname-OST0000
 		if [ "$mds1_FSTYPE" == ldiskfs ]; then
@@ -1885,23 +2055,26 @@ t32_test() {
 	shall_cleanup_mdt=true
 
 	if $mdt2_is_available; then
+		echo "== mdt2 available =="
 		mopts=mgsnode=$nid,$mopts
 		$r $MOUNT_CMD -o $mopts $mdt2_dev $tmp/mnt/mdt1 || {
 			$r losetup -a
 			error_noexit "Mounting the MDT"
 			return 1
 		}
+		shall_cleanup_mdt1=true
+		echo "== mount new MDT $mdt2_dev =="
 
-		echo "mount new MDT....$mdt2_dev"
-		$r $LCTL set_param -n mdt.${fsname}*.enable_remote_dir=1 ||
+		$r $LCTL set_param -n mdt.${fsname}*.enable_remote_dir=1 || {
 			error_noexit "enable remote dir create failed"
+			return 1
+		}
 
-		shall_cleanup_mdt1=true
 	elif [ "$dne_upgrade" != "no" ]; then
 		local fs2mdsdev=$(mdsdevname 1_2)
 		local fs2mdsvdev=$(mdsvdevname 1_2)
 
-		echo "mkfs new MDT on ${fs2mdsdev}...."
+		echo "== mkfs new MDT on ${fs2mdsdev} =="
 		if [ "$mds1_FSTYPE" == ldiskfs ]; then
 			mkfsoptions="--mkfsoptions=\\\"-J size=8\\\""
 		fi
@@ -1920,7 +2093,7 @@ t32_test() {
 			return 1
 		}
 
-		echo "mount new MDT....$fs2mdsdev"
+		echo "== mount new MDT....$fs2mdsdev =="
 		$r $MOUNT_CMD -o $mopts $fs2mdsdev $tmp/mnt/mdt1 || {
 			error_noexit "mount mdt1 failed"
 			return 1
@@ -1945,15 +2118,50 @@ t32_test() {
 		error_noexit "tunefs.lustre before mounting the OST"
 		return 1
 	}
+
+	if $ost2_is_available; then
+		$r $TUNEFS --dryrun $ost2_dev || {
+			error_noexit "tunefs.lustre before mounting the OST"
+			return 1
+		}
+	fi
+
 	if [ "$writeconf" ]; then
+		echo "== writeconf and ost quota tunefs =="
 		mopts=mgsnode=$nid,$writeconf
-		if [ "$mds1_FSTYPE" == ldiskfs ]; then
+		if [ "$ost1_FSTYPE" == ldiskfs ]; then
 			mopts="loop,$mopts"
 			$r $TUNEFS --quota $ost_dev || {
 				$r losetup -a
 				error_noexit "Enable ost quota feature"
 				return 1
 			}
+
+			if [ "$project_quota_upgrade" != "no" ]; then
+				echo "== enable ost project quota =="
+				$r $TUNE2FS -O project $ost_dev || {
+					$r losetup -a
+					error_noexit "tune2fs $ost_dev failed"
+					return 1
+				}
+			fi
+
+			if $ost2_is_available; then
+				$r $TUNEFS --quota $ost2_dev || {
+					$r losetup -a
+					error_noexit "Enable ost2 quota feature"
+					return 1
+				}
+				if [ "$project_quota_upgrade" != "no" ]; then
+					echo "== enable ost2 project quota =="
+					$r $TUNE2FS -O project $ost2_dev || {
+						$r losetup -a
+						error_noexit \
+						"tune2fs $ost2_dev failed"
+						return 1
+					}
+				fi
+			fi
 		fi
 	else
 		mopts=mgsnode=$nid
@@ -1966,6 +2174,15 @@ t32_test() {
 		error_noexit "Mounting the OST"
 		return 1
 	}
+
+	if $ost2_is_available; then
+		$r $MOUNT_CMD -onomgs -o$mopts $ost2_dev $tmp/mnt/ost1 || {
+			error_noexit "Mounting the OST2"
+			return 1
+		}
+		shall_cleanup_ost1=true
+	fi
+
 	shall_cleanup_ost=true
 
 	uuid=$($r $LCTL get_param -n obdfilter.$fsname-OST0000.uuid) || {
@@ -1977,7 +2194,19 @@ t32_test() {
 		return 1
 	fi
 
-	if [[ $PERM_CMD == *"set_param -P"* ]]; then
+	if $ost2_is_available; then
+		uuid=$($r $LCTL get_param -n obdfilter.$fsname-OST0001.uuid) ||{
+			error_noexit "Getting OST1 UUID"
+			return 1
+		}
+		if [ "$uuid" != $fsname-OST0001_UUID ]; then
+			error_noexit "Unexpected OST1 UUID: \"$uuid\""
+			return 1
+		fi
+	fi
+
+	if [[ $PERM_CMD =~ "set_param" ]]; then
+		echo "== perm cmd = '$PERM_CMD' =="
 		$r $PERM_CMD osc.$fsname-OST0000*.import=connection=$nid || {
 			error_noexit "Setting OST \"failover.node\""
 			return 1
@@ -2003,6 +2232,7 @@ t32_test() {
 			return 1
 		}
 	else
+		echo "== perm cmd = '$PERM_CMD' =="
 		$r $PERM_CMD $fsname-OST0000.failover.node=$nid || {
 			error_noexit "Setting OST \"failover.node\""
 			return 1
@@ -2037,6 +2267,7 @@ t32_test() {
 	}
 
 	if [ "$ff_convert" != "no" -a "$ost1_FSTYPE" == ldiskfs ]; then
+		echo "== ff convert ($ff_convert) =="
 		$r $LCTL lfsck_start -M $fsname-OST0000 || {
 			error_noexit "Start OI scrub on OST0"
 			return 1
@@ -2060,8 +2291,9 @@ t32_test() {
 		}
 	fi
 
-	if [ "$dne_upgrade" != "no" ]; then
-		if [[ $PERM_CMD == *"set_param -P"* ]]; then
+	if [[ "$dne_upgrade" != "no" ]] || $mdt2_is_available; then
+		echo "== dne upgrate ($dne_upgrade) or mdt2 available =="
+		if [[ $PERM_CMD =~ "set_param" ]]; then
 			$r $PERM_CMD mdc.$fsname-MDT0001*.import=connection=$nid || {
 				error_noexit "Setting MDT1 \"failover.node\""
 				return 1
@@ -2092,6 +2324,7 @@ t32_test() {
 	fi
 
 	if [ "$writeconf" ]; then
+		echo "== writeconf and client mount =="
 		$MOUNT_CMD $nid:/$fsname $tmp/mnt/lustre || {
 			error_noexit "Mounting the client"
 			return 1
@@ -2106,6 +2339,7 @@ t32_test() {
 		}
 
 		if $r test -f $tmp/list; then
+			echo "== list verification =="
 			#
 			# There is not a Test Framework API to copy files to or
 			# from a remote node.
@@ -2153,11 +2387,12 @@ t32_test() {
 				return 1
 			fi
 		else
-			echo "list verification skipped"
+			echo "== list verification skipped =="
 		fi
 
+		#non-dom upgrade to dom
 		if [ "$dom_upgrade" != "no" ]; then
-			echo "Check DoM file can be created"
+			echo "== dom upgrade =="
 			$LFS setstripe -E 1M -L mdt -E EOF $tmp/mnt/lustre/dom || {
 				error_noexit "Verify DoM creation"
 				return 1
@@ -2185,12 +2420,41 @@ t32_test() {
 			}
 		fi
 
+		#dom upgrade
+		#$LFS setstripe -E 1M -L mdt -E -1 -S 4M $dom_file
+		if [ "$dom_new_upgrade" != "no" ]; then
+			if ! $mdt2_is_available; then
+				dom_file=$tmp/mnt/lustre/dom_dir/dom_file
+			fi
+			echo "== check DoM file can be accessed =="
+			[ $($LFS getstripe -I1 -L $dom_file) == "mdt" ] || {
+				error_noexit "Verify a DoM file"
+				return 1
+			}
+			[ $(stat -c%s $dom_file) == $((2 * 1024 * 1024)) ] || {
+				error_noexit "DoM: bad size after write"
+				return 1
+			}
+		fi
+
+		if [ "$flr_upgrade" != "no" ]; then
+			if ! $mdt2_is_available; then
+				flr_file=$tmp/mnt/lustre/flr_dir/flr_file
+			fi
+			local mirror_count=$($LFS getstripe -N $flr_file)
+			echo "== check FLR file =="
+			[ $mirror_count == 2 ] || {
+				error_noexit "FLR mirror count wrong"
+				return 1
+			}
+		fi
+
 		if [ "$dne_upgrade" != "no" ]; then
+			echo "== dne upgrade striped_dir  =="
 			$LFS mkdir -i 1 -c2 $tmp/mnt/lustre/striped_dir || {
 				error_noexit "set striped dir failed"
 				return 1
 			}
-
 			$LFS setdirstripe -D -c2 $tmp/mnt/lustre/striped_dir
 
 			pushd $tmp/mnt/lustre
@@ -2204,9 +2468,73 @@ t32_test() {
 			popd
 		fi
 
+		if [ "$pfl_upgrade" != "no" ]; then
+			local comp_size
+			local comp_cnt
+			local stripe_size
+			local stripe_cnt
+			local comp_id
+
+			echo "== check PFL file =="
+			if ! $mdt2_is_available; then
+				pfl_file=$tmp/mnt/lustre/pfl_dir/pfl_file
+			fi
+			comp_cnt=$($LFS getstripe --component-count $pfl_file)
+			[ $comp_cnt == 2 ] || {
+				error_noexit "wrong comp_cnt $comp_cnt"
+				return 1
+			}
+
+			comp_size=$($LFS getstripe -I1 -E $pfl_file)
+			[ $comp_size == 2097152 ] || {
+				error_noexit "wrong component size $comp_size"
+				return 1
+			}
+
+			comp_id=$($LFS getstripe -I1 -i $pfl_file)
+			[ $comp_id == 0 ] || {
+				error_noexit "wrong comp id $comp_id"
+				return 1
+			}
+
+			comp_id=$($LFS getstripe -I2 -i $pfl_file)
+			[ $comp_id -eq 1 ] || {
+				error_noexit "wrong comp id $comp_id"
+				return 1
+			}
+
+			stripe_size=$($LFS getstripe -I1 -S $pfl_file)
+			[ $stripe_size -eq 1048576 ] || {
+				error_noexit "wrong stripe size $stripe_size"
+				return 1
+			}
+
+			stripe_size=$($LFS getstripe -I2 -S $pfl_file)
+			[ $comp_size -eq 2097152 ] || {
+				error_noexit "wrong component size $comp_size"
+				return 1
+			}
+		fi
+
+		if [ "$project_quota_upgrade" != "no" ]; then
+			if ! $mdt2_is_available; then
+				quota_dir=$tmp/mnt/lustre/project_quota_dir
+			fi
+			local hardlimit
+
+			echo "== check Project Quota =="
+			hardlimit=$(getquota -p $T32_PRJID global 3 \
+							${tmp}/mnt/lustre)
+			[ $hardlimit == $T32_PROLIMIT ] || {
+				error_noexit "wrong hardlimit $hardlimit"
+				return 1
+			}
+		fi
+
 		# If it is upgrade from DNE (2.5), then rename the remote dir,
 		# which is created in 2.5 to striped dir.
 		if $mdt2_is_available && [[ "$dne_upgrade" != "no" ]]; then
+			echo "== rename remote_dir =="
 			stripe_index=$($LFS getdirstripe -i	\
 				       $tmp/mnt/lustre/remote_dir)
 
@@ -2225,6 +2553,7 @@ t32_test() {
 		# If it is upgraded from DNE (2.7), then move the striped dir
 		# which was created in 2.7 to the new striped dir.
 		if $mdt2_is_available && [[ "$dne_upgrade" != "no" ]] &&
+			echo "== rename striped dir =="
 			[[ -d $tmp/mnt/lustre/striped_dir_old ]]; then
 			stripe_count=$($LFS getdirstripe -c	\
 				       $tmp/mnt/lustre/striped_dir_old)
@@ -2252,24 +2581,40 @@ t32_test() {
 		}
 
 		if $r test -f $tmp/sha1sums; then
+			echo "== checking sha1sums =="
 			# LU-2393 - do both sorts on same node to ensure locale
 			# is identical
 			$r cat $tmp/sha1sums | sort -k 2 >$tmp/sha1sums.orig
-			if [ "$dne_upgrade" != "no" ]; then
-				pushd $tmp/mnt/lustre/striped_dir
+			if [[ "$dne_upgrade" != "no" ]]; then
+				if [[ -d $tmp/mnt/lustre/striped_dir/remote_dir ]]; then
+					pushd $tmp/mnt/lustre/striped_dir/remote_dir
+				else
+					pushd $tmp/mnt/lustre/striped_dir
+				fi
+			elif [[ "$pfl_upgrade" != "no" ]] ||
+				[[ "$flr_upgrade" != "no" ]] ||
+				[[ "$dom_new_upgrade" != "no" ]] ||
+				[[ "$project_quota_upgrade" != "no" ]] ||
+				[[ -d $tmp/mnt/lustre/remote_dir ]]; then
+				pushd $tmp/mnt/lustre/remote_dir
 			else
 				pushd $tmp/mnt/lustre
 			fi
 
 			find ! -path "*remote_dir*" ! -path "*striped_dir*" \
-				! -name .lustre -type f -exec sha1sum {} \; |
+			     ! -name .lustre -type f -exec sha1sum {} \; |
 				sort -k 2 >$tmp/sha1sums || {
 				popd
 				error_noexit "sha1sum"
 				return 1
 			}
+			local save=$PWD
+
 			popd
 			if ! diff -ub $tmp/sha1sums.orig $tmp/sha1sums; then
+				echo "==** find $save files start **=="
+				find $save -type f -ls
+				echo "==** find $save files finish **=="
 				error_noexit "sha1sum verification failed"
 				return 1
 			fi
@@ -2278,6 +2623,7 @@ t32_test() {
 			# if upgrade from DNE(2.7), then check striped directory
 			if $mdt2_is_available &&
 			   [[ "$dne_upgrade" != "no" ]]; then
+				echo "== mdt2 upgrade sha1sum check =="
 				local new_dir="$tmp/mnt/lustre/striped_dir"
 				local striped_dir_old="$new_dir/striped_dir_old"
 
@@ -2286,10 +2632,12 @@ t32_test() {
 					dir_list+=" $striped_dir_old"
 
 				for dir in $dir_list; do
+					echo "== dir = $dir =="
 					pushd $dir
-					find ! -name .lustre -type f	\
+					find ! -name .lustre -type f \
 						-exec sha1sum {} \; |
 						sort -k 2 >$tmp/sha1sums || {
+							ls -alR
 							popd
 							error_noexit "sha1sum"
 							return 1
@@ -2297,6 +2645,7 @@ t32_test() {
 					popd
 					if ! diff -ub $tmp/sha1sums.orig \
 						$tmp/sha1sums; then
+						ls -alR $dir
 						error_noexit "sha1sum $dir" \
 							     "failed"
 						return 1
@@ -2304,7 +2653,20 @@ t32_test() {
 				done
 			fi
 		else
-			echo "sha1sum verification skipped"
+			echo "== sha1sum verification skipped =="
+		fi
+
+		# PFL write test after sha1sum check
+		if [ "$pfl_upgrade" != "no" ]; then
+			local rw_len=$((3 * 1034 * 1024))
+
+			if ! $mdt2_is_available; then
+				pfl_file=$tmp/mnt/lustre/pfl_dir/pfl_file
+			fi
+			small_write $pfl_file $rw_len || {
+				error_noexit "PFL RW Failed"
+				return 1
+			}
 		fi
 
 		if [ "$dne_upgrade" != "no" ]; then
@@ -2320,7 +2682,7 @@ t32_test() {
 			$r $LCTL set_param -n	\
 				mdt.${fsname}*.enable_remote_dir=1 2>/dev/null
 
-			echo "test migration"
+			echo "== test migration =="
 			pushd $tmp/mnt/lustre
 			for dir in $(find ! -name .lustre ! -name . -type d); do
 				mdt_index=$($LFS getdirstripe -i $dir)
@@ -2419,8 +2781,19 @@ t32_test() {
 			error_noexit "Unmounting the OST"
 			return 1
 		}
-		if [[ "$mds1_FSTYPE" == zfs ]]; then
-		    $r "$ZPOOL export t32fs-ost1"
+		if $ost2_is_available; then
+			$r $UMOUNT $tmp/mnt/ost1 || {
+				error_noexit "Unmounting the OST1"
+				return 1
+			}
+			shall_cleanup_ost1=false
+		fi
+
+		if [[ $ost1_FSTYPE == zfs ]]; then
+			$r "$ZPOOL export t32fs-ost1"
+			if $ost2_is_available; then
+				$r "$ZPOOL export t32fs-ost2"
+			fi
 		fi
 		shall_cleanup_ost=false
 
@@ -2492,12 +2865,8 @@ test_32c() {
 	local tarball
 	local rc=0
 
-	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
 	t32_check
 	for tarball in $tarballs; do
-		# Do not support 1_8 and 2_1 direct upgrade to DNE2 anymore */
-		[[ "$tarball" =~ "1_8" ]] && echo "skip $tarball" && continue
-		[[ "$tarball" =~ "2_1" ]] && echo "skip $tarball" && continue
 		banner "testing $tarball upgrade with DNE"
 		load_modules
 		dne_upgrade=yes t32_test $tarball writeconf || rc=$?
@@ -2513,7 +2882,7 @@ test_32d() {
 
 	t32_check
 	for tarball in $tarballs; do
-		banner "testing $tarball upgrade with ff convert and project upgrade"
+		banner "testing $tarball upgrade with ff and project upgrade"
 		project_upgrade="no"
 		[[ "$MDS1_VERSION" -ge $(version_code 2.13.54) ]] &&
 			[[ "$tarball" =~ "disk2_4-ldiskfs" ]] &&
@@ -2535,7 +2904,8 @@ test_32e() {
 
 	t32_check
 	for tarball in $tarballs; do
-		[[ "$tarball" =~ "2_9" ]] || continue
+		[[ "$tarball" =~ "2_9" ]] ||
+			{ echo "skip $(basename $tarball)"; continue; }
 		#load_modules
 		banner "testing $tarball upgrade with DoM"
 		dom_upgrade=yes t32_test $tarball writeconf || let "rc += $?"
@@ -2544,8 +2914,47 @@ test_32e() {
 }
 run_test 32e "dom upgrade test"
 
+test_32f() {
+	[[ $MDS1_VERSION -ge $(version_code 2.10.56) ]] ||
+		skip "Need MDS version at least 2.10.56"
+
+	local tarballs
+	local tarball
+	local rc=0
+
+	t32_check
+	for tarball in $tarballs; do
+		[[ "$tarball" =~ "2_10" ]] ||
+			{ echo "skip $(basename $tarball)"; continue; }
+		pfl_upgrade=yes project_quota_upgrade=yes \
+		t32_test $tarball writeconf || let "rc += $?"
+	done
+	return $rc
+}
+run_test 32f "pfl upgrade test"
+
+test_32g() {
+	[[ $MDS1_VERSION -ge $(version_code 2.10.56) ]] ||
+		skip "Need MDS version at least 2.10.56"
+
+	local tarballs
+	local tarball
+	local rc=0
+
+	t32_check
+	for tarball in $tarballs; do
+		[[ $tarball =~ "2_12" ]] ||
+			{ echo "skip $(basename $tarball)"; continue; }
+		flr_upgrade=yes dom_new_upgrade=yes \
+		t32_test $tarball writeconf || let "rc += $?"
+	done
+	return $rc
+}
+run_test 32g "flr/dom upgrade test"
+
 test_33a() { # bug 12333, was test_33
-	local FSNAME2=test-123
+	local FSNAME2=test-$testnum
+
 	local MDSDEV=$(mdsdevname ${SINGLEMDS//mds/})
 	local mkfsoptions
 
@@ -2554,6 +2963,7 @@ test_33a() { # bug 12333, was test_33
 	if [ -z "$fs2ost_DEV" -o -z "$fs2mds_DEV" ]; then
 		local dev=${SINGLEMDS}_dev
 		local MDSDEV=${!dev}
+
 		is_blkdev $SINGLEMDS $MDSDEV &&
 			skip_env "mixed loopback and real device not working"
 	fi
@@ -2575,7 +2985,7 @@ test_33a() { # bug 12333, was test_33
 		--reformat $mgs_flag $mkfsoptions $fs2mdsdev $fs2mdsvdev ||
 		exit 10
 	add fs2ost $(mkfs_opts ost1 ${fs2ostdev}) --mgsnode=$MGSNID \
-		--fsname=${FSNAME2} --index=8191 --reformat $fs2ostdev \
+		--fsname=${FSNAME2} --index=0x1fff --reformat $fs2ostdev \
 		$fs2ostvdev || exit 10
 
 	start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_fs2 EXIT INT
@@ -2597,6 +3007,15 @@ test_33a() { # bug 12333, was test_33
 		error "$LFS getstripe $MOUNT2/hosts failed"
 
 	umount $MOUNT2
+
+	# test lctl del_ost on large index
+	do_facet mgs "$LCTL del_ost -t ${FSNAME2}-OST1fff" ||
+		error "del_ost failed with $?"
+	$MOUNT_CMD $MGSNID:/${FSNAME2} $MOUNT2 || error "$MOUNT_CMD failed"
+	echo "ok."
+	$LFS df | grep -q OST1fff && error "del_ost did not remove OST1fff!"
+	umount $MOUNT2
+
 	stop fs2ost -f
 	stop fs2mds -f
 	cleanup_nocli || error "cleanup_nocli failed with $?"
@@ -2604,10 +3023,10 @@ test_33a() { # bug 12333, was test_33
 run_test 33a "Mount ost with a large index number"
 
 test_33b() {	# was test_34
-        setup
+	setup
 
-        do_facet client dd if=/dev/zero of=$MOUNT/24 bs=1024k count=1
-        # Drop lock cancelation reply during umount
+	do_facet client dd if=/dev/zero of=$MOUNT/24 bs=1024k count=1
+	# Drop lock cancelation reply during umount
 	#define OBD_FAIL_LDLM_CANCEL_NET			0x304
 	do_facet client $LCTL set_param fail_loc=0x80000304
 	#lctl set_param debug=-1
@@ -2617,7 +3036,7 @@ test_33b() {	# was test_34
 run_test 33b "Drop cancel during umount"
 
 test_34a() {
-        setup
+	setup
 	do_facet client "sh runmultiop_bg_pause $DIR/file O_c"
 	manual_umount_client
 	rc=$?
@@ -2859,23 +3278,23 @@ test_36() { # 12743
 	DFAVAIL=$(echo $STRING | cut -d, -f3)
 	DFFREE=$(($DFTOTAL - $DFUSED))
 
-        ALLOWANCE=$((64 * $OSTCOUNT))
-
-        if [ $DFTOTAL -lt $(($BKTOTAL - $ALLOWANCE)) ] ||
-           [ $DFTOTAL -gt $(($BKTOTAL + $ALLOWANCE)) ] ; then
-                echo "**** FAIL: df total($DFTOTAL) mismatch OST total($BKTOTAL)"
-                rc=1
-        fi
-        if [ $DFFREE -lt $(($BKFREE - $ALLOWANCE)) ] ||
-           [ $DFFREE -gt $(($BKFREE + $ALLOWANCE)) ] ; then
-                echo "**** FAIL: df free($DFFREE) mismatch OST free($BKFREE)"
-                rc=2
-        fi
-        if [ $DFAVAIL -lt $(($BKAVAIL - $ALLOWANCE)) ] ||
-           [ $DFAVAIL -gt $(($BKAVAIL + $ALLOWANCE)) ] ; then
-                echo "**** FAIL: df avail($DFAVAIL) mismatch OST avail($BKAVAIL)"
-                rc=3
-       fi
+	ALLOWANCE=$((64 * $OSTCOUNT))
+
+	if [ $DFTOTAL -lt $(($BKTOTAL - $ALLOWANCE)) ] ||
+	   [ $DFTOTAL -gt $(($BKTOTAL + $ALLOWANCE)) ] ; then
+		echo "**** FAIL: df total($DFTOTAL) mismatch OST total($BKTOTAL)"
+		rc=1
+	fi
+	if [ $DFFREE -lt $(($BKFREE - $ALLOWANCE)) ] ||
+	   [ $DFFREE -gt $(($BKFREE + $ALLOWANCE)) ] ; then
+		echo "**** FAIL: df free($DFFREE) mismatch OST free($BKFREE)"
+		rc=2
+	fi
+	if [ $DFAVAIL -lt $(($BKAVAIL - $ALLOWANCE)) ] ||
+	   [ $DFAVAIL -gt $(($BKAVAIL + $ALLOWANCE)) ] ; then
+		echo "**** FAIL: df avail($DFAVAIL) mismatch OST avail($BKAVAIL)"
+		rc=3
+	fi
 
 	$UMOUNT $MOUNT2
 	stop fs3ost -f || error "unable to stop OST3"
@@ -2934,6 +3353,7 @@ test_38() { # bug 14222
 	local COUNT=10
 	local SRC="/etc /bin"
 	local FILES=$(find $SRC -type f -mtime +1 | head -n $COUNT)
+
 	log "copying $(echo $FILES | wc -w) files to $DIR/$tdir"
 	mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
 	tar cf - $FILES | tar xf - -C $DIR/$tdir ||
@@ -3129,6 +3549,7 @@ test_41c() {
 	local rc2=$?
 	wait $pid
 	local rc=$?
+
 	do_facet mds1 "$LCTL set_param fail_loc=0x0"
 	if [ $rc -eq 0 ] && [ $rc2 -ne 0 ]; then
 		echo "1st MDT start succeed"
@@ -3441,12 +3862,12 @@ test_44() { # 16317
 	check_mount || error "check_mount"
 	UUID=$($LCTL get_param llite.${FSNAME}*.uuid | cut -d= -f2)
 	STATS_FOUND=no
-        UUIDS=$(do_facet $SINGLEMDS "$LCTL get_param mdt.${FSNAME}*.exports.*.uuid")
-        for VAL in $UUIDS; do
-                NID=$(echo $VAL | cut -d= -f1)
-                CLUUID=$(echo $VAL | cut -d= -f2)
-                [ "$UUID" = "$CLUUID" ] && STATS_FOUND=yes && break
-        done
+	UUIDS=$(do_facet $SINGLEMDS "$LCTL get_param mdt.${FSNAME}*.exports.*.uuid")
+	for VAL in $UUIDS; do
+		NID=$(echo $VAL | cut -d= -f1)
+		CLUUID=$(echo $VAL | cut -d= -f2)
+		[ "$UUID" = "$CLUUID" ] && STATS_FOUND=yes && break
+	done
 	[ "$STATS_FOUND" = "no" ] && error "stats not found for client"
 	cleanup || error "cleanup failed with $?"
 }
@@ -3495,7 +3916,7 @@ test_46a() {
 	start_mds || error "unable to start MDS"
 	#first client should see only one ost
 	start_ost || error "Unable to start OST1"
-        wait_osc_import_state mds ost FULL
+	wait_osc_import_state mds ost FULL
 	#start_client
 	mount_client $MOUNT || error "mount_client $MOUNT failed"
 	trap "cleanup_46a $OSTCOUNT" EXIT ERR
@@ -3541,33 +3962,33 @@ test_47() { #17674
 	check_mount || error "check_mount failed"
 	$LCTL set_param ldlm.namespaces.$FSNAME-*-*-*.lru_size=100
 
-        local lru_size=[]
-        local count=0
-        for ns in $($LCTL get_param ldlm.namespaces.$FSNAME-*-*-*.lru_size); do
-            if echo $ns | grep "MDT[[:digit:]]*"; then
-                continue
-            fi
-            lrs=$(echo $ns | sed 's/.*lru_size=//')
-            lru_size[count]=$lrs
-            let count=count+1
-        done
+	local lru_size=[]
+	local count=0
+	for ns in $($LCTL get_param ldlm.namespaces.$FSNAME-*-*-*.lru_size); do
+		if echo $ns | grep "MDT[[:digit:]]*"; then
+			continue
+		fi
+		lrs=$(echo $ns | sed 's/.*lru_size=//')
+		lru_size[count]=$lrs
+		let count=count+1
+	done
 
 	facet_failover ost1
 	facet_failover $SINGLEMDS
 	client_up || error "client_up failed"
 
-        count=0
-        for ns in $($LCTL get_param ldlm.namespaces.$FSNAME-*-*-*.lru_size); do
-            if echo $ns | grep "MDT[[:digit:]]*"; then
-                continue
-            fi
-            lrs=$(echo $ns | sed 's/.*lru_size=//')
-            if ! test "$lrs" -eq "${lru_size[count]}"; then
-                n=$(echo $ns | sed -e 's/ldlm.namespaces.//' -e 's/.lru_size=.*//')
-                error "$n has lost lru_size: $lrs vs. ${lru_size[count]}"
-            fi
-            let count=count+1
-        done
+	count=0
+	for ns in $($LCTL get_param ldlm.namespaces.$FSNAME-*-*-*.lru_size); do
+		if echo $ns | grep "MDT[[:digit:]]*"; then
+			continue
+		fi
+		lrs=$(echo $ns | sed 's/.*lru_size=//')
+		if ! test "$lrs" -eq "${lru_size[count]}"; then
+			n=$(echo $ns | sed -e 's/ldlm.namespaces.//' -e 's/.lru_size=.*//')
+			error "$n has lost lru_size: $lrs vs. ${lru_size[count]}"
+		fi
+		let count=count+1
+	done
 
 	cleanup || error "cleanup failed with $?"
 }
@@ -3618,6 +4039,7 @@ test_48() { # bz-17636 LU-7473
 	cancel_lru_locks mdc
 	stat $MOUNT/widestripe || error "stat $MOUNT/widestripe failed"
 	local r_count=$(getfacl $MOUNT/widestripe | grep "user:" | wc -l)
+
 	count=$((count + 1)) # for the entry "user::rw-"
 
 	[ $count -eq $r_count ] ||
@@ -3696,7 +4118,7 @@ run_test 49b "check PARAM_SYS_LDLM_TIMEOUT option of mkfs.lustre"
 lazystatfs() {
 	# wait long enough to exceed OBD_STATFS_CACHE_SECONDS = 1
 	sleep 2
-        # Test both statfs and lfs df and fail if either one fails
+	# Test both statfs and lfs df and fail if either one fails
 	multiop_bg_pause $1 f_
 	RC=$?
 	PID=$!
@@ -3757,7 +4179,7 @@ test_50c() {
 
 	# Wait for client to detect down OST
 	stop_ost || error "Unable to stop OST1"
-        wait_osc_import_state mds ost DISCONN
+	wait_osc_import_state mds ost DISCONN
 	lazystatfs $MOUNT || error "lazystatfs failed with one down server"
 
 	umount_client $MOUNT || error "Unable to unmount client"
@@ -3836,10 +4258,10 @@ test_50f() {
 	start_mds || error "Unable to start mds"
 	#first client should see only one ost
 	start_ost || error "Unable to start OST1"
-        wait_osc_import_state mds ost FULL
+	wait_osc_import_state mds ost FULL
 
-        start_ost2 || error "Unable to start OST2"
-        wait_osc_import_state mds ost2 FULL
+	start_ost2 || error "Unable to start OST2"
+	wait_osc_import_state mds ost2 FULL
 
 	# Wait for client to detect down OST
 	stop_ost2 || error "Unable to stop OST2"
@@ -4042,11 +4464,11 @@ copy_files_xattrs()
 	do_node $node mkdir -p $dest
 	[ $? -eq 0 ] || { error "Unable to create directory"; return 1; }
 
-	do_node $node  'tar cf - '$@' | tar xf - -C '$dest';
+	do_node $node  'tar cf - '$*' | tar xf - -C '$dest';
 			[ \"\${PIPESTATUS[*]}\" = \"0 0\" ] || exit 1'
 	[ $? -eq 0 ] || { error "Unable to tar files"; return 2; }
 
-	do_node $node 'getfattr -d -m "[a-z]*\\." '$@' > '$xattrs
+	do_node $node 'getfattr -d -m "[a-z]*\\." '$*' > '$xattrs
 	[ $? -eq 0 ] || { error "Unable to read xattrs"; return 3; }
 }
 
@@ -4062,7 +4484,7 @@ diff_files_xattrs()
 	do_node $node mkdir -p $backup2
 	[ $? -eq 0 ] || { error "Unable to create directory"; return 1; }
 
-	do_node $node  'tar cf - '$@' | tar xf - -C '$backup2';
+	do_node $node  'tar cf - '$*' | tar xf - -C '$backup2';
 			[ \"\${PIPESTATUS[*]}\" = \"0 0\" ] || exit 1'
 	[ $? -eq 0 ] || { error "Unable to tar files to diff"; return 2; }
 
@@ -4070,7 +4492,7 @@ diff_files_xattrs()
 	[ $? -eq 0 ] || { error "contents differ"; return 3; }
 
 	local xattrs2=${TMP}/xattrs2
-	do_node $node 'getfattr -d -m "[a-z]*\\." '$@' > '$xattrs2
+	do_node $node 'getfattr -d -m "[a-z]*\\." '$*' > '$xattrs2
 	[ $? -eq 0 ] || { error "Unable to read xattrs to diff"; return 4; }
 
 	do_node $node "diff $xattrs $xattrs2"
@@ -4175,30 +4597,30 @@ run_test 52 "check recovering objects from lost+found"
 # Arguments: service name (OST or MDT), facet (e.g., ost1, $SINGLEMDS), and a
 # parameter pattern prefix like 'ost.*.ost'.
 thread_sanity() {
-        local modname=$1
-        local facet=$2
-        local parampat=$3
-        local opts=$4
+	local modname=$1
+	local facet=$2
+	local parampat=$3
+	local opts=$4
 	local basethr=$5
-        local tmin
-        local tmin2
-        local tmax
-        local tmax2
-        local tstarted
-        local paramp
-        local msg="Insane $modname thread counts"
+	local tmin
+	local tmin2
+	local tmax
+	local tmax2
+	local tstarted
+	local paramp
+	local msg="Insane $modname thread counts"
 	local ncpts=$(check_cpt_number $facet)
 	local nthrs
-        shift 4
+	shift 4
 
-        check_mount || return 41
+	check_mount || return 41
 
-        # We need to expand $parampat, but it may match multiple parameters, so
-        # we'll pick the first one
-        if ! paramp=$(do_facet $facet "lctl get_param -N ${parampat}.threads_min"|head -1); then
-                error "Couldn't expand ${parampat}.threads_min parameter name"
-                return 22
-        fi
+	# We need to expand $parampat, but it may match multiple parameters, so
+	# we'll pick the first one
+	if ! paramp=$(do_facet $facet "lctl get_param -N ${parampat}.threads_min"|head -1); then
+		error "Couldn't expand ${parampat}.threads_min parameter name"
+		return 22
+	fi
 
 	# Remove the .threads_min part
 	paramp=${paramp%.threads_min}
@@ -4260,6 +4682,7 @@ thread_sanity() {
 	cleanup
 	local oldvalue
 	local newvalue="${opts}=$(expr $basethr \* $ncpts)"
+
 	setmodopts -a $modname "$newvalue" oldvalue
 
 	setup
@@ -4328,6 +4751,7 @@ run_test 54b "test llverfs and partial verify of filesystem"
 lov_objid_size()
 {
 	local max_ost_index=$1
+
 	echo -n $(((max_ost_index + 1) * 8))
 }
 
@@ -4388,6 +4812,12 @@ test_56a() {
 	echo ok
 	$LFS osts
 
+	# test instantiating PFL components with sparse index LU-15513
+	mkdir -p $MOUNT/$tdir
+	$LFS setstripe -E 4M -c 1 -E 1G -c 4 -S4M -E eof -c -1 $MOUNT/$tdir
+	dd if=/dev/zero of=$MOUNT/$tdir/$tfile bs=4K count=1 seek=10k ||
+		error "dd to second component failed"
+
 	if [[ "$MDS1_VERSION" -ge $(version_code 2.6.54) ]] ||
 	   [[ "$MDS1_VERSION" -ge $(version_code 2.5.4) &&
 	      "$MDS1_VERSION" -lt $(version_code 2.5.11) ]]; then
@@ -4532,7 +4962,7 @@ test_57b() {
 run_test 57b "initial registration from servicenode should not fail"
 
 count_osts() {
-        do_facet mgs $LCTL get_param mgs.MGS.live.$FSNAME | grep OST | wc -l
+	do_facet mgs $LCTL get_param mgs.MGS.live.$FSNAME | grep OST | wc -l
 }
 
 test_58() { # bug 22658
@@ -4592,7 +5022,7 @@ test_59() {
 }
 run_test 59 "writeconf mount option"
 
-test_60() { # LU-471
+test_60a() { # LU-471
 	if [ "$mds1_FSTYPE" != ldiskfs ]; then
 		skip "ldiskfs only test"
 	fi
@@ -4618,9 +5048,24 @@ test_60() { # LU-471
 	stop_mds
 	reformat_and_config
 }
-run_test 60 "check mkfs.lustre --mkfsoptions -E -O options setting"
+run_test 60a "check mkfs.lustre --mkfsoptions -E -O options setting"
+
+test_60b() {
+	[[ "$mds1_FSTYPE" == ldiskfs ]] || skip "ldiskfs only test"
 
-test_61() { # LU-80
+	local features=$(do_facet $SINGLEMDS $DUMPE2FS $(mdsdevname 1) |
+			 grep features)
+	[ ${PIPESTATUS[0]} -eq 0 ] || error "$DUMPE2FS $(mdsdevname 1) failed"
+
+	echo $features
+	# ea_inode feature should be enabled by default for MDTs
+	[[ "$features" =~ "ea_inode" ]] || error "ea_inode is not set"
+	# large_dir feature should be enabled by default for MDTs
+	[[ "$features" =~ "large_dir" ]] || error "large_dir is not set"
+}
+run_test 60b "check mkfs.lustre MDT default features"
+
+test_61a() { # LU-80
 	local lxattr=$(large_xattr_enabled)
 
 	[ "$MDS1_VERSION" -ge $(version_code 2.1.53) ] ||
@@ -4694,7 +5139,79 @@ test_61() { # LU-80
 	rm -f $file
 	cleanup || error "stopping systems failed"
 }
-run_test 61 "large xattr"
+run_test 61a "large xattr"
+
+test_61b() { # LU-80
+	local lxattr=$(large_xattr_enabled)
+
+	[ "$MDS1_VERSION" -ge $(version_code 2.1.53) ] ||
+		skip "Need MDS version at least 2.1.53"
+
+	if [ "$mds1_FSTYPE" != ldiskfs ]; then
+		skip "ldiskfs specific bug"
+	fi
+
+	if ! large_xattr_enabled; then
+		lxattr=true
+
+		for num in $(seq $MDSCOUNT); do
+			do_facet mds${num} $TUNE2FS -O ea_inode \
+				$(mdsdevname $num) ||
+				error "tune2fs on mds $num failed"
+		done
+	fi
+
+	setup || error "setting up the filesystem failed"
+	client_up || error "starting client failed"
+
+	local _file=$MOUNT/panda
+	local large_value="$(generate_string $(max_xattr_size))"
+	local name="trusted.big"
+
+	touch ${_file} || error "touch ${_file} failed"
+	setfattr -n $name -v $large_value ${_file} ||
+		error "saving $name on $file failed"
+
+	MDT_DEV="${FSNAME}-MDT0000"
+	MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
+
+	stopall || error "stopping for e2fsck run"
+
+
+	ino=$(do_facet $SINGLEMDS "$DEBUGFS -R 'stat /ROOT/panda' \
+		${MDT_DEVNAME} | grep trusted.big")
+	ino=$(echo "${ino}" | awk '{print $2;}')
+	echo "large ea "${ino}
+
+	do_facet $SINGLEMDS "$DEBUGFS -w -R \\\"ln $ino /lost+found\\\" \
+		 ${MDT_DEVNAME}"
+
+	setup_noconfig || error "remounting the filesystem failed"
+
+	do_facet $SINGLEMDS $LCTL lfsck_start -M ${MDT_DEV} -t namespace || {
+		error "can't start lfsck namespace"
+	}
+
+	sleep 5
+	wait_update_facet $SINGLEMDS "$LCTL get_param -n \
+		mdd.${MDT_DEV}.lfsck_namespace |
+		awk '/^status/ { print \\\$2 }'" "completed" 32 || {
+		error "(2) unexpected status"
+	}
+
+	stopall || error "stopping for e2fsck run"
+	for num in $(seq $MDSCOUNT); do
+		run_e2fsck $(facet_active_host mds$num) \
+			$(mdsdevname $num) "-y" ||
+			error "e2fsck MDT$num failed"
+	done
+	setup_noconfig || error "remounting the filesystem failed"
+
+	# need to delete this file to avoid problems in other tests
+	rm -f $file
+	cleanup || error "stopping systems failed"
+}
+run_test 61b "large xattr"
 
 test_62() {
 	if [ "$mds1_FSTYPE" != ldiskfs ]; then
@@ -5068,6 +5585,7 @@ test_69() {
 	start_ost || error "OST0 restart failure"
 	wait_osc_import_state mds ost FULL
 
+	sleep $((TIMEOUT/2)) #object recreation requires some time
 	mount_client $MOUNT || error "mount client failed"
 	touch $DIR/$tdir/$tfile-last || error "create file after reformat"
 	local idx=$($LFS getstripe -i $DIR/$tdir/$tfile-last)
@@ -5076,7 +5594,7 @@ test_69() {
 	local iused=$($LFS df -i $MOUNT |
 		awk '/OST0000/ { print $3 }'; exit ${PIPESTATUS[0]})
 	log "On OST0, $iused used inodes rc=$?"
-	[ $iused -ge $((ost_max_pre/2 + 1000)) ] &&
+	[ $iused -ge $((ost_max_pre + 1000)) ] &&
 		error "OST replacement created too many inodes; $iused"
 	cleanup || error "cleanup failed with $?"
 }
@@ -8467,7 +8985,7 @@ cleanup_115()
 {
 	trap 0
 	stopall
-	rm -f $TMP/$tdir/lustre-mdt
+	do_facet mds1 rm -f $1
 }
 
 test_115() {
@@ -8476,75 +8994,79 @@ test_115() {
 	fi
 	[ -n "$FILESET" ] && skip "Not functional for FILESET set"
 
-	local dbfs_ver=$(do_facet $SINGLEMDS $DEBUGFS -V 2>&1)
+	local dbfs_ver=$(do_facet mds1 $DEBUGFS -V 2>&1)
 
 	echo "debugfs version: $dbfs_ver"
 	echo "$dbfs_ver" | egrep -w "1.44.3.wc1|1.44.5.wc1|1.45.2.wc1" &&
 		skip_env "This version of debugfs doesn't show inode number"
 
-	IMAGESIZE=$((3072 << 30)) # 3072 GiB
+	local IMAGESIZE=$((3072 << 30)) # 3072 GiB
 
 	stopall
 
-	local saved_flakey=${FLAKEY}
-	stack_trap "FLAKEY=$saved_flakey" EXIT
 	FLAKEY=false
 
 	echo "client1: "
-	lctl dl
+	$LCTL dl
 	mount | grep lustre
 	echo "mds1: "
-	do_facet mds1 "hostname; lctl dl; mount"
+	do_facet mds1 "hostname; $LCTL dl; mount"
 	echo "ost1: "
-	do_facet ost1 "hostname; lctl dl; mount"
+	do_facet ost1 "hostname; $LCTL dl; mount"
 	# We need MDT size 3072GB, because it is smallest
 	# partition that can store 2B inodes
-	do_facet $SINGLEMDS "mkdir -p $TMP/$tdir"
+	do_facet mds1 "mkdir -p $TMP/$tdir"
 	local mdsimgname=$TMP/$tdir/lustre-mdt
-	do_facet $SINGLEMDS "rm -f $mdsimgname"
-	do_facet $SINGLEMDS "touch $mdsimgname"
-	trap cleanup_115 RETURN EXIT
-	do_facet $SINGLEMDS "$TRUNCATE $mdsimgname $IMAGESIZE" ||
+
+	do_facet mds1 "rm -f $mdsimgname"
+	do_facet mds1 "touch $mdsimgname"
+	stack_trap "cleanup_115 $mdsimgname" EXIT
+	do_facet mds1 "$TRUNCATE $mdsimgname $IMAGESIZE" ||
 		skip "Backend FS doesn't support sparse files"
-	local mdsdev=$(do_facet $SINGLEMDS "losetup -f")
-	do_facet $SINGLEMDS "losetup $mdsdev $mdsimgname"
+	local mdsdev=$(do_facet mds1 "losetup -f")
+
+	do_facet mds1 "losetup $mdsdev $mdsimgname"
 
 	local mds_opts="$(mkfs_opts mds1 $(mdsdevname 1))	 \
 		--mkfsoptions='-O ea_inode,^resize_inode,meta_bg \
 		-N 2247484000 -E lazy_itable_init' --device-size=$IMAGESIZE"
 	add mds1 $mds_opts --mgs --reformat $mdsdev ||
 		skip_env "format large MDT failed"
-	opts="$(mkfs_opts ost1 $(ostdevname 1)) \
-		$replace --reformat $(ostdevname 1) $(ostvdevname 1)"
+
+	local ostdev=$(ostdevname 1)
+
+	local opts="$(mkfs_opts ost1 $ostdev) \
+		--reformat $ostdev $ostdev"
 	add ost1 $opts || error "add ost1 failed with new params"
-	start $SINGLEMDS  $mdsdev $MDS_MOUNT_OPTS || error "start MDS failed"
+	start mds1  $mdsdev $MDS_MOUNT_OPTS || error "start MDS failed"
 	start_ost || error "start OSS failed"
 	mount_client $MOUNT || error "mount client failed"
 
 	mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir fail"
-	goal="/sys/fs/ldiskfs/$(basename $mdsdev)/inode_goal"
+	local goal="/sys/fs/ldiskfs/$(basename $mdsdev)/inode_goal"
 	echo goal: $goal
 	# 2147483648 is 0x80000000
-	do_facet $SINGLEMDS "echo 2147483648 >> $goal; grep . $goal"
+	do_facet mds1 "echo 2147483648 >> $goal; grep . $goal"
 	touch $DIR/$tdir/$tfile
 
 	# attrs from 1 to 16 go to block, 17th - to inode
+	local i
+
 	for i in {1..17}; do
 		local nm="trusted.ea$i"
 		setfattr -n $nm -v $(printf "xattr%0250d" $i) $DIR/$tdir/$tfile
 	done
 
-	do_facet $SINGLEMDS $DEBUGFS -c -R "stat ROOT/$tdir/$tfile" $mdsdev
+	do_facet mds1 "$DEBUGFS -c -R 'stat ROOT/$tdir/$tfile' $mdsdev"
 
 	# inode <2147483649> trusted.ea16 (255)
-	local inode_num=$(do_facet $SINGLEMDS \
+	local inode_num=$(do_facet mds1 \
 			"$DEBUGFS -c -R 'stat ROOT/$tdir/$tfile' $mdsdev" |
 			 awk '/ea17/ { print $2 }' |
 			 sed -e 's/>//' -e 's/<//' -e 's/\"//')
 	echo "inode num: $inode_num"
 	[ $inode_num -ge 2147483648 ] || error "inode $inode_num too small"
-	do_facet $SINGLEMDS "losetup -d $mdsdev"
-	cleanup_115
+	do_facet mds1 "losetup -d $mdsdev"
 }
 run_test 115 "Access large xattr with inodes number over 2TB"
 
@@ -8966,7 +9488,7 @@ test_123af() { #LU-13609
 		fi
 		orig_clist=($(do_facet $facet $LCTL $cmd | awk '{ print $2 }'))
 		orig_count=${#orig_clist[@]}
-		echo "orig_clist: ${orig_clist[@]}"
+		echo "orig_clist: ${orig_clist[*]}"
 
 		#define OBD_FAIL_CATLIST 0x131b
 		#fetch to llog records from the second one
@@ -8974,7 +9496,7 @@ test_123af() { #LU-13609
 
 		new_clist=($(do_facet $facet $LCTL $cmd | awk '{ print $2 }'))
 		new_count=${#new_clist[@]}
-		echo "new_clist: ${new_clist[@]}"
+		echo "new_clist: ${new_clist[*]}"
 
 		[ $new_count -eq $((orig_count - 1)) ] ||
 			error "$new_count != $orig_count - 1"
@@ -9019,17 +9541,74 @@ test_123ag() { # LU-15142
 }
 run_test 123ag "llog_print skips values deleted by set_param -P -d"
 
+test_123ah() { #LU-7668 del_ost
+	[ "$MGS_VERSION" -ge $(version_code 2.15.50) -a \
+	   "$MDS1_VERSION" -ge $(version_code 2.15.50) ] ||
+		skip "Need both MGS and MDS version at least 2.15.50"
+
+	[ -d $MOUNT/.lustre ] || setupall
+	stack_trap "do_facet mds1 $LCTL set_param fail_loc=0" EXIT
+
+	local cmd="--device MGS llog_print"
+
+	cli_llogcnt_orig=$(do_facet mgs $LCTL $cmd $FSNAME-client |
+				grep -c $FSNAME-OST0000)
+	mds1_llogcnt_orig=$(do_facet mgs $LCTL $cmd $FSNAME-MDT0000 |
+				grep -c $FSNAME-OST0000)
+
+	[ $cli_llogcnt_orig -gt 0 ] ||
+		error "$FSNAME-OST0000 not found (client)"
+	[ $mds1_llogcnt_orig -gt 0 ] || error "$FSNAME-OST0000 not found (MDT)"
+
+	# -n/--dryrun should NOT modify catalog
+	do_facet mgs "$LCTL del_ost -n -t $FSNAME-OST0000" ||
+		error "del_ost --dryrun failed with $?"
+
+	local cli_llogcnt=$(do_facet mgs $LCTL $cmd $FSNAME-client |
+			grep -c $FSNAME-OST0000)
+	local mds1_llogcnt=$(do_facet mgs $LCTL $cmd $FSNAME-MDT0000 |
+			grep -c $FSNAME-OST0000)
+
+	[ $cli_llogcnt -eq $cli_llogcnt_orig ] ||
+		error "--dryrun error: $cli_llogcnt != $cli_llogcnt_orig"
+	[ $mds1_llogcnt -eq $mds1_llogcnt_orig ] ||
+		error "--dryrun error: $mds1_llogcnt != $mds1_llogcnt_orig"
+
+	# actual run
+	do_facet mgs "$LCTL del_ost --target $FSNAME-OST0000" ||
+		error "del_ost failed with $?"
+
+	local cli_llogcnt=$(do_facet mgs $LCTL $cmd $FSNAME-client |
+			grep -c $FSNAME-OST0000)
+	local mds1_llogcnt=$(do_facet mgs $LCTL $cmd $FSNAME-MDT0000 |
+			grep -c $FSNAME-OST0000)
+
+	# every catalog entry for OST0000 should have been cancelled
+	[ $cli_llogcnt -eq 0 ] || error "$FSNAME-OST0000 not cancelled (cli)"
+	[ $mds1_llogcnt -eq 0 ] || error "$FSNAME-OST0000 not cancelled (MDT)"
+
+	umount_client $MOUNT
+	mount_client $MOUNT
+
+	$LFS df | grep -q OST0000 && error "del_ost did not remove OST0000!"
+	cleanup
+	reformat_and_config
+}
+run_test 123ah "del_ost cancels config log entries correctly"
+
 test_123F() {
 	remote_mgs_nodsh && skip "remote MGS with nodsh"
 
 	[ -d $MOUNT/.lustre ] || setup
 	local yaml_file="$TMP/$tfile.yaml"
+
 	do_facet mgs rm "$yaml_file"
 	local cfgfiles=$(do_facet mgs "lctl --device MGS llog_catlist" |
 			 sed 's/config_log://')
 
 	# set jobid_var to a different value for test
 	local orig_val=$(do_facet mgs $LCTL get_param jobid_var)
+
 	do_facet mgs $LCTL set_param -P jobid_var="TESTNAME"
 
 	for i in $cfgfiles params; do
@@ -9048,6 +9627,7 @@ test_123F() {
 	do_facet mgs "lctl set_param -F $yaml_file"
 
 	local set_val=$(do_facet mgs $LCTL get_param jobid_var)
+
 	do_facet mgs $LCTL set_param -P $orig_val
 
 	[ $set_val == "jobid_var=TESTNAME" ] ||
@@ -9079,6 +9659,7 @@ test_124()
 	fi
 	local nid=$(do_facet mds2 $LCTL list_nids | head -1)
 	local failover_nid=$(do_node $mds2failover_HOST $LCTL list_nids | head -1)
+
 	do_facet mgs $LCTL replace_nids $FSNAME-MDT0001 $nid:$failover_nid ||
 		error "replace_nids execution error"
 
@@ -9142,6 +9723,7 @@ check_slaves_max_sectors_kb()
 
 	local slave max_sectors new_max_sectors max_hw_sectors path
 	local rc=0
+
 	for slave in ${slave_devices}; do
 		path="/dev/${slave}"
 		! is_blkdev ${facet} ${path} && continue
@@ -9419,6 +10001,99 @@ test_132() {
 }
 run_test 132 "hsm_actions processed after failover"
 
+# This test verifies we do RR allocation within a pool even if there is a
+# significant imbalance vs an OST outside the pool
+test_133() {
+	[[ $OSTCOUNT -lt 4 ]] && skip_env "needs >= 4 OSTs"
+	# This is the easiest way to ensure OSTs start out balanced
+	reformat_and_config
+	setupall
+
+	check_set_fallocate_or_skip
+
+	local testfile=$DIR/$tdir/$tfile
+	local pool="testpool"
+	local ostrange=$((OSTCOUNT - 1))
+	# Select all but the last OST to add to the pool
+	local poolostrange=$((OSTCOUNT - 2))
+	local filenum=20
+	local filecount
+	local stripecount
+
+	declare -a AVAIL
+	free_min_max
+
+	[ $MINV -eq 0 ] && error "no free space in OST$MINI"
+	[ $MAXV -gt $((2 * $MINV)) ] &&
+		error "OSTs badly unbalanced after reformat"
+
+	create_pool $FSNAME.$pool || error "failed to create a pool"
+	do_facet mgs $LCTL pool_add $FSNAME.$pool OST[0-$poolostrange] ||
+		error "failed to add OST[0-$poolostrange] to the pool"
+
+	test_mkdir -p $DIR/$tdir || error "failed to mkdir $DIR/$tdir"
+	# Consume space on the OSTs in the pool so they are unbalanced with the
+	# OST outside of the pool
+	# fill each OST 90% with fallocate so they are widely
+	# imbalanced
+	local size=$(((MINV * 9 / 10) * 1024))
+	for ((i = 0; i <= poolostrange; i++)); do
+		$LFS setstripe -c 1 -i $i $testfile$i ||
+			error "failed to setstripe $testfile$i"
+		fallocate -l $size $testfile$i || error "fallocate failed"
+	done
+	ls -la $DIR/$tdir
+	sleep_maxage
+	$LFS df
+
+	# Create files in the pool now that there is an imbalance
+	filecount=$(((OSTCOUNT - 1) * filenum))
+	for ((i = 0; i < filecount; i++)); do
+		$LFS setstripe -p $pool $testfile-$i ||
+			error "failed to setstripe -p $pool $testfile-$i"
+	done
+	$LFS getstripe -i $testfile-* > /tmp/$tfile.log
+	# Count the number of files with a stripe on each OST to verify the
+	# pool allocated with round-robin
+	for ((i = 0; i <= poolostrange; i++)); do
+		stripecount=$(grep -c $i /tmp/$tfile.log)
+		# Allow a little leeway
+		if (( stripecount < filenum - 1 ||
+		      stripecount > filenum + 1 )); then
+			cat /tmp/$tfile.log
+			error "$stripecount != $filenum files on OST$i"
+		fi
+	done
+
+	# Create files across the system now that there is an imbalance
+	filecount=$((OSTCOUNT * filenum))
+	for ((i = 1; i < filecount; i++)); do
+		$LFS setstripe $testfile-$i.2 ||
+			error "failed to setstripe $testilfe-$i.2"
+	done
+	$LFS getstripe -i $testfile-*.2 > /tmp/$tfile.log
+	local qos_used=""
+	# Count the number of files with a stripe on each OST to verify the
+	# files are *NOT* allocated with round-robin
+	for ((i = 0; i <= ostrange; i++)); do
+		stripecount=$(grep -c $i /tmp/$tfile.log)
+		if [[ $stripecount -ne $filenum ]]; then
+			qos_used="true"
+			echo "QOS: $stripecount != $filenum files on OST$i"
+		fi
+	done
+	if [ -z "$qos_used" ]; then
+		error "QOS not used on imbalanced OSTs!"
+	fi
+
+	rm -rf /tmp/$tfile.log $DIR/$tdir
+	do_facet mgs $LCTL pool_remove $FSNAME.$pool OST[0-$poolostrange] ||
+		"failed to remove OST[0-$poolostrange] from the pool"
+	do_facet mgs $LCTL pool_destroy $FSNAME.$pool ||
+		error "failed to destroy pool"
+}
+run_test 133 "stripe QOS: free space balance in a pool"
+
 if ! combined_mgs_mds ; then
 	stop mgs
 fi