X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Fsanity.sh;h=4d50dcfab8877826132194e9dcfb059c9f72282c;hp=1b7f3e32404709d59854ff00e1050c7cf9514a8c;hb=777b04a093;hpb=9a0a864112550047ae7236c7a904dc7a9955880e

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
old mode 100644
new mode 100755
index 1b7f3e3..4d50dcf
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -31,8 +31,8 @@ GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""}
 export PARALLEL=${PARALLEL:-"no"}
 
 TRACE=${TRACE:-""}
-LUSTRE_TESTS_API_DIR=${LUSTRE_TESTS_API_DIR:-${LUSTRE}/tests/clientapi}
 LUSTRE=${LUSTRE:-$(dirname $0)/..}
+LUSTRE_TESTS_API_DIR=${LUSTRE_TESTS_API_DIR:-${LUSTRE}/tests/clientapi}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 
@@ -41,28 +41,46 @@ init_logging
 ALWAYS_EXCEPT="$SANITY_EXCEPT "
 # bug number for skipped test: LU-9693 LU-6493 LU-9693
 ALWAYS_EXCEPT+="               42a     42b     42c "
-# bug number:    LU-8411 LU-9054
-ALWAYS_EXCEPT+=" 407     312 "
+# bug number:    LU-8411 LU-9054 LU-13314
+ALWAYS_EXCEPT+=" 407     312	56ob"
 
 if $SHARED_KEY; then
 	# bug number:    LU-9795 LU-9795 LU-9795 LU-9795
-	ALWAYS_EXCEPT+=" 17n     60a     133g    300f "
+	ALWAYS_EXCEPT+=" 17n     60a     133g    300f"
+fi
+
+selinux_status=$(getenforce)
+if [ "$selinux_status" != "Disabled" ]; then
+	# bug number:
+	ALWAYS_EXCEPT+=""
 fi
 
 # skip the grant tests for ARM until they are fixed
 if [[ $(uname -m) = aarch64 ]]; then
 	# bug number:	 LU-11596
 	ALWAYS_EXCEPT+=" $GRANT_CHECK_LIST"
-	# bug number:	 LU-11671 LU-11594 LU-11667 LU-11729
-	ALWAYS_EXCEPT+=" 45	  103a	    317      810"
+	# bug number:	 LU-11671 LU-11667
+	ALWAYS_EXCEPT+=" 45	  317"
 fi
 
-#                                  5          12          (min)"
-[ "$SLOW" = "no" ] && EXCEPT_SLOW="27m 64b 68 71 115 300o"
+# skip nfs tests on kernels >= 4.14.0 until they are fixed
+if [ $LINUX_VERSION_CODE -ge $(version_code 4.14.0) ]; then
+	# bug number:	LU-12661
+	ALWAYS_EXCEPT+=" 817"
+fi
+# skip cgroup tests on RHEL8.1 kernels until they are fixed
+if (( $LINUX_VERSION_CODE >= $(version_code 4.18.0) &&
+      $LINUX_VERSION_CODE <  $(version_code 5.4.0) )); then
+	# bug number:	LU-13063
+	ALWAYS_EXCEPT+=" 411"
+fi
+
+#                                  5          12     8   12  (min)"
+[ "$SLOW" = "no" ] && EXCEPT_SLOW="27m 64b 68 71 115 135 136 300o"
 
 if [ "$mds1_FSTYPE" = "zfs" ]; then
-	# bug number for skipped test: LU-1957
-	ALWAYS_EXCEPT="$ALWAYS_EXCEPT  180"
+	# bug number for skipped test:
+	ALWAYS_EXCEPT="$ALWAYS_EXCEPT  "
 	#                                               13    (min)"
 	[ "$SLOW" = "no" ] && EXCEPT_SLOW="$EXCEPT_SLOW 51b"
 fi
@@ -538,11 +556,6 @@ test_17g() {
 	[ $MDS1_VERSION -le $(version_code 2.3.55) ] &&
 		TESTS="4094 4095"
 
-	# skip long symlink name for rhel6.5.
-	# rhel6.5 has a limit (PATH_MAX - sizeof(struct filename))
-	grep -q '6.5' /etc/redhat-release &>/dev/null &&
-		TESTS="59 60 61 4062 4063"
-
 	for i in $TESTS; do
 		local SYMNAME=$(str_repeat 'x' $i)
 		ln -s $SYMNAME $DIR/$tdir/f$i || error "failed $i-char symlink"
@@ -2458,10 +2471,6 @@ test_27D() {
 	local ost_list=$(seq $first_ost $ost_step $last_ost)
 	local ost_range="$first_ost $last_ost $ost_step"
 
-	if ! combined_mgs_mds ; then
-		mount_mgs_client
-	fi
-
 	test_mkdir $DIR/$tdir
 	pool_add $POOL || error "pool_add failed"
 	pool_add_targets $POOL $ost_range || error "pool_add_targets failed"
@@ -2472,19 +2481,15 @@ test_27D() {
 	[ $MDS1_VERSION -lt $(version_code 2.9.55) ] ||
 		[ $CLIENT_VERSION -lt $(version_code 2.9.55) ] &&
 			skip27D+=" -s 30,31"
-	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code $SEL_VER) ] &&
-		skip27D+="-s 32"
 	[[ ! $($LCTL get_param mdc.*.import) =~ connect_flags.*overstriping ||
 	  $OSTCOUNT -ge $(($LOV_MAX_STRIPE_COUNT / 2)) ]] &&
 		skip27D+=" -s 32,33"
+	[[ $MDS_VERSION -lt $(version_code $SEL_VER) ]] &&
+		skip27D+=" -s 34"
 	llapi_layout_test -d$DIR/$tdir -p$POOL -o$OSTCOUNT $skip27D ||
 		error "llapi_layout_test failed"
 
 	destroy_test_pools || error "destroy test pools failed"
-
-	if ! combined_mgs_mds ; then
-		umount_mgs_client
-	fi
 }
 run_test 27D "validate llapi_layout API"
 
@@ -2620,6 +2625,8 @@ run_test 27H "Set specific OSTs stripe"
 test_27I() {
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
 	[[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
+	[[ $MDS1_VERSION -gt $(version_code 2.12.52) ]] ||
+		skip "Need MDS version newer than 2.12.52"
 	local pool=$TESTNAME
 	local ostrange="1 1 1"
 
@@ -2635,7 +2642,7 @@ test_27I() {
 run_test 27I "check that root dir striping does not break parent dir one"
 
 test_27J() {
-	[[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.12.51) ]] &&
+	[[ $MDS1_VERSION -le $(version_code 2.12.51) ]] &&
 		skip "Need MDS version newer than 2.12.51"
 
 	test_mkdir $DIR/$tdir
@@ -2730,7 +2737,7 @@ test_27J() {
 run_test 27J "basic ops on file with foreign LOV"
 
 test_27K() {
-	[[ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.12.49) ]] &&
+	[[ $MDS1_VERSION -le $(version_code 2.12.49) ]] &&
 		skip "Need MDS version newer than 2.12.49"
 
 	test_mkdir $DIR/$tdir
@@ -2815,11 +2822,6 @@ test_27L() {
 
 	local POOL=${POOL:-$TESTNAME}
 
-	if ! combined_mgs_mds ; then
-		mount_mgs_client
-		trap umount_mgs_client EXIT
-	fi
-
 	pool_add $POOL || error "pool_add failed"
 
 	lfs pool_list $MOUNT | grep -Fx "${FSNAME}.${POOL}" ||
@@ -2828,6 +2830,152 @@ test_27L() {
 }
 run_test 27L "lfs pool_list gives correct pool name"
 
+test_27M() {
+	[[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.12.57) ]] &&
+		skip "Need MDS version >= than 2.12.57"
+	remote_mds_nodsh && skip "remote MDS with nodsh"
+	[[ $OSTCOUNT -lt 2 ]] && skip_env "need > 1 OST"
+
+	test_mkdir $DIR/$tdir
+
+	# Set default striping on directory
+	$LFS setstripe -C 4 $DIR/$tdir
+
+	echo 1 > $DIR/$tdir/${tfile}.1
+	local count=$($LFS getstripe -c $DIR/$tdir/${tfile}.1)
+	local setcount=4
+	[ $count -eq $setcount ] ||
+		error "(1) stripe count $count, should be $setcount"
+
+	# Capture existing append_stripe_count setting for restore
+	local orig_count=$(do_facet mds1 $LCTL get_param -n mdd.$FSNAME-MDT0000.append_stripe_count)
+	local mdts=$(comma_list $(mdts_nodes))
+	stack_trap "do_nodes $mdts $LCTL set_param mdd.*.append_stripe_count=$orig_count" EXIT
+
+	local appendcount=$orig_count
+	echo 1 >> $DIR/$tdir/${tfile}.2_append
+	count=$($LFS getstripe -c $DIR/$tdir/${tfile}.2_append)
+	[ $count -eq $appendcount ] ||
+		error "(2)stripe count $count, should be $appendcount for append"
+
+	# Disable O_APPEND striping, verify it works
+	do_nodes $mdts $LCTL set_param mdd.*.append_stripe_count=0
+
+	# Should now get the default striping, which is 4
+	setcount=4
+	echo 1 >> $DIR/$tdir/${tfile}.3_append
+	count=$($LFS getstripe -c $DIR/$tdir/${tfile}.3_append)
+	[ $count -eq $setcount ] ||
+		error "(3) stripe count $count, should be $setcount"
+
+	# Try changing the stripe count for append files
+	do_nodes $mdts $LCTL set_param mdd.*.append_stripe_count=2
+
+	# Append striping is now 2 (directory default is still 4)
+	appendcount=2
+	echo 1 >> $DIR/$tdir/${tfile}.4_append
+	count=$($LFS getstripe -c $DIR/$tdir/${tfile}.4_append)
+	[ $count -eq $appendcount ] ||
+		error "(4) stripe count $count, should be $appendcount for append"
+
+	# Test append stripe count of -1
+	do_nodes $mdts $LCTL set_param mdd.*.append_stripe_count=-1
+	appendcount=$OSTCOUNT
+	echo 1 >> $DIR/$tdir/${tfile}.5
+	count=$($LFS getstripe -c $DIR/$tdir/${tfile}.5)
+	[ $count -eq $appendcount ] ||
+		error "(5) stripe count $count, should be $appendcount for append"
+
+	# Set append striping back to default of 1
+	do_nodes $mdts $LCTL set_param mdd.*.append_stripe_count=1
+
+	# Try a new default striping, PFL + DOM
+	$LFS setstripe -L mdt -E 1M -E -1 -c 2 $DIR/$tdir
+
+	# Create normal DOM file, DOM returns stripe count == 0
+	setcount=0
+	touch $DIR/$tdir/${tfile}.6
+	count=$($LFS getstripe -c $DIR/$tdir/${tfile}.6)
+	[ $count -eq $setcount ] ||
+		error "(6) stripe count $count, should be $setcount"
+
+	# Show
+	appendcount=1
+	echo 1 >> $DIR/$tdir/${tfile}.7_append
+	count=$($LFS getstripe -c $DIR/$tdir/${tfile}.7_append)
+	[ $count -eq $appendcount ] ||
+		error "(7) stripe count $count, should be $appendcount for append"
+
+	# Clean up DOM layout
+	$LFS setstripe -d $DIR/$tdir
+
+	# Now test that append striping works when layout is from root
+	$LFS setstripe -c 2 $MOUNT
+	# Make a special directory for this
+	mkdir $DIR/${tdir}/${tdir}.2
+	stack_trap "$LFS setstripe -d $MOUNT" EXIT
+
+	# Verify for normal file
+	setcount=2
+	echo 1 > $DIR/${tdir}/${tdir}.2/${tfile}.8
+	count=$($LFS getstripe -c $DIR/$tdir/${tdir}.2/${tfile}.8)
+	[ $count -eq $setcount ] ||
+		error "(8) stripe count $count, should be $setcount"
+
+	appendcount=1
+	echo 1 >> $DIR/${tdir}/${tdir}.2/${tfile}.9_append
+	count=$($LFS getstripe -c $DIR/${tdir}/${tdir}.2/${tfile}.9_append)
+	[ $count -eq $appendcount ] ||
+		error "(9) stripe count $count, should be $appendcount for append"
+
+	# Now test O_APPEND striping with pools
+	do_nodes $mdts $LCTL set_param mdd.*.append_pool="$TESTNAME"
+	stack_trap "do_nodes $mdts $LCTL set_param mdd.*.append_pool='none'" EXIT
+
+	# Create the pool
+	pool_add $TESTNAME || error "pool creation failed"
+	pool_add_targets $TESTNAME 0 1 || error "Pool add targets failed"
+
+	echo 1 >> $DIR/$tdir/${tfile}.10_append
+
+	pool=$($LFS getstripe -p $DIR/$tdir/${tfile}.10_append)
+	[ "$pool" = "$TESTNAME" ] || error "(10) incorrect pool: $pool"
+
+	# Check that count is still correct
+	appendcount=1
+	echo 1 >> $DIR/$tdir/${tfile}.11_append
+	count=$($LFS getstripe -c $DIR/$tdir/${tfile}.11_append)
+	[ $count -eq $appendcount ] ||
+		error "(11) stripe count $count, should be $appendcount for append"
+
+	# Disable O_APPEND stripe count, verify pool works separately
+	do_nodes $mdts $LCTL set_param mdd.*.append_stripe_count=0
+
+	echo 1 >> $DIR/$tdir/${tfile}.12_append
+
+	pool=$($LFS getstripe -p $DIR/$tdir/${tfile}.12_append)
+	[ "$pool" = "$TESTNAME" ] || error "(12) incorrect pool: $pool"
+
+	# Remove pool setting, verify it's not applied
+	do_nodes $mdts $LCTL set_param mdd.*.append_pool='none'
+
+	echo 1 >> $DIR/$tdir/${tfile}.13_append
+
+	pool=$($LFS getstripe -p $DIR/$tdir/${tfile}.13_append)
+	[ "$pool" = "" ] || error "(13) pool found: $pool"
+}
+run_test 27M "test O_APPEND striping"
+
+test_27N() {
+	combined_mgs_mds && skip "needs separate MGS/MDT"
+
+	pool_add $TESTNAME || error "pool_add failed"
+	do_facet mgs "$LCTL pool_list $FSNAME" |
+		grep -Fx "${FSNAME}.${TESTNAME}" ||
+		error "lctl pool_list on MGS failed"
+}
+run_test 27N "lctl pool_list on separate MGS gives correct pool name"
+
 # createtest also checks that device nodes are created and
 # then visible correctly (#2091)
 test_28() { # bug 2091
@@ -3082,10 +3230,10 @@ test_31n() {
 run_test 31n "check link count of unlinked file"
 
 link_one() {
-	local TEMPNAME=$(mktemp $1_XXXXXX)
-	mlink $TEMPNAME $1 2> /dev/null &&
-		echo "$BASHPID: link $TEMPNAME to $1 succeeded"
-	munlink $TEMPNAME
+	local tempfile=$(mktemp $1_XXXXXX)
+	mlink $tempfile $1 2> /dev/null &&
+		echo "$BASHPID: link $tempfile to $1 succeeded"
+	munlink $tempfile
 }
 
 test_31o() { # LU-2901
@@ -3601,6 +3749,46 @@ test_33g() {
 }
 run_test 33g "nonroot user create already existing root created file"
 
+test_33h() {
+	[ $MDSCOUNT -lt 2 ] && skip_env "needs >= 2 MDTs"
+	[ $MDS1_VERSION -lt $(version_code 2.13.50) ] &&
+		skip "Need MDS version at least 2.13.50"
+
+	test_mkdir -c $MDSCOUNT -H crush $DIR/$tdir ||
+		error "mkdir $tdir failed"
+	touch $DIR/$tdir/$tfile || error "touch $tfile failed"
+
+	local index=$($LFS getstripe -m $DIR/$tdir/$tfile)
+	local index2
+
+	for fname in $DIR/$tdir/$tfile.bak \
+		     $DIR/$tdir/$tfile.SAV \
+		     $DIR/$tdir/$tfile.orig \
+		     $DIR/$tdir/$tfile~; do
+		touch $fname  || error "touch $fname failed"
+		index2=$($LFS getstripe -m $fname)
+		[ $index -eq $index2 ] ||
+			error "$fname MDT index mismatch $index != $index2"
+	done
+
+	local failed=0
+	for i in {1..50}; do
+		for fname in $(mktemp -u $DIR/$tdir/.$tfile.XXXXXX) \
+			     $(mktemp $DIR/$tdir/$tfile.XXXXXXXX); do
+			touch $fname  || error "touch $fname failed"
+			index2=$($LFS getstripe -m $fname)
+			if [[ $index != $index2 ]]; then
+				failed=$((failed + 1))
+				echo "$fname MDT index mismatch $index != $index2"
+			fi
+		done
+	done
+	echo "$failed MDT index mismatches"
+	(( failed < 4 )) || error "MDT index mismatch $failed times"
+
+}
+run_test 33h "temp file is located on the same MDT as target"
+
 TEST_34_SIZE=${TEST_34_SIZE:-2000000000000}
 test_34a() {
 	rm -f $DIR/f34
@@ -4133,7 +4321,7 @@ test_39j() {
 			error "mtime is lost on close: $mtime2, " \
 			      "should be $mtime1"
 
-		cancel_lru_locks $OSC
+		cancel_lru_locks
 		if [ $i = 0 ] ; then echo "repeat after cancel_lru_locks"; fi
 	done
 	lctl set_param fail_loc=0
@@ -4162,7 +4350,7 @@ test_39k() {
 		[ "$mtime2" = $TEST_39_MTIME ] || \
 			error "mtime is lost on close: $mtime2, should be $TEST_39_MTIME"
 
-		cancel_lru_locks osc
+		cancel_lru_locks
 		if [ $i = 0 ] ; then echo "repeat after cancel_lru_locks"; fi
 	done
 }
@@ -4840,7 +5028,7 @@ test_49() { # LU-1030
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
 	remote_ost_nodsh && skip "remote OST with nodsh"
 
-	# get ost1 size - lustre-OST0000
+	# get ost1 size - $FSNAME-OST0000
 	ost1_size=$(do_facet ost1 $LFS df | grep ${ost1_svc} |
 		awk '{ print $4 }')
 	# write 800M at maximum
@@ -5360,19 +5548,27 @@ test_56c() {
 		skip_env "OST $ost_name is in $old_status status"
 
 	do_facet ost1 $LCTL set_param -n obdfilter.$ost_name.degraded=1
+	[[ $OST1_VERSION -ge $(version_code 2.12.55) ]] && do_facet ost1 \
+		$LCTL set_param -n obdfilter.$ost_name.no_precreate=1
 	sleep_maxage
 
 	local new_status=$(ost_dev_status $ost_idx)
 
-	[[ "$new_status" = "D" ]] ||
-		error "OST $ost_name is in status of '$new_status', not 'D'"
+	[[ "$new_status" =~ "D" ]] ||
+		error "$ost_name status is '$new_status', missing 'D'"
+	if [[ $OST1_VERSION -ge $(version_code 2.12.55) ]]; then
+		[[ "$new_status" =~ "N" ]] ||
+			error "$ost_name status is '$new_status', missing 'N'"
+	fi
 
 	do_facet ost1 $LCTL set_param -n obdfilter.$ost_name.degraded=0
+	[[ $OST1_VERSION -ge $(version_code 2.12.55) ]] && do_facet ost1 \
+		$LCTL set_param -n obdfilter.$ost_name.no_precreate=0
 	sleep_maxage
 
 	new_status=$(ost_dev_status $ost_idx)
-	[[ -z "$new_status" ]] ||
-		error "OST $ost_name is in status of '$new_status', not ''"
+	[[ ! "$new_status" =~ "D" && ! "$new_status" =~ "N" ]] ||
+		error "$ost_name status is '$new_status', has 'D' and/or 'N'"
 }
 run_test 56c "check 'lfs df' showing device status"
 
@@ -5595,6 +5791,65 @@ test_56ob() {
 }
 run_test 56ob "check lfs find -atime -mtime -ctime with units"
 
+test_newerXY_base() {
+	local x=$1
+	local y=$2
+	local dir=$DIR/$tdir
+	local ref
+	local negref
+
+	if [ $y == "t" ]; then
+		ref="\"$(date +"%Y-%m-%d %H:%M:%S")\""
+	else
+		ref=$DIR/$tfile.newer
+		touch $ref || error "touch $ref failed"
+	fi
+	sleep 2
+	setup_56 $dir $NUMFILES $NUMDIRS "-i0 -c1" "-i0 -c1"
+	sleep 2
+	if [ $y == "t" ]; then
+		negref="\"$(date +"%Y-%m-%d %H:%M:%S")\""
+	else
+		negref=$DIR/$tfile.newerneg
+		touch $negref || error "touch $negref failed"
+	fi
+
+	local cmd="$LFS find $dir -newer$x$y $ref"
+	local nums=$(eval $cmd | wc -l)
+	local expected=$(((NUMFILES + 2) * NUMDIRS + 1))
+
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+
+	cmd="$LFS find $dir ! -newer$x$y $negref"
+	nums=$(eval $cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+
+	cmd="$LFS find $dir -newer$x$y $ref ! -newer$x$y $negref"
+	nums=$(eval $cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+
+	rm -rf $DIR/*
+}
+
+test_56oc() {
+	test_newerXY_base "a" "a"
+	test_newerXY_base "a" "m"
+	test_newerXY_base "a" "c"
+	test_newerXY_base "m" "a"
+	test_newerXY_base "m" "m"
+	test_newerXY_base "m" "c"
+	test_newerXY_base "c" "a"
+	test_newerXY_base "c" "m"
+	test_newerXY_base "c" "c"
+	test_newerXY_base "a" "t"
+	test_newerXY_base "m" "t"
+	test_newerXY_base "c" "t"
+}
+run_test 56oc "check lfs find -newerXY work"
+
 test_56p() {
 	[ $RUNAS_ID -eq $UID ] &&
 		skip_env "RUNAS_ID = UID = $UID -- skipping"
@@ -5649,39 +5904,73 @@ test_56r() {
 	setup_56 $dir $NUMFILES $NUMDIRS
 
 	local expected=12
-	local cmd="$LFS find -size 0 -type f $dir"
+	local cmd="$LFS find -size 0 -type f -lazy $dir"
 	local nums=$($cmd | wc -l)
 
 	[ $nums -eq $expected ] ||
 		error "'$cmd' wrong: found $nums, expected $expected"
+	cmd="$LFS find -size 0 -type f $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+
 	expected=0
+	cmd="$LFS find ! -size 0 -type f -lazy $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
 	cmd="$LFS find ! -size 0 -type f $dir"
 	nums=$($cmd | wc -l)
 	[ $nums -eq $expected ] ||
 		error "'$cmd' wrong: found $nums, expected $expected"
+
 	echo "test" > $dir/$tfile
 	echo "test2" > $dir/$tfile.2 && sync
 	expected=1
+	cmd="$LFS find -size 5 -type f -lazy $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
 	cmd="$LFS find -size 5 -type f $dir"
 	nums=$($cmd | wc -l)
 	[ $nums -eq $expected ] ||
 		error "'$cmd' wrong: found $nums, expected $expected"
+
 	expected=1
+	cmd="$LFS find -size +5 -type f -lazy $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
 	cmd="$LFS find -size +5 -type f $dir"
 	nums=$($cmd | wc -l)
 	[ $nums -eq $expected ] ||
 		error "'$cmd' wrong: found $nums, expected $expected"
+
 	expected=2
+	cmd="$LFS find -size +0 -type f -lazy $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
 	cmd="$LFS find -size +0 -type f $dir"
 	nums=$($cmd | wc -l)
 	[ $nums -eq $expected ] ||
 		error "'$cmd' wrong: found $nums, expected $expected"
+
 	expected=2
+	cmd="$LFS find ! -size -5 -type f -lazy $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
 	cmd="$LFS find ! -size -5 -type f $dir"
 	nums=$($cmd | wc -l)
 	[ $nums -eq $expected ] ||
 		error "'$cmd' wrong: found $nums, expected $expected"
+
 	expected=12
+	cmd="$LFS find -size -5 -type f -lazy $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
 	cmd="$LFS find -size -5 -type f $dir"
 	nums=$($cmd | wc -l)
 	[ $nums -eq $expected ] ||
@@ -5689,6 +5978,157 @@ test_56r() {
 }
 run_test 56r "check lfs find -size works"
 
+test_56ra() {
+	[[ $MDS1_VERSION -gt $(version_code 2.12.58) ]] ||
+		skip "MDS < 2.12.58 doesn't return LSOM data"
+	local dir=$DIR/$tdir
+
+	[[ $OSC == "mdc" ]] && skip "DoM files" && return
+
+	setup_56 $dir $NUMFILES $NUMDIRS "-c 1"
+
+	cancel_lru_locks $OSC
+
+	local rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	local expected=12
+	local cmd="$LFS find -size 0 -type f -lazy $dir"
+	local nums=$($cmd | wc -l)
+
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+
+	local rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	[ $rpcs_before -eq $rpcs_after ] ||
+		error "'$cmd' should not send glimpse RPCs to OST"
+	cmd="$LFS find -size 0 -type f $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+	rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	echo "Before: $rpcs_before After: $rpcs_after $NUMFILES"
+	$LCTL get_param osc.*.stats
+	[ $rpcs_after -eq $((rpcs_before + 12)) ] ||
+		error "'$cmd' should send 12 glimpse RPCs to OST"
+
+	cancel_lru_locks $OSC
+	rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	expected=0
+	cmd="$LFS find ! -size 0 -type f -lazy $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+	rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	$LCTL get_param mdc.*.stats
+	[ $rpcs_before -eq $rpcs_after ] ||
+		error "'$cmd' should not send glimpse RPCs to OST"
+	cmd="$LFS find ! -size 0 -type f $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+	rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	echo "Before: $rpcs_before After: $rpcs_after $NUMFILES"
+	[ $rpcs_after -eq $((rpcs_before + 12)) ] ||
+		error "'$cmd' should send 12 glimpse RPCs to OST"
+
+	echo "test" > $dir/$tfile
+	echo "test2" > $dir/$tfile.2 && sync
+	cancel_lru_locks $OSC
+	rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	expected=1
+	cmd="$LFS find -size 5 -type f -lazy $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+	rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	[ $rpcs_before -eq $rpcs_after ] ||
+		error "'$cmd' should not send glimpse RPCs to OST"
+	cmd="$LFS find -size 5 -type f $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+	rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	echo "Before: $rpcs_before After: $rpcs_after $NUMFILES"
+	[ $rpcs_after -eq $((rpcs_before + 14)) ] ||
+		error "'$cmd' should send 14 glimpse RPCs to OST"
+
+	cancel_lru_locks $OSC
+	rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	expected=1
+	cmd="$LFS find -size +5 -type f -lazy $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+	rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	[ $rpcs_before -eq $rpcs_after ] ||
+		error "'$cmd' should not send glimpse RPCs to OST"
+	cmd="$LFS find -size +5 -type f $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+	rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	echo "Before: $rpcs_before After: $rpcs_after $NUMFILES"
+	[ $rpcs_after -eq $((rpcs_before + 14)) ] ||
+		error "'$cmd' should send 14 glimpse RPCs to OST"
+
+	cancel_lru_locks $OSC
+	rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	expected=2
+	cmd="$LFS find -size +0 -type f -lazy $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+	rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	[ $rpcs_before -eq $rpcs_after ] ||
+		error "'$cmd' should not send glimpse RPCs to OST"
+	cmd="$LFS find -size +0 -type f $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+	rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	echo "Before: $rpcs_before After: $rpcs_after $NUMFILES"
+	[ $rpcs_after -eq $((rpcs_before + 14)) ] ||
+		error "'$cmd' should send 14 glimpse RPCs to OST"
+
+	cancel_lru_locks $OSC
+	rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	expected=2
+	cmd="$LFS find ! -size -5 -type f -lazy $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+	rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	[ $rpcs_before -eq $rpcs_after ] ||
+		error "'$cmd' should not send glimpse RPCs to OST"
+	cmd="$LFS find ! -size -5 -type f $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+	rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	echo "Before: $rpcs_before After: $rpcs_after $NUMFILES"
+	[ $rpcs_after -eq $((rpcs_before + 14)) ] ||
+		error "'$cmd' should send 14 glimpse RPCs to OST"
+
+	cancel_lru_locks $OSC
+	rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	expected=12
+	cmd="$LFS find -size -5 -type f -lazy $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+	rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	[ $rpcs_before -eq $rpcs_after ] ||
+		error "'$cmd' should not send glimpse RPCs to OST"
+	cmd="$LFS find -size -5 -type f $dir"
+	nums=$($cmd | wc -l)
+	[ $nums -eq $expected ] ||
+		error "'$cmd' wrong: found $nums, expected $expected"
+	rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+	echo "Before: $rpcs_before After: $rpcs_after $NUMFILES"
+	[ $rpcs_after -eq $((rpcs_before + 14)) ] ||
+		error "'$cmd' should send 14 glimpse RPCs to OST"
+}
+run_test 56ra "check lfs find -size -lazy works for data on OSTs"
+
 test_56s() { # LU-611 #LU-9369
 	[[ $OSTCOUNT -lt 2 ]] && skip_env "need at least 2 OSTs"
 
@@ -5911,7 +6351,7 @@ test_56w() {
 	done
 
 	# $LFS_MIGRATE will fail if hard link migration is unsupported
-	if [[ $(lustre_version_code mds1) -gt $(version_code 2.5.55) ]]; then
+	if [[ $MDS1_VERSION -gt $(version_code 2.5.55) ]]; then
 		createmany -l$dir/dir1/file1 $dir/dir1/link 200 ||
 			error "creating links to $dir/dir1/file1 failed"
 	fi
@@ -6038,7 +6478,9 @@ test_56wb() {
 	echo "done."
 
 	echo -n "Removing test file from pool '$pool'..."
-	$LFS migrate $file1 &> /dev/null ||
+	# "lfs migrate $file" won't remove the file from the pool
+	# until some striping information is changed.
+	$LFS migrate -c 1 $file1 &> /dev/null ||
 		error "cannot remove from pool"
 	[ "$($LFS getstripe -p $file1)" ] &&
 		error "pool still set"
@@ -6060,19 +6502,23 @@ run_test 56wb "check lfs_migrate pool support"
 
 test_56wc() {
 	local file1="$DIR/$tdir/file1"
+	local parent_ssize
+	local parent_scount
+	local cur_ssize
+	local cur_scount
+	local orig_ssize
 
 	echo -n "Creating test dir..."
 	test_mkdir $DIR/$tdir &> /dev/null || error "cannot create dir"
-	local def_stripe_size=$($LFS getstripe -S $DIR/$tdir 2>/dev/null)
 	$LFS setstripe -S 1M -c 1 "$DIR/$tdir" &> /dev/null ||
-		error "cannot set stripe"
+		error "cannot set stripe by '-S 1M -c 1'"
 	echo "done"
 
 	echo -n "Setting initial stripe for test file..."
 	$LFS setstripe -S 512K -c 1 "$file1" &> /dev/null ||
 		error "cannot set stripe"
-	[ $($LFS getstripe -S "$file1") -eq 524288 ] ||
-		error "stripe size not set"
+	cur_ssize=$($LFS getstripe -S "$file1")
+	[ $cur_ssize -eq 524288 ] || error "setstripe -S $cur_ssize != 524288"
 	echo "done."
 
 	# File currently set to -S 512K -c 1
@@ -6089,8 +6535,8 @@ test_56wc() {
 	echo -n "Verifying -S option is passed through to lfs migrate..."
 	$LFS_MIGRATE -y -S 1M "$file1" &> /dev/null ||
 		error "migration failed"
-	[ $($LFS getstripe -S "$file1") -eq 1048576 ] ||
-		error "file was not restriped"
+	cur_ssize=$($LFS getstripe -S "$file1")
+	[ $cur_ssize -eq 1048576 ] || error "migrate -S $cur_ssize != 1048576"
 	echo "done."
 
 	# File currently set to -S 1M -c 1
@@ -6101,8 +6547,9 @@ test_56wc() {
 		error "long option without argument not supported"
 	$LFS_MIGRATE -y --stripe-size 512K "$file1" &> /dev/null ||
 		error "long option with argument not supported"
-	[ $($LFS getstripe -S "$file1") -eq 524288 ] ||
-		error "file not restriped with --stripe-size option"
+	cur_ssize=$($LFS getstripe -S "$file1")
+	[ $cur_ssize -eq 524288 ] ||
+		error "migrate --stripe-size $cur_ssize != 524288"
 	echo "done."
 
 	# File currently set to -S 512K -c 1
@@ -6111,8 +6558,8 @@ test_56wc() {
 		echo -n "Verifying explicit stripe count can be set..."
 		$LFS_MIGRATE -y -c 2 "$file1" &> /dev/null ||
 			error "migrate failed"
-		[ $($LFS getstripe -c "$file1") -eq 2 ] ||
-			error "file not restriped to explicit count"
+		cur_scount=$($LFS getstripe -c "$file1")
+		[ $cur_scount -eq 2 ] || error "migrate -c $cur_scount != 2"
 		echo "done."
 	fi
 
@@ -6121,17 +6568,21 @@ test_56wc() {
 	# Ensure parent striping is used if -R is set, and no stripe
 	# count or size is specified
 	echo -n "Setting stripe for parent directory..."
-	$LFS setstripe -S 1M -c 1 "$DIR/$tdir" &> /dev/null ||
-		error "cannot set stripe"
+	$LFS setstripe -S 2M -c 1 "$DIR/$tdir" &> /dev/null ||
+		error "cannot set stripe '-S 2M -c 1'"
 	echo "done."
 
 	echo -n "Verifying restripe option uses parent stripe settings..."
+	parent_ssize=$($LFS getstripe -S $DIR/$tdir 2>/dev/null)
+	parent_scount=$($LFS getstripe -c $DIR/$tdir 2>/dev/null)
 	$LFS_MIGRATE -y -R "$file1" &> /dev/null ||
 		error "migrate failed"
-	[ $($LFS getstripe -S "$file1") -eq $def_stripe_size ] ||
-		error "file not restriped to parent settings"
-	[ $($LFS getstripe -c "$file1") -eq 1 ] ||
-		error "file not restriped to parent settings"
+	cur_ssize=$($LFS getstripe -S "$file1")
+	[ $cur_ssize -eq $parent_ssize ] ||
+		error "migrate -R stripe_size $cur_ssize != $parent_ssize"
+	cur_scount=$($LFS getstripe -c "$file1")
+	[ $cur_scount -eq $parent_scount ] ||
+		error "migrate -R stripe_count $cur_scount != $parent_scount"
 	echo "done."
 
 	# File currently set to -S 1M -c 1
@@ -6139,13 +6590,14 @@ test_56wc() {
 	# Ensure striping is preserved if -R is not set, and no stripe
 	# count or size is specified
 	echo -n "Verifying striping size preserved when not specified..."
-	local orig_stripe_size=$($LFS getstripe -S "$file1" 2>/dev/null)
+	orig_ssize=$($LFS getstripe -S "$file1" 2>/dev/null)
 	$LFS setstripe -S 2M -c 1 "$DIR/$tdir" &> /dev/null ||
 		error "cannot set stripe on parent directory"
 	$LFS_MIGRATE -y "$file1" &> /dev/null ||
 		error "migrate failed"
-	[ $($LFS getstripe -S "$file1") -eq $orig_stripe_size ] ||
-		error "file was restriped"
+	cur_ssize=$($LFS getstripe -S "$file1")
+	[ $cur_ssize -eq $orig_ssize ] ||
+		error "migrate by default $cur_ssize != $orig_ssize"
 	echo "done."
 
 	# Ensure file name properly detected when final option has no argument
@@ -6249,6 +6701,7 @@ check_migrate_links() {
 	local file1="$dir/file1"
 	local begin="$2"
 	local count="$3"
+	local runas="$4"
 	local total_count=$(($begin + $count - 1))
 	local symlink_count=10
 	local uniq_count=10
@@ -6293,7 +6746,7 @@ check_migrate_links() {
 	fi
 
 	echo -n "migrating files..."
-	local migrate_out=$($LFS_MIGRATE -y -S '1m' $dir)
+	local migrate_out=$($runas $LFS_MIGRATE -y -S '1m' $dir)
 	local rc=$?
 	[ $rc -eq 0 ] || error "migrate failed rc = $rc"
 	echo "done"
@@ -6312,7 +6765,7 @@ check_migrate_links() {
 	# make sure hard links were properly detected, and migration was
 	# performed only once for the entire link set; nonlinked files should
 	# also be migrated
-	local actual=$(grep -c 'done migrate' <<< "$migrate_out")
+	local actual=$(grep -c 'done' <<< "$migrate_out")
 	local expected=$(($uniq_count + 1))
 
 	[ "$actual" -eq  "$expected" ] ||
@@ -6348,6 +6801,9 @@ test_56xb() {
 	echo "testing rsync mode when all links do not fit within xattrs"
 	LFS_MIGRATE_RSYNC_MODE=true check_migrate_links "$dir" 101 100
 
+	chown -R $RUNAS_ID $dir
+	echo "testing non-root lfs migrate mode when not all links are in xattr"
+	LFS_MIGRATE_RSYNC_MODE=false check_migrate_links "$dir" 101 100 "$RUNAS"
 
 	# clean up
 	rm -rf $dir
@@ -6363,13 +6819,14 @@ test_56xc() {
 
 	# Test 1: ensure file < 1 GB is always migrated with 1 stripe
 	echo -n "Setting initial stripe for 20MB test file..."
-	$LFS setstripe -c 2 -i 0 "$dir/20mb" || error "cannot setstripe"
+	$LFS setstripe -c 2 -i 0 "$dir/20mb" ||
+		error "cannot setstripe 20MB file"
 	echo "done"
 	echo -n "Sizing 20MB test file..."
 	truncate "$dir/20mb" 20971520 || error "cannot create 20MB test file"
 	echo "done"
 	echo -n "Verifying small file autostripe count is 1..."
-	$LFS_MIGRATE -y -A -C 1 "$dir/20mb" &> /dev/null ||
+	$LFS_MIGRATE -y -A -C 1 "$dir/20mb" ||
 		error "cannot migrate 20MB file"
 	local stripe_count=$($LFS getstripe -c "$dir/20mb") ||
 		error "cannot get stripe for $dir/20mb"
@@ -6382,24 +6839,28 @@ test_56xc() {
 	# sqrt(size_in_gb) + 1 OSTs but is larger than 1GB.  The file must
 	# have at least an additional 1KB for each desired stripe for test 3
 	echo -n "Setting stripe for 1GB test file..."
-	$LFS setstripe -c 1 -i 0 "$dir/1gb" || error "cannot setstripe"
+	$LFS setstripe -c 1 -i 0 "$dir/1gb" || error "cannot setstripe 1GB file"
 	echo "done"
 	echo -n "Sizing 1GB test file..."
 	# File size is 1GB + 3KB
-	truncate "$dir/1gb" 1073744896 &> /dev/null ||
-		error "cannot create 1GB test file"
-	echo "done"
-	echo -n "Migrating 1GB file..."
-	$LFS_MIGRATE -y -A -C 1 "$dir/1gb" &> /dev/null ||
-		error "cannot migrate file"
-	echo "done"
-	echo -n "Verifying autostripe count is sqrt(n) + 1..."
-	stripe_count=$($LFS getstripe -c "$dir/1gb") ||
-		error "cannot get stripe for $dir/1gb"
-	[ $stripe_count -eq 2 ] ||
-		error "unexpected stripe count $stripe_count (expected 2)"
+	truncate "$dir/1gb" 1073744896 || error "cannot create 1GB test file"
 	echo "done"
 
+	# need at least 512MB per OST for 1GB file to fit in 2 stripes
+	local avail=$($LCTL get_param -n llite.$FSNAME*.kbytesavail)
+	if (( avail > 524288 * OSTCOUNT )); then
+		echo -n "Migrating 1GB file..."
+		$LFS_MIGRATE -y -A -C 1 "$dir/1gb" ||
+			error "cannot migrate 1GB file"
+		echo "done"
+		echo -n "Verifying autostripe count is sqrt(n) + 1..."
+		stripe_count=$($LFS getstripe -c "$dir/1gb") ||
+			error "cannot getstripe for 1GB file"
+		[ $stripe_count -eq 2 ] ||
+			error "unexpected stripe count $stripe_count != 2"
+		echo "done"
+	fi
+
 	# Test 3: File is too large to fit within the available space on
 	# sqrt(n) + 1 OSTs.  Simulate limited available space with -X
 	if [ $OSTCOUNT -ge 3 ]; then
@@ -6407,15 +6868,15 @@ test_56xc() {
 		# file size (1GB + 3KB) / OST count (3).
 		local kb_per_ost=349526
 
-		echo -n "Migrating 1GB file..."
-		$LFS_MIGRATE -y -A -C 1 -X $kb_per_ost "$dir/1gb" &>> \
-			/dev/null || error "cannot migrate file"
+		echo -n "Migrating 1GB file with limit..."
+		$LFS_MIGRATE -y -A -C 1 -X $kb_per_ost "$dir/1gb" ||
+			error "cannot migrate 1GB file with limit"
 		echo "done"
 
 		stripe_count=$($LFS getstripe -c "$dir/1gb")
-		echo -n "Verifying autostripe count with limited space..."
-		[ "$stripe_count" -a $stripe_count -eq 3 ] ||
-			error "unexpected stripe count $stripe_count (wanted 3)"
+		echo -n "Verifying 1GB autostripe count with limited space..."
+		[ "$stripe_count" -a $stripe_count -ge 3 ] ||
+			error "unexpected stripe count $stripe_count (min 3)"
 		echo "done"
 	fi
 
@@ -6424,6 +6885,75 @@ test_56xc() {
 }
 run_test 56xc "lfs migration autostripe"
 
+test_56xd() {
+	[[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
+
+	local dir=$DIR/$tdir
+	local f_mgrt=$dir/$tfile.mgrt
+	local f_yaml=$dir/$tfile.yaml
+	local f_copy=$dir/$tfile.copy
+	local layout_yaml="-E 1M -S 512K -c 1 -E -1 -S 1M -c 2 -i 0"
+	local layout_copy="-c 2 -S 2M -i 1"
+	local yamlfile=$dir/yamlfile
+	local layout_before;
+	local layout_after;
+
+	test_mkdir "$dir" || error "cannot create dir $dir"
+	$LFS setstripe $layout_yaml $f_yaml ||
+		error "cannot setstripe $f_yaml with layout $layout_yaml"
+	$LFS getstripe --yaml $f_yaml > $yamlfile
+	$LFS setstripe $layout_copy $f_copy ||
+		error "cannot setstripe $f_copy with layout $layout_copy"
+	touch $f_mgrt
+	dd if=/dev/zero of=$f_mgrt bs=1M count=4
+
+	# 1. test option --yaml
+	$LFS_MIGRATE -y --yaml $yamlfile $f_mgrt ||
+		error "cannot migrate $f_mgrt with --yaml $yamlfile"
+	layout_before=$(get_layout_param $f_yaml)
+	layout_after=$(get_layout_param $f_mgrt)
+	[ "$layout_after" == "$layout_before" ] ||
+		error "lfs_migrate --yaml: $layout_after != $layout_before"
+
+	# 2. test option --copy
+	$LFS_MIGRATE -y --copy $f_copy $f_mgrt ||
+		error "cannot migrate $f_mgrt with --copy $f_copy"
+	layout_before=$(get_layout_param $f_copy)
+	layout_after=$(get_layout_param $f_mgrt)
+	[ "$layout_after" == "$layout_before" ] ||
+		error "lfs_migrate --copy: $layout_after != $layout_before"
+}
+run_test 56xd "check lfs_migrate --yaml and --copy support"
+
+test_56xe() {
+	[[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
+
+	local dir=$DIR/$tdir
+	local f_comp=$dir/$tfile
+	local layout="-E 1M -S 512K -c 1 -E -1 -S 1M -c 2 -i 0"
+	local layout_before=""
+	local layout_after=""
+
+	test_mkdir "$dir" || error "cannot create dir $dir"
+	$LFS setstripe $layout $f_comp ||
+		error "cannot setstripe $f_comp with layout $layout"
+	layout_before=$(get_layout_param $f_comp)
+	dd if=/dev/zero of=$f_comp bs=1M count=4
+
+	# 1. migrate a comp layout file by lfs_migrate
+	$LFS_MIGRATE -y $f_comp || error "cannot migrate $f_comp by lfs_migrate"
+	layout_after=$(get_layout_param $f_comp)
+	[ "$layout_before" == "$layout_after" ] ||
+		error "lfs_migrate: $layout_before != $layout_after"
+
+	# 2. migrate a comp layout file by lfs migrate
+	$LFS migrate $f_comp || error "cannot migrate $f_comp by lfs migrate"
+	layout_after=$(get_layout_param $f_comp)
+	[ "$layout_before" == "$layout_after" ] ||
+		error "lfs migrate: $layout_before != $layout_after"
+}
+run_test 56xe "migrate a composite layout file"
+
 test_56y() {
 	[ $MDS1_VERSION -lt $(version_code 2.4.53) ] &&
 		skip "No HSM $(lustre_build_version $SINGLEMDS) MDS < 2.4.53"
@@ -6466,18 +6996,20 @@ test_56z() { # LU-4824
 	test_mkdir $dir
 	for i in d{0..9}; do
 		test_mkdir $dir/$i
+		touch $dir/$i/$tfile
 	done
-	touch $dir/d{0..9}/$tfile
 	$LFS find $DIR/non_existent_dir $dir &&
 		error "$LFS find did not return an error"
 	# Make a directory unsearchable. This should NOT be the last entry in
 	# directory order.  Arbitrarily pick the 6th entry
 	chmod 700 $($LFS find $dir -type d | sed '6!d')
 
+	$RUNAS $LFS find $DIR/non_existent $dir
 	local count=$($RUNAS $LFS find $DIR/non_existent $dir | wc -l)
 
 	# The user should be able to see 10 directories and 9 files
-	[ $count == 19 ] || error "$LFS find did not continue after error"
+	(( count == 19 )) ||
+		error "$LFS find found $count != 19 entries after error"
 }
 run_test 56z "lfs find should continue after an error"
 
@@ -6800,9 +7332,9 @@ test_60a() {
 	local pass=true
 
 	#get fid and record list
-	fid_list=($(awk '/9_sub.*record/ { print $NF }' /$TMP/$tfile |
+	fid_list=($(awk '/9_sub.*record/ { print $NF }' $TMP/$tfile |
 		tail -n 4))
-	rec_list=($(awk '/9_sub.*record/ { print $((NF-3)) }' /$TMP/$tfile |
+	rec_list=($(awk '/9_sub.*record/ { print $((NF-3)) }' $TMP/$tfile |
 		tail -n 4))
 	#remount mgs as ldiskfs or zfs type
 	stop mgs || error "stop mgs failed"
@@ -6921,13 +7453,16 @@ run_test 60e "no space while new llog is being created"
 
 test_60g() {
 	local pid
+	local i
 
 	test_mkdir -c $MDSCOUNT $DIR/$tdir
-	$LFS setdirstripe -D -i -1 -c $MDSCOUNT $DIR/$tdir
 
 	(
 		local index=0
 		while true; do
+			$LFS setdirstripe -i $(($index % $MDSCOUNT)) \
+				-c $MDSCOUNT $DIR/$tdir/subdir$index \
+				2>/dev/null
 			mkdir $DIR/$tdir/subdir$index 2>/dev/null
 			rmdir $DIR/$tdir/subdir$index 2>/dev/null
 			index=$((index + 1))
@@ -6936,16 +7471,34 @@ test_60g() {
 
 	pid=$!
 
-	for i in $(seq 100); do 
+	for i in {0..100}; do
 		# define OBD_FAIL_OSD_TXN_START    0x19a
-		do_facet mds1 lctl set_param fail_loc=0x8000019a
+		local index=$((i % MDSCOUNT + 1))
+
+		do_facet mds$index $LCTL set_param fail_loc=0x8000019a \
+			> /dev/null
 		usleep 100
 	done
 
 	kill -9 $pid
 
+	for i in $(seq $MDSCOUNT); do
+		do_facet mds$i $LCTL set_param fail_loc=0 > /dev/null
+	done
+
 	mkdir $DIR/$tdir/new || error "mkdir failed"
 	rmdir $DIR/$tdir/new || error "rmdir failed"
+
+	do_facet mds1 $LCTL lfsck_start -M $(facet_svc mds1) -A -C \
+		-t namespace
+	for i in $(seq $MDSCOUNT); do
+		wait_update_facet mds$i "$LCTL get_param -n \
+			mdd.$(facet_svc mds$i).lfsck_namespace |
+			awk '/^status/ { print \\\$2 }'" "completed"
+	done
+
+	ls -R $DIR/$tdir || error "ls failed"
+	rm -rf $DIR/$tdir || error "rmdir failed"
 }
 run_test 60g "transaction abort won't cause MDT hung"
 
@@ -7060,8 +7613,8 @@ run_test 63b "async write errors should be returned to fsync ==="
 test_64a () {
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
 
-	df $DIR
-	lctl get_param -n osc.*[oO][sS][cC][_-]*.cur* | grep "[0-9]"
+	lfs df $DIR
+	lctl get_param osc.*[oO][sS][cC][_-]*.cur* | grep "=[1-9]"
 }
 run_test 64a "verify filter grant calculations (in kernel) ====="
 
@@ -7321,7 +7874,7 @@ cleanup_65k() {
 	do_facet $SINGLEMDS lctl --device  %$INACTIVE_OSC activate
 	echo $INACTIVE_OSC "is Activate"
 
-	wait_osc_import_state mds ost$ostnum FULL
+	wait_osc_import_state mds ost$(( ostnum + 1 )) FULL
 }
 
 test_65k() { # bug11679
@@ -7381,7 +7934,7 @@ test_65k() { # bug11679
 		do_facet $SINGLEMDS lctl --device  %$INACTIVE_OSC activate
 		echo $INACTIVE_OSC "is Activate"
 
-		wait_osc_import_state mds ost$ostnum FULL
+		wait_osc_import_state mds ost$(( ostnum + 1 )) FULL
 	done
 }
 run_test 65k "validate manual striping works properly with deactivated OSCs"
@@ -7407,7 +7960,7 @@ run_test 65m "normal user can't set filesystem default stripe"
 
 test_65n() {
 	[ -n "$FILESET" ] && skip "Not functional for FILESET set"
-	[[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.12.50) ]] ||
+	[[ $MDS1_VERSION -ge $(version_code 2.12.50) ]] ||
 		skip "Need MDS version at least 2.12.50"
 	[[ $PARALLEL != "yes" ]] || skip "skip parallel run"
 
@@ -7450,8 +8003,13 @@ test_65n() {
 
 	local dir3=$MOUNT/$tdir-3
 	mkdir $dir3 || error "mkdir $dir3 failed"
-	! getfattr -n trusted.lov $dir3 &> /dev/null ||
-		error "$dir3 shouldn't have LOV EA"
+	# $dir3 shouldn't have LOV EA, but "lfs getstripe -d $dir3" should show
+	# the root layout, which is the actual default layout that will be used
+	# when new files are created in $dir3.
+	local dir3_layout=$(get_layout_param $dir3)
+	local root_dir_layout=$(get_layout_param $MOUNT)
+	[[ "$dir3_layout" = "$root_dir_layout" ]] ||
+		error "$dir3 should show the default layout from $MOUNT"
 
 	# set OST pool on root directory
 	local pool=$TESTNAME
@@ -7473,8 +8031,14 @@ test_65n() {
 
 	local dir4=$MOUNT/$tdir-4
 	mkdir $dir4 || error "mkdir $dir4 failed"
-	! getfattr -n trusted.lov $dir4 &> /dev/null ||
-		error "$dir4 shouldn't have LOV EA"
+	local dir4_layout=$(get_layout_param $dir4)
+	root_dir_layout=$(get_layout_param $MOUNT)
+	echo "$LFS getstripe -d $dir4"
+	$LFS getstripe -d $dir4
+	echo "$LFS getstripe -d $MOUNT"
+	$LFS getstripe -d $MOUNT
+	[[ "$dir4_layout" = "$root_dir_layout" ]] ||
+		error "$dir4 should show the default layout from $MOUNT"
 
 	# new file created in $dir4 should inherit the pool from
 	# the filesystem default
@@ -7493,10 +8057,33 @@ test_65n() {
 	local dir5=$dir4/$tdir-5
 	mkdir $dir5 || error "mkdir $dir5 failed"
 
-	local dir4_layout=$(get_layout_param $dir4)
+	dir4_layout=$(get_layout_param $dir4)
 	local dir5_layout=$(get_layout_param $dir5)
 	[[ "$dir4_layout" = "$dir5_layout" ]] ||
 		error "$dir5 should inherit the default layout from $dir4"
+
+	# though subdir under ROOT doesn't inherit default layout, but
+	# its sub dir/file should be created with default layout.
+	[[ $MDSCOUNT -ge 2 ]] || skip_env "needs >= 2 MDTs"
+	[[ $MDS1_VERSION -ge $(version_code 2.12.59) ]] ||
+		skip "Need MDS version at least 2.12.59"
+
+	local default_lmv_count=$($LFS getdirstripe -D -c $MOUNT)
+	local default_lmv_index=$($LFS getdirstripe -D -i $MOUNT)
+	local default_lmv_hash=$($LFS getdirstripe -D -H $MOUNT)
+
+	if [ $default_lmv_hash == "none" ]; then
+		stack_trap "$LFS setdirstripe -D -d $MOUNT" EXIT
+	else
+		stack_trap "$LFS setdirstripe -D -i $default_lmv_index \
+			-c $default_lmv_count -H $default_lmv_hash $MOUNT" EXIT
+	fi
+
+	$LFS setdirstripe -D -c 2 $MOUNT ||
+		error "setdirstripe -D -c 2 failed"
+	mkdir $MOUNT/$tdir-6 || error "mkdir $tdir-6 failed"
+	local lmv_count=$($LFS getdirstripe -c $MOUNT/$tdir-6)
+	[ $lmv_count -eq 2 ] || error "$tdir-6 stripe count $lmv_count"
 }
 run_test 65n "don't inherit default layout from root for new subdirectories"
 
@@ -7690,28 +8277,27 @@ num_inodes() {
 test_76() { # Now for bug 20433, added originally in bug 1443
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
 
-	local CPUS=$(getconf _NPROCESSORS_ONLN 2>/dev/null)
-
 	cancel_lru_locks osc
-	BEFORE_INODES=$(num_inodes)
-	echo "before inodes: $BEFORE_INODES"
-	local COUNT=1000
-	[ "$SLOW" = "no" ] && COUNT=100
-	for i in $(seq $COUNT); do
+	local cpus=$(getconf _NPROCESSORS_ONLN 2>/dev/null)
+	local before=$(num_inodes)
+	local count=$((512 * cpus))
+	[ "$SLOW" = "no" ] && count=$((64 * cpus))
+
+	echo "before inodes: $before"
+	for i in $(seq $count); do
 		touch $DIR/$tfile
 		rm -f $DIR/$tfile
 	done
 	cancel_lru_locks osc
-	AFTER_INODES=$(num_inodes)
-	echo "after inodes: $AFTER_INODES"
-	local wait=0
-	while [[ $((AFTER_INODES-1*${CPUS:-1})) -gt $BEFORE_INODES ]]; do
-		sleep 2
-		AFTER_INODES=$(num_inodes)
-		wait=$((wait+2))
-		echo "wait $wait seconds inodes: $AFTER_INODES"
-		if [ $wait -gt 30 ]; then
-			error "inode slab grew from $BEFORE_INODES to $AFTER_INODES"
+	local after=$(num_inodes)
+	echo "after inodes: $after"
+	while (( after > before + 8 * ${cpus:-1} )); do
+		sleep 1
+		after=$(num_inodes)
+		wait=$((wait + 1))
+		(( wait % 5 == 0 )) && echo "wait $wait seconds inodes: $after"
+		if (( wait > 30 )); then
+			error "inode slab grew from $before to $after"
 		fi
 	done
 }
@@ -7740,9 +8326,22 @@ CKSUM_TYPES=${CKSUM_TYPES:-$(lctl get_param -n osc.*osc-[^mM]*.checksum_type |
 set_checksum_type()
 {
 	lctl set_param -n osc.*osc-[^mM]*.checksum_type $1
-	log "set checksum type to $1"
-	return 0
+	rc=$?
+	log "set checksum type to $1, rc = $rc"
+	return $rc
+}
+
+get_osc_checksum_type()
+{
+	# arugment 1: OST name, like OST0000
+	ost=$1
+	checksum_type=$(lctl get_param -n osc.*${ost}-osc-[^mM]*.checksum_type |
+			sed 's/.*\[\(.*\)\].*/\1/g')
+	rc=$?
+	[ $rc -ne 0 ] && error "failed to get checksum type of $ost, rc = $rc, output = $checksum_type"
+	echo $checksum_type
 }
+
 F77_TMP=$TMP/f77-temp
 F77SZ=8
 setup_f77() {
@@ -7971,7 +8570,7 @@ test_77k() { # LU-10906
 		wait_update $HOSTNAME "$get_checksum" $i
 		#remount
 		echo "remount client, checksum should be $i"
-		remount_client $MOUNT || "failed to remount client"
+		remount_client $MOUNT || error "failed to remount client"
 		checksum=$(eval $get_checksum)
 		[ $checksum -eq $i ] || error "checksum($checksum) != $i"
 	done
@@ -7982,20 +8581,52 @@ test_77k() { # LU-10906
 	for opt in "checksum" "nochecksum"; do
 		#remount with mount option
 		echo "remount client with option $opt, checksum should be $i"
-		umount_client $MOUNT || "failed to umount client"
+		umount_client $MOUNT || error "failed to umount client"
 		mount_client $MOUNT "$MOUNT_OPTS,$opt" ||
-			"failed to mount client with option '$opt'"
+			error "failed to mount client with option '$opt'"
 		checksum=$(eval $get_checksum)
 		[ $checksum -eq $i ] || error "checksum($checksum) != $i"
 		i=$((i - 1))
 	done
 
-	remount_client $MOUNT || "failed to remount client"
+	remount_client $MOUNT || error "failed to remount client"
 }
 run_test 77k "enable/disable checksum correctly"
 
-[ "$ORIG_CSUM" ] && set_checksums $ORIG_CSUM || true
-rm -f $F77_TMP
+test_77l() {
+	[ $PARALLEL == "yes" ] && skip "skip parallel run"
+	$GSS && skip_env "could not run with gss"
+
+	set_checksums 1
+	stack_trap "set_checksums $ORIG_CSUM" EXIT
+	stack_trap "set_checksum_type $ORIG_CSUM_TYPE" EXIT
+
+	set_checksum_type invalid && error "unexpected success of invalid checksum type"
+
+	$LFS setstripe -c 1 -i 0 $DIR/$tfile
+	for algo in $CKSUM_TYPES; do
+		set_checksum_type $algo || error "fail to set checksum type $algo"
+		osc_algo=$(get_osc_checksum_type OST0000)
+		[ "$osc_algo" != "$algo" ] && error "checksum type is $osc_algo after setting it to $algo"
+
+		# no locks, no reqs to let the connection idle
+		cancel_lru_locks osc
+		lru_resize_disable osc
+		wait_osc_import_state client ost1 IDLE
+
+		# ensure ost1 is connected
+		stat $DIR/$tfile >/dev/null || error "can't stat"
+		wait_osc_import_state client ost1 FULL
+
+		osc_algo=$(get_osc_checksum_type OST0000)
+		[ "$osc_algo" != "$algo" ] && error "checksum type changed from $algo to $osc_algo after reconnection"
+	done
+	return 0
+}
+run_test 77l "preferred checksum type is remembered after reconnected"
+
+[ "$ORIG_CSUM" ] && set_checksums $ORIG_CSUM || true
+rm -f $F77_TMP
 unset F77_TMP
 
 cleanup_test_78() {
@@ -8087,31 +8718,35 @@ test_80() { # bug 10718
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
 
 	# relax strong synchronous semantics for slow backends like ZFS
-	local soc="obdfilter.*.sync_on_lock_cancel"
-	local soc_old=$(do_facet ost1 lctl get_param -n $soc | head -n1)
-	local hosts=
-	if [ "$soc_old" != "never" ] &&
-		[ "$ost1_FSTYPE" != "ldiskfs" ]; then
-			hosts=$(for host in $(seq -f "ost%g" 1 $OSTCOUNT); do
-				facet_active_host $host; done | sort -u)
-			do_nodes $hosts lctl set_param $soc=never
-	fi
+	if [ "$ost1_FSTYPE" != "ldiskfs" ]; then
+		local soc="obdfilter.*.sync_lock_cancel"
+		local save=$(do_facet ost1 $LCTL get_param -n $soc | head -n1)
 
-        dd if=/dev/zero of=$DIR/$tfile bs=1M count=1 seek=1M
-        sync; sleep 1; sync
-        local BEFORE=`date +%s`
-        cancel_lru_locks osc
-        local AFTER=`date +%s`
-        local DIFF=$((AFTER-BEFORE))
-        if [ $DIFF -gt 1 ] ; then
-                error "elapsed for 1M@1T = $DIFF"
-        fi
+		# "sync_on_lock_cancel" was broken by v2_11_55_0-26-g7059644e9a
+		if [ -z "$save" ]; then
+			soc="obdfilter.*.sync_on_lock_cancel"
+			save=$(do_facet ost1 $LCTL get_param -n $soc | head -n1)
+		fi
 
-        [ -n "$hosts" ] && do_nodes $hosts lctl set_param $soc=$soc_old
+		if [ "$save" != "never" ]; then
+			local hosts=$(comma_list $(osts_nodes))
 
-        rm -f $DIR/$tfile
+			do_nodes $hosts $LCTL set_param $soc=never
+			stack_trap "do_nodes $hosts $LCTL set_param $soc=$save"
+		fi
+	fi
+
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=1 seek=1M
+	sync; sleep 1; sync
+	local before=$(date +%s)
+	cancel_lru_locks osc
+	local after=$(date +%s)
+	local diff=$((after - before))
+	[ $diff -le 1 ] || error "elapsed for 1M@1T = $diff"
+
+	rm -f $DIR/$tfile
 }
-run_test 80 "Page eviction is equally fast at high offsets too  ===="
+run_test 80 "Page eviction is equally fast at high offsets too"
 
 test_81a() { # LU-456
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
@@ -8255,7 +8890,6 @@ cleanup_101a() {
 
 test_101a() {
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
-	[ $MDSCOUNT -ge 2 ] && skip_env "needs < 2 MDTs" #LU-4322
 
 	local s
 	local discard
@@ -8280,9 +8914,12 @@ test_101a() {
 	done
 	cleanup_101a
 
-	if [[ $(($discard * 10)) -gt $nreads ]]; then
-		$LCTL get_param osc.*-osc*.rpc_stats
-		$LCTL get_param llite.*.read_ahead_stats
+	$LCTL get_param osc.*-osc*.rpc_stats
+	$LCTL get_param llite.*.read_ahead_stats
+
+	# Discard is generally zero, but sometimes a few random reads line up
+	# and trigger larger readahead, which is wasted & leads to discards.
+	if [[ $(($discard)) -gt $nreads ]]; then
 		error "too many ($discard) discarded pages"
 	fi
 	rm -f $DIR/$tfile || true
@@ -8392,12 +9029,14 @@ test_101c() {
 	cancel_lru_locks osc
 	$LCTL set_param osc.*.rpc_stats 0
 	$READS -f $DIR/$tfile -s$FILE_LENGTH -b$rsize -n$nreads -t 180
+	$LCTL get_param osc.*.rpc_stats
 	for osc_rpc_stats in $($LCTL get_param -N osc.*.rpc_stats); do
 		local stats=$($LCTL get_param -n $osc_rpc_stats)
 		local lines=$(echo "$stats" | awk 'END {print NR;}')
 		local size
 
 		if [ $lines -le 20 ]; then
+			echo "continue debug"
 			continue
 		fi
 		for size in 1 2 4 8; do
@@ -8413,11 +9052,6 @@ test_101c() {
 }
 run_test 101c "check stripe_size aligned read-ahead ================="
 
-set_read_ahead() {
-	$LCTL get_param -n llite.*.max_read_ahead_mb | head -n 1
-	$LCTL set_param -n llite.*.max_read_ahead_mb $1 > /dev/null 2>&1
-}
-
 test_101d() {
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
 
@@ -8437,7 +9071,10 @@ test_101d() {
 	cancel_lru_locks osc
 
 	echo Disable read-ahead
-	local old_READAHEAD=$(set_read_ahead 0)
+	local old_RA=$($LCTL get_param -n llite.*.max_read_ahead_mb | head -n 1)
+	$LCTL set_param -n llite.*.max_read_ahead_mb=0
+	stack_trap "$LCTL set_param -n llite.*.max_read_ahead_mb $old_RA" EXIT
+	$LCTL get_param -n llite.*.max_read_ahead_mb
 
 	echo Reading the test file $file with read-ahead disabled
 	local raOFF=$(do_and_time "dd if=$file of=/dev/null bs=1M count=$sz_MB")
@@ -8445,7 +9082,7 @@ test_101d() {
 	echo Cancel LRU locks on lustre client to flush the client cache
 	cancel_lru_locks osc
 	echo Enable read-ahead with ${ra_MB}MB
-	set_read_ahead $ra_MB
+	$LCTL set_param -n llite.*.max_read_ahead_mb=$ra_MB
 
 	echo Reading the test file $file with read-ahead enabled
 	local raON=$(do_and_time "dd if=$file of=/dev/null bs=1M count=$sz_MB")
@@ -8453,7 +9090,6 @@ test_101d() {
 	echo "read-ahead disabled time read $raOFF"
 	echo "read-ahead enabled  time read $raON"
 
-	set_read_ahead $old_READAHEAD
 	rm -f $file
 	wait_delete_completed
 
@@ -8637,6 +9273,30 @@ test_101h() {
 }
 run_test 101h "Readahead should cover current read window"
 
+test_101i() {
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=10 ||
+		error "dd 10M file failed"
+
+	local max_per_file_mb=$($LCTL get_param -n \
+		llite.*.max_read_ahead_per_file_mb 2>/dev/null)
+	cancel_lru_locks osc
+	stack_trap "$LCTL set_param llite.*.max_read_ahead_per_file_mb=$max_per_file_mb"
+	$LCTL set_param llite.*.max_read_ahead_per_file_mb=1 ||
+		error "set max_read_ahead_per_file_mb to 1 failed"
+
+	echo "Reset readahead stats"
+	$LCTL set_param llite.*.read_ahead_stats=0
+
+	dd if=$DIR/$tfile of=/dev/null bs=2M
+
+	$LCTL get_param llite.*.read_ahead_stats
+	local miss=$($LCTL get_param -n llite.*.read_ahead_stats |
+		     awk '/misses/ { print $2 }')
+	[ $miss -eq 5 ] || error "expected misses 5 but got $miss"
+	rm -f $DIR/$tfile
+}
+run_test 101i "allow current readahead to exceed reservation"
+
 setup_test102() {
 	test_mkdir $DIR/$tdir
 	chown $RUNAS_ID $DIR/$tdir
@@ -10644,6 +11304,27 @@ test_123b () { # statahead(bug 15027)
 }
 run_test 123b "not panic with network error in statahead enqueue (bug 15027)"
 
+test_123c() {
+	[[ $MDSCOUNT -lt 2 ]] && skip_env "needs >= 2 MDTs"
+
+	test_mkdir -i 0 -c 1 $DIR/$tdir.0
+	test_mkdir -i 1 -c 1 $DIR/$tdir.1
+	touch $DIR/$tdir.1/{1..3}
+	mv $DIR/$tdir.1/{1..3} $DIR/$tdir.0
+
+	remount_client $MOUNT
+
+	$MULTIOP $DIR/$tdir.0 Q
+
+	# let statahead to complete
+	ls -l $DIR/$tdir.0 > /dev/null
+
+	testid=$(echo $TESTNAME | tr '_' ' ')
+	dmesg | tac | sed "/$testid/,$ d" | grep "Can not initialize inode" &&
+		error "statahead warning" || true
+}
+run_test 123c "Can not initialize inode warning on DNE statahead"
+
 test_124a() {
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
 	$LCTL get_param -n mdc.*.connect_flags | grep -q lru_resize ||
@@ -10923,41 +11604,52 @@ run_test 126 "check that the fsgid provided by the client is taken into account"
 
 test_127a() { # bug 15521
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
+	local name count samp unit min max sum sumsq
 
 	$LFS setstripe -i 0 -c 1 $DIR/$tfile || error "setstripe failed"
+	echo "stats before reset"
+	$LCTL get_param osc.*.stats
 	$LCTL set_param osc.*.stats=0
-	FSIZE=$((2048 * 1024))
-	dd if=/dev/zero of=$DIR/$tfile bs=$FSIZE count=1
+	local fsize=$((2048 * 1024))
+
+	dd if=/dev/zero of=$DIR/$tfile bs=$fsize count=1
 	cancel_lru_locks osc
-	dd if=$DIR/$tfile of=/dev/null bs=$FSIZE
-
-	$LCTL get_param osc.*0000-osc-*.stats | grep samples > $DIR/${tfile}.tmp
-        while read NAME COUNT SAMP UNIT MIN MAX SUM SUMSQ; do
-                echo "got $COUNT $NAME"
-                [ ! $MIN ] && error "Missing min value for $NAME proc entry"
-                eval $NAME=$COUNT || error "Wrong proc format"
-
-                case $NAME in
-                        read_bytes|write_bytes)
-                        [ $MIN -lt 4096 ] && error "min is too small: $MIN"
-                        [ $MIN -gt $FSIZE ] && error "min is too big: $MIN"
-                        [ $MAX -lt 4096 ] && error "max is too small: $MAX"
-                        [ $MAX -gt $FSIZE ] && error "max is too big: $MAX"
-                        [ $SUM -ne $FSIZE ] && error "sum is wrong: $SUM"
-                        [ $SUMSQ -lt $(((FSIZE /4096) * (4096 * 4096))) ] &&
-                                error "sumsquare is too small: $SUMSQ"
-                        [ $SUMSQ -gt $((FSIZE * FSIZE)) ] &&
-                                error "sumsquare is too big: $SUMSQ"
-                        ;;
-                        *) ;;
-                esac
-        done < $DIR/${tfile}.tmp
-
-        #check that we actually got some stats
-        [ "$read_bytes" ] || error "Missing read_bytes stats"
-        [ "$write_bytes" ] || error "Missing write_bytes stats"
-        [ "$read_bytes" != 0 ] || error "no read done"
-        [ "$write_bytes" != 0 ] || error "no write done"
+	dd if=$DIR/$tfile of=/dev/null bs=$fsize
+
+	$LCTL get_param osc.*0000-osc-*.stats | grep samples > $DIR/$tfile.tmp
+	stack_trap "rm -f $TMP/$tfile.tmp"
+	while read name count samp unit min max sum sumsq; do
+		echo "got name=$name count=$count unit=$unit min=$min max=$max"
+		[ ! $min ] && error "Missing min value for $name proc entry"
+		eval $name=$count || error "Wrong proc format"
+
+		case $name in
+		read_bytes|write_bytes)
+			[[ "$unit" =~ "bytes" ]] ||
+				error "unit is not 'bytes': $unit"
+			(( $min >= 4096 )) || error "min is too small: $min"
+			(( $min <= $fsize )) || error "min is too big: $min"
+			(( $max >= 4096 )) || error "max is too small: $max"
+			(( $max <= $fsize )) || error "max is too big: $max"
+			(( $sum == $fsize )) || error "sum is wrong: $sum"
+			(( $sumsq >= ($fsize / 4096) * (4096 * 4096) )) ||
+				error "sumsquare is too small: $sumsq"
+			(( $sumsq <= $fsize * $fsize )) ||
+				error "sumsquare is too big: $sumsq"
+			;;
+		ost_read|ost_write)
+			[[ "$unit" =~ "usec" ]] ||
+				error "unit is not 'usec': $unit"
+			;;
+		*)	;;
+		esac
+	done < $DIR/$tfile.tmp
+
+	#check that we actually got some stats
+	[ "$read_bytes" ] || error "Missing read_bytes stats"
+	[ "$write_bytes" ] || error "Missing write_bytes stats"
+	[ "$read_bytes" != 0 ] || error "no read done"
+	[ "$write_bytes" != 0 ] || error "no write done"
 }
 run_test 127a "verify the client stats are sane"
 
@@ -10965,6 +11657,8 @@ test_127b() { # bug LU-333
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
 	local name count samp unit min max sum sumsq
 
+	echo "stats before reset"
+	$LCTL get_param llite.*.stats
 	$LCTL set_param llite.*.stats=0
 
 	# perform 2 reads and writes so MAX is different from SUM.
@@ -10975,30 +11669,28 @@ test_127b() { # bug LU-333
 	dd if=$DIR/$tfile of=/dev/null bs=$PAGE_SIZE count=1
 
 	$LCTL get_param llite.*.stats | grep samples > $TMP/$tfile.tmp
+	stack_trap "rm -f $TMP/$tfile.tmp"
 	while read name count samp unit min max sum sumsq; do
-		echo "got $count $name"
+		echo "got name=$name count=$count unit=$unit min=$min max=$max"
 		eval $name=$count || error "Wrong proc format"
 
 		case $name in
-		read_bytes)
-			[ $count -ne 2 ] && error "count is not 2: $count"
-			[ $min -ne $PAGE_SIZE ] &&
+		read_bytes|write_bytes)
+			[[ "$unit" =~ "bytes" ]] ||
+				error "unit is not 'bytes': $unit"
+			(( $count == 2 )) || error "count is not 2: $count"
+			(( $min == $PAGE_SIZE )) ||
 				error "min is not $PAGE_SIZE: $min"
-			[ $max -ne $PAGE_SIZE ] &&
-				error "max is incorrect: $max"
-			[ $sum -ne $((PAGE_SIZE * 2)) ] &&
-				error "sum is wrong: $sum"
+			(( $max == $PAGE_SIZE )) ||
+				error "max is not $PAGE_SIZE: $max"
+			(( $sum == $PAGE_SIZE * 2 )) ||
+				error "sum is not $((PAGE_SIZE * 2)): $sum"
 			;;
-		write_bytes)
-			[ $count -ne 2 ] && error "count is not 2: $count"
-			[ $min -ne $PAGE_SIZE ] &&
-				error "min is not $PAGE_SIZE: $min"
-			[ $max -ne $PAGE_SIZE ] &&
-				error "max is incorrect: $max"
-			[ $sum -ne $((PAGE_SIZE * 2)) ] &&
-				error "sum is wrong: $sum"
+		read|write)
+			[[ "$unit" =~ "usec" ]] ||
+				error "unit is not 'usec': $unit"
 			;;
-		*) ;;
+		*)	;;
 		esac
 	done < $TMP/$tfile.tmp
 
@@ -11007,11 +11699,78 @@ test_127b() { # bug LU-333
 	[ "$write_bytes" ] || error "Missing write_bytes stats"
 	[ "$read_bytes" != 0 ] || error "no read done"
 	[ "$write_bytes" != 0 ] || error "no write done"
-
-	rm -f $TMP/${tfile}.tmp
 }
 run_test 127b "verify the llite client stats are sane"
 
+test_127c() { # LU-12394
+	[ "$OSTCOUNT" -lt "2" ] && skip_env "needs >= 2 OSTs"
+	local size
+	local bsize
+	local reads
+	local writes
+	local count
+
+	$LCTL set_param llite.*.extents_stats=1
+	stack_trap "$LCTL set_param llite.*.extents_stats=0" EXIT
+
+	# Use two stripes so there is enough space in default config
+	$LFS setstripe -c 2 $DIR/$tfile
+
+	# Extent stats start at 0-4K and go in power of two buckets
+	# LL_HIST_START = 12 --> 2^12 = 4K
+	# We do 3K*2^i, so 3K, 6K, 12K, 24K... hitting each bucket.
+	# We do not do buckets larger than 64 MiB to avoid ENOSPC issues on
+	# small configs
+	for size in 3K 6K 12K 24K 48K 96K 192K 384K 768K 1536K 3M 6M 12M 24M 48M;
+		do
+		# Write and read, 2x each, second time at a non-zero offset
+		dd if=/dev/zero of=$DIR/$tfile bs=$size count=1
+		dd if=/dev/zero of=$DIR/$tfile bs=$size count=1 seek=10
+		dd if=$DIR/$tfile of=/dev/null bs=$size count=1
+		dd if=$DIR/$tfile of=/dev/null bs=$size count=1 seek=10
+		rm -f $DIR/$tfile
+	done
+
+	$LCTL get_param llite.*.extents_stats
+
+	count=2
+	for bsize in 4K 8K 16K 32K 64K 128K 256K 512K 1M 2M 4M 8M 16M 32M 64M;
+		do
+		local bucket=$($LCTL get_param -n llite.*.extents_stats |
+				grep -m 1 $bsize)
+		reads=$(echo $bucket | awk '{print $5}')
+		writes=$(echo $bucket | awk '{print $9}')
+		[ "$reads" -eq $count ] ||
+			error "$reads reads in < $bsize bucket, expect $count"
+		[ "$writes" -eq $count ] ||
+			error "$writes writes in < $bsize bucket, expect $count"
+	done
+
+	# Test mmap write and read
+	$LCTL set_param llite.*.extents_stats=c
+	size=512
+	dd if=/dev/zero of=$DIR/$tfile bs=${size}K count=1
+	$MULTIOP $DIR/$tfile OSMRUc || error "$MULTIOP $DIR/$tfile failed"
+	$MULTIOP $DIR/$tfile OSMWUc || error "$MULTIOP $DIR/$tfile failed"
+
+	$LCTL get_param llite.*.extents_stats
+
+	count=$(((size*1024) / PAGE_SIZE))
+
+	bsize=$((2 * PAGE_SIZE / 1024))K
+
+	bucket=$($LCTL get_param -n llite.*.extents_stats |
+			grep -m 1 $bsize)
+	reads=$(echo $bucket | awk '{print $5}')
+	writes=$(echo $bucket | awk '{print $9}')
+	# mmap writes fault in the page first, creating an additonal read
+	[ "$reads" -eq $((2 * count)) ] ||
+		error "$reads reads in < $bsize bucket, expect $count"
+	[ "$writes" -eq $count ] ||
+		error "$writes writes in < $bsize bucket, expect $count"
+}
+run_test 127c "test llite extent stats with regular & mmap i/o"
+
 test_128() { # bug 15212
 	touch $DIR/$tfile
 	$LFS 2>&1 <<-EOF | tee $TMP/$tfile.log
@@ -11876,33 +12635,22 @@ test_133g() {
 	remote_mds_nodsh && skip "remote MDS with nodsh"
 	remote_ost_nodsh && skip "remote OST with nodsh"
 
-	# eventually, this can also be replaced with "lctl get_param -R",
-	# but not until that option is always available on the server
 	local facet
 	for facet in mds1 ost1; do
-		[ $(lustre_version_code $facet) -le $(version_code 2.5.54) ] &&
-			skip_noexit "Too old lustre on $facet"
-		local facet_proc_dirs=$(do_facet $facet \
-					\\\ls -d $proc_regexp 2>/dev/null)
-		echo "${facet}_proc_dirs='$facet_proc_dirs'"
-		[ -z "$facet_proc_dirs" ] && error "no proc_dirs on $facet"
-		do_facet $facet find $facet_proc_dirs \
-			! -name req_history \
-			-exec cat '{}' \\\; &> /dev/null
-
-		do_facet $facet find $facet_proc_dirs \
-			! -name req_history \
-			-type f \
-			-exec cat '{}' \\\; &> /dev/null ||
-				error "proc file read failed"
-
-		do_facet $facet find $facet_proc_dirs \
-			-ignore_readdir_race \
-			-type f \
-			-not -name force_lbug \
-			-not -name changelog_mask \
-			-exec badarea_io '{}' \\\; ||
-				error_133 "$facet find $facet_proc_dirs failed"
+		local facet_ver=$(lustre_version_code $facet)
+		if [ $facet_ver -ge $(version_code 2.7.65) ]; then
+			do_facet $facet "$LCTL get_param -R '*'" &> /dev/null
+		else
+			log "$facet: too old lustre for get_param -R"
+		fi
+		if [ $facet_ver -ge $(version_code 2.5.54) ]; then
+			do_facet $facet "$LCTL list_param -R '*' | grep '=' |
+				tr -d= | egrep -v 'force_lbug|changelog_mask' |
+				xargs badarea_io" ||
+					error_133 "$facet badarea_io failed"
+		else
+			skip_noexit "$facet: too old lustre for get_param -R"
+		fi
 	done
 
 	# remount the FS in case writes/reads /proc break the FS
@@ -12018,6 +12766,73 @@ test_134b() {
 }
 run_test 134b "Server rejects lock request when reaching lock_limit_mb"
 
+test_135() {
+	remote_mds_nodsh && skip "remote MDS with nodsh"
+	[[ $MDS1_VERSION -lt $(version_code 2.13.50) ]] &&
+		skip "Need MDS version at least 2.13.50"
+	local fname
+
+	mkdir -p $DIR/$tdir || error "failed to create $DIR/$tdir"
+
+#define OBD_FAIL_PLAIN_RECORDS 0x1319
+	#set only one record at plain llog
+	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x1319 fail_val=1
+
+	#fill already existed plain llog each 64767
+	#wrapping whole catalog
+	createmany -o -u $DIR/$tdir/$tfile- $((64767 * 1))
+
+	createmany -o $DIR/$tdir/$tfile_ 64700
+	for (( i = 0; i < 64700; i = i + 2 ))
+	do
+		rm $DIR/$tdir/$tfile_$i &
+		rm $DIR/$tdir/$tfile_$((i + 1)) &
+		local pid=$!
+		wait $pid
+	done
+
+	#waiting osp synchronization
+	wait_delete_completed
+}
+run_test 135 "Race catalog processing"
+
+test_136() {
+	remote_mds_nodsh && skip "remote MDS with nodsh"
+	[[ $MDS1_VERSION -lt $(version_code 2.13.50) ]] &&
+		skip "Need MDS version at least 2.13.50"
+	local fname
+
+	mkdir -p $DIR/$tdir || error "failed to create $DIR/$tdir"
+	$LFS setstripe -c 1 -i 0 $DIR/$tdir || error "failed to set striping"
+	#set only one record at plain llog
+#define OBD_FAIL_CATALOG_FULL_CHECK                0x131a
+	do_facet $SINGLEMDS $LCTL set_param fail_loc=0x131a fail_val=1
+
+	#fill already existed 2 plain llogs each 64767
+	#wrapping whole catalog
+	createmany -o -u $DIR/$tdir/$tfile- $((64767 * 1))
+	createmany -o -u $DIR/$tdir/$tfile- $((64767 * 3 / 2))
+	wait_delete_completed
+
+	createmany -o $DIR/$tdir/$tfile_ 10
+	sleep 25
+
+	do_facet $SINGLEMDS $LCTL set_param fail_val=3
+	for (( i = 0; i < 10; i = i + 3 ))
+	do
+		rm $DIR/$tdir/$tfile_$i &
+		rm $DIR/$tdir/$tfile_$((i + 1)) &
+		local pid=$!
+		wait $pid
+		sleep 7
+		rm $DIR/$tdir/$tfile_$((i + 2)) &
+	done
+
+	#waiting osp synchronization
+	wait_delete_completed
+}
+run_test 136 "Race catalog processing 2"
+
 test_140() { #bug-17379
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
 
@@ -12191,8 +13006,9 @@ test_151() {
 		error "NOT IN CACHE: before: $BEFORE, after: $AFTER"
 	fi
 
-        # the following read invalidates the cache
         cancel_lru_locks osc
+	# invalidates OST cache
+	do_nodes $list "echo 1 > /proc/sys/vm/drop_caches"
 	set_osd_param $list '' read_cache_enable 0
         cat $DIR/$tfile >/dev/null
 
@@ -12837,9 +13653,9 @@ test_156() {
 	cat $file >/dev/null
 	AFTER=$(roc_hit)
 	if ! let "AFTER - BEFORE == CPAGES"; then
-		error "NOT IN CACHE: before: $BEFORE, after: $AFTER"
+		error "NOT IN CACHE (2): before: $BEFORE, after: $AFTER"
 	else
-		log "cache hits:: before: $BEFORE, after: $AFTER"
+		log "cache hits: before: $BEFORE, after: $AFTER"
 	fi
 
 	log "Read again; it should be satisfied from the cache."
@@ -12848,7 +13664,7 @@ test_156() {
 	cat $file >/dev/null
 	AFTER=$(roc_hit)
 	if ! let "AFTER - BEFORE == CPAGES"; then
-		error "NOT IN CACHE: before: $BEFORE, after: $AFTER"
+		error "NOT IN CACHE (3): before: $BEFORE, after: $AFTER"
 	else
 		log "cache hits:: before: $BEFORE, after: $AFTER"
 	fi
@@ -12863,20 +13679,23 @@ test_156() {
 	cat $file >/dev/null
 	AFTER=$(roc_hit)
 	if ! let "AFTER - BEFORE == CPAGES"; then
-		error "NOT IN CACHE: before: $BEFORE, after: $AFTER"
+		error "NOT IN CACHE (4): before: $BEFORE, after: $AFTER"
 	else
 		log "cache hits:: before: $BEFORE, after: $AFTER"
 	fi
 
-	log "Read again; it should not be satisfied from the cache."
-	BEFORE=$AFTER
-	cancel_lru_locks osc
-	cat $file >/dev/null
-	AFTER=$(roc_hit)
-	if ! let "AFTER - BEFORE == 0"; then
-		error "IN CACHE: before: $BEFORE, after: $AFTER"
-	else
-		log "cache hits:: before: $BEFORE, after: $AFTER"
+	if [ $OST1_VERSION -lt $(version_code 2.12.55) ]; then
+		# > 2.12.56 uses pagecache if cached
+		log "Read again; it should not be satisfied from the cache."
+		BEFORE=$AFTER
+		cancel_lru_locks osc
+		cat $file >/dev/null
+		AFTER=$(roc_hit)
+		if ! let "AFTER - BEFORE == 0"; then
+			error "IN CACHE (5): before: $BEFORE, after: $AFTER"
+		else
+			log "cache hits:: before: $BEFORE, after: $AFTER"
+		fi
 	fi
 
 	log "Write data and read it back."
@@ -12887,20 +13706,23 @@ test_156() {
 	cat $file >/dev/null
 	AFTER=$(roc_hit)
 	if ! let "AFTER - BEFORE == CPAGES"; then
-		error "NOT IN CACHE: before: $BEFORE, after: $AFTER"
+		error "NOT IN CACHE (6): before: $BEFORE, after: $AFTER"
 	else
 		log "cache hits:: before: $BEFORE, after: $AFTER"
 	fi
 
-	log "Read again; it should not be satisfied from the cache."
-	BEFORE=$AFTER
-	cancel_lru_locks osc
-	cat $file >/dev/null
-	AFTER=$(roc_hit)
-	if ! let "AFTER - BEFORE == 0"; then
-		error "IN CACHE: before: $BEFORE, after: $AFTER"
-	else
-		log "cache hits:: before: $BEFORE, after: $AFTER"
+	if [ $OST1_VERSION -lt $(version_code 2.12.55) ]; then
+		# > 2.12.56 uses pagecache if cached
+		log "Read again; it should not be satisfied from the cache."
+		BEFORE=$AFTER
+		cancel_lru_locks osc
+		cat $file >/dev/null
+		AFTER=$(roc_hit)
+		if ! let "AFTER - BEFORE == 0"; then
+			error "IN CACHE (7): before: $BEFORE, after: $AFTER"
+		else
+			log "cache hits:: before: $BEFORE, after: $AFTER"
+		fi
 	fi
 
 	log "Turn off read and write cache"
@@ -12916,7 +13738,7 @@ test_156() {
 	cat $file >/dev/null
 	AFTER=$(roc_hit)
 	if ! let "AFTER - BEFORE == 0"; then
-		error_ignore bz20762 "IN CACHE: before: $BEFORE, after: $AFTER"
+		error_ignore bz20762 "IN CACHE (8):before:$BEFORE,after:$AFTER"
 	else
 		log "cache hits:: before: $BEFORE, after: $AFTER"
 	fi
@@ -12934,7 +13756,7 @@ test_156() {
 	cat $file >/dev/null
 	AFTER=$(roc_hit)
 	if ! let "AFTER - BEFORE == 0"; then
-		error_ignore bz20762 "IN CACHE: before: $BEFORE, after: $AFTER"
+		error_ignore bz20762 "IN CACHE (9):before:$BEFORE,after:$AFTER"
 	else
 		log "cache hits:: before: $BEFORE, after: $AFTER"
 	fi
@@ -12945,7 +13767,7 @@ test_156() {
 	cat $file >/dev/null
 	AFTER=$(roc_hit)
 	if ! let "AFTER - BEFORE == CPAGES"; then
-		error "NOT IN CACHE: before: $BEFORE, after: $AFTER"
+		error "NOT IN CACHE (1): before: $BEFORE, after: $AFTER"
 	else
 		log "cache hits:: before: $BEFORE, after: $AFTER"
 	fi
@@ -13194,15 +14016,20 @@ test_160f() {
 	local i
 
 	# generate some changelog records to accumulate on each MDT
-	test_mkdir -c $MDSCOUNT $DIR/$tdir || error "test_mkdir $tdir failed"
+	# use fnv1a because created files should be evenly distributed
+	test_mkdir -c $MDSCOUNT -H fnv_1a_64 $DIR/$tdir ||
+		error "test_mkdir $tdir failed"
+	log "$(date +%s): creating first files"
 	createmany -m $DIR/$tdir/$tfile $((MDSCOUNT * 2)) ||
 		error "create $DIR/$tdir/$tfile failed"
 
 	# check changelogs have been generated
+	local start=$SECONDS
+	local idle_time=$((MDSCOUNT * 5 + 5))
 	local nbcl=$(changelog_dump | wc -l)
 	[[ $nbcl -eq 0 ]] && error "no changelogs found"
 
-	for param in "changelog_max_idle_time=10" \
+	for param in "changelog_max_idle_time=$idle_time" \
 		     "changelog_gc=1" \
 		     "changelog_min_gc_interval=2" \
 		     "changelog_min_free_cat_entries=3"; do
@@ -13214,8 +14041,11 @@ test_160f() {
 		do_nodes $mdts $LCTL set_param mdd.*.$param
 	done
 
-	# force cl_user2 to be idle (1st part)
-	sleep 9
+	# force cl_user2 to be idle (1st part), but also cancel the
+	# cl_user1 records so that it is not evicted later in the test.
+	local sleep1=$((idle_time / 2))
+	echo "$(date +%s): sleep1 $sleep1/${idle_time}s"
+	sleep $sleep1
 
 	# simulate changelog catalog almost full
 	#define OBD_FAIL_CAT_FREE_RECORDS	0x1313
@@ -13251,13 +14081,16 @@ test_160f() {
 			      "$user_rec1, but is $user_rec2"
 	done
 
-	# force cl_user2 to be idle (2nd part) and to reach
-	# changelog_max_idle_time
-	sleep 2
+	# force cl_user2 idle (2nd part) to just exceed changelog_max_idle_time
+	local sleep2=$((idle_time - (SECONDS - start) + 1))
+	echo "$(date +%s): sleep2 $sleep2/${idle_time}s"
+	sleep $sleep2
 
-	# generate one more changelog to trigger fail_loc
-	createmany -m $DIR/$tdir/${tfile}bis $((MDSCOUNT * 2)) ||
-		error "create $DIR/$tdir/${tfile}bis failed"
+	# Generate one more changelog to trigger GC at fail_loc for cl_user2.
+	# cl_user1 should be OK because it recently processed records.
+	echo "$(date +%s): creating $((MDSCOUNT * 2)) files"
+	createmany -m $DIR/$tdir/${tfile}b $((MDSCOUNT * 2)) ||
+		error "create $DIR/$tdir/${tfile}b failed"
 
 	# ensure gc thread is done
 	for i in $(mdts_nodes); do
@@ -13311,7 +14144,9 @@ test_160g() {
 	local i
 
 	# generate some changelog records to accumulate on each MDT
-	test_mkdir -c $MDSCOUNT $DIR/$tdir || error "mkdir $tdir failed"
+	# use fnv1a because created files should be evenly distributed
+	test_mkdir -c $MDSCOUNT -H fnv_1a_64 $DIR/$tdir ||
+		error "mkdir $tdir failed"
 	createmany -m $DIR/$tdir/$tfile $((MDSCOUNT * 2)) ||
 		error "create $DIR/$tdir/$tfile failed"
 
@@ -13425,7 +14260,9 @@ test_160h() {
 	local i
 
 	# generate some changelog records to accumulate on each MDT
-	test_mkdir -c $MDSCOUNT $DIR/$tdir || error "test_mkdir $tdir failed"
+	# use fnv1a because created files should be evenly distributed
+	test_mkdir -c $MDSCOUNT -H fnv_1a_64 $DIR/$tdir ||
+		error "test_mkdir $tdir failed"
 	createmany -m $DIR/$tdir/$tfile $((MDSCOUNT * 2)) ||
 		error "create $DIR/$tdir/$tfile failed"
 
@@ -13574,7 +14411,9 @@ test_160i() {
 	changelog_register || error "first changelog_register failed"
 
 	# generate some changelog records to accumulate on each MDT
-	test_mkdir -c $MDSCOUNT $DIR/$tdir || error "mkdir $tdir failed"
+	# use fnv1a because created files should be evenly distributed
+	test_mkdir -c $MDSCOUNT -H fnv_1a_64 $DIR/$tdir ||
+		error "mkdir $tdir failed"
 	createmany -m $DIR/$tdir/$tfile $((MDSCOUNT * 2)) ||
 		error "create $DIR/$tdir/$tfile failed"
 
@@ -13627,6 +14466,74 @@ test_160i() {
 }
 run_test 160i "changelog user register/unregister race"
 
+test_160j() {
+	remote_mds_nodsh && skip "remote MDS with nodsh"
+	[[ $MDS1_VERSION -lt $(version_code 2.12.56) ]] &&
+		skip "Need MDS version at least 2.12.56"
+
+	mount_client $MOUNT2 || error "mount_client on $MOUNT2 failed"
+	stack_trap "umount $MOUNT2" EXIT
+
+	changelog_register || error "first changelog_register failed"
+	stack_trap "changelog_deregister" EXIT
+
+	# generate some changelog
+	# use fnv1a because created files should be evenly distributed
+	test_mkdir -c $MDSCOUNT -H fnv_1a_64 $DIR/$tdir ||
+		error "mkdir $tdir failed"
+	createmany -m $DIR/$tdir/${tfile}bis $((MDSCOUNT * 2)) ||
+		error "create $DIR/$tdir/${tfile}bis failed"
+
+	# open the changelog device
+	exec 3>/dev/changelog-$FSNAME-MDT0000
+	stack_trap "exec 3>&-" EXIT
+	exec 4</dev/changelog-$FSNAME-MDT0000
+	stack_trap "exec 4<&-" EXIT
+
+	# umount the first lustre mount
+	umount $MOUNT
+	stack_trap "mount_client $MOUNT" EXIT
+
+	# read changelog
+	cat <&4 >/dev/null || error "read changelog failed"
+
+	# clear changelog
+	local cl_user="${CL_USERS[$SINGLEMDS]%% *}"
+	changelog_users $SINGLEMDS | grep -q $cl_user ||
+		error "User $cl_user not found in changelog_users"
+
+	printf 'clear:'$cl_user':0' >&3
+}
+run_test 160j "client can be umounted  while its chanangelog is being used"
+
+test_160k() {
+	[ $PARALLEL == "yes" ] && skip "skip parallel run"
+	remote_mds_nodsh && skip "remote MDS with nodsh"
+
+	mkdir -p $DIR/$tdir/1/1
+
+	changelog_register || error "changelog_register failed"
+	local cl_user="${CL_USERS[$SINGLEMDS]%% *}"
+
+	changelog_users $SINGLEMDS | grep -q $cl_user ||
+		error "User '$cl_user' not found in changelog_users"
+#define OBD_FAIL_MDS_CHANGELOG_REORDER 0x15d
+	do_facet mds1 $LCTL set_param fail_loc=0x8000015d fail_val=3
+	rmdir $DIR/$tdir/1/1 & sleep 1
+	mkdir $DIR/$tdir/2
+	touch $DIR/$tdir/2/2
+	rm -rf $DIR/$tdir/2
+
+	wait
+	sleep 4
+
+	changelog_dump | grep rmdir || error "rmdir not recorded"
+
+	rm -rf $DIR/$tdir
+	changelog_deregister
+}
+run_test 160k "Verify that changelog records are not lost"
+
 test_161a() {
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
 
@@ -13823,7 +14730,8 @@ test_161d() {
 	ps -p $pid
 	[[ $? -eq 0 ]] || error "create should be blocked"
 
-	local tempfile=$(mktemp)
+	local tempfile="$(mktemp --tmpdir $tfile.XXXXXX)"
+	stack_trap "rm -f $tempfile"
 	fid=$(changelog_extract_field "CREAT" "$tfile" "t=")
 	cat $MOUNT/.lustre/fid/$fid 2>/dev/null >$tempfile || error "cat failed"
 	# some delay may occur during ChangeLog publishing and file read just
@@ -14578,10 +15486,6 @@ test_200() {
 	local subdir=$test_path/subdir
 	local rc=0
 
-	if ! combined_mgs_mds ; then
-		mount_mgs_client
-	fi
-
 	while : ; do
 		# former test_200a test_200b
 		pool_add $POOL				|| { rc=$? ; break; }
@@ -14615,9 +15519,6 @@ test_200() {
 
 	destroy_test_pools
 
-	if ! combined_mgs_mds ; then
-		umount_mgs_client
-	fi
 	return $rc
 }
 run_test 200 "OST pools"
@@ -14795,7 +15696,7 @@ jobstats_set() {
 		"$FSNAME.sys.jobid_var" $new_jobenv
 }
 
-test_205() { # Job stats
+test_205a() { # Job stats
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
 	[[ $MDS1_VERSION -ge $(version_code 2.7.1) ]] ||
 		skip "Need MDS version with at least 2.7.1"
@@ -14897,7 +15798,18 @@ test_205() { # Job stats
 
 	verify_jobstats "touch $DIR/$tfile" $SINGLEMDS
 }
-run_test 205 "Verify job stats"
+run_test 205a "Verify job stats"
+
+# LU-13117
+test_205b() {
+	$LCTL set_param jobid_var=USER jobid_name="%e.%u"
+	env -i USERTESTJOBSTATS=foolish touch $DIR/$tfile.1
+	do_facet $SINGLEMDS $LCTL get_param mdt.*.job_stats |
+		grep job_id: | grep foolish &&
+			error "Unexpected jobid found"
+	true
+}
+run_test 205b "Verify job stats jobid parsing"
 
 # LU-1480, LU-1773 and LU-1657
 test_206() {
@@ -15299,10 +16211,6 @@ test_220() { #LU-325
 
 	$LFS df -i
 
-	if ! combined_mgs_mds ; then
-		mount_mgs_client
-	fi
-
 	do_facet ost$((OSTIDX + 1)) lctl set_param fail_val=-1
 	#define OBD_FAIL_OST_ENOINO              0x229
 	do_facet ost$((OSTIDX + 1)) lctl set_param fail_loc=0x229
@@ -15339,10 +16247,6 @@ test_220() { #LU-325
 		error "$LCTL pool_destroy $FSNAME.$TESTNAME failed"
 	echo "unlink $MDSOBJS files @$next_id..."
 	unlinkmany $DIR/$tdir/f $MDSOBJS || error "unlinkmany failed"
-
-	if ! combined_mgs_mds ; then
-		umount_mgs_client
-	fi
 }
 run_test 220 "preallocated MDS objects still used if ENOSPC from OST"
 
@@ -15442,8 +16346,7 @@ test_224c() { # LU-6441
 	save_writethrough $p
 	set_cache writethrough on
 
-	local pages_per_rpc=$($LCTL get_param \
-				osc.*.max_pages_per_rpc)
+	local pages_per_rpc=$($LCTL get_param osc.*.max_pages_per_rpc)
 	local at_max=$($LCTL get_param -n at_max)
 	local timeout=$($LCTL get_param -n timeout)
 	local test_at="at_max"
@@ -15852,6 +16755,27 @@ test_230b() {
 	ln -s $migrate_dir/$tfile $migrate_dir/${tfile}_ln
 	ln -s $other_dir/$tfile $migrate_dir/${tfile}_ln_other
 
+	local len
+	local lnktgt
+
+	# inline symlink
+	for len in 58 59 60; do
+		lnktgt=$(str_repeat 'l' $len)
+		touch $migrate_dir/$lnktgt
+		ln -s $lnktgt $migrate_dir/${len}char_ln
+	done
+
+	# PATH_MAX
+	for len in 4094 4095; do
+		lnktgt=$(str_repeat 'l' $len)
+		ln -s $lnktgt $migrate_dir/${len}char_ln
+	done
+
+	# NAME_MAX
+	for len in 254 255; do
+		touch $migrate_dir/$(str_repeat 'l' $len)
+	done
+
 	$LFS migrate -m $MDTIDX $migrate_dir ||
 		error "fails on migrating remote dir to MDT1"
 
@@ -15859,7 +16783,8 @@ test_230b() {
 	for ((i = 0; i < 10; i++)); do
 		for file in $(find $migrate_dir/dir_${i}); do
 			mdt_index=$($LFS getstripe -m $file)
-			[ $mdt_index == $MDTIDX ] ||
+			# broken symlink getstripe will fail
+			[ $mdt_index -ne $MDTIDX ] && stat -L $file &&
 				error "$file is not on MDT${MDTIDX}"
 		done
 	done
@@ -15923,7 +16848,7 @@ test_230b() {
 	echo "migrate back to MDT0, checking.."
 	for file in $(find $migrate_dir); do
 		mdt_index=$($LFS getstripe -m $file)
-		[ $mdt_index == $MDTIDX ] ||
+		[ $mdt_index -ne $MDTIDX ] && stat -L $file &&
 			error "$file is not on MDT${MDTIDX}"
 	done
 
@@ -16346,6 +17271,61 @@ test_230l() {
 }
 run_test 230l "readdir between MDTs won't crash"
 
+test_230m() {
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
+	[ $MDS1_VERSION -lt $(version_code 2.11.56) ] &&
+		skip "Need MDS version at least 2.11.56"
+
+	local MDTIDX=1
+	local mig_dir=$DIR/$tdir/migrate_dir
+	local longstr="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	local shortstr="b"
+	local val
+
+	echo "Creating files and dirs with xattrs"
+	test_mkdir $DIR/$tdir
+	test_mkdir -i0 -c1 $mig_dir
+	mkdir $mig_dir/dir
+	setfattr -n user.attr1 -v $longstr $mig_dir/dir ||
+		error "cannot set xattr attr1 on dir"
+	setfattr -n user.attr2 -v $shortstr $mig_dir/dir ||
+		error "cannot set xattr attr2 on dir"
+	touch $mig_dir/dir/f0
+	setfattr -n user.attr1 -v $longstr $mig_dir/dir/f0 ||
+		error "cannot set xattr attr1 on file"
+	setfattr -n user.attr2 -v $shortstr $mig_dir/dir/f0 ||
+		error "cannot set xattr attr2 on file"
+	sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
+	val=$(getfattr --only-values -n user.attr1 $mig_dir/dir 2>/dev/null)
+	[ "$val" = $longstr ] || error "xattr attr1 not set properly on dir"
+	val=$(getfattr --only-values -n user.attr2 $mig_dir/dir 2>/dev/null)
+	[ "$val" = $shortstr ] || error "xattr attr2 not set properly on dir"
+	val=$(getfattr --only-values -n user.attr1 $mig_dir/dir/f0 2>/dev/null)
+	[ "$val" = $longstr ] || error "xattr attr1 not set properly on file"
+	val=$(getfattr --only-values -n user.attr2 $mig_dir/dir/f0 2>/dev/null)
+	[ "$val" = $shortstr ] || error "xattr attr2 not set properly on file"
+
+	echo "Migrating to MDT1"
+	$LFS migrate -m $MDTIDX $mig_dir ||
+		error "fails on migrating dir to MDT1"
+
+	sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
+	echo "Checking xattrs"
+	val=$(getfattr --only-values -n user.attr1 $mig_dir/dir 2>/dev/null)
+	[ "$val" = $longstr ] ||
+		error "expecting xattr1 $longstr on dir, found $val"
+	val=$(getfattr --only-values -n user.attr2 $mig_dir/dir 2>/dev/null)
+	[ "$val" = $shortstr ] ||
+		error "expecting xattr2 $shortstr on dir, found $val"
+	val=$(getfattr --only-values -n user.attr1 $mig_dir/dir/f0 2>/dev/null)
+	[ "$val" = $longstr ] ||
+		error "expecting xattr1 $longstr on file, found $val"
+	val=$(getfattr --only-values -n user.attr2 $mig_dir/dir/f0 2>/dev/null)
+	[ "$val" = $shortstr ] ||
+		error "expecting xattr2 $shortstr on file, found $val"
+}
+run_test 230m "xattrs not changed after dir migration"
+
 test_231a()
 {
 	# For simplicity this test assumes that max_pages_per_rpc
@@ -16696,7 +17676,7 @@ test_243()
 }
 run_test 243 "various group lock tests"
 
-test_244()
+test_244a()
 {
 	test_mkdir $DIR/$tdir
 	dd if=/dev/zero of=$DIR/$tdir/$tfile bs=1M count=35
@@ -16704,7 +17684,26 @@ test_244()
 		error "sendfile+grouplock failed"
 	rm -rf $DIR/$tdir
 }
-run_test 244 "sendfile with group lock tests"
+run_test 244a "sendfile with group lock tests"
+
+test_244b()
+{
+	[ $PARALLEL == "yes" ] && skip "skip parallel run" && return
+
+	local threads=50
+	local size=$((1024*1024))
+
+	test_mkdir $DIR/$tdir
+	for i in $(seq 1 $threads); do
+		local file=$DIR/$tdir/file_$((i / 10))
+		$MULTIOP $file OG1234w$size_$((i % 3))w$size_$((i % 4))g1234c &
+		local pids[$i]=$!
+	done
+	for i in $(seq 1 $threads); do
+		wait ${pids[$i]}
+	done
+}
+run_test 244b "multi-threaded write with group lock"
 
 test_245() {
 	local flagname="multi_mod_rpcs"
@@ -16731,21 +17730,6 @@ test_245() {
 }
 run_test 245 "check mdc connection flag/data: multiple modify RPCs"
 
-test_246() { # LU-7371
-	remote_ost_nodsh && skip "remote OST with nodsh"
-	[ $OST1_VERSION -lt $(version_code 2.7.62) ] &&
-		skip "Need OST version >= 2.7.62"
-
-	do_facet ost1 $LCTL set_param fail_val=4095
-#define OBD_FAIL_OST_READ_SIZE		0x234
-	do_facet ost1 $LCTL set_param fail_loc=0x234
-	$LFS setstripe $DIR/$tfile -i 0 -c 1
-	dd if=/dev/zero of=$DIR/$tfile bs=4095 count=1 > /dev/null 2>&1
-	cancel_lru_locks $FSNAME-OST0000
-	dd if=$DIR/$tfile of=/dev/null bs=1048576 || error "Read failed"
-}
-run_test 246 "Read file of size 4095 should return right length"
-
 cleanup_247() {
 	local submount=$1
 
@@ -16838,7 +17822,7 @@ test_247e() {
 }
 run_test 247e "mount .. as fileset"
 
-test_248() {
+test_248a() {
 	local fast_read_sav=$($LCTL get_param -n llite.*.fast_read 2>/dev/null)
 	[ -z "$fast_read_sav" ] && skip "no fast read support"
 
@@ -16889,7 +17873,89 @@ test_248() {
 	$LCTL set_param -n llite.*.fast_read=$fast_read_sav
 	rm -f $DIR/$tfile
 }
-run_test 248 "fast read verification"
+run_test 248a "fast read verification"
+
+test_248b() {
+	# Default short_io_bytes=16384, try both smaller and larger sizes.
+	# Lustre O_DIRECT read and write needs to be a multiple of PAGE_SIZE.
+	# 6017024 = 2^12*13*113 = 47008*128 = 11752*512 = 4096*1469 = 53248*113
+	echo "bs=53248 count=113 normal buffered write"
+	dd if=/dev/urandom of=$TMP/$tfile.0 bs=53248 count=113 ||
+		error "dd of initial data file failed"
+	stack_trap "rm -f $DIR/$tfile.[0-3] $TMP/$tfile.[0-3]" EXIT
+
+	echo "bs=47008 count=128 oflag=dsync normal write $tfile.0"
+	dd if=$TMP/$tfile.0 of=$DIR/$tfile.0 bs=47008 count=128 oflag=dsync ||
+		error "dd with sync normal writes failed"
+	cmp $TMP/$tfile.0 $DIR/$tfile.0 || error "compare $DIR/$tfile.0 failed"
+
+	echo "bs=11752 count=512 oflag=dsync small write $tfile.1"
+	dd if=$TMP/$tfile.0 of=$DIR/$tfile.1 bs=11752 count=512 oflag=dsync ||
+		error "dd with sync small writes failed"
+	cmp $TMP/$tfile.0 $DIR/$tfile.1 || error "compare $DIR/$tfile.1 failed"
+
+	cancel_lru_locks osc
+
+	# calculate the small O_DIRECT size and count for the client PAGE_SIZE
+	local num=$((13 * 113 / (PAGE_SIZE / 4096)))
+	echo "bs=$PAGE_SIZE count=$num iflag=direct small read $tfile.1"
+	dd if=$DIR/$tfile.1 of=$TMP/$tfile.1 bs=$PAGE_SIZE count=$num \
+		iflag=direct || error "dd with O_DIRECT small read failed"
+	# adjust bytes checked to handle larger PAGE_SIZE for ARM/PPC
+	cmp --bytes=$((PAGE_SIZE * num)) $TMP/$tfile.0 $TMP/$tfile.1 ||
+		error "compare $TMP/$tfile.1 failed"
+
+	local save=$($LCTL get_param -n osc.*OST000*.short_io_bytes | head -n 1)
+	stack_trap "$LCTL set_param osc.$FSNAME-*.short_io_bytes=$save" EXIT
+
+	# just to see what the maximum tunable value is, and test parsing
+	echo "test invalid parameter 2MB"
+	$LCTL set_param osc.$FSNAME-OST0000*.short_io_bytes=2M &&
+		error "too-large short_io_bytes allowed"
+	echo "test maximum parameter 512KB"
+	# if we can set a larger short_io_bytes, run test regardless of version
+	if ! $LCTL set_param osc.$FSNAME-OST0000*.short_io_bytes=512K; then
+		# older clients may not allow setting it this large, that's OK
+		[ $CLIENT_VERSION -ge $(version_code 2.13.50) ] ||
+			skip "Need at least client version 2.13.50"
+		error "medium short_io_bytes failed"
+	fi
+	$LCTL get_param osc.$FSNAME-OST0000*.short_io_bytes
+	size=$($LCTL get_param -n osc.$FSNAME-OST0000*.short_io_bytes)
+
+	echo "test large parameter 64KB"
+	$LCTL set_param osc.$FSNAME-*.short_io_bytes=65536
+	$LCTL get_param osc.$FSNAME-OST0000*.short_io_bytes
+
+	echo "bs=47008 count=128 oflag=dsync large write $tfile.2"
+	dd if=$TMP/$tfile.0 of=$DIR/$tfile.2 bs=47008 count=128 oflag=dsync ||
+		error "dd with sync large writes failed"
+	cmp $TMP/$tfile.0 $DIR/$tfile.2 || error "compare $DIR/$tfile.2 failed"
+
+	# calculate the large O_DIRECT size and count for the client PAGE_SIZE
+	local size=$(((4096 * 13 + PAGE_SIZE - 1) / PAGE_SIZE * PAGE_SIZE))
+	num=$((113 * 4096 / PAGE_SIZE))
+	echo "bs=$size count=$num oflag=direct large write $tfile.3"
+	dd if=$TMP/$tfile.0 of=$DIR/$tfile.3 bs=$size count=$num oflag=direct ||
+		error "dd with O_DIRECT large writes failed"
+	cmp --bytes=$((size * num)) $TMP/$tfile.0 $DIR/$tfile.3 ||
+		error "compare $DIR/$tfile.3 failed"
+
+	cancel_lru_locks osc
+
+	echo "bs=$size count=$num iflag=direct large read $tfile.2"
+	dd if=$DIR/$tfile.2 of=$TMP/$tfile.2 bs=$size count=$num iflag=direct ||
+		error "dd with O_DIRECT large read failed"
+	cmp --bytes=$((size * num)) $TMP/$tfile.0 $TMP/$tfile.2 ||
+		error "compare $TMP/$tfile.2 failed"
+
+	echo "bs=$size count=$num iflag=direct large read $tfile.3"
+	dd if=$DIR/$tfile.3 of=$TMP/$tfile.3 bs=$size count=$num iflag=direct ||
+		error "dd with O_DIRECT large read failed"
+	cmp --bytes=$((size * num)) $TMP/$tfile.0 $TMP/$tfile.3 ||
+		error "compare $TMP/$tfile.3 failed"
+}
+run_test 248b "test short_io read and write for both small and large sizes"
 
 test_249() { # LU-7890
 	[ $MDS1_VERSION -lt $(version_code 2.8.53) ] &&
@@ -17004,9 +18070,6 @@ test_253() {
 	wait_delete_completed
 	mkdir $DIR/$tdir
 
-	if ! combined_mgs_mds ; then
-		mount_mgs_client
-	fi
 	pool_add $TESTNAME || error "Pool creation failed"
 	pool_add_targets $TESTNAME 0 || error "Pool add targets failed"
 
@@ -17052,10 +18115,6 @@ test_253() {
 	if (( oa_status != 0 )); then
 		error "Object allocation still disable after rm"
 	fi
-
-	if ! combined_mgs_mds ; then
-		umount_mgs_client
-	fi
 }
 run_test 253 "Check object allocation limit"
 
@@ -17361,7 +18420,7 @@ run_test 255b "check 'lfs ladvise -a dontneed'"
 
 test_255c() {
 	[ $OST1_VERSION -lt $(version_code 2.10.50) ] &&
-		skip "lustre < 2.10.53 does not support lockahead"
+		skip "lustre < 2.10.50 does not support lockahead"
 
 	local count
 	local new_count
@@ -17476,7 +18535,8 @@ test_256() {
 
 	#after mount new plainllog is used
 	touch $DIR/$tdir/{11..19}
-	local tmpfile=$(mktemp -u $tfile.XXXXXX)
+	local tmpfile="$(mktemp --tmpdir -u $tfile.XXXXXX)"
+	stack_trap "rm -f $tmpfile"
 	cat_sl=$(do_facet $SINGLEMDS "sync; \
 		 $DEBUGFS -c -R 'dump changelog_catalog $tmpfile' $mdt_dev; \
 		 llog_reader $tmpfile | grep -c type=1064553b")
@@ -17488,7 +18548,7 @@ test_256() {
 
 	cat_sl=$(do_facet $SINGLEMDS "sync; \
 		 $DEBUGFS -c -R 'dump changelog_catalog $tmpfile' $mdt_dev; \
-		 llog_reader $tmpfile | grep -c type=1064553b; rm -f $tmpfile")
+		 llog_reader $tmpfile | grep -c type=1064553b")
 
 	if (( cat_sl == 2 )); then
 		error "Empty plain llog was not deleted from changelog catalog"
@@ -18112,8 +19172,8 @@ test_271f() {
 	local mdtidx=$($LFS getstripe --mdt-index $DIR/$tdir)
 
 	cancel_lru_locks mdc
-	dd if=/dev/urandom of=$tmp bs=200000 count=1
-	dd if=$tmp of=$dom bs=200000 count=1
+	dd if=/dev/urandom of=$tmp bs=265000 count=1
+	dd if=$tmp of=$dom bs=265000 count=1
 	cancel_lru_locks mdc
 	cat /etc/hosts >> $tmp
 	lctl set_param -n mdc.*.stats=clear
@@ -18140,6 +19200,7 @@ test_271f() {
 	local ra=$(get_mdc_stats $mdtidx req_active)
 	local rw=$(get_mdc_stats $mdtidx req_waittime)
 
+	[ -z $num ] && num=0
 	[ $num -eq 1 ] || error "expect 1 READ RPC, $num occured"
 	[ $ra == $rw ] || error "$((ra - rw)) resend occured"
 	echo "... DONE"
@@ -18192,7 +19253,7 @@ test_272a() {
 		error "md5sum differ: $old_md5, $new_md5"
 
 	[ $($LFS getstripe -c $dom) -eq 2 ] ||
-		error "migrate stripe count bad: $(LFS getstripe -c $dom) != 2"
+		error "bad final stripe count: $($LFS getstripe -c $dom) != 2"
 }
 run_test 272a "DoM migration: new layout with the same DOM component"
 
@@ -18219,12 +19280,12 @@ test_272b() {
 
 	$LFS migrate -c2 $dom ||
 		error "failed to migrate to the new composite layout"
-	[ $($LFS getstripe -L $dom) == 'mdt' ] &&
+	[ $($LFS getstripe -L $dom) != 'mdt' ] ||
 		error "MDT stripe was not removed"
 
 	cancel_lru_locks mdc
 	local new_md5=$(md5sum $dom)
-	[ "$old_md5" != "$new_md5" ] &&
+	[ "$old_md5" == "$new_md5" ] ||
 		error "$old_md5 != $new_md5"
 
 	# Skip free space checks with ZFS
@@ -18264,7 +19325,7 @@ test_272c() {
 
 	cancel_lru_locks mdc
 	local new_md5=$(md5sum $dom)
-	[ "$old_md5" != "$new_md5" ] &&
+	[ "$old_md5" == "$new_md5" ] ||
 		error "$old_md5 != $new_md5"
 
 	# Skip free space checks with ZFS
@@ -18278,6 +19339,108 @@ test_272c() {
 }
 run_test 272c "DoM migration: DOM file to the OST-striped file (composite)"
 
+test_272d() {
+	[ $MDS1_VERSION -lt $(version_code 2.12.55) ] &&
+		skip "Need MDS version at least 2.12.55"
+
+	local dom=$DIR/$tdir/$tfile
+	mkdir -p $DIR/$tdir
+	$LFS setstripe -E 1M -L mdt -E -1 -c1 $dom
+
+	local mdtidx=$($LFS getstripe -m $dom)
+	local mdtname=MDT$(printf %04x $mdtidx)
+	local facet=mds$((mdtidx + 1))
+
+	dd if=/dev/urandom of=$dom bs=2M count=1 oflag=direct ||
+		error "failed to write data into $dom"
+	local old_md5=$(md5sum $dom)
+	cancel_lru_locks mdc
+	local mdtfree1=$(do_facet $facet \
+		lctl get_param -n osd*.*$mdtname.kbytesfree)
+
+	$LFS mirror extend -N -E 2M -c1 -E -1 -c2 $dom ||
+		error "failed mirroring to the new composite layout"
+	$LFS mirror resync $dom ||
+		error "failed mirror resync"
+	$LFS mirror split --mirror-id 1 -d $dom ||
+		error "failed mirror split"
+
+	[ $($LFS getstripe -L $dom) != 'mdt' ] ||
+		error "MDT stripe was not removed"
+
+	cancel_lru_locks mdc
+	local new_md5=$(md5sum $dom)
+	[ "$old_md5" == "$new_md5" ] ||
+		error "$old_md5 != $new_md5"
+
+	# Skip free space checks with ZFS
+	if [ "$(facet_fstype $facet)" != "zfs" ]; then
+		local mdtfree2=$(do_facet $facet \
+				lctl get_param -n osd*.*$mdtname.kbytesfree)
+		[ $mdtfree2 -gt $mdtfree1 ] ||
+			error "MDS space is not freed after DOM mirror deletion"
+	fi
+	return 0
+}
+run_test 272d "DoM mirroring: OST-striped mirror to DOM file"
+
+test_272e() {
+	[ $MDS1_VERSION -lt $(version_code 2.12.55) ] &&
+		skip "Need MDS version at least 2.12.55"
+
+	local dom=$DIR/$tdir/$tfile
+	mkdir -p $DIR/$tdir
+	$LFS setstripe -c 2 $dom
+
+	dd if=/dev/urandom of=$dom bs=512K count=1 oflag=direct ||
+		error "failed to write data into $dom"
+	local old_md5=$(md5sum $dom)
+	cancel_lru_locks mdc
+
+	$LFS mirror extend -N -E 1M -L mdt -E eof -c2 $dom ||
+		error "failed mirroring to the DOM layout"
+	$LFS mirror resync $dom ||
+		error "failed mirror resync"
+	$LFS mirror split --mirror-id 1 -d $dom ||
+		error "failed mirror split"
+
+	[ $($LFS getstripe -L $dom) != 'mdt' ] ||
+		error "MDT stripe was not removed"
+
+	cancel_lru_locks mdc
+	local new_md5=$(md5sum $dom)
+	[ "$old_md5" == "$new_md5" ] ||
+		error "$old_md5 != $new_md5"
+
+	return 0
+}
+run_test 272e "DoM mirroring: DOM mirror to the OST-striped file"
+
+test_272f() {
+	[ $MDS1_VERSION -lt $(version_code 2.12.55) ] &&
+		skip "Need MDS version at least 2.12.55"
+
+	local dom=$DIR/$tdir/$tfile
+	mkdir -p $DIR/$tdir
+	$LFS setstripe -c 2 $dom
+
+	dd if=/dev/urandom of=$dom bs=512K count=1 oflag=direct ||
+		error "failed to write data into $dom"
+	local old_md5=$(md5sum $dom)
+	cancel_lru_locks mdc
+
+	$LFS migrate -E 1M -L mdt -E eof -c2 -v $dom ||
+		error "failed migrating to the DOM file"
+
+	cancel_lru_locks mdc
+	local new_md5=$(md5sum $dom)
+	[ "$old_md5" != "$new_md5" ] &&
+		error "$old_md5 != $new_md5"
+
+	return 0
+}
+run_test 272f "DoM migration: OST-striped file to DOM file"
+
 test_273a() {
 	[ $MDS1_VERSION -lt $(version_code 2.11.50) ] &&
 		skip "Need MDS version at least 2.11.50"
@@ -18356,6 +19519,67 @@ test_276() {
 }
 run_test 276 "Race between mount and obd_statfs"
 
+test_277() {
+	$LCTL set_param ldlm.namespaces.*.lru_size=0
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=1
+	local cached_mb=$($LCTL get_param llite.*.max_cached_mb |
+			grep ^used_mb | awk '{print $2}')
+	[ $cached_mb -eq 1 ] || error "expected mb 1 got $cached_mb"
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=1 \
+		oflag=direct conv=notrunc
+	cached_mb=$($LCTL get_param llite.*.max_cached_mb |
+			grep ^used_mb | awk '{print $2}')
+	[ $cached_mb -eq 0 ] || error "expected mb 0 got $cached_mb"
+}
+run_test 277 "Direct IO shall drop page cache"
+
+test_278() {
+	[ $PARALLEL == "yes" ] && skip "skip parallel run" && return
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+	[[ "$(facet_host mds1)" != "$(facet_host mds2)" ]] &&
+		skip "needs the same host for mdt1 mdt2" && return
+
+	local pid1
+	local pid2
+
+#define OBD_FAIL_OBD_STOP_MDS_RACE     0x60b
+	do_facet mds2 $LCTL set_param fail_loc=0x8000060c
+	stop mds2 &
+	pid2=$!
+
+	stop mds1
+
+	echo "Starting MDTs"
+	start mds1 $(mdsdevname 1) $MDS_MOUNT_OPTS
+	wait $pid2
+#For the error assertion will happen. lu_env_get_key(..., &mdt_thread_key)
+#will return NULL
+	do_facet mds2 $LCTL set_param fail_loc=0
+
+	start mds2 $(mdsdevname 2) $MDS_MOUNT_OPTS
+	wait_recovery_complete mds2
+}
+run_test 278 "Race starting MDS between MDTs stop/start"
+
+test_280() {
+	[ $MGS_VERSION -lt $(version_code 2.13.52) ] &&
+		skip "Need MGS version at least 2.13.52"
+	[ $PARALLEL == "yes" ] && skip "skip parallel run"
+	combined_mgs_mds || skip "needs combined MGS/MDT"
+
+	umount_client $MOUNT
+#define OBD_FAIL_MDS_LLOG_UMOUNT_RACE   0x15e
+	do_facet mgs $LCTL set_param fail_loc=0x8000015e fail_val=0
+
+	mount_client $MOUNT &
+	sleep 1
+	stop mgs || error "stop mgs failed"
+	#for a race mgs would crash
+	start mgs $(mgsdevname) $MGS_MOUNT_OPTS || error "start mgs failed"
+	mount_client $MOUNT || error "mount client failed"
+}
+run_test 280 "Race between MGS umount and client llog processing"
+
 cleanup_test_300() {
 	trap 0
 	umask $SAVE_UMASK
@@ -18509,12 +19733,22 @@ test_300d() {
 	#local striped directory
 	$LFS setdirstripe -i 0 -c 2 -H all_char $DIR/$tdir/striped_dir ||
 		error "set striped dir error"
+        #look at the directories for debug purposes
+	ls -l $DIR/$tdir
+	$LFS getdirstripe $DIR/$tdir
+	ls -l $DIR/$tdir/striped_dir
+	$LFS getdirstripe $DIR/$tdir/striped_dir
 	createmany -o $DIR/$tdir/striped_dir/f 10 ||
 		error "create 10 files failed"
 
 	#remote striped directory
 	$LFS setdirstripe -i 1 -c 2 $DIR/$tdir/remote_striped_dir ||
 		error "set striped dir error"
+        #look at the directories for debug purposes
+	ls -l $DIR/$tdir
+	$LFS getdirstripe $DIR/$tdir
+	ls -l $DIR/$tdir/remote_striped_dir
+	$LFS getdirstripe $DIR/$tdir/remote_striped_dir
 	createmany -o $DIR/$tdir/remote_striped_dir/f 10 ||
 		error "create 10 files failed"
 
@@ -18838,8 +20072,9 @@ test_300k() {
 
 	# this test needs a huge transaction
 	local kb
-	kb=$(do_facet $SINGLEMDS lctl get_param -n osd*.lustre-MDT0000.kbytestotal)
-	[ $kb -lt $((1024*1024)) ] && skip "too small mds: $kb"
+	kb=$(do_facet $SINGLEMDS "$LCTL get_param -n \
+	     osd*.$FSNAME-MDT0000.kbytestotal")
+	[ $kb -lt $((1024*1024)) ] && skip "MDT0 too small: $kb"
 
 	local stripe_count
 	local file
@@ -19047,7 +20282,7 @@ test_300q() {
 run_test 300q "create remote directory under orphan directory"
 
 test_300r() {
-	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.55) ] &&
+	[ $MDS1_VERSION -lt $(version_code 2.7.55) ] &&
 		skip "Need MDS version at least 2.7.55" && return
 	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
 
@@ -19362,21 +20597,20 @@ test_316() {
 	chown nobody $DIR/$tdir/d
 	touch $DIR/$tdir/d/file
 
-	$LFS mv -M1 $DIR/$tdir/d || error "lfs mv failed"
+	$LFS mv -m1 $DIR/$tdir/d || error "lfs mv failed"
 }
 run_test 316 "lfs mv"
 
 test_317() {
 	[ $MDS1_VERSION -lt $(version_code 2.11.53) ] &&
 		skip "Need MDS version at least 2.11.53"
+	if [ "$ost1_FSTYPE" == "zfs" ]; then
+		skip "LU-10370: no implementation for ZFS"
+	fi
+
 	local trunc_sz
 	local grant_blk_size
 
-	if [ "$(facet_fstype $facet)" == "zfs" ]; then
-		skip "LU-10370: no implementation for ZFS" && return
-	fi
-
-	stack_trap "rm -f $DIR/$tfile" EXIT
 	grant_blk_size=$($LCTL get_param osc.$FSNAME*.import |
 			awk '/grant_block_size:/ { print $2; exit; }')
 	#
@@ -19384,7 +20618,8 @@ test_317() {
 	# blocks count.
 	#
 	dd if=/dev/zero of=$DIR/$tfile bs=5M count=1 conv=fsync ||
-		error "Create file : $DIR/$tfile"
+		error "Create file $DIR/$tfile failed"
+	stack_trap "rm -f $DIR/$tfile" EXIT
 
 	for trunc_sz in 2097152 4097 4000 509 0; do
 		$TRUNCATE $DIR/$tfile $trunc_sz ||
@@ -19444,9 +20679,8 @@ test_318() {
 			   llite.*.max_read_ahead_async_active 2>/dev/null)
 	[ $max_active -ne 256 ] && error "expected 256 but got $max_active"
 
-	# currently reset to 0 is unsupported, leave it 512 for now.
-	$LCTL set_param llite.*.max_read_ahead_async_active=0 &&
-		error "set max_read_ahead_async_active should fail"
+	$LCTL set_param llite.*.max_read_ahead_async_active=0 ||
+		error "set max_read_ahead_async_active should succeed"
 
 	$LCTL set_param llite.*.max_read_ahead_async_active=512
 	max_active=$($LCTL get_param -n \
@@ -19505,6 +20739,105 @@ test_319() {
 }
 run_test 319 "lost lease lock on migrate error"
 
+test_398a() { # LU-4198
+	$LFS setstripe -c 1 -i 0 $DIR/$tfile
+	$LCTL set_param ldlm.namespaces.*.lru_size=clear
+
+	# request a new lock on client
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=1
+
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=1 oflag=direct conv=notrunc
+	local lock_count=$($LCTL get_param -n \
+			   ldlm.namespaces.*-OST0000-osc-ffff*.lru_size)
+	[[ $lock_count -eq 0 ]] || error "lock should be cancelled by direct IO"
+
+	$LCTL set_param ldlm.namespaces.*-OST0000-osc-ffff*.lru_size=clear
+
+	# no lock cached, should use lockless IO and not enqueue new lock
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=1 oflag=direct conv=notrunc
+	lock_count=$($LCTL get_param -n \
+		     ldlm.namespaces.*-OST0000-osc-ffff*.lru_size)
+	[[ $lock_count -eq 0 ]] || error "no lock should be held by direct IO"
+}
+run_test 398a "direct IO should cancel lock otherwise lockless"
+
+test_398b() { # LU-4198
+	which fio || skip_env "no fio installed"
+	$LFS setstripe -c -1 $DIR/$tfile
+
+	local size=12
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=$size
+
+	local njobs=4
+	echo "mix direct rw ${size}M to OST0 by fio with $njobs jobs..."
+	fio --name=rand-rw --rw=randrw --bs=$PAGE_SIZE --direct=1 \
+		--numjobs=$njobs --fallocate=none \
+		--iodepth=16 --allow_file_create=0 --size=$((size/njobs))M \
+		--filename=$DIR/$tfile &
+	bg_pid=$!
+
+	echo "mix buffer rw ${size}M to OST0 by fio with $njobs jobs..."
+	fio --name=rand-rw --rw=randrw --bs=$PAGE_SIZE \
+		--numjobs=$njobs --fallocate=none \
+		--iodepth=16 --allow_file_create=0 --size=$((size/njobs))M \
+		--filename=$DIR/$tfile || true
+	wait $bg_pid
+
+	rm -rf $DIR/$tfile
+}
+run_test 398b "DIO and buffer IO race"
+
+test_398c() { # LU-4198
+	which fio || skip_env "no fio installed"
+
+	saved_debug=$($LCTL get_param -n debug)
+	$LCTL set_param debug=0
+
+	local size=$(lctl get_param -n osc.$FSNAME-OST0000*.kbytesavail | head -1)
+	((size /= 1024)) # by megabytes
+	((size /= 2)) # write half of the OST at most
+	[ $size -gt 40 ] && size=40 #reduce test time anyway
+
+	$LFS setstripe -c 1 $DIR/$tfile
+
+	# it seems like ldiskfs reserves more space than necessary if the
+	# writing blocks are not mapped, so it extends the file firstly
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=$size && sync
+	cancel_lru_locks osc
+
+	# clear and verify rpc_stats later
+	$LCTL set_param osc.${FSNAME}-OST0000-osc-ffff*.rpc_stats=clear
+
+	local njobs=4
+	echo "writing ${size}M to OST0 by fio with $njobs jobs..."
+	fio --name=rand-write --rw=randwrite --bs=$PAGE_SIZE --direct=1 \
+		--numjobs=$njobs --fallocate=none --ioengine=libaio \
+		--iodepth=16 --allow_file_create=0 --size=$((size/njobs))M \
+		--filename=$DIR/$tfile
+	[ $? -eq 0 ] || error "fio write error"
+
+	[ $($LCTL get_param -n \
+	 ldlm.namespaces.${FSNAME}-OST0000-osc-ffff*.lock_count) -eq 0 ] ||
+		error "Locks were requested while doing AIO"
+
+	# get the percentage of 1-page I/O
+	pct=$($LCTL get_param osc.${FSNAME}-OST0000-osc-ffff*.rpc_stats |
+		grep -A 1 'pages per rpc' | grep -v 'pages per rpc' |
+		awk '{print $7}')
+	[ $pct -le 50 ] || error "$pct% of I/O are 1-page"
+
+	echo "mix rw ${size}M to OST0 by fio with $njobs jobs..."
+	fio --name=rand-rw --rw=randrw --bs=$PAGE_SIZE --direct=1 \
+		--numjobs=$njobs --fallocate=none --ioengine=libaio \
+		--iodepth=16 --allow_file_create=0 --size=$((size/njobs))M \
+		--filename=$DIR/$tfile
+	[ $? -eq 0 ] || error "fio mixed read write error"
+
+	rm -rf $DIR/$tfile
+	$LCTL set_param debug="$saved_debug"
+}
+run_test 398c "run fio to test AIO"
+
 test_fake_rw() {
 	local read_write=$1
 	if [ "$read_write" = "write" ]; then
@@ -19521,7 +20854,7 @@ test_fake_rw() {
 
 	$LFS setstripe -c 1 -i 0 $DIR/$tfile
 
-	# get ost1 size - lustre-OST0000
+	# get ost1 size - $FSNAME-OST0000
 	local ost1_avail_size=$($LFS df | awk /${ost1_svc}/'{ print $4 }')
 	local blocks=$((ost1_avail_size/2/1024)) # half avail space by megabytes
 	[ $blocks -gt 1000 ] && blocks=1000 # 1G in maximum
@@ -19590,6 +20923,10 @@ test_400a() { # LU-1606, was conf-sanity test_74
 	local prefix=/usr/include/lustre
 	local prog
 
+	# Oleg removes c files in his test rig so test if any c files exist
+	[ -z "$(ls -A $LUSTRE_TESTS_API_DIR)" ] && \
+		skip_env "Needed c test files are missing"
+
 	if ! [[ -d $prefix ]]; then
 		# Assume we're running in tree and fixup the include path.
 		extra_flags+=" -I$LUSTRE/../lnet/include/uapi -I$LUSTRE/include/uapi -I$LUSTRE/include"
@@ -19597,7 +20934,7 @@ test_400a() { # LU-1606, was conf-sanity test_74
 	fi
 
 	for prog in $LUSTRE_TESTS_API_DIR/*.c; do
-		$CC -Wall -Werror $extra_flags -o $out $prog -llustreapi ||
+		$CC -Wall -Werror -std=c99 $extra_flags -o $out $prog -llustreapi ||
 			error "client api broken"
 	done
 	rm -f $out
@@ -19629,7 +20966,7 @@ test_400b() { # LU-1606, LU-5011
 			continue # lustre_ioctl.h is internal header
 		fi
 
-		$CC -Wall -Werror -include $header -c -x c /dev/null -o $out ||
+		$CC -Wall -Werror -std=c99 -include $header -c -x c /dev/null -o $out ||
 			error "cannot compile '$header'"
 	done
 	rm -f $out
@@ -19832,9 +21169,6 @@ test_406() {
 	local def_stripe_size=$($LFS getstripe -S $MOUNT)
 	local test_pool=$TESTNAME
 
-	if ! combined_mgs_mds ; then
-		mount_mgs_client
-	fi
 	pool_add $test_pool || error "pool_add failed"
 	pool_add_targets $test_pool 0 $(($OSTCOUNT - 1)) 1 ||
 		error "pool_add_targets failed"
@@ -19887,10 +21221,6 @@ test_406() {
 	local f=$DIR/$tdir/$tfile
 	pool_remove_all_targets $test_pool $f
 	pool_remove $test_pool $f
-
-	if ! combined_mgs_mds ; then
-		umount_mgs_client
-	fi
 }
 run_test 406 "DNE support fs default striping"
 
@@ -20025,7 +21355,7 @@ run_test 411 "Slab allocation error with cgroup does not LBUG"
 
 test_412() {
 	[ $MDSCOUNT -lt 2 ] && skip_env "needs >= 2 MDTs"
-	if [ $(lustre_version_code mds1) -lt $(version_code 2.10.55) ]; then
+	if [ $MDS1_VERSION -lt $(version_code 2.10.55) ]; then
 		skip "Need server version at least 2.10.55"
 	fi
 
@@ -20041,174 +21371,247 @@ test_412() {
 }
 run_test 412 "mkdir on specific MDTs"
 
-test_413a() {
-	[ $MDSCOUNT -lt 2 ] &&
-		skip "We need at least 2 MDTs for this test"
-
-	if [ $(lustre_version_code mds1) -lt $(version_code 2.10.55) ]; then
-		skip "Need server version at least 2.10.55"
-	fi
-
-	mkdir $DIR/$tdir || error "mkdir failed"
-
-	# find MDT that is the most full
-	local max=$($LFS df | grep MDT |
-		awk 'BEGIN { a=0 }
-			{ sub("%", "", $5)
-			  if (0+$5 >= a)
-			  {
-				a = $5
-				b = $6
-			  }
-			}
-		     END { split(b, c, ":")
-			   sub("]", "", c[2])
-			   print c[2]
-			 }')
-
-	for i in $(seq $((MDSCOUNT - 1))); do
-		$LFS mkdir -c $i $DIR/$tdir/d$i ||
-			error "mkdir d$i failed"
-		$LFS getdirstripe $DIR/$tdir/d$i
-		local stripe_index=$($LFS getdirstripe -i $DIR/$tdir/d$i)
-		[ $stripe_index -ne $max ] ||
-			error "don't expect $max"
-	done
-}
-run_test 413a "mkdir on less full MDTs"
-
-test_413b() {
-	[ $MDSCOUNT -lt 2 ] &&
-		skip "We need at least 2 MDTs for this test"
-
-	[ $MDS1_VERSION -lt $(version_code 2.12.52) ] &&
-		skip "Need server version at least 2.12.52"
-
-	mkdir $DIR/$tdir || error "mkdir failed"
-	$LFS setdirstripe -D -i -1 -H space $DIR/$tdir ||
-		error "setdirstripe failed"
+test_qos_mkdir() {
+	local mkdir_cmd=$1
+	local stripe_count=$2
+	local mdts=$(comma_list $(mdts_nodes))
 
-	local qos_prio_free
-	local qos_threshold_rr
+	local testdir
+	local lmv_qos_prio_free
+	local lmv_qos_threshold_rr
+	local lmv_qos_maxage
+	local lod_qos_prio_free
+	local lod_qos_threshold_rr
+	local lod_qos_maxage
 	local count
+	local i
 
-	qos_prio_free=$($LCTL get_param -n lmv.*.qos_prio_free | head -n1)
-	qos_prio_free=${qos_prio_free%%%}
-	qos_threshold_rr=$($LCTL get_param -n lmv.*.qos_threshold_rr | head -n1)
-	qos_threshold_rr=${qos_threshold_rr%%%}
-	qos_maxage=$($LCTL get_param -n lmv.*.qos_maxage)
-
-	stack_trap "$LCTL set_param lmv.*.qos_prio_free=$qos_prio_free" EXIT
-	stack_trap "$LCTL set_param lmv.*.qos_threshold_rr=$qos_threshold_rr" \
+	lmv_qos_prio_free=$($LCTL get_param -n lmv.*.qos_prio_free | head -n1)
+	lmv_qos_prio_free=${lmv_qos_prio_free%%%}
+	lmv_qos_threshold_rr=$($LCTL get_param -n lmv.*.qos_threshold_rr |
+		head -n1)
+	lmv_qos_threshold_rr=${lmv_qos_threshold_rr%%%}
+	lmv_qos_maxage=$($LCTL get_param -n lmv.*.qos_maxage)
+	stack_trap "$LCTL set_param \
+		lmv.*.qos_prio_free=$lmv_qos_prio_free > /dev/null" EXIT
+	stack_trap "$LCTL set_param \
+		lmv.*.qos_threshold_rr=$lmv_qos_threshold_rr > /dev/null" EXIT
+	stack_trap "$LCTL set_param \
+		lmv.*.qos_maxage=$lmv_qos_maxage > /dev/null" EXIT
+
+	lod_qos_prio_free=$(do_facet mds1 $LCTL get_param -n \
+		lod.$FSNAME-MDT0000-mdtlov.mdt_qos_prio_free | head -n1)
+	lod_qos_prio_free=${lod_qos_prio_free%%%}
+	lod_qos_threshold_rr=$(do_facet mds1 $LCTL get_param -n \
+		lod.$FSNAME-MDT0000-mdtlov.mdt_qos_threshold_rr | head -n1)
+	lod_qos_threshold_rr=${lod_qos_threshold_rr%%%}
+	lod_qos_maxage=$(do_facet mds1 $LCTL get_param -n \
+		lod.$FSNAME-MDT0000-mdtlov.qos_maxage | awk '{ print $1 }')
+	stack_trap "do_nodes $mdts $LCTL set_param \
+		lod.*.mdt_qos_prio_free=$lod_qos_prio_free > /dev/null" EXIT
+	stack_trap "do_nodes $mdts $LCTL set_param \
+		lod.*.mdt_qos_threshold_rr=$lod_qos_threshold_rr > /dev/null" \
 		EXIT
-	stack_trap "$LCTL set_param lmv.*.qos_maxage=$qos_maxage" EXIT
+	stack_trap "do_nodes $mdts $LCTL set_param \
+		lod.*.mdt_qos_maxage=$lod_qos_maxage > /dev/null" EXIT
 
-	echo "mkdir with roundrobin"
+	echo
+	echo "Mkdir (stripe_count $stripe_count) roundrobin:"
+
+	$LCTL set_param lmv.*.qos_threshold_rr=100 > /dev/null
+	do_nodes $mdts $LCTL set_param lod.*.mdt_qos_threshold_rr=100 > /dev/null
+
+	testdir=$DIR/$tdir-s$stripe_count/rr
 
-	$LCTL set_param lmv.*.qos_threshold_rr=100
 	for i in $(seq $((100 * MDSCOUNT))); do
-		mkdir $DIR/$tdir/subdir$i || error "mkdir subdir$i failed"
+		eval $mkdir_cmd $testdir/subdir$i ||
+			error "$mkdir_cmd subdir$i failed"
 	done
+
 	for i in $(seq $MDSCOUNT); do
-		count=$($LFS getdirstripe -i $DIR/$tdir/* | grep ^$((i - 1))$ |
-			wc -w)
+		count=$($LFS getdirstripe -i $testdir/* |
+				grep ^$((i - 1))$ | wc -l)
 		echo "$count directories created on MDT$((i - 1))"
 		[ $count -eq 100 ] || error "subdirs are not evenly distributed"
+
+		if [ $stripe_count -gt 1 ]; then
+			count=$($LFS getdirstripe $testdir/* |
+				grep -P "^\s+$((i - 1))\t" | wc -l)
+			echo "$count stripes created on MDT$((i - 1))"
+			# deviation should < 5% of average
+			[ $count -lt $((95 * stripe_count)) ] ||
+			[ $count -gt $((105 * stripe_count)) ] &&
+				error "stripes are not evenly distributed"
+		fi
 	done
 
-	rm -rf $DIR/$tdir/*
+	$LCTL set_param lmv.*.qos_threshold_rr=$lmv_qos_threshold_rr > /dev/null
+	do_nodes $mdts $LCTL set_param \
+		lod.*.mdt_qos_threshold_rr=$lod_qos_threshold_rr > /dev/null
 
-	$LCTL set_param lmv.*.qos_threshold_rr=$qos_threshold_rr
-	# Shorten statfs result age, so that it can be updated in time
-	$LCTL set_param lmv.*.qos_maxage=1
-	sleep_maxage
+	echo
+	echo "Check for uneven MDTs: "
 
 	local ffree
+	local bavail
 	local max
 	local min
 	local max_index
 	local min_index
+	local tmp
 
-	ffree=($(lctl get_param -n mdc.*[mM][dD][cC]-[^M]*.filesfree | uniq))
-	echo "MDT filesfree available: ${ffree[@]}"
-	max=${ffree[0]}
-	min=${ffree[0]}
+	ffree=($(lctl get_param -n mdc.*[mM][dD][cC]-[^M]*.filesfree))
+	bavail=($(lctl get_param -n mdc.*[mM][dD][cC]-[^M]*.kbytesavail))
+	bsize=$(lctl get_param -n mdc.*MDT0000*.blocksize)
+
+	max=$(((${ffree[0]} >> 8) * (${bavail[0]} * bsize >> 16)))
+	min=$(((${ffree[0]} >> 8) * (${bavail[0]} * bsize >> 16)))
 	max_index=0
 	min_index=0
-	for ((i = 0; i < ${#ffree[@]}; i++)); do
-		if [[ ${ffree[i]} -gt $max ]]; then
-			max=${ffree[i]}
+	for ((i = 1; i < ${#ffree[@]}; i++)); do
+		tmp=$(((${ffree[i]} >> 8) * (${bavail[i]} * bsize >> 16)))
+		if [ $tmp -gt $max ]; then
+			max=$tmp
 			max_index=$i
 		fi
-		if [[ ${ffree[i]} -lt $min ]]; then
-			min=${ffree[i]}
+		if [ $tmp -lt $min ]; then
+			min=$tmp
 			min_index=$i
 		fi
 	done
-	echo "Min free files: MDT$min_index: $min"
-	echo "Max free files: MDT$max_index: $max"
 
-	[ $min -eq 0 ] && skip "no free files in MDT$min_index"
-	[ $min -gt 10000000 ] && skip "too much free files in MDT$min_index"
+	[ ${ffree[min_index]} -eq 0 ] &&
+		skip "no free files in MDT$min_index"
+	[ ${ffree[min_index]} -gt 100000000 ] &&
+		skip "too much free files in MDT$min_index"
 
 	# Check if we need to generate uneven MDTs
-	test_mkdir -i $min_index -c 1 -p $DIR/$tdir-MDT$min_index
-	local threshold=10
-	local diff=$((max - min))
-	local diff2=$((diff * 100 / min))
-
-	echo -n "Check for uneven MDTs: "
-	echo -n "diff=$diff files ($diff2%) must be > $threshold% ..."
-
-	if [ $diff2 -gt $threshold ]; then
-		echo "ok"
-		echo "Don't need to fill MDT$min_index"
-	else
-		# generate uneven MDTs, create till 25% diff
-		echo "no"
-		diff2=$((threshold - diff2))
-		diff=$((min * diff2 / 100))
+	local threshold=50
+	local diff=$(((max - min) * 100 / min))
+	local value="$(generate_string 1024)"
+
+	while [ $diff -lt $threshold ]; do
+		# generate uneven MDTs, create till $threshold% diff
+		echo -n "weight diff=$diff% must be > $threshold% ..."
+		count=$((${ffree[min_index]} / 10))
 		# 50 sec per 10000 files in vm
-		[ $diff -gt 40000 ] && [ "$SLOW" = "no" ] &&
-			skip "$diff files to create"
-		echo "Fill $diff2% diff in MDT$min_index with $diff files"
-		local i
-		local value="$(generate_string 1024)"
-		for i in $(seq $diff); do
+		[ $count -gt 40000 ] && [ "$SLOW" = "no" ] &&
+			skip "$count files to create"
+		echo "Fill MDT$min_index with $count files"
+		[ -d $DIR/$tdir-MDT$min_index ] ||
+			$LFS mkdir -i $min_index $DIR/$tdir-MDT$min_index ||
+			error "mkdir $tdir-MDT$min_index failed"
+		for i in $(seq $count); do
 			$OPENFILE -f O_CREAT:O_LOV_DELAY_CREATE \
-				$DIR/$tdir-MDT$min_index/f$i > /dev/null ||
-				error "create f$i failed"
+				$DIR/$tdir-MDT$min_index/f$j_$i > /dev/null ||
+				error "create f$j_$i failed"
 			setfattr -n user.413b -v $value \
-				$DIR/$tdir-MDT$min_index/f$i ||
-				error "setfattr f$i failed"
+				$DIR/$tdir-MDT$min_index/f$j_$i ||
+				error "setfattr f$j_$i failed"
 		done
-	fi
 
-	min=$((100 *MDSCOUNT))
-	max=0
+		ffree=($(lctl get_param -n mdc.*[mM][dD][cC]-*.filesfree))
+		bavail=($(lctl get_param -n mdc.*[mM][dD][cC]-*.kbytesavail))
+		max=$(((${ffree[max_index]} >> 8) * \
+			(${bavail[max_index]} * bsize >> 16)))
+		min=$(((${ffree[min_index]} >> 8) * \
+			(${bavail[min_index]} * bsize >> 16)))
+		diff=$(((max - min) * 100 / min))
+	done
+
+	echo "MDT filesfree available: ${ffree[@]}"
+	echo "MDT blocks available: ${bavail[@]}"
+	echo "weight diff=$diff%"
+
+	echo
+	echo "Mkdir (stripe_count $stripe_count) with balanced space usage:"
+
+	$LCTL set_param lmv.*.qos_prio_free=100 > /dev/null
+	do_nodes $mdts $LCTL set_param lod.*.mdt_qos_prio_free=100 > /dev/null
+	# decrease statfs age, so that it can be updated in time
+	$LCTL set_param lmv.*.qos_maxage=1 > /dev/null
+	do_nodes $mdts $LCTL set_param lod.*.mdt_qos_maxage=1 > /dev/null
+
+	sleep 1
+
+	testdir=$DIR/$tdir-s$stripe_count/qos
 
-	echo "mkdir with balanced space usage"
-	$LCTL set_param lmv.*.qos_prio_free=100
 	for i in $(seq $((100 * MDSCOUNT))); do
-		mkdir $DIR/$tdir/subdir$i || error "mkdir subdir$i failed"
+		eval $mkdir_cmd $testdir/subdir$i ||
+			error "$mkdir_cmd subdir$i failed"
 	done
+
 	for i in $(seq $MDSCOUNT); do
-		count=$($LFS getdirstripe -i $DIR/$tdir/* | grep ^$((i - 1))$ |
-			wc -w)
+		count=$($LFS getdirstripe -i $testdir/* | grep ^$((i - 1))$ |
+			wc -l)
 		echo "$count directories created on MDT$((i - 1))"
-		[ $min -gt $count ] && min=$count
-		[ $max -lt $count ] && max=$count
+
+		if [ $stripe_count -gt 1 ]; then
+			count=$($LFS getdirstripe $testdir/* |
+				grep -P "^\s+$((i - 1))\t" | wc -l)
+			echo "$count stripes created on MDT$((i - 1))"
+		fi
 	done
-	[ $((max - min)) -gt $MDSCOUNT ] ||
+
+	max=$($LFS getdirstripe -i $testdir/* | grep ^$max_index$ | wc -l)
+	min=$($LFS getdirstripe -i $testdir/* | grep ^$min_index$ | wc -l)
+
+	# D-value should > 10% of averge
+	[ $((max - min)) -lt 10 ] &&
 		error "subdirs shouldn't be evenly distributed"
 
-	which getfattr > /dev/null 2>&1 || skip_env "no getfattr command"
+	# ditto
+	if [ $stripe_count -gt 1 ]; then
+		max=$($LFS getdirstripe $testdir/* |
+			grep -P "^\s+$max_index\t" | wc -l)
+		min=$($LFS getdirstripe $testdir/* |
+			grep -P "^\s+$min_index\t" | wc -l)
+		[ $((max - min)) -le $((10 * stripe_count)) ] &&
+			error "stripes shouldn't be evenly distributed"|| true
+	fi
+}
 
-	$LFS setdirstripe -D -d $DIR/$tdir || error "setdirstripe -d failed"
-	getfattr -n trusted.dmv $DIR/$tdir && error "default dir layout exists"
-	true
+test_413a() {
+	[ $MDSCOUNT -lt 2 ] &&
+		skip "We need at least 2 MDTs for this test"
+
+	[ $MDS1_VERSION -lt $(version_code 2.12.52) ] &&
+		skip "Need server version at least 2.12.52"
+
+	local stripe_count
+
+	for stripe_count in $(seq 1 $((MDSCOUNT - 1))); do
+		mkdir $DIR/$tdir-s$stripe_count || error "mkdir failed"
+		mkdir $DIR/$tdir-s$stripe_count/rr || error "mkdir failed"
+		mkdir $DIR/$tdir-s$stripe_count/qos || error "mkdir failed"
+		test_qos_mkdir "$LFS mkdir -c $stripe_count" $stripe_count
+	done
 }
-run_test 413b "mkdir with balanced space usage"
+run_test 413a "QoS mkdir with 'lfs mkdir -i -1'"
+
+test_413b() {
+	[ $MDSCOUNT -lt 2 ] &&
+		skip "We need at least 2 MDTs for this test"
+
+	[ $MDS1_VERSION -lt $(version_code 2.12.52) ] &&
+		skip "Need server version at least 2.12.52"
+
+	local stripe_count
+
+	for stripe_count in $(seq 1 $((MDSCOUNT - 1))); do
+		mkdir $DIR/$tdir-s$stripe_count || error "mkdir failed"
+		mkdir $DIR/$tdir-s$stripe_count/rr || error "mkdir failed"
+		mkdir $DIR/$tdir-s$stripe_count/qos || error "mkdir failed"
+		$LFS setdirstripe -D -c $stripe_count \
+			$DIR/$tdir-s$stripe_count/rr ||
+			error "setdirstripe failed"
+		$LFS setdirstripe -D -c $stripe_count \
+			$DIR/$tdir-s$stripe_count/qos ||
+			error "setdirstripe failed"
+		test_qos_mkdir "mkdir" $stripe_count
+	done
+}
+run_test 413b "QoS mkdir under dir whose default LMV starting MDT offset is -1"
 
 test_414() {
 #define OBD_FAIL_PTLRPC_BULK_ATTACH      0x521
@@ -20220,7 +21623,7 @@ run_test 414 "simulate ENOMEM in ptlrpc_register_bulk()"
 
 test_415() {
 	[ $PARALLEL == "yes" ] && skip "skip parallel run"
-	[ $(lustre_version_code mds1) -lt $(version_code 2.11.52) ] &&
+	[ $MDS1_VERSION -lt $(version_code 2.11.52) ] &&
 		skip "Need server version at least 2.11.52"
 
 	# LU-11102
@@ -20262,7 +21665,7 @@ test_415() {
 run_test 415 "lock revoke is not missing"
 
 test_416() {
-	[ $(lustre_version_code mds1) -lt $(version_code 2.11.55) ] &&
+	[ $MDS1_VERSION -lt $(version_code 2.11.55) ] &&
 		skip "Need server version at least 2.11.55"
 
 	# define OBD_FAIL_OSD_TXN_START    0x19a
@@ -20444,8 +21847,301 @@ test_420()
 }
 run_test 420 "clear SGID bit on non-directories for non-members"
 
+test_421a() {
+	local cnt
+	local fid1
+	local fid2
+
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	test_mkdir $DIR/$tdir
+	createmany -o $DIR/$tdir/f 3
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 3 ] && error "unexpected #files: $cnt"
+
+	fid1=$(lfs path2fid $DIR/$tdir/f1)
+	fid2=$(lfs path2fid $DIR/$tdir/f2)
+	$LFS rmfid $DIR $fid1 $fid2 || error "rmfid failed"
+
+	stat $DIR/$tdir/f1 && error "f1 still visible on the client"
+	stat $DIR/$tdir/f2 && error "f2 still visible on the client"
+
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt == 1 ] || error "unexpected #files after: $cnt"
+
+	rm -f $DIR/$tdir/f3 || error "can't remove f3"
+	createmany -o $DIR/$tdir/f 3
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 3 ] && error "unexpected #files: $cnt"
+
+	fid1=$(lfs path2fid $DIR/$tdir/f1)
+	fid2=$(lfs path2fid $DIR/$tdir/f2)
+	echo "remove using fsname $FSNAME"
+	$LFS rmfid $FSNAME $fid1 $fid2 || error "rmfid with fsname failed"
+
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt == 1 ] || error "unexpected #files after: $cnt"
+}
+run_test 421a "simple rm by fid"
+
+test_421b() {
+	local cnt
+	local FID1
+	local FID2
+
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	test_mkdir $DIR/$tdir
+	createmany -o $DIR/$tdir/f 3
+	multiop_bg_pause $DIR/$tdir/f1 o_c || error "multiop failed to start"
+	MULTIPID=$!
+
+	FID1=$(lfs path2fid $DIR/$tdir/f1)
+	FID2=$(lfs path2fid $DIR/$tdir/f2)
+	$LFS rmfid $DIR $FID1 $FID2 && error "rmfid didn't fail"
+
+	kill -USR1 $MULTIPID
+	wait
+
+	cnt=$(ls $DIR/$tdir | wc -l)
+	[ $cnt == 2 ] || error "unexpected #files after: $cnt"
+}
+run_test 421b "rm by fid on open file"
+
+test_421c() {
+	local cnt
+	local FIDS
+
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	test_mkdir $DIR/$tdir
+	createmany -o $DIR/$tdir/f 3
+	touch $DIR/$tdir/$tfile
+	createmany -l$DIR/$tdir/$tfile $DIR/$tdir/h 180
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 184 ] && error "unexpected #files: $cnt"
+
+	FID1=$(lfs path2fid $DIR/$tdir/$tfile)
+	$LFS rmfid $DIR $FID1 || error "rmfid failed"
+
+	cnt=$(ls $DIR/$tdir | wc -l)
+	[ $cnt == 3 ] || error "unexpected #files after: $cnt"
+}
+run_test 421c "rm by fid against hardlinked files"
+
+test_421d() {
+	local cnt
+	local FIDS
+
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	test_mkdir $DIR/$tdir
+	createmany -o $DIR/$tdir/f 4097
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 4097 ] && error "unexpected #files: $cnt"
+
+	FIDS=$(lfs path2fid $DIR/$tdir/f* | sed "s/[/][^:]*://g")
+	$LFS rmfid $DIR $FIDS || error "rmfid failed"
+
+	cnt=$(ls $DIR/$tdir | wc -l)
+	rm -rf $DIR/$tdir
+	[ $cnt == 0 ] || error "unexpected #files after: $cnt"
+}
+run_test 421d "rmfid en masse"
+
+test_421e() {
+	local cnt
+	local FID
+
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	mkdir -p $DIR/$tdir
+	$LFS setdirstripe -c$MDSCOUNT $DIR/$tdir/striped_dir
+	createmany -o $DIR/$tdir/striped_dir/f 512
+	cnt=$(ls -1 $DIR/$tdir/striped_dir | wc -l)
+	[ $cnt != 512 ] && error "unexpected #files: $cnt"
+
+	FIDS=$(lfs path2fid $DIR/$tdir/striped_dir/f* |
+		sed "s/[/][^:]*://g")
+	$LFS rmfid $DIR $FIDS || error "rmfid failed"
+
+	cnt=$(ls $DIR/$tdir/striped_dir | wc -l)
+	rm -rf $DIR/$tdir
+	[ $cnt == 0 ] || error "unexpected #files after: $cnt"
+}
+run_test 421e "rmfid in DNE"
+
+test_421f() {
+	local cnt
+	local FID
+
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	test_mkdir $DIR/$tdir
+	touch $DIR/$tdir/f
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 1 ] && error "unexpected #files: $cnt"
+
+	FID=$(lfs path2fid $DIR/$tdir/f)
+	$RUNAS $LFS rmfid $DIR $FID && error "rmfid didn't fail (1)"
+	# rmfid should fail
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 1 ] && error "unexpected #files after (2): $cnt"
+
+	chmod a+rw $DIR/$tdir
+	ls -la $DIR/$tdir
+	$RUNAS $LFS rmfid $DIR $FID && error "rmfid didn't fail (2)"
+	# rmfid should fail
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 1 ] && error "unexpected #files after (3): $cnt"
+
+	rm -f $DIR/$tdir/f
+	$RUNAS touch $DIR/$tdir/f
+	FID=$(lfs path2fid $DIR/$tdir/f)
+	echo "rmfid as root"
+	$LFS rmfid $DIR $FID || error "rmfid as root failed"
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt == 0 ] || error "unexpected #files after (4): $cnt"
+
+	rm -f $DIR/$tdir/f
+	$RUNAS touch $DIR/$tdir/f
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt != 1 ] && error "unexpected #files (4): $cnt"
+	FID=$(lfs path2fid $DIR/$tdir/f)
+	# rmfid w/o user_fid2path mount option should fail
+	$RUNAS $LFS rmfid $DIR $FID && error "rmfid didn't fail(3)"
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt == 1 ] || error "unexpected #files after (5): $cnt"
+
+	umount_client $MOUNT || error "failed to umount client"
+	mount_client $MOUNT "$MOUNT_OPTS,user_fid2path" ||
+		error "failed to mount client'"
+
+	$RUNAS $LFS rmfid $DIR $FID || error "rmfid failed"
+	# rmfid should succeed
+	cnt=$(ls -1 $DIR/$tdir | wc -l)
+	[ $cnt == 0 ] || error "unexpected #files after (6): $cnt"
+
+	# rmfid shouldn't allow to remove files due to dir's permission
+	chmod a+rwx $DIR/$tdir
+	touch $DIR/$tdir/f
+	ls -la $DIR/$tdir
+	FID=$(lfs path2fid $DIR/$tdir/f)
+	$RUNAS $LFS rmfid $DIR $FID && error "rmfid didn't fail"
+
+	umount_client $MOUNT || error "failed to umount client"
+	mount_client $MOUNT "$MOUNT_OPTS" ||
+		error "failed to mount client'"
+
+}
+run_test 421f "rmfid checks permissions"
+
+test_421g() {
+	local cnt
+	local FIDS
+
+	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
+	[ $MDS1_VERSION -lt $(version_code 2.12.54) ] &&
+		skip "Need MDS version at least 2.12.54"
+
+	mkdir -p $DIR/$tdir
+	$LFS setdirstripe -c$MDSCOUNT $DIR/$tdir/striped_dir
+	createmany -o $DIR/$tdir/striped_dir/f 512
+	cnt=$(ls -1 $DIR/$tdir/striped_dir | wc -l)
+	[ $cnt != 512 ] && error "unexpected #files: $cnt"
+
+	FIDS=$(lfs path2fid $DIR/$tdir/striped_dir/f* |
+		sed "s/[/][^:]*://g")
+
+	rm -f $DIR/$tdir/striped_dir/f1*
+	cnt=$(ls -1 $DIR/$tdir/striped_dir | wc -l)
+	removed=$((512 - cnt))
+
+	# few files have been just removed, so we expect
+	# rmfid to fail on their fids
+	errors=$($LFS rmfid $DIR $FIDS 2>&1 | wc -l)
+	[ $removed != $errors ] && error "$errors != $removed"
+
+	cnt=$(ls $DIR/$tdir/striped_dir | wc -l)
+	rm -rf $DIR/$tdir
+	[ $cnt == 0 ] || error "unexpected #files after: $cnt"
+}
+run_test 421g "rmfid to return errors properly"
+
+test_422() {
+	test_mkdir -i 0 -c 1 -p $DIR/$tdir/d1
+	test_mkdir -i 0 -c 1 -p $DIR/$tdir/d2
+	test_mkdir -i 0 -c 1 -p $DIR/$tdir/d3
+	dd if=/dev/zero of=$DIR/$tdir/d1/file1 bs=1k count=1
+	dd if=/dev/zero of=$DIR/$tdir/d2/file1 bs=1k count=1
+
+	local amc=$(at_max_get client)
+	local amo=$(at_max_get mds1)
+	local timeout=`lctl get_param -n timeout`
+
+	at_max_set 0 client
+	at_max_set 0 mds1
+
+#define OBD_FAIL_PTLRPC_PAUSE_REQ        0x50a
+	do_facet mds1 $LCTL set_param fail_loc=0x8000050a \
+			fail_val=$(((2*timeout + 10)*1000))
+	touch $DIR/$tdir/d3/file &
+	sleep 2
+#define OBD_FAIL_TGT_REPLY_DATA_RACE	 0x722
+	do_facet mds1 $LCTL set_param fail_loc=0x80000722 \
+			fail_val=$((2*timeout + 5))
+	mv $DIR/$tdir/d1/file1 $DIR/$tdir/d1/file2 &
+	local pid=$!
+	sleep 1
+	kill -9 $pid
+	sleep $((2 * timeout))
+	echo kill $pid
+	kill -9 $pid
+	lctl mark touch
+	touch $DIR/$tdir/d2/file3
+	touch $DIR/$tdir/d2/file4
+	touch $DIR/$tdir/d2/file5
+
+	wait
+	at_max_set $amc client
+	at_max_set $amo mds1
+
+	# LU-12838 - verify the ptlrpc thread watchdog is not always throttled
+	do_facet mds1 "dmesg | grep 'Dumping the stack trace for debugging'" ||
+		error "Watchdog is always throttled"
+}
+run_test 422 "kill a process with RPC in progress"
+
+stat_test() {
+    df -h $MOUNT &
+    df -h $MOUNT &
+    df -h $MOUNT &
+    df -h $MOUNT &
+    df -h $MOUNT &
+    df -h $MOUNT &
+}
+
+test_423() {
+    local _stats
+    # ensure statfs cache is expired
+    sleep 2;
+
+    _stats=$(stat_test | grep $MOUNT | sort -u | wc -l)
+    [[ ${_stats} -ne 1 ]] && error "statfs wrong"
+
+    return 0
+}
+run_test 423 "statfs should return a right data"
+
 prep_801() {
-	[[ $(lustre_version_code mds1) -lt $(version_code 2.9.55) ]] ||
+	[[ $MDS1_VERSION -lt $(version_code 2.9.55) ]] ||
 	[[ $OST1_VERSION -lt $(version_code 2.9.55) ]] &&
 		skip "Need server version at least 2.9.55"
 
@@ -20486,7 +22182,8 @@ test_801a() {
 	echo "Start barrier_freeze at: $(date)"
 	#define OBD_FAIL_BARRIER_DELAY		0x2202
 	do_facet mgs $LCTL set_param fail_val=5 fail_loc=0x2202
-	do_facet mgs $LCTL barrier_freeze $FSNAME 10 &
+	# Do not reduce barrier time - See LU-11873
+	do_facet mgs $LCTL barrier_freeze $FSNAME 20 &
 
 	sleep 2
 	local b_status=$(barrier_stat)
@@ -20508,7 +22205,8 @@ test_801a() {
 	[ "$b_status" = "'expired'" ] ||
 		error "(3) unexpected barrier status $b_status"
 
-	do_facet mgs $LCTL barrier_freeze $FSNAME 10 ||
+	# Do not reduce barrier time - See LU-11873
+	do_facet mgs $LCTL barrier_freeze $FSNAME 20 ||
 		error "(4) fail to freeze barrier"
 
 	b_status=$(barrier_stat)
@@ -20633,7 +22331,8 @@ test_801c() {
 	do_facet mgs $LCTL barrier_rescan $FSNAME ||
 		error "(3) Fail to rescan barrier bitmap"
 
-	do_facet mgs $LCTL barrier_freeze $FSNAME 10
+	# Do not reduce barrier time - See LU-11873
+	do_facet mgs $LCTL barrier_freeze $FSNAME 20
 
 	b_status=$(barrier_stat)
 	[ "$b_status" = "'frozen'" ] ||
@@ -20672,8 +22371,8 @@ cleanup_802a() {
 }
 
 test_802a() {
-
-	[[ $(lustre_version_code mds1) -lt $(version_code 2.9.55) ]] ||
+	[[ $mds1_FSTYPE = zfs ]] || skip "ZFS specific test"
+	[[ $MDS1_VERSION -lt $(version_code 2.9.55) ]] ||
 	[[ $OST1_VERSION -lt $(version_code 2.9.55) ]] &&
 		skip "Need server version at least 2.9.55"
 
@@ -20888,7 +22587,7 @@ cleanup_805() {
 }
 
 test_805() {
-	local zfs_version=$(do_node $SINGLEMDS cat /sys/module/zfs/version)
+	local zfs_version=$(do_facet mds1 cat /sys/module/zfs/version)
 	[ "$mds1_FSTYPE" != "zfs" ] && skip "ZFS specific test"
 	[ $(version_code $zfs_version) -lt $(version_code 0.7.2) ] &&
 		skip "netfree not implemented before 0.7"
@@ -20900,7 +22599,7 @@ test_805() {
 	local usedkb
 	local old
 	local quota
-	local pref="osd-zfs.lustre-MDT0000."
+	local pref="osd-zfs.$FSNAME-MDT0000."
 
 	# limit available space on MDS dataset to meet nospace issue
 	# quickly. then ZFS 0.7.2 can use reserved space if asked
@@ -20998,7 +22697,7 @@ test_806() {
 	done
 	check_lsom_size $DIR/$tfile $size
 
-	# multi-client wirtes
+	# multi-client writes
 	num=$(get_node_count ${CLIENTS//,/ })
 	size=$(($num * $bs))
 	offset=0
@@ -21163,25 +22862,36 @@ test_809() {
 run_test 809 "Verify no SOM xattr store for DoM-only files"
 
 test_810() {
-	local ORIG
-	local CSUM
-
-	# t10 seem to dislike partial pages
-	lctl set_param osc.*.checksum_type=adler
-	lctl set_param fail_loc=0x411
-	dd if=/dev/urandom of=$DIR/$tfile bs=10240 count=2
-	ORIG=$(md5sum $DIR/$tfile)
-	lctl set_param ldlm.namespaces.*osc*.lru_size=clear
-	CSUM=$(md5sum $DIR/$tfile)
-	set_checksum_type adler
-	if [ "$ORIG" != "$CSUM" ]; then
-		error "$ORIG != $CSUM"
-	fi
+	[ $PARALLEL == "yes" ] && skip "skip parallel run"
+	$GSS && skip_env "could not run with gss"
+	[[ $OST1_VERSION -gt $(version_code 2.12.58) ]] ||
+		skip "OST < 2.12.58 doesn't align checksum"
+
+	set_checksums 1
+	stack_trap "set_checksums $ORIG_CSUM" EXIT
+	stack_trap "set_checksum_type $ORIG_CSUM_TYPE" EXIT
+
+	local csum
+	local before
+	local after
+	for csum in $CKSUM_TYPES; do
+		#define OBD_FAIL_OSC_NO_GRANT	0x411
+		$LCTL set_param osc.*.checksum_type=$csum fail_loc=0x411
+		for i in "10240 0" "10000 0" "4000 1" "500 1"; do
+			eval set -- $i
+			dd if=/dev/urandom of=$DIR/$tfile bs=$1 count=2 seek=$2
+			before=$(md5sum $DIR/$tfile)
+			$LCTL set_param ldlm.namespaces.*osc*.lru_size=clear
+			after=$(md5sum $DIR/$tfile)
+			[ "$before" == "$after" ] ||
+				error "$csum: $before != $after bs=$1 seek=$2"
+		done
+	done
 }
 run_test 810 "partial page writes on ZFS (LU-11663)"
 
 test_811() {
-	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.11.56) ] &&
+	[ $MDS1_VERSION -lt $(version_code 2.11.56) ] &&
 		skip "Need MDS version at least 2.11.56"
 
 	#define OBD_FAIL_MDS_ORPHAN_DELETE	0x165
@@ -21191,13 +22901,12 @@ test_811() {
 	stop mds1
 	start mds1 $(mdsdevname 1) $MDS_MOUNT_OPTS
 
-	sleep 5
-	[[ $(do_facet mds1 pgrep orph_.*-MDD | wc -l) -eq 0 ]] ||
+	wait_update_facet mds1 "pgrep orph_.*-MDD | wc -l" "0" ||
 		error "MDD orphan cleanup thread not quit"
 }
 run_test 811 "orphan name stub can be cleaned up in startup"
 
-test_812() {
+test_812a() {
 	[ $OST1_VERSION -lt $(version_code 2.12.51) ] &&
 		skip "OST < 2.12.51 doesn't support this fail_loc"
 	[ "$SHARED_KEY" = true ] &&
@@ -21218,7 +22927,31 @@ test_812() {
 
 	stat $DIR/$tfile >/dev/null || error "can't stat file"
 }
-run_test 812 "do not drop reqs generated when imp is going to idle (LU-11951)"
+run_test 812a "do not drop reqs generated when imp is going to idle (LU-11951)"
+
+test_812b() { # LU-12378
+	[ $OST1_VERSION -lt $(version_code 2.12.51) ] &&
+		skip "OST < 2.12.51 doesn't support this fail_loc"
+	[ "$SHARED_KEY" = true ] &&
+		skip "OSC connections never go IDLE with Shared-Keys enabled"
+
+	$LFS setstripe -c 1 -i 0 $DIR/$tfile || error "setstripe failed"
+	# ensure ost1 is connected
+	stat $DIR/$tfile >/dev/null || error "can't stat"
+	wait_osc_import_state client ost1 FULL
+	# no locks, no reqs to let the connection idle
+	cancel_lru_locks osc
+
+	# delay OST_DISCONNECT on OST1 to put OSC into intermediate state
+#define OBD_FAIL_OST_DISCONNECT_DELAY	 0x245
+	do_facet ost1 "$LCTL set_param fail_loc=0x245 fail_val=8"
+	wait_osc_import_state client ost1 CONNECTING
+	do_facet ost1 "$LCTL set_param fail_loc=0 fail_val=0"
+
+	$LFS quota -u 0 $DIR/ || error "lfs quota should succeed"
+	wait_osc_import_state client ost1 IDLE
+}
+run_test 812b "do not drop no resend request for idle connect"
 
 test_813() {
 	local file_heat_sav=$($LCTL get_param -n llite.*.file_heat 2>/dev/null)
@@ -21408,6 +23141,9 @@ test_815()
 run_test 815 "zero byte tiny write doesn't hang (LU-12382)"
 
 test_816() {
+	[ "$SHARED_KEY" = true ] &&
+		skip "OSC connections never go IDLE with Shared-Keys enabled"
+
 	$LFS setstripe -c 1 -i 0 $DIR/$tfile
 	# ensure ost1 is connected
 	stat $DIR/$tfile >/dev/null || error "can't stat"
@@ -21428,6 +23164,99 @@ test_816() {
 }
 run_test 816 "do not reset lru_resize on idle reconnect"
 
+cleanup_817() {
+	umount $tmpdir
+	exportfs -u localhost:$DIR/nfsexp
+	rm -rf $DIR/nfsexp
+}
+
+test_817() {
+	systemctl restart nfs-server.service || skip "failed to restart nfsd"
+
+	mkdir -p $DIR/nfsexp
+	exportfs -orw,no_root_squash localhost:$DIR/nfsexp ||
+		error "failed to export nfs"
+
+	tmpdir=$(mktemp -d /tmp/nfs-XXXXXX)
+	stack_trap cleanup_817 EXIT
+
+	mount -t nfs -orw localhost:$DIR/nfsexp $tmpdir ||
+		error "failed to mount nfs to $tmpdir"
+
+	cp /bin/true $tmpdir
+	$DIR/nfsexp/true || error "failed to execute 'true' command"
+}
+run_test 817 "nfsd won't cache write lock for exec file"
+
+test_818() {
+	mkdir $DIR/$tdir
+	$LFS setstripe -c1 -i0 $DIR/$tfile
+	$LFS setstripe -c1 -i1 $DIR/$tfile
+	stop $SINGLEMDS
+	#define OBD_FAIL_OSP_CANT_PROCESS_LLOG		0x2105
+	do_facet $SINGLEMDS lctl set_param fail_loc=0x80002105
+	start $SINGLEMDS $(mdsdevname ${SINGLEMDS//mds/}) $MDS_MOUNT_OPTS ||
+		error "start $SINGLEMDS failed"
+	rm -rf $DIR/$tdir
+}
+run_test 818 "unlink with failed llog"
+
+test_819a() {
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=1
+	cancel_lru_locks osc
+	#define OBD_FAIL_OST_2BIG_NIOBUF		0x248
+	do_facet $SINGLEMDS lctl set_param fail_loc=0x80000248
+	dd if=$DIR/$tfile of=/dev/null bs=1M count=1
+	rm -f $TDIR/$tfile
+}
+run_test 819a "too big niobuf in read"
+
+test_819b() {
+	#define OBD_FAIL_OST_2BIG_NIOBUF		0x248
+	do_facet $SINGLEMDS lctl set_param fail_loc=0x80000248
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=1
+	cancel_lru_locks osc
+	sleep 1
+	rm -f $TDIR/$tfile
+}
+run_test 819b "too big niobuf in write"
+
+
+function test_820_start_ost() {
+	sleep 5
+
+	for num in $(seq $OSTCOUNT); do
+		start ost$num $(ostdevname $num) $OST_MOUNT_OPTS
+	done
+}
+
+test_820() {
+	[[ $MDSCOUNT -lt 2 ]] && skip_env "needs >= 2 MDTs"
+
+	mkdir $DIR/$tdir
+	umount_client $MOUNT || error "umount failed"
+	for num in $(seq $OSTCOUNT); do
+		stop ost$num
+	done
+
+	# mount client with no active OSTs
+	# so that the client can't initialize max LOV EA size
+	# from OSC notifications
+	mount_client $MOUNT || error "mount failed"
+	# delay OST starting to keep this 0 max EA size for a while
+	test_820_start_ost &
+
+	# create a directory on MDS2
+	test_mkdir -i 1 -c1 $DIR/$tdir/mds2 ||
+		error "Failed to create directory"
+	# open intent should update default EA size
+	# see mdc_update_max_ea_from_body()
+	# notice this is the very first RPC to MDS2
+	cp /etc/services $DIR/$tdir/mds2 ||
+		error "Failed to copy files to mds$n"
+}
+run_test 820 "update max EA from open intent"
+
 #
 # tests that do cleanup/setup should be run at the end
 #
@@ -21446,6 +23275,43 @@ test_900() {
 }
 run_test 900 "umount should not race with any mgc requeue thread"
 
+# LUS-6253/LU-11185
+test_901() {
+	local oldc
+	local newc
+	local olds
+	local news
+	[ $PARALLEL == "yes" ] && skip "skip parallel run"
+
+	# some get_param have a bug to handle dot in param name
+	cancel_lru_locks MGC
+	oldc=$($LCTL get_param -n 'ldlm.namespaces.MGC*.lock_count')
+	olds=$(do_facet mgs $LCTL get_param -n 'ldlm.namespaces.MGS*.lock_count')
+	umount_client $MOUNT || error "umount failed"
+	mount_client $MOUNT || error "mount failed"
+	cancel_lru_locks MGC
+	newc=$($LCTL get_param -n 'ldlm.namespaces.MGC*.lock_count')
+	news=$(do_facet mgs $LCTL get_param -n 'ldlm.namespaces.MGS*.lock_count')
+
+	[ $oldc -lt $newc ] && error "mgc lock leak ($oldc != $newc)"
+	[ $olds -lt $news ] && error "mgs lock leak ($olds != $news)"
+
+	return 0
+}
+run_test 901 "don't leak a mgc lock on client umount"
+
+# LU-13377
+test_902() {
+	[ $CLIENT_VERSION -lt $(version_code 2.13.52) ] &&
+		skip "client does not have LU-13377 fix"
+	#define OBD_FAIL_LLITE_SHORT_COMMIT 0x1415
+	$LCTL set_param fail_loc=0x1415
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=1
+	cancel_lru_locks osc
+	rm -f $DIR/$tfile
+}
+run_test 902 "test short write doesn't hang lustre"
+
 complete $SECONDS
 [ -f $EXT2_DEV ] && rm $EXT2_DEV || true
 check_and_cleanup_lustre