X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Fsanity-quota.sh;h=86c54febbaf2b9792d1feb3ba6e00600e6d6cffe;hp=af980c8b113f7addb1c2f6e90a5b72783fd256f8;hb=7b46ff54afb23c04dec74ef162ed9052f7259e4a;hpb=e2f467190ec7bc3c0102f699d22b4891fa7217f1

diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh
old mode 100644
new mode 100755
index af980c8..86c54fe
--- a/lustre/tests/sanity-quota.sh
+++ b/lustre/tests/sanity-quota.sh
@@ -6,15 +6,16 @@
 # Run test by setting NOSETUP=true when ltest has setup env for us
 set -e
 
-SRCDIR=`dirname $0`
+SRCDIR=$(dirname $0)
 export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin
 
 ONLY=${ONLY:-"$*"}
-ALWAYS_EXCEPT="$SANITY_QUOTA_EXCEPT"
+# Bug number for skipped test:      LU-5152
+ALWAYS_EXCEPT="$SANITY_QUOTA_EXCEPT 55"
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
 
 [ "$ALWAYS_EXCEPT$EXCEPT" ] &&
-	echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT`"
+	echo "Skipping tests: $ALWAYS_EXCEPT $EXCEPT"
 
 TMP=${TMP:-/tmp}
 
@@ -23,9 +24,11 @@ TSTID=${TSTID:-60000}
 TSTID2=${TSTID2:-60001}
 TSTUSR=${TSTUSR:-"quota_usr"}
 TSTUSR2=${TSTUSR2:-"quota_2usr"}
+TSTPRJID=${TSTPRJID:-1000}
 BLK_SZ=1024
 MAX_DQ_TIME=604800
 MAX_IQ_TIME=604800
+QTYPE="ugp"
 
 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 . $LUSTRE/tests/test-framework.sh
@@ -37,26 +40,24 @@ DIRECTIO=${DIRECTIO:-$LUSTRE/tests/directio}
 require_dsh_mds || exit 0
 require_dsh_ost || exit 0
 
-# XXX Once we drop the interoperability with old server (< 2.3.50), the
-#     sanity-quota-old.sh should be removed.
-if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then
-	exec $LUSTRE/tests/sanity-quota-old.sh
+# Does e2fsprogs support quota feature?
+if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&
+	do_facet $SINGLEMDS "! $DEBUGFS -c -R supported_features |
+		grep -q 'quota'"; then
+	skip_env "e2fsprogs doesn't support quota" && exit 0
 fi
 
-# if e2fsprogs support quota feature?
-if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] && \
-	! $DEBUGFS -c -R supported_features | grep -q 'quota'; then
-	skip "e2fsprogs doesn't support quota" && exit 0
-fi
+# Test duration:                   30 min
+[ "$SLOW" = "no" ] && EXCEPT_SLOW="61"
 
 if [ $(facet_fstype $SINGLEMDS) = "zfs" ]; then
-# bug number for skipped test:        LU-2836 LU-2836 LU-2059
-	ALWAYS_EXCEPT="$ALWAYS_EXCEPT 3       6       7d"
-# bug number:     LU-2887
-	ZFS_SLOW="12a"
-fi
+	# bug number for skipped test: LU-6836
+	ALWAYS_EXCEPT="$ALWAYS_EXCEPT  4a"
 
-[ "$SLOW" = "no" ] && EXCEPT_SLOW="$ZFS_SLOW 9 18 21"
+	# bug number:                        LU-2887
+	# Test duration:                     21      9 min"
+	[ "$SLOW" = "no" ] && EXCEPT_SLOW+=" 12a     9"
+fi
 
 QUOTALOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
 
@@ -70,36 +71,55 @@ export QUOTA_AUTO=0
 
 check_and_setup_lustre
 
-LOVNAME=`lctl get_param -n llite.*.lov.common_name | tail -n 1`
-OSTCOUNT=`lctl get_param -n lov.$LOVNAME.numobd`
+ENABLE_PROJECT_QUOTAS=${ENABLE_PROJECT_QUOTAS:-true}
+is_project_quota_supported() {
+	$ENABLE_PROJECT_QUOTAS || return 1
+	[ "$(facet_fstype $SINGLEMDS)" == "ldiskfs" ] &&
+		[ $(lustre_version_code $SINGLEMDS) -gt \
+		$(version_code 2.9.55) ] &&
+		lfs --help | grep project >&/dev/null &&
+		egrep -q "7." /etc/redhat-release && return 0
+
+	if [ "$(facet_fstype $SINGLEMDS)" == "zfs" ]; then
+		[ $(lustre_version_code $SINGLEMDS) -le \
+			$(version_code 2.10.53) ] && return 1
+
+		$ZPOOL upgrade -v | grep project_quota && return 0
+	fi
+
+	return 1
+}
 
 SHOW_QUOTA_USER="$LFS quota -v -u $TSTUSR $DIR"
 SHOW_QUOTA_USERID="$LFS quota -v -u $TSTID $DIR"
-SHOW_QUOTA_USER2="$LFS quota -v -u $TSTUSR2 $DIR"
 SHOW_QUOTA_GROUP="$LFS quota -v -g $TSTUSR $DIR"
 SHOW_QUOTA_GROUPID="$LFS quota -v -g $TSTID $DIR"
-SHOW_QUOTA_GROUP2="$LFS quota -v -g $TSTUSR2 $DIR"
+SHOW_QUOTA_PROJID="eval is_project_quota_supported && $LFS quota -v -p $TSTPRJID $DIR"
 SHOW_QUOTA_INFO_USER="$LFS quota -t -u $DIR"
 SHOW_QUOTA_INFO_GROUP="$LFS quota -t -g $DIR"
+SHOW_QUOTA_INFO_PROJID="eval is_project_quota_supported && $LFS quota -t -p $DIR"
 
 build_test_filter
 
 lustre_fail() {
-        local fail_node=$1
+	local fail_node=$1
 	local fail_loc=$2
 	local fail_val=${3:-0}
+	local NODES=
 
-	if [ $fail_node == "mds" ] || [ $fail_node == "mds_ost" ]; then
-		do_facet $SINGLEMDS "lctl set_param fail_val=$fail_val"
-		do_facet $SINGLEMDS "lctl set_param fail_loc=$fail_loc"
-	fi
+	case $fail_node in
+	mds_ost|mdt_ost) NODES="$(comma_list $(mdts_nodes) $(osts_nodes))";;
+	mds|mdt) NODES="$(comma_list $(mdts_nodes))";;
+	ost) NODES="$(comma_list $(osts_nodes))";;
+	esac
 
-	if [ $fail_node == "ost" ] || [ $fail_node == "mds_ost" ]; then
-		for num in `seq $OSTCOUNT`; do
-			do_facet ost$num "lctl set_param fail_val=$fail_val"
-			do_facet ost$num "lctl set_param fail_loc=$fail_loc"
-		done
-	fi
+	do_nodes $NODES "lctl set_param fail_val=$fail_val fail_loc=$fail_loc"
+}
+
+change_project()
+{
+	echo "lfs project $*"
+	lfs project $* || error "lfs project $* failed"
 }
 
 RUNAS="runas -u $TSTID -g $TSTID"
@@ -108,20 +128,20 @@ DD="dd if=/dev/zero bs=1M"
 
 FAIL_ON_ERROR=false
 
-check_runas_id_ret $TSTUSR $TSTUSR $RUNAS ||
-	error "Please create user $TSTUSR($TSTID) and group $TSTUSR($TSTID)"
-check_runas_id_ret $TSTUSR2 $TSTUSR2 $RUNAS2 ||
-	error "Please create user $TSTUSR2($TSTID2) and group $TSTUSR2($TSTID2)"
-
 # clear quota limits for a user or a group
 # usage: resetquota -u username
 #        resetquota -g groupname
+#	 resetquota -p projid
 
 resetquota() {
 	[ "$#" != 2 ] && error "resetquota: wrong number of arguments: $#"
-	[ "$1" != "-u" -a "$1" != "-g" ] &&
+	[ "$1" != "-u" -a "$1" != "-g" -a "$1" != "-p" ] &&
 		error "resetquota: wrong specifier $1 passed"
 
+	if [ $1 == "-p" ]; then
+		is_project_quota_supported || return 0
+	fi
+
 	$LFS setquota "$1" "$2" -b 0 -B 0 -i 0 -I 0 $MOUNT ||
 		error "clear quota for [type:$1 name:$2] failed"
 	# give a chance to slave to release space
@@ -129,20 +149,27 @@ resetquota() {
 }
 
 quota_scan() {
-	local LOCAL_UG=$1
-	local LOCAL_ID=$2
+	local local_ugp=$1
+	local local_id=$2
+
+	if [ "$local_ugp" == "a" -o "$local_ugp" == "u" ]; then
+		$LFS quota -v -u $local_id $DIR
+		log "Files for user ($local_id):"
+		($LFS find --user $local_id $DIR | head -n 4 |
+			xargs stat 2>/dev/null)
+	fi
 
-	if [ "$LOCAL_UG" == "a" -o "$LOCAL_UG" == "u" ]; then
-		$LFS quota -v -u $LOCAL_ID $DIR
-		log "Files for user ($LOCAL_ID):"
-		($LFS find -user $LOCAL_ID $DIR | head -n 4 |
+	if [ "$local_ugp" == "a" -o "$local_ugp" == "g" ]; then
+		$LFS quota -v -g $local_id $DIR
+		log "Files for group ($local_id):"
+		($LFS find --group $local_id $DIR | head -n 4 |
 			xargs stat 2>/dev/null)
 	fi
 
-	if [ "$LOCAL_UG" == "a" -o "$LOCAL_UG" == "g" ]; then
-		$LFS quota -v -u $LOCAL_ID $DIR
-		log "Files for group ($LOCAL_ID):"
-		($LFS find -group $LOCAL_ID $DIR | head -n 4 |
+	if [ "$local_ugp" == "a" -o "$local_ugp" == "p" ]; then
+		$LFS quota -v -p $TSTPRJID $DIR
+		log "Files for project ($TSTPRJID):"
+		($LFS find --projid $TSTPRJID $DIR | head -n 4 |
 			xargs stat 2>/dev/null)
 	fi
 }
@@ -160,16 +187,17 @@ quota_log() {
 }
 
 # get quota for a user or a group
-# usage: getquota -u|-g <username>|<groupname> global|<obd_uuid> \
+# usage: getquota -u|-g|-p <username>|<groupname>|<projid> global|<obd_uuid> \
 #		  bhardlimit|bsoftlimit|bgrace|ihardlimit|isoftlimit|igrace
-#
 getquota() {
 	local spec
 	local uuid
 
+	sync_all_data > /dev/null 2>&1 || true
+
 	[ "$#" != 4 ] && error "getquota: wrong number of arguments: $#"
-	[ "$1" != "-u" -a "$1" != "-g" ] &&
-		error "getquota: wrong u/g specifier $1 passed"
+	[ "$1" != "-u" -a "$1" != "-g" -a "$1" != "-p" ] &&
+		error "getquota: wrong u/g/p specifier $1 passed"
 
 	uuid="$3"
 
@@ -194,13 +222,21 @@ getquota() {
 }
 
 # set mdt quota type
-# usage: set_mdt_qtype ug|u|g|none
+# usage: set_mdt_qtype ugp|u|g|p|none
 set_mdt_qtype() {
 	local qtype=$1
 	local varsvc
 	local mdts=$(get_facets MDS)
 	local cmd
-	do_facet mgs $LCTL conf_param $FSNAME.quota.mdt=$qtype
+	[[ "$qtype" =~ "p" ]] && ! is_project_quota_supported &&
+		qtype=$(tr -d 'p' <<<$qtype)
+
+	if [[ $PERM_CMD = *"set_param -P"* ]]; then
+		do_facet mgs $PERM_CMD \
+			osd-*.$FSNAME-MDT*.quota_slave.enable=$qtype
+	else
+		do_facet mgs $PERM_CMD $FSNAME.quota.mdt=$qtype
+	fi
 	# we have to make sure each MDT received config changes
 	for mdt in ${mdts//,/ }; do
 		varsvc=${mdt}_svc
@@ -216,13 +252,21 @@ set_mdt_qtype() {
 }
 
 # set ost quota type
-# usage: set_ost_qtype ug|u|g|none
+# usage: set_ost_qtype ugp|u|g|p|none
 set_ost_qtype() {
 	local qtype=$1
 	local varsvc
 	local osts=$(get_facets OST)
 	local cmd
-	do_facet mgs $LCTL conf_param $FSNAME.quota.ost=$qtype
+	[[ "$qtype" =~ "p" ]] && ! is_project_quota_supported &&
+		qtype=$(tr -d 'p' <<<$qtype)
+
+	if [[ $PERM_CMD = *"set_param -P"* ]]; then
+		do_facet mgs $PERM_CMD \
+			osd-*.$FSNAME-OST*.quota_slave.enable=$qtype
+	else
+		do_facet mgs $PERM_CMD $FSNAME.quota.ost=$qtype
+	fi
 	# we have to make sure each OST received config changes
 	for ost in ${osts//,/ }; do
 		varsvc=${ost}_svc
@@ -271,13 +315,18 @@ wait_mdt_reint() {
 	local qtype=$1
 	local max=${2:-90}
 
-	if [ $qtype == "u" ] || [ $qtype == "ug" ]; then
+	if [[ "$qtype" =~ "u" ]]; then
 		wait_reintegration "mdt" "user" $max || return 1
 	fi
 
-	if [ $qtype == "g" ] || [ $qtype == "ug" ]; then
+	if [[ "$qtype" =~ "g" ]]; then
 		wait_reintegration "mdt" "group" $max || return 1
 	fi
+
+	if [[ "$qtype" =~ "p" ]]; then
+		! is_project_quota_supported && return 0
+		wait_reintegration "mdt" "project" $max || return 1
+	fi
 	return 0
 }
 
@@ -285,22 +334,45 @@ wait_ost_reint() {
 	local qtype=$1
 	local max=${2:-90}
 
-	if [ $qtype == "u" ] || [ $qtype == "ug" ]; then
+	if [[ "$qtype" =~ "u" ]]; then
 		wait_reintegration "ost" "user" $max || return 1
 	fi
 
-	if [ $qtype == "g" ] || [ $qtype == "ug" ]; then
+	if [[ "$qtype" =~ "g" ]]; then
 		wait_reintegration "ost" "group" $max || return 1
 	fi
+
+	if [[ "$qtype" =~ "p" ]]; then
+		! is_project_quota_supported && return 0
+		wait_reintegration "ost" "project" $max || return 1
+	fi
 	return 0
 }
 
+disable_project_quota() {
+	is_project_quota_supported || return 0
+	[ "$(facet_fstype $SINGLEMDS)" != "ldiskfs" ] && return 0
+	stopall || error "failed to stopall (1)"
+
+	for num in $(seq $MDSCOUNT); do
+		do_facet mds$num $TUNE2FS -Q ^prj $(mdsdevname $num) ||
+			error "tune2fs $(mdsdevname $num) failed"
+	done
+
+	for num in $(seq $OSTCOUNT); do
+		do_facet ost$num $TUNE2FS -Q ^prj $(ostdevname $num) ||
+			error "tune2fs $(ostdevname $num) failed"
+	done
+
+	mount
+	setupall
+}
+
 setup_quota_test() {
-	rm -rf $DIR/$tdir
 	wait_delete_completed
 	echo "Creating test directory"
-	mkdir -p $DIR/$tdir
-	chmod 0777 $DIR/$tdir
+	mkdir $DIR/$tdir || return 1
+	chmod 0777 $DIR/$tdir || return 2
 	# always clear fail_loc in case of fail_loc isn't cleared
 	# properly when previous test failed
 	lustre_fail mds_ost 0
@@ -317,45 +389,74 @@ cleanup_quota_test() {
 
 quota_show_check() {
 	local bf=$1
-	local ug=$2
+	local ugp=$2
 	local qid=$3
 	local usage
 
-	$LFS quota -v -$ug $qid $DIR
+	$LFS quota -v -$ugp $qid $DIR
 
 	if [ "$bf" == "a" -o "$bf" == "b" ]; then
-		usage=$(getquota -$ug $qid global curspace)
+		usage=$(getquota -$ugp $qid global curspace)
 		if [ -z $usage ]; then
-			quota_error $ug $qid \
-				"Query block quota failed ($ug:$qid)."
+			quota_error $ugp $qid \
+				"Query block quota failed ($ugp:$qid)."
 		else
-			[ $usage -ne 0 ] && quota_log $ug $qid \
-				"Block quota isn't 0 ($ug:$qid:$usage)."
+			[ $usage -ne 0 ] && quota_log $ugp $qid \
+				"Block quota isn't 0 ($ugp:$qid:$usage)."
 		fi
 	fi
 
 	if [ "$bf" == "a" -o "$bf" == "f" ]; then
-		usage=$(getquota -$ug $qid global curinodes)
+		usage=$(getquota -$ugp $qid global curinodes)
 		if [ -z $usage ]; then
-			quota_error $ug $qid \
-				"Query file quota failed ($ug:$qid)."
+			quota_error $ugp $qid \
+				"Query file quota failed ($ugp:$qid)."
 		else
-			[ $usage -ne 0 ] && quota_log $ug $qid \
-				"File quota isn't 0 ($ug:$qid:$usage)."
+			[ $usage -ne 0 ] && quota_log $ugp $qid \
+				"File quota isn't 0 ($ugp:$qid:$usage)."
 		fi
 	fi
 }
 
+enable_project_quota() {
+	is_project_quota_supported || return 0
+	[ "$(facet_fstype $SINGLEMDS)" != "ldiskfs" ] && return 0
+	stopall || error "failed to stopall (1)"
+
+	for num in $(seq $MDSCOUNT); do
+		do_facet mds$num $TUNE2FS -O project $(mdsdevname $num) ||
+			error "tune2fs $(mdsdevname $num) failed"
+	done
+
+	for num in $(seq $OSTCOUNT); do
+		do_facet ost$num $TUNE2FS -O project $(ostdevname $num) ||
+			error "tune2fs $(ostdevname $num) failed"
+	done
+
+	mount
+	setupall
+}
+enable_project_quota
+
+reset_quota_settings() {
+	resetquota -u $TSTUSR
+	resetquota -g $TSTUSR
+	resetquota -u $TSTUSR2
+	resetquota -g $TSTUSR2
+	resetquota -p $TSTPRJID
+}
+
 # enable quota debug
 quota_init() {
 	do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=+quota"
 }
 quota_init
+reset_quota_settings
 
-resetquota -u $TSTUSR
-resetquota -g $TSTUSR
-resetquota -u $TSTUSR2
-resetquota -g $TSTUSR2
+check_runas_id_ret $TSTUSR $TSTUSR $RUNAS ||
+	error "Please create user $TSTUSR($TSTID) and group $TSTUSR($TSTID)"
+check_runas_id_ret $TSTUSR2 $TSTUSR2 $RUNAS2 ||
+	error "Please create user $TSTUSR2($TSTID2) and group $TSTUSR2($TSTID2)"
 
 test_quota_performance() {
 	local TESTFILE="$DIR/$tdir/$tfile-0"
@@ -367,8 +468,14 @@ test_quota_performance() {
 	delta=$((etime - stime))
 	if [ $delta -gt 0 ]; then
 	    rate=$((size * 1024 / delta))
-	    [ $rate -gt 1024 ] ||
-		error "SLOW IO for $TSTUSR (user): $rate KB/sec"
+	    if [ $(facet_fstype $SINGLEMDS) = "zfs" ]; then
+		# LU-2872 - see LU-2887 for fix
+		[ $rate -gt 64 ] ||
+			error "SLOW IO for $TSTUSR (user): $rate KB/sec"
+	    else
+		[ $rate -gt 1024 ] ||
+			error "SLOW IO for $TSTUSR (user): $rate KB/sec"
+	    fi
 	fi
 	rm -f $TESTFILE
 }
@@ -381,14 +488,14 @@ test_0() {
 	local free_space=$(lfs_df | grep "summary" | awk '{print $4}')
 	[ $free_space -le $((MB * 1024)) ] &&
 		skip "not enough space ${free_space} KB, " \
-			"required $((MB * 1024)) KB" && return
-	setup_quota_test
+			"required $((MB * 1024)) KB"
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	set_ost_qtype "none" || error "disable ost quota failed"
 	test_quota_performance $MB
 
-	set_ost_qtype "ug" || error "enable ost quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
 	$LFS setquota -u $TSTUSR -b 0 -B 10G -i 0 -I 0 $DIR ||
 		error "set quota failed"
 	test_quota_performance $MB
@@ -403,11 +510,11 @@ test_1() {
 	local LIMIT=10  # 10M
 	local TESTFILE="$DIR/$tdir/$tfile-0"
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	# enable ost quota
-	set_ost_qtype "ug" || "enable ost quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
 
 	# test for user
 	log "User quota (block hardlimit:$LIMIT MB)"
@@ -418,8 +525,8 @@ test_1() {
 	local USED=$(getquota -u $TSTUSR global curspace)
 	[ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0."
 
-	$LFS setstripe $TESTFILE -c 1
-	chown $TSTUSR.$TSTUSR $TESTFILE
+	$SETSTRIPE $TESTFILE -c 1 || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
 
 	log "Write..."
 	$RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) ||
@@ -429,11 +536,12 @@ test_1() {
 	$RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) seek=$((LIMIT/2)) || true
 	# flush cache, ensure noquota flag is set on client
 	cancel_lru_locks osc
+	sync; sync_all_data || true
 	$RUNAS $DD of=$TESTFILE count=1 seek=$LIMIT &&
 		quota_error u $TSTUSR "user write success, but expect EDQUOT"
 
 	rm -f $TESTFILE
-	wait_delete_completed
+	wait_delete_completed || error "wait_delete_completed failed"
 	sync_all_data || true
 	USED=$(getquota -u $TSTUSR global curspace)
 	[ $USED -ne 0 ] && quota_error u $TSTUSR \
@@ -449,29 +557,70 @@ test_1() {
 	TESTFILE="$DIR/$tdir/$tfile-1"
 	# make sure the system is clean
 	USED=$(getquota -g $TSTUSR global curspace)
-	[ $USED -ne 0 ] && error "Used space($USED) for group $TSTUSR isn't 0"
+	[ $USED -ne 0 ] && error "Used space ($USED) for group $TSTUSR isn't 0"
 
-	$LFS setstripe $TESTFILE -c 1
-	chown $TSTUSR.$TSTUSR $TESTFILE
+	$SETSTRIPE $TESTFILE -c 1 || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
 
 	log "Write ..."
 	$RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) ||
-		quota_error g $TSTUSR "group write failure, but expect success"
+		quota_error g $TSTUSR "Group write failure, but expect success"
 	log "Write out of block quota ..."
 	# this time maybe cache write, ignore it's failure
 	$RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) seek=$((LIMIT/2)) || true
 	cancel_lru_locks osc
+	sync; sync_all_data || true
 	$RUNAS $DD of=$TESTFILE count=10 seek=$LIMIT &&
-		quota_error g $TSTUSR "group write success, but expect EDQUOT"
+		quota_error g $TSTUSR "Group write success, but expect EDQUOT"
+	rm -f $TESTFILE
+	wait_delete_completed || error "wait_delete_completed failed"
+	sync_all_data || true
+	USED=$(getquota -g $TSTUSR global curspace)
+	[ $USED -ne 0 ] && quota_error g $TSTUSR \
+				"Group quota isn't released after deletion"
+	resetquota -g $TSTUSR
+
+	if ! is_project_quota_supported; then
+		echo "Project quota is not supported"
+		cleanup_quota_test
+		return 0
+	fi
+
+	TESTFILE="$DIR/$tdir/$tfile-2"
+	# make sure the system is clean
+	USED=$(getquota -p $TSTPRJID global curspace)
+	[ $USED -ne 0 ] &&
+		error "used space($USED) for project $TSTPRJID isn't 0"
+
+	# test for Project
+	log "--------------------------------------"
+	log "Project quota (block hardlimit:$LIMIT mb)"
+	$LFS setquota -p $TSTPRJID -b 0 -B ${LIMIT}M -i 0 -I 0 $DIR ||
+		error "set project quota failed"
+
+	$SETSTRIPE $TESTFILE -c 1 || error "setstripe $TESTFILE failed"
+	chown $TSTUSR:$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
+	change_project -p $TSTPRJID $TESTFILE
+
+	log "write ..."
+	$RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) || quota_error p $TSTPRJID \
+		"project write failure, but expect success"
+	log "write out of block quota ..."
+	# this time maybe cache write, ignore it's failure
+	$RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) seek=$((LIMIT/2)) || true
+	cancel_lru_locks osc
+	sync; sync_all_data || true
+	$RUNAS $DD of=$TESTFILE count=10 seek=$LIMIT && quota_error p \
+		$TSTPRJID "project write success, but expect EDQUOT"
 
 	# cleanup
 	cleanup_quota_test
 
-	USED=$(getquota -g $TSTUSR global curspace)
-	[ $USED -ne 0 ] && quota_error g $TSTUSR \
-		"group quota isn't released after deletion"
+	USED=$(getquota -p $TSTPRJID global curspace)
+	[ $USED -ne 0 ] && quota_error p $TSTPRJID \
+		"project quota isn't released after deletion"
 
-	resetquota -g $TSTUSR
+	resetquota -p $TSTPRJID
 }
 run_test 1 "Block hard limit (normal use and out of quota)"
 
@@ -482,16 +631,16 @@ test_2() {
 
 	[ "$SLOW" = "no" ] && LIMIT=1024 # 1k inodes
 
-	local FREE_INODES=$(lfs_df -i | grep "summary" | awk '{print $4}')
+	local FREE_INODES=$(mdt_free_inodes 0)
+	echo "$FREE_INODES free inodes on master MDT"
 	[ $FREE_INODES -lt $LIMIT ] &&
-		skip "not enough free inodes $FREE_INODES required $LIMIT" &&
-		return
+		skip "not enough free inodes $FREE_INODES required $LIMIT"
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	# enable mdt quota
-	set_mdt_qtype "ug" || "enable mdt quota failed"
+	set_mdt_qtype $QTYPE || error "enable mdt quota failed"
 
 	# test for user
 	log "User quota (inode hardlimit:$LIMIT files)"
@@ -503,14 +652,14 @@ test_2() {
 	[ $USED -ne 0 ] && error "Used inodes($USED) for user $TSTUSR isn't 0."
 
 	log "Create $LIMIT files ..."
-	$RUNAS createmany -m ${TESTFILE} $LIMIT || \
+	$RUNAS createmany -m ${TESTFILE} $LIMIT ||
 		quota_error u $TSTUSR "user create failure, but expect success"
 	log "Create out of file quota ..."
-	$RUNAS touch ${TESTFILE}_xxx && \
+	$RUNAS touch ${TESTFILE}_xxx &&
 		quota_error u $TSTUSR "user create success, but expect EDQUOT"
 
 	# cleanup
-	unlinkmany ${TESTFILE} $LIMIT
+	unlinkmany ${TESTFILE} $LIMIT || error "unlinkmany $TESTFILE failed"
 	rm -f ${TESTFILE}_xxx
 	wait_delete_completed
 
@@ -525,7 +674,7 @@ test_2() {
 	$LFS setquota -g $TSTUSR -b 0 -B 0 -i 0 -I $LIMIT $DIR ||
 		error "set group quota failed"
 
-        TESTFILE=$DIR/$tdir/$tfile-1
+	TESTFILE=$DIR/$tdir/$tfile-1
 	# make sure the system is clean
 	USED=$(getquota -g $TSTUSR global curinodes)
 	[ $USED -ne 0 ] && error "Used inodes($USED) for group $TSTUSR isn't 0."
@@ -534,11 +683,11 @@ test_2() {
 	$RUNAS createmany -m ${TESTFILE} $LIMIT ||
 		quota_error g $TSTUSR "group create failure, but expect success"
 	log "Create out of file quota ..."
-	$RUNAS touch ${TESTFILE}_xxx && \
+	$RUNAS touch ${TESTFILE}_xxx &&
 		quota_error g $TSTUSR "group create success, but expect EDQUOT"
 
 	# cleanup
-	unlinkmany ${TESTFILE} $LIMIT
+	unlinkmany ${TESTFILE} $LIMIT || error "unlinkmany $TESTFILE failed"
 	rm -f ${TESTFILE}_xxx
 	wait_delete_completed
 
@@ -546,8 +695,38 @@ test_2() {
 	[ $USED -ne 0 ] && quota_error g $TSTUSR \
 		"user quota isn't released after deletion"
 
-	cleanup_quota_test
 	resetquota -g $TSTUSR
+	! is_project_quota_supported && cleanup_quota_test &&
+		echo "Skip project quota is not supported" && return 0
+
+	# test for project
+	log "--------------------------------------"
+	log "Project quota (inode hardlimit:$LIMIT files)"
+	$LFS setquota -p $TSTPRJID -b 0 -B 0 -i 0 -I $LIMIT $DIR ||
+		error "set project quota failed"
+
+	TESTFILE=$DIR/$tdir/$tfile-1
+	# make sure the system is clean
+	USED=$(getquota -p $TSTPRJID global curinodes)
+	[ $USED -ne 0 ] &&
+		error "Used inodes($USED) for project $TSTPRJID isn't 0"
+
+	change_project -sp $TSTPRJID $DIR/$tdir
+	log "Create $LIMIT files ..."
+	$RUNAS createmany -m ${TESTFILE} $((LIMIT-1)) || quota_error p \
+		$TSTPRJID "project create fail, but expect success"
+	log "Create out of file quota ..."
+	$RUNAS touch ${TESTFILE}_xxx && quota_error p $TSTPRJID \
+		"project create success, but expect EDQUOT"
+	change_project -C $DIR/$tdir
+
+	cleanup_quota_test
+	USED=$(getquota -p $TSTPRJID global curinodes)
+	[ $USED -ne 0 ] && quota_error p $TSTPRJID \
+		"project quota isn't released after deletion"
+
+	resetquota -p $TSTPRJID
+
 }
 run_test 2 "File hard limit (normal use and out of quota)"
 
@@ -556,12 +735,15 @@ test_block_soft() {
 	local TIMER=$(($2 * 3 / 2))
 	local LIMIT=$3
 	local OFFSET=0
+	local qtype=$4
 
 	setup_quota_test
 	trap cleanup_quota_test EXIT
 
-	$LFS setstripe $TESTFILE -c 1 -i 0
+	$SETSTRIPE $TESTFILE -c 1 -i 0
 	chown $TSTUSR.$TSTUSR $TESTFILE
+	[ "$qtype" == "p" ] && is_project_quota_supported &&
+		change_project -p $TSTPRJID $TESTFILE
 
 	echo "Write up to soft limit"
 	$RUNAS $DD of=$TESTFILE count=$LIMIT ||
@@ -577,8 +759,10 @@ test_block_soft() {
 
 	$SHOW_QUOTA_USER
 	$SHOW_QUOTA_GROUP
+	$SHOW_QUOTA_PROJID
 	$SHOW_QUOTA_INFO_USER
 	$SHOW_QUOTA_INFO_GROUP
+	$SHOW_QUOTA_INFO_PROJID
 
 	echo "Write before timer goes off"
 	$RUNAS dd if=/dev/zero of=$TESTFILE bs=1K count=10 seek=$OFFSET ||
@@ -591,8 +775,10 @@ test_block_soft() {
 
 	$SHOW_QUOTA_USER
 	$SHOW_QUOTA_GROUP
+	$SHOW_QUOTA_PROJID
 	$SHOW_QUOTA_INFO_USER
 	$SHOW_QUOTA_INFO_GROUP
+	$SHOW_QUOTA_INFO_PROJID
 
 	echo "Write after timer goes off"
 	# maybe cache write, ignore.
@@ -604,8 +790,10 @@ test_block_soft() {
 
 	$SHOW_QUOTA_USER
 	$SHOW_QUOTA_GROUP
+	$SHOW_QUOTA_PROJID
 	$SHOW_QUOTA_INFO_USER
 	$SHOW_QUOTA_INFO_GROUP
+	$SHOW_QUOTA_INFO_PROJID
 
 	echo "Unlink file to stop timer"
 	rm -f $TESTFILE
@@ -614,11 +802,14 @@ test_block_soft() {
 
 	$SHOW_QUOTA_USER
 	$SHOW_QUOTA_GROUP
+	$SHOW_QUOTA_PROJID
 	$SHOW_QUOTA_INFO_USER
 	$SHOW_QUOTA_INFO_GROUP
+	$SHOW_QUOTA_INFO_PROJID
 
-	$LFS setstripe $TESTFILE -c 1 -i 0
+	$SETSTRIPE $TESTFILE -c 1 -i 0
 	chown $TSTUSR.$TSTUSR $TESTFILE
+	[ "$qtype" == "p" ] && change_project -p $TSTPRJID $TESTFILE
 
 	echo "Write ..."
 	$RUNAS $DD of=$TESTFILE count=$LIMIT ||
@@ -633,7 +824,7 @@ test_3() {
 	local GRACE=20 # 20s
 	local TESTFILE=$DIR/$tdir/$tfile-0
 
-	set_ost_qtype "ug" || error "enable ost quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
 
 	echo "User quota (soft limit:$LIMIT MB  grace:$GRACE seconds)"
 	# make sure the system is clean
@@ -645,7 +836,7 @@ test_3() {
 	$LFS setquota -u $TSTUSR -b ${LIMIT}M -B 0 -i 0 -I 0 $DIR ||
 		error "set user quota failed"
 
-	test_block_soft $TESTFILE $GRACE $LIMIT
+	test_block_soft $TESTFILE $GRACE $LIMIT "u"
 	resetquota -u $TSTUSR
 
 	echo "Group quota (soft limit:$LIMIT MB  grace:$GRACE seconds)"
@@ -659,9 +850,30 @@ test_3() {
 	$LFS setquota -g $TSTUSR -b ${LIMIT}M -B 0 -i 0 -I 0 $DIR ||
 		error "set group quota failed"
 
-	test_block_soft $TESTFILE $GRACE $LIMIT
+	test_block_soft $TESTFILE $GRACE $LIMIT "g"
 	resetquota -g $TSTUSR
 
+	if is_project_quota_supported; then
+		echo "Project quota (soft limit:$LIMIT MB  grace:$GRACE sec)"
+		TESTFILE=$DIR/$tdir/$tfile-2
+		# make sure the system is clean
+		USED=$(getquota -p $TSTPRJID global curspace)
+		[ $USED -ne 0 ] && error \
+			"Used space($USED) for project $TSTPRJID isn't 0."
+
+		$LFS setquota -t -p --block-grace $GRACE --inode-grace \
+			$MAX_IQ_TIME $DIR ||
+				error "set project grace time failed"
+		$LFS setquota -p $TSTPRJID -b ${LIMIT}M -B 0 -i 0 -I 0 \
+			$DIR || error "set project quota failed"
+
+		test_block_soft $TESTFILE $GRACE $LIMIT "p"
+		resetquota -p $TSTPRJID
+		$LFS setquota -t -p --block-grace $MAX_DQ_TIME --inode-grace \
+			$MAX_IQ_TIME $DIR ||
+				error "restore project grace time failed"
+	fi
+
 	# cleanup
 	$LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace \
 		$MAX_IQ_TIME $DIR || error "restore user grace time failed"
@@ -673,45 +885,57 @@ run_test 3 "Block soft limit (start timer, timer goes off, stop timer)"
 test_file_soft() {
 	local TESTFILE=$1
 	local LIMIT=$2
-	local TIMER=$(($3 * 3 / 2))
+	local grace=$3
+	local TIMER=$(($grace * 3 / 2))
 
 	setup_quota_test
 	trap cleanup_quota_test EXIT
+	is_project_quota_supported && change_project -sp $TSTPRJID $DIR/$tdir
 
 	echo "Create files to exceed soft limit"
 	$RUNAS createmany -m ${TESTFILE}_ $((LIMIT + 1)) ||
 		quota_error a $TSTUSR "create failure, but expect success"
-	sync; sleep 1; sync
+	local trigger_time=$(date +%s)
+
+	sync_all_data || true
+
+	local cur_time=$(date +%s)
+	[ $(($cur_time - $trigger_time)) -ge $grace ] &&
+		error "Passed grace time $grace, $trigger_time, $cur_time"
 
 	echo "Create file before timer goes off"
 	$RUNAS touch ${TESTFILE}_before ||
 		quota_error a $TSTUSR "failed create before timer expired," \
-			"but expect success"
-	sync; sleep 1; sync
+			"but expect success. $trigger_time, $cur_time"
+	sync_all_data || true
 
 	echo "Sleep $TIMER seconds ..."
 	sleep $TIMER
 
 	$SHOW_QUOTA_USER
 	$SHOW_QUOTA_GROUP
+	$SHOW_QUOTA_PROJID
 	$SHOW_QUOTA_INFO_USER
 	$SHOW_QUOTA_INFO_GROUP
+	$SHOW_QUOTA_INFO_PROJID
 
 	echo "Create file after timer goes off"
 	# There is a window that space is accounted in the quota usage but
 	# hasn't been decreased from the pending write, if we acquire quota
 	# in this window, we'll acquire more than we needed.
 	$RUNAS touch ${TESTFILE}_after_1 ${TESTFILE}_after_2 || true
-	sync; sleep 1; sync
+	sync_all_data || true
 	$RUNAS touch ${TESTFILE}_after_3 &&
 		quota_error a $TSTUSR "create after timer expired," \
 			"but expect EDQUOT"
-	sync; sleep 1; sync
+	sync_all_data || true
 
 	$SHOW_QUOTA_USER
 	$SHOW_QUOTA_GROUP
+	$SHOW_QUOTA_PROJID
 	$SHOW_QUOTA_INFO_USER
 	$SHOW_QUOTA_INFO_GROUP
+	$SHOW_QUOTA_INFO_PROJID
 
 	echo "Unlink files to stop timer"
 	find $(dirname $TESTFILE) -name "$(basename ${TESTFILE})*" | xargs rm -f
@@ -721,7 +945,7 @@ test_file_soft() {
 	$RUNAS touch ${TESTFILE}_xxx ||
 		quota_error a $TSTUSR "touch after timer stop failure," \
 			"but expect success"
-	sync; sleep 1; sync
+	sync_all_data || true
 
 	# cleanup
 	cleanup_quota_test
@@ -731,9 +955,9 @@ test_file_soft() {
 test_4a() {
 	local LIMIT=10 # inodes
 	local TESTFILE=$DIR/$tdir/$tfile-0
-	local GRACE=5
+	local GRACE=12
 
-	set_mdt_qtype "ug" || error "enable mdt quota failed"
+	set_mdt_qtype $QTYPE || error "enable mdt quota failed"
 
 	echo "User quota (soft limit:$LIMIT files  grace:$GRACE seconds)"
 	# make sure the system is clean
@@ -762,6 +986,27 @@ test_4a() {
 	test_file_soft $TESTFILE $LIMIT $GRACE
 	resetquota -g $TSTUSR
 
+	if is_project_quota_supported; then
+		echo "Project quota (soft limit:$LIMIT files grace:$GRACE sec)"
+		# make sure the system is clean
+		USED=$(getquota -p $TSTPRJID global curinodes)
+		[ $USED -ne 0 ] && error \
+			"Used space($USED) for project $TSTPRJID isn't 0."
+
+		$LFS setquota -t -p --block-grace $MAX_DQ_TIME --inode-grace \
+			$GRACE $DIR || error "set project grace time failed"
+		$LFS setquota -p $TSTPRJID -b 0 -B 0 -i $LIMIT -I 0 $DIR ||
+			error "set project quota failed"
+
+		TESTFILE=$DIR/$tdir/$tfile-1
+		# one less than limit, because of parent directory included.
+		test_file_soft $TESTFILE $((LIMIT-1)) $GRACE
+		resetquota -p $TSTPRJID
+		$LFS setquota -t -p --block-grace $MAX_DQ_TIME --inode-grace \
+			$MAX_IQ_TIME $DIR ||
+				error "restore project grace time failed"
+	fi
+
 	# cleanup
 	$LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace \
 		$MAX_IQ_TIME $DIR || error "restore user grace time failed"
@@ -776,7 +1021,8 @@ test_4b() {
 	local GR_STR3="5s"
 	local GR_STR4="1w2d3h4m5s"
 	local GR_STR5="5c"
-	local GR_STR6="1111111111111111"
+	local GR_STR6="18446744073709551615"
+	local GR_STR7="-1"
 
 	wait_delete_completed
 
@@ -793,6 +1039,8 @@ test_4b() {
 	echo "  Invalid grace strings test"
 	! $LFS setquota -t -u --block-grace $GR_STR4 --inode-grace $GR_STR5 $DIR
 	! $LFS setquota -t -g --block-grace $GR_STR4 --inode-grace $GR_STR6 $DIR
+	! $LFS setquota -t -g --block-grace $GR_STR4 --inode-grace \
+		$GR_STR7 $DIR
 
 	# cleanup
 	$LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace \
@@ -807,17 +1055,21 @@ test_5() {
 	local BLIMIT=10 # 10M
 	local ILIMIT=10 # 10 inodes
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
-	set_mdt_qtype "ug" || error "enable mdt quota failed"
-	set_ost_qtype "ug" || error "enable ost quota failed"
+	set_mdt_qtype $QTYPE || error "enable mdt quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
 
 	echo "Set quota limit (0 ${BLIMIT}M 0 $ILIMIT) for $TSTUSR.$TSTUSR"
 	$LFS setquota -u $TSTUSR -b 0 -B ${BLIMIT}M -i 0 -I $ILIMIT $DIR ||
 		error "set user quota failed"
 	$LFS setquota -g $TSTUSR -b 0 -B ${BLIMIT}M -i 0 -I $ILIMIT $DIR ||
+	if is_project_quota_supported; then
 		error "set group quota failed"
+		$LFS setquota -p $TSTPRJID -b 0 -B ${BLIMIT}M -i 0 \
+			-I $ILIMIT $DIR || error "set project quota failed"
+	fi
 
 	# make sure the system is clean
 	local USED=$(getquota -u $TSTUSR global curinodes)
@@ -828,25 +1080,39 @@ test_5() {
 	[ $USED -ne 0 ] && error "Used block($USED) for user $TSTUSR isn't 0."
 	USED=$(getquota -g $TSTUSR global curspace)
 	[ $USED -ne 0 ] && error "Used block($USED) for group $TSTUSR isn't 0."
+	if is_project_quota_supported; then
+		USED=$(getquota -p $TSTPRJID global curinodes)
+		[ $USED -ne 0 ] &&
+			error "Used inode($USED) for project $TSTPRJID isn't 0."
+		USED=$(getquota -p $TSTPRJID global curspace)
+		[ $USED -ne 0 ] &&
+			error "Used block($USED) for project $TSTPRJID isn't 0."
+	fi
 
 	echo "Create more than $ILIMIT files and more than $BLIMIT MB ..."
 	createmany -m $DIR/$tdir/$tfile-0_ $((ILIMIT + 1)) ||
 		error "create failure, expect success"
+	if is_project_quota_supported; then
+		touch $DIR/$tdir/$tfile-0_1
+		change_project -p $TSTPRJID $DIR/$tdir/$tfile-0_1
+	fi
 	$DD of=$DIR/$tdir/$tfile-0_1 count=$((BLIMIT+1)) ||
 		error "write failure, expect success"
 
 	echo "Chown files to $TSTUSR.$TSTUSR ..."
-	for i in `seq 0 $ILIMIT`; do
+	for i in $(seq 0 $ILIMIT); do
 		chown $TSTUSR.$TSTUSR $DIR/$tdir/$tfile-0_$i ||
 			quota_error a $TSTUSR "chown failure, expect success"
 	done
 
 	# cleanup
-	unlinkmany $DIR/$tdir/$tfile-0_ $((ILIMIT + 1))
+	unlinkmany $DIR/$tdir/$tfile-0_ $((ILIMIT + 1)) ||
+		error "unlinkmany $DIR/$tdir/$tfile-0_ failed"
 	cleanup_quota_test
 
 	resetquota -u $TSTUSR
 	resetquota -g $TSTUSR
+	resetquota -p $TSTPRJID
 }
 run_test 5 "Chown & chgrp successfully even out of block/file quota"
 
@@ -854,7 +1120,11 @@ run_test 5 "Chown & chgrp successfully even out of block/file quota"
 test_6() {
 	local LIMIT=3 # 3M
 
-	setup_quota_test
+	# Clear dmesg so watchdog is not triggered by previous
+	# test output
+	do_facet ost1 dmesg -c > /dev/null
+
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	# make sure the system is clean
@@ -862,18 +1132,18 @@ test_6() {
 	[ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0."
 
 	# make sure no granted quota on ost
-	set_ost_qtype "ug" || error "enable ost quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
 	resetquota -u $TSTUSR
 
 	# create file for $TSTUSR
 	local TESTFILE=$DIR/$tdir/$tfile-$TSTUSR
-	$LFS setstripe $TESTFILE -c 1 -i 0
-	chown $TSTUSR.$TSTUSR $TESTFILE
+	$SETSTRIPE $TESTFILE -c 1 -i 0 || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
 
 	# create file for $TSTUSR2
 	local TESTFILE2=$DIR/$tdir/$tfile-$TSTUSR2
-	$LFS setstripe $TESTFILE2 -c 1 -i 0
-	chown $TSTUSR2.$TSTUSR2 $TESTFILE2
+	$SETSTRIPE $TESTFILE2 -c 1 -i 0 || error "setstripe $TESTFILE2 failed"
+	chown $TSTUSR2.$TSTUSR2 $TESTFILE2 || error "chown $TESTFILE2 failed"
 
 	# cache per-ID lock for $TSTUSR on slave
 	$LFS setquota -u $TSTUSR -b 0 -B ${LIMIT}M -i 0 -I 0 $DIR ||
@@ -912,19 +1182,14 @@ test_6() {
 
 	# no watchdog is triggered
 	do_facet ost1 dmesg > $TMP/lustre-log-${TESTNAME}.log
-	watchdog=$(awk '/sanity-quota test 6/ {start = 1;}
-	               /Service thread pid/ && /was inactive/ {
-			       if (start) {
-				       print;
-			       }
-		       }' $TMP/lustre-log-${TESTNAME}.log)
+	watchdog=$(awk '/Service thread pid/ && /was inactive/ \
+			{ print; }' $TMP/lustre-log-${TESTNAME}.log)
 	[ -z "$watchdog" ] || error "$watchdog"
 
 	rm -f $TMP/lustre-log-${TESTNAME}.log
 
-	# write should continue & succeed
+	# write should continue then fail with EDQUOT
 	local count=0
-	local o_size=$(stat -c %s $TESTFILE)
 	local c_size
 	while [ true ]; do
 		if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi
@@ -934,13 +1199,8 @@ test_6() {
 		count=$((count + 1))
 		if [ $((count % 30)) -eq 0 ]; then
 			c_size=$(stat -c %s $TESTFILE)
-			if [ $c_size -eq $o_size ]; then
-				quota_error u $TSTUSR "file not growed" \
-				"in 30 seconds $o_size/$c_size"
-			else
-				echo "Waiting $count secs. $o_size/$c_size"
-				o_size=$c_size
-			fi
+			echo "Waiting $count secs. $c_size"
+			$SHOW_QUOTA_USER
 		fi
 		sleep 1
 	done
@@ -957,7 +1217,7 @@ test_7a() {
 
 	[ "$SLOW" = "no" ] && LIMIT=5
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	# make sure the system is clean
@@ -965,32 +1225,32 @@ test_7a() {
 	[ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0."
 
 	# make sure no granted quota on ost1
-	set_ost_qtype "ug" || error "enable ost quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
 	resetquota -u $TSTUSR
 	set_ost_qtype "none" || error "disable ost quota failed"
 
 	local OSTUUID=$(ostuuid_from_index 0)
 	USED=$(getquota -u $TSTUSR $OSTUUID bhardlimit)
-	[ $USED -ne 0 ] && error "limit($USED) on $OSTUUID for user" \
-		"$TSTUSR isn't 0."
+	[ $USED -ne 0 ] &&
+		error "limit($USED) on $OSTUUID for user $TSTUSR isn't 0"
 
 	# create test file
-	$LFS setstripe $TESTFILE -c 1 -i 0
-	chown $TSTUSR.$TSTUSR $TESTFILE
+	$SETSTRIPE $TESTFILE -c 1 -i 0 || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
 
 	echo "Stop ost1..."
 	stop ost1
 
 	echo "Enable quota & set quota limit for $TSTUSR"
-	set_ost_qtype "ug" || error "enable ost quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
 	$LFS setquota -u $TSTUSR -b 0 -B ${LIMIT}M -i 0 -I 0 $DIR ||
 		error "set quota failed"
 
 	echo "Start ost1..."
-	start ost1 $(ostdevname 1) $OST_MOUNT_OPTS
+	start ost1 $(ostdevname 1) $OST_MOUNT_OPTS || error "start ost1 failed"
 	quota_init
 
-	wait_ost_reint "ug" || error "reintegration failed"
+	wait_ost_reint $QTYPE || error "reintegration failed"
 
 	# hardlimit should have been fetched by slave during global
 	# reintegration, write will exceed quota
@@ -1009,10 +1269,10 @@ test_7a() {
 		error "clear quota failed"
 
 	echo "Start ost1..."
-	start ost1 $(ostdevname 1) $OST_MOUNT_OPTS
+	start ost1 $(ostdevname 1) $OST_MOUNT_OPTS || error "start ost1 failed"
 	quota_init
 
-	wait_ost_reint "ug" || error "reintegration failed"
+	wait_ost_reint $QTYPE || error "reintegration failed"
 
 	# hardlimit should be cleared on slave during reintegration
 	$RUNAS $DD of=$TESTFILE count=$((LIMIT + 1)) oflag=sync ||
@@ -1028,7 +1288,7 @@ test_7b() {
 	local LIMIT="100G"
 	local TESTFILE=$DIR/$tdir/$tfile
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	# make sure the system is clean
@@ -1036,18 +1296,18 @@ test_7b() {
 	[ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0."
 
 	# make sure no granted quota on ost1
-	set_ost_qtype "ug" || error "enable ost quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
 	resetquota -u $TSTUSR
 	set_ost_qtype "none" || error "disable ost quota failed"
 
 	local OSTUUID=$(ostuuid_from_index 0)
 	USED=$(getquota -u $TSTUSR $OSTUUID bhardlimit)
-	[ $USED -ne 0 ] && error "limit($USED) on $OSTUUID for user" \
-		"$TSTUSR isn't 0."
+	[ $USED -ne 0 ] &&
+		error "limit($USED) on $OSTUUID for user $TSTUSR isn't 0"
 
 	# create test file
-	$LFS setstripe $TESTFILE -c 1 -i 0
-	chown $TSTUSR.$TSTUSR $TESTFILE
+	$SETSTRIPE $TESTFILE -c 1 -i 0 || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
 
 	# consume some space to make sure the granted space will not
 	# be released during reconciliation
@@ -1057,7 +1317,7 @@ test_7b() {
 	# define OBD_FAIL_QUOTA_EDQUOT 0xa02
 	lustre_fail mds 0xa02
 
-	set_ost_qtype "ug" || error "enable ost quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
 	$LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR ||
 		error "set quota failed"
 
@@ -1070,10 +1330,10 @@ test_7b() {
 
 	echo "Restart ost to trigger reintegration..."
 	stop ost1
-	start ost1 $(ostdevname 1) $OST_MOUNT_OPTS
+	start ost1 $(ostdevname 1) $OST_MOUNT_OPTS || error "start ost1 failed"
 	quota_init
 
-	wait_ost_reint "ug" || error "reintegration failed"
+	wait_ost_reint $QTYPE || error "reintegration failed"
 
 	USED=$(getquota -u $TSTUSR $OSTUUID bhardlimit)
 	[ $USED -gt $old_used ] || error "limit on $OSTUUID $USED <= $old_used"
@@ -1091,7 +1351,7 @@ test_7c() {
 
 	[ "$SLOW" = "no" ] && LIMIT=5
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	# make sure the system is clean
@@ -1106,11 +1366,12 @@ test_7c() {
 	lustre_fail ost 0xa03
 
 	# enable ost quota
-	set_ost_qtype "ug" || error "enable ost quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
 	# trigger reintegration
 	local procf="osd-$(facet_fstype ost1).$FSNAME-OST*."
 	procf=${procf}quota_slave.force_reint
-	$LCTL set_param $procf=1 || "force reintegration failed"
+	do_facet ost1 $LCTL set_param $procf=1 ||
+		error "force reintegration failed"
 
 	echo "Stop mds..."
 	stop mds1
@@ -1123,7 +1384,7 @@ test_7c() {
 
 	# wait longer than usual to make sure the reintegration
 	# is triggered by quota wb thread.
-	wait_ost_reint "ug" 200 || error "reintegration failed"
+	wait_ost_reint $QTYPE 200 || error "reintegration failed"
 
 	# hardlimit should have been fetched by slave during global
 	# reintegration, write will exceed quota
@@ -1141,7 +1402,7 @@ test_7d(){
 	local TESTFILE1="$DIR/$tdir/$tfile"-1
 	local limit=20 #20M
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	set_ost_qtype "none" || error "disable ost quota failed"
@@ -1175,12 +1436,23 @@ run_test 7d "Quota reintegration (Transfer index in multiple bulks)"
 
 # quota reintegration (inode limits)
 test_7e() {
-	[ "$MDSCOUNT" -lt "2" ] && skip "Required more MDTs" && return
+	[ "$MDSCOUNT" -lt "2" ] && skip "needs >= 2 MDTs"
+
+	# LU-2435: skip this quota test if underlying zfs version has not
+	# supported native dnode accounting
+	[ "$(facet_fstype mds1)" == "zfs" ] && {
+		local F="feature@userobj_accounting"
+		local pool=$(zpool_name mds1)
+		local feature=$(do_facet mds1 $ZPOOL get -H $F $pool)
+
+		[[ "$feature" != *" active "* ]] &&
+			skip "requires zpool with active userobj_accounting"
+	}
 
 	local ilimit=$((1024 * 2)) # 2k inodes
 	local TESTFILE=$DIR/${tdir}-1/$tfile
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	# make sure the system is clean
@@ -1188,7 +1460,7 @@ test_7e() {
 	[ $USED -ne 0 ] && error "Used inode($USED) for user $TSTUSR isn't 0."
 
 	# make sure no granted quota on mdt1
-	set_mdt_qtype "ug" || error "enable mdt quota failed"
+	set_mdt_qtype $QTYPE || error "enable mdt quota failed"
 	resetquota -u $TSTUSR
 	set_mdt_qtype "none" || error "disable mdt quota failed"
 
@@ -1201,15 +1473,15 @@ test_7e() {
 	stop mds${MDSCOUNT}
 
 	echo "Enable quota & set quota limit for $TSTUSR"
-	set_mdt_qtype "ug" || error "enable mdt quota failed"
-	$LFS setquota -u $TSTUSR -b 0 -B 0 -i 0 -I $ilimit  $DIR ||
+	set_mdt_qtype $QTYPE || error "enable mdt quota failed"
+	$LFS setquota -u $TSTUSR -b 0 -B 0 -i 0 -I $ilimit $DIR ||
 		error "set quota failed"
 
 	echo "Start mds${MDSCOUNT}..."
 	start mds${MDSCOUNT} $(mdsdevname $MDSCOUNT) $MDS_MOUNT_OPTS
 	quota_init
 
-	wait_mdt_reint "ug" || error "reintegration failed"
+	wait_mdt_reint $QTYPE || error "reintegration failed"
 
 	echo "create remote dir"
 	$LFS mkdir -i $((MDSCOUNT - 1)) $DIR/${tdir}-1 ||
@@ -1221,7 +1493,7 @@ test_7e() {
 	$RUNAS createmany -m $TESTFILE $((ilimit + 1)) &&
 		quota_error u $TSTUSR "create succeeded, expect EDQUOT"
 
-	$RUNAS unlinkmany $TESTFILE $ilimit || "unlink files failed"
+	$RUNAS unlinkmany $TESTFILE $ilimit || error "unlink files failed"
 	wait_delete_completed
 	sync_all_data || true
 
@@ -1235,14 +1507,14 @@ test_7e() {
 	start mds${MDSCOUNT} $(mdsdevname $MDSCOUNT) $MDS_MOUNT_OPTS
 	quota_init
 
-	wait_mdt_reint "ug" || error "reintegration failed"
+	wait_mdt_reint $QTYPE || error "reintegration failed"
 
 	# hardlimit should be cleared on slave during reintegration
 	$RUNAS createmany -m $TESTFILE $((ilimit + 1)) ||
-		quota_error -u $TSTUSR "create failed, expect success"
+		quota_error u $TSTUSR "create failed, expect success"
 
-	$RUNAS unlinkmany $TESTFILE $((ilimit + 1)) || "unlink failed"
-	$LFS rmdir $DIR/${tdir}-1 || "unlink remote dir failed"
+	$RUNAS unlinkmany $TESTFILE $((ilimit + 1)) || error "unlink failed"
+	rmdir $DIR/${tdir}-1 || error "unlink remote dir failed"
 
 	cleanup_quota_test
 	resetquota -u $TSTUSR
@@ -1254,11 +1526,11 @@ test_8() {
 	local BLK_LIMIT="100g" #100G
 	local FILE_LIMIT=1000000
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
-	set_mdt_qtype "ug" || error "enable mdt quota failed"
-	set_ost_qtype "ug" || error "enable ost quota failed"
+	set_mdt_qtype $QTYPE || error "enable mdt quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
 
 	echo "Set enough high limit for user: $TSTUSR"
 	$LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR ||
@@ -1266,15 +1538,24 @@ test_8() {
 	echo "Set enough high limit for group: $TSTUSR"
 	$LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR ||
 		error "set group quota failed"
+	if is_project_quota_supported; then
+		change_project -sp $TSTPRJID $DIR/$tdir
+		echo "Set enough high limit for project: $TSTPRJID"
+		$LFS setquota -p $TSTPRJID -b 0 \
+			-B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR ||
+			error "set project quota failed"
+	fi
 
 	local duration=""
 	[ "$SLOW" = "no" ] && duration=" -t 120"
 	$RUNAS bash rundbench -D $DIR/$tdir 3 $duration ||
 		quota_error a $TSTUSR "dbench failed!"
 
+	is_project_quota_supported && change_project -C $DIR/$tdir
 	cleanup_quota_test
 	resetquota -u $TSTUSR
 	resetquota -g $TSTUSR
+	resetquota -p $TSTPRJID
 }
 run_test 8 "Run dbench with quota enabled"
 
@@ -1298,7 +1579,7 @@ test_9() {
 
 	check_whether_skip && return 0
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	set_ost_qtype "ug" || error "enable ost quota failed"
@@ -1323,8 +1604,8 @@ test_9() {
 	quota_show_check a g $TSTUSR
 
 	echo "Create test file"
-	$LFS setstripe $TESTFILE -c 1 -i 0
-	chown $TSTUSR.$TSTUSR $TESTFILE
+	$SETSTRIPE $TESTFILE -c 1 -i 0 || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
 
 	log "Write the big file of 4.5G ..."
 	$RUNAS $DD of=$TESTFILE count=$filesize ||
@@ -1345,7 +1626,7 @@ run_test 9 "Block limit larger than 4GB (b10707)"
 test_10() {
 	local TESTFILE=$DIR/$tdir/$tfile
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	# set limit to root user should fail
@@ -1353,16 +1634,18 @@ test_10() {
 		error "set limit for root user successfully, expect failure"
 	$LFS setquota -g root -b 1T -B 10T -i 5K -I 100M $DIR &&
 		error "set limit for root group successfully, expect failure"
+	$LFS setquota -p 0 -b 1T -B 10T -i 5K -I 100M $DIR &&
+		error "set limit for project 0 successfully, expect failure"
 
 	# root user can overrun quota
-	set_ost_qtype "ug" || "enable ost quota failed"
+	set_ost_qtype "ug" || error "enable ost quota failed"
 
 	$LFS setquota -u $TSTUSR -b 0 -B 2M -i 0 -I 0 $DIR ||
 		error "set quota failed"
 	quota_show_check b u $TSTUSR
 
-	$LFS setstripe $TESTFILE -c 1
-	chown $TSTUSR.$TSTUSR $TESTFILE
+	$SETSTRIPE $TESTFILE -c 1 || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
 
 	runas -u 0 -g 0 $DD of=$TESTFILE count=3 oflag=sync ||
 		error "write failure, expect success"
@@ -1374,18 +1657,18 @@ run_test 10 "Test quota for root user"
 
 test_11() {
 	local TESTFILE=$DIR/$tdir/$tfile
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
-	set_mdt_qtype "ug" || "enable mdt quota failed"
+	set_mdt_qtype "ug" || error "enable mdt quota failed"
 	$LFS setquota -u $TSTUSR -b 0 -B 0 -i 0 -I 1 $DIR ||
 		error "set quota failed"
 
-	touch "$TESTFILE"-0
-	touch "$TESTFILE"-1
+	touch "$TESTFILE"-0 || error "touch $TESTFILE-0 failed"
+	touch "$TESTFILE"-1 || error "touch $TESTFILE-0 failed"
 
-	chown $TSTUSR.$TSTUSR "$TESTFILE"-0
-	chown $TSTUSR.$TSTUSR "$TESTFILE"-1
+	chown $TSTUSR.$TSTUSR "$TESTFILE"-0 || error "chown $TESTFILE-0 failed"
+	chown $TSTUSR.$TSTUSR "$TESTFILE"-1 || error "chown $TESTFILE-1 failed"
 
 	$SHOW_QUOTA_USER
 	local USED=$(getquota -u $TSTUSR global curinodes)
@@ -1397,26 +1680,26 @@ test_11() {
 run_test 11 "Chown/chgrp ignores quota"
 
 test_12a() {
-	[ "$OSTCOUNT" -lt "2" ] && skip "skipping rebalancing test" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "needs >= 2 OSTs"
 
 	local blimit=22 # 22M
 	local blk_cnt=$((blimit - 5))
 	local TESTFILE0="$DIR/$tdir/$tfile"-0
 	local TESTFILE1="$DIR/$tdir/$tfile"-1
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
-	set_ost_qtype "u" || "enable ost quota failed"
+	set_ost_qtype "u" || error "enable ost quota failed"
 	quota_show_check b u $TSTUSR
 
 	$LFS setquota -u $TSTUSR -b 0 -B "$blimit"M -i 0 -I 0 $DIR ||
 		error "set quota failed"
 
-	$LFS setstripe $TESTFILE0 -c 1 -i 0
-	$LFS setstripe $TESTFILE1 -c 1 -i 1
-	chown $TSTUSR.$TSTUSR $TESTFILE0
-	chown $TSTUSR.$TSTUSR $TESTFILE1
+	$SETSTRIPE $TESTFILE0 -c 1 -i 0 || error "setstripe $TESTFILE0 failed"
+	$SETSTRIPE $TESTFILE1 -c 1 -i 1 || error "setstripe $TESTFILE1 failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE0 || error "chown $TESTFILE0 failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE1 || error "chown $TESTFILE1 failed"
 
 	echo "Write to ost0..."
 	$RUNAS $DD of=$TESTFILE0 count=$blk_cnt oflag=sync ||
@@ -1441,19 +1724,19 @@ test_12a() {
 run_test 12a "Block quota rebalancing"
 
 test_12b() {
-	[ "$MDSCOUNT" -lt "2" ] && skip "skipping rebalancing test" && return
+	[ "$MDSCOUNT" -lt "2" ] && skip "needs >= 2 MDTs"
 
 	local ilimit=$((1024 * 2)) # 2k inodes
 	local TESTFILE0=$DIR/$tdir/$tfile
 	local TESTFILE1=$DIR/${tdir}-1/$tfile
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	$LFS mkdir -i 1 $DIR/${tdir}-1 || error "create remote dir failed"
 	chmod 0777 $DIR/${tdir}-1
 
-	set_mdt_qtype "u" || "enable mdt quota failed"
+	set_mdt_qtype "u" || error "enable mdt quota failed"
 	quota_show_check f u $TSTUSR
 
 	$LFS setquota -u $TSTUSR -b 0 -B 0 -i 0 -I $ilimit $DIR ||
@@ -1478,7 +1761,7 @@ test_12b() {
 
 	$RUNAS unlinkmany $TESTFILE1 $((ilimit / 2)) ||
 		error "unlink mdt1 files failed"
-	$LFS rmdir $DIR/${tdir}-1 || error "unlink remote dir failed"
+	rmdir $DIR/${tdir}-1 || error "unlink remote dir failed"
 
 	cleanup_quota_test
 	resetquota -u $TSTUSR
@@ -1490,16 +1773,16 @@ test_13(){
 	# the name of lwp on ost1 name is MDT0000-lwp-OST0000
 	local procf="ldlm.namespaces.*MDT0000-lwp-OST0000.lru_size"
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
-	set_ost_qtype "u" || "enable ost quota failed"
+	set_ost_qtype "u" || error "enable ost quota failed"
 	quota_show_check b u $TSTUSR
 
 	$LFS setquota -u $TSTUSR -b 0 -B 10M -i 0 -I 0 $DIR ||
 		error "set quota failed"
-	$LFS setstripe $TESTFILE -c 1 -i 0
-	chown $TSTUSR.$TSTUSR $TESTFILE
+	$SETSTRIPE $TESTFILE -c 1 -i 0 || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
 
 	# clear the locks in cache first
 	do_facet ost1 $LCTL set_param -n $procf=clear
@@ -1540,7 +1823,7 @@ test_15(){
 	wait_delete_completed
 	sync_all_data || true
 
-        # test for user
+	# test for user
 	$LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR ||
 		error "set user quota failed"
 	local TOTAL_LIMIT=$(getquota -u $TSTUSR global bhardlimit)
@@ -1563,7 +1846,7 @@ test_17sub() {
 	local BLKS=1    # 1M less than limit
 	local TESTFILE=$DIR/$tdir/$tfile
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	# make sure the system is clean
@@ -1608,7 +1891,7 @@ test_17sub() {
 	sync; sync_all_data || true
 
 	USED=$(getquota -u $TSTUSR global curspace)
-	[ $USED -ge $(($BLKS * 1024)) ] || quota_error u $TSTUSR \
+	[ $USED -ge $((BLKS * 1024)) ] || quota_error u $TSTUSR \
 		"Used space(${USED}K) is less than ${BLKS}M"
 
 	cleanup_quota_test
@@ -1641,7 +1924,7 @@ test_18_sub () {
 	local blimit="200m" # 200M
 	local TESTFILE="$DIR/$tdir/$tfile"
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	set_ost_qtype "u" || error "enable ost quota failed"
@@ -1650,8 +1933,8 @@ test_18_sub () {
 		error "set quota failed"
 	quota_show_check b u $TSTUSR
 
-	$LFS setstripe $TESTFILE -i 0 -c 1
-	chown $TSTUSR.$TSTUSR $TESTFILE
+	$SETSTRIPE $TESTFILE -i 0 -c 1 || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
 
 	local timeout=$(sysctl -n lustre.timeout)
 
@@ -1691,7 +1974,7 @@ test_18_sub () {
 
 	local testfile_size=$(stat -c %s $TESTFILE)
 	if [ $testfile_size -ne $((BLK_SZ * 1024 * 100)) ] ; then
-		quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100)),"
+		quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100))," \
 			"got ${testfile_size}. Verifying file failed!"
 	fi
 	cleanup_quota_test
@@ -1701,17 +1984,17 @@ test_18_sub () {
 # test when mds does failover, the ost still could work well
 # this test shouldn't trigger watchdog b=14840
 test_18() {
+	# Clear dmesg so watchdog is not triggered by previous
+	# test output
+	do_facet ost1 dmesg -c > /dev/null
+
 	test_18_sub normal
 	test_18_sub directio
 
 	# check if watchdog is triggered
 	do_facet ost1 dmesg > $TMP/lustre-log-${TESTNAME}.log
-	local watchdog=$(awk '/sanity-quota test 18/ {start = 1;}
-	               /Service thread pid/ && /was inactive/ {
-			       if (start) {
-				       print;
-			       }
-		       }' $TMP/lustre-log-${TESTNAME}.log)
+	local watchdog=$(awk '/Service thread pid/ && /was inactive/ \
+			{ print; }' $TMP/lustre-log-${TESTNAME}.log)
 	[ -z "$watchdog" ] || error "$watchdog"
 	rm -f $TMP/lustre-log-${TESTNAME}.log
 }
@@ -1721,16 +2004,16 @@ test_19() {
 	local blimit=5 # 5M
 	local TESTFILE=$DIR/$tdir/$tfile
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
-	set_ost_qtype "ug" || error "enable ost quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
 
 	# bind file to a single OST
-	$LFS setstripe -c 1 $TESTFILE
-	chown $TSTUSR.$TSTUSR $TESTFILE
+	$SETSTRIPE -c 1 $TESTFILE || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
 
-	echo "Set user quota (limit: "$blimit"M)"
+	echo "Set user quota (limit: ${blimit}M)"
 	$LFS setquota -u $TSTUSR -b 0 -B "$blimit"M -i 0 -I 0 $MOUNT ||
 		error "set user quota failed"
 	quota_show_check b u $TSTUSR
@@ -1755,7 +2038,8 @@ run_test 19 "Updating admin limits doesn't zero operational limits(b14790)"
 test_20() { # b15754
 	local LSTR=(2g 1t 4k 3m) # limits strings
 	# limits values
-	local LVAL=($[2*1024*1024] $[1*1024*1024*1024] $[4*1024] $[3*1024*1024])
+	local LVAL=($((2*1024*1024)) $((1*1024*1024*1024)) $((4*1024)) \
+		    $((3*1024*1024)))
 
 	resetquota -u $TSTUSR
 
@@ -1797,10 +2081,10 @@ test_21() {
 	local BLIMIT=10 # 10G
 	local ILIMIT=1000000
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
-	set_ost_qtype "ug" || error "Enable ost quota failed"
+	set_ost_qtype $QTYPE || error "Enable ost quota failed"
 
 	log "Set limit(block:${BLIMIT}G; file:$ILIMIT) for user: $TSTUSR"
 	$LFS setquota -u $TSTUSR -b 0 -B ${BLIMIT}G -i 0 -I $ILIMIT $MOUNT ||
@@ -1808,6 +2092,12 @@ test_21() {
 	log "Set limit(block:${BLIMIT}G; file:$ILIMIT) for group: $TSTUSR"
 	$LFS setquota -g $TSTUSR -b 0 -B $BLIMIT -i 0 -I $ILIMIT $MOUNT ||
 		error "set group quota failed"
+	if is_project_quota_supported; then
+		log "Set limit(block:${BLIMIT}G; file:$LIMIT) for " \
+			"project: $TSTPRJID"
+		$LFS setquota -p $TSTPRJID -b 0 -B $BLIMIT -i 0 -I $ILIMIT \
+			 $MOUNT || error "set project quota failed"
+	fi
 
 	# repeat writing on a 1M file
 	test_21_sub ${TESTFILE}_1 1 30 &
@@ -1826,6 +2116,11 @@ test_21() {
 		$LFS setquota -g $TSTUSR -b 0 -B "$((BLIMIT + i))G" -i 0 \
 			-I $((ILIMIT + i)) $MOUNT ||
 				error "Set group quota failed"
+		if is_project_quota_supported; then
+			$LFS setquota -p $TSTPRJID -b 0 -B \
+			"$((BLIMIT + i))G"  -i 0 -I $((ILIMIT + i)) $MOUNT ||
+				error "Set project quota failed"
+		fi
 		i=$((i+1))
 		sleep 1
 	done
@@ -1855,32 +2150,35 @@ test_21() {
 	cleanup_quota_test
 	resetquota -u $TSTUSR
 	resetquota -g $TSTUSR
+	resetquota -p $TSTPRJID
 }
 run_test 21 "Setquota while writing & deleting (b16053)"
 
 # enable/disable quota enforcement permanently
 test_22() {
 	echo "Set both mdt & ost quota type as ug"
-	set_mdt_qtype "ug" || error "enable mdt quota failed"
-	set_ost_qtype "ug" || error "enable ost quota failed"
+	local qtype="ug"
+	is_project_quota_supported && qtype=$QTYPE
+	set_mdt_qtype $qtype || error "enable mdt quota failed"
+	set_ost_qtype $qtype || error "enable ost quota failed"
 
 	echo "Restart..."
-	stopall
+	stopall || error "failed to stopall (1)"
 	mount
 	setupall
 
 	echo "Verify if quota is enabled"
-	local qtype=$(mdt_quota_type)
-	[ $qtype != "ug" ] && error "mdt quota setting is lost"
+	local qtype1=$(mdt_quota_type)
+	[ $qtype1 != $qtype ] && error "mdt quota setting is lost"
 	qtype=$(ost_quota_type)
-	[ $qtype != "ug" ] && error "ost quota setting is lost"
+	[ $qtype1 != $qtype ] && error "ost quota setting is lost"
 
 	echo "Set both mdt & ost quota type as none"
 	set_mdt_qtype "none" || error "disable mdt quota failed"
 	set_ost_qtype "none" || error "disable ost quota failed"
 
 	echo "Restart..."
-	stopall
+	stopall || error "failed to stopall (2)"
 	mount
 	setupall
 	quota_init
@@ -1893,16 +2191,16 @@ test_22() {
 
 	return 0
 }
-run_test 22 "enable/disable quota by 'lctl conf_param'"
+run_test 22 "enable/disable quota by 'lctl conf_param/set_param -P'"
 
 test_23_sub() {
 	local TESTFILE="$DIR/$tdir/$tfile"
 	local LIMIT=$1
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
-	set_ost_qtype "ug" || error "Enable ost quota failed"
+	set_ost_qtype $QTYPE || error "Enable ost quota failed"
 
 	# test for user
 	log "User quota (limit: $LIMIT MB)"
@@ -1910,8 +2208,8 @@ test_23_sub() {
 		error "set quota failed"
 	quota_show_check b u $TSTUSR
 
-	$LFS setstripe $TESTFILE -c 1 -i 0
-	chown $TSTUSR.$TSTUSR $TESTFILE
+	$SETSTRIPE $TESTFILE -c 1 -i 0 || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
 
 	log "Step1: trigger EDQUOT with O_DIRECT"
 	log "Write half of file"
@@ -1943,6 +2241,10 @@ test_23_sub() {
 }
 
 test_23() {
+	[ $(facet_fstype ost1) == "zfs" ] &&
+		skip "Overwrite in place is not guaranteed to be " \
+		"space neutral on ZFS"
+
 	local OST0_MIN=$((6 * 1024)) # 6MB, extra space for meta blocks.
 	check_whether_skip && return 0
 	log "run for 4MB test file"
@@ -1959,16 +2261,16 @@ test_24() {
 	local blimit=5 # 5M
 	local TESTFILE="$DIR/$tdir/$tfile"
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
-	set_ost_qtype "ug" || error "enable ost quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
 
 	# bind file to a single OST
-	$LFS setstripe -c 1 $TESTFILE
-	chown $TSTUSR.$TSTUSR $TESTFILE
+	$SETSTRIPE -c 1 $TESTFILE || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
 
-	echo "Set user quota (limit: "$blimit"M)"
+	echo "Set user quota (limit: ${blimit}M)"
 	$LFS setquota -u $TSTUSR -b 0 -B "$blimit"M -i 0 -I 0 $MOUNT ||
 		error "set quota failed"
 
@@ -1999,27 +2301,78 @@ test_27b() { # b20200
 		error "lfs setquota failed with uid argument"
 	$LFS setquota -g $TSTID -b 1000 -B 1000 -i 1000 -I 1000 $DIR ||
 		error "lfs stequota failed with gid argument"
+	if is_project_quota_supported; then
+		$LFS setquota -p $TSTPRJID -b 1000 -B 1000 -i 1000 -I \
+			1000 $DIR || error \
+				"lfs stequota failed with projid argument"
+	fi
 	$SHOW_QUOTA_USERID || error "lfs quota failed with uid argument"
 	$SHOW_QUOTA_GROUPID || error "lfs quota failed with gid argument"
+	if is_project_quota_supported; then
+		$SHOW_QUOTA_PROJID ||
+			error "lfs quota failed with projid argument"
+	fi
 	resetquota -u $TSTUSR
 	resetquota -g $TSTUSR
+	resetquota -p $TSTPRJID
 	return 0
 }
-run_test 27b "lfs quota/setquota should handle user/group ID (b20200)"
+run_test 27b "lfs quota/setquota should handle user/group/project ID (b20200)"
+
+test_27c() {
+	local limit
+
+	$LFS setquota -u $TSTID -b 30M -B 3T $DIR ||
+		error "lfs setquota failed"
+
+	limit=$($LFS quota -u $TSTID -v -h $DIR | grep $DIR | awk '{print $3}')
+	[ $limit != "30M" ] && error "softlimit $limit isn't human-readable"
+	limit=$($LFS quota -u $TSTID -v -h $DIR | grep $DIR | awk '{print $4}')
+	[ $limit != "3T" ] && error "hardlimit $limit isn't human-readable"
+
+	$LFS setquota -u $TSTID -b 1500M -B 18500G $DIR ||
+		error "lfs setquota for $TSTID failed"
+
+	limit=$($LFS quota -u $TSTID -v -h $DIR | grep $DIR | awk '{print $3}')
+	[ $limit != "1.465G" ] && error "wrong softlimit $limit"
+	limit=$($LFS quota -u $TSTID -v -h $DIR | grep $DIR | awk '{print $4}')
+	[ $limit != "18.07T" ] && error "wrong hardlimit $limit"
+
+	$LFS quota -u $TSTID -v -h $DIR | grep -q "Total allocated" ||
+		error "total allocated inode/block limit not printed"
+
+	resetquota -u $TSTUSR
+}
+run_test 27c "lfs quota should support human-readable output"
+
+test_27d() {
+	local softlimit=1.5
+	local hardlimit=2.3
+	local limit
+
+	$LFS setquota -u $TSTID -b ${softlimit}p -B ${hardlimit}P $DIR ||
+		error "set fraction block limit failed"
+	limit=$($LFS quota -u $TSTID -h $DIR | grep $DIR | awk '{print $3}')
+	[ $limit == ${softlimit}P ] || error "get fraction softlimit failed"
+	limit=$($LFS quota -u $TSTID -h $DIR | grep $DIR | awk '{print $4}')
+	[ $limit == ${hardlimit}P ] || error "get fraction hardlimit failed"
+
+	resetquota -u $TSTUSR
+}
+run_test 27d "lfs setquota should support fraction block limit"
 
 test_30() {
-	local output
 	local LIMIT=4 # 4MB
 	local TESTFILE="$DIR/$tdir/$tfile"
 	local GRACE=10
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	set_ost_qtype "u" || error "enable ost quota failed"
 
-	$LFS setstripe $TESTFILE -i 0 -c 1
-	chown $TSTUSR.$TSTUSR $TESTFILE
+	$SETSTRIPE $TESTFILE -i 0 -c 1 || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
 
 	$LFS setquota -t -u --block-grace $GRACE --inode-grace \
 		$MAX_IQ_TIME $DIR || error "set grace time failed"
@@ -2032,11 +2385,11 @@ test_30() {
 	# over-quota flag has not yet settled since we do not trigger async
 	# events based on grace time period expiration
 	$SHOW_QUOTA_USER
-	$RUNAS $DD of=$TESTFILE conv=notrunc oflag=append count=1 || true
+	$RUNAS $DD of=$TESTFILE conv=notrunc oflag=append count=4 || true
 	cancel_lru_locks osc
 	# now over-quota flag should be settled and further writes should fail
 	$SHOW_QUOTA_USER
-	$RUNAS $DD of=$TESTFILE conv=notrunc oflag=append count=1 &&
+	$RUNAS $DD of=$TESTFILE conv=notrunc oflag=append count=4 &&
 		error "grace times were reset"
 	# cleanup
 	cleanup_quota_test
@@ -2052,7 +2405,7 @@ test_33() {
 	local BLK_CNT=2 # of 2M each
 	local TOTAL_BLKS=$((INODES * BLK_CNT * 1024))
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	# make sure the system is clean
@@ -2062,14 +2415,25 @@ test_33() {
 	USED=$(getquota -g $TSTID global curspace)
 	[ $USED -ne 0 ] &&
 		error "Used space ($USED) for group $TSTID isn't 0."
+	if is_project_quota_supported; then
+		USED=$(getquota -p $TSTPRJID global curspace)
+		[ $USED -ne 0 ] && error \
+			"Used space ($USED) for project $TSTPRJID isn't 0."
+	fi
 
 	echo "Write files..."
-	for i in `seq 0 $INODES`; do
+	for i in $(seq 0 $INODES); do
 		$RUNAS $DD of=$DIR/$tdir/$tfile-$i count=$BLK_CNT 2>/dev/null ||
 			error "write failed"
+			is_project_quota_supported &&
+				change_project -p $TSTPRJID $DIR/$tdir/$tfile-$i
 		echo "Iteration $i/$INODES completed"
 	done
 	cancel_lru_locks osc
+
+	echo "Wait for setattr on objects finished..."
+	wait_delete_completed
+
 	sync; sync_all_data || true
 
 	echo "Verify disk usage after write"
@@ -2079,6 +2443,11 @@ test_33() {
 	USED=$(getquota -g $TSTID global curspace)
 	[ $USED -lt $TOTAL_BLKS ] &&
 		error "Used space for group $TSTID:$USED, expected:$TOTAL_BLKS"
+	if is_project_quota_supported; then
+		USED=$(getquota -p $TSTPRJID global curspace)
+		[ $USED -lt $TOTAL_BLKS ] && error \
+			"Used space for project $TSTPRJID:$USED, expected:$TOTAL_BLKS"
+	fi
 
 	echo "Verify inode usage after write"
 	USED=$(getquota -u $TSTID global curinodes)
@@ -2087,6 +2456,11 @@ test_33() {
 	USED=$(getquota -g $TSTID global curinodes)
 	[ $USED -lt $INODES ] &&
 		error "Used inode for group $TSTID is $USED, expected $INODES"
+	if is_project_quota_supported; then
+		USED=$(getquota -p $TSTPRJID global curinodes)
+		[ $USED -lt $INODES ] && error \
+			"Used inode for project $TSTPRJID is $USED, expected $INODES"
+	fi
 
 	cleanup_quota_test
 
@@ -2099,15 +2473,25 @@ test_33() {
 	[ $USED -eq 0 ] || error "Used space for group $TSTID isn't 0. $USED"
 	USED=$(getquota -g $TSTID global curinodes)
 	[ $USED -eq 0 ] || error "Used inodes for group $TSTID isn't 0. $USED"
+	if is_project_quota_supported; then
+		USED=$(getquota -p $TSTPRJID global curspace)
+		[ $USED -eq 0 ] ||
+			error "Used space for project $TSTPRJID isn't 0. $USED"
+		USED=$(getquota -p $TSTPRJID global curinodes)
+		[ $USED -eq 0 ] ||
+			error "Used inodes for project $TSTPRJID isn't 0. $USED"
+	fi
 }
-run_test 33 "Basic usage tracking for user & group"
+run_test 33 "Basic usage tracking for user & group & project"
 
-# usage transfer test for user & group
+# usage transfer test for user & group & project
 test_34() {
-        local BLK_CNT=2 # 2MB
+	local BLK_CNT=2 # 2MB
+	local project_supported="no"
 
-	setup_quota_test
-        trap cleanup_quota_test EXIT
+	is_project_quota_supported && project_supported="yes"
+	setup_quota_test || error "setup quota failed with $?"
+	trap cleanup_quota_test EXIT
 
 	# make sure the system is clean
 	local USED=$(getquota -u $TSTID global curspace)
@@ -2115,6 +2499,14 @@ test_34() {
 	USED=$(getquota -g $TSTID global curspace)
 	[ $USED -ne 0 ] && error "Used space ($USED) for group $TSTID isn't 0."
 
+	local USED=$(getquota -u $TSTID2 global curspace)
+	[ $USED -ne 0 ] && error "Used space ($USED) for user $TSTID2 isn't 0."
+	if [ $project_supported == "yes" ]; then
+		USED=$(getquota -p $TSTPRJID global curspace)
+		[ $USED -ne 0 ] && error \
+			"Used space ($USED) for Project $TSTPRJID isn't 0."
+	fi
+
 	echo "Write file..."
 	$DD of=$DIR/$tdir/$tfile count=$BLK_CNT 2>/dev/null ||
 		error "write failed"
@@ -2151,21 +2543,56 @@ test_34() {
 	[ $USED -eq 1 ] ||
 		error "Used inodes for group $TSTID is $USED, expected 1"
 
+	# chown won't change the ost object group. LU-4345 */
+	echo "chown the file to user $TSTID2"
+	chown $TSTID2 $DIR/$tdir/$tfile || error "chown to $TSTID2 failed"
+
+	echo "Wait for setattr on objects finished..."
+	wait_delete_completed
+
+	echo "change_project project id to $TSTPRJID"
+	[ $project_supported == "yes" ] &&
+		change_project -p $TSTPRJID $DIR/$tdir/$tfile
+	echo "Wait for setattr on objects finished..."
+	wait_delete_completed
+
+	echo "Verify disk usage for user $TSTID2/$TSTID and group $TSTID"
+	USED=$(getquota -u $TSTID2 global curspace)
+	[ $USED -lt $BLK_CNT ] &&
+		error "Used space for user $TSTID2 is $USED, expected $BLK_CNT"
+	USED=$(getquota -u $TSTID global curspace)
+	[ $USED -ne 0 ] &&
+		error "Used space for user $TSTID is $USED, expected 0"
+	USED=$(getquota -g $TSTID global curspace)
+	[ $USED -lt $BLK_CNT ] &&
+		error "Used space for group $TSTID is $USED, expected $BLK_CNT"
+	if [ $project_supported == "yes" ]; then
+		USED=$(getquota -p $TSTPRJID global curspace)
+		[ $USED -lt $BLK_CNT ] && error \
+			"Used space for group $TSTPRJID is $USED, expected $BLK_CNT"
+	fi
+
 	cleanup_quota_test
 }
-run_test 34 "Usage transfer for user & group"
+run_test 34 "Usage transfer for user & group & project"
 
 # usage is still accessible across restart
 test_35() {
 	local BLK_CNT=2 # 2 MB
 
-	setup_quota_test
+	setup_quota_test || error "setup quota failed with $?"
 	trap cleanup_quota_test EXIT
 
 	echo "Write file..."
 	$RUNAS $DD of=$DIR/$tdir/$tfile count=$BLK_CNT 2>/dev/null ||
 		error "write failed"
+	is_project_quota_supported &&
+		change_project -p $TSTPRJID $DIR/$tdir/$tfile
 	cancel_lru_locks osc
+
+	echo "Wait for setattr on objects finished..."
+	wait_delete_completed
+
 	sync; sync_all_data || true
 
 	echo "Save disk usage before restart"
@@ -2184,11 +2611,19 @@ test_35() {
 		error "Used inodes for group $TSTID is 0, expected 1"
 	echo "Group $TSTID: ${ORIG_GRP_SPACE}KB $ORIG_GRP_INODES inodes"
 
+	if is_project_quota_supported; then
+		local ORIG_PRJ_SPACE=$(getquota -p $TSTPRJID global curspace)
+		[ $ORIG_PRJ_SPACE -eq 0 ] && error \
+			"Used space for project $TSTPRJID is 0, expected ${BLK_CNT}M"
+		local ORIG_PRJ_INODES=$(getquota -p $TSTPRJID global curinodes)
+		[ $ORIG_PRJ_INODES -eq 0 ] && error \
+			"Used inodes for project $TSTPRJID is 0, expected 1"
+		echo "Project $TSTPRJID: ${ORIG_PRJ_SPACE}KB $ORIG_PRJ_INODES inodes"
+	fi
+
 	log "Restart..."
-	local ORIG_REFORMAT=$REFORMAT
-	REFORMAT=""
-	cleanup_and_setup_lustre
-	REFORMAT=$ORIG_REFORMAT
+	stopall
+	setupall
 	quota_init
 
 	echo "Verify disk usage after restart"
@@ -2208,6 +2643,16 @@ test_35() {
 	[ $USED -eq $ORIG_GRP_INODES ] ||
 		error "Used inodes for group $TSTID changed from " \
 			"$ORIG_GRP_INODES to $USED"
+	if [ $project_supported == "yes" ]; then
+		USED=$(getquota -p $TSTPRJID global curinodes)
+		[ $USED -eq $ORIG_PRJ_INODES ] ||
+			error "Used inodes for project $TSTPRJID " \
+				"changed from $ORIG_PRJ_INODES to $USED"
+		USED=$(getquota -p $TSTPRJID global curspace)
+		[ $USED -eq $ORIG_PRJ_SPACE ] ||
+			error "Used space for project $TSTPRJID "\
+				"changed from $ORIG_PRJ_SPACE to $USED"
+	fi
 
 	# check if the vfs_dq_init() is called before writing
 	echo "Append to the same file..."
@@ -2225,87 +2670,774 @@ test_35() {
 	[ $USED -gt $ORIG_GRP_SPACE ] ||
 		error "Used space for group $TSTID isn't increased" \
 			"orig:$ORIG_GRP_SPACE, now:$USED"
+	if [ $project_supported == "yes" ]; then
+		USED=$(getquota -p $TSTPRJID global curspace)
+		[ $USED -gt $ORIG_PRJ_SPACE ] ||
+			error "Used space for project $TSTPRJID isn't " \
+				"increased orig:$ORIG_PRJ_SPACE, now:$USED"
+	fi
 
 	cleanup_quota_test
 }
 run_test 35 "Usage is still accessible across reboot"
 
-# test migrating old amdin quota files (in Linux quota file format v2) into new
-# quota global index (in IAM format)
-test_36() {
-	[ $(facet_fstype $SINGLEMDS) != ldiskfs ] && \
-		skip "skipping migration test" && return
-
-	# get the mdt0 device name
-	local mdt0_node=$(facet_active_host $SINGLEMDS)
-	local mdt0_dev=$(mdsdevname ${SINGLEMDS//mds/})
-
-	echo "Reformat..."
-	formatall
-
-	echo "Copy admin quota files into MDT0..."
-	local mntpt=$(facet_mntpt $SINGLEMDS)
-	local mdt0_fstype=$(facet_fstype $SINGLEMDS)
-	local opt
-	if ! do_node $mdt0_node test -b $mdt0_fstype; then
-		opt="-o loop"
-	fi
-	echo "$mdt0_node, $mdt0_dev, $mntpt, $opt"
-	do_node $mdt0_node mount -t $mdt0_fstype $opt $mdt0_dev $mntpt
-	do_node $mdt0_node mkdir $mntpt/OBJECTS
-	do_node $mdt0_node cp $LUSTRE/tests/admin_quotafile_v2.usr $mntpt/OBJECTS
-	do_node $mdt0_node cp $LUSTRE/tests/admin_quotafile_v2.grp $mntpt/OBJECTS
-	do_node $mdt0_node umount -f $mntpt
-
-	echo "Setup all..."
-	setupall
+# chown/chgrp to the file created with MDS_OPEN_DELAY_CREATE
+# LU-5006
+test_37() {
+	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.6.93) ] &&
+		skip "Old server doesn't have LU-5006 fix."
 
-	echo "Verify global limits..."
-	local id_cnt
-	local limit
+	setup_quota_test || error "setup quota failed with $?"
+	trap cleanup_quota_test EXIT
+
+	# make sure the system is clean
+	local USED=$(getquota -u $TSTID global curspace)
+	[ $USED -ne 0 ] &&
+		error "Used space ($USED) for user $TSTID isn't 0."
+
+	# create file with MDS_OPEN_DELAY_CREATE flag
+	$SETSTRIPE -c 1 -i 0 $DIR/$tdir/$tfile ||
+		error "Create file failed"
+	# write to file
+	dd if=/dev/zero of=$DIR/$tdir/$tfile bs=1M count=1 conv=notrunc \
+		oflag=sync || error "Write file failed"
+	# chown to the file
+	chown $TSTID $DIR/$tdir/$tfile || error "Chown to file failed"
+
+	# wait for setattr on objects finished..."
+	wait_delete_completed
+
+	USED=$(getquota -u $TSTID global curspace)
+	[ $USED -ne 0 ] || quota_error u $TSTUSR "Used space is 0"
+
+	cleanup_quota_test
+}
+run_test 37 "Quota accounted properly for file created by 'lfs setstripe'"
+
+# LU-8801
+test_38() {
+	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.8.60) ] &&
+		skip "Old server doesn't have LU-8801 fix."
+
+	[ "$UID" != 0 ] && skip_env "must run as root" && return
+
+	setup_quota_test || error "setup quota failed with $?"
+	trap cleanup_quota_test EXIT
+
+	# make sure the system is clean
+	local USED=$(getquota -u $TSTID global curspace)
+	[ $USED -ne 0 ] &&
+		error "Used space ($USED) for user $TSTID isn't 0."
+	USED=$(getquota -u $TSTID2 global curspace)
+	[ $USED -ne 0 ] &&
+		error "Used space ($USED) for user $TSTID2 isn't 0."
 
-	local proc="qmt.*.md-0x0.glb-usr"
-	id_cnt=$(do_node $mdt0_node $LCTL get_param -n $proc | wc -l)
-	[ $id_cnt -eq 403 ] || error "Migrate inode user limit failed: $id_cnt"
-	limit=$(getquota -u 1 global isoftlimit)
-	[ $limit -eq 1024 ] || error "User inode softlimit: $limit"
-	limit=$(getquota -u 1 global ihardlimit)
-	[ $limit -eq 2048 ] || error "User inode hardlimit: $limit"
-
-	proc="qmt.*.md-0x0.glb-grp"
-	id_cnt=$(do_node $mdt0_node $LCTL get_param -n $proc | wc -l)
-	[ $id_cnt -eq 403 ] || error "Migrate inode group limit failed: $id_cnt"
-	limit=$(getquota -g 1 global isoftlimit)
-	[ $limit -eq 1024 ] || error "Group inode softlimit: $limit"
-	limit=$(getquota -g 1 global ihardlimit)
-	[ $limit -eq 2048 ] || error "Group inode hardlimit: $limit"
-
-	proc=" qmt.*.dt-0x0.glb-usr"
-	id_cnt=$(do_node $mdt0_node $LCTL get_param -n $proc | wc -l)
-	[ $id_cnt -eq 403 ] || error "Migrate block user limit failed: $id_cnt"
-	limit=$(getquota -u 60001 global bsoftlimit)
-	[ $limit -eq 10485760 ] || error "User block softlimit: $limit"
-	limit=$(getquota -u 60001 global bhardlimit)
-	[ $limit -eq 20971520 ] || error "User block hardlimit: $limit"
-
-	proc="qmt.*.dt-0x0.glb-grp"
-	id_cnt=$(do_node $mdt0_node $LCTL get_param -n $proc | wc -l)
-	[ $id_cnt -eq 403 ] || error "Migrate block user limit failed: $id_cnt"
-	limit=$(getquota -g 60001 global bsoftlimit)
-	[ $limit -eq 10485760 ] || error "Group block softlimit: $limit"
-	limit=$(getquota -g 60001 global bhardlimit)
-	[ $limit -eq 20971520 ] || error "Group block hardlimit: $limit"
-
-	echo "Cleanup..."
-	formatall
+	local TESTFILE="$DIR/$tdir/$tfile"
+	local file_cnt=10000
+
+	# Generate id entries in accounting file
+	echo "Create $file_cnt files..."
+	for i in `seq $file_cnt`; do
+		touch $TESTFILE-$i
+		chown $((file_cnt - i)):$((file_cnt - i)) $TESTFILE-$i ||
+			error "failed to chown $TESTFILE-$i"
+	done
+	cancel_lru_locks osc
+	sync; sync_all_data || true
+
+	local procf="osd-$(facet_fstype $SINGLEMDS).$FSNAME-MDT0000"
+	procf=${procf}.quota_slave.acct_user
+	local accnt_cnt
+
+	acct_cnt=$(do_facet mds1 $LCTL get_param $procf | grep "id:" | \
+		   awk '{if ($3 < 10000) {print $3}}' | wc -l)
+	echo "Found $acct_cnt id entries"
+
+	[ $file_cnt -eq $acct_cnt ] || {
+		do_facet mds1 $LCTL get_param $procf
+		error "skipped id entries"
+	}
+
+	cleanup_quota_test
+}
+run_test 38 "Quota accounting iterator doesn't skip id entries"
+
+test_39() {
+	local TESTFILE="$DIR/$tdir/project"
+	! is_project_quota_supported &&
+		skip "Project quota is not supported"
+
+	setup_quota_test || error "setup quota failed with $?"
+
+	touch $TESTFILE
+	projectid=$(lfs project $TESTFILE | awk '{print $1}')
+	[ $projectid -ne 0 ] &&
+		error "Project id should be 0 not $projectid"
+	change_project -p 1024 $TESTFILE
+	projectid=$(lfs project $TESTFILE | awk '{print $1}')
+	[ $projectid -ne 1024 ] &&
+		error "Project id should be 1024 not $projectid"
+
+	stopall || error "failed to stopall (1)"
+	mount
 	setupall
+	projectid=$(lfs project $TESTFILE | awk '{print $1}')
+	[ $projectid -ne 1024 ] &&
+		error "Project id should be 1024 not $projectid"
+
+	cleanup_quota_test
+}
+run_test 39 "Project ID interface works correctly"
+
+test_40a() {
+	! is_project_quota_supported &&
+		skip "Project quota is not supported"
+	local dir1="$DIR/$tdir/dir1"
+	local dir2="$DIR/$tdir/dir2"
+
+	setup_quota_test || error "setup quota failed with $?"
+
+	mkdir -p $dir1 $dir2
+	change_project -sp 1 $dir1 && touch $dir1/1
+	change_project -sp 2 $dir2
+
+	ln $dir1/1 $dir2/1_link &&
+		error "Hard link across different project quota should fail"
+	rm -rf $dir1 $dir2
+
+	cleanup_quota_test
+}
+run_test 40a "Hard link across different project ID"
+
+test_40b() {
+	! is_project_quota_supported &&
+		skip "Project quota is not supported"
+	local dir1="$DIR/$tdir/dir1"
+	local dir2="$DIR/$tdir/dir2"
+
+	setup_quota_test || error "setup quota failed with $?"
+	mkdir -p $dir1 $dir2
+	change_project -sp 1 $dir1 && touch $dir1/1
+	change_project -sp 2 $dir2
+
+	mv $dir1/1 $dir2/2 || error "mv failed $?"
+	local projid=$(lfs project $dir2/2 | awk '{print $1}')
+	if [ "$projid" != "2" ]; then
+		error "project id expected 2 not $projid"
+	fi
+	rm -rf $dir1 $dir2
+	cleanup_quota_test
+}
+run_test 40b "Mv across different project ID"
+
+test_40c() {
+	[ "$MDSCOUNT" -lt "2" ] && skip "needs >= 2 MDTs"
+		! is_project_quota_supported &&
+			skip "Project quota is not supported"
+
+	setup_quota_test || error "setup quota failed with $?"
+	local dir="$DIR/$tdir/dir"
+
+	mkdir -p $dir && change_project -sp 1 $dir
+	$LFS mkdir -i 1 $dir/remote_dir || error "create remote dir failed"
+	local projid=$(lfs project -d $dir/remote_dir | awk '{print $1}')
+	[ "$projid" != "1" ] && error "projid id expected 1 not $projid"
+	touch $dir/remote_dir/file
+	#verify inherit works file for remote dir.
+	local projid=$(lfs project -d $dir/remote_dir/file | awk '{print $1}')
+	[ "$projid" != "1" ] &&
+		error "file under remote dir expected 1 not $projid"
+
+	#Agent inode should be ignored for project quota
+	USED=$(getquota -p 1 global curinodes)
+	[ "$USED" != "3" ] &&
+		error "file count expected 3 got $USED"
+
+	rm -rf $dir
+	cleanup_quota_test
+	return 0
+}
+run_test 40c "Remote child Dir inherit project quota properly"
+
+test_50() {
+	! is_project_quota_supported &&
+		skip "Project quota is not supported"
+
+	setup_quota_test || error "setup quota failed with $?"
+	local dir="$DIR/$tdir/dir"
+
+	mkdir $dir && change_project -p 1 $dir
+	count=$($LFS find --projid 1 $DIR | wc -l)
+	[ "$count" != 1 ] && error "expected 1 but got $count"
+
+	rm -rf $dir
+	cleanup_quota_test
+}
+run_test 50 "Test if lfs find --projid works"
+
+test_51() {
+	! is_project_quota_supported &&
+		skip "Project quota is not supported"
+	setup_quota_test || error "setup quota failed with $?"
+	local dir="$DIR/$tdir/dir"
+
+	mkdir $dir && change_project -sp 1 $dir
+	local used=$(getquota -p 1 global curinodes)
+	[ $used != "1" ] && error "expected 1 got $used"
+
+	touch $dir/1
+	touch $dir/2
+	cp $dir/2 $dir/3
+	used=$(getquota -p 1 global curinodes)
+	[ $used != "4" ] && error "expected 4 got $used"
+
+	$DD if=/dev/zero of=$DIR/$tdir/6 bs=1M count=1
+	#try cp to dir
+	cp $DIR/$tdir/6 $dir/6
+	used=$(getquota -p 1 global curinodes)
+	[ $used != "5" ] && error "expected 5 got $used"
+
+	#try mv to dir
+	mv $DIR/$tdir/6 $dir/7
+	used=$(getquota -p 1 global curinodes)
+	[ $used != "6" ] && error "expected 6 got $used"
+
+	rm -rf $dir
+	cleanup_quota_test
+}
+run_test 51 "Test project accounting with mv/cp"
+
+test_52() {
+	! is_project_quota_supported &&
+		skip "Project quota is not supported"
+	setup_quota_test || error "setup quota failed with $?"
+	local dir="$DIR/$tdir/dir"
+	mkdir $dir && change_project -sp 1 $dir
+
+	touch $DIR/$tdir/file
+	#Try renaming a file into the project.  This should fail.
+	for num in $(seq 1 2000); do
+		mrename $DIR/$tdir/file $dir/file >&/dev/null &&
+			error "rename should fail"
+	done
+	rm -rf $dir
+	cleanup_quota_test
+}
+run_test 52 "Rename across different project ID"
+
+test_53() {
+	! is_project_quota_supported &&
+		skip "Project quota is not supported"
+	setup_quota_test || error "setup quota failed with $?"
+	local dir="$DIR/$tdir/dir"
+	mkdir $dir && change_project -s $dir
+	lfs project -d $dir | grep P || error "inherit attribute should be set"
+
+	change_project -C $dir
+	lfs project -d $dir | grep P &&
+		error "inherit attribute should be cleared"
+
+	rm -rf $dir
+	cleanup_quota_test
+}
+run_test 53 "Project inherit attribute could be cleared"
+
+test_54() {
+	! is_project_quota_supported &&
+		skip "Project quota is not supported"
+	setup_quota_test || error "setup quota failed with $?"
+	trap cleanup_quota_test EXIT
+	local testfile="$DIR/$tdir/$tfile-0"
+
+	#set project ID/inherit attribute
+	change_project -sp $TSTPRJID $DIR/$tdir
+	$RUNAS createmany -m ${testfile} 100 ||
+		error "create many files failed"
+
+	local proj_count=$(lfs project -r $DIR/$tdir | wc -l)
+	# one more count for directory itself */
+	((proj_count++))
+
+	#check project
+	local proj_count1=$(lfs project -rcp $TSTPRJID $DIR/$tdir | wc -l)
+	[ $proj_count1 -eq 0 ] || error "c1: expected 0 got $proj_count1"
+
+	proj_count1=$(lfs project -rcp $((TSTPRJID+1)) $DIR/$tdir | wc -l)
+	[ $proj_count1 -eq $proj_count ] ||
+			error "c2: expected $proj_count got $proj_count1"
+
+	#clear project but with kept projid
+	change_project -rCk $DIR/$tdir
+	proj_count1=$(lfs project -rcp $TSTPRJID $DIR/$tdir | wc -l)
+	[ $proj_count1 -eq $proj_count ] ||
+			error "c3: expected $proj_count got $proj_count1"
+
+	#verify projid untouched.
+	proj_count1=$(lfs project -r $DIR/$tdir | grep -c $TSTPRJID)
+	((proj_count1++))
+	[ $proj_count1 -eq $proj_count ] ||
+			error "c4: expected $proj_count got $proj_count1"
+
+	# test -0 option
+	lfs project $DIR/$tdir -cr -0 | xargs -0 lfs project -s
+	proj_count1=$(lfs project -rcp $TSTPRJID $DIR/$tdir | wc -l)
+	[ $proj_count1 -eq 0 ] || error "c5: expected 0 got $proj_count1"
+
+	#this time clear all
+	change_project -rC $DIR/$tdir
+	proj_count1=$(lfs project -r $DIR/$tdir | grep -c $TSTPRJID)
+	[ $proj_count1 -eq 0 ] ||
+			error "c6: expected 0 got $proj_count1"
+	#cleanup
+	unlinkmany ${testfile} 100 ||
+		error "unlink many files failed"
+
+	cleanup_quota_test
+}
+run_test 54 "basic lfs project interface test"
+
+test_55() {
+	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.58) ] &&
+		skip "Not supported before 2.10.58."
+	setup_quota_test || error "setup quota failed with $?"
+
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
+	quota_init
+
+	#add second group to TSTUSR
+	usermod -G $TSTUSR,$TSTUSR2 $TSTUSR
+
+	#prepare test file
+	$RUNAS dd if=/dev/zero of=$DIR/$tdir/$tfile bs=1024 count=100000 ||
+	error "failed to dd"
+
+	cancel_lru_locks osc
+	sync; sync_all_data || true
+
+	$LFS setquota -g $TSTUSR2 -b 0 -B 50M $DIR ||
+	error "failed to setquota on group $TSTUSR2"
+
+	$LFS quota -v -g $TSTUSR2 $DIR
+
+	runas -u $TSTUSR -g $TSTUSR2 chgrp $TSTUSR2 $DIR/$tdir/$tfile &&
+	error "chgrp should failed with -EDQUOT"
+
+	USED=$(getquota -g $TSTUSR2 global curspace)
+	echo "$USED"
+
+	$LFS setquota -g $TSTUSR2 -b 0 -B 300M $DIR ||
+	error "failed to setquota on group $TSTUSR2"
+
+	$LFS quota -v -g $TSTUSR2 $DIR
+
+	runas -u $TSTUSR -g $TSTUSR2 chgrp $TSTUSR2 $DIR/$tdir/$tfile ||
+	error "chgrp should succeed"
+
+	$LFS quota -v -g $TSTUSR2 $DIR
+
+	resetquota -g $TSTUSR2
+	cleanup_quota_test
+}
+run_test 55 "Chgrp should be affected by group quota"
+
+test_56() {
+	setup_quota_test || error "setup quota failed with $?"
+
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
+	quota_init
+
+	$LFS setquota -t -u -b 10 -i 10 $DIR ||
+		erro "failed to set grace time for usr quota"
+	grace_time=$($LFS quota -t -u $DIR | grep "Block grace time:" |
+		     awk '{print $4 $8}')
+	if [ "x$grace_time" != "x10s;10s" ]; then
+		$LFS quota -t -u $DIR
+		error "expected grace time: 10s;10s, got:$grace_time"
+	fi
+
+	cleanup_quota_test
+}
+run_test 56 "lfs quota -t should work well"
+
+test_57() {
+	setup_quota_test || error "setup quota failed with $?"
+
+	local dir="$DIR/$tdir/dir"
+	mkdir -p $dir
+	mkfifo $dir/pipe
+	#try to change pipe file should not hang and return failure
+	wait_update_facet client "$LFS project -sp 1 $dir/pipe 2>&1 |
+		awk -F ':' '{ print \\\$2 }'" \
+			" failed to get xattr for '$dir/pipe'" || return 1
+	#command can process further if it hit some errors
+	touch $dir/aaa $dir/bbb
+	#create one invalid link file
+	ln -s $dir/not_exist_file $dir/ccc
+	local cnt=$(lfs project -r $dir 2>/dev/null | wc -l)
+	[ $cnt -eq 2 ] || error "expected 2 got $cnt"
+
+	cleanup_quota_test
+}
+run_test 57 "lfs project could tolerate errors"
+
+test_59() {
+	[ "$(facet_fstype $SINGLEMDS)" != "ldiskfs" ] &&
+		skip "ldiskfs only test"
+	disable_project_quota
+	setup_quota_test || error "setup quota failed with $?"
+	quota_init
+
+	local testfile="$DIR/$tdir/$tfile-0"
+	#make sure it did not crash kernel
+	touch $testfile && lfs project -sp 1 $testfile
+
+	enable_project_quota
+	cleanup_quota_test
+}
+run_test 59 "lfs project dosen't crash kernel with project disabled"
+
+test_60() {
+	setup_quota_test || error "setup quota failed with $?"
+	trap cleanup_quota_test EXIT
+	local testfile=$DIR/$tdir/$tfile
+	local limit=100
+
+	set_mdt_qtype "ug" || error "enable mdt quota failed"
+
+	$LFS setquota -g $TSTUSR -b 0 -B 0 -i 0 -I $limit $DIR ||
+		error "set quota failed"
+	quota_show_check a g $TSTUSR
+
+	chown $TSTUSR.$TSTUSR $DIR/$tdir || error "chown $DIR/$tdir failed"
+	chmod g+s $DIR/$tdir || error "chmod g+s failed"
+	$RUNAS createmany -m ${testfile} $((limit-1)) ||
+		error "create many files failed"
+
+	$RUNAS touch $DIR/$tdir/foo && error "regular user should fail"
+
+	# root user can overrun quota
+	runas -u 0 -g 0 touch $DIR/$tdir/foo ||
+		error "root user should succeed"
+
+	cleanup_quota_test
+	resetquota -g $TSTUSR
+}
+run_test 60 "Test quota for root with setgid"
+
+# test default quota
+test_default_quota() {
+	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.11.51) ] &&
+		skip "Not supported before 2.11.51."
+
+	local qtype=$1
+	local qpool=$2
+	local qid=$TSTUSR
+	local qprjid=$TSTPRJID
+	local qdtype="-U"
+	local qs="-b"
+	local qh="-B"
+	local LIMIT=20480 #20M disk space
+	local TESTFILE="$DIR/$tdir/$tfile-0"
+
+	[ $qtype == "-p" ] && ! is_project_quota_supported &&
+		echo "Project quota is not supported" && return 0
+
+	[ $qtype == "-u" ] && qdtype="-U"
+	[ $qtype == "-g" ] && qdtype="-G"
+	[ $qtype == "-p" ] && {
+		qdtype="-P"
+		qid=$qprjid
+	}
+
+	[ $qpool == "meta" ] && {
+		LIMIT=10240 #10K inodes
+		qs="-i"
+		qh="-I"
+	}
+
+	setup_quota_test || error "setup quota failed with $?"
+	trap cleanup_quota_test EXIT
+
+	quota_init
+
+	# enable mdt/ost quota
+	set_mdt_qtype $QTYPE || error "enable mdt quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
+
+	log "set to use default quota"
+	$LFS setquota $qtype $qid -d $DIR ||
+		error "set $qid to use default quota failed"
+
+	log "set default quota"
+	$LFS setquota $qdtype $qs ${LIMIT} $qh ${LIMIT} $DIR ||
+		error "set $qid default quota failed"
+
+	log "get default quota"
+	$LFS quota $qdtype $DIR || error "get default quota failed"
+
+	if [ $qpool == "data" ]; then
+		local SLIMIT=$($LFS quota $qdtype $DIR | grep "$MOUNT" | \
+							awk '{print $2}')
+		[ $SLIMIT -eq $LIMIT ] ||
+			error "the returned default quota is wrong"
+	else
+		local SLIMIT=$($LFS quota $qdtype $DIR | grep "$MOUNT" | \
+							awk '{print $5}')
+		[ $SLIMIT -eq $LIMIT ] ||
+			error "the returned default quota is wrong"
+	fi
+
+	# make sure the system is clean
+	local USED=$(getquota $qtype $qid global curspace)
+	[ $USED -ne 0 ] && error "Used space for $qid isn't 0."
+
+	$SETSTRIPE $TESTFILE -c 1 || error "setstripe $TESTFILE failed"
+	chown $TSTUSR.$TSTUSR $TESTFILE || error "chown $TESTFILE failed"
+
+	[ $qtype == "-p" ] && change_project -sp $TSTPRJID $DIR/$tdir
+
+	log "Test not out of quota"
+	if [ $qpool == "data" ]; then
+		$RUNAS $DD of=$TESTFILE count=$((LIMIT/2 >> 10)) oflag=sync ||
+			quota_error $qtype $qid "write failed, expect succeed"
+	else
+		$RUNAS createmany -m $TESTFILE $((LIMIT/2)) ||
+			quota_error $qtype $qid "create failed, expect succeed"
+
+		unlinkmany $TESTFILE $((LIMIT/2))
+	fi
+
+	log "Test out of quota"
+	# flush cache, ensure noquota flag is set on client
+	cancel_lru_locks osc
+	cancel_lru_locks mdc
+	sync; sync_all_data || true
+	if [ $qpool == "data" ]; then
+		$RUNAS $DD of=$TESTFILE count=$((LIMIT*2 >> 10)) oflag=sync &&
+			quota_error $qtype $qid "write succeed, expect EDQUOT"
+	else
+		$RUNAS createmany -m $TESTFILE $((LIMIT*2)) &&
+			quota_error $qtype $qid "create succeed, expect EDQUOT"
+
+		unlinkmany $TESTFILE $((LIMIT*2))
+	fi
+
+	log "Increase default quota"
+	# increase default quota
+	$LFS setquota $qdtype $qs $((LIMIT*3)) $qh $((LIMIT*3)) $DIR ||
+		error "set default quota failed"
+
+	cancel_lru_locks osc
+	cancel_lru_locks mdc
+	sync; sync_all_data || true
+	if [ $qpool == "data" ]; then
+		$RUNAS $DD of=$TESTFILE count=$((LIMIT*2 >> 10)) oflag=sync ||
+			quota_error $qtype $qid "write failed, expect succeed"
+	else
+		$RUNAS createmany -m $TESTFILE $((LIMIT*2)) ||
+			quota_error $qtype $qid "create failed, expect succeed"
+
+		unlinkmany $TESTFILE $((LIMIT*2))
+	fi
+
+	log "Set quota to override default quota"
+	$LFS setquota $qtype $qid $qs ${LIMIT} $qh ${LIMIT} $DIR ||
+		error "set $qid quota failed"
+
+	cancel_lru_locks osc
+	cancel_lru_locks mdc
+	sync; sync_all_data || true
+	if [ $qpool == "data" ]; then
+		$RUNAS $DD of=$TESTFILE count=$((LIMIT*2 >> 10)) oflag=sync &&
+			quota_error $qtype $qid "write succeed, expect EQUOT"
+	else
+		$RUNAS createmany -m $TESTFILE $((LIMIT*2)) &&
+			quota_error $qtype $qid "create succeed, expect EQUOT"
+
+		unlinkmany $TESTFILE $((LIMIT*2))
+	fi
+
+	log "Set to use default quota again"
+	$LFS setquota $qtype $qid -d $DIR ||
+		error "set $qid to use default quota failed"
+
+	cancel_lru_locks osc
+	cancel_lru_locks mdc
+	sync; sync_all_data || true
+	if [ $qpool == "data" ]; then
+		$RUNAS $DD of=$TESTFILE count=$((LIMIT*2 >> 10)) oflag=sync ||
+			quota_error $qtype $qid "write failed, expect succeed"
+	else
+		$RUNAS createmany -m $TESTFILE $((LIMIT*2)) ||
+			quota_error $qtype $qid "create failed, expect succeed"
+
+		unlinkmany $TESTFILE $((LIMIT*2))
+	fi
+
+	log "Cleanup"
+	rm -f $TESTFILE
+	wait_delete_completed || error "wait_delete_completed failed"
+	sync_all_data || true
+	$LFS setquota $qdtype -b 0 -B 0 -i 0 -I 0 $DIR ||
+		error "reset default quota failed"
+	$LFS setquota $qtype $qid -b 0 -B 0 -i 0 -I 0 $DIR ||
+		error "reset quota failed"
+
+	cleanup_quota_test
+}
+
+test_61() {
+	test_default_quota "-u" "data"
+	test_default_quota "-u" "meta"
+	test_default_quota "-g" "data"
+	test_default_quota "-g" "meta"
+	test_default_quota "-p" "data"
+	test_default_quota "-p" "meta"
+}
+run_test 61 "default quota tests"
+
+test_62() {
+	! is_project_quota_supported &&
+		skip "Project quota is not supported"
+	 [[ "$(chattr -h 2>&1)" =~ "project" ]] ||
+		skip "chattr did not support project quota"
+	setup_quota_test || error "setup quota failed with $?"
+	local testdir=$DIR/$tdir/
+
+	$RUNAS mkdir -p $testdir || error "failed to mkdir"
+	change_project -s $testdir
+	[[ $($LFS project -d $testdir) =~ "P" ]] ||
+		error "inherit attribute should be set"
+	# chattr used FS_IOC_SETFLAGS ioctl
+	$RUNAS chattr -P $testdir &&
+		error "regular user clear inherit should fail"
+	[[ $($LFS project -d $testdir) =~ "P" ]] ||
+		error "inherit attribute should still be set"
+	chattr -P $testdir || error "root failed to clear inherit"
+	[[ $($LFS project -d $testdir) =~ "P" ]] &&
+		error "inherit attribute should be cleared"
+	cleanup_quota_test
+}
+run_test 62 "Project inherit should be only changed by root"
+
+test_dom() {
+	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.11.55) ] &&
+		skip "Not supported before 2.11.55" && return
+
+	local qtype=$1
+	local qid=$TSTUSR
+	local dd_failed=false
+	local tdir_dom=${tdir}_dom
+	local LIMIT=20480 #20M
+
+	[ $qtype == "p" ] && ! is_project_quota_supported &&
+		echo "Project quota is not supported" && return 0
+
+	[ $qtype == "p" ] && qid=$TSTPRJID
+
+	setup_quota_test || error "setup quota failed with $?"
+	trap cleanup_quota_test EXIT
+
+	quota_init
+
+	# enable mdt/ost quota
+	set_mdt_qtype $QTYPE || error "enable mdt quota failed"
+	set_ost_qtype $QTYPE || error "enable ost quota failed"
+
+	# make sure the system is clean
+	local USED=$(getquota -$qtype $qid global curspace)
+	[ $USED -ne 0 ] && error "Used space for $qid isn't 0."
+
+	chown $TSTUSR.$TSTUSR $DIR/$tdir || error "chown $tdir failed"
+
+	mkdir $DIR/$tdir_dom || error "mkdir $tdir_dom failed"
+	$SETSTRIPE -E 1M -L mdt $DIR/$tdir_dom ||
+		error "setstripe $tdir_dom failed"
+	chown $TSTUSR.$TSTUSR $DIR/$tdir_dom || error "chown $tdir_dom failed"
+
+	[ $qtype == "p" ] && {
+		change_project -sp $TSTPRJID $DIR/$tdir
+		change_project -sp $TSTPRJID $DIR/$tdir_dom
+	}
+
+	$LFS setquota -$qtype $qid -b $LIMIT -B $LIMIT $DIR ||
+		error "set $qid quota failed"
+
+	for ((i = 0; i < $((LIMIT/2048)); i++)); do
+		$RUNAS $DD of=$DIR/$tdir_dom/$tfile-$i count=1 oflag=sync ||
+								dd_failed=true
+	done
+
+	$dd_failed && quota_error $qtype $qid "write failed, expect succeed"
+
+	for ((i = $((LIMIT/2048)); i < $((LIMIT/1024 + 10)); i++)); do
+		$RUNAS $DD of=$DIR/$tdir_dom/$tfile-$i count=1 oflag=sync ||
+								dd_failed=true
+	done
+
+	$dd_failed || quota_error $qtype $qid "write succeed, expect EDQUOT"
+
+	rm -f $DIR/$tdir_dom/*
+
+	# flush cache, ensure noquota flag is set on client
+	cancel_lru_locks osc
+	cancel_lru_locks mdc
+	sync; sync_all_data || true
+
+	dd_failed=false
+
+	$RUNAS $DD of=$DIR/$tdir/file count=$((LIMIT/2048)) oflag=sync ||
+		quota_error $qtype $qid "write failed, expect succeed"
+
+	for ((i = 0; i < $((LIMIT/2048 + 10)); i++)); do
+		$RUNAS $DD of=$DIR/$tdir_dom/$tfile-$i count=1 oflag=sync ||
+								dd_failed=true
+	done
+
+	$dd_failed || quota_error $qtype $TSTID "write succeed, expect EDQUOT"
+
+	rm -f $DIR/$tdir/*
+	rm -f $DIR/$tdir_dom/*
+
+	# flush cache, ensure noquota flag is set on client
+	cancel_lru_locks osc
+	cancel_lru_locks mdc
+	sync; sync_all_data || true
+
+	dd_failed=false
+
+	for ((i = 0; i < $((LIMIT/2048)); i++)); do
+		$RUNAS $DD of=$DIR/$tdir_dom/$tfile-$i count=1 oflag=sync ||
+								dd_failed=true
+	done
+
+	$dd_failed && quota_error $qtype $qid "write failed, expect succeed"
+
+	$RUNAS $DD of=$DIR/$tdir/file count=$((LIMIT/2048 + 10)) oflag=sync &&
+		quota_error $qtype $qid "write succeed, expect EDQUOT"
+
+	rm -f $DIR/$tdir/*
+	rm -fr $DIR/$tdir_dom
+
+	$LFS setquota -u $TSTUSR -b 0 -B 0 -i 0 -I 0 $DIR ||
+		error "reset usr quota failed"
+
+	cleanup_quota_test
+}
+
+test_63() {
+	test_dom "u"
+	test_dom "g"
+	test_dom "p"
 }
-run_test 36 "Migrate old admin files into new global indexes"
+run_test 63 "quota on DoM tests"
 
 quota_fini()
 {
-        do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=-quota"
+	do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=-quota"
+	disable_project_quota
 }
+reset_quota_settings
 quota_fini
 
 cd $ORIG_PWD