From d6f2a9fcfd8d3b14e05e3f1f660ba2ead9f81879 Mon Sep 17 00:00:00 2001 From: Niu Yawei Date: Thu, 20 Sep 2012 04:41:37 -0400 Subject: [PATCH] LU-1842 quota: new sanity-quota tests Added new sanity-quota.sh for the new quota architecture, meanwhile, the old sanity-quota.sh is reserved for the interoprability with old server. Removed the quotacheck tests in conf-sanity, since quotacheck is deprecated in the new quota architecture. Test-Parameters: envdefinitions=SLOW=yes testlist=sanity-quota Signed-off-by: Niu Yawei Change-Id: Ia3e3248a167e9f312a782aefea7af0aeced52220 Reviewed-on: http://review.whamcloud.com/4040 Tested-by: Hudson Reviewed-by: Yu Jian Reviewed-by: Fan Yong Reviewed-by: Johann Lombardi Tested-by: Maloo --- lustre/tests/Makefile.am | 1 + lustre/tests/admin_quotafile_v2.grp | Bin 0 -> 21504 bytes lustre/tests/admin_quotafile_v2.usr | Bin 0 -> 21504 bytes lustre/tests/conf-sanity.sh | 7 - lustre/tests/ost-pools.sh | 28 +- lustre/tests/sanity-quota-old.sh | 2411 +++++++++++++++++++++++++ lustre/tests/sanity-quota.sh | 3284 ++++++++++++++++------------------- lustre/tests/test-framework.sh | 16 +- 8 files changed, 3909 insertions(+), 1838 deletions(-) create mode 100644 lustre/tests/admin_quotafile_v2.grp create mode 100644 lustre/tests/admin_quotafile_v2.usr create mode 100755 lustre/tests/sanity-quota-old.sh diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am index 0836686..9eb7d6d 100644 --- a/lustre/tests/Makefile.am +++ b/lustre/tests/Makefile.am @@ -31,6 +31,7 @@ noinst_SCRIPTS += sgpdd-survey.sh maloo_upload.sh auster setup-nfs.sh noinst_SCRIPTS += mds-survey.sh parallel-scale-nfs.sh noinst_SCRIPTS += parallel-scale-nfsv3.sh parallel-scale-nfsv4.sh noinst_SCRIPTS += posix.sh sanity-scrub.sh scrub-performance.sh +noinst_SCRIPTS += sanity-quota-old.sh nobase_noinst_SCRIPTS = cfg/local.sh nobase_noinst_SCRIPTS += test-groups/regression test-groups/regression-mpi nobase_noinst_SCRIPTS += acl/make-tree acl/run cfg/ncli.sh diff --git a/lustre/tests/admin_quotafile_v2.grp b/lustre/tests/admin_quotafile_v2.grp new file mode 100644 index 0000000000000000000000000000000000000000..b5081a0d38ee8a0e8b59c9db4e91a622c4514448 GIT binary patch literal 21504 zcmeI(^>Wof6vy$4(_+ObTC~NBLyHtB4yCwD1$TESUfkV^ySo<|-ho%(CCdJio5DX! zXUG?rx#Y}jZ+5qn`P`G7>=Gb8Hvem}P$-m>6y-E?7^avd?&m!=7syATHe9WpU(RUz>_JDIgejY;h&fE;8Xl&ezG7>;brjY$%Z^yM~M4p z2IO(pP2m2S33;@Z68V`0dBh3g-)6wJ_tLm+@1=3u-b>@Qy_d$Lf6;7vFOA#wUK+RU zy)@Qy_d#qdoPXu&-*WL{dblT;CVS4dOqoXC7zdaAdhN? z@XUogs!PH%5Ax_)KzQau9zA~u&jQHf)EQ7;3*|B4cR%Z%#eGKWSkGM|UyGo8=~?fD z&;9LU$fNaz$j=hUqcx52EQRhTt)qlz8RT)^rvUY~9J*f4zpVj0E1>-7aw)v^zlJ)1 zZzYtknviD|A#+aXUg$g@Kp6MpN>`D+#6dhLYrlb|mJcyb|6bI7v`^0a_FyCIJ_3a|cK zssmiVJ&>;zi%a8hR zuK}R`4@3Ft0C|o;o{o^`DCFq`d5%Gz&XDIggGhCELok2ngi{>P{T+)q!T>o*qiJcIHz4)Q#Q zJmVqH3&=A8^1OsR6Cuwl$TJD@yoNmDD7^ZgtPW8BZy?_k$nzHROocq}AkQ?&^B(d{ zhdlX^Cj;_)fIOLy=c7C({O%v!vlRX})=-uPfcpOgb?c9K?Z^hL}&(zN9gwo zzE+TdAOl5bphVF>68uDvfgl4Bnt_rL`n`g$6=WciGEn*tewk9{+O|^0^w~;T(`PGD zrq5QQO`ollGkvyF-t^f@1=D9M6-}S5R5E?G5@Y&oCD!!WN@de$D^*ONt>}46@XW;| zA&9@80nHDEo&`FgiLD495K_F31h?614}}t)hA*kf>ERydE#sSc%PlB~Pjg#inuU8c z@tyFlm=~^xYI+}jes`WvnCmzTjXHkQlL>j8-O!9NkVhPaH~;IY0*vns zkgq=E*$8>!AkQYq6AyWEAWs8%6h8Oc&5);|EWmi)B995b=j}$i2;kWYwSqi*Ax~?_vk&sLfjs-=G2wTAYpaU@y0?pJ5Xa~#S~7szu0@^pnfCn1kG3UB^* zQw11LryySvF0~6kd0s-EiIC?NmtDT$%pcj0eL<^o=nK|5%P$m@ctiZmMXyg`vm#2A /dev/null + for j in `seq $OSTCOUNT`; do + do_facet ost$j "lctl set_param lquota.${FSNAME}-OST*.stats=0" > /dev/null + done + run_test "$@" + if [ ${STAT:-"yes"} != "no" -a -z "$LAST_SKIPPED" ]; then + echo "statistics info begin ***************************************" + do_facet $SINGLEMDS "lctl get_param lquota.mdd_obd-${FSNAME}-MDT*.stats" + for j in `seq $OSTCOUNT`; do + do_facet ost$j "lctl get_param lquota.${FSNAME}-OST*.stats" + done + echo "statistics info end ***************************************" + fi +} + +# +# clear quota limits for a user or a group +# usage: resetquota -u username +# resetquota -g groupname + +resetquota() { + [ "$#" != 2 ] && error "resetquota: wrong number of arguments: $#" + [ "$1" != "-u" -a "$1" != "-g" ] && error "resetquota: wrong specifier $1 passed" + + count=0 + if at_is_enabled; then + timeout=$(at_max_get mds) + else + timeout=$(lctl get_param -n timeout) + fi + + while [ $((count++)) -lt $timeout ]; do + local RC=0 + OUTPUT=`$LFS setquota "$1" "$2" -b 0 -B 0 -i 0 -I 0 $MOUNT 2>&1` || RC=${PIPESTATUS[0]} + if [ $RC -ne 0 ]; then + if echo "$OUTPUT" | grep -q busy; then + log "resetquota is blocked for quota master recovery, retry after $((count * 3)) sec" + sleep 3 + continue + else + error "resetquota failed" + fi + fi + break + done + + [ $count -lt $timeout ] || error "resetquota timeout: $timeout" +} + +quota_scan() { + LOCAL_UG=$1 + LOCAL_ID=$2 + + if [ "$LOCAL_UG" == "a" -o "$LOCAL_UG" == "u" ]; then + log "Files for user ($LOCAL_ID):" + ($LFS find -user $LOCAL_ID $DIR | xargs stat 2>/dev/null) + fi + + if [ "$LOCAL_UG" == "a" -o "$LOCAL_UG" == "g" ]; then + log "Files for group ($LOCAL_ID):" + ($LFS find -group $LOCAL_ID $DIR | xargs stat 2>/dev/null) + fi +} + +quota_error() { + quota_scan $1 $2 + shift 2 + error "$*" +} + +quota_log() { + quota_scan $1 $2 + shift 2 + log "$*" +} + +# +# get quota info for a user or a group +# usage: getquota -u|-g | global| bhardlimit|bsoftlimit|bgrace|ihardlimit|isoftlimit|igrace +# +getquota() { + local spec + local uuid + + [ "$#" != 4 ] && error "getquota: wrong number of arguments: $#" + [ "$1" != "-u" -a "$1" != "-g" ] && error "getquota: wrong u/g specifier $1 passed" + + uuid="$3" + + case "$4" in + curspace) spec=1;; + bsoftlimit) spec=2;; + bhardlimit) spec=3;; + bgrace) spec=4;; + curinodes) spec=5;; + isoftlimit) spec=6;; + ihardlimit) spec=7;; + igrace) spec=8;; + *) error "unknown quota parameter $4";; + esac + + [ "$uuid" = "global" ] && uuid=$DIR + + $LFS quota -v "$1" "$2" $DIR | awk 'BEGIN { num='$spec' } { if ($1 == "'$uuid'") { if (NF == 1) { getline } else { num++ } ; print $num;} }' | tr -d "*" +} + +quota_show_check() { + LOCAL_BF=$1 + LOCAL_UG=$2 + LOCAL_ID=$3 + PATTERN="`echo $DIR | sed 's/\//\\\\\//g'`" + + $LFS quota -v -$LOCAL_UG $LOCAL_ID $DIR + + if [ "$LOCAL_BF" == "a" -o "$LOCAL_BF" == "b" ]; then + USAGE=`getquota -$LOCAL_UG $LOCAL_ID global curspace` + if [ -z $USAGE ]; then + quota_error $LOCAL_UG $LOCAL_ID "System is error when query quota for block ($LOCAL_UG:$LOCAL_ID)." + else + [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for block ($LOCAL_UG:$LOCAL_ID:$USAGE)." + fi + fi + + if [ "$LOCAL_BF" == "a" -o "$LOCAL_BF" == "f" ]; then + USAGE=`getquota -$LOCAL_UG $LOCAL_ID global curinodes` + if [ -z $USAGE ]; then + quota_error $LOCAL_UG $LOCAL_ID "System is error when query quota for file ($LOCAL_UG:$LOCAL_ID)." + else + [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for file ($LOCAL_UG:$LOCAL_ID:$USAGE)." + fi + fi +} + +# set quota +quota_init() { + do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=+quota" + + log "do the quotacheck ..." + $LFS quotacheck -ug $DIR + + resetquota -u $TSTUSR + resetquota -g $TSTUSR +} +quota_init + +test_quota_performance() { + TESTFILE="$DIR/$tdir/$tfile-0" + local size=$1 + local stime=`date +%s` + $RUNAS dd if=/dev/zero of=$TESTFILE bs=1M count=$size || quota_error u $TSTUSR "write failure" + local etime=`date +%s` + delta=$((etime - stime)) + if [ $delta -gt 0 ]; then + rate=$((size * 1024 / delta)) + [ $rate -gt 1024 ] || error "SLOW IO for $TSTUSR (user): $rate KB/sec" + fi + rm -f $TESTFILE +} + +# test basic quota performance b=21696 +test_0() { + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + MB=100 + [ "$SLOW" = "no" ] && MB=10 + + test_quota_performance $MB + + $LFS setquota -u $TSTUSR -b 0 -B $((1024*1024)) -i 0 -I 0 $DIR + test_quota_performance $MB + + resetquota -u $TSTUSR +} +run_test_with_stat 0 "Test basic quota performance ===" + +# test for specific quota limitation, qunit, qtune $1=block_quota_limit +test_1_sub() { + LIMIT=$1 + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + TESTFILE="$DIR/$tdir/$tfile-0" + + wait_delete_completed + + # test for user + log " User quota (limit: $LIMIT kbytes)" + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + sleep 3 + quota_show_check b u $TSTUSR + + $LFS setstripe $TESTFILE -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + + log " Write ..." + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT/2)) || quota_error u $TSTUSR "(usr) write failure, but expect success" + log " Done" + log " Write out of block quota ..." + # this time maybe cache write, ignore it's failure + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT/2)) seek=$(($LIMIT/2)) || true + # flush cache, ensure noquota flag is setted on client + cancel_lru_locks osc + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$BUNIT_SZ seek=$LIMIT && quota_error u $TSTUSR "(usr) write success, but expect EDQUOT" + + rm -f $TESTFILE + sync; sleep 1; sync; + OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` + OST0_QUOTA_USED=`getquota -u $TSTUSR $OST0_UUID curspace` + echo $OST0_QUOTA_USED + [ $OST0_QUOTA_USED -ne 0 ] && \ + ($SHOW_QUOTA_USER; quota_error u $TSTUSR "(usr) quota deleted isn't released") + $SHOW_QUOTA_USER + resetquota -u $TSTUSR + + # test for group + log "--------------------------------------" + log " Group quota (limit: $LIMIT kbytes)" + $LFS setquota -g $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + sleep 3 + quota_show_check b g $TSTUSR + TESTFILE="$DIR/$tdir/$tfile-1" + + $LFS setstripe $TESTFILE -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + + log " Write ..." + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT/2)) || quota_error g $TSTUSR "(grp) write failure, but expect success" + log " Done" + log " Write out of block quota ..." + # this time maybe cache write, ignore it's failure + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT/2)) seek=$(($LIMIT/2)) || true + cancel_lru_locks osc + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$BUNIT_SZ seek=$LIMIT && quota_error g $TSTUSR "(grp) write success, but expect EDQUOT" + + # cleanup + rm -f $TESTFILE + sync; sleep 1; sync; + OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` + OST0_QUOTA_USED=`getquota -g $TSTUSR $OST0_UUID curspace` + echo $OST0_QUOTA_USED + [ $OST0_QUOTA_USED -ne 0 ] && \ + ($SHOW_QUOTA_GROUP; quota_error g $TSTUSR "(grp) quota deleted isn't released") + $SHOW_QUOTA_GROUP + resetquota -g $TSTUSR +} + +# block hard limit (normal use and out of quota) +test_1() { + for i in `seq 1 $cycle`; do + # define blk_qunit is between 1M and 4M + blk_qunit=$(( $RANDOM % 3072 + 1024 )) + blk_qtune=$(( $RANDOM % $blk_qunit )) + # other osts and mds will occupy at 1M blk quota + b_limit=$(( ($RANDOM - 16384) / 8 + ($OSTCOUNT + 1) * $blk_qunit * 4 )) + set_blk_tunesz $blk_qtune + set_blk_unitsz $blk_qunit + echo "cycle: $i(total $cycle) bunit:$blk_qunit, btune:$blk_qtune, blimit:$b_limit" + test_1_sub $b_limit + echo "==================================================" + set_blk_unitsz $((128 * 1024)) + set_blk_tunesz $((128 * 1024 / 2)) + done +} +run_test_with_stat 1 "Block hard limit (normal use and out of quota) ===" + +# test for specific quota limitation, qunit, qtune $1=block_quota_limit +test_2_sub() { + LIMIT=$1 + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + TESTFILE="$DIR/$tdir/$tfile-0" + + wait_delete_completed + + # test for user + log " User quota (limit: $LIMIT files)" + $LFS setquota -u $TSTUSR -b 0 -B 0 -i 0 -I $LIMIT $DIR + sleep 3 + quota_show_check f u $TSTUSR + + log " Create $LIMIT files ..." + $RUNAS createmany -m ${TESTFILE} $LIMIT || \ + quota_error u $TSTUSR "(usr) create failure, but expect success" + log " Done" + log " Create out of file quota ..." + $RUNAS touch ${TESTFILE}_xxx && \ + quota_error u $TSTUSR "(usr) touch success, but expect EDQUOT" + + unlinkmany ${TESTFILE} $LIMIT + rm -f ${TESTFILE}_xxx + sync; sleep 1; sync; + + MDS_UUID=`do_facet $SINGLEMDS $LCTL dl | grep -m1 " mdt " | awk '{print $((NF-1))}'` + MDS_QUOTA_USED=`getquota -u $TSTUSR $MDS_UUID curinodes` + echo $MDS_QUOTA_USED + [ $MDS_QUOTA_USED -ne 0 ] && \ + ($SHOW_QUOTA_USER; quota_error u $TSTUSR "(usr) quota deleted isn't released") + $SHOW_QUOTA_USER + resetquota -u $TSTUSR + + # test for group + log "--------------------------------------" + log " Group quota (limit: $LIMIT FILE)" + $LFS setquota -g $TSTUSR -b 0 -B 0 -i 0 -I $LIMIT $DIR + sleep 3 + quota_show_check f g $TSTUSR + TESTFILE=$DIR/$tdir/$tfile-1 + + log " Create $LIMIT files ..." + $RUNAS createmany -m ${TESTFILE} $LIMIT || \ + quota_error g $TSTUSR "(grp) create failure, but expect success" + log " Done" + log " Create out of file quota ..." + $RUNAS touch ${TESTFILE}_xxx && \ + quota_error g $TSTUSR "(grp) touch success, but expect EDQUOT" + + unlinkmany ${TESTFILE} $LIMIT + rm -f ${TESTFILE}_xxx + sync; sleep 1; sync; + + MDS_UUID=`do_facet $SINGLEMDS $LCTL dl | grep -m1 " mdt " | awk '{print $((NF-1))}'` + MDS_QUOTA_USED=`getquota -g $TSTUSR $MDS_UUID curinodes` + echo $MDS_QUOTA_USED + [ $MDS_QUOTA_USED -ne 0 ] && \ + ($SHOW_QUOTA_GROUP; quota_error g $TSTUSR "(grp) quota deleted isn't released") + $SHOW_QUOTA_GROUP + resetquota -g $TSTUSR +} + +# file hard limit (normal use and out of quota) +test_2() { + for i in `seq 1 $cycle`; do + if [ $i -eq 1 ]; then + ino_qunit=52 + ino_qtune=41 + i_limit=11 + else + # define ino_qunit is between 10 and 100 + ino_qunit=$(( $RANDOM % 90 + 10 )) + ino_qtune=$(( $RANDOM % $ino_qunit )) + # RANDOM's maxium is 32767 + i_limit=$(( $RANDOM % 990 + 10 )) + fi + + set_file_tunesz $ino_qtune + set_file_unitsz $ino_qunit + echo "cycle: $i(total $cycle) iunit:$ino_qunit, itune:$ino_qtune, ilimit:$i_limit" + test_2_sub $i_limit + echo "==================================================" + set_file_unitsz 5120 + set_file_tunesz 2560 + done +} +run_test_with_stat 2 "File hard limit (normal use and out of quota) ===" + +test_block_soft() { + TESTFILE=$1 + TIMER=$(($2 * 3 / 2)) + OFFSET=0 + + wait_delete_completed + + echo " Write to exceed soft limit" + RUNDD="$RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ" + $RUNDD count=$((BUNIT_SZ+1)) || \ + quota_error a $TSTUSR "write failure, but expect success" + OFFSET=$((OFFSET + BUNIT_SZ + 1)) + cancel_lru_locks osc + + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $SHOW_QUOTA_INFO_USER + $SHOW_QUOTA_INFO_GROUP + + echo " Write before timer goes off" + $RUNDD count=$BUNIT_SZ seek=$OFFSET || \ + quota_error a $TSTUSR "write failure, but expect success" + OFFSET=$((OFFSET + BUNIT_SZ)) + cancel_lru_locks osc + echo " Done" + + echo " Sleep $TIMER seconds ..." + sleep $TIMER + + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $SHOW_QUOTA_INFO_USER + $SHOW_QUOTA_INFO_GROUP + + echo " Write after timer goes off" + # maybe cache write, ignore. + $RUNDD count=$BUNIT_SZ seek=$OFFSET || true + OFFSET=$((OFFSET + BUNIT_SZ)) + cancel_lru_locks osc + $RUNDD count=$BUNIT_SZ seek=$OFFSET && \ + quota_error a $TSTUSR "write success, but expect EDQUOT" + + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $SHOW_QUOTA_INFO_USER + $SHOW_QUOTA_INFO_GROUP + + echo " Unlink file to stop timer" + rm -f $TESTFILE + sync; sleep 1; sync + echo " Done" + + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $SHOW_QUOTA_INFO_USER + $SHOW_QUOTA_INFO_GROUP + + echo " Write ..." + $RUNDD count=$BUNIT_SZ || quota_error a $TSTUSR "write failure, but expect success" + echo " Done" + + # cleanup + rm -f $TESTFILE + sync; sleep 3; sync; +} + +# block soft limit (start timer, timer goes off, stop timer) +test_3() { + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + + # 1 bunit on mds and 1 bunit on every ost + LIMIT=$(( $BUNIT_SZ * ($OSTCOUNT + 1) )) + GRACE=10 + + echo " User quota (soft limit: $LIMIT kbytes grace: $GRACE seconds)" + TESTFILE=$DIR/$tdir/$tfile-0 + + $LFS setstripe $TESTFILE -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + + $LFS setquota -t -u --block-grace $GRACE --inode-grace $MAX_IQ_TIME $DIR + $LFS setquota -u $TSTUSR -b $LIMIT -B 0 -i 0 -I 0 $DIR + + test_block_soft $TESTFILE $GRACE + resetquota -u $TSTUSR + + echo " Group quota (soft limit: $LIMIT kbytes grace: $GRACE seconds)" + TESTFILE=$DIR/$tdir/$tfile-1 + + $LFS setstripe $TESTFILE -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + + $LFS setquota -t -g --block-grace $GRACE --inode-grace $MAX_IQ_TIME $DIR + $LFS setquota -g $TSTUSR -b $LIMIT -B 0 -i 0 -I 0 $DIR + + test_block_soft $TESTFILE $GRACE + resetquota -g $TSTUSR +} +run_test_with_stat 3 "Block soft limit (start timer, timer goes off, stop timer) ===" + +test_file_soft() { + TESTFILE=$1 + LIMIT=$2 + TIMER=$(($3 * 3 / 2)) + + wait_delete_completed + + echo " Create files to exceed soft limit" + $RUNAS createmany -m ${TESTFILE}_ $((LIMIT + 1)) || \ + quota_error a $TSTUSR "create failure, but expect success" + sync; sleep 1; sync + echo " Done" + + echo " Create file before timer goes off" + $RUNAS touch ${TESTFILE}_before || \ + quota_error a $TSTUSR "failed create before timer expired, but expect success" + sync; sleep 1; sync + echo " Done" + + echo " Sleep $TIMER seconds ..." + sleep $TIMER + + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $SHOW_QUOTA_INFO_USER + $SHOW_QUOTA_INFO_GROUP + + echo " Create file after timer goes off" + # the least of inode qunit is 2, so there are at most 3(qunit:2+qtune:1) + # inode quota left here + $RUNAS touch ${TESTFILE}_after ${TESTFILE}_after1 ${TESTFILE}_after2 || true + sync; sleep 1; sync + $RUNAS touch ${TESTFILE}_after3 && \ + quota_error a $TSTUSR "create after timer expired, but expect EDQUOT" + sync; sleep 1; sync + + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $SHOW_QUOTA_INFO_USER + $SHOW_QUOTA_INFO_GROUP + + echo " Unlink files to stop timer" + find `dirname $TESTFILE` -name "`basename ${TESTFILE}`*" | xargs rm -f + echo " Done" + + echo " Create file" + $RUNAS touch ${TESTFILE}_xxx || \ + quota_error a $TSTUSR "touch after timer stop failure, but expect success" + sync; sleep 1; sync + echo " Done" + + # cleanup + rm -f ${TESTFILE}_xxx + sync; sleep 3; sync; +} + +# file soft limit (start timer, timer goes off, stop timer) +test_4a() { # was test_4 + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + LIMIT=$(($IUNIT_SZ * 10)) # 10 iunits on mds + TESTFILE=$DIR/$tdir/$tfile-0 + + GRACE=5 + + echo " User quota (soft limit: $LIMIT files grace: $GRACE seconds)" + $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace $GRACE $DIR + $LFS setquota -u $TSTUSR -b 0 -B 0 -i $LIMIT -I 0 $DIR + quota_show_check f u $TSTUSR + + test_file_soft $TESTFILE $LIMIT $GRACE + resetquota -u $TSTUSR + + echo " Group quota (soft limit: $LIMIT files grace: $GRACE seconds)" + $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace $GRACE $DIR + $LFS setquota -g $TSTUSR -b 0 -B 0 -i $LIMIT -I 0 $DIR + quota_show_check f g $TSTUSR + TESTFILE=$DIR/$tdir/$tfile-1 + + test_file_soft $TESTFILE $LIMIT $GRACE + resetquota -g $TSTUSR + + # cleanup + $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR + $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR +} +run_test_with_stat 4a "File soft limit (start timer, timer goes off, stop timer) ===" + +test_4b() { # was test_4a + GR_STR1="1w3d" + GR_STR2="1000s" + GR_STR3="5s" + GR_STR4="1w2d3h4m5s" + GR_STR5="5c" + GR_STR6="1111111111111111" + + wait_delete_completed + + # test of valid grace strings handling + echo " Valid grace strings test" + $LFS setquota -t -u --block-grace $GR_STR1 --inode-grace $GR_STR2 $DIR + $LFS quota -u -t $DIR | grep "Block grace time: $GR_STR1" + $LFS setquota -t -g --block-grace $GR_STR3 --inode-grace $GR_STR4 $DIR + $LFS quota -g -t $DIR | grep "Inode grace time: $GR_STR4" + + # test of invalid grace strings handling + echo " Invalid grace strings test" + ! $LFS setquota -t -u --block-grace $GR_STR4 --inode-grace $GR_STR5 $DIR + ! $LFS setquota -t -g --block-grace $GR_STR4 --inode-grace $GR_STR6 $DIR + + # cleanup + $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR + $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR +} +run_test_with_stat 4b "Grace time strings handling ===" + +# chown & chgrp (chown & chgrp successfully even out of block/file quota) +test_5() { + mkdir -p $DIR/$tdir + BLIMIT=$(( $BUNIT_SZ * $((OSTCOUNT + 1)) * 10)) # 10 bunits on each server + ILIMIT=$(( $IUNIT_SZ * 10 )) # 10 iunits on mds + + wait_delete_completed + + echo " Set quota limit (0 $BLIMIT 0 $ILIMIT) for $TSTUSR.$TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B $BLIMIT -i 0 -I $ILIMIT $DIR + $LFS setquota -g $TSTUSR -b 0 -B $BLIMIT -i 0 -I $ILIMIT $DIR + quota_show_check a u $TSTUSR + quota_show_check a g $TSTUSR + + echo " Create more than $ILIMIT files and more than $BLIMIT kbytes ..." + createmany -m $DIR/$tdir/$tfile-0_ $((ILIMIT + 1)) || \ + error "touch failure, expect success" + dd if=/dev/zero of=$DIR/$tdir/$tfile-0_1 bs=$BLK_SZ count=$((BLIMIT+1)) || error "write failure, expect success" + + echo " Chown files to $TSTUSR.$TSTUSR ..." + for i in `seq 0 $ILIMIT`; do + chown $TSTUSR.$TSTUSR $DIR/$tdir/$tfile-0_$i || \ + quota_error a $TSTUSR "chown failure, but expect success" + done + + # cleanup + unlinkmany $DIR/$tdir/$tfile-0_ $((ILIMIT + 1)) + sync; sleep 3; sync; + + resetquota -u $TSTUSR + resetquota -g $TSTUSR +} +run_test_with_stat 5 "Chown & chgrp successfully even out of block/file quota ===" + +# block quota acquire & release +test_6() { + if [ $OSTCOUNT -lt 2 ]; then + skip_env "$OSTCOUNT < 2, too few osts" + return 0; + fi + + wait_delete_completed + + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + + LIMIT=$((BUNIT_SZ * (OSTCOUNT + 1) * 5)) # 5 bunits per server + FILEA="$DIR/$tdir/$tfile-0_a" + FILEB="$DIR/$tdir/$tfile-0_b" + + echo " Set block limit $LIMIT kbytes to $TSTUSR.$TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + $LFS setquota -g $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + quota_show_check b u $TSTUSR + quota_show_check b g $TSTUSR + + echo " Create filea on OST0 and fileb on OST1" + $LFS setstripe $FILEA -i 0 -c 1 + $LFS setstripe $FILEB -i 1 -c 1 + chown $TSTUSR.$TSTUSR $FILEA + chown $TSTUSR.$TSTUSR $FILEB + + echo " Exceed quota limit ..." + RUNDD="$RUNAS dd if=/dev/zero of=$FILEA bs=$BLK_SZ" + $RUNDD count=$((LIMIT - BUNIT_SZ * OSTCOUNT)) || \ + quota_error a $TSTUSR "write filea failure, but expect success" + + cancel_lru_locks osc + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $RUNDD seek=$LIMIT count=$((BUNIT_SZ * OSTCOUNT)) && \ + quota_error a $TSTUSR "write filea success, but expect EDQUOT" + cancel_lru_locks osc + echo " Write to OST1 return EDQUOT" + # this write maybe cache write, ignore it's failure + RUNDD="$RUNAS dd if=/dev/zero of=$FILEB bs=$BLK_SZ" + $RUNDD count=$(($BUNIT_SZ * 2)) || true + cancel_lru_locks osc + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $RUNDD count=$((BUNIT_SZ * 2)) seek=$((BUNIT_SZ *2)) && \ + quota_error a $TSTUSR "write fileb success, but expect EDQUOT" + + echo " Remove filea to let OST0 release quota" + rm -f $FILEA + + if at_is_enabled; then + timeout=$(at_max_get mds) + else + timeout=$(lctl get_param -n timeout) + fi + count=$((timeout / 5)) + OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` + + while [ $((count--)) -gt 0 ]; do + sync && sleep 5 + + OST0_QUOTA_HOLD=`getquota -u $TSTUSR $OST0_UUID bhardlimit` + if [ -z $OST0_QUOTA_HOLD ]; then + error "System is error when query quota for block (U:$TSTUSR)." + else + [ $OST0_QUOTA_HOLD -gt $BUNIT_SZ ] && continue + fi + + break + done + + [ ! $count -gt 0 ] && error "Release quota for block timeout (U:$TSTUSR)." + $SHOW_QUOTA_USER + + while [ $((count--)) -gt 0 ]; do + sync && sleep 5 + + OST0_QUOTA_HOLD=`getquota -g $TSTUSR $OST0_UUID bhardlimit` + if [ -z $OST0_QUOTA_HOLD ]; then + error "System is error when query quota for block (G:$TSTUSR)." + else + [ $OST0_QUOTA_HOLD -gt $BUNIT_SZ ] && continue + fi + + break + done + + [ ! $count -gt 0 ] && error "Release quota for block timeout (G:$TSTUSR)." + $SHOW_QUOTA_GROUP + + echo " Write to OST1" + $RUNDD count=$((LIMIT - BUNIT_SZ * OSTCOUNT)) || \ + quota_error a $TSTUSR "write fileb failure, expect success" + echo " Done" + + # cleanup + rm -f $FILEB + sync; sleep 3; sync; + + resetquota -u $TSTUSR + resetquota -g $TSTUSR + return 0 +} +run_test_with_stat 6 "Block quota acquire & release =========" + +# quota recovery (block quota only by now) +test_7() +{ + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + + wait_delete_completed + + LIMIT=$(( $BUNIT_SZ * $(($OSTCOUNT + 1)) )) + TESTFILE="$DIR/$tdir/$tfile-0" + + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + + $LFS setstripe $TESTFILE -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + + echo " Write to OST0..." + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$BUNIT_SZ || \ + quota_error u $TSTUSR "write failure, but expect success" + + #define OBD_FAIL_OBD_DQACQ 0x604 + lustre_fail mds 0x604 + echo " Remove files on OST0" + rm -f $TESTFILE + lustre_fail mds 0 + + echo " Trigger recovery..." + OSC0_UUID="`$LCTL dl | awk '$3 ~ /osc/ { print $1 }'`" + for i in $OSC0_UUID; do + $LCTL --device $i activate || error "activate osc failed!" + done + + # sleep a while to wait for recovery done + sleep 20 + + # check limits + PATTERN="`echo $DIR | sed 's/\//\\\\\//g'`" + TOTAL_LIMIT=`getquota -u $TSTUSR global bhardlimit` + [ $TOTAL_LIMIT -eq $LIMIT ] || error "total limits not recovery!" + echo " total limits = $TOTAL_LIMIT" + + OST0_UUID=`do_facet ost1 "$LCTL dl | grep -m1 obdfilter" | awk '{print $((NF-1))}'` + [ -z "$OST0_UUID" ] && OST0_UUID=`do_facet ost1 "$LCTL dl | grep -m1 obdfilter" | awk '{print $((NF-1))}'` + OST0_LIMIT=`getquota -u $TSTUSR $OST0_UUID bhardlimit` + [ $OST0_LIMIT -eq $BUNIT_SZ ] || error "high limits not released!" + echo " limits on $OST0_UUID = $OST0_LIMIT" + + # cleanup + resetquota -u $TSTUSR +} +run_test_with_stat 7 "Quota recovery (only block limit) ======" + +# run dbench with quota enabled +test_8() { + mkdir -p $DIR/$tdir + BLK_LIMIT=$((100 * 1024 * 1024)) # 100G + FILE_LIMIT=1000000 + + wait_delete_completed + + echo " Set enough high limit for user: $TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + echo " Set enough high limit for group: $TSTUSR" + $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + + chmod 0777 $DIR/$tdir + local duration="" + [ "$SLOW" = "no" ] && duration=" -t 120" + $RUNAS bash rundbench -D $DIR/$tdir 3 $duration || quota_error a $TSTUSR "dbench failed!" + + rm -rf $DIR/$tdir + sync; sleep 3; sync; + + return 0 +} +run_test_with_stat 8 "Run dbench with quota enabled ===========" + +# run for fixing bug10707, it needs a big room. test for 64bit +KB=1024 +GB=$((KB * 1024 * 1024)) +# Use this as dd bs to decrease time +# inode->i_blkbits = min(PTLRPC_MAX_BRW_BITS+1, LL_MAX_BLKSIZE_BITS); +blksize=$((1 << 21)) # 2Mb +size_file=$((GB * 9 / 2)) +# this check is just for test_9 +OST0_MIN=4900000 #4.67G +check_whether_skip () { + OST0_SIZE=`$LFS df $DIR | awk '/\[OST:0\]/ {print $4}'` + log "OST0_SIZE: $OST0_SIZE required: $OST0_MIN" + if [ $OST0_SIZE -lt $OST0_MIN ]; then + echo "WARN: OST0 has less than $OST0_MIN free, skip this test." + return 0 + else + return 1 + fi +} + +test_9() { + check_whether_skip && return 0 + + wait_delete_completed + + set_blk_tunesz 512 + set_blk_unitsz 1024 + + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + TESTFILE="$DIR/$tdir/$tfile-0" + + BLK_LIMIT=$((100 * KB * KB)) # 100G + FILE_LIMIT=1000000 + echo " Set block limit $BLK_LIMIT kbytes to $TSTUSR.$TSTUSR" + + log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for user: $TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for group: $TSTUSR" + $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + + quota_show_check a u $TSTUSR + quota_show_check a g $TSTUSR + + echo " Set stripe" + $LFS setstripe $TESTFILE -c 1 -i 0 + touch $TESTFILE + chown $TSTUSR.$TSTUSR $TESTFILE + + log " Write the big file of 4.5G ..." + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$blksize count=$((size_file / blksize)) || \ + quota_error a $TSTUSR "(usr) write 4.5G file failure, but expect success" + + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + + log " delete the big file of 4.5G..." + $RUNAS rm -f $TESTFILE + sync; sleep 3; sync; + + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + + RC=$? + + set_blk_unitsz $((128 * 1024)) + set_blk_tunesz $((128 * 1024 / 2)) + + return $RC +} +run_test_with_stat 9 "run for fixing bug10707(64bit) ===========" + +# 2.0 version does not support 32 bit qd_count, +# test_10 "run for fixing bug10707(32bit) " is obsolete + +# test a deadlock between quota and journal b=11693 +test_12() { + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + + [ "$(grep $DIR2 /proc/mounts)" ] || mount_client $DIR2 || \ + { skip_env "Need lustre mounted on $MOUNT2 " && retutn 0; } + + LIMIT=$(( $BUNIT_SZ * $(($OSTCOUNT + 1)) * 10)) # 10 bunits each sever + TESTFILE="$DIR/$tdir/$tfile-0" + TESTFILE2="$DIR2/$tdir/$tfile-1" + + wait_delete_completed + + echo " User quota (limit: $LIMIT kbytes)" + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + + $LFS setstripe $TESTFILE -i 0 -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + $LFS setstripe $TESTFILE2 -i 0 -c 1 + chown $TSTUSR2.$TSTUSR2 $TESTFILE2 + + #define OBD_FAIL_OST_HOLD_WRITE_RPC 0x21f + #define OBD_FAIL_SOME 0x10000000 /* fail N times */ + lustre_fail ost $((0x0000021f | 0x10000000)) 1 + + echo " step1: write out of block quota ..." + $RUNAS2 dd if=/dev/zero of=$TESTFILE2 bs=$BLK_SZ count=102400 & + DDPID1=$! + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT*2)) & + DDPID=$! + + echo " step2: testing ......" + local last_size=$(stat -c %s $TESTFILE2) + local stall_secs=0 + local start_secs=$SECONDS + while [ -d /proc/${DDPID1} ]; do + local size=$(stat -c %s $TESTFILE2) + if [ $size -eq $last_size ]; then + stall_secs=$[stall_secs+1] + else + stall_secs=0 + fi + if [ $stall_secs -gt 30 ]; then + lustre_fail ost 0 + quota_error u $TSTUSR2 "giving up: dd stalled (i.e. made no progress) for 30 seconds!" + fi + last_size=$size + sleep 1 + done + echo "(dd_pid=$DDPID1, time=$((SECONDS-start_secs)))successful" + + #Recover fail_loc and dd will finish soon + lustre_fail ost 0 + + echo " step3: testing ......" + count=0 + while [ true ]; do + if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi + count=$[count+1] + if [ $count -gt 150 ]; then + quota_error u $TSTUSR "dd should be finished!" + fi + sleep 1 + done + echo "(dd_pid=$DDPID, time=$count)successful" + + rm -f $TESTFILE $TESTFILE2 + sync; sleep 3; sync; + + resetquota -u $TSTUSR +} +run_test_with_stat 12 "test a deadlock between quota and journal ===" + +# test multiple clients write block quota b=11693 +test_13() { + mkdir -p $DIR/$tdir + wait_delete_completed + + # one OST * 10 + (mds + other OSTs) + LIMIT=$((BUNIT_SZ * 10 + (BUNIT_SZ * OSTCOUNT))) + TESTFILE="$DIR/$tdir/$tfile" + + echo " User quota (limit: $LIMIT kbytes)" + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + quota_show_check b u $TSTUSR + + $LFS setstripe $TESTFILE -i 0 -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + $LFS setstripe $TESTFILE.2 -i 0 -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE.2 + + echo " step1: write out of block quota ..." + # one bunit will give mds + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$[($LIMIT - $BUNIT_SZ) / 2] & + DDPID=$! + $RUNAS dd if=/dev/zero of=$TESTFILE.2 bs=$BLK_SZ count=$[($LIMIT - $BUNIT_SZ) / 2] & + DDPID1=$! + + echo " step2: testing ......" + count=0 + while [ true ]; do + if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi + count=$[count+1] + if [ $count -gt 64 ]; then + quota_error u $TSTUSR "dd should be finished!" + fi + sleep 1 + done + echo "(dd_pid=$DDPID, time=$count)successful" + + count=0 + while [ true ]; do + if ! ps -p ${DDPID1} > /dev/null 2>&1 ; then break; fi + count=$[count+1] + if [ $count -gt 64 ]; then + quota_error u $TSTUSR "dd should be finished!" + fi + sleep 1 + done + echo "(dd_pid=$DDPID1, time=$count)successful" + + sync; sleep 5; sync; + + echo " step3: checking ......" + fz=`stat -c %s $TESTFILE` + fz2=`stat -c %s $TESTFILE.2` + $SHOW_QUOTA_USER + [ $((fz + fz2)) -lt $((BUNIT_SZ * BLK_SZ * 10)) ] && \ + quota_error u $TSTUSR "files too small $fz + $fz2 < $((BUNIT_SZ * BLK_SZ * 10))" + + rm -f $TESTFILE $TESTFILE.2 + sync; sleep 3; sync; + + resetquota -u $TSTUSR +} +run_test_with_stat 13 "test multiple clients write block quota ===" + +check_if_quota_zero(){ + line=`$LFS quota -v -$1 $2 $DIR | wc -l` + for i in `seq 3 $line`; do + if [ $i -eq 3 ]; then + field="3 4 7 8" + else + field="3 6" + fi + for j in $field; do + tmp=`$LFS quota -v -$1 $2 $DIR | sed -n ${i}p | + awk '{print $'"$j"'}'` + [ -n "$tmp" ] && [ $tmp -ne 0 ] && $LFS quota -v -$1 $2 $DIR && \ + error "quota on $2 isn't clean" + done + done + echo "pass check_if_quota_zero" +} + +test_14a() { # was test_14 b=12223 -- setting quota on root + TESTFILE="$DIR/$tdir/$tfile" + + # reboot the lustre + sync; sleep 5; sync + cleanup_and_setup_lustre + quota_init + + mkdir -p $DIR/$tdir + + # out of root's file and block quota + $LFS setquota -u root -b 10 -B 10 -i 10 -I 10 $DIR + createmany -m ${TESTFILE} 20 || \ + quota_error u root "unexpected: user(root) create files failly!" + $MULTIOP ${TESTFILE} oO_CREAT:O_WRONLY:O_DIRECT:w$((4096 * 4096))c || \ + quota_error u root "unexpected: user(root) write files failly!" + chmod 666 $TESTFILE + $RUNAS $MULTIOP ${TESTFILE} oO_WRONLY:O_APPEND:O_DIRECT:w$((4096 * 4096))c && \ + quota_error u root "unexpected: user(quota_usr) write a file successfully!" + + # trigger the llog + chmod 777 $DIR + for i in `seq 1 10`; do $RUNAS touch ${TESTFILE}a_$i; done + for i in `seq 1 10`; do $RUNAS rm -f ${TESTFILE}a_$i; done + + # do the check + dmesg | tail | grep "\-122" |grep llog_obd_origin_add && error "err -122 not found in dmesg" + resetquota -u root + #check_if_quota_zero u root + + # clean + unlinkmany ${TESTFILE} 15 + rm -f $TESTFILE + sync; sleep 3; sync; +} +run_test_with_stat 14a "test setting quota on root ===" + +test_15(){ + LIMIT=$((24 * 1024 * 1024 * 1024 * 1024)) # 24 TB + PATTERN="`echo $DIR | sed 's/\//\\\\\//g'`" + + wait_delete_completed + + # test for user + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + TOTAL_LIMIT=`getquota -u $TSTUSR global bhardlimit` + [ $TOTAL_LIMIT -eq $LIMIT ] || error " (user)total limits = $TOTAL_LIMIT; limit = $LIMIT, failed!" + echo " (user)total limits = $TOTAL_LIMIT; limit = $LIMIT, successful!" + resetquota -u $TSTUSR + + # test for group + $LFS setquota -g $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + TOTAL_LIMIT=`getquota -g $TSTUSR global bhardlimit` + [ $TOTAL_LIMIT -eq $LIMIT ] || error " (group)total limits = $TOTAL_LIMIT; limit = $LIMIT, failed!" + echo " (group)total limits = $TOTAL_LIMIT; limit = $LIMIT, successful!" + resetquota -g $TSTUSR + $LFS quotaoff -ug $DIR + do_facet $SINGLEMDS "lctl set_param lquota.mdd_obd-${FSNAME}-MDT*.quota_type=ug" | grep "error writing" && \ + error "fail to set version for $SINGLEMDS" + for j in `seq $OSTCOUNT`; do + do_facet ost$j "lctl set_param lquota.${FSNAME}-OST*.quota_type=ug" | grep "error writing" && \ + error "fail to set version for ost$j" + done + + $LFS quotacheck -ug $DIR +} +run_test_with_stat 15 "set block quota more than 4T ===" + +# 2.0 version does not support WITHOUT_CHANGE_QS, +# test_16 "test without adjusting qunit" is obsolete + +# run for fixing bug14526, failed returned quota reqs shouldn't ruin lustre. +test_17() { + set_blk_tunesz 512 + set_blk_unitsz 1024 + + wait_delete_completed + + #define OBD_FAIL_QUOTA_RET_QDATA | OBD_FAIL_ONCE + lustre_fail ost 0x80000A02 + + TESTFILE="$DIR/$tdir/$tfile-a" + TESTFILE2="$DIR/$tdir/$tfile-b" + mkdir -p $DIR/$tdir + + BLK_LIMIT=$((100 * 1024)) # 100M + + log " Set enough high limit(block:$BLK_LIMIT) for user: $TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR + log " Set enough high limit(block:$BLK_LIMIT) for group: $TSTUSR" + $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR + + quota_show_check b u $TSTUSR + quota_show_check b g $TSTUSR + + touch $TESTFILE + chown $TSTUSR.$TSTUSR $TESTFILE + touch $TESTFILE2 + chown $TSTUSR.$TSTUSR $TESTFILE2 + + log " Write the test file1 ..." + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(( 10 * 1024 )) \ + || quota_error a $TSTUSR "write 10M file failure" + + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + + log " write the test file2 ..." + $RUNAS dd if=/dev/zero of=$TESTFILE2 bs=$BLK_SZ count=$(( 10 * 1024 )) \ + || quota_error a $TSTUSR "write 10M file failure" + + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + + rm -f $TESTFILE $TESTFILE2 + RC=$? + sync; sleep 3; sync; + + # make qd_count 64 bit + lustre_fail ost 0 + + set_blk_unitsz $((128 * 1024)) + set_blk_tunesz $((128 * 1024 / 2)) + + resetquota -u $TSTUSR + resetquota -g $TSTUSR + + return $RC +} +run_test_with_stat 17 "run for fixing bug14526 ===========" + +# test when mds takes a long time to handle a quota req so that +# the ost has dropped it, the ost still could work well b=14840 +test_18() { + LIMIT=$((100 * 1024 * 1024)) # 100G + TESTFILE="$DIR/$tdir/$tfile" + mkdir -p $DIR/$tdir + + wait_delete_completed + + set_blk_tunesz 512 + set_blk_unitsz 1024 + + log " User quota (limit: $LIMIT kbytes)" + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT + quota_show_check b u $TSTUSR + + $LFS setstripe $TESTFILE -i 0 -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + + #define OBD_FAIL_MDS_BLOCK_QUOTA_REQ 0x13c + lustre_fail mds 0x13c + + log " step1: write 100M block ..." + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$((1024 * 100)) & + DDPID=$! + + sleep 5 + lustre_fail mds 0 + + echo " step2: testing ......" + count=0 + if at_is_enabled; then + timeout=$(at_max_get mds) + else + timeout=$(lctl get_param -n timeout) + fi + while [ true ]; do + if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi + count=$[count+1] + if [ $count -gt $((4 * $timeout)) ]; then + quota_error u $TSTUSR "count=$count dd should be finished!" + fi + sleep 1 + done + log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" + sync + cancel_lru_locks mdc + cancel_lru_locks osc + + testfile_size=$(stat -c %s $TESTFILE) + [ $testfile_size -ne $((BLK_SZ * 1024 * 100)) ] && \ + quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100)), got ${testfile_size}. Verifying file failed!" + $SHOW_QUOTA_USER + rm -f $TESTFILE + sync + + resetquota -u $TSTUSR + set_blk_unitsz $((128 * 1024)) + set_blk_tunesz $((128 * 1024 / 2)) +} +run_test_with_stat 18 "run for fixing bug14840 ===========" + +# test when mds drops a quota req, the ost still could work well b=14840 +test_18a() { + LIMIT=$((100 * 1024 * 1024)) # 100G + TESTFILE="$DIR/$tdir/$tfile-a" + mkdir -p $DIR/$tdir + + wait_delete_completed + + set_blk_tunesz 512 + set_blk_unitsz 1024 + + log " User quota (limit: $LIMIT kbytes)" + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT + quota_show_check b u $TSTUSR + + $LFS setstripe $TESTFILE -i 0 -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + + #define OBD_FAIL_MDS_DROP_QUOTA_REQ | OBD_FAIL_ONCE 0x8000013d + lustre_fail mds 0x8000013d + + log " step1: write 100M block ..." + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$((1024 * 100)) & + DDPID=$! + + echo " step2: testing ......" + count=0 + if at_is_enabled; then + timeout=$(at_max_get mds) + else + timeout=$(lctl get_param -n timeout) + fi + while [ true ]; do + if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi + count=$[count+1] + if [ $count -gt $((6 * $timeout)) ]; then + lustre_fail mds 0 + quota_error u $TSTUSR "count=$count dd should be finished!" + fi + sleep 1 + done + log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" + + lustre_fail mds 0 + rm -f $TESTFILE + sync + + resetquota -u $TSTUSR + set_blk_unitsz $((128 * 1024)) + set_blk_tunesz $((128 * 1024 / 2)) +} +run_test_with_stat 18a "run for fixing bug14840 ===========" + +# test when mds do failover, the ost still could work well without trigger +# watchdog b=14840 +test_18bc_sub() { + type=$1 + + LIMIT=$(((100 + $OSTCOUNT * 3) * 1024)) + TESTFILE="$DIR/$tdir/$tfile" + mkdir -p $DIR/$tdir + + wait_delete_completed + + set_blk_tunesz 512 + set_blk_unitsz 1024 + + log " User quota (limit: $LIMIT kbytes)" + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT + quota_show_check b u $TSTUSR + + $LFS setstripe $TESTFILE -i 0 -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + + timeout=$(sysctl -n lustre.timeout) + + if [ $type = "directio" ]; then + log " write 100M block(directio) ..." + $RUNAS $DIRECTIO write $TESTFILE 0 100 $((BLK_SZ * 1024)) & + else + log " write 100M block(normal) ..." + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$((BLK_SZ * 1024)) count=100 & + fi + + DDPID=$! + do_facet $SINGLEMDS "$LCTL conf_param ${FSNAME}-MDT*.mdd.quota_type=ug" + + replay_barrier $SINGLEMDS + + log "failing mds for $((2 * timeout)) seconds" + fail $SINGLEMDS $((2 * timeout)) + + # check if quotaon successful + $LFS quota -u $TSTUSR $MOUNT 2>&1 | grep -q "quotas are not enabled" + if [ $? -eq 0 ]; then + rm -rf $TESTFILE + error "quotaon failed!" + return + fi + + count=0 + if at_is_enabled; then + timeout=$(at_max_get mds) + else + timeout=$(lctl get_param -n timeout) + fi + while [ true ]; do + if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi + if [ $((++count % (2 * timeout) )) -eq 0 ]; then + log "it took $count second" + fi + sleep 1 + done + log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" + sync + cancel_lru_locks mdc + cancel_lru_locks osc + $SHOW_QUOTA_USER + + resetquota -u $TSTUSR + set_blk_unitsz $((128 * 1024)) + set_blk_tunesz $((128 * 1024 / 2)) + testfile_size=$(stat -c %s $TESTFILE) + if [ $testfile_size -ne $((BLK_SZ * 1024 * 100)) ] ; then + rm -f $TESTFILE + quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100)), got ${testfile_size}. Verifying file failed!" + fi + rm -f $TESTFILE + +} + +# test when mds does failover, the ost still could work well +# this test shouldn't trigger watchdog b=14840 +test_18b() { + test_18bc_sub normal + test_18bc_sub directio + # check if watchdog is triggered + do_facet ost1 dmesg > $TMP/lustre-log-${TESTNAME}.log + watchdog=`awk '/test 18b/ {start = 1;} + /Service thread pid/ && /was inactive/ { + if (start) { + print; + } + }' $TMP/lustre-log-${TESTNAME}.log` + [ `echo "$watchdog" | wc -l` -ge 3 ] && error "$watchdog" + rm -f $TMP/lustre-log-${TESTNAME}.log +} +run_test_with_stat 18b "run for fixing bug14840(mds failover, no watchdog) ===========" + +# test when mds does failover, the ost still could work well +# this test will prevent OST_DISCONNET from happening b=14840 +test_18c() { + # define OBD_FAIL_OST_DISCONNECT_NET 0x202(disable ost_disconnect for osts) + lustre_fail ost 0x202 + test_18bc_sub normal + test_18bc_sub directio + lustre_fail ost 0 +} +run_test_with_stat 18c "run for fixing bug14840(mds failover, OST_DISCONNECT is disabled) ===========" + +run_to_block_limit() { + local LIMIT=$((($OSTCOUNT + 1) * $BUNIT_SZ)) + local TESTFILE=$1 + wait_delete_completed + + # set 1 Mb quota unit size + set_blk_tunesz 512 + set_blk_unitsz 1024 + + # bind file to a single OST + $LFS setstripe -c 1 $TESTFILE + chown $TSTUSR.$TSTUSR $TESTFILE + + echo " User quota (limit: $LIMIT kbytes)" + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT + quota_show_check b u $TSTUSR + echo " Updating quota limits" + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT + quota_show_check b u $TSTUSR + + RUNDD="$RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ" + $RUNDD count=$BUNIT_SZ || quota_error u $TSTUSR "(usr) write failure, but expect success" + # for now page cache of TESTFILE may still be dirty, + # let's push it to the corresponding OST, this will also + # cache NOQUOTA on the client from OST's reply + cancel_lru_locks osc + $RUNDD seek=$BUNIT_SZ && quota_error u $TSTUSR "(usr) write success, should be EDQUOT" +} + +test_19() { + # 1 Mb bunit per each MDS/OSS + local TESTFILE="$DIR/$tdir/$tfile" + mkdir -p $DIR/$tdir + + run_to_block_limit $TESTFILE + $SHOW_QUOTA_USER + + # cleanup + rm -f $TESTFILE + resetquota -u $TSTUSR + + set_blk_unitsz $((128 * 1024)) + set_blk_tunesz $((128 * 1024 / 2)) + +} +run_test_with_stat 19 "test if administrative limits updates do not zero operational limits (14790) ===" + +test_20() +{ + LSTR=(1t 2g 3m 4k) # limits strings + LVAL=($[1*1024*1024*1024] $[2*1024*1024] $[3*1024*1024] $[4*1024]) # limits values + + $LFS setquota -u $TSTUSR --block-softlimit ${LSTR[0]} \ + $MOUNT || error "could not set quota limits" + + $LFS setquota -u $TSTUSR --block-hardlimit ${LSTR[1]} \ + --inode-softlimit ${LSTR[2]} \ + --inode-hardlimit ${LSTR[3]} \ + $MOUNT || error "could not set quota limits" + + [ "`getquota -u $TSTUSR global bsoftlimit`" = "${LVAL[0]}" ] || error "bsoftlimit was not set properly" + [ "`getquota -u $TSTUSR global bhardlimit`" = "${LVAL[1]}" ] || error "bhardlimit was not set properly" + [ "`getquota -u $TSTUSR global isoftlimit`" = "${LVAL[2]}" ] || error "isoftlimit was not set properly" + [ "`getquota -u $TSTUSR global ihardlimit`" = "${LVAL[3]}" ] || error "ihardlimit was not set properly" + + resetquota -u $TSTUSR +} +run_test_with_stat 20 "test if setquota specifiers work properly (15754)" + +test_21_sub() { + local testfile=$1 + local blk_number=$2 + local seconds=$3 + + time=$(($(date +%s) + seconds)) + while [ $(date +%s) -lt $time ]; do + $RUNAS dd if=/dev/zero of=$testfile bs=$BLK_SZ count=$blk_number > /dev/null 2>&1 + rm -f $testfile + done +} + +# run for fixing bug16053, setquota shouldn't fail when writing and +# deleting are happening +test_21() { + set_blk_tunesz 512 + set_blk_unitsz 1024 + + wait_delete_completed + + TESTFILE="$DIR/$tdir/$tfile" + + BLK_LIMIT=$((10 * 1024 * 1024)) # 10G + FILE_LIMIT=1000000 + + log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for user: $TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $MOUNT + log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for group: $TSTUSR" + $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $MOUNT + + # repeat writing on a 1M file + test_21_sub ${TESTFILE}_1 1024 30 & + DDPID1=$! + # repeat writing on a 128M file + test_21_sub ${TESTFILE}_2 $((1024 * 128)) 30 & + DDPID2=$! + + time=$(($(date +%s) + 30)) + i=1 + while [ $(date +%s) -lt $time ]; do + log " Set quota for $i times" + $LFS setquota -u $TSTUSR -b 0 -B $((BLK_LIMIT + 1024 * i)) -i 0 -I $((FILE_LIMIT + i)) $MOUNT + $LFS setquota -g $TSTUSR -b 0 -B $((BLK_LIMIT + 1024 * i)) -i 0 -I $((FILE_LIMIT + i)) $MOUNT + i=$((i+1)) + sleep 1 + done + + count=0 + while [ true ]; do + if ! ps -p ${DDPID1} > /dev/null 2>&1; then break; fi + count=$[count+1] + if [ $count -gt 60 ]; then + quota_error a $TSTUSR "dd should be finished!" + fi + sleep 1 + done + echo "(dd_pid=$DDPID1, time=$count)successful" + + count=0 + while [ true ]; do + if ! ps -p ${DDPID2} > /dev/null 2>&1; then break; fi + count=$[count+1] + if [ $count -gt 60 ]; then + quota_error a $TSTUSR "dd should be finished!" + fi + sleep 1 + done + echo "(dd_pid=$DDPID2, time=$count)successful" + + set_blk_unitsz $((128 * 1024)) + set_blk_tunesz $((128 * 1024 / 2)) + resetquota -u $TSTUSR + resetquota -g $TSTUSR + + return $RC +} +run_test_with_stat 21 "run for fixing bug16053 ===========" + +test_22() { + quota_save_version "ug3" + + stopall + mount + setupall + + echo "checking parameters" + + do_facet $SINGLEMDS "lctl get_param mdd.${FSNAME}-MDT*.quota_type" | grep "ug3" || error "admin failure" + do_facet ost1 "lctl get_param obdfilter.*.quota_type" | grep "ug3" || error "op failure" + + quota_init +} +run_test_with_stat 22 "test if quota_type saved as permanent parameter ====" + +test_23_sub() { + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + TESTFILE="$DIR/$tdir/$tfile-0" + rm -f $TESTFILE + local bs_unit=$((1024*1024)) + LIMIT=$1 + + wait_delete_completed + + # test for user + log " User quota (limit: $LIMIT kbytes)" + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + sleep 3 + quota_show_check b u $TSTUSR + + $LFS setstripe $TESTFILE -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE + + log " Step1: trigger quota with 0_DIRECT" + log " Write half of file" + $RUNAS $DIRECTIO write $TESTFILE 0 $(($LIMIT/1024/2)) $bs_unit || \ + quota_error u $TSTUSR "(1) write failure, but expect success: $LIMIT" + log " Write out of block quota ..." + $RUNAS $DIRECTIO write $TESTFILE $(($LIMIT/1024/2)) $(($LIMIT/1024/2)) $bs_unit && \ + quota_error u $TSTUSR "(2) write success, but expect EDQUOT: $LIMIT" + log " Step1: done" + + log " Step2: rewrite should succeed" + $RUNAS $DIRECTIO write $TESTFILE 0 1 $bs_unit || \ + quota_error u $TSTUSR "(3) write failure, but expect success: $LIMIT" + log " Step2: done" + + rm -f $TESTFILE + wait_delete_completed + OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` + OST0_QUOTA_USED=`getquota -u $TSTUSR $OST0_UUID curspace` + echo $OST0_QUOTA_USED + [ $OST0_QUOTA_USED -ne 0 ] && \ + ($SHOW_QUOTA_USER; quota_error u $TSTUSR "quota deleted isn't released") + $SHOW_QUOTA_USER + resetquota -u $TSTUSR +} + +test_23() { + local slave_cnt=$((OSTCOUNT + 1)) # 1 mds, n osts + + OST0_MIN=$((6 * $slave_cnt * 1024)) # extra space for meta blocks. + check_whether_skip && return 0 + log "run for $((4 * $slave_cnt))MB test file" + test_23_sub $((4 * $slave_cnt * 1024)) + + OST0_MIN=$((60 * $slave_cnt * 1024)) # extra space for meta blocks. + check_whether_skip && return 0 + log "run for $((40 * $slave_cnt))MB test file" + test_23_sub $((40 * $slave_cnt * 1024)) +} +run_test_with_stat 23 "run for fixing bug16125 ===========" + +test_24() { + local TESTFILE="$DIR/$tdir/$tfile" + mkdir -p $DIR/$tdir + + run_to_block_limit $TESTFILE + $SHOW_QUOTA_USER | grep '*' || error "no matching *" + + # cleanup + rm -f $TESTFILE + resetquota -u $TSTUSR + + set_blk_unitsz $((128 * 1024)) + set_blk_tunesz $((128 * 1024 / 2)) + +} +run_test_with_stat 24 "test if lfs draws an asterix when limit is reached (16646) ===========" + +show_quota() { + if [ $1 = "-u" ]; then + if [ $2 = "$TSTUSR" ]; then + $SHOW_QUOTA_USER + else + $SHOW_QUOTA_USER2 + fi + else + if [ $2 = "$TSTUSR" ]; then + $SHOW_QUOTA_GROUP + else + $SHOW_QUOTA_GROUP2 + fi + fi +} + +test_25_sub() { + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + TESTFILE="$DIR/$tdir/$tfile-0" + rm -f $TESTFILE + LIMIT=$(( $BUNIT_SZ * ($OSTCOUNT + 1) + 4096 )) + + wait_delete_completed + + # set quota for $TSTUSR + log "setquota for $TSTUSR" + $LFS setquota $1 $TSTUSR -b $LIMIT -B $LIMIT -i 10 -I 10 $DIR + sleep 3 + if [ "$1" == "-u" ]; then + quota_show_check a u $TSTUSR + else + quota_show_check a g $TSTUSR + fi + + # set quota for $TSTUSR2 + log "setquota for $TSTUSR2" + $LFS setquota $1 $TSTUSR2 -b $LIMIT -B $LIMIT -i 10 -I 10 $DIR + sleep 3 + if [ "$1" == "-u" ]; then + quota_show_check a u $TSTUSR2 + else + quota_show_check a g $TSTUSR2 + fi + + # set stripe index to 0 + log "setstripe for $DIR/$tdir to 0" + $LFS setstripe $DIR/$tdir -c 1 -i 0 + MDS_UUID=`do_facet $SINGLEMDS $LCTL dl | grep -m1 " mdt " | awk '{print $((NF-1))}'` + OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` + MDS_QUOTA_USED_OLD=`getquota $1 $TSTUSR $MDS_UUID curinodes` + OST0_QUOTA_USED_OLD=`getquota $1 $TSTUSR $OST0_UUID curspace` + MDS_QUOTA_USED2_OLD=`getquota $1 $TSTUSR2 $MDS_UUID curinodes` + OST0_QUOTA_USED2_OLD=`getquota $1 $TSTUSR2 $OST0_UUID curspace` + + # TSTUSR write 4M + log "$TSTUSR write 4M to $TESTFILE" + $RUNAS dd if=/dev/zero of=$TESTFILE bs=4K count=1K || quota_error a $TSTUSR "dd failed" + sync + show_quota $1 $TSTUSR + show_quota $1 $TSTUSR2 + MDS_QUOTA_USED_NEW=`getquota $1 $TSTUSR $MDS_UUID curinodes` + [ $MDS_QUOTA_USED_NEW -ne $((MDS_QUOTA_USED_OLD + 1)) ] && \ + quota_error a $TSTUSR "$TSTUSR inode quota usage error: [$MDS_QUOTA_USED_OLD|$MDS_QUOTA_USED_NEW]" + OST0_QUOTA_USED_NEW=`getquota $1 $TSTUSR $OST0_UUID curspace` + OST0_QUOTA_USED_DELTA=$((OST0_QUOTA_USED_NEW - OST0_QUOTA_USED_OLD)) + [ $OST0_QUOTA_USED_DELTA -lt 4096 ] && \ + quota_error a $TSTUSR "$TSTUSR block quota usage error: [$OST0_QUOTA_USED_OLD|$OST0_QUOTA_USED_NEW]" + + # chown/chgrp from $TSTUSR to $TSTUSR2 + if [ $1 = "-u" ]; then + log "chown from $TSTUSR to $TSTUSR2" + chown $TSTUSR2 $TESTFILE || quota_error u $TSTUSR2 "chown failed" + else + log "chgrp from $TSTUSR to $TSTUSR2" + chgrp $TSTUSR2 $TESTFILE || quota_error g $TSTUSR2 "chgrp failed" + fi + sync + show_quota $1 $TSTUSR + show_quota $1 $TSTUSR2 + MDS_QUOTA_USED2_NEW=`getquota $1 $TSTUSR2 $MDS_UUID curinodes` + [ $MDS_QUOTA_USED2_NEW -ne $((MDS_QUOTA_USED2_OLD + 1)) ] && \ + quota_error a $TSTUSR2 "$TSTUSR2 inode quota usage transfer from $TSTUSR to $TSTUSR2 failed: [$MDS_QUOTA_USED2_OLD|$MDS_QUOTA_USED2_NEW]" + OST0_QUOTA_USED2_NEW=`getquota $1 $TSTUSR2 $OST0_UUID curspace` + # when chown, the quota on ost could be displayed out of quota temporarily. Delete the '*' in this situation. b=20433 + OST0_QUOTA_USED2_NEW=${OST0_QUOTA_USED2_NEW%\*} + OST0_QUOTA_USED2_DELTA=$((OST0_QUOTA_USED2_NEW - OST0_QUOTA_USED2_OLD)) + [ $OST0_QUOTA_USED2_DELTA -ne $OST0_QUOTA_USED_DELTA ] && \ + quota_error a $TSTUSR2 "$TSTUSR2 block quota usage transfer from $TSTUSR to $TSTUSR2 failed: [$OST0_QUOTA_USED2_OLD|$OST0_QUOTA_USED2_NEW]" + MDS_QUOTA_USED_NEW=`getquota $1 $TSTUSR $MDS_UUID curinodes` + [ $MDS_QUOTA_USED_NEW -ne $MDS_QUOTA_USED_OLD ] && \ + quota_error a $TSTUSR "$TSTUSR inode quota usage transfer from $TSTUSR to $TSTUSR2 failed: [$MDS_QUOTA_USED_OLD|$MDS_QUOTA_USED_NEW]" + OST0_QUOTA_USED_NEW=`getquota $1 $TSTUSR $OST0_UUID curspace` + [ $OST0_QUOTA_USED_NEW -ne $OST0_QUOTA_USED_OLD ] && \ + quota_error a $TSTUSR "$TSTUSR block quota usage transfer from $TSTUSR to $TSTUSR2 failed: [$OST0_QUOTA_USED_OLD|$OST0_QUOTA_USED_NEW]" + + rm -f $TESTFILE + wait_delete_completed + resetquota $1 $TSTUSR + resetquota $1 $TSTUSR2 +} + +test_25() { + log "run for chown case" + test_25_sub -u + + log "run for chgrp case" + test_25_sub -g +} +run_test_with_stat 25 "test whether quota usage is transfered when chown/chgrp (18081) ===========" + +test_26() { + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + TESTFILE="$DIR/$tdir/$tfile-0" + TESTFILE2="$DIR/$tdir/$tfile-1" + set_blk_tunesz 512 + set_blk_unitsz 1024 + + wait_delete_completed + + # every quota slave gets 20MB + b_limit=$(((OSTCOUNT + 1) * 20 * 1024)) + log "limit: ${b_limit}KB" + $LFS setquota -u $TSTUSR -b 0 -B $b_limit -i 0 -I 0 $DIR + sleep 3 + quota_show_check b u $TSTUSR + + $LFS setstripe $TESTFILE -c 1 -i 0 + $LFS setstripe $TESTFILE2 -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE + chown $TSTUSR.$TSTUSR $TESTFILE2 + + #define OBD_FAIL_QUOTA_DELAY_REL 0xA03 + lustre_fail ost 0xA03 + + log " Write the first file..." + $RUNAS $DIRECTIO write $TESTFILE 0 10 $((BLK_SZ * 1024)) || quota_error u $TSTUSR "write failure, but expect success" + log " Delete the first file..." + rm -f $TESTFILE + + + wait_delete_completed + + log " Write the second file..." + $RUNAS $DIRECTIO write $TESTFILE2 0 10 $((BLK_SZ * 1024)) || quota_error u $TSTUSR "write failure, but expect success" + log " Delete the second file..." + rm -f $TESTFILE2 + + lustre_fail ost 0 + set_blk_unitsz $((128 * 1024)) + set_blk_tunesz $((128 * 1024 / 2)) + resetquota -u $TSTUSR +} +run_test_with_stat 26 "test for false quota error(bz18491) ======================================" + +test_27a() { + $LFS quota $TSTUSR $DIR && error "lfs succeeded with no type, but should have failed" + $LFS setquota $TSTUSR $DIR && error "lfs succeeded with no type, but should have failed" + return 0 +} +run_test_with_stat 27a "lfs quota/setquota should handle wrong arguments (19612) =================" + +test_27b() { + $LFS setquota -u $TSTID -b 1000 -B 1000 -i 1000 -I 1000 $DIR || \ + error "lfs setquota failed with uid argument" + $LFS setquota -g $TSTID -b 1000 -B 1000 -i 1000 -I 1000 $DIR || \ + error "lfs stequota failed with gid argument" + $SHOW_QUOTA_USERID || error "lfs quota failed with uid argument" + $SHOW_QUOTA_GROUPID || error "lfs quota failed with gid argument" + resetquota -u $TSTUSR + resetquota -g $TSTUSR + return 0 +} +run_test 27b "lfs quota/setquota should handle user/group ID (20200) =================" + +test_28() { + BLK_LIMIT=$((100 * 1024 * 1024)) # 100G + echo "Step 1: set enough high limit for user [$TSTUSR:$BLK_LIMIT]" + $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR + $SHOW_QUOTA_USER + + echo "Step 2: reset system ..." + cleanup_and_setup_lustre + quota_init + + echo "Step 3: change qunit for user [$TSTUSR:512:1024]" + set_blk_tunesz 512 + set_blk_unitsz 1024 + + wait_delete_completed + + #define OBD_FAIL_QUOTA_RET_QDATA | OBD_FAIL_ONCE + lustre_fail ost 0x80000A02 + + TESTFILE="$DIR/$tdir/$tfile" + mkdir -p $DIR/$tdir + + BLK_LIMIT=$((100 * 1024)) # 100M + echo "Step 4: set enough high limit for user [$TSTUSR:$BLK_LIMIT]" + $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR + $SHOW_QUOTA_USER + + touch $TESTFILE + chown $TSTUSR.$TSTUSR $TESTFILE + + echo "Step 5: write the test file1 [10M] ..." + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(( 10 * 1024 )) \ + || quota_error a $TSTUSR "write 10M file failure" + $SHOW_QUOTA_USER + + rm -f $TESTFILE + sync; sleep 3; sync; + + # make qd_count 64 bit + lustre_fail ost 0 + + set_blk_unitsz $((128 * 1024)) + set_blk_tunesz $((128 * 1024 / 2)) + + resetquota -u $TSTUSR +} +run_test_with_stat 28 "test for consistency for qunit when setquota (18574) ===========" + +test_29() +{ + local BLK_LIMIT=$((100 * 1024 * 1024)) # 100G + local timeout + local at_max_val + local pid + local waited + local deadline + + at_max_val=10 + + if at_is_enabled; then + timeout=$(at_max_get client) + [ $(at_min_get client) -gt $at_max_val ] && + at_max_val=$(at_min_get client) + at_max_set $at_max_val client + else + timeout=$(lctl get_param -n timeout) + lctl set_param timeout=$at_max_val + fi + # actually send a RPC to make service at_current confined within at_max + $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR || + error "should succeed" + + #define OBD_FAIL_MDS_QUOTACTL_NET 0x12e + lustre_fail mds 0x12e + + $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR & pid=$! + + deadline=$((2 * (($at_max_val << 1) + ($at_max_val >> 2) + 5))) + echo "wait at most 2 * ($at_max_val * 2.25 + 5) = $deadline seconds," \ + "it is server process time add the network latency." + + waited=0 + while [ $waited -lt $deadline ]; do + echo -n "." + sleep 1 + waited=$(($waited + 1)) + ps -p $pid > /dev/null || break + done + echo "waited $waited seconds" + ps -p $pid && error "lfs hadn't finished by $deadline seconds" + wait $pid && error "succeeded, but should have failed" + + lustre_fail mds 0 + + if at_is_enabled; then + at_max_set $timeout client + else + lctl set_param timeout=$timeout + fi + + resetquota -u $TSTUSR +} +run_test_with_stat 29 "unhandled quotactls must not hang lustre client (19778) ========" + +test_30() +{ + local output + local LIMIT=1024 + local TESTFILE="$DIR/$tdir/$tfile" + local GRACE=10 + + set_blk_tunesz 512 + set_blk_unitsz 1024 + + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + + $LFS setstripe $TESTFILE -i 0 -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + + $LFS setquota -t -u --block-grace $GRACE --inode-grace $MAX_IQ_TIME $DIR + $LFS setquota -u $TSTUSR -b $LIMIT -B 0 -i 0 -I 0 $DIR + $RUNAS dd if=/dev/zero of=$TESTFILE bs=1024 count=$((LIMIT * 2)) || true + cancel_lru_locks osc + sleep $GRACE + $LFS setquota -u $TSTUSR -B 0 $DIR + # over-quota flag has not yet settled since we do not trigger async events + # based on grace time period expiration + $SHOW_QUOTA_USER + $RUNAS dd if=/dev/zero of=$TESTFILE conv=notrunc oflag=append bs=1048576 count=1 || true + cancel_lru_locks osc + # now over-quota flag should be settled and further writes should fail + $SHOW_QUOTA_USER + $RUNAS dd if=/dev/zero of=$TESTFILE conv=notrunc oflag=append bs=1048576 count=1 && error "grace times were reset" + rm -f $TESTFILE + resetquota -u $TSTUSR + $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR + + set_blk_unitsz $((128 * 1024)) + set_blk_tunesz $((128 * 1024 / 2)) +} +run_test_with_stat 30 "hard limit updates should not reset grace times ================" + +# test duplicate quota releases b=18630 +test_31() { + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + + LIMIT=$(( $BUNIT_SZ * $(($OSTCOUNT + 1)) * 10)) # 10 bunits each sever + TESTFILE="$DIR/$tdir/$tfile-0" + TESTFILE2="$DIR/$tdir/$tfile-1" + + wait_delete_completed + + log " User quota (limit: $LIMIT kbytes)" + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + + $LFS setstripe $TESTFILE -i 0 -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + $LFS setstripe $TESTFILE2 -i 0 -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE2 + + log " step1: write out of block quota ..." + $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=5120 + $RUNAS dd if=/dev/zero of=$TESTFILE2 bs=$BLK_SZ count=5120 + + #define OBD_FAIL_QUOTA_DELAY_SD 0xA04 + #define OBD_FAIL_SOME 0x10000000 /* fail N times */ + lustre_fail ost $((0x00000A04 | 0x10000000)) 1 + + log " step2: delete two files so that triggering duplicate quota release ..." + rm -f $TESTFILE $TESTFILE2 + sync; sleep 5; sync # OBD_FAIL_QUOTA_DELAY_SD will delay for 5 seconds + wait_delete_completed + + log " step3: verify if the ost failed" + do_facet ost1 dmesg > $TMP/lustre-log-${TESTNAME}.log + watchdog=`awk '/test 31/ {start = 1;} + /release quota error/ { + if (start) { + print; + } + }' $TMP/lustre-log-${TESTNAME}.log` + [ "$watchdog" ] && error "$watchdog" + rm -f $TMP/lustre-log-${TESTNAME}.log + + lustre_fail ost 0 + resetquota -u $TSTUSR +} +run_test_with_stat 31 "test duplicate quota releases ===" + +# check hash_cur_bits +check_quota_hash_cur_bits() { + local bits=$1 + + # check quota_hash_cur_bits on all obdfilters + for num in `seq $OSTCOUNT`; do + cb=`do_facet ost$num "cat /sys/module/lquota/parameters/hash_lqs_cur_bits"` + if [ $cb -gt $bits ]; then + echo "hash_lqs_cur_bits of ost$num is too large(cur_bits=$cb)" + return 1; + fi + done + # check quota_hash_cur_bits on mds + cb=`do_facet $SINGLEMDS "cat /sys/module/lquota/parameters/hash_lqs_cur_bits"` + if [ $cb -gt $bits ]; then + echo "hash_lqs_cur_bits of mds is too large(cur_bits=$cb)" + return 1; + fi + return 0; +} + +# check lqs hash +check_lqs_hash() { + # check distribution of all obdfilters + for num in `seq $OSTCOUNT`; do + do_facet ost$num "lctl get_param obdfilter.${FSNAME}-OST*.hash_stats | grep LQS_HASH" | while read line; do + rehash_count=`echo $line | awk '{print $9}'` + if [ $rehash_count -eq 0 ]; then + echo -e "ost$num:\n $line" + error "Rehearsh didn't happen" + fi + done + done + # check distribution of mds + do_facet $SINGLEMDS "lctl get_param mdt.${FSNAME}-MDT*.hash_stats | grep LQS_HASH" | while read line; do + rehash_count=`echo $line | awk '{print $9}'` + if [ $rehash_count -eq 0 ]; then + echo -e "mdt:\n $line" + error "Rehearsh didn't happen" + fi + done +} + +test_32() +{ + # reset system so that quota_hash_cur_bits==3 + echo "Reset system ..." + local LMR_orig=$LOAD_MODULES_REMOTE + LOAD_MODULES_REMOTE=true + cleanup_and_setup_lustre + LOAD_MODULES_REMOTE=$LMR_orig + + client_up + wait_mds_ost_sync + quota_init + + for user in $SANITY_QUOTA_USERS; do + check_runas_id_ret $user quota_usr "runas -u $user -g quota_usr" >/dev/null 2>/dev/null || \ + missing_users="$missing_users $user" + done + [ -n "$missing_users" ] && { skip_env "the following users are missing: $missing_users" ; return 0 ; } + check_quota_hash_cur_bits 3 || { skip_env "hash_lqs_cur_bits isn't set properly"; return 0;} + + $LFS quotaoff -ug $DIR + $LFS quotacheck -ug $DIR + + for user in $SANITY_QUOTA_USERS; do + $LFS setquota -u $user --block-hardlimit 1048576 $DIR + done + + check_lqs_hash + + for user in $SANITY_QUOTA_USERS; do + resetquota -u $user + done +} +run_test 32 "check lqs hash(bug 21846) ==========================================" + +cleanup_quota_test() { + trap 0 + echo "Delete files..." + rm -rf $DIR/$tdir +} + +# basic usage tracking for user & group +test_33() { + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + INODES=10 + BLK_CNT=1024 + TOTAL_BLKS=$(($INODES * $BLK_CNT)) + + trap cleanup_quota_test EXIT + + # make sure the system is clean + USED=`getquota -u $TSTID global curspace` + [ $USED -ne 0 ] && \ + error "Used space ($USED) for user $TSTID isn't 0." + USED=`getquota -g $TSTID global curspace` + [ $USED -ne 0 ] && \ + error "Used space ($USED) for group $TSTID isn't 0." + + for i in `seq 0 $INODES`; do + $RUNAS dd if=/dev/zero of=$DIR/$tdir/$tfile-$i conv=fsync \ + bs=$((BLK_SZ * BLK_CNT)) count=1 2>/dev/null || + error "write failed" + echo "Iteration $i/$INODES completed" + done + sync; sync_all_data; + + echo "Verify disk usage after write" + USED=`getquota -u $TSTID global curspace` + [ $USED -lt $TOTAL_BLKS ] && \ + error "Used space for user $TSTID is $USED, expected $TOTAL_BLKS" + USED=`getquota -u $TSTID global curinodes` + [ $USED -lt $INODES ] && \ + error "Used inode for user $TSTID is $USED, expected $INODES" + + USED=`getquota -g $TSTID global curspace` + [ $USED -lt $TOTAL_BLKS ] && \ + error "Used space for group $TSTID is $USED, expected $TOTAL_BLKS" + USED=`getquota -g $TSTID global curinodes` + [ $USED -lt $INODES ] && \ + error "Used inode for group $TSTID is $USED, expected $INODES" + + cleanup_quota_test + + echo "Verify disk usage after delete" + wait_delete_completed + sync; sync_all_data; + + USED=`getquota -u $TSTID global curspace` + [ $USED -eq 0 ] || error "Used space for user $TSTID isn't 0. $USED" + USED=`getquota -u $TSTID global curinodes` + [ $USED -eq 0 ] || error "Used inodes for user $TSTID isn't 0. $USED" + USED=`getquota -g $TSTID global curspace` + [ $USED -eq 0 ] || error "Used space for group $TSTID isn't 0. $USED" + USED=`getquota -g $TSTID global curinodes` + [ $USED -eq 0 ] || error "Used inodes for group $TSTID isn't 0. $USED" +} +run_test 33 "basic usage tracking for user & group ==============================" + +# usage transfer test for user & group +test_34() { + BLK_CNT=1024 + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + + trap cleanup_quota_test EXIT + + # make sure the system is clean + USED=`getquota -u $TSTID global curspace` + [ $USED -ne 0 ] && error "Used space ($USED) for user $TSTID isn't 0." + USED=`getquota -g $TSTID global curspace` + [ $USED -ne 0 ] && error "Used space ($USED) for group $TSTID isn't 0." + + echo "Write file..." + dd if=/dev/zero of=$DIR/$tdir/$tfile bs=$((BLK_SZ * BLK_CNT)) count=1 \ + conv=fsync 2>/dev/null || error "write failed" + sync; sync_all_data; + + echo "chown the file to user $TSTID" + chown $TSTID $DIR/$tdir/$tfile || error "chown failed" + + echo "Wait for setattr on objects finished..." + wait_delete_completed + + echo "Verify disk usage for user $TSTID" + USED=`getquota -u $TSTID global curspace` + [ $USED -lt $BLK_CNT ] && \ + error "Used space for user $TSTID is $USED, expected $BLK_CNT" + USED=`getquota -u $TSTID global curinodes` + [ $USED -ne 1 ] && \ + error "Used inodes for user $TSTID is $USED, expected 1" + + echo "chgrp the file to group $TSTID" + chgrp $TSTID $DIR/$tdir/$tfile || error "chgrp failed" + + echo "Wait for setattr on objects finished..." + wait_delete_completed + + echo "Verify disk usage for group $TSTID" + USED=`getquota -g $TSTID global curspace` + [ $USED -ge $BLK_CNT ] || \ + error "Used space for group $TSTID is $USED, expected $BLK_CNT" + USED=`getquota -g $TSTID global curinodes` + [ $USED -eq 1 ] || \ + error "Used inodes for group $TSTID is $USED, expected 1" + + cleanup_quota_test +} +run_test 34 "usage transfer for user & group ====================================" + +# usage is still accessible across restart +test_35() { + mkdir -p $DIR/$tdir + chmod 0777 $DIR/$tdir + BLK_CNT=1024 + + trap cleanup_quota_test EXIT + + echo "Write file..." + $RUNAS dd if=/dev/zero of=$DIR/$tdir/$tfile bs=$((BLK_SZ * BLK_CNT)) \ + count=1 conv=fsync 2>/dev/null || error "write failed" + sync; sync_all_data; + + echo "Save disk usage before restart" + ORIG_USR_SPACE=`getquota -u $TSTID global curspace` + [ $ORIG_USR_SPACE -eq 0 ] && \ + error "Used space for user $TSTID is 0, expected $BLK_CNT" + ORIG_USR_INODES=`getquota -u $TSTID global curinodes` + [ $ORIG_USR_INODES -eq 0 ] && \ + error "Used inodes for user $TSTID is 0, expected 1" + ORIG_GRP_SPACE=`getquota -g $TSTID global curspace` + [ $ORIG_GRP_SPACE -eq 0 ] && \ + error "Used space for group $TSTID is 0, expected $BLK_CNT" + ORIG_GRP_INODES=`getquota -g $TSTID global curinodes` + [ $ORIG_GRP_INODES -eq 0 ] && \ + error "Used inodes for group $TSTID is 0, expected 1" + + log "Restart..." + local ORIG_REFORMAT=$REFORMAT + REFORMAT="" + cleanup_and_setup_lustre + REFORMAT=$ORIG_REFORMAT + quota_init + + echo "Verify disk usage after restart" + USED=`getquota -u $TSTID global curspace` + [ $USED -eq $ORIG_USR_SPACE ] || \ + error "Used space for user $TSTID changed from " \ + "$ORIG_USR_SPACE to $USED" + USED=`getquota -u $TSTID global curinodes` + [ $USED -eq $ORIG_USR_INODES ] || \ + error "Used inodes for user $TSTID changed from " \ + "$ORIG_USR_INODES to $USED" + USED=`getquota -g $TSTID global curspace` + [ $USED -eq $ORIG_GRP_SPACE ] || \ + error "Used space for group $TSTID changed from " \ + "$ORIG_GRP_SPACE to $USED" + USED=`getquota -g $TSTID global curinodes` + [ $USED -eq $ORIG_GRP_INODES ] || \ + error "Used inodes for group $TSTID changed from " \ + "$ORIG_GRP_INODES to $USED" + + # check if the vfs_dq_init() is called before writing + echo "Append to the same file..." + $RUNAS dd if=/dev/zero of=$DIR/$tdir/$tfile bs=$((BLK_SZ * BLK_CNT)) \ + count=1 seek=1 conv=fsync 2>/dev/null || error "write failed" + sync; sync_all_data; + + echo "Verify space usage is increased" + USED=`getquota -u $TSTID global curspace` + [ $USED -gt $ORIG_USR_SPACE ] || + error "Used space for user $TSTID isn't increased" \ + "orig:$ORIG_USR_SPACE, now:$USED" + USED=`getquota -g $TSTID global curspace` + [ $USED -gt $ORIG_GRP_SPACE ] || + error "Used space for group $TSTID isn't increased" \ + "orig:$ORIG_GRP_SPACE, now:$USED" + + cleanup_quota_test +} +run_test 35 "usage is still accessible across reboot ============================" + +# turn off quota +quota_fini() +{ + $LFS quotaoff $DIR + do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=-quota" +} +quota_fini + +cd $ORIG_PWD +complete sanity-quota.sh $SECONDS +check_and_cleanup_lustre +export QUOTA_AUTO=$QUOTA_AUTO_OLD +exit_status diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index 2387913..4029290 100644 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -13,14 +13,10 @@ export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin ONLY="33 34 35" ONLY=${ONLY:-"$*"} -# test_11 has been used to protect a kernel bug(bz10912), now it isn't -# useful any more. Then add it to ALWAYS_EXCEPT. b=19835 -# We have changed the mechanism of quota, test_12 is meanless now. -# b=20877 -ALWAYS_EXCEPT="10 12 $SANITY_QUOTA_EXCEPT" +ALWAYS_EXCEPT="$SANITY_QUOTA_EXCEPT" # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! -[ "$ALWAYS_EXCEPT$EXCEPT" ] && \ +[ "$ALWAYS_EXCEPT$EXCEPT" ] && echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT`" TMP=${TMP:-/tmp} @@ -31,19 +27,10 @@ TSTID2=${TSTID2:-60001} TSTUSR=${TSTUSR:-"quota_usr"} TSTUSR2=${TSTUSR2:-"quota_2usr"} BLK_SZ=1024 -BUNIT_SZ=${BUNIT_SZ:-1024} # min block quota unit(kB) -IUNIT_SZ=${IUNIT_SZ:-10} # min inode quota unit MAX_DQ_TIME=604800 MAX_IQ_TIME=604800 -SANITY_QUOTA_USERS="quota15_1 quota15_2 quota15_3 quota15_4 quota15_5 quota15_6 \ - quota15_7 quota15_8 quota15_9 quota15_10 quota15_11 quota15_12 \ - quota15_13 quota15_14 quota15_15 quota15_16 quota15_17 quota15_18 \ - quota15_19 quota15_20 quota15_21 quota15_22 quota15_23 quota15_24 \ - quota15_25 quota15_26 quota15_27 quota15_28 quota15_29 quota15_30" - -export MULTIOP=${MULTIOP:-multiop} -TRACE=${TRACE:-""} -LUSTRE=${LUSTRE:-`dirname $0`/..} + +LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} . $LUSTRE/tests/test-framework.sh init_test_env $@ . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} @@ -55,7 +42,19 @@ DIRECTIO=${DIRECTIO:-$LUSTRE/tests/directio} require_dsh_mds || exit 0 require_dsh_ost || exit 0 -[ "$SLOW" = "no" ] && EXCEPT_SLOW="9 10 11 18b 21" +# XXX Once we drop the interoperability with old server (< 2.3.50), the +# sanity-quota-old.sh should be removed. +if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then + exec $LUSTRE/tests/sanity-quota-old.sh +fi + +# if e2fsprogs support quota feature? +if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] && \ + ! $DEBUGFS -c -R supported_features | grep -q 'quota'; then + skip "e2fsprogs doesn't support quota" && exit 0 +fi + +[ "$SLOW" = "no" ] && EXCEPT_SLOW="9 18 21" QUOTALOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} @@ -81,50 +80,15 @@ SHOW_QUOTA_GROUP2="$LFS quota -v -g $TSTUSR2 $DIR" SHOW_QUOTA_INFO_USER="$LFS quota -t -u $DIR" SHOW_QUOTA_INFO_GROUP="$LFS quota -t -g $DIR" -# control the time of tests -cycle=30 -[ "$SLOW" = "no" ] && cycle=10 - build_test_filter # Use OFD for all quota testing +cleanupall USE_OFD_OLD="$USE_OFD" LOAD_MODULES_REMOTE_OLD="$LOAD_MODULES_REMOTE" export USE_OFD="yes" export LOAD_MODULES_REMOTE="true" -cleanup_and_setup_lustre - -# set_blk_tunables(btune_sz) -set_blk_tunesz() { - local btune=$(($1 * BLK_SZ)) - # set btune size on all obdfilters - do_nodes $(comma_list $(osts_nodes)) "lctl set_param lquota.${FSNAME}-OST*.quota_btune_sz=$btune" - # set btune size on mds - do_facet $SINGLEMDS "lctl set_param lquota.mdd_obd-${FSNAME}-MDT*.quota_btune_sz=$btune" -} - -# set_blk_unitsz(bunit_sz) -set_blk_unitsz() { - local bunit=$(($1 * BLK_SZ)) - # set bunit size on all obdfilters - do_nodes $(comma_list $(osts_nodes)) "lctl set_param lquota.${FSNAME}-OST*.quota_bunit_sz=$bunit" - # set bunit size on mds - do_facet $SINGLEMDS "lctl set_param lquota.mdd_obd-${FSNAME}-MDT*.quota_bunit_sz=$bunit" -} - -# set_file_tunesz(itune_sz) -set_file_tunesz() { - local itune=$1 - # set itune size on mds - do_facet $SINGLEMDS "lctl set_param lquota.mdd_obd-${FSNAME}-MDT*.quota_itune_sz=$itune" -} - -# set_file_unitsz(iunit_sz) -set_file_unitsz() { - local iunit=$1 - # set iunit size on mds - do_facet $SINGLEMDS "lctl set_param lquota.mdd_obd-${FSNAME}-MDT*.quota_iunit_sz=$iunit" -} +setupall lustre_fail() { local fail_node=$1 @@ -132,20 +96,15 @@ lustre_fail() { local fail_val=${3:-0} if [ $fail_node == "mds" ] || [ $fail_node == "mds_ost" ]; then - if [ $((fail_loc & 0x10000000)) -ne 0 -a $fail_val -gt 0 ] || \ - [ $((fail_loc)) -eq 0 ]; then do_facet $SINGLEMDS "lctl set_param fail_val=$fail_val" - fi - do_facet $SINGLEMDS "lctl set_param fail_loc=$fail_loc" + do_facet $SINGLEMDS "lctl set_param fail_loc=$fail_loc" fi + if [ $fail_node == "ost" ] || [ $fail_node == "mds_ost" ]; then - for num in `seq $OSTCOUNT`; do - if [ $((fail_loc & 0x10000000)) -ne 0 -a $fail_val -gt 0 ] || \ - [ $((fail_loc)) -eq 0 ]; then - do_facet ost$num "lctl set_param fail_val=$fail_val" - fi - do_facet ost$num "lctl set_param fail_loc=$fail_loc" - done + for num in `seq $OSTCOUNT`; do + do_facet ost$num "lctl set_param fail_val=$fail_val" + do_facet ost$num "lctl set_param fail_loc=$fail_loc" + done fi } @@ -153,492 +112,553 @@ RUNAS="runas -u $TSTID -g $TSTID" RUNAS2="runas -u $TSTID2 -g $TSTID2" FAIL_ON_ERROR=true check_runas_id $TSTID $TSTID $RUNAS FAIL_ON_ERROR=true check_runas_id $TSTID2 $TSTID2 $RUNAS2 +DD="dd if=/dev/zero bs=1M" FAIL_ON_ERROR=false -run_test_with_stat() { - (($# != 2)) && error "the number of arguments is wrong" - - if [ "$USE_OFD" == "yes" ]; then - run_test "$@" - return - fi - - do_facet $SINGLEMDS "lctl set_param lquota.mdd_obd-${FSNAME}-MDT*.stats=0" > /dev/null - for j in `seq $OSTCOUNT`; do - do_facet ost$j "lctl set_param lquota.${FSNAME}-OST*.stats=0" > /dev/null - done - run_test "$@" - if [ ${STAT:-"yes"} != "no" -a -z "$LAST_SKIPPED" ]; then - echo "statistics info begin ***************************************" - do_facet $SINGLEMDS "lctl get_param lquota.mdd_obd-${FSNAME}-MDT*.stats" - for j in `seq $OSTCOUNT`; do - do_facet ost$j "lctl get_param lquota.${FSNAME}-OST*.stats" - done - echo "statistics info end ***************************************" - fi -} - -# # clear quota limits for a user or a group # usage: resetquota -u username # resetquota -g groupname resetquota() { - [ "$#" != 2 ] && error "resetquota: wrong number of arguments: $#" - [ "$1" != "-u" -a "$1" != "-g" ] && error "resetquota: wrong specifier $1 passed" - - count=0 - if at_is_enabled; then - timeout=$(at_max_get mds) - else - timeout=$(lctl get_param -n timeout) - fi - - while [ $((count++)) -lt $timeout ]; do - local RC=0 - OUTPUT=`$LFS setquota "$1" "$2" -b 0 -B 0 -i 0 -I 0 $MOUNT 2>&1` || RC=${PIPESTATUS[0]} - if [ $RC -ne 0 ]; then - if echo "$OUTPUT" | grep -q busy; then - log "resetquota is blocked for quota master recovery, retry after $((count * 3)) sec" - sleep 3 - continue - else - error "resetquota failed" - fi - fi - break - done - - [ $count -lt $timeout ] || error "resetquota timeout: $timeout" + [ "$#" != 2 ] && error "resetquota: wrong number of arguments: $#" + [ "$1" != "-u" -a "$1" != "-g" ] && + error "resetquota: wrong specifier $1 passed" + + $LFS setquota "$1" "$2" -b 0 -B 0 -i 0 -I 0 $MOUNT + # The usage should be 0 now, enforce quota to trigger quota + # release for each slave, sleep 3 seconds to wait for the release + # done, then we can make sure granted quota for each slave is 0. + $LFS setquota "$1" "$2" -b 0 -B 1 -i 0 -I 1 $MOUNT + sleep 3 + $LFS setquota "$1" "$2" -b 0 -B 0 -i 0 -I 0 $MOUNT } quota_scan() { - LOCAL_UG=$1 - LOCAL_ID=$2 - - if [ "$LOCAL_UG" == "a" -o "$LOCAL_UG" == "u" ]; then - log "Files for user ($LOCAL_ID):" - ($LFS find -user $LOCAL_ID $DIR | xargs stat 2>/dev/null) - fi - - if [ "$LOCAL_UG" == "a" -o "$LOCAL_UG" == "g" ]; then - log "Files for group ($LOCAL_ID):" - ($LFS find -group $LOCAL_ID $DIR | xargs stat 2>/dev/null) - fi + local LOCAL_UG=$1 + local LOCAL_ID=$2 + + if [ "$LOCAL_UG" == "a" -o "$LOCAL_UG" == "u" ]; then + $LFS quota -v -u $LOCAL_ID $DIR + log "Files for user ($LOCAL_ID):" + ($LFS find -user $LOCAL_ID $DIR | head -n 4 | + xargs stat 2>/dev/null) + fi + + if [ "$LOCAL_UG" == "a" -o "$LOCAL_UG" == "g" ]; then + $LFS quota -v -u $LOCAL_ID $DIR + log "Files for group ($LOCAL_ID):" + ($LFS find -group $LOCAL_ID $DIR | head -n 4 | + xargs stat 2>/dev/null) + fi } quota_error() { - quota_scan $1 $2 - shift 2 - error "$*" + quota_scan $1 $2 + shift 2 + error "$*" } quota_log() { - quota_scan $1 $2 - shift 2 - log "$*" + quota_scan $1 $2 + shift 2 + log "$*" } -# -# get quota info for a user or a group -# usage: getquota -u|-g | global| bhardlimit|bsoftlimit|bgrace|ihardlimit|isoftlimit|igrace +# get quota for a user or a group +# usage: getquota -u|-g | global| \ +# bhardlimit|bsoftlimit|bgrace|ihardlimit|isoftlimit|igrace # getquota() { - local spec - local uuid + local spec + local uuid + + [ "$#" != 4 ] && error "getquota: wrong number of arguments: $#" + [ "$1" != "-u" -a "$1" != "-g" ] && + error "getquota: wrong u/g specifier $1 passed" + + uuid="$3" + + case "$4" in + curspace) spec=1;; + bsoftlimit) spec=2;; + bhardlimit) spec=3;; + bgrace) spec=4;; + curinodes) spec=5;; + isoftlimit) spec=6;; + ihardlimit) spec=7;; + igrace) spec=8;; + *) error "unknown quota parameter $4";; + esac + + [ "$uuid" = "global" ] && uuid=$DIR + + $LFS quota -v "$1" "$2" $DIR | + awk 'BEGIN { num='$spec' } { if ($1 == "'$uuid'") \ + { if (NF == 1) { getline } else { num++ } ; print $num;} }' \ + | tr -d "*" +} + +# set mdt quota type +# usage: set_mdt_qtype ug|u|g|none +set_mdt_qtype() { + local qtype=$1 + local varsvc=${SINGLEMDS}_svc + do_facet mgs $LCTL conf_param $FSNAME.quota.mdt=$qtype + if $(facet_up $SINGLEMDS); then + wait_update_facet $SINGLEMDS "$LCTL get_param -n \ + osd-$FSTYPE.${!varsvc}.quota_slave.enabled" "$qtype" || + return 1 + fi + return 0 +} + +# set ost quota type +# usage: set_ost_quota_type ug|u|g|none +set_ost_qtype() { + local qtype=$1 + local varsvc + local osts=$(get_facets OST) + do_facet mgs $LCTL conf_param $FSNAME.quota.ost=$qtype + # we have to make sure each OST received config changes + for ost in ${osts//,/ }; do + varsvc=${ost}_svc + if $(facet_up $ost); then + wait_update_facet $ost "$LCTL get_param -n \ + osd-$FSTYPE.${!varsvc}.quota_slave.enabled" \ + "$qtype" || return 1 + fi + done + return 0 +} + +wait_reintegration() { + local ntype=$1 + local qtype=$2 + local result="glb[1],slv[1],reint[0]" + local varsvc + + if [ $ntype == "mdt" ]; then + varsvc=${SINGLEMDS}_svc + if $(facet_up $SINGLEMDS); then + wait_update_facet $SINGLEMDS "$LCTL get_param -n \ + osd-$FSTYPE.${!varsvc}.quota_slave.info | + grep "$qtype" | awk '{ print \\\$3 }'" "$result" || + return 1 + fi + else + local osts=$(get_facets OST) + for ost in ${osts//,/ }; do + varsvc=${ost}_svc + if $(facet_up $ost); then + wait_update_facet $ost "$LCTL get_param -n \ + osd-$FSTYPE.${!varsvc}.quota_slave.info | + grep "$qtype" | awk '{ print \\\$3 }'" \ + "$result" || return 1 + fi + done + fi + return 0 +} + +wait_mdt_reint() { + local qtype=$1 + + if [ $qtype == "u" ] || [ $qtype == "ug" ]; then + wait_reintegration "mdt" "user" || return 1 + fi + + if [ $qtype == "g" ] || [ $qtype == "ug" ]; then + wait_reintegration "mdt" "group" || return 1 + fi + return 0 +} - [ "$#" != 4 ] && error "getquota: wrong number of arguments: $#" - [ "$1" != "-u" -a "$1" != "-g" ] && error "getquota: wrong u/g specifier $1 passed" +wait_ost_reint() { + local qtype=$1 - uuid="$3" + if [ $qtype == "u" ] || [ $qtype == "ug" ]; then + wait_reintegration "ost" "user" || return 1 + fi - case "$4" in - curspace) spec=1;; - bsoftlimit) spec=2;; - bhardlimit) spec=3;; - bgrace) spec=4;; - curinodes) spec=5;; - isoftlimit) spec=6;; - ihardlimit) spec=7;; - igrace) spec=8;; - *) error "unknown quota parameter $4";; - esac + if [ $qtype == "g" ] || [ $qtype == "ug" ]; then + wait_reintegration "ost" "group" || return 1 + fi + return 0 +} - [ "$uuid" = "global" ] && uuid=$DIR +setup_quota_test() { + rm -rf $DIR/$tdir + wait_delete_completed + echo "Creating test directory" + mkdir -p $DIR/$tdir + chmod 077 $DIR/$tdir + # always clear fail_loc in case of fail_loc isn't cleared + # properly when previous test failed + lustre_fail mds_ost 0 +} - $LFS quota -v "$1" "$2" $DIR | awk 'BEGIN { num='$spec' } { if ($1 == "'$uuid'") { if (NF == 1) { getline } else { num++ } ; print $num;} }' | tr -d "*" +cleanup_quota_test() { + trap 0 + echo "Delete files..." + rm -rf $DIR/$tdir + echo "Wait for unlink objects finished..." + wait_delete_completed + sync_all_data } quota_show_check() { - LOCAL_BF=$1 - LOCAL_UG=$2 - LOCAL_ID=$3 - PATTERN="`echo $DIR | sed 's/\//\\\\\//g'`" - - $LFS quota -v -$LOCAL_UG $LOCAL_ID $DIR - - if [ "$LOCAL_BF" == "a" -o "$LOCAL_BF" == "b" ]; then - USAGE=`getquota -$LOCAL_UG $LOCAL_ID global curspace` - if [ -z $USAGE ]; then - quota_error $LOCAL_UG $LOCAL_ID "System is error when query quota for block ($LOCAL_UG:$LOCAL_ID)." - else - [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for block ($LOCAL_UG:$LOCAL_ID:$USAGE)." - fi - fi - - if [ "$LOCAL_BF" == "a" -o "$LOCAL_BF" == "f" ]; then - USAGE=`getquota -$LOCAL_UG $LOCAL_ID global curinodes` - if [ -z $USAGE ]; then - quota_error $LOCAL_UG $LOCAL_ID "System is error when query quota for file ($LOCAL_UG:$LOCAL_ID)." - else - [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for file ($LOCAL_UG:$LOCAL_ID:$USAGE)." - fi - fi + local bf=$1 + local ug=$2 + local qid=$3 + local usage + + $LFS quota -v -$ug $qid $DIR + + if [ "$bf" == "a" -o "$bf" == "b" ]; then + usage=$(getquota -$ug $qid global curspace) + if [ -z $usage ]; then + quota_error $ug $qid \ + "Query block quota failed ($ug:$qid)." + else + [ $usage -ne 0 ] && quota_log $ug $qid \ + "Block quota isn't 0 ($ug:$qid:$usage)." + fi + fi + + if [ "$bf" == "a" -o "$bf" == "f" ]; then + usage=$(getquota -$ug $qid global curinodes) + if [ -z $usage ]; then + quota_error $ug $qid \ + "Query file quota failed ($ug:$qid)." + else + [ $usage -ne 0 ] && quota_log $ug $qid \ + "File quota isn't 0 ($ug:$qid:$usage)." + fi + fi } -# set quota +# enalbe quota debug quota_init() { do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=+quota" - - # log "do the quotacheck ..." - # $LFS quotacheck -ug $DIR - - # resetquota -u $TSTUSR - # resetquota -g $TSTUSR } quota_init test_quota_performance() { - TESTFILE="$DIR/$tdir/$tfile-0" - local size=$1 - local stime=`date +%s` - $RUNAS dd if=/dev/zero of=$TESTFILE bs=1M count=$size || quota_error u $TSTUSR "write failure" - local etime=`date +%s` + local TESTFILE="$DIR/$tdir/$tfile-0" + local size=$1 # in MB + local stime=$(date +%s) + $RUNAS $DD of=$TESTFILE count=$size conv=fsync || + quota_error u $TSTUSR "write failure" + local etime=$(date +%s) delta=$((etime - stime)) if [ $delta -gt 0 ]; then rate=$((size * 1024 / delta)) - [ $rate -gt 1024 ] || error "SLOW IO for $TSTUSR (user): $rate KB/sec" + [ $rate -gt 1024 ] || + error "SLOW IO for $TSTUSR (user): $rate KB/sec" fi rm -f $TESTFILE } # test basic quota performance b=21696 test_0() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - MB=100 + setup_quota_test + trap cleanup_quota_test EXIT + + local MB=100 # 100M [ "$SLOW" = "no" ] && MB=10 + set_ost_qtype "none" || error "disable ost quota failed" test_quota_performance $MB - $LFS setquota -u $TSTUSR -b 0 -B $((1024*1024)) -i 0 -I 0 $DIR + set_ost_qtype "ug" || error "enable ost quota failed" + $LFS setquota -u $TSTUSR -b 0 -B 10G -i 0 -I 0 $DIR test_quota_performance $MB + cleanup_quota_test resetquota -u $TSTUSR } -run_test_with_stat 0 "Test basic quota performance ===" +run_test 0 "Test basic quota performance" -# test for specific quota limitation, qunit, qtune $1=block_quota_limit -test_1_sub() { - LIMIT=$1 - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - TESTFILE="$DIR/$tdir/$tfile-0" +# test block hardlimit +test_1() { + local LIMIT=10 # 10M + local TESTFILE="$DIR/$tdir/$tfile-0" - wait_delete_completed + setup_quota_test + trap cleanup_quota_test EXIT - # test for user - log " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - sleep 3 - quota_show_check b u $TSTUSR + # enable ost quota + set_ost_qtype "ug" || "enable ost quota failed" - $LFS setstripe $TESTFILE -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE + # test for user + log "User quota (block hardlimit:$LIMIT MB)" + $LFS setquota -u $TSTUSR -b 0 -B ${LIMIT}M -i 0 -I 0 $DIR + + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." + + $LFS setstripe $TESTFILE -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE - log " Write ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT/2)) || quota_error u $TSTUSR "(usr) write failure, but expect success" - log " Done" - log " Write out of block quota ..." + log "Write..." + $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) || + quota_error u $TSTUSR "user write failure, but expect success" + log "Write out of block quota ..." # this time maybe cache write, ignore it's failure - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT/2)) seek=$(($LIMIT/2)) || true - # flush cache, ensure noquota flag is setted on client - cancel_lru_locks osc - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$BUNIT_SZ seek=$LIMIT && quota_error u $TSTUSR "(usr) write success, but expect EDQUOT" - - rm -f $TESTFILE - sync; sleep 1; sync; - OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` - OST0_QUOTA_USED=`getquota -u $TSTUSR $OST0_UUID curspace` - echo $OST0_QUOTA_USED - [ $OST0_QUOTA_USED -ne 0 ] && \ - ($SHOW_QUOTA_USER; quota_error u $TSTUSR "(usr) quota deleted isn't released") - $SHOW_QUOTA_USER - resetquota -u $TSTUSR + $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) seek=$((LIMIT/2)) || true + # flush cache, ensure noquota flag is set on client + cancel_lru_locks osc + $RUNAS $DD of=$TESTFILE count=1 seek=$LIMIT && + quota_error u $TSTUSR "user write success, but expect EDQUOT" + + rm -f $TESTFILE + wait_delete_completed + sync_all_data + USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && quota_error u $TSTUSR \ + "user quota isn't released after deletion" + resetquota -u $TSTUSR - # test for group + # test for group log "--------------------------------------" - log " Group quota (limit: $LIMIT kbytes)" - $LFS setquota -g $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - sleep 3 - quota_show_check b g $TSTUSR - TESTFILE="$DIR/$tdir/$tfile-1" + log "Group quota (block hardlimit:$LIMIT MB)" + $LFS setquota -g $TSTUSR -b 0 -B ${LIMIT}M -i 0 -I 0 $DIR - $LFS setstripe $TESTFILE -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE + TESTFILE="$DIR/$tdir/$tfile-1" + # make sure the system is clean + USED=$(getquota -g $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for group $TSTUSR isn't 0" - log " Write ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT/2)) || quota_error g $TSTUSR "(grp) write failure, but expect success" - log " Done" - log " Write out of block quota ..." + $LFS setstripe $TESTFILE -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + + log "Write ..." + $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) || + quota_error g $TSTUSR "group write failure, but expect success" + log "Write out of block quota ..." # this time maybe cache write, ignore it's failure - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT/2)) seek=$(($LIMIT/2)) || true - cancel_lru_locks osc - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$BUNIT_SZ seek=$LIMIT && quota_error g $TSTUSR "(grp) write success, but expect EDQUOT" + $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) seek=$((LIMIT/2)) || true + cancel_lru_locks osc + $RUNAS $DD of=$TESTFILE count=10 seek=$LIMIT && + quota_error g $TSTUSR "group write success, but expect EDQUOT" # cleanup - rm -f $TESTFILE - sync; sleep 1; sync; - OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` - OST0_QUOTA_USED=`getquota -g $TSTUSR $OST0_UUID curspace` - echo $OST0_QUOTA_USED - [ $OST0_QUOTA_USED -ne 0 ] && \ - ($SHOW_QUOTA_GROUP; quota_error g $TSTUSR "(grp) quota deleted isn't released") - $SHOW_QUOTA_GROUP - resetquota -g $TSTUSR -} + cleanup_quota_test -# block hard limit (normal use and out of quota) -test_1() { - for i in `seq 1 $cycle`; do - # define blk_qunit is between 1M and 4M - blk_qunit=$(( $RANDOM % 3072 + 1024 )) - blk_qtune=$(( $RANDOM % $blk_qunit )) - # other osts and mds will occupy at 1M blk quota - b_limit=$(( ($RANDOM - 16384) / 8 + ($OSTCOUNT + 1) * $blk_qunit * 4 )) - set_blk_tunesz $blk_qtune - set_blk_unitsz $blk_qunit - echo "cycle: $i(total $cycle) bunit:$blk_qunit, btune:$blk_qtune, blimit:$b_limit" - test_1_sub $b_limit - echo "==================================================" - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) - done + USED=$(getquota -g $TSTUSR global curspace) + [ $USED -ne 0 ] && quota_error g $TSTUSR \ + "group quota isn't released after deletion" + + resetquota -g $TSTUSR } -run_test_with_stat 1 "Block hard limit (normal use and out of quota) ===" +run_test 1 "Block hard limit (normal use and out of quota)" -# test for specific quota limitation, qunit, qtune $1=block_quota_limit -test_2_sub() { - LIMIT=$1 - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - TESTFILE="$DIR/$tdir/$tfile-0" +# test inode hardlimit +test_2() { + local LIMIT=$((1024 * 1024)) # 1M inodes + local TESTFILE="$DIR/$tdir/$tfile-0" - wait_delete_completed + [ "$SLOW" = "no" ] && LIMIT=1024 # 1k inodes - # test for user - log " User quota (limit: $LIMIT files)" - $LFS setquota -u $TSTUSR -b 0 -B 0 -i 0 -I $LIMIT $DIR - sleep 3 - quota_show_check f u $TSTUSR + setup_quota_test + trap cleanup_quota_test EXIT + + # enable mdt quota + set_mdt_qtype "ug" || "enable mdt quota failed" + + # test for user + log "User quota (inode hardlimit:$LIMIT files)" + $LFS setquota -u $TSTUSR -b 0 -B 0 -i 0 -I $LIMIT $DIR + + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curinodes) + [ $USED -ne 0 ] && error "Used inodes($USED) for user $TSTUSR isn't 0." - log " Create $LIMIT files ..." + log "Create $LIMIT files ..." $RUNAS createmany -m ${TESTFILE} $LIMIT || \ - quota_error u $TSTUSR "(usr) create failure, but expect success" - log " Done" - log " Create out of file quota ..." + quota_error u $TSTUSR "user create failure, but expect success" + log "Create out of file quota ..." $RUNAS touch ${TESTFILE}_xxx && \ - quota_error u $TSTUSR "(usr) touch success, but expect EDQUOT" + quota_error u $TSTUSR "user create success, but expect EDQUOT" + # cleanup unlinkmany ${TESTFILE} $LIMIT rm -f ${TESTFILE}_xxx - sync; sleep 1; sync; + wait_delete_completed - MDS_UUID=`do_facet $SINGLEMDS $LCTL dl | grep -m1 " mdt " | awk '{print $((NF-1))}'` - MDS_QUOTA_USED=`getquota -u $TSTUSR $MDS_UUID curinodes` - echo $MDS_QUOTA_USED - [ $MDS_QUOTA_USED -ne 0 ] && \ - ($SHOW_QUOTA_USER; quota_error u $TSTUSR "(usr) quota deleted isn't released") - $SHOW_QUOTA_USER + USED=$(getquota -u $TSTUSR global curinodes) + [ $USED -ne 0 ] && quota_error u $TSTUSR \ + "user quota isn't released after deletion" resetquota -u $TSTUSR - # test for group + # test for group log "--------------------------------------" - log " Group quota (limit: $LIMIT FILE)" - $LFS setquota -g $TSTUSR -b 0 -B 0 -i 0 -I $LIMIT $DIR - sleep 3 - quota_show_check f g $TSTUSR - TESTFILE=$DIR/$tdir/$tfile-1 + log "Group quota (inode hardlimit:$LIMIT files)" + $LFS setquota -g $TSTUSR -b 0 -B 0 -i 0 -I $LIMIT $DIR - log " Create $LIMIT files ..." - $RUNAS createmany -m ${TESTFILE} $LIMIT || \ - quota_error g $TSTUSR "(grp) create failure, but expect success" - log " Done" - log " Create out of file quota ..." + TESTFILE=$DIR/$tdir/$tfile-1 + # make sure the system is clean + USED=$(getquota -g $TSTUSR global curinodes) + [ $USED -ne 0 ] && error "Used inodes($USED) for group $TSTUSR isn't 0." + + log "Create $LIMIT files ..." + $RUNAS createmany -m ${TESTFILE} $LIMIT || + quota_error g $TSTUSR "group create failure, but expect success" + log "Create out of file quota ..." $RUNAS touch ${TESTFILE}_xxx && \ - quota_error g $TSTUSR "(grp) touch success, but expect EDQUOT" + quota_error g $TSTUSR "group create success, but expect EDQUOT" + # cleanup unlinkmany ${TESTFILE} $LIMIT rm -f ${TESTFILE}_xxx - sync; sleep 1; sync; + wait_delete_completed - MDS_UUID=`do_facet $SINGLEMDS $LCTL dl | grep -m1 " mdt " | awk '{print $((NF-1))}'` - MDS_QUOTA_USED=`getquota -g $TSTUSR $MDS_UUID curinodes` - echo $MDS_QUOTA_USED - [ $MDS_QUOTA_USED -ne 0 ] && \ - ($SHOW_QUOTA_GROUP; quota_error g $TSTUSR "(grp) quota deleted isn't released") - $SHOW_QUOTA_GROUP - resetquota -g $TSTUSR -} + USED=$(getquota -g $TSTUSR global curinodes) + [ $USED -ne 0 ] && quota_error g $TSTUSR \ + "user quota isn't released after deletion" -# file hard limit (normal use and out of quota) -test_2() { - for i in `seq 1 $cycle`; do - if [ $i -eq 1 ]; then - ino_qunit=52 - ino_qtune=41 - i_limit=11 - else - # define ino_qunit is between 10 and 100 - ino_qunit=$(( $RANDOM % 90 + 10 )) - ino_qtune=$(( $RANDOM % $ino_qunit )) - # RANDOM's maxium is 32767 - i_limit=$(( $RANDOM % 990 + 10 )) - fi - - set_file_tunesz $ino_qtune - set_file_unitsz $ino_qunit - echo "cycle: $i(total $cycle) iunit:$ino_qunit, itune:$ino_qtune, ilimit:$i_limit" - test_2_sub $i_limit - echo "==================================================" - set_file_unitsz 5120 - set_file_tunesz 2560 - done + cleanup_quota_test + resetquota -g $TSTUSR } -run_test_with_stat 2 "File hard limit (normal use and out of quota) ===" +run_test 2 "File hard limit (normal use and out of quota)" test_block_soft() { - TESTFILE=$1 - TIMER=$(($2 * 3 / 2)) - OFFSET=0 + local TESTFILE=$1 + local TIMER=$(($2 * 3 / 2)) + local LIMIT=$3 + local OFFSET=0 - wait_delete_completed + setup_quota_test + trap cleanup_quota_test EXIT + + $LFS setstripe $TESTFILE -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE - echo " Write to exceed soft limit" - RUNDD="$RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ" - $RUNDD count=$((BUNIT_SZ+1)) || \ + echo "Write up to soft limit" + $RUNAS $DD of=$TESTFILE count=$LIMIT || quota_error a $TSTUSR "write failure, but expect success" - OFFSET=$((OFFSET + BUNIT_SZ + 1)) - cancel_lru_locks osc + OFFSET=$((LIMIT * 1024)) + cancel_lru_locks osc + + echo "Write to exceed soft limit" + $RUNAS dd if=/dev/zero of=$TESTFILE bs=1K count=10 seek=$OFFSET || + quota_error a $TSTUSR "write failure, but expect success" + OFFSET=$((OFFSET + 1024)) # make sure we don't write to same block + cancel_lru_locks osc $SHOW_QUOTA_USER $SHOW_QUOTA_GROUP $SHOW_QUOTA_INFO_USER $SHOW_QUOTA_INFO_GROUP - echo " Write before timer goes off" - $RUNDD count=$BUNIT_SZ seek=$OFFSET || \ + echo "Write before timer goes off" + $RUNAS dd if=/dev/zero of=$TESTFILE bs=1K count=10 seek=$OFFSET || quota_error a $TSTUSR "write failure, but expect success" - OFFSET=$((OFFSET + BUNIT_SZ)) - cancel_lru_locks osc - echo " Done" + OFFSET=$((OFFSET + 1024)) + cancel_lru_locks osc - echo " Sleep $TIMER seconds ..." - sleep $TIMER + echo "Sleep $TIMER seconds ..." + sleep $TIMER - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP - $SHOW_QUOTA_INFO_USER - $SHOW_QUOTA_INFO_GROUP + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $SHOW_QUOTA_INFO_USER + $SHOW_QUOTA_INFO_GROUP - echo " Write after timer goes off" + echo "Write after timer goes off" # maybe cache write, ignore. - $RUNDD count=$BUNIT_SZ seek=$OFFSET || true - OFFSET=$((OFFSET + BUNIT_SZ)) - cancel_lru_locks osc - $RUNDD count=$BUNIT_SZ seek=$OFFSET && \ + $RUNAS dd if=/dev/zero of=$TESTFILE bs=1K count=10 seek=$OFFSET || true + OFFSET=$((OFFSET + 1024)) + cancel_lru_locks osc + $RUNAS dd if=/dev/zero of=$TESTFILE bs=1K count=10 seek=$OFFSET && quota_error a $TSTUSR "write success, but expect EDQUOT" - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP - $SHOW_QUOTA_INFO_USER - $SHOW_QUOTA_INFO_GROUP + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $SHOW_QUOTA_INFO_USER + $SHOW_QUOTA_INFO_GROUP - echo " Unlink file to stop timer" + echo "Unlink file to stop timer" rm -f $TESTFILE - sync; sleep 1; sync - echo " Done" + wait_delete_completed + sync_all_data - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP - $SHOW_QUOTA_INFO_USER - $SHOW_QUOTA_INFO_GROUP + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $SHOW_QUOTA_INFO_USER + $SHOW_QUOTA_INFO_GROUP - echo " Write ..." - $RUNDD count=$BUNIT_SZ || quota_error a $TSTUSR "write failure, but expect success" - echo " Done" + $LFS setstripe $TESTFILE -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE + echo "Write ..." + $RUNAS $DD of=$TESTFILE count=$LIMIT || + quota_error a $TSTUSR "write failure, but expect success" # cleanup - rm -f $TESTFILE - sync; sleep 3; sync; + cleanup_quota_test } -# block soft limit (start timer, timer goes off, stop timer) +# block soft limit test_3() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir + local LIMIT=1 # 1MB + local GRACE=10 # 10s + local TESTFILE=$DIR/$tdir/$tfile-0 - # 1 bunit on mds and 1 bunit on every ost - LIMIT=$(( $BUNIT_SZ * ($OSTCOUNT + 1) )) - GRACE=10 + set_ost_qtype "ug" || error "enable ost quota failed" - echo " User quota (soft limit: $LIMIT kbytes grace: $GRACE seconds)" - TESTFILE=$DIR/$tdir/$tfile-0 - - $LFS setstripe $TESTFILE -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE + echo "User quota (soft limit:$LIMIT MB grace:$GRACE seconds)" + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." $LFS setquota -t -u --block-grace $GRACE --inode-grace $MAX_IQ_TIME $DIR - $LFS setquota -u $TSTUSR -b $LIMIT -B 0 -i 0 -I 0 $DIR + $LFS setquota -u $TSTUSR -b ${LIMIT}M -B 0 -i 0 -I 0 $DIR - test_block_soft $TESTFILE $GRACE + test_block_soft $TESTFILE $GRACE $LIMIT resetquota -u $TSTUSR - echo " Group quota (soft limit: $LIMIT kbytes grace: $GRACE seconds)" + echo "Group quota (soft limit:$LIMIT MB grace:$GRACE seconds)" TESTFILE=$DIR/$tdir/$tfile-1 - - $LFS setstripe $TESTFILE -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE + # make sure the system is clean + USED=$(getquota -g $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for group $TSTUSR isn't 0." $LFS setquota -t -g --block-grace $GRACE --inode-grace $MAX_IQ_TIME $DIR - $LFS setquota -g $TSTUSR -b $LIMIT -B 0 -i 0 -I 0 $DIR + $LFS setquota -g $TSTUSR -b ${LIMIT}M -B 0 -i 0 -I 0 $DIR - test_block_soft $TESTFILE $GRACE + test_block_soft $TESTFILE $GRACE $LIMIT resetquota -g $TSTUSR + + # cleanup + $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR + $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR } -run_test_with_stat 3 "Block soft limit (start timer, timer goes off, stop timer) ===" +run_test 3 "Block soft limit (start timer, timer goes off, stop timer)" test_file_soft() { - TESTFILE=$1 - LIMIT=$2 - TIMER=$(($3 * 3 / 2)) + local TESTFILE=$1 + local LIMIT=$2 + local TIMER=$(($3 * 3 / 2)) - wait_delete_completed + setup_quota_test + trap cleanup_quota_test EXIT - echo " Create files to exceed soft limit" - $RUNAS createmany -m ${TESTFILE}_ $((LIMIT + 1)) || \ + echo "Create files to exceed soft limit" + $RUNAS createmany -m ${TESTFILE}_ $((LIMIT + 1)) || quota_error a $TSTUSR "create failure, but expect success" sync; sleep 1; sync - echo " Done" - echo " Create file before timer goes off" - $RUNAS touch ${TESTFILE}_before || \ - quota_error a $TSTUSR "failed create before timer expired, but expect success" + echo "Create file before timer goes off" + $RUNAS touch ${TESTFILE}_before || + quota_error a $TSTUSR "failed create before timer expired," \ + "but expect success" sync; sleep 1; sync - echo " Done" - echo " Sleep $TIMER seconds ..." + echo "Sleep $TIMER seconds ..." sleep $TIMER $SHOW_QUOTA_USER @@ -646,13 +666,15 @@ test_file_soft() { $SHOW_QUOTA_INFO_USER $SHOW_QUOTA_INFO_GROUP - echo " Create file after timer goes off" - # the least of inode qunit is 2, so there are at most 3(qunit:2+qtune:1) - # inode quota left here - $RUNAS touch ${TESTFILE}_after ${TESTFILE}_after1 ${TESTFILE}_after2 || true + echo "Create file after timer goes off" + # There is a window that space is accounted in the quota usage but + # hasn't been decreased from the pending write, if we acquire quota + # in this window, we'll acquire more than we needed. + $RUNAS touch ${TESTFILE}_after_1 ${TESTFILE}_after_2 || true sync; sleep 1; sync - $RUNAS touch ${TESTFILE}_after3 && \ - quota_error a $TSTUSR "create after timer expired, but expect EDQUOT" + $RUNAS touch ${TESTFILE}_after_3 && + quota_error a $TSTUSR "create after timer expired," \ + "but expect EDQUOT" sync; sleep 1; sync $SHOW_QUOTA_USER @@ -660,1764 +682,1382 @@ test_file_soft() { $SHOW_QUOTA_INFO_USER $SHOW_QUOTA_INFO_GROUP - echo " Unlink files to stop timer" - find `dirname $TESTFILE` -name "`basename ${TESTFILE}`*" | xargs rm -f - echo " Done" + echo "Unlink files to stop timer" + find $(dirname $TESTFILE) -name "$(basename ${TESTFILE})*" | xargs rm -f + wait_delete_completed - echo " Create file" - $RUNAS touch ${TESTFILE}_xxx || \ - quota_error a $TSTUSR "touch after timer stop failure, but expect success" + echo "Create file" + $RUNAS touch ${TESTFILE}_xxx || + quota_error a $TSTUSR "touch after timer stop failure," \ + "but expect success" sync; sleep 1; sync - echo " Done" # cleanup - rm -f ${TESTFILE}_xxx - sync; sleep 3; sync; + cleanup_quota_test } -# file soft limit (start timer, timer goes off, stop timer) -test_4a() { # was test_4 - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - LIMIT=$(($IUNIT_SZ * 10)) # 10 iunits on mds - TESTFILE=$DIR/$tdir/$tfile-0 +# file soft limit +test_4a() { + local LIMIT=10 # inodes + local TESTFILE=$DIR/$tdir/$tfile-0 + local GRACE=5 + + set_mdt_qtype "ug" || error "enable mdt quota failed" - GRACE=5 + echo "User quota (soft limit:$LIMIT files grace:$GRACE seconds)" + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curinodes) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." - echo " User quota (soft limit: $LIMIT files grace: $GRACE seconds)" $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace $GRACE $DIR $LFS setquota -u $TSTUSR -b 0 -B 0 -i $LIMIT -I 0 $DIR - quota_show_check f u $TSTUSR test_file_soft $TESTFILE $LIMIT $GRACE resetquota -u $TSTUSR - echo " Group quota (soft limit: $LIMIT files grace: $GRACE seconds)" + echo "Group quota (soft limit:$LIMIT files grace:$GRACE seconds)" + # make sure the system is clean + USED=$(getquota -g $TSTUSR global curinodes) + [ $USED -ne 0 ] && error "Used space($USED) for group $TSTUSR isn't 0." + $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace $GRACE $DIR $LFS setquota -g $TSTUSR -b 0 -B 0 -i $LIMIT -I 0 $DIR - quota_show_check f g $TSTUSR TESTFILE=$DIR/$tdir/$tfile-1 test_file_soft $TESTFILE $LIMIT $GRACE resetquota -g $TSTUSR # cleanup - $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR - $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR + $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR + $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR } -run_test_with_stat 4a "File soft limit (start timer, timer goes off, stop timer) ===" +run_test 4a "File soft limit (start timer, timer goes off, stop timer)" -test_4b() { # was test_4a - GR_STR1="1w3d" - GR_STR2="1000s" - GR_STR3="5s" - GR_STR4="1w2d3h4m5s" - GR_STR5="5c" - GR_STR6="1111111111111111" +test_4b() { + local GR_STR1="1w3d" + local GR_STR2="1000s" + local GR_STR3="5s" + local GR_STR4="1w2d3h4m5s" + local GR_STR5="5c" + local GR_STR6="1111111111111111" wait_delete_completed - # test of valid grace strings handling - echo " Valid grace strings test" - $LFS setquota -t -u --block-grace $GR_STR1 --inode-grace $GR_STR2 $DIR - $LFS quota -u -t $DIR | grep "Block grace time: $GR_STR1" - $LFS setquota -t -g --block-grace $GR_STR3 --inode-grace $GR_STR4 $DIR - $LFS quota -g -t $DIR | grep "Inode grace time: $GR_STR4" - - # test of invalid grace strings handling - echo " Invalid grace strings test" - ! $LFS setquota -t -u --block-grace $GR_STR4 --inode-grace $GR_STR5 $DIR - ! $LFS setquota -t -g --block-grace $GR_STR4 --inode-grace $GR_STR6 $DIR - - # cleanup - $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR - $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR + # test of valid grace strings handling + echo "Valid grace strings test" + $LFS setquota -t -u --block-grace $GR_STR1 --inode-grace $GR_STR2 $DIR + $LFS quota -u -t $DIR | grep "Block grace time: $GR_STR1" + $LFS setquota -t -g --block-grace $GR_STR3 --inode-grace $GR_STR4 $DIR + $LFS quota -g -t $DIR | grep "Inode grace time: $GR_STR4" + + # test of invalid grace strings handling + echo " Invalid grace strings test" + ! $LFS setquota -t -u --block-grace $GR_STR4 --inode-grace $GR_STR5 $DIR + ! $LFS setquota -t -g --block-grace $GR_STR4 --inode-grace $GR_STR6 $DIR + + # cleanup + $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR + $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR } -run_test_with_stat 4b "Grace time strings handling ===" +run_test 4b "Grace time strings handling" # chown & chgrp (chown & chgrp successfully even out of block/file quota) test_5() { - mkdir -p $DIR/$tdir - BLIMIT=$(( $BUNIT_SZ * $((OSTCOUNT + 1)) * 10)) # 10 bunits on each server - ILIMIT=$(( $IUNIT_SZ * 10 )) # 10 iunits on mds - - wait_delete_completed - - echo " Set quota limit (0 $BLIMIT 0 $ILIMIT) for $TSTUSR.$TSTUSR" - $LFS setquota -u $TSTUSR -b 0 -B $BLIMIT -i 0 -I $ILIMIT $DIR - $LFS setquota -g $TSTUSR -b 0 -B $BLIMIT -i 0 -I $ILIMIT $DIR - quota_show_check a u $TSTUSR - quota_show_check a g $TSTUSR - - echo " Create more than $ILIMIT files and more than $BLIMIT kbytes ..." - createmany -m $DIR/$tdir/$tfile-0_ $((ILIMIT + 1)) || \ - error "touch failure, expect success" - dd if=/dev/zero of=$DIR/$tdir/$tfile-0_1 bs=$BLK_SZ count=$((BLIMIT+1)) || error "write failure, expect success" - - echo " Chown files to $TSTUSR.$TSTUSR ..." + local BLIMIT=10 # 10M + local ILIMIT=10 # 10 inodes + + setup_quota_test + trap cleanup_quota_test EXIT + + set_mdt_qtype "ug" || error "enable mdt quota failed" + set_ost_qtype "ug" || error "enable ost quota failed" + + echo "Set quota limit (0 ${BLIMIT}M 0 $ILIMIT) for $TSTUSR.$TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B ${BLIMIT}M -i 0 -I $ILIMIT $DIR + $LFS setquota -g $TSTUSR -b 0 -B ${BLIMIT}M -i 0 -I $ILIMIT $DIR + + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curinodes) + [ $USED -ne 0 ] && error "Used inode($USED) for user $TSTUSR isn't 0." + USED=$(getquota -g $TSTUSR global curinodes) + [ $USED -ne 0 ] && error "Used inode($USED) for group $TSTUSR isn't 0." + USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used block($USED) for user $TSTUSR isn't 0." + USED=$(getquota -g $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used block($USED) for group $TSTUSR isn't 0." + + echo "Create more than $ILIMIT files and more than $BLIMIT MB ..." + createmany -m $DIR/$tdir/$tfile-0_ $((ILIMIT + 1)) || + error "create failure, expect success" + $DD of=$DIR/$tdir/$tfile-0_1 count=$((BLIMIT+1)) || + error "write failure, expect success" + + echo "Chown files to $TSTUSR.$TSTUSR ..." for i in `seq 0 $ILIMIT`; do - chown $TSTUSR.$TSTUSR $DIR/$tdir/$tfile-0_$i || \ - quota_error a $TSTUSR "chown failure, but expect success" + chown $TSTUSR.$TSTUSR $DIR/$tdir/$tfile-0_$i || + quota_error a $TSTUSR "chown failure, expect success" done # cleanup unlinkmany $DIR/$tdir/$tfile-0_ $((ILIMIT + 1)) - sync; sleep 3; sync; + cleanup_quota_test resetquota -u $TSTUSR resetquota -g $TSTUSR } -run_test_with_stat 5 "Chown & chgrp successfully even out of block/file quota ===" +run_test 5 "Chown & chgrp successfully even out of block/file quota" -# block quota acquire & release +# test dropping acquire request on master test_6() { - if [ $OSTCOUNT -lt 2 ]; then - skip_env "$OSTCOUNT < 2, too few osts" - return 0; - fi + setup_quota_test + trap cleanup_quota_test EXIT - wait_delete_completed + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir + # make sure no granted quota on ost + set_ost_qtype "ug" || error "enable ost quota failed" + resetquota -u $TSTUSR - LIMIT=$((BUNIT_SZ * (OSTCOUNT + 1) * 5)) # 5 bunits per server - FILEA="$DIR/$tdir/$tfile-0_a" - FILEB="$DIR/$tdir/$tfile-0_b" + # create file for $TSTUSR + local TESTFILE=$DIR/$tdir/$tfile-$TSTUSR + $LFS setstripe $TESTFILE -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE - echo " Set block limit $LIMIT kbytes to $TSTUSR.$TSTUSR" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - $LFS setquota -g $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - quota_show_check b u $TSTUSR - quota_show_check b g $TSTUSR - - echo " Create filea on OST0 and fileb on OST1" - $LFS setstripe $FILEA -i 0 -c 1 - $LFS setstripe $FILEB -i 1 -c 1 - chown $TSTUSR.$TSTUSR $FILEA - chown $TSTUSR.$TSTUSR $FILEB - - echo " Exceed quota limit ..." - RUNDD="$RUNAS dd if=/dev/zero of=$FILEA bs=$BLK_SZ" - $RUNDD count=$((LIMIT - BUNIT_SZ * OSTCOUNT)) || \ - quota_error a $TSTUSR "write filea failure, but expect success" - - cancel_lru_locks osc - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP - $RUNDD seek=$LIMIT count=$((BUNIT_SZ * OSTCOUNT)) && \ - quota_error a $TSTUSR "write filea success, but expect EDQUOT" - cancel_lru_locks osc - echo " Write to OST1 return EDQUOT" - # this write maybe cache write, ignore it's failure - RUNDD="$RUNAS dd if=/dev/zero of=$FILEB bs=$BLK_SZ" - $RUNDD count=$(($BUNIT_SZ * 2)) || true - cancel_lru_locks osc - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP - $RUNDD count=$((BUNIT_SZ * 2)) seek=$((BUNIT_SZ *2)) && \ - quota_error a $TSTUSR "write fileb success, but expect EDQUOT" - - echo " Remove filea to let OST0 release quota" - rm -f $FILEA - - if at_is_enabled; then - timeout=$(at_max_get mds) - else - timeout=$(lctl get_param -n timeout) - fi - count=$((timeout / 5)) - OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` - - while [ $((count--)) -gt 0 ]; do - sync && sleep 5 - - OST0_QUOTA_HOLD=`getquota -u $TSTUSR $OST0_UUID bhardlimit` - if [ -z $OST0_QUOTA_HOLD ]; then - error "System is error when query quota for block (U:$TSTUSR)." - else - [ $OST0_QUOTA_HOLD -gt $BUNIT_SZ ] && continue - fi - - break - done - - [ ! $count -gt 0 ] && error "Release quota for block timeout (U:$TSTUSR)." - $SHOW_QUOTA_USER - - while [ $((count--)) -gt 0 ]; do - sync && sleep 5 - - OST0_QUOTA_HOLD=`getquota -g $TSTUSR $OST0_UUID bhardlimit` - if [ -z $OST0_QUOTA_HOLD ]; then - error "System is error when query quota for block (G:$TSTUSR)." - else - [ $OST0_QUOTA_HOLD -gt $BUNIT_SZ ] && continue - fi - - break - done - - [ ! $count -gt 0 ] && error "Release quota for block timeout (G:$TSTUSR)." - $SHOW_QUOTA_GROUP - - echo " Write to OST1" - $RUNDD count=$((LIMIT - BUNIT_SZ * OSTCOUNT)) || \ - quota_error a $TSTUSR "write fileb failure, expect success" - echo " Done" + # create file for $TSTUSR2 + local TESTFILE2=$DIR/$tdir/$tfile-$TSTUSR2 + $LFS setstripe $TESTFILE2 -c 1 -i 0 + chown $TSTUSR2.$TSTUSR2 $TESTFILE2 + + # cache per-ID lock for $TSTUSR on slave + $LFS setquota -u $TSTUSR -b 0 -B 2M -i 0 -I 0 $DIR + $RUNAS $DD of=$TESTFILE count=1 || + error "write $TESTFILE failure, expect success" + $RUNAS2 $DD of=$TESTFILE2 count=1 || + error "write $TESTFILE2 failure, expect success" + sync; sync + sync_all_data + + # set a small timeout value, to make the quota RPC timedout before + # the watchdog triggered. + local timeout + if at_is_enabled; then + timeout=$(at_max_get ost) + at_max_set 20 ost + else + timeout=$(do_facet ost1 $LCTL get_param -n timeout) + do_facet ost1 $LCTL set_param timeout=20 + fi - # cleanup - rm -f $FILEB - sync; sleep 3; sync; + #define QUOTA_DQACQ 601 + #define OBD_FAIL_PTLRPC_DROP_REQ_OPC 0x513 + lustre_fail mds 0x513 601 - resetquota -u $TSTUSR - resetquota -g $TSTUSR - return 0 -} -run_test_with_stat 6 "Block quota acquire & release =========" + # write to un-enforced ID ($TSTUSR2) should succeed + $RUNAS2 $DD of=$TESTFILE2 count=1 seek=1 oflag=sync conv=notrunc || + error "write failure, expect success" -# quota recovery (block quota only by now) -test_7() -{ - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir + # write to enforced ID ($TSTUSR) in background, exceeding limit + # to make sure DQACQ is sent + $RUNAS $DD of=$TESTFILE count=2 seek=1 oflag=sync conv=notrunc & + DDPID=$! - wait_delete_completed + echo "Sleep for $timeout" + sleep $timeout - LIMIT=$(( $BUNIT_SZ * $(($OSTCOUNT + 1)) )) - TESTFILE="$DIR/$tdir/$tfile-0" + # write should be blocked and never finished + if ! ps -p $DDPID > /dev/null 2>&1; then + lustre_fail mds 0 0 + error "write finished incorrectly!" + fi - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + if at_is_enabled; then + at_max_set $timeout ost + else + do_facet ost1 $LCTL set_param timeout=$timeout + fi - $LFS setstripe $TESTFILE -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE + lustre_fail mds 0 0 - echo " Write to OST0..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$BUNIT_SZ || \ - quota_error u $TSTUSR "write failure, but expect success" + # no watchdog is triggered + do_facet ost1 dmesg > $TMP/lustre-log-${TESTNAME}.log + watchdog=$(awk '/sanity-quota test 6/ {start = 1;} + /Service thread pid/ && /was inactive/ { + if (start) { + print; + } + }' $TMP/lustre-log-${TESTNAME}.log) + [ -z "$watchdog" ] || error "$watchdog" - #define OBD_FAIL_OBD_DQACQ 0x604 - lustre_fail mds 0x604 - echo " Remove files on OST0" - rm -f $TESTFILE - lustre_fail mds 0 + rm -f $TMP/lustre-log-${TESTNAME}.log - echo " Trigger recovery..." - OSC0_UUID="`$LCTL dl | awk '$3 ~ /osc/ { print $1 }'`" - for i in $OSC0_UUID; do - $LCTL --device $i activate || error "activate osc failed!" + # write should continue & succeed + local count=0 + while [ true ]; do + if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi + if [ $count -ge 120 ]; then + quota_error u $TSTUSR "dd not finished in $count secs" + fi + count=$((count + 1)) + [ $((count % 10)) -eq 0 ] && echo "Waiting $count secs" + sleep 1 done - # sleep a while to wait for recovery done - sleep 20 + cleanup_quota_test + resetquota -u $TSTUSR +} +run_test 6 "Test dropping acquire request on master" - # check limits - PATTERN="`echo $DIR | sed 's/\//\\\\\//g'`" - TOTAL_LIMIT=`getquota -u $TSTUSR global bhardlimit` - [ $TOTAL_LIMIT -eq $LIMIT ] || error "total limits not recovery!" - echo " total limits = $TOTAL_LIMIT" +# quota reintegration (global index) +test_7a() { + local TESTFILE=$DIR/$tdir/$tfile + local LIMIT=20 # 20M - OST0_UUID=`do_facet ost1 "$LCTL dl | grep -m1 obdfilter" | awk '{print $((NF-1))}'` - [ -z "$OST0_UUID" ] && OST0_UUID=`do_facet ost1 "$LCTL dl | grep -m1 obdfilter" | awk '{print $((NF-1))}'` - OST0_LIMIT=`getquota -u $TSTUSR $OST0_UUID bhardlimit` - [ $OST0_LIMIT -eq $BUNIT_SZ ] || error "high limits not released!" - echo " limits on $OST0_UUID = $OST0_LIMIT" + [ "$SLOW" = "no" ] && LIMIT=5 - # cleanup + setup_quota_test + trap cleanup_quota_test EXIT + + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." + + # make sure no granted quota on ost1 + set_ost_qtype "ug" || error "enable ost quota failed" resetquota -u $TSTUSR -} -run_test_with_stat 7 "Quota recovery (only block limit) ======" + set_ost_qtype "none" || error "disable ost quota failed" -# run dbench with quota enabled -test_8() { - mkdir -p $DIR/$tdir - BLK_LIMIT=$((100 * 1024 * 1024)) # 100G - FILE_LIMIT=1000000 + local OSTUUID=$(ostuuid_from_index 0) + USED=$(getquota -u $TSTUSR $OSTUUID bhardlimit) + [ $USED -ne 0 ] && error "limit($USED) on $OSTUUID for user" \ + "$TSTUSR isn't 0." - wait_delete_completed + # create test file + $LFS setstripe $TESTFILE -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE - echo " Set enough high limit for user: $TSTUSR" - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR - echo " Set enough high limit for group: $TSTUSR" - $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + echo "Stop ost1..." + stop ost1 - chmod 0777 $DIR/$tdir - local duration="" - [ "$SLOW" = "no" ] && duration=" -t 120" - $RUNAS bash rundbench -D $DIR/$tdir 3 $duration || quota_error a $TSTUSR "dbench failed!" + echo "Enable quota & set quota limit for $TSTUSR" + set_ost_qtype "ug" || error "enable ost quota failed" + $LFS setquota -u $TSTUSR -b 0 -B ${LIMIT}M -i 0 -I 0 $DIR - rm -rf $DIR/$tdir - sync; sleep 3; sync; + echo "Start ost1..." + start ost1 $(ostdevname 1) $OST_MOUNT_OPTS - return 0 -} -run_test_with_stat 8 "Run dbench with quota enabled ===========" + wait_ost_reint "ug" || error "reintegration failed" -# run for fixing bug10707, it needs a big room. test for 64bit -KB=1024 -GB=$((KB * 1024 * 1024)) -# Use this as dd bs to decrease time -# inode->i_blkbits = min(PTLRPC_MAX_BRW_BITS+1, LL_MAX_BLKSIZE_BITS); -blksize=$((1 << 21)) # 2Mb -size_file=$((GB * 9 / 2)) -# this check is just for test_9 -OST0_MIN=4900000 #4.67G -check_whether_skip () { - OST0_SIZE=`$LFS df $DIR | awk '/\[OST:0\]/ {print $4}'` - log "OST0_SIZE: $OST0_SIZE required: $OST0_MIN" - if [ $OST0_SIZE -lt $OST0_MIN ]; then - echo "WARN: OST0 has less than $OST0_MIN free, skip this test." - return 0 - else - return 1 - fi + # hardlimit should have been fetched by slave during global + # reintegration, write will exceed quota + $RUNAS $DD of=$TESTFILE count=$((LIMIT + 1)) oflag=sync && + quota_error u $TSTUSR "write success, but expect EDQUOT" + + cleanup_quota_test + resetquota -u $TSTUSR } +run_test 7a "Quota reintegration (global index)" -test_9() { - check_whether_skip && return 0 +# quota reintegration (slave index) +test_7b() { + local LIMIT="100G" + local TESTFILE=$DIR/$tdir/$tfile - wait_delete_completed + setup_quota_test + trap cleanup_quota_test EXIT - set_blk_tunesz 512 - set_blk_unitsz 1024 + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - TESTFILE="$DIR/$tdir/$tfile-0" + # make sure no granted quota on ost1 + set_ost_qtype "ug" || error "enable ost quota failed" + resetquota -u $TSTUSR + set_ost_qtype "none" || error "disable ost quota failed" + + local OSTUUID=$(ostuuid_from_index 0) + USED=$(getquota -u $TSTUSR $OSTUUID bhardlimit) + [ $USED -ne 0 ] && error "limit($USED) on $OSTUUID for user" \ + "$TSTUSR isn't 0." - BLK_LIMIT=$((100 * KB * KB)) # 100G - FILE_LIMIT=1000000 - echo " Set block limit $BLK_LIMIT kbytes to $TSTUSR.$TSTUSR" + # create test file + $LFS setstripe $TESTFILE -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE - log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for user: $TSTUSR" - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR - log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for group: $TSTUSR" - $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + # consume some space to make sure the granted space will not + # be released during reconciliation + $RUNAS $DD of=$TESTFILE count=1 oflag=sync || + error "consume space failure, expect success" - quota_show_check a u $TSTUSR - quota_show_check a g $TSTUSR + # define OBD_FAIL_QUOTA_EDQUOT 0xa02 + lustre_fail mds 0xa02 - echo " Set stripe" - $LFS setstripe $TESTFILE -c 1 -i 0 - touch $TESTFILE - chown $TSTUSR.$TSTUSR $TESTFILE + set_ost_qtype "ug" || error "enable ost quota failed" + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - log " Write the big file of 4.5G ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$blksize count=$((size_file / blksize)) || \ - quota_error a $TSTUSR "(usr) write 4.5G file failure, but expect success" + # ignore the write error + $RUNAS $DD of=$TESTFILE count=1 seek=1 oflag=sync conv=notrunc - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP + local old_used=$(getquota -u $TSTUSR $OSTUUID bhardlimit) - log " delete the big file of 4.5G..." - $RUNAS rm -f $TESTFILE - sync; sleep 3; sync; + lustre_fail mds 0 - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP + echo "Restart ost to trigger reintegration..." + stop ost1 + start ost1 $(ostdevname 1) $OST_MOUNT_OPTS - RC=$? + wait_ost_reint "ug" || error "reintegration failed" - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) + USED=$(getquota -u $TSTUSR $OSTUUID bhardlimit) + [ $USED -gt $old_used ] || error "limit on $OSTUUID $USED <= $old_used" - return $RC + cleanup_quota_test + resetquota -u $TSTUSR + $SHOW_QUOTA_USER } -run_test_with_stat 9 "run for fixing bug10707(64bit) ===========" +run_test 7b "Quota reintegration (slave index)" -# 2.0 version does not support 32 bit qd_count, -# test_10 "run for fixing bug10707(32bit) " is obsolete +# quota reintegration (restart mds during reintegration) +test_7c() { + local LIMIT=20 # 20M + local TESTFILE=$DIR/$tdir/$tfile -# test a deadlock between quota and journal b=11693 -test_12() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir + [ "$SLOW" = "no" ] && LIMIT=5 - [ "$(grep $DIR2 /proc/mounts)" ] || mount_client $DIR2 || \ - { skip_env "Need lustre mounted on $MOUNT2 " && retutn 0; } + setup_quota_test + trap cleanup_quota_test EXIT - LIMIT=$(( $BUNIT_SZ * $(($OSTCOUNT + 1)) * 10)) # 10 bunits each sever - TESTFILE="$DIR/$tdir/$tfile-0" - TESTFILE2="$DIR2/$tdir/$tfile-1" + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." - wait_delete_completed + set_ost_qtype "none" || error "disable ost quota failed" + $LFS setquota -u $TSTUSR -b 0 -B ${LIMIT}M -i 0 -I 0 $DIR - echo " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + # define OBD_FAIL_QUOTA_DELAY_REINT 0xa03 + lustre_fail ost 0xa03 - $LFS setstripe $TESTFILE -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE - $LFS setstripe $TESTFILE2 -i 0 -c 1 - chown $TSTUSR2.$TSTUSR2 $TESTFILE2 + # enable ost quota to trigger reintegration + set_ost_qtype "ug" || error "enable ost quota failed" - #define OBD_FAIL_OST_HOLD_WRITE_RPC 0x21f - #define OBD_FAIL_SOME 0x10000000 /* fail N times */ - lustre_fail ost $((0x0000021f | 0x10000000)) 1 + echo "Stop mds..." + stop mds1 - echo " step1: write out of block quota ..." - $RUNAS2 dd if=/dev/zero of=$TESTFILE2 bs=$BLK_SZ count=102400 & - DDPID1=$! - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT*2)) & - DDPID=$! + lustre_fail ost 0 - echo " step2: testing ......" - local last_size=$(stat -c %s $TESTFILE2) - local stall_secs=0 - local start_secs=$SECONDS - while [ -d /proc/${DDPID1} ]; do - local size=$(stat -c %s $TESTFILE2) - if [ $size -eq $last_size ]; then - stall_secs=$[stall_secs+1] - else - stall_secs=0 - fi - if [ $stall_secs -gt 30 ]; then - lustre_fail ost 0 - quota_error u $TSTUSR2 "giving up: dd stalled (i.e. made no progress) for 30 seconds!" - fi - last_size=$size - sleep 1 - done - echo "(dd_pid=$DDPID1, time=$((SECONDS-start_secs)))successful" + echo "Start mds..." + start mds1 $(mdsdevname 1) $MDS_MOUNT_OPTS - #Recover fail_loc and dd will finish soon - lustre_fail ost 0 + wait_ost_reint "ug" || error "reintegration failed" - echo " step3: testing ......" - count=0 - while [ true ]; do - if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi - count=$[count+1] - if [ $count -gt 150 ]; then - quota_error u $TSTUSR "dd should be finished!" - fi - sleep 1 - done - echo "(dd_pid=$DDPID, time=$count)successful" + # hardlimit should have been fetched by slave during global + # reintegration, write will exceed quota + $RUNAS $DD of=$TESTFILE count=$((LIMIT + 1)) oflag=sync && + quota_error u $TSTUSR "write success, but expect EDQUOT" + + cleanup_quota_test + resetquota -u $TSTUSR +} +run_test 7c "Quota reintegration (restart mds during reintegration)" + +# run dbench with quota enabled +test_8() { + local BLK_LIMIT="100g" #100G + local FILE_LIMIT=1000000 - rm -f $TESTFILE $TESTFILE2 - sync; sleep 3; sync; + setup_quota_test + trap cleanup_quota_test EXIT - resetquota -u $TSTUSR + set_mdt_qtype "ug" || error "enable mdt quota failed" + set_ost_qtype "ug" || error "enable ost quota failed" + + echo "Set enough high limit for user: $TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + echo "Set enough high limit for group: $TSTUSR" + $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + + local duration="" + [ "$SLOW" = "no" ] && duration=" -t 120" + $RUNAS bash rundbench -D $DIR/$tdir 3 $duration || + quota_error a $TSTUSR "dbench failed!" + + cleanup_quota_test + resetquota -u $TSTUSR + resetquota -g $TSTUSR } -run_test_with_stat 12 "test a deadlock between quota and journal ===" +run_test 8 "Run dbench with quota enabled" -# test multiple clients write block quota b=11693 -test_13() { - mkdir -p $DIR/$tdir - wait_delete_completed +# this check is just for test_9 +OST0_MIN=4900000 #4.67G - # one OST * 10 + (mds + other OSTs) - LIMIT=$((BUNIT_SZ * 10 + (BUNIT_SZ * OSTCOUNT))) - TESTFILE="$DIR/$tdir/$tfile" +check_whether_skip () { + local OST0_SIZE=$($LFS df $DIR | awk '/\[OST:0\]/ {print $4}') + log "OST0_SIZE: $OST0_SIZE required: $OST0_MIN" + if [ $OST0_SIZE -lt $OST0_MIN ]; then + echo "WARN: OST0 has less than $OST0_MIN free, skip this test." + return 0 + else + return 1 + fi +} - echo " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - quota_show_check b u $TSTUSR +# run for fixing bug10707, it needs a big room. test for 64bit +test_9() { + local filesize=$((1024 * 1024 * 1024 * 9 / 2)) # 4.5G - $LFS setstripe $TESTFILE -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE - $LFS setstripe $TESTFILE.2 -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE.2 + check_whether_skip && return 0 - echo " step1: write out of block quota ..." - # one bunit will give mds - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$[($LIMIT - $BUNIT_SZ) / 2] & - DDPID=$! - $RUNAS dd if=/dev/zero of=$TESTFILE.2 bs=$BLK_SZ count=$[($LIMIT - $BUNIT_SZ) / 2] & - DDPID1=$! + setup_quota_test + trap cleanup_quota_test EXIT - echo " step2: testing ......" - count=0 - while [ true ]; do - if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi - count=$[count+1] - if [ $count -gt 64 ]; then - quota_error u $TSTUSR "dd should be finished!" - fi - sleep 1 - done - echo "(dd_pid=$DDPID, time=$count)successful" + set_ost_qtype "ug" || error "enable ost quota failed" - count=0 - while [ true ]; do - if ! ps -p ${DDPID1} > /dev/null 2>&1 ; then break; fi - count=$[count+1] - if [ $count -gt 64 ]; then - quota_error u $TSTUSR "dd should be finished!" - fi - sleep 1 - done - echo "(dd_pid=$DDPID1, time=$count)successful" + local TESTFILE="$DIR/$tdir/$tfile-0" + local BLK_LIMIT=100G #100G + local FILE_LIMIT=1000000 - sync; sleep 5; sync; + echo "Set block limit $BLK_LIMIT bytes to $TSTUSR.$TSTUSR" - echo " step3: checking ......" - fz=`stat -c %s $TESTFILE` - fz2=`stat -c %s $TESTFILE.2` - $SHOW_QUOTA_USER - [ $((fz + fz2)) -lt $((BUNIT_SZ * BLK_SZ * 10)) ] && \ - quota_error u $TSTUSR "files too small $fz + $fz2 < $((BUNIT_SZ * BLK_SZ * 10))" + log "Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT)" \ + "for user: $TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + + log "Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT)" \ + "for group: $TSTUSR" + $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + + quota_show_check a u $TSTUSR + quota_show_check a g $TSTUSR + + echo "Create test file" + $LFS setstripe $TESTFILE -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE - rm -f $TESTFILE $TESTFILE.2 - sync; sleep 3; sync; + log "Write the big file of 4.5G ..." + $RUNAS $DD of=$TESTFILE count=$filesize || + quota_error a $TSTUSR "write 4.5G file failure, expect success" + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + + cleanup_quota_test resetquota -u $TSTUSR + resetquota -g $TSTUSR + + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP } -run_test_with_stat 13 "test multiple clients write block quota ===" - -check_if_quota_zero(){ - line=`$LFS quota -v -$1 $2 $DIR | wc -l` - for i in `seq 3 $line`; do - if [ $i -eq 3 ]; then - field="3 4 7 8" - else - field="3 6" - fi - for j in $field; do - tmp=`$LFS quota -v -$1 $2 $DIR | sed -n ${i}p | - awk '{print $'"$j"'}'` - [ -n "$tmp" ] && [ $tmp -ne 0 ] && $LFS quota -v -$1 $2 $DIR && \ - error "quota on $2 isn't clean" - done - done - echo "pass check_if_quota_zero" -} +run_test 9 "Block limit larger than 4GB (b10707)" -test_14a() { # was test_14 b=12223 -- setting quota on root - TESTFILE="$DIR/$tdir/$tfile" +test_10() { + local TESTFILE=$DIR/$tdir/$tfile - # reboot the lustre - sync; sleep 5; sync - cleanup_and_setup_lustre - quota_init + setup_quota_test + trap cleanup_quota_test EXIT - mkdir -p $DIR/$tdir + # set limit to root user should fail + $LFS setquota -u root -b 100G -B 500G -i 1K -I 1M $DIR && + error "set limit for root user successfully, expect failure" + $LFS setquota -g root -b 1T -B 10T -i 5K -I 100M $DIR && + error "set limit for root group successfully, expect failure" - # out of root's file and block quota - $LFS setquota -u root -b 10 -B 10 -i 10 -I 10 $DIR - createmany -m ${TESTFILE} 20 || \ - quota_error u root "unexpected: user(root) create files failly!" - $MULTIOP ${TESTFILE} oO_CREAT:O_WRONLY:O_DIRECT:w$((4096 * 4096))c || \ - quota_error u root "unexpected: user(root) write files failly!" - chmod 666 $TESTFILE - $RUNAS $MULTIOP ${TESTFILE} oO_WRONLY:O_APPEND:O_DIRECT:w$((4096 * 4096))c && \ - quota_error u root "unexpected: user(quota_usr) write a file successfully!" - - # trigger the llog - chmod 777 $DIR - for i in `seq 1 10`; do $RUNAS touch ${TESTFILE}a_$i; done - for i in `seq 1 10`; do $RUNAS rm -f ${TESTFILE}a_$i; done - - # do the check - dmesg | tail | grep "\-122" |grep llog_obd_origin_add && error "err -122 not found in dmesg" - resetquota -u root - #check_if_quota_zero u root - - # clean - unlinkmany ${TESTFILE} 15 - rm -f $TESTFILE - sync; sleep 3; sync; -} -run_test_with_stat 14a "test setting quota on root ===" + # root user can overrun quota + set_ost_qtype "ug" || "enable ost quota failed" -test_15(){ - LIMIT=$((24 * 1024 * 1024 * 1024 * 1024)) # 24 TB - PATTERN="`echo $DIR | sed 's/\//\\\\\//g'`" + $LFS setquota -u $TSTUSR -b 0 -B 2M -i 0 -I 0 $DIR + quota_show_check b u $TSTUSR - wait_delete_completed + $LFS setstripe $TESTFILE -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE - # test for user - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - TOTAL_LIMIT=`getquota -u $TSTUSR global bhardlimit` - [ $TOTAL_LIMIT -eq $LIMIT ] || error " (user)total limits = $TOTAL_LIMIT; limit = $LIMIT, failed!" - echo " (user)total limits = $TOTAL_LIMIT; limit = $LIMIT, successful!" - resetquota -u $TSTUSR - - # test for group - $LFS setquota -g $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - TOTAL_LIMIT=`getquota -g $TSTUSR global bhardlimit` - [ $TOTAL_LIMIT -eq $LIMIT ] || error " (group)total limits = $TOTAL_LIMIT; limit = $LIMIT, failed!" - echo " (group)total limits = $TOTAL_LIMIT; limit = $LIMIT, successful!" - resetquota -g $TSTUSR - $LFS quotaoff -ug $DIR - do_facet $SINGLEMDS "lctl set_param lquota.mdd_obd-${FSNAME}-MDT*.quota_type=ug" | grep "error writing" && \ - error "fail to set version for $SINGLEMDS" - for j in `seq $OSTCOUNT`; do - do_facet ost$j "lctl set_param lquota.${FSNAME}-OST*.quota_type=ug" | grep "error writing" && \ - error "fail to set version for ost$j" - done - - $LFS quotacheck -ug $DIR + runas -u 0 -g 0 $DD of=$TESTFILE count=3 oflag=sync || + error "write failure, expect success" + + cleanup_quota_test + resetquota -u $TSTUSR } -run_test_with_stat 15 "set block quota more than 4T ===" +run_test 10 "Test quota for root user" -# 2.0 version does not support WITHOUT_CHANGE_QS, -# test_16 "test without adjusting qunit" is obsolete +test_11() { + local TESTFILE=$DIR/$tdir/$tfile + setup_quota_test + trap cleanup_quota_test EXIT -# run for fixing bug14526, failed returned quota reqs shouldn't ruin lustre. -test_17() { - set_blk_tunesz 512 - set_blk_unitsz 1024 + set_mdt_qtype "ug" || "enable mdt quota failed" + $LFS setquota -u $TSTUSR -b 0 -B 0 -i 0 -I 1 $DIR - wait_delete_completed + touch "$TESTFILE"-0 + touch "$TESTFILE"-1 - #define OBD_FAIL_QUOTA_RET_QDATA | OBD_FAIL_ONCE - lustre_fail ost 0x80000A02 + chown $TSTUSR.$TSTUSR "$TESTFILE"-0 + chown $TSTUSR.$TSTUSR "$TESTFILE"-1 - TESTFILE="$DIR/$tdir/$tfile-a" - TESTFILE2="$DIR/$tdir/$tfile-b" - mkdir -p $DIR/$tdir + $SHOW_QUOTA_USER + local USED=$(getquota -u $TSTUSR global curinodes) + [ $USED -ge 2 ] || error "Used inodes($USED) is less than 2" - BLK_LIMIT=$((100 * 1024)) # 100M + cleanup_quota_test + resetquota -u $TSTUSR +} +run_test 11 "Chown/chgrp ignores quota" - log " Set enough high limit(block:$BLK_LIMIT) for user: $TSTUSR" - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR - log " Set enough high limit(block:$BLK_LIMIT) for group: $TSTUSR" - $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR +test_12() { + [ "$OSTCOUNT" -lt "2" ] && skip "skipping rebalancing test" && return - quota_show_check b u $TSTUSR - quota_show_check b g $TSTUSR + local blimit=22 # 22M + local blk_cnt=$((blimit - 5)) + local TESTFILE0="$DIR/$tdir/$tfile"-0 + local TESTFILE1="$DIR/$tdir/$tfile"-1 - touch $TESTFILE - chown $TSTUSR.$TSTUSR $TESTFILE - touch $TESTFILE2 - chown $TSTUSR.$TSTUSR $TESTFILE2 + setup_quota_test + trap cleanup_quota_test EXIT - log " Write the test file1 ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(( 10 * 1024 )) \ - || quota_error a $TSTUSR "write 10M file failure" + set_ost_qtype "u" || "enable ost quota failed" + quota_show_check b u $TSTUSR - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP + $LFS setquota -u $TSTUSR -b 0 -B "$blimit"M -i 0 -I 0 $DIR - log " write the test file2 ..." - $RUNAS dd if=/dev/zero of=$TESTFILE2 bs=$BLK_SZ count=$(( 10 * 1024 )) \ - || quota_error a $TSTUSR "write 10M file failure" + $LFS setstripe $TESTFILE0 -c 1 -i 0 + $LFS setstripe $TESTFILE1 -c 1 -i 1 + chown $TSTUSR.$TSTUSR $TESTFILE0 + chown $TSTUSR.$TSTUSR $TESTFILE1 - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP + echo "Write to ost0..." + $RUNAS $DD of=$TESTFILE0 count=$blk_cnt oflag=sync || + quota_error a $TSTUSR "dd failed" - rm -f $TESTFILE $TESTFILE2 - RC=$? - sync; sleep 3; sync; + echo "Write to ost1..." + $RUNAS $DD of=$TESTFILE1 count=$blk_cnt oflag=sync && + quota_error a $TSTUSR "dd succeed, expect EDQUOT" - # make qd_count 64 bit - lustre_fail ost 0 + echo "Free space from ost0..." + rm -f $TESTFILE0 + wait_delete_completed + sync_all_data - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) + echo "Write to ost1 after space freed from ost0..." + $RUNAS $DD of=$TESTFILE1 count=$blk_cnt oflag=sync || + quota_error a $TSTUSR "rebalancing failed" + cleanup_quota_test resetquota -u $TSTUSR - resetquota -g $TSTUSR - - return $RC } -run_test_with_stat 17 "run for fixing bug14526 ===========" - -# test when mds takes a long time to handle a quota req so that -# the ost has dropped it, the ost still could work well b=14840 -test_18() { - LIMIT=$((100 * 1024 * 1024)) # 100G - TESTFILE="$DIR/$tdir/$tfile" - mkdir -p $DIR/$tdir +run_test 12 "Block quota rebalancing" - wait_delete_completed +test_13(){ + local TESTFILE=$DIR/$tdir/$tfile + # the name of osp on ost1 name is MDT0000-osp-OST0000 + local procf="ldlm.namespaces.*MDT0000-osp-OST0000.lru_size" - set_blk_tunesz 512 - set_blk_unitsz 1024 + setup_quota_test + trap cleanup_quota_test EXIT - log " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT - quota_show_check b u $TSTUSR + set_ost_qtype "u" || "enable ost quota failed" + quota_show_check b u $TSTUSR - $LFS setstripe $TESTFILE -i 0 -c 1 + $LFS setquota -u $TSTUSR -b 0 -B 10M -i 0 -I 0 $DIR + $LFS setstripe $TESTFILE -c 1 -i 0 chown $TSTUSR.$TSTUSR $TESTFILE - #define OBD_FAIL_MDS_BLOCK_QUOTA_REQ 0x13c - lustre_fail mds 0x13c - - log " step1: write 100M block ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$((1024 * 100)) & - DDPID=$! + # clear the locks in cache first + do_facet ost1 $LCTL set_param -n $procf=clear + local nlock=$(do_facet ost1 $LCTL get_param -n $procf) + [ $nlock -eq 0 ] || error "$nlock cached locks" - sleep 5 - lustre_fail mds 0 + # write to acquire the per-ID lock + $RUNAS $DD of=$TESTFILE count=1 oflag=sync || + quota_error a $TSTUSR "dd failed" - echo " step2: testing ......" - count=0 - if at_is_enabled; then - timeout=$(at_max_get mds) - else - timeout=$(lctl get_param -n timeout) - fi - while [ true ]; do - if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi - count=$[count+1] - if [ $count -gt $((4 * $timeout)) ]; then - quota_error u $TSTUSR "count=$count dd should be finished!" - fi - sleep 1 - done - log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" - sync - cancel_lru_locks mdc - cancel_lru_locks osc - - testfile_size=$(stat -c %s $TESTFILE) - [ $testfile_size -ne $((BLK_SZ * 1024 * 100)) ] && \ - quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100)), got ${testfile_size}. Verifying file failed!" - $SHOW_QUOTA_USER - rm -f $TESTFILE - sync + nlock=$(do_facet ost1 $LCTL get_param -n $procf) + [ $nlock -eq 1 ] || error "lock count($nlock) isn't 1" + # clear quota doesn't trigger per-ID lock cancellation resetquota -u $TSTUSR - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) + nlock=$(do_facet ost1 $LCTL get_param -n $procf) + [ $nlock -eq 1 ] || error "per-ID lock is lost on quota clear" + + # clear the per-ID lock + do_facet ost1 $LCTL set_param -n $procf=clear + nlock=$(do_facet ost1 $LCTL get_param -n $procf) + [ $nlock -eq 0 ] || error "per-ID lock isn't cleared" + + # spare quota should be released + local OSTUUID=$(ostuuid_from_index 0) + local limit=$(getquota -u $TSTUSR $OSTUUID bhardlimit) + local space=$(getquota -u $TSTUSR $OSTUUID curspace) + [ $limit -le $space ] || + error "spare quota isn't released, limit:$limit, space:$space" + + cleanup_quota_test } -run_test_with_stat 18 "run for fixing bug14840 ===========" +run_test 13 "Cancel per-ID lock in the LRU list" -# test when mds drops a quota req, the ost still could work well b=14840 -test_18a() { - LIMIT=$((100 * 1024 * 1024)) # 100G - TESTFILE="$DIR/$tdir/$tfile-a" - mkdir -p $DIR/$tdir +test_15(){ + local LIMIT=$((24 * 1024 * 1024 * 1024 * 1024)) # 24 TB wait_delete_completed + sync_all_data - set_blk_tunesz 512 - set_blk_unitsz 1024 + # test for user + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + local TOTAL_LIMIT=$(getquota -u $TSTUSR global bhardlimit) + [ $TOTAL_LIMIT -eq $LIMIT ] || + error "(user) limit:$TOTAL_LIMIT, expect:$LIMIT, failed!" + resetquota -u $TSTUSR - log " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT - quota_show_check b u $TSTUSR + # test for group + $LFS setquota -g $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + TOTAL_LIMIT=$(getquota -g $TSTUSR global bhardlimit) + [ $TOTAL_LIMIT -eq $LIMIT ] || + error "(group) limits:$TOTAL_LIMIT, expect:$LIMIT, failed!" + resetquota -g $TSTUSR +} +run_test 15 "Set over 4T block quota" - $LFS setstripe $TESTFILE -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE +test_17sub() { + local err_code=$1 + local BLKS=1 # 1M less than limit + local TESTFILE=$DIR/$tdir/$tfile - #define OBD_FAIL_MDS_DROP_QUOTA_REQ | OBD_FAIL_ONCE 0x8000013d - lustre_fail mds 0x8000013d + setup_quota_test + trap cleanup_quota_test EXIT - log " step1: write 100M block ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$((1024 * 100)) & - DDPID=$! + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." - echo " step2: testing ......" - count=0 - if at_is_enabled; then - timeout=$(at_max_get mds) - else - timeout=$(lctl get_param -n timeout) + set_ost_qtype "ug" || error "enable ost quota failed" + # make sure no granted quota on ost + resetquota -u $TSTUSR + $LFS setquota -u $TSTUSR -b 0 -B 10M -i 0 -I 0 $DIR + + quota_show_check b u $TSTUSR + + #define OBD_FAIL_QUOTA_RECOVERABLE_ERR 0xa04 + lustre_fail mds 0xa04 $err_code + + # write in background + $RUNAS $DD of=$TESTFILE count=$BLKS oflag=direct & + local DDPID=$! + + sleep 2 + # write should be blocked and never finished + if ! ps -p $DDPID > /dev/null 2>&1; then + lustre_fail mds 0 0 + quota_error u $TSTUSR "write finished incorrectly!" fi + + lustre_fail mds 0 0 + + local count=0 + local timeout=30 while [ true ]; do - if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi - count=$[count+1] - if [ $count -gt $((6 * $timeout)) ]; then - lustre_fail mds 0 - quota_error u $TSTUSR "count=$count dd should be finished!" - fi - sleep 1 + if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi + count=$((count+1)) + if [ $count -gt $timeout ]; then + quota_error u $TSTUSR "dd is not finished!" + fi + sleep 1 done - log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" - lustre_fail mds 0 - rm -f $TESTFILE - sync + sync; sync_all_data + + USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ge $(($BLKS * 1024)) ] || quota_error u $TSTUSR \ + "Used space(${USED}K) is less than ${BLKS}M" + cleanup_quota_test resetquota -u $TSTUSR - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) } -run_test_with_stat 18a "run for fixing bug14840 ===========" -# test when mds do failover, the ost still could work well without trigger -# watchdog b=14840 -test_18bc_sub() { - type=$1 +# DQACQ return recoverable error +test_17() { + echo "DQACQ return -ENOLCK" + #define ENOLCK 37 + test_17sub 37 || error "Handle -ENOLCK failed" + + echo "DQACQ return -EAGAIN" + #define EAGAIN 11 + test_17sub 11 || error "Handle -EAGAIN failed" + + echo "DQACQ return -ETIMEDOUT" + #define ETIMEDOUT 110 + test_17sub 110 || error "Handle -ETIMEDOUT failed" + + echo "DQACQ return -ENOTCONN" + #define ENOTCONN 107 + test_17sub 107 || error "Handle -ENOTCONN failed" +} - LIMIT=$(((100 + $OSTCOUNT * 3) * 1024)) - TESTFILE="$DIR/$tdir/$tfile" - mkdir -p $DIR/$tdir +run_test 17 "DQACQ return recoverable error" - wait_delete_completed +test_18_sub () { + local io_type=$1 + local blimit="200m" # 200M + local TESTFILE="$DIR/$tdir/$tfile" - set_blk_tunesz 512 - set_blk_unitsz 1024 + setup_quota_test + trap cleanup_quota_test EXIT - log " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT - quota_show_check b u $TSTUSR + set_ost_qtype "u" || error "enable ost quota failed" + log "User quota (limit: $blimit)" + $LFS setquota -u $TSTUSR -b 0 -B $blimit -i 0 -I 0 $MOUNT + quota_show_check b u $TSTUSR - $LFS setstripe $TESTFILE -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE + $LFS setstripe $TESTFILE -i 0 -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE - timeout=$(sysctl -n lustre.timeout) + local timeout=$(sysctl -n lustre.timeout) - if [ $type = "directio" ]; then - log " write 100M block(directio) ..." - $RUNAS $DIRECTIO write $TESTFILE 0 100 $((BLK_SZ * 1024)) & + if [ $io_type = "directio" ]; then + log "Write 100M (directio) ..." + $RUNAS $DD of=$TESTFILE count=100 oflag=direct & else - log " write 100M block(normal) ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$((BLK_SZ * 1024)) count=100 & + log "Write 100M (buffered) ..." + $RUNAS $DD of=$TESTFILE count=100 & fi - - DDPID=$! - do_facet $SINGLEMDS "$LCTL conf_param ${FSNAME}-MDT*.mdd.quota_type=ug" + local DDPID=$! replay_barrier $SINGLEMDS + log "Fail mds for $((2 * timeout)) seconds" + fail $SINGLEMDS $((2 * timeout)) - log "failing mds for $((2 * timeout)) seconds" - fail $SINGLEMDS $((2 * timeout)) + local count=0 + if at_is_enabled; then + timeout=$(at_max_get mds) + else + timeout=$(lctl get_param -n timeout) + fi - # check if quotaon successful - $LFS quota -u $TSTUSR $MOUNT 2>&1 | grep -q "quotas are not enabled" - if [ $? -eq 0 ]; then - rm -rf $TESTFILE - error "quotaon failed!" - return - fi + while [ true ]; do + if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi + if [ $((++count % (2 * timeout) )) -eq 0 ]; then + log "it took $count second" + fi + sleep 1 + done - count=0 - if at_is_enabled; then - timeout=$(at_max_get mds) - else - timeout=$(lctl get_param -n timeout) - fi - while [ true ]; do - if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi - if [ $((++count % (2 * timeout) )) -eq 0 ]; then - log "it took $count second" - fi - sleep 1 - done - log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" - sync - cancel_lru_locks mdc - cancel_lru_locks osc - $SHOW_QUOTA_USER + log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" + sync + cancel_lru_locks mdc + cancel_lru_locks osc + $SHOW_QUOTA_USER - resetquota -u $TSTUSR - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) - testfile_size=$(stat -c %s $TESTFILE) + local testfile_size=$(stat -c %s $TESTFILE) if [ $testfile_size -ne $((BLK_SZ * 1024 * 100)) ] ; then - rm -f $TESTFILE - quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100)), got ${testfile_size}. Verifying file failed!" + quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100))," + "got ${testfile_size}. Verifying file failed!" fi - rm -f $TESTFILE - + cleanup_quota_test + resetquota -u $TSTUSR } # test when mds does failover, the ost still could work well # this test shouldn't trigger watchdog b=14840 -test_18b() { - test_18bc_sub normal - test_18bc_sub directio +test_18() { + test_18_sub normal + test_18_sub directio + # check if watchdog is triggered do_facet ost1 dmesg > $TMP/lustre-log-${TESTNAME}.log - watchdog=`awk '/test 18b/ {start = 1;} + local watchdog=$(awk '/test 18b/ {start = 1;} /Service thread pid/ && /was inactive/ { if (start) { print; } - }' $TMP/lustre-log-${TESTNAME}.log` - [ `echo "$watchdog" | wc -l` -ge 3 ] && error "$watchdog" + }' $TMP/lustre-log-${TESTNAME}.log) + [ -z "$watchdog" ] || error "$watchdog" rm -f $TMP/lustre-log-${TESTNAME}.log } -run_test_with_stat 18b "run for fixing bug14840(mds failover, no watchdog) ===========" +run_test 18 "MDS failover while writing, no watchdog triggered (b14840)" -# test when mds does failover, the ost still could work well -# this test will prevent OST_DISCONNET from happening b=14840 -test_18c() { - # define OBD_FAIL_OST_DISCONNECT_NET 0x202(disable ost_disconnect for osts) - lustre_fail ost 0x202 - test_18bc_sub normal - test_18bc_sub directio - lustre_fail ost 0 -} -run_test_with_stat 18c "run for fixing bug14840(mds failover, OST_DISCONNECT is disabled) ===========" +test_19() { + local blimit=5 # 5M + local TESTFILE=$DIR/$tdir/$tfile -run_to_block_limit() { - local LIMIT=$((($OSTCOUNT + 1) * $BUNIT_SZ)) - local TESTFILE=$1 - wait_delete_completed + setup_quota_test + trap cleanup_quota_test EXIT - # set 1 Mb quota unit size - set_blk_tunesz 512 - set_blk_unitsz 1024 + set_ost_qtype "ug" || error "enable ost quota failed" # bind file to a single OST $LFS setstripe -c 1 $TESTFILE chown $TSTUSR.$TSTUSR $TESTFILE - echo " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT - quota_show_check b u $TSTUSR - echo " Updating quota limits" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT - quota_show_check b u $TSTUSR - - RUNDD="$RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ" - $RUNDD count=$BUNIT_SZ || quota_error u $TSTUSR "(usr) write failure, but expect success" - # for now page cache of TESTFILE may still be dirty, - # let's push it to the corresponding OST, this will also - # cache NOQUOTA on the client from OST's reply - cancel_lru_locks osc - $RUNDD seek=$BUNIT_SZ && quota_error u $TSTUSR "(usr) write success, should be EDQUOT" -} - -test_19() { - # 1 Mb bunit per each MDS/OSS - local TESTFILE="$DIR/$tdir/$tfile" - mkdir -p $DIR/$tdir + echo "Set user quota (limit: "$blimit"M)" + $LFS setquota -u $TSTUSR -b 0 -B "$blimit"M -i 0 -I 0 $MOUNT + quota_show_check b u $TSTUSR + echo "Update quota limits" + $LFS setquota -u $TSTUSR -b 0 -B "$blimit"M -i 0 -I 0 $MOUNT + quota_show_check b u $TSTUSR - run_to_block_limit $TESTFILE + # first wirte might be cached + $RUNAS $DD of=$TESTFILE count=$((blimit + 1)) + cancel_lru_locks osc + $SHOW_QUOTA_USER + $RUNAS $DD of=$TESTFILE count=$((blimit + 1)) seek=$((blimit + 1)) && + quota_error u $TSTUSR "Write success, expect failure" $SHOW_QUOTA_USER - # cleanup - rm -f $TESTFILE + cleanup_quota_test resetquota -u $TSTUSR - - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) - } -run_test_with_stat 19 "test if administrative limits updates do not zero operational limits (14790) ===" - -test_20() -{ - LSTR=(1t 2g 3m 4k) # limits strings - LVAL=($[1*1024*1024*1024] $[2*1024*1024] $[3*1024*1024] $[4*1024]) # limits values +run_test 19 "Updating admin limits doesn't zero operational limits(b14790)" - $LFS setquota -u $TSTUSR --block-softlimit ${LSTR[0]} \ - $MOUNT || error "could not set quota limits" +test_20() { # b15754 + local LSTR=(2g 1t 4k 3m) # limits strings + # limits values + local LVAL=($[2*1024*1024] $[1*1024*1024*1024] $[4*1024] $[3*1024*1024]) - $LFS setquota -u $TSTUSR --block-hardlimit ${LSTR[1]} \ - --inode-softlimit ${LSTR[2]} \ - --inode-hardlimit ${LSTR[3]} \ - $MOUNT || error "could not set quota limits" + resetquota -u $TSTUSR - [ "`getquota -u $TSTUSR global bsoftlimit`" = "${LVAL[0]}" ] || error "bsoftlimit was not set properly" - [ "`getquota -u $TSTUSR global bhardlimit`" = "${LVAL[1]}" ] || error "bhardlimit was not set properly" - [ "`getquota -u $TSTUSR global isoftlimit`" = "${LVAL[2]}" ] || error "isoftlimit was not set properly" - [ "`getquota -u $TSTUSR global ihardlimit`" = "${LVAL[3]}" ] || error "ihardlimit was not set properly" + $LFS setquota -u $TSTUSR --block-softlimit ${LSTR[0]} \ + $MOUNT || error "could not set quota limits" + $LFS setquota -u $TSTUSR --block-hardlimit ${LSTR[1]} \ + --inode-softlimit ${LSTR[2]} \ + --inode-hardlimit ${LSTR[3]} \ + $MOUNT || error "could not set quota limits" + + [ "$(getquota -u $TSTUSR global bsoftlimit)" = "${LVAL[0]}" ] || + error "bsoftlimit was not set properly" + [ "$(getquota -u $TSTUSR global bhardlimit)" = "${LVAL[1]}" ] || + error "bhardlimit was not set properly" + [ "$(getquota -u $TSTUSR global isoftlimit)" = "${LVAL[2]}" ] || + error "isoftlimit was not set properly" + [ "$(getquota -u $TSTUSR global ihardlimit)" = "${LVAL[3]}" ] || + error "ihardlimit was not set properly" - resetquota -u $TSTUSR + resetquota -u $TSTUSR } -run_test_with_stat 20 "test if setquota specifiers work properly (15754)" +run_test 20 "Test if setquota specifiers work properly (b15754)" test_21_sub() { local testfile=$1 local blk_number=$2 local seconds=$3 - time=$(($(date +%s) + seconds)) + local time=$(($(date +%s) + seconds)) while [ $(date +%s) -lt $time ]; do - $RUNAS dd if=/dev/zero of=$testfile bs=$BLK_SZ count=$blk_number > /dev/null 2>&1 - rm -f $testfile + $RUNAS $DD of=$testfile count=$blk_number > /dev/null 2>&1 done } # run for fixing bug16053, setquota shouldn't fail when writing and # deleting are happening test_21() { - set_blk_tunesz 512 - set_blk_unitsz 1024 - - wait_delete_completed + local TESTFILE="$DIR/$tdir/$tfile" + local BLIMIT=10 # 10G + local ILIMIT=1000000 - TESTFILE="$DIR/$tdir/$tfile" + setup_quota_test + trap cleanup_quota_test EXIT - BLK_LIMIT=$((10 * 1024 * 1024)) # 10G - FILE_LIMIT=1000000 + set_ost_qtype "ug" || error "Enable ost quota failed" - log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for user: $TSTUSR" - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $MOUNT - log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for group: $TSTUSR" - $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $MOUNT + log "Set limit(block:${BLIMIT}G; file:$ILIMIT) for user: $TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B ${BLIMIT}G -i 0 -I $ILIMIT $MOUNT + log "Set limit(block:${BLIMIT}G; file:$ILIMIT) for group: $TSTUSR" + $LFS setquota -g $TSTUSR -b 0 -B $BLIMIT -i 0 -I $ILIMIT $MOUNT # repeat writing on a 1M file - test_21_sub ${TESTFILE}_1 1024 30 & - DDPID1=$! + test_21_sub ${TESTFILE}_1 1 30 & + local DDPID1=$! # repeat writing on a 128M file - test_21_sub ${TESTFILE}_2 $((1024 * 128)) 30 & - DDPID2=$! + test_21_sub ${TESTFILE}_2 128 30 & + local DDPID2=$! - time=$(($(date +%s) + 30)) - i=1 + local time=$(($(date +%s) + 30)) + local i=1 while [ $(date +%s) -lt $time ]; do - log " Set quota for $i times" - $LFS setquota -u $TSTUSR -b 0 -B $((BLK_LIMIT + 1024 * i)) -i 0 -I $((FILE_LIMIT + i)) $MOUNT - $LFS setquota -g $TSTUSR -b 0 -B $((BLK_LIMIT + 1024 * i)) -i 0 -I $((FILE_LIMIT + i)) $MOUNT - i=$((i+1)) - sleep 1 + log "Set quota for $i times" + $LFS setquota -u $TSTUSR -b 0 -B "$((BLIMIT + i))G" -i 0 \ + -I $((ILIMIT + i)) $MOUNT || + error "Set user quota failed" + $LFS setquota -g $TSTUSR -b 0 -B "$((BLIMIT + i))G" -i 0 \ + -I $((ILIMIT + i)) $MOUNT || + error "Set group quota failed" + i=$((i+1)) + sleep 1 done - count=0 + local count=0 while [ true ]; do - if ! ps -p ${DDPID1} > /dev/null 2>&1; then break; fi - count=$[count+1] - if [ $count -gt 60 ]; then - quota_error a $TSTUSR "dd should be finished!" - fi - sleep 1 + if ! ps -p ${DDPID1} > /dev/null 2>&1; then break; fi + count=$((count+1)) + if [ $count -gt 60 ]; then + quota_error a $TSTUSR "dd should be finished!" + fi + sleep 1 done echo "(dd_pid=$DDPID1, time=$count)successful" count=0 while [ true ]; do - if ! ps -p ${DDPID2} > /dev/null 2>&1; then break; fi - count=$[count+1] - if [ $count -gt 60 ]; then - quota_error a $TSTUSR "dd should be finished!" - fi - sleep 1 + if ! ps -p ${DDPID2} > /dev/null 2>&1; then break; fi + count=$((count+1)) + if [ $count -gt 60 ]; then + quota_error a $TSTUSR "dd should be finished!" + fi + sleep 1 done echo "(dd_pid=$DDPID2, time=$count)successful" - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) + cleanup_quota_test resetquota -u $TSTUSR resetquota -g $TSTUSR - - return $RC } -run_test_with_stat 21 "run for fixing bug16053 ===========" +run_test 21 "Setquota while writing & deleting (b16053)" +# enable/disable quota enforcement permanently test_22() { - quota_save_version "ug3" + echo "Set both mdt & ost quota type as ug" + set_mdt_qtype "ug" || error "enable mdt quota failed" + set_ost_qtype "ug" || error "enable ost quota failed" + + echo "Restart..." + stopall + mount + setupall - stopall - mount - setupall + echo "Verify if quota is enabled" + local qtype=$(mdt_quota_type) + [ $qtype != "ug" ] && error "mdt quota setting is lost" + qtype=$(ost_quota_type) + [ $qtype != "ug" ] && error "ost quota setting is lost" - echo "checking parameters" + echo "Set both mdt & ost quota type as none" + set_mdt_qtype "none" || error "disable mdt quota failed" + set_ost_qtype "none" || error "disable ost quota failed" - do_facet $SINGLEMDS "lctl get_param mdd.${FSNAME}-MDT*.quota_type" | grep "ug3" || error "admin failure" - do_facet ost1 "lctl get_param obdfilter.*.quota_type" | grep "ug3" || error "op failure" + echo "Restart..." + stopall + mount + setupall - quota_init + echo "Verify if quota is disabled" + qtype=$(mdt_quota_type) + [ $qtype != "none" ] && error "mdt quota setting is lost" + qtype=$(ost_quota_type) + [ $qtype != "none" ] && error "ost quota setting is lost" + + quota_init } -run_test_with_stat 22 "test if quota_type saved as permanent parameter ====" +run_test 22 "enable/disable quota by 'lctl conf_param'" test_23_sub() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - TESTFILE="$DIR/$tdir/$tfile-0" - rm -f $TESTFILE - local bs_unit=$((1024*1024)) - LIMIT=$1 + local TESTFILE="$DIR/$tdir/$tfile" + local LIMIT=$1 - wait_delete_completed + setup_quota_test + trap cleanup_quota_test EXIT + + set_ost_qtype "ug" || error "Enable ost quota failed" # test for user - log " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - sleep 3 - quota_show_check b u $TSTUSR + log "User quota (limit: $LIMIT MB)" + $LFS setquota -u $TSTUSR -b 0 -B "$LIMIT"M -i 0 -I 0 $DIR + quota_show_check b u $TSTUSR $LFS setstripe $TESTFILE -c 1 -i 0 chown $TSTUSR.$TSTUSR $TESTFILE - log " Step1: trigger quota with 0_DIRECT" - log " Write half of file" - $RUNAS $DIRECTIO write $TESTFILE 0 $(($LIMIT/1024/2)) $bs_unit || \ - quota_error u $TSTUSR "(1) write failure, but expect success: $LIMIT" - log " Write out of block quota ..." - $RUNAS $DIRECTIO write $TESTFILE $(($LIMIT/1024/2)) $(($LIMIT/1024/2)) $bs_unit && \ - quota_error u $TSTUSR "(2) write success, but expect EDQUOT: $LIMIT" - log " Step1: done" - - log " Step2: rewrite should succeed" - $RUNAS $DIRECTIO write $TESTFILE 0 1 $bs_unit || \ - quota_error u $TSTUSR "(3) write failure, but expect success: $LIMIT" - log " Step2: done" - - rm -f $TESTFILE - wait_delete_completed - OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` - OST0_QUOTA_USED=`getquota -u $TSTUSR $OST0_UUID curspace` - echo $OST0_QUOTA_USED - [ $OST0_QUOTA_USED -ne 0 ] && \ - ($SHOW_QUOTA_USER; quota_error u $TSTUSR "quota deleted isn't released") + log "Step1: trigger EDQUOT with O_DIRECT" + log "Write half of file" + $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) oflag=direct || + quota_error u $TSTUSR "(1) Write failure, expect success." \ + "limit=$LIMIT" + log "Write out of block quota ..." + $RUNAS $DD of=$TESTFILE count=$((LIMIT/2 + 1)) seek=$((LIMIT/2)) \ + oflag=direct conv=notrunc && + quota_error u $TSTUSR "(2) Write success, expect EDQUOT." \ + "limit=$LIMIT" + log "Step1: done" + + log "Step2: rewrite should succeed" + $RUNAS $DD of=$TESTFILE count=1 oflag=direct conv=notrunc|| + quota_error u $TSTUSR "(3) Write failure, expect success." \ + "limit=$LIMIT" + log "Step2: done" + + cleanup_quota_test + + local OST0_UUID=$(ostuuid_from_index 0) + local OST0_QUOTA_USED=$(getquota -u $TSTUSR $OST0_UUID curspace) + [ $OST0_QUOTA_USED -ne 0 ] && + ($SHOW_QUOTA_USER; \ + quota_error u $TSTUSR "quota isn't released") $SHOW_QUOTA_USER resetquota -u $TSTUSR } test_23() { - local slave_cnt=$((OSTCOUNT + 1)) # 1 mds, n osts - - OST0_MIN=$((6 * $slave_cnt * 1024)) # extra space for meta blocks. + local OST0_MIN=$((6 * 1024)) # 6MB, extra space for meta blocks. check_whether_skip && return 0 - log "run for $((4 * $slave_cnt))MB test file" - test_23_sub $((4 * $slave_cnt * 1024)) + log "run for 4MB test file" + test_23_sub 4 - OST0_MIN=$((60 * $slave_cnt * 1024)) # extra space for meta blocks. + OST0_MIN=$((60 * 1024)) # 60MB, extra space for meta blocks. check_whether_skip && return 0 - log "run for $((40 * $slave_cnt))MB test file" - test_23_sub $((40 * $slave_cnt * 1024)) + log "run for 40MB test file" + test_23_sub 40 } -run_test_with_stat 23 "run for fixing bug16125 ===========" +run_test 23 "Quota should be honored with directIO (b16125)" test_24() { + local blimit=5 # 5M local TESTFILE="$DIR/$tdir/$tfile" - mkdir -p $DIR/$tdir - - run_to_block_limit $TESTFILE - $SHOW_QUOTA_USER | grep '*' || error "no matching *" - - # cleanup - rm -f $TESTFILE - resetquota -u $TSTUSR - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) - -} -run_test_with_stat 24 "test if lfs draws an asterix when limit is reached (16646) ===========" - -show_quota() { - if [ $1 = "-u" ]; then - if [ $2 = "$TSTUSR" ]; then - $SHOW_QUOTA_USER - else - $SHOW_QUOTA_USER2 - fi - else - if [ $2 = "$TSTUSR" ]; then - $SHOW_QUOTA_GROUP - else - $SHOW_QUOTA_GROUP2 - fi - fi -} + setup_quota_test + trap cleanup_quota_test EXIT -test_25_sub() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - TESTFILE="$DIR/$tdir/$tfile-0" - rm -f $TESTFILE - LIMIT=$(( $BUNIT_SZ * ($OSTCOUNT + 1) + 4096 )) + set_ost_qtype "ug" || error "enable ost quota failed" - wait_delete_completed - - # set quota for $TSTUSR - log "setquota for $TSTUSR" - $LFS setquota $1 $TSTUSR -b $LIMIT -B $LIMIT -i 10 -I 10 $DIR - sleep 3 - if [ "$1" == "-u" ]; then - quota_show_check a u $TSTUSR - else - quota_show_check a g $TSTUSR - fi - - # set quota for $TSTUSR2 - log "setquota for $TSTUSR2" - $LFS setquota $1 $TSTUSR2 -b $LIMIT -B $LIMIT -i 10 -I 10 $DIR - sleep 3 - if [ "$1" == "-u" ]; then - quota_show_check a u $TSTUSR2 - else - quota_show_check a g $TSTUSR2 - fi - - # set stripe index to 0 - log "setstripe for $DIR/$tdir to 0" - $LFS setstripe $DIR/$tdir -c 1 -i 0 - MDS_UUID=`do_facet $SINGLEMDS $LCTL dl | grep -m1 " mdt " | awk '{print $((NF-1))}'` - OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` - MDS_QUOTA_USED_OLD=`getquota $1 $TSTUSR $MDS_UUID curinodes` - OST0_QUOTA_USED_OLD=`getquota $1 $TSTUSR $OST0_UUID curspace` - MDS_QUOTA_USED2_OLD=`getquota $1 $TSTUSR2 $MDS_UUID curinodes` - OST0_QUOTA_USED2_OLD=`getquota $1 $TSTUSR2 $OST0_UUID curspace` - - # TSTUSR write 4M - log "$TSTUSR write 4M to $TESTFILE" - $RUNAS dd if=/dev/zero of=$TESTFILE bs=4K count=1K || quota_error a $TSTUSR "dd failed" - sync - show_quota $1 $TSTUSR - show_quota $1 $TSTUSR2 - MDS_QUOTA_USED_NEW=`getquota $1 $TSTUSR $MDS_UUID curinodes` - [ $MDS_QUOTA_USED_NEW -ne $((MDS_QUOTA_USED_OLD + 1)) ] && \ - quota_error a $TSTUSR "$TSTUSR inode quota usage error: [$MDS_QUOTA_USED_OLD|$MDS_QUOTA_USED_NEW]" - OST0_QUOTA_USED_NEW=`getquota $1 $TSTUSR $OST0_UUID curspace` - OST0_QUOTA_USED_DELTA=$((OST0_QUOTA_USED_NEW - OST0_QUOTA_USED_OLD)) - [ $OST0_QUOTA_USED_DELTA -lt 4096 ] && \ - quota_error a $TSTUSR "$TSTUSR block quota usage error: [$OST0_QUOTA_USED_OLD|$OST0_QUOTA_USED_NEW]" - - # chown/chgrp from $TSTUSR to $TSTUSR2 - if [ $1 = "-u" ]; then - log "chown from $TSTUSR to $TSTUSR2" - chown $TSTUSR2 $TESTFILE || quota_error u $TSTUSR2 "chown failed" - else - log "chgrp from $TSTUSR to $TSTUSR2" - chgrp $TSTUSR2 $TESTFILE || quota_error g $TSTUSR2 "chgrp failed" - fi - sync - show_quota $1 $TSTUSR - show_quota $1 $TSTUSR2 - MDS_QUOTA_USED2_NEW=`getquota $1 $TSTUSR2 $MDS_UUID curinodes` - [ $MDS_QUOTA_USED2_NEW -ne $((MDS_QUOTA_USED2_OLD + 1)) ] && \ - quota_error a $TSTUSR2 "$TSTUSR2 inode quota usage transfer from $TSTUSR to $TSTUSR2 failed: [$MDS_QUOTA_USED2_OLD|$MDS_QUOTA_USED2_NEW]" - OST0_QUOTA_USED2_NEW=`getquota $1 $TSTUSR2 $OST0_UUID curspace` - # when chown, the quota on ost could be displayed out of quota temporarily. Delete the '*' in this situation. b=20433 - OST0_QUOTA_USED2_NEW=${OST0_QUOTA_USED2_NEW%\*} - OST0_QUOTA_USED2_DELTA=$((OST0_QUOTA_USED2_NEW - OST0_QUOTA_USED2_OLD)) - [ $OST0_QUOTA_USED2_DELTA -ne $OST0_QUOTA_USED_DELTA ] && \ - quota_error a $TSTUSR2 "$TSTUSR2 block quota usage transfer from $TSTUSR to $TSTUSR2 failed: [$OST0_QUOTA_USED2_OLD|$OST0_QUOTA_USED2_NEW]" - MDS_QUOTA_USED_NEW=`getquota $1 $TSTUSR $MDS_UUID curinodes` - [ $MDS_QUOTA_USED_NEW -ne $MDS_QUOTA_USED_OLD ] && \ - quota_error a $TSTUSR "$TSTUSR inode quota usage transfer from $TSTUSR to $TSTUSR2 failed: [$MDS_QUOTA_USED_OLD|$MDS_QUOTA_USED_NEW]" - OST0_QUOTA_USED_NEW=`getquota $1 $TSTUSR $OST0_UUID curspace` - [ $OST0_QUOTA_USED_NEW -ne $OST0_QUOTA_USED_OLD ] && \ - quota_error a $TSTUSR "$TSTUSR block quota usage transfer from $TSTUSR to $TSTUSR2 failed: [$OST0_QUOTA_USED_OLD|$OST0_QUOTA_USED_NEW]" - - rm -f $TESTFILE - wait_delete_completed - resetquota $1 $TSTUSR - resetquota $1 $TSTUSR2 -} - -test_25() { - log "run for chown case" - test_25_sub -u - - log "run for chgrp case" - test_25_sub -g -} -run_test_with_stat 25 "test whether quota usage is transfered when chown/chgrp (18081) ===========" - -test_26() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - TESTFILE="$DIR/$tdir/$tfile-0" - TESTFILE2="$DIR/$tdir/$tfile-1" - set_blk_tunesz 512 - set_blk_unitsz 1024 - - wait_delete_completed - - # every quota slave gets 20MB - b_limit=$(((OSTCOUNT + 1) * 20 * 1024)) - log "limit: ${b_limit}KB" - $LFS setquota -u $TSTUSR -b 0 -B $b_limit -i 0 -I 0 $DIR - sleep 3 - quota_show_check b u $TSTUSR - - $LFS setstripe $TESTFILE -c 1 -i 0 - $LFS setstripe $TESTFILE2 -c 1 -i 0 + # bind file to a single OST + $LFS setstripe -c 1 $TESTFILE chown $TSTUSR.$TSTUSR $TESTFILE - chown $TSTUSR.$TSTUSR $TESTFILE2 - - #define OBD_FAIL_QUOTA_DELAY_REL 0xA03 - lustre_fail ost 0xA03 - - log " Write the first file..." - $RUNAS $DIRECTIO write $TESTFILE 0 10 $((BLK_SZ * 1024)) || quota_error u $TSTUSR "write failure, but expect success" - log " Delete the first file..." - rm -f $TESTFILE + echo "Set user quota (limit: "$blimit"M)" + $LFS setquota -u $TSTUSR -b 0 -B "$blimit"M -i 0 -I 0 $MOUNT - wait_delete_completed + # overrun quota by root user + runas -u 0 -g 0 $DD of=$TESTFILE count=$((blimit + 1)) || + error "write failure, expect success" + cancel_lru_locks osc + sync_all_data - log " Write the second file..." - $RUNAS $DIRECTIO write $TESTFILE2 0 10 $((BLK_SZ * 1024)) || quota_error u $TSTUSR "write failure, but expect success" - log " Delete the second file..." - rm -f $TESTFILE2 + $SHOW_QUOTA_USER | grep '*' || error "no matching *" - lustre_fail ost 0 - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) + cleanup_quota_test resetquota -u $TSTUSR } -run_test_with_stat 26 "test for false quota error(bz18491) ======================================" +run_test 24 "lfs draws an asterix when limit is reached (b16646)" -test_27a() { - $LFS quota $TSTUSR $DIR && error "lfs succeeded with no type, but should have failed" - $LFS setquota $TSTUSR $DIR && error "lfs succeeded with no type, but should have failed" - return 0 -} -run_test_with_stat 27a "lfs quota/setquota should handle wrong arguments (19612) =================" - -test_27b() { - $LFS setquota -u $TSTID -b 1000 -B 1000 -i 1000 -I 1000 $DIR || \ - error "lfs setquota failed with uid argument" - $LFS setquota -g $TSTID -b 1000 -B 1000 -i 1000 -I 1000 $DIR || \ - error "lfs stequota failed with gid argument" - $SHOW_QUOTA_USERID || error "lfs quota failed with uid argument" - $SHOW_QUOTA_GROUPID || error "lfs quota failed with gid argument" - resetquota -u $TSTUSR - resetquota -g $TSTUSR - return 0 +test_27a() { # b19612 + $LFS quota $TSTUSR $DIR && + error "lfs succeeded with no type, but should have failed" + $LFS setquota $TSTUSR $DIR && + error "lfs succeeded with no type, but should have failed" + return 0 } -run_test 27b "lfs quota/setquota should handle user/group ID (20200) =================" - -test_28() { - BLK_LIMIT=$((100 * 1024 * 1024)) # 100G - echo "Step 1: set enough high limit for user [$TSTUSR:$BLK_LIMIT]" - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR - $SHOW_QUOTA_USER - - echo "Step 2: reset system ..." - cleanup_and_setup_lustre - quota_init - - echo "Step 3: change qunit for user [$TSTUSR:512:1024]" - set_blk_tunesz 512 - set_blk_unitsz 1024 - - wait_delete_completed - - #define OBD_FAIL_QUOTA_RET_QDATA | OBD_FAIL_ONCE - lustre_fail ost 0x80000A02 - - TESTFILE="$DIR/$tdir/$tfile" - mkdir -p $DIR/$tdir - - BLK_LIMIT=$((100 * 1024)) # 100M - echo "Step 4: set enough high limit for user [$TSTUSR:$BLK_LIMIT]" - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR - $SHOW_QUOTA_USER - - touch $TESTFILE - chown $TSTUSR.$TSTUSR $TESTFILE - - echo "Step 5: write the test file1 [10M] ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(( 10 * 1024 )) \ - || quota_error a $TSTUSR "write 10M file failure" - $SHOW_QUOTA_USER +run_test 27a "lfs quota/setquota should handle wrong arguments (b19612)" - rm -f $TESTFILE - sync; sleep 3; sync; - - # make qd_count 64 bit - lustre_fail ost 0 - - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) - - resetquota -u $TSTUSR +test_27b() { # b20200 + $LFS setquota -u $TSTID -b 1000 -B 1000 -i 1000 -I 1000 $DIR || + error "lfs setquota failed with uid argument" + $LFS setquota -g $TSTID -b 1000 -B 1000 -i 1000 -I 1000 $DIR || + error "lfs stequota failed with gid argument" + $SHOW_QUOTA_USERID || error "lfs quota failed with uid argument" + $SHOW_QUOTA_GROUPID || error "lfs quota failed with gid argument" + resetquota -u $TSTUSR + resetquota -g $TSTUSR + return 0 } -run_test_with_stat 28 "test for consistency for qunit when setquota (18574) ===========" - -test_29() -{ - local BLK_LIMIT=$((100 * 1024 * 1024)) # 100G - local timeout - local at_max_val - local pid - local waited - local deadline - - at_max_val=10 +run_test 27b "lfs quota/setquota should handle user/group ID (b20200)" - if at_is_enabled; then - timeout=$(at_max_get client) - [ $(at_min_get client) -gt $at_max_val ] && - at_max_val=$(at_min_get client) - at_max_set $at_max_val client - else - timeout=$(lctl get_param -n timeout) - lctl set_param timeout=$at_max_val - fi - # actually send a RPC to make service at_current confined within at_max - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR || - error "should succeed" - - #define OBD_FAIL_MDS_QUOTACTL_NET 0x12e - lustre_fail mds 0x12e - - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR & pid=$! - - deadline=$((2 * (($at_max_val << 1) + ($at_max_val >> 2) + 5))) - echo "wait at most 2 * ($at_max_val * 2.25 + 5) = $deadline seconds," \ - "it is server process time add the network latency." +test_30() { + local output + local LIMIT=4 # 4MB + local TESTFILE="$DIR/$tdir/$tfile" + local GRACE=10 - waited=0 - while [ $waited -lt $deadline ]; do - echo -n "." - sleep 1 - waited=$(($waited + 1)) - ps -p $pid > /dev/null || break - done - echo "waited $waited seconds" - ps -p $pid && error "lfs hadn't finished by $deadline seconds" - wait $pid && error "succeeded, but should have failed" + setup_quota_test + trap cleanup_quota_test EXIT - lustre_fail mds 0 + set_ost_qtype "u" || error "enable ost quota failed" - if at_is_enabled; then - at_max_set $timeout client - else - lctl set_param timeout=$timeout - fi + $LFS setstripe $TESTFILE -i 0 -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + $LFS setquota -t -u --block-grace $GRACE --inode-grace $MAX_IQ_TIME $DIR + $LFS setquota -u $TSTUSR -b ${LIMIT}M -B 0 -i 0 -I 0 $DIR + $RUNAS $DD of=$TESTFILE count=$((LIMIT * 2)) || true + cancel_lru_locks osc + sleep $GRACE + $LFS setquota -u $TSTUSR -B 0 $DIR + # over-quota flag has not yet settled since we do not trigger async + # events based on grace time period expiration + $SHOW_QUOTA_USER + $RUNAS $DD of=$TESTFILE conv=notrunc oflag=append count=1 || true + cancel_lru_locks osc + # now over-quota flag should be settled and further writes should fail + $SHOW_QUOTA_USER + $RUNAS $DD of=$TESTFILE conv=notrunc oflag=append count=1 && + error "grace times were reset" + # cleanup + cleanup_quota_test resetquota -u $TSTUSR + $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR } -run_test_with_stat 29 "unhandled quotactls must not hang lustre client (19778) ========" - -test_30() -{ - local output - local LIMIT=1024 - local TESTFILE="$DIR/$tdir/$tfile" - local GRACE=10 - - set_blk_tunesz 512 - set_blk_unitsz 1024 - - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - - $LFS setstripe $TESTFILE -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE - - $LFS setquota -t -u --block-grace $GRACE --inode-grace $MAX_IQ_TIME $DIR - $LFS setquota -u $TSTUSR -b $LIMIT -B 0 -i 0 -I 0 $DIR - $RUNAS dd if=/dev/zero of=$TESTFILE bs=1024 count=$((LIMIT * 2)) || true - cancel_lru_locks osc - sleep $GRACE - $LFS setquota -u $TSTUSR -B 0 $DIR - # over-quota flag has not yet settled since we do not trigger async events - # based on grace time period expiration - $SHOW_QUOTA_USER - $RUNAS dd if=/dev/zero of=$TESTFILE conv=notrunc oflag=append bs=1048576 count=1 || true - cancel_lru_locks osc - # now over-quota flag should be settled and further writes should fail - $SHOW_QUOTA_USER - $RUNAS dd if=/dev/zero of=$TESTFILE conv=notrunc oflag=append bs=1048576 count=1 && error "grace times were reset" - rm -f $TESTFILE - resetquota -u $TSTUSR - $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR - - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) -} -run_test_with_stat 30 "hard limit updates should not reset grace times ================" - -# test duplicate quota releases b=18630 -test_31() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - - LIMIT=$(( $BUNIT_SZ * $(($OSTCOUNT + 1)) * 10)) # 10 bunits each sever - TESTFILE="$DIR/$tdir/$tfile-0" - TESTFILE2="$DIR/$tdir/$tfile-1" - - wait_delete_completed - - log " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - - $LFS setstripe $TESTFILE -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE - $LFS setstripe $TESTFILE2 -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE2 - - log " step1: write out of block quota ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=5120 - $RUNAS dd if=/dev/zero of=$TESTFILE2 bs=$BLK_SZ count=5120 - - #define OBD_FAIL_QUOTA_DELAY_SD 0xA04 - #define OBD_FAIL_SOME 0x10000000 /* fail N times */ - lustre_fail ost $((0x00000A04 | 0x10000000)) 1 - - log " step2: delete two files so that triggering duplicate quota release ..." - rm -f $TESTFILE $TESTFILE2 - sync; sleep 5; sync # OBD_FAIL_QUOTA_DELAY_SD will delay for 5 seconds - wait_delete_completed - - log " step3: verify if the ost failed" - do_facet ost1 dmesg > $TMP/lustre-log-${TESTNAME}.log - watchdog=`awk '/test 31/ {start = 1;} - /release quota error/ { - if (start) { - print; - } - }' $TMP/lustre-log-${TESTNAME}.log` - [ "$watchdog" ] && error "$watchdog" - rm -f $TMP/lustre-log-${TESTNAME}.log - - lustre_fail ost 0 - resetquota -u $TSTUSR -} -run_test_with_stat 31 "test duplicate quota releases ===" - -# check hash_cur_bits -check_quota_hash_cur_bits() { - local bits=$1 - - # check quota_hash_cur_bits on all obdfilters - for num in `seq $OSTCOUNT`; do - cb=`do_facet ost$num "cat /sys/module/lquota/parameters/hash_lqs_cur_bits"` - if [ $cb -gt $bits ]; then - echo "hash_lqs_cur_bits of ost$num is too large(cur_bits=$cb)" - return 1; - fi - done - # check quota_hash_cur_bits on mds - cb=`do_facet $SINGLEMDS "cat /sys/module/lquota/parameters/hash_lqs_cur_bits"` - if [ $cb -gt $bits ]; then - echo "hash_lqs_cur_bits of mds is too large(cur_bits=$cb)" - return 1; - fi - return 0; -} - -# check lqs hash -check_lqs_hash() { - # check distribution of all obdfilters - for num in `seq $OSTCOUNT`; do - do_facet ost$num "lctl get_param obdfilter.${FSNAME}-OST*.hash_stats | grep LQS_HASH" | while read line; do - rehash_count=`echo $line | awk '{print $9}'` - if [ $rehash_count -eq 0 ]; then - echo -e "ost$num:\n $line" - error "Rehearsh didn't happen" - fi - done - done - # check distribution of mds - do_facet $SINGLEMDS "lctl get_param mdt.${FSNAME}-MDT*.hash_stats | grep LQS_HASH" | while read line; do - rehash_count=`echo $line | awk '{print $9}'` - if [ $rehash_count -eq 0 ]; then - echo -e "mdt:\n $line" - error "Rehearsh didn't happen" - fi - done -} - -test_32() -{ - # reset system so that quota_hash_cur_bits==3 - echo "Reset system ..." - local LMR_orig=$LOAD_MODULES_REMOTE - LOAD_MODULES_REMOTE=true - cleanup_and_setup_lustre - LOAD_MODULES_REMOTE=$LMR_orig - - client_up - wait_mds_ost_sync - quota_init - - for user in $SANITY_QUOTA_USERS; do - check_runas_id_ret $user quota_usr "runas -u $user -g quota_usr" >/dev/null 2>/dev/null || \ - missing_users="$missing_users $user" - done - [ -n "$missing_users" ] && { skip_env "the following users are missing: $missing_users" ; return 0 ; } - check_quota_hash_cur_bits 3 || { skip_env "hash_lqs_cur_bits isn't set properly"; return 0;} - - $LFS quotaoff -ug $DIR - $LFS quotacheck -ug $DIR - - for user in $SANITY_QUOTA_USERS; do - $LFS setquota -u $user --block-hardlimit 1048576 $DIR - done - - check_lqs_hash - - for user in $SANITY_QUOTA_USERS; do - resetquota -u $user - done -} -run_test 32 "check lqs hash(bug 21846) ==========================================" - -cleanup_quota_test() { - trap 0 - echo "Delete files..." - rm -rf $DIR/$tdir -} +run_test 30 "Hard limit updates should not reset grace times" # basic usage tracking for user & group test_33() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - INODES=10 - BLK_CNT=1024 - TOTAL_BLKS=$(($INODES * $BLK_CNT)) - - trap cleanup_quota_test EXIT - - # make sure the system is clean - USED=`getquota -u $TSTID global curspace` - [ $USED -ne 0 ] && \ - error "Used space ($USED) for user $TSTID isn't 0." - USED=`getquota -g $TSTID global curspace` - [ $USED -ne 0 ] && \ - error "Used space ($USED) for group $TSTID isn't 0." - + local INODES=10 # 10 files + local BLK_CNT=2 # of 2M each + local TOTAL_BLKS=$((INODES * BLK_CNT * 1024)) + + setup_quota_test + trap cleanup_quota_test EXIT + + # make sure the system is clean + local USED=$(getquota -u $TSTID global curspace) + [ $USED -ne 0 ] && + error "Used space ($USED) for user $TSTID isn't 0." + USED=$(getquota -g $TSTID global curspace) + [ $USED -ne 0 ] && + error "Used space ($USED) for group $TSTID isn't 0." + + echo "Write files..." for i in `seq 0 $INODES`; do - $RUNAS dd if=/dev/zero of=$DIR/$tdir/$tfile-$i conv=fsync \ - bs=$((BLK_SZ * BLK_CNT)) count=1 2>/dev/null || + $RUNAS $DD of=$DIR/$tdir/$tfile-$i count=$BLK_CNT 2>/dev/null || error "write failed" echo "Iteration $i/$INODES completed" done - sync; sync_all_data; - - echo "Verify disk usage after write" - USED=`getquota -u $TSTID global curspace` - [ $USED -lt $TOTAL_BLKS ] && \ - error "Used space for user $TSTID is $USED, expected $TOTAL_BLKS" - USED=`getquota -u $TSTID global curinodes` - [ $USED -lt $INODES ] && \ - error "Used inode for user $TSTID is $USED, expected $INODES" - - USED=`getquota -g $TSTID global curspace` - [ $USED -lt $TOTAL_BLKS ] && \ - error "Used space for group $TSTID is $USED, expected $TOTAL_BLKS" - USED=`getquota -g $TSTID global curinodes` - [ $USED -lt $INODES ] && \ - error "Used inode for group $TSTID is $USED, expected $INODES" - - cleanup_quota_test - - echo "Verify disk usage after delete" - wait_delete_completed - USED=`getquota -u $TSTID global curspace` - [ $USED -eq 0 ] || error "Used space for user $TSTID isn't 0. $USED" - USED=`getquota -u $TSTID global curinodes` - [ $USED -eq 0 ] || error "Used inodes for user $TSTID isn't 0. $USED" - USED=`getquota -g $TSTID global curspace` - [ $USED -eq 0 ] || error "Used space for group $TSTID isn't 0. $USED" - USED=`getquota -g $TSTID global curinodes` - [ $USED -eq 0 ] || error "Used inodes for group $TSTID isn't 0. $USED" -} -run_test 33 "basic usage tracking for user & group ==============================" + cancel_lru_locks osc + sync; sync_all_data + + echo "Verify disk usage after write" + USED=$(getquota -u $TSTID global curspace) + [ $USED -lt $TOTAL_BLKS ] && + error "Used space for user $TSTID:$USED, expected:$TOTAL_BLKS" + USED=$(getquota -g $TSTID global curspace) + [ $USED -lt $TOTAL_BLKS ] && + error "Used space for group $TSTID:$USED, expected:$TOTAL_BLKS" + + echo "Verify inode usage after write" + USED=$(getquota -u $TSTID global curinodes) + [ $USED -lt $INODES ] && + error "Used inode for user $TSTID is $USED, expected $INODES" + USED=$(getquota -g $TSTID global curinodes) + [ $USED -lt $INODES ] && + error "Used inode for group $TSTID is $USED, expected $INODES" + + cleanup_quota_test + + echo "Verify disk usage after delete" + USED=$(getquota -u $TSTID global curspace) + [ $USED -eq 0 ] || error "Used space for user $TSTID isn't 0. $USED" + USED=$(getquota -u $TSTID global curinodes) + [ $USED -eq 0 ] || error "Used inodes for user $TSTID isn't 0. $USED" + USED=$(getquota -g $TSTID global curspace) + [ $USED -eq 0 ] || error "Used space for group $TSTID isn't 0. $USED" + USED=$(getquota -g $TSTID global curinodes) + [ $USED -eq 0 ] || error "Used inodes for group $TSTID isn't 0. $USED" +} +run_test 33 "Basic usage tracking for user & group" # usage transfer test for user & group test_34() { - BLK_CNT=1024 - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir + local BLK_CNT=2 # 2MB + setup_quota_test trap cleanup_quota_test EXIT - # make sure the system is clean - USED=`getquota -u $TSTID global curspace` - [ $USED -ne 0 ] && error "Used space ($USED) for user $TSTID isn't 0." - USED=`getquota -g $TSTID global curspace` - [ $USED -ne 0 ] && error "Used space ($USED) for group $TSTID isn't 0." + # make sure the system is clean + local USED=$(getquota -u $TSTID global curspace) + [ $USED -ne 0 ] && error "Used space ($USED) for user $TSTID isn't 0." + USED=$(getquota -g $TSTID global curspace) + [ $USED -ne 0 ] && error "Used space ($USED) for group $TSTID isn't 0." echo "Write file..." - dd if=/dev/zero of=$DIR/$tdir/$tfile bs=$((BLK_SZ * BLK_CNT)) count=1 \ - conv=fsync 2>/dev/null || error "write failed" - sync; sync_all_data; + $DD of=$DIR/$tdir/$tfile count=$BLK_CNT 2>/dev/null || + error "write failed" + cancel_lru_locks osc + sync; sync_all_data - echo "chown the file to user $TSTID" - chown $TSTID $DIR/$tdir/$tfile || error "chown failed" + echo "chown the file to user $TSTID" + chown $TSTID $DIR/$tdir/$tfile || error "chown failed" echo "Wait for setattr on objects finished..." wait_delete_completed - echo "Verify disk usage for user $TSTID" - USED=`getquota -u $TSTID global curspace` - [ $USED -lt $BLK_CNT ] && \ - error "Used space for user $TSTID is $USED, expected $BLK_CNT" - USED=`getquota -u $TSTID global curinodes` - [ $USED -ne 1 ] && \ - error "Used inodes for user $TSTID is $USED, expected 1" + BLK_CNT=$((BLK_CNT * 1024)) + + echo "Verify disk usage for user $TSTID" + USED=$(getquota -u $TSTID global curspace) + [ $USED -lt $BLK_CNT ] && + error "Used space for user $TSTID is ${USED}, expected $BLK_CNT" + USED=$(getquota -u $TSTID global curinodes) + [ $USED -ne 1 ] && + error "Used inodes for user $TSTID is $USED, expected 1" - echo "chgrp the file to group $TSTID" - chgrp $TSTID $DIR/$tdir/$tfile || error "chgrp failed" + echo "chgrp the file to group $TSTID" + chgrp $TSTID $DIR/$tdir/$tfile || error "chgrp failed" echo "Wait for setattr on objects finished..." wait_delete_completed - echo "Verify disk usage for group $TSTID" - USED=`getquota -g $TSTID global curspace` - [ $USED -ge $BLK_CNT ] || \ - error "Used space for group $TSTID is $USED, expected $BLK_CNT" - USED=`getquota -g $TSTID global curinodes` - [ $USED -eq 1 ] || \ - error "Used inodes for group $TSTID is $USED, expected 1" + echo "Verify disk usage for group $TSTID" + USED=$(getquota -g $TSTID global curspace) + [ $USED -ge $BLK_CNT ] || + error "Used space for group $TSTID is $USED, expected $BLK_CNT" + USED=$(getquota -g $TSTID global curinodes) + [ $USED -eq 1 ] || + error "Used inodes for group $TSTID is $USED, expected 1" - cleanup_quota_test + cleanup_quota_test } -run_test 34 "usage transfer for user & group ====================================" +run_test 34 "Usage transfer for user & group" # usage is still accessible across restart test_35() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - BLK_CNT=1024 + local BLK_CNT=2 # 2 MB - trap cleanup_quota_test EXIT + setup_quota_test + trap cleanup_quota_test EXIT echo "Write file..." - $RUNAS dd if=/dev/zero of=$DIR/$tdir/$tfile bs=$((BLK_SZ * BLK_CNT)) \ - count=1 conv=fsync 2>/dev/null || error "write failed" - sync; sync_all_data; - - echo "Save disk usage before restart" - ORIG_USR_SPACE=`getquota -u $TSTID global curspace` - [ $ORIG_USR_SPACE -eq 0 ] && \ - error "Used space for user $TSTID is 0, expected $BLK_CNT" - ORIG_USR_INODES=`getquota -u $TSTID global curinodes` - [ $ORIG_USR_INODES -eq 0 ] && \ - error "Used inodes for user $TSTID is 0, expected 1" - ORIG_GRP_SPACE=`getquota -g $TSTID global curspace` - [ $ORIG_GRP_SPACE -eq 0 ] && \ - error "Used space for group $TSTID is 0, expected $BLK_CNT" - ORIG_GRP_INODES=`getquota -g $TSTID global curinodes` - [ $ORIG_GRP_INODES -eq 0 ] && \ - error "Used inodes for group $TSTID is 0, expected 1" - - log "Restart..." - local ORIG_REFORMAT=$REFORMAT - REFORMAT="" - cleanup_and_setup_lustre - REFORMAT=$ORIG_REFORMAT - quota_init - - echo "Verify disk usage after restart" - USED=`getquota -u $TSTID global curspace` - [ $USED -eq $ORIG_USR_SPACE ] || \ - error "Used space for user $TSTID changed from " \ - "$ORIG_USR_SPACE to $USED" - USED=`getquota -u $TSTID global curinodes` - [ $USED -eq $ORIG_USR_INODES ] || \ - error "Used inodes for user $TSTID changed from " \ - "$ORIG_USR_INODES to $USED" - USED=`getquota -g $TSTID global curspace` - [ $USED -eq $ORIG_GRP_SPACE ] || \ - error "Used space for group $TSTID changed from " \ - "$ORIG_GRP_SPACE to $USED" - USED=`getquota -g $TSTID global curinodes` - [ $USED -eq $ORIG_GRP_INODES ] || \ - error "Used inodes for group $TSTID changed from " \ - "$ORIG_GRP_INODES to $USED" + $RUNAS $DD of=$DIR/$tdir/$tfile count=$BLK_CNT 2>/dev/null || + error "write failed" + cancel_lru_locks osc + sync; sync_all_data + + echo "Save disk usage before restart" + local ORIG_USR_SPACE=$(getquota -u $TSTID global curspace) + [ $ORIG_USR_SPACE -eq 0 ] && + error "Used space for user $TSTID is 0, expected ${BLK_CNT}M" + local ORIG_USR_INODES=$(getquota -u $TSTID global curinodes) + [ $ORIG_USR_INODES -eq 0 ] && + error "Used inodes for user $TSTID is 0, expected 1" + echo "User $TSTID: ${ORIG_USR_SPACE}KB $ORIG_USR_INODES inodes" + local ORIG_GRP_SPACE=$(getquota -g $TSTID global curspace) + [ $ORIG_GRP_SPACE -eq 0 ] && + error "Used space for group $TSTID is 0, expected ${BLK_CNT}M" + local ORIG_GRP_INODES=$(getquota -g $TSTID global curinodes) + [ $ORIG_GRP_INODES -eq 0 ] && + error "Used inodes for group $TSTID is 0, expected 1" + echo "Group $TSTID: ${ORIG_GRP_SPACE}KB $ORIG_GRP_INODES inodes" + + log "Restart..." + local ORIG_REFORMAT=$REFORMAT + REFORMAT="" + cleanup_and_setup_lustre + REFORMAT=$ORIG_REFORMAT + quota_init + + echo "Verify disk usage after restart" + local USED=$(getquota -u $TSTID global curspace) + [ $USED -eq $ORIG_USR_SPACE ] || + error "Used space for user $TSTID changed from " \ + "$ORIG_USR_SPACE to $USED" + USED=$(getquota -u $TSTID global curinodes) + [ $USED -eq $ORIG_USR_INODES ] || + error "Used inodes for user $TSTID changed from " \ + "$ORIG_USR_INODES to $USED" + USED=$(getquota -g $TSTID global curspace) + [ $USED -eq $ORIG_GRP_SPACE ] || + error "Used space for group $TSTID changed from " \ + "$ORIG_GRP_SPACE to $USED" + USED=$(getquota -g $TSTID global curinodes) + [ $USED -eq $ORIG_GRP_INODES ] || + error "Used inodes for group $TSTID changed from " \ + "$ORIG_GRP_INODES to $USED" # check if the vfs_dq_init() is called before writing echo "Append to the same file..." - $RUNAS dd if=/dev/zero of=$DIR/$tdir/$tfile bs=$((BLK_SZ * BLK_CNT)) \ - count=1 seek=1 conv=fsync 2>/dev/null || error "write failed" - sync; sync_all_data; + $RUNAS $DD of=$DIR/$tdir/$tfile count=$BLK_CNT seek=1 2>/dev/null || + error "write failed" + cancel_lru_locks osc + sync; sync_all_data echo "Verify space usage is increased" - USED=`getquota -u $TSTID global curspace` + USED=$(getquota -u $TSTID global curspace) [ $USED -gt $ORIG_USR_SPACE ] || error "Used space for user $TSTID isn't increased" \ "orig:$ORIG_USR_SPACE, now:$USED" - USED=`getquota -g $TSTID global curspace` + USED=$(getquota -g $TSTID global curspace) [ $USED -gt $ORIG_GRP_SPACE ] || error "Used space for group $TSTID isn't increased" \ "orig:$ORIG_GRP_SPACE, now:$USED" - cleanup_quota_test -} -run_test 35 "usage is still accessible across reboot ============================" + cleanup_quota_test +} +run_test 35 "Usage is still accessible across reboot" + +# test migrating old amdin quota files (in Linux quota file format v2) into new +# quota global index (in IAM format) +test_36() { + [ $(facet_fstype $SINGLEMDS) != ldiskfs ] && \ + skip "skipping migration test" && return + + # get the mdt0 device name + local mdt0_node=$(facet_active_host $SINGLEMDS) + local mdt0_dev=$(mdsdevname ${SINGLEMDS//mds/}) + + echo "Reformat..." + formatall + + echo "Copy admin quota files into MDT0..." + local mntpt=$(facet_mntpt $SINGLEMDS) + local mdt0_fstype=$(facet_fstype $SINGLEMDS) + echo "$mdt0_node, $mdt0_dev, $mntpt" + do_node $mdt0_node mount -t $mdt0_fstype $MDS_MOUNT_OPTS $mdt0_dev $mntpt + do_node $mdt0_node mkdir $mntpt/OBJECTS + do_node $mdt0_node cp $LUSTRE/tests/admin_quotafile_v2.usr $mntpt/OBJECTS + do_node $mdt0_node cp $LUSTRE/tests/admin_quotafile_v2.grp $mntpt/OBJECTS + do_node $mdt0_node umount -f $mntpt + + echo "Setup all..." + setupall + + echo "Verify global limits..." + local id_cnt + local limit + + local proc="qmt.*.md-0x0.glb-usr" + id_cnt=$(do_node $mdt0_node $LCTL get_param -n $proc | wc -l) + [ $id_cnt -eq 201 ] || error "Migrate inode user limit failed" + limit=$(getquota -u 1 global isoftlimit) + [ $limit -eq 1024 ] || error "User inode softlimit: $limit" + limit=$(getquota -u 1 global ihardlimit) + [ $limit -eq 2048 ] || error "User inode hardlimit: $limit" + + proc="qmt.*.md-0x0.glb-grp" + id_cnt=$(do_node $mdt0_node $LCTL get_param -n $proc | wc -l) + [ $id_cnt -eq 201 ] || error "Migrate inode group limit failed" + limit=$(getquota -g 1 global isoftlimit) + [ $limit -eq 1024 ] || error "Group inode softlimit: $limit" + limit=$(getquota -g 1 global ihardlimit) + [ $limit -eq 2048 ] || error "Group inode hardlimit: $limit" + + proc=" qmt.*.dt-0x0.glb-usr" + id_cnt=$(do_node $mdt0_node $LCTL get_param -n $proc | wc -l) + [ $id_cnt -eq 201 ] || error "Migrate block user limit failed" + limit=$(getquota -u 60001 global bsoftlimit) + [ $limit -eq 10485760 ] || error "User block softlimit: $limit" + limit=$(getquota -u 60001 global bhardlimit) + [ $limit -eq 20971520 ] || error "User block hardlimit: $limit" + + proc="qmt.*.dt-0x0.glb-grp" + id_cnt=$(do_node $mdt0_node $LCTL get_param -n $proc | wc -l) + [ $id_cnt -eq 201 ] || error "Migrate block user limit failed" + limit=$(getquota -g 60001 global bsoftlimit) + [ $limit -eq 10485760 ] || error "Group block softlimit: $limit" + limit=$(getquota -g 60001 global bhardlimit) + [ $limit -eq 20971520 ] || error "Group block hardlimit: $limit" + + echo "Cleanup..." + formatall + setupall +} +run_test 36 "Migrate old admin files into new global indexes" -# turn off quota quota_fini() { - #$LFS quotaoff $DIR do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=-quota" } quota_fini cd $ORIG_PWD complete $(basename $0) $SECONDS +check_and_cleanup_lustre export USE_OFD="$USE_OFD_OLD" export LOAD_MODULES_REMOTE="$LOAD_MODULES_REMOTE_OLD" export QUOTA_AUTO=$QUOTA_AUTO_OLD -cleanup_and_setup_lustre exit_status diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 7cfe4fc..78e2db4 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -981,7 +981,7 @@ stop() { # save quota version (both administrative and operational quotas) # add an additional parameter if mountpoint is ever different from $MOUNT # -# XXX This function is kept for interoperability with old server (< 2.3.51), +# XXX This function is kept for interoperability with old server (< 2.3.50), # it should be removed whenever we drop the interoperability for such # server. quota_save_version() { @@ -1005,7 +1005,7 @@ quota_save_version() { # client could mount several lustre # -# XXX This function is kept for interoperability with old server (< 2.3.51), +# XXX This function is kept for interoperability with old server (< 2.3.50), # it should be removed whenever we drop the interoperability for such # server. quota_type () { @@ -1017,7 +1017,7 @@ quota_type () { return $rc } -# XXX This function is kept for interoperability with old server (< 2.3.51), +# XXX This function is kept for interoperability with old server (< 2.3.50), # it should be removed whenever we drop the interoperability for such # server. restore_quota_old() { @@ -1030,7 +1030,7 @@ restore_quota_old() { quota_save_version $old_QUOTA_TYPE } -# XXX This function is kept for interoperability with old server (< 2.3.51), +# XXX This function is kept for interoperability with old server (< 2.3.50), # it should be removed whenever we drop the interoperability for such # server. setup_quota_old(){ @@ -1098,7 +1098,7 @@ ost_quota_type() { # restore old quota type settings restore_quota() { - if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.51) ]; then + if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then restore_quota_old return fi @@ -1114,11 +1114,15 @@ restore_quota() { } setup_quota(){ - if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.51) ]; then + if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then setup_quota_old $1 return fi + # XXX remove it once all quota code landed + echo "skip quota setup" + return + local mntpt=$1 # save old quota type & set new quota type -- 1.8.3.1