X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Fsanity-quota.sh;h=09806a504fcc28cd8bb71ebcc83f9edfc6dfa9d0;hp=3ef3ca71bf1337d8e14d5d595398e50d17944d6d;hb=f424516984cd4e1aab0dc2070b971744c9ff811b;hpb=2650455e68fea4442b43f39acc7e56fdc934867e diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index 3ef3ca7..09806a5 100644 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -6,28 +6,14 @@ # Run test by setting NOSETUP=true when ltest has setup env for us set -e -#kernel 2.4.x doesn't support quota -K_VER=`uname --kernel-release | cut -b 1-3` -if [ $K_VER = "2.4" ]; then - echo "Kernel 2.4 doesn't support quota" - exit 0 -fi - SRCDIR=`dirname $0` export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin ONLY=${ONLY:-"$*"} -# test_11 has been used to protect a kernel bug(bz10912), now it isn't -# useful any more. Then add it to ALWAYS_EXCEPT. b=19835 -ALWAYS_EXCEPT="10 $SANITY_QUOTA_EXCEPT" +ALWAYS_EXCEPT="$SANITY_QUOTA_EXCEPT" # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! -case `uname -r` in -2.6*) FSTYPE=${FSTYPE:-ldiskfs};; -*) error "unsupported kernel" ;; -esac - -[ "$ALWAYS_EXCEPT$EXCEPT" ] && \ +[ "$ALWAYS_EXCEPT$EXCEPT" ] && echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT`" TMP=${TMP:-/tmp} @@ -38,24 +24,34 @@ TSTID2=${TSTID2:-60001} TSTUSR=${TSTUSR:-"quota_usr"} TSTUSR2=${TSTUSR2:-"quota_2usr"} BLK_SZ=1024 -BUNIT_SZ=${BUNIT_SZ:-1024} # min block quota unit(kB) -IUNIT_SZ=${IUNIT_SZ:-10} # min inode quota unit MAX_DQ_TIME=604800 MAX_IQ_TIME=604800 -TRACE=${TRACE:-""} -LUSTRE=${LUSTRE:-`dirname $0`/..} +LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} . $LUSTRE/tests/test-framework.sh init_test_env $@ . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} +init_logging DIRECTIO=${DIRECTIO:-$LUSTRE/tests/directio} [ $MDSCOUNT -gt 1 ] && skip "CMD case" && exit 0 -remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0 -remote_ost_nodsh && skip "remote OST with nodsh" && exit 0 +require_dsh_mds || exit 0 +require_dsh_ost || exit 0 + +# XXX Once we drop the interoperability with old server (< 2.3.50), the +# sanity-quota-old.sh should be removed. +if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then + exec $LUSTRE/tests/sanity-quota-old.sh +fi + +# if e2fsprogs support quota feature? +if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] && \ + ! $DEBUGFS -c -R supported_features | grep -q 'quota'; then + skip "e2fsprogs doesn't support quota" && exit 0 +fi -[ "$SLOW" = "no" ] && EXCEPT_SLOW="9 10 11 18b 21" +[ "$SLOW" = "no" ] && EXCEPT_SLOW="9 18 21" QUOTALOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} @@ -69,13 +65,6 @@ export QUOTA_AUTO=0 check_and_setup_lustre -if [ x"$(som_check)" = x"enabled" ]; then - echo "Som is enabled, Quota is temporary conflicts with it" - check_and_cleanup_lustre - export QUOTA_AUTO=$QUOTA_AUTO_OLD - exit 0 -fi - LOVNAME=`lctl get_param -n llite.*.lov.common_name | tail -n 1` OSTCOUNT=`lctl get_param -n lov.$LOVNAME.numobd` @@ -88,68 +77,23 @@ SHOW_QUOTA_GROUP2="$LFS quota -v -g $TSTUSR2 $DIR" SHOW_QUOTA_INFO_USER="$LFS quota -t -u $DIR" SHOW_QUOTA_INFO_GROUP="$LFS quota -t -g $DIR" -# control the time of tests -cycle=30 -[ "$SLOW" = "no" ] && cycle=10 - build_test_filter -# set_blk_tunables(btune_sz) -set_blk_tunesz() { - local btune=$(($1 * BLK_SZ)) - # set btune size on all obdfilters - do_facet ost1 "lctl set_param lquota.${FSNAME}-OST*.quota_btune_sz=$btune" - # set btune size on mds - do_facet $SINGLEMDS "lctl set_param lquota.mdd_obd-${FSNAME}-MDT*.quota_btune_sz=$btune" -} - -# set_blk_unitsz(bunit_sz) -set_blk_unitsz() { - local bunit=$(($1 * BLK_SZ)) - # set bunit size on all obdfilters - do_facet ost1 "lctl set_param lquota.${FSNAME}-OST*.quota_bunit_sz=$bunit" - # set bunit size on mds - do_facet $SINGLEMDS "lctl set_param lquota.mdd_obd-${FSNAME}-MDT*.quota_bunit_sz=$bunit" -} - -# set_file_tunesz(itune_sz) -set_file_tunesz() { - local itune=$1 - # set itune size on all obdfilters - do_facet ost1 "lctl set_param lquota.${FSNAME}-OST*.quota_itune_sz=$itune" - # set itune size on mds - do_facet $SINGLEMDS "lctl set_param lquota.mdd_obd-${FSNAME}-MDT*.quota_itune_sz=$itune" -} - -# set_file_unitsz(iunit_sz) -set_file_unitsz() { - local iunit=$1 - # set iunit size on all obdfilters - do_facet ost1 "lctl set_param lquota.${FSNAME}-OST*.quota_iunit_sz=$iunit" - # set iunit size on mds - do_facet $SINGLEMDS "lctl set_param lquota.mdd_obd-${FSNAME}-MDT*.quota_iunit_sz=$iunit" -} - lustre_fail() { local fail_node=$1 local fail_loc=$2 local fail_val=${3:-0} if [ $fail_node == "mds" ] || [ $fail_node == "mds_ost" ]; then - if [ $((fail_loc & 0x10000000)) -ne 0 -a $fail_val -gt 0 ] || \ - [ $((fail_loc)) -eq 0 ]; then do_facet $SINGLEMDS "lctl set_param fail_val=$fail_val" - fi - do_facet $SINGLEMDS "lctl set_param fail_loc=$fail_loc" + do_facet $SINGLEMDS "lctl set_param fail_loc=$fail_loc" fi + if [ $fail_node == "ost" ] || [ $fail_node == "mds_ost" ]; then - for num in `seq $OSTCOUNT`; do - if [ $((fail_loc & 0x10000000)) -ne 0 -a $fail_val -gt 0 ] || \ - [ $((fail_loc)) -eq 0 ]; then - do_facet ost$num "lctl set_param fail_val=$fail_val" - fi - do_facet ost$num "lctl set_param fail_loc=$fail_loc" - done + for num in `seq $OSTCOUNT`; do + do_facet ost$num "lctl set_param fail_val=$fail_val" + do_facet ost$num "lctl set_param fail_loc=$fail_loc" + done fi } @@ -157,440 +101,553 @@ RUNAS="runas -u $TSTID -g $TSTID" RUNAS2="runas -u $TSTID2 -g $TSTID2" FAIL_ON_ERROR=true check_runas_id $TSTID $TSTID $RUNAS FAIL_ON_ERROR=true check_runas_id $TSTID2 $TSTID2 $RUNAS2 +DD="dd if=/dev/zero bs=1M" FAIL_ON_ERROR=false -run_test_with_stat() { - (($# != 2)) && error "the number of arguments is wrong" - - do_facet $SINGLEMDS "lctl set_param lquota.mdd_obd-${FSNAME}-MDT*.stats=0" > /dev/null - for j in `seq $OSTCOUNT`; do - do_facet ost$j "lctl set_param lquota.${FSNAME}-OST*.stats=0" > /dev/null - done - run_test "$@" - if [ ${STAT:-"yes"} != "no" -a -z "$LAST_SKIPPED" ]; then - echo "statistics info begin ***************************************" - do_facet $SINGLEMDS "lctl get_param lquota.mdd_obd-${FSNAME}-MDT*.stats" - for j in `seq $OSTCOUNT`; do - do_facet ost$j "lctl get_param lquota.${FSNAME}-OST*.stats" - done - echo "statistics info end ***************************************" - fi -} - -# # clear quota limits for a user or a group # usage: resetquota -u username # resetquota -g groupname resetquota() { - [ "$#" != 2 ] && error "resetquota: wrong number of arguments: $#" - [ "$1" != "-u" -a "$1" != "-g" ] && error "resetquota: wrong specifier $1 passed" - - count=0 - if at_is_enabled; then - timeout=$(at_max_get mds) - else - timeout=$(lctl get_param -n timeout) - fi - - while [ $((count++)) -lt $timeout ]; do - $LFS setquota "$1" "$2" -b 0 -B 0 -i 0 -I 0 $MOUNT - RC=$? - if [ $RC -ne 0 ]; then - if [ $RC -eq 240 ]; then # 240 means -EBUSY - log "resetquota is blocked for quota master recovery, retry after 1 sec" - sleep 1 - continue - else - error "resetquota failed: $RC" - fi - fi - break - done - - [ $count -lt $timeout ] || error "resetquota timeout: $timeout" + [ "$#" != 2 ] && error "resetquota: wrong number of arguments: $#" + [ "$1" != "-u" -a "$1" != "-g" ] && + error "resetquota: wrong specifier $1 passed" + + $LFS setquota "$1" "$2" -b 0 -B 0 -i 0 -I 0 $MOUNT + # The usage should be 0 now, enforce quota to trigger quota + # release for each slave, sleep 3 seconds to wait for the release + # done, then we can make sure granted quota for each slave is 0. + $LFS setquota "$1" "$2" -b 0 -B 1 -i 0 -I 1 $MOUNT + sleep 3 + $LFS setquota "$1" "$2" -b 0 -B 0 -i 0 -I 0 $MOUNT } quota_scan() { - LOCAL_UG=$1 - LOCAL_ID=$2 - - if [ "$LOCAL_UG" == "a" -o "$LOCAL_UG" == "u" ]; then - log "Files for user ($LOCAL_ID):" - ($LFS find -user $LOCAL_ID $DIR | xargs stat 2>/dev/null) - fi - - if [ "$LOCAL_UG" == "a" -o "$LOCAL_UG" == "g" ]; then - log "Files for group ($LOCAL_ID):" - ($LFS find -group $LOCAL_ID $DIR | xargs stat 2>/dev/null) - fi + local LOCAL_UG=$1 + local LOCAL_ID=$2 + + if [ "$LOCAL_UG" == "a" -o "$LOCAL_UG" == "u" ]; then + $LFS quota -v -u $LOCAL_ID $DIR + log "Files for user ($LOCAL_ID):" + ($LFS find -user $LOCAL_ID $DIR | head -n 4 | + xargs stat 2>/dev/null) + fi + + if [ "$LOCAL_UG" == "a" -o "$LOCAL_UG" == "g" ]; then + $LFS quota -v -u $LOCAL_ID $DIR + log "Files for group ($LOCAL_ID):" + ($LFS find -group $LOCAL_ID $DIR | head -n 4 | + xargs stat 2>/dev/null) + fi } quota_error() { - quota_scan $1 $2 - shift 2 - error "$*" + quota_scan $1 $2 + shift 2 + error "$*" } quota_log() { - quota_scan $1 $2 - shift 2 - log "$*" + quota_scan $1 $2 + shift 2 + log "$*" } -quota_show_check() { - LOCAL_BF=$1 - LOCAL_UG=$2 - LOCAL_ID=$3 - PATTERN="`echo $DIR | sed 's/\//\\\\\//g'`" - - $LFS quota -v -$LOCAL_UG $LOCAL_ID $DIR - - if [ "$LOCAL_BF" == "a" -o "$LOCAL_BF" == "b" ]; then - USAGE="`$LFS quota -$LOCAL_UG $LOCAL_ID $DIR | awk '/^.*'$PATTERN'.*[[:digit:]+][[:space:]+]/ { print $2 }'`" - if [ -z $USAGE ]; then - quota_error $LOCAL_UG $LOCAL_ID "System is error when query quota for block ($LOCAL_UG:$LOCAL_ID)." - else - [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for block ($LOCAL_UG:$LOCAL_ID:$USAGE)." - fi - fi - - if [ "$LOCAL_BF" == "a" -o "$LOCAL_BF" == "f" ]; then - USAGE="`$LFS quota -$LOCAL_UG $LOCAL_ID $DIR | awk '/^.*'$PATTERN'.*[[:digit:]+][[:space:]+]/ { print $5 }'`" - if [ -z $USAGE ]; then - quota_error $LOCAL_UG $LOCAL_ID "System is error when query quota for file ($LOCAL_UG:$LOCAL_ID)." - else - [ $USAGE -ne 0 ] && quota_log $LOCAL_UG $LOCAL_ID "System is not clean for file ($LOCAL_UG:$LOCAL_ID:$USAGE)." - fi - fi +# get quota for a user or a group +# usage: getquota -u|-g | global| \ +# bhardlimit|bsoftlimit|bgrace|ihardlimit|isoftlimit|igrace +# +getquota() { + local spec + local uuid + + [ "$#" != 4 ] && error "getquota: wrong number of arguments: $#" + [ "$1" != "-u" -a "$1" != "-g" ] && + error "getquota: wrong u/g specifier $1 passed" + + uuid="$3" + + case "$4" in + curspace) spec=1;; + bsoftlimit) spec=2;; + bhardlimit) spec=3;; + bgrace) spec=4;; + curinodes) spec=5;; + isoftlimit) spec=6;; + ihardlimit) spec=7;; + igrace) spec=8;; + *) error "unknown quota parameter $4";; + esac + + [ "$uuid" = "global" ] && uuid=$DIR + + $LFS quota -v "$1" "$2" $DIR | + awk 'BEGIN { num='$spec' } { if ($1 == "'$uuid'") \ + { if (NF == 1) { getline } else { num++ } ; print $num;} }' \ + | tr -d "*" } -# set quota -quota_init() { - $LFS quotacheck -ug $DIR +# set mdt quota type +# usage: set_mdt_qtype ug|u|g|none +set_mdt_qtype() { + local qtype=$1 + local varsvc=${SINGLEMDS}_svc + do_facet mgs $LCTL conf_param $FSNAME.quota.mdt=$qtype + if $(facet_up $SINGLEMDS); then + wait_update_facet $SINGLEMDS "$LCTL get_param -n \ + osd-$FSTYPE.${!varsvc}.quota_slave.enabled" "$qtype" || + return 1 + fi + return 0 +} - resetquota -u $TSTUSR - resetquota -g $TSTUSR +# set ost quota type +# usage: set_ost_quota_type ug|u|g|none +set_ost_qtype() { + local qtype=$1 + local varsvc + local osts=$(get_facets OST) + do_facet mgs $LCTL conf_param $FSNAME.quota.ost=$qtype + # we have to make sure each OST received config changes + for ost in ${osts//,/ }; do + varsvc=${ost}_svc + if $(facet_up $ost); then + wait_update_facet $ost "$LCTL get_param -n \ + osd-$FSTYPE.${!varsvc}.quota_slave.enabled" \ + "$qtype" || return 1 + fi + done + return 0 +} - do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=+quota" +wait_reintegration() { + local ntype=$1 + local qtype=$2 + local result="glb[1],slv[1],reint[0]" + local varsvc + + if [ $ntype == "mdt" ]; then + varsvc=${SINGLEMDS}_svc + if $(facet_up $SINGLEMDS); then + wait_update_facet $SINGLEMDS "$LCTL get_param -n \ + osd-$FSTYPE.${!varsvc}.quota_slave.info | + grep "$qtype" | awk '{ print \\\$3 }'" "$result" || + return 1 + fi + else + local osts=$(get_facets OST) + for ost in ${osts//,/ }; do + varsvc=${ost}_svc + if $(facet_up $ost); then + wait_update_facet $ost "$LCTL get_param -n \ + osd-$FSTYPE.${!varsvc}.quota_slave.info | + grep "$qtype" | awk '{ print \\\$3 }'" \ + "$result" || return 1 + fi + done + fi + return 0 } -quota_init -# test for specific quota limitation, qunit, qtune $1=block_quota_limit -test_1_sub() { - LIMIT=$1 +wait_mdt_reint() { + local qtype=$1 + + if [ $qtype == "u" ] || [ $qtype == "ug" ]; then + wait_reintegration "mdt" "user" || return 1 + fi + + if [ $qtype == "g" ] || [ $qtype == "ug" ]; then + wait_reintegration "mdt" "group" || return 1 + fi + return 0 +} + +wait_ost_reint() { + local qtype=$1 + + if [ $qtype == "u" ] || [ $qtype == "ug" ]; then + wait_reintegration "ost" "user" || return 1 + fi + + if [ $qtype == "g" ] || [ $qtype == "ug" ]; then + wait_reintegration "ost" "group" || return 1 + fi + return 0 +} + +setup_quota_test() { + rm -rf $DIR/$tdir + wait_delete_completed + echo "Creating test directory" mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - TESTFILE="$DIR/$tdir/$tfile-0" + chmod 077 $DIR/$tdir + # always clear fail_loc in case of fail_loc isn't cleared + # properly when previous test failed + lustre_fail mds_ost 0 +} +cleanup_quota_test() { + trap 0 + echo "Delete files..." + rm -rf $DIR/$tdir + echo "Wait for unlink objects finished..." wait_delete_completed + sync_all_data || true +} - # test for user - log " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - sleep 3 - quota_show_check b u $TSTUSR +quota_show_check() { + local bf=$1 + local ug=$2 + local qid=$3 + local usage + + $LFS quota -v -$ug $qid $DIR + + if [ "$bf" == "a" -o "$bf" == "b" ]; then + usage=$(getquota -$ug $qid global curspace) + if [ -z $usage ]; then + quota_error $ug $qid \ + "Query block quota failed ($ug:$qid)." + else + [ $usage -ne 0 ] && quota_log $ug $qid \ + "Block quota isn't 0 ($ug:$qid:$usage)." + fi + fi + + if [ "$bf" == "a" -o "$bf" == "f" ]; then + usage=$(getquota -$ug $qid global curinodes) + if [ -z $usage ]; then + quota_error $ug $qid \ + "Query file quota failed ($ug:$qid)." + else + [ $usage -ne 0 ] && quota_log $ug $qid \ + "File quota isn't 0 ($ug:$qid:$usage)." + fi + fi +} + +# enable quota debug +quota_init() { + do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=+quota" +} +quota_init + +test_quota_performance() { + local TESTFILE="$DIR/$tdir/$tfile-0" + local size=$1 # in MB + local stime=$(date +%s) + $RUNAS $DD of=$TESTFILE count=$size conv=fsync || + quota_error u $TSTUSR "write failure" + local etime=$(date +%s) + delta=$((etime - stime)) + if [ $delta -gt 0 ]; then + rate=$((size * 1024 / delta)) + [ $rate -gt 1024 ] || + error "SLOW IO for $TSTUSR (user): $rate KB/sec" + fi + rm -f $TESTFILE +} + +# test basic quota performance b=21696 +test_0() { + setup_quota_test + trap cleanup_quota_test EXIT + + local MB=100 # 100M + [ "$SLOW" = "no" ] && MB=10 + + set_ost_qtype "none" || error "disable ost quota failed" + test_quota_performance $MB + + set_ost_qtype "ug" || error "enable ost quota failed" + $LFS setquota -u $TSTUSR -b 0 -B 10G -i 0 -I 0 $DIR + test_quota_performance $MB + + cleanup_quota_test + resetquota -u $TSTUSR +} +run_test 0 "Test basic quota performance" + +# test block hardlimit +test_1() { + local LIMIT=10 # 10M + local TESTFILE="$DIR/$tdir/$tfile-0" + + setup_quota_test + trap cleanup_quota_test EXIT + + # enable ost quota + set_ost_qtype "ug" || "enable ost quota failed" + + # test for user + log "User quota (block hardlimit:$LIMIT MB)" + $LFS setquota -u $TSTUSR -b 0 -B ${LIMIT}M -i 0 -I 0 $DIR + + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." + + $LFS setstripe $TESTFILE -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE - $LFS setstripe $TESTFILE -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE - - log " Write ..." - stime=`date +%s` - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT/2)) || quota_error u $TSTUSR "(usr) write failure, but expect success" - etime=`date +%s` - delta=$((etime - stime)) - if [ $delta -gt 0 ]; then - rate=$((BLK_SZ * LIMIT / 2 / delta / 1024)) - [ $rate -gt 1024 ] || error "SLOW IO for $TSTUSR (user): $rate KB/sec" - fi - log " Done" - log " Write out of block quota ..." + log "Write..." + $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) || + quota_error u $TSTUSR "user write failure, but expect success" + log "Write out of block quota ..." # this time maybe cache write, ignore it's failure - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT/2)) seek=$(($LIMIT/2)) || true - # flush cache, ensure noquota flag is setted on client - cancel_lru_locks osc - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$BUNIT_SZ seek=$LIMIT && quota_error u $TSTUSR "(usr) write success, but expect EDQUOT" - - rm -f $TESTFILE - sync; sleep 1; sync; - OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` - OST0_QUOTA_USED=`$LFS quota -o $OST0_UUID -u $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $1 }'` - echo $OST0_QUOTA_USED - [ $OST0_QUOTA_USED -ne 0 ] && \ - ($SHOW_QUOTA_USER; quota_error u $TSTUSR "(usr) quota deleted isn't released") - $SHOW_QUOTA_USER - resetquota -u $TSTUSR + $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) seek=$((LIMIT/2)) || true + # flush cache, ensure noquota flag is set on client + cancel_lru_locks osc + $RUNAS $DD of=$TESTFILE count=1 seek=$LIMIT && + quota_error u $TSTUSR "user write success, but expect EDQUOT" + + rm -f $TESTFILE + wait_delete_completed + sync_all_data || true + USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && quota_error u $TSTUSR \ + "user quota isn't released after deletion" + resetquota -u $TSTUSR - # test for group + # test for group log "--------------------------------------" - log " Group quota (limit: $LIMIT kbytes)" - $LFS setquota -g $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - sleep 3 - quota_show_check b g $TSTUSR - TESTFILE="$DIR/$tdir/$tfile-1" - - $LFS setstripe $TESTFILE -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE - - log " Write ..." - stime=`date +%s` - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT/2)) || quota_error g $TSTUSR "(grp) write failure, but expect success" - etime=`date +%s` - delta=$((etime - stime)) - if [ $delta -gt 0 ]; then - rate=$((BLK_SZ * LIMIT / 2 / delta / 1024)) - [ $rate -gt 1024 ] || error "SLOW IO for $TSTUSR (group): $rate KB/sec" - fi - log " Done" - log " Write out of block quota ..." + log "Group quota (block hardlimit:$LIMIT MB)" + $LFS setquota -g $TSTUSR -b 0 -B ${LIMIT}M -i 0 -I 0 $DIR + + TESTFILE="$DIR/$tdir/$tfile-1" + # make sure the system is clean + USED=$(getquota -g $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for group $TSTUSR isn't 0" + + $LFS setstripe $TESTFILE -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + + log "Write ..." + $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) || + quota_error g $TSTUSR "group write failure, but expect success" + log "Write out of block quota ..." # this time maybe cache write, ignore it's failure - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT/2)) seek=$(($LIMIT/2)) || true - cancel_lru_locks osc - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$BUNIT_SZ seek=$LIMIT && quota_error g $TSTUSR "(grp) write success, but expect EDQUOT" + $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) seek=$((LIMIT/2)) || true + cancel_lru_locks osc + $RUNAS $DD of=$TESTFILE count=10 seek=$LIMIT && + quota_error g $TSTUSR "group write success, but expect EDQUOT" # cleanup - rm -f $TESTFILE - sync; sleep 1; sync; - OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` - OST0_QUOTA_USED=`$LFS quota -o $OST0_UUID -g $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $1 }'` - echo $OST0_QUOTA_USED - [ $OST0_QUOTA_USED -ne 0 ] && \ - ($SHOW_QUOTA_GROUP; quota_error g $TSTUSR "(grp) quota deleted isn't released") - $SHOW_QUOTA_GROUP - resetquota -g $TSTUSR -} + cleanup_quota_test -# block hard limit (normal use and out of quota) -test_1() { - for i in `seq 1 $cycle`; do - # define blk_qunit is between 1M and 4M - blk_qunit=$(( $RANDOM % 3072 + 1024 )) - blk_qtune=$(( $RANDOM % $blk_qunit )) - # other osts and mds will occupy at 1M blk quota - b_limit=$(( ($RANDOM - 16384) / 8 + ($OSTCOUNT + 1) * $blk_qunit * 4 )) - set_blk_tunesz $blk_qtune - set_blk_unitsz $blk_qunit - echo "cycle: $i(total $cycle) bunit:$blk_qunit, btune:$blk_qtune, blimit:$b_limit" - test_1_sub $b_limit - echo "==================================================" - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) - done + USED=$(getquota -g $TSTUSR global curspace) + [ $USED -ne 0 ] && quota_error g $TSTUSR \ + "group quota isn't released after deletion" + + resetquota -g $TSTUSR } -run_test_with_stat 1 "Block hard limit (normal use and out of quota) ===" +run_test 1 "Block hard limit (normal use and out of quota)" -# test for specific quota limitation, qunit, qtune $1=block_quota_limit -test_2_sub() { - LIMIT=$1 - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - TESTFILE="$DIR/$tdir/$tfile-0" +# test inode hardlimit +test_2() { + local LIMIT=$((1024 * 1024)) # 1M inodes + local TESTFILE="$DIR/$tdir/$tfile-0" - wait_delete_completed + [ "$SLOW" = "no" ] && LIMIT=1024 # 1k inodes - # test for user - log " User quota (limit: $LIMIT files)" - $LFS setquota -u $TSTUSR -b 0 -B 0 -i 0 -I $LIMIT $DIR - sleep 3 - quota_show_check f u $TSTUSR + setup_quota_test + trap cleanup_quota_test EXIT + + # enable mdt quota + set_mdt_qtype "ug" || "enable mdt quota failed" + + # test for user + log "User quota (inode hardlimit:$LIMIT files)" + $LFS setquota -u $TSTUSR -b 0 -B 0 -i 0 -I $LIMIT $DIR + + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curinodes) + [ $USED -ne 0 ] && error "Used inodes($USED) for user $TSTUSR isn't 0." - log " Create $LIMIT files ..." + log "Create $LIMIT files ..." $RUNAS createmany -m ${TESTFILE} $LIMIT || \ - quota_error u $TSTUSR "(usr) create failure, but expect success" - log " Done" - log " Create out of file quota ..." + quota_error u $TSTUSR "user create failure, but expect success" + log "Create out of file quota ..." $RUNAS touch ${TESTFILE}_xxx && \ - quota_error u $TSTUSR "(usr) touch success, but expect EDQUOT" + quota_error u $TSTUSR "user create success, but expect EDQUOT" + # cleanup unlinkmany ${TESTFILE} $LIMIT rm -f ${TESTFILE}_xxx - sync; sleep 1; sync; + wait_delete_completed - MDS_UUID=`do_facet $SINGLEMDS $LCTL dl | grep -m1 " mdt " | awk '{print $((NF-1))}'` - MDS_QUOTA_USED=`$LFS quota -o $MDS_UUID -u $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $4 }'` - echo $MDS_QUOTA_USED - [ $MDS_QUOTA_USED -ne 0 ] && \ - ($SHOW_QUOTA_USER; quota_error u $TSTUSR "(usr) quota deleted isn't released") - $SHOW_QUOTA_USER + USED=$(getquota -u $TSTUSR global curinodes) + [ $USED -ne 0 ] && quota_error u $TSTUSR \ + "user quota isn't released after deletion" resetquota -u $TSTUSR - # test for group + # test for group log "--------------------------------------" - log " Group quota (limit: $LIMIT FILE)" - $LFS setquota -g $TSTUSR -b 0 -B 0 -i 0 -I $LIMIT $DIR - sleep 3 - quota_show_check f g $TSTUSR - TESTFILE=$DIR/$tdir/$tfile-1 + log "Group quota (inode hardlimit:$LIMIT files)" + $LFS setquota -g $TSTUSR -b 0 -B 0 -i 0 -I $LIMIT $DIR - log " Create $LIMIT files ..." - $RUNAS createmany -m ${TESTFILE} $LIMIT || \ - quota_error g $TSTUSR "(grp) create failure, but expect success" - log " Done" - log " Create out of file quota ..." + TESTFILE=$DIR/$tdir/$tfile-1 + # make sure the system is clean + USED=$(getquota -g $TSTUSR global curinodes) + [ $USED -ne 0 ] && error "Used inodes($USED) for group $TSTUSR isn't 0." + + log "Create $LIMIT files ..." + $RUNAS createmany -m ${TESTFILE} $LIMIT || + quota_error g $TSTUSR "group create failure, but expect success" + log "Create out of file quota ..." $RUNAS touch ${TESTFILE}_xxx && \ - quota_error g $TSTUSR "(grp) touch success, but expect EDQUOT" + quota_error g $TSTUSR "group create success, but expect EDQUOT" + # cleanup unlinkmany ${TESTFILE} $LIMIT rm -f ${TESTFILE}_xxx - sync; sleep 1; sync; + wait_delete_completed - MDS_UUID=`do_facet $SINGLEMDS $LCTL dl | grep -m1 " mdt " | awk '{print $((NF-1))}'` - MDS_QUOTA_USED=`$LFS quota -o $MDS_UUID -g $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $4 }'` - echo $MDS_QUOTA_USED - [ $MDS_QUOTA_USED -ne 0 ] && \ - ($SHOW_QUOTA_GROUP; quota_error g $TSTUSR "(grp) quota deleted isn't released") - $SHOW_QUOTA_GROUP - resetquota -g $TSTUSR -} + USED=$(getquota -g $TSTUSR global curinodes) + [ $USED -ne 0 ] && quota_error g $TSTUSR \ + "user quota isn't released after deletion" -# file hard limit (normal use and out of quota) -test_2() { - for i in `seq 1 $cycle`; do - if [ $i -eq 1 ]; then - ino_qunit=52 - ino_qtune=41 - i_limit=11 - else - # define ino_qunit is between 10 and 100 - ino_qunit=$(( $RANDOM % 90 + 10 )) - ino_qtune=$(( $RANDOM % $ino_qunit )) - # RANDOM's maxium is 32767 - i_limit=$(( $RANDOM % 990 + 10 )) - fi - - set_file_tunesz $ino_qtune - set_file_unitsz $ino_qunit - echo "cycle: $i(total $cycle) iunit:$ino_qunit, itune:$ino_qtune, ilimit:$i_limit" - test_2_sub $i_limit - echo "==================================================" - set_file_unitsz 5120 - set_file_tunesz 2560 - done + cleanup_quota_test + resetquota -g $TSTUSR } -run_test_with_stat 2 "File hard limit (normal use and out of quota) ===" +run_test 2 "File hard limit (normal use and out of quota)" test_block_soft() { - TESTFILE=$1 - TIMER=$(($2 * 3 / 2)) - OFFSET=0 + local TESTFILE=$1 + local TIMER=$(($2 * 3 / 2)) + local LIMIT=$3 + local OFFSET=0 - wait_delete_completed + setup_quota_test + trap cleanup_quota_test EXIT + + $LFS setstripe $TESTFILE -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE + + echo "Write up to soft limit" + $RUNAS $DD of=$TESTFILE count=$LIMIT || + quota_error a $TSTUSR "write failure, but expect success" + OFFSET=$((LIMIT * 1024)) + cancel_lru_locks osc - echo " Write to exceed soft limit" - RUNDD="$RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ" - $RUNDD count=$((BUNIT_SZ+1)) || \ + echo "Write to exceed soft limit" + $RUNAS dd if=/dev/zero of=$TESTFILE bs=1K count=10 seek=$OFFSET || quota_error a $TSTUSR "write failure, but expect success" - OFFSET=$((OFFSET + BUNIT_SZ + 1)) - cancel_lru_locks osc + OFFSET=$((OFFSET + 1024)) # make sure we don't write to same block + cancel_lru_locks osc $SHOW_QUOTA_USER $SHOW_QUOTA_GROUP $SHOW_QUOTA_INFO_USER $SHOW_QUOTA_INFO_GROUP - echo " Write before timer goes off" - $RUNDD count=$BUNIT_SZ seek=$OFFSET || \ + echo "Write before timer goes off" + $RUNAS dd if=/dev/zero of=$TESTFILE bs=1K count=10 seek=$OFFSET || quota_error a $TSTUSR "write failure, but expect success" - OFFSET=$((OFFSET + BUNIT_SZ)) - cancel_lru_locks osc - echo " Done" + OFFSET=$((OFFSET + 1024)) + cancel_lru_locks osc - echo " Sleep $TIMER seconds ..." - sleep $TIMER + echo "Sleep $TIMER seconds ..." + sleep $TIMER - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP - $SHOW_QUOTA_INFO_USER - $SHOW_QUOTA_INFO_GROUP + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $SHOW_QUOTA_INFO_USER + $SHOW_QUOTA_INFO_GROUP - echo " Write after timer goes off" + echo "Write after timer goes off" # maybe cache write, ignore. - $RUNDD count=$BUNIT_SZ seek=$OFFSET || true - OFFSET=$((OFFSET + BUNIT_SZ)) - cancel_lru_locks osc - $RUNDD count=$BUNIT_SZ seek=$OFFSET && \ + $RUNAS dd if=/dev/zero of=$TESTFILE bs=1K count=10 seek=$OFFSET || true + OFFSET=$((OFFSET + 1024)) + cancel_lru_locks osc + $RUNAS dd if=/dev/zero of=$TESTFILE bs=1K count=10 seek=$OFFSET && quota_error a $TSTUSR "write success, but expect EDQUOT" - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP - $SHOW_QUOTA_INFO_USER - $SHOW_QUOTA_INFO_GROUP + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $SHOW_QUOTA_INFO_USER + $SHOW_QUOTA_INFO_GROUP - echo " Unlink file to stop timer" + echo "Unlink file to stop timer" rm -f $TESTFILE - sync; sleep 1; sync - echo " Done" + wait_delete_completed + sync_all_data || true - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP - $SHOW_QUOTA_INFO_USER - $SHOW_QUOTA_INFO_GROUP + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + $SHOW_QUOTA_INFO_USER + $SHOW_QUOTA_INFO_GROUP - echo " Write ..." - $RUNDD count=$BUNIT_SZ || quota_error a $TSTUSR "write failure, but expect success" - echo " Done" + $LFS setstripe $TESTFILE -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE + echo "Write ..." + $RUNAS $DD of=$TESTFILE count=$LIMIT || + quota_error a $TSTUSR "write failure, but expect success" # cleanup - rm -f $TESTFILE - sync; sleep 3; sync; + cleanup_quota_test } -# block soft limit (start timer, timer goes off, stop timer) +# block soft limit test_3() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir + local LIMIT=1 # 1MB + local GRACE=10 # 10s + local TESTFILE=$DIR/$tdir/$tfile-0 - # 1 bunit on mds and 1 bunit on every ost - LIMIT=$(( $BUNIT_SZ * ($OSTCOUNT + 1) )) - GRACE=10 + set_ost_qtype "ug" || error "enable ost quota failed" - echo " User quota (soft limit: $LIMIT kbytes grace: $GRACE seconds)" - TESTFILE=$DIR/$tdir/$tfile-0 - - $LFS setstripe $TESTFILE -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE + echo "User quota (soft limit:$LIMIT MB grace:$GRACE seconds)" + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." $LFS setquota -t -u --block-grace $GRACE --inode-grace $MAX_IQ_TIME $DIR - $LFS setquota -u $TSTUSR -b $LIMIT -B 0 -i 0 -I 0 $DIR + $LFS setquota -u $TSTUSR -b ${LIMIT}M -B 0 -i 0 -I 0 $DIR - test_block_soft $TESTFILE $GRACE + test_block_soft $TESTFILE $GRACE $LIMIT resetquota -u $TSTUSR - echo " Group quota (soft limit: $LIMIT kbytes grace: $GRACE seconds)" + echo "Group quota (soft limit:$LIMIT MB grace:$GRACE seconds)" TESTFILE=$DIR/$tdir/$tfile-1 - - $LFS setstripe $TESTFILE -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE + # make sure the system is clean + USED=$(getquota -g $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for group $TSTUSR isn't 0." $LFS setquota -t -g --block-grace $GRACE --inode-grace $MAX_IQ_TIME $DIR - $LFS setquota -g $TSTUSR -b $LIMIT -B 0 -i 0 -I 0 $DIR + $LFS setquota -g $TSTUSR -b ${LIMIT}M -B 0 -i 0 -I 0 $DIR - test_block_soft $TESTFILE $GRACE + test_block_soft $TESTFILE $GRACE $LIMIT resetquota -g $TSTUSR + + # cleanup + $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR + $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR } -run_test_with_stat 3 "Block soft limit (start timer, timer goes off, stop timer) ===" +run_test 3 "Block soft limit (start timer, timer goes off, stop timer)" test_file_soft() { - TESTFILE=$1 - LIMIT=$2 - TIMER=$(($3 * 3 / 2)) + local TESTFILE=$1 + local LIMIT=$2 + local TIMER=$(($3 * 3 / 2)) - wait_delete_completed + setup_quota_test + trap cleanup_quota_test EXIT - echo " Create files to exceed soft limit" - $RUNAS createmany -m ${TESTFILE}_ $((LIMIT + 1)) || \ + echo "Create files to exceed soft limit" + $RUNAS createmany -m ${TESTFILE}_ $((LIMIT + 1)) || quota_error a $TSTUSR "create failure, but expect success" sync; sleep 1; sync - echo " Done" - echo " Create file before timer goes off" - $RUNAS touch ${TESTFILE}_before || \ - quota_error a $TSTUSR "failed create before timer expired, but expect success" + echo "Create file before timer goes off" + $RUNAS touch ${TESTFILE}_before || + quota_error a $TSTUSR "failed create before timer expired," \ + "but expect success" sync; sleep 1; sync - echo " Done" - echo " Sleep $TIMER seconds ..." + echo "Sleep $TIMER seconds ..." sleep $TIMER $SHOW_QUOTA_USER @@ -598,13 +655,15 @@ test_file_soft() { $SHOW_QUOTA_INFO_USER $SHOW_QUOTA_INFO_GROUP - echo " Create file after timer goes off" - # the least of inode qunit is 2, so there are at most 3(qunit:2+qtune:1) - # inode quota left here - $RUNAS touch ${TESTFILE}_after ${TESTFILE}_after1 ${TESTFILE}_after2 || true + echo "Create file after timer goes off" + # There is a window that space is accounted in the quota usage but + # hasn't been decreased from the pending write, if we acquire quota + # in this window, we'll acquire more than we needed. + $RUNAS touch ${TESTFILE}_after_1 ${TESTFILE}_after_2 || true sync; sleep 1; sync - $RUNAS touch ${TESTFILE}_after3 && \ - quota_error a $TSTUSR "create after timer expired, but expect EDQUOT" + $RUNAS touch ${TESTFILE}_after_3 && + quota_error a $TSTUSR "create after timer expired," \ + "but expect EDQUOT" sync; sleep 1; sync $SHOW_QUOTA_USER @@ -612,1582 +671,1367 @@ test_file_soft() { $SHOW_QUOTA_INFO_USER $SHOW_QUOTA_INFO_GROUP - echo " Unlink files to stop timer" - find `dirname $TESTFILE` -name "`basename ${TESTFILE}`*" | xargs rm -f - echo " Done" + echo "Unlink files to stop timer" + find $(dirname $TESTFILE) -name "$(basename ${TESTFILE})*" | xargs rm -f + wait_delete_completed - echo " Create file" - $RUNAS touch ${TESTFILE}_xxx || \ - quota_error a $TSTUSR "touch after timer stop failure, but expect success" + echo "Create file" + $RUNAS touch ${TESTFILE}_xxx || + quota_error a $TSTUSR "touch after timer stop failure," \ + "but expect success" sync; sleep 1; sync - echo " Done" # cleanup - rm -f ${TESTFILE}_xxx - sync; sleep 3; sync; + cleanup_quota_test } -# file soft limit (start timer, timer goes off, stop timer) -test_4a() { # was test_4 - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - LIMIT=$(($IUNIT_SZ * 10)) # 10 iunits on mds - TESTFILE=$DIR/$tdir/$tfile-0 +# file soft limit +test_4a() { + local LIMIT=10 # inodes + local TESTFILE=$DIR/$tdir/$tfile-0 + local GRACE=5 + + set_mdt_qtype "ug" || error "enable mdt quota failed" - GRACE=5 + echo "User quota (soft limit:$LIMIT files grace:$GRACE seconds)" + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curinodes) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." - echo " User quota (soft limit: $LIMIT files grace: $GRACE seconds)" $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace $GRACE $DIR $LFS setquota -u $TSTUSR -b 0 -B 0 -i $LIMIT -I 0 $DIR - quota_show_check f u $TSTUSR test_file_soft $TESTFILE $LIMIT $GRACE resetquota -u $TSTUSR - echo " Group quota (soft limit: $LIMIT files grace: $GRACE seconds)" + echo "Group quota (soft limit:$LIMIT files grace:$GRACE seconds)" + # make sure the system is clean + USED=$(getquota -g $TSTUSR global curinodes) + [ $USED -ne 0 ] && error "Used space($USED) for group $TSTUSR isn't 0." + $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace $GRACE $DIR $LFS setquota -g $TSTUSR -b 0 -B 0 -i $LIMIT -I 0 $DIR - quota_show_check f g $TSTUSR TESTFILE=$DIR/$tdir/$tfile-1 test_file_soft $TESTFILE $LIMIT $GRACE resetquota -g $TSTUSR # cleanup - $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR - $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR + $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR + $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR } -run_test_with_stat 4a "File soft limit (start timer, timer goes off, stop timer) ===" +run_test 4a "File soft limit (start timer, timer goes off, stop timer)" -test_4b() { # was test_4a - GR_STR1="1w3d" - GR_STR2="1000s" - GR_STR3="5s" - GR_STR4="1w2d3h4m5s" - GR_STR5="5c" - GR_STR6="1111111111111111" +test_4b() { + local GR_STR1="1w3d" + local GR_STR2="1000s" + local GR_STR3="5s" + local GR_STR4="1w2d3h4m5s" + local GR_STR5="5c" + local GR_STR6="1111111111111111" wait_delete_completed - # test of valid grace strings handling - echo " Valid grace strings test" - $LFS setquota -t -u --block-grace $GR_STR1 --inode-grace $GR_STR2 $DIR - $LFS quota -u -t $DIR | grep "Block grace time: $GR_STR1" - $LFS setquota -t -g --block-grace $GR_STR3 --inode-grace $GR_STR4 $DIR - $LFS quota -g -t $DIR | grep "Inode grace time: $GR_STR4" - - # test of invalid grace strings handling - echo " Invalid grace strings test" - ! $LFS setquota -t -u --block-grace $GR_STR4 --inode-grace $GR_STR5 $DIR - ! $LFS setquota -t -g --block-grace $GR_STR4 --inode-grace $GR_STR6 $DIR - - # cleanup - $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR - $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR + # test of valid grace strings handling + echo "Valid grace strings test" + $LFS setquota -t -u --block-grace $GR_STR1 --inode-grace $GR_STR2 $DIR + $LFS quota -u -t $DIR | grep "Block grace time: $GR_STR1" + $LFS setquota -t -g --block-grace $GR_STR3 --inode-grace $GR_STR4 $DIR + $LFS quota -g -t $DIR | grep "Inode grace time: $GR_STR4" + + # test of invalid grace strings handling + echo " Invalid grace strings test" + ! $LFS setquota -t -u --block-grace $GR_STR4 --inode-grace $GR_STR5 $DIR + ! $LFS setquota -t -g --block-grace $GR_STR4 --inode-grace $GR_STR6 $DIR + + # cleanup + $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR + $LFS setquota -t -g --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR } -run_test_with_stat 4b "Grace time strings handling ===" +run_test 4b "Grace time strings handling" # chown & chgrp (chown & chgrp successfully even out of block/file quota) test_5() { - mkdir -p $DIR/$tdir - BLIMIT=$(( $BUNIT_SZ * $((OSTCOUNT + 1)) * 10)) # 10 bunits on each server - ILIMIT=$(( $IUNIT_SZ * 10 )) # 10 iunits on mds - - wait_delete_completed - - echo " Set quota limit (0 $BLIMIT 0 $ILIMIT) for $TSTUSR.$TSTUSR" - $LFS setquota -u $TSTUSR -b 0 -B $BLIMIT -i 0 -I $ILIMIT $DIR - $LFS setquota -g $TSTUSR -b 0 -B $BLIMIT -i 0 -I $ILIMIT $DIR - quota_show_check a u $TSTUSR - quota_show_check a g $TSTUSR - - echo " Create more than $ILIMIT files and more than $BLIMIT kbytes ..." - createmany -m $DIR/$tdir/$tfile-0_ $((ILIMIT + 1)) || \ - error "touch failure, expect success" - dd if=/dev/zero of=$DIR/$tdir/$tfile-0_1 bs=$BLK_SZ count=$((BLIMIT+1)) || error "write failure, expect success" - - echo " Chown files to $TSTUSR.$TSTUSR ..." + local BLIMIT=10 # 10M + local ILIMIT=10 # 10 inodes + + setup_quota_test + trap cleanup_quota_test EXIT + + set_mdt_qtype "ug" || error "enable mdt quota failed" + set_ost_qtype "ug" || error "enable ost quota failed" + + echo "Set quota limit (0 ${BLIMIT}M 0 $ILIMIT) for $TSTUSR.$TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B ${BLIMIT}M -i 0 -I $ILIMIT $DIR + $LFS setquota -g $TSTUSR -b 0 -B ${BLIMIT}M -i 0 -I $ILIMIT $DIR + + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curinodes) + [ $USED -ne 0 ] && error "Used inode($USED) for user $TSTUSR isn't 0." + USED=$(getquota -g $TSTUSR global curinodes) + [ $USED -ne 0 ] && error "Used inode($USED) for group $TSTUSR isn't 0." + USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used block($USED) for user $TSTUSR isn't 0." + USED=$(getquota -g $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used block($USED) for group $TSTUSR isn't 0." + + echo "Create more than $ILIMIT files and more than $BLIMIT MB ..." + createmany -m $DIR/$tdir/$tfile-0_ $((ILIMIT + 1)) || + error "create failure, expect success" + $DD of=$DIR/$tdir/$tfile-0_1 count=$((BLIMIT+1)) || + error "write failure, expect success" + + echo "Chown files to $TSTUSR.$TSTUSR ..." for i in `seq 0 $ILIMIT`; do - chown $TSTUSR.$TSTUSR $DIR/$tdir/$tfile-0_$i || \ - quota_error a $TSTUSR "chown failure, but expect success" + chown $TSTUSR.$TSTUSR $DIR/$tdir/$tfile-0_$i || + quota_error a $TSTUSR "chown failure, expect success" done # cleanup unlinkmany $DIR/$tdir/$tfile-0_ $((ILIMIT + 1)) - sync; sleep 3; sync; + cleanup_quota_test resetquota -u $TSTUSR resetquota -g $TSTUSR } -run_test_with_stat 5 "Chown & chgrp successfully even out of block/file quota ===" +run_test 5 "Chown & chgrp successfully even out of block/file quota" -# block quota acquire & release +# test dropping acquire request on master test_6() { - if [ $OSTCOUNT -lt 2 ]; then - skip_env "$OSTCOUNT < 2, too few osts" - return 0; - fi + setup_quota_test + trap cleanup_quota_test EXIT - wait_delete_completed + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir + # make sure no granted quota on ost + set_ost_qtype "ug" || error "enable ost quota failed" + resetquota -u $TSTUSR - LIMIT=$((BUNIT_SZ * (OSTCOUNT + 1) * 5)) # 5 bunits per server - FILEA="$DIR/$tdir/$tfile-0_a" - FILEB="$DIR/$tdir/$tfile-0_b" + # create file for $TSTUSR + local TESTFILE=$DIR/$tdir/$tfile-$TSTUSR + $LFS setstripe $TESTFILE -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE - echo " Set block limit $LIMIT kbytes to $TSTUSR.$TSTUSR" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - $LFS setquota -g $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - quota_show_check b u $TSTUSR - quota_show_check b g $TSTUSR - - echo " Create filea on OST0 and fileb on OST1" - $LFS setstripe $FILEA -i 0 -c 1 - $LFS setstripe $FILEB -i 1 -c 1 - chown $TSTUSR.$TSTUSR $FILEA - chown $TSTUSR.$TSTUSR $FILEB - - echo " Exceed quota limit ..." - RUNDD="$RUNAS dd if=/dev/zero of=$FILEA bs=$BLK_SZ" - $RUNDD count=$((LIMIT - BUNIT_SZ * OSTCOUNT)) || \ - quota_error a $TSTUSR "write filea failure, but expect success" - - cancel_lru_locks osc - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP - $RUNDD seek=$LIMIT count=$((BUNIT_SZ * OSTCOUNT)) && \ - quota_error a $TSTUSR "write filea success, but expect EDQUOT" - cancel_lru_locks osc - echo " Write to OST1 return EDQUOT" - # this write maybe cache write, ignore it's failure - RUNDD="$RUNAS dd if=/dev/zero of=$FILEB bs=$BLK_SZ" - $RUNDD count=$(($BUNIT_SZ * 2)) || true - cancel_lru_locks osc - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP - $RUNDD count=$((BUNIT_SZ * 2)) seek=$((BUNIT_SZ *2)) && \ - quota_error a $TSTUSR "write fileb success, but expect EDQUOT" - - echo " Remove filea to let OST0 release quota" - rm -f $FILEA - - if at_is_enabled; then - timeout=$(at_max_get mds) - else - timeout=$(lctl get_param -n timeout) - fi - count=$((timeout / 5)) - OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` - - while [ $((count--)) -gt 0 ]; do - sync && sleep 5 - - OST0_QUOTA_HOLD=`$LFS quota -o $OST0_UUID -u $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $3 }'` - if [ -z $OST0_QUOTA_HOLD ]; then - error "System is error when query quota for block (U:$TSTUSR)." - else - [ $OST0_QUOTA_HOLD -gt $BUNIT_SZ ] && continue - fi - - break - done - - [ ! $count -gt 0 ] && error "Release quota for block timeout (U:$TSTUSR)." - $SHOW_QUOTA_USER - - while [ $((count--)) -gt 0 ]; do - sync && sleep 5 - - OST0_QUOTA_HOLD=`$LFS quota -o $OST0_UUID -g $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $3 }'` - if [ -z $OST0_QUOTA_HOLD ]; then - error "System is error when query quota for block (G:$TSTUSR)." - else - [ $OST0_QUOTA_HOLD -gt $BUNIT_SZ ] && continue - fi - - break - done - - [ ! $count -gt 0 ] && error "Release quota for block timeout (G:$TSTUSR)." - $SHOW_QUOTA_GROUP - - echo " Write to OST1" - $RUNDD count=$((LIMIT - BUNIT_SZ * OSTCOUNT)) || \ - quota_error a $TSTUSR "write fileb failure, expect success" - echo " Done" + # create file for $TSTUSR2 + local TESTFILE2=$DIR/$tdir/$tfile-$TSTUSR2 + $LFS setstripe $TESTFILE2 -c 1 -i 0 + chown $TSTUSR2.$TSTUSR2 $TESTFILE2 + + # cache per-ID lock for $TSTUSR on slave + $LFS setquota -u $TSTUSR -b 0 -B 2M -i 0 -I 0 $DIR + $RUNAS $DD of=$TESTFILE count=1 || + error "write $TESTFILE failure, expect success" + $RUNAS2 $DD of=$TESTFILE2 count=1 || + error "write $TESTFILE2 failure, expect success" + sync; sync + sync_all_data || true + + #define QUOTA_DQACQ 601 + #define OBD_FAIL_PTLRPC_DROP_REQ_OPC 0x513 + lustre_fail mds 0x513 601 + + # write to un-enforced ID ($TSTUSR2) should succeed + $RUNAS2 $DD of=$TESTFILE2 count=1 seek=1 oflag=sync conv=notrunc || + error "write failure, expect success" + + # write to enforced ID ($TSTUSR) in background, exceeding limit + # to make sure DQACQ is sent + $RUNAS $DD of=$TESTFILE count=2 seek=1 oflag=sync conv=notrunc & + DDPID=$! - # cleanup - rm -f $FILEB - sync; sleep 3; sync; + echo "Sleep for $TIMEOUT" + sleep $TIMEOUT - resetquota -u $TSTUSR - resetquota -g $TSTUSR - return 0 -} -run_test_with_stat 6 "Block quota acquire & release =========" + # write should be blocked and never finished + if ! ps -p $DDPID > /dev/null 2>&1; then + lustre_fail mds 0 0 + error "write finished incorrectly!" + fi -# quota recovery (block quota only by now) -test_7() -{ - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir + lustre_fail mds 0 0 - wait_delete_completed + # no watchdog is triggered + do_facet ost1 dmesg > $TMP/lustre-log-${TESTNAME}.log + watchdog=$(awk '/sanity-quota test 6/ {start = 1;} + /Service thread pid/ && /was inactive/ { + if (start) { + print; + } + }' $TMP/lustre-log-${TESTNAME}.log) + [ -z "$watchdog" ] || error "$watchdog" - LIMIT=$(( $BUNIT_SZ * $(($OSTCOUNT + 1)) )) - TESTFILE="$DIR/$tdir/$tfile-0" + rm -f $TMP/lustre-log-${TESTNAME}.log - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + # write should continue & succeed + local count=0 + while [ true ]; do + if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi + if [ $count -ge 120 ]; then + quota_error u $TSTUSR "dd not finished in $count secs" + fi + count=$((count + 1)) + [ $((count % 10)) -eq 0 ] && echo "Waiting $count secs" + sleep 1 + done - $LFS setstripe $TESTFILE -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE + cleanup_quota_test + resetquota -u $TSTUSR +} +run_test 6 "Test dropping acquire request on master" - echo " Write to OST0..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$BUNIT_SZ || \ - quota_error u $TSTUSR "write failure, but expect success" +# quota reintegration (global index) +test_7a() { + local TESTFILE=$DIR/$tdir/$tfile + local LIMIT=20 # 20M - #define OBD_FAIL_OBD_DQACQ 0x604 - lustre_fail mds 0x604 - echo " Remove files on OST0" - rm -f $TESTFILE - lustre_fail mds 0 + [ "$SLOW" = "no" ] && LIMIT=5 - echo " Trigger recovery..." - OSC0_UUID="`$LCTL dl | awk '$3 ~ /osc/ { print $1 }'`" - for i in $OSC0_UUID; do - $LCTL --device $i activate || error "activate osc failed!" - done + setup_quota_test + trap cleanup_quota_test EXIT - # sleep a while to wait for recovery done - sleep 20 + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." - # check limits - PATTERN="`echo $DIR | sed 's/\//\\\\\//g'`" - TOTAL_LIMIT="`$LFS quota -v -u $TSTUSR $DIR | awk '/^.*'$PATTERN'.*[[:digit:]+][[:space:]+]/ { print $4 }'`" - [ $TOTAL_LIMIT -eq $LIMIT ] || error "total limits not recovery!" - echo " total limits = $TOTAL_LIMIT" + # make sure no granted quota on ost1 + set_ost_qtype "ug" || error "enable ost quota failed" + resetquota -u $TSTUSR + set_ost_qtype "none" || error "disable ost quota failed" - OST0_UUID=`do_facet ost1 "$LCTL dl | grep -m1 obdfilter" | awk '{print $((NF-1))}'` - [ -z "$OST0_UUID" ] && OST0_UUID=`do_facet ost1 "$LCTL dl | grep -m1 obdfilter" | awk '{print $((NF-1))}'` - OST0_LIMIT="`$LFS quota -o $OST0_UUID -u $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $3 }'`" - [ $OST0_LIMIT -eq $BUNIT_SZ ] || error "high limits not released!" - echo " limits on $OST0_UUID = $OST0_LIMIT" + local OSTUUID=$(ostuuid_from_index 0) + USED=$(getquota -u $TSTUSR $OSTUUID bhardlimit) + [ $USED -ne 0 ] && error "limit($USED) on $OSTUUID for user" \ + "$TSTUSR isn't 0." - # cleanup - resetquota -u $TSTUSR -} -run_test_with_stat 7 "Quota recovery (only block limit) ======" + # create test file + $LFS setstripe $TESTFILE -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE -# run dbench with quota enabled -test_8() { - mkdir -p $DIR/$tdir - BLK_LIMIT=$((100 * 1024 * 1024)) # 100G - FILE_LIMIT=1000000 + echo "Stop ost1..." + stop ost1 - wait_delete_completed + echo "Enable quota & set quota limit for $TSTUSR" + set_ost_qtype "ug" || error "enable ost quota failed" + $LFS setquota -u $TSTUSR -b 0 -B ${LIMIT}M -i 0 -I 0 $DIR - echo " Set enough high limit for user: $TSTUSR" - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR - echo " Set enough high limit for group: $TSTUSR" - $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + echo "Start ost1..." + start ost1 $(ostdevname 1) $OST_MOUNT_OPTS - chmod 0777 $DIR/$tdir - local duration="" - [ "$SLOW" = "no" ] && duration=" -t 120" - $RUNAS bash rundbench -D $DIR/$tdir 3 $duration || quota_error a $TSTUSR "dbench failed!" + wait_ost_reint "ug" || error "reintegration failed" - rm -rf $DIR/$tdir - sync; sleep 3; sync; + # hardlimit should have been fetched by slave during global + # reintegration, write will exceed quota + $RUNAS $DD of=$TESTFILE count=$((LIMIT + 1)) oflag=sync && + quota_error u $TSTUSR "write success, but expect EDQUOT" - return 0 + cleanup_quota_test + resetquota -u $TSTUSR } -run_test_with_stat 8 "Run dbench with quota enabled ===========" +run_test 7a "Quota reintegration (global index)" -# run for fixing bug10707, it needs a big room. test for 64bit -KB=1024 -GB=$((KB * 1024 * 1024)) -# Use this as dd bs to decrease time -# inode->i_blkbits = min(PTLRPC_MAX_BRW_BITS+1, LL_MAX_BLKSIZE_BITS); -blksize=$((1 << 21)) # 2Mb -size_file=$((GB * 9 / 2)) -# this check is just for test9 and test10 -OST0_MIN=4900000 #4.67G -check_whether_skip () { - OST0_SIZE=`$LFS df $DIR | awk '/\[OST:0\]/ {print $4}'` - log "OST0_SIZE: $OST0_SIZE required: $OST0_MIN" - if [ $OST0_SIZE -lt $OST0_MIN ]; then - echo "WARN: OST0 has less than $OST0_MIN free, skip this test." - return 0 - else - return 1 - fi -} +# quota reintegration (slave index) +test_7b() { + local LIMIT="100G" + local TESTFILE=$DIR/$tdir/$tfile -test_9() { - check_whether_skip && return 0 + setup_quota_test + trap cleanup_quota_test EXIT - wait_delete_completed + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." - set_blk_tunesz 512 - set_blk_unitsz 1024 + # make sure no granted quota on ost1 + set_ost_qtype "ug" || error "enable ost quota failed" + resetquota -u $TSTUSR + set_ost_qtype "none" || error "disable ost quota failed" - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - TESTFILE="$DIR/$tdir/$tfile-0" + local OSTUUID=$(ostuuid_from_index 0) + USED=$(getquota -u $TSTUSR $OSTUUID bhardlimit) + [ $USED -ne 0 ] && error "limit($USED) on $OSTUUID for user" \ + "$TSTUSR isn't 0." - BLK_LIMIT=$((100 * KB * KB)) # 100G - FILE_LIMIT=1000000 - echo " Set block limit $BLK_LIMIT kbytes to $TSTUSR.$TSTUSR" + # create test file + $LFS setstripe $TESTFILE -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE - log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for user: $TSTUSR" - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR - log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for group: $TSTUSR" - $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + # consume some space to make sure the granted space will not + # be released during reconciliation + $RUNAS $DD of=$TESTFILE count=1 oflag=sync || + error "consume space failure, expect success" - quota_show_check a u $TSTUSR - quota_show_check a g $TSTUSR + # define OBD_FAIL_QUOTA_EDQUOT 0xa02 + lustre_fail mds 0xa02 - echo " Set stripe" - $LFS setstripe $TESTFILE -c 1 - touch $TESTFILE - chown $TSTUSR.$TSTUSR $TESTFILE + set_ost_qtype "ug" || error "enable ost quota failed" + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - log " Write the big file of 4.5G ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$blksize count=$((size_file / blksize)) || \ - quota_error a $TSTUSR "(usr) write 4.5G file failure, but expect success" + # ignore the write error + $RUNAS $DD of=$TESTFILE count=1 seek=1 oflag=sync conv=notrunc - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP + local old_used=$(getquota -u $TSTUSR $OSTUUID bhardlimit) - log " delete the big file of 4.5G..." - $RUNAS rm -f $TESTFILE - sync; sleep 3; sync; + lustre_fail mds 0 - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP + echo "Restart ost to trigger reintegration..." + stop ost1 + start ost1 $(ostdevname 1) $OST_MOUNT_OPTS - RC=$? + wait_ost_reint "ug" || error "reintegration failed" - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) + USED=$(getquota -u $TSTUSR $OSTUUID bhardlimit) + [ $USED -gt $old_used ] || error "limit on $OSTUUID $USED <= $old_used" - return $RC + cleanup_quota_test + resetquota -u $TSTUSR + $SHOW_QUOTA_USER } -run_test_with_stat 9 "run for fixing bug10707(64bit) ===========" +run_test 7b "Quota reintegration (slave index)" -# run for fixing bug10707, it need a big room. test for 32bit -# 2.0 version does not support 32 bit qd_count, so such test is obsolete. -test_10() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - check_whether_skip && return 0 - - wait_delete_completed +# quota reintegration (restart mds during reintegration) +test_7c() { + local LIMIT=20 # 20M + local TESTFILE=$DIR/$tdir/$tfile - set_blk_tunesz 512 - set_blk_unitsz 1024 + [ "$SLOW" = "no" ] && LIMIT=5 - # make qd_count 32 bit - lustre_fail mds_ost 0xA00 - - TESTFILE="$DIR/$tdir/$tfile-0" - - BLK_LIMIT=$((100 * KB * KB)) # 100G - FILE_LIMIT=1000000 - - log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for user: $TSTUSR" - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR - log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for group: $TSTUSR" - $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + setup_quota_test + trap cleanup_quota_test EXIT - quota_show_check a u $TSTUSR - quota_show_check a g $TSTUSR + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." - echo " Set stripe" - $LFS setstripe $TESTFILE -c 1 - touch $TESTFILE - chown $TSTUSR.$TSTUSR $TESTFILE + set_ost_qtype "none" || error "disable ost quota failed" + $LFS setquota -u $TSTUSR -b 0 -B ${LIMIT}M -i 0 -I 0 $DIR - log " Write the big file of 4.5 G ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$blksize count=$((size_file / blksize)) || \ - quota_error a $TSTUSR "(usr) write 4.5 G file failure, but expect success" + # define OBD_FAIL_QUOTA_DELAY_REINT 0xa03 + lustre_fail ost 0xa03 - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP + # enable ost quota to trigger reintegration + set_ost_qtype "ug" || error "enable ost quota failed" - log " delete the big file of 4.5 G..." - $RUNAS rm -f $TESTFILE - sync; sleep 3; sync; + echo "Stop mds..." + stop mds1 - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP + lustre_fail ost 0 - RC=$? + echo "Start mds..." + start mds1 $(mdsdevname 1) $MDS_MOUNT_OPTS - # make qd_count 64 bit - lustre_fail mds_ost 0 + wait_ost_reint "ug" || error "reintegration failed" - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) + # hardlimit should have been fetched by slave during global + # reintegration, write will exceed quota + $RUNAS $DD of=$TESTFILE count=$((LIMIT + 1)) oflag=sync && + quota_error u $TSTUSR "write success, but expect EDQUOT" - return $RC + cleanup_quota_test + resetquota -u $TSTUSR } -#run_test_with_stat 10 "run for fixing bug10707(32bit) ===========" - -# test a deadlock between quota and journal b=11693 -test_12() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir +run_test 7c "Quota reintegration (restart mds during reintegration)" - [ "$(grep $DIR2 /proc/mounts)" ] || mount_client $DIR2 || \ - { skip_env "Need lustre mounted on $MOUNT2 " && retutn 0; } +# run dbench with quota enabled +test_8() { + local BLK_LIMIT="100g" #100G + local FILE_LIMIT=1000000 - LIMIT=$(( $BUNIT_SZ * $(($OSTCOUNT + 1)) * 10)) # 10 bunits each sever - TESTFILE="$DIR/$tdir/$tfile-0" - TESTFILE2="$DIR2/$tdir/$tfile-1" + setup_quota_test + trap cleanup_quota_test EXIT - wait_delete_completed - - echo " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + set_mdt_qtype "ug" || error "enable mdt quota failed" + set_ost_qtype "ug" || error "enable ost quota failed" - $LFS setstripe $TESTFILE -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE - $LFS setstripe $TESTFILE2 -i 0 -c 1 - chown $TSTUSR2.$TSTUSR2 $TESTFILE2 + echo "Set enough high limit for user: $TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR + echo "Set enough high limit for group: $TSTUSR" + $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR - #define OBD_FAIL_OST_HOLD_WRITE_RPC 0x21f - #define OBD_FAIL_SOME 0x10000000 /* fail N times */ - lustre_fail ost $((0x0000021f | 0x10000000)) 1 + local duration="" + [ "$SLOW" = "no" ] && duration=" -t 120" + $RUNAS bash rundbench -D $DIR/$tdir 3 $duration || + quota_error a $TSTUSR "dbench failed!" - echo " step1: write out of block quota ..." - $RUNAS2 dd if=/dev/zero of=$TESTFILE2 bs=$BLK_SZ count=102400 & - DDPID1=$! - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT*2)) & - DDPID=$! + cleanup_quota_test + resetquota -u $TSTUSR + resetquota -g $TSTUSR +} +run_test 8 "Run dbench with quota enabled" - echo " step2: testing ......" - local last_size=$(stat -c %s $TESTFILE2) - local stall_secs=0 - local start_secs=$SECONDS - while [ -d /proc/${DDPID1} ]; do - local size=$(stat -c %s $TESTFILE2) - if [ $size -eq $last_size ]; then - stall_secs=$[stall_secs+1] - else - stall_secs=0 - fi - if [ $stall_secs -gt 30 ]; then - lustre_fail ost 0 - quota_error u $TSTUSR2 "giving up: dd stalled (i.e. made no progress) for 30 seconds!" - fi - last_size=$size - sleep 1 - done - echo "(dd_pid=$DDPID1, time=$((SECONDS-start_secs)))successful" +# this check is just for test_9 +OST0_MIN=4900000 #4.67G - #Recover fail_loc and dd will finish soon - lustre_fail ost 0 +check_whether_skip () { + local OST0_SIZE=$($LFS df $DIR | awk '/\[OST:0\]/ {print $4}') + log "OST0_SIZE: $OST0_SIZE required: $OST0_MIN" + if [ $OST0_SIZE -lt $OST0_MIN ]; then + echo "WARN: OST0 has less than $OST0_MIN free, skip this test." + return 0 + else + return 1 + fi +} - echo " step3: testing ......" - count=0 - while [ true ]; do - if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi - count=$[count+1] - if [ $count -gt 150 ]; then - quota_error u $TSTUSR "dd should be finished!" - fi - sleep 1 - done - echo "(dd_pid=$DDPID, time=$count)successful" +# run for fixing bug10707, it needs a big room. test for 64bit +test_9() { + local filesize=$((1024 * 1024 * 1024 * 9 / 2)) # 4.5G - rm -f $TESTFILE $TESTFILE2 - sync; sleep 3; sync; + check_whether_skip && return 0 - resetquota -u $TSTUSR -} -run_test_with_stat 12 "test a deadlock between quota and journal ===" + setup_quota_test + trap cleanup_quota_test EXIT -# test multiple clients write block quota b=11693 -test_13() { - mkdir -p $DIR/$tdir - wait_delete_completed + set_ost_qtype "ug" || error "enable ost quota failed" - # one OST * 10 + (mds + other OSTs) - LIMIT=$((BUNIT_SZ * 10 + (BUNIT_SZ * OSTCOUNT))) - TESTFILE="$DIR/$tdir/$tfile" + local TESTFILE="$DIR/$tdir/$tfile-0" + local BLK_LIMIT=100G #100G + local FILE_LIMIT=1000000 - echo " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - quota_show_check b u $TSTUSR + echo "Set block limit $BLK_LIMIT bytes to $TSTUSR.$TSTUSR" - $LFS setstripe $TESTFILE -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE - $LFS setstripe $TESTFILE.2 -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE.2 + log "Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT)" \ + "for user: $TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR - echo " step1: write out of block quota ..." - # one bunit will give mds - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$[($LIMIT - $BUNIT_SZ) / 2] & - DDPID=$! - $RUNAS dd if=/dev/zero of=$TESTFILE.2 bs=$BLK_SZ count=$[($LIMIT - $BUNIT_SZ) / 2] & - DDPID1=$! + log "Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT)" \ + "for group: $TSTUSR" + $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $DIR - echo " step2: testing ......" - count=0 - while [ true ]; do - if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi - count=$[count+1] - if [ $count -gt 64 ]; then - quota_error u $TSTUSR "dd should be finished!" - fi - sleep 1 - done - echo "(dd_pid=$DDPID, time=$count)successful" + quota_show_check a u $TSTUSR + quota_show_check a g $TSTUSR - count=0 - while [ true ]; do - if ! ps -p ${DDPID1} > /dev/null 2>&1 ; then break; fi - count=$[count+1] - if [ $count -gt 64 ]; then - quota_error u $TSTUSR "dd should be finished!" - fi - sleep 1 - done - echo "(dd_pid=$DDPID1, time=$count)successful" + echo "Create test file" + $LFS setstripe $TESTFILE -c 1 -i 0 + chown $TSTUSR.$TSTUSR $TESTFILE - sync; sleep 5; sync; + log "Write the big file of 4.5G ..." + $RUNAS $DD of=$TESTFILE count=$filesize || + quota_error a $TSTUSR "write 4.5G file failure, expect success" - echo " step3: checking ......" - fz=`stat -c %s $TESTFILE` - fz2=`stat -c %s $TESTFILE.2` $SHOW_QUOTA_USER - [ $((fz + fz2)) -lt $((BUNIT_SZ * BLK_SZ * 10)) ] && \ - quota_error u $TSTUSR "files too small $fz + $fz2 < $((BUNIT_SZ * BLK_SZ * 10))" - - rm -f $TESTFILE $TESTFILE.2 - sync; sleep 3; sync; + $SHOW_QUOTA_GROUP + cleanup_quota_test resetquota -u $TSTUSR -} -run_test_with_stat 13 "test multiple clients write block quota ===" - -check_if_quota_zero(){ - line=`$LFS quota -v -$1 $2 $DIR | wc -l` - for i in `seq 3 $line`; do - if [ $i -eq 3 ]; then - field="3 4 6 7" - else - field="3 5" - fi - for j in $field; do - tmp=`$LFS quota -v -$1 $2 $DIR | sed -n ${i}p | - awk '{print $'"$j"'}'` - [ -n "$tmp" ] && [ $tmp -ne 0 ] && $LFS quota -v -$1 $2 $DIR && \ - error "quota on $2 isn't clean" - done - done - echo "pass check_if_quota_zero" -} - -test_14a() { # was test_14 b=12223 -- setting quota on root - TESTFILE="$DIR/$tdir/$tfile" - - # reboot the lustre - sync; sleep 5; sync - cleanup_and_setup_lustre - quota_init - - mkdir -p $DIR/$tdir + resetquota -g $TSTUSR - # out of root's file and block quota - $LFS setquota -u root -b 10 -B 10 -i 10 -I 10 $DIR - createmany -m ${TESTFILE} 20 || \ - quota_error u root "unexpected: user(root) create files failly!" - dd if=/dev/zero of=$TESTFILE bs=4k count=4096 || \ - quota_error u root "unexpected: user(root) write files failly!" - chmod 666 $TESTFILE - $RUNAS dd if=/dev/zero of=${TESTFILE} seek=4096 bs=4k count=4096 && \ - quota_error u root "unexpected: user(quota_usr) write a file successfully!" - - # trigger the llog - chmod 777 $DIR - for i in `seq 1 10`; do $RUNAS touch ${TESTFILE}a_$i; done - for i in `seq 1 10`; do $RUNAS rm -f ${TESTFILE}a_$i; done - - # do the check - dmesg | tail | grep "\-122" |grep llog_obd_origin_add && error "err -122 not found in dmesg" - resetquota -u root - #check_if_quota_zero u root - - # clean - unlinkmany ${TESTFILE} 15 - rm -f $TESTFILE - sync; sleep 3; sync; + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP } -run_test_with_stat 14a "test setting quota on root ===" - -test_15(){ - LIMIT=$((24 * 1024 * 1024 * 1024 * 1024)) # 24 TB - PATTERN="`echo $DIR | sed 's/\//\\\\\//g'`" +run_test 9 "Block limit larger than 4GB (b10707)" - wait_delete_completed +test_10() { + local TESTFILE=$DIR/$tdir/$tfile - # test for user - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - TOTAL_LIMIT="`$LFS quota -v -u $TSTUSR $DIR | awk '/^.*'$PATTERN'.*[[:digit:]+][[:space:]+]/ { print $4 }'`" - [ $TOTAL_LIMIT -eq $LIMIT ] || error " (user)total limits = $TOTAL_LIMIT; limit = $LIMIT, failed!" - echo " (user)total limits = $TOTAL_LIMIT; limit = $LIMIT, successful!" - resetquota -u $TSTUSR - - # test for group - $LFS setquota -g $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - TOTAL_LIMIT="`$LFS quota -v -g $TSTUSR $DIR | awk '/^.*'$PATTERN'.*[[:digit:]+][[:space:]+]/ { print $4 }'`" - [ $TOTAL_LIMIT -eq $LIMIT ] || error " (group)total limits = $TOTAL_LIMIT; limit = $LIMIT, failed!" - echo " (group)total limits = $TOTAL_LIMIT; limit = $LIMIT, successful!" - resetquota -g $TSTUSR - $LFS quotaoff -ug $DIR - do_facet $SINGLEMDS "lctl set_param lquota.mdd_obd-${FSNAME}-MDT*.quota_type=ug" | grep "error writing" && \ - error "fail to set version for $SINGLEMDS" - for j in `seq $OSTCOUNT`; do - do_facet ost$j "lctl set_param lquota.${FSNAME}-OST*.quota_type=ug" | grep "error writing" && \ - error "fail to set version for ost$j" - done - - echo "invalidating quota files" - $LFS quotainv -ug $DIR - $LFS quotainv -ugf $DIR - $LFS quotacheck -ug $DIR -} -run_test_with_stat 15 "set block quota more than 4T ===" + setup_quota_test + trap cleanup_quota_test EXIT -# $1=u/g $2=with qunit adjust or not -test_16_tub() { - LIMIT=$(( $BUNIT_SZ * $(($OSTCOUNT + 1)) * 4)) - TESTFILE="$DIR/$tdir/$tfile" - mkdir -p $DIR/$tdir + # set limit to root user should fail + $LFS setquota -u root -b 100G -B 500G -i 1K -I 1M $DIR && + error "set limit for root user successfully, expect failure" + $LFS setquota -g root -b 1T -B 10T -i 5K -I 100M $DIR && + error "set limit for root group successfully, expect failure" - wait_delete_completed + # root user can overrun quota + set_ost_qtype "ug" || "enable ost quota failed" - echo " User quota (limit: $LIMIT kbytes)" - if [ $1 == "u" ]; then - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - quota_show_check b u $TSTUSR - else - $LFS setquota -g $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - quota_show_check b g $TSTUSR - fi + $LFS setquota -u $TSTUSR -b 0 -B 2M -i 0 -I 0 $DIR + quota_show_check b u $TSTUSR $LFS setstripe $TESTFILE -c 1 chown $TSTUSR.$TSTUSR $TESTFILE - echo " Write ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$((BUNIT_SZ * 4)) || \ - quota_error a $TSTUSR "(usr) write failure, but expect success" - echo " Done" - echo " Write out of block quota ..." - # this time maybe cache write, ignore it's failure - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$BUNIT_SZ seek=$((BUNIT_SZ * 4)) || true - # flush cache, ensure noquota flag is setted on client - cancel_lru_locks osc - if [ $2 -eq 1 ]; then - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$BUNIT_SZ seek=$((BUNIT_SZ * 4)) || \ - quota_error a $TSTUSR "(write failure, but expect success" - else - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$BUNIT_SZ seek=$((BUNIT_SZ * 4)) && \ - quota_error a $TSTUSR "(write success, but expect EDQUOT" - fi + runas -u 0 -g 0 $DD of=$TESTFILE count=3 oflag=sync || + error "write failure, expect success" - rm -f $TESTFILE - sync; sleep 3; sync; - resetquota -$1 $TSTUSR + cleanup_quota_test + resetquota -u $TSTUSR } +run_test 10 "Test quota for root user" -# test without adjusting qunit -# 2.0 version does not support WITHOUT_CHANGE_QS, so such test is obsolete -test_16 () { - set_blk_tunesz $((BUNIT_SZ * 2)) - set_blk_unitsz $((BUNIT_SZ * 4)) - for i in u g; do - for j in 0 1; do - # define OBD_FAIL_QUOTA_WITHOUT_CHANGE_QS 0xA01 - echo " grp/usr: $i, adjust qunit: $j" - echo "-------------------------------" - [ $j -eq 1 ] && lustre_fail mds_ost 0 - [ $j -eq 0 ] && lustre_fail mds_ost 0xA01 - test_16_tub $i $j - done - done - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) -} -#run_test_with_stat 16 "test without adjusting qunit" +test_11() { + local TESTFILE=$DIR/$tdir/$tfile + setup_quota_test + trap cleanup_quota_test EXIT -# run for fixing bug14526, failed returned quota reqs shouldn't ruin lustre. -test_17() { - set_blk_tunesz 512 - set_blk_unitsz 1024 + set_mdt_qtype "ug" || "enable mdt quota failed" + $LFS setquota -u $TSTUSR -b 0 -B 0 -i 0 -I 1 $DIR - wait_delete_completed + touch "$TESTFILE"-0 + touch "$TESTFILE"-1 - #define OBD_FAIL_QUOTA_RET_QDATA | OBD_FAIL_ONCE - lustre_fail ost 0x80000A02 + chown $TSTUSR.$TSTUSR "$TESTFILE"-0 + chown $TSTUSR.$TSTUSR "$TESTFILE"-1 - TESTFILE="$DIR/$tdir/$tfile-a" - TESTFILE2="$DIR/$tdir/$tfile-b" - mkdir -p $DIR/$tdir + $SHOW_QUOTA_USER + local USED=$(getquota -u $TSTUSR global curinodes) + [ $USED -ge 2 ] || error "Used inodes($USED) is less than 2" - BLK_LIMIT=$((100 * 1024)) # 100M + cleanup_quota_test + resetquota -u $TSTUSR +} +run_test 11 "Chown/chgrp ignores quota" - log " Set enough high limit(block:$BLK_LIMIT) for user: $TSTUSR" - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR - log " Set enough high limit(block:$BLK_LIMIT) for group: $TSTUSR" - $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR +test_12() { + [ "$OSTCOUNT" -lt "2" ] && skip "skipping rebalancing test" && return - quota_show_check b u $TSTUSR - quota_show_check b g $TSTUSR + local blimit=22 # 22M + local blk_cnt=$((blimit - 5)) + local TESTFILE0="$DIR/$tdir/$tfile"-0 + local TESTFILE1="$DIR/$tdir/$tfile"-1 - touch $TESTFILE - chown $TSTUSR.$TSTUSR $TESTFILE - touch $TESTFILE2 - chown $TSTUSR.$TSTUSR $TESTFILE2 + setup_quota_test + trap cleanup_quota_test EXIT - log " Write the test file1 ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(( 10 * 1024 )) \ - || quota_error a $TSTUSR "write 10M file failure" + set_ost_qtype "u" || "enable ost quota failed" + quota_show_check b u $TSTUSR - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP + $LFS setquota -u $TSTUSR -b 0 -B "$blimit"M -i 0 -I 0 $DIR - log " write the test file2 ..." - $RUNAS dd if=/dev/zero of=$TESTFILE2 bs=$BLK_SZ count=$(( 10 * 1024 )) \ - || quota_error a $TSTUSR "write 10M file failure" + $LFS setstripe $TESTFILE0 -c 1 -i 0 + $LFS setstripe $TESTFILE1 -c 1 -i 1 + chown $TSTUSR.$TSTUSR $TESTFILE0 + chown $TSTUSR.$TSTUSR $TESTFILE1 - $SHOW_QUOTA_USER - $SHOW_QUOTA_GROUP + echo "Write to ost0..." + $RUNAS $DD of=$TESTFILE0 count=$blk_cnt oflag=sync || + quota_error a $TSTUSR "dd failed" - rm -f $TESTFILE $TESTFILE2 - RC=$? - sync; sleep 3; sync; + echo "Write to ost1..." + $RUNAS $DD of=$TESTFILE1 count=$blk_cnt oflag=sync && + quota_error a $TSTUSR "dd succeed, expect EDQUOT" - # make qd_count 64 bit - lustre_fail ost 0 + echo "Free space from ost0..." + rm -f $TESTFILE0 + wait_delete_completed + sync_all_data || true - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) + echo "Write to ost1 after space freed from ost0..." + $RUNAS $DD of=$TESTFILE1 count=$blk_cnt oflag=sync || + quota_error a $TSTUSR "rebalancing failed" + cleanup_quota_test resetquota -u $TSTUSR - resetquota -g $TSTUSR - - return $RC } -run_test_with_stat 17 "run for fixing bug14526 ===========" +run_test 12 "Block quota rebalancing" -# test when mds takes a long time to handle a quota req so that -# the ost has dropped it, the ost still could work well b=14840 -test_18() { - LIMIT=$((100 * 1024 * 1024)) # 100G - TESTFILE="$DIR/$tdir/$tfile" - mkdir -p $DIR/$tdir +test_13(){ + local TESTFILE=$DIR/$tdir/$tfile + # the name of osp on ost1 name is MDT0000-osp-OST0000 + local procf="ldlm.namespaces.*MDT0000-osp-OST0000.lru_size" - wait_delete_completed + setup_quota_test + trap cleanup_quota_test EXIT - set_blk_tunesz 512 - set_blk_unitsz 1024 - - log " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT - quota_show_check b u $TSTUSR + set_ost_qtype "u" || "enable ost quota failed" + quota_show_check b u $TSTUSR - $LFS setstripe $TESTFILE -i 0 -c 1 + $LFS setquota -u $TSTUSR -b 0 -B 10M -i 0 -I 0 $DIR + $LFS setstripe $TESTFILE -c 1 -i 0 chown $TSTUSR.$TSTUSR $TESTFILE - #define OBD_FAIL_MDS_BLOCK_QUOTA_REQ 0x13c - lustre_fail mds 0x13c - - log " step1: write 100M block ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$((1024 * 100)) & - DDPID=$! + # clear the locks in cache first + do_facet ost1 $LCTL set_param -n $procf=clear + local nlock=$(do_facet ost1 $LCTL get_param -n $procf) + [ $nlock -eq 0 ] || error "$nlock cached locks" - sleep 5 - lustre_fail mds 0 + # write to acquire the per-ID lock + $RUNAS $DD of=$TESTFILE count=1 oflag=sync || + quota_error a $TSTUSR "dd failed" - echo " step2: testing ......" - count=0 - if at_is_enabled; then - timeout=$(at_max_get mds) - else - timeout=$(lctl get_param -n timeout) - fi - while [ true ]; do - if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi - count=$[count+1] - if [ $count -gt $((4 * $timeout)) ]; then - quota_error u $TSTUSR "count=$count dd should be finished!" - fi - sleep 1 - done - log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" - sync - cancel_lru_locks mdc - cancel_lru_locks osc - - testfile_size=$(stat -c %s $TESTFILE) - [ $testfile_size -ne $((BLK_SZ * 1024 * 100)) ] && \ - quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100)), got ${testfile_size}. Verifying file failed!" - $SHOW_QUOTA_USER - rm -f $TESTFILE - sync + nlock=$(do_facet ost1 $LCTL get_param -n $procf) + [ $nlock -eq 1 ] || error "lock count($nlock) isn't 1" + # clear quota doesn't trigger per-ID lock cancellation resetquota -u $TSTUSR - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) + nlock=$(do_facet ost1 $LCTL get_param -n $procf) + [ $nlock -eq 1 ] || error "per-ID lock is lost on quota clear" + + # clear the per-ID lock + do_facet ost1 $LCTL set_param -n $procf=clear + nlock=$(do_facet ost1 $LCTL get_param -n $procf) + [ $nlock -eq 0 ] || error "per-ID lock isn't cleared" + + # spare quota should be released + local OSTUUID=$(ostuuid_from_index 0) + local limit=$(getquota -u $TSTUSR $OSTUUID bhardlimit) + local space=$(getquota -u $TSTUSR $OSTUUID curspace) + [ $limit -le $space ] || + error "spare quota isn't released, limit:$limit, space:$space" + + cleanup_quota_test } -run_test_with_stat 18 "run for fixing bug14840 ===========" +run_test 13 "Cancel per-ID lock in the LRU list" -# test when mds drops a quota req, the ost still could work well b=14840 -test_18a() { - LIMIT=$((100 * 1024 * 1024)) # 100G - TESTFILE="$DIR/$tdir/$tfile-a" - mkdir -p $DIR/$tdir +test_15(){ + local LIMIT=$((24 * 1024 * 1024 * 1024 * 1024)) # 24 TB wait_delete_completed + sync_all_data || true - set_blk_tunesz 512 - set_blk_unitsz 1024 + # test for user + $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + local TOTAL_LIMIT=$(getquota -u $TSTUSR global bhardlimit) + [ $TOTAL_LIMIT -eq $LIMIT ] || + error "(user) limit:$TOTAL_LIMIT, expect:$LIMIT, failed!" + resetquota -u $TSTUSR - log " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT - quota_show_check b u $TSTUSR + # test for group + $LFS setquota -g $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR + TOTAL_LIMIT=$(getquota -g $TSTUSR global bhardlimit) + [ $TOTAL_LIMIT -eq $LIMIT ] || + error "(group) limits:$TOTAL_LIMIT, expect:$LIMIT, failed!" + resetquota -g $TSTUSR +} +run_test 15 "Set over 4T block quota" - $LFS setstripe $TESTFILE -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE +test_17sub() { + local err_code=$1 + local BLKS=1 # 1M less than limit + local TESTFILE=$DIR/$tdir/$tfile - #define OBD_FAIL_MDS_DROP_QUOTA_REQ | OBD_FAIL_ONCE 0x8000013d - lustre_fail mds 0x8000013d + setup_quota_test + trap cleanup_quota_test EXIT - log " step1: write 100M block ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$((1024 * 100)) & - DDPID=$! + # make sure the system is clean + local USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ne 0 ] && error "Used space($USED) for user $TSTUSR isn't 0." - echo " step2: testing ......" - count=0 - if at_is_enabled; then - timeout=$(at_max_get mds) - else - timeout=$(lctl get_param -n timeout) + set_ost_qtype "ug" || error "enable ost quota failed" + # make sure no granted quota on ost + resetquota -u $TSTUSR + $LFS setquota -u $TSTUSR -b 0 -B 10M -i 0 -I 0 $DIR + + quota_show_check b u $TSTUSR + + #define OBD_FAIL_QUOTA_RECOVERABLE_ERR 0xa04 + lustre_fail mds 0xa04 $err_code + + # write in background + $RUNAS $DD of=$TESTFILE count=$BLKS oflag=direct & + local DDPID=$! + + sleep 2 + # write should be blocked and never finished + if ! ps -p $DDPID > /dev/null 2>&1; then + lustre_fail mds 0 0 + quota_error u $TSTUSR "write finished incorrectly!" fi + + lustre_fail mds 0 0 + + local count=0 + local timeout=30 while [ true ]; do - if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi - count=$[count+1] - if [ $count -gt $((6 * $timeout)) ]; then - lustre_fail mds 0 - quota_error u $TSTUSR "count=$count dd should be finished!" - fi - sleep 1 + if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi + count=$((count+1)) + if [ $count -gt $timeout ]; then + quota_error u $TSTUSR "dd is not finished!" + fi + sleep 1 done - log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" - lustre_fail mds 0 - rm -f $TESTFILE - sync + sync; sync_all_data || true + + USED=$(getquota -u $TSTUSR global curspace) + [ $USED -ge $(($BLKS * 1024)) ] || quota_error u $TSTUSR \ + "Used space(${USED}K) is less than ${BLKS}M" + cleanup_quota_test resetquota -u $TSTUSR - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) } -run_test_with_stat 18a "run for fixing bug14840 ===========" -# test when mds do failover, the ost still could work well without trigger -# watchdog b=14840 -test_18bc_sub() { - type=$1 +# DQACQ return recoverable error +test_17() { + echo "DQACQ return -ENOLCK" + #define ENOLCK 37 + test_17sub 37 || error "Handle -ENOLCK failed" + + echo "DQACQ return -EAGAIN" + #define EAGAIN 11 + test_17sub 11 || error "Handle -EAGAIN failed" - LIMIT=$((110 * 1024 )) # 110M - TESTFILE="$DIR/$tdir/$tfile" - mkdir -p $DIR/$tdir + echo "DQACQ return -ETIMEDOUT" + #define ETIMEDOUT 110 + test_17sub 110 || error "Handle -ETIMEDOUT failed" - wait_delete_completed + echo "DQACQ return -ENOTCONN" + #define ENOTCONN 107 + test_17sub 107 || error "Handle -ENOTCONN failed" +} - set_blk_tunesz 512 - set_blk_unitsz 1024 +run_test 17 "DQACQ return recoverable error" - log " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT - quota_show_check b u $TSTUSR +test_18_sub () { + local io_type=$1 + local blimit="200m" # 200M + local TESTFILE="$DIR/$tdir/$tfile" - $LFS setstripe $TESTFILE -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE + setup_quota_test + trap cleanup_quota_test EXIT - timeout=$(sysctl -n lustre.timeout) + set_ost_qtype "u" || error "enable ost quota failed" + log "User quota (limit: $blimit)" + $LFS setquota -u $TSTUSR -b 0 -B $blimit -i 0 -I 0 $MOUNT + quota_show_check b u $TSTUSR - if [ $type = "directio" ]; then - log " write 100M block(directio) ..." - $RUNAS $DIRECTIO write $TESTFILE 0 100 $((BLK_SZ * 1024)) & + $LFS setstripe $TESTFILE -i 0 -c 1 + chown $TSTUSR.$TSTUSR $TESTFILE + + local timeout=$(sysctl -n lustre.timeout) + + if [ $io_type = "directio" ]; then + log "Write 100M (directio) ..." + $RUNAS $DD of=$TESTFILE count=100 oflag=direct & else - log " write 100M block(normal) ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$((BLK_SZ * 1024)) count=100 & + log "Write 100M (buffered) ..." + $RUNAS $DD of=$TESTFILE count=100 & fi + local DDPID=$! - DDPID=$! - do_facet $SINGLEMDS "$LCTL conf_param ${FSNAME}-MDT*.mdd.quota_type=ug" + replay_barrier $SINGLEMDS + log "Fail mds for $((2 * timeout)) seconds" + fail $SINGLEMDS $((2 * timeout)) - log "failing mds for $((2 * timeout)) seconds" - fail $SINGLEMDS $((2 * timeout)) + local count=0 + if at_is_enabled; then + timeout=$(at_max_get mds) + else + timeout=$(lctl get_param -n timeout) + fi - # check if quotaon successful - $LFS quota -u $TSTUSR $MOUNT 2>&1 | grep -q "quotas are not enabled" - if [ $? -eq 0 ]; then - error "quotaon failed!" - rm -rf $TESTFILE - return - fi + while [ true ]; do + if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi + if [ $((++count % (2 * timeout) )) -eq 0 ]; then + log "it took $count second" + fi + sleep 1 + done - count=0 - if at_is_enabled; then - timeout=$(at_max_get mds) - else - timeout=$(lctl get_param -n timeout) - fi - while [ true ]; do - if ! ps -p ${DDPID} > /dev/null 2>&1; then break; fi - if [ $((++count % (2 * timeout) )) -eq 0 ]; then - log "it took $count second" - fi - sleep 1 - done - log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" - sync - cancel_lru_locks mdc - cancel_lru_locks osc - - testfile_size=$(stat -c %s $TESTFILE) - [ $testfile_size -ne $((BLK_SZ * 1024 * 100)) ] && \ - quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100)), got ${testfile_size}. Verifying file failed!" - $SHOW_QUOTA_USER - rm -f $TESTFILE - sync + log "(dd_pid=$DDPID, time=$count, timeout=$timeout)" + sync + cancel_lru_locks mdc + cancel_lru_locks osc + $SHOW_QUOTA_USER + local testfile_size=$(stat -c %s $TESTFILE) + if [ $testfile_size -ne $((BLK_SZ * 1024 * 100)) ] ; then + quota_error u $TSTUSR "expect $((BLK_SZ * 1024 * 100))," + "got ${testfile_size}. Verifying file failed!" + fi + cleanup_quota_test resetquota -u $TSTUSR - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) } # test when mds does failover, the ost still could work well # this test shouldn't trigger watchdog b=14840 -test_18b() { - test_18bc_sub normal - test_18bc_sub directio +test_18() { + test_18_sub normal + test_18_sub directio + # check if watchdog is triggered do_facet ost1 dmesg > $TMP/lustre-log-${TESTNAME}.log - watchdog=`awk '/test 18b/ {start = 1;} + local watchdog=$(awk '/test 18b/ {start = 1;} /Service thread pid/ && /was inactive/ { if (start) { print; } - }' $TMP/lustre-log-${TESTNAME}.log` - [ `echo "$watchdog" | wc -l` -ge 3 ] && error "$watchdog" + }' $TMP/lustre-log-${TESTNAME}.log) + [ -z "$watchdog" ] || error "$watchdog" rm -f $TMP/lustre-log-${TESTNAME}.log } -run_test_with_stat 18b "run for fixing bug14840(mds failover, no watchdog) ===========" +run_test 18 "MDS failover while writing, no watchdog triggered (b14840)" -# test when mds does failover, the ost still could work well -# this test will prevent OST_DISCONNET from happening b=14840 -test_18c() { - # define OBD_FAIL_OST_DISCONNECT_NET 0x202(disable ost_disconnect for osts) - lustre_fail ost 0x202 - test_18bc_sub normal - test_18bc_sub directio - lustre_fail ost 0 -} -run_test_with_stat 18c "run for fixing bug14840(mds failover, OST_DISCONNECT is disabled) ===========" +test_19() { + local blimit=5 # 5M + local TESTFILE=$DIR/$tdir/$tfile -run_to_block_limit() { - local LIMIT=$((($OSTCOUNT + 1) * $BUNIT_SZ)) - local TESTFILE=$1 - wait_delete_completed + setup_quota_test + trap cleanup_quota_test EXIT - # set 1 Mb quota unit size - set_blk_tunesz 512 - set_blk_unitsz 1024 + set_ost_qtype "ug" || error "enable ost quota failed" # bind file to a single OST $LFS setstripe -c 1 $TESTFILE chown $TSTUSR.$TSTUSR $TESTFILE - echo " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT - quota_show_check b u $TSTUSR - echo " Updating quota limits" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $MOUNT - quota_show_check b u $TSTUSR - - RUNDD="$RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ" - $RUNDD count=$BUNIT_SZ || quota_error u $TSTUSR "(usr) write failure, but expect success" - # for now page cache of TESTFILE may still be dirty, - # let's push it to the corresponding OST, this will also - # cache NOQUOTA on the client from OST's reply - cancel_lru_locks osc - $RUNDD seek=$BUNIT_SZ && quota_error u $TSTUSR "(usr) write success, should be EDQUOT" -} - -test_19() { - # 1 Mb bunit per each MDS/OSS - local TESTFILE="$DIR/$tdir/$tfile" - mkdir -p $DIR/$tdir + echo "Set user quota (limit: "$blimit"M)" + $LFS setquota -u $TSTUSR -b 0 -B "$blimit"M -i 0 -I 0 $MOUNT + quota_show_check b u $TSTUSR + echo "Update quota limits" + $LFS setquota -u $TSTUSR -b 0 -B "$blimit"M -i 0 -I 0 $MOUNT + quota_show_check b u $TSTUSR - run_to_block_limit $TESTFILE + # first wirte might be cached + $RUNAS $DD of=$TESTFILE count=$((blimit + 1)) + cancel_lru_locks osc + $SHOW_QUOTA_USER + $RUNAS $DD of=$TESTFILE count=$((blimit + 1)) seek=$((blimit + 1)) && + quota_error u $TSTUSR "Write success, expect failure" $SHOW_QUOTA_USER - # cleanup - rm -f $TESTFILE + cleanup_quota_test resetquota -u $TSTUSR - - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) - } -run_test_with_stat 19 "test if administrative limits updates do not zero operational limits (14790) ===" +run_test 19 "Updating admin limits doesn't zero operational limits(b14790)" -test_20() -{ - LSTR=(1t 2g 3m 4k) # limits strings - LVAL=($[1*1024*1024*1024] $[2*1024*1024] $[3*1024*1024] $[4*1024]) # limits values +test_20() { # b15754 + local LSTR=(2g 1t 4k 3m) # limits strings + # limits values + local LVAL=($[2*1024*1024] $[1*1024*1024*1024] $[4*1024] $[3*1024*1024]) - $LFS setquota -u $TSTUSR --block-softlimit ${LSTR[0]} \ - $MOUNT || error "could not set quota limits" - - $LFS setquota -u $TSTUSR --block-hardlimit ${LSTR[1]} \ - --inode-softlimit ${LSTR[2]} \ - --inode-hardlimit ${LSTR[3]} \ - $MOUNT || error "could not set quota limits" + resetquota -u $TSTUSR - ($LFS quota -v -u $TSTUSR $MOUNT | \ - grep -E '^ *'$MOUNT' *[0-9]+\** *'${LVAL[0]}' *'${LVAL[1]}' *[0-9]+\** *'${LVAL[2]}' *'${LVAL[3]}) \ - || error "lfs quota output is unexpected" + $LFS setquota -u $TSTUSR --block-softlimit ${LSTR[0]} \ + $MOUNT || error "could not set quota limits" + $LFS setquota -u $TSTUSR --block-hardlimit ${LSTR[1]} \ + --inode-softlimit ${LSTR[2]} \ + --inode-hardlimit ${LSTR[3]} \ + $MOUNT || error "could not set quota limits" + + [ "$(getquota -u $TSTUSR global bsoftlimit)" = "${LVAL[0]}" ] || + error "bsoftlimit was not set properly" + [ "$(getquota -u $TSTUSR global bhardlimit)" = "${LVAL[1]}" ] || + error "bhardlimit was not set properly" + [ "$(getquota -u $TSTUSR global isoftlimit)" = "${LVAL[2]}" ] || + error "isoftlimit was not set properly" + [ "$(getquota -u $TSTUSR global ihardlimit)" = "${LVAL[3]}" ] || + error "ihardlimit was not set properly" - resetquota -u $TSTUSR + resetquota -u $TSTUSR } -run_test_with_stat 20 "test if setquota specifiers work properly (15754)" +run_test 20 "Test if setquota specifiers work properly (b15754)" test_21_sub() { local testfile=$1 local blk_number=$2 local seconds=$3 - time=$(($(date +%s) + seconds)) + local time=$(($(date +%s) + seconds)) while [ $(date +%s) -lt $time ]; do - $RUNAS dd if=/dev/zero of=$testfile bs=$BLK_SZ count=$blk_number > /dev/null 2>&1 - rm -f $testfile + $RUNAS $DD of=$testfile count=$blk_number > /dev/null 2>&1 done } # run for fixing bug16053, setquota shouldn't fail when writing and # deleting are happening test_21() { - set_blk_tunesz 512 - set_blk_unitsz 1024 - - wait_delete_completed + local TESTFILE="$DIR/$tdir/$tfile" + local BLIMIT=10 # 10G + local ILIMIT=1000000 - TESTFILE="$DIR/$tdir/$tfile" + setup_quota_test + trap cleanup_quota_test EXIT - BLK_LIMIT=$((10 * 1024 * 1024)) # 10G - FILE_LIMIT=1000000 + set_ost_qtype "ug" || error "Enable ost quota failed" - log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for user: $TSTUSR" - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $MOUNT - log " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for group: $TSTUSR" - $LFS setquota -g $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I $FILE_LIMIT $MOUNT + log "Set limit(block:${BLIMIT}G; file:$ILIMIT) for user: $TSTUSR" + $LFS setquota -u $TSTUSR -b 0 -B ${BLIMIT}G -i 0 -I $ILIMIT $MOUNT + log "Set limit(block:${BLIMIT}G; file:$ILIMIT) for group: $TSTUSR" + $LFS setquota -g $TSTUSR -b 0 -B $BLIMIT -i 0 -I $ILIMIT $MOUNT # repeat writing on a 1M file - test_21_sub ${TESTFILE}_1 1024 30 & - DDPID1=$! + test_21_sub ${TESTFILE}_1 1 30 & + local DDPID1=$! # repeat writing on a 128M file - test_21_sub ${TESTFILE}_2 $((1024 * 128)) 30 & - DDPID2=$! + test_21_sub ${TESTFILE}_2 128 30 & + local DDPID2=$! - time=$(($(date +%s) + 30)) - i=1 + local time=$(($(date +%s) + 30)) + local i=1 while [ $(date +%s) -lt $time ]; do - log " Set quota for $i times" - $LFS setquota -u $TSTUSR -b 0 -B $((BLK_LIMIT + 1024 * i)) -i 0 -I $((FILE_LIMIT + i)) $MOUNT - $LFS setquota -g $TSTUSR -b 0 -B $((BLK_LIMIT + 1024 * i)) -i 0 -I $((FILE_LIMIT + i)) $MOUNT - i=$((i+1)) - sleep 1 + log "Set quota for $i times" + $LFS setquota -u $TSTUSR -b 0 -B "$((BLIMIT + i))G" -i 0 \ + -I $((ILIMIT + i)) $MOUNT || + error "Set user quota failed" + $LFS setquota -g $TSTUSR -b 0 -B "$((BLIMIT + i))G" -i 0 \ + -I $((ILIMIT + i)) $MOUNT || + error "Set group quota failed" + i=$((i+1)) + sleep 1 done - count=0 + local count=0 while [ true ]; do - if ! ps -p ${DDPID1} > /dev/null 2>&1; then break; fi - count=$[count+1] - if [ $count -gt 60 ]; then - quota_error a $TSTUSR "dd should be finished!" - fi - sleep 1 + if ! ps -p ${DDPID1} > /dev/null 2>&1; then break; fi + count=$((count+1)) + if [ $count -gt 60 ]; then + quota_error a $TSTUSR "dd should be finished!" + fi + sleep 1 done echo "(dd_pid=$DDPID1, time=$count)successful" count=0 while [ true ]; do - if ! ps -p ${DDPID2} > /dev/null 2>&1; then break; fi - count=$[count+1] - if [ $count -gt 60 ]; then - quota_error a $TSTUSR "dd should be finished!" - fi - sleep 1 + if ! ps -p ${DDPID2} > /dev/null 2>&1; then break; fi + count=$((count+1)) + if [ $count -gt 60 ]; then + quota_error a $TSTUSR "dd should be finished!" + fi + sleep 1 done echo "(dd_pid=$DDPID2, time=$count)successful" - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) + cleanup_quota_test resetquota -u $TSTUSR resetquota -g $TSTUSR - - return $RC } -run_test_with_stat 21 "run for fixing bug16053 ===========" +run_test 21 "Setquota while writing & deleting (b16053)" +# enable/disable quota enforcement permanently test_22() { - quota_save_version "ug3" - - stopall - mount - setupall - - echo "checking parameters" - - do_facet $SINGLEMDS "lctl get_param mdd.${FSNAME}-MDT*.quota_type" | grep "ug3" || error "admin failure" - do_facet ost1 "lctl get_param obdfilter.*.quota_type" | grep "ug3" || error "op failure" - - quota_init + echo "Set both mdt & ost quota type as ug" + set_mdt_qtype "ug" || error "enable mdt quota failed" + set_ost_qtype "ug" || error "enable ost quota failed" + + echo "Restart..." + stopall + mount + setupall + + echo "Verify if quota is enabled" + local qtype=$(mdt_quota_type) + [ $qtype != "ug" ] && error "mdt quota setting is lost" + qtype=$(ost_quota_type) + [ $qtype != "ug" ] && error "ost quota setting is lost" + + echo "Set both mdt & ost quota type as none" + set_mdt_qtype "none" || error "disable mdt quota failed" + set_ost_qtype "none" || error "disable ost quota failed" + + echo "Restart..." + stopall + mount + setupall + + echo "Verify if quota is disabled" + qtype=$(mdt_quota_type) + [ $qtype != "none" ] && error "mdt quota setting is lost" + qtype=$(ost_quota_type) + [ $qtype != "none" ] && error "ost quota setting is lost" + + quota_init } -run_test_with_stat 22 "test if quota_type saved as permanent parameter ====" +run_test 22 "enable/disable quota by 'lctl conf_param'" test_23_sub() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - TESTFILE="$DIR/$tdir/$tfile-0" - rm -f $TESTFILE - local bs_unit=$((1024*1024)) - LIMIT=$1 + local TESTFILE="$DIR/$tdir/$tfile" + local LIMIT=$1 - wait_delete_completed + setup_quota_test + trap cleanup_quota_test EXIT + + set_ost_qtype "ug" || error "Enable ost quota failed" # test for user - log " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - sleep 3 - quota_show_check b u $TSTUSR + log "User quota (limit: $LIMIT MB)" + $LFS setquota -u $TSTUSR -b 0 -B "$LIMIT"M -i 0 -I 0 $DIR + quota_show_check b u $TSTUSR - $LFS setstripe $TESTFILE -c 1 + $LFS setstripe $TESTFILE -c 1 -i 0 chown $TSTUSR.$TSTUSR $TESTFILE - log " Step1: trigger quota with 0_DIRECT" - log " Write half of file" - $RUNAS $DIRECTIO write $TESTFILE 0 $(($LIMIT/1024/2)) $bs_unit || \ - quota_error u $TSTUSR "(1) write failure, but expect success: $LIMIT" - log " Write out of block quota ..." - $RUNAS $DIRECTIO write $TESTFILE $(($LIMIT/1024/2)) $(($LIMIT/1024/2)) $bs_unit && \ - quota_error u $TSTUSR "(2) write success, but expect EDQUOT: $LIMIT" - log " Step1: done" - - log " Step2: rewrite should succeed" - $RUNAS $DIRECTIO write $TESTFILE 0 1 $bs_unit || \ - quota_error u $TSTUSR "(3) write failure, but expect success: $LIMIT" - log " Step2: done" - - rm -f $TESTFILE - wait_delete_completed - OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` - OST0_QUOTA_USED=`$LFS quota -o $OST0_UUID -u $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $1 }'` - echo $OST0_QUOTA_USED - [ $OST0_QUOTA_USED -ne 0 ] && \ - ($SHOW_QUOTA_USER; quota_error u $TSTUSR "quota deleted isn't released") + log "Step1: trigger EDQUOT with O_DIRECT" + log "Write half of file" + $RUNAS $DD of=$TESTFILE count=$((LIMIT/2)) oflag=direct || + quota_error u $TSTUSR "(1) Write failure, expect success." \ + "limit=$LIMIT" + log "Write out of block quota ..." + $RUNAS $DD of=$TESTFILE count=$((LIMIT/2 + 1)) seek=$((LIMIT/2)) \ + oflag=direct conv=notrunc && + quota_error u $TSTUSR "(2) Write success, expect EDQUOT." \ + "limit=$LIMIT" + log "Step1: done" + + log "Step2: rewrite should succeed" + $RUNAS $DD of=$TESTFILE count=1 oflag=direct conv=notrunc|| + quota_error u $TSTUSR "(3) Write failure, expect success." \ + "limit=$LIMIT" + log "Step2: done" + + cleanup_quota_test + + local OST0_UUID=$(ostuuid_from_index 0) + local OST0_QUOTA_USED=$(getquota -u $TSTUSR $OST0_UUID curspace) + [ $OST0_QUOTA_USED -ne 0 ] && + ($SHOW_QUOTA_USER; \ + quota_error u $TSTUSR "quota isn't released") $SHOW_QUOTA_USER resetquota -u $TSTUSR } test_23() { - log "run for $((OSTCOUNT * 4))MB test file" - test_23_sub $((OSTCOUNT * 4 * 1024)) + local OST0_MIN=$((6 * 1024)) # 6MB, extra space for meta blocks. + check_whether_skip && return 0 + log "run for 4MB test file" + test_23_sub 4 - OST0_MIN=120000 + OST0_MIN=$((60 * 1024)) # 60MB, extra space for meta blocks. check_whether_skip && return 0 - log "run for $((OSTCOUNT * 40))MB test file" - test_23_sub $((OSTCOUNT * 40 * 1024)) + log "run for 40MB test file" + test_23_sub 40 } -run_test_with_stat 23 "run for fixing bug16125 ===========" +run_test 23 "Quota should be honored with directIO (b16125)" test_24() { + local blimit=5 # 5M local TESTFILE="$DIR/$tdir/$tfile" - mkdir -p $DIR/$tdir - run_to_block_limit $TESTFILE - $SHOW_QUOTA_USER | grep '*' || error "no matching *" + setup_quota_test + trap cleanup_quota_test EXIT - # cleanup - rm -f $TESTFILE - resetquota -u $TSTUSR + set_ost_qtype "ug" || error "enable ost quota failed" - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) - -} -run_test_with_stat 24 "test if lfs draws an asterix when limit is reached (16646) ===========" - -show_quota() { - if [ $1 = "-u" ]; then - if [ $2 = "$TSTUSR" ]; then - $SHOW_QUOTA_USER - else - $SHOW_QUOTA_USER2 - fi - else - if [ $2 = "$TSTUSR" ]; then - $SHOW_QUOTA_GROUP - else - $SHOW_QUOTA_GROUP2 - fi - fi -} + # bind file to a single OST + $LFS setstripe -c 1 $TESTFILE + chown $TSTUSR.$TSTUSR $TESTFILE -test_25_sub() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - TESTFILE="$DIR/$tdir/$tfile-0" - rm -f $TESTFILE - LIMIT=$(( $BUNIT_SZ * ($OSTCOUNT + 1) + 4096 )) + echo "Set user quota (limit: "$blimit"M)" + $LFS setquota -u $TSTUSR -b 0 -B "$blimit"M -i 0 -I 0 $MOUNT - wait_delete_completed + # overrun quota by root user + runas -u 0 -g 0 $DD of=$TESTFILE count=$((blimit + 1)) || + error "write failure, expect success" + cancel_lru_locks osc + sync_all_data || true - # set quota for $TSTUSR - log "setquota for $TSTUSR" - $LFS setquota $1 $TSTUSR -b $LIMIT -B $LIMIT -i 10 -I 10 $DIR - sleep 3 - if [ "$1" == "-u" ]; then - quota_show_check a u $TSTUSR - else - quota_show_check a g $TSTUSR - fi - - # set quota for $TSTUSR2 - log "setquota for $TSTUSR2" - $LFS setquota $1 $TSTUSR2 -b $LIMIT -B $LIMIT -i 10 -I 10 $DIR - sleep 3 - if [ "$1" == "-u" ]; then - quota_show_check a u $TSTUSR2 - else - quota_show_check a g $TSTUSR2 - fi - - # set stripe index to 0 - log "setstripe for $DIR/$tdir to 0" - $LFS setstripe $DIR/$tdir -c 1 -i 0 - MDS_UUID=`do_facet $SINGLEMDS $LCTL dl | grep -m1 " mdt " | awk '{print $((NF-1))}'` - OST0_UUID=`do_facet ost1 $LCTL dl | grep -m1 obdfilter | awk '{print $((NF-1))}'` - MDS_QUOTA_USED_OLD=`$LFS quota -o $MDS_UUID $1 $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $4 }'` - OST0_QUOTA_USED_OLD=`$LFS quota -o $OST0_UUID $1 $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $1 }'` - MDS_QUOTA_USED2_OLD=`$LFS quota -o $MDS_UUID $1 $TSTUSR2 $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $4 }'` - OST0_QUOTA_USED2_OLD=`$LFS quota -o $OST0_UUID $1 $TSTUSR2 $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $1 }'` - - # TSTUSR write 4M - log "$TSTUSR write 4M to $TESTFILE" - $RUNAS dd if=/dev/zero of=$TESTFILE bs=4K count=1K || quota_error a $TSTUSR "dd failed" - sync - show_quota $1 $TSTUSR - show_quota $1 $TSTUSR2 - MDS_QUOTA_USED_NEW=`$LFS quota -o $MDS_UUID $1 $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $4 }'` - [ $MDS_QUOTA_USED_NEW -ne $((MDS_QUOTA_USED_OLD + 1)) ] && \ - quota_error a $TSTUSR "$TSTUSR inode quota usage error: [$MDS_QUOTA_USED_OLD|$MDS_QUOTA_USED_NEW]" - OST0_QUOTA_USED_NEW=`$LFS quota -o $OST0_UUID $1 $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $1 }'` - OST0_QUOTA_USED_DELTA=$((OST0_QUOTA_USED_NEW - OST0_QUOTA_USED_OLD)) - [ $OST0_QUOTA_USED_DELTA -lt 4096 ] && \ - quota_error a $TSTUSR "$TSTUSR block quota usage error: [$OST0_QUOTA_USED_OLD|$OST0_QUOTA_USED_NEW]" - - # chown/chgrp from $TSTUSR to $TSTUSR2 - if [ $1 = "-u" ]; then - log "chown from $TSTUSR to $TSTUSR2" - chown $TSTUSR2 $TESTFILE || quota_error u $TSTUSR2 "chown failed" - else - log "chgrp from $TSTUSR to $TSTUSR2" - chgrp $TSTUSR2 $TESTFILE || quota_error g $TSTUSR2 "chgrp failed" - fi - sync - show_quota $1 $TSTUSR - show_quota $1 $TSTUSR2 - MDS_QUOTA_USED2_NEW=`$LFS quota -o $MDS_UUID $1 $TSTUSR2 $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $4 }'` - [ $MDS_QUOTA_USED2_NEW -ne $((MDS_QUOTA_USED2_OLD + 1)) ] && \ - quota_error a $TSTUSR2 "$TSTUSR2 inode quota usage transfer from $TSTUSR to $TSTUSR2 failed: [$MDS_QUOTA_USED2_OLD|$MDS_QUOTA_USED2_NEW]" - OST0_QUOTA_USED2_NEW=`$LFS quota -o $OST0_UUID $1 $TSTUSR2 $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $1 }'` - # when chown, the quota on ost could be displayed out of quota temporarily. Delete the '*' in this situation. b=20433 - OST0_QUOTA_USED2_NEW=${OST0_QUOTA_USED2_NEW%\*} - OST0_QUOTA_USED2_DELTA=$((OST0_QUOTA_USED2_NEW - OST0_QUOTA_USED2_OLD)) - [ $OST0_QUOTA_USED2_DELTA -ne $OST0_QUOTA_USED_DELTA ] && \ - quota_error a $TSTUSR2 "$TSTUSR2 block quota usage transfer from $TSTUSR to $TSTUSR2 failed: [$OST0_QUOTA_USED2_OLD|$OST0_QUOTA_USED2_NEW]" - MDS_QUOTA_USED_NEW=`$LFS quota -o $MDS_UUID $1 $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $4 }'` - [ $MDS_QUOTA_USED_NEW -ne $MDS_QUOTA_USED_OLD ] && \ - quota_error a $TSTUSR "$TSTUSR inode quota usage transfer from $TSTUSR to $TSTUSR2 failed: [$MDS_QUOTA_USED_OLD|$MDS_QUOTA_USED_NEW]" - OST0_QUOTA_USED_NEW=`$LFS quota -o $OST0_UUID $1 $TSTUSR $DIR | awk '/^.*[[:digit:]+][[:space:]+]/ { print $1 }'` - [ $OST0_QUOTA_USED_NEW -ne $OST0_QUOTA_USED_OLD ] && \ - quota_error a $TSTUSR "$TSTUSR block quota usage transfer from $TSTUSR to $TSTUSR2 failed: [$OST0_QUOTA_USED_OLD|$OST0_QUOTA_USED_NEW]" + $SHOW_QUOTA_USER | grep '*' || error "no matching *" - rm -f $TESTFILE - wait_delete_completed - resetquota $1 $TSTUSR - resetquota $1 $TSTUSR2 + cleanup_quota_test + resetquota -u $TSTUSR } +run_test 24 "lfs draws an asterix when limit is reached (b16646)" -test_25() { - log "run for chown case" - test_25_sub -u - - log "run for chgrp case" - test_25_sub -g +test_27a() { # b19612 + $LFS quota $TSTUSR $DIR && + error "lfs succeeded with no type, but should have failed" + $LFS setquota $TSTUSR $DIR && + error "lfs succeeded with no type, but should have failed" + return 0 +} +run_test 27a "lfs quota/setquota should handle wrong arguments (b19612)" + +test_27b() { # b20200 + $LFS setquota -u $TSTID -b 1000 -B 1000 -i 1000 -I 1000 $DIR || + error "lfs setquota failed with uid argument" + $LFS setquota -g $TSTID -b 1000 -B 1000 -i 1000 -I 1000 $DIR || + error "lfs stequota failed with gid argument" + $SHOW_QUOTA_USERID || error "lfs quota failed with uid argument" + $SHOW_QUOTA_GROUPID || error "lfs quota failed with gid argument" + resetquota -u $TSTUSR + resetquota -g $TSTUSR + return 0 } -run_test_with_stat 25 "test whether quota usage is transfered when chown/chgrp (18081) ===========" +run_test 27b "lfs quota/setquota should handle user/group ID (b20200)" -test_26() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - TESTFILE="$DIR/$tdir/$tfile-0" - TESTFILE2="$DIR/$tdir/$tfile-1" - set_blk_tunesz 512 - set_blk_unitsz 1024 +test_30() { + local output + local LIMIT=4 # 4MB + local TESTFILE="$DIR/$tdir/$tfile" + local GRACE=10 - wait_delete_completed + setup_quota_test + trap cleanup_quota_test EXIT - # every quota slave gets 20MB - b_limit=$(((OSTCOUNT + 1) * 20 * 1024)) - log "limit: ${b_limit}KB" - $LFS setquota -u $TSTUSR -b 0 -B $b_limit -i 0 -I 0 $DIR - sleep 3 - quota_show_check b u $TSTUSR + set_ost_qtype "u" || error "enable ost quota failed" - $LFS setstripe $TESTFILE -c 1 -i 0 - $LFS setstripe $TESTFILE2 -c 1 -i 0 + $LFS setstripe $TESTFILE -i 0 -c 1 chown $TSTUSR.$TSTUSR $TESTFILE - chown $TSTUSR.$TSTUSR $TESTFILE2 - - #define OBD_FAIL_QUOTA_DELAY_REL 0xA03 - lustre_fail ost 0xA03 - - log " Write the first file..." - $RUNAS $DIRECTIO write $TESTFILE 0 10 $((BLK_SZ * 1024)) || quota_error u $TSTUSR "write failure, but expect success" - log " Delete the first file..." - rm -f $TESTFILE - - wait_delete_completed - - log " Write the second file..." - $RUNAS $DIRECTIO write $TESTFILE2 0 10 $((BLK_SZ * 1024)) || quota_error u $TSTUSR "write failure, but expect success" - log " Delete the second file..." - rm -f $TESTFILE2 - - lustre_fail ost 0 - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) + $LFS setquota -t -u --block-grace $GRACE --inode-grace $MAX_IQ_TIME $DIR + $LFS setquota -u $TSTUSR -b ${LIMIT}M -B 0 -i 0 -I 0 $DIR + $RUNAS $DD of=$TESTFILE count=$((LIMIT * 2)) || true + cancel_lru_locks osc + sleep $GRACE + $LFS setquota -u $TSTUSR -B 0 $DIR + # over-quota flag has not yet settled since we do not trigger async + # events based on grace time period expiration + $SHOW_QUOTA_USER + $RUNAS $DD of=$TESTFILE conv=notrunc oflag=append count=1 || true + cancel_lru_locks osc + # now over-quota flag should be settled and further writes should fail + $SHOW_QUOTA_USER + $RUNAS $DD of=$TESTFILE conv=notrunc oflag=append count=1 && + error "grace times were reset" + # cleanup + cleanup_quota_test resetquota -u $TSTUSR + $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace \ + $MAX_IQ_TIME $DIR } -run_test_with_stat 26 "test for false quota error(bz18491) ======================================" - -test_27a() { - $LFS quota $TSTUSR $DIR && error "lfs succeeded with no type, but should have failed" - $LFS setquota $TSTUSR $DIR && error "lfs succeeded with no type, but should have failed" - return 0 -} -run_test_with_stat 27a "lfs quota/setquota should handle wrong arguments (19612) =================" - -test_27b() { - $LFS setquota -u $TSTID -b 1000 -B 1000 -i 1000 -I 1000 $DIR || \ - error "lfs setquota failed with uid argument" - $LFS setquota -g $TSTID -b 1000 -B 1000 -i 1000 -I 1000 $DIR || \ - error "lfs stequota failed with gid argument" - $SHOW_QUOTA_USERID || error "lfs quota failed with uid argument" - $SHOW_QUOTA_GROUPID || error "lfs quota failed with gid argument" - resetquota -u $TSTUSR - resetquota -g $TSTUSR - return 0 +run_test 30 "Hard limit updates should not reset grace times" + +# basic usage tracking for user & group +test_33() { + local INODES=10 # 10 files + local BLK_CNT=2 # of 2M each + local TOTAL_BLKS=$((INODES * BLK_CNT * 1024)) + + setup_quota_test + trap cleanup_quota_test EXIT + + # make sure the system is clean + local USED=$(getquota -u $TSTID global curspace) + [ $USED -ne 0 ] && + error "Used space ($USED) for user $TSTID isn't 0." + USED=$(getquota -g $TSTID global curspace) + [ $USED -ne 0 ] && + error "Used space ($USED) for group $TSTID isn't 0." + + echo "Write files..." + for i in `seq 0 $INODES`; do + $RUNAS $DD of=$DIR/$tdir/$tfile-$i count=$BLK_CNT 2>/dev/null || + error "write failed" + echo "Iteration $i/$INODES completed" + done + cancel_lru_locks osc + sync; sync_all_data || true + + echo "Verify disk usage after write" + USED=$(getquota -u $TSTID global curspace) + [ $USED -lt $TOTAL_BLKS ] && + error "Used space for user $TSTID:$USED, expected:$TOTAL_BLKS" + USED=$(getquota -g $TSTID global curspace) + [ $USED -lt $TOTAL_BLKS ] && + error "Used space for group $TSTID:$USED, expected:$TOTAL_BLKS" + + echo "Verify inode usage after write" + USED=$(getquota -u $TSTID global curinodes) + [ $USED -lt $INODES ] && + error "Used inode for user $TSTID is $USED, expected $INODES" + USED=$(getquota -g $TSTID global curinodes) + [ $USED -lt $INODES ] && + error "Used inode for group $TSTID is $USED, expected $INODES" + + cleanup_quota_test + + echo "Verify disk usage after delete" + USED=$(getquota -u $TSTID global curspace) + [ $USED -eq 0 ] || error "Used space for user $TSTID isn't 0. $USED" + USED=$(getquota -u $TSTID global curinodes) + [ $USED -eq 0 ] || error "Used inodes for user $TSTID isn't 0. $USED" + USED=$(getquota -g $TSTID global curspace) + [ $USED -eq 0 ] || error "Used space for group $TSTID isn't 0. $USED" + USED=$(getquota -g $TSTID global curinodes) + [ $USED -eq 0 ] || error "Used inodes for group $TSTID isn't 0. $USED" } -run_test 27b "lfs quota/setquota should handle user/group ID (20200) =================" - -test_28() { - BLK_LIMIT=$((100 * 1024 * 1024)) # 100G - echo "Step 1: set enough high limit for user [$TSTUSR:$BLK_LIMIT]" - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR - $SHOW_QUOTA_USER +run_test 33 "Basic usage tracking for user & group" - echo "Step 2: reset system ..." - cleanup_and_setup_lustre - quota_init +# usage transfer test for user & group +test_34() { + local BLK_CNT=2 # 2MB - echo "Step 3: change qunit for user [$TSTUSR:512:1024]" - set_blk_tunesz 512 - set_blk_unitsz 1024 + setup_quota_test + trap cleanup_quota_test EXIT - wait_delete_completed + # make sure the system is clean + local USED=$(getquota -u $TSTID global curspace) + [ $USED -ne 0 ] && error "Used space ($USED) for user $TSTID isn't 0." + USED=$(getquota -g $TSTID global curspace) + [ $USED -ne 0 ] && error "Used space ($USED) for group $TSTID isn't 0." - #define OBD_FAIL_QUOTA_RET_QDATA | OBD_FAIL_ONCE - lustre_fail ost 0x80000A02 + echo "Write file..." + $DD of=$DIR/$tdir/$tfile count=$BLK_CNT 2>/dev/null || + error "write failed" + cancel_lru_locks osc + sync; sync_all_data || true - TESTFILE="$DIR/$tdir/$tfile" - mkdir -p $DIR/$tdir + echo "chown the file to user $TSTID" + chown $TSTID $DIR/$tdir/$tfile || error "chown failed" - BLK_LIMIT=$((100 * 1024)) # 100M - echo "Step 4: set enough high limit for user [$TSTUSR:$BLK_LIMIT]" - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR - $SHOW_QUOTA_USER + echo "Wait for setattr on objects finished..." + wait_delete_completed - touch $TESTFILE - chown $TSTUSR.$TSTUSR $TESTFILE + BLK_CNT=$((BLK_CNT * 1024)) - echo "Step 5: write the test file1 [10M] ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(( 10 * 1024 )) \ - || quota_error a $TSTUSR "write 10M file failure" - $SHOW_QUOTA_USER + echo "Verify disk usage for user $TSTID" + USED=$(getquota -u $TSTID global curspace) + [ $USED -lt $BLK_CNT ] && + error "Used space for user $TSTID is ${USED}, expected $BLK_CNT" + USED=$(getquota -u $TSTID global curinodes) + [ $USED -ne 1 ] && + error "Used inodes for user $TSTID is $USED, expected 1" - rm -f $TESTFILE - sync; sleep 3; sync; + echo "chgrp the file to group $TSTID" + chgrp $TSTID $DIR/$tdir/$tfile || error "chgrp failed" - # make qd_count 64 bit - lustre_fail ost 0 + echo "Wait for setattr on objects finished..." + wait_delete_completed - set_blk_unitsz $((128 * 1024)) - set_blk_tunesz $((128 * 1024 / 2)) + echo "Verify disk usage for group $TSTID" + USED=$(getquota -g $TSTID global curspace) + [ $USED -ge $BLK_CNT ] || + error "Used space for group $TSTID is $USED, expected $BLK_CNT" + USED=$(getquota -g $TSTID global curinodes) + [ $USED -eq 1 ] || + error "Used inodes for group $TSTID is $USED, expected 1" - resetquota -u $TSTUSR + cleanup_quota_test } -run_test_with_stat 28 "test for consistency for qunit when setquota (18574) ===========" +run_test 34 "Usage transfer for user & group" -test_29() -{ - local BLK_LIMIT=$((100 * 1024 * 1024)) # 100G - local timeout - local pid - local origin_resends - - if at_is_enabled; then - timeout=$(at_max_get client) - at_max_set 10 client - else - timeout=$(lctl get_param -n timeout) - lctl set_param timeout=10 - fi - - origin_resends=$(lctl get_param -n mdc.${FSNAME}-*.quota_resend_count | head -1) - lctl set_param -n mdc.${FSNAME}-*.quota_resend_count 0 - - #define OBD_FAIL_MDS_QUOTACTL_NET 0x12e - lustre_fail mds 0x12e - - $LFS setquota -u $TSTUSR -b 0 -B $BLK_LIMIT -i 0 -I 0 $DIR & pid=$! - - echo "sleeping for $((10 * 2)) seconds" - sleep $((10 * 2)) - ps -p $pid && error "lfs hadn't finished by timeout" - wait $pid && error "succeeded, but should have failed" - - lustre_fail mds 0 - - if at_is_enabled; then - at_max_set $timeout client - else - lctl set_param timeout=$timeout - fi - - lctl set_param -n mdc.${FSNAME}-*.quota_resend_count $origin_resends - resetquota -u $TSTUSR -} -run_test_with_stat 29 "unhandled quotactls must not hang lustre client (19778) ========" +# usage is still accessible across restart +test_35() { + local BLK_CNT=2 # 2 MB -test_30() -{ - local output - local LIMIT=1024 - local TESTFILE="$DIR/$tdir/$tfile" - local GRACE=10 - - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - - $LFS setquota -t -u --block-grace $GRACE --inode-grace $MAX_IQ_TIME $DIR - $LFS setquota -u $TSTUSR -b $LIMIT -B 0 -i 0 -I 0 $DIR - $RUNAS dd if=/dev/zero of=$TESTFILE bs=1024 count=$((LIMIT * 2)) || true - cancel_lru_locks osc - sleep 5 - $LFS setquota -u $TSTUSR -B 0 $DIR - $SHOW_QUOTA_USER - output=`$SHOW_QUOTA_USER | grep $MOUNT | awk '{ print $5 }' | tr -d s` - [ "$output" -le "$((GRACE - 5))" ] || error "grace times were reset or unexpectedly high latency" - rm -f $TESTFILE - resetquota -u $TSTUSR - $LFS setquota -t -u --block-grace $MAX_DQ_TIME --inode-grace $MAX_IQ_TIME $DIR + setup_quota_test + trap cleanup_quota_test EXIT + + echo "Write file..." + $RUNAS $DD of=$DIR/$tdir/$tfile count=$BLK_CNT 2>/dev/null || + error "write failed" + cancel_lru_locks osc + sync; sync_all_data || true + + echo "Save disk usage before restart" + local ORIG_USR_SPACE=$(getquota -u $TSTID global curspace) + [ $ORIG_USR_SPACE -eq 0 ] && + error "Used space for user $TSTID is 0, expected ${BLK_CNT}M" + local ORIG_USR_INODES=$(getquota -u $TSTID global curinodes) + [ $ORIG_USR_INODES -eq 0 ] && + error "Used inodes for user $TSTID is 0, expected 1" + echo "User $TSTID: ${ORIG_USR_SPACE}KB $ORIG_USR_INODES inodes" + local ORIG_GRP_SPACE=$(getquota -g $TSTID global curspace) + [ $ORIG_GRP_SPACE -eq 0 ] && + error "Used space for group $TSTID is 0, expected ${BLK_CNT}M" + local ORIG_GRP_INODES=$(getquota -g $TSTID global curinodes) + [ $ORIG_GRP_INODES -eq 0 ] && + error "Used inodes for group $TSTID is 0, expected 1" + echo "Group $TSTID: ${ORIG_GRP_SPACE}KB $ORIG_GRP_INODES inodes" + + log "Restart..." + local ORIG_REFORMAT=$REFORMAT + REFORMAT="" + cleanup_and_setup_lustre + REFORMAT=$ORIG_REFORMAT + quota_init + + echo "Verify disk usage after restart" + local USED=$(getquota -u $TSTID global curspace) + [ $USED -eq $ORIG_USR_SPACE ] || + error "Used space for user $TSTID changed from " \ + "$ORIG_USR_SPACE to $USED" + USED=$(getquota -u $TSTID global curinodes) + [ $USED -eq $ORIG_USR_INODES ] || + error "Used inodes for user $TSTID changed from " \ + "$ORIG_USR_INODES to $USED" + USED=$(getquota -g $TSTID global curspace) + [ $USED -eq $ORIG_GRP_SPACE ] || + error "Used space for group $TSTID changed from " \ + "$ORIG_GRP_SPACE to $USED" + USED=$(getquota -g $TSTID global curinodes) + [ $USED -eq $ORIG_GRP_INODES ] || + error "Used inodes for group $TSTID changed from " \ + "$ORIG_GRP_INODES to $USED" + + # check if the vfs_dq_init() is called before writing + echo "Append to the same file..." + $RUNAS $DD of=$DIR/$tdir/$tfile count=$BLK_CNT seek=1 2>/dev/null || + error "write failed" + cancel_lru_locks osc + sync; sync_all_data || true + + echo "Verify space usage is increased" + USED=$(getquota -u $TSTID global curspace) + [ $USED -gt $ORIG_USR_SPACE ] || + error "Used space for user $TSTID isn't increased" \ + "orig:$ORIG_USR_SPACE, now:$USED" + USED=$(getquota -g $TSTID global curspace) + [ $USED -gt $ORIG_GRP_SPACE ] || + error "Used space for group $TSTID isn't increased" \ + "orig:$ORIG_GRP_SPACE, now:$USED" + + cleanup_quota_test } -run_test_with_stat 30 "hard limit updates should not reset grace times ================" - -# test duplicate quota releases b=18630 -test_31() { - mkdir -p $DIR/$tdir - chmod 0777 $DIR/$tdir - - LIMIT=$(( $BUNIT_SZ * $(($OSTCOUNT + 1)) * 10)) # 10 bunits each sever - TESTFILE="$DIR/$tdir/$tfile-0" - TESTFILE2="$DIR/$tdir/$tfile-1" - - wait_delete_completed - - log " User quota (limit: $LIMIT kbytes)" - $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR - - $LFS setstripe $TESTFILE -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE - $LFS setstripe $TESTFILE2 -i 0 -c 1 - chown $TSTUSR.$TSTUSR $TESTFILE2 - - log " step1: write out of block quota ..." - $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=5120 - $RUNAS dd if=/dev/zero of=$TESTFILE2 bs=$BLK_SZ count=5120 - - #define OBD_FAIL_QUOTA_DELAY_SD 0xA04 - #define OBD_FAIL_SOME 0x10000000 /* fail N times */ - lustre_fail ost $((0x00000A04 | 0x10000000)) 1 - - log " step2: delete two files so that triggering duplicate quota release ..." - rm -f $TESTFILE $TESTFILE2 - sync; sleep 5; sync # OBD_FAIL_QUOTA_DELAY_SD will delay for 5 seconds - wait_delete_completed - - log " step3: verify if the ost failed" - do_facet ost1 dmesg > $TMP/lustre-log-${TESTNAME}.log - watchdog=`awk '/test 31/ {start = 1;} - /release quota error/ { - if (start) { - print; - } - }' $TMP/lustre-log-${TESTNAME}.log` - [ "$watchdog" ] && error "$watchdog" - rm -f $TMP/lustre-log-${TESTNAME}.log - - lustre_fail ost 0 - resetquota -u $TSTUSR +run_test 35 "Usage is still accessible across reboot" + +# test migrating old amdin quota files (in Linux quota file format v2) into new +# quota global index (in IAM format) +test_36() { + [ $(facet_fstype $SINGLEMDS) != ldiskfs ] && \ + skip "skipping migration test" && return + + # get the mdt0 device name + local mdt0_node=$(facet_active_host $SINGLEMDS) + local mdt0_dev=$(mdsdevname ${SINGLEMDS//mds/}) + + echo "Reformat..." + formatall + + echo "Copy admin quota files into MDT0..." + local mntpt=$(facet_mntpt $SINGLEMDS) + local mdt0_fstype=$(facet_fstype $SINGLEMDS) + local opt + if ! do_node $mdt0_node test -b $mdt0_fstype; then + opt="-o loop" + fi + echo "$mdt0_node, $mdt0_dev, $mntpt, $opt" + do_node $mdt0_node mount -t $mdt0_fstype $opt $mdt0_dev $mntpt + do_node $mdt0_node mkdir $mntpt/OBJECTS + do_node $mdt0_node cp $LUSTRE/tests/admin_quotafile_v2.usr $mntpt/OBJECTS + do_node $mdt0_node cp $LUSTRE/tests/admin_quotafile_v2.grp $mntpt/OBJECTS + do_node $mdt0_node umount -f $mntpt + + echo "Setup all..." + setupall + + echo "Verify global limits..." + local id_cnt + local limit + + local proc="qmt.*.md-0x0.glb-usr" + id_cnt=$(do_node $mdt0_node $LCTL get_param -n $proc | wc -l) + [ $id_cnt -eq 401 ] || error "Migrate inode user limit failed: $id_cnt" + limit=$(getquota -u 1 global isoftlimit) + [ $limit -eq 1024 ] || error "User inode softlimit: $limit" + limit=$(getquota -u 1 global ihardlimit) + [ $limit -eq 2048 ] || error "User inode hardlimit: $limit" + + proc="qmt.*.md-0x0.glb-grp" + id_cnt=$(do_node $mdt0_node $LCTL get_param -n $proc | wc -l) + [ $id_cnt -eq 401 ] || error "Migrate inode group limit failed: $id_cnt" + limit=$(getquota -g 1 global isoftlimit) + [ $limit -eq 1024 ] || error "Group inode softlimit: $limit" + limit=$(getquota -g 1 global ihardlimit) + [ $limit -eq 2048 ] || error "Group inode hardlimit: $limit" + + proc=" qmt.*.dt-0x0.glb-usr" + id_cnt=$(do_node $mdt0_node $LCTL get_param -n $proc | wc -l) + [ $id_cnt -eq 401 ] || error "Migrate block user limit failed: $id_cnt" + limit=$(getquota -u 60001 global bsoftlimit) + [ $limit -eq 10485760 ] || error "User block softlimit: $limit" + limit=$(getquota -u 60001 global bhardlimit) + [ $limit -eq 20971520 ] || error "User block hardlimit: $limit" + + proc="qmt.*.dt-0x0.glb-grp" + id_cnt=$(do_node $mdt0_node $LCTL get_param -n $proc | wc -l) + [ $id_cnt -eq 401 ] || error "Migrate block user limit failed: $id_cnt" + limit=$(getquota -g 60001 global bsoftlimit) + [ $limit -eq 10485760 ] || error "Group block softlimit: $limit" + limit=$(getquota -g 60001 global bhardlimit) + [ $limit -eq 20971520 ] || error "Group block hardlimit: $limit" + + echo "Cleanup..." + formatall + setupall } -run_test_with_stat 31 "test duplicate quota releases ===" +run_test 36 "Migrate old admin files into new global indexes" -# turn off quota quota_fini() { - $LFS quotaoff $DIR do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=-quota" } quota_fini -log "cleanup: ======================================================" cd $ORIG_PWD +complete $(basename $0) $SECONDS check_and_cleanup_lustre -echo '=========================== finished ===============================' export QUOTA_AUTO=$QUOTA_AUTO_OLD -[ -f "$QUOTALOG" ] && cat $QUOTALOG && grep -q FAIL $QUOTALOG && exit 1 || true -echo "$0: completed" +exit_status