+test_65() {
+ local SIZE=10 #10M
+ local TESTFILE="$DIR/$tdir/$tfile-0"
+
+ setup_quota_test || error "setup quota failed with $?"
+ set_ost_qtype $QTYPE || error "enable ost quota failed"
+ quota_init
+
+ echo "Write..."
+ $RUNAS $DD of=$TESTFILE count=$SIZE ||
+ error "failed to write"
+ # flush cache, ensure noquota flag is set on client
+ cancel_lru_locks osc
+ sync; sync_all_data || true
+
+ local quota_u=$($LFS quota -u $TSTUSR $DIR)
+ local quota_g=$($LFS quota -g $TSTUSR $DIR)
+ local quota_all=$($RUNAS $LFS quota $DIR)
+
+ [ "$(echo "$quota_all" | head -n3)" != "$quota_u" ] &&
+ error "usr quota not match"
+ [ "$(echo "$quota_all" | tail -n3)" != "$quota_g" ] &&
+ error "grp quota not match"
+
+ rm -f $TESTFILE
+ # cleanup
+ cleanup_quota_test
+}
+run_test 65 "Check lfs quota result"
+
+test_66() {
+ ! is_project_quota_supported &&
+ skip "Project quota is not supported"
+ [ "$MDS1_VERSION" -lt $(version_code 2.12.4) ] &&
+ skip "Not supported before 2.12.4"
+ setup_quota_test || error "setup quota failed with $?"
+ stack_trap cleanup_quota_test EXIT
+ local old=$(do_facet mds1 $LCTL get_param -n \
+ mdt.*.enable_chprojid_gid | head -1)
+ local testdir=$DIR/$tdir/foo
+
+ do_facet mds1 $LCTL set_param mdt.*.enable_chprojid_gid=0
+ stack_trap "do_facet mds1 $LCTL set_param mdt.*.enable_chprojid_gid=0" \
+ EXIT
+
+ test_mkdir -i 0 -c 1 $testdir || error "failed to mkdir"
+ chown -R $TSTID:$TSTID $testdir
+ change_project -sp $TSTPRJID $testdir
+ $RUNAS mkdir $testdir/foo || error "failed to mkdir foo"
+
+ $RUNAS lfs project -p 0 $testdir/foo &&
+ error "nonroot user should fail to set projid"
+
+ $RUNAS lfs project -C $testdir/foo &&
+ error "nonroot user should fail to clear projid"
+
+ change_project -C $testdir/foo || error "failed to clear project"
+
+ do_facet mds1 $LCTL set_param mdt.*.enable_chprojid_gid=-1
+ $RUNAS lfs project -p $TSTPRJID $testdir/foo || error \
+ "failed to set projid with normal user when enable_chprojid_gid=-1"
+
+ $RUNAS lfs project -rC $testdir/ || error \
+"failed to clear project state with normal user when enable_chprojid_gid=-1"
+
+ touch $testdir/bar || error "failed touch $testdir/bar"
+ $RUNAS lfs project -p $TSTPRJID $testdir/bar && error \
+ "normal user should not be able to set projid on root owned file"
+
+ change_project -p $TSTPRJID $testdir/bar || error \
+ "root should be able to change its own file's projid"
+
+ cleanup_quota_test
+}
+run_test 66 "nonroot user can not change project state in default"
+
+test_67_write() {
+ local file="$1"
+ local qtype="$2"
+ local size=$3
+ local _runas=""
+ local short_qtype=${qtype:0:1}
+
+ echo "file "$file
+ echo "0 $0 1 $1 2 $2 3 $3 4 $4"
+ case "$4" in
+ quota_usr) _runas=$RUNAS;;
+ quota_2usr) _runas=$RUNAS2;;
+ *) error "unknown quota parameter $4";;
+ esac
+
+ log "Write..."
+ date
+ $_runas $DD of=$file count=$size ||
+ quota_error $short_qtype $TSTUSR \
+ "$qtype write failure, but expect success"
+ date
+ cancel_lru_locks osc
+ date
+ sync; sync_all_data || true
+ date
+}
+
+getgranted() {
+ local pool=$1
+ local ptype=$2
+ local userid=$3
+ local qtype=$4
+ local param=qmt.$FSNAME-QMT0000.$ptype-$pool.glb-$qtype
+
+ do_facet mds1 $LCTL get_param $param |
+ grep -A2 $userid | awk -F'[, ]*' 'NR==2{print $9}'
+}
+
+test_67() {
+ local limit=20 # 20M
+ local testfile="$DIR/$tdir/$tfile-0"
+ local testfile2="$DIR/$tdir/$tfile-1"
+ local testfile3="$DIR/$tdir/$tfile-2"
+ local qpool="qpool1"
+ local used
+ local granted
+ local granted_mb
+
+ mds_supports_qp
+ [ "$ost1_FSTYPE" == zfs ] &&
+ skip "ZFS grants some block space together with inode"
+
+ setup_quota_test || error "setup quota failed with $?"
+ trap cleanup_quota_test EXIT
+
+ # enable ost quota
+ set_ost_qtype $QTYPE || error "enable ost quota failed"
+
+ # test for user
+ log "User quota (block hardlimit:$limit MB)"
+ $LFS setquota -u $TSTUSR -b 0 -B ${limit}M -i 0 -I 0 $DIR ||
+ error "set user quota failed"
+
+ # make sure the system is clean
+ used=$(getquota -u $TSTUSR global curspace)
+ [ $used -ne 0 ] && error "Used space($used) for user $TSTUSR isn't 0."
+
+ granted=$(getgranted "0x0" "dt" $TSTID "usr")
+ echo "granted 0x0 before write $granted"
+
+ # trigger reintegration
+ local procf="osd-$(facet_fstype ost1).$FSNAME-OST*."
+ procf=${procf}quota_slave.force_reint
+ do_facet ost1 $LCTL set_param $procf=1 ||
+ error "force reintegration failed"
+ wait_ost_reint "u" || error "reintegration failed"
+ granted=$(getgranted "0x0" "dt" $TSTID "usr")
+ [ $granted -ne 0 ] &&
+ error "Granted($granted) for $TSTUSR in $qpool isn't 0."
+
+ $LFS setstripe $testfile -c 1 -i 0 || error "setstripe $testfile failed"
+ chown $TSTUSR.$TSTUSR $testfile || error "chown $testfile failed"
+
+ # write 10 MB to testfile
+ test_67_write "$testfile" "user" 10 "quota_usr"
+
+ # create qpool and add OST1
+ pool_add $qpool || error "pool_add failed"
+ pool_add_targets $qpool 1 1 || error "pool_add_targets failed"
+ # as quota_usr hasn't limits, lqe may absent. But it should be
+ # created after the 1st direct qmt_get.
+ used=$(getquota -u $TSTUSR global bhardlimit $qpool)
+
+ # check granted - should be 0, as testfile is located only on OST0
+ granted=$(getgranted "0x0" "dt" $TSTID "usr")
+ echo "global granted $granted"
+ granted=$(getgranted $qpool "dt" $TSTID "usr")
+ echo "$qpool granted $granted"
+ [ $granted -ne 0 ] &&
+ error "Granted($granted) for $TSTUSR in $qpool isn't 0."
+
+ # add OST0 to qpool and check granted space
+ pool_add_targets $qpool 0 1 ||
+ error "pool_add_targets failed"
+ granted_mb=$(($(getgranted $qpool "dt" $TSTID "usr")/1024))
+ echo "Granted $granted_mb MB"
+ #should be 10M + qunit for each OST
+ [ $granted_mb -ge 10 -a $granted_mb -lt $limit ] ||
+ error "Granted($granted_mb) for $TSTUSR in $qpool is wrong."
+
+ $LFS setstripe $testfile2 -c 1 -i 1 ||
+ error "setstripe $testfile2 failed"
+ chown $TSTUSR2.$TSTUSR2 $testfile2 || error "chown $testfile2 failed"
+ # Write from another user and check that qpool1
+ # shows correct granted, despite quota_2usr hasn't limits in qpool1.
+ test_67_write "$testfile2" "user" 10 "quota_2usr"
+ used=$(getquota -u $TSTUSR2 global curspace $qpool)
+ granted=$(getgranted $qpool "dt" $TSTID2 "usr")
+ [ $granted -ne 0 ] &&
+ error "Granted($granted) for $TSTUSR2 in $qpool isn't 0."
+
+ # Granted space for quota_2usr in qpool1 should appear only
+ # when global lqe for this user becomes enforced.
+ $LFS setquota -u $TSTUSR2 -B ${limit}M $DIR ||
+ error "set user quota failed"
+ granted_mb=$(($(getgranted $qpool "dt" $TSTID2 "usr")/1024))
+ echo "granted_mb $granted_mb"
+ [ $granted_mb -ge 10 -a $granted_mb -lt $limit ] ||
+ error "Granted($granted) for $TSTUSR in $qpool is wrong."
+
+ $LFS setstripe $testfile3 -c 1 -i 0 ||
+ error "setstripe $testfile3 failed"
+ chown $TSTUSR2.$TSTUSR2 $testfile3 || error "chown $testfile3 failed"
+ test_67_write "$testfile3" "user" 10 "quota_2usr"
+ granted_mb=$(($(getgranted $qpool "dt" $TSTID2 "usr")/1024))
+ echo "$testfile3 granted_mb $granted_mb"
+ [ $granted_mb -eq $limit ] ||
+ error "Granted($granted_mb) for $TSTUSR2 is not equal to 20M"
+
+ # remove OST1 from the qpool1 and check granted space
+ # should be 0 for TSTUSR and 10M for TSTUSR2
+ pool_remove_target $qpool 0
+ granted_mb=$(($(getgranted $qpool "dt" $TSTID "usr")/1024))
+ [ $granted_mb -eq 0 ] ||
+ error "Granted($granted_mb) for $TSTUSR in $qpool != 0."
+ granted_mb=$(($(getgranted $qpool "dt" $TSTID2 "usr")/1024))
+ [ $granted_mb -eq 10 ] ||
+ error "Granted($granted_mb) for $TSTUSR2 is not equal to 10M"
+
+ rm -f $testfile
+ wait_delete_completed || error "wait_delete_completed failed"
+ sync_all_data || true
+ used=$(getquota -u $TSTUSR global curspace)
+ [ $used -ne 0 ] && quota_error u $TSTUSR \
+ "user quota isn't released after deletion"
+ resetquota -u $TSTUSR
+
+ cleanup_quota_test
+}
+run_test 67 "quota pools recalculation"
+
+get_slave_nr() {
+ local pool=$1
+ local qtype=$2
+ local nr
+
+ do_facet mds1 $LCTL get_param -n qmt.$FSNAME-QMT0000.dt-$pool.info |
+ awk '/usr/ {getline; print $2}'
+}
+
+test_68()
+{
+ local qpool="qpool1"
+
+ mds_supports_qp
+ setup_quota_test || error "setup quota failed with $?"
+ stack_trap cleanup_quota_test EXIT
+
+ # enable ost quota
+ set_ost_qtype $QTYPE || error "enable ost quota failed"
+
+ # check slave number for glbal pool
+ local nr=$(get_slave_nr "0x0" "usr")
+ echo "nr result $nr"
+ [[ $nr != $((OSTCOUNT + MDSCOUNT)) ]] &&
+ error "Slave_nr $nr for global pool != ($OSTCOUNT + $MDSCOUNT)"
+
+ # create qpool and add OST1
+ pool_add $qpool || error "pool_add failed"
+ nr=$(get_slave_nr $qpool "usr")
+ [[ $nr != 0 ]] && error "Slave number $nr for $qpool != 0"
+
+ # add OST1 to qpool
+ pool_add_targets $qpool 1 1 || error "pool_add_targets failed"
+ nr=$(get_slave_nr $qpool "usr")
+ [[ $nr != 1 ]] && error "Slave number $nr for $qpool != 1"
+
+ # add OST0 to qpool
+ pool_add_targets $qpool 0 1 || error "pool_add_targets failed"
+ nr=$(get_slave_nr $qpool "usr")
+ [[ $nr != 2 ]] && error "Slave number $nr for $qpool != 2"
+
+ # remove OST0
+ pool_remove_target $qpool 0
+ nr=$(get_slave_nr $qpool "usr")
+ [[ $nr != 1 ]] && error "Slave number $nr for $qpool != 1"
+
+ # remove OST1
+ pool_remove_target $qpool 1
+ nr=$(get_slave_nr $qpool "usr")
+ [[ $nr != 0 ]] && error "Slave number $nr for $qpool != 0"
+
+ # Check again that all is fine with global pool
+ nr=$(get_slave_nr "0x0" "usr")
+ [[ $nr != $((OSTCOUNT + MDSCOUNT)) ]] &&
+ error "Slave_nr $nr for global pool != ($OSTCOUNT + $MDSCOUNT)"
+
+ cleanup_quota_test
+}
+run_test 68 "slave number in quota pool changed after each add/remove OST"
+
+test_69()
+{
+ local global_limit=200 # 200M
+ local limit=10 # 10M
+ local testfile="$DIR/$tdir/$tfile-0"
+ local dom0="$DIR/$tdir/dom0"
+ local qpool="qpool1"
+
+ mds_supports_qp
+ setup_quota_test || error "setup quota failed with $?"
+ stack_trap cleanup_quota_test EXIT
+
+ # enable ost quota
+ set_ost_qtype $QTYPE || error "enable ost quota failed"
+ set_mdt_qtype $QTYPE || error "enable mdt quota failed"
+
+ # Save DOM only at MDT0
+ $LFS setdirstripe -c 1 -i 0 $dom0 || error "cannot create $dom0"
+ $LFS setstripe -E 1M $dom0 -L mdt || error "setstripe to $dom0 failed"
+ chmod 0777 $dom0
+ $LFS setstripe -c 1 -i 0 "$DIR/$tdir/"
+
+ # create qpool and add OST0
+ pool_add $qpool || error "pool_add failed"
+ pool_add_targets $qpool 0 0 || error "pool_add_targets failed"
+
+ log "User quota (block hardlimit:$global_limit MB)"
+ $LFS setquota -u $TSTUSR -b 0 -B ${global_limit}M -i 0 -I 0 $DIR ||
+ error "set user quota failed"
+
+ log "User quota (block hardlimit:$limit MB)"
+ $LFS setquota -u $TSTUSR -B ${limit}M -o $qpool $DIR ||
+ error "set user quota failed"
+
+ $RUNAS dd if=/dev/zero of="$dom0/f1" bs=1K count=512 oflag=sync ||
+ quota_error u $TSTUSR "write failed"
+
+ $RUNAS dd if=/dev/zero of="$dom0/f1" bs=1K count=512 seek=512 \
+ oflag=sync || quota_error u $TSTUSR "write failed"
+
+ $RUNAS $DD of=$testfile count=$limit || true
+
+ # flush cache, ensure noquota flag is set on client
+ cancel_lru_locks osc
+ sync; sync_all_data || true
+
+ # MDT0 shouldn't get EDQUOT with glimpse.
+ $RUNAS $DD of=$testfile count=$limit seek=$limit &&
+ quota_error u $TSTUSR \
+ "user write success, but expect EDQUOT"
+
+ # Now all members of qpool1 should get EDQUOT. Expect success
+ # when write to DOM on MDT0, as it belongs to global pool.
+ $RUNAS dd if=/dev/zero of="$dom0/f1" bs=1K count=512 \
+ oflag=sync || quota_error u $TSTUSR "write failed"
+
+ $RUNAS dd if=/dev/zero of="$dom0/f1" bs=1K count=512 seek=512 \
+ oflag=sync || quota_error u $TSTUSR "write failed"
+
+ cleanup_quota_test
+}
+run_test 69 "EDQUOT at one of pools shouldn't affect DOM"
+
+test_70()
+{
+ local qpool="qpool1"
+ local limit=20
+ local err=0
+ local bhard
+
+ [[ CLIENT_VERSION -lt $(version_code $VERSION_WITH_QP) ]] &&
+ skip "Needs a client >= $VERSION_WITH_QP"
+
+ setup_quota_test || error "setup quota failed with $?"
+ stack_trap cleanup_quota_test EXIT
+
+ # MDS returns EFAULT for unsupported quotactl command
+ [[ $MDS1_VERSION -lt $(version_code $VERSION_WITH_QP) ]] && err=14
+
+ # create qpool and add OST0
+ pool_add $qpool || error "pool_add failed"
+ pool_add_targets $qpool 0 0 || error "pool_add_targets failed"
+
+ $LFS setquota -u $TSTUSR -B ${limit}M -o $qpool $DIR
+ rc=$?
+ [ $rc -eq $err ] || error "setquota res $rc != $err"
+
+ # If MDS supports QP, check that limit was set properly.
+ if [[ $MDS1_VERSION -ge $(version_code $VERSION_WITH_QP) ]]; then
+ bhard=$(getquota -u $TSTUSR global bhardlimit $qpool)
+ echo "hard limit $bhard limit $limit"
+ [ $bhard -ne $((limit*1024)) ] &&
+ error "bhard:$bhard for $qpool!=$((limit*1024))"
+ fi
+
+ $LFS quota -u $TSTUSR --pool $qpool $DIR
+ rc=$?
+ [ $rc -eq $err ] || error "quota res $rc != $err"
+
+ cleanup_quota_test
+}
+run_test 70 "check lfs setquota/quota with a pool option"
+
+test_71a()
+{
+ local limit=10 # 10M
+ local global_limit=100 # 100M
+ local testfile="$DIR/$tdir/$tfile-0"
+ local qpool="qpool1"
+ local qpool2="qpool2"
+
+ [ "$ost1_FSTYPE" == zfs ] &&
+ skip "ZFS grants some block space together with inode"
+ [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs"
+ mds_supports_qp
+ setup_quota_test || error "setup quota failed with $?"
+ stack_trap cleanup_quota_test EXIT
+
+ # enable ost quota
+ set_ost_qtype $QTYPE || error "enable ost quota failed"
+
+ # test for user
+ log "User quota (block hardlimit:$global_limit MB)"
+ $LFS setquota -u $TSTUSR -b 0 -B ${global_limit}M -i 0 -I 0 $DIR ||
+ error "set user quota failed"
+
+ pool_add $qpool || error "pool_add failed"
+ pool_add_targets $qpool 0 1 ||
+ error "pool_add_targets failed"
+
+ $LFS setquota -u $TSTUSR -B ${limit}M -o $qpool $DIR ||
+ error "set user quota failed"
+
+ pool_add $qpool2 || error "pool_add failed"
+ pool_add_targets $qpool2 1 1 ||
+ error "pool_add_targets failed"
+
+ $LFS setquota -u $TSTUSR -B ${limit}M -o $qpool2 $DIR ||
+ error "set user quota failed"
+
+ # make sure the system is clean
+ local used=$(getquota -u $TSTUSR global curspace)
+
+ echo "used $used"
+ [ $used -ne 0 ] && error "Used space($used) for user $TSTUSR isn't 0."
+
+ # create 1st component 1-10M
+ $LFS setstripe -E 10M -S 1M -c 1 -i 0 $testfile
+ #create 2nd component 10-30M
+ $LFS setstripe --component-add -E 30M -c 1 -i 1 $testfile
+ chown $TSTUSR.$TSTUSR $testfile || error "chown $testfile failed"
+
+ # сheck normal use and out of quota with PFL
+ # 1st element is in qppol1(OST0), 2nd in qpool2(OST2).
+ test_1_check_write $testfile "user" $((limit*2))
+ rm -f $testfile
+ wait_delete_completed || error "wait_delete_completed failed"
+ sync_all_data || true
+ used=$(getquota -u $TSTUSR global curspace)
+ [ $used -ne 0 ] && quota_error u $TSTUSR \
+ "user quota isn't released after deletion"
+
+ # create 1st component 1-10M
+ $LFS setstripe -E 10M -S 1M -c 1 -i 0 $testfile
+ # create 2nd component 10-30M
+ $LFS setstripe --component-add -E 30M -c 1 -i 1 $testfile
+ chown $TSTUSR.$TSTUSR $testfile || error "chown $testfile failed"
+
+ # write to the 2nd component
+ $RUNAS $DD of=$testfile count=$limit seek=10 ||
+ quota_error $short_qtype $TSTUSR \
+ "write failure, but expect success"
+ # this time maybe cache write, ignore it's failure
+ $RUNAS $DD of=$testfile count=$((2*limit)) seek=10 || true
+ cancel_lru_locks osc
+ sync; sync_all_data || true
+ # write over limit in qpool2(2nd component 10-30M)
+ $RUNAS $DD of=$testfile count=1 seek=$((10 + 2*limit)) &&
+ quota_error u $TSTUSR "user write success, but expect EDQUOT"
+ # write to the 1st component - OST0 is empty
+ $RUNAS $DD of=$testfile count=$limit seek=0 ||
+ quota_error u $TSTUSR "write failed"
+
+ cleanup_quota_test
+}
+run_test 71a "Check PFL with quota pools"
+
+test_71b()
+{
+ local global_limit=1000 # 1G
+ local limit1=160 # 160M
+ local limit2=10 # 10M
+ local testfile="$DIR/$tdir/$tfile-0"
+ local qpool="qpool1"
+ local qpool2="qpool2"
+
+ [ "$ost1_FSTYPE" == zfs ] &&
+ skip "ZFS grants some block space together with inode"
+ [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
+ mds_supports_qp
+ setup_quota_test || error "setup quota failed with $?"
+ stack_trap cleanup_quota_test EXIT
+
+ # enable ost quota
+ set_ost_qtype $QTYPE || error "enable ost quota failed"
+
+ # test for user
+ log "User quota (block hardlimit:$global_limit MB)"
+ $LFS setquota -u $TSTUSR -b 0 -B ${global_limit}M -i 0 -I 0 $DIR ||
+ error "set user quota failed"
+
+ pool_add $qpool || error "pool_add failed"
+ pool_add_targets $qpool 0 1 ||
+ error "pool_add_targets failed"
+
+ $LFS setquota -u $TSTUSR -B ${limit1}M -o $qpool $DIR ||
+ error "set user quota failed"
+
+ pool_add $qpool2 || error "pool_add failed"
+ pool_add_targets $qpool2 1 1 ||
+ error "pool_add_targets failed"
+
+ $LFS setquota -u $TSTUSR -B ${limit2}M -o $qpool2 $DIR ||
+ error "set user quota failed"
+
+ # make sure the system is clean
+ local used=$(getquota -u $TSTUSR global curspace)
+
+ echo "used $used"
+ [ $used -ne 0 ] && error "Used space($used) for user $TSTUSR isn't 0."
+
+ # First component is on OST0, 2nd on OST1
+ $LFS setstripe -E 128M -i 0 -z 64M -E -1 -i 1 -z 64M $testfile
+ chown $TSTUSR.$TSTUSR $testfile || error "chown $testfile failed"
+
+ # fill the 1st component on OST0
+ $RUNAS $DD of=$testfile count=128 ||
+ quota_error u $TSTUSR "write failed"
+ # write to the 2nd cmpnt on OST1
+ $RUNAS $DD of=$testfile count=$((limit2/2)) seek=128 ||
+ quota_error u $TSTUSR "write failed"
+ # this time maybe cache write, ignore it's failure
+ $RUNAS $DD of=$testfile count=$((limit2/2)) seek=$((128 + limit2/2)) ||
+ true
+ cancel_lru_locks osc
+ sync; sync_all_data || true
+ # write over limit in qpool2
+ $RUNAS $DD of=$testfile count=2 seek=$((128 + limit2)) &&
+ quota_error u $TSTUSR "user write success, but expect EDQUOT"
+
+ cleanup_quota_test
+}
+run_test 71b "Check SEL with quota pools"
+