From cbb24dbb4d90e4d28b4f5b242db747b754c7c630 Mon Sep 17 00:00:00 2001 From: tianzy Date: Wed, 5 Sep 2007 09:50:17 +0000 Subject: [PATCH] Branch b1_6 when mds and osts use different quota unit(32bit and 64bit), quota will be released repeatly. This patch voids sending multiple quota reqs to mds, which will keep the status between the reqs. b=12588 i=johann i=wangdi --- lustre/ChangeLog | 7 +++++ lustre/include/lustre/lustre_idl.h | 2 +- lustre/ptlrpc/pack_generic.c | 1 - lustre/quota/quota_context.c | 57 ++++++++++++++++++++++----------- lustre/quota/quota_internal.h | 2 +- lustre/tests/sanity-quota.sh | 64 ++++++++++++++++++++++---------------- 6 files changed, 85 insertions(+), 48 deletions(-) diff --git a/lustre/ChangeLog b/lustre/ChangeLog index df278f0..944024f 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -152,6 +152,13 @@ Bugzilla : 11401 Description: client-side metadata stat-ahead during readdir(directory readahead) Details : perform client-side metadata stat-ahead when the client detects readdir and sequential stat of dir entries therein + +Severity : minor +Bugzilla : 12588 +Description: when mds and osts use different quota unit(32bit and 64bit), + quota will be released repeatly. +Details : void sending multiple quota reqs to mds, which will keep the status + between the reqs. -------------------------------------------------------------------------------- 2007-08-27 Cluster File Systems, Inc. diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h index 094998e..bb4a2af 100644 --- a/lustre/include/lustre/lustre_idl.h +++ b/lustre/include/lustre/lustre_idl.h @@ -303,7 +303,7 @@ extern void lustre_swab_ptlrpc_body(struct ptlrpc_body *pb); #define ECHO_CONNECT_SUPPORTED (0) #define MGS_CONNECT_SUPPORTED (OBD_CONNECT_VERSION | OBD_CONNECT_AT) -#define MAX_QUOTA_COUNT32 ((0xffffffffULL >> QUOTABLOCK_BITS) << QUOTABLOCK_BITS) +#define MAX_QUOTA_COUNT32 (0xffffffffULL) #define OBD_OCD_VERSION(major,minor,patch,fix) (((major)<<24) + ((minor)<<16) +\ ((patch)<<8) + (fix)) diff --git a/lustre/ptlrpc/pack_generic.c b/lustre/ptlrpc/pack_generic.c index 09ec8e1..83bbe3a 100644 --- a/lustre/ptlrpc/pack_generic.c +++ b/lustre/ptlrpc/pack_generic.c @@ -2263,7 +2263,6 @@ struct qunit_data_old *lustre_quota_new_to_old(struct qunit_data *d) if (!d) return NULL; - LASSERT(d->qd_count <= MAX_QUOTA_COUNT32); tmp = *d; ret = (struct qunit_data_old *)d; ret->qd_id = tmp.qd_id; diff --git a/lustre/quota/quota_context.c b/lustre/quota/quota_context.c index 2f0cb2e..5418f0e 100644 --- a/lustre/quota/quota_context.c +++ b/lustre/quota/quota_context.c @@ -359,27 +359,31 @@ schedule_dqacq(struct obd_device *obd, struct lustre_quota_ctxt *qctxt, static int split_before_schedule_dqacq(struct obd_device *obd, struct lustre_quota_ctxt *qctxt, struct qunit_data *qdata, int opc, int wait) { - int rc = 0, ret; + int rc = 0; + unsigned long factor; struct qunit_data tmp_qdata; ENTRY; - LASSERT(qdata); - if (qctxt->lqc_import) - while (should_translate_quota(qctxt->lqc_import) && - qdata->qd_count > MAX_QUOTA_COUNT32) { + LASSERT(qdata && qdata->qd_count); + QDATA_DEBUG(qdata, "%s quota split.\n", + (qdata->qd_flags & QUOTA_IS_BLOCK) ? "block" : "inode"); + if (qdata->qd_flags & QUOTA_IS_BLOCK) + factor = MAX_QUOTA_COUNT32 / qctxt->lqc_bunit_sz * + qctxt->lqc_bunit_sz; + else + factor = MAX_QUOTA_COUNT32 / qctxt->lqc_iunit_sz * + qctxt->lqc_iunit_sz; + if (qctxt->lqc_import && should_translate_quota(qctxt->lqc_import) && + qdata->qd_count > factor) { tmp_qdata = *qdata; - tmp_qdata.qd_count = MAX_QUOTA_COUNT32; + tmp_qdata.qd_count = factor; qdata->qd_count -= tmp_qdata.qd_count; - ret = schedule_dqacq(obd, qctxt, &tmp_qdata, opc, wait); - if (!rc) - rc = ret; - } - - if (qdata->qd_count){ - ret = schedule_dqacq(obd, qctxt, qdata, opc, wait); - if (!rc) - rc = ret; + QDATA_DEBUG((&tmp_qdata), "be split.\n"); + rc = schedule_dqacq(obd, qctxt, &tmp_qdata, opc, wait); + } else{ + QDATA_DEBUG(qdata, "don't be split.\n"); + rc = schedule_dqacq(obd, qctxt, qdata, opc, wait); } RETURN(rc); @@ -404,7 +408,8 @@ dqacq_completion(struct obd_device *obd, LASSERT(qdata); qunit_sz = is_blk ? qctxt->lqc_bunit_sz : qctxt->lqc_iunit_sz; div_r = do_div(qd_tmp, qunit_sz); - LASSERT(!div_r); + LASSERTF(!div_r, "qunit_sz: %lu, return qunit_sz: "LPU64"\n", + qunit_sz, qd_tmp); /* update local operational quota file */ if (rc == 0) { @@ -438,10 +443,18 @@ dqacq_completion(struct obd_device *obd, switch (opc) { case QUOTA_DQACQ: + CDEBUG(D_QUOTA, "%s(acq):count: %d, hardlimt: "LPU64 + ",type: %s.\n", obd->obd_name, count, *hardlimit, + qdata_type ? "grp": "usr"); INC_QLIMIT(*hardlimit, count); break; case QUOTA_DQREL: - LASSERT(count < *hardlimit); + CDEBUG(D_QUOTA, "%s(rel):count: %d, hardlimt: "LPU64 + ",type: %s.\n", obd->obd_name, count, *hardlimit, + qdata_type ? "grp": "usr"); + LASSERTF(count < *hardlimit, + "count: %d, hardlimit: "LPU64".\n", + count, *hardlimit); *hardlimit -= count; break; default: @@ -584,6 +597,7 @@ schedule_dqacq(struct obd_device *obd, struct dqacq_async_args *aa; int size[2] = { sizeof(struct ptlrpc_body), sizeof(*reqdata) }; struct obd_import *imp = NULL; + unsigned long factor; int rc = 0; ENTRY; @@ -650,8 +664,15 @@ schedule_dqacq(struct obd_device *obd, RETURN(-ENOMEM); } + if (qdata->qd_flags & QUOTA_IS_BLOCK) + factor = MAX_QUOTA_COUNT32 / qctxt->lqc_bunit_sz * + qctxt->lqc_bunit_sz; + else + factor = MAX_QUOTA_COUNT32 / qctxt->lqc_iunit_sz * + qctxt->lqc_iunit_sz; + LASSERTF(!should_translate_quota(imp) || - qdata->qd_count <= MAX_QUOTA_COUNT32, + qdata->qd_count <= factor, "qd_count: "LPU64"; should_translate_quota: %d.\n", qdata->qd_count, should_translate_quota(imp)); if (should_translate_quota(imp)) diff --git a/lustre/quota/quota_internal.h b/lustre/quota/quota_internal.h index 9550f52..fcfee55 100644 --- a/lustre/quota/quota_internal.h +++ b/lustre/quota/quota_internal.h @@ -45,7 +45,7 @@ qinfo->qi_info[1].dqi_free_entry, ## arg); #define QDATA_DEBUG(qd, fmt, arg...) \ - CDEBUG(D_QUOTA, "id(%u) type(%lu) count(%llu) isblk(%lu):" \ + CDEBUG(D_QUOTA, "id(%u) type(%lu) count("LPU64") isblk(%lu):" \ fmt, qd->qd_id, qd->qd_flags & QUOTA_IS_GRP, qd->qd_count, \ (qd->qd_flags & QUOTA_IS_BLOCK) >> 1, \ ## arg); diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index 4cc9cf9..5b3ebfc 100644 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -16,14 +16,6 @@ fi SRCDIR=`dirname $0` export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin -if [ "$1" == "9_10" ]; then - echo "only run for test9 and test10" - shift - TEST_9_10=1 - ONLY="9 10" -else - TEST_9_10=0 -fi ONLY=${ONLY:-"$*"} ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-""} # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! @@ -152,9 +144,7 @@ MOUNT_HINT=$MOUNT MOUNT_HINT2=$MOUNT_2 MOUNT="`mounted_lustre_filesystems 1`" MOUNT_2="`mounted_lustre_filesystems 2`" -if [ $TEST_9_10 -eq 1 -a "$MOUNT" ]; then - echo "test9 and test10 will run on $MOUNT" -elif [ "$MOUNT" -a "$MOUNT_2" ]; then +if [ "$MOUNT" -a "$MOUNT_2" ]; then echo "testing on $MOUNT and $MOUNT_2" elif [ "$MOUNT" -o "$MOUNT_2" ]; then error "test needs two mounts, only found $MOUNT $MOUNT_2!" @@ -179,11 +169,7 @@ DIR2=${DIR2:-$MOUNT_2} LPROC=/proc/fs/lustre LOVNAME=`cat $LPROC/llite/*/lov/common_name | tail -n 1` -if [ $TEST_9_10 -eq 1 ]; then - OSTCOUNT=2 -else - OSTCOUNT=`cat $LPROC/lov/$LOVNAME/numobd` -fi +OSTCOUNT=`cat $LPROC/lov/$LOVNAME/numobd` STRIPECOUNT=`cat $LPROC/lov/$LOVNAME/stripecount` STRIPESIZE=`cat $LPROC/lov/$LOVNAME/stripesize` ORIGFREE=`cat $LPROC/lov/$LOVNAME/kbytesavail` @@ -257,9 +243,7 @@ pre_test() { set_file_unitsz $IUNIT_SZ fi } -if [ $TEST_9_10 -eq 0 ]; then - pre_test -fi +pre_test post_test() { if [ -z "$NOSETUP" ]; then @@ -760,19 +744,21 @@ test_9() { return 0; fi + set_blk_unitsz $((1024 * 100)) + set_blk_tunesz $((1024 * 50)) + # set the D_QUOTA flag debugsave sysctl -w lnet.debug="+quota" TESTFILE="$TSTDIR/quota_tst90" - echo " Set block limit $LIMIT kbytes to $TSTUSR.$TSTUSR" BLK_LIMIT=$((100 * 1024 * 1024)) # 100G FILE_LIMIT=1000000 - echo " Set enough high limit for user: $TSTUSR" + echo " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for user: $TSTUSR" $LFS setquota -u $TSTUSR 0 $BLK_LIMIT 0 $FILE_LIMIT $MOUNT - echo " Set enough high limit for group: $TSTUSR" + echo " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for group: $TSTUSR" $LFS setquota -g $TSTUSR 0 $BLK_LIMIT 0 $FILE_LIMIT $MOUNT echo " Set stripe" @@ -780,12 +766,21 @@ test_9() { touch $TESTFILE chown $TSTUSR.$TSTUSR $TESTFILE + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + echo " Write the big file of $(($OSTCOUNT * 9 / 2 ))G ..." $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$size_file || error "(usr) write $((9 / 2 * $OSTCOUNT))G file failure, but expect success" + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + echo " delete the big file of $(($OSTCOUNT * 9 / 2))G..." $RUNAS rm -f $TESTFILE + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + echo " write the big file of 2G..." $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$((1024 * 1024 * 2)) || error "(usr) write $((9 / 2 * $OSTCOUNT))G file failure, but expect seccess" @@ -793,6 +788,9 @@ test_9() { $RUNAS rm -f $TESTFILE RC=$? + set_blk_tunesz $BTUNE_SZ + set_blk_unitsz $BUNIT_SZ + debugrestore return $RC } @@ -815,6 +813,9 @@ test_10() { sync; sleep 10; sync; + set_blk_unitsz $((1024 * 100)) + set_blk_tunesz $((1024 * 50)) + # set the D_QUOTA flag debugsave sysctl -w lnet.debug="+quota" @@ -824,13 +825,12 @@ test_10() { TESTFILE="$TSTDIR/quota_tst100" - echo " Set block limit $LIMIT kbytes to $TSTUSR.$TSTUSR" BLK_LIMIT=$((100 * 1024 * 1024)) # 100G FILE_LIMIT=1000000 - echo " Set enough high limit for user: $TSTUSR" + echo " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for user: $TSTUSR" $LFS setquota -u $TSTUSR 0 $BLK_LIMIT 0 $FILE_LIMIT $MOUNT - echo " Set enough high limit for group: $TSTUSR" + echo " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for group: $TSTUSR" $LFS setquota -g $TSTUSR 0 $BLK_LIMIT 0 $FILE_LIMIT $MOUNT echo " Set stripe" @@ -838,12 +838,21 @@ test_10() { touch $TESTFILE chown $TSTUSR.$TSTUSR $TESTFILE + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + echo " Write the big file of $(($OSTCOUNT * 9 / 2 ))G ..." $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$size_file || error "(usr) write $((9 / 2 * $OSTCOUNT))G file failure, but expect success" + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + echo " delete the big file of $(($OSTCOUNT * 9 / 2))G..." $RUNAS rm -f $TESTFILE + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + echo " write the big file of 2G..." $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$((1024 * 1024 * 2)) || error "(usr) write $((9 / 2 * $OSTCOUNT))G file failure, but expect success" @@ -858,6 +867,9 @@ test_10() { # make qd_count 64 bit sysctl -w lustre.fail_loc=0 + set_blk_tunesz $BTUNE_SZ + set_blk_unitsz $BUNIT_SZ + return $RC } run_test 10 "run for fixing bug10707(32bit) ===========" @@ -1108,9 +1120,7 @@ run_test 99 "Quota off ===============================" log "cleanup: ======================================================" if [ "`mount | grep ^$NAME`" ]; then rm -fr $TSTDIR - if [ $TEST_9_10 -eq 0 ]; then post_test - fi # delete test user and group userdel "$TSTUSR" userdel "$TSTUSR2" -- 1.8.3.1