From 14d1efa1ecda80f11f6cc1d42c517dcfdf914ce5 Mon Sep 17 00:00:00 2001 From: tianzy Date: Wed, 5 Sep 2007 10:35:39 +0000 Subject: [PATCH] Branch HEAD when mds and osts use different quota unit(32bit and 64bit), quota will be released repeatly. This patch voids sending multiple quota reqs to mds, which will keep the status between the reqs. b=12588 i=johann i=wangdi --- lustre/ChangeLog | 6 ++++ lustre/include/lustre/lustre_idl.h | 2 +- lustre/ptlrpc/pack_generic.c | 1 - lustre/quota/quota_context.c | 57 ++++++++++++++++++++++----------- lustre/quota/quota_internal.h | 2 +- lustre/tests/sanity-quota.sh | 64 ++++++++++++++++++++++---------------- 6 files changed, 84 insertions(+), 48 deletions(-) diff --git a/lustre/ChangeLog b/lustre/ChangeLog index bacd497..9e11a86 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -208,6 +208,12 @@ Details : after a connection loss, the lustre client should attempt to reconnect to the last active server first before trying the other potential connections. +Severity : minor +Bugzilla : 12588 +Description: when mds and osts use different quota unit(32bit and 64bit), + quota will be released repeatly. +Details : void sending multiple quota reqs to mds, which will keep the status + between the reqs. -------------------------------------------------------------------------------- 2007-08-10 Cluster File Systems, Inc. diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h index 56910b0..9cd7877 100644 --- a/lustre/include/lustre/lustre_idl.h +++ b/lustre/include/lustre/lustre_idl.h @@ -543,7 +543,7 @@ extern void lustre_swab_ptlrpc_body(struct ptlrpc_body *pb); #define ECHO_CONNECT_SUPPORTED (0) #define MGS_CONNECT_SUPPORTED (OBD_CONNECT_VERSION) -#define MAX_QUOTA_COUNT32 ((0xffffffffULL >> QUOTABLOCK_BITS) << QUOTABLOCK_BITS) +#define MAX_QUOTA_COUNT32 (0xffffffffULL) #define OBD_OCD_VERSION(major,minor,patch,fix) (((major)<<24) + ((minor)<<16) +\ ((patch)<<8) + (fix)) diff --git a/lustre/ptlrpc/pack_generic.c b/lustre/ptlrpc/pack_generic.c index 08d5192..3e9c269 100644 --- a/lustre/ptlrpc/pack_generic.c +++ b/lustre/ptlrpc/pack_generic.c @@ -2242,7 +2242,6 @@ struct qunit_data_old *lustre_quota_new_to_old(struct qunit_data *d) if (!d) return NULL; - LASSERT(d->qd_count <= MAX_QUOTA_COUNT32); tmp = *d; ret = (struct qunit_data_old *)d; ret->qd_id = tmp.qd_id; diff --git a/lustre/quota/quota_context.c b/lustre/quota/quota_context.c index 65c727e..9eb829b 100644 --- a/lustre/quota/quota_context.c +++ b/lustre/quota/quota_context.c @@ -358,27 +358,31 @@ schedule_dqacq(struct obd_device *obd, struct lustre_quota_ctxt *qctxt, static int split_before_schedule_dqacq(struct obd_device *obd, struct lustre_quota_ctxt *qctxt, struct qunit_data *qdata, int opc, int wait) { - int rc = 0, ret; + int rc = 0; + unsigned long factor; struct qunit_data tmp_qdata; ENTRY; - LASSERT(qdata); - if (qctxt->lqc_import) - while (should_translate_quota(qctxt->lqc_import) && - qdata->qd_count > MAX_QUOTA_COUNT32) { + LASSERT(qdata && qdata->qd_count); + QDATA_DEBUG(qdata, "%s quota split.\n", + (qdata->qd_flags & QUOTA_IS_BLOCK) ? "block" : "inode"); + if (qdata->qd_flags & QUOTA_IS_BLOCK) + factor = MAX_QUOTA_COUNT32 / qctxt->lqc_bunit_sz * + qctxt->lqc_bunit_sz; + else + factor = MAX_QUOTA_COUNT32 / qctxt->lqc_iunit_sz * + qctxt->lqc_iunit_sz; + if (qctxt->lqc_import && should_translate_quota(qctxt->lqc_import) && + qdata->qd_count > factor) { tmp_qdata = *qdata; - tmp_qdata.qd_count = MAX_QUOTA_COUNT32; + tmp_qdata.qd_count = factor; qdata->qd_count -= tmp_qdata.qd_count; - ret = schedule_dqacq(obd, qctxt, &tmp_qdata, opc, wait); - if (!rc) - rc = ret; - } - - if (qdata->qd_count){ - ret = schedule_dqacq(obd, qctxt, qdata, opc, wait); - if (!rc) - rc = ret; + QDATA_DEBUG((&tmp_qdata), "be split.\n"); + rc = schedule_dqacq(obd, qctxt, &tmp_qdata, opc, wait); + } else{ + QDATA_DEBUG(qdata, "don't be split.\n"); + rc = schedule_dqacq(obd, qctxt, qdata, opc, wait); } RETURN(rc); @@ -403,7 +407,8 @@ dqacq_completion(struct obd_device *obd, LASSERT(qdata); qunit_sz = is_blk ? qctxt->lqc_bunit_sz : qctxt->lqc_iunit_sz; div_r = do_div(qd_tmp, qunit_sz); - LASSERT(!div_r); + LASSERTF(!div_r, "qunit_sz: %lu, return qunit_sz: "LPU64"\n", + qunit_sz, qd_tmp); /* update local operational quota file */ if (rc == 0) { @@ -437,10 +442,18 @@ dqacq_completion(struct obd_device *obd, switch (opc) { case QUOTA_DQACQ: + CDEBUG(D_QUOTA, "%s(acq):count: %d, hardlimt: "LPU64 + ",type: %s.\n", obd->obd_name, count, *hardlimit, + qdata_type ? "grp": "usr"); INC_QLIMIT(*hardlimit, count); break; case QUOTA_DQREL: - LASSERT(count < *hardlimit); + CDEBUG(D_QUOTA, "%s(rel):count: %d, hardlimt: "LPU64 + ",type: %s.\n", obd->obd_name, count, *hardlimit, + qdata_type ? "grp": "usr"); + LASSERTF(count < *hardlimit, + "count: %d, hardlimit: "LPU64".\n", + count, *hardlimit); *hardlimit -= count; break; default: @@ -582,6 +595,7 @@ schedule_dqacq(struct obd_device *obd, struct qunit_data *reqdata; struct dqacq_async_args *aa; int size[2] = { sizeof(struct ptlrpc_body), sizeof(*reqdata) }; + unsigned long factor; int rc = 0; ENTRY; @@ -630,8 +644,15 @@ schedule_dqacq(struct obd_device *obd, RETURN(-ENOMEM); } + if (qdata->qd_flags & QUOTA_IS_BLOCK) + factor = MAX_QUOTA_COUNT32 / qctxt->lqc_bunit_sz * + qctxt->lqc_bunit_sz; + else + factor = MAX_QUOTA_COUNT32 / qctxt->lqc_iunit_sz * + qctxt->lqc_iunit_sz; + LASSERT(!should_translate_quota(qctxt->lqc_import) || - qdata->qd_count <= MAX_QUOTA_COUNT32); + qdata->qd_count <= factor); if (should_translate_quota(qctxt->lqc_import)) { struct qunit_data_old *reqdata_old, *tmp; diff --git a/lustre/quota/quota_internal.h b/lustre/quota/quota_internal.h index 9550f52..fcfee55 100644 --- a/lustre/quota/quota_internal.h +++ b/lustre/quota/quota_internal.h @@ -45,7 +45,7 @@ qinfo->qi_info[1].dqi_free_entry, ## arg); #define QDATA_DEBUG(qd, fmt, arg...) \ - CDEBUG(D_QUOTA, "id(%u) type(%lu) count(%llu) isblk(%lu):" \ + CDEBUG(D_QUOTA, "id(%u) type(%lu) count("LPU64") isblk(%lu):" \ fmt, qd->qd_id, qd->qd_flags & QUOTA_IS_GRP, qd->qd_count, \ (qd->qd_flags & QUOTA_IS_BLOCK) >> 1, \ ## arg); diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index f491b0e..0e1e3f1 100644 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -16,14 +16,6 @@ fi SRCDIR=`dirname $0` export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin -if [ "$1" == "9_10" ]; then - echo "only run for test9 and test10" - shift - TEST_9_10=1 - ONLY="9 10" -else - TEST_9_10=0 -fi ONLY=${ONLY:-"$*"} ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-""} # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! @@ -147,9 +139,7 @@ MOUNT_HINT=$MOUNT MOUNT_HINT2=$MOUNT_2 MOUNT="`mounted_lustre_filesystems 1`" MOUNT_2="`mounted_lustre_filesystems 2`" -if [ $TEST_9_10 -eq 1 -a "$MOUNT" ]; then - echo "test9 and test10 will run on $MOUNT" -elif [ "$MOUNT" -a "$MOUNT_2" ]; then +if [ "$MOUNT" -a "$MOUNT_2" ]; then echo "testing on $MOUNT and $MOUNT_2" elif [ "$MOUNT" -o "$MOUNT_2" ]; then error "test needs two mounts, only found $MOUNT $MOUNT_2!" @@ -174,11 +164,7 @@ DIR2=${DIR2:-$MOUNT_2} LPROC=/proc/fs/lustre LOVNAME=`cat $LPROC/llite/*/lov/common_name | tail -n 1` -if [ $TEST_9_10 -eq 1 ]; then - OSTCOUNT=2 -else - OSTCOUNT=`cat $LPROC/lov/$LOVNAME/numobd` -fi +OSTCOUNT=`cat $LPROC/lov/$LOVNAME/numobd` STRIPECOUNT=`cat $LPROC/lov/$LOVNAME/stripecount` STRIPESIZE=`cat $LPROC/lov/$LOVNAME/stripesize` ORIGFREE=`cat $LPROC/lov/$LOVNAME/kbytesavail` @@ -252,9 +238,7 @@ pre_test() { set_file_unitsz $IUNIT_SZ fi } -if [ $TEST_9_10 -eq 0 ]; then - pre_test -fi +pre_test post_test() { if [ -z "$NOSETUP" ]; then @@ -750,19 +734,21 @@ test_9() { return 0; fi + set_blk_unitsz $((1024 * 100)) + set_blk_tunesz $((1024 * 50)) + # set the D_QUOTA flag DBG_SAVE="`sysctl -n lnet.debug`" sysctl -w lnet.debug="$DBG_SAVE quota" TESTFILE="$TSTDIR/quota_tst90" - echo " Set block limit $LIMIT kbytes to $TSTUSR.$TSTUSR" BLK_LIMIT=$((100 * 1024 * 1024)) # 100G FILE_LIMIT=1000000 - echo " Set enough high limit for user: $TSTUSR" + echo " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for user: $TSTUSR" $LFS setquota -u $TSTUSR 0 $BLK_LIMIT 0 $FILE_LIMIT $MOUNT - echo " Set enough high limit for group: $TSTUSR" + echo " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for group: $TSTUSR" $LFS setquota -g $TSTUSR 0 $BLK_LIMIT 0 $FILE_LIMIT $MOUNT echo " Set stripe" @@ -770,12 +756,21 @@ test_9() { touch $TESTFILE chown $TSTUSR.$TSTUSR $TESTFILE + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + echo " Write the big file of $(($OSTCOUNT * 9 / 2 ))G ..." $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$size_file || error "(usr) write $((9 / 2 * $OSTCOUNT))G file failure, but expect success" + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + echo " delete the big file of $(($OSTCOUNT * 9 / 2))G..." $RUNAS rm -f $TESTFILE + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + echo " write the big file of 2G..." $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$((1024 * 1024 * 2)) || error "(usr) write $((9 / 2 * $OSTCOUNT))G file failure, but expect seccess" @@ -783,6 +778,9 @@ test_9() { $RUNAS rm -f $TESTFILE RC=$? + set_blk_tunesz $BTUNE_SZ + set_blk_unitsz $BUNIT_SZ + sysctl -w lnet.debug="$DBG_SAVE" return $RC } @@ -805,6 +803,9 @@ test_10() { sync; sleep 10; sync; + set_blk_unitsz $((1024 * 100)) + set_blk_tunesz $((1024 * 50)) + # set the D_QUOTA flag set_flag=0 if [ -z "`sysctl lnet.debug | grep quota`" ]; then @@ -817,13 +818,12 @@ test_10() { TESTFILE="$TSTDIR/quota_tst100" - echo " Set block limit $LIMIT kbytes to $TSTUSR.$TSTUSR" BLK_LIMIT=$((100 * 1024 * 1024)) # 100G FILE_LIMIT=1000000 - echo " Set enough high limit for user: $TSTUSR" + echo " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for user: $TSTUSR" $LFS setquota -u $TSTUSR 0 $BLK_LIMIT 0 $FILE_LIMIT $MOUNT - echo " Set enough high limit for group: $TSTUSR" + echo " Set enough high limit(block:$BLK_LIMIT; file: $FILE_LIMIT) for group: $TSTUSR" $LFS setquota -g $TSTUSR 0 $BLK_LIMIT 0 $FILE_LIMIT $MOUNT echo " Set stripe" @@ -831,12 +831,21 @@ test_10() { touch $TESTFILE chown $TSTUSR.$TSTUSR $TESTFILE + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + echo " Write the big file of $(($OSTCOUNT * 9 / 2 ))G ..." $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$size_file || error "(usr) write $((9 / 2 * $OSTCOUNT))G file failure, but expect success" + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + echo " delete the big file of $(($OSTCOUNT * 9 / 2))G..." $RUNAS rm -f $TESTFILE + $SHOW_QUOTA_USER + $SHOW_QUOTA_GROUP + echo " write the big file of 2G..." $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$((1024 * 1024 * 2)) || error "(usr) write $((9 / 2 * $OSTCOUNT))G file failure, but expect success" @@ -853,6 +862,9 @@ test_10() { # make qd_count 64 bit sysctl -w lustre.fail_loc=0 + set_blk_tunesz $BTUNE_SZ + set_blk_unitsz $BUNIT_SZ + return $RC } run_test 10 "run for fixing bug10707(32bit) ===========" @@ -1098,9 +1110,7 @@ run_test 99 "Quota off ===============================" log "cleanup: ======================================================" if [ "`mount | grep ^$NAME`" ]; then rm -fr $TSTDIR - if [ $TEST_9_10 -eq 0 ]; then post_test - fi # delete test user and group userdel "$TSTUSR" userdel "$TSTUSR2" -- 1.8.3.1