From: tianzy Date: Mon, 15 Dec 2008 09:26:51 +0000 (+0000) Subject: Branch HEAD X-Git-Tag: v1_9_130~33 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=e9dc8feb37943024452dcc5acb3f96b95c6a518d Branch HEAD meta blocks ignored can lead to occasional -EDQUOT occasionally b=16542 i=johann i=alex.zhuravlev --- diff --git a/lustre/include/linux/lustre_fsfilt.h b/lustre/include/linux/lustre_fsfilt.h index 17e30a5..484d267 100644 --- a/lustre/include/linux/lustre_fsfilt.h +++ b/lustre/include/linux/lustre_fsfilt.h @@ -121,6 +121,8 @@ struct fsfilt_operations { int cmd); int (* fs_qids)(struct file *file, struct inode *inode, int type, struct list_head *list); + int (* fs_get_mblk)(struct super_block *sb, int *count, + struct inode *inode, int frags); int (* fs_dquot)(struct lustre_dquot *dquot, int cmd); lvfs_sbdev_type (* fs_journal_sbdev)(struct super_block *sb); }; @@ -443,6 +445,15 @@ static inline int fsfilt_dquot(struct obd_device *obd, return -ENOTSUPP; } +static inline int fsfilt_get_mblk(struct obd_device *obd, + struct super_block *sb, int *count, + struct inode *inode, int frags) +{ + if (obd->obd_fsops->fs_get_mblk) + return obd->obd_fsops->fs_get_mblk(sb, count, inode, frags); + return -ENOTSUPP; +} + static inline int fsfilt_map_inode_pages(struct obd_device *obd, struct inode *inode, struct page **page, int pages, diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h index 8cac1e6..c80dd5e 100644 --- a/lustre/include/lustre/lustre_idl.h +++ b/lustre/include/lustre/lustre_idl.h @@ -2489,7 +2489,6 @@ typedef enum { #define QUOTA_RET_NOQUOTA 1 /**< not support quota */ #define QUOTA_RET_NOLIMIT 2 /**< quota limit isn't set */ #define QUOTA_RET_ACQUOTA 4 /**< need to acquire extra quota */ -#define QUOTA_RET_INC_PENDING 8 /**< pending value is increased */ /* security opcodes */ typedef enum { diff --git a/lustre/include/lustre_quota.h b/lustre/include/lustre_quota.h index b0dc442..9dd419c 100644 --- a/lustre/include/lustre_quota.h +++ b/lustre/include/lustre_quota.h @@ -499,6 +499,7 @@ typedef struct { */ int (*quota_getflag) (struct obd_device *, struct obdo *); +#ifdef __KERNEL__ /** * For quota slave, acquire/release quota from master if needed */ @@ -512,7 +513,15 @@ typedef struct { */ int (*quota_chkquota) (struct obd_device *, unsigned int, unsigned int, int, int *, quota_acquire, - struct obd_trans_info *, int); + struct obd_trans_info *, int, struct inode *, + int); + + /** + * For quota client, the actions after the pending write is committed + */ + int (*quota_pending_commit) (struct obd_device *, unsigned int, + unsigned int, int, int); +#endif /** * For quota client, poll if the quota check done @@ -525,12 +534,6 @@ typedef struct { int (*quota_chkdq) (struct client_obd *, unsigned int, unsigned int); /** - * For quota client, the actions after the pending write is committed - */ - int (*quota_pending_commit) (struct obd_device *, unsigned int, - unsigned int, int, int); - - /** * For quota client, set over quota flag for specifed uid/gid */ int (*quota_setdq) (struct client_obd *, unsigned int, unsigned int, @@ -746,11 +749,12 @@ static inline int lquota_getflag(quota_interface_t *interface, RETURN(rc); } +#ifdef __KERNEL__ static inline int lquota_chkquota(quota_interface_t *interface, struct obd_device *obd, unsigned int uid, unsigned int gid, int count, int *flag, struct obd_trans_info *oti, - int isblk) + int isblk, struct inode *inode, int frags) { int rc; ENTRY; @@ -759,22 +763,23 @@ static inline int lquota_chkquota(quota_interface_t *interface, QUOTA_CHECK_OP(interface, acquire); rc = QUOTA_OP(interface, chkquota)(obd, uid, gid, count, flag, QUOTA_OP(interface, acquire), oti, - isblk); + isblk, inode, frags); RETURN(rc); } static inline int lquota_pending_commit(quota_interface_t *interface, struct obd_device *obd, unsigned int uid, unsigned int gid, - int npage, int isblk) + int pending, int isblk) { int rc; ENTRY; QUOTA_CHECK_OP(interface, pending_commit); - rc = QUOTA_OP(interface, pending_commit)(obd, uid, gid, npage, isblk); + rc = QUOTA_OP(interface, pending_commit)(obd, uid, gid, pending, isblk); RETURN(rc); } +#endif #ifndef __KERNEL__ extern quota_interface_t osc_quota_interface; diff --git a/lustre/lvfs/fsfilt_ext3.c b/lustre/lvfs/fsfilt_ext3.c index 26ed65b..97758a6 100644 --- a/lustre/lvfs/fsfilt_ext3.c +++ b/lustre/lvfs/fsfilt_ext3.c @@ -2094,6 +2094,16 @@ static int fsfilt_ext3_dquot(struct lustre_dquot *dquot, int cmd) } RETURN(rc); } + +static int fsfilt_ext3_get_mblk(struct super_block *sb, int *count, + struct inode *inode, int frags) +{ + /* for an ost_write request, it needs <#fragments> * + * metablocks at maxium b=16542 */ + *count = frags * (EXT_DEPTH(inode) + 1) * EXT3_BLOCK_SIZE(sb); + return 0; +} + #endif lvfs_sbdev_type fsfilt_ext3_journal_sbdev(struct super_block *sb) @@ -2138,6 +2148,7 @@ static struct fsfilt_operations fsfilt_ext3_ops = { .fs_quotainfo = fsfilt_ext3_quotainfo, .fs_qids = fsfilt_ext3_qids, .fs_dquot = fsfilt_ext3_dquot, + .fs_get_mblk = fsfilt_ext3_get_mblk, #endif .fs_journal_sbdev = fsfilt_ext3_journal_sbdev, }; diff --git a/lustre/mdd/mdd_dir.c b/lustre/mdd/mdd_dir.c index f00d307..c6b57d8 100644 --- a/lustre/mdd/mdd_dir.c +++ b/lustre/mdd/mdd_dir.c @@ -680,7 +680,8 @@ static int mdd_link(const struct lu_env *env, struct md_object *tgt_obj, /* get block quota for parent */ lquota_chkquota(mds_quota_interface_ref, obd, qids[USRQUOTA], qids[GRPQUOTA], 1, - &rec_pending, NULL, LQUOTA_FLAGS_BLK); + &rec_pending, NULL, LQUOTA_FLAGS_BLK, + NULL, 0); } } #endif @@ -735,7 +736,7 @@ out_pending: if (rec_pending) lquota_pending_commit(mds_quota_interface_ref, obd, qids[USRQUOTA], qids[GRPQUOTA], - 1, 1); + rec_pending, 1); /* Trigger dqacq for the parent owner. If failed, * the next call for lquota_chkquota will process it. */ lquota_adjust(mds_quota_interface_ref, obd, 0, qids, rc, @@ -965,7 +966,7 @@ static int mdd_name_insert(const struct lu_env *env, lquota_chkquota(mds_quota_interface_ref, obd, qids[USRQUOTA], qids[GRPQUOTA], 1, &rec_pending, NULL, - LQUOTA_FLAGS_BLK); + LQUOTA_FLAGS_BLK, NULL, 0); } } else { uc->mu_cap |= CFS_CAP_SYS_RESOURCE_MASK; @@ -1014,7 +1015,8 @@ out_pending: if (rec_pending) lquota_pending_commit(mds_quota_interface_ref, obd, qids[USRQUOTA], - qids[GRPQUOTA], 1, 1); + qids[GRPQUOTA], + rec_pending, 1); /* Trigger dqacq for the parent owner. If failed, * the next call for lquota_chkquota will process it*/ lquota_adjust(mds_quota_interface_ref, obd, 0, qids, @@ -1187,7 +1189,8 @@ static int mdd_rename_tgt(const struct lu_env *env, /* get block quota for target parent */ lquota_chkquota(mds_quota_interface_ref, obd, qpids[USRQUOTA], qpids[GRPQUOTA], 1, - &rec_pending, NULL, LQUOTA_FLAGS_BLK); + &rec_pending, NULL, LQUOTA_FLAGS_BLK, + NULL, 0); } } #endif @@ -1276,7 +1279,7 @@ out_pending: lquota_pending_commit(mds_quota_interface_ref, obd, qpids[USRQUOTA], qpids[GRPQUOTA], - 1, 1); + rec_pending, 1); if (quota_opc) /* Trigger dqrel/dqacq on the target owner of child and * parent. If failed, the next call for lquota_chkquota @@ -1636,7 +1639,7 @@ static int mdd_create(const struct lu_env *env, /* get file quota for child */ lquota_chkquota(mds_quota_interface_ref, obd, qcids[USRQUOTA], qcids[GRPQUOTA], 1, - &inode_pending, NULL, 0); + &inode_pending, NULL, 0, NULL, 0); switch (ma->ma_attr.la_mode & S_IFMT) { case S_IFLNK: case S_IFDIR: @@ -1657,12 +1660,12 @@ static int mdd_create(const struct lu_env *env, qcids[USRQUOTA], qcids[GRPQUOTA], block_count, &block_pending, NULL, - LQUOTA_FLAGS_BLK); + LQUOTA_FLAGS_BLK, NULL, 0); if (!same) lquota_chkquota(mds_quota_interface_ref, obd, qpids[USRQUOTA], qpids[GRPQUOTA], 1, &parent_pending, NULL, - LQUOTA_FLAGS_BLK); + LQUOTA_FLAGS_BLK, NULL, 0); } } #endif @@ -1834,15 +1837,15 @@ out_pending: if (inode_pending) lquota_pending_commit(mds_quota_interface_ref, obd, qcids[USRQUOTA], qcids[GRPQUOTA], - 1, 0); + inode_pending, 0); if (block_pending) lquota_pending_commit(mds_quota_interface_ref, obd, qcids[USRQUOTA], qcids[GRPQUOTA], - block_count, 1); + block_pending, 1); if (parent_pending) lquota_pending_commit(mds_quota_interface_ref, obd, qpids[USRQUOTA], qpids[GRPQUOTA], - 1, 1); + parent_pending, 1); /* Trigger dqacq on the owner of child and parent. If failed, * the next call for lquota_chkquota will process it. */ lquota_adjust(mds_quota_interface_ref, obd, qcids, qpids, rc, @@ -1989,7 +1992,8 @@ static int mdd_rename(const struct lu_env *env, obd, qtpids[USRQUOTA], qtpids[GRPQUOTA], 1, &rec_pending, NULL, - LQUOTA_FLAGS_BLK); + LQUOTA_FLAGS_BLK, + NULL, 0); } } } @@ -2159,7 +2163,7 @@ out_pending: lquota_pending_commit(mds_quota_interface_ref, obd, qtpids[USRQUOTA], qtpids[GRPQUOTA], - 1, 1); + rec_pending, 1); /* Trigger dqrel on the source owner of parent. * If failed, the next call for lquota_chkquota will * process it. */ diff --git a/lustre/mdd/mdd_object.c b/lustre/mdd/mdd_object.c index 1efe365..56f595d 100644 --- a/lustre/mdd/mdd_object.c +++ b/lustre/mdd/mdd_object.c @@ -1244,7 +1244,7 @@ static int mdd_attr_set(const struct lu_env *env, struct md_object *obj, /* get file quota for new owner */ lquota_chkquota(mds_quota_interface_ref, obd, qnids[USRQUOTA], qnids[GRPQUOTA], 1, - &inode_pending, NULL, 0); + &inode_pending, NULL, 0, NULL, 0); block_count = (la_tmp->la_blocks + 7) >> 3; if (block_count) /* get block quota for new owner */ @@ -1252,7 +1252,8 @@ static int mdd_attr_set(const struct lu_env *env, struct md_object *obj, qnids[USRQUOTA], qnids[GRPQUOTA], block_count, &block_pending, - NULL, LQUOTA_FLAGS_BLK); + NULL, LQUOTA_FLAGS_BLK, + NULL, 0); } } #endif @@ -1307,11 +1308,11 @@ cleanup: if (inode_pending) lquota_pending_commit(mds_quota_interface_ref, obd, qnids[USRQUOTA], qnids[GRPQUOTA], - 1, 0); + inode_pending, 0); if (block_pending) lquota_pending_commit(mds_quota_interface_ref, obd, qnids[USRQUOTA], qnids[GRPQUOTA], - block_count, 1); + block_pending, 1); /* Trigger dqrel/dqacq for original owner and new owner. * If failed, the next call for lquota_chkquota will * process it. */ @@ -1561,7 +1562,8 @@ static int mdd_object_create(const struct lu_env *env, mdd_quota_wrapper(&ma->ma_attr, qids); /* get file quota for child */ lquota_chkquota(mds_quota_interface_ref, obd, qids[USRQUOTA], - qids[GRPQUOTA], 1, &inode_pending, NULL, 0); + qids[GRPQUOTA], 1, &inode_pending, NULL, 0, + NULL, 0); switch (ma->ma_attr.la_mode & S_IFMT) { case S_IFLNK: case S_IFDIR: @@ -1576,7 +1578,7 @@ static int mdd_object_create(const struct lu_env *env, lquota_chkquota(mds_quota_interface_ref, obd, qids[USRQUOTA], qids[GRPQUOTA], block_count, &block_pending, NULL, - LQUOTA_FLAGS_BLK); + LQUOTA_FLAGS_BLK, NULL, 0); } #endif @@ -1646,11 +1648,11 @@ out_pending: if (inode_pending) lquota_pending_commit(mds_quota_interface_ref, obd, qids[USRQUOTA], qids[GRPQUOTA], - 1, 0); + inode_pending, 0); if (block_pending) lquota_pending_commit(mds_quota_interface_ref, obd, qids[USRQUOTA], qids[GRPQUOTA], - block_count, 1); + block_pending, 1); /* Trigger dqacq on the owner of child. If failed, * the next call for lquota_chkquota will process it. */ lquota_adjust(mds_quota_interface_ref, obd, qids, 0, rc, diff --git a/lustre/obdfilter/filter_io_26.c b/lustre/obdfilter/filter_io_26.c index aa49c15..44178ee 100644 --- a/lustre/obdfilter/filter_io_26.c +++ b/lustre/obdfilter/filter_io_26.c @@ -551,7 +551,7 @@ int filter_commitrw_write(struct obd_export *exp, struct obdo *oa, struct lvfs_run_ctxt saved; struct fsfilt_objinfo fso; struct iattr iattr = { 0 }; - struct inode *inode = NULL; + struct inode *inode = res->dentry->d_inode; unsigned long now = jiffies; int i, err, cleanup_phase = 0; struct obd_device *obd = exp->exp_obd; @@ -573,7 +573,7 @@ int filter_commitrw_write(struct obd_export *exp, struct obdo *oa, * decide if it is out of quota or not b=14783 */ lquota_chkquota(filter_quota_interface_ref, obd, oa->o_uid, oa->o_gid, niocount, &rec_pending, oti, - LQUOTA_FLAGS_BLK); + LQUOTA_FLAGS_BLK, inode, obj->ioo_bufcnt); iobuf = filter_iobuf_get(&obd->u.filter, oti); if (IS_ERR(iobuf)) @@ -582,7 +582,6 @@ int filter_commitrw_write(struct obd_export *exp, struct obdo *oa, fso.fso_dentry = res->dentry; fso.fso_bufcnt = obj->ioo_bufcnt; - inode = res->dentry->d_inode; iobuf->dr_ignore_quota = 0; for (i = 0, lnb = res; i < niocount; i++, lnb++) { @@ -721,7 +720,7 @@ int filter_commitrw_write(struct obd_export *exp, struct obdo *oa, cleanup: if (rec_pending) lquota_pending_commit(filter_quota_interface_ref, obd, oa->o_uid, - oa->o_gid, niocount, 1); + oa->o_gid, rec_pending, 1); filter_grant_commit(exp, niocount, res); diff --git a/lustre/quota/quota_context.c b/lustre/quota/quota_context.c index 1e928c3..bb0bbd2 100644 --- a/lustre/quota/quota_context.c +++ b/lustre/quota/quota_context.c @@ -302,7 +302,7 @@ check_cur_qunit(struct obd_device *obd, if (QDATA_IS_BLK(qdata)) { qunit_sz = lqs->lqs_bunit_sz; tune_sz = lqs->lqs_btune_sz; - pending_write = lqs->lqs_bwrite_pending * CFS_PAGE_SIZE; + pending_write = lqs->lqs_bwrite_pending; record = lqs->lqs_blk_rec; LASSERT(!(qunit_sz % QUOTABLOCK_SIZE)); } else { diff --git a/lustre/quota/quota_interface.c b/lustre/quota/quota_interface.c index 5e1c51b..5cf9505 100644 --- a/lustre/quota/quota_interface.c +++ b/lustre/quota/quota_interface.c @@ -240,12 +240,14 @@ static int filter_quota_getflag(struct obd_device *obd, struct obdo *oa) * or inode_create rpc. When need to acquire quota, return QUOTA_RET_ACQUOTA */ static int quota_check_common(struct obd_device *obd, unsigned int uid, - unsigned int gid, int count, int cycle, int isblk) + unsigned int gid, int count, int cycle, int isblk, + struct inode *inode, int frags, int *pending) { struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt; int i; __u32 id[MAXQUOTAS] = { uid, gid }; struct qunit_data qdata[MAXQUOTAS]; + int mb = 0; int rc = 0, rc2[2] = { 0, 0 }; ENTRY; @@ -280,25 +282,37 @@ static int quota_check_common(struct obd_device *obd, unsigned int uid, rc2[i] = compute_remquota(obd, qctxt, &qdata[i], isblk); spin_lock(&lqs->lqs_lock); if (!cycle) { - rc = QUOTA_RET_INC_PENDING; - if (isblk) - lqs->lqs_bwrite_pending += count; - else - lqs->lqs_iwrite_pending += count; + if (isblk) { + *pending = count * CFS_PAGE_SIZE; + /* in order to complete this write, we need extra + * meta blocks. This function can get it through + * data needed to be written b=16542 */ + mb = *pending; + LASSERT(inode && frags > 0); + if (fsfilt_get_mblk(obd, qctxt->lqc_sb, &mb, + inode, frags) < 0) + CDEBUG(D_ERROR, + "can't get extra meta blocks.\n"); + else + *pending += mb; + lqs->lqs_bwrite_pending += *pending; + } else { + *pending = count; + lqs->lqs_iwrite_pending += *pending; + } } if (rc2[i] == QUOTA_RET_OK) { - if (isblk && qdata[i].qd_count < - lqs->lqs_bwrite_pending * CFS_PAGE_SIZE) + if (isblk && qdata[i].qd_count < lqs->lqs_bwrite_pending) rc2[i] = QUOTA_RET_ACQUOTA; if (!isblk && qdata[i].qd_count < lqs->lqs_iwrite_pending) rc2[i] = QUOTA_RET_ACQUOTA; } spin_unlock(&lqs->lqs_lock); - CDEBUG(D_QUOTA, "count: %d, write pending: %lu, qd_count: "LPU64 - ".\n", count, + CDEBUG(D_QUOTA, "count: %d, lqs pending: %lu, qd_count: "LPU64 + ", metablocks: %d, isblk: %d, pending: %d.\n", count, isblk ? lqs->lqs_bwrite_pending : lqs->lqs_iwrite_pending, - qdata[i].qd_count); + qdata[i].qd_count, mb, isblk, *pending); /* When cycle is zero, lqs_*_pending will be changed. We will * get reference of the lqs here and put reference of lqs in @@ -311,7 +325,7 @@ static int quota_check_common(struct obd_device *obd, unsigned int uid, } if (rc2[0] == QUOTA_RET_ACQUOTA || rc2[1] == QUOTA_RET_ACQUOTA) - RETURN(rc | QUOTA_RET_ACQUOTA); + RETURN(QUOTA_RET_ACQUOTA); else RETURN(rc); } @@ -319,7 +333,8 @@ static int quota_check_common(struct obd_device *obd, unsigned int uid, static int quota_chk_acq_common(struct obd_device *obd, unsigned int uid, unsigned int gid, int count, int *pending, quota_acquire acquire, - struct obd_trans_info *oti, int isblk) + struct obd_trans_info *oti, int isblk, + struct inode *inode, int frags) { struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt; struct timeval work_start; @@ -336,7 +351,8 @@ static int quota_chk_acq_common(struct obd_device *obd, unsigned int uid, * have to wait for the completion of in flight dqacq/dqrel, * in order to get enough quota for write b=12588 */ do_gettimeofday(&work_start); - while ((rc = quota_check_common(obd, uid, gid, count, cycle, isblk)) & + while ((rc = quota_check_common(obd, uid, gid, count, cycle, isblk, + inode, frags, pending)) & QUOTA_RET_ACQUOTA) { spin_lock(&qctxt->lqc_lock); @@ -356,9 +372,6 @@ static int quota_chk_acq_common(struct obd_device *obd, unsigned int uid, spin_unlock(&qctxt->lqc_lock); } - if (rc & QUOTA_RET_INC_PENDING) - *pending = 1; - cycle++; if (isblk) OBD_FAIL_TIMEOUT(OBD_FAIL_OST_HOLD_WRITE_RPC, 90); @@ -414,10 +427,6 @@ static int quota_chk_acq_common(struct obd_device *obd, unsigned int uid, CDEBUG(D_QUOTA, "recheck quota with rc: %d, cycle: %d\n", rc, cycle); } - - if (!cycle && rc & QUOTA_RET_INC_PENDING) - *pending = 1; - do_gettimeofday(&work_end); timediff = cfs_timeval_sub(&work_end, &work_start, NULL); lprocfs_counter_add(qctxt->lqc_stats, @@ -433,7 +442,7 @@ static int quota_chk_acq_common(struct obd_device *obd, unsigned int uid, * pending blocks and inodes */ static int quota_pending_commit(struct obd_device *obd, unsigned int uid, - unsigned int gid, int count, int isblk) + unsigned int gid, int pending, int isblk) { struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt; struct timeval work_start; @@ -465,13 +474,10 @@ static int quota_pending_commit(struct obd_device *obd, unsigned int uid, quota_search_lqs(&qdata[i], NULL, qctxt, &lqs); if (lqs) { int flag = 0; - CDEBUG(D_QUOTA, "pending: %lu, count: %d.\n", - isblk ? lqs->lqs_bwrite_pending : - lqs->lqs_iwrite_pending, count); spin_lock(&lqs->lqs_lock); if (isblk) { - if (lqs->lqs_bwrite_pending >= count) { - lqs->lqs_bwrite_pending -= count; + if (lqs->lqs_bwrite_pending >= pending) { + lqs->lqs_bwrite_pending -= pending; spin_unlock(&lqs->lqs_lock); flag = 1; } else { @@ -480,8 +486,8 @@ static int quota_pending_commit(struct obd_device *obd, unsigned int uid, "there are too many blocks!\n"); } } else { - if (lqs->lqs_iwrite_pending >= count) { - lqs->lqs_iwrite_pending -= count; + if (lqs->lqs_iwrite_pending >= pending) { + lqs->lqs_iwrite_pending -= pending; spin_unlock(&lqs->lqs_lock); flag = 1; } else { @@ -490,6 +496,10 @@ static int quota_pending_commit(struct obd_device *obd, unsigned int uid, "there are too many files!\n"); } } + CDEBUG(D_QUOTA, "lqs pending: %lu, pending: %d, " + "isblk: %d.\n", + isblk ? lqs->lqs_bwrite_pending : + lqs->lqs_iwrite_pending, pending, isblk); lqs_putref(lqs); /* When lqs_*_pening is changed back, we'll putref lqs