From: fanyong Date: Wed, 4 Nov 2009 02:20:51 +0000 (+0000) Subject: b=20530 X-Git-Tag: GIT_EPOCH_B_HD_KDMU~128 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=9587ede5df1cd99c74dd732b84d885106af57ca0 b=20530 i=johann i=tianzy In quota slaves, quota_chk_acq_common() shouldn't retry indefinitely if relative clients are evicted or reconnect already. --- diff --git a/lustre/cmm/cmm_device.c b/lustre/cmm/cmm_device.c index deda49e..8867fd8 100644 --- a/lustre/cmm/cmm_device.c +++ b/lustre/cmm/cmm_device.c @@ -190,7 +190,7 @@ static int cmm_quota_recovery(const struct lu_env *env, struct md_device *m) } static int cmm_quota_check(const struct lu_env *env, struct md_device *m, - struct obd_export *exp, __u32 type) + __u32 type) { struct cmm_device *cmm_dev = md2cmm_dev(m); int rc; @@ -202,7 +202,7 @@ static int cmm_quota_check(const struct lu_env *env, struct md_device *m, rc = cmm_child_ops(cmm_dev)->mdo_quota.mqo_check(env, cmm_dev->cmm_child, - exp, type); + type); RETURN(rc); } diff --git a/lustre/include/lustre_quota.h b/lustre/include/lustre_quota.h index 7179ed8..5331f7c 100644 --- a/lustre/include/lustre_quota.h +++ b/lustre/include/lustre_quota.h @@ -526,10 +526,10 @@ typedef struct { * can finish a block_write or inode_create rpc. It updates the pending * record of block and inode, acquires quota if necessary */ - int (*quota_chkquota) (struct obd_device *, const unsigned int [], - int [], int, quota_acquire, - struct obd_trans_info *, int, struct inode *, - int); + int (*quota_chkquota) (struct obd_device *, struct obd_export *, + const unsigned int [], int [], + int, quota_acquire, struct obd_trans_info *, + int, struct inode *, int); /** * For quota client, the actions after the pending write is committed @@ -765,6 +765,7 @@ static inline int lquota_getflag(quota_interface_t *interface, #ifdef __KERNEL__ static inline int lquota_chkquota(quota_interface_t *interface, struct obd_device *obd, + struct obd_export *exp, const unsigned int id[], int pending[], int count, struct obd_trans_info *oti, int isblk, void *data, int frags) @@ -774,7 +775,7 @@ static inline int lquota_chkquota(quota_interface_t *interface, QUOTA_CHECK_OP(interface, chkquota); QUOTA_CHECK_OP(interface, acquire); - rc = QUOTA_OP(interface, chkquota)(obd, id, pending, count, + rc = QUOTA_OP(interface, chkquota)(obd, exp, id, pending, count, QUOTA_OP(interface, acquire), oti, isblk, (struct inode *)data, frags); RETURN(rc); diff --git a/lustre/include/md_object.h b/lustre/include/md_object.h index 6f818b5..bfa7de4 100644 --- a/lustre/include/md_object.h +++ b/lustre/include/md_object.h @@ -101,6 +101,10 @@ struct md_capainfo { struct lustre_capa *mc_capa[MD_CAPAINFO_MAX]; }; +struct md_quota { + struct obd_export *mq_exp; +}; + /** * Implemented in mdd/mdd_handler.c. * @@ -109,6 +113,7 @@ struct md_capainfo { */ struct md_ucred *md_ucred(const struct lu_env *env); struct md_capainfo *md_capainfo(const struct lu_env *env); +struct md_quota *md_quota(const struct lu_env *env); /** metadata attributes */ enum ma_valid { @@ -352,7 +357,6 @@ struct md_device_operations { int (*mqo_check)(const struct lu_env *env, struct md_device *m, - struct obd_export *exp, __u32 type); int (*mqo_on)(const struct lu_env *env, diff --git a/lustre/mdd/mdd_device.c b/lustre/mdd/mdd_device.c index 48da062..7dc67d1 100644 --- a/lustre/mdd/mdd_device.c +++ b/lustre/mdd/mdd_device.c @@ -1270,6 +1270,25 @@ struct md_capainfo *md_capainfo(const struct lu_env *env) } EXPORT_SYMBOL(md_capainfo); +/* + * context key constructor/destructor: + * mdd_quota_key_init, mdd_quota_key_fini + */ +LU_KEY_INIT_FINI(mdd_quota, struct md_quota); + +struct lu_context_key mdd_quota_key = { + .lct_tags = LCT_SESSION, + .lct_init = mdd_quota_key_init, + .lct_fini = mdd_quota_key_fini +}; + +struct md_quota *md_quota(const struct lu_env *env) +{ + LASSERT(env->le_ses != NULL); + return lu_context_key_get(env->le_ses, &mdd_quota_key); +} +EXPORT_SYMBOL(md_quota); + static int mdd_changelog_user_register(struct mdd_device *mdd, int *id) { struct llog_ctxt *ctxt; @@ -1491,7 +1510,8 @@ static int mdd_iocontrol(const struct lu_env *env, struct md_device *m, } /* type constructor/destructor: mdd_type_init, mdd_type_fini */ -LU_TYPE_INIT_FINI(mdd, &mdd_thread_key, &mdd_ucred_key, &mdd_capainfo_key); +LU_TYPE_INIT_FINI(mdd, &mdd_thread_key, &mdd_ucred_key, &mdd_capainfo_key, + &mdd_quota_key); const struct md_device_operations mdd_ops = { .mdo_statfs = mdd_statfs, diff --git a/lustre/mdd/mdd_dir.c b/lustre/mdd/mdd_dir.c index e053f61..fb0c0f3 100644 --- a/lustre/mdd/mdd_dir.c +++ b/lustre/mdd/mdd_dir.c @@ -680,6 +680,7 @@ static int mdd_link(const struct lu_env *env, struct md_object *tgt_obj, struct thandle *handle; #ifdef HAVE_QUOTA_SUPPORT struct obd_device *obd = mdd->mdd_obd_dev; + struct obd_export *exp = md_quota(env)->mq_exp; struct mds_obd *mds = &obd->u.mds; unsigned int qids[MAXQUOTAS] = { 0, 0 }; int quota_opc = 0, rec_pending[MAXQUOTAS] = { 0, 0 }; @@ -698,7 +699,7 @@ static int mdd_link(const struct lu_env *env, struct md_object *tgt_obj, quota_opc = FSFILT_OP_LINK; mdd_quota_wrapper(la_tmp, qids); /* get block quota for parent */ - lquota_chkquota(mds_quota_interface_ref, obd, + lquota_chkquota(mds_quota_interface_ref, obd, exp, qids, rec_pending, 1, NULL, LQUOTA_FLAGS_BLK, data, 1); } @@ -966,6 +967,7 @@ static int mdd_name_insert(const struct lu_env *env, #ifdef HAVE_QUOTA_SUPPORT struct md_ucred *uc = md_ucred(env); struct obd_device *obd = mdd->mdd_obd_dev; + struct obd_export *exp = md_quota(env)->mq_exp; struct mds_obd *mds = &obd->u.mds; unsigned int qids[MAXQUOTAS] = { 0, 0 }; int quota_opc = 0, rec_pending[MAXQUOTAS] = { 0, 0 }; @@ -987,7 +989,7 @@ static int mdd_name_insert(const struct lu_env *env, mdd_quota_wrapper(la_tmp, qids); /* get block quota for parent */ lquota_chkquota(mds_quota_interface_ref, obd, - qids, rec_pending, 1, NULL, + exp, qids, rec_pending, 1, NULL, LQUOTA_FLAGS_BLK, data, 1); } } else { @@ -1189,6 +1191,7 @@ static int mdd_rename_tgt(const struct lu_env *env, struct thandle *handle; #ifdef HAVE_QUOTA_SUPPORT struct obd_device *obd = mdd->mdd_obd_dev; + struct obd_export *exp = md_quota(env)->mq_exp; struct mds_obd *mds = &obd->u.mds; unsigned int qcids[MAXQUOTAS] = { 0, 0 }; unsigned int qpids[MAXQUOTAS] = { 0, 0 }; @@ -1209,7 +1212,7 @@ static int mdd_rename_tgt(const struct lu_env *env, quota_popc = FSFILT_OP_LINK; mdd_quota_wrapper(la_tmp, qpids); /* get block quota for target parent */ - lquota_chkquota(mds_quota_interface_ref, obd, + lquota_chkquota(mds_quota_interface_ref, obd, exp, qpids, rec_pending, 1, NULL, LQUOTA_FLAGS_BLK, data, 1); } @@ -1587,6 +1590,7 @@ static int mdd_create(const struct lu_env *env, int got_def_acl = 0; #ifdef HAVE_QUOTA_SUPPORT struct obd_device *obd = mdd->mdd_obd_dev; + struct obd_export *exp = md_quota(env)->mq_exp; struct mds_obd *mds = &obd->u.mds; unsigned int qcids[MAXQUOTAS] = { 0, 0 }; unsigned int qpids[MAXQUOTAS] = { 0, 0 }; @@ -1650,8 +1654,9 @@ static int mdd_create(const struct lu_env *env, mdd_quota_wrapper(&ma->ma_attr, qcids); mdd_quota_wrapper(la_tmp, qpids); /* get file quota for child */ - lquota_chkquota(mds_quota_interface_ref, obd, qcids, - inode_pending, 1, NULL, 0, NULL, 0); + lquota_chkquota(mds_quota_interface_ref, obd, exp, + qcids, inode_pending, 1, NULL, 0, NULL, + 0); switch (ma->ma_attr.la_mode & S_IFMT) { case S_IFLNK: case S_IFDIR: @@ -1669,13 +1674,14 @@ static int mdd_create(const struct lu_env *env, /* get block quota for child and parent */ if (block_count) lquota_chkquota(mds_quota_interface_ref, obd, - qcids, block_pending, + exp, qcids, block_pending, block_count, NULL, LQUOTA_FLAGS_BLK, NULL, 0); if (!same) lquota_chkquota(mds_quota_interface_ref, obd, - qpids, parent_pending, 1, NULL, - LQUOTA_FLAGS_BLK, NULL, 0); + exp, qpids, parent_pending, 1, + NULL, LQUOTA_FLAGS_BLK, NULL, + 0); } } #endif @@ -1967,6 +1973,7 @@ static int mdd_rename(const struct lu_env *env, #ifdef HAVE_QUOTA_SUPPORT struct obd_device *obd = mdd->mdd_obd_dev; + struct obd_export *exp = md_quota(env)->mq_exp; struct mds_obd *mds = &obd->u.mds; unsigned int qspids[MAXQUOTAS] = { 0, 0 }; unsigned int qtcids[MAXQUOTAS] = { 0, 0 }; @@ -1999,7 +2006,7 @@ static int mdd_rename(const struct lu_env *env, mdd_quota_wrapper(la_tmp, qtpids); /* get block quota for target parent */ lquota_chkquota(mds_quota_interface_ref, - obd, qtpids, + obd, exp, qtpids, rec_pending, 1, NULL, LQUOTA_FLAGS_BLK, data, 1); diff --git a/lustre/mdd/mdd_internal.h b/lustre/mdd/mdd_internal.h index 65ee175..2b79611 100644 --- a/lustre/mdd/mdd_internal.h +++ b/lustre/mdd/mdd_internal.h @@ -387,7 +387,7 @@ int mdd_quota_setup(const struct lu_env *env, struct md_device *m, int mdd_quota_cleanup(const struct lu_env *env, struct md_device *m); int mdd_quota_recovery(const struct lu_env *env, struct md_device *m); int mdd_quota_check(const struct lu_env *env, struct md_device *m, - struct obd_export *exp, __u32 type); + __u32 type); int mdd_quota_on(const struct lu_env *env, struct md_device *m, __u32 type); int mdd_quota_off(const struct lu_env *env, struct md_device *m, diff --git a/lustre/mdd/mdd_object.c b/lustre/mdd/mdd_object.c index 29a6d79..55b9f7b 100644 --- a/lustre/mdd/mdd_object.c +++ b/lustre/mdd/mdd_object.c @@ -1229,6 +1229,7 @@ static int mdd_attr_set(const struct lu_env *env, struct md_object *obj, struct lu_attr *la_copy = &mdd_env_info(env)->mti_la_for_fix; #ifdef HAVE_QUOTA_SUPPORT struct obd_device *obd = mdd->mdd_obd_dev; + struct obd_export *exp = md_quota(env)->mq_exp; struct mds_obd *mds = &obd->u.mds; unsigned int qnids[MAXQUOTAS] = { 0, 0 }; unsigned int qoids[MAXQUOTAS] = { 0, 0 }; @@ -1278,15 +1279,16 @@ static int mdd_attr_set(const struct lu_env *env, struct md_object *obj, mdd_quota_wrapper(la_copy, qnids); mdd_quota_wrapper(la_tmp, qoids); /* get file quota for new owner */ - lquota_chkquota(mds_quota_interface_ref, obd, qnids, - inode_pending, 1, NULL, 0, NULL, 0); + lquota_chkquota(mds_quota_interface_ref, obd, exp, + qnids, inode_pending, 1, NULL, 0, + NULL, 0); block_count = (la_tmp->la_blocks + 7) >> 3; if (block_count) { void *data = NULL; mdd_data_get(env, mdd_obj, &data); /* get block quota for new owner */ lquota_chkquota(mds_quota_interface_ref, obd, - qnids, block_pending, + exp, qnids, block_pending, block_count, NULL, LQUOTA_FLAGS_BLK, data, 1); } @@ -1585,6 +1587,7 @@ static int mdd_object_create(const struct lu_env *env, struct thandle *handle; #ifdef HAVE_QUOTA_SUPPORT struct obd_device *obd = mdd->mdd_obd_dev; + struct obd_export *exp = md_quota(env)->mq_exp; struct mds_obd *mds = &obd->u.mds; unsigned int qids[MAXQUOTAS] = { 0, 0 }; int quota_opc = 0, block_count = 0; @@ -1599,8 +1602,9 @@ static int mdd_object_create(const struct lu_env *env, quota_opc = FSFILT_OP_CREATE_PARTIAL_CHILD; mdd_quota_wrapper(&ma->ma_attr, qids); /* get file quota for child */ - lquota_chkquota(mds_quota_interface_ref, obd, qids, - inode_pending, 1, NULL, 0, NULL, 0); + lquota_chkquota(mds_quota_interface_ref, obd, exp, + qids, inode_pending, 1, NULL, 0, + NULL, 0); switch (ma->ma_attr.la_mode & S_IFMT) { case S_IFLNK: case S_IFDIR: @@ -1612,9 +1616,9 @@ static int mdd_object_create(const struct lu_env *env, } /* get block quota for child */ if (block_count) - lquota_chkquota(mds_quota_interface_ref, obd, qids, - block_pending, block_count, NULL, - LQUOTA_FLAGS_BLK, NULL, 0); + lquota_chkquota(mds_quota_interface_ref, obd, exp, + qids, block_pending, block_count, + NULL, LQUOTA_FLAGS_BLK, NULL, 0); } #endif diff --git a/lustre/mdd/mdd_quota.c b/lustre/mdd/mdd_quota.c index d246a07..0601b2e 100644 --- a/lustre/mdd/mdd_quota.c +++ b/lustre/mdd/mdd_quota.c @@ -93,10 +93,11 @@ int mdd_quota_recovery(const struct lu_env *env, struct md_device *m) } int mdd_quota_check(const struct lu_env *env, struct md_device *m, - struct obd_export *exp, __u32 type) + __u32 type) { struct mdd_device *mdd = lu2mdd_dev(&m->md_lu_dev); struct obd_device *obd = mdd->mdd_obd_dev; + struct obd_export *exp = md_quota(env)->mq_exp; struct obd_quotactl *oqctl = &mdd_env_info(env)->mti_oqctl; int rc; ENTRY; diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 436523c..760b0cb 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -1490,6 +1490,7 @@ static int mdt_reint_internal(struct mdt_thread_info *info, { struct req_capsule *pill = info->mti_pill; struct mdt_device *mdt = info->mti_mdt; + struct md_quota *mq = md_quota(info->mti_env); struct mdt_body *repbody; int rc = 0; ENTRY; @@ -1548,6 +1549,7 @@ static int mdt_reint_internal(struct mdt_thread_info *info, rc = lustre_msg_get_status(mdt_info_req(info)->rq_repmsg); GOTO(out_ucred, rc); } + mq->mq_exp = info->mti_exp; rc = mdt_reint_rec(info, lhc); EXIT; out_ucred: @@ -1697,6 +1699,7 @@ static int mdt_quotacheck_handle(struct mdt_thread_info *info) struct obd_quotactl *oqctl; struct req_capsule *pill = info->mti_pill; struct obd_export *exp = info->mti_exp; + struct md_quota *mq = md_quota(info->mti_env); struct md_device *next = info->mti_mdt->mdt_child; int rc; ENTRY; @@ -1713,7 +1716,8 @@ static int mdt_quotacheck_handle(struct mdt_thread_info *info) if (rc) RETURN(rc); - rc = next->md_ops->mdo_quota.mqo_check(info->mti_env, next, exp, + mq->mq_exp = exp; + rc = next->md_ops->mdo_quota.mqo_check(info->mti_env, next, oqctl->qc_type); RETURN(rc); } @@ -1723,6 +1727,7 @@ static int mdt_quotactl_handle(struct mdt_thread_info *info) struct obd_quotactl *oqctl, *repoqc; struct req_capsule *pill = info->mti_pill; struct obd_export *exp = info->mti_exp; + struct md_quota *mq = md_quota(info->mti_env); struct md_device *next = info->mti_mdt->mdt_child; const struct md_quota_operations *mqo = &next->md_ops->mdo_quota; int id, rc; @@ -1766,6 +1771,7 @@ static int mdt_quotactl_handle(struct mdt_thread_info *info) repoqc = req_capsule_server_get(pill, &RMF_OBD_QUOTACTL); LASSERT(repoqc != NULL); + mq->mq_exp = exp; switch (oqctl->qc_cmd) { case Q_QUOTAON: rc = mqo->mqo_on(info->mti_env, next, oqctl->qc_type); diff --git a/lustre/obdfilter/filter_io_26.c b/lustre/obdfilter/filter_io_26.c index 56547b3..72e0d65 100644 --- a/lustre/obdfilter/filter_io_26.c +++ b/lustre/obdfilter/filter_io_26.c @@ -644,10 +644,11 @@ int filter_commitrw_write(struct obd_export *exp, struct obdo *oa, /* we try to get enough quota to write here, and let ldiskfs * decide if it is out of quota or not b=14783 */ - lquota_chkquota(filter_quota_interface_ref, obd, qcids, rec_pending, - quota_pages, oti, LQUOTA_FLAGS_BLK, (void *)inode, - obj->ioo_bufcnt); - + rc = lquota_chkquota(filter_quota_interface_ref, obd, exp, qcids, + rec_pending, quota_pages, oti, LQUOTA_FLAGS_BLK, + (void *)inode, obj->ioo_bufcnt); + if (rc == -ENOTCONN) + GOTO(cleanup, rc); push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); cleanup_phase = 2; diff --git a/lustre/ost/ost_handler.c b/lustre/ost/ost_handler.c index 46c5996..2ee5f2b 100644 --- a/lustre/ost/ost_handler.c +++ b/lustre/ost/ost_handler.c @@ -1086,6 +1086,12 @@ static int ost_brw_write(struct ptlrpc_request *req, struct obd_trans_info *oti) /* Must commit after prep above in all cases */ rc = obd_commitrw(OBD_BRW_WRITE, exp, &repbody->oa, objcount, ioo, remote_nb, npages, local_nb, oti, rc); + if (rc == -ENOTCONN) + /* quota acquire process has been given up because + * either the client has been evicted or the client + * has timed out the request already */ + no_reply = 1; + if (exp_connect_rmtclient(exp)) { repbody->oa.o_uid = o_uid; repbody->oa.o_gid = o_gid; diff --git a/lustre/quota/quota_interface.c b/lustre/quota/quota_interface.c index 3c3b16f..cc103d1 100644 --- a/lustre/quota/quota_interface.c +++ b/lustre/quota/quota_interface.c @@ -396,8 +396,9 @@ int quota_is_set(struct obd_device *obd, const unsigned int id[], int flag) return q_set; } -static int quota_chk_acq_common(struct obd_device *obd, const unsigned int id[], - int pending[], int count, quota_acquire acquire, +static int quota_chk_acq_common(struct obd_device *obd, struct obd_export *exp, + const unsigned int id[], int pending[], + int count, quota_acquire acquire, struct obd_trans_info *oti, int isblk, struct inode *inode, int frags) { @@ -412,6 +413,12 @@ static int quota_chk_acq_common(struct obd_device *obd, const unsigned int id[], if (!quota_is_set(obd, id, isblk ? QB_SET : QI_SET)) RETURN(0); + if (isblk && (exp->exp_failed || exp->exp_abort_active_req)) + /* If the client has been evicted or if it + * timed out and tried to reconnect already, + * abort the request immediately */ + RETURN(-ENOTCONN); + CDEBUG(D_QUOTA, "check quota for %s\n", obd->obd_name); pending[USRQUOTA] = pending[GRPQUOTA] = 0; /* Unfortunately, if quota master is too busy to handle the @@ -468,6 +475,11 @@ static int quota_chk_acq_common(struct obd_device *obd, const unsigned int id[], break; } + if (isblk && (exp->exp_failed || exp->exp_abort_active_req)) + /* The client has been evicted or tried to + * to reconnect already, abort the request */ + RETURN(-ENOTCONN); + /* -EBUSY and others, wait a second and try again */ if (rc < 0) { cfs_waitq_t waitq; @@ -511,6 +523,8 @@ static int quota_chk_acq_common(struct obd_device *obd, const unsigned int id[], LQUOTA_WAIT_FOR_CHK_INO, timediff); + if (rc > 0) + rc = 0; RETURN(rc); } diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index fe14e19..84f6679 100644 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -338,8 +338,10 @@ test_1_sub() { $RUNAS dd if=/dev/zero of=$TESTFILE bs=$BLK_SZ count=$(($LIMIT/2)) || quota_error g $TSTUSR "(grp) write failure, but expect success" etime=`date +%s` delta=$((etime - stime)) - rate=$((BLK_SZ * LIMIT / 2 / delta / 1024)) - [ $rate -gt 1024 ] || error "SLOW IO for $TSTUSR (group): $rate KB/sec" + if [ $delta -gt 0 ]; then + rate=$((BLK_SZ * LIMIT / 2 / delta / 1024)) + [ $rate -gt 1024 ] || error "SLOW IO for $TSTUSR (group): $rate KB/sec" + fi log " Done" log " Write out of block quota ..." # this time maybe cache write, ignore it's failure