From 498ee35323fcacf829f4e77c91e7700cb3660111 Mon Sep 17 00:00:00 2001 From: fanyong Date: Fri, 17 Jul 2009 02:17:21 +0000 Subject: [PATCH] Branch HEAD b=20082 i=robert.read i=pravin.shelar Sanity check for connection between client and OSS established before quotacheck b=20152 i=tianzy i=robert.read use "obt->obt_quotachecking" to serializate quotacheck from client and auto_quota_on from MDS itself --- lustre/llite/dir.c | 2 +- lustre/quota/lproc_quota.c | 53 +++++++++++++++++++++------------- lustre/quota/quota_check.c | 21 ++++++++++++++ lustre/quota/quota_context.c | 53 ++++++++++++++++++++++++++++++++++ lustre/quota/quota_ctl.c | 64 ++++++------------------------------------ lustre/quota/quota_internal.h | 1 + lustre/quota/quota_master.c | 22 +++++++++++++-- lustre/tests/sanity-quota.sh | 3 +- lustre/tests/test-framework.sh | 12 ++++---- lustre/utils/lfs.c | 9 ------ 10 files changed, 144 insertions(+), 96 deletions(-) diff --git a/lustre/llite/dir.c b/lustre/llite/dir.c index 3f393ae..2549009 100644 --- a/lustre/llite/dir.c +++ b/lustre/llite/dir.c @@ -1149,7 +1149,7 @@ static int ll_dir_ioctl(struct inode *inode, struct file *file, QCTL_COPY(oqctl, qctl); rc = obd_quotactl(sbi->ll_md_exp, oqctl); if (rc) { - if (rc != -EBUSY && cmd == Q_QUOTAON) { + if (rc != -EALREADY && cmd == Q_QUOTAON) { oqctl->qc_cmd = Q_QUOTAOFF; obd_quotactl(sbi->ll_md_exp, oqctl); } diff --git a/lustre/quota/lproc_quota.c b/lustre/quota/lproc_quota.c index 7e607f9..ce0f8ba 100644 --- a/lustre/quota/lproc_quota.c +++ b/lustre/quota/lproc_quota.c @@ -207,8 +207,10 @@ static int auto_quota_on(struct obd_device *obd, int type, { struct obd_quotactl *oqctl; struct lvfs_run_ctxt saved; - int rc = 0, id; + int rc = 0, rc1 = 0, id; struct obd_device_target *obt = &obd->u.obt; + struct lustre_quota_ctxt *qctxt = &obt->obt_qctxt; + struct mds_obd *mds = NULL; ENTRY; LASSERT(type == USRQUOTA || type == GRPQUOTA || type == UGQUOTA); @@ -234,31 +236,44 @@ static int auto_quota_on(struct obd_device *obd, int type, oqctl->qc_id = obt->obt_qfmt; push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - if (is_master) { - struct mds_obd *mds = &obd->u.mds; + if (is_master) { + mds = &obd->u.mds; down(&mds->mds_qonoff_sem); /* turn on cluster wide quota */ - rc = mds_admin_quota_on(obd, oqctl); - if (rc) - CDEBUG(rc == -ENOENT ? D_QUOTA : D_ERROR, - "auto-enable admin quota failed. rc=%d\n", rc); - up(&mds->mds_qonoff_sem); - + rc1 = mds_admin_quota_on(obd, oqctl); + if (rc1 && rc1 != -EALREADY) { + CDEBUG(rc1 == -ENOENT ? D_QUOTA : D_ERROR, + "auto-enable admin quota failed. rc=%d\n", rc1); + GOTO(out_ctxt, rc1); + } } + + /* turn on local quota */ + rc = fsfilt_quotactl(obd, sb, oqctl); if (!rc) { - /* turn on local quota */ - rc = fsfilt_quotactl(obd, sb, oqctl); - if (rc) - CDEBUG(rc == -ENOENT ? D_QUOTA : D_ERROR, - "auto-enable local quota failed. rc=%d\n", rc); - else - obt->obt_qctxt.lqc_flags |= UGQUOTA2LQC(type); + obt->obt_qctxt.lqc_flags |= UGQUOTA2LQC(type); + build_lqs(obd); + } else if (rc == -EBUSY && quota_is_on(qctxt, oqctl)) { + CWARN("mds local quota[%d] is on already\n", oqctl->qc_type); + rc = -EALREADY; + } else { + CDEBUG(rc == -ENOENT ? D_QUOTA : D_ERROR, + "auto-enable local quota failed. rc=%d\n", rc); + if (rc1 == -EALREADY) { + oqctl->qc_cmd = Q_QUOTAOFF; + mds_admin_quota_off(obd, oqctl); + } + GOTO(out_ctxt, rc); } - pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); EXIT; +out_ctxt: + if (mds != NULL) + up(&mds->mds_qonoff_sem); + pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + out: up(&obt->obt_quotachecking); OBD_FREE_PTR(oqctl); @@ -308,9 +323,7 @@ int lprocfs_quota_wr_type(struct file *file, const char *buffer, if (type != 0) { int rc = auto_quota_on(obd, type - 1, obt->obt_sb, is_mds); - if (rc == 0) - build_lqs(obd); - else if (rc != -EALREADY) + if (rc && rc != -EALREADY) return rc; } diff --git a/lustre/quota/quota_check.c b/lustre/quota/quota_check.c index dab8a89..93b578c 100644 --- a/lustre/quota/quota_check.c +++ b/lustre/quota/quota_check.c @@ -141,13 +141,34 @@ int target_quota_check(struct obd_device *obd, struct obd_export *exp, qta->qta_sb = obt->obt_sb; qta->qta_sem = &obt->obt_quotachecking; + /* quotaoff firstly */ + oqctl->qc_cmd = Q_QUOTAOFF; if (!strcmp(obd->obd_type->typ_name, LUSTRE_MDS_NAME)) { + rc = do_mds_quota_off(obd, oqctl); + if (rc && rc != -EALREADY) { + CERROR("off quota on MDS failed: %d\n", rc); + GOTO(out, rc); + } + /* quota master */ rc = init_admin_quotafiles(obd, &qta->qta_oqctl); if (rc) { CERROR("init_admin_quotafiles failed: %d\n", rc); GOTO(out, rc); } + } else { + struct lvfs_run_ctxt saved; + struct lustre_quota_ctxt *qctxt = &obt->obt_qctxt; + + push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + rc = fsfilt_quotactl(obd, obt->obt_sb, oqctl); + pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + if (!rc) { + qctxt->lqc_flags &= ~UGQUOTA2LQC(oqctl->qc_type); + } else if (!quota_is_off(qctxt, oqctl)) { + CERROR("off quota on OSS failed: %d\n", rc); + GOTO(out, rc); + } } /* we get ref for exp because target_quotacheck_callback() will use this diff --git a/lustre/quota/quota_context.c b/lustre/quota/quota_context.c index dec240a..01f09e9 100644 --- a/lustre/quota/quota_context.c +++ b/lustre/quota/quota_context.c @@ -1478,6 +1478,59 @@ int quota_is_off(struct lustre_quota_ctxt *qctxt, struct obd_quotactl *oqctl) } /** + * When quotaon, build a lqs for every uid/gid who has been set limitation + * for quota. After quota_search_lqs, it will hold one ref for the lqs. + * It will be released when qctxt_cleanup() is executed b=18574 + * + * Should be called with obt->obt_quotachecking held. b=20152 + */ +void build_lqs(struct obd_device *obd) +{ + struct obd_device_target *obt = &obd->u.obt; + struct lustre_quota_ctxt *qctxt = &obt->obt_qctxt; + struct list_head id_list; + int i, rc; + + LASSERT_SEM_LOCKED(&obt->obt_quotachecking); + INIT_LIST_HEAD(&id_list); + for (i = 0; i < MAXQUOTAS; i++) { + struct dquot_id *dqid, *tmp; + +#ifndef KERNEL_SUPPORTS_QUOTA_READ + rc = fsfilt_qids(obd, sb_dqopt(qctxt->lqc_sb)->files[i], NULL, + i, &id_list); +#else + rc = fsfilt_qids(obd, NULL, sb_dqopt(qctxt->lqc_sb)->files[i], + i, &id_list); +#endif + if (rc) { + CDEBUG(D_ERROR, "fail to get %s qids!\n", + i ? "group" : "user"); + continue; + } + + list_for_each_entry_safe(dqid, tmp, &id_list, + di_link) { + struct lustre_qunit_size *lqs; + + list_del_init(&dqid->di_link); + lqs = quota_search_lqs(LQS_KEY(i, dqid->di_id), + qctxt, 1); + if (lqs && !IS_ERR(lqs)) { + lqs->lqs_flags |= dqid->di_flag; + lqs_putref(lqs); + } else { + CDEBUG(D_ERROR, "fail to create a lqs" + "(%s id: %u)!\n", i ? "group" : "user", + dqid->di_id); + } + + OBD_FREE_PTR(dqid); + } + } +} + +/** * lqs<->qctxt hash operations */ diff --git a/lustre/quota/quota_ctl.c b/lustre/quota/quota_ctl.c index cae30c4..75ab78d 100644 --- a/lustre/quota/quota_ctl.c +++ b/lustre/quota/quota_ctl.c @@ -65,53 +65,6 @@ #ifdef HAVE_QUOTA_SUPPORT #ifdef __KERNEL__ -/* When quotaon, build a lqs for every uid/gid who has been set limitation - * for quota. After quota_search_lqs, it will hold one ref for the lqs. - * It will be released when qctxt_cleanup() is executed b=18574 */ -void build_lqs(struct obd_device *obd) -{ - struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt; - struct list_head id_list; - int i, rc; - - INIT_LIST_HEAD(&id_list); - for (i = 0; i < MAXQUOTAS; i++) { - struct dquot_id *dqid, *tmp; - -#ifndef KERNEL_SUPPORTS_QUOTA_READ - rc = fsfilt_qids(obd, sb_dqopt(qctxt->lqc_sb)->files[i], NULL, - i, &id_list); -#else - rc = fsfilt_qids(obd, NULL, sb_dqopt(qctxt->lqc_sb)->files[i], - i, &id_list); -#endif - if (rc) { - CDEBUG(D_ERROR, "fail to get %s qids!\n", - i ? "group" : "user"); - continue; - } - - list_for_each_entry_safe(dqid, tmp, &id_list, - di_link) { - struct lustre_qunit_size *lqs; - - list_del_init(&dqid->di_link); - lqs = quota_search_lqs(LQS_KEY(i, dqid->di_id), - qctxt, 1); - if (lqs && !IS_ERR(lqs)) { - lqs->lqs_flags |= dqid->di_flag; - lqs_putref(lqs); - } else { - CDEBUG(D_ERROR, "fail to create a lqs" - "(%s id: %u)!\n", i ? "group" : "user", - dqid->di_id); - } - - OBD_FREE_PTR(dqid); - } - } -} - int mds_quota_ctl(struct obd_device *obd, struct obd_export *unused, struct obd_quotactl *oqctl) { @@ -128,8 +81,6 @@ int mds_quota_ctl(struct obd_device *obd, struct obd_export *unused, case Q_QUOTAON: oqctl->qc_id = obt->obt_qfmt; /* override qfmt version */ rc = mds_quota_on(obd, oqctl); - /* when quotaon, create lqs for every quota uid/gid b=18574 */ - build_lqs(obd); break; case Q_QUOTAOFF: oqctl->qc_id = obt->obt_qfmt; /* override qfmt version */ @@ -181,7 +132,7 @@ int filter_quota_ctl(struct obd_device *unused, struct obd_export *exp, struct obd_device *obd = exp->exp_obd; struct obd_device_target *obt = &obd->u.obt; struct lvfs_run_ctxt saved; - struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt; + struct lustre_quota_ctxt *qctxt = &obt->obt_qctxt; struct lustre_qunit_size *lqs; void *handle = NULL; struct timeval work_start; @@ -224,12 +175,16 @@ int filter_quota_ctl(struct obd_device *unused, struct obd_export *exp, if (oqctl->qc_cmd == Q_QUOTAON || oqctl->qc_cmd == Q_QUOTAOFF || oqctl->qc_cmd == Q_FINVALIDATE) { if (oqctl->qc_cmd == Q_QUOTAON) { - if (!rc) + if (!rc) { obt->obt_qctxt.lqc_flags |= UGQUOTA2LQC(oqctl->qc_type); - else if (rc == -EBUSY && - quota_is_on(qctxt, oqctl)) + /* when quotaon, create lqs for every + * quota uid/gid b=18574 */ + build_lqs(obd); + } else if (rc == -EBUSY && + quota_is_on(qctxt, oqctl)) { rc = -EALREADY; + } } else if (oqctl->qc_cmd == Q_QUOTAOFF) { if (!rc) obt->obt_qctxt.lqc_flags &= @@ -240,9 +195,6 @@ int filter_quota_ctl(struct obd_device *unused, struct obd_export *exp, up(&obt->obt_quotachecking); } - /* when quotaon, create lqs for every quota uid/gid b=18574 */ - if (oqctl->qc_cmd == Q_QUOTAON) - build_lqs(obd); break; case Q_SETQUOTA: /* currently, it is only used for nullifying the quota */ diff --git a/lustre/quota/quota_internal.h b/lustre/quota/quota_internal.h index 54dc220..a6db1ec 100644 --- a/lustre/quota/quota_internal.h +++ b/lustre/quota/quota_internal.h @@ -135,6 +135,7 @@ int mds_quota_finvalidate(struct obd_device *obd, struct obd_quotactl *oqctl); int mds_admin_quota_on(struct obd_device *obd, struct obd_quotactl *oqctl); int mds_quota_on(struct obd_device *obd, struct obd_quotactl *oqctl); int mds_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl); +int do_mds_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl); int mds_admin_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl); int mds_set_dqinfo(struct obd_device *obd, struct obd_quotactl *oqctl); int mds_get_dqinfo(struct obd_device *obd, struct obd_quotactl *oqctl); diff --git a/lustre/quota/quota_master.c b/lustre/quota/quota_master.c index 21265c2..51b6d65 100644 --- a/lustre/quota/quota_master.c +++ b/lustre/quota/quota_master.c @@ -872,6 +872,8 @@ int mds_quota_on(struct obd_device *obd, struct obd_quotactl *oqctl) rc1 = fsfilt_quotactl(obd, obd->u.obt.obt_sb, oqctl); if (!rc1) { qctxt->lqc_flags |= UGQUOTA2LQC(oqctl->qc_type); + /* when quotaon, create lqs for every quota uid/gid b=18574 */ + build_lqs(obd); } else if (rc1 == -EBUSY && quota_is_on(qctxt, oqctl)) { CWARN("mds local quota[%d] is on already\n", oqctl->qc_type); rc1 = -EALREADY; @@ -899,6 +901,7 @@ int mds_quota_on(struct obd_device *obd, struct obd_quotactl *oqctl) } oqctl->qc_cmd = Q_QUOTAON; } + EXIT; out: @@ -908,7 +911,8 @@ out: return rc ? : (rc1 ? : rc2); } -int mds_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl) +/* with obt->obt_quotachecking held */ +int do_mds_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl) { struct mds_obd *mds = &obd->u.mds; struct obd_device_target *obt = &obd->u.obt; @@ -917,6 +921,8 @@ int mds_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl) int rc = 0, rc1 = 0, rc2 = 0, imm; ENTRY; + LASSERT_SEM_LOCKED(&obt->obt_quotachecking); + imm = oqctl->qc_type & IMMQUOTA; oqctl->qc_type &= ~IMMQUOTA; @@ -925,7 +931,6 @@ int mds_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl) oqctl->qc_type != UGQUOTA) RETURN(-EINVAL); - down(&obt->obt_quotachecking); push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); down(&mds->mds_qonoff_sem); /* close admin quota files */ @@ -974,10 +979,21 @@ int mds_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl) out: up(&mds->mds_qonoff_sem); pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - up(&obt->obt_quotachecking); return rc ? : (rc1 ? : rc2); } +int mds_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl) +{ + struct obd_device_target *obt = &obd->u.obt; + int rc; + ENTRY; + + down(&obt->obt_quotachecking); + rc = do_mds_quota_off(obd, oqctl); + up(&obt->obt_quotachecking); + RETURN(rc); +} + int mds_set_dqinfo(struct obd_device *obd, struct obd_quotactl *oqctl) { struct mds_obd *mds = &obd->u.mds; diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index 31311bd..9b4c876 100644 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -261,7 +261,6 @@ quota_show_check() { # set quota quota_init() { - $LFS quotaoff -ug $DIR $LFS quotacheck -ug $DIR resetquota -u $TSTUSR @@ -2106,7 +2105,7 @@ run_test_with_stat 28 "test for consistency for qunit when setquota (18574) ==== quota_fini() { $LFS quotaoff $DIR - do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=-quota" + do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=-quota" } quota_fini diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 44141c2..4dc3cae 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -527,7 +527,6 @@ quota_save_version() { [ -n "$ver" -a "$ver" != "3" ] && error "wrong quota version specifier" - $LFS quotaoff -ug $MOUNT # just in case [ -n "$type" ] && { $LFS quotacheck -$type $MOUNT || error "quotacheck has failed"; } do_facet mgs "lctl conf_param ${fsname}-MDT*.mdd.quota_type=$spec" @@ -1734,12 +1733,14 @@ init_facets_vars () { done } -mds_sanity_check () { - local timeout=$1 +osc_ensure_active () { + local facet=$1 + local type=$2 + local timeout=$3 local period=0 while [ $period -lt $timeout ]; do - count=$(do_facet $SINGLEMDS "lctl dl | grep 'osc.*mdtlov_UUID' | grep ' IN ' 2>/dev/null | wc -l") + count=$(do_facet $facet "lctl dl | grep '${FSNAME}-OST.*-osc-${type}' | grep ' IN ' 2>/dev/null | wc -l") if [ $count -eq 0 ]; then break fi @@ -1769,7 +1770,8 @@ init_param_vars () { log "Using TIMEOUT=$TIMEOUT" - mds_sanity_check $TIMEOUT + osc_ensure_active $SINGLEMDS M $TIMEOUT + osc_ensure_active client c $TIMEOUT if [ x"$(som_check)" = x"enabled" ]; then ENABLE_QUOTA="" diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 69433cd..816d921 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -1414,15 +1414,6 @@ static int lfs_quotacheck(int argc, char **argv) mnt = argv[optind]; - memset(&qctl, 0, sizeof(qctl)); - qctl.qc_cmd = LUSTRE_Q_QUOTAOFF; - qctl.qc_type = check_type; - rc = llapi_quotactl(mnt, &qctl); - if (rc && errno != EALREADY) { - fprintf(stderr, "quota off failed: %s\n", strerror(errno)); - return rc; - } - rc = llapi_quotacheck(mnt, check_type); if (rc) { fprintf(stderr, "quotacheck failed: %s\n", strerror(errno)); -- 1.8.3.1