X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fquota%2Fquota_master.c;h=05a8dac9bf080226104b7e922afbaecf3830d598;hp=512fc49cb068736aa677ca3cd3630ba0a753358e;hb=82b79742c8e86650d1441828ef727c21e88faff8;hpb=cefa8cda2ba2d288ccaa4ec077a6c627592503ea diff --git a/lustre/quota/quota_master.c b/lustre/quota/quota_master.c index 512fc49..05a8dac 100644 --- a/lustre/quota/quota_master.c +++ b/lustre/quota/quota_master.c @@ -26,7 +26,7 @@ * GPL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved + * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. */ /* @@ -64,9 +64,9 @@ #ifdef HAVE_QUOTA_SUPPORT -/* lock ordering: mds->mds_qonoff_sem > dquot->dq_sem */ -static struct list_head lustre_dquot_hash[NR_DQHASH]; -static spinlock_t dquot_hash_lock = SPIN_LOCK_UNLOCKED; +/* lock ordering: mds->mds_qonoff_sem > dquot->dq_sem > lqs->lqs_lock */ +static cfs_list_t lustre_dquot_hash[NR_DQHASH]; +static cfs_rwlock_t dquot_hash_lock = CFS_RW_LOCK_UNLOCKED; cfs_mem_cache_t *lustre_dquot_cachep; @@ -95,7 +95,7 @@ void lustre_dquot_exit(void) /* FIXME cleanup work ?? */ for (i = 0; i < NR_DQHASH; i++) { - LASSERT(list_empty(lustre_dquot_hash + i)); + LASSERT(cfs_list_empty(lustre_dquot_hash + i)); } if (lustre_dquot_cachep) { int rc; @@ -126,8 +126,7 @@ static struct lustre_dquot *find_dquot(int hashent, struct lustre_dquot *dquot; ENTRY; - LASSERT_SPIN_LOCKED(&dquot_hash_lock); - list_for_each_entry(dquot, &lustre_dquot_hash[hashent], dq_hash) { + cfs_list_for_each_entry(dquot, &lustre_dquot_hash[hashent], dq_hash) { if (dquot->dq_info == lqi && dquot->dq_id == id && dquot->dq_type == type) RETURN(dquot); @@ -141,17 +140,16 @@ static struct lustre_dquot *alloc_dquot(struct lustre_quota_info *lqi, struct lustre_dquot *dquot = NULL; ENTRY; - OBD_SLAB_ALLOC(dquot, lustre_dquot_cachep, CFS_ALLOC_IO, sizeof(*dquot)); + OBD_SLAB_ALLOC_PTR_GFP(dquot, lustre_dquot_cachep, CFS_ALLOC_IO); if (dquot == NULL) RETURN(NULL); CFS_INIT_LIST_HEAD(&dquot->dq_hash); - init_mutex_locked(&dquot->dq_sem); - dquot->dq_refcnt = 1; + cfs_init_mutex_locked(&dquot->dq_sem); + cfs_atomic_set(&dquot->dq_refcnt, 1); dquot->dq_info = lqi; dquot->dq_id = id; dquot->dq_type = type; - dquot->dq_status = DQ_STATUS_AVAIL; RETURN(dquot); } @@ -163,67 +161,87 @@ static void free_dquot(struct lustre_dquot *dquot) static void insert_dquot_nolock(struct lustre_dquot *dquot) { - struct list_head *head = lustre_dquot_hash + + cfs_list_t *head = lustre_dquot_hash + dquot_hashfn(dquot->dq_info, dquot->dq_id, dquot->dq_type); - LASSERT(list_empty(&dquot->dq_hash)); - list_add(&dquot->dq_hash, head); + LASSERT(cfs_list_empty(&dquot->dq_hash)); + cfs_list_add(&dquot->dq_hash, head); } static void remove_dquot_nolock(struct lustre_dquot *dquot) { - LASSERT(!list_empty(&dquot->dq_hash)); - list_del_init(&dquot->dq_hash); + LASSERT(!cfs_list_empty(&dquot->dq_hash)); + cfs_list_del_init(&dquot->dq_hash); } static void lustre_dqput(struct lustre_dquot *dquot) { ENTRY; - spin_lock(&dquot_hash_lock); - LASSERT(dquot->dq_refcnt); - dquot->dq_refcnt--; - if (!dquot->dq_refcnt) { + cfs_write_lock(&dquot_hash_lock); + LASSERT(cfs_atomic_read(&dquot->dq_refcnt)); + cfs_atomic_dec(&dquot->dq_refcnt); + if (cfs_atomic_read(&dquot->dq_refcnt) == 0) { remove_dquot_nolock(dquot); free_dquot(dquot); } - spin_unlock(&dquot_hash_lock); + cfs_write_unlock(&dquot_hash_lock); EXIT; } static struct lustre_dquot *lustre_dqget(struct obd_device *obd, struct lustre_quota_info *lqi, - qid_t id, int type) + qid_t id, int type, int can_fake) { unsigned int hashent = dquot_hashfn(lqi, id, type); struct lustre_dquot *dquot, *empty; + int free_dq = 0; ENTRY; if ((empty = alloc_dquot(lqi, id, type)) == NULL) RETURN(ERR_PTR(-ENOMEM)); - spin_lock(&dquot_hash_lock); + cfs_read_lock(&dquot_hash_lock); if ((dquot = find_dquot(hashent, lqi, id, type)) != NULL) { - dquot->dq_refcnt++; - spin_unlock(&dquot_hash_lock); - free_dquot(empty); + cfs_atomic_inc(&dquot->dq_refcnt); + cfs_read_unlock(&dquot_hash_lock); + free_dq = 1; } else { int rc; - dquot = empty; - insert_dquot_nolock(dquot); - spin_unlock(&dquot_hash_lock); + cfs_read_unlock(&dquot_hash_lock); + dquot = empty; rc = fsfilt_dquot(obd, dquot, QFILE_RD_DQUOT); - up(&dquot->dq_sem); + cfs_up(&dquot->dq_sem); if (rc) { CERROR("can't read dquot from admin quotafile! " "(rc:%d)\n", rc); - lustre_dqput(dquot); + free_dquot(dquot); RETURN(ERR_PTR(rc)); + } else { + cfs_write_lock(&dquot_hash_lock); + if ((dquot = find_dquot(hashent, lqi, id, type)) != NULL) { + cfs_atomic_inc(&dquot->dq_refcnt); + free_dq = 1; + } else { + dquot = empty; + insert_dquot_nolock(dquot); + } + cfs_write_unlock(&dquot_hash_lock); } } LASSERT(dquot); + if (!can_fake && cfs_test_bit(DQ_FAKE_B, &dquot->dq_flags)) { + DQUOT_DEBUG(dquot, "It is a fake dquot: unexpected!\n"); + lustre_dqput(dquot); + dquot = ERR_PTR(-ENOENT); + } + + if (free_dq) + free_dquot(empty); + + RETURN(dquot); } @@ -235,13 +253,13 @@ static void init_oqaq(struct quota_adjust_qunit *oqaq, oqaq->qaq_id = id; oqaq->qaq_flags = type; - quota_search_lqs(NULL, oqaq, qctxt, &lqs); - if (lqs) { - spin_lock(&lqs->lqs_lock); + lqs = quota_search_lqs(LQS_KEY(type, id), qctxt, 0); + if (lqs && !IS_ERR(lqs)) { + cfs_spin_lock(&lqs->lqs_lock); oqaq->qaq_bunit_sz = lqs->lqs_bunit_sz; oqaq->qaq_iunit_sz = lqs->lqs_iunit_sz; oqaq->qaq_flags = lqs->lqs_flags; - spin_unlock(&lqs->lqs_lock); + cfs_spin_unlock(&lqs->lqs_lock); lqs_putref(lqs); } else { CDEBUG(D_QUOTA, "Can't find the lustre qunit size!\n"); @@ -255,11 +273,9 @@ int dqacq_adjust_qunit_sz(struct obd_device *obd, qid_t id, int type, { struct mds_obd *mds = &obd->u.mds; struct lustre_quota_ctxt *qctxt = &mds->mds_obt.obt_qctxt; - struct obd_device *lov_mds_obd = class_exp2obd(mds->mds_osc_exp); - struct lov_obd *lov = &lov_mds_obd->u.lov; - __u32 ost_num = lov->desc.ld_tgt_count, mdt_num = 1; + __u32 ost_num = mds->mds_lov_objid_count, mdt_num = 1; struct quota_adjust_qunit *oqaq = NULL; - unsigned int uid = 0, gid = 0; + unsigned int qid[MAXQUOTAS] = { 0, 0 }; struct lustre_quota_info *info = &mds->mds_quota_info; struct lustre_dquot *dquot = NULL; int adjust_res = 0; @@ -267,15 +283,17 @@ int dqacq_adjust_qunit_sz(struct obd_device *obd, qid_t id, int type, ENTRY; LASSERT(mds); - dquot = lustre_dqget(obd, info, id, type); + cfs_down_read(&mds->mds_qonoff_sem); + dquot = lustre_dqget(obd, info, id, type, 0); if (IS_ERR(dquot)) RETURN(PTR_ERR(dquot)); + cfs_up_read(&mds->mds_qonoff_sem); OBD_ALLOC_PTR(oqaq); if (!oqaq) GOTO(out, rc = -ENOMEM); - down(&dquot->dq_sem); + cfs_down(&dquot->dq_sem); init_oqaq(oqaq, qctxt, id, type); rc = dquot_create_oqaq(qctxt, dquot, ost_num, mdt_num, @@ -283,7 +301,7 @@ int dqacq_adjust_qunit_sz(struct obd_device *obd, qid_t id, int type, LQUOTA_FLAGS_ADJINO, oqaq); if (rc < 0) { - CDEBUG(D_ERROR, "create oqaq failed! (rc:%d)\n", rc); + CERROR("create oqaq failed! (rc:%d)\n", rc); GOTO(out_sem, rc); } QAQ_DEBUG(oqaq, "show oqaq.\n") @@ -296,8 +314,8 @@ int dqacq_adjust_qunit_sz(struct obd_device *obd, qid_t id, int type, if (adjust_res <= 0) { if (adjust_res < 0) { rc = adjust_res; - CDEBUG(D_ERROR, "adjust mds slave's qunit size failed! \ - (rc:%d)\n", rc); + CERROR("adjust mds slave's qunit size failed! " + "(rc:%d)\n", rc); } else { CDEBUG(D_QUOTA, "qunit doesn't need to be adjusted.\n"); } @@ -305,26 +323,26 @@ int dqacq_adjust_qunit_sz(struct obd_device *obd, qid_t id, int type, } if (type) - gid = dquot->dq_id; + qid[GRPQUOTA] = dquot->dq_id; else - uid = dquot->dq_id; + qid[USRQUOTA] = dquot->dq_id; - up(&dquot->dq_sem); + cfs_up(&dquot->dq_sem); - rc = qctxt_adjust_qunit(obd, qctxt, uid, gid, is_blk, 0, NULL); + rc = qctxt_adjust_qunit(obd, qctxt, qid, is_blk, 0, NULL); if (rc == -EDQUOT || rc == -EBUSY) { CDEBUG(D_QUOTA, "rc: %d.\n", rc); rc = 0; } if (rc) { - CDEBUG(D_ERROR, "mds fail to adjust file quota! \ - (rc:%d)\n", rc); + CERROR("%s: mds fail to adjust file quota! (rc:%d)\n", + obd->obd_name, rc); GOTO(out, rc); } /* only when block qunit is reduced, boardcast to osts */ if ((adjust_res & LQS_BLK_DECREASE) && QAQ_IS_ADJBLK(oqaq)) - rc = obd_quota_adjust_qunit(mds->mds_osc_exp, oqaq, qctxt); + rc = obd_quota_adjust_qunit(mds->mds_lov_exp, oqaq, qctxt, NULL); out: lustre_dqput(dquot); @@ -333,7 +351,7 @@ out: RETURN(rc); out_sem: - up(&dquot->dq_sem); + cfs_up(&dquot->dq_sem); goto out; } @@ -354,21 +372,39 @@ int dqacq_handler(struct obd_device *obd, struct qunit_data *qdata, int opc) if (OBD_FAIL_CHECK(OBD_FAIL_OBD_DQACQ)) RETURN(-EIO); - dquot = lustre_dqget(obd, info, qdata->qd_id, QDATA_IS_GRP(qdata)); - if (IS_ERR(dquot)) - RETURN(PTR_ERR(dquot)); + if (!ll_sb_has_quota_active(qctxt->lqc_sb, + QDATA_IS_GRP(qdata) ? GRPQUOTA : USRQUOTA)) + RETURN(-EIO); - DQUOT_DEBUG(dquot, "get dquot in dqacq_handler\n"); - QINFO_DEBUG(dquot->dq_info, "get dquot in dqadq_handler\n"); + lqs = quota_search_lqs(LQS_KEY(QDATA_IS_GRP(qdata), qdata->qd_id), + qctxt, 0); + if (lqs == NULL) + rc = -ENOENT; + if (IS_ERR(lqs)) + rc = PTR_ERR(lqs); + if (rc) + RETURN(rc); - down(&mds->mds_qonoff_sem); - down(&dquot->dq_sem); + cfs_spin_lock(&lqs->lqs_lock); + if (LQS_IS_RECOVERY(lqs)) { + cfs_spin_unlock(&lqs->lqs_lock); + LQS_DEBUG(lqs, "this lqs is under recovery\n"); + GOTO(skip, rc = -EBUSY); + } + cfs_spin_unlock(&lqs->lqs_lock); - if (dquot->dq_status & DQ_STATUS_RECOVERY) { - DQUOT_DEBUG(dquot, "this dquot is under recovering.\n"); - GOTO(out, rc = -EBUSY); + cfs_down_write(&mds->mds_qonoff_sem); + dquot = lustre_dqget(obd, info, qdata->qd_id, QDATA_IS_GRP(qdata), 0); + if (IS_ERR(dquot)) { + cfs_up_write(&mds->mds_qonoff_sem); + GOTO(skip, rc = PTR_ERR(dquot)); } + DQUOT_DEBUG(dquot, "get dquot in dqacq_handler\n"); + QINFO_DEBUG(dquot->dq_info, "get dquot in dqadq_handler\n"); + + cfs_down(&dquot->dq_sem); + if (QDATA_IS_BLK(qdata)) { grace = info->qi_info[QDATA_IS_GRP(qdata)].dqi_bgrace; usage = &dquot->dq_dqb.dqb_curspace; @@ -418,7 +454,7 @@ int dqacq_handler(struct obd_device *obd, struct qunit_data *qdata, int opc) case QUOTA_DQREL: /* The usage in administrative file might be incorrect before * recovery done */ - if (*usage - qdata->qd_count < 0) + if (*usage < qdata->qd_count) *usage = 0; else *usage -= qdata->qd_count; @@ -434,45 +470,32 @@ int dqacq_handler(struct obd_device *obd, struct qunit_data *qdata, int opc) rc = fsfilt_dquot(obd, dquot, QFILE_WR_DQUOT); EXIT; out: - up(&dquot->dq_sem); - up(&mds->mds_qonoff_sem); + cfs_up(&dquot->dq_sem); + cfs_up_write(&mds->mds_qonoff_sem); lustre_dqput(dquot); if (rc != -EDQUOT) dqacq_adjust_qunit_sz(obd, qdata->qd_id, QDATA_IS_GRP(qdata), QDATA_IS_BLK(qdata)); - quota_search_lqs(qdata, NULL, qctxt, &lqs); - if (QDATA_IS_BLK(qdata)) { - if (!lqs) { - CDEBUG(D_INFO, "Can't find the lustre qunit size!\n"); - qdata->qd_qunit = qctxt->lqc_bunit_sz; - } else { - spin_lock(&lqs->lqs_lock); - qdata->qd_qunit = lqs->lqs_bunit_sz; - spin_unlock(&lqs->lqs_lock); - } + cfs_spin_lock(&lqs->lqs_lock); + qdata->qd_qunit = QDATA_IS_BLK(qdata) ? lqs->lqs_bunit_sz : + lqs->lqs_iunit_sz; + cfs_spin_unlock(&lqs->lqs_lock); + + if (QDATA_IS_BLK(qdata)) QDATA_SET_ADJBLK(qdata); - } else { - if (!lqs) { - CDEBUG(D_INFO, "Can't find the lustre qunit size!\n"); - qdata->qd_qunit = qctxt->lqc_iunit_sz; - } else { - spin_lock(&lqs->lqs_lock); - qdata->qd_qunit = lqs->lqs_iunit_sz; - spin_unlock(&lqs->lqs_lock); - } + else QDATA_SET_ADJINO(qdata); - } QDATA_DEBUG(qdata, "alloc/release qunit in dqacq_handler\n"); - if (lqs) - lqs_putref(lqs); +skip: + lqs_putref(lqs); return rc; } -int mds_quota_adjust(struct obd_device *obd, unsigned int qcids[], - unsigned int qpids[], int rc, int opc) +int mds_quota_adjust(struct obd_device *obd, const unsigned int qcids[], + const unsigned int qpids[], int rc, int opc) { struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt; int rc2 = 0; @@ -484,67 +507,53 @@ int mds_quota_adjust(struct obd_device *obd, unsigned int qcids[], switch (opc) { case FSFILT_OP_SETATTR: /* release file quota on original owner */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qpids[0], qpids[1], 0, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qpids, 0, 0, NULL); /* release block quota on original owner */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qpids[0], qpids[1], 1, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qpids, 1, 0, NULL); /* acquire file quota on current owner */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids[0], qcids[1], 0, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids, 0, 0, NULL); /* acquire block quota on current owner */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids[0], qcids[1], 1, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids, 1, 0, NULL); break; case FSFILT_OP_UNLINK_PARTIAL_CHILD: /* release file quota on child */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids[0], qcids[1], 0, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids, 0, 0, NULL); /* rlease block quota on child */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids[0], qcids[1], 1, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids, 1, 0, NULL); break; case FSFILT_OP_CREATE_PARTIAL_CHILD: /* acquire file quota on child */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids[0], qcids[1], 0, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids, 0, 0, NULL); /* acquire block quota on child */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids[0], qcids[1], 1, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids, 1, 0, NULL); break; case FSFILT_OP_LINK: /* acquire block quota on parent */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qpids[0], qpids[1], 1, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qpids, 1, 0, NULL); break; case FSFILT_OP_UNLINK: /* release block quota on parent */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qpids[0], qpids[1], 1, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qpids, 1, 0, NULL); /* release file quota on child */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids[0], qcids[1], 0, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids, 0, 0, NULL); if (qpids[0] != qcids[0] || qpids[1] != qcids[1]) /* release block quota on child */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids[0], - qcids[1], 1, 0, NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids, 1, 0, + NULL); break; case FSFILT_OP_UNLINK_PARTIAL_PARENT: /* release block quota on parent */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qpids[0], qpids[1], 1, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qpids, 1, 0, NULL); break; case FSFILT_OP_CREATE: /* acquire block quota on parent */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qpids[0], qpids[1], 1, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qpids, 1, 0, NULL); /* acquire file quota on child */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids[0], qcids[1], 0, 0, - NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids, 0, 0, NULL); if (qpids[0] != qcids[0] || qpids[1] != qcids[1]) /* acquire block quota on child */ - rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids[0], - qcids[1], 1, 0, NULL); + rc2 |= qctxt_adjust_qunit(obd, qctxt, qcids, 1, 0, + NULL); break; default: LBUG(); @@ -552,13 +561,14 @@ int mds_quota_adjust(struct obd_device *obd, unsigned int qcids[], } if (rc2) - CDEBUG(rc2 == -EAGAIN ? D_QUOTA: D_ERROR, - "mds adjust qunit failed! (opc:%d rc:%d)\n", opc, rc2); + CDEBUG(D_QUOTA, + "mds adjust qunit %ssuccessfully! (opc:%d rc:%d)\n", + rc2 == QUOTA_REQ_RETURNED ? "" : "un", opc, rc2); RETURN(0); } -int filter_quota_adjust(struct obd_device *obd, unsigned int qcids[], - unsigned int qpids[], int rc, int opc) +int filter_quota_adjust(struct obd_device *obd, const unsigned int qcids[], + const unsigned int qpids[], int rc, int opc) { struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt; int rc2 = 0; @@ -570,17 +580,14 @@ int filter_quota_adjust(struct obd_device *obd, unsigned int qcids[], switch (opc) { case FSFILT_OP_SETATTR: /* acquire/release block quota on original & current owner */ - rc = qctxt_adjust_qunit(obd, qctxt, qcids[0], qcids[1], 1, 0, - NULL); - rc2 = qctxt_adjust_qunit(obd, qctxt, qpids[0], qpids[1], 1, 0, - NULL); + rc = qctxt_adjust_qunit(obd, qctxt, qcids, 1, 0, NULL); + rc2 = qctxt_adjust_qunit(obd, qctxt, qpids, 1, 0, NULL); break; case FSFILT_OP_UNLINK: /* release block quota on this owner */ case FSFILT_OP_CREATE: /* XXX for write operation on obdfilter */ /* acquire block quota on this owner */ - rc = qctxt_adjust_qunit(obd, qctxt, qcids[0], qcids[1], 1, 0, - NULL); + rc = qctxt_adjust_qunit(obd, qctxt, qcids, 1, 0, NULL); break; default: LBUG(); @@ -590,9 +597,9 @@ int filter_quota_adjust(struct obd_device *obd, unsigned int qcids[], if (rc || rc2) { if (!rc) rc = rc2; - CDEBUG(rc == -EAGAIN ? D_QUOTA: D_ERROR, - "filter adjust qunit failed! (opc:%d rc%d)\n", - opc, rc); + CDEBUG(D_QUOTA, + "filter adjust qunit %ssuccessfully! (opc:%d rc%d)\n", + rc == QUOTA_REQ_RETURNED ? "" : "un", opc, rc); } RETURN(0); @@ -604,16 +611,23 @@ int mds_quota_invalidate(struct obd_device *obd, struct obd_quotactl *oqctl) { struct mds_obd *mds = &obd->u.mds; struct lustre_quota_info *qinfo = &mds->mds_quota_info; - int rc = 0, i; + struct obd_device_target *obt = &obd->u.obt; + int rc = 0, i, rc1 = 0; char *quotafile[] = LUSTRE_ADMIN_QUOTAFILES_V2; char name[64]; struct lvfs_run_ctxt saved; + ENTRY; LASSERT(qinfo->qi_version == LUSTRE_QUOTA_V2); - push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + if (oqctl->qc_type != USRQUOTA && + oqctl->qc_type != GRPQUOTA && + oqctl->qc_type != UGQUOTA) + RETURN(-EINVAL); - down(&mds->mds_qonoff_sem); + cfs_down(&obt->obt_quotachecking); + push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + cfs_down_write(&mds->mds_qonoff_sem); for (i = 0; i < MAXQUOTAS; i++) { struct file *fp; @@ -623,8 +637,9 @@ int mds_quota_invalidate(struct obd_device *obd, struct obd_quotactl *oqctl) /* quota file has been opened ? */ if (qinfo->qi_files[i]) { - rc = -EBUSY; - goto out; + CWARN("quota[%d] is on yet\n", i); + rc1 = -EBUSY; + continue; } LASSERT(strlen(quotafile[i]) + sizeof(prefix) <= sizeof(name)); @@ -633,40 +648,48 @@ int mds_quota_invalidate(struct obd_device *obd, struct obd_quotactl *oqctl) fp = filp_open(name, O_CREAT | O_TRUNC | O_RDWR, 0644); if (IS_ERR(fp)) { rc = PTR_ERR(fp); - CERROR("error invalidating admin quotafile %s (rc:%d)\n", - name, rc); + CERROR("%s: error invalidating admin quotafile %s (rc:%d)\n", + obd->obd_name, name, rc); } else filp_close(fp, 0); } -out: - up(&mds->mds_qonoff_sem); - + cfs_up_write(&mds->mds_qonoff_sem); pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - - return rc; + cfs_up(&obt->obt_quotachecking); + RETURN(rc ? : rc1); } int mds_quota_finvalidate(struct obd_device *obd, struct obd_quotactl *oqctl) { struct mds_obd *mds = &obd->u.mds; + struct obd_device_target *obt = &obd->u.obt; int rc; struct lvfs_run_ctxt saved; + ENTRY; + + if (oqctl->qc_type != USRQUOTA && + oqctl->qc_type != GRPQUOTA && + oqctl->qc_type != UGQUOTA) + RETURN(-EINVAL); + cfs_down(&obt->obt_quotachecking); + if (obt->obt_qctxt.lqc_flags & UGQUOTA2LQC(oqctl->qc_type)) + GOTO(out, rc = -EBUSY); push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - down(&mds->mds_qonoff_sem); + cfs_down_write(&mds->mds_qonoff_sem); oqctl->qc_cmd = Q_FINVALIDATE; - oqctl->qc_id = obd->u.obt.obt_qfmt; rc = fsfilt_quotactl(obd, obd->u.obt.obt_sb, oqctl); if (!rc) - rc = obd_quotactl(mds->mds_osc_exp, oqctl); + rc = obd_quotactl(mds->mds_lov_exp, oqctl); - up(&mds->mds_qonoff_sem); + cfs_up_write(&mds->mds_qonoff_sem); pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - - return rc; +out: + cfs_up(&obt->obt_quotachecking); + RETURN(rc); } int init_admin_quotafiles(struct obd_device *obd, struct obd_quotactl *oqctl) @@ -682,8 +705,7 @@ int init_admin_quotafiles(struct obd_device *obd, struct obd_quotactl *oqctl) LASSERT(qinfo->qi_version == LUSTRE_QUOTA_V2); push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - - down(&mds->mds_qonoff_sem); + cfs_down_write(&mds->mds_qonoff_sem); for (i = 0; i < MAXQUOTAS && !rc; i++) { struct file *fp; @@ -714,7 +736,7 @@ int init_admin_quotafiles(struct obd_device *obd, struct obd_quotactl *oqctl) } qinfo->qi_files[i] = fp; rc = fsfilt_quotainfo(obd, qinfo, i, QFILE_CHK); - qinfo->qi_files[i] = 0; + qinfo->qi_files[i] = NULL; filp_close(fp, 0); } else @@ -725,8 +747,8 @@ int init_admin_quotafiles(struct obd_device *obd, struct obd_quotactl *oqctl) /* -EINVAL may be returned by quotainfo for bad quota file */ if (rc != -ENOENT && rc != -EINVAL) { - CERROR("error opening old quota file %s (%d)\n", - name, rc); + CERROR("%s: error opening old quota file %s (%d)\n", + obd->obd_name, name, rc); break; } @@ -737,8 +759,8 @@ int init_admin_quotafiles(struct obd_device *obd, struct obd_quotactl *oqctl) fp = filp_open(name, O_CREAT | O_TRUNC | O_RDWR, 0644); if (IS_ERR(fp)) { rc = PTR_ERR(fp); - CERROR("error creating admin quotafile %s (rc:%d)\n", - name, rc); + CERROR("%s: error creating admin quotafile %s (rc:%d)\n", + obd->obd_name, name, rc); break; } @@ -752,8 +774,8 @@ int init_admin_quotafiles(struct obd_device *obd, struct obd_quotactl *oqctl) filp_close(fp, 0); qinfo->qi_files[i] = NULL; } - up(&mds->mds_qonoff_sem); + cfs_up_write(&mds->mds_qonoff_sem); pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); RETURN(rc); } @@ -768,7 +790,8 @@ static int close_quota_files(struct obd_quotactl *oqctl, if (!Q_TYPESET(oqctl, i)) continue; if (qinfo->qi_files[i] == NULL) { - rc = -ESRCH; + CWARN("quota[%d] is off already\n", i); + rc = -EALREADY; continue; } filp_close(qinfo->qi_files[i], 0); @@ -783,7 +806,7 @@ int mds_admin_quota_on(struct obd_device *obd, struct obd_quotactl *oqctl) struct lustre_quota_info *qinfo = &mds->mds_quota_info; const char *quotafile[] = LUSTRE_ADMIN_QUOTAFILES_V2; char name[64]; - int i, rc = 0; + int i, rc = 0, rc1 = 0; ENTRY; LASSERT(qinfo->qi_version == LUSTRE_QUOTA_V2); @@ -792,18 +815,12 @@ int mds_admin_quota_on(struct obd_device *obd, struct obd_quotactl *oqctl) for (i = 0; i < MAXQUOTAS; i++) { struct file *fp; - if (!Q_TYPESET(oqctl, i)) + if (!Q_TYPESET(oqctl, i) || qinfo->qi_files[i] != NULL) continue; LASSERT(strlen(quotafile[i]) + sizeof(prefix) <= sizeof(name)); sprintf(name, "%s%s", prefix, quotafile[i]); - - if (qinfo->qi_files[i] != NULL) { - rc = -EBUSY; - break; - } - fp = filp_open(name, O_RDWR, 0); if (IS_ERR(fp) || !S_ISREG(fp->f_dentry->d_inode->i_mode)) { rc = IS_ERR(fp) ? PTR_ERR(fp) : -EINVAL; @@ -826,12 +843,28 @@ int mds_admin_quota_on(struct obd_device *obd, struct obd_quotactl *oqctl) } } - if (rc && rc != -EBUSY) + if (rc && rc1 != -EALREADY) close_quota_files(oqctl, qinfo); + RETURN(rc ? : rc1); +} + +int mds_quota_on(struct obd_device *obd, struct obd_quotactl *oqctl) +{ + int rc; + ENTRY; + + if (oqctl->qc_type != USRQUOTA && + oqctl->qc_type != GRPQUOTA && + oqctl->qc_type != UGQUOTA) + RETURN(-EINVAL); + + rc = generic_quota_on(obd, oqctl, 1); + RETURN(rc); } + int mds_admin_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl) { @@ -845,69 +878,83 @@ int mds_admin_quota_off(struct obd_device *obd, RETURN(rc); } -int mds_quota_on(struct obd_device *obd, struct obd_quotactl *oqctl) + +/* with obt->obt_quotachecking held */ +int do_mds_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl) { struct mds_obd *mds = &obd->u.mds; struct obd_device_target *obt = &obd->u.obt; + struct lustre_quota_ctxt *qctxt = &obt->obt_qctxt; struct lvfs_run_ctxt saved; - int rc; + int rc = 0, rc1 = 0, rc2 = 0; ENTRY; - if (!atomic_dec_and_test(&obt->obt_quotachecking)) { - CDEBUG(D_INFO, "other people are doing quotacheck\n"); - atomic_inc(&obt->obt_quotachecking); - RETURN(-EBUSY); - } + LASSERT_SEM_LOCKED(&obt->obt_quotachecking); + + if (oqctl->qc_type != USRQUOTA && + oqctl->qc_type != GRPQUOTA && + oqctl->qc_type != UGQUOTA) + RETURN(-EINVAL); - down(&mds->mds_qonoff_sem); push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - rc = mds_admin_quota_on(obd, oqctl); - if (rc) - goto out; + cfs_down_write(&mds->mds_qonoff_sem); + /* close admin quota files */ + rc2 = mds_admin_quota_off(obd, oqctl); + if (rc2 && rc2 != -EALREADY) { + CWARN("mds quota[%d] is failed to be off for %d\n", oqctl->qc_type, rc2); + GOTO(out, rc2); + } - rc = obd_quotactl(mds->mds_osc_exp, oqctl); - if (rc) - goto out; + rc1 = fsfilt_quotactl(obd, obd->u.obt.obt_sb, oqctl); + if (!rc1) { + obt->obt_qctxt.lqc_flags &= ~UGQUOTA2LQC(oqctl->qc_type); + } else if (quota_is_off(qctxt, oqctl)) { + CWARN("mds local quota[%d] is off already\n", oqctl->qc_type); + rc1 = -EALREADY; + } else { + if (rc2 != -EALREADY) { + CWARN("mds local quota[%d] is failed to be off for %d\n", + oqctl->qc_type, rc1); + oqctl->qc_cmd = Q_QUOTAON; + mds_admin_quota_on(obd, oqctl); + oqctl->qc_cmd = Q_QUOTAOFF; + } + GOTO(out, rc1); + } + + rc = obd_quotactl(mds->mds_lov_exp, oqctl); + if (rc && rc != -EALREADY) { + CWARN("mds remote quota[%d] is failed to be off for %d\n", + oqctl->qc_type, rc); + oqctl->qc_cmd = Q_QUOTAON; + if (rc2 != -EALREADY) + mds_admin_quota_on(obd, oqctl); + if (rc1 != -EALREADY) { + fsfilt_quotactl(obd, obd->u.obt.obt_sb, oqctl); + qctxt->lqc_flags |= UGQUOTA2LQC(oqctl->qc_type); + } + oqctl->qc_cmd = Q_QUOTAOFF; + } + EXIT; - rc = fsfilt_quotactl(obd, obd->u.obt.obt_sb, oqctl); - if (!rc) - obt->obt_qctxt.lqc_flags |= UGQUOTA2LQC(oqctl->qc_type); out: + CDEBUG(D_QUOTA, "%s: quotaoff type:flags:rc %u:%lu:%d\n", + obd->obd_name, oqctl->qc_type, qctxt->lqc_flags, rc); + cfs_up_write(&mds->mds_qonoff_sem); pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - up(&mds->mds_qonoff_sem); - atomic_inc(&obt->obt_quotachecking); - RETURN(rc); + return rc ? : (rc1 ? : rc2); } int mds_quota_off(struct obd_device *obd, struct obd_quotactl *oqctl) { - struct mds_obd *mds = &obd->u.mds; struct obd_device_target *obt = &obd->u.obt; - struct lvfs_run_ctxt saved; - int rc, rc2; + int rc; ENTRY; - if (!atomic_dec_and_test(&obt->obt_quotachecking)) { - CDEBUG(D_INFO, "other people are doing quotacheck\n"); - atomic_inc(&obt->obt_quotachecking); - RETURN(-EBUSY); - } - - down(&mds->mds_qonoff_sem); - /* close admin quota files */ - push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - mds_admin_quota_off(obd, oqctl); - - rc = obd_quotactl(mds->mds_osc_exp, oqctl); - rc2 = fsfilt_quotactl(obd, obd->u.obt.obt_sb, oqctl); - if (!rc2) - obt->obt_qctxt.lqc_flags &= ~UGQUOTA2LQC(oqctl->qc_type); - - pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - up(&mds->mds_qonoff_sem); - atomic_inc(&obt->obt_quotachecking); - - RETURN(rc ?: rc2); + cfs_down(&obt->obt_quotachecking); + rc = do_mds_quota_off(obd, oqctl); + cfs_up(&obt->obt_quotachecking); + RETURN(rc); } int mds_set_dqinfo(struct obd_device *obd, struct obd_quotactl *oqctl) @@ -918,10 +965,14 @@ int mds_set_dqinfo(struct obd_device *obd, struct obd_quotactl *oqctl) int rc; ENTRY; - down(&mds->mds_qonoff_sem); + if (oqctl->qc_type != USRQUOTA && + oqctl->qc_type != GRPQUOTA) + RETURN(-EINVAL); + + cfs_down_write(&mds->mds_qonoff_sem); if (qinfo->qi_files[oqctl->qc_type] == NULL) { - rc = -ESRCH; - goto out; + CWARN("quota[%u] is off\n", oqctl->qc_type); + GOTO(out, rc = -ESRCH); } qinfo->qi_info[oqctl->qc_type].dqi_bgrace = dqinfo->dqi_bgrace; @@ -929,10 +980,11 @@ int mds_set_dqinfo(struct obd_device *obd, struct obd_quotactl *oqctl) qinfo->qi_info[oqctl->qc_type].dqi_flags = dqinfo->dqi_flags; rc = fsfilt_quotainfo(obd, qinfo, oqctl->qc_type, QFILE_WR_INFO); + EXIT; out: - up(&mds->mds_qonoff_sem); - RETURN(rc); + cfs_up_write(&mds->mds_qonoff_sem); + return rc; } int mds_get_dqinfo(struct obd_device *obd, struct obd_quotactl *oqctl) @@ -943,19 +995,24 @@ int mds_get_dqinfo(struct obd_device *obd, struct obd_quotactl *oqctl) int rc = 0; ENTRY; - down(&mds->mds_qonoff_sem); + if (oqctl->qc_type != USRQUOTA && + oqctl->qc_type != GRPQUOTA) + RETURN(-EINVAL); + + cfs_down_read(&mds->mds_qonoff_sem); if (qinfo->qi_files[oqctl->qc_type] == NULL) { - rc = -ESRCH; - goto out; + CWARN("quota[%u] is off\n", oqctl->qc_type); + GOTO(out, rc = -ESRCH); } dqinfo->dqi_bgrace = qinfo->qi_info[oqctl->qc_type].dqi_bgrace; dqinfo->dqi_igrace = qinfo->qi_info[oqctl->qc_type].dqi_igrace; dqinfo->dqi_flags = qinfo->qi_info[oqctl->qc_type].dqi_flags; + EXIT; out: - up(&mds->mds_qonoff_sem); - RETURN(rc); + cfs_up_read(&mds->mds_qonoff_sem); + return rc; } int dquot_create_oqaq(struct lustre_quota_ctxt *qctxt, @@ -990,14 +1047,16 @@ int dquot_create_oqaq(struct lustre_quota_ctxt *qctxt, if ((type & LQUOTA_FLAGS_ADJBLK) && blimit) { __u64 b_limitation = - oqaq->qaq_bunit_sz * ost_num * shrink_qunit_limit; + oqaq->qaq_bunit_sz * (ost_num + 1) * shrink_qunit_limit; /* enlarge block qunit size */ while (blimit > QUSG(dquot->dq_dqb.dqb_curspace + 2 * b_limitation, 1)) { oqaq->qaq_bunit_sz = QUSG(oqaq->qaq_bunit_sz * cqs_factor, 1) << QUOTABLOCK_BITS; - b_limitation = oqaq->qaq_bunit_sz * ost_num * + if (oqaq->qaq_bunit_sz >= qctxt->lqc_bunit_sz) + break; + b_limitation = oqaq->qaq_bunit_sz * (ost_num + 1) * shrink_qunit_limit; } @@ -1010,7 +1069,7 @@ int dquot_create_oqaq(struct lustre_quota_ctxt *qctxt, do_div(oqaq->qaq_bunit_sz , cqs_factor); oqaq->qaq_bunit_sz = QUSG(oqaq->qaq_bunit_sz, 1) << QUOTABLOCK_BITS; - b_limitation = oqaq->qaq_bunit_sz * ost_num * + b_limitation = oqaq->qaq_bunit_sz * (ost_num + 1) * shrink_qunit_limit; if (oqaq->qaq_bunit_sz < qctxt->lqc_cqs_least_bunit) break; @@ -1031,6 +1090,8 @@ int dquot_create_oqaq(struct lustre_quota_ctxt *qctxt, while (ilimit > dquot->dq_dqb.dqb_curinodes + 2 * i_limitation) { oqaq->qaq_iunit_sz = oqaq->qaq_iunit_sz * cqs_factor; + if (oqaq->qaq_iunit_sz >= qctxt->lqc_iunit_sz) + break; i_limitation = oqaq->qaq_iunit_sz * mdt_num * shrink_qunit_limit; } @@ -1056,28 +1117,20 @@ int dquot_create_oqaq(struct lustre_quota_ctxt *qctxt, } - if (!dquot->dq_dqb.dqb_bhardlimit && !dquot->dq_dqb.dqb_bsoftlimit && - !dquot->dq_dqb.dqb_ihardlimit && !dquot->dq_dqb.dqb_isoftlimit) { - oqaq->qaq_bunit_sz = 0; - oqaq->qaq_iunit_sz = 0; - QAQ_SET_ADJBLK(oqaq); - QAQ_SET_ADJINO(oqaq); - } - QAQ_DEBUG(oqaq, "the oqaq computed\n"); RETURN(rc); } static int mds_init_slave_ilimits(struct obd_device *obd, - struct obd_quotactl *oqctl, int set, - struct quota_adjust_qunit *oqaq) + struct obd_quotactl *oqctl, int set) { /* XXX: for file limits only adjust local now */ struct obd_device_target *obt = &obd->u.obt; struct lustre_quota_ctxt *qctxt = &obt->obt_qctxt; - unsigned int uid = 0, gid = 0; + unsigned int id[MAXQUOTAS] = { 0, 0 }; struct obd_quotactl *ioqc = NULL; + struct lustre_qunit_size *lqs; int flag; int rc; ENTRY; @@ -1099,12 +1152,21 @@ static int mds_init_slave_ilimits(struct obd_device *obd, ioqc->qc_dqblk.dqb_valid = QIF_ILIMITS; ioqc->qc_dqblk.dqb_ihardlimit = flag ? MIN_QLIMIT : 0; - if (QAQ_IS_ADJINO(oqaq)) { - /* adjust the mds slave's inode qunit size */ - rc = quota_adjust_slave_lqs(oqaq, qctxt); - if (rc < 0) - CDEBUG(D_ERROR, "adjust mds slave's inode qunit size \ - failed! (rc:%d)\n", rc); + /* build lqs for mds */ + lqs = quota_search_lqs(LQS_KEY(oqctl->qc_type, oqctl->qc_id), + qctxt, flag ? 1 : 0); + if (lqs && !IS_ERR(lqs)) { + if (flag) + lqs->lqs_flags |= QI_SET; + else + lqs->lqs_flags &= ~QI_SET; + lqs_putref(lqs); + } else { + CERROR("fail to %s lqs for inode(%s id: %u)!\n", + flag ? "create" : "search", + oqctl->qc_type ? "group" : "user", + oqctl->qc_id); + GOTO(out, rc = PTR_ERR(lqs)); } /* set local limit to MIN_QLIMIT */ @@ -1114,12 +1176,11 @@ static int mds_init_slave_ilimits(struct obd_device *obd, /* trigger local qunit pre-acquire */ if (oqctl->qc_type == USRQUOTA) - uid = oqctl->qc_id; + id[USRQUOTA] = oqctl->qc_id; else - gid = oqctl->qc_id; + id[GRPQUOTA] = oqctl->qc_id; - rc = qctxt_adjust_qunit(obd, &obd->u.obt.obt_qctxt, uid, gid, 0, 0, - NULL); + rc = qctxt_adjust_qunit(obd, &obd->u.obt.obt_qctxt, id, 0, 0, NULL); if (rc == -EDQUOT || rc == -EBUSY) { CDEBUG(D_QUOTA, "rc: %d.\n", rc); rc = 0; @@ -1138,15 +1199,15 @@ out: } static int mds_init_slave_blimits(struct obd_device *obd, - struct obd_quotactl *oqctl, int set, - struct quota_adjust_qunit *oqaq) + struct obd_quotactl *oqctl, int set) { struct obd_device_target *obt = &obd->u.obt; struct lustre_quota_ctxt *qctxt = &obt->obt_qctxt; struct mds_obd *mds = &obd->u.mds; struct obd_quotactl *ioqc; - unsigned int uid = 0, gid = 0; - int rc, rc1 = 0; + struct lustre_qunit_size *lqs; + unsigned int id[MAXQUOTAS] = { 0, 0 }; + int rc; int flag; ENTRY; @@ -1166,12 +1227,22 @@ static int mds_init_slave_blimits(struct obd_device *obd, ioqc->qc_type = oqctl->qc_type; ioqc->qc_dqblk.dqb_valid = QIF_BLIMITS; ioqc->qc_dqblk.dqb_bhardlimit = flag ? MIN_QLIMIT : 0; - if (QAQ_IS_ADJBLK(oqaq)) { - /* adjust the mds slave's block qunit size */ - rc1 = quota_adjust_slave_lqs(oqaq, qctxt); - if (rc1 < 0) - CERROR("adjust mds slave's block qunit size failed!" - "(rc:%d)\n", rc1); + + /* build lqs for mds */ + lqs = quota_search_lqs(LQS_KEY(oqctl->qc_type, oqctl->qc_id), + qctxt, flag ? 1 : 0); + if (lqs && !IS_ERR(lqs)) { + if (flag) + lqs->lqs_flags |= QB_SET; + else + lqs->lqs_flags &= ~QB_SET; + lqs_putref(lqs); + } else { + CERROR("fail to %s lqs for block(%s id: %u)!\n", + flag ? "create" : "search", + oqctl->qc_type ? "group" : "user", + oqctl->qc_id); + GOTO(out, rc = PTR_ERR(lqs)); } rc = fsfilt_quotactl(obd, obd->u.obt.obt_sb, ioqc); @@ -1180,15 +1251,14 @@ static int mds_init_slave_blimits(struct obd_device *obd, /* trigger local qunit pre-acquire */ if (oqctl->qc_type == USRQUOTA) - uid = oqctl->qc_id; + id[USRQUOTA] = oqctl->qc_id; else - gid = oqctl->qc_id; + id[GRPQUOTA] = oqctl->qc_id; /* initialize all slave's limit */ - rc = obd_quotactl(mds->mds_osc_exp, ioqc); + rc = obd_quotactl(mds->mds_lov_exp, ioqc); - rc = qctxt_adjust_qunit(obd, &obd->u.obt.obt_qctxt, uid, gid, 1, 0, - NULL); + rc = qctxt_adjust_qunit(obd, &obd->u.obt.obt_qctxt, id, 1, 0, NULL); if (rc == -EDQUOT || rc == -EBUSY) { CDEBUG(D_QUOTA, "rc: %d.\n", rc); rc = 0; @@ -1198,23 +1268,37 @@ static int mds_init_slave_blimits(struct obd_device *obd, GOTO(out, rc); } - /* adjust all slave's qunit size when setting quota - * this is will create a lqs for every ost, which will present - * certain uid/gid is set quota or not */ - QAQ_SET_ADJBLK(oqaq); - rc = obd_quota_adjust_qunit(mds->mds_osc_exp, oqaq, qctxt); - EXIT; out: OBD_FREE_PTR(ioqc); return rc; } +static void adjust_lqs(struct obd_device *obd, struct quota_adjust_qunit *qaq) +{ + struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt; + int rc = 0; + + QAQ_SET_CREATE_LQS(qaq); + /* adjust local lqs */ + rc = quota_adjust_slave_lqs(qaq, qctxt); + if (rc < 0) + CERROR("adjust master's qunit size failed!(rc=%d)\n", rc); + + /* adjust remote lqs */ + if (QAQ_IS_ADJBLK(qaq)) { + rc = obd_quota_adjust_qunit(obd->u.mds.mds_lov_exp, qaq, qctxt, NULL); + if (rc < 0) + CERROR("adjust slaves' qunit size failed!(rc=%d)\n", rc); + + } +} + int mds_set_dqblk(struct obd_device *obd, struct obd_quotactl *oqctl) { struct mds_obd *mds = &obd->u.mds; struct lustre_quota_ctxt *qctxt = &mds->mds_obt.obt_qctxt; - struct obd_device *lov_obd = class_exp2obd(mds->mds_osc_exp); + struct obd_device *lov_obd = class_exp2obd(mds->mds_lov_exp); struct lov_obd *lov = &lov_obd->u.lov; struct quota_adjust_qunit *oqaq = NULL; struct lustre_quota_info *qinfo = &mds->mds_quota_info; @@ -1225,32 +1309,49 @@ int mds_set_dqblk(struct obd_device *obd, struct obd_quotactl *oqctl) /* orig_set means if quota was set before; now_set means we are * setting/cancelling quota */ int orig_set, now_set; - int rc, rc2 = 0, flag = 0; + struct lustre_qunit_size *lqs; + int rc = 0, rc2 = 0, flag = 0; ENTRY; + if (oqctl->qc_type != USRQUOTA && + oqctl->qc_type != GRPQUOTA) + RETURN(-EINVAL); + OBD_ALLOC_PTR(oqaq); if (!oqaq) RETURN(-ENOMEM); - down(&mds->mds_qonoff_sem); + + cfs_down_write(&mds->mds_qonoff_sem); init_oqaq(oqaq, qctxt, oqctl->qc_id, oqctl->qc_type); - if (qinfo->qi_files[oqctl->qc_type] == NULL) + if (qinfo->qi_files[oqctl->qc_type] == NULL) { + CWARN("quota[%u] is off\n", oqctl->qc_type); GOTO(out_sem, rc = -ESRCH); + } - dquot = lustre_dqget(obd, qinfo, oqctl->qc_id, oqctl->qc_type); + dquot = lustre_dqget(obd, qinfo, oqctl->qc_id, oqctl->qc_type, 1); if (IS_ERR(dquot)) GOTO(out_sem, rc = PTR_ERR(dquot)); DQUOT_DEBUG(dquot, "get dquot in mds_set_blk\n"); QINFO_DEBUG(dquot->dq_info, "get dquot in mds_set_blk\n"); - down(&dquot->dq_sem); + lqs = quota_search_lqs(LQS_KEY(oqctl->qc_type, oqctl->qc_id), qctxt, 1); + if (lqs == NULL) + rc = -ENOENT; + if (IS_ERR(lqs)) + rc = PTR_ERR(lqs); + if (rc) + GOTO(out, rc); - if (dquot->dq_status) { - up(&dquot->dq_sem); - lustre_dqput(dquot); - GOTO(out_sem, rc = -EBUSY); + cfs_down(&dquot->dq_sem); + cfs_spin_lock(&lqs->lqs_lock); + if (LQS_IS_SETQUOTA(lqs) || LQS_IS_RECOVERY(lqs)) { + cfs_spin_unlock(&lqs->lqs_lock); + cfs_up(&dquot->dq_sem); + GOTO(skip, rc = -EBUSY); } - dquot->dq_status |= DQ_STATUS_SET; + LQS_SET_SETQUOTA(lqs); + cfs_spin_unlock(&lqs->lqs_lock); ihardlimit = dquot->dq_dqb.dqb_ihardlimit; isoftlimit = dquot->dq_dqb.dqb_isoftlimit; @@ -1311,21 +1412,23 @@ int mds_set_dqblk(struct obd_device *obd, struct obd_quotactl *oqctl) rc = fsfilt_dquot(obd, dquot, QFILE_WR_DQUOT); - up(&dquot->dq_sem); + cfs_up(&dquot->dq_sem); if (rc) { CERROR("set limit failed! (rc:%d)\n", rc); - goto out; + GOTO(update_fail, rc); } - up(&mds->mds_qonoff_sem); + cfs_up_write(&mds->mds_qonoff_sem); + adjust_lqs(obd, oqaq); + orig_set = ihardlimit || isoftlimit; now_set = dqblk->dqb_ihardlimit || dqblk->dqb_isoftlimit; if (dqblk->dqb_valid & QIF_ILIMITS && orig_set != now_set) { - down(&dquot->dq_sem); + cfs_down(&dquot->dq_sem); dquot->dq_dqb.dqb_curinodes = 0; - up(&dquot->dq_sem); - rc = mds_init_slave_ilimits(obd, oqctl, orig_set, oqaq); + cfs_up(&dquot->dq_sem); + rc = mds_init_slave_ilimits(obd, oqctl, orig_set); if (rc) { CERROR("init slave ilimits failed! (rc:%d)\n", rc); goto revoke_out; @@ -1335,10 +1438,10 @@ int mds_set_dqblk(struct obd_device *obd, struct obd_quotactl *oqctl) orig_set = bhardlimit || bsoftlimit; now_set = dqblk->dqb_bhardlimit || dqblk->dqb_bsoftlimit; if (dqblk->dqb_valid & QIF_BLIMITS && orig_set != now_set) { - down(&dquot->dq_sem); + cfs_down(&dquot->dq_sem); dquot->dq_dqb.dqb_curspace = 0; - up(&dquot->dq_sem); - rc = mds_init_slave_blimits(obd, oqctl, orig_set, oqaq); + cfs_up(&dquot->dq_sem); + rc = mds_init_slave_blimits(obd, oqctl, orig_set); if (rc) { CERROR("init slave blimits failed! (rc:%d)\n", rc); goto revoke_out; @@ -1346,8 +1449,8 @@ int mds_set_dqblk(struct obd_device *obd, struct obd_quotactl *oqctl) } revoke_out: - down(&mds->mds_qonoff_sem); - down(&dquot->dq_sem); + cfs_down_write(&mds->mds_qonoff_sem); + cfs_down(&dquot->dq_sem); if (rc) { /* cancel previous setting */ dquot->dq_dqb.dqb_ihardlimit = ihardlimit; @@ -1358,16 +1461,18 @@ revoke_out: dquot->dq_dqb.dqb_itime = itime; } rc2 = fsfilt_dquot(obd, dquot, QFILE_WR_DQUOT); - up(&dquot->dq_sem); - + cfs_up(&dquot->dq_sem); +update_fail: + cfs_spin_lock(&lqs->lqs_lock); + LQS_CLEAR_SETQUOTA(lqs); + cfs_spin_unlock(&lqs->lqs_lock); +skip: + lqs_putref(lqs); out: - down(&dquot->dq_sem); - dquot->dq_status &= ~DQ_STATUS_SET; - up(&dquot->dq_sem); lustre_dqput(dquot); EXIT; out_sem: - up(&mds->mds_qonoff_sem); + cfs_up_write(&mds->mds_qonoff_sem); if (oqaq) OBD_FREE_PTR(oqaq); @@ -1392,8 +1497,8 @@ static int mds_get_space(struct obd_device *obd, struct obd_quotactl *oqctl) /* get block usage from OSS */ soqc->qc_dqblk.dqb_curspace = 0; - rc = obd_quotactl(obd->u.mds.mds_osc_exp, soqc); - if (!rc) { + rc = obd_quotactl(obd->u.mds.mds_lov_exp, soqc); + if (!rc || rc == -EREMOTEIO) { oqctl->qc_dqblk.dqb_curspace = soqc->qc_dqblk.dqb_curspace; oqctl->qc_dqblk.dqb_valid |= QIF_SPACE; } @@ -1424,16 +1529,22 @@ int mds_get_dqblk(struct obd_device *obd, struct obd_quotactl *oqctl) int rc; ENTRY; - down(&mds->mds_qonoff_sem); + if (oqctl->qc_type != USRQUOTA && + oqctl->qc_type != GRPQUOTA) + RETURN(-EINVAL); + + cfs_down_read(&mds->mds_qonoff_sem); dqblk->dqb_valid = 0; - if (qinfo->qi_files[oqctl->qc_type] == NULL) + if (qinfo->qi_files[oqctl->qc_type] == NULL) { + CWARN("quota[%u] is off\n", oqctl->qc_type); GOTO(out, rc = -ESRCH); + } - dquot = lustre_dqget(obd, qinfo, oqctl->qc_id, oqctl->qc_type); + dquot = lustre_dqget(obd, qinfo, oqctl->qc_id, oqctl->qc_type, 1); if (IS_ERR(dquot)) GOTO(out, rc = PTR_ERR(dquot)); - down(&dquot->dq_sem); + cfs_down(&dquot->dq_sem); dqblk->dqb_ihardlimit = dquot->dq_dqb.dqb_ihardlimit; dqblk->dqb_isoftlimit = dquot->dq_dqb.dqb_isoftlimit; dqblk->dqb_bhardlimit = dquot->dq_dqb.dqb_bhardlimit; @@ -1441,17 +1552,31 @@ int mds_get_dqblk(struct obd_device *obd, struct obd_quotactl *oqctl) dqblk->dqb_btime = dquot->dq_dqb.dqb_btime; dqblk->dqb_itime = dquot->dq_dqb.dqb_itime; dqblk->dqb_valid |= QIF_LIMITS | QIF_TIMES; - up(&dquot->dq_sem); + cfs_up(&dquot->dq_sem); lustre_dqput(dquot); + cfs_up_read(&mds->mds_qonoff_sem); /* the usages in admin quota file is inaccurate */ dqblk->dqb_curinodes = 0; dqblk->dqb_curspace = 0; rc = mds_get_space(obd, oqctl); - EXIT; + + /* + * Querying of curinodes and/or curspace may have failed, administrative + * quota data are likely to be better approximation to the real usage in + * this case. + */ + if (!(dqblk->dqb_valid & QIF_INODES) && dquot->dq_dqb.dqb_curinodes > 0) + dqblk->dqb_curinodes = dquot->dq_dqb.dqb_curinodes; + + if (!(dqblk->dqb_valid & QIF_SPACE) && dquot->dq_dqb.dqb_curspace > 0) + dqblk->dqb_curspace = dquot->dq_dqb.dqb_curspace; + + RETURN(rc); + out: - up(&mds->mds_qonoff_sem); + cfs_up_read(&mds->mds_qonoff_sem); return rc; } @@ -1475,40 +1600,59 @@ static int dquot_recovery(struct obd_device *obd, unsigned int id, unsigned short type) { struct mds_obd *mds = &obd->u.mds; - struct lustre_quota_info *qinfo= &mds->mds_quota_info; + struct lustre_quota_ctxt *qctxt = &mds->mds_obt.obt_qctxt; + struct lustre_quota_info *qinfo = &mds->mds_quota_info; + struct lustre_qunit_size *lqs; struct lustre_dquot *dquot; struct obd_quotactl *qctl; __u64 total_limits = 0; - int rc; + int rc = 0; ENTRY; OBD_ALLOC_PTR(qctl); if (qctl == NULL) RETURN(-ENOMEM); - dquot = lustre_dqget(obd, qinfo, id, type); + dquot = lustre_dqget(obd, qinfo, id, type, 0); if (IS_ERR(dquot)) { CERROR("Get dquot failed. (rc:%ld)\n", PTR_ERR(dquot)); OBD_FREE_PTR(qctl); RETURN(PTR_ERR(dquot)); } - down(&dquot->dq_sem); + lqs = quota_search_lqs(LQS_KEY(type, id), qctxt, 1); + if (lqs == NULL) + rc = -ENOENT; + if (IS_ERR(lqs)) + rc = PTR_ERR(lqs); + if (rc) + GOTO(skip, rc); + + cfs_down(&dquot->dq_sem); - /* don't recovery the dquot without limits or under setting */ + /* don't recover the dquot without limits or quota is setting or + * another recovery is already going on */ if (!(dquot->dq_dqb.dqb_bhardlimit || dquot->dq_dqb.dqb_bsoftlimit) || - dquot->dq_status) - GOTO(skip, rc = 0); - dquot->dq_status |= DQ_STATUS_RECOVERY; + LQS_IS_SETQUOTA(lqs) || LQS_IS_RECOVERY(lqs)) { + cfs_up(&dquot->dq_sem); + GOTO(skip1, rc = 0); + } + + cfs_spin_lock(&lqs->lqs_lock); + LQS_SET_RECOVERY(lqs); + cfs_spin_unlock(&lqs->lqs_lock); + cfs_up(&dquot->dq_sem); - up(&dquot->dq_sem); + /* release mds_qonoff_sem during obd_quotactl ops here */ + cfs_up_write(&mds->mds_qonoff_sem); /* get real bhardlimit from all slaves. */ qctl->qc_cmd = Q_GETOQUOTA; qctl->qc_type = type; qctl->qc_id = id; qctl->qc_stat = QUOTA_RECOVERING; - rc = obd_quotactl(mds->mds_osc_exp, qctl); + rc = obd_quotactl(mds->mds_lov_exp, qctl); + cfs_down_write(&mds->mds_qonoff_sem); if (rc) GOTO(out, rc); total_limits = qctl->qc_dqblk.dqb_bhardlimit; @@ -1520,8 +1664,7 @@ dquot_recovery(struct obd_device *obd, unsigned int id, unsigned short type) total_limits += qctl->qc_dqblk.dqb_bhardlimit; /* amend the usage of the administrative quotafile */ - down(&mds->mds_qonoff_sem); - down(&dquot->dq_sem); + cfs_down(&dquot->dq_sem); dquot->dq_dqb.dqb_curspace = total_limits << QUOTABLOCK_BITS; @@ -1529,15 +1672,15 @@ dquot_recovery(struct obd_device *obd, unsigned int id, unsigned short type) if (rc) CERROR("write dquot failed! (rc:%d)\n", rc); - up(&dquot->dq_sem); - up(&mds->mds_qonoff_sem); + cfs_up(&dquot->dq_sem); EXIT; out: - down(&dquot->dq_sem); - dquot->dq_status &= ~DQ_STATUS_RECOVERY; + cfs_spin_lock(&lqs->lqs_lock); + LQS_CLEAR_RECOVERY(lqs); + cfs_spin_unlock(&lqs->lqs_lock); +skip1: + lqs_putref(lqs); skip: - up(&dquot->dq_sem); - lustre_dqput(dquot); OBD_FREE_PTR(qctl); return rc; @@ -1545,79 +1688,95 @@ skip: struct qmaster_recov_thread_data { struct obd_device *obd; - struct completion comp; + cfs_completion_t comp; }; static int qmaster_recovery_main(void *arg) { struct qmaster_recov_thread_data *data = arg; struct obd_device *obd = data->obd; + struct mds_obd *mds = &obd->u.mds; + struct lustre_quota_info *qinfo = &mds->mds_quota_info; int rc = 0; unsigned short type; ENTRY; - ptlrpc_daemonize("qmaster_recovd"); + cfs_daemonize_ctxt("qmaster_recovd"); + + /* for mds */ + class_incref(obd, "qmaster_recovd_mds", obd); + /* for lov */ + class_incref(mds->mds_lov_obd, "qmaster_recovd_lov", mds->mds_lov_obd); - complete(&data->comp); + cfs_complete(&data->comp); + cfs_down_write(&mds->mds_qonoff_sem); for (type = USRQUOTA; type < MAXQUOTAS; type++) { - struct mds_obd *mds = &obd->u.mds; - struct lustre_quota_info *qinfo = &mds->mds_quota_info; - struct list_head id_list; + cfs_list_t id_list; struct dquot_id *dqid, *tmp; - down(&mds->mds_qonoff_sem); - if (qinfo->qi_files[type] == NULL) { - up(&mds->mds_qonoff_sem); + if (qinfo->qi_files[type] == NULL) continue; - } + CFS_INIT_LIST_HEAD(&id_list); rc = fsfilt_qids(obd, qinfo->qi_files[type], NULL, type, &id_list); - up(&mds->mds_qonoff_sem); - if (rc) CERROR("error get ids from admin quotafile.(%d)\n", rc); - list_for_each_entry_safe(dqid, tmp, &id_list, di_link) { - list_del_init(&dqid->di_link); + cfs_list_for_each_entry_safe(dqid, tmp, &id_list, di_link) { + cfs_list_del_init(&dqid->di_link); if (rc) goto free; rc = dquot_recovery(obd, dqid->di_id, type); if (rc) - CERROR("qmaster recovery failed! (id:%d type:%d" - " rc:%d)\n", dqid->di_id, type, rc); + CERROR("%s: qmaster recovery failed for %sid %d" + " rc:%d)\n", obd->obd_name, + type ? "g" : "u", dqid->di_id, rc); free: - kfree(dqid); + OBD_FREE_PTR(dqid); } } + cfs_up_write(&mds->mds_qonoff_sem); + class_decref(mds->mds_lov_obd, "qmaster_recovd_lov", mds->mds_lov_obd); + class_decref(obd, "qmaster_recovd_mds", obd); RETURN(rc); } int mds_quota_recovery(struct obd_device *obd) { - struct lov_obd *lov = &obd->u.mds.mds_osc_obd->u.lov; + struct mds_obd *mds = &obd->u.mds; struct qmaster_recov_thread_data data; int rc = 0; ENTRY; - mutex_down(&lov->lov_lock); - if (lov->desc.ld_tgt_count != lov->desc.ld_active_tgt_count) { - CWARN("Not all osts are active, abort quota recovery\n"); - mutex_up(&lov->lov_lock); + if (!ll_sb_any_quota_active(obd->u.obt.obt_qctxt.lqc_sb)) + RETURN(0); + + if (unlikely(!mds->mds_quota || obd->obd_stopping)) + RETURN(rc); + + cfs_mutex_down(&obd->obd_dev_sem); + if (mds->mds_lov_desc.ld_active_tgt_count != mds->mds_lov_objid_count) { + CWARN("Only %u/%u OSTs are active, abort quota recovery\n", + mds->mds_lov_desc.ld_active_tgt_count, + mds->mds_lov_objid_count); + cfs_mutex_up(&obd->obd_dev_sem); RETURN(rc); } - mutex_up(&lov->lov_lock); + cfs_mutex_up(&obd->obd_dev_sem); data.obd = obd; - init_completion(&data.comp); + cfs_init_completion(&data.comp); - rc = kernel_thread(qmaster_recovery_main, &data, CLONE_VM|CLONE_FILES); + rc = cfs_create_thread(qmaster_recovery_main, &data, + CFS_DAEMON_FLAGS); if (rc < 0) - CERROR("Cannot start quota recovery thread: rc %d\n", rc); + CERROR("%s: cannot start quota recovery thread: rc %d\n", + obd->obd_name, rc); - wait_for_completion(&data.comp); + cfs_wait_for_completion(&data.comp); RETURN(rc); }