From c40c7939c0110ca53151c47249d85e56a6e0f24d Mon Sep 17 00:00:00 2001 From: Johann Lombardi Date: Mon, 1 Oct 2012 21:17:32 +0200 Subject: [PATCH] LU-1842 quota: add per-filesystem information Add a global list to track per-filesystem quota information which will be used by slaves to determine whether quota enforcement is enabled. This patch also enables test 22 in sanity-quota.sh which should now pass and avoids spurious test failures due to sync_all_data. Signed-off-by: Johann Lombardi Change-Id: I336e7704d3e16dc3f5a92350c652cd6f2f25f25a Reviewed-on: http://review.whamcloud.com/4156 Reviewed-by: Niu Yawei Tested-by: Hudson Reviewed-by: Mike Pershin Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/include/lquota.h | 16 +++- lustre/mgs/mgs_llog.c | 17 ++-- lustre/osd-ldiskfs/osd_handler.c | 21 +++-- lustre/osd-zfs/osd_handler.c | 18 ++-- lustre/quota/Makefile.in | 2 +- lustre/quota/lquota_entry.c | 4 +- lustre/quota/lquota_lib.c | 22 +++-- lustre/quota/qsd_config.c | 174 +++++++++++++++++++++++++++++++++++++++ lustre/quota/qsd_entry.c | 4 +- lustre/quota/qsd_internal.h | 53 +++++++++++- lustre/quota/qsd_lib.c | 156 +++++++++++++++++++++++++++++------ lustre/quota/qsd_lock.c | 4 +- lustre/quota/qsd_reint.c | 4 +- lustre/quota/qsd_request.c | 4 +- lustre/quota/qsd_writeback.c | 4 +- lustre/tests/sanity-quota.sh | 32 ++++--- 16 files changed, 446 insertions(+), 89 deletions(-) create mode 100644 lustre/quota/qsd_config.c diff --git a/lustre/include/lquota.h b/lustre/include/lquota.h index 689bfc1..7af5baa 100644 --- a/lustre/include/lquota.h +++ b/lustre/include/lquota.h @@ -111,10 +111,16 @@ struct qsd_instance; * The API is the following: * * - qsd_init(): the user (mostly the OSD layer) should first allocate a qsd - * instance via qsd_init(). This sets up on-disk objects - * associated with the quota slave feature and initiates the quota - * reintegration procedure if needed. qsd_init() should typically - * be called when ->ldo_start is invoked. + * instance via qsd_init(). This creates all required structures + * to manage quota enforcement for this target and performs all + * low-level initialization which does not involve any lustre + * object. qsd_init() should typically be called when the OSD + * is being set up. + * + * - qsd_prepare(): This sets up on-disk objects associated with the quota slave + * feature and initiates the quota reintegration procedure if + * needed. qsd_prepare() should typically be called when + * ->ldo_prepare is invoked. * * - qsd_fini(): is used to release a qsd_instance structure allocated with * qsd_init(). This releases all quota slave objects and frees the @@ -126,6 +132,8 @@ struct qsd_instance; struct qsd_instance *qsd_init(const struct lu_env *, char *, struct dt_device *, cfs_proc_dir_entry_t *); +int qsd_prepare(const struct lu_env *, struct qsd_instance *); + void qsd_fini(const struct lu_env *, struct qsd_instance *); /* XXX: dummy qsd_op_begin() & qsd_op_end(), will be replaced with the real diff --git a/lustre/mgs/mgs_llog.c b/lustre/mgs/mgs_llog.c index 230a5e9..8cc4d64 100644 --- a/lustre/mgs/mgs_llog.c +++ b/lustre/mgs/mgs_llog.c @@ -2264,12 +2264,11 @@ static int mgs_write_log_quota(const struct lu_env *env, struct mgs_device *mgs, struct fs_db *fsdb, struct mgs_target_info *mti, char *quota, char *ptr) { - struct lustre_cfg_bufs bufs; - struct lustre_cfg *lcfg; - char *tmp; - char sep; - int cmd = LCFG_PARAM; - int rc; + struct mgs_thread_info *mgi = mgs_env_info(env); + struct lustre_cfg *lcfg; + char *tmp; + char sep; + int rc, cmd = LCFG_PARAM; /* support only 'meta' and 'data' pools so far */ if (class_match_param(ptr, QUOTA_METAPOOL_NAME, &tmp) != 0 && @@ -2291,9 +2290,9 @@ static int mgs_write_log_quota(const struct lu_env *env, struct mgs_device *mgs, } } - lustre_cfg_bufs_reset(&bufs, NULL); - lustre_cfg_bufs_set_string(&bufs, 1, quota); - lcfg = lustre_cfg_new(cmd, &bufs); + lustre_cfg_bufs_reset(&mgi->mgi_bufs, mti->mti_fsname); + lustre_cfg_bufs_set_string(&mgi->mgi_bufs, 1, quota); + lcfg = lustre_cfg_new(cmd, &mgi->mgi_bufs); /* truncate the comment to the parameter name */ ptr = tmp - 1; sep = *ptr; diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c index 32ff79b..ac8d51c 100644 --- a/lustre/osd-ldiskfs/osd_handler.c +++ b/lustre/osd-ldiskfs/osd_handler.c @@ -4532,7 +4532,18 @@ static int osd_device_init0(const struct lu_env *env, LASSERT(l->ld_site->ls_linkage.next && l->ld_site->ls_linkage.prev); + /* initialize quota slave instance */ + o->od_quota_slave = qsd_init(env, o->od_svname, &o->od_dt_dev, + o->od_proc_entry); + if (IS_ERR(o->od_quota_slave)) { + rc = PTR_ERR(o->od_quota_slave); + o->od_quota_slave = NULL; + GOTO(out_procfs, rc); + } + RETURN(0); +out_procfs: + osd_procfs_fini(o); out_site: lu_site_fini(&o->od_site); out_compat: @@ -4690,13 +4701,9 @@ static int osd_prepare(const struct lu_env *env, struct lu_device *pdev, RETURN(result); } - /* 2. setup quota slave instance */ - osd->od_quota_slave = qsd_init(env, osd->od_svname, &osd->od_dt_dev, - osd->od_proc_entry); - if (IS_ERR(osd->od_quota_slave)) { - result = PTR_ERR(osd->od_quota_slave); - osd->od_quota_slave = NULL; - } + if (osd->od_quota_slave != NULL) + /* set up quota slave objects */ + result = qsd_prepare(env, osd->od_quota_slave); RETURN(result); } diff --git a/lustre/osd-zfs/osd_handler.c b/lustre/osd-zfs/osd_handler.c index 7b85f40..6da92fa 100644 --- a/lustre/osd-zfs/osd_handler.c +++ b/lustre/osd-zfs/osd_handler.c @@ -579,6 +579,14 @@ static int osd_mount(const struct lu_env *env, o->arc_prune_cb = arc_add_prune_callback(arc_prune_func, o); + /* initialize quota slave instance */ + o->od_quota_slave = qsd_init(env, o->od_svname, &o->od_dt_dev, + o->od_proc_entry); + if (IS_ERR(o->od_quota_slave)) { + rc = PTR_ERR(o->od_quota_slave); + o->od_quota_slave = NULL; + GOTO(err, rc); + } err: RETURN(rc); } @@ -819,13 +827,9 @@ static int osd_prepare(const struct lu_env *env, struct lu_device *pdev, RETURN(rc); } - /* initialize quota slave instance */ - osd->od_quota_slave = qsd_init(env, osd->od_svname, &osd->od_dt_dev, - osd->od_proc_entry); - if (IS_ERR(osd->od_quota_slave)) { - rc = PTR_ERR(osd->od_quota_slave); - osd->od_quota_slave = NULL; - } + if (osd->od_quota_slave != NULL) + /* set up quota slave objects */ + rc = qsd_prepare(env, osd->od_quota_slave); RETURN(rc); } diff --git a/lustre/quota/Makefile.in b/lustre/quota/Makefile.in index b8745cf..193f727 100644 --- a/lustre/quota/Makefile.in +++ b/lustre/quota/Makefile.in @@ -5,7 +5,7 @@ qmt-objs := qmt_dev.o qmt_handler.o qmt_lock.o quota-objs := lproc_quota.o lquota_lib.o lquota_disk.o lquota_entry.o qsd-objs := qsd_lib.o qsd_request.o qsd_entry.o qsd_lock.o -qsd-objs += qsd_reint.o qsd_writeback.o +qsd-objs += qsd_reint.o qsd_writeback.o qsd_config.o lquota-objs := $(quota-objs) $(qsd-objs) diff --git a/lustre/quota/lquota_entry.c b/lustre/quota/lquota_entry.c index 7ccdd17..830e9cf 100644 --- a/lustre/quota/lquota_entry.c +++ b/lustre/quota/lquota_entry.c @@ -23,8 +23,8 @@ * Copyright (c) 2011, 2012, Intel, Inc. * Use is subject to license terms. * - * Author: Johann Lombardi - * Author: Niu Yawei + * Author: Johann Lombardi + * Author: Niu Yawei */ #ifndef EXPORT_SYMTAB diff --git a/lustre/quota/lquota_lib.c b/lustre/quota/lquota_lib.c index 6ab1f79..67145e3 100644 --- a/lustre/quota/lquota_lib.c +++ b/lustre/quota/lquota_lib.c @@ -326,14 +326,11 @@ const struct dt_index_features *glb_idx_feature(struct lu_fid *fid) static int __init init_lquota(void) { int rc; + ENTRY; lquota_key_init_generic(&lquota_thread_key, NULL); lu_context_key_register(&lquota_thread_key); - rc = qmt_glb_init(); - if (rc) - return rc; - #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2,7,50,0) dt_quota_iusr_features = dt_quota_busr_features = dt_quota_glb_features; dt_quota_igrp_features = dt_quota_bgrp_features = dt_quota_glb_features; @@ -341,11 +338,26 @@ static int __init init_lquota(void) #warning "remove old quota compatibility code" #endif - return 0; + rc = qmt_glb_init(); + if (rc) + GOTO(out_key, rc); + + rc = qsd_glb_init(); + if (rc) + GOTO(out_qmt, rc); + + RETURN(0); + +out_qmt: + qmt_glb_fini(); +out_key: + lu_context_key_degister(&lquota_thread_key); + return rc; } static void exit_lquota(void) { + qsd_glb_fini(); qmt_glb_fini(); lu_context_key_degister(&lquota_thread_key); } diff --git a/lustre/quota/qsd_config.c b/lustre/quota/qsd_config.c new file mode 100644 index 0000000..26c06bb --- /dev/null +++ b/lustre/quota/qsd_config.c @@ -0,0 +1,174 @@ +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 021110-1307, USA + * + * GPL HEADER END + */ +/* + * Copyright (c) 2012 Intel, Inc. + * Use is subject to license terms. + * + * Author: Johann Lombardi + * Author: Niu Yawei + */ + +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif + +#define DEBUG_SUBSYSTEM S_LQUOTA + +#include +#include + +#include "qsd_internal.h" + +static CFS_LIST_HEAD(qfs_list); +/* protect the qfs_list */ +static DEFINE_SPINLOCK(qfs_list_lock); + +/* + * Put reference of qsd_fsinfo. + * + * \param qfs - the qsd_fsinfo to be put + */ +void qsd_put_fsinfo(struct qsd_fsinfo *qfs) +{ + ENTRY; + LASSERT(qfs != NULL); + + cfs_spin_lock(&qfs_list_lock); + LASSERT(qfs->qfs_ref > 0); + qfs->qfs_ref--; + if (qfs->qfs_ref == 0) { + LASSERT(cfs_list_empty(&qfs->qfs_qsd_list)); + cfs_list_del(&qfs->qfs_link); + OBD_FREE_PTR(qfs); + } + cfs_spin_unlock(&qfs_list_lock); + EXIT; +} + +/* + * Find or create a qsd_fsinfo + * + * \param name - filesystem name + * \param create - when @create is non-zero, create new one if fail to + * find existing qfs by @name + * + * \retval qsd_fsinfo - success + * \retval NULL - failure + */ +struct qsd_fsinfo *qsd_get_fsinfo(char *name, bool create) +{ + struct qsd_fsinfo *qfs, *new = NULL; + ENTRY; + + if (name == NULL || strlen(name) >= MTI_NAME_MAXLEN) + RETURN(NULL); + + if (create) { + /* pre-allocate a qsd_fsinfo in case there isn't one already. + * we can afford the extra cost since qsd_get_fsinfo() isn't + * called very often with create = true */ + + OBD_ALLOC_PTR(new); + if (new == NULL) + RETURN(NULL); + + cfs_sema_init(&new->qfs_sem, 1); + CFS_INIT_LIST_HEAD(&new->qfs_qsd_list); + strcpy(new->qfs_name, name); + new->qfs_ref = 1; + } + + /* search in the fsinfo list */ + cfs_spin_lock(&qfs_list_lock); + cfs_list_for_each_entry(qfs, &qfs_list, qfs_link) { + if (!strcmp(qfs->qfs_name, name)) { + qfs->qfs_ref++; + goto out; + } + } + + qfs = NULL; /* not found */ + + if (new) { + /* not found, but we were asked to create a new one */ + cfs_list_add_tail(&new->qfs_link, &qfs_list); + qfs = new; + new = NULL; + } +out: + cfs_spin_unlock(&qfs_list_lock); + + if (new) + OBD_FREE_PTR(new); + RETURN(qfs); +} + +/* + * Quota configuration handlers in charge of processing all per-filesystem quota + * parameters set via conf_param. + * + * \param lcfg - quota configuration log to be processed + */ +int qsd_process_config(struct lustre_cfg *lcfg) +{ + struct qsd_fsinfo *qfs; + char *fsname = lustre_cfg_string(lcfg, 0); + char *cfgstr = lustre_cfg_string(lcfg, 1); + char *keystr, *valstr; + int rc, pool, enabled = 0; + ENTRY; + + CDEBUG(D_QUOTA, "processing quota parameter: fs:%s cfgstr:%s\n", fsname, + cfgstr); + + if (class_match_param(cfgstr, PARAM_QUOTA, &keystr) != 0) + RETURN(-EINVAL); + + if (!class_match_param(keystr, QUOTA_METAPOOL_NAME, &valstr)) + pool = LQUOTA_RES_MD; + else if (!class_match_param(keystr, QUOTA_DATAPOOL_NAME, &valstr)) + pool = LQUOTA_RES_DT; + else + RETURN(-EINVAL); + + qfs = qsd_get_fsinfo(fsname, 0); + if (qfs == NULL) { + CERROR("Fail to find quota filesystem information for %s\n", + fsname); + RETURN(-ENOENT); + } + + if (strchr(valstr, 'u')) + enabled |= 1 << USRQUOTA; + if (strchr(valstr, 'g')) + enabled |= 1 << GRPQUOTA; + + if (qfs->qfs_enabled[pool - LQUOTA_FIRST_RES] == enabled) + /* no change required */ + GOTO(out, rc = 0); + + qfs->qfs_enabled[pool - LQUOTA_FIRST_RES] = enabled; +out: + qsd_put_fsinfo(qfs); + RETURN(0); +} diff --git a/lustre/quota/qsd_entry.c b/lustre/quota/qsd_entry.c index f66df72..0970a42 100644 --- a/lustre/quota/qsd_entry.c +++ b/lustre/quota/qsd_entry.c @@ -24,8 +24,8 @@ * Copyright (c) 2011, 2012, Intel, Inc. * Use is subject to license terms. * - * Author: Johann Lombardi - * Author: Niu Yawei + * Author: Johann Lombardi + * Author: Niu Yawei */ #ifndef EXPORT_SYMTAB diff --git a/lustre/quota/qsd_internal.h b/lustre/quota/qsd_internal.h index c829588..f37a5a8 100644 --- a/lustre/quota/qsd_internal.h +++ b/lustre/quota/qsd_internal.h @@ -31,6 +31,7 @@ #define _QSD_INTERNAL_H struct qsd_type_info; +struct qsd_fsinfo; /* * A QSD instance implements quota enforcement support for a given OSD. @@ -73,7 +74,13 @@ struct qsd_instance { * * probably way too much :( */ - cfs_rwlock_t qsd_lock; + cfs_rwlock_t qsd_lock; + + /* per-filesystem quota information */ + struct qsd_fsinfo *qsd_fsinfo; + + /* link into qfs_qsd_list of qfs_fsinfo */ + cfs_list_t qsd_link; unsigned long qsd_is_md:1, /* managing quota for mdt */ qsd_stopping:1; /* qsd_instance is stopping */ @@ -124,6 +131,28 @@ struct qsd_qtype_info { }; /* + * Per-filesystem quota information + * Structure tracking quota enforcement status on a per-filesystem basis + */ +struct qsd_fsinfo { + /* filesystem name */ + char qfs_name[MTI_NAME_MAXLEN]; + + /* what type of quota is enabled for each resource type. */ + unsigned int qfs_enabled[LQUOTA_NR_RES]; + + /* list of all qsd_instance for this fs */ + cfs_list_t qfs_qsd_list; + cfs_semaphore_t qfs_sem; + + /* link to the global quota fsinfo list. */ + cfs_list_t qfs_link; + + /* reference count */ + int qfs_ref; +}; + +/* * Helper functions & prototypes */ @@ -190,6 +219,24 @@ struct qsd_thread_info *qsd_info(const struct lu_env *env) return info; } +/* helper function to check whether a given quota type is enabled */ +static inline int qsd_type_enabled(struct qsd_instance *qsd, int type) +{ + int enabled, pool; + + LASSERT(qsd != NULL); + LASSERT(type < MAXQUOTAS); + + if (qsd->qsd_fsinfo == NULL) + return 0; + + pool = qsd->qsd_is_md ? LQUOTA_RES_MD : LQUOTA_RES_DT; + enabled = qsd->qsd_fsinfo->qfs_enabled[pool - LQUOTA_FIRST_RES]; + + return enabled & (1 << type); +} + +/* helper function to set new qunit and compute associated qtune value */ static inline void qsd_set_qunit(struct lquota_entry *lqe, __u64 qunit) { if (lqe->lqe_qunit == qunit) @@ -258,6 +305,10 @@ int qsd_fetch_index(const struct lu_env *, struct obd_export *, void qsd_bump_version(struct qsd_qtype_info *, __u64, bool); void qsd_upd_schedule(struct qsd_qtype_info *, struct lquota_entry *, union lquota_id *, union lquota_rec *, __u64, bool); +/* qsd_config.c */ +struct qsd_fsinfo *qsd_get_fsinfo(char *, bool); +void qsd_put_fsinfo(struct qsd_fsinfo *); +int qsd_process_config(struct lustre_cfg *); /* qsd_handler.c */ /* XXX to be replaced with real function once qsd_handler landed */ diff --git a/lustre/quota/qsd_lib.c b/lustre/quota/qsd_lib.c index 16a5cc4..65208d2 100644 --- a/lustre/quota/qsd_lib.c +++ b/lustre/quota/qsd_lib.c @@ -21,15 +21,32 @@ * GPL HEADER END */ /* - * Copyright (c) 2012 Whamcloud, Inc. + * Copyright (c) 2012 Intel, Inc. * Use is subject to license terms. * - * Author: Johann Lombardi - * Author: Niu Yawei + * Author: Johann Lombardi + * Author: Niu Yawei */ /* * Quota Slave Driver (QSD) management. + * + * The quota slave feature is implemented under the form of a library called + * QSD. Each OSD device should create a QSD instance via qsd_init() which will + * be used to manage quota enforcement for this device. This implies: + * - completing the reintegration procedure with the quota master (aka QMT, see + * qmt_dev.c) to retrieve the latest quota settings and space distribution. + * - managing quota locks in order to be notified of configuration changes. + * - acquiring space from the QMT when quota space for a given user/group is + * close to exhaustion. + * - allocating quota space to service threads for local request processing. + * + * Once the QSD instance created, the OSD device should invoke qsd_start() + * when recovery is completed. This notifies the QSD that we are about to + * process new requests on which quota should be strictly enforced. + * Then, qsd_op_begin/end can be used to reserve/release/pre-acquire quota space + * for/after each operation until shutdown where the QSD instance should be + * freed via qsd_fini(). */ #ifndef EXPORT_SYMTAB @@ -38,6 +55,7 @@ #define DEBUG_SUBSYSTEM S_LQUOTA +#include #include "qsd_internal.h" /* define qsd thread key */ @@ -62,8 +80,27 @@ static int lprocfs_qsd_rd_state(char *page, char **start, off_t off, qsd->qsd_is_md ? "md" : "dt"); } +static int lprocfs_qsd_rd_enabled(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct qsd_instance *qsd = (struct qsd_instance *)data; + char enabled[5]; + LASSERT(qsd != NULL); + + memset(enabled, 0, sizeof(enabled)); + if (qsd_type_enabled(qsd, USRQUOTA)) + strcat(enabled, "u"); + if (qsd_type_enabled(qsd, GRPQUOTA)) + strcat(enabled, "g"); + if (strlen(enabled) == 0) + strcat(enabled, "none"); + + return snprintf(page, count, "%s\n", enabled); +} + static struct lprocfs_vars lprocfs_quota_qsd_vars[] = { { "info", lprocfs_qsd_rd_state, 0, 0}, + { "enabled", lprocfs_qsd_rd_enabled, 0, 0}, { NULL } }; @@ -119,7 +156,8 @@ static void qsd_qtype_fini(const struct lu_env *env, struct qsd_instance *qsd, /* * Allocate and initialize a qsd_qtype_info structure for quota type \qtype. * This opens the accounting object and initializes the proc file. - * It's called on OSD start when the qsd instance is created. + * It's called on OSD start when the qsd_prepare() is invoked on the qsd + * instance. * * \param env - the environment passed by the caller * \param qsd - is the qsd instance which will be in charge of the new @@ -229,8 +267,16 @@ void qsd_fini(const struct lu_env *env, struct qsd_instance *qsd) CDEBUG(D_QUOTA, "%s: initiating QSD shutdown\n", qsd->qsd_svname); qsd->qsd_stopping = true; + /* remove from the list of fsinfo */ + if (!cfs_list_empty(&qsd->qsd_link)) { + LASSERT(qsd->qsd_fsinfo != NULL); + cfs_down(&qsd->qsd_fsinfo->qfs_sem); + cfs_list_del_init(&qsd->qsd_link); + cfs_up(&qsd->qsd_fsinfo->qfs_sem); + } + /* remove qsd proc entry */ - if (qsd->qsd_proc != NULL && !IS_ERR(qsd->qsd_proc)) { + if (qsd->qsd_proc != NULL) { lprocfs_remove(&qsd->qsd_proc); qsd->qsd_proc = NULL; } @@ -239,8 +285,12 @@ void qsd_fini(const struct lu_env *env, struct qsd_instance *qsd) for (qtype = USRQUOTA; qtype < MAXQUOTAS; qtype++) qsd_qtype_fini(env, qsd, qtype); + /* release per-filesystem information */ + if (qsd->qsd_fsinfo != NULL) + qsd_put_fsinfo(qsd->qsd_fsinfo); + /* release quota root directory */ - if (qsd->qsd_root != NULL && !IS_ERR(qsd->qsd_root)) { + if (qsd->qsd_root != NULL) { lu_object_put(env, &qsd->qsd_root->do_lu); qsd->qsd_root = NULL; } @@ -259,8 +309,7 @@ EXPORT_SYMBOL(qsd_fini); /* * Create a new qsd_instance to be associated with backend osd device - * identified by \dev. For now, this function just create procfs files which - * dumps the accounting information + * identified by \dev. * * \param env - the environment passed by the caller * \param svname - is the service name of the OSD device creating this instance @@ -275,8 +324,9 @@ struct qsd_instance *qsd_init(const struct lu_env *env, char *svname, struct dt_device *dev, cfs_proc_dir_entry_t *osd_proc) { + struct qsd_thread_info *qti = qsd_info(env); struct qsd_instance *qsd; - int rc, qtype; + int rc; ENTRY; /* allocate qsd instance */ @@ -285,6 +335,7 @@ struct qsd_instance *qsd_init(const struct lu_env *env, char *svname, RETURN(ERR_PTR(-ENOMEM)); cfs_rwlock_init(&qsd->qsd_lock); + CFS_INIT_LIST_HEAD(&qsd->qsd_link); /* copy service name */ strncpy(qsd->qsd_svname, svname, MAX_OBD_NAME); @@ -298,35 +349,35 @@ struct qsd_instance *qsd_init(const struct lu_env *env, char *svname, * the configuration log in the future */ qsd->qsd_pool_id = 0; - /* Record whether this qsd instance is managing quota enforcement for a - * MDT (i.e. inode quota) or OST (block quota) */ - qsd->qsd_is_md = lu_device_is_md(dev->dd_lu_dev.ld_site->ls_top_dev); - - /* look-up on-disk directory for the quota slave */ - qsd->qsd_root = lquota_disk_dir_find_create(env, dev, NULL, QSD_DIR); - if (IS_ERR(qsd->qsd_root)) { - rc = PTR_ERR(qsd->qsd_root); - CERROR("%s: failed to create quota slave root dir (%d)\n", - svname, rc); + /* get fsname from svname */ + rc = server_name2fsname(svname, qti->qti_buf, NULL); + if (rc) { + CERROR("%s: fail to extract filesystem name\n", svname); GOTO(out, rc); } + /* look up quota setting for the filesystem the target belongs to */ + qsd->qsd_fsinfo = qsd_get_fsinfo(qti->qti_buf, 1); + if (qsd->qsd_fsinfo == NULL) { + CERROR("%s: failed to locate filesystem information\n", svname); + GOTO(out, rc = -EINVAL); + } + + /* add in the list of lquota_fsinfo */ + cfs_down(&qsd->qsd_fsinfo->qfs_sem); + list_add_tail(&qsd->qsd_link, &qsd->qsd_fsinfo->qfs_qsd_list); + cfs_up(&qsd->qsd_fsinfo->qfs_sem); + /* register procfs directory */ qsd->qsd_proc = lprocfs_register(QSD_DIR, osd_proc, lprocfs_quota_qsd_vars, qsd); if (IS_ERR(qsd->qsd_proc)) { rc = PTR_ERR(qsd->qsd_proc); + qsd->qsd_proc = NULL; CERROR("%s: fail to create quota slave proc entry (%d)\n", svname, rc); GOTO(out, rc); } - - /* initialize per-quota type data */ - for (qtype = USRQUOTA; qtype < MAXQUOTAS; qtype++) { - rc = qsd_qtype_init(env, qsd, qtype); - if (rc) - GOTO(out, rc); - } out: if (rc) { qsd_fini(env, qsd); @@ -337,12 +388,64 @@ out: EXPORT_SYMBOL(qsd_init); /* + * Initialize on-disk structures in order to manage quota enforcement for + * the target associated with the qsd instance \qsd and starts the reintegration + * procedure for each quota type as soon as possible. + * The last step of the reintegration will be completed once qsd_start() is + * called, at which points the space reconciliation with the master will be + * executed. + * This function must be called when the server stack is fully configured, + * typically when ->ldo_prepare is called across the stack. + * + * \param env - the environment passed by the caller + * \param qsd - is qsd_instance to prepare + * + * \retval - 0 on success, appropriate error on failure + */ +int qsd_prepare(const struct lu_env *env, struct qsd_instance *qsd) +{ + int rc, qtype; + ENTRY; + + LASSERT(qsd != NULL); + + /* Record whether this qsd instance is managing quota enforcement for a + * MDT (i.e. inode quota) or OST (block quota) */ + if (lu_device_is_md(qsd->qsd_dev->dd_lu_dev.ld_site->ls_top_dev)) + qsd->qsd_is_md = true; + + /* look-up on-disk directory for the quota slave */ + qsd->qsd_root = lquota_disk_dir_find_create(env, qsd->qsd_dev, NULL, + QSD_DIR); + if (IS_ERR(qsd->qsd_root)) { + rc = PTR_ERR(qsd->qsd_root); + qsd->qsd_root = NULL; + CERROR("%s: failed to create quota slave root dir (%d)\n", + qsd->qsd_svname, rc); + RETURN(rc); + } + + /* initialize per-quota type data */ + for (qtype = USRQUOTA; qtype < MAXQUOTAS; qtype++) { + rc = qsd_qtype_init(env, qsd, qtype); + if (rc) + RETURN(rc); + } + + RETURN(0); +} +EXPORT_SYMBOL(qsd_prepare); + +void lustre_register_quota_process_config(int (*qpc)(struct lustre_cfg *lcfg)); + +/* * Global initialization performed at module load time */ int qsd_glb_init(void) { qsd_key_init_generic(&qsd_thread_key, NULL); lu_context_key_register(&qsd_thread_key); + lustre_register_quota_process_config(qsd_process_config); return 0; } @@ -351,5 +454,6 @@ int qsd_glb_init(void) */ void qsd_glb_fini(void) { + lustre_register_quota_process_config(NULL); lu_context_key_degister(&qsd_thread_key); } diff --git a/lustre/quota/qsd_lock.c b/lustre/quota/qsd_lock.c index 679cbb7..7c17a62 100644 --- a/lustre/quota/qsd_lock.c +++ b/lustre/quota/qsd_lock.c @@ -24,8 +24,8 @@ * Copyright (c) 2011, 2012, Intel, Inc. * Use is subject to license terms. * - * Author: Johann Lombardi - * Author: Niu Yawei + * Author: Johann Lombardi + * Author: Niu Yawei */ #ifndef EXPORT_SYMTAB diff --git a/lustre/quota/qsd_reint.c b/lustre/quota/qsd_reint.c index 64405fc..fdbfdc8 100644 --- a/lustre/quota/qsd_reint.c +++ b/lustre/quota/qsd_reint.c @@ -24,8 +24,8 @@ * Copyright (c) 2011, 2012, Intel, Inc. * Use is subject to license terms. * - * Author: Johann Lombardi - * Author: Niu Yawei + * Author: Johann Lombardi + * Author: Niu Yawei */ #ifndef EXPORT_SYMTAB diff --git a/lustre/quota/qsd_request.c b/lustre/quota/qsd_request.c index 57574a1..a0c830c 100644 --- a/lustre/quota/qsd_request.c +++ b/lustre/quota/qsd_request.c @@ -24,8 +24,8 @@ * Copyright (c) 2012 Intel, Inc. * Use is subject to license terms. * - * Author: Johann Lombardi - * Author: Niu Yawei + * Author: Johann Lombardi + * Author: Niu Yawei */ #ifndef EXPORT_SYMTAB diff --git a/lustre/quota/qsd_writeback.c b/lustre/quota/qsd_writeback.c index 40b064e..4ed599b 100644 --- a/lustre/quota/qsd_writeback.c +++ b/lustre/quota/qsd_writeback.c @@ -24,8 +24,8 @@ * Copyright (c) 2011, 2012, Intel, Inc. * Use is subject to license terms. * - * Author: Johann Lombardi - * Author: Niu Yawei + * Author: Johann Lombardi + * Author: Niu Yawei */ #ifndef EXPORT_SYMTAB diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index 4029290..c0912ae 100644 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -9,11 +9,9 @@ set -e SRCDIR=`dirname $0` export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin -# only accounting tests are supported for the time being -ONLY="33 34 35" - ONLY=${ONLY:-"$*"} -ALWAYS_EXCEPT="$SANITY_QUOTA_EXCEPT" +# only accounting tests and test 22 are supported for the time being +ALWAYS_EXCEPT="0 1 2 3 4 5 6 7 8 9 10 11 12 13 15 17 18 19 20 21 23 24 27 30 36 $SANITY_QUOTA_EXCEPT" # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! [ "$ALWAYS_EXCEPT$EXCEPT" ] && @@ -304,7 +302,7 @@ cleanup_quota_test() { rm -rf $DIR/$tdir echo "Wait for unlink objects finished..." wait_delete_completed - sync_all_data + sync_all_data || true } quota_show_check() { @@ -338,7 +336,7 @@ quota_show_check() { fi } -# enalbe quota debug +# enable quota debug quota_init() { do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=+quota" } @@ -415,7 +413,7 @@ test_1() { rm -f $TESTFILE wait_delete_completed - sync_all_data + sync_all_data || true USED=$(getquota -u $TSTUSR global curspace) [ $USED -ne 0 ] && quota_error u $TSTUSR \ "user quota isn't released after deletion" @@ -583,7 +581,7 @@ test_block_soft() { echo "Unlink file to stop timer" rm -f $TESTFILE wait_delete_completed - sync_all_data + sync_all_data || true $SHOW_QUOTA_USER $SHOW_QUOTA_GROUP @@ -841,7 +839,7 @@ test_6() { $RUNAS2 $DD of=$TESTFILE2 count=1 || error "write $TESTFILE2 failure, expect success" sync; sync - sync_all_data + sync_all_data || true # set a small timeout value, to make the quota RPC timedout before # the watchdog triggered. @@ -1238,7 +1236,7 @@ test_12() { echo "Free space from ost0..." rm -f $TESTFILE0 wait_delete_completed - sync_all_data + sync_all_data || true echo "Write to ost1 after space freed from ost0..." $RUNAS $DD of=$TESTFILE1 count=$blk_cnt oflag=sync || @@ -1301,7 +1299,7 @@ test_15(){ local LIMIT=$((24 * 1024 * 1024 * 1024 * 1024)) # 24 TB wait_delete_completed - sync_all_data + sync_all_data || true # test for user $LFS setquota -u $TSTUSR -b 0 -B $LIMIT -i 0 -I 0 $DIR @@ -1365,7 +1363,7 @@ test_17sub() { sleep 1 done - sync; sync_all_data + sync; sync_all_data || true USED=$(getquota -u $TSTUSR global curspace) [ $USED -ge $(($BLKS * 1024)) ] || quota_error u $TSTUSR \ @@ -1728,7 +1726,7 @@ test_24() { runas -u 0 -g 0 $DD of=$TESTFILE count=$((blimit + 1)) || error "write failure, expect success" cancel_lru_locks osc - sync_all_data + sync_all_data || true $SHOW_QUOTA_USER | grep '*' || error "no matching *" @@ -1820,7 +1818,7 @@ test_33() { echo "Iteration $i/$INODES completed" done cancel_lru_locks osc - sync; sync_all_data + sync; sync_all_data || true echo "Verify disk usage after write" USED=$(getquota -u $TSTID global curspace) @@ -1869,7 +1867,7 @@ test_34() { $DD of=$DIR/$tdir/$tfile count=$BLK_CNT 2>/dev/null || error "write failed" cancel_lru_locks osc - sync; sync_all_data + sync; sync_all_data || true echo "chown the file to user $TSTID" chown $TSTID $DIR/$tdir/$tfile || error "chown failed" @@ -1916,7 +1914,7 @@ test_35() { $RUNAS $DD of=$DIR/$tdir/$tfile count=$BLK_CNT 2>/dev/null || error "write failed" cancel_lru_locks osc - sync; sync_all_data + sync; sync_all_data || true echo "Save disk usage before restart" local ORIG_USR_SPACE=$(getquota -u $TSTID global curspace) @@ -1964,7 +1962,7 @@ test_35() { $RUNAS $DD of=$DIR/$tdir/$tfile count=$BLK_CNT seek=1 2>/dev/null || error "write failed" cancel_lru_locks osc - sync; sync_all_data + sync; sync_all_data || true echo "Verify space usage is increased" USED=$(getquota -u $TSTID global curspace) -- 1.8.3.1