From d943c6afe74f6e0d371120ade8bed55c5d60b67a Mon Sep 17 00:00:00 2001 From: Niu Yawei Date: Fri, 28 Sep 2012 05:34:48 -0400 Subject: [PATCH] LU-1842 quota: qsd entry qsd entry operations. Signed-off-by: Johann Lombardi Signed-off-by: Niu Yawei Change-Id: I88c8b0658798d0b2bea62ef80c79b28badc289d2 Reviewed-on: http://review.whamcloud.com/4119 Tested-by: Hudson Tested-by: Maloo Reviewed-by: Johann Lombardi Reviewed-by: Alex Zhuravlev --- lustre/quota/Makefile.in | 2 +- lustre/quota/qsd_entry.c | 346 ++++++++++++++++++++++++++++++++++++++++++++ lustre/quota/qsd_internal.h | 56 +++++++ 3 files changed, 403 insertions(+), 1 deletion(-) create mode 100644 lustre/quota/qsd_entry.c diff --git a/lustre/quota/Makefile.in b/lustre/quota/Makefile.in index 140a1d6..93dc089 100644 --- a/lustre/quota/Makefile.in +++ b/lustre/quota/Makefile.in @@ -2,7 +2,7 @@ MODULES := lquota quota-objs := lproc_quota.o lquota_lib.o lquota_disk.o lquota_entry.o -qsd-objs := qsd_lib.o qsd_request.o +qsd-objs := qsd_lib.o qsd_request.o qsd_entry.o lquota-objs := $(quota-objs) $(qsd-objs) diff --git a/lustre/quota/qsd_entry.c b/lustre/quota/qsd_entry.c new file mode 100644 index 0000000..b27d329 --- /dev/null +++ b/lustre/quota/qsd_entry.c @@ -0,0 +1,346 @@ +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 021110-1307, USA + * + * GPL HEADER END + */ +/* + * Copyright (c) 2011, 2012, Intel, Inc. + * Use is subject to license terms. + * + * Author: Johann Lombardi + * Author: Niu Yawei + */ + +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif + +#define DEBUG_SUBSYSTEM S_LQUOTA + +#include "qsd_internal.h" + +/* + * Initialize qsd-specific fields of quota entry. + * + * \param lqe - is the quota entry to initialize + * \param arg - is the pointer to the qsd_qtype_info structure + */ +static void qsd_lqe_init(struct lquota_entry *lqe, void *arg) +{ + LASSERT(!lqe_is_master(lqe)); + + /* initialize slave parameters */ + cfs_rwlock_init(&lqe->lqe_lock); + memset(&lqe->lqe_lockh, 0, sizeof(lqe->lqe_lockh)); + lqe->lqe_pending_write = 0; + lqe->lqe_pending_req = 0; + cfs_waitq_init(&lqe->lqe_waiters); + lqe->lqe_usage = 0; + lqe->lqe_nopreacq = false; +} + +/* + * Update a slave quota entry. This is done by reading enforcement status from + * the copy of the global index and then how much is the slave currenly owns + * for this user from the slave index copy. + * + * \param env - the environment passed by the caller + * \param lqe - is the quota entry to refresh + * \param arg - is the pointer to the qsd_qtype_info structure + */ +static int qsd_lqe_read(const struct lu_env *env, struct lquota_entry *lqe, + void *arg) +{ + struct qsd_thread_info *qti = qsd_info(env); + struct qsd_qtype_info *qqi = (struct qsd_qtype_info *)arg; + int rc; + + LASSERT(!lqe_is_master(lqe)); + + /* read record from global index copy to know whether quota is + * enforced for this user */ + rc = lquota_disk_read(env, qqi->qqi_glb_obj, &lqe->lqe_id, + (struct dt_rec *)&qti->qti_glb_rec); + + switch(rc) { + case -ENOENT: + /* no such entry, assume quota isn't enforced for this user */ + lqe->lqe_enforced = false; + break; + case 0: + if (qti->qti_glb_rec.qbr_hardlimit == 0 && + qti->qti_glb_rec.qbr_softlimit == 0) + /* quota isn't enforced for this use */ + lqe->lqe_enforced = false; + else + lqe->lqe_enforced = true; + break; + default: + LQUOTA_ERROR(lqe, "failed to read quota entry from global index" + "copy, rc:%d", rc); + return rc; + } + + /* read record from slave index copy to find out how much space is + * currently owned by this slave */ + rc = lquota_disk_read(env, qqi->qqi_slv_obj, &lqe->lqe_id, + (struct dt_rec *)&qti->qti_slv_rec); + switch(rc) { + case -ENOENT: + lqe->lqe_granted = 0; + break; + case 0: + lqe->lqe_granted = qti->qti_slv_rec.qsr_granted; + break; + default: + LQUOTA_ERROR(lqe, "failed to read quota entry from slave index" + "copy, rc:%d", rc); + return rc; + } + + /* don't know what the qunit value is yet */ + qsd_set_qunit(lqe, 0); + + /* read current disk-usage from disk */ + rc = qsd_refresh_usage(env, lqe); + if (rc) + return rc; + + LQUOTA_DEBUG(lqe, "successfully read from disk"); + return 0; +} + +/* + * Print lqe information for debugging. + * + * \param lqe - is the quota entry to debug + * \param arg - is the pointer to the qsd_qtype_info structure + * \param msgdata - debug message + * \param fmt - format of debug message + */ +static void qsd_lqe_debug(struct lquota_entry *lqe, void *arg, + struct libcfs_debug_msg_data *msgdata, + const char *fmt, va_list args) +{ + struct qsd_qtype_info *qqi = (struct qsd_qtype_info *)arg; + + libcfs_debug_vmsg2(msgdata, fmt, args, + "qsd:%s qtype:%s id:"LPU64" enforced:%d granted:" + LPU64" pending:"LPU64" waiting:"LPU64" req:%d usage:" + LPU64" qunit:"LPU64" qtune:"LPU64" edquot:%d\n", + qqi->qqi_qsd->qsd_svname, QTYPE_NAME(qqi->qqi_qtype), + lqe->lqe_id.qid_uid, lqe->lqe_enforced, + lqe->lqe_granted, lqe->lqe_pending_write, + lqe->lqe_waiting_write, lqe->lqe_pending_req, + lqe->lqe_usage, lqe->lqe_qunit, lqe->lqe_qtune, + lqe->lqe_edquot); +} + +/* + * Vector of quota entry operations supported on the slave + */ +struct lquota_entry_operations qsd_lqe_ops = { + .lqe_init = qsd_lqe_init, + .lqe_read = qsd_lqe_read, + .lqe_debug = qsd_lqe_debug, +}; + +int qsd_write_version(const struct lu_env *env, struct qsd_qtype_info *qqi, + __u64 ver, bool global) +{ + struct qsd_instance *qsd = qqi->qqi_qsd; + struct dt_object *obj = global ? qqi->qqi_glb_obj : + qqi->qqi_slv_obj; + int rc; + ENTRY; + + rc = lquota_disk_update_ver(env, qsd->qsd_dev, obj, ver); + if (rc) + RETURN(rc); + + qsd_bump_version(qqi, ver, global); + RETURN(0); +} + +/* + * Consult current disk space consumed by a given identifier. + * + * \param env - the environment passed by the caller + * \param qqi - is the pointer to the qsd_qtype_info structure associated + * with the identifier. + * \param lqe - is the quota entry associated with the identifier + */ +int qsd_refresh_usage(const struct lu_env *env, struct lquota_entry *lqe) +{ + struct qsd_thread_info *qti = qsd_info(env); + struct lquota_acct_rec *rec = &qti->qti_acct_rec; + struct qsd_qtype_info *qqi = lqe2qqi(lqe); + int rc = 0; + ENTRY; + + LASSERT(qqi->qqi_acct_obj); + + /* read disk usage */ + rc = lquota_disk_read(env, qqi->qqi_acct_obj, &lqe->lqe_id, + (struct dt_rec *)rec); + switch(rc) { + case -ENOENT: + lqe->lqe_usage = 0; + rc = 0; + break; + case 0: + if (qqi->qqi_qsd->qsd_is_md) + lqe->lqe_usage = rec->ispace; + else + lqe->lqe_usage = toqb(rec->bspace); + break; + default: + LQUOTA_ERROR(lqe, "failed to read disk usage, rc:%d", rc); + RETURN(rc); + } + + LQUOTA_DEBUG(lqe, "disk usage: "LPU64, lqe->lqe_usage); + RETURN(0); +} + +/* + * Update slave or global index copy. + * + * \param env - the environment passed by the caller + * \param qqi - is the qsd_type_info structure managing the index to be + * update + * \param qid - is the identifier for which we need to update the quota + * settings + * \param global - is set to true when updating the global index copy and to + * false for the slave index copy. + * \param ver - is the new version of the index. If equal to 0, the version + * of the index isn't changed + * \param rec - is the updated record to insert in the index file + */ +int qsd_update_index(const struct lu_env *env, struct qsd_qtype_info *qqi, + union lquota_id *qid, bool global, __u64 ver, void *rec) +{ + struct thandle *th = NULL; + struct dt_object *obj; + __u64 *new_verp = NULL; + int flags = 0; + int rc; + ENTRY; + + obj = global ? qqi->qqi_glb_obj : qqi->qqi_slv_obj; + + /* allocate transaction */ + th = dt_trans_create(env, qqi->qqi_qsd->qsd_dev); + if (IS_ERR(th)) + RETURN(PTR_ERR(th)); + + /* reserve enough credits to update record in index file */ + rc = lquota_disk_declare_write(env, th, obj, qid); + if (rc) + GOTO(out, rc); + + /* start local transaction */ + rc = dt_trans_start_local(env, qqi->qqi_qsd->qsd_dev, th); + if (rc) + GOTO(out, rc); + + /* write lock lquota entry */ + if (global) { + /* Update record in global index copy */ + struct lquota_glb_rec *glb_rec = (struct lquota_glb_rec *)rec; + + CDEBUG(D_QUOTA, "%s: updating global index hardlimit: "LPU64", " + "softlimit: "LPU64" for id "LPU64"\n", + qqi->qqi_qsd->qsd_svname, glb_rec->qbr_hardlimit, + glb_rec->qbr_softlimit, qid->qid_uid); + } else { + /* Update record in slave index copy */ + struct lquota_slv_rec *slv_rec = (struct lquota_slv_rec *)rec; + + CDEBUG(D_QUOTA, "%s: update granted to "LPU64" for id "LPU64 + "\n", qqi->qqi_qsd->qsd_svname, slv_rec->qsr_granted, + qid->qid_uid); + } + + if (ver != 0) { + new_verp = &ver; + flags = LQUOTA_SET_VER; + } + + /* write new record to index file */ + rc = lquota_disk_write(env, th, obj, qid, (struct dt_rec *)rec, flags, + new_verp); + EXIT; +out: + dt_trans_stop(env, qqi->qqi_qsd->qsd_dev, th); + if (rc) + CERROR("%s: failed to update %s index copy for id "LPU64", rc:" + "%d\n", qqi->qqi_qsd->qsd_svname, + global ? "global" : "slave", qid->qid_uid, rc); + else if (flags == LQUOTA_SET_VER) + qsd_bump_version(qqi, ver, global); + return rc; +} + +/* + * Update in-memory lquota entry with new quota setting from record \rec. + * The record can either be a global record (i.e. lquota_glb_rec) or a slave + * index record (i.e. lquota_slv_rec). In the former case, \global should be + * set to true. + * + * \param env - the environment passed by the caller + * \param lqe - is the quota entry associated with the identifier + * \param global - is set to true when updating the record is of type + * lquota_glb_rec. Otherwise, it is a lquota_slv_rec record. + * \param rec - is the updated record received from the master. + */ +int qsd_update_lqe(const struct lu_env *env, struct lquota_entry *lqe, + bool global, void *rec) +{ + struct qsd_qtype_info *qqi; + ENTRY; + + LASSERT(lqe != NULL); + LASSERT(!lqe_is_master(lqe)); + + qqi = lqe2qqi(lqe); + + /* updating lqe is always serialized, no locking needed. */ + if (global) { + struct lquota_glb_rec *glb_rec = (struct lquota_glb_rec *)rec; + + /* change enforcement status based on new hard/soft limit */ + lqe->lqe_enforced = (glb_rec->qbr_hardlimit || + glb_rec->qbr_softlimit) ? true : false; + + LQUOTA_DEBUG(lqe, "updating global index hardlimit: "LPU64", " + "softlimit: "LPU64"\n", glb_rec->qbr_hardlimit, + glb_rec->qbr_softlimit); + } else { + struct lquota_slv_rec *slv_rec = (struct lquota_slv_rec *)rec; + + lqe->lqe_granted = slv_rec->qsr_granted; + + LQUOTA_DEBUG(lqe, "updating slave index, granted:"LPU64"", + slv_rec->qsr_granted); + } + + RETURN(0); +} diff --git a/lustre/quota/qsd_internal.h b/lustre/quota/qsd_internal.h index 2af0307..ac39419 100644 --- a/lustre/quota/qsd_internal.h +++ b/lustre/quota/qsd_internal.h @@ -103,6 +103,13 @@ struct qsd_qtype_info { * Helper functions & prototypes */ +/* helper routine to find qsd_instance associated a lquota_entry */ +static inline struct qsd_qtype_info *lqe2qqi(struct lquota_entry *lqe) +{ + LASSERT(!lqe_is_master(lqe)); + return (struct qsd_qtype_info *)lqe->lqe_site->lqs_parent; +} + /* qqi_getref/putref is used to track users of a qqi structure */ static inline void qqi_getref(struct qsd_qtype_info *qqi) { @@ -143,10 +150,52 @@ struct qsd_thread_info *qsd_info(const struct lu_env *env) struct qsd_thread_info *info; info = lu_context_key_get(&env->le_ctx, &qsd_thread_key); + if (info == NULL) { + lu_env_refill((struct lu_env *)env); + info = lu_context_key_get(&env->le_ctx, &qsd_thread_key); + } LASSERT(info); return info; } +static inline void qsd_set_qunit(struct lquota_entry *lqe, __u64 qunit) +{ + if (lqe->lqe_qunit == qunit) + return; + + lqe->lqe_qunit = qunit; + + /* With very large qunit support, we can't afford to have a static + * qtune value, e.g. with a 1PB qunit and qtune set to 50%, we would + * start pre-allocation when 512TB of free quota space remains. + * Therefore, we adapt qtune depending on the actual qunit value */ + if (qunit == 0) /* if qunit is NULL */ + lqe->lqe_qtune = 0; /* qtune = 0 */ + else if (qunit == 1024) /* if 1MB or 1K inodes */ + lqe->lqe_qtune = qunit >> 1; /* => 50% */ + else if (qunit <= 1024 * 1024) /* up to 1GB or 1M inodes */ + lqe->lqe_qtune = qunit >> 2; /* => 25% */ + else if (qunit <= 4 * 1024 * 1024) /* up to 16GB or 16M inodes */ + lqe->lqe_qtune = qunit >> 3; /* => 12.5% */ + else /* above 4GB/4M */ + lqe->lqe_qtune = 1024 * 1024; /* value capped to 1GB/1M */ + + LQUOTA_DEBUG(lqe, "changing qunit & qtune"); + + /* turn on pre-acquire when qunit is modified */ + lqe->lqe_nopreacq = false; +} + +/* qsd_entry.c */ +extern struct lquota_entry_operations qsd_lqe_ops; +int qsd_refresh_usage(const struct lu_env *, struct lquota_entry *); +int qsd_update_index(const struct lu_env *, struct qsd_qtype_info *, + union lquota_id *, bool, __u64, void *); +int qsd_update_lqe(const struct lu_env *, struct lquota_entry *, bool, + void *); +int qsd_write_version(const struct lu_env *, struct qsd_qtype_info *, + __u64, bool); + /* qsd_request.c */ typedef void (*qsd_req_completion_t) (const struct lu_env *, struct qsd_qtype_info *, @@ -163,4 +212,11 @@ int qsd_intent_lock(const struct lu_env *, struct obd_export *, int qsd_fetch_index(const struct lu_env *, struct obd_export *, struct idx_info *, unsigned int, cfs_page_t **, bool *); +/* qsd_writeback.c */ +/* XXX to be replaced with real function when reintegration landed. */ +static inline void qsd_bump_version(struct qsd_qtype_info *qqi, __u64 ver, + bool global) +{ +} + #endif /* _QSD_INTERNAL_H */ -- 1.8.3.1